[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"skill:01M6F66DTNK4643PPM02HMW5QP":3,"skill-same:01M6F66DTNK4643PPM02HMW5QP":80},{"skillNo":4,"name":5,"nameZh":6,"description":7,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":12,"repositoryNo":13,"starsCount":14,"licenseSpdxId":15,"licenseFileUrl":6,"sourceHtmlUrl":16,"categories":17,"keywords":23,"publishedAt":48,"linkOnly":49,"markdownContent":50,"markdownBody":51,"frontMatter":52,"refName":57,"commitSha":58,"skillDirectory":59,"forksCount":60,"repositoryHtmlUrl":61,"repositoryHomepageUrl":6,"ownerHtmlUrl":62,"ownerBlogUrl":63,"ownerDescription":64,"updatedAt":65,"install":66,"resources":79},"01M6F66DTNK4643PPM02HMW5QP","databricks-data-handling",null,"Implement Delta Lake data management patterns including GDPR, PII handling, and data lifecycle.\nUse when implementing data retention, handling GDPR requests,\nor managing data lifecycle in Delta Lake.\nTrigger with phrases like \"databricks GDPR\", \"databricks PII\",\n\"databricks data retention\", \"databricks data lifecycle\", \"delete user data\".","Dicklesworthstone\u002Fpi_agent_rust","Dicklesworthstone","https:\u002F\u002Favatars.githubusercontent.com\u002Fu\u002F35050222?v=4","01M6CG26AR942CSVMN1M5E3DSE","Jeff Emanuel","01M6CG26EPP71PSHW1VDS71121",1992,"MIT","https:\u002F\u002Fgithub.com\u002FDicklesworthstone\u002Fpi_agent_rust\u002Fblob\u002Fmain\u002Ftests\u002Fext_conformance\u002Fartifacts\u002Fplugins-community\u002Fplugins\u002Fsaas-packs\u002Fdatabricks-pack\u002Fskills\u002Fdatabricks-data-handling\u002FSKILL.md",[18],{"categoryNo":19,"categoryName":20,"taxonomyType":21,"taxonomyTypeDesc":22},"01JBSEEDCAT000000000000033","数据分析师","PROFESSION","适用职业",[24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47],"databricks","databricks data handling","data","when","implement","implementing","with","retention","data management","trigger","io","patterns","delta","implement delta","use","edit","write","read","Bash","lifecycle","pii","gdpr","delta lake","delete","2026-11-10 10:12:30",false,"---\nname: databricks-data-handling\ndescription: |\n  Implement Delta Lake data management patterns including GDPR, PII handling, and data lifecycle.\n  Use when implementing data retention, handling GDPR requests,\n  or managing data lifecycle in Delta Lake.\n  Trigger with phrases like \"databricks GDPR\", \"databricks PII\",\n  \"databricks data retention\", \"databricks data lifecycle\", \"delete user data\".\nallowed-tools: Read, Write, Edit, Bash(databricks:*)\nversion: 1.0.0\nlicense: MIT\nauthor: Jeremy Longshore \u003Cjeremy@intentsolutions.io>\n---\n\n# Databricks Data Handling\n\n## Overview\nImplement data management patterns for compliance, privacy, and lifecycle in Delta Lake.\n\n## Prerequisites\n- Unity Catalog configured\n- Understanding of Delta Lake features\n- Compliance requirements documented\n- Data classification in place\n\n## Instructions\n\n### Step 1: Data Classification and Tagging\n\n```sql\n-- Tag tables with data classification\nALTER TABLE catalog.schema.customers\nSET TAGS ('data_classification' = 'PII', 'retention_days' = '365');\n\nALTER TABLE catalog.schema.orders\nSET TAGS ('data_classification' = 'CONFIDENTIAL', 'retention_days' = '2555');\n\nALTER TABLE catalog.schema.analytics_events\nSET TAGS ('data_classification' = 'INTERNAL', 'retention_days' = '90');\n\n-- Tag columns with sensitivity\nALTER TABLE catalog.schema.customers\nALTER COLUMN email SET TAGS ('pii' = 'true', 'pii_type' = 'email');\n\nALTER TABLE catalog.schema.customers\nALTER COLUMN phone SET TAGS ('pii' = 'true', 'pii_type' = 'phone');\n\n-- Query classified data\nSELECT\n    table_catalog,\n    table_schema,\n    table_name,\n    tag_name,\n    tag_value\nFROM system.information_schema.table_tags\nWHERE tag_name = 'data_classification';\n```\n\n### Step 2: GDPR Right to Deletion (RTBF)\n\n```python\n# src\u002Fcompliance\u002Fgdpr.py\nfrom pyspark.sql import SparkSession\nfrom delta.tables import DeltaTable\nfrom datetime import datetime\nimport logging\n\nlogger = logging.getLogger(__name__)\n\nclass GDPRHandler:\n    \"\"\"Handle GDPR data subject requests.\"\"\"\n\n    def __init__(self, spark: SparkSession, catalog: str):\n        self.spark = spark\n        self.catalog = catalog\n\n    def process_deletion_request(\n        self,\n        user_id: str,\n        request_id: str,\n        dry_run: bool = True,\n    ) -> dict:\n        \"\"\"\n        Process GDPR deletion request for a user.\n\n        Args:\n            user_id: User identifier to delete\n            request_id: GDPR request tracking ID\n            dry_run: If True, only report what would be deleted\n\n        Returns:\n            Deletion report\n        \"\"\"\n        report = {\n            \"request_id\": request_id,\n            \"user_id\": user_id,\n            \"timestamp\": datetime.utcnow().isoformat(),\n            \"dry_run\": dry_run,\n            \"tables_processed\": [],\n            \"total_rows_deleted\": 0,\n        }\n\n        # Find all tables with PII tag\n        pii_tables = self._get_pii_tables()\n\n        for table_info in pii_tables:\n            table_name = f\"{table_info['catalog']}.{table_info['schema']}.{table_info['table']}\"\n            user_column = self._get_user_column(table_name)\n\n            if not user_column:\n                continue\n\n            # Count rows to be deleted\n            count_query = f\"SELECT COUNT(*) FROM {table_name} WHERE {user_column} = '{user_id}'\"\n            row_count = self.spark.sql(count_query).first()[0]\n\n            if row_count > 0:\n                table_report = {\n                    \"table\": table_name,\n                    \"rows_to_delete\": row_count,\n                    \"deleted\": False,\n                }\n\n                if not dry_run:\n                    # Delete using Delta Lake DELETE\n                    self.spark.sql(f\"\"\"\n                        DELETE FROM {table_name}\n                        WHERE {user_column} = '{user_id}'\n                    \"\"\")\n                    table_report[\"deleted\"] = True\n\n                    # Log deletion for audit\n                    self._log_deletion(request_id, table_name, user_id, row_count)\n\n                report[\"tables_processed\"].append(table_report)\n                report[\"total_rows_deleted\"] += row_count\n\n        return report\n\n    def _get_pii_tables(self) -> list[dict]:\n        \"\"\"Get all tables tagged as containing PII.\"\"\"\n        query = f\"\"\"\n            SELECT DISTINCT\n                table_catalog as catalog,\n                table_schema as schema,\n                table_name as table\n            FROM {self.catalog}.information_schema.table_tags\n            WHERE tag_name = 'data_classification'\n              AND tag_value = 'PII'\n        \"\"\"\n        return [row.asDict() for row in self.spark.sql(query).collect()]\n\n    def _get_user_column(self, table_name: str) -> str:\n        \"\"\"Determine user identifier column for table.\"\"\"\n        # Check for common user ID columns\n        columns = [c.name for c in self.spark.table(table_name).schema]\n        user_columns = ['user_id', 'customer_id', 'account_id', 'member_id']\n\n        for uc in user_columns:\n            if uc in columns:\n                return uc\n        return None\n\n    def _log_deletion(\n        self,\n        request_id: str,\n        table_name: str,\n        user_id: str,\n        row_count: int\n    ):\n        \"\"\"Log deletion for compliance audit.\"\"\"\n        self.spark.sql(f\"\"\"\n            INSERT INTO {self.catalog}.compliance.deletion_log\n            VALUES (\n                '{request_id}',\n                '{table_name}',\n                '{user_id}',\n                {row_count},\n                current_timestamp()\n            )\n        \"\"\")\n\n# Usage\ngdpr = GDPRHandler(spark, \"prod_catalog\")\nreport = gdpr.process_deletion_request(\n    user_id=\"user-12345\",\n    request_id=\"GDPR-2024-001\",\n    dry_run=True  # Set to False to actually delete\n)\nprint(report)\n```\n\n### Step 3: Data Retention Policies\n\n```python\n# src\u002Fcompliance\u002Fretention.py\nfrom pyspark.sql import SparkSession\nfrom datetime import datetime, timedelta\n\nclass DataRetentionManager:\n    \"\"\"Manage data retention and cleanup.\"\"\"\n\n    def __init__(self, spark: SparkSession, catalog: str):\n        self.spark = spark\n        self.catalog = catalog\n\n    def apply_retention_policies(self, dry_run: bool = True) -> list[dict]:\n        \"\"\"\n        Apply retention policies based on table tags.\n\n        Returns:\n            List of tables processed with row counts\n        \"\"\"\n        results = []\n\n        # Get tables with retention tags\n        tables = self.spark.sql(f\"\"\"\n            SELECT\n                table_catalog,\n                table_schema,\n                table_name,\n                CAST(tag_value AS INT) as retention_days\n            FROM {self.catalog}.information_schema.table_tags\n            WHERE tag_name = 'retention_days'\n        \"\"\").collect()\n\n        for table in tables:\n            full_name = f\"{table.table_catalog}.{table.table_schema}.{table.table_name}\"\n            cutoff_date = datetime.now() - timedelta(days=table.retention_days)\n\n            # Find date column\n            date_col = self._get_date_column(full_name)\n            if not date_col:\n                continue\n\n            # Count rows to delete\n            count = self.spark.sql(f\"\"\"\n                SELECT COUNT(*) FROM {full_name}\n                WHERE {date_col} \u003C '{cutoff_date.strftime('%Y-%m-%d')}'\n            \"\"\").first()[0]\n\n            result = {\n                \"table\": full_name,\n                \"retention_days\": table.retention_days,\n                \"cutoff_date\": cutoff_date.strftime('%Y-%m-%d'),\n                \"rows_to_delete\": count,\n                \"deleted\": False,\n            }\n\n            if not dry_run and count > 0:\n                self.spark.sql(f\"\"\"\n                    DELETE FROM {full_name}\n                    WHERE {date_col} \u003C '{cutoff_date.strftime('%Y-%m-%d')}'\n                \"\"\")\n                result[\"deleted\"] = True\n\n            results.append(result)\n\n        return results\n\n    def vacuum_tables(self, retention_hours: int = 168) -> list[dict]:\n        \"\"\"\n        Vacuum Delta tables to remove old files.\n\n        Args:\n            retention_hours: Hours of history to retain (default 7 days)\n        \"\"\"\n        results = []\n\n        # Get all Delta tables\n        tables = self.spark.sql(f\"\"\"\n            SELECT table_catalog, table_schema, table_name\n            FROM {self.catalog}.information_schema.tables\n            WHERE table_type = 'MANAGED'\n        \"\"\").collect()\n\n        for table in tables:\n            full_name = f\"{table.table_catalog}.{table.table_schema}.{table.table_name}\"\n\n            try:\n                self.spark.sql(f\"VACUUM {full_name} RETAIN {retention_hours} HOURS\")\n                results.append({\"table\": full_name, \"status\": \"vacuumed\"})\n            except Exception as e:\n                results.append({\"table\": full_name, \"status\": \"error\", \"error\": str(e)})\n\n        return results\n\n    def _get_date_column(self, table_name: str) -> str:\n        \"\"\"Find appropriate date column for retention.\"\"\"\n        columns = [c.name for c in self.spark.table(table_name).schema]\n        date_columns = ['created_at', 'event_date', 'timestamp', 'date', 'updated_at']\n\n        for dc in date_columns:\n            if dc in columns:\n                return dc\n        return None\n\n# Scheduled job for retention\ndef run_daily_retention(spark):\n    \"\"\"Run as scheduled job.\"\"\"\n    manager = DataRetentionManager(spark, \"prod_catalog\")\n\n    # Apply retention policies\n    retention_results = manager.apply_retention_policies(dry_run=False)\n    print(f\"Retention applied: {len(retention_results)} tables processed\")\n\n    # Vacuum tables\n    vacuum_results = manager.vacuum_tables()\n    print(f\"Vacuum completed: {len(vacuum_results)} tables\")\n```\n\n### Step 4: PII Masking and Anonymization\n\n```python\n# src\u002Fcompliance\u002Fmasking.py\nfrom pyspark.sql import DataFrame\nfrom pyspark.sql.functions import (\n    col, sha2, concat, lit, regexp_replace,\n    when, substring, length\n)\n\nclass PIIMasker:\n    \"\"\"Mask PII data for analytics and testing.\"\"\"\n\n    @staticmethod\n    def mask_email(df: DataFrame, column: str) -> DataFrame:\n        \"\"\"Mask email addresses: john.doe@company.com -> j***@***.com\"\"\"\n        return df.withColumn(\n            column,\n            concat(\n                substring(col(column), 1, 1),\n                lit(\"***@***.\"),\n                regexp_replace(col(column), r\".*\\.(\\w+)$\", \"$1\")\n            )\n        )\n\n    @staticmethod\n    def mask_phone(df: DataFrame, column: str) -> DataFrame:\n        \"\"\"Mask phone numbers: +1-555-123-4567 -> +1-555-***-****\"\"\"\n        return df.withColumn(\n            column,\n            regexp_replace(col(column), r\"(\\d{3})-(\\d{4})$\", \"***-****\")\n        )\n\n    @staticmethod\n    def hash_identifier(df: DataFrame, column: str, salt: str = \"\") -> DataFrame:\n        \"\"\"Hash identifiers for pseudonymization.\"\"\"\n        return df.withColumn(\n            column,\n            sha2(concat(col(column), lit(salt)), 256)\n        )\n\n    @staticmethod\n    def mask_name(df: DataFrame, column: str) -> DataFrame:\n        \"\"\"Mask names: John Smith -> J*** S***\"\"\"\n        return df.withColumn(\n            column,\n            regexp_replace(col(column), r\"(\\w)\\w+\", \"$1***\")\n        )\n\n    @staticmethod\n    def create_masked_view(\n        spark,\n        source_table: str,\n        view_name: str,\n        masking_rules: dict[str, str],\n    ) -> None:\n        \"\"\"\n        Create a view with masked PII columns.\n\n        Args:\n            spark: SparkSession\n            source_table: Source table name\n            view_name: Name for masked view\n            masking_rules: Dict of {column: masking_type}\n                          Types: email, phone, hash, name, redact\n        \"\"\"\n        df = spark.table(source_table)\n\n        for column, mask_type in masking_rules.items():\n            if mask_type == \"email\":\n                df = PIIMasker.mask_email(df, column)\n            elif mask_type == \"phone\":\n                df = PIIMasker.mask_phone(df, column)\n            elif mask_type == \"hash\":\n                df = PIIMasker.hash_identifier(df, column)\n            elif mask_type == \"name\":\n                df = PIIMasker.mask_name(df, column)\n            elif mask_type == \"redact\":\n                df = df.withColumn(column, lit(\"[REDACTED]\"))\n\n        df.createOrReplaceTempView(view_name)\n\n# Usage\nPIIMasker.create_masked_view(\n    spark,\n    \"prod_catalog.customers.users\",\n    \"masked_users\",\n    {\n        \"email\": \"email\",\n        \"phone\": \"phone\",\n        \"full_name\": \"name\",\n        \"ssn\": \"redact\",\n    }\n)\n```\n\n### Step 5: Row-Level Security\n\n```sql\n-- Create row filter function\nCREATE OR REPLACE FUNCTION catalog.security.region_filter(region STRING)\nRETURNS BOOLEAN\nRETURN (\n    -- Allow access if user is admin\n    IS_ACCOUNT_GROUP_MEMBER('data-admins')\n    OR\n    -- Or if region matches user's assigned region\n    region = current_user_attribute('region')\n);\n\n-- Apply row filter to table\nALTER TABLE catalog.schema.sales\nSET ROW FILTER catalog.security.region_filter ON (region);\n\n-- Column masking function\nCREATE OR REPLACE FUNCTION catalog.security.mask_salary(salary DECIMAL)\nRETURNS DECIMAL\nRETURN CASE\n    WHEN IS_ACCOUNT_GROUP_MEMBER('hr-team') THEN salary\n    ELSE NULL\nEND;\n\n-- Apply column mask\nALTER TABLE catalog.schema.employees\nALTER COLUMN salary SET MASK catalog.security.mask_salary;\n```\n\n## Output\n- Data classification tags applied\n- GDPR deletion process implemented\n- Retention policies enforced\n- PII masking configured\n- Row-level security enabled\n\n## Error Handling\n| Issue | Cause | Solution |\n|-------|-------|----------|\n| Vacuum fails | Retention too short | Ensure > 7 days retention |\n| Delete timeout | Large table | Partition deletes over time |\n| Missing user column | Non-standard schema | Map user columns manually |\n| Mask function error | Invalid regex | Test masking functions |\n\n## Examples\n\n### GDPR Subject Access Request\n```python\ndef generate_sar_report(spark, user_id: str) -> dict:\n    \"\"\"Generate Subject Access Request report.\"\"\"\n    pii_tables = get_pii_tables(spark)\n    report = {\"user_id\": user_id, \"data\": {}}\n\n    for table in pii_tables:\n        user_col = get_user_column(table)\n        if user_col:\n            data = spark.sql(f\"\"\"\n                SELECT * FROM {table}\n                WHERE {user_col} = '{user_id}'\n            \"\"\").toPandas().to_dict('records')\n            report[\"data\"][table] = data\n\n    return report\n```\n\n## Resources\n- [Delta Lake Privacy](https:\u002F\u002Fdocs.databricks.com\u002Fdelta\u002Fprivacy.html)\n- [Unity Catalog Security](https:\u002F\u002Fdocs.databricks.com\u002Fdata-governance\u002Funity-catalog\u002Frow-level-security.html)\n- [GDPR Compliance](https:\u002F\u002Fdocs.databricks.com\u002Fsecurity\u002Fprivacy\u002Fgdpr.html)\n\n## Next Steps\nFor enterprise RBAC, see `databricks-enterprise-rbac`.\n","# Databricks Data Handling\n\n## Overview\nImplement data management patterns for compliance, privacy, and lifecycle in Delta Lake.\n\n## Prerequisites\n- Unity Catalog configured\n- Understanding of Delta Lake features\n- Compliance requirements documented\n- Data classification in place\n\n## Instructions\n\n### Step 1: Data Classification and Tagging\n\n```sql\n-- Tag tables with data classification\nALTER TABLE catalog.schema.customers\nSET TAGS ('data_classification' = 'PII', 'retention_days' = '365');\n\nALTER TABLE catalog.schema.orders\nSET TAGS ('data_classification' = 'CONFIDENTIAL', 'retention_days' = '2555');\n\nALTER TABLE catalog.schema.analytics_events\nSET TAGS ('data_classification' = 'INTERNAL', 'retention_days' = '90');\n\n-- Tag columns with sensitivity\nALTER TABLE catalog.schema.customers\nALTER COLUMN email SET TAGS ('pii' = 'true', 'pii_type' = 'email');\n\nALTER TABLE catalog.schema.customers\nALTER COLUMN phone SET TAGS ('pii' = 'true', 'pii_type' = 'phone');\n\n-- Query classified data\nSELECT\n    table_catalog,\n    table_schema,\n    table_name,\n    tag_name,\n    tag_value\nFROM system.information_schema.table_tags\nWHERE tag_name = 'data_classification';\n```\n\n### Step 2: GDPR Right to Deletion (RTBF)\n\n```python\n# src\u002Fcompliance\u002Fgdpr.py\nfrom pyspark.sql import SparkSession\nfrom delta.tables import DeltaTable\nfrom datetime import datetime\nimport logging\n\nlogger = logging.getLogger(__name__)\n\nclass GDPRHandler:\n    \"\"\"Handle GDPR data subject requests.\"\"\"\n\n    def __init__(self, spark: SparkSession, catalog: str):\n        self.spark = spark\n        self.catalog = catalog\n\n    def process_deletion_request(\n        self,\n        user_id: str,\n        request_id: str,\n        dry_run: bool = True,\n    ) -> dict:\n        \"\"\"\n        Process GDPR deletion request for a user.\n\n        Args:\n            user_id: User identifier to delete\n            request_id: GDPR request tracking ID\n            dry_run: If True, only report what would be deleted\n\n        Returns:\n            Deletion report\n        \"\"\"\n        report = {\n            \"request_id\": request_id,\n            \"user_id\": user_id,\n            \"timestamp\": datetime.utcnow().isoformat(),\n            \"dry_run\": dry_run,\n            \"tables_processed\": [],\n            \"total_rows_deleted\": 0,\n        }\n\n        # Find all tables with PII tag\n        pii_tables = self._get_pii_tables()\n\n        for table_info in pii_tables:\n            table_name = f\"{table_info['catalog']}.{table_info['schema']}.{table_info['table']}\"\n            user_column = self._get_user_column(table_name)\n\n            if not user_column:\n                continue\n\n            # Count rows to be deleted\n            count_query = f\"SELECT COUNT(*) FROM {table_name} WHERE {user_column} = '{user_id}'\"\n            row_count = self.spark.sql(count_query).first()[0]\n\n            if row_count > 0:\n                table_report = {\n                    \"table\": table_name,\n                    \"rows_to_delete\": row_count,\n                    \"deleted\": False,\n                }\n\n                if not dry_run:\n                    # Delete using Delta Lake DELETE\n                    self.spark.sql(f\"\"\"\n                        DELETE FROM {table_name}\n                        WHERE {user_column} = '{user_id}'\n                    \"\"\")\n                    table_report[\"deleted\"] = True\n\n                    # Log deletion for audit\n                    self._log_deletion(request_id, table_name, user_id, row_count)\n\n                report[\"tables_processed\"].append(table_report)\n                report[\"total_rows_deleted\"] += row_count\n\n        return report\n\n    def _get_pii_tables(self) -> list[dict]:\n        \"\"\"Get all tables tagged as containing PII.\"\"\"\n        query = f\"\"\"\n            SELECT DISTINCT\n                table_catalog as catalog,\n                table_schema as schema,\n                table_name as table\n            FROM {self.catalog}.information_schema.table_tags\n            WHERE tag_name = 'data_classification'\n              AND tag_value = 'PII'\n        \"\"\"\n        return [row.asDict() for row in self.spark.sql(query).collect()]\n\n    def _get_user_column(self, table_name: str) -> str:\n        \"\"\"Determine user identifier column for table.\"\"\"\n        # Check for common user ID columns\n        columns = [c.name for c in self.spark.table(table_name).schema]\n        user_columns = ['user_id', 'customer_id', 'account_id', 'member_id']\n\n        for uc in user_columns:\n            if uc in columns:\n                return uc\n        return None\n\n    def _log_deletion(\n        self,\n        request_id: str,\n        table_name: str,\n        user_id: str,\n        row_count: int\n    ):\n        \"\"\"Log deletion for compliance audit.\"\"\"\n        self.spark.sql(f\"\"\"\n            INSERT INTO {self.catalog}.compliance.deletion_log\n            VALUES (\n                '{request_id}',\n                '{table_name}',\n                '{user_id}',\n                {row_count},\n                current_timestamp()\n            )\n        \"\"\")\n\n# Usage\ngdpr = GDPRHandler(spark, \"prod_catalog\")\nreport = gdpr.process_deletion_request(\n    user_id=\"user-12345\",\n    request_id=\"GDPR-2024-001\",\n    dry_run=True  # Set to False to actually delete\n)\nprint(report)\n```\n\n### Step 3: Data Retention Policies\n\n```python\n# src\u002Fcompliance\u002Fretention.py\nfrom pyspark.sql import SparkSession\nfrom datetime import datetime, timedelta\n\nclass DataRetentionManager:\n    \"\"\"Manage data retention and cleanup.\"\"\"\n\n    def __init__(self, spark: SparkSession, catalog: str):\n        self.spark = spark\n        self.catalog = catalog\n\n    def apply_retention_policies(self, dry_run: bool = True) -> list[dict]:\n        \"\"\"\n        Apply retention policies based on table tags.\n\n        Returns:\n            List of tables processed with row counts\n        \"\"\"\n        results = []\n\n        # Get tables with retention tags\n        tables = self.spark.sql(f\"\"\"\n            SELECT\n                table_catalog,\n                table_schema,\n                table_name,\n                CAST(tag_value AS INT) as retention_days\n            FROM {self.catalog}.information_schema.table_tags\n            WHERE tag_name = 'retention_days'\n        \"\"\").collect()\n\n        for table in tables:\n            full_name = f\"{table.table_catalog}.{table.table_schema}.{table.table_name}\"\n            cutoff_date = datetime.now() - timedelta(days=table.retention_days)\n\n            # Find date column\n            date_col = self._get_date_column(full_name)\n            if not date_col:\n                continue\n\n            # Count rows to delete\n            count = self.spark.sql(f\"\"\"\n                SELECT COUNT(*) FROM {full_name}\n                WHERE {date_col} \u003C '{cutoff_date.strftime('%Y-%m-%d')}'\n            \"\"\").first()[0]\n\n            result = {\n                \"table\": full_name,\n                \"retention_days\": table.retention_days,\n                \"cutoff_date\": cutoff_date.strftime('%Y-%m-%d'),\n                \"rows_to_delete\": count,\n                \"deleted\": False,\n            }\n\n            if not dry_run and count > 0:\n                self.spark.sql(f\"\"\"\n                    DELETE FROM {full_name}\n                    WHERE {date_col} \u003C '{cutoff_date.strftime('%Y-%m-%d')}'\n                \"\"\")\n                result[\"deleted\"] = True\n\n            results.append(result)\n\n        return results\n\n    def vacuum_tables(self, retention_hours: int = 168) -> list[dict]:\n        \"\"\"\n        Vacuum Delta tables to remove old files.\n\n        Args:\n            retention_hours: Hours of history to retain (default 7 days)\n        \"\"\"\n        results = []\n\n        # Get all Delta tables\n        tables = self.spark.sql(f\"\"\"\n            SELECT table_catalog, table_schema, table_name\n            FROM {self.catalog}.information_schema.tables\n            WHERE table_type = 'MANAGED'\n        \"\"\").collect()\n\n        for table in tables:\n            full_name = f\"{table.table_catalog}.{table.table_schema}.{table.table_name}\"\n\n            try:\n                self.spark.sql(f\"VACUUM {full_name} RETAIN {retention_hours} HOURS\")\n                results.append({\"table\": full_name, \"status\": \"vacuumed\"})\n            except Exception as e:\n                results.append({\"table\": full_name, \"status\": \"error\", \"error\": str(e)})\n\n        return results\n\n    def _get_date_column(self, table_name: str) -> str:\n        \"\"\"Find appropriate date column for retention.\"\"\"\n        columns = [c.name for c in self.spark.table(table_name).schema]\n        date_columns = ['created_at', 'event_date', 'timestamp', 'date', 'updated_at']\n\n        for dc in date_columns:\n            if dc in columns:\n                return dc\n        return None\n\n# Scheduled job for retention\ndef run_daily_retention(spark):\n    \"\"\"Run as scheduled job.\"\"\"\n    manager = DataRetentionManager(spark, \"prod_catalog\")\n\n    # Apply retention policies\n    retention_results = manager.apply_retention_policies(dry_run=False)\n    print(f\"Retention applied: {len(retention_results)} tables processed\")\n\n    # Vacuum tables\n    vacuum_results = manager.vacuum_tables()\n    print(f\"Vacuum completed: {len(vacuum_results)} tables\")\n```\n\n### Step 4: PII Masking and Anonymization\n\n```python\n# src\u002Fcompliance\u002Fmasking.py\nfrom pyspark.sql import DataFrame\nfrom pyspark.sql.functions import (\n    col, sha2, concat, lit, regexp_replace,\n    when, substring, length\n)\n\nclass PIIMasker:\n    \"\"\"Mask PII data for analytics and testing.\"\"\"\n\n    @staticmethod\n    def mask_email(df: DataFrame, column: str) -> DataFrame:\n        \"\"\"Mask email addresses: john.doe@company.com -> j***@***.com\"\"\"\n        return df.withColumn(\n            column,\n            concat(\n                substring(col(column), 1, 1),\n                lit(\"***@***.\"),\n                regexp_replace(col(column), r\".*\\.(\\w+)$\", \"$1\")\n            )\n        )\n\n    @staticmethod\n    def mask_phone(df: DataFrame, column: str) -> DataFrame:\n        \"\"\"Mask phone numbers: +1-555-123-4567 -> +1-555-***-****\"\"\"\n        return df.withColumn(\n            column,\n            regexp_replace(col(column), r\"(\\d{3})-(\\d{4})$\", \"***-****\")\n        )\n\n    @staticmethod\n    def hash_identifier(df: DataFrame, column: str, salt: str = \"\") -> DataFrame:\n        \"\"\"Hash identifiers for pseudonymization.\"\"\"\n        return df.withColumn(\n            column,\n            sha2(concat(col(column), lit(salt)), 256)\n        )\n\n    @staticmethod\n    def mask_name(df: DataFrame, column: str) -> DataFrame:\n        \"\"\"Mask names: John Smith -> J*** S***\"\"\"\n        return df.withColumn(\n            column,\n            regexp_replace(col(column), r\"(\\w)\\w+\", \"$1***\")\n        )\n\n    @staticmethod\n    def create_masked_view(\n        spark,\n        source_table: str,\n        view_name: str,\n        masking_rules: dict[str, str],\n    ) -> None:\n        \"\"\"\n        Create a view with masked PII columns.\n\n        Args:\n            spark: SparkSession\n            source_table: Source table name\n            view_name: Name for masked view\n            masking_rules: Dict of {column: masking_type}\n                          Types: email, phone, hash, name, redact\n        \"\"\"\n        df = spark.table(source_table)\n\n        for column, mask_type in masking_rules.items():\n            if mask_type == \"email\":\n                df = PIIMasker.mask_email(df, column)\n            elif mask_type == \"phone\":\n                df = PIIMasker.mask_phone(df, column)\n            elif mask_type == \"hash\":\n                df = PIIMasker.hash_identifier(df, column)\n            elif mask_type == \"name\":\n                df = PIIMasker.mask_name(df, column)\n            elif mask_type == \"redact\":\n                df = df.withColumn(column, lit(\"[REDACTED]\"))\n\n        df.createOrReplaceTempView(view_name)\n\n# Usage\nPIIMasker.create_masked_view(\n    spark,\n    \"prod_catalog.customers.users\",\n    \"masked_users\",\n    {\n        \"email\": \"email\",\n        \"phone\": \"phone\",\n        \"full_name\": \"name\",\n        \"ssn\": \"redact\",\n    }\n)\n```\n\n### Step 5: Row-Level Security\n\n```sql\n-- Create row filter function\nCREATE OR REPLACE FUNCTION catalog.security.region_filter(region STRING)\nRETURNS BOOLEAN\nRETURN (\n    -- Allow access if user is admin\n    IS_ACCOUNT_GROUP_MEMBER('data-admins')\n    OR\n    -- Or if region matches user's assigned region\n    region = current_user_attribute('region')\n);\n\n-- Apply row filter to table\nALTER TABLE catalog.schema.sales\nSET ROW FILTER catalog.security.region_filter ON (region);\n\n-- Column masking function\nCREATE OR REPLACE FUNCTION catalog.security.mask_salary(salary DECIMAL)\nRETURNS DECIMAL\nRETURN CASE\n    WHEN IS_ACCOUNT_GROUP_MEMBER('hr-team') THEN salary\n    ELSE NULL\nEND;\n\n-- Apply column mask\nALTER TABLE catalog.schema.employees\nALTER COLUMN salary SET MASK catalog.security.mask_salary;\n```\n\n## Output\n- Data classification tags applied\n- GDPR deletion process implemented\n- Retention policies enforced\n- PII masking configured\n- Row-level security enabled\n\n## Error Handling\n| Issue | Cause | Solution |\n|-------|-------|----------|\n| Vacuum fails | Retention too short | Ensure > 7 days retention |\n| Delete timeout | Large table | Partition deletes over time |\n| Missing user column | Non-standard schema | Map user columns manually |\n| Mask function error | Invalid regex | Test masking functions |\n\n## Examples\n\n### GDPR Subject Access Request\n```python\ndef generate_sar_report(spark, user_id: str) -> dict:\n    \"\"\"Generate Subject Access Request report.\"\"\"\n    pii_tables = get_pii_tables(spark)\n    report = {\"user_id\": user_id, \"data\": {}}\n\n    for table in pii_tables:\n        user_col = get_user_column(table)\n        if user_col:\n            data = spark.sql(f\"\"\"\n                SELECT * FROM {table}\n                WHERE {user_col} = '{user_id}'\n            \"\"\").toPandas().to_dict('records')\n            report[\"data\"][table] = data\n\n    return report\n```\n\n## Resources\n- [Delta Lake Privacy](https:\u002F\u002Fdocs.databricks.com\u002Fdelta\u002Fprivacy.html)\n- [Unity Catalog Security](https:\u002F\u002Fdocs.databricks.com\u002Fdata-governance\u002Funity-catalog\u002Frow-level-security.html)\n- [GDPR Compliance](https:\u002F\u002Fdocs.databricks.com\u002Fsecurity\u002Fprivacy\u002Fgdpr.html)\n\n## Next Steps\nFor enterprise RBAC, see `databricks-enterprise-rbac`.\n",{"name":5,"author":53,"license":15,"version":54,"description":55,"allowed-tools":56},"Jeremy Longshore \u003Cjeremy@intentsolutions.io>","1.0.0","Implement Delta Lake data management patterns including GDPR, PII handling, and data lifecycle.\nUse when implementing data retention, handling GDPR requests,\nor managing data lifecycle in Delta Lake.\nTrigger with phrases like \"databricks GDPR\", \"databricks PII\",\n\"databricks data retention\", \"databricks data lifecycle\", \"delete user data\".\n","Read, Write, Edit, Bash(databricks:*)","main","90b50ed0778745a80fd5bfb13b1843d45b9e82f7","tests\u002Fext_conformance\u002Fartifacts\u002Fplugins-community\u002Fplugins\u002Fsaas-packs\u002Fdatabricks-pack\u002Fskills\u002Fdatabricks-data-handling",233,"https:\u002F\u002Fgithub.com\u002FDicklesworthstone\u002Fpi_agent_rust","https:\u002F\u002Fgithub.com\u002FDicklesworthstone","https:\u002F\u002Fwww.jeffreyemanuel.com\u002F","Building in NY","2026-11-15 01:54:15",{"source":8,"skillName":5,"commands":67,"prompt":77,"zipAvailable":78},[68,71,74],{"tool":69,"command":70},"npx","npx skills add Dicklesworthstone\u002Fpi_agent_rust --skill databricks-data-handling",{"tool":72,"command":73},"bunx","bunx skills add Dicklesworthstone\u002Fpi_agent_rust --skill databricks-data-handling",{"tool":75,"command":76},"pnpm","pnpm dlx skills add Dicklesworthstone\u002Fpi_agent_rust --skill databricks-data-handling","安装技能，技能地址：https:\u002F\u002Fgithub.com\u002FDicklesworthstone\u002Fpi_agent_rust\u002Fblob\u002Fmain\u002Ftests\u002Fext_conformance\u002Fartifacts\u002Fplugins-community\u002Fplugins\u002Fsaas-packs\u002Fdatabricks-pack\u002Fskills\u002Fdatabricks-data-handling\u002FSKILL.md",true,[],[81,114,149,172,197,224],{"skillNo":82,"name":83,"nameZh":6,"description":84,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":12,"repositoryNo":13,"starsCount":14,"licenseSpdxId":85,"licenseFileUrl":86,"sourceHtmlUrl":87,"categories":88,"keywords":94,"publishedAt":113},"01M6FCR7G5A005P364Z6ZWCJDP","theme-factory","Toolkit for styling artifacts with a theme. These artifacts can be slides, docs, reportings, HTML landing pages, etc. There are 10 pre-set themes with colors\u002Ffonts that you can apply to any artifact that has been creating, or can generate a new theme on-the-fly.","Apache-2.0","https:\u002F\u002Fgithub.com\u002FDicklesworthstone\u002Fpi_agent_rust\u002Fblob\u002Fmain\u002Ftests\u002Fext_conformance\u002Fartifacts\u002Ftemplates-davila7\u002Fcli-tool\u002Fcomponents\u002Fskills\u002Fcreative-design\u002Ftheme-factory\u002FLICENSE.txt","https:\u002F\u002Fgithub.com\u002FDicklesworthstone\u002Fpi_agent_rust\u002Fblob\u002Fmain\u002Ftests\u002Fext_conformance\u002Fartifacts\u002Ftemplates-davila7\u002Fcli-tool\u002Fcomponents\u002Fskills\u002Fcreative-design\u002Ftheme-factory\u002FSKILL.md",[89],{"categoryNo":90,"categoryName":91,"taxonomyType":92,"taxonomyTypeDesc":93},"01JBSEEDCAT000000000000015","视觉设计","SCENARIO","使用场景",[95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,30,111,112],"theme factory","factory","theme","docs","toolkit","generate","artifacts","SET","fly","slides","landing","html","styling","artifact","pages","themes","you","fonts","2026-11-11 15:11:14",{"skillNo":115,"name":116,"nameZh":6,"description":117,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":12,"repositoryNo":13,"starsCount":14,"licenseSpdxId":85,"licenseFileUrl":118,"sourceHtmlUrl":119,"categories":120,"keywords":124,"publishedAt":48},"01M6FEGPPK0ZTKCCM0R3E3KS33","web-artifacts-builder","Suite of tools for creating elaborate, multi-component claude.ai HTML artifacts using modern frontend web technologies (React, Tailwind CSS, shadcn\u002Fui). Use for complex artifacts requiring state management, routing, or shadcn\u002Fui components - not for simple single-file HTML\u002FJSX artifacts.","https:\u002F\u002Fgithub.com\u002FDicklesworthstone\u002Fpi_agent_rust\u002Fblob\u002Fmain\u002Ftests\u002Fext_conformance\u002Fartifacts\u002Ftemplates-davila7\u002Fcli-tool\u002Fcomponents\u002Fskills\u002Futilities\u002Fweb-artifacts-builder\u002FLICENSE.txt","https:\u002F\u002Fgithub.com\u002FDicklesworthstone\u002Fpi_agent_rust\u002Fblob\u002Fmain\u002Ftests\u002Fext_conformance\u002Fartifacts\u002Ftemplates-davila7\u002Fcli-tool\u002Fcomponents\u002Fskills\u002Futilities\u002Fweb-artifacts-builder\u002FSKILL.md",[121],{"categoryNo":122,"categoryName":123,"taxonomyType":92,"taxonomyTypeDesc":93},"01JBSEEDCAT000000000000009","前端开发",[125,126,101,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,38,106,142,143,144,145,146,147,148],"web artifacts builder","web","artifacts builder","builder","frontend","css","components","react","component","routing","state management","state","shadcn","file","ui","Tools","Tailwind","single-file","simple","ui components","multi","for","jsx","suite",{"skillNo":150,"name":151,"nameZh":6,"description":152,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":12,"repositoryNo":13,"starsCount":14,"licenseSpdxId":85,"licenseFileUrl":153,"sourceHtmlUrl":154,"categories":155,"keywords":160,"publishedAt":48},"01M6FEE9VTK1W4GY21AKREJXRG","webapp-testing","Toolkit for interacting with and testing local web applications using Playwright. Supports verifying frontend functionality, debugging UI behavior, capturing browser screenshots, and viewing browser logs.","https:\u002F\u002Fgithub.com\u002FDicklesworthstone\u002Fpi_agent_rust\u002Fblob\u002Fmain\u002Ftests\u002Fext_conformance\u002Fartifacts\u002Ftemplates-davila7\u002Fcli-tool\u002Fcomponents\u002Fskills\u002Fsecurity\u002Fwebapp-testing\u002FLICENSE.txt","https:\u002F\u002Fgithub.com\u002FDicklesworthstone\u002Fpi_agent_rust\u002Fblob\u002Fmain\u002Ftests\u002Fext_conformance\u002Fartifacts\u002Ftemplates-davila7\u002Fcli-tool\u002Fcomponents\u002Fskills\u002Fsecurity\u002Fwebapp-testing\u002FSKILL.md",[156,159],{"categoryNo":157,"categoryName":158,"taxonomyType":92,"taxonomyTypeDesc":93},"01JBSEEDCAT000000000000008","测试与质量",{"categoryNo":122,"categoryName":123,"taxonomyType":92,"taxonomyTypeDesc":93},[161,162,163,129,164,165,126,166,99,167,139,168,169,170,171,30,146],"testing","webapp testing","webapp","screenshots","playwright","debugging","browser","logs","applications","behavior","local",{"skillNo":173,"name":174,"nameZh":6,"description":175,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":12,"repositoryNo":13,"starsCount":14,"licenseSpdxId":15,"licenseFileUrl":176,"sourceHtmlUrl":177,"categories":178,"keywords":179,"publishedAt":48},"01M6FE20F7ZTQBFWKF4MPJ2SBX","neuropixels-analysis","Neuropixels neural recording analysis. Load SpikeGLX\u002FOpenEphys data, preprocess, motion correction, Kilosort4 spike sorting, quality metrics, Allen\u002FIBL curation, AI-assisted visual analysis, for Neuropixels 1.0\u002F2.0 extracellular electrophysiology. Use when working with neural recordings, spike sorting, extracellular electrophysiology, or when the user mentions Neuropixels, SpikeGLX, Open Ephys, Kilosort, quality metrics, or unit curation.","https:\u002F\u002Fgithub.com\u002FDicklesworthstone\u002Fpi_agent_rust\u002Fblob\u002Fmain\u002Ftests\u002Fext_conformance\u002Fartifacts\u002Ftemplates-davila7\u002Fcli-tool\u002Fcomponents\u002Fskills\u002Fscientific\u002Fneuropixels-analysis\u002FLICENSE.txt","https:\u002F\u002Fgithub.com\u002FDicklesworthstone\u002Fpi_agent_rust\u002Fblob\u002Fmain\u002Ftests\u002Fext_conformance\u002Fartifacts\u002Ftemplates-davila7\u002Fcli-tool\u002Fcomponents\u002Fskills\u002Fscientific\u002Fneuropixels-analysis\u002FSKILL.md",[],[180,181,182,183,26,184,185,186,38,187,188,189,190,191,192,193,30,194,195,196,146,27],"analysis","neuropixels analysis","spike","metrics","open","quality","user","recordings","Neural","motion","visual","quality metrics","unit","sorting","curation","recording","load",{"skillNo":198,"name":199,"nameZh":6,"description":200,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":12,"repositoryNo":13,"starsCount":14,"licenseSpdxId":85,"licenseFileUrl":201,"sourceHtmlUrl":202,"categories":203,"keywords":207,"publishedAt":48},"01M6FDP0PDK6F1GGX395C7625Z","skill-creator","Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Claude's capabilities with specialized knowledge, workflows, or tool integrations.","https:\u002F\u002Fgithub.com\u002FDicklesworthstone\u002Fpi_agent_rust\u002Fblob\u002Fmain\u002Ftests\u002Fext_conformance\u002Fartifacts\u002Ftemplates-davila7\u002Fcli-tool\u002Fcomponents\u002Fskills\u002Fproductivity\u002Fskill-creator\u002FLICENSE.txt","https:\u002F\u002Fgithub.com\u002FDicklesworthstone\u002Fpi_agent_rust\u002Fblob\u002Fmain\u002Ftests\u002Fext_conformance\u002Fartifacts\u002Ftemplates-davila7\u002Fcli-tool\u002Fcomponents\u002Fskills\u002Fproductivity\u002Fskill-creator\u002FSKILL.md",[204],{"categoryNo":205,"categoryName":206,"taxonomyType":92,"taxonomyTypeDesc":93},"01JBSEEDCAT000000000000023","Skill与智能体开发",[208,209,210,211,212,213,214,215,216,217,218,219,220,221,30,146,222,27,223],"skill creator","creator","workflows","create","update","capabilities","knowledge","tool","integrations","specialized","guide","new skill","existing skill","users","Existing","new",{"skillNo":225,"name":226,"nameZh":6,"description":227,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":12,"repositoryNo":13,"starsCount":14,"licenseSpdxId":15,"licenseFileUrl":228,"sourceHtmlUrl":229,"categories":230,"keywords":232,"publishedAt":48},"01M6FDM8Y7VK7NYG9SM7QPA3PW","notebooklm","Use this skill to query your Google NotebookLM notebooks directly from Claude Code for source-grounded, citation-backed answers from Gemini. Browser automation, library management, persistent auth. Drastically reduced hallucinations through document-only responses.","https:\u002F\u002Fgithub.com\u002FDicklesworthstone\u002Fpi_agent_rust\u002Fblob\u002Fmain\u002Ftests\u002Fext_conformance\u002Fartifacts\u002Ftemplates-davila7\u002Fcli-tool\u002Fcomponents\u002Fskills\u002Fproductivity\u002Fnotebooklm\u002FLICENSE","https:\u002F\u002Fgithub.com\u002FDicklesworthstone\u002Fpi_agent_rust\u002Fblob\u002Fmain\u002Ftests\u002Fext_conformance\u002Fartifacts\u002Ftemplates-davila7\u002Fcli-tool\u002Fcomponents\u002Fskills\u002Fproductivity\u002Fnotebooklm\u002FSKILL.md",[231],{"categoryNo":205,"categoryName":206,"taxonomyType":92,"taxonomyTypeDesc":93},[226,233,234,235,236,237,238,167,239,240,38,241,242,243,244,146,245],"google","browser automation","gemini","automation","auth","code","library","management","only","document","query","Citation","source"]