[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"skill:01M6CWJNKWAQM7K2W99Y277NFS":3,"skill-same:01M6CWJNKWAQM7K2W99Y277NFS":51},{"skillNo":4,"name":5,"nameZh":6,"description":7,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":15,"categories":16,"keywords":17,"publishedAt":23,"linkOnly":24,"markdownContent":25,"markdownBody":26,"frontMatter":27,"refName":31,"commitSha":32,"skillDirectory":5,"forksCount":33,"repositoryHtmlUrl":34,"repositoryHomepageUrl":6,"ownerHtmlUrl":35,"ownerBlogUrl":6,"ownerDescription":6,"updatedAt":36,"install":37,"resources":50},"01M6CWJNKWAQM7K2W99Y277NFS","bulk-operations",null,"Bulk data processing and operations","ffsshhttiikk\u002Fopencode-agents-skills","ffsshhttiikk","https:\u002F\u002Favatars.githubusercontent.com\u002Fu\u002F226346827?v=4","01M6CWD9RR3Y7VRQX3MXYC8FYH","01M6CWD9WVPYAZW0J82G1VKGHY",3,"MIT","https:\u002F\u002Fgithub.com\u002Fffsshhttiikk\u002Fopencode-agents-skills\u002Fblob\u002Fmain\u002Fbulk-operations\u002FSKILL.md",[],[18,19,20,21,22],"operations","bulk operations","data","data processing","opencode","2026-11-10 10:12:30",false,"---\nname: bulk-operations\ndescription: Bulk data processing and operations\nlicense: MIT\ncompatibility: opencode\nmetadata:\n  audience: developers\n  category: utilities\n---\n## What I do\n- Implement bulk database operations\n- Process large datasets efficiently\n- Handle batch processing\n- Implement progress tracking\n- Handle errors in bulk operations\n- Optimize bulk imports\u002Fexports\n- Manage memory for large files\n- Implement parallel processing\n\n## When to use me\nWhen implementing bulk data operations or processing large datasets.\n\n## Bulk Database Operations\n```python\nfrom typing import List, Dict, Any, Callable\nfrom dataclasses import dataclass\nfrom contextlib import contextmanager\nimport itertools\n\n\n@dataclass\nclass BulkOperationResult:\n    \"\"\"Result of bulk operation.\"\"\"\n    total: int\n    successful: int\n    failed: int\n    errors: List[Dict[str, Any]]\n    duration_seconds: float\n\n\nclass BulkDatabaseOperations:\n    \"\"\"Bulk database operations manager.\"\"\"\n    \n    def __init__(self, db_session):\n        self.session = db_session\n    \n    @contextmanager\n    def bulk_insert(\n        self,\n        batch_size: int = 1000\n    ):\n        \"\"\"Context manager for bulk inserts.\"\"\"\n        objects = []\n        \n        try:\n            yield objects\n        finally:\n            if objects:\n                self._flush_insert(objects)\n                objects.clear()\n    \n    def _flush_insert(self, objects: List[Any]) -> int:\n        \"\"\"Flush pending inserts to database.\"\"\"\n        self.session.add_all(objects)\n        self.session.commit()\n        return len(objects)\n    \n    def bulk_insert_records(\n        self,\n        model_class,\n        records: List[Dict[str, Any]]\n    ) -> BulkOperationResult:\n        \"\"\"Bulk insert records efficiently.\"\"\"\n        start_time = datetime.utcnow()\n        successful = 0\n        failed = 0\n        errors = []\n        \n        # Process in batches\n        batch_size = 1000\n        \n        for i in range(0, len(records), batch_size):\n            batch = records[i:i + batch_size]\n            \n            try:\n                self.session.bulk_insert_mappings(\n                    model_class,\n                    batch\n                )\n                self.session.commit()\n                successful += len(batch)\n            except Exception as e:\n                # Rollback and retry individually\n                self.session.rollback()\n                \n                for record in batch:\n                    try:\n                        obj = model_class(**record)\n                        self.session.add(obj)\n                        self.session.commit()\n                        successful += 1\n                    except Exception as record_error:\n                        failed += 1\n                        errors.append({\n                            \"record\": record,\n                            \"error\": str(record_error)\n                        })\n        \n        return BulkOperationResult(\n            total=len(records),\n            successful=successful,\n            failed=failed,\n            errors=errors,\n            duration_seconds=(datetime.utcnow() - start_time).total_seconds()\n        )\n    \n    def bulk_update(\n        self,\n        model_class,\n        updates: List[Dict[str, Any]],\n        match_field: str = \"id\"\n    ) -> BulkOperationResult:\n        \"\"\"Bulk update records efficiently.\"\"\"\n        start_time = datetime.utcnow()\n        successful = 0\n        failed = 0\n        errors = []\n        \n        for update in updates:\n            try:\n                match_value = update.pop(match_field)\n                \n                self.session.query(model_class).filter(\n                    getattr(model_class, match_field) == match_value\n                ).update(update, synchronize_session=False)\n                \n                successful += 1\n            except Exception as e:\n                failed += 1\n                errors.append({\n                    \"update\": update,\n                    \"error\": str(e)\n                })\n        \n        self.session.commit()\n        \n        return BulkOperationResult(\n            total=len(updates),\n            successful=successful,\n            failed=failed,\n            errors=errors,\n            duration_seconds=(datetime.utcnow() - start_time).total_seconds()\n        )\n    \n    def bulk_upsert(\n        self,\n        model_class,\n        records: List[Dict[str, Any]],\n        unique_fields: List[str]\n    ) -> BulkOperationResult:\n        \"\"\"Bulk upsert (insert or update) records.\"\"\"\n        start_time = datetime.utcnow()\n        successful = 0\n        failed = 0\n        errors = []\n        \n        for record in records:\n            try:\n                # Build lookup query\n                filters = {\n                    field: record.get(field)\n                    for field in unique_fields\n                }\n                \n                existing = self.session.query(model_class).filter_by(**filters).first()\n                \n                if existing:\n                    # Update\n                    for key, value in record.items():\n                        setattr(existing, key, value)\n                else:\n                    # Insert\n                    obj = model_class(**record)\n                    self.session.add(obj)\n                \n                successful += 1\n            except Exception as e:\n                failed += 1\n                errors.append({\n                    \"record\": record,\n                    \"error\": str(e)\n                })\n        \n        self.session.commit()\n        \n        return BulkOperationResult(\n            total=len(records),\n            successful=successful,\n            failed=failed,\n            errors=errors,\n            duration_seconds=(datetime.utcnow() - start_time).total_seconds()\n        )\n```\n\n## Chunked Processing\n```python\nfrom typing import Iterator, Callable, List\nfrom dataclasses import dataclass\nimport asyncio\n\n\n@dataclass\nclass ProcessingProgress:\n    \"\"\"Progress of chunked processing.\"\"\"\n    total_items: int\n    processed_items: int\n    failed_items: int\n    current_chunk: int\n    total_chunks: int\n    percent_complete: float\n    elapsed_seconds: float\n    estimated_remaining_seconds: float\n    \n    def to_dict(self) -> dict:\n        return {\n            \"total\": self.total_items,\n            \"processed\": self.processed_items,\n            \"failed\": self.failed_items,\n            \"percent\": self.percent_complete,\n            \"elapsed\": f\"{self.elapsed_seconds:.2f}s\",\n            \"remaining\": f\"{self.estimated_remaining_seconds:.2f}s\",\n        }\n\n\nclass ChunkedProcessor:\n    \"\"\"Process data in chunks with progress tracking.\"\"\"\n    \n    def __init__(\n        self,\n        chunk_size: int = 1000,\n        progress_callback: Callable[[ProcessingProgress], None] = None\n    ):\n        self.chunk_size = chunk_size\n        self.progress_callback = progress_callback\n        self.start_time = None\n    \n    def process_in_chunks(\n        self,\n        items: List[Any],\n        process_func: Callable[[List[Any]], List[Any]],\n        max_chunks: int = None\n    ) -> tuple[List[Any], ProcessingProgress]:\n        \"\"\"Process items in chunks.\"\"\"\n        self.start_time = datetime.utcnow()\n        \n        total_items = len(items)\n        total_chunks = (total_items + self.chunk_size - 1) \u002F\u002F self.chunk_size\n        \n        if max_chunks:\n            total_chunks = min(total_chunks, max_chunks)\n        \n        processed_items = 0\n        failed_items = 0\n        all_results = []\n        \n        for chunk_idx in range(total_chunks):\n            start_idx = chunk_idx * self.chunk_size\n            end_idx = min(start_idx + self.chunk_size, total_items)\n            chunk = items[start_idx:end_idx]\n            \n            try:\n                results = process_func(chunk)\n                all_results.extend(results)\n                processed_items += len(results)\n            except Exception as e:\n                failed_items += len(chunk)\n                # Log error but continue\n                print(f\"Error processing chunk {chunk_idx}: {e}\")\n            \n            # Report progress\n            progress = self._calculate_progress(\n                total_items,\n                processed_items,\n                failed_items,\n                chunk_idx + 1,\n                total_chunks\n            )\n            \n            if self.progress_callback:\n                self.progress_callback(progress)\n        \n        return all_results, progress\n    \n    def _calculate_progress(\n        self,\n        total_items: int,\n        processed_items: int,\n        failed_items: int,\n        current_chunk: int,\n        total_chunks: int\n    ) -> ProcessingProgress:\n        \"\"\"Calculate current progress.\"\"\"\n        elapsed = (datetime.utcnow() - self.start_time).total_seconds()\n        \n        items_processed = processed_items + failed_items\n        percent_complete = (items_processed \u002F total_items * 100) if total_items else 0\n        \n        if elapsed > 0 and processed_items > 0:\n            items_per_second = processed_items \u002F elapsed\n            remaining_items = total_items - items_processed\n            estimated_remaining = remaining_items \u002F items_per_second\n        else:\n            estimated_remaining = 0\n        \n        return ProcessingProgress(\n            total_items=total_items,\n            processed_items=processed_items,\n            failed_items=failed_items,\n            current_chunk=current_chunk,\n            total_chunks=total_chunks,\n            percent_complete=percent_complete,\n            elapsed_seconds=elapsed,\n            estimated_remaining_seconds=estimated_remaining\n        )\n\n\n# Streaming iterator for large files\ndef stream_large_file(\n    file_path: str,\n    chunk_size: int = 10000,\n    parse_func: Callable[[str], Dict] = None\n) -> Iterator[List[Dict]]:\n    \"\"\"Stream large file in chunks.\"\"\"\n    parse_func = parse_func or json.loads\n    \n    with open(file_path, 'r', encoding='utf-8') as f:\n        chunk = []\n        \n        for line in f:\n            parsed = parse_func(line.strip())\n            chunk.append(parsed)\n            \n            if len(chunk) >= chunk_size:\n                yield chunk\n                chunk = []\n        \n        # Yield remaining items\n        if chunk:\n            yield chunk\n```\n\n## Parallel Processing\n```python\nimport multiprocessing\nfrom concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor\nfrom typing import List, Callable\n\n\nclass ParallelProcessor:\n    \"\"\"Parallel processing with configurable workers.\"\"\"\n    \n    def __init__(\n        self,\n        max_workers: int = None,\n        use_processes: bool = False\n    ):\n        self.max_workers = max_workers or multiprocessing.cpu_count()\n        self.use_processes = use_processes\n    \n    def execute(\n        self,\n        func: Callable,\n        items: List[Any],\n        progress_callback: Callable[[int, int], None] = None\n    ) -> List[Any]:\n        \"\"\"Execute function on items in parallel.\"\"\"\n        ExecutorClass = (\n            ProcessPoolExecutor\n            if self.use_processes\n            else ThreadPoolExecutor\n        )\n        \n        with ExecutorClass(max_workers=self.max_workers) as executor:\n            futures = {\n                executor.submit(func, item): item\n                for item in items\n            }\n            \n            results = []\n            for i, future in enumerate(\n                as_completed(futures.keys()),\n                1\n            ):\n                try:\n                    result = future.result()\n                    results.append(result)\n                except Exception as e:\n                    item = futures[future]\n                    print(f\"Error processing {item}: {e}\")\n                \n                if progress_callback:\n                    progress_callback(i, len(items))\n            \n            return results\n    \n    def map_chunks(\n        self,\n        func: Callable,\n        chunks: List[List[Any]],\n        progress_callback: Callable[[int, int], None] = None\n    ) -> List[Any]:\n        \"\"\"Process chunks in parallel.\"\"\"\n        with ThreadPoolExecutor(max_workers=self.max_workers) as executor:\n            futures = {\n                executor.submit(func, chunk): idx\n                for idx, chunk in enumerate(chunks)\n            }\n            \n            results = [None] * len(chunks)\n            \n            for future in as_completed(futures.keys()):\n                idx = futures[future]\n                try:\n                    results[idx] = future.result()\n                except Exception as e:\n                    print(f\"Error processing chunk {idx}: {e}\")\n                    results[idx] = None\n                \n                if progress_callback:\n                    progress_callback(idx + 1, len(chunks))\n            \n            return results\n```\n\n## Batch Processing Queue\n```python\nfrom dataclasses import dataclass\nfrom typing import Callable, Any\nfrom datetime import datetime\nimport json\n\n\n@dataclass\nclass BatchJob:\n    \"\"\"Batch job configuration.\"\"\"\n    job_id: str\n    job_type: str\n    payload: dict\n    status: str = \"pending\"\n    created_at: datetime = None\n    started_at: datetime = None\n    completed_at: datetime = None\n    result: Any = None\n    error: str = None\n    retry_count: int = 0\n    max_retries: int = 3\n\n\nclass BatchJobProcessor:\n    \"\"\"Process batch jobs from queue.\"\"\"\n    \n    def __init__(self, queue, storage, max_concurrent: int = 5):\n        self.queue = queue\n        self.storage = storage\n        self.max_concurrent = max_concurrent\n        self.active_jobs = set()\n    \n    async def process_queue(self) -> None:\n        \"\"\"Process jobs from queue.\"\"\"\n        while True:\n            # Get next job\n            job_data = await self.queue.get()\n            \n            if job_data is None:\n                break\n            \n            job = BatchJob(**job_data)\n            \n            # Check concurrent limit\n            while len(self.active_jobs) >= self.max_concurrent:\n                await asyncio.sleep(1)\n            \n            # Process job\n            self.active_jobs.add(job.job_id)\n            \n            try:\n                result = await self._execute_job(job)\n                \n                job.status = \"completed\"\n                job.result = result\n                job.completed_at = datetime.utcnow()\n                \n                await self.storage.save_job(job)\n                \n            except Exception as e:\n                job.retry_count += 1\n                \n                if job.retry_count >= job.max_retries:\n                    job.status = \"failed\"\n                    job.error = str(e)\n                    job.completed_at = datetime.utcnow()\n                else:\n                    # Requeue with backoff\n                    job.status = \"pending\"\n                    await self.queue.put(job.to_dict())\n                \n                await self.storage.save_job(job)\n            \n            finally:\n                self.active_jobs.discard(job.job_id)\n    \n    async def _execute_job(self, job: BatchJob) -> Any:\n        \"\"\"Execute a single job.\"\"\"\n        job_handlers = {\n            \"import\": self._handle_import,\n            \"export\": self._handle_export,\n            \"process\": self._handle_process,\n        }\n        \n        handler = job_handlers.get(job.job_type)\n        \n        if not handler:\n            raise ValueError(f\"Unknown job type: {job.job_type}\")\n        \n        return await handler(job.payload)\n    \n    async def _handle_import(self, payload: dict) -> dict:\n        \"\"\"Handle import job.\"\"\"\n        # Import logic\n        return {\"imported\": 100}\n    \n    async def _handle_export(self, payload: dict) -> dict:\n        \"\"\"Handle export job.\"\"\"\n        # Export logic\n        return {\"exported\": 100}\n    \n    async def _handle_process(self, payload: dict) -> dict:\n        \"\"\"Handle processing job.\"\"\"\n        # Processing logic\n        return {\"processed\": 100}\n```\n\n## Best Practices\n```\nBulk Operations Best Practices:\n\n1. Use batch operations\n   Bulk inserts instead of individual\n   Reduces round trips\n\n2. Process in chunks\n   Memory efficient\n   Progress tracking\n\n3. Handle errors gracefully\n   Skip failed items\n   Log for review\n\n4. Monitor progress\n   Real-time updates\n   ETA calculations\n\n5. Use transactions appropriately\n   Commit frequently\n   Balance performance\n\n6. Parallelize I\u002FO bound\n   Thread pool for I\u002FO\n   Process pool for CPU\n\n7. Queue for reliability\n   Persistent queue\n   Retry with backoff\n\n8. Validate input\n   Check before processing\n   Sanitize data\n\n9. Resource limits\n   Memory constraints\n   Timeout handling\n\n10. Clean up after\n     Temporary files\n     Progress markers\n```\n","## What I do\n- Implement bulk database operations\n- Process large datasets efficiently\n- Handle batch processing\n- Implement progress tracking\n- Handle errors in bulk operations\n- Optimize bulk imports\u002Fexports\n- Manage memory for large files\n- Implement parallel processing\n\n## When to use me\nWhen implementing bulk data operations or processing large datasets.\n\n## Bulk Database Operations\n```python\nfrom typing import List, Dict, Any, Callable\nfrom dataclasses import dataclass\nfrom contextlib import contextmanager\nimport itertools\n\n\n@dataclass\nclass BulkOperationResult:\n    \"\"\"Result of bulk operation.\"\"\"\n    total: int\n    successful: int\n    failed: int\n    errors: List[Dict[str, Any]]\n    duration_seconds: float\n\n\nclass BulkDatabaseOperations:\n    \"\"\"Bulk database operations manager.\"\"\"\n    \n    def __init__(self, db_session):\n        self.session = db_session\n    \n    @contextmanager\n    def bulk_insert(\n        self,\n        batch_size: int = 1000\n    ):\n        \"\"\"Context manager for bulk inserts.\"\"\"\n        objects = []\n        \n        try:\n            yield objects\n        finally:\n            if objects:\n                self._flush_insert(objects)\n                objects.clear()\n    \n    def _flush_insert(self, objects: List[Any]) -> int:\n        \"\"\"Flush pending inserts to database.\"\"\"\n        self.session.add_all(objects)\n        self.session.commit()\n        return len(objects)\n    \n    def bulk_insert_records(\n        self,\n        model_class,\n        records: List[Dict[str, Any]]\n    ) -> BulkOperationResult:\n        \"\"\"Bulk insert records efficiently.\"\"\"\n        start_time = datetime.utcnow()\n        successful = 0\n        failed = 0\n        errors = []\n        \n        # Process in batches\n        batch_size = 1000\n        \n        for i in range(0, len(records), batch_size):\n            batch = records[i:i + batch_size]\n            \n            try:\n                self.session.bulk_insert_mappings(\n                    model_class,\n                    batch\n                )\n                self.session.commit()\n                successful += len(batch)\n            except Exception as e:\n                # Rollback and retry individually\n                self.session.rollback()\n                \n                for record in batch:\n                    try:\n                        obj = model_class(**record)\n                        self.session.add(obj)\n                        self.session.commit()\n                        successful += 1\n                    except Exception as record_error:\n                        failed += 1\n                        errors.append({\n                            \"record\": record,\n                            \"error\": str(record_error)\n                        })\n        \n        return BulkOperationResult(\n            total=len(records),\n            successful=successful,\n            failed=failed,\n            errors=errors,\n            duration_seconds=(datetime.utcnow() - start_time).total_seconds()\n        )\n    \n    def bulk_update(\n        self,\n        model_class,\n        updates: List[Dict[str, Any]],\n        match_field: str = \"id\"\n    ) -> BulkOperationResult:\n        \"\"\"Bulk update records efficiently.\"\"\"\n        start_time = datetime.utcnow()\n        successful = 0\n        failed = 0\n        errors = []\n        \n        for update in updates:\n            try:\n                match_value = update.pop(match_field)\n                \n                self.session.query(model_class).filter(\n                    getattr(model_class, match_field) == match_value\n                ).update(update, synchronize_session=False)\n                \n                successful += 1\n            except Exception as e:\n                failed += 1\n                errors.append({\n                    \"update\": update,\n                    \"error\": str(e)\n                })\n        \n        self.session.commit()\n        \n        return BulkOperationResult(\n            total=len(updates),\n            successful=successful,\n            failed=failed,\n            errors=errors,\n            duration_seconds=(datetime.utcnow() - start_time).total_seconds()\n        )\n    \n    def bulk_upsert(\n        self,\n        model_class,\n        records: List[Dict[str, Any]],\n        unique_fields: List[str]\n    ) -> BulkOperationResult:\n        \"\"\"Bulk upsert (insert or update) records.\"\"\"\n        start_time = datetime.utcnow()\n        successful = 0\n        failed = 0\n        errors = []\n        \n        for record in records:\n            try:\n                # Build lookup query\n                filters = {\n                    field: record.get(field)\n                    for field in unique_fields\n                }\n                \n                existing = self.session.query(model_class).filter_by(**filters).first()\n                \n                if existing:\n                    # Update\n                    for key, value in record.items():\n                        setattr(existing, key, value)\n                else:\n                    # Insert\n                    obj = model_class(**record)\n                    self.session.add(obj)\n                \n                successful += 1\n            except Exception as e:\n                failed += 1\n                errors.append({\n                    \"record\": record,\n                    \"error\": str(e)\n                })\n        \n        self.session.commit()\n        \n        return BulkOperationResult(\n            total=len(records),\n            successful=successful,\n            failed=failed,\n            errors=errors,\n            duration_seconds=(datetime.utcnow() - start_time).total_seconds()\n        )\n```\n\n## Chunked Processing\n```python\nfrom typing import Iterator, Callable, List\nfrom dataclasses import dataclass\nimport asyncio\n\n\n@dataclass\nclass ProcessingProgress:\n    \"\"\"Progress of chunked processing.\"\"\"\n    total_items: int\n    processed_items: int\n    failed_items: int\n    current_chunk: int\n    total_chunks: int\n    percent_complete: float\n    elapsed_seconds: float\n    estimated_remaining_seconds: float\n    \n    def to_dict(self) -> dict:\n        return {\n            \"total\": self.total_items,\n            \"processed\": self.processed_items,\n            \"failed\": self.failed_items,\n            \"percent\": self.percent_complete,\n            \"elapsed\": f\"{self.elapsed_seconds:.2f}s\",\n            \"remaining\": f\"{self.estimated_remaining_seconds:.2f}s\",\n        }\n\n\nclass ChunkedProcessor:\n    \"\"\"Process data in chunks with progress tracking.\"\"\"\n    \n    def __init__(\n        self,\n        chunk_size: int = 1000,\n        progress_callback: Callable[[ProcessingProgress], None] = None\n    ):\n        self.chunk_size = chunk_size\n        self.progress_callback = progress_callback\n        self.start_time = None\n    \n    def process_in_chunks(\n        self,\n        items: List[Any],\n        process_func: Callable[[List[Any]], List[Any]],\n        max_chunks: int = None\n    ) -> tuple[List[Any], ProcessingProgress]:\n        \"\"\"Process items in chunks.\"\"\"\n        self.start_time = datetime.utcnow()\n        \n        total_items = len(items)\n        total_chunks = (total_items + self.chunk_size - 1) \u002F\u002F self.chunk_size\n        \n        if max_chunks:\n            total_chunks = min(total_chunks, max_chunks)\n        \n        processed_items = 0\n        failed_items = 0\n        all_results = []\n        \n        for chunk_idx in range(total_chunks):\n            start_idx = chunk_idx * self.chunk_size\n            end_idx = min(start_idx + self.chunk_size, total_items)\n            chunk = items[start_idx:end_idx]\n            \n            try:\n                results = process_func(chunk)\n                all_results.extend(results)\n                processed_items += len(results)\n            except Exception as e:\n                failed_items += len(chunk)\n                # Log error but continue\n                print(f\"Error processing chunk {chunk_idx}: {e}\")\n            \n            # Report progress\n            progress = self._calculate_progress(\n                total_items,\n                processed_items,\n                failed_items,\n                chunk_idx + 1,\n                total_chunks\n            )\n            \n            if self.progress_callback:\n                self.progress_callback(progress)\n        \n        return all_results, progress\n    \n    def _calculate_progress(\n        self,\n        total_items: int,\n        processed_items: int,\n        failed_items: int,\n        current_chunk: int,\n        total_chunks: int\n    ) -> ProcessingProgress:\n        \"\"\"Calculate current progress.\"\"\"\n        elapsed = (datetime.utcnow() - self.start_time).total_seconds()\n        \n        items_processed = processed_items + failed_items\n        percent_complete = (items_processed \u002F total_items * 100) if total_items else 0\n        \n        if elapsed > 0 and processed_items > 0:\n            items_per_second = processed_items \u002F elapsed\n            remaining_items = total_items - items_processed\n            estimated_remaining = remaining_items \u002F items_per_second\n        else:\n            estimated_remaining = 0\n        \n        return ProcessingProgress(\n            total_items=total_items,\n            processed_items=processed_items,\n            failed_items=failed_items,\n            current_chunk=current_chunk,\n            total_chunks=total_chunks,\n            percent_complete=percent_complete,\n            elapsed_seconds=elapsed,\n            estimated_remaining_seconds=estimated_remaining\n        )\n\n\n# Streaming iterator for large files\ndef stream_large_file(\n    file_path: str,\n    chunk_size: int = 10000,\n    parse_func: Callable[[str], Dict] = None\n) -> Iterator[List[Dict]]:\n    \"\"\"Stream large file in chunks.\"\"\"\n    parse_func = parse_func or json.loads\n    \n    with open(file_path, 'r', encoding='utf-8') as f:\n        chunk = []\n        \n        for line in f:\n            parsed = parse_func(line.strip())\n            chunk.append(parsed)\n            \n            if len(chunk) >= chunk_size:\n                yield chunk\n                chunk = []\n        \n        # Yield remaining items\n        if chunk:\n            yield chunk\n```\n\n## Parallel Processing\n```python\nimport multiprocessing\nfrom concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor\nfrom typing import List, Callable\n\n\nclass ParallelProcessor:\n    \"\"\"Parallel processing with configurable workers.\"\"\"\n    \n    def __init__(\n        self,\n        max_workers: int = None,\n        use_processes: bool = False\n    ):\n        self.max_workers = max_workers or multiprocessing.cpu_count()\n        self.use_processes = use_processes\n    \n    def execute(\n        self,\n        func: Callable,\n        items: List[Any],\n        progress_callback: Callable[[int, int], None] = None\n    ) -> List[Any]:\n        \"\"\"Execute function on items in parallel.\"\"\"\n        ExecutorClass = (\n            ProcessPoolExecutor\n            if self.use_processes\n            else ThreadPoolExecutor\n        )\n        \n        with ExecutorClass(max_workers=self.max_workers) as executor:\n            futures = {\n                executor.submit(func, item): item\n                for item in items\n            }\n            \n            results = []\n            for i, future in enumerate(\n                as_completed(futures.keys()),\n                1\n            ):\n                try:\n                    result = future.result()\n                    results.append(result)\n                except Exception as e:\n                    item = futures[future]\n                    print(f\"Error processing {item}: {e}\")\n                \n                if progress_callback:\n                    progress_callback(i, len(items))\n            \n            return results\n    \n    def map_chunks(\n        self,\n        func: Callable,\n        chunks: List[List[Any]],\n        progress_callback: Callable[[int, int], None] = None\n    ) -> List[Any]:\n        \"\"\"Process chunks in parallel.\"\"\"\n        with ThreadPoolExecutor(max_workers=self.max_workers) as executor:\n            futures = {\n                executor.submit(func, chunk): idx\n                for idx, chunk in enumerate(chunks)\n            }\n            \n            results = [None] * len(chunks)\n            \n            for future in as_completed(futures.keys()):\n                idx = futures[future]\n                try:\n                    results[idx] = future.result()\n                except Exception as e:\n                    print(f\"Error processing chunk {idx}: {e}\")\n                    results[idx] = None\n                \n                if progress_callback:\n                    progress_callback(idx + 1, len(chunks))\n            \n            return results\n```\n\n## Batch Processing Queue\n```python\nfrom dataclasses import dataclass\nfrom typing import Callable, Any\nfrom datetime import datetime\nimport json\n\n\n@dataclass\nclass BatchJob:\n    \"\"\"Batch job configuration.\"\"\"\n    job_id: str\n    job_type: str\n    payload: dict\n    status: str = \"pending\"\n    created_at: datetime = None\n    started_at: datetime = None\n    completed_at: datetime = None\n    result: Any = None\n    error: str = None\n    retry_count: int = 0\n    max_retries: int = 3\n\n\nclass BatchJobProcessor:\n    \"\"\"Process batch jobs from queue.\"\"\"\n    \n    def __init__(self, queue, storage, max_concurrent: int = 5):\n        self.queue = queue\n        self.storage = storage\n        self.max_concurrent = max_concurrent\n        self.active_jobs = set()\n    \n    async def process_queue(self) -> None:\n        \"\"\"Process jobs from queue.\"\"\"\n        while True:\n            # Get next job\n            job_data = await self.queue.get()\n            \n            if job_data is None:\n                break\n            \n            job = BatchJob(**job_data)\n            \n            # Check concurrent limit\n            while len(self.active_jobs) >= self.max_concurrent:\n                await asyncio.sleep(1)\n            \n            # Process job\n            self.active_jobs.add(job.job_id)\n            \n            try:\n                result = await self._execute_job(job)\n                \n                job.status = \"completed\"\n                job.result = result\n                job.completed_at = datetime.utcnow()\n                \n                await self.storage.save_job(job)\n                \n            except Exception as e:\n                job.retry_count += 1\n                \n                if job.retry_count >= job.max_retries:\n                    job.status = \"failed\"\n                    job.error = str(e)\n                    job.completed_at = datetime.utcnow()\n                else:\n                    # Requeue with backoff\n                    job.status = \"pending\"\n                    await self.queue.put(job.to_dict())\n                \n                await self.storage.save_job(job)\n            \n            finally:\n                self.active_jobs.discard(job.job_id)\n    \n    async def _execute_job(self, job: BatchJob) -> Any:\n        \"\"\"Execute a single job.\"\"\"\n        job_handlers = {\n            \"import\": self._handle_import,\n            \"export\": self._handle_export,\n            \"process\": self._handle_process,\n        }\n        \n        handler = job_handlers.get(job.job_type)\n        \n        if not handler:\n            raise ValueError(f\"Unknown job type: {job.job_type}\")\n        \n        return await handler(job.payload)\n    \n    async def _handle_import(self, payload: dict) -> dict:\n        \"\"\"Handle import job.\"\"\"\n        # Import logic\n        return {\"imported\": 100}\n    \n    async def _handle_export(self, payload: dict) -> dict:\n        \"\"\"Handle export job.\"\"\"\n        # Export logic\n        return {\"exported\": 100}\n    \n    async def _handle_process(self, payload: dict) -> dict:\n        \"\"\"Handle processing job.\"\"\"\n        # Processing logic\n        return {\"processed\": 100}\n```\n\n## Best Practices\n```\nBulk Operations Best Practices:\n\n1. Use batch operations\n   Bulk inserts instead of individual\n   Reduces round trips\n\n2. Process in chunks\n   Memory efficient\n   Progress tracking\n\n3. Handle errors gracefully\n   Skip failed items\n   Log for review\n\n4. Monitor progress\n   Real-time updates\n   ETA calculations\n\n5. Use transactions appropriately\n   Commit frequently\n   Balance performance\n\n6. Parallelize I\u002FO bound\n   Thread pool for I\u002FO\n   Process pool for CPU\n\n7. Queue for reliability\n   Persistent queue\n   Retry with backoff\n\n8. Validate input\n   Check before processing\n   Sanitize data\n\n9. Resource limits\n   Memory constraints\n   Timeout handling\n\n10. Clean up after\n     Temporary files\n     Progress markers\n```\n",{"name":5,"license":14,"metadata":28,"description":7,"compatibility":22},{"audience":29,"category":30},"developers","utilities","main","5f7a3e453c98261f93d19a6a7de44ad8f7b72d41",2,"https:\u002F\u002Fgithub.com\u002Fffsshhttiikk\u002Fopencode-agents-skills","https:\u002F\u002Fgithub.com\u002Fffsshhttiikk","2026-03-01 06:55:40",{"source":8,"skillName":5,"commands":38,"prompt":48,"zipAvailable":49},[39,42,45],{"tool":40,"command":41},"npx","npx skills add ffsshhttiikk\u002Fopencode-agents-skills --skill bulk-operations",{"tool":43,"command":44},"bunx","bunx skills add ffsshhttiikk\u002Fopencode-agents-skills --skill bulk-operations",{"tool":46,"command":47},"pnpm","pnpm dlx skills add ffsshhttiikk\u002Fopencode-agents-skills --skill bulk-operations","安装技能，技能地址：https:\u002F\u002Fgithub.com\u002Fffsshhttiikk\u002Fopencode-agents-skills\u002Fblob\u002Fmain\u002Fbulk-operations\u002FSKILL.md",true,[],[52,63,72,81,91,100],{"skillNo":53,"name":54,"nameZh":6,"description":55,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":56,"categories":57,"keywords":58,"publishedAt":23},"01M6CXT9TXDF79BWJTMQAPB49G","xamarin","Xamarin cross-platform development","https:\u002F\u002Fgithub.com\u002Fffsshhttiikk\u002Fopencode-agents-skills\u002Fblob\u002Fmain\u002Fxamarin\u002FSKILL.md",[],[54,59,60,61,22,62],"platform","cross-platform","mobile","development",{"skillNo":64,"name":65,"nameZh":6,"description":66,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":67,"categories":68,"keywords":69,"publishedAt":23},"01M6CXT8AD04MFYDA32VW3FKAG","wrangling","Data wrangling fundamentals","https:\u002F\u002Fgithub.com\u002Fffsshhttiikk\u002Fopencode-agents-skills\u002Fblob\u002Fmain\u002Fwrangling\u002FSKILL.md",[],[65,70,22,71,20],"data wrangling","science",{"skillNo":73,"name":74,"nameZh":6,"description":75,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":76,"categories":77,"keywords":78,"publishedAt":23},"01M6CXT717GCN7SRRNPBDP4508","windows","Windows operating system administration and scripting","https:\u002F\u002Fgithub.com\u002Fffsshhttiikk\u002Fopencode-agents-skills\u002Fblob\u002Fmain\u002Fwindows\u002FSKILL.md",[],[74,79,22,80],"system","scripting",{"skillNo":82,"name":83,"nameZh":6,"description":84,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":85,"categories":86,"keywords":87,"publishedAt":23},"01M6CXT54K794NMEM3XAPZZDKT","wide-column","Wide-column data model","https:\u002F\u002Fgithub.com\u002Fffsshhttiikk\u002Fopencode-agents-skills\u002Fblob\u002Fmain\u002Fwide-column\u002FSKILL.md",[],[88,89,22,90,20],"wide column","databases","model",{"skillNo":92,"name":93,"nameZh":6,"description":94,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":95,"categories":96,"keywords":97,"publishedAt":23},"01M6CXT28FMT2P4V9GZE67WA8E","wide-column-stores","Wide-column database stores","https:\u002F\u002Fgithub.com\u002Fffsshhttiikk\u002Fopencode-agents-skills\u002Fblob\u002Fmain\u002Fwide-column-stores\u002FSKILL.md",[],[98,89,22,99],"wide column stores","database",{"skillNo":101,"name":102,"nameZh":6,"description":103,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":104,"categories":105,"keywords":106,"publishedAt":23},"01M6CXT0D0M0Q4T2N7C2RX7DSN","websockets","WebSocket implementation and best practices","https:\u002F\u002Fgithub.com\u002Fffsshhttiikk\u002Fopencode-agents-skills\u002Fblob\u002Fmain\u002Fwebsockets\u002FSKILL.md",[],[102,107,108,109,22,110,111],"websocket","real-time","time","best practices","implementation"]