[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"skill:01M6NTPTR4JWA8D8DGZK3W5VAP":3,"skill-same:01M6NTPTR4JWA8D8DGZK3W5VAP":71},{"skillNo":4,"name":5,"nameZh":6,"description":7,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":15,"categories":16,"keywords":17,"publishedAt":38,"linkOnly":39,"markdownContent":40,"markdownBody":41,"frontMatter":42,"refName":30,"commitSha":52,"skillDirectory":53,"forksCount":13,"repositoryHtmlUrl":54,"repositoryHomepageUrl":6,"ownerHtmlUrl":55,"ownerBlogUrl":6,"ownerDescription":6,"updatedAt":56,"install":57,"resources":70},"01M6NTPTR4JWA8D8DGZK3W5VAP","ai-sre-incident-response",null,"Build AI-focused SRE incident response practices for LLM outages, degraded quality, runaway cost events, and safety regressions.","duc-duy-1612\u002FOnline-Delivery-Systems-Web","duc-duy-1612","https:\u002F\u002Favatars.githubusercontent.com\u002Fu\u002F192231384?v=4","01M6NSDKQQDKVTHZ8DZMZ1VYY6","01M6NSDKW8DDVTQHBT1HBDDJPT",0,"MIT","https:\u002F\u002Fgithub.com\u002Fduc-duy-1612\u002FOnline-Delivery-Systems-Web\u002Fblob\u002Fmain\u002F.agents\u002Fskills_archive\u002Fai-sre-incident-response\u002FSKILL.md",[],[18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37],"incident response","ai sre","incident","ai sre incident response","sre","Safety","devops-security","access","kubectl","only","ci","community","main","git","platform","security","critical","scripts","events","relevant","2026-11-10 10:12:30",false,"---\nname: ai-sre-incident-response\ndescription: Build AI-focused SRE incident response practices for LLM outages, degraded\n  quality, runaway cost events, and safety regressions.\ncategory: devops\nrisk: critical\nsource: https:\u002F\u002Fgithub.com\u002FBagelHole\u002FDevOps-Security-Agent-Skills\nsource_repo: BagelHole\u002FDevOps-Security-Agent-Skills\nsource_type: community\ndate_added: '2026-09-20'\nlicense: MIT\nlicense_source: https:\u002F\u002Fgithub.com\u002FBagelHole\u002FDevOps-Security-Agent-Skills\u002Fblob\u002Fmain\u002FLICENSE\ncompatibility: Requires the relevant platform CLIs (kubectl, helm, terraform, git,\n  CI runners) and authorized access to the target environment. Docs-only; helper scripts\n  and templates not bundled.\nmetadata:\n  author: devops-skills\n  version: '1.0'\n---\n\n# AI SRE Incident Response\n\nApply SRE rigor to AI systems where incidents include quality regressions, unsafe outputs, and budget explosions.\n\n## When to Use This Skill\n\n- An LLM endpoint begins returning degraded or hallucinated answers\n- Token spend spikes beyond budget thresholds\n- A model provider goes down and traffic must fail over\n- Safety guardrails fire at abnormal rates\n- A new model deployment causes latency or accuracy regression\n\n## Prerequisites\n\n- Prometheus and Alertmanager deployed with scrape targets for AI services\n- Grafana dashboards for golden signals (latency, error rate, cost, quality)\n- On-call rotation configured in PagerDuty, Opsgenie, or equivalent\n- Runbook repository accessible to responders\n- Rollback mechanism for model and prompt versions (GitOps or feature flags)\n\n## AI Incident Classes\n\n- **Availability incident**: model\u002Fprovider unavailable, timeout storm.\n- **Quality incident**: answer accuracy or tool success drops below SLO.\n- **Safety incident**: harmful or policy-violating outputs increase.\n- **Cost incident**: unexpected token or provider spend spike.\n\n## Severity Framework\n\n| Severity | Criteria | Response Time | Notification |\n|----------|----------|---------------|--------------|\n| SEV1 | User-facing outage, compliance risk, data leak | 5 min | Page on-call + incident commander |\n| SEV2 | Major degradation in key flows | 15 min | Page on-call |\n| SEV3 | Limited impact or internal-only issue | 1 hour | Slack alert |\n| SEV4 | Cosmetic or low-priority regression | Next business day | Ticket |\n\n## Golden Signals for AI Services\n\n- Request success rate\n- Latency (queue + generation + tool execution)\n- Hallucination\u002Fgroundedness proxy metrics\n- Cost per minute and per tenant\n- Guardrail violation rate\n\n## Prometheus Alert Rules\n\n```yaml\n# prometheus-ai-alerts.yaml\ngroups:\n  - name: ai-service-alerts\n    rules:\n      - alert: ModelEndpointDown\n        expr: up{job=\"llm-inference\"} == 0\n        for: 2m\n        labels:\n          severity: sev1\n        annotations:\n          summary: \"LLM inference endpoint {{ $labels.instance }} is down\"\n          runbook_url: \"https:\u002F\u002Frunbooks.internal\u002Fai\u002Fmodel-outage\"\n\n      - alert: HighHallucinationRate\n        expr: |\n          rate(llm_hallucination_detected_total[10m])\n          \u002F rate(llm_requests_total[10m]) > 0.15\n        for: 5m\n        labels:\n          severity: sev2\n        annotations:\n          summary: \"Hallucination rate above 15% for {{ $labels.model }}\"\n          runbook_url: \"https:\u002F\u002Frunbooks.internal\u002Fai\u002Fquality-regression\"\n\n      - alert: TokenCostExplosion\n        expr: |\n          sum(rate(llm_token_cost_dollars[5m])) by (tenant)\n          > 0.50\n        for: 3m\n        labels:\n          severity: sev2\n        annotations:\n          summary: \"Token spend exceeds $0.50\u002Fmin for tenant {{ $labels.tenant }}\"\n          runbook_url: \"https:\u002F\u002Frunbooks.internal\u002Fai\u002Fcost-spike\"\n\n      - alert: LatencyP95Exceeded\n        expr: |\n          histogram_quantile(0.95,\n            rate(llm_request_duration_seconds_bucket[5m])\n          ) > 5\n        for: 5m\n        labels:\n          severity: sev2\n        annotations:\n          summary: \"LLM p95 latency exceeds 5s for {{ $labels.service }}\"\n\n      - alert: GuardrailViolationSpike\n        expr: |\n          rate(llm_guardrail_violations_total[10m])\n          \u002F rate(llm_requests_total[10m]) > 0.05\n        for: 5m\n        labels:\n          severity: sev1\n        annotations:\n          summary: \"Guardrail violations above 5% for {{ $labels.model }}\"\n          runbook_url: \"https:\u002F\u002Frunbooks.internal\u002Fai\u002Fsafety-incident\"\n\n      - alert: ModelQualityDrop\n        expr: |\n          llm_eval_score{metric=\"groundedness\"} \u003C 0.70\n        for: 10m\n        labels:\n          severity: sev2\n        annotations:\n          summary: \"Groundedness score dropped below 0.70 for {{ $labels.model }}\"\n\n      - alert: ProviderErrorRateHigh\n        expr: |\n          rate(llm_provider_errors_total[5m])\n          \u002F rate(llm_provider_requests_total[5m]) > 0.10\n        for: 3m\n        labels:\n          severity: sev2\n        annotations:\n          summary: \"Provider {{ $labels.provider }} error rate above 10%\"\n```\n\n## Response Playbooks\n\n### Model Outage Runbook\n\n```text\nTRIGGER: ModelEndpointDown fires for > 2 minutes\nRESPONDER: On-call AI platform engineer\n\n1. Acknowledge alert in PagerDuty.\n2. Check provider status page (e.g., status.openai.com).\n3. Verify network connectivity:\n     curl -s -o \u002Fdev\u002Fnull -w \"%{http_code}\" https:\u002F\u002Fapi.provider.com\u002Fhealth\n4. If provider is down:\n     a. Enable fallback model route in gateway config.\n     b. kubectl set env deployment\u002Fllm-gateway FALLBACK_ENABLED=true\n     c. Verify fallback traffic is flowing via Grafana dashboard.\n5. If self-hosted model is down:\n     a. Check pod status: kubectl get pods -l app=llm-inference -n ai\n     b. Check GPU health: kubectl logs -l app=llm-inference --tail=50\n     c. Restart if OOM: kubectl rollout restart deployment\u002Fllm-inference -n ai\n6. Freeze all deployments:\n     kubectl annotate deployment --all deploy-freeze=true -n ai\n7. Communicate ETA in #incident-channel.\n8. When resolved, unfreeze and run smoke tests.\n```\n\n### Quality Regression Runbook (Hallucination Spike)\n\n```text\nTRIGGER: HighHallucinationRate or ModelQualityDrop fires\nRESPONDER: On-call AI engineer + ML lead\n\n1. Acknowledge alert. Open incident ticket.\n2. Identify scope:\n     - Which model version? Check deployment metadata.\n     - Which routes\u002Ftenants affected? Filter by labels in Grafana.\n3. Check recent changes:\n     - Model version promotion in last 24h?\n     - Prompt template changes in last 24h?\n     - Retrieval index rebuild in last 24h?\n4. If recent model change:\n     kubectl rollout undo deployment\u002Fllm-inference -n ai\n5. If recent prompt change:\n     git revert \u003Ccommit> && git push  # triggers GitOps redeploy\n6. Increase trace sampling to 100% for affected route:\n     kubectl set env deployment\u002Fllm-gateway TRACE_SAMPLE_RATE=1.0\n7. Run offline eval suite against current production:\n     python run_evals.py --target prod --suite quality --compare baseline\n8. Confirm metrics return to baseline before closing.\n```\n\n### Token Cost Explosion Runbook\n\n```text\nTRIGGER: TokenCostExplosion fires\nRESPONDER: On-call platform engineer\n\n1. Identify top consumers:\n     Query: topk(10, sum(rate(llm_token_cost_dollars[15m])) by (tenant, model, route))\n2. Check for runaway loops:\n     - Agent retry storms (exponential token growth per request)\n     - Missing max_tokens caps on new routes\n     - Cache bypass due to config change\n3. Apply immediate caps:\n     kubectl patch configmap llm-quotas -n ai --patch '\n       data:\n         max_tokens_per_request: \"4096\"\n         rpm_limit: \"60\"\n     '\n4. Enable semantic cache if disabled:\n     kubectl set env deployment\u002Fllm-gateway CACHE_ENABLED=true\n5. Route traffic to cheaper model tier:\n     kubectl set env deployment\u002Fllm-gateway DEFAULT_MODEL=gpt-4o-mini\n6. Notify affected tenants of temporary limits.\n7. Open postmortem with cost attribution analysis.\n```\n\n## Escalation Procedures\n\n```text\nLevel 1 (0-15 min):  On-call AI platform engineer\nLevel 2 (15-30 min): AI platform team lead + affected product owner\nLevel 3 (30-60 min): Engineering director + security (if safety incident)\nLevel 4 (60+ min):   VP Engineering + legal (if compliance\u002Fdata incident)\n\nSafety incidents always start at Level 2 minimum.\nProvider-side incidents: open support ticket immediately at Level 1.\n```\n\n## Detection Queries (PromQL)\n\n```promql\n# Request success rate by model\n1 - (\n  sum(rate(llm_requests_total{status=\"error\"}[5m])) by (model)\n  \u002F sum(rate(llm_requests_total[5m])) by (model)\n)\n\n# Cost per successful answer\nsum(rate(llm_token_cost_dollars[5m])) by (route)\n\u002F sum(rate(llm_requests_total{status=\"success\"}[5m])) by (route)\n\n# Hallucination rate trend (1h window, 5m steps)\nrate(llm_hallucination_detected_total[1h])\n\u002F rate(llm_requests_total[1h])\n\n# Latency breakdown by stage\nhistogram_quantile(0.95, rate(llm_retrieval_duration_seconds_bucket[5m]))\nhistogram_quantile(0.95, rate(llm_generation_duration_seconds_bucket[5m]))\nhistogram_quantile(0.95, rate(llm_tool_execution_duration_seconds_bucket[5m]))\n\n# Tenant cost leaderboard\ntopk(10, sum(rate(llm_token_cost_dollars[1h])) by (tenant))\n```\n\n## Postmortem Requirements\n\n- Timeline with detector and responder timestamps\n- Blast radius by tenant and feature\n- Missed signals and alert tuning actions\n- Concrete hardening tasks with owners and due dates\n- Cost impact (dollars, tokens, affected requests)\n- Customer communication log\n\n## Postmortem Template\n\n```markdown\n## Incident Summary\n- **Severity**: SEVx\n- **Duration**: start_time - end_time (Xh Ym)\n- **Detection**: How was it detected? (alert \u002F customer report \u002F manual)\n- **Impact**: X tenants, Y requests, $Z cost\n\n## Timeline\n| Time (UTC) | Event |\n|------------|-------|\n| HH:MM | Alert fired |\n| HH:MM | Responder acknowledged |\n| HH:MM | Root cause identified |\n| HH:MM | Mitigation applied |\n| HH:MM | Incident resolved |\n\n## Root Cause\n[Description]\n\n## Action Items\n| Action | Owner | Due Date | Status |\n|--------|-------|----------|--------|\n| Tune alert threshold | @engineer | YYYY-MM-DD | Open |\n| Add fallback route | @platform | YYYY-MM-DD | Open |\n```\n\n## Chaos Engineering for AI Systems\n\nRegularly test incident readiness:\n\n- **Provider failover drill**: block provider API at network level, verify fallback activates within SLO.\n- **Model rollback drill**: deploy known-bad model version, verify automated quality gate catches it.\n- **Cost cap drill**: simulate runaway token usage, verify quotas trigger before budget threshold.\n- **Cache failure drill**: disable semantic cache, verify system degrades gracefully.\n\n## Troubleshooting\n\n| Symptom | Check | Fix |\n|---------|-------|-----|\n| All requests timing out | Provider status page, DNS resolution | Enable fallback provider |\n| Gradual quality decline | Recent model\u002Fprompt deployments | Roll back to last known good |\n| Sudden cost spike | Per-tenant token usage dashboard | Apply emergency token caps |\n| Guardrail violations spike | Model version, prompt injection logs | Enable stricter input filtering |\n| Intermittent 503 errors | Pod restarts, GPU OOM events | Increase memory limits or reduce batch size |\n\n## Related Skills\n\n- incident-response (`incident-response`) - Standard incident process and evidence\n- alerting-oncall (`alerting-oncall`) - Paging and escalation policy\n- llm-cost-optimization (`llm-cost-optimization`) - Spend controls and efficiency patterns\n- agent-observability (`agent-observability`) - Instrument requests, traces, and costs\n- rag-observability-evals (`rag-observability-evals`) - RAG quality monitoring\n\n## Limitations\n\n- Guidance executes against real environments: confirm target, blast radius, and rollback plan before applying anything.\n- Never deploy to production without explicit approval. Docs-only import: upstream scripts and templates not bundled.\n\n### Example\n\n```bash\ngit status && git diff --stat\nkubectl diff -f manifest.yaml\n```\n\n> Adapted from [BagelHole\u002FDevOps-Security-Agent-Skills](https:\u002F\u002Fgithub.com\u002FBagelHole\u002FDevOps-Security-Agent-Skills) (MIT); frontmatter, When to Use\u002FLimitations, and safety boundaries added for upstream compliance. Docs-only import: helper scripts and templates not bundled.\n","# AI SRE Incident Response\n\nApply SRE rigor to AI systems where incidents include quality regressions, unsafe outputs, and budget explosions.\n\n## When to Use This Skill\n\n- An LLM endpoint begins returning degraded or hallucinated answers\n- Token spend spikes beyond budget thresholds\n- A model provider goes down and traffic must fail over\n- Safety guardrails fire at abnormal rates\n- A new model deployment causes latency or accuracy regression\n\n## Prerequisites\n\n- Prometheus and Alertmanager deployed with scrape targets for AI services\n- Grafana dashboards for golden signals (latency, error rate, cost, quality)\n- On-call rotation configured in PagerDuty, Opsgenie, or equivalent\n- Runbook repository accessible to responders\n- Rollback mechanism for model and prompt versions (GitOps or feature flags)\n\n## AI Incident Classes\n\n- **Availability incident**: model\u002Fprovider unavailable, timeout storm.\n- **Quality incident**: answer accuracy or tool success drops below SLO.\n- **Safety incident**: harmful or policy-violating outputs increase.\n- **Cost incident**: unexpected token or provider spend spike.\n\n## Severity Framework\n\n| Severity | Criteria | Response Time | Notification |\n|----------|----------|---------------|--------------|\n| SEV1 | User-facing outage, compliance risk, data leak | 5 min | Page on-call + incident commander |\n| SEV2 | Major degradation in key flows | 15 min | Page on-call |\n| SEV3 | Limited impact or internal-only issue | 1 hour | Slack alert |\n| SEV4 | Cosmetic or low-priority regression | Next business day | Ticket |\n\n## Golden Signals for AI Services\n\n- Request success rate\n- Latency (queue + generation + tool execution)\n- Hallucination\u002Fgroundedness proxy metrics\n- Cost per minute and per tenant\n- Guardrail violation rate\n\n## Prometheus Alert Rules\n\n```yaml\n# prometheus-ai-alerts.yaml\ngroups:\n  - name: ai-service-alerts\n    rules:\n      - alert: ModelEndpointDown\n        expr: up{job=\"llm-inference\"} == 0\n        for: 2m\n        labels:\n          severity: sev1\n        annotations:\n          summary: \"LLM inference endpoint {{ $labels.instance }} is down\"\n          runbook_url: \"https:\u002F\u002Frunbooks.internal\u002Fai\u002Fmodel-outage\"\n\n      - alert: HighHallucinationRate\n        expr: |\n          rate(llm_hallucination_detected_total[10m])\n          \u002F rate(llm_requests_total[10m]) > 0.15\n        for: 5m\n        labels:\n          severity: sev2\n        annotations:\n          summary: \"Hallucination rate above 15% for {{ $labels.model }}\"\n          runbook_url: \"https:\u002F\u002Frunbooks.internal\u002Fai\u002Fquality-regression\"\n\n      - alert: TokenCostExplosion\n        expr: |\n          sum(rate(llm_token_cost_dollars[5m])) by (tenant)\n          > 0.50\n        for: 3m\n        labels:\n          severity: sev2\n        annotations:\n          summary: \"Token spend exceeds $0.50\u002Fmin for tenant {{ $labels.tenant }}\"\n          runbook_url: \"https:\u002F\u002Frunbooks.internal\u002Fai\u002Fcost-spike\"\n\n      - alert: LatencyP95Exceeded\n        expr: |\n          histogram_quantile(0.95,\n            rate(llm_request_duration_seconds_bucket[5m])\n          ) > 5\n        for: 5m\n        labels:\n          severity: sev2\n        annotations:\n          summary: \"LLM p95 latency exceeds 5s for {{ $labels.service }}\"\n\n      - alert: GuardrailViolationSpike\n        expr: |\n          rate(llm_guardrail_violations_total[10m])\n          \u002F rate(llm_requests_total[10m]) > 0.05\n        for: 5m\n        labels:\n          severity: sev1\n        annotations:\n          summary: \"Guardrail violations above 5% for {{ $labels.model }}\"\n          runbook_url: \"https:\u002F\u002Frunbooks.internal\u002Fai\u002Fsafety-incident\"\n\n      - alert: ModelQualityDrop\n        expr: |\n          llm_eval_score{metric=\"groundedness\"} \u003C 0.70\n        for: 10m\n        labels:\n          severity: sev2\n        annotations:\n          summary: \"Groundedness score dropped below 0.70 for {{ $labels.model }}\"\n\n      - alert: ProviderErrorRateHigh\n        expr: |\n          rate(llm_provider_errors_total[5m])\n          \u002F rate(llm_provider_requests_total[5m]) > 0.10\n        for: 3m\n        labels:\n          severity: sev2\n        annotations:\n          summary: \"Provider {{ $labels.provider }} error rate above 10%\"\n```\n\n## Response Playbooks\n\n### Model Outage Runbook\n\n```text\nTRIGGER: ModelEndpointDown fires for > 2 minutes\nRESPONDER: On-call AI platform engineer\n\n1. Acknowledge alert in PagerDuty.\n2. Check provider status page (e.g., status.openai.com).\n3. Verify network connectivity:\n     curl -s -o \u002Fdev\u002Fnull -w \"%{http_code}\" https:\u002F\u002Fapi.provider.com\u002Fhealth\n4. If provider is down:\n     a. Enable fallback model route in gateway config.\n     b. kubectl set env deployment\u002Fllm-gateway FALLBACK_ENABLED=true\n     c. Verify fallback traffic is flowing via Grafana dashboard.\n5. If self-hosted model is down:\n     a. Check pod status: kubectl get pods -l app=llm-inference -n ai\n     b. Check GPU health: kubectl logs -l app=llm-inference --tail=50\n     c. Restart if OOM: kubectl rollout restart deployment\u002Fllm-inference -n ai\n6. Freeze all deployments:\n     kubectl annotate deployment --all deploy-freeze=true -n ai\n7. Communicate ETA in #incident-channel.\n8. When resolved, unfreeze and run smoke tests.\n```\n\n### Quality Regression Runbook (Hallucination Spike)\n\n```text\nTRIGGER: HighHallucinationRate or ModelQualityDrop fires\nRESPONDER: On-call AI engineer + ML lead\n\n1. Acknowledge alert. Open incident ticket.\n2. Identify scope:\n     - Which model version? Check deployment metadata.\n     - Which routes\u002Ftenants affected? Filter by labels in Grafana.\n3. Check recent changes:\n     - Model version promotion in last 24h?\n     - Prompt template changes in last 24h?\n     - Retrieval index rebuild in last 24h?\n4. If recent model change:\n     kubectl rollout undo deployment\u002Fllm-inference -n ai\n5. If recent prompt change:\n     git revert \u003Ccommit> && git push  # triggers GitOps redeploy\n6. Increase trace sampling to 100% for affected route:\n     kubectl set env deployment\u002Fllm-gateway TRACE_SAMPLE_RATE=1.0\n7. Run offline eval suite against current production:\n     python run_evals.py --target prod --suite quality --compare baseline\n8. Confirm metrics return to baseline before closing.\n```\n\n### Token Cost Explosion Runbook\n\n```text\nTRIGGER: TokenCostExplosion fires\nRESPONDER: On-call platform engineer\n\n1. Identify top consumers:\n     Query: topk(10, sum(rate(llm_token_cost_dollars[15m])) by (tenant, model, route))\n2. Check for runaway loops:\n     - Agent retry storms (exponential token growth per request)\n     - Missing max_tokens caps on new routes\n     - Cache bypass due to config change\n3. Apply immediate caps:\n     kubectl patch configmap llm-quotas -n ai --patch '\n       data:\n         max_tokens_per_request: \"4096\"\n         rpm_limit: \"60\"\n     '\n4. Enable semantic cache if disabled:\n     kubectl set env deployment\u002Fllm-gateway CACHE_ENABLED=true\n5. Route traffic to cheaper model tier:\n     kubectl set env deployment\u002Fllm-gateway DEFAULT_MODEL=gpt-4o-mini\n6. Notify affected tenants of temporary limits.\n7. Open postmortem with cost attribution analysis.\n```\n\n## Escalation Procedures\n\n```text\nLevel 1 (0-15 min):  On-call AI platform engineer\nLevel 2 (15-30 min): AI platform team lead + affected product owner\nLevel 3 (30-60 min): Engineering director + security (if safety incident)\nLevel 4 (60+ min):   VP Engineering + legal (if compliance\u002Fdata incident)\n\nSafety incidents always start at Level 2 minimum.\nProvider-side incidents: open support ticket immediately at Level 1.\n```\n\n## Detection Queries (PromQL)\n\n```promql\n# Request success rate by model\n1 - (\n  sum(rate(llm_requests_total{status=\"error\"}[5m])) by (model)\n  \u002F sum(rate(llm_requests_total[5m])) by (model)\n)\n\n# Cost per successful answer\nsum(rate(llm_token_cost_dollars[5m])) by (route)\n\u002F sum(rate(llm_requests_total{status=\"success\"}[5m])) by (route)\n\n# Hallucination rate trend (1h window, 5m steps)\nrate(llm_hallucination_detected_total[1h])\n\u002F rate(llm_requests_total[1h])\n\n# Latency breakdown by stage\nhistogram_quantile(0.95, rate(llm_retrieval_duration_seconds_bucket[5m]))\nhistogram_quantile(0.95, rate(llm_generation_duration_seconds_bucket[5m]))\nhistogram_quantile(0.95, rate(llm_tool_execution_duration_seconds_bucket[5m]))\n\n# Tenant cost leaderboard\ntopk(10, sum(rate(llm_token_cost_dollars[1h])) by (tenant))\n```\n\n## Postmortem Requirements\n\n- Timeline with detector and responder timestamps\n- Blast radius by tenant and feature\n- Missed signals and alert tuning actions\n- Concrete hardening tasks with owners and due dates\n- Cost impact (dollars, tokens, affected requests)\n- Customer communication log\n\n## Postmortem Template\n\n```markdown\n## Incident Summary\n- **Severity**: SEVx\n- **Duration**: start_time - end_time (Xh Ym)\n- **Detection**: How was it detected? (alert \u002F customer report \u002F manual)\n- **Impact**: X tenants, Y requests, $Z cost\n\n## Timeline\n| Time (UTC) | Event |\n|------------|-------|\n| HH:MM | Alert fired |\n| HH:MM | Responder acknowledged |\n| HH:MM | Root cause identified |\n| HH:MM | Mitigation applied |\n| HH:MM | Incident resolved |\n\n## Root Cause\n[Description]\n\n## Action Items\n| Action | Owner | Due Date | Status |\n|--------|-------|----------|--------|\n| Tune alert threshold | @engineer | YYYY-MM-DD | Open |\n| Add fallback route | @platform | YYYY-MM-DD | Open |\n```\n\n## Chaos Engineering for AI Systems\n\nRegularly test incident readiness:\n\n- **Provider failover drill**: block provider API at network level, verify fallback activates within SLO.\n- **Model rollback drill**: deploy known-bad model version, verify automated quality gate catches it.\n- **Cost cap drill**: simulate runaway token usage, verify quotas trigger before budget threshold.\n- **Cache failure drill**: disable semantic cache, verify system degrades gracefully.\n\n## Troubleshooting\n\n| Symptom | Check | Fix |\n|---------|-------|-----|\n| All requests timing out | Provider status page, DNS resolution | Enable fallback provider |\n| Gradual quality decline | Recent model\u002Fprompt deployments | Roll back to last known good |\n| Sudden cost spike | Per-tenant token usage dashboard | Apply emergency token caps |\n| Guardrail violations spike | Model version, prompt injection logs | Enable stricter input filtering |\n| Intermittent 503 errors | Pod restarts, GPU OOM events | Increase memory limits or reduce batch size |\n\n## Related Skills\n\n- incident-response (`incident-response`) - Standard incident process and evidence\n- alerting-oncall (`alerting-oncall`) - Paging and escalation policy\n- llm-cost-optimization (`llm-cost-optimization`) - Spend controls and efficiency patterns\n- agent-observability (`agent-observability`) - Instrument requests, traces, and costs\n- rag-observability-evals (`rag-observability-evals`) - RAG quality monitoring\n\n## Limitations\n\n- Guidance executes against real environments: confirm target, blast radius, and rollback plan before applying anything.\n- Never deploy to production without explicit approval. Docs-only import: upstream scripts and templates not bundled.\n\n### Example\n\n```bash\ngit status && git diff --stat\nkubectl diff -f manifest.yaml\n```\n\n> Adapted from [BagelHole\u002FDevOps-Security-Agent-Skills](https:\u002F\u002Fgithub.com\u002FBagelHole\u002FDevOps-Security-Agent-Skills) (MIT); frontmatter, When to Use\u002FLimitations, and safety boundaries added for upstream compliance. Docs-only import: helper scripts and templates not bundled.\n",{"name":5,"risk":34,"source":43,"license":14,"category":44,"metadata":45,"date_added":48,"description":7,"source_repo":49,"source_type":29,"compatibility":50,"license_source":51},"https:\u002F\u002Fgithub.com\u002FBagelHole\u002FDevOps-Security-Agent-Skills","devops",{"author":46,"version":47},"devops-skills","1.0","2026-09-20","BagelHole\u002FDevOps-Security-Agent-Skills","Requires the relevant platform CLIs (kubectl, helm, terraform, git, CI runners) and authorized access to the target environment. Docs-only; helper scripts and templates not bundled.","https:\u002F\u002Fgithub.com\u002FBagelHole\u002FDevOps-Security-Agent-Skills\u002Fblob\u002Fmain\u002FLICENSE","f3312ba749561b6159f477dbfc9313c875d40a56",".agents\u002Fskills_archive\u002Fai-sre-incident-response","https:\u002F\u002Fgithub.com\u002Fduc-duy-1612\u002FOnline-Delivery-Systems-Web","https:\u002F\u002Fgithub.com\u002Fduc-duy-1612","2026-10-25 23:50:58",{"source":8,"skillName":5,"commands":58,"prompt":68,"zipAvailable":69},[59,62,65],{"tool":60,"command":61},"npx","npx skills add duc-duy-1612\u002FOnline-Delivery-Systems-Web --skill ai-sre-incident-response",{"tool":63,"command":64},"bunx","bunx skills add duc-duy-1612\u002FOnline-Delivery-Systems-Web --skill ai-sre-incident-response",{"tool":66,"command":67},"pnpm","pnpm dlx skills add duc-duy-1612\u002FOnline-Delivery-Systems-Web --skill ai-sre-incident-response","安装技能，技能地址：https:\u002F\u002Fgithub.com\u002Fduc-duy-1612\u002FOnline-Delivery-Systems-Web\u002Fblob\u002Fmain\u002F.agents\u002Fskills_archive\u002Fai-sre-incident-response\u002FSKILL.md",true,[],[72,95,119,144,172,196],{"skillNo":73,"name":74,"nameZh":6,"description":75,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":76,"categories":77,"keywords":83,"publishedAt":38},"01M6Q0FA5FZ410Z33Z3TFQ4DBA","zero-trust","Implement zero-trust network architecture. Configure identity-based access, micro-segmentation, and continuous verification. Use when implementing modern security architectures.","https:\u002F\u002Fgithub.com\u002Fduc-duy-1612\u002FOnline-Delivery-Systems-Web\u002Fblob\u002Fmain\u002F.agents\u002Fskills_archive\u002Fzero-trust\u002FSKILL.md",[78],{"categoryNo":79,"categoryName":80,"taxonomyType":81,"taxonomyTypeDesc":82},"01JBSEEDCAT000000000000011","运维与部署","SCENARIO","使用场景",[84,85,86,87,88,89,74,90,24,25,91,27,29,30,92,34,93,35,37,94],"ZeRO","trust","zero trust","architecture","assessment","verification","segmentation","configure","active","scope","implementing",{"skillNo":96,"name":97,"nameZh":6,"description":98,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":99,"categories":100,"keywords":101,"publishedAt":38},"01M6Q0F1YZJW3KX5FBKEVZ8CV4","zcode-delegate","Delegate coding tasks to the Z.AI ZCode CLI only when the user explicitly requests it, while the orchestrator retains review and landing responsibility.","https:\u002F\u002Fgithub.com\u002Fduc-duy-1612\u002FOnline-Delivery-Systems-Web\u002Fblob\u002Fmain\u002F.agents\u002Fskills_archive\u002Fzcode-delegate\u002FSKILL.md",[],[102,103,104,105,106,107,108,109,110,111,112,113,114,27,115,29,116,117,34,118],"delegate","zcode","zcode delegate","review","cli","orchestration","tasks","run","Bash","model","node","read","orchestrator","files","desktop app","app","agent-orchestration",{"skillNo":120,"name":121,"nameZh":6,"description":122,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":123,"categories":124,"keywords":125,"publishedAt":38},"01M6Q0ETK41VGNMYEXW0T00AJN","youtube-transcript","Fetch YouTube transcripts through DeepAPI or local fallback tooling and save clean text output.","https:\u002F\u002Fgithub.com\u002Fduc-duy-1612\u002FOnline-Delivery-Systems-Web\u002Fblob\u002Fmain\u002F.agents\u002Fskills_archive\u002Fyoutube-transcript\u002FSKILL.md",[],[126,127,128,129,130,131,132,133,134,135,136,137,138,29,30,139,140,141,142,143],"transcript","youtube transcript","youtube","safe","github","codex","research","license","https","text","fetch","tooling","clean","local","output","deepapi","youtube transcripts","blob",{"skillNo":145,"name":146,"nameZh":6,"description":147,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":148,"categories":149,"keywords":161,"publishedAt":38},"01M6Q0EPPH2PVMDW40017F8WX0","youtube-transcript-skills","Fetch YouTube video transcripts, search videos\u002Fchannels, browse channels, and extract playlists via the getyoutubetranscript.com API - free tier, no card required.","https:\u002F\u002Fgithub.com\u002Fduc-duy-1612\u002FOnline-Delivery-Systems-Web\u002Fblob\u002Fmain\u002F.agents\u002Fskills_archive\u002Fyoutube-transcript-skills\u002FSKILL.md",[150,153,156],{"categoryNo":151,"categoryName":152,"taxonomyType":81,"taxonomyTypeDesc":82},"01JBSEEDCAT000000000000010","后端与接口",{"categoryNo":154,"categoryName":155,"taxonomyType":81,"taxonomyTypeDesc":82},"01JBSEEDCAT000000000000016","图像与多媒体",{"categoryNo":157,"categoryName":158,"taxonomyType":159,"taxonomyTypeDesc":160},"01JBSEEDCAT000000000000030","开发工程师","PROFESSION","适用职业",[126,127,128,162,129,163,130,131,133,164,165,166,167,136,168,29,30,169,170,171],"youtube transcript skills","gemini","search","api","Video","browse","api-integration","free","card","channels",{"skillNo":173,"name":174,"nameZh":6,"description":175,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":176,"categories":177,"keywords":182,"publishedAt":38},"01M6Q0EAB96PX83W5PTE8A0R8D","youtube-notetaker","Turn YouTube talks into local study notes with slides, transcripts, editable annotations, and a markdown-backed viewer.","https:\u002F\u002Fgithub.com\u002Fduc-duy-1612\u002FOnline-Delivery-Systems-Web\u002Fblob\u002Fmain\u002F.agents\u002Fskills_archive\u002Fyoutube-notetaker\u002FSKILL.md",[178,181],{"categoryNo":179,"categoryName":180,"taxonomyType":81,"taxonomyTypeDesc":82},"01JBSEEDCAT000000000000003","演示文稿",{"categoryNo":154,"categoryName":155,"taxonomyType":81,"taxonomyTypeDesc":82},[128,183,184,106,129,185,186,187,188,133,166,189,190,191,192,30,193,139,194,195],"youtube notetaker","notetaker","workflow","official","notes","code","markdown","slides","annotations","README","study","dair-academy","viewer",{"skillNo":197,"name":198,"nameZh":6,"description":199,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":200,"categories":201,"keywords":203,"publishedAt":38},"01M6Q0E55F0GW5MX48V8HXS8B5","youtube-full","Fetch YouTube transcripts, search videos, browse channels, and extract playlists via TranscriptAPI — no yt-dlp, no Google API key, works from any cloud server.","https:\u002F\u002Fgithub.com\u002Fduc-duy-1612\u002FOnline-Delivery-Systems-Web\u002Fblob\u002Fmain\u002F.agents\u002Fskills_archive\u002Fyoutube-full\u002FSKILL.md",[202],{"categoryNo":151,"categoryName":152,"taxonomyType":81,"taxonomyTypeDesc":82},[204,128,205,206,129,163,164,207,208,165,166,167,136,209,168,29,30,169,210,142],"youtube full","full","google","server","dlp","credits","automatic"]