[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"skill:01M6R5GSEPBHHD2NTHZGYDWJEQ":3,"skill-same:01M6R5GSEPBHHD2NTHZGYDWJEQ":68},{"skillNo":4,"name":5,"nameZh":6,"description":7,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":15,"categories":16,"keywords":22,"publishedAt":43,"linkOnly":44,"markdownContent":45,"markdownBody":46,"frontMatter":47,"refName":48,"commitSha":49,"skillDirectory":50,"forksCount":13,"repositoryHtmlUrl":51,"repositoryHomepageUrl":6,"ownerHtmlUrl":52,"ownerBlogUrl":6,"ownerDescription":6,"updatedAt":53,"install":54,"resources":67},"01M6R5GSEPBHHD2NTHZGYDWJEQ","monitor-services",null,"Health check and auto-restart all Pollinations GPU services (Flux on Vast.ai, Z-Image on RunPod, LTX-2 on GH200, Klein on RunPod, legacy image on OVH, Sana on Oracle Cloud). Use with \u002Floop for recurring checks.","APPLEPIE6969\u002FPollinations","APPLEPIE6969","https:\u002F\u002Favatars.githubusercontent.com\u002Fu\u002F242827480?v=4","01M6R5D0EAJN2DFJB3X6QTDANJ","01M6R5D0K2TSEDG15DK4M26BHH",0,"MIT","https:\u002F\u002Fgithub.com\u002FAPPLEPIE6969\u002FPollinations\u002Fblob\u002Fwan-aspect-ratio-fix\u002F.claude\u002Fskills\u002Fmonitor-services\u002FSKILL.md",[17],{"categoryNo":18,"categoryName":19,"taxonomyType":20,"taxonomyTypeDesc":21},"01JBSEEDCAT000000000000016","图像与多媒体","SCENARIO","使用场景",[23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42],"monitor services","monitor","services","use","for","with","ltx","pollinations","runpod","Legacy","health check","auto","image","recurring","check","loop","health","cloud","gpu","oracle","2026-11-10 10:12:30",false,"---\nname: monitor-services\ndescription: \"Health check and auto-restart all Pollinations GPU services (Flux on Vast.ai, Z-Image on RunPod, LTX-2 on GH200, Klein on RunPod, legacy image on OVH, Sana on Oracle Cloud). Use with \u002Floop for recurring checks.\"\n---\n\n# Monitor Services\n\nCheck health of all Pollinations GPU backend services and auto-restart if unhealthy.\n\n## Quick Start\n\n```\n\u002Floop 30m \u002Fmonitor-services\n```\n\nOr run once: `\u002Fmonitor-services`\n\n---\n\n## Services\n\n### 1. LTX-2.3 Video (GH200 - Lambda Labs)\n\n| Property | Value |\n|----------|-------|\n| **Host** | `192.222.51.105` |\n| **Port** | `8765` |\n| **Provider** | Lambda Labs (NVIDIA GH200) |\n| **SSH** | `ssh -i \u003CSSH_LAMBDA_SANA_LTX2_ACESTEP from SOPS> ubuntu@192.222.51.105` |\n\n**Health check:**\n```bash\ncurl -s --connect-timeout 5 --max-time 10 http:\u002F\u002F192.222.51.105:8765\u002Fhealth\n```\nExpected: `{\"status\":\"healthy\",\"model\":\"ltx-2-comfyui\"}`\n\n**E2E test (through prod):**\n```bash\ncurl -s -o \u002Fdev\u002Fnull -w \"HTTP %{http_code}, Size: %{size_download}, Time: %{time_total}s\" \\\n  --max-time 120 \\\n  \"https:\u002F\u002Fgen.pollinations.ai\u002Fvideo\u002Fhealth_check_$(date +%s)?model=ltx-2&duration=3\" \\\n  -H \"Authorization: Bearer $TEST_TOKEN\"\n```\nExpected: HTTP 200, ~500-800KB, ~11-13s\n\n**Restart:**\n```bash\nssh -i \u003CSOPS:SSH_LAMBDA_SANA_LTX2_ACESTEP> ubuntu@192.222.51.105 \"bash \u002Fhome\u002Fubuntu\u002Fstart_ltx2.sh\"\n```\nWait ~60s after restart, then re-check health.\n\n---\n\n### 2. ACE-Step Music (GH200 - same host as LTX-2)\n\n| Property | Value |\n|----------|-------|\n| **Host** | `192.222.51.105` |\n| **Port** | `8189` |\n| **SSH** | `ssh -i \u003CSSH_LAMBDA_SANA_LTX2_ACESTEP from SOPS> ubuntu@192.222.51.105` |\n| **Systemd** | `acestep.service` |\n| **Auth** | `ACESTEP_API_KEY` env var (Bearer token) |\n\n**Health check:**\n```bash\ncurl -s --connect-timeout 5 --max-time 10 http:\u002F\u002F192.222.51.105:8189\u002Fhealth\n```\nExpected: `{\"status\":\"ok\",\"models_initialized\":true}`\n\n**Restart:**\n```bash\nssh -i \u003CSOPS:SSH_LAMBDA_SANA_LTX2_ACESTEP> ubuntu@192.222.51.105 \"sudo systemctl restart acestep\"\n```\nWait ~50s for model initialization, then re-check health.\n\n**Notes:**\n- Token auth via `Authorization: Bearer \u003Ctoken>` — token stored in encrypted secrets as `PLN_GPU_TOKEN`\n- Server-side token set via `ACESTEP_API_KEY` env var in systemd unit\n- Runs on port 8189 (port 8188 is ComfyUI\u002FLTX-2)\n\n---\n\n### 3. Legacy Image Service (OVH)\n\n| Property | Value |\n|----------|-------|\n| **Host** | `57.130.31.42` |\n| **Port** | `16384` |\n| **SSH** | `ssh -i ~\u002F.ssh\u002Fid_rsa_ovh ubuntu@57.130.31.42` |\n| **Branch** | `master` (separate from main) |\n\n**Health check:**\n```bash\nssh -i ~\u002F.ssh\u002Fid_rsa_ovh -o ConnectTimeout=5 ubuntu@57.130.31.42 \"systemctl is-active image-pollinations\"\n```\nExpected: `active`\n\n**Restart:**\n```bash\nssh -i ~\u002F.ssh\u002Fid_rsa_ovh ubuntu@57.130.31.42 \"sudo systemctl restart image-pollinations\"\n```\n\n---\n\n### 4. Klein 4B (RunPod Pod)\n\n| Property | Value |\n|----------|-------|\n| **Pod ID** | `jmrbmje2fyuy46` (current — pod ID changes if recreated) |\n| **Host** | `\u003Cpod-id>-8000.proxy.runpod.net` |\n| **Port** | `8000` |\n| **Provider** | RunPod (RTX A5000 secure cloud, $0.27\u002Fhr via API) |\n| **SSH** | Full SSH with `SSH_RUNPOD_KLEIN` from SOPS; get current public host\u002Fport from RunPod runtime ports |\n| **Auth** | `x-backend-token` header with `PLN_GPU_TOKEN` |\n| **Config** | `KLEIN_URL` in `gen.pollinations.ai\u002Fsecrets\u002Fprod.vars.json` (sops); fallback in `gen.pollinations.ai\u002Fsrc\u002Fimage\u002Fmodels\u002FfluxKleinModel.ts` |\n\n**Health check:**\n```bash\ncurl -s --connect-timeout 5 --max-time 10 https:\u002F\u002Fjmrbmje2fyuy46-8000.proxy.runpod.net\u002Fhealth\n```\nExpected: `{\"status\":\"ok\",\"model\":\"black-forest-labs\u002FFLUX.2-klein-4B\"}`\n\n**Restart (in-pod):**\n```bash\n# Open SSH from dashboard, then:\nbash \u002Fworkspace\u002Frestart.sh\n```\nWait ~30s for model load, then re-check health.\n\n**Recovery from RunPod host outage:**\n\nSymptom: dashboard banner \"*This server has recently suffered a network outage*\"; control plane reports RUNNING but HTTPS proxy \u002F SSH \u002F ICMP all unreachable. Restart\u002Freset reschedules onto the same broken host. Recreate on a different host:\n\n1. Create cheap replacement capacity first: prefer secure A5000 at \u003C= $0.30\u002Fhr, fallback to community 3090 at \u003C= $0.23\u002Fhr. Reject any assigned host with `machine.note` or `maintenanceNote` mentioning an outage. Do not use a 4090 unless the higher cost is explicitly accepted.\n2. Use `runpod\u002Fpytorch:1.0.2-cu1281-torch280-ubuntu2404`, `8000\u002Fhttp,22\u002Ftcp`, `20GB` container disk, `100GB` volume, and `\u002Fworkspace`.\n3. Get SSH runtime port from RunPod GraphQL and connect with `SSH_RUNPOD_KLEIN`:\n   ```bash\n   sops -d enter.pollinations.ai\u002Fsecrets\u002Fprod.vars.json | jq -r .SSH_RUNPOD_KLEIN > \u002Ftmp\u002Fklein-key\n   chmod 600 \u002Ftmp\u002Fklein-key\n   ssh -i \u002Ftmp\u002Fklein-key -p \u003CpublicPort> root@\u003CpublicIp>\n   ```\n4. Copy `image.pollinations.ai\u002Fklein-runpod\u002Fhandler.py` and `requirements.txt` to `\u002Fworkspace`.\n5. Install runtime packages without replacing the base CUDA torch:\n   ```bash\n   python -m venv --system-site-packages \u002Fworkspace\u002Fvenv\n   \u002Fworkspace\u002Fvenv\u002Fbin\u002Fpython -m pip install --upgrade pip\n   \u002Fworkspace\u002Fvenv\u002Fbin\u002Fpython -m pip install --no-cache-dir -r \u002Fworkspace\u002Frequirements.txt\n   ```\n6. Put `PLN_GPU_TOKEN` in `\u002Froot\u002F.gpu_token` (not `\u002Fworkspace`, which may ignore Unix mode bits) and create `\u002Fworkspace\u002Frestart.sh`:\n   ```bash\n   export HF_HUB_CACHE=\u002Fworkspace\u002Fhf-cache\n   export HF_XET_HIGH_PERFORMANCE=1\n   export PLN_GPU_TOKEN=\"$(cat \u002Froot\u002F.gpu_token)\"\n   pkill -f \"\u002Fworkspace\u002Fhandler.py\" 2>\u002Fdev\u002Fnull || true\n   nohup \u002Fworkspace\u002Fvenv\u002Fbin\u002Fpython -u \u002Fworkspace\u002Fhandler.py > \u002Fworkspace\u002Fklein.log 2>&1 &\n   echo $! > \u002Fworkspace\u002Fklein.pid\n   ```\n7. Run `bash \u002Fworkspace\u002Frestart.sh`; first startup downloads ~15-24GB of model files and should end with `Model loaded and ready!`.\n8. Update `KLEIN_URL` in `gen.pollinations.ai\u002Fsecrets\u002Fprod.vars.json` and the hardcoded fallback in `gen.pollinations.ai\u002Fsrc\u002Fimage\u002Fmodels\u002FfluxKleinModel.ts`.\n9. Push the production Worker secret:\n   ```bash\n   tmp=$(mktemp)\n   sops -d gen.pollinations.ai\u002Fsecrets\u002Fprod.vars.json > \"$tmp\"\n   (cd gen.pollinations.ai && node scripts\u002Fpush-generation-secrets.mjs \"$tmp\" production)\n   rm -f \"$tmp\"\n   ```\n10. Verify direct pod `\u002Fhealth`, direct authenticated `\u002Fgenerate`, and production `gen.pollinations.ai\u002Fimage\u002F...model=klein`, remove `\u002Ftmp\u002Fklein-key`, then terminate the old outage pod.\n\nNote: the pod uses a generic `runpod\u002Fpytorch` image; `handler.py` and `restart.sh` live on the pod volume only (not baked into a Docker image despite `image.pollinations.ai\u002Fklein-runpod\u002FDockerfile`). The pod volume is destroyed on terminate.\n\n---\n\n### 5. Z-Image Workers (RunPod, multiple single-GPU pods)\n\n> ⚠️ **Pod IDs, hostnames, SSH ports, and the worker count all change over time — DISCOVER them live, never trust hardcoded values here.** As of 2026-06-16, zimage runs as **3 separate single-GPU pods** (one 4090 + two 3090s), each serving on port 8767. The old `hsl3ksl31lvrcc` 4x-4090 pod is gone. Flux runs on Vast.ai (§8) — unrelated to zimage.\n\n**Step 1 — discover what's actually deployed (the source of truth):**\n```bash\n# (a) Who is registered + heartbeating (= actually taking prod traffic):\ncurl -s --max-time 10 https:\u002F\u002Fgen.pollinations.ai\u002Fregister | python3 -c \"import sys,json,time; [print(f\\\"{w['type']:\u003C7} {w['url']} hb={(time.time()*1000-w['lastHeartbeat'])\u002F1000:.0f}s\\\") for w in json.load(sys.stdin)]\"\n\n# (b) ALL RunPod pods + their GPU util (catch idle\u002Fdead pods NOT in the registry):\nRUNPOD_TOKEN=$(cat ~\u002F.runpod\u002Fconfig.toml | grep apikey | cut -d\\' -f2)\nrunpodctl pod list\ncurl -s -X POST \"https:\u002F\u002Fapi.runpod.io\u002Fgraphql?api_key=$RUNPOD_TOKEN\" -H \"Content-Type: application\u002Fjson\" \\\n  -d '{\"query\":\"{myself{pods{id name desiredStatus machine{gpuDisplayName} runtime{gpus{gpuUtilPercent}}}}}\"}' \\\n  | python3 -c \"import sys,json; [print(p['name'], p['id'], p['desiredStatus'], (p.get('runtime') or {}).get('gpus')) for p in json.load(sys.stdin)['data']['myself']['pods']]\"\n```\n**Key failure mode (this caused the 2026-06-16 incident):** a pod can be RUNNING + costing money but **0% util and absent from `\u002Fregister`** because its `server.py` died or its GPU fell off the bus. All traffic then piles onto the remaining worker(s) → 524 timeouts + 100% util on the survivor. Always cross-check (a) vs (b): every RUNNING zimage pod should appear in the registry.\n\n**SSH key:** the working key is **`SSH_RUNPOD_KLEIN`** (the documented `SSH_RUNPOD_FLUX_ZIMAGE` does NOT auth against these pods). Get the rotating SSH port per pod:\n```bash\nsops -d enter.pollinations.ai\u002Fsecrets\u002Fprod.vars.json | jq -r .SSH_RUNPOD_KLEIN > \u002Ftmp\u002Fzk; chmod 600 \u002Ftmp\u002Fzk\ncurl -s -X POST \"https:\u002F\u002Fapi.runpod.io\u002Fgraphql?api_key=$RUNPOD_TOKEN\" -H \"Content-Type: application\u002Fjson\" \\\n  -d '{\"query\":\"{pod(input:{podId:\\\"\u003CPOD_ID>\\\"}){runtime{ports{ip publicPort type}}}}\"}' \\\n  | python3 -c \"import sys,json; [print(p) for p in json.load(sys.stdin)['data']['pod']['runtime']['ports'] if p['type']=='tcp']\"\n```\n\n**Diagnose a single pod (SSH in on the discovered tcp port):**\n```bash\nssh -i \u002Ftmp\u002Fzk -p \u003CPORT> -o StrictHostKeyChecking=no root@\u003CIP> \\\n  \"nvidia-smi --query-gpu=utilization.gpu,memory.used --format=csv,noheader; ps aux | grep 'z-image\u002Fserver.py' | grep -v grep; ss -ltn | grep 8767\"\n```\nCode lives at `\u002Froot\u002Fpollinations\u002Fimage.pollinations.ai\u002Fz-image\u002F`; service auto-sources env from PID 1 (port 8767, `ZIMAGE_MODEL_ID`, `PLN_GPU_TOKEN`, `HF_TOKEN`). Direct `curl \u002Fgenerate` returns **403 without the GPU token** — a 403 still proves the server is up.\n\n**Recovery decision tree:**\n\n| Symptom | Fix |\n|---------|-----|\n| `server.py` not running, but `nvidia-smi` works | **Process relaunch** (cheap): `ssh ... \"bash \u002Froot\u002Frelaunch-zimage.sh\"` — kills + relaunches via `\u002Froot\u002Flaunch.sh`, survives SSH disconnect. Verify `Heartbeat sent successfully` in `\u002Froot\u002Flogs\u002Fzimage.log` and that the pod reappears in `\u002Fregister`. |\n| `nvidia-smi` → `Failed to initialize NVML: Unknown Error` (`\u002Fdev\u002Fnvidia*` owned by `nobody`) | **GPU off the bus — needs pod stop\u002Fstart** (NOT recoverable in-container). See below. |\n| Worker in `\u002Fregister` + heartbeating, but **524 timeouts route only to that one host** (and it's slow \u002F inference rate degrades mid-request \u002F huge RSS from long uptime) | **Stale degraded process — relaunch it, but confirm the OLD pid actually died.** A heartbeat thread keeps firing even when generation hangs, so the pod looks healthy. See pkill caveat below. |\n| All workers up + 100% util + still 524s | **Genuine capacity deficit** — add a GPU. Note 401\u002F402 are rejected at the edge (~100ms) and do NOT load the GPU, so it's real paid demand, not abuse. |\n\n> ⚠️ **`relaunch-zimage.sh` pkill caveat (2026-06-16):** the script kills by pattern `pkill -f 'z-image\u002Fserver.py'`, but the running process command is often just `python server.py` (launched from within the z-image dir), so the pattern **does not match** and the relaunch silently no-ops — the new process then fails with `address already in use` and exits while the old, degraded process keeps serving. **Always verify the old PID actually died** (`ps -o pid,etime -C python` — elapsed time should reset to seconds). If it didn't: `kill -9 \u003Cpid>; fuser -k 8767\u002Ftcp; bash \u002Froot\u002Frelaunch-zimage.sh`.\n\n**Pod stop\u002Fstart (for NVML-broken GPU) — WIPES the container overlay disk.** Code is on `\u002Froot` (overlay), there is **no persistent `\u002Fworkspace` volume**. So back up the provision script first, then rebuild:\n```bash\n# 1. Back up provision script BEFORE stopping:\nssh -i \u002Ftmp\u002Fzk -p \u003CPORT> root@\u003CIP> \"cat \u002Froot\u002Fprovision-zimage.sh\" > \u002Ftmp\u002Fprovision-zimage.sh\n# 2. Stop + start (port rotates; GPU re-attaches on fresh container):\nrunpodctl pod stop \u003CPOD_ID>;  sleep 10;  runpodctl pod start \u003CPOD_ID>\n# 3. Wait for RUNNING + tcp port, confirm GPU healthy (nvidia-smi shows the card), then re-provision:\nscp -i \u002Ftmp\u002Fzk -P \u003CNEW_PORT> \u002Ftmp\u002Fprovision-zimage.sh root@\u003CIP>:\u002Froot\u002F\nssh -i \u002Ftmp\u002Fzk -p \u003CNEW_PORT> root@\u003CIP> \\\n  \"export POD_ID=\u003CPOD_ID>; setsid bash \u002Froot\u002Fprovision-zimage.sh > \u002Froot\u002Fprovision.log 2>&1 \u003C\u002Fdev\u002Fnull & disown\"\n# provision: re-clones branch feat\u002Fzimage-nf4-model-option, builds venv, downloads model (~9GB) + SPAN\n#            upscaler, writes launch.sh, starts server. ~3-6 min. Tokens come from PID1 env (auto re-injected).\n# 4. Verify: server LISTENING :8767, model loaded, heartbeat sent, pod appears in \u002Fregister, e2e 200 via prod.\n```\n\n**Verify recovery end-to-end (prod success + load now shared):**\n```bash\nTOKEN=$(grep ENTER_API_TOKEN_REMOTE enter.pollinations.ai\u002F.testingtokens | cut -d= -f2)\nfor i in 1 2 3 4 5; do curl -s -o \u002Fdev\u002Fnull -w \"HTTP %{http_code}\\n\" --max-time 60 \\\n  \"https:\u002F\u002Fgen.pollinations.ai\u002Fimage\u002Fverify_${i}_$(date +%s%N)?model=zimage&width=512&height=512&seed=$i\" \\\n  -H \"Authorization: Bearer $TOKEN\"; done\n# Then confirm the 524 trend dropped in Tinybird (model_health \u002F generation_event_v2, model_requested='zimage').\n```\n\nThe registry is Cloudflare KV-backed (`image:server:\u003Cenv>:\u003Ctype>:\u003Chash>`, 240s TTL); workers heartbeat to `gen.pollinations.ai\u002Fregister`.\n\n---\n\n### 6. Sana Sprint 1.6B Worker (GH200 - same host as LTX-2)\n\nOne worker registered as `sana` type with OVH legacy service via heartbeat.\n\n| Instance | GPU | Host | Port | SSH |\n|----------|-----|------|------|-----|\n| Lambda GH200 | GH200 (96GB) | `192.222.51.105` | `8766` | `ssh -i \u003CSOPS:SSH_LAMBDA_SANA_LTX2_ACESTEP> ubuntu@192.222.51.105` |\n\n**Health check:**\n```bash\ncurl -s --connect-timeout 5 --max-time 10 http:\u002F\u002F192.222.51.105:8766\u002Fhealth\n```\nExpected: `{\"status\":\"healthy\",\"model\":\"Efficient-Large-Model\u002FSana_Sprint_1.6B_1024px_diffusers\"}`\n\n**Sana registry check (OVH side):**\n```bash\nssh -i ~\u002F.ssh\u002Fid_rsa_ovh -o ConnectTimeout=5 ubuntu@57.130.31.42 \"curl -s http:\u002F\u002Flocalhost:16384\u002Fregister\"\n```\nExpected: 1 worker with 0% error rate\n\n**Restart:**\n```bash\nssh -i \u003CSOPS:SSH_LAMBDA_SANA_LTX2_ACESTEP> ubuntu@192.222.51.105 \"sudo systemctl restart sana\"\n```\n\n**Notes:**\n- GH200 generates at ~0.165s\u002Fimg\n- Runs alongside LTX-2 (port 8765) and ACE-Step (port 8189) on the same host\n- Oracle A10\u002FA100 instances decommissioned on 2026-04-12\n- Server code: `image.pollinations.ai\u002Fsana\u002Fserver.py` (MAX_DIM=768, MAX_PIXELS=512*512)\n- Systemd service: `sana.service`\n\n---\n\n### 7. OVH Disk Space\n\n**Check:**\n```bash\nssh -i ~\u002F.ssh\u002Fid_rsa_ovh -o ConnectTimeout=5 ubuntu@57.130.31.42 \"df -h \u002F | tail -1\"\n```\nAlert if usage > 85%.\n\n**Fix (if full):**\n```bash\nssh -i ~\u002F.ssh\u002Fid_rsa_ovh ubuntu@57.130.31.42 \"sudo truncate -s 0 \u002Fvar\u002Flog\u002Fsyslog && sudo journalctl --vacuum-size=100M\"\n```\n\n---\n\n### 8. Flux Workers (Vast.ai RTX 5090s)\n\nFlux is self-hosted on Vast.ai 5090s since 2026-07-02 (PR #12171) with automatic\nFireworks fallback: the worker sheds load with 503 beyond `QUEUE_LIMIT`, and the\ngateway retries those requests on Fireworks. Instance coordinates, provisioning\none-liner, and host gotchas live in `image.pollinations.ai\u002FGPU_INSTANCES.md`\n(source of truth — instance IDs\u002FIPs\u002Fports change on recreate).\n\n**Check:**\n```bash\n# flux worker(s) registered — the `url` field is the tunnel to probe, and\n# `lastMs` is the last request's duration (a healthy 5090 is single-digit\n# seconds; tens of seconds means the tunnel is degrading, see below):\ncurl -s https:\u002F\u002Fgen.pollinations.ai\u002Fregister | grep -o '\"type\":\"flux\"[^}]*'\n# then probe THAT url (expect 200 in ~1s):\ncurl -s -o \u002Fdev\u002Fnull -w \"%{http_code} %{time_total}s\\n\" \"\u003Curl-from-registry>\u002Fdocs\"\n```\n\nRegistry health is NOT data-path health: the worker heartbeats fine even when\nthe tunnel in front of it is too slow to serve. Always time the probe. If\n`\u002Fdocs` (a static page) takes >5s, the tunnel is the problem, not the GPU —\ncompare against the worker on localhost via SSH to confirm.\n\n**Pool vs Fireworks split** — invisible in Tinybird (`model_provider_used` is\nstatic registry data); read it from the worker log (SSH coords from\nGPU_INSTANCES.md \u002F `vastai show instances`):\n```bash\nssh -p \u003CPORT> -i ~\u002F.ssh\u002Fpollinations_services_2026 root@\u003CIP> \\\n  'grep -c \"\\\" 200\" \u002Ftmp\u002Fflux.log; grep -c \"\\\" 503\" \u002Ftmp\u002Fflux.log'   # served \u002F shed\n```\n\n**Fix:** on the instance, `screen -r flux` and `screen -r cloudflared` run in\nrestart loops (logs `\u002Ftmp\u002Fflux.log`, `\u002Ftmp\u002Fcloudflared.log`); restart commands in\nGPU_INSTANCES.md. If flux is fully down, all traffic falls back to Fireworks\nautomatically — users are unaffected, but it costs ~$70\u002Fday, so fix promptly.\nVast SSH gotcha: long-lived sessions get dropped — use short commands, never\n`sleep` inside a session.\n\n---\n\n## Procedure\n\nWhen invoked, run checks in this order:\n\n1. **gen.pollinations.ai registry** - `curl https:\u002F\u002Fgen.pollinations.ai\u002Fregister` (KV-backed), check worker count and error rates\n2. **Flux (Vast.ai) + Z-Image (RunPod)** - flux worker registered + tunnel healthy (§8); every RUNNING zimage pod registered (§5)\n3. **LTX-2 health** - curl health endpoint\n4. **LTX-2 e2e** - if healthy, test through gen.pollinations.ai (use test token from `.testingtokens`)\n5. **ACE-Step health** - curl health endpoint on port 8189\n6. **Klein health** - curl RunPod proxy health endpoint\n7. **Legacy image service** - check systemctl status on OVH\n8. **Sana worker** - curl health on GH200 port 8766\n9. **Sana registry** - check OVH legacy registry for 1 worker with 0% errors\n10. **Disk space** - check OVH disk usage\n\nFor each:\n- If healthy: report OK with latency\n- If unhealthy: attempt restart, wait, re-check, report result\n\n## Auth\n\n- **Test token**: Read from `enter.pollinations.ai\u002F.testingtokens` (ENTER_API_TOKEN_REMOTE)\n- **SSH keys**: Stored in SOPS (`enter.pollinations.ai\u002Fsecrets\u002Fprod.vars.json`):\n  - `SSH_LAMBDA_SANA_LTX2_ACESTEP` — Lambda GH200 (LTX-2, ACE-Step, Sana)\n  - Vast.ai flux workers use the local key `~\u002F.ssh\u002Fpollinations_services_2026` (attached per-instance via `vastai attach ssh`; not in SOPS)\n  - Klein uses the RunPod relay (`ssh.runpod.io`) with `~\u002F.ssh\u002Fid_ed25519` — get the full command from the dashboard \"Connect\" tab\n  - Extract: `sops -d enter.pollinations.ai\u002Fsecrets\u002Fprod.vars.json | jq -r '.KEY_NAME' > \u002Ftmp\u002Fkey && chmod 600 \u002Ftmp\u002Fkey`\n- **OVH**: `~\u002F.ssh\u002Fid_rsa_ovh` (not in SOPS)\n\n## Output\n\nReport a brief status table:\n\n```\n| Service | Status | Latency | Notes |\n|---------|--------|---------|-------|\n| gen registry | OK | 0.1s | 4 workers, 0% errors |\n| Flux Vast (flux-vast-01) | OK | 2.3s | tunnel 200, 1.2% shed to Fireworks |\n| Z-Image RunPod ×3 | OK | 1.5s | pods discovered via runpodctl |\n| LTX-2 health | OK | 0.2s | |\n| LTX-2 e2e | OK | 11.3s | 682KB |\n| ACE-Step | OK | 0.1s | |\n| Klein 4B | OK | 0.3s | RunPod |\n| Legacy image | OK | - | active |\n| Sana (GH200) | OK | 0.2s | 1.6B, ~0.165s\u002Fimg |\n| Sana registry | OK | - | 1 worker, 0% errors |\n| OVH disk | OK | - | 45% used |\n```\n","# Monitor Services\n\nCheck health of all Pollinations GPU backend services and auto-restart if unhealthy.\n\n## Quick Start\n\n```\n\u002Floop 30m \u002Fmonitor-services\n```\n\nOr run once: `\u002Fmonitor-services`\n\n---\n\n## Services\n\n### 1. LTX-2.3 Video (GH200 - Lambda Labs)\n\n| Property | Value |\n|----------|-------|\n| **Host** | `192.222.51.105` |\n| **Port** | `8765` |\n| **Provider** | Lambda Labs (NVIDIA GH200) |\n| **SSH** | `ssh -i \u003CSSH_LAMBDA_SANA_LTX2_ACESTEP from SOPS> ubuntu@192.222.51.105` |\n\n**Health check:**\n```bash\ncurl -s --connect-timeout 5 --max-time 10 http:\u002F\u002F192.222.51.105:8765\u002Fhealth\n```\nExpected: `{\"status\":\"healthy\",\"model\":\"ltx-2-comfyui\"}`\n\n**E2E test (through prod):**\n```bash\ncurl -s -o \u002Fdev\u002Fnull -w \"HTTP %{http_code}, Size: %{size_download}, Time: %{time_total}s\" \\\n  --max-time 120 \\\n  \"https:\u002F\u002Fgen.pollinations.ai\u002Fvideo\u002Fhealth_check_$(date +%s)?model=ltx-2&duration=3\" \\\n  -H \"Authorization: Bearer $TEST_TOKEN\"\n```\nExpected: HTTP 200, ~500-800KB, ~11-13s\n\n**Restart:**\n```bash\nssh -i \u003CSOPS:SSH_LAMBDA_SANA_LTX2_ACESTEP> ubuntu@192.222.51.105 \"bash \u002Fhome\u002Fubuntu\u002Fstart_ltx2.sh\"\n```\nWait ~60s after restart, then re-check health.\n\n---\n\n### 2. ACE-Step Music (GH200 - same host as LTX-2)\n\n| Property | Value |\n|----------|-------|\n| **Host** | `192.222.51.105` |\n| **Port** | `8189` |\n| **SSH** | `ssh -i \u003CSSH_LAMBDA_SANA_LTX2_ACESTEP from SOPS> ubuntu@192.222.51.105` |\n| **Systemd** | `acestep.service` |\n| **Auth** | `ACESTEP_API_KEY` env var (Bearer token) |\n\n**Health check:**\n```bash\ncurl -s --connect-timeout 5 --max-time 10 http:\u002F\u002F192.222.51.105:8189\u002Fhealth\n```\nExpected: `{\"status\":\"ok\",\"models_initialized\":true}`\n\n**Restart:**\n```bash\nssh -i \u003CSOPS:SSH_LAMBDA_SANA_LTX2_ACESTEP> ubuntu@192.222.51.105 \"sudo systemctl restart acestep\"\n```\nWait ~50s for model initialization, then re-check health.\n\n**Notes:**\n- Token auth via `Authorization: Bearer \u003Ctoken>` — token stored in encrypted secrets as `PLN_GPU_TOKEN`\n- Server-side token set via `ACESTEP_API_KEY` env var in systemd unit\n- Runs on port 8189 (port 8188 is ComfyUI\u002FLTX-2)\n\n---\n\n### 3. Legacy Image Service (OVH)\n\n| Property | Value |\n|----------|-------|\n| **Host** | `57.130.31.42` |\n| **Port** | `16384` |\n| **SSH** | `ssh -i ~\u002F.ssh\u002Fid_rsa_ovh ubuntu@57.130.31.42` |\n| **Branch** | `master` (separate from main) |\n\n**Health check:**\n```bash\nssh -i ~\u002F.ssh\u002Fid_rsa_ovh -o ConnectTimeout=5 ubuntu@57.130.31.42 \"systemctl is-active image-pollinations\"\n```\nExpected: `active`\n\n**Restart:**\n```bash\nssh -i ~\u002F.ssh\u002Fid_rsa_ovh ubuntu@57.130.31.42 \"sudo systemctl restart image-pollinations\"\n```\n\n---\n\n### 4. Klein 4B (RunPod Pod)\n\n| Property | Value |\n|----------|-------|\n| **Pod ID** | `jmrbmje2fyuy46` (current — pod ID changes if recreated) |\n| **Host** | `\u003Cpod-id>-8000.proxy.runpod.net` |\n| **Port** | `8000` |\n| **Provider** | RunPod (RTX A5000 secure cloud, $0.27\u002Fhr via API) |\n| **SSH** | Full SSH with `SSH_RUNPOD_KLEIN` from SOPS; get current public host\u002Fport from RunPod runtime ports |\n| **Auth** | `x-backend-token` header with `PLN_GPU_TOKEN` |\n| **Config** | `KLEIN_URL` in `gen.pollinations.ai\u002Fsecrets\u002Fprod.vars.json` (sops); fallback in `gen.pollinations.ai\u002Fsrc\u002Fimage\u002Fmodels\u002FfluxKleinModel.ts` |\n\n**Health check:**\n```bash\ncurl -s --connect-timeout 5 --max-time 10 https:\u002F\u002Fjmrbmje2fyuy46-8000.proxy.runpod.net\u002Fhealth\n```\nExpected: `{\"status\":\"ok\",\"model\":\"black-forest-labs\u002FFLUX.2-klein-4B\"}`\n\n**Restart (in-pod):**\n```bash\n# Open SSH from dashboard, then:\nbash \u002Fworkspace\u002Frestart.sh\n```\nWait ~30s for model load, then re-check health.\n\n**Recovery from RunPod host outage:**\n\nSymptom: dashboard banner \"*This server has recently suffered a network outage*\"; control plane reports RUNNING but HTTPS proxy \u002F SSH \u002F ICMP all unreachable. Restart\u002Freset reschedules onto the same broken host. Recreate on a different host:\n\n1. Create cheap replacement capacity first: prefer secure A5000 at \u003C= $0.30\u002Fhr, fallback to community 3090 at \u003C= $0.23\u002Fhr. Reject any assigned host with `machine.note` or `maintenanceNote` mentioning an outage. Do not use a 4090 unless the higher cost is explicitly accepted.\n2. Use `runpod\u002Fpytorch:1.0.2-cu1281-torch280-ubuntu2404`, `8000\u002Fhttp,22\u002Ftcp`, `20GB` container disk, `100GB` volume, and `\u002Fworkspace`.\n3. Get SSH runtime port from RunPod GraphQL and connect with `SSH_RUNPOD_KLEIN`:\n   ```bash\n   sops -d enter.pollinations.ai\u002Fsecrets\u002Fprod.vars.json | jq -r .SSH_RUNPOD_KLEIN > \u002Ftmp\u002Fklein-key\n   chmod 600 \u002Ftmp\u002Fklein-key\n   ssh -i \u002Ftmp\u002Fklein-key -p \u003CpublicPort> root@\u003CpublicIp>\n   ```\n4. Copy `image.pollinations.ai\u002Fklein-runpod\u002Fhandler.py` and `requirements.txt` to `\u002Fworkspace`.\n5. Install runtime packages without replacing the base CUDA torch:\n   ```bash\n   python -m venv --system-site-packages \u002Fworkspace\u002Fvenv\n   \u002Fworkspace\u002Fvenv\u002Fbin\u002Fpython -m pip install --upgrade pip\n   \u002Fworkspace\u002Fvenv\u002Fbin\u002Fpython -m pip install --no-cache-dir -r \u002Fworkspace\u002Frequirements.txt\n   ```\n6. Put `PLN_GPU_TOKEN` in `\u002Froot\u002F.gpu_token` (not `\u002Fworkspace`, which may ignore Unix mode bits) and create `\u002Fworkspace\u002Frestart.sh`:\n   ```bash\n   export HF_HUB_CACHE=\u002Fworkspace\u002Fhf-cache\n   export HF_XET_HIGH_PERFORMANCE=1\n   export PLN_GPU_TOKEN=\"$(cat \u002Froot\u002F.gpu_token)\"\n   pkill -f \"\u002Fworkspace\u002Fhandler.py\" 2>\u002Fdev\u002Fnull || true\n   nohup \u002Fworkspace\u002Fvenv\u002Fbin\u002Fpython -u \u002Fworkspace\u002Fhandler.py > \u002Fworkspace\u002Fklein.log 2>&1 &\n   echo $! > \u002Fworkspace\u002Fklein.pid\n   ```\n7. Run `bash \u002Fworkspace\u002Frestart.sh`; first startup downloads ~15-24GB of model files and should end with `Model loaded and ready!`.\n8. Update `KLEIN_URL` in `gen.pollinations.ai\u002Fsecrets\u002Fprod.vars.json` and the hardcoded fallback in `gen.pollinations.ai\u002Fsrc\u002Fimage\u002Fmodels\u002FfluxKleinModel.ts`.\n9. Push the production Worker secret:\n   ```bash\n   tmp=$(mktemp)\n   sops -d gen.pollinations.ai\u002Fsecrets\u002Fprod.vars.json > \"$tmp\"\n   (cd gen.pollinations.ai && node scripts\u002Fpush-generation-secrets.mjs \"$tmp\" production)\n   rm -f \"$tmp\"\n   ```\n10. Verify direct pod `\u002Fhealth`, direct authenticated `\u002Fgenerate`, and production `gen.pollinations.ai\u002Fimage\u002F...model=klein`, remove `\u002Ftmp\u002Fklein-key`, then terminate the old outage pod.\n\nNote: the pod uses a generic `runpod\u002Fpytorch` image; `handler.py` and `restart.sh` live on the pod volume only (not baked into a Docker image despite `image.pollinations.ai\u002Fklein-runpod\u002FDockerfile`). The pod volume is destroyed on terminate.\n\n---\n\n### 5. Z-Image Workers (RunPod, multiple single-GPU pods)\n\n> ⚠️ **Pod IDs, hostnames, SSH ports, and the worker count all change over time — DISCOVER them live, never trust hardcoded values here.** As of 2026-06-16, zimage runs as **3 separate single-GPU pods** (one 4090 + two 3090s), each serving on port 8767. The old `hsl3ksl31lvrcc` 4x-4090 pod is gone. Flux runs on Vast.ai (§8) — unrelated to zimage.\n\n**Step 1 — discover what's actually deployed (the source of truth):**\n```bash\n# (a) Who is registered + heartbeating (= actually taking prod traffic):\ncurl -s --max-time 10 https:\u002F\u002Fgen.pollinations.ai\u002Fregister | python3 -c \"import sys,json,time; [print(f\\\"{w['type']:\u003C7} {w['url']} hb={(time.time()*1000-w['lastHeartbeat'])\u002F1000:.0f}s\\\") for w in json.load(sys.stdin)]\"\n\n# (b) ALL RunPod pods + their GPU util (catch idle\u002Fdead pods NOT in the registry):\nRUNPOD_TOKEN=$(cat ~\u002F.runpod\u002Fconfig.toml | grep apikey | cut -d\\' -f2)\nrunpodctl pod list\ncurl -s -X POST \"https:\u002F\u002Fapi.runpod.io\u002Fgraphql?api_key=$RUNPOD_TOKEN\" -H \"Content-Type: application\u002Fjson\" \\\n  -d '{\"query\":\"{myself{pods{id name desiredStatus machine{gpuDisplayName} runtime{gpus{gpuUtilPercent}}}}}\"}' \\\n  | python3 -c \"import sys,json; [print(p['name'], p['id'], p['desiredStatus'], (p.get('runtime') or {}).get('gpus')) for p in json.load(sys.stdin)['data']['myself']['pods']]\"\n```\n**Key failure mode (this caused the 2026-06-16 incident):** a pod can be RUNNING + costing money but **0% util and absent from `\u002Fregister`** because its `server.py` died or its GPU fell off the bus. All traffic then piles onto the remaining worker(s) → 524 timeouts + 100% util on the survivor. Always cross-check (a) vs (b): every RUNNING zimage pod should appear in the registry.\n\n**SSH key:** the working key is **`SSH_RUNPOD_KLEIN`** (the documented `SSH_RUNPOD_FLUX_ZIMAGE` does NOT auth against these pods). Get the rotating SSH port per pod:\n```bash\nsops -d enter.pollinations.ai\u002Fsecrets\u002Fprod.vars.json | jq -r .SSH_RUNPOD_KLEIN > \u002Ftmp\u002Fzk; chmod 600 \u002Ftmp\u002Fzk\ncurl -s -X POST \"https:\u002F\u002Fapi.runpod.io\u002Fgraphql?api_key=$RUNPOD_TOKEN\" -H \"Content-Type: application\u002Fjson\" \\\n  -d '{\"query\":\"{pod(input:{podId:\\\"\u003CPOD_ID>\\\"}){runtime{ports{ip publicPort type}}}}\"}' \\\n  | python3 -c \"import sys,json; [print(p) for p in json.load(sys.stdin)['data']['pod']['runtime']['ports'] if p['type']=='tcp']\"\n```\n\n**Diagnose a single pod (SSH in on the discovered tcp port):**\n```bash\nssh -i \u002Ftmp\u002Fzk -p \u003CPORT> -o StrictHostKeyChecking=no root@\u003CIP> \\\n  \"nvidia-smi --query-gpu=utilization.gpu,memory.used --format=csv,noheader; ps aux | grep 'z-image\u002Fserver.py' | grep -v grep; ss -ltn | grep 8767\"\n```\nCode lives at `\u002Froot\u002Fpollinations\u002Fimage.pollinations.ai\u002Fz-image\u002F`; service auto-sources env from PID 1 (port 8767, `ZIMAGE_MODEL_ID`, `PLN_GPU_TOKEN`, `HF_TOKEN`). Direct `curl \u002Fgenerate` returns **403 without the GPU token** — a 403 still proves the server is up.\n\n**Recovery decision tree:**\n\n| Symptom | Fix |\n|---------|-----|\n| `server.py` not running, but `nvidia-smi` works | **Process relaunch** (cheap): `ssh ... \"bash \u002Froot\u002Frelaunch-zimage.sh\"` — kills + relaunches via `\u002Froot\u002Flaunch.sh`, survives SSH disconnect. Verify `Heartbeat sent successfully` in `\u002Froot\u002Flogs\u002Fzimage.log` and that the pod reappears in `\u002Fregister`. |\n| `nvidia-smi` → `Failed to initialize NVML: Unknown Error` (`\u002Fdev\u002Fnvidia*` owned by `nobody`) | **GPU off the bus — needs pod stop\u002Fstart** (NOT recoverable in-container). See below. |\n| Worker in `\u002Fregister` + heartbeating, but **524 timeouts route only to that one host** (and it's slow \u002F inference rate degrades mid-request \u002F huge RSS from long uptime) | **Stale degraded process — relaunch it, but confirm the OLD pid actually died.** A heartbeat thread keeps firing even when generation hangs, so the pod looks healthy. See pkill caveat below. |\n| All workers up + 100% util + still 524s | **Genuine capacity deficit** — add a GPU. Note 401\u002F402 are rejected at the edge (~100ms) and do NOT load the GPU, so it's real paid demand, not abuse. |\n\n> ⚠️ **`relaunch-zimage.sh` pkill caveat (2026-06-16):** the script kills by pattern `pkill -f 'z-image\u002Fserver.py'`, but the running process command is often just `python server.py` (launched from within the z-image dir), so the pattern **does not match** and the relaunch silently no-ops — the new process then fails with `address already in use` and exits while the old, degraded process keeps serving. **Always verify the old PID actually died** (`ps -o pid,etime -C python` — elapsed time should reset to seconds). If it didn't: `kill -9 \u003Cpid>; fuser -k 8767\u002Ftcp; bash \u002Froot\u002Frelaunch-zimage.sh`.\n\n**Pod stop\u002Fstart (for NVML-broken GPU) — WIPES the container overlay disk.** Code is on `\u002Froot` (overlay), there is **no persistent `\u002Fworkspace` volume**. So back up the provision script first, then rebuild:\n```bash\n# 1. Back up provision script BEFORE stopping:\nssh -i \u002Ftmp\u002Fzk -p \u003CPORT> root@\u003CIP> \"cat \u002Froot\u002Fprovision-zimage.sh\" > \u002Ftmp\u002Fprovision-zimage.sh\n# 2. Stop + start (port rotates; GPU re-attaches on fresh container):\nrunpodctl pod stop \u003CPOD_ID>;  sleep 10;  runpodctl pod start \u003CPOD_ID>\n# 3. Wait for RUNNING + tcp port, confirm GPU healthy (nvidia-smi shows the card), then re-provision:\nscp -i \u002Ftmp\u002Fzk -P \u003CNEW_PORT> \u002Ftmp\u002Fprovision-zimage.sh root@\u003CIP>:\u002Froot\u002F\nssh -i \u002Ftmp\u002Fzk -p \u003CNEW_PORT> root@\u003CIP> \\\n  \"export POD_ID=\u003CPOD_ID>; setsid bash \u002Froot\u002Fprovision-zimage.sh > \u002Froot\u002Fprovision.log 2>&1 \u003C\u002Fdev\u002Fnull & disown\"\n# provision: re-clones branch feat\u002Fzimage-nf4-model-option, builds venv, downloads model (~9GB) + SPAN\n#            upscaler, writes launch.sh, starts server. ~3-6 min. Tokens come from PID1 env (auto re-injected).\n# 4. Verify: server LISTENING :8767, model loaded, heartbeat sent, pod appears in \u002Fregister, e2e 200 via prod.\n```\n\n**Verify recovery end-to-end (prod success + load now shared):**\n```bash\nTOKEN=$(grep ENTER_API_TOKEN_REMOTE enter.pollinations.ai\u002F.testingtokens | cut -d= -f2)\nfor i in 1 2 3 4 5; do curl -s -o \u002Fdev\u002Fnull -w \"HTTP %{http_code}\\n\" --max-time 60 \\\n  \"https:\u002F\u002Fgen.pollinations.ai\u002Fimage\u002Fverify_${i}_$(date +%s%N)?model=zimage&width=512&height=512&seed=$i\" \\\n  -H \"Authorization: Bearer $TOKEN\"; done\n# Then confirm the 524 trend dropped in Tinybird (model_health \u002F generation_event_v2, model_requested='zimage').\n```\n\nThe registry is Cloudflare KV-backed (`image:server:\u003Cenv>:\u003Ctype>:\u003Chash>`, 240s TTL); workers heartbeat to `gen.pollinations.ai\u002Fregister`.\n\n---\n\n### 6. Sana Sprint 1.6B Worker (GH200 - same host as LTX-2)\n\nOne worker registered as `sana` type with OVH legacy service via heartbeat.\n\n| Instance | GPU | Host | Port | SSH |\n|----------|-----|------|------|-----|\n| Lambda GH200 | GH200 (96GB) | `192.222.51.105` | `8766` | `ssh -i \u003CSOPS:SSH_LAMBDA_SANA_LTX2_ACESTEP> ubuntu@192.222.51.105` |\n\n**Health check:**\n```bash\ncurl -s --connect-timeout 5 --max-time 10 http:\u002F\u002F192.222.51.105:8766\u002Fhealth\n```\nExpected: `{\"status\":\"healthy\",\"model\":\"Efficient-Large-Model\u002FSana_Sprint_1.6B_1024px_diffusers\"}`\n\n**Sana registry check (OVH side):**\n```bash\nssh -i ~\u002F.ssh\u002Fid_rsa_ovh -o ConnectTimeout=5 ubuntu@57.130.31.42 \"curl -s http:\u002F\u002Flocalhost:16384\u002Fregister\"\n```\nExpected: 1 worker with 0% error rate\n\n**Restart:**\n```bash\nssh -i \u003CSOPS:SSH_LAMBDA_SANA_LTX2_ACESTEP> ubuntu@192.222.51.105 \"sudo systemctl restart sana\"\n```\n\n**Notes:**\n- GH200 generates at ~0.165s\u002Fimg\n- Runs alongside LTX-2 (port 8765) and ACE-Step (port 8189) on the same host\n- Oracle A10\u002FA100 instances decommissioned on 2026-04-12\n- Server code: `image.pollinations.ai\u002Fsana\u002Fserver.py` (MAX_DIM=768, MAX_PIXELS=512*512)\n- Systemd service: `sana.service`\n\n---\n\n### 7. OVH Disk Space\n\n**Check:**\n```bash\nssh -i ~\u002F.ssh\u002Fid_rsa_ovh -o ConnectTimeout=5 ubuntu@57.130.31.42 \"df -h \u002F | tail -1\"\n```\nAlert if usage > 85%.\n\n**Fix (if full):**\n```bash\nssh -i ~\u002F.ssh\u002Fid_rsa_ovh ubuntu@57.130.31.42 \"sudo truncate -s 0 \u002Fvar\u002Flog\u002Fsyslog && sudo journalctl --vacuum-size=100M\"\n```\n\n---\n\n### 8. Flux Workers (Vast.ai RTX 5090s)\n\nFlux is self-hosted on Vast.ai 5090s since 2026-07-02 (PR #12171) with automatic\nFireworks fallback: the worker sheds load with 503 beyond `QUEUE_LIMIT`, and the\ngateway retries those requests on Fireworks. Instance coordinates, provisioning\none-liner, and host gotchas live in `image.pollinations.ai\u002FGPU_INSTANCES.md`\n(source of truth — instance IDs\u002FIPs\u002Fports change on recreate).\n\n**Check:**\n```bash\n# flux worker(s) registered — the `url` field is the tunnel to probe, and\n# `lastMs` is the last request's duration (a healthy 5090 is single-digit\n# seconds; tens of seconds means the tunnel is degrading, see below):\ncurl -s https:\u002F\u002Fgen.pollinations.ai\u002Fregister | grep -o '\"type\":\"flux\"[^}]*'\n# then probe THAT url (expect 200 in ~1s):\ncurl -s -o \u002Fdev\u002Fnull -w \"%{http_code} %{time_total}s\\n\" \"\u003Curl-from-registry>\u002Fdocs\"\n```\n\nRegistry health is NOT data-path health: the worker heartbeats fine even when\nthe tunnel in front of it is too slow to serve. Always time the probe. If\n`\u002Fdocs` (a static page) takes >5s, the tunnel is the problem, not the GPU —\ncompare against the worker on localhost via SSH to confirm.\n\n**Pool vs Fireworks split** — invisible in Tinybird (`model_provider_used` is\nstatic registry data); read it from the worker log (SSH coords from\nGPU_INSTANCES.md \u002F `vastai show instances`):\n```bash\nssh -p \u003CPORT> -i ~\u002F.ssh\u002Fpollinations_services_2026 root@\u003CIP> \\\n  'grep -c \"\\\" 200\" \u002Ftmp\u002Fflux.log; grep -c \"\\\" 503\" \u002Ftmp\u002Fflux.log'   # served \u002F shed\n```\n\n**Fix:** on the instance, `screen -r flux` and `screen -r cloudflared` run in\nrestart loops (logs `\u002Ftmp\u002Fflux.log`, `\u002Ftmp\u002Fcloudflared.log`); restart commands in\nGPU_INSTANCES.md. If flux is fully down, all traffic falls back to Fireworks\nautomatically — users are unaffected, but it costs ~$70\u002Fday, so fix promptly.\nVast SSH gotcha: long-lived sessions get dropped — use short commands, never\n`sleep` inside a session.\n\n---\n\n## Procedure\n\nWhen invoked, run checks in this order:\n\n1. **gen.pollinations.ai registry** - `curl https:\u002F\u002Fgen.pollinations.ai\u002Fregister` (KV-backed), check worker count and error rates\n2. **Flux (Vast.ai) + Z-Image (RunPod)** - flux worker registered + tunnel healthy (§8); every RUNNING zimage pod registered (§5)\n3. **LTX-2 health** - curl health endpoint\n4. **LTX-2 e2e** - if healthy, test through gen.pollinations.ai (use test token from `.testingtokens`)\n5. **ACE-Step health** - curl health endpoint on port 8189\n6. **Klein health** - curl RunPod proxy health endpoint\n7. **Legacy image service** - check systemctl status on OVH\n8. **Sana worker** - curl health on GH200 port 8766\n9. **Sana registry** - check OVH legacy registry for 1 worker with 0% errors\n10. **Disk space** - check OVH disk usage\n\nFor each:\n- If healthy: report OK with latency\n- If unhealthy: attempt restart, wait, re-check, report result\n\n## Auth\n\n- **Test token**: Read from `enter.pollinations.ai\u002F.testingtokens` (ENTER_API_TOKEN_REMOTE)\n- **SSH keys**: Stored in SOPS (`enter.pollinations.ai\u002Fsecrets\u002Fprod.vars.json`):\n  - `SSH_LAMBDA_SANA_LTX2_ACESTEP` — Lambda GH200 (LTX-2, ACE-Step, Sana)\n  - Vast.ai flux workers use the local key `~\u002F.ssh\u002Fpollinations_services_2026` (attached per-instance via `vastai attach ssh`; not in SOPS)\n  - Klein uses the RunPod relay (`ssh.runpod.io`) with `~\u002F.ssh\u002Fid_ed25519` — get the full command from the dashboard \"Connect\" tab\n  - Extract: `sops -d enter.pollinations.ai\u002Fsecrets\u002Fprod.vars.json | jq -r '.KEY_NAME' > \u002Ftmp\u002Fkey && chmod 600 \u002Ftmp\u002Fkey`\n- **OVH**: `~\u002F.ssh\u002Fid_rsa_ovh` (not in SOPS)\n\n## Output\n\nReport a brief status table:\n\n```\n| Service | Status | Latency | Notes |\n|---------|--------|---------|-------|\n| gen registry | OK | 0.1s | 4 workers, 0% errors |\n| Flux Vast (flux-vast-01) | OK | 2.3s | tunnel 200, 1.2% shed to Fireworks |\n| Z-Image RunPod ×3 | OK | 1.5s | pods discovered via runpodctl |\n| LTX-2 health | OK | 0.2s | |\n| LTX-2 e2e | OK | 11.3s | 682KB |\n| ACE-Step | OK | 0.1s | |\n| Klein 4B | OK | 0.3s | RunPod |\n| Legacy image | OK | - | active |\n| Sana (GH200) | OK | 0.2s | 1.6B, ~0.165s\u002Fimg |\n| Sana registry | OK | - | 1 worker, 0% errors |\n| OVH disk | OK | - | 45% used |\n```\n",{"name":5,"description":7},"wan-aspect-ratio-fix","0cf9c44b2cb868c1431babf98007a269e7a2b66b",".claude\u002Fskills\u002Fmonitor-services","https:\u002F\u002Fgithub.com\u002FAPPLEPIE6969\u002FPollinations","https:\u002F\u002Fgithub.com\u002FAPPLEPIE6969","2026-07-26 00:50:46",{"source":8,"skillName":5,"commands":55,"prompt":65,"zipAvailable":66},[56,59,62],{"tool":57,"command":58},"npx","npx skills add APPLEPIE6969\u002FPollinations --skill monitor-services",{"tool":60,"command":61},"bunx","bunx skills add APPLEPIE6969\u002FPollinations --skill monitor-services",{"tool":63,"command":64},"pnpm","pnpm dlx skills add APPLEPIE6969\u002FPollinations --skill monitor-services","安装技能，技能地址：https:\u002F\u002Fgithub.com\u002FAPPLEPIE6969\u002FPollinations\u002Fblob\u002Fwan-aspect-ratio-fix\u002F.claude\u002Fskills\u002Fmonitor-services\u002FSKILL.md",true,[],[69,99,119,143,166,180],{"skillNo":70,"name":71,"nameZh":6,"description":72,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":73,"categories":74,"keywords":79,"publishedAt":43},"01M6R5KBQ6ARHS4YP4XZX06DJC","polli","Generate images, text, audio, video, and transcribe speech via the Pollinations API using the polli CLI. Use when asked to generate media, call pollinations.ai, check pollen balance, list models, manage API keys, inspect quests, manage my-models, or run polli commands.","https:\u002F\u002Fgithub.com\u002FAPPLEPIE6969\u002FPollinations\u002Fblob\u002Fwan-aspect-ratio-fix\u002Fpackages\u002Fpolli-cli\u002FSKILL.md",[75,78],{"categoryNo":76,"categoryName":77,"taxonomyType":20,"taxonomyTypeDesc":21},"01JBSEEDCAT000000000000010","后端与接口",{"categoryNo":18,"categoryName":19,"taxonomyType":20,"taxonomyTypeDesc":21},[71,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98],"cli","generate","run","speech","api","Video","Media","Audio","Bash","text","list","images","commands","balance","quests","my","inspect","api keys","when",{"skillNo":100,"name":101,"nameZh":6,"description":102,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":103,"categories":104,"keywords":105,"publishedAt":43},"01M6R5K5FEQFK9BRNY2S06M73S","founder-meditation","When something goes wrong — build failures, crashes, errors, test failures, deployment issues — generate a short calming meditation with TTS audio to reassure the user that everything will be okay.","https:\u002F\u002Fgithub.com\u002FAPPLEPIE6969\u002FPollinations\u002Fblob\u002Fwan-aspect-ratio-fix\u002Fapps\u002Fopenclaw\u002Fskills\u002Ffounder-meditation\u002FSKILL.md",[],[106,107,108,109,81,110,111,84,87,88,112,113,114,115,116,30,28,117,118,98],"founder","founder meditation","test","deployment","issues","https","tts","curl","short","user","build","errors","Wrong",{"skillNo":120,"name":121,"nameZh":6,"description":122,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":123,"categories":124,"keywords":128,"publishedAt":43},"01M6R5JT4Q4B2SE36ZP416AJ4Q","web-research","Query Pollinations text API with web-search models (gemini-search, perplexity-fast, nomnom, etc.). Use when you need web search grounded answers via Pollinations.","https:\u002F\u002Fgithub.com\u002FAPPLEPIE6969\u002FPollinations\u002Fblob\u002Fwan-aspect-ratio-fix\u002F.claude\u002Fskills\u002Fweb-research\u002FSKILL.md",[125],{"categoryNo":126,"categoryName":127,"taxonomyType":20,"taxonomyTypeDesc":21},"01JBSEEDCAT000000000000021","资料检索与研究",[129,130,131,132,133,134,135,84,136,89,26,137,30,138,139,140,28,141,142,98],"research","web","web research","gemini","search","fast","perplexity","web search","models","web-search","query","via","you","nomnom",{"skillNo":144,"name":145,"nameZh":6,"description":146,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":147,"categories":148,"keywords":149,"publishedAt":43},"01M6R5JMK3CJN4VEKVX23745BR","voting-status","Create and post ASCII art voting status diagrams to GitHub issues and Discord. Use when asked to update voting, show voting results, or announce voting status.","https:\u002F\u002Fgithub.com\u002FAPPLEPIE6969\u002FPollinations\u002Fblob\u002Fwan-aspect-ratio-fix\u002F.claude\u002Fskills\u002Fvoting-status\u002FSKILL.md",[],[150,151,152,153,154,155,156,157,110,158,26,159,160,161,162,163,164,165,98],"status","voting","voting status","discord","github","create","update","github issues","results","ascii art","diagrams","Art","show","ascii","post","announce",{"skillNo":167,"name":168,"nameZh":6,"description":169,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":170,"categories":171,"keywords":175,"publishedAt":43},"01M6R5J6ZK52VS1783EJH36137","tinybird-deploy","Deploy Tinybird pipes and datasources for enter.pollinations.ai observability with the Tinybird Forward CLI.","https:\u002F\u002Fgithub.com\u002FAPPLEPIE6969\u002FPollinations\u002Fblob\u002Fwan-aspect-ratio-fix\u002F.claude\u002Fskills\u002Ftinybird-deploy\u002FSKILL.md",[172],{"categoryNo":173,"categoryName":174,"taxonomyType":20,"taxonomyTypeDesc":21},"01JBSEEDCAT000000000000011","运维与部署",[176,177,178,80,179,30,28,27],"deploy","tinybird","tinybird deploy","observability",{"skillNo":181,"name":182,"nameZh":6,"description":183,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":184,"categories":185,"keywords":186,"publishedAt":43},"01M6R5HYNSNJBPMC3SQ7PZVM5Y","sync-production","Sync the main branch into production via the GitHub merges API. Pre-checks for merge conflicts and bails out with a clear reason if the merge isn't clean. Use when asked to merge main into production, deploy main to production, or sync production.","https:\u002F\u002Fgithub.com\u002FAPPLEPIE6969\u002FPollinations\u002Fblob\u002Fwan-aspect-ratio-fix\u002F.claude\u002Fskills\u002Fsync-production\u002FSKILL.md",[],[187,188,189,154,190,176,84,191,26,192,193,194,195,196,140,28,27,197,98],"sync","Production","sync production","merge","branch","merge conflicts","conflicts","clean","main","clear","reason"]