feat(github-runner): harden Linux runner fleet (#5)

apps/monitoring/noc-monitoring.yaml

@@ -723,6 +723,24 @@ data:
               summary: "Mac mini GitHub runner offline ({{ $labels.runner }})"
               description: "A macmini-* GitHub Actions runner has not reported online for more than 10 minutes. Puppet manages its LaunchDaemon under /Library/LaunchDaemons/io.flowercore.github-runner-<slug>.plist; runners survive reboot and do not require a GUI session."
 
+      - name: linux-runners
+        rules:
+          - alert: LinuxRunnerOffline
+            expr: |
+              kube_deployment_status_replicas_ready{
+                namespace="github-runner",
+                deployment=~"github-runner(|-(sharedpos|puppet|signage|dms|telephony|print-web|chat|mysql|kiosk-linux))"
+              } == 0
+            for: 5m
+            labels:
+              severity: warning
+              alert_channel: irc
+              service: github-runner
+              team: ci
+            annotations:
+              summary: "Linux CI runner offline: {{ $labels.deployment }}"
+              description: "Deployment {{ $labels.deployment }} in namespace github-runner has 0 ready replicas for more than 5 minutes. CI jobs targeting this repo will queue until the runner pod restarts and re-registers with GitHub. Check pods with: kubectl -n github-runner get pods -l app.kubernetes.io/name={{ $labels.deployment }}. Check logs with: kubectl -n github-runner logs -l app.kubernetes.io/name={{ $labels.deployment }} --tail=50. Common causes: PAT missing repo access, runner CrashLoopBackOff, or node/resource pressure."
+
       - name: remote-desktop
         rules:
           - alert: RemoteDesktopWebDown
@@ -3421,6 +3439,39 @@ data:
                 relativeTimeRange: {from: 120, to: 0}
                 datasourceUid: __expr__
                 model: {type: threshold, expression: B, conditions: [{evaluator: {params: [600], type: gt}}], refId: C}
+      - orgId: 1
+        name: CI Runners
+        folder: CI Alerts
+        interval: 1m
+        rules:
+          - uid: linux-runner-offline
+            title: LinuxRunnerOffline
+            condition: C
+            for: 5m
+            noDataState: OK
+            execErrState: Error
+            annotations:
+              summary: "Linux CI runner offline: {{ $labels.deployment }}"
+              description: "A github-runner namespace Deployment has 0 ready replicas for more than 5 minutes. CI jobs targeting that repo will queue until the runner pod restarts and re-registers."
+              runbook: "1. kubectl -n github-runner get pods -l app.kubernetes.io/name={{ $labels.deployment }} 2. kubectl -n github-runner logs -l app.kubernetes.io/name={{ $labels.deployment }} --tail=50 3. Verify PAT repo access if registration returns 404 4. Verify no RWO PVC is shared by scaled runners"
+            labels:
+              severity: warning
+              service: github-runner
+              alert_channel: irc
+              team: ci
+            data:
+              - refId: A
+                relativeTimeRange: {from: 300, to: 0}
+                datasourceUid: prometheus
+                model: {expr: 'kube_deployment_status_replicas_ready{namespace="github-runner",deployment=~"github-runner(|-(sharedpos|puppet|signage|dms|telephony|print-web|chat|mysql|kiosk-linux))"} == 0', instant: true, refId: A}
+              - refId: B
+                relativeTimeRange: {from: 300, to: 0}
+                datasourceUid: __expr__
+                model: {type: reduce, expression: A, reducer: last, refId: B}
+              - refId: C
+                relativeTimeRange: {from: 300, to: 0}
+                datasourceUid: __expr__
+                model: {type: threshold, expression: B, conditions: [{evaluator: {params: [0], type: gt}}], refId: C}
       - orgId: 1
         name: Infrastructure
         folder: AI Stack Alerts