feat(github-runner): add top Linux repo runners
This commit is contained in:
@@ -974,6 +974,19 @@ data:
|
||||
summary: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} replica mismatch"
|
||||
description: "Spec wants {{ $labels.spec_replicas }} but only {{ $value }} available. Likely a rollout stuck on probe failure, scheduling, or PVC."
|
||||
|
||||
- alert: LinuxRunnerOffline
|
||||
expr: |
|
||||
kube_deployment_status_replicas_available{namespace="github-runner",deployment=~"github-runner(|-(puppet|signage|dms|telephony|print-web|chat|mysql|kiosk-linux))"} < 1
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
service: github-runner
|
||||
alert_channel: thermal_print
|
||||
annotations:
|
||||
summary: "Linux GitHub Actions runner offline: {{ $labels.deployment }}"
|
||||
description: "{{ $labels.deployment }} has no available runner pod for 10 minutes. GitHub jobs using [self-hosted, linux, fc-build-linux] for its repo will queue at $0 until the runner returns."
|
||||
runbook_url: "https://gitea.iamworkin.lan/bluejay/FlowerCore.Notes/src/branch/master/docs/infrastructure/self-hosted-runner-fleet.md"
|
||||
|
||||
# Q-MR-3 (2026-05-11): multus memory pressure — catches the next OOM
|
||||
# cascade BEFORE multus is OOM-killed cluster-wide. The 2026-05-10
|
||||
# outage (21h) hit because no alert fired on the rising multus working
|
||||
@@ -3427,6 +3440,33 @@ data:
|
||||
relativeTimeRange: {from: 120, to: 0}
|
||||
datasourceUid: __expr__
|
||||
model: {type: threshold, expression: B, conditions: [{evaluator: {params: [1], type: lt}}], refId: C}
|
||||
- uid: linux-runner-offline
|
||||
title: LinuxRunnerOffline
|
||||
condition: C
|
||||
for: 10m
|
||||
noDataState: Alerting
|
||||
execErrState: OK
|
||||
annotations:
|
||||
summary: Linux GitHub Actions runner offline
|
||||
description: "A repo-scoped fc-build-linux runner deployment has no available pod. Jobs will queue at $0 until ArgoCD/K8s returns the runner."
|
||||
runbook_url: "https://gitea.iamworkin.lan/bluejay/FlowerCore.Notes/src/branch/master/docs/infrastructure/self-hosted-runner-fleet.md"
|
||||
labels:
|
||||
severity: warning
|
||||
service: github-runner
|
||||
alert_channel: thermal_print
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: {from: 600, to: 0}
|
||||
datasourceUid: prometheus
|
||||
model: {expr: 'min by(deployment) (kube_deployment_status_replicas_available{namespace="github-runner",deployment=~"github-runner(|-(puppet|signage|dms|telephony|print-web|chat|mysql|kiosk-linux))"})', instant: true, refId: A}
|
||||
- refId: B
|
||||
relativeTimeRange: {from: 600, to: 0}
|
||||
datasourceUid: __expr__
|
||||
model: {type: reduce, expression: A, reducer: last, refId: B}
|
||||
- refId: C
|
||||
relativeTimeRange: {from: 600, to: 0}
|
||||
datasourceUid: __expr__
|
||||
model: {type: threshold, expression: B, conditions: [{evaluator: {params: [1], type: lt}}], refId: C}
|
||||
- uid: high-cpu
|
||||
title: High CPU (>85%)
|
||||
condition: C
|
||||
|
||||
Reference in New Issue
Block a user