From c23e903ba7bd6acbb211e07b179242461af1a5d2 Mon Sep 17 00:00:00 2001 From: Andrew Stoltz Date: Fri, 24 Apr 2026 00:57:26 -0500 Subject: [PATCH] feat(monitoring): Grafana alert rules route RemoteDesktop to IRC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Companion to the Prometheus alert rules landed in e44e9a0. The Prometheus rules were loading but never delivered — the monitoring stack has no Alertmanager configured; **Grafana** owns alert routing via its built-in engine + webhook contact point to irc-notify.monitoring.svc:9119. Without a matching Grafana alert, the Prometheus rules just show up in the Prometheus UI and page no one. Adds 6 Grafana alert rules in a new `RemoteDesktop` group under the AI Stack Alerts folder: - remotedesktop-web-down (3m) — probe_success{job="probe-remotedesktop"} < 1 - remotedesktop-metrics-stale (10m) — fc_desktop_session_events_total series absent - remotedesktop-pool-depleted (5m) — fc_desktop_pool_depleted > 0 - remotedesktop-pool-deficit-sustained (10m info) — fc_desktop_pool_deficit > 0 - remotedesktop-session-churn-spike (5m info) — launch rate > 20/min - remotedesktop-tls-expiry (6h critical) — cert < 2 days to expiry Each uses the standard Grafana 3-stage pipeline (query → reduce → threshold) matching the existing AI Stack + Infrastructure alert patterns. Labels: service=remotedesktop + severity (warning/info/critical). Default route is `IRC #alerts` via the existing webhook contact point. Parity with the Prometheus rules (which already fire internally for the Prometheus UI + any future Alertmanager integration). Grafana restart picks up the new provisioning on next reload. Co-Authored-By: Claude Opus 4.7 (1M context) --- apps/monitoring/noc-monitoring.yaml | 166 ++++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) diff --git a/apps/monitoring/noc-monitoring.yaml b/apps/monitoring/noc-monitoring.yaml index 0d578d4..ad4f791 100644 --- a/apps/monitoring/noc-monitoring.yaml +++ b/apps/monitoring/noc-monitoring.yaml @@ -3139,6 +3139,172 @@ data: relativeTimeRange: {from: 600, to: 0} datasourceUid: __expr__ model: {type: threshold, expression: B, conditions: [{evaluator: {params: [85], type: gt}}], refId: C} + - orgId: 1 + name: RemoteDesktop + folder: AI Stack Alerts + interval: 1m + rules: + - uid: remotedesktop-web-down + title: RemoteDesktop Web DOWN + condition: C + for: 3m + noDataState: Alerting + execErrState: OK + annotations: + summary: FlowerCore RemoteDesktop /health probe failing + description: "https://desktop.iamworkin.lan/health has failed for 3 minutes. Catalog + session launch surface offline." + runbook: "1. kubectl -n fc-desktop get pods -l app.kubernetes.io/name=remotedesktop-web 2. kubectl -n fc-desktop logs deploy/remotedesktop-web --tail=50 3. Check Traefik IngressRoute + step-ca cert 4. Rollout restart if pod is stuck" + labels: + severity: warning + service: remotedesktop + data: + - refId: A + relativeTimeRange: {from: 180, to: 0} + datasourceUid: prometheus + model: {expr: 'probe_success{job="probe-remotedesktop"}', instant: true, refId: A} + - refId: B + relativeTimeRange: {from: 180, to: 0} + datasourceUid: __expr__ + model: {type: reduce, expression: A, reducer: last, refId: B} + - refId: C + relativeTimeRange: {from: 180, to: 0} + datasourceUid: __expr__ + model: {type: threshold, expression: B, conditions: [{evaluator: {params: [1], type: lt}}], refId: C} + + - uid: remotedesktop-metrics-stale + title: RemoteDesktop metrics stale + condition: C + for: 10m + noDataState: Alerting + execErrState: OK + annotations: + summary: RemoteDesktop /metrics returning no series + description: "No fc_desktop_session_events_total series for 10 minutes. Either the Prometheus scrape is misconfigured or the web deployment stopped exporting metrics. Cross-checked by Zabbix template's identical 10m no-data trigger." + runbook: "1. curl -sk https://desktop.iamworkin.lan/metrics | head 2. kubectl -n monitoring exec deploy/prometheus -- wget -qO- localhost:9090/api/v1/targets?scrapePool=fc-remotedesktop 3. Check monitoring-netpol egress allows to fc-desktop:8080" + labels: + severity: warning + service: remotedesktop + data: + - refId: A + relativeTimeRange: {from: 600, to: 0} + datasourceUid: prometheus + model: {expr: 'count(fc_desktop_session_events_total) or vector(0)', instant: true, refId: A} + - refId: B + relativeTimeRange: {from: 600, to: 0} + datasourceUid: __expr__ + model: {type: reduce, expression: A, reducer: last, refId: B} + - refId: C + relativeTimeRange: {from: 600, to: 0} + datasourceUid: __expr__ + model: {type: threshold, expression: B, conditions: [{evaluator: {params: [1], type: lt}}], refId: C} + + - uid: remotedesktop-pool-depleted + title: RemoteDesktop pool depleted + condition: C + for: 5m + noDataState: OK + execErrState: OK + annotations: + summary: RemoteDesktop warm pool depleted for 5m + description: "A RemoteDesktop warm pool has fc_desktop_pool_depleted=1 for 5 minutes. New launches will cold-start. Check pod scheduling, image pull, node capacity." + runbook: "1. kubectl -n fc-desktop get pods -l app.kubernetes.io/name=remote-desktop --sort-by=.status.startTime 2. kubectl -n fc-desktop describe desktoppool 3. Verify localhost/fc-desktop:* images imported on all 3 RKE2 nodes" + labels: + severity: warning + service: remotedesktop + data: + - refId: A + relativeTimeRange: {from: 300, to: 0} + datasourceUid: prometheus + model: {expr: 'max(fc_desktop_pool_depleted)', instant: true, refId: A} + - refId: B + relativeTimeRange: {from: 300, to: 0} + datasourceUid: __expr__ + model: {type: reduce, expression: A, reducer: last, refId: B} + - refId: C + relativeTimeRange: {from: 300, to: 0} + datasourceUid: __expr__ + model: {type: threshold, expression: B, conditions: [{evaluator: {params: [0.5], type: gt}}], refId: C} + + - uid: remotedesktop-pool-deficit-sustained + title: RemoteDesktop pool below desired + condition: C + for: 10m + noDataState: OK + execErrState: OK + annotations: + summary: RemoteDesktop pool sustained deficit + description: "A pool has fc_desktop_pool_deficit>0 for 10 minutes. Operator is reconciling but can't reach desired size — likely image pull, NFS affinity, or claim-init issue." + runbook: "1. kubectl -n fc-desktop get pods -l flowercore.io/pool= 2. kubectl logs -n fc-desktop deploy/remotedesktop-operator 3. Check claim-init hook env on template" + labels: + severity: info + service: remotedesktop + data: + - refId: A + relativeTimeRange: {from: 600, to: 0} + datasourceUid: prometheus + model: {expr: 'max(fc_desktop_pool_deficit)', instant: true, refId: A} + - refId: B + relativeTimeRange: {from: 600, to: 0} + datasourceUid: __expr__ + model: {type: reduce, expression: A, reducer: last, refId: B} + - refId: C + relativeTimeRange: {from: 600, to: 0} + datasourceUid: __expr__ + model: {type: threshold, expression: B, conditions: [{evaluator: {params: [0], type: gt}}], refId: C} + + - uid: remotedesktop-session-churn-spike + title: RemoteDesktop launch rate spike + condition: C + for: 5m + noDataState: OK + execErrState: OK + annotations: + summary: RemoteDesktop launch rate exceeds 20/min + description: "Launch events >20/min for 5 minutes. Could be a user-facing feature launch, pooled template thrashing, or runaway automation loop." + runbook: "1. kubectl -n fc-desktop get pods -l app.kubernetes.io/name=remote-desktop -o wide | wc -l 2. curl -sk https://desktop.iamworkin.lan/api/sessions/active 3. Check operator logs for reconcile loops" + labels: + severity: info + service: remotedesktop + data: + - refId: A + relativeTimeRange: {from: 300, to: 0} + datasourceUid: prometheus + model: {expr: 'sum(rate(fc_desktop_session_events_total{event="launch"}[5m])) * 60', instant: true, refId: A} + - refId: B + relativeTimeRange: {from: 300, to: 0} + datasourceUid: __expr__ + model: {type: reduce, expression: A, reducer: last, refId: B} + - refId: C + relativeTimeRange: {from: 300, to: 0} + datasourceUid: __expr__ + model: {type: threshold, expression: B, conditions: [{evaluator: {params: [20], type: gt}}], refId: C} + + - uid: remotedesktop-tls-expiry + title: RemoteDesktop TLS cert expiring + condition: C + for: 6h + noDataState: OK + execErrState: OK + annotations: + summary: desktop.iamworkin.lan cert <2d to expiry + description: "The desktop.iamworkin.lan certificate is inside the 2-day renewal window and cert-manager has not renewed. Check cert-manager logs, step-ca reachability, FlowerCore.DNS preflight for dnsNames." + runbook: "1. kubectl -n fc-desktop get certificate remotedesktop-web-tls 2. kubectl -n cert-manager logs deploy/cert-manager --tail=50 3. Verify pfSense DNS override for desktop.iamworkin.lan" + labels: + severity: critical + service: remotedesktop + data: + - refId: A + relativeTimeRange: {from: 21600, to: 0} + datasourceUid: prometheus + model: {expr: '(probe_ssl_earliest_cert_expiry{job="probe-remotedesktop"} - time()) / 86400', instant: true, refId: A} + - refId: B + relativeTimeRange: {from: 21600, to: 0} + datasourceUid: __expr__ + model: {type: reduce, expression: A, reducer: last, refId: B} + - refId: C + relativeTimeRange: {from: 21600, to: 0} + datasourceUid: __expr__ + model: {type: threshold, expression: B, conditions: [{evaluator: {params: [2], type: lt}}], refId: C} # ============================================================================= # Deployment: Grafana