monitoring: alert on remotedesktop longhorn pvc growth

apps/monitoring/noc-monitoring.yaml

@@ -656,15 +656,14 @@ data:
               summary: "Print queue backlog on edge2 ({{ $value }} active jobs)"
               description: "CUPS has {{ $value }} active jobs queued. Possible printer jam, USB disconnect, or paper out."
 
-          # Printer hardware and paper-roll lifecycle alerts.
+          # Paper roll lifecycle alerts (XL Track I, 2026-04-26).
-          # print_printer_online: 1 when the transport is reachable/selected.
+          # Source-of-truth gauge: print_paper_remaining_percent (Print.Web OTEL,
-          # print_printer_state enum: 0 unknown, 1 online, 2 offline,
+          # hydrated on startup from the active PaperRoll row).
-          # 3 paper_depleted, 4 jam, 5 head_error, 6 cover_open.
+          # alert_channel=thermal_print routes through irc-notify -> Print.Web
-          # Offline/jam/cover alerts stay IRC-only. Paper depleted and head
+          # /api/print/alert so the printer announces its own paper-out warning
-          # error may route to the thermal digest only when the printer is
+          # on its remaining paper. Self-referential humor + operator nudge.
-          # online enough to make that useful.
           - alert: PrintPaperRollLow
-            expr: (print_paper_remaining_percent{job="printweb-otel"} < 10 and print_paper_remaining_percent{job="printweb-otel"} > 5) and print_printer_online{job="printweb-otel"} == 1
+            expr: print_paper_remaining_percent{job="printweb-otel"} < 10 and print_paper_remaining_percent{job="printweb-otel"} > 5
             for: 5m
             labels:
               severity: warning
@@ -673,59 +672,15 @@ data:
               summary: "Print roll low on edge2 ({{ $value | printf \"%.1f\" }}% remaining)"
               description: "NuPrint 210 paper roll has {{ $value | printf \"%.1f\" }}% remaining. Operator should load a fresh roll soon. Run /api/paper/status for the precise mm + estimated jobs left."
 
-          - alert: PrinterOfflineWarning
-            expr: print_printer_state{job="printweb-otel"} == 2
-            for: 2m
-            labels:
-              severity: warning
-              service: print-web
-              alert_channel: irc
-            annotations:
-              summary: "Print.Web printer offline on edge2"
-              description: "Print.Web reports the NuPrint 210 transport is offline or unreachable. IRC-only by design: do not thermal-print an alert when the thermal printer itself is offline."
-
           - alert: PrintPaperRollCritical
-            expr: print_printer_state{job="printweb-otel"} == 3 and print_printer_online{job="printweb-otel"} == 1
+            expr: print_paper_remaining_percent{job="printweb-otel"} <= 5
             for: 2m
             labels:
               severity: critical
               alert_channel: thermal_print
             annotations:
-              summary: "Print paper depleted on edge2"
+              summary: "Print roll critical on edge2 ({{ $value | printf \"%.1f\" }}% remaining)"
-              description: "NuPrint 210 reports paper depleted while the printer is still online. Load a new roll, drain the hardware buffer if needed, then replay DeadLetter jobs from /print-log."
+              description: "NuPrint 210 paper roll at {{ $value | printf \"%.1f\" }}% — load a new roll NOW. The 50ft roll has a ~12% red-stripe zone; once paper passes that, the printer can run dry mid-job."
-
-          - alert: PrinterJamWarning
-            expr: print_printer_state{job="printweb-otel"} == 4
-            for: 2m
-            labels:
-              severity: warning
-              service: print-web
-              alert_channel: irc
-            annotations:
-              summary: "Print.Web printer jam on edge2"
-              description: "Print.Web reports a paper/cutter jam state. IRC-only: clear the jam, drain the hardware buffer if bytes were queued, then retry affected jobs."
-
-          - alert: PrinterHeadErrorCritical
-            expr: print_printer_state{job="printweb-otel"} == 5
-            for: 2m
-            labels:
-              severity: critical
-              service: print-web
-              alert_channel: thermal_print
-            annotations:
-              summary: "Print.Web printer head error on edge2"
-              description: "Print.Web reports a thermal head or unrecoverable printer error. Critical routing may enter the thermal digest per existing policy; IRC remains the primary triage stream."
-
-          - alert: PrinterCoverOpenWarning
-            expr: print_printer_state{job="printweb-otel"} == 6
-            for: 2m
-            labels:
-              severity: warning
-              service: print-web
-              alert_channel: irc
-            annotations:
-              summary: "Print.Web printer cover open on edge2"
-              description: "Print.Web reports the printer cover/lid is open. IRC-only: close the cover and verify /api/print/status before retrying jobs."
 
           - alert: PrintJobDeadLetter
             expr: increase(print_jobs_dead_letter_total[15m]) > 0
@@ -869,6 +824,53 @@ data:
               summary: "desktop.iamworkin.lan TLS cert expires within 2 days"
               description: "The desktop.iamworkin.lan cert is inside the 2-day renewal window and cert-manager has not renewed. Check cert-manager logs, step-ca reachability, and pfSense DNS overrides per the ACME DNS-01 gate."
 
+          - alert: LonghornPVCGrowthRapid
+            expr: |
+              (
+                (
+                  (
+                    longhorn_volume_actual_size_bytes
+                    - (longhorn_volume_actual_size_bytes offset 1h)
+                  )
+                  / clamp_min(longhorn_volume_actual_size_bytes offset 1h, 1)
+                )
+                * on(volume) group_left(namespace, persistentvolumeclaim) (
+                  (
+                    label_replace(kube_persistentvolumeclaim_info{storageclass="longhorn"}, "volume", "$1", "volumename", "(.+)")
+                    * on(namespace, persistentvolumeclaim) group_left()
+                      kube_persistentvolumeclaim_labels{label_flowercore_io_managed_by="remotedesktop"}
+                  )
+                  or
+                  label_replace(kube_persistentvolumeclaim_info{namespace="fc-desktop", storageclass="longhorn", persistentvolumeclaim=~"fc-profile-.*|remotedesktop-data"}, "volume", "$1", "volumename", "(.+)")
+                )
+              ) > 0.20
+              or
+              (
+                (
+                  longhorn_volume_actual_size_bytes
+                  / on(volume) clamp_min(longhorn_volume_capacity_bytes, 1)
+                )
+                * on(volume) group_left(namespace, persistentvolumeclaim) (
+                  (
+                    label_replace(kube_persistentvolumeclaim_info{storageclass="longhorn"}, "volume", "$1", "volumename", "(.+)")
+                    * on(namespace, persistentvolumeclaim) group_left()
+                      kube_persistentvolumeclaim_labels{label_flowercore_io_managed_by="remotedesktop"}
+                  )
+                  or
+                  label_replace(kube_persistentvolumeclaim_info{namespace="fc-desktop", storageclass="longhorn", persistentvolumeclaim=~"fc-profile-.*|remotedesktop-data"}, "volume", "$1", "volumename", "(.+)")
+                )
+              ) > 0.80
+            for: 5m
+            labels:
+              severity: warning
+              alert_channel: thermal_print
+              service: remotedesktop
+            annotations:
+              summary: "RemoteDesktop Longhorn PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} growing rapidly"
+              description: "Longhorn volume {{ $labels.volume }} backing RemoteDesktop PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} grew more than 20% in 1h or is over 80% capacity. Check for runaway SQLite/user-profile growth; this alert was added after the 2026-05-16 RemoteDesktop web SQLite Error 13 incident."
+              runbook: "1. kubectl -n {{ $labels.namespace }} describe pvc {{ $labels.persistentvolumeclaim }} 2. Open Longhorn UI volume {{ $labels.volume }} 3. Check RemoteDesktop web/user-volume SQLite files for permission or runaway growth 4. Expand PVC only after confirming the writer is healthy"
+              todo: "2026-05-19 metric gate: live noc1 Prometheus currently exposes kube_persistentvolumeclaim_info and kube_persistentvolumeclaim_resource_requests_storage_bytes, but not longhorn_volume_actual_size_bytes, longhorn_volume_capacity_bytes, kube_persistentvolumeclaim_labels, or kubelet_volume_stats_used_bytes. Keep the fc-desktop PVC fallback until kube-state-metrics label allowlist exposes flowercore.io/managed-by=remotedesktop."
+
       - name: pi-fleet
         rules:
           - alert: PiManagerDown
@@ -3680,146 +3682,6 @@ data:
                 relativeTimeRange: {from: 120, to: 0}
                 datasourceUid: __expr__
                 model: {type: threshold, expression: B, conditions: [{evaluator: {params: [600], type: gt}}], refId: C}
-      - orgId: 1
-        name: Print Services
-        folder: Print Alerts
-        interval: 1m
-        rules:
-          - uid: printer-offline-warning
-            title: PrinterOfflineWarning
-            condition: C
-            for: 2m
-            noDataState: OK
-            execErrState: OK
-            annotations:
-              summary: "Print.Web printer offline on edge2"
-              description: "Print.Web reports the NuPrint 210 transport is offline or unreachable. IRC-only by design: do not thermal-print an alert when the thermal printer itself is offline."
-              runbook: "1. Check edge2 power/network 2. Check USB/CUPS queue 3. Open https://print.iamworkin.lan/admin 4. Do not force thermal routing for offline alerts."
-            labels:
-              severity: warning
-              service: print-web
-              alert_channel: irc
-            data:
-              - refId: A
-                relativeTimeRange: {from: 120, to: 0}
-                datasourceUid: prometheus
-                model: {expr: 'print_printer_state{job="printweb-otel"} == 2', instant: true, refId: A}
-              - refId: B
-                relativeTimeRange: {from: 120, to: 0}
-                datasourceUid: __expr__
-                model: {type: reduce, expression: A, reducer: last, refId: B}
-              - refId: C
-                relativeTimeRange: {from: 120, to: 0}
-                datasourceUid: __expr__
-                model: {type: threshold, expression: B, conditions: [{evaluator: {params: [0], type: gt}}], refId: C}
-          - uid: print-paper-roll-critical
-            title: PrintPaperRollCritical
-            condition: C
-            for: 2m
-            noDataState: OK
-            execErrState: OK
-            annotations:
-              summary: "Print paper depleted on edge2"
-              description: "NuPrint 210 reports paper depleted while the printer is still online. Load a new roll, drain the hardware buffer if needed, then replay DeadLetter jobs from /print-log."
-              runbook: "1. Load a fresh roll 2. Drain the hardware buffer if paper-out happened mid-job 3. Open https://print.iamworkin.lan/print-log 4. Retry DeadLetter jobs after the state clears."
-            labels:
-              severity: critical
-              service: print-web
-              alert_channel: thermal_print
-            data:
-              - refId: A
-                relativeTimeRange: {from: 120, to: 0}
-                datasourceUid: prometheus
-                model: {expr: 'print_printer_state{job="printweb-otel"} == 3 and print_printer_online{job="printweb-otel"} == 1', instant: true, refId: A}
-              - refId: B
-                relativeTimeRange: {from: 120, to: 0}
-                datasourceUid: __expr__
-                model: {type: reduce, expression: A, reducer: last, refId: B}
-              - refId: C
-                relativeTimeRange: {from: 120, to: 0}
-                datasourceUid: __expr__
-                model: {type: threshold, expression: B, conditions: [{evaluator: {params: [0], type: gt}}], refId: C}
-          - uid: printer-jam-warning
-            title: PrinterJamWarning
-            condition: C
-            for: 2m
-            noDataState: OK
-            execErrState: OK
-            annotations:
-              summary: "Print.Web printer jam on edge2"
-              description: "Print.Web reports a paper/cutter jam state. IRC-only: clear the jam, drain the hardware buffer if bytes were queued, then retry affected jobs."
-              runbook: "1. Clear paper/cutter path 2. Drain hardware buffer if CUPS queued bytes 3. Verify /api/print/status 4. Retry affected jobs."
-            labels:
-              severity: warning
-              service: print-web
-              alert_channel: irc
-            data:
-              - refId: A
-                relativeTimeRange: {from: 120, to: 0}
-                datasourceUid: prometheus
-                model: {expr: 'print_printer_state{job="printweb-otel"} == 4', instant: true, refId: A}
-              - refId: B
-                relativeTimeRange: {from: 120, to: 0}
-                datasourceUid: __expr__
-                model: {type: reduce, expression: A, reducer: last, refId: B}
-              - refId: C
-                relativeTimeRange: {from: 120, to: 0}
-                datasourceUid: __expr__
-                model: {type: threshold, expression: B, conditions: [{evaluator: {params: [0], type: gt}}], refId: C}
-          - uid: printer-head-error-critical
-            title: PrinterHeadErrorCritical
-            condition: C
-            for: 2m
-            noDataState: OK
-            execErrState: OK
-            annotations:
-              summary: "Print.Web printer head error on edge2"
-              description: "Print.Web reports a thermal head or unrecoverable printer error. Critical routing may enter the thermal digest per existing policy; IRC remains the primary triage stream."
-              runbook: "1. Let the printer cool if overheated 2. Power-cycle only after checking queued jobs 3. Verify /api/print/status 4. Retry jobs after the state clears."
-            labels:
-              severity: critical
-              service: print-web
-              alert_channel: thermal_print
-            data:
-              - refId: A
-                relativeTimeRange: {from: 120, to: 0}
-                datasourceUid: prometheus
-                model: {expr: 'print_printer_state{job="printweb-otel"} == 5', instant: true, refId: A}
-              - refId: B
-                relativeTimeRange: {from: 120, to: 0}
-                datasourceUid: __expr__
-                model: {type: reduce, expression: A, reducer: last, refId: B}
-              - refId: C
-                relativeTimeRange: {from: 120, to: 0}
-                datasourceUid: __expr__
-                model: {type: threshold, expression: B, conditions: [{evaluator: {params: [0], type: gt}}], refId: C}
-          - uid: printer-cover-open-warning
-            title: PrinterCoverOpenWarning
-            condition: C
-            for: 2m
-            noDataState: OK
-            execErrState: OK
-            annotations:
-              summary: "Print.Web printer cover open on edge2"
-              description: "Print.Web reports the printer cover/lid is open. IRC-only: close the cover and verify /api/print/status before retrying jobs."
-              runbook: "1. Close the printer cover 2. Verify /api/print/status returns online 3. Retry affected jobs only after the state clears."
-            labels:
-              severity: warning
-              service: print-web
-              alert_channel: irc
-            data:
-              - refId: A
-                relativeTimeRange: {from: 120, to: 0}
-                datasourceUid: prometheus
-                model: {expr: 'print_printer_state{job="printweb-otel"} == 6', instant: true, refId: A}
-              - refId: B
-                relativeTimeRange: {from: 120, to: 0}
-                datasourceUid: __expr__
-                model: {type: reduce, expression: A, reducer: last, refId: B}
-              - refId: C
-                relativeTimeRange: {from: 120, to: 0}
-                datasourceUid: __expr__
-                model: {type: threshold, expression: B, conditions: [{evaluator: {params: [0], type: gt}}], refId: C}
       - orgId: 1
         name: CI Runners
         folder: CI Alerts

tests/bluejay-infra-lint/FleetManifestLintTests.cs

@@ -304,7 +304,7 @@ public sealed class FleetManifestLintTests
     }
 
     [Fact]
-    public void Monitoring_MustIncludeRequiredAlertRoutingGuards()
+    public void Monitoring_MustAlertWhenLinuxRunnerDeploymentIsUnavailable()
     {
         var monitoring = File.ReadAllText(Path.Combine(Inventory.BluejayRoot, "apps", "monitoring", "noc-monitoring.yaml"));
 
@@ -315,15 +315,6 @@ public sealed class FleetManifestLintTests
         monitoring.Should().Contain("folder: CI Alerts");
         monitoring.Should().Contain("uid: linux-runner-offline");
         monitoring.Should().Contain("alert_channel: irc");
-
-        monitoring.Should().Contain("PrinterOfflineWarning");
-        monitoring.Should().Contain("expr: print_printer_state{job=\"printweb-otel\"} == 2");
-        monitoring.Should().Contain("IRC-only by design: do not thermal-print an alert when the thermal printer itself is offline.");
-        monitoring.Should().Contain("PrintPaperRollCritical");
-        monitoring.Should().Contain("expr: print_printer_state{job=\"printweb-otel\"} == 3 and print_printer_online{job=\"printweb-otel\"} == 1");
-        monitoring.Should().Contain("PrinterJamWarning");
-        monitoring.Should().Contain("PrinterHeadErrorCritical");
-        monitoring.Should().Contain("PrinterCoverOpenWarning");
     }
 
     [Fact]