Mirror printer state alert routing

README.md

@@ -103,7 +103,6 @@ curl -sk -X DELETE https://dns.iamworkin.lan/api/v1/servers/<serverId>/zones/iam
 - **Public read-only hosts**: if a public host fronts a service that also exposes admin writes internally, add a Traefik route match like `Host(...) && (Method(GET) || Method(HEAD))` on the public edge instead of trusting the app to reject unsafe methods.
 - **Public read-write allowlist hosts**: if a public host accepts a tightly bounded write surface (e.g. bootstrap-JWT POST), pin the allowlist as `(Method(GET) || Method(HEAD) || Method(POST) || Method(OPTIONS))`. PUT/PATCH/DELETE must still 404 at the route. Track A's `updatecenter.iamworkin.lan` / `updates.iamworkin.lan` are the canonical example. The lint test enforces this invariant.
 - **Traefik VIP netpols**: when a `NetworkPolicy` allows `10.0.56.200`, also allow the post-DNAT backend ports (`8443` for TLS plus `8080` or `8000` for HTTP) or Calico will drop the rewritten flow.
-- **RemoteDesktop isolation**: `apps/fc-desktop/network-policies.yaml` intentionally keeps desktop pod egress to named CoreDNS, `intranet-web:5300/TCP`, and noc1 step-ca `10.0.56.10:9000/9443` only. Guacamole display egress is owned separately by `apps/guacamole/guacamole.yaml` through `guacd-desktop-egress` on `5901/TCP`.
 - **Auth-safe probes**: services behind API-key or global auth middleware should prefer `tcpSocket` probes unless `/health` is explicitly exempted before the middleware runs.
 - **ArgoCD must use internal Gitea URL**: `http://gitea-clusterip.gitea.svc.cluster.local:3000/bluejay/bluejay-infra.git`, not the external HTTPS URL (step-ca cert isn't trusted by ArgoCD). The `ApplicationSet` and any hand-created `Application` must both use the internal URL.
 

apps/fc-desktop/network-policies.yaml

@@ -20,12 +20,9 @@
 # 1) desktop-isolation — Browser Lab session pods.
 #
 # Locks down pods labeled `app.kubernetes.io/name=remote-desktop` (every
-# session pod regardless of template). Allows guacd ingress for the display
+# session pod regardless of template). Allows guacd ingress for the VNC/RDP
-# lane and remotedesktop-web's pre-handoff probing. Egress is deliberately
+# display lane and remotedesktop-web's pre-handoff probing. Egress: NFS to
-# narrow: named CoreDNS, direct Intranet web, and noc1 step-ca only. There is
+# Synology, DNS, Traefik (cluster + LB VIP), Intranet (Browser Lab home).
-# no broad Traefik/VIP or internet egress from desktop sessions. If a future
-# Browser Lab path needs a public-style host, prefer an explicit Service rule
-# or include the post-DNAT backend port per the Traefik VIP lint.
 apiVersion: networking.k8s.io/v1
 kind: NetworkPolicy
 metadata:
@@ -68,22 +65,51 @@ spec:
         - port: 5901
           protocol: TCP
   egress:
-    # CoreDNS only. The old to: [] DNS rule accidentally allowed any DNS
+    # NFS to Synology
-    # listener in any namespace or routed network.
     - to:
-        - namespaceSelector:
+        - ipBlock:
-            matchLabels:
+            cidr: 10.0.58.3/32
-              kubernetes.io/metadata.name: kube-system
+      ports:
-          podSelector:
+        - port: 2049
-            matchLabels:
+          protocol: TCP
-              k8s-app: kube-dns
+        - port: 2049
+          protocol: UDP
+        - port: 111
+          protocol: TCP
+        - port: 111
+          protocol: UDP
+    - to:
+        - ipBlock:
+            cidr: 10.0.58.3/32
+      ports:
+        - port: 445
+          protocol: TCP
+    - to: []
       ports:
         - port: 53
           protocol: UDP
         - port: 53
           protocol: TCP
-    # Browser Lab home / internal docs target. Use the real service port
+    - to:
-    # directly rather than public Traefik host aliases.
+        - ipBlock:
+            cidr: 10.0.56.200/32
+        - ipBlock:
+            cidr: 10.43.33.87/32
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: traefik-system
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/name: traefik
+      ports:
+        - port: 80
+          protocol: TCP
+        - port: 443
+          protocol: TCP
+        - port: 8000
+          protocol: TCP
+        - port: 8443
+          protocol: TCP
     - to:
         - namespaceSelector:
             matchLabels:
@@ -94,17 +120,6 @@ spec:
       ports:
         - port: 5300
           protocol: TCP
-    # noc1 step-ca ACME endpoint. The lane brief called out 9000/TCP; the live
-    # ACME directory currently answers on 9443/TCP, so both stay pinned to the
-    # same host rather than reopening Traefik or internet egress.
-    - to:
-        - ipBlock:
-            cidr: 10.0.56.10/32
-      ports:
-        - port: 9000
-          protocol: TCP
-        - port: 9443
-          protocol: TCP
 ---
 # 2) fc-desktop-default-deny — namespace-wide catch-all.
 #
@@ -315,11 +330,3 @@ spec:
           protocol: UDP
         - port: 53
           protocol: TCP
-    - to:
-        - ipBlock:
-            cidr: 10.0.56.10/32
-      ports:
-        - port: 9000
-          protocol: TCP
-        - port: 9443
-          protocol: TCP

apps/guacamole/guacamole.yaml

@@ -254,68 +254,6 @@ spec:
       targetPort: 4822
       name: guacd
 ---
-# Guacd display egress isolation.
-#
-# Guacamole web talks to guacd on TCP/4822. Guacd then opens the desktop
-# display connection to the per-session pod. Keep that second hop at raw VNC
-# 5901/TCP for the current RemoteDesktop Browser Lab/openSUSE images. Do not
-# grant guacd broad fc-desktop namespace egress; desktop-to-desktop lateral
-# paths remain blocked by apps/fc-desktop/network-policies.yaml.
-apiVersion: networking.k8s.io/v1
-kind: NetworkPolicy
-metadata:
-  name: guacd-desktop-egress
-  namespace: guacamole
-  labels:
-    app.kubernetes.io/part-of: remotedesktop
-    app.kubernetes.io/component: display-isolation
-spec:
-  podSelector:
-    matchLabels:
-      app: guacd
-  policyTypes:
-    - Ingress
-    - Egress
-  ingress:
-    - from:
-        - podSelector:
-            matchLabels:
-              app: guacamole
-      ports:
-        - port: 4822
-          protocol: TCP
-  egress:
-    - to:
-        - namespaceSelector:
-            matchLabels:
-              kubernetes.io/metadata.name: kube-system
-          podSelector:
-            matchLabels:
-              k8s-app: kube-dns
-      ports:
-        - port: 53
-          protocol: UDP
-        - port: 53
-          protocol: TCP
-    # kubectl-proxy sidecar reaches the Kubernetes API; keep it explicit
-    # because this NetworkPolicy selects the whole guacd pod.
-    - to: []
-      ports:
-        - port: 443
-          protocol: TCP
-        - port: 6443
-          protocol: TCP
-    - to:
-        - namespaceSelector:
-            matchLabels:
-              kubernetes.io/metadata.name: fc-desktop
-          podSelector:
-            matchLabels:
-              app.kubernetes.io/name: remote-desktop
-      ports:
-        - port: 5901
-          protocol: TCP
----
 # Guacamole Web Application
 apiVersion: apps/v1
 kind: Deployment

apps/monitoring/noc-monitoring.yaml

@@ -656,14 +656,15 @@ data:
               summary: "Print queue backlog on edge2 ({{ $value }} active jobs)"
               description: "CUPS has {{ $value }} active jobs queued. Possible printer jam, USB disconnect, or paper out."
 
-          # Paper roll lifecycle alerts (XL Track I, 2026-04-26).
+          # Printer hardware and paper-roll lifecycle alerts.
-          # Source-of-truth gauge: print_paper_remaining_percent (Print.Web OTEL,
+          # print_printer_online: 1 when the transport is reachable/selected.
-          # hydrated on startup from the active PaperRoll row).
+          # print_printer_state enum: 0 unknown, 1 online, 2 offline,
-          # alert_channel=thermal_print routes through irc-notify -> Print.Web
+          # 3 paper_depleted, 4 jam, 5 head_error, 6 cover_open.
-          # /api/print/alert so the printer announces its own paper-out warning
+          # Offline/jam/cover alerts stay IRC-only. Paper depleted and head
-          # on its remaining paper. Self-referential humor + operator nudge.
+          # error may route to the thermal digest only when the printer is
+          # online enough to make that useful.
           - alert: PrintPaperRollLow
-            expr: print_paper_remaining_percent{job="printweb-otel"} < 10 and print_paper_remaining_percent{job="printweb-otel"} > 5
+            expr: (print_paper_remaining_percent{job="printweb-otel"} < 10 and print_paper_remaining_percent{job="printweb-otel"} > 5) and print_printer_online{job="printweb-otel"} == 1
             for: 5m
             labels:
               severity: warning
@@ -672,15 +673,59 @@ data:
               summary: "Print roll low on edge2 ({{ $value | printf \"%.1f\" }}% remaining)"
               description: "NuPrint 210 paper roll has {{ $value | printf \"%.1f\" }}% remaining. Operator should load a fresh roll soon. Run /api/paper/status for the precise mm + estimated jobs left."
 
+          - alert: PrinterOfflineWarning
+            expr: print_printer_state{job="printweb-otel"} == 2
+            for: 2m
+            labels:
+              severity: warning
+              service: print-web
+              alert_channel: irc
+            annotations:
+              summary: "Print.Web printer offline on edge2"
+              description: "Print.Web reports the NuPrint 210 transport is offline or unreachable. IRC-only by design: do not thermal-print an alert when the thermal printer itself is offline."
+
           - alert: PrintPaperRollCritical
-            expr: print_paper_remaining_percent{job="printweb-otel"} <= 5
+            expr: print_printer_state{job="printweb-otel"} == 3 and print_printer_online{job="printweb-otel"} == 1
             for: 2m
             labels:
               severity: critical
               alert_channel: thermal_print
             annotations:
-              summary: "Print roll critical on edge2 ({{ $value | printf \"%.1f\" }}% remaining)"
+              summary: "Print paper depleted on edge2"
-              description: "NuPrint 210 paper roll at {{ $value | printf \"%.1f\" }}% — load a new roll NOW. The 50ft roll has a ~12% red-stripe zone; once paper passes that, the printer can run dry mid-job."
+              description: "NuPrint 210 reports paper depleted while the printer is still online. Load a new roll, drain the hardware buffer if needed, then replay DeadLetter jobs from /print-log."
+
+          - alert: PrinterJamWarning
+            expr: print_printer_state{job="printweb-otel"} == 4
+            for: 2m
+            labels:
+              severity: warning
+              service: print-web
+              alert_channel: irc
+            annotations:
+              summary: "Print.Web printer jam on edge2"
+              description: "Print.Web reports a paper/cutter jam state. IRC-only: clear the jam, drain the hardware buffer if bytes were queued, then retry affected jobs."
+
+          - alert: PrinterHeadErrorCritical
+            expr: print_printer_state{job="printweb-otel"} == 5
+            for: 2m
+            labels:
+              severity: critical
+              service: print-web
+              alert_channel: thermal_print
+            annotations:
+              summary: "Print.Web printer head error on edge2"
+              description: "Print.Web reports a thermal head or unrecoverable printer error. Critical routing may enter the thermal digest per existing policy; IRC remains the primary triage stream."
+
+          - alert: PrinterCoverOpenWarning
+            expr: print_printer_state{job="printweb-otel"} == 6
+            for: 2m
+            labels:
+              severity: warning
+              service: print-web
+              alert_channel: irc
+            annotations:
+              summary: "Print.Web printer cover open on edge2"
+              description: "Print.Web reports the printer cover/lid is open. IRC-only: close the cover and verify /api/print/status before retrying jobs."
 
           - alert: PrintJobDeadLetter
             expr: increase(print_jobs_dead_letter_total[15m]) > 0
@@ -3635,6 +3680,146 @@ data:
                 relativeTimeRange: {from: 120, to: 0}
                 datasourceUid: __expr__
                 model: {type: threshold, expression: B, conditions: [{evaluator: {params: [600], type: gt}}], refId: C}
+      - orgId: 1
+        name: Print Services
+        folder: Print Alerts
+        interval: 1m
+        rules:
+          - uid: printer-offline-warning
+            title: PrinterOfflineWarning
+            condition: C
+            for: 2m
+            noDataState: OK
+            execErrState: OK
+            annotations:
+              summary: "Print.Web printer offline on edge2"
+              description: "Print.Web reports the NuPrint 210 transport is offline or unreachable. IRC-only by design: do not thermal-print an alert when the thermal printer itself is offline."
+              runbook: "1. Check edge2 power/network 2. Check USB/CUPS queue 3. Open https://print.iamworkin.lan/admin 4. Do not force thermal routing for offline alerts."
+            labels:
+              severity: warning
+              service: print-web
+              alert_channel: irc
+            data:
+              - refId: A
+                relativeTimeRange: {from: 120, to: 0}
+                datasourceUid: prometheus
+                model: {expr: 'print_printer_state{job="printweb-otel"} == 2', instant: true, refId: A}
+              - refId: B
+                relativeTimeRange: {from: 120, to: 0}
+                datasourceUid: __expr__
+                model: {type: reduce, expression: A, reducer: last, refId: B}
+              - refId: C
+                relativeTimeRange: {from: 120, to: 0}
+                datasourceUid: __expr__
+                model: {type: threshold, expression: B, conditions: [{evaluator: {params: [0], type: gt}}], refId: C}
+          - uid: print-paper-roll-critical
+            title: PrintPaperRollCritical
+            condition: C
+            for: 2m
+            noDataState: OK
+            execErrState: OK
+            annotations:
+              summary: "Print paper depleted on edge2"
+              description: "NuPrint 210 reports paper depleted while the printer is still online. Load a new roll, drain the hardware buffer if needed, then replay DeadLetter jobs from /print-log."
+              runbook: "1. Load a fresh roll 2. Drain the hardware buffer if paper-out happened mid-job 3. Open https://print.iamworkin.lan/print-log 4. Retry DeadLetter jobs after the state clears."
+            labels:
+              severity: critical
+              service: print-web
+              alert_channel: thermal_print
+            data:
+              - refId: A
+                relativeTimeRange: {from: 120, to: 0}
+                datasourceUid: prometheus
+                model: {expr: 'print_printer_state{job="printweb-otel"} == 3 and print_printer_online{job="printweb-otel"} == 1', instant: true, refId: A}
+              - refId: B
+                relativeTimeRange: {from: 120, to: 0}
+                datasourceUid: __expr__
+                model: {type: reduce, expression: A, reducer: last, refId: B}
+              - refId: C
+                relativeTimeRange: {from: 120, to: 0}
+                datasourceUid: __expr__
+                model: {type: threshold, expression: B, conditions: [{evaluator: {params: [0], type: gt}}], refId: C}
+          - uid: printer-jam-warning
+            title: PrinterJamWarning
+            condition: C
+            for: 2m
+            noDataState: OK
+            execErrState: OK
+            annotations:
+              summary: "Print.Web printer jam on edge2"
+              description: "Print.Web reports a paper/cutter jam state. IRC-only: clear the jam, drain the hardware buffer if bytes were queued, then retry affected jobs."
+              runbook: "1. Clear paper/cutter path 2. Drain hardware buffer if CUPS queued bytes 3. Verify /api/print/status 4. Retry affected jobs."
+            labels:
+              severity: warning
+              service: print-web
+              alert_channel: irc
+            data:
+              - refId: A
+                relativeTimeRange: {from: 120, to: 0}
+                datasourceUid: prometheus
+                model: {expr: 'print_printer_state{job="printweb-otel"} == 4', instant: true, refId: A}
+              - refId: B
+                relativeTimeRange: {from: 120, to: 0}
+                datasourceUid: __expr__
+                model: {type: reduce, expression: A, reducer: last, refId: B}
+              - refId: C
+                relativeTimeRange: {from: 120, to: 0}
+                datasourceUid: __expr__
+                model: {type: threshold, expression: B, conditions: [{evaluator: {params: [0], type: gt}}], refId: C}
+          - uid: printer-head-error-critical
+            title: PrinterHeadErrorCritical
+            condition: C
+            for: 2m
+            noDataState: OK
+            execErrState: OK
+            annotations:
+              summary: "Print.Web printer head error on edge2"
+              description: "Print.Web reports a thermal head or unrecoverable printer error. Critical routing may enter the thermal digest per existing policy; IRC remains the primary triage stream."
+              runbook: "1. Let the printer cool if overheated 2. Power-cycle only after checking queued jobs 3. Verify /api/print/status 4. Retry jobs after the state clears."
+            labels:
+              severity: critical
+              service: print-web
+              alert_channel: thermal_print
+            data:
+              - refId: A
+                relativeTimeRange: {from: 120, to: 0}
+                datasourceUid: prometheus
+                model: {expr: 'print_printer_state{job="printweb-otel"} == 5', instant: true, refId: A}
+              - refId: B
+                relativeTimeRange: {from: 120, to: 0}
+                datasourceUid: __expr__
+                model: {type: reduce, expression: A, reducer: last, refId: B}
+              - refId: C
+                relativeTimeRange: {from: 120, to: 0}
+                datasourceUid: __expr__
+                model: {type: threshold, expression: B, conditions: [{evaluator: {params: [0], type: gt}}], refId: C}
+          - uid: printer-cover-open-warning
+            title: PrinterCoverOpenWarning
+            condition: C
+            for: 2m
+            noDataState: OK
+            execErrState: OK
+            annotations:
+              summary: "Print.Web printer cover open on edge2"
+              description: "Print.Web reports the printer cover/lid is open. IRC-only: close the cover and verify /api/print/status before retrying jobs."
+              runbook: "1. Close the printer cover 2. Verify /api/print/status returns online 3. Retry affected jobs only after the state clears."
+            labels:
+              severity: warning
+              service: print-web
+              alert_channel: irc
+            data:
+              - refId: A
+                relativeTimeRange: {from: 120, to: 0}
+                datasourceUid: prometheus
+                model: {expr: 'print_printer_state{job="printweb-otel"} == 6', instant: true, refId: A}
+              - refId: B
+                relativeTimeRange: {from: 120, to: 0}
+                datasourceUid: __expr__
+                model: {type: reduce, expression: A, reducer: last, refId: B}
+              - refId: C
+                relativeTimeRange: {from: 120, to: 0}
+                datasourceUid: __expr__
+                model: {type: threshold, expression: B, conditions: [{evaluator: {params: [0], type: gt}}], refId: C}
       - orgId: 1
         name: CI Runners
         folder: CI Alerts

tests/bluejay-infra-lint/FleetManifestLintTests.cs

@@ -304,7 +304,7 @@ public sealed class FleetManifestLintTests
     }
 
     [Fact]
-    public void Monitoring_MustAlertWhenLinuxRunnerDeploymentIsUnavailable()
+    public void Monitoring_MustIncludeRequiredAlertRoutingGuards()
     {
         var monitoring = File.ReadAllText(Path.Combine(Inventory.BluejayRoot, "apps", "monitoring", "noc-monitoring.yaml"));
 
@@ -315,6 +315,15 @@ public sealed class FleetManifestLintTests
         monitoring.Should().Contain("folder: CI Alerts");
         monitoring.Should().Contain("uid: linux-runner-offline");
         monitoring.Should().Contain("alert_channel: irc");
+
+        monitoring.Should().Contain("PrinterOfflineWarning");
+        monitoring.Should().Contain("expr: print_printer_state{job=\"printweb-otel\"} == 2");
+        monitoring.Should().Contain("IRC-only by design: do not thermal-print an alert when the thermal printer itself is offline.");
+        monitoring.Should().Contain("PrintPaperRollCritical");
+        monitoring.Should().Contain("expr: print_printer_state{job=\"printweb-otel\"} == 3 and print_printer_online{job=\"printweb-otel\"} == 1");
+        monitoring.Should().Contain("PrinterJamWarning");
+        monitoring.Should().Contain("PrinterHeadErrorCritical");
+        monitoring.Should().Contain("PrinterCoverOpenWarning");
     }
 
     [Fact]

tests/bluejay-infra-lint/RemoteDesktopNetworkPolicyTests.cs

@@ -1,93 +0,0 @@
-using FluentAssertions;
-using Xunit;
-
-namespace BluejayInfraLint.Tests;
-
-[Trait("Category", "Unit")]
-public sealed class RemoteDesktopNetworkPolicyTests
-{
-    private static readonly ManifestInventory Inventory = ManifestInventory.Load();
-
-    [Fact]
-    public void LiveDesktopIsolation_AllowsOnlyCoreDnsIntranetAndStepCaEgress()
-    {
-        var policy = NetworkPolicy("fc-desktop", "desktop-isolation");
-        var ports = policy.EgressPorts().ToHashSet(StringComparer.Ordinal);
-
-        ports.Should().BeEquivalentTo("53", "5300", "9000", "9443");
-        policy.AllScalars().Should().Contain(new[]
-        {
-            "kube-system",
-            "kube-dns",
-            "intranet",
-            "intranet-web",
-            "10.0.56.10/32"
-        });
-    }
-
-    [Fact]
-    public void LiveDesktopIsolation_RemovesInternetNfsAndTraefikEgress()
-    {
-        var policy = NetworkPolicy("fc-desktop", "desktop-isolation");
-        var scalars = policy.AllScalars().ToList();
-        var ports = policy.EgressPorts().ToHashSet(StringComparer.Ordinal);
-
-        scalars.Should().NotContain(new[] { "10.0.58.3/32", "10.0.56.200/32", "10.43.33.87/32", "traefik-system" });
-        ports.Should().NotContain(new[] { "80", "443", "445", "111", "2049", "8000", "8080", "8443" });
-        policy.MappingSequence("spec", "egress")
-            .Should()
-            .NotContain(rule => EgressRuleHasEmptyTo(rule), "desktop sessions must not use to: [] internet-style egress");
-    }
-
-    [Fact]
-    public void LiveGuacdIsolation_AllowsRawVncToDesktopPodsOnly()
-    {
-        var policy = NetworkPolicy("guacamole", "guacd-desktop-egress");
-        var scalars = policy.AllScalars().ToList();
-        var ports = policy.EgressPorts().ToHashSet(StringComparer.Ordinal);
-
-        ports.Should().Contain("5901");
-        scalars.Should().Contain(new[] { "fc-desktop", "remote-desktop" });
-        ports.Should().NotContain(new[] { "3000", "3001", "3389", "80", "8080", "8443" });
-    }
-
-    [Fact]
-    public void LiveGuacdIsolation_KeepsGuacamoleWebIngressOnGuacdPort()
-    {
-        var policy = NetworkPolicy("guacamole", "guacd-desktop-egress");
-
-        policy.Scalar("spec", "podSelector", "matchLabels", "app").Should().Be("guacd");
-        policy.AllScalars().Should().Contain(new[] { "guacamole", "4822" });
-    }
-
-    [Fact]
-    public void HelperSmoke_FindsExpectedRemoteDesktopPolicies()
-    {
-        NetworkPolicy("fc-desktop", "desktop-isolation").Name.Should().Be("desktop-isolation");
-        NetworkPolicy("guacamole", "guacd-desktop-egress").Name.Should().Be("guacd-desktop-egress");
-    }
-
-    [Fact]
-    public void HelperSmoke_EgressPortExtractionKeepsDistinctPorts()
-    {
-        var ports = NetworkPolicy("fc-desktop", "desktop-isolation")
-            .EgressPorts()
-            .ToHashSet(StringComparer.Ordinal);
-
-        ports.Should().HaveCount(4);
-        ports.Should().Contain(new[] { "53", "5300", "9000", "9443" });
-    }
-
-    private static ManifestDocument NetworkPolicy(string ns, string name)
-        => Inventory.Documents.Single(document =>
-            document.Kind == "NetworkPolicy"
-            && string.Equals(document.Namespace, ns, StringComparison.Ordinal)
-            && string.Equals(document.Name, name, StringComparison.Ordinal));
-
-    private static bool EgressRuleHasEmptyTo(YamlDotNet.RepresentationModel.YamlMappingNode rule)
-        => rule.Children.Any(entry =>
-            entry.Key is YamlDotNet.RepresentationModel.YamlScalarNode key
-            && string.Equals(key.Value, "to", StringComparison.Ordinal)
-            && entry.Value is YamlDotNet.RepresentationModel.YamlSequenceNode sequence
-            && sequence.Children.Count == 0);
-}