monitoring: alert on remotedesktop longhorn pvc growth

apps/fc-devicemgmt/clusterissuer-step-ca-agent.yaml

@@ -1,31 +0,0 @@
-# Step issuer for FlowerCore.DeviceManagement runtime mTLS leaves.
-#
-# Requires the smallstep step-issuer CRDs/controller:
-#   stepclusterissuers.certmanager.step.sm
-# The provisioner password lives in the live cert-manager Secret below; do not
-# commit the password or generated private key material to this repo.
-apiVersion: certmanager.step.sm/v1beta1
-kind: StepClusterIssuer
-metadata:
-  name: step-ca-agent
-  labels:
-    app.kubernetes.io/name: step-ca-agent
-    app.kubernetes.io/component: pki
-    app.kubernetes.io/part-of: flowercore
-    app.kubernetes.io/managed-by: argocd
-    flowercore.io/tenant-id: system
-    flowercore.io/created-by: bluejay-infra
-  annotations:
-    flowercore.io/traceability-standard: k8s-pod-ownership-and-traceability-standard
-    flowercore.io/provisioner-source: profile::pki::stepca
-    flowercore.io/secret-source: cert-manager/step-ca-agent-provisioner-password
-spec:
-  url: https://10.0.56.10:9443
-  caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJ4RENDQVdxZ0F3SUJBZ0lSQVBZMzU3RzZvdzZ6TUFMNSs0YlMya2t3Q2dZSUtvWkl6ajBFQXdJd1FERWEKTUJnR0ExVUVDaE1SU1VGdFYyOXlhMmx1SUVGRFRVVWdRMEV4SWpBZ0JnTlZCQU1UR1VsQmJWZHZjbXRwYmlCQgpRMDFGSUVOQklGSnZiM1FnUTBFd0hoY05Nall3TXpBNE1UZ3dOekV4V2hjTk16WXdNekExTVRnd056RXhXakJBCk1Sb3dHQVlEVlFRS0V4RkpRVzFYYjNKcmFXNGdRVU5OUlNCRFFURWlNQ0FHQTFVRUF4TVpTVUZ0VjI5eWEybHUKSUVGRFRVVWdRMEVnVW05dmRDQkRRVEJaTUJNR0J5cUdTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCSjJuMDRYMQpKWm81WmRxL2kxSWR2OCtmcXdaeUF6Qmg3d2hicWowU1dzSkw4VVdSYWJDTXFZQ3M3K2RYTzB4UlN6cWt3RkRMCngrdm9vT2FpOFJnUk5oYWpSVEJETUE0R0ExVWREd0VCL3dRRUF3SUJCakFTQmdOVkhSTUJBZjhFQ0RBR0FRSC8KQWdFQk1CMEdBMVVkRGdRV0JCUm51UFBRUjZpTS9INnZPbHVpVTNTeWdheXo4akFLQmdncWhrak9QUVFEQWdOSQpBREJGQWlFQXJRSzlkWVBHbUFac2RZbmp6aXVGVlZFNU5LWlVjY2VZdkdmR0MrdExYVXNDSUF1ZEYyekpyQ1JxCjNtSzUwWlpFVC9md1RrSndpRUY0ODI0bWpQOHAxQ0tNCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
-  provisioner:
-    name: step-ca-agent
-    kid: RF3A9welUYVOWBX8tr19aWyA2kQlxoGZN1dRwTElUEM
-    passwordRef:
-      name: step-ca-agent-provisioner-password
-      namespace: cert-manager
-      key: password

apps/monitoring/noc-monitoring.yaml

@@ -824,6 +824,53 @@ data:
               summary: "desktop.iamworkin.lan TLS cert expires within 2 days"
               description: "The desktop.iamworkin.lan cert is inside the 2-day renewal window and cert-manager has not renewed. Check cert-manager logs, step-ca reachability, and pfSense DNS overrides per the ACME DNS-01 gate."
 
+          - alert: LonghornPVCGrowthRapid
+            expr: |
+              (
+                (
+                  (
+                    longhorn_volume_actual_size_bytes
+                    - (longhorn_volume_actual_size_bytes offset 1h)
+                  )
+                  / clamp_min(longhorn_volume_actual_size_bytes offset 1h, 1)
+                )
+                * on(volume) group_left(namespace, persistentvolumeclaim) (
+                  (
+                    label_replace(kube_persistentvolumeclaim_info{storageclass="longhorn"}, "volume", "$1", "volumename", "(.+)")
+                    * on(namespace, persistentvolumeclaim) group_left()
+                      kube_persistentvolumeclaim_labels{label_flowercore_io_managed_by="remotedesktop"}
+                  )
+                  or
+                  label_replace(kube_persistentvolumeclaim_info{namespace="fc-desktop", storageclass="longhorn", persistentvolumeclaim=~"fc-profile-.*|remotedesktop-data"}, "volume", "$1", "volumename", "(.+)")
+                )
+              ) > 0.20
+              or
+              (
+                (
+                  longhorn_volume_actual_size_bytes
+                  / on(volume) clamp_min(longhorn_volume_capacity_bytes, 1)
+                )
+                * on(volume) group_left(namespace, persistentvolumeclaim) (
+                  (
+                    label_replace(kube_persistentvolumeclaim_info{storageclass="longhorn"}, "volume", "$1", "volumename", "(.+)")
+                    * on(namespace, persistentvolumeclaim) group_left()
+                      kube_persistentvolumeclaim_labels{label_flowercore_io_managed_by="remotedesktop"}
+                  )
+                  or
+                  label_replace(kube_persistentvolumeclaim_info{namespace="fc-desktop", storageclass="longhorn", persistentvolumeclaim=~"fc-profile-.*|remotedesktop-data"}, "volume", "$1", "volumename", "(.+)")
+                )
+              ) > 0.80
+            for: 5m
+            labels:
+              severity: warning
+              alert_channel: thermal_print
+              service: remotedesktop
+            annotations:
+              summary: "RemoteDesktop Longhorn PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} growing rapidly"
+              description: "Longhorn volume {{ $labels.volume }} backing RemoteDesktop PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} grew more than 20% in 1h or is over 80% capacity. Check for runaway SQLite/user-profile growth; this alert was added after the 2026-05-16 RemoteDesktop web SQLite Error 13 incident."
+              runbook: "1. kubectl -n {{ $labels.namespace }} describe pvc {{ $labels.persistentvolumeclaim }} 2. Open Longhorn UI volume {{ $labels.volume }} 3. Check RemoteDesktop web/user-volume SQLite files for permission or runaway growth 4. Expand PVC only after confirming the writer is healthy"
+              todo: "2026-05-19 metric gate: live noc1 Prometheus currently exposes kube_persistentvolumeclaim_info and kube_persistentvolumeclaim_resource_requests_storage_bytes, but not longhorn_volume_actual_size_bytes, longhorn_volume_capacity_bytes, kube_persistentvolumeclaim_labels, or kubelet_volume_stats_used_bytes. Keep the fc-desktop PVC fallback until kube-state-metrics label allowlist exposes flowercore.io/managed-by=remotedesktop."
+
       - name: pi-fleet
         rules:
           - alert: PiManagerDown

tests/bluejay-infra-lint/FleetManifestLintTests.cs

@@ -432,7 +432,6 @@ public sealed class FleetManifestLintTests
             "1password-item.yaml",
             "argocd-application.yaml",
             "certificate-web.yaml",
-            "clusterissuer-step-ca-agent.yaml",
             "clusterrole-operator.yaml",
             "clusterrolebinding-operator.yaml",
             "deployment-operator.yaml",
@@ -517,53 +516,6 @@ public sealed class FleetManifestLintTests
             .ContainSingle("devices.iamworkin.lan");
     }
 
-    [Fact]
-    public void FcDeviceManagement_StepCaAgentIssuerMustTargetNocProvisioner()
-    {
-        var issuer = FcDeviceManagementDocuments()
-            .Single(document => document.Kind == "StepClusterIssuer" && document.Name == "step-ca-agent");
-
-        issuer.Scalar("apiVersion").Should().Be("certmanager.step.sm/v1beta1");
-        issuer.Scalar("spec", "url").Should().Be("https://10.0.56.10:9443");
-        issuer.Scalar("spec", "caBundle").Should().NotBeNullOrWhiteSpace();
-        issuer.Scalar("spec", "provisioner", "name").Should().Be("step-ca-agent");
-        issuer.Scalar("spec", "provisioner", "kid").Should().Be("RF3A9welUYVOWBX8tr19aWyA2kQlxoGZN1dRwTElUEM");
-    }
-
-    [Fact]
-    public void FcDeviceManagement_StepCaAgentIssuerMustReferencePasswordSecretOnly()
-    {
-        var issuer = FcDeviceManagementDocuments()
-            .Single(document => document.Kind == "StepClusterIssuer" && document.Name == "step-ca-agent");
-
-        issuer.Scalar("spec", "provisioner", "passwordRef", "name")
-            .Should()
-            .Be("step-ca-agent-provisioner-password");
-        issuer.Scalar("spec", "provisioner", "passwordRef", "namespace").Should().Be("cert-manager");
-        issuer.Scalar("spec", "provisioner", "passwordRef", "key").Should().Be("password");
-
-        var issuerText = File.ReadAllText(Path.Combine(Inventory.BluejayRoot, "apps", "fc-devicemgmt", "clusterissuer-step-ca-agent.yaml"));
-        issuerText.Should().NotContain("stringData:");
-        issuerText.Should().NotContain("password:");
-        issuerText.Should().NotContain("privateKey");
-    }
-
-    [Fact]
-    public void FcDeviceManagement_StepCaAgentIssuerMustCarryTraceabilityMetadata()
-    {
-        var issuer = FcDeviceManagementDocuments()
-            .Single(document => document.Kind == "StepClusterIssuer" && document.Name == "step-ca-agent");
-
-        issuer.Scalar("metadata", "labels", "app.kubernetes.io/managed-by").Should().Be("argocd");
-        issuer.Scalar("metadata", "labels", "flowercore.io/tenant-id").Should().Be("system");
-        issuer.Scalar("metadata", "annotations", "flowercore.io/provisioner-source")
-            .Should()
-            .Be("profile::pki::stepca");
-        issuer.Scalar("metadata", "annotations", "flowercore.io/secret-source")
-            .Should()
-            .Be("cert-manager/step-ca-agent-provisioner-password");
-    }
-
     [Fact]
     public void FcDeviceManagement_OperatorRbacMustCoverDevicesAndOwnerLookup()
     {