feat(github-runner): harden Linux runner fleet

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

apps/github-runner/README.md

@@ -0,0 +1,61 @@
+# GitHub Runner Fleet
+
+ArgoCD owns `apps/github-runner/github-runner.yaml`. Do not patch live runner
+Deployments with `kubectl`; update this manifest and let ArgoCD reconcile.
+
+## Runner Shape
+
+All repo-scoped Linux runners use:
+
+- `ACCESS_TOKEN` from the `github-runner-token` Secret
+- `RUN_AS_ROOT=false`
+- `EPHEMERAL=true`
+- `LABELS=self-hosted,linux,fc-build-linux`
+- writable non-root paths under `/home/runner` for .NET, NuGet, XDG cache, and
+  Actions tool cache
+
+`github-runner` for `FlowerCore.Common` is single-replica because it retains the
+original Longhorn ReadWriteOnce NuGet PVC. `github-runner-sharedpos` and the top
+Linux-cost repo runners use two replicas with per-pod `emptyDir` caches. That is
+the safe backlog-drain strategy: no two pods share one RWO PVC.
+
+## Post-Merge Proof
+
+After the PR is merged and ArgoCD syncs, verify the runner fleet:
+
+```bash
+kubectl -n github-runner get deploy,pods,pvc
+```
+
+Verify GitHub registration for the repo-scoped runners:
+
+```bash
+for repo in FlowerCore.Common FlowerCore.Shared.Pos FlowerCore.Puppet FlowerCore.Signage \
+            FlowerCore.DMS FlowerCore.Telephony FlowerCore.Print.Web FlowerCore.Chat \
+            FlowerCore.MySQL FlowerCore.Kiosk.Linux; do
+  echo "=== $repo ==="
+  gh api "/repos/astoltz/$repo/actions/runners" \
+    --jq '.runners[] | select(.labels[].name == "fc-build-linux") | {name,status,busy,labels:[.labels[].name]}'
+done
+```
+
+Shared.Pos publish proof after the runner pod is online:
+
+```bash
+gh run list --repo astoltz/FlowerCore.Shared.Pos \
+  --workflow "Build, Test & Publish" --branch main --limit 5
+```
+
+If the latest run is still queued after runner registration, rerun the workflow
+from GitHub Actions and verify it lands on an `rke2-linux-*` runner.
+
+## Failure Notes
+
+- `actions/setup-dotnet` permission error at `/usr/share/dotnet`: check that
+  `DOTNET_INSTALL_DIR=/home/runner/.dotnet` and related cache env vars are
+  present on the runner pod.
+- `404` during runner registration: the fine-grained PAT is valid but missing
+  repository access for that repo. Add the repo to the PAT access list; the PAT
+  value does not change.
+- `Multi-Attach` volume error: only the Common runner uses a RWO PVC and it must
+  stay single-replica. New multi-replica runners use `emptyDir`.

apps/monitoring/noc-monitoring.yaml

@@ -75,6 +75,20 @@ data:
               cluster: "rke2"
               role: "agent"
 
+      # Mac mini macOS runner node (INFRA VLAN)
+      - job_name: "macmini-node"
+        scrape_timeout: 15s
+        static_configs:
+          - targets: ["10.0.56.115:9100"]
+            labels:
+              instance: "macmini"
+              host: "macmini.iamworkin.lan"
+              vlan: "infra"
+              arch: "arm64"
+              role: "macos-runner"
+              puppet_managed: "true"
+              puppet_server: "puppet.iamworkin.lan"
+
       # In-cluster node-exporter DaemonSet
       - job_name: "k8s-node-exporter"
         kubernetes_sd_configs:
@@ -697,6 +711,36 @@ data:
               summary: "Print.Web Ollama runner held for >10m ({{ $labels.model }})"
               description: "Print.Web reports model {{ $labels.model }} with {{ $value | printf \"%.0f\" }}s of keep-alive remaining. Check concurrent requests before the Pi 5 Ollama lane thrashes."
 
+      - name: macmini-runners
+        rules:
+          - alert: MacMiniRunnerOffline
+            expr: (flowercore_github_runner_online{runner=~"macmini-.*"} == 0) or absent(flowercore_github_runner_online{runner=~"macmini-.*"})
+            for: 10m
+            labels:
+              severity: warning
+              service: github-runner
+            annotations:
+              summary: "Mac mini GitHub runner offline ({{ $labels.runner }})"
+              description: "A macmini-* GitHub Actions runner has not reported online for more than 10 minutes. Puppet manages its LaunchDaemon under /Library/LaunchDaemons/io.flowercore.github-runner-<slug>.plist; runners survive reboot and do not require a GUI session."
+
+      - name: linux-runners
+        rules:
+          - alert: LinuxRunnerOffline
+            expr: |
+              kube_deployment_status_replicas_ready{
+                namespace="github-runner",
+                deployment=~"github-runner(|-(sharedpos|puppet|signage|dms|telephony|print-web|chat|mysql|kiosk-linux))"
+              } == 0
+            for: 5m
+            labels:
+              severity: warning
+              alert_channel: irc
+              service: github-runner
+              team: ci
+            annotations:
+              summary: "Linux CI runner offline: {{ $labels.deployment }}"
+              description: "Deployment {{ $labels.deployment }} in namespace github-runner has 0 ready replicas for more than 5 minutes. CI jobs targeting this repo will queue until the runner pod restarts and re-registers with GitHub. Check pods with: kubectl -n github-runner get pods -l app.kubernetes.io/name={{ $labels.deployment }}. Check logs with: kubectl -n github-runner logs -l app.kubernetes.io/name={{ $labels.deployment }} --tail=50. Common causes: PAT missing repo access, runner CrashLoopBackOff, or node/resource pressure."
+
       - name: remote-desktop
         rules:
           - alert: RemoteDesktopWebDown
@@ -3395,6 +3439,39 @@ data:
                 relativeTimeRange: {from: 120, to: 0}
                 datasourceUid: __expr__
                 model: {type: threshold, expression: B, conditions: [{evaluator: {params: [600], type: gt}}], refId: C}
+      - orgId: 1
+        name: CI Runners
+        folder: CI Alerts
+        interval: 1m
+        rules:
+          - uid: linux-runner-offline
+            title: LinuxRunnerOffline
+            condition: C
+            for: 5m
+            noDataState: OK
+            execErrState: Error
+            annotations:
+              summary: "Linux CI runner offline: {{ $labels.deployment }}"
+              description: "A github-runner namespace Deployment has 0 ready replicas for more than 5 minutes. CI jobs targeting that repo will queue until the runner pod restarts and re-registers."
+              runbook: "1. kubectl -n github-runner get pods -l app.kubernetes.io/name={{ $labels.deployment }} 2. kubectl -n github-runner logs -l app.kubernetes.io/name={{ $labels.deployment }} --tail=50 3. Verify PAT repo access if registration returns 404 4. Verify no RWO PVC is shared by scaled runners"
+            labels:
+              severity: warning
+              service: github-runner
+              alert_channel: irc
+              team: ci
+            data:
+              - refId: A
+                relativeTimeRange: {from: 300, to: 0}
+                datasourceUid: prometheus
+                model: {expr: 'kube_deployment_status_replicas_ready{namespace="github-runner",deployment=~"github-runner(|-(sharedpos|puppet|signage|dms|telephony|print-web|chat|mysql|kiosk-linux))"} == 0', instant: true, refId: A}
+              - refId: B
+                relativeTimeRange: {from: 300, to: 0}
+                datasourceUid: __expr__
+                model: {type: reduce, expression: A, reducer: last, refId: B}
+              - refId: C
+                relativeTimeRange: {from: 300, to: 0}
+                datasourceUid: __expr__
+                model: {type: threshold, expression: B, conditions: [{evaluator: {params: [0], type: gt}}], refId: C}
       - orgId: 1
         name: Infrastructure
         folder: AI Stack Alerts
@@ -3427,6 +3504,32 @@ data:
                 relativeTimeRange: {from: 120, to: 0}
                 datasourceUid: __expr__
                 model: {type: threshold, expression: B, conditions: [{evaluator: {params: [1], type: lt}}], refId: C}
+          - uid: macmini-runner-offline
+            title: MacMiniRunnerOffline
+            condition: C
+            for: 10m
+            noDataState: Alerting
+            execErrState: OK
+            annotations:
+              summary: Mac mini GitHub runner offline
+              description: "One or more macmini-* GitHub Actions runners have not reported online for more than 10 minutes. LaunchDaemons survive reboot and do not require the bluejay GUI session."
+              runbook: "1. ssh fcadmin@macmini.iamworkin.lan 2. launchctl print system/io.flowercore.github-runner-<slug> 3. Check /Users/fcadmin/Library/Logs/github-runners/<slug>/stderr.log 4. Re-register the repo runner if .runner is missing"
+            labels:
+              severity: warning
+              service: github-runner
+            data:
+              - refId: A
+                relativeTimeRange: {from: 600, to: 0}
+                datasourceUid: prometheus
+                model: {expr: 'min(flowercore_github_runner_online{runner=~"macmini-.*"} or vector(0))', instant: true, refId: A}
+              - refId: B
+                relativeTimeRange: {from: 600, to: 0}
+                datasourceUid: __expr__
+                model: {type: reduce, expression: A, reducer: last, refId: B}
+              - refId: C
+                relativeTimeRange: {from: 600, to: 0}
+                datasourceUid: __expr__
+                model: {type: threshold, expression: B, conditions: [{evaluator: {params: [1], type: lt}}], refId: C}
           - uid: high-cpu
             title: High CPU (>85%)
             condition: C

tests/bluejay-infra-lint/FleetManifestLintTests.cs

@@ -54,6 +54,43 @@ public sealed class FleetManifestLintTests
         "ttsreader-piper",
     };
 
+    private static readonly IReadOnlyDictionary<string, string> LinuxRunnerRepos = new Dictionary<string, string>(StringComparer.Ordinal)
+    {
+        ["github-runner"] = "https://github.com/astoltz/FlowerCore.Common",
+        ["github-runner-sharedpos"] = "https://github.com/astoltz/FlowerCore.Shared.Pos",
+        ["github-runner-puppet"] = "https://github.com/astoltz/FlowerCore.Puppet",
+        ["github-runner-signage"] = "https://github.com/astoltz/FlowerCore.Signage",
+        ["github-runner-dms"] = "https://github.com/astoltz/FlowerCore.DMS",
+        ["github-runner-telephony"] = "https://github.com/astoltz/FlowerCore.Telephony",
+        ["github-runner-print-web"] = "https://github.com/astoltz/FlowerCore.Print.Web",
+        ["github-runner-chat"] = "https://github.com/astoltz/FlowerCore.Chat",
+        ["github-runner-mysql"] = "https://github.com/astoltz/FlowerCore.MySQL",
+        ["github-runner-kiosk-linux"] = "https://github.com/astoltz/FlowerCore.Kiosk.Linux",
+    };
+
+    private static readonly HashSet<string> ScaledLinuxRunnerDeployments = new(StringComparer.Ordinal)
+    {
+        "github-runner-sharedpos",
+        "github-runner-puppet",
+        "github-runner-signage",
+        "github-runner-dms",
+        "github-runner-telephony",
+        "github-runner-print-web",
+        "github-runner-chat",
+        "github-runner-mysql",
+        "github-runner-kiosk-linux",
+    };
+
+    private static readonly IReadOnlyDictionary<string, string> WritableRunnerEnv = new Dictionary<string, string>(StringComparer.Ordinal)
+    {
+        ["HOME"] = "/home/runner",
+        ["DOTNET_INSTALL_DIR"] = "/home/runner/.dotnet",
+        ["DOTNET_CLI_HOME"] = "/home/runner",
+        ["NUGET_PACKAGES"] = "/home/runner/.nuget/packages",
+        ["XDG_CACHE_HOME"] = "/home/runner/.cache",
+        ["RUNNER_TOOL_CACHE"] = "/home/runner/_tool",
+    };
+
     [Fact]
     public void IngressRoutes_MustKeepServiceReferencesInTheSameNamespace()
     {
@@ -187,6 +224,98 @@ public sealed class FleetManifestLintTests
         violations.Should().BeEmpty();
     }
 
+    [Fact]
+    public void GitHubRunnerFleet_MustRegisterRequiredReposAsRepoScopedDeployments()
+    {
+        var deployments = GitHubRunnerDeployments();
+
+        foreach (var expectedRunner in LinuxRunnerRepos)
+        {
+            deployments.Should().ContainKey(expectedRunner.Key);
+
+            var container = deployments[expectedRunner.Key].ContainerMappings().Should().ContainSingle().Subject;
+            EnvValue(container, "REPO_URL").Should().Be(expectedRunner.Value);
+            EnvValue(container, "EPHEMERAL").Should().Be("true");
+            EnvValue(container, "LABELS").Should().Be("self-hosted,linux,fc-build-linux");
+            EnvValue(container, "RUN_AS_ROOT").Should().Be("false");
+            EnvValue(container, "ACCESS_TOKEN").Should().BeNull("ACCESS_TOKEN must come from github-runner-token Secret, not a literal");
+            EnvSecretName(container, "ACCESS_TOKEN").Should().Be("github-runner-token");
+            EnvSecretKey(container, "ACCESS_TOKEN").Should().Be("credential");
+        }
+    }
+
+    [Fact]
+    public void GitHubRunnerFleet_MustSetWritableNonRootDotnetAndCachePaths()
+    {
+        foreach (var deployment in GitHubRunnerDeployments().Values)
+        {
+            var container = deployment.ContainerMappings().Should().ContainSingle().Subject;
+
+            foreach (var expectedEnv in WritableRunnerEnv)
+            {
+                EnvValue(container, expectedEnv.Key).Should().Be(expectedEnv.Value, $"{deployment.Name} must keep .NET paths writable for uid 1001");
+            }
+
+            var mounts = ManifestNodeExtensions.MappingSequence(container, "volumeMounts")
+                .ToDictionary(
+                    mount => ManifestNodeExtensions.Scalar(mount, "name") ?? string.Empty,
+                    mount => ManifestNodeExtensions.Scalar(mount, "mountPath") ?? string.Empty,
+                    StringComparer.Ordinal);
+
+            mounts.Should().Contain("runner-home", "/home/runner");
+            mounts.Should().Contain("nuget-cache", "/home/runner/.nuget/packages");
+            mounts.Should().Contain("tmp", "/tmp");
+        }
+    }
+
+    [Fact]
+    public void GitHubRunnerFleet_MustAvoidRwoMultiAttachForScaledDeployments()
+    {
+        var deployments = GitHubRunnerDeployments();
+
+        foreach (var deploymentName in ScaledLinuxRunnerDeployments)
+        {
+            var deployment = deployments[deploymentName];
+            ReplicaCount(deployment).Should().Be(2);
+
+            var volumes = deployment.MappingSequence("spec", "template", "spec", "volumes");
+            var claimNames = volumes
+                .Select(volume => ManifestNodeExtensions.Scalar(volume, "persistentVolumeClaim", "claimName"))
+                .Where(value => !string.IsNullOrWhiteSpace(value))
+                .ToList();
+
+            claimNames.Should().BeEmpty($"{deploymentName} is scaled and must not share a RWO PVC");
+            volumes.Should().Contain(volume =>
+                string.Equals(ManifestNodeExtensions.Scalar(volume, "name"), "nuget-cache", StringComparison.Ordinal)
+                && ManifestNodeExtensions.Mapping(volume, "emptyDir") != null);
+        }
+
+        var common = deployments["github-runner"];
+        ReplicaCount(common).Should().Be(1);
+        common.MappingSequence("spec", "template", "spec", "volumes")
+            .Select(volume => ManifestNodeExtensions.Scalar(volume, "persistentVolumeClaim", "claimName"))
+            .Where(value => !string.IsNullOrWhiteSpace(value))
+            .Should()
+            .ContainSingle()
+            .Which
+            .Should()
+            .Be("github-runner-nuget-cache");
+    }
+
+    [Fact]
+    public void Monitoring_MustAlertWhenLinuxRunnerDeploymentIsUnavailable()
+    {
+        var monitoring = File.ReadAllText(Path.Combine(Inventory.BluejayRoot, "apps", "monitoring", "noc-monitoring.yaml"));
+
+        monitoring.Should().Contain("MacMiniRunnerOffline");
+        monitoring.Should().Contain("LinuxRunnerOffline");
+        monitoring.Should().Contain("kube_deployment_status_replicas_ready");
+        monitoring.Should().Contain("github-runner(|-(sharedpos|puppet|signage|dms|telephony|print-web|chat|mysql|kiosk-linux))");
+        monitoring.Should().Contain("folder: CI Alerts");
+        monitoring.Should().Contain("uid: linux-runner-offline");
+        monitoring.Should().Contain("alert_channel: irc");
+    }
+
     [Fact]
     public void StatefulSets_WithVolumeClaimTemplates_MustDeclareFilesystemDefaults()
     {
@@ -314,6 +443,44 @@ public sealed class FleetManifestLintTests
             $"{document.Descriptor} container '{containerName}' still uses {probeKey}.httpGet on /health.",
         };
     }
+
+    private static IReadOnlyDictionary<string, ManifestDocument> GitHubRunnerDeployments()
+    {
+        return Inventory.Documents
+            .Where(document => document.Kind == "Deployment")
+            .Where(document => document.Namespace == "github-runner")
+            .ToDictionary(document => document.Name, StringComparer.Ordinal);
+    }
+
+    private static int ReplicaCount(ManifestDocument document)
+    {
+        return int.TryParse(document.Scalar("spec", "replicas"), out var replicas) ? replicas : 1;
+    }
+
+    private static string? EnvValue(YamlMappingNode container, string name)
+    {
+        return EnvMapping(container, name) is { } env ? ManifestNodeExtensions.Scalar(env, "value") : null;
+    }
+
+    private static string? EnvSecretName(YamlMappingNode container, string name)
+    {
+        return EnvMapping(container, name) is { } env
+            ? ManifestNodeExtensions.Scalar(env, "valueFrom", "secretKeyRef", "name")
+            : null;
+    }
+
+    private static string? EnvSecretKey(YamlMappingNode container, string name)
+    {
+        return EnvMapping(container, name) is { } env
+            ? ManifestNodeExtensions.Scalar(env, "valueFrom", "secretKeyRef", "key")
+            : null;
+    }
+
+    private static YamlMappingNode? EnvMapping(YamlMappingNode container, string name)
+    {
+        return ManifestNodeExtensions.MappingSequence(container, "env")
+            .SingleOrDefault(env => string.Equals(ManifestNodeExtensions.Scalar(env, "name"), name, StringComparison.Ordinal));
+    }
 }
 
 internal sealed class ManifestInventory