feat(github-runner): add top Linux repo runners

2026-05-17 13:55:55 -05:00
4 changed files with 290 additions and 620 deletions
--- a/apps/github-runner/README.md
+++ b/apps/github-runner/README.md
@@ -1,61 +1,38 @@
-# GitHub Runner Fleet
+# github-runner

-ArgoCD owns `apps/github-runner/github-runner.yaml`. Do not patch live runner
-Deployments with `kubectl`; update this manifest and let ArgoCD reconcile.
+ArgoCD-managed repo-scoped Linux GitHub Actions runners for FlowerCore.

-## Runner Shape
+`astoltz` is a GitHub user account, not an organization, so each repository
+needs its own runner registration. The existing Common runner remains
+`Deployment/github-runner`; Sprint 29 adds one single-replica Deployment for
+each top Linux-cost repo:

-All repo-scoped Linux runners use:
+- `FlowerCore.Puppet`
+- `FlowerCore.Signage`
+- `FlowerCore.DMS`
+- `FlowerCore.Telephony`
+- `FlowerCore.Print.Web`
+- `FlowerCore.Chat`
+- `FlowerCore.MySQL`
+- `FlowerCore.Kiosk.Linux`

- `ACCESS_TOKEN` from the `github-runner-token` Secret
- `RUN_AS_ROOT=false`
- `EPHEMERAL=true`
- `LABELS=self-hosted,linux,fc-build-linux`
- writable non-root paths under `/home/runner` for .NET, NuGet, XDG cache, and
-  Actions tool cache
+Each runner uses `myoung34/github-runner:latest`, `EPHEMERAL=true`, and labels
+`self-hosted,linux,fc-build-linux`. The shared `github-runner-token` Secret is
+synced from the existing 1Password item `GitHub PAT (Runner Registration)` and
+is consumed as `ACCESS_TOKEN`.

-`github-runner` for `FlowerCore.Common` is single-replica because it retains the
-original Longhorn ReadWriteOnce NuGet PVC. `github-runner-sharedpos` and the top
-Linux-cost repo runners use two replicas with per-pod `emptyDir` caches. That is
-the safe backlog-drain strategy: no two pods share one RWO PVC.
-
-## Post-Merge Proof
-
-After the PR is merged and ArgoCD syncs, verify the runner fleet:
+Do not `kubectl apply` this app over ArgoCD. Merge to `main`, let
+`infra-github-runner` sync, then verify from `noc1`:

 ```bash
 kubectl -n github-runner get deploy,pods,pvc
-```

-Verify GitHub registration for the repo-scoped runners:
-
-```bash
-for repo in FlowerCore.Common FlowerCore.Shared.Pos FlowerCore.Puppet FlowerCore.Signage \
-            FlowerCore.DMS FlowerCore.Telephony FlowerCore.Print.Web FlowerCore.Chat \
-            FlowerCore.MySQL FlowerCore.Kiosk.Linux; do
-  echo "=== $repo ==="
+for repo in FlowerCore.Puppet FlowerCore.Signage FlowerCore.DMS FlowerCore.Telephony FlowerCore.Print.Web FlowerCore.Chat FlowerCore.MySQL FlowerCore.Kiosk.Linux; do
  gh api "/repos/astoltz/$repo/actions/runners" \
-    --jq '.runners[] | select(.labels[].name == "fc-build-linux") | {name,status,busy,labels:[.labels[].name]}'
+    --jq '.runners[] | select((.labels[].name == "fc-build-linux") and (.status == "online")) | {name,status,busy,labels:[.labels[].name]}'
 done
 ```

-Shared.Pos publish proof after the runner pod is online:
-
-```bash
-gh run list --repo astoltz/FlowerCore.Shared.Pos \
-  --workflow "Build, Test & Publish" --branch main --limit 5
-```
-
-If the latest run is still queued after runner registration, rerun the workflow
-from GitHub Actions and verify it lands on an `rke2-linux-*` runner.
-
-## Failure Notes
-
- `actions/setup-dotnet` permission error at `/usr/share/dotnet`: check that
-  `DOTNET_INSTALL_DIR=/home/runner/.dotnet` and related cache env vars are
-  present on the runner pod.
- `404` during runner registration: the fine-grained PAT is valid but missing
-  repository access for that repo. Add the repo to the PAT access list; the PAT
-  value does not change.
- `Multi-Attach` volume error: only the Common runner uses a RWO PVC and it must
-  stay single-replica. New multi-replica runners use `emptyDir`.
+`LinuxRunnerOffline` is declared in `apps/monitoring/noc-monitoring.yaml` and
+fires when any Common or top-8 Linux runner deployment has no available replica
+for 10 minutes.
--- a/apps/github-runner/github-runner.yaml
+++ b/apps/github-runner/github-runner.yaml
--- a/apps/monitoring/noc-monitoring.yaml
+++ b/apps/monitoring/noc-monitoring.yaml
@@ -75,20 +75,6 @@ data:
              cluster: "rke2"
              role: "agent"

-      # Mac mini macOS runner node (INFRA VLAN)
-      - job_name: "macmini-node"
-        scrape_timeout: 15s
-        static_configs:
-          - targets: ["10.0.56.115:9100"]
-            labels:
-              instance: "macmini"
-              host: "macmini.iamworkin.lan"
-              vlan: "infra"
-              arch: "arm64"
-              role: "macos-runner"
-              puppet_managed: "true"
-              puppet_server: "puppet.iamworkin.lan"
-
      # In-cluster node-exporter DaemonSet
      - job_name: "k8s-node-exporter"
        kubernetes_sd_configs:
@@ -711,36 +697,6 @@ data:
              summary: "Print.Web Ollama runner held for >10m ({{ $labels.model }})"
              description: "Print.Web reports model {{ $labels.model }} with {{ $value | printf \"%.0f\" }}s of keep-alive remaining. Check concurrent requests before the Pi 5 Ollama lane thrashes."

-      - name: macmini-runners
-        rules:
-          - alert: MacMiniRunnerOffline
-            expr: (flowercore_github_runner_online{runner=~"macmini-.*"} == 0) or absent(flowercore_github_runner_online{runner=~"macmini-.*"})
-            for: 10m
-            labels:
-              severity: warning
-              service: github-runner
-            annotations:
-              summary: "Mac mini GitHub runner offline ({{ $labels.runner }})"
-              description: "A macmini-* GitHub Actions runner has not reported online for more than 10 minutes. Puppet manages its LaunchDaemon under /Library/LaunchDaemons/io.flowercore.github-runner-<slug>.plist; runners survive reboot and do not require a GUI session."
-
-      - name: linux-runners
-        rules:
-          - alert: LinuxRunnerOffline
-            expr: |
-              kube_deployment_status_replicas_ready{
-                namespace="github-runner",
-                deployment=~"github-runner(|-(sharedpos|puppet|signage|dms|telephony|print-web|chat|mysql|kiosk-linux))"
-              } == 0
-            for: 5m
-            labels:
-              severity: warning
-              alert_channel: irc
-              service: github-runner
-              team: ci
-            annotations:
-              summary: "Linux CI runner offline: {{ $labels.deployment }}"
-              description: "Deployment {{ $labels.deployment }} in namespace github-runner has 0 ready replicas for more than 5 minutes. CI jobs targeting this repo will queue until the runner pod restarts and re-registers with GitHub. Check pods with: kubectl -n github-runner get pods -l app.kubernetes.io/name={{ $labels.deployment }}. Check logs with: kubectl -n github-runner logs -l app.kubernetes.io/name={{ $labels.deployment }} --tail=50. Common causes: PAT missing repo access, runner CrashLoopBackOff, or node/resource pressure."
-
      - name: remote-desktop
        rules:
          - alert: RemoteDesktopWebDown
@@ -1018,6 +974,19 @@ data:
              summary: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} replica mismatch"
              description: "Spec wants {{ $labels.spec_replicas }} but only {{ $value }} available. Likely a rollout stuck on probe failure, scheduling, or PVC."

+          - alert: LinuxRunnerOffline
+            expr: |
+              kube_deployment_status_replicas_available{namespace="github-runner",deployment=~"github-runner(|-(puppet|signage|dms|telephony|print-web|chat|mysql|kiosk-linux))"} < 1
+            for: 10m
+            labels:
+              severity: warning
+              service: github-runner
+              alert_channel: thermal_print
+            annotations:
+              summary: "Linux GitHub Actions runner offline: {{ $labels.deployment }}"
+              description: "{{ $labels.deployment }} has no available runner pod for 10 minutes. GitHub jobs using [self-hosted, linux, fc-build-linux] for its repo will queue at $0 until the runner returns."
+              runbook_url: "https://gitea.iamworkin.lan/bluejay/FlowerCore.Notes/src/branch/master/docs/infrastructure/self-hosted-runner-fleet.md"
+
          # Q-MR-3 (2026-05-11): multus memory pressure — catches the next OOM
          # cascade BEFORE multus is OOM-killed cluster-wide. The 2026-05-10
          # outage (21h) hit because no alert fired on the rising multus working
@@ -3439,39 +3408,6 @@ data:
                relativeTimeRange: {from: 120, to: 0}
                datasourceUid: __expr__
                model: {type: threshold, expression: B, conditions: [{evaluator: {params: [600], type: gt}}], refId: C}
-      - orgId: 1
-        name: CI Runners
-        folder: CI Alerts
-        interval: 1m
-        rules:
-          - uid: linux-runner-offline
-            title: LinuxRunnerOffline
-            condition: C
-            for: 5m
-            noDataState: OK
-            execErrState: Error
-            annotations:
-              summary: "Linux CI runner offline: {{ $labels.deployment }}"
-              description: "A github-runner namespace Deployment has 0 ready replicas for more than 5 minutes. CI jobs targeting that repo will queue until the runner pod restarts and re-registers."
-              runbook: "1. kubectl -n github-runner get pods -l app.kubernetes.io/name={{ $labels.deployment }} 2. kubectl -n github-runner logs -l app.kubernetes.io/name={{ $labels.deployment }} --tail=50 3. Verify PAT repo access if registration returns 404 4. Verify no RWO PVC is shared by scaled runners"
-            labels:
-              severity: warning
-              service: github-runner
-              alert_channel: irc
-              team: ci
-            data:
-              - refId: A
-                relativeTimeRange: {from: 300, to: 0}
-                datasourceUid: prometheus
-                model: {expr: 'kube_deployment_status_replicas_ready{namespace="github-runner",deployment=~"github-runner(|-(sharedpos|puppet|signage|dms|telephony|print-web|chat|mysql|kiosk-linux))"} == 0', instant: true, refId: A}
-              - refId: B
-                relativeTimeRange: {from: 300, to: 0}
-                datasourceUid: __expr__
-                model: {type: reduce, expression: A, reducer: last, refId: B}
-              - refId: C
-                relativeTimeRange: {from: 300, to: 0}
-                datasourceUid: __expr__
-                model: {type: threshold, expression: B, conditions: [{evaluator: {params: [0], type: gt}}], refId: C}
      - orgId: 1
        name: Infrastructure
        folder: AI Stack Alerts
@@ -3504,24 +3440,25 @@ data:
                relativeTimeRange: {from: 120, to: 0}
                datasourceUid: __expr__
                model: {type: threshold, expression: B, conditions: [{evaluator: {params: [1], type: lt}}], refId: C}
-          - uid: macmini-runner-offline
-            title: MacMiniRunnerOffline
+          - uid: linux-runner-offline
+            title: LinuxRunnerOffline
            condition: C
            for: 10m
            noDataState: Alerting
            execErrState: OK
            annotations:
-              summary: Mac mini GitHub runner offline
-              description: "One or more macmini-* GitHub Actions runners have not reported online for more than 10 minutes. LaunchDaemons survive reboot and do not require the bluejay GUI session."
-              runbook: "1. ssh fcadmin@macmini.iamworkin.lan 2. launchctl print system/io.flowercore.github-runner-<slug> 3. Check /Users/fcadmin/Library/Logs/github-runners/<slug>/stderr.log 4. Re-register the repo runner if .runner is missing"
+              summary: Linux GitHub Actions runner offline
+              description: "A repo-scoped fc-build-linux runner deployment has no available pod. Jobs will queue at $0 until ArgoCD/K8s returns the runner."
+              runbook_url: "https://gitea.iamworkin.lan/bluejay/FlowerCore.Notes/src/branch/master/docs/infrastructure/self-hosted-runner-fleet.md"
            labels:
              severity: warning
              service: github-runner
+              alert_channel: thermal_print
            data:
              - refId: A
                relativeTimeRange: {from: 600, to: 0}
                datasourceUid: prometheus
-                model: {expr: 'min(flowercore_github_runner_online{runner=~"macmini-.*"} or vector(0))', instant: true, refId: A}
+                model: {expr: 'min by(deployment) (kube_deployment_status_replicas_available{namespace="github-runner",deployment=~"github-runner(|-(puppet|signage|dms|telephony|print-web|chat|mysql|kiosk-linux))"})', instant: true, refId: A}
              - refId: B
                relativeTimeRange: {from: 600, to: 0}
                datasourceUid: __expr__
--- a/tests/bluejay-infra-lint/FleetManifestLintTests.cs
+++ b/tests/bluejay-infra-lint/FleetManifestLintTests.cs
@@ -54,10 +54,8 @@ public sealed class FleetManifestLintTests
        "ttsreader-piper",
    };

-    private static readonly IReadOnlyDictionary<string, string> LinuxRunnerRepos = new Dictionary<string, string>(StringComparer.Ordinal)
+    private static readonly IReadOnlyDictionary<string, string> TopLinuxRunnerRepos = new Dictionary<string, string>(StringComparer.Ordinal)
    {
-        ["github-runner"] = "https://github.com/astoltz/FlowerCore.Common",
-        ["github-runner-sharedpos"] = "https://github.com/astoltz/FlowerCore.Shared.Pos",
        ["github-runner-puppet"] = "https://github.com/astoltz/FlowerCore.Puppet",
        ["github-runner-signage"] = "https://github.com/astoltz/FlowerCore.Signage",
        ["github-runner-dms"] = "https://github.com/astoltz/FlowerCore.DMS",
@@ -68,29 +66,6 @@ public sealed class FleetManifestLintTests
        ["github-runner-kiosk-linux"] = "https://github.com/astoltz/FlowerCore.Kiosk.Linux",
    };

-    private static readonly HashSet<string> ScaledLinuxRunnerDeployments = new(StringComparer.Ordinal)
-    {
-        "github-runner-sharedpos",
-        "github-runner-puppet",
-        "github-runner-signage",
-        "github-runner-dms",
-        "github-runner-telephony",
-        "github-runner-print-web",
-        "github-runner-chat",
-        "github-runner-mysql",
-        "github-runner-kiosk-linux",
-    };
-
-    private static readonly IReadOnlyDictionary<string, string> WritableRunnerEnv = new Dictionary<string, string>(StringComparer.Ordinal)
-    {
-        ["HOME"] = "/home/runner",
-        ["DOTNET_INSTALL_DIR"] = "/home/runner/.dotnet",
-        ["DOTNET_CLI_HOME"] = "/home/runner",
-        ["NUGET_PACKAGES"] = "/home/runner/.nuget/packages",
-        ["XDG_CACHE_HOME"] = "/home/runner/.cache",
-        ["RUNNER_TOOL_CACHE"] = "/home/runner/_tool",
-    };
-
    [Fact]
    public void IngressRoutes_MustKeepServiceReferencesInTheSameNamespace()
    {
@@ -225,11 +200,14 @@ public sealed class FleetManifestLintTests
    }

    [Fact]
-    public void GitHubRunnerFleet_MustRegisterRequiredReposAsRepoScopedDeployments()
+    public void GitHubRunnerFleet_MustRegisterTopLinuxReposAsRepoScopedDeployments()
    {
-        var deployments = GitHubRunnerDeployments();
+        var deployments = Inventory.Documents
+            .Where(document => document.Kind == "Deployment")
+            .Where(document => document.Namespace == "github-runner")
+            .ToDictionary(document => document.Name, StringComparer.Ordinal);

-        foreach (var expectedRunner in LinuxRunnerRepos)
+        foreach (var expectedRunner in TopLinuxRunnerRepos)
        {
            deployments.Should().ContainKey(expectedRunner.Key);

@@ -237,7 +215,6 @@ public sealed class FleetManifestLintTests
            EnvValue(container, "REPO_URL").Should().Be(expectedRunner.Value);
            EnvValue(container, "EPHEMERAL").Should().Be("true");
            EnvValue(container, "LABELS").Should().Be("self-hosted,linux,fc-build-linux");
-            EnvValue(container, "RUN_AS_ROOT").Should().Be("false");
            EnvValue(container, "ACCESS_TOKEN").Should().BeNull("ACCESS_TOKEN must come from github-runner-token Secret, not a literal");
            EnvSecretName(container, "ACCESS_TOKEN").Should().Be("github-runner-token");
            EnvSecretKey(container, "ACCESS_TOKEN").Should().Be("credential");
@@ -245,75 +222,51 @@ public sealed class FleetManifestLintTests
    }

    [Fact]
-    public void GitHubRunnerFleet_MustSetWritableNonRootDotnetAndCachePaths()
+    public void GitHubRunnerFleet_MustPreserveExistingCommonRunnerShape()
    {
-        foreach (var deployment in GitHubRunnerDeployments().Values)
-        {
-            var container = deployment.ContainerMappings().Should().ContainSingle().Subject;
+        var common = Inventory.Documents
+            .Single(document => document.Kind == "Deployment"
+                && document.Namespace == "github-runner"
+                && document.Name == "github-runner");

-            foreach (var expectedEnv in WritableRunnerEnv)
-            {
-                EnvValue(container, expectedEnv.Key).Should().Be(expectedEnv.Value, $"{deployment.Name} must keep .NET paths writable for uid 1001");
-            }
+        var container = common.ContainerMappings().Should().ContainSingle().Subject;
+        EnvValue(container, "REPO_URL").Should().Be("https://github.com/astoltz/FlowerCore.Common");
+        EnvValue(container, "RUNNER_NAME_PREFIX").Should().Be("rke2-linux");
+        EnvValue(container, "LABELS").Should().Be("self-hosted,linux,fc-build-linux");

-            var mounts = ManifestNodeExtensions.MappingSequence(container, "volumeMounts")
-                .ToDictionary(
-                    mount => ManifestNodeExtensions.Scalar(mount, "name") ?? string.Empty,
-                    mount => ManifestNodeExtensions.Scalar(mount, "mountPath") ?? string.Empty,
-                    StringComparer.Ordinal);
-
-            mounts.Should().Contain("runner-home", "/home/runner");
-            mounts.Should().Contain("nuget-cache", "/home/runner/.nuget/packages");
-            mounts.Should().Contain("tmp", "/tmp");
-        }
-    }
-
-    [Fact]
-    public void GitHubRunnerFleet_MustAvoidRwoMultiAttachForScaledDeployments()
-    {
-        var deployments = GitHubRunnerDeployments();
-
-        foreach (var deploymentName in ScaledLinuxRunnerDeployments)
-        {
-            var deployment = deployments[deploymentName];
-            ReplicaCount(deployment).Should().Be(2);
-
-            var volumes = deployment.MappingSequence("spec", "template", "spec", "volumes");
-            var claimNames = volumes
+        var claimNames = common.MappingSequence("spec", "template", "spec", "volumes")
            .Select(volume => ManifestNodeExtensions.Scalar(volume, "persistentVolumeClaim", "claimName"))
            .Where(value => !string.IsNullOrWhiteSpace(value))
            .ToList();

-            claimNames.Should().BeEmpty($"{deploymentName} is scaled and must not share a RWO PVC");
-            volumes.Should().Contain(volume =>
-                string.Equals(ManifestNodeExtensions.Scalar(volume, "name"), "nuget-cache", StringComparison.Ordinal)
-                && ManifestNodeExtensions.Mapping(volume, "emptyDir") != null);
-        }
-
-        var common = deployments["github-runner"];
-        ReplicaCount(common).Should().Be(1);
-        common.MappingSequence("spec", "template", "spec", "volumes")
-            .Select(volume => ManifestNodeExtensions.Scalar(volume, "persistentVolumeClaim", "claimName"))
-            .Where(value => !string.IsNullOrWhiteSpace(value))
-            .Should()
-            .ContainSingle()
-            .Which
-            .Should()
-            .Be("github-runner-nuget-cache");
+        claimNames.Should().Contain("github-runner-nuget-cache");
    }

    [Fact]
-    public void Monitoring_MustAlertWhenLinuxRunnerDeploymentIsUnavailable()
+    public void GitHubRunnerFleet_MustUseOneRwoCachePerRepoScopedDeployment()
+    {
+        var pvcNames = Inventory.Documents
+            .Where(document => document.Kind == "PersistentVolumeClaim")
+            .Where(document => document.Namespace == "github-runner")
+            .Select(document => document.Name)
+            .ToHashSet(StringComparer.Ordinal);
+
+        foreach (var deploymentName in TopLinuxRunnerRepos.Keys)
+        {
+            var suffix = deploymentName["github-runner-".Length..];
+            pvcNames.Should().Contain($"github-runner-{suffix}-nuget-cache");
+        }
+    }
+
+    [Fact]
+    public void Monitoring_MustAlertWhenTopLinuxRunnerDeploymentIsUnavailable()
    {
        var monitoring = File.ReadAllText(Path.Combine(Inventory.BluejayRoot, "apps", "monitoring", "noc-monitoring.yaml"));

-        monitoring.Should().Contain("MacMiniRunnerOffline");
        monitoring.Should().Contain("LinuxRunnerOffline");
-        monitoring.Should().Contain("kube_deployment_status_replicas_ready");
-        monitoring.Should().Contain("github-runner(|-(sharedpos|puppet|signage|dms|telephony|print-web|chat|mysql|kiosk-linux))");
-        monitoring.Should().Contain("folder: CI Alerts");
-        monitoring.Should().Contain("uid: linux-runner-offline");
-        monitoring.Should().Contain("alert_channel: irc");
+        monitoring.Should().Contain("kube_deployment_status_replicas_available{namespace=\"github-runner\"");
+        monitoring.Should().Contain("github-runner(|-(puppet|signage|dms|telephony|print-web|chat|mysql|kiosk-linux))");
+        monitoring.Should().Contain("runbook_url: \"https://gitea.iamworkin.lan/bluejay/FlowerCore.Notes/src/branch/master/docs/infrastructure/self-hosted-runner-fleet.md\"");
    }

    [Fact]
@@ -444,19 +397,6 @@ public sealed class FleetManifestLintTests
        };
    }

-    private static IReadOnlyDictionary<string, ManifestDocument> GitHubRunnerDeployments()
-    {
-        return Inventory.Documents
-            .Where(document => document.Kind == "Deployment")
-            .Where(document => document.Namespace == "github-runner")
-            .ToDictionary(document => document.Name, StringComparer.Ordinal);
-    }
-
-    private static int ReplicaCount(ManifestDocument document)
-    {
-        return int.TryParse(document.Scalar("spec", "replicas"), out var replicas) ? replicas : 1;
-    }
-
    private static string? EnvValue(YamlMappingNode container, string name)
    {
        return EnvMapping(container, name) is { } env ? ManifestNodeExtensions.Scalar(env, "value") : null;