Compare commits

..

3 Commits

Author SHA1 Message Date
Andrew Stoltz
ec78175526 tests: add bluejay-ws runner-exclusion lint + fix 3 stale runner-fleet assertions
Adds Runners_MustNotPinToOperatorWorkstationHosts lint test enforcing
operator directive 2026-05-26: BLUEJAY-WS / iamworkin-ws must never be
a fleet GitHub Actions runner. Build-side analog of the Sprint 9 NEW
safe-account exclusion gate (Puppet GPO/AppLocker/WDAC/audit-forwarder
modules refuse to apply on BLUEJAY-WS). Scans every github-runner
Deployment for forbidden nodeName, nodeSelector, nodeAffinity match
expressions, and toleration key/value pinning. See CLAUDE.md "Common
Mistakes" entry and feedback_bluejay_ws_never_public_runner.md.

Also fixes 3 pre-existing GitHubRunnerFleet_* lint failures that broke
when the runner image bumped to v20260525-ruby3.3.11-stepca (added a
setup-runner-home initContainer):

  * Add MainContainerMappings() helper (containers only, excludes
    initContainers) and switch
    GitHubRunnerFleet_MustRegisterRequiredReposAsRepoScopedDeployments
    + GitHubRunnerFleet_MustSetWritableNonRootDotnetAndCachePaths
    over to it. Without this, ContainerMappings().Should().ContainSingle()
    found the initContainer + runner = 2 containers and failed.

  * Loosen GitHubRunnerFleet_MustAvoidRwoMultiAttachForScaledDeployments
    ReplicaCount assertion from Be(2) to BeGreaterOrEqualTo(2). The
    semantic invariant is "at least 2 replicas so no single-pod
    bottleneck"; deployments tuned upward per 14d CI activity (e.g.
    github-runner-print-web at replicas: 3, see commit 1f1f682 PR #24)
    are valid.

Lint baseline: 6 failed -> 3 failed (the 3 remaining are unrelated:
PublicReadWriteIngressRoutes_* lives in FlowerCore.Updater/k8s/
ingressroute.yaml — separate PR; FcDeviceManagement_* needs operator
domain decision on the missing apps/fc-devicemgmt/argocd-application.yaml).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-25 22:41:00 -05:00
Andrew Stoltz
2cc91b6df0 runners: bump tts-reader memory limit 4Gi -> 8Gi
The github-runner-tts-reader pod was being OOMKilled (exit 137)
mid-`dotnet test` on the TtsReader 1000+ test suite. PR #21 CI
(the Windows -> Linux runner migration) flapped twice with the
"self-hosted runner lost communication" annotation before the
K8s-side symptoms surfaced via kubectl describe pod.

Requests bumped 1Gi -> 2Gi, limits 4Gi -> 8Gi. Comment added
inline so future fleet runs don't trip the same wall.

Unblocks PR #21 + the 9 other open TtsReader PRs that all rebase
through it.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-25 22:31:48 -05:00
0d2090fe81 runners: add github-runner-updater Deployment (#29)
Close runner-fleet gap for FlowerCore.Updater. Matches Sprint 32 long-tail pattern; registers entry in fleet-lint required-set.
2026-05-26 03:24:13 +00:00
2 changed files with 130 additions and 5 deletions

View File

@@ -1726,13 +1726,17 @@ spec:
key: credential key: credential
- name: RUN_AS_ROOT - name: RUN_AS_ROOT
value: "false" value: "false"
# Bumped 2026-05-25: previous 4Gi limit caused OOMKill (exit 137)
# mid-`dotnet test` on TtsReader's 1000+ test suite. PR #21 CI flapped
# twice with "runner lost communication" before the K8s side
# symptoms surfaced. 8Gi gives ~30% headroom over peak observed.
resources: resources:
requests: requests:
cpu: "500m" cpu: "500m"
memory: "1Gi" memory: "2Gi"
limits: limits:
cpu: "2000m" cpu: "2000m"
memory: "4Gi" memory: "8Gi"
volumeMounts: volumeMounts:
- name: runner-home - name: runner-home
mountPath: /home/runner mountPath: /home/runner

View File

@@ -236,7 +236,7 @@ public sealed class FleetManifestLintTests
{ {
deployments.Should().ContainKey(expectedRunner.Key); deployments.Should().ContainKey(expectedRunner.Key);
var container = deployments[expectedRunner.Key].ContainerMappings().Should().ContainSingle().Subject; var container = deployments[expectedRunner.Key].MainContainerMappings().Should().ContainSingle().Subject;
EnvValue(container, "REPO_URL").Should().Be(expectedRunner.Value); EnvValue(container, "REPO_URL").Should().Be(expectedRunner.Value);
EnvValue(container, "EPHEMERAL").Should().Be("true"); EnvValue(container, "EPHEMERAL").Should().Be("true");
EnvValue(container, "LABELS").Should().Be("self-hosted,linux,fc-build-linux"); EnvValue(container, "LABELS").Should().Be("self-hosted,linux,fc-build-linux");
@@ -252,7 +252,7 @@ public sealed class FleetManifestLintTests
{ {
foreach (var deployment in GitHubRunnerDeployments().Values) foreach (var deployment in GitHubRunnerDeployments().Values)
{ {
var container = deployment.ContainerMappings().Should().ContainSingle().Subject; var container = deployment.MainContainerMappings().Should().ContainSingle().Subject;
foreach (var expectedEnv in WritableRunnerEnv) foreach (var expectedEnv in WritableRunnerEnv)
{ {
@@ -279,7 +279,10 @@ public sealed class FleetManifestLintTests
foreach (var deploymentName in ScaledLinuxRunnerDeployments) foreach (var deploymentName in ScaledLinuxRunnerDeployments)
{ {
var deployment = deployments[deploymentName]; var deployment = deployments[deploymentName];
ReplicaCount(deployment).Should().Be(2); // Scaled runners must have >= 2 replicas (avoid single-pod bottleneck).
// Individual deployments may be tuned upward per CI activity — see
// "runners: right-size replica counts per 14d CI activity (#24)".
ReplicaCount(deployment).Should().BeGreaterOrEqualTo(2, $"{deploymentName} is in the scaled set and must run with at least 2 replicas");
var volumes = deployment.MappingSequence("spec", "template", "spec", "volumes"); var volumes = deployment.MappingSequence("spec", "template", "spec", "volumes");
var claimNames = volumes var claimNames = volumes
@@ -305,6 +308,108 @@ public sealed class FleetManifestLintTests
.Be("github-runner-nuget-cache"); .Be("github-runner-nuget-cache");
} }
[Fact]
public void Runners_MustNotPinToOperatorWorkstationHosts()
{
// CRITICAL SAFETY (operator directive 2026-05-26): BLUEJAY-WS is the
// operator's primary workstation — host of the 1Password Connect
// bearer token, fcadmin SSH keys to noc1, signing CA private keys,
// and source for every FC repo. A self-hosted GitHub Actions runner
// there would execute arbitrary PR code with that local access.
// Build-side analog of the Sprint 9 NEW safe-account exclusion gate
// (Puppet GPO/AppLocker/WDAC/audit-forwarder modules refuse to apply
// on BLUEJAY-WS). This lint asserts no GitHub-runner Deployment in
// apps/github-runner/ pins to a forbidden operator-workstation host
// via nodeName, nodeSelector, nodeAffinity, or tolerations.
// Existing legacy `bluejay-ws-sandbox-1` GitHub-registered runner is
// out of scope here (it's a runtime registration, not a K8s
// Deployment) — see CLAUDE.md "Common Mistakes" entry and
// feedback_bluejay_ws_never_public_runner.md.
var forbiddenHostPatterns = new[]
{
"bluejay-ws",
"BLUEJAY-WS",
"bluejay-ws.iamworkin.lan",
"iamworkin-ws",
};
bool ContainsForbidden(string? value) =>
!string.IsNullOrWhiteSpace(value)
&& forbiddenHostPatterns.Any(pattern => value!.Contains(pattern, StringComparison.OrdinalIgnoreCase));
var violations = GitHubRunnerDeployments().Values.SelectMany(deployment =>
{
var local = new List<string>();
var podSpec = ManifestNodeExtensions.Mapping(deployment.Root, "spec", "template", "spec");
if (podSpec is null)
{
return local;
}
// nodeName: pins the pod to a specific node by name.
var nodeName = ManifestNodeExtensions.Scalar(podSpec, "nodeName");
if (ContainsForbidden(nodeName))
{
local.Add($"{deployment.Name} sets nodeName='{nodeName}' which targets a forbidden operator-workstation host.");
}
// nodeSelector: dict of label → value pinning the pod to nodes
// carrying matching labels. Examples that would trip this:
// kubernetes.io/hostname: bluejay-ws
// flowercore.io/host: bluejay-ws.iamworkin.lan
var nodeSelector = ManifestNodeExtensions.Mapping(podSpec, "nodeSelector");
if (nodeSelector is not null)
{
foreach (var entry in nodeSelector.Children)
{
var key = entry.Key is YamlScalarNode keyScalar ? keyScalar.Value : null;
var value = entry.Value is YamlScalarNode valueScalar ? valueScalar.Value : null;
if (ContainsForbidden(value))
{
local.Add($"{deployment.Name} has nodeSelector entry '{key}: {value}' which targets a forbidden operator-workstation host.");
}
}
}
// nodeAffinity: matchExpressions over node labels.
foreach (var term in ManifestNodeExtensions.MappingSequence(podSpec, "affinity", "nodeAffinity", "requiredDuringSchedulingIgnoredDuringExecution", "nodeSelectorTerms"))
{
foreach (var expr in ManifestNodeExtensions.MappingSequence(term, "matchExpressions"))
{
var key = ManifestNodeExtensions.Scalar(expr, "key");
foreach (var valueNode in ManifestNodeExtensions.ScalarSequence(expr, "values"))
{
if (ContainsForbidden(valueNode))
{
local.Add($"{deployment.Name} has nodeAffinity matchExpression '{key}' value '{valueNode}' which targets a forbidden operator-workstation host.");
}
}
}
}
// tolerations: scheduling onto a tainted operator-workstation
// node would let the runner run there. Forbid any toleration
// value that names the workstation.
foreach (var toleration in ManifestNodeExtensions.MappingSequence(podSpec, "tolerations"))
{
var key = ManifestNodeExtensions.Scalar(toleration, "key");
var value = ManifestNodeExtensions.Scalar(toleration, "value");
if (ContainsForbidden(key))
{
local.Add($"{deployment.Name} has toleration key '{key}' which targets a forbidden operator-workstation host.");
}
if (ContainsForbidden(value))
{
local.Add($"{deployment.Name} has toleration value '{value}' which targets a forbidden operator-workstation host.");
}
}
return local;
}).ToList();
violations.Should().BeEmpty("BLUEJAY-WS / iamworkin-ws must never host a fleet GitHub Actions runner; see CLAUDE.md 'Registering BLUEJAY-WS as a fleet GitHub Actions runner' and feedback_bluejay_ws_never_public_runner.md");
}
[Fact] [Fact]
public void Monitoring_MustAlertWhenLinuxRunnerDeploymentIsUnavailable() public void Monitoring_MustAlertWhenLinuxRunnerDeploymentIsUnavailable()
{ {
@@ -892,6 +997,22 @@ internal sealed record ManifestDocument(
.ToList(); .ToList();
} }
// MainContainerMappings excludes initContainers. Use this when asserting
// properties of the primary container (env, image, volumeMounts) where an
// initContainer would be a false-positive match — e.g. the GitHub runner
// image's `setup-runner-home` initContainer should not count toward the
// single-container assertions on the runner deployments.
public IReadOnlyList<YamlMappingNode> MainContainerMappings()
{
var podSpec = PodSpec();
if (podSpec is null)
{
return Array.Empty<YamlMappingNode>();
}
return ManifestNodeExtensions.MappingSequence(podSpec, "containers").ToList();
}
public IReadOnlyList<ContainerSpec> ContainerSpecs() public IReadOnlyList<ContainerSpec> ContainerSpecs()
{ {
return ContainerMappings() return ContainerMappings()