feat(github-runner): harden Linux runner fleet (#5)
This commit was merged in pull request #5.
This commit is contained in:
@@ -54,6 +54,43 @@ public sealed class FleetManifestLintTests
|
||||
"ttsreader-piper",
|
||||
};
|
||||
|
||||
private static readonly IReadOnlyDictionary<string, string> LinuxRunnerRepos = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["github-runner"] = "https://github.com/astoltz/FlowerCore.Common",
|
||||
["github-runner-sharedpos"] = "https://github.com/astoltz/FlowerCore.Shared.Pos",
|
||||
["github-runner-puppet"] = "https://github.com/astoltz/FlowerCore.Puppet",
|
||||
["github-runner-signage"] = "https://github.com/astoltz/FlowerCore.Signage",
|
||||
["github-runner-dms"] = "https://github.com/astoltz/FlowerCore.DMS",
|
||||
["github-runner-telephony"] = "https://github.com/astoltz/FlowerCore.Telephony",
|
||||
["github-runner-print-web"] = "https://github.com/astoltz/FlowerCore.Print.Web",
|
||||
["github-runner-chat"] = "https://github.com/astoltz/FlowerCore.Chat",
|
||||
["github-runner-mysql"] = "https://github.com/astoltz/FlowerCore.MySQL",
|
||||
["github-runner-kiosk-linux"] = "https://github.com/astoltz/FlowerCore.Kiosk.Linux",
|
||||
};
|
||||
|
||||
private static readonly HashSet<string> ScaledLinuxRunnerDeployments = new(StringComparer.Ordinal)
|
||||
{
|
||||
"github-runner-sharedpos",
|
||||
"github-runner-puppet",
|
||||
"github-runner-signage",
|
||||
"github-runner-dms",
|
||||
"github-runner-telephony",
|
||||
"github-runner-print-web",
|
||||
"github-runner-chat",
|
||||
"github-runner-mysql",
|
||||
"github-runner-kiosk-linux",
|
||||
};
|
||||
|
||||
private static readonly IReadOnlyDictionary<string, string> WritableRunnerEnv = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["HOME"] = "/home/runner",
|
||||
["DOTNET_INSTALL_DIR"] = "/home/runner/.dotnet",
|
||||
["DOTNET_CLI_HOME"] = "/home/runner",
|
||||
["NUGET_PACKAGES"] = "/home/runner/.nuget/packages",
|
||||
["XDG_CACHE_HOME"] = "/home/runner/.cache",
|
||||
["RUNNER_TOOL_CACHE"] = "/home/runner/_tool",
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public void IngressRoutes_MustKeepServiceReferencesInTheSameNamespace()
|
||||
{
|
||||
@@ -187,6 +224,98 @@ public sealed class FleetManifestLintTests
|
||||
violations.Should().BeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GitHubRunnerFleet_MustRegisterRequiredReposAsRepoScopedDeployments()
|
||||
{
|
||||
var deployments = GitHubRunnerDeployments();
|
||||
|
||||
foreach (var expectedRunner in LinuxRunnerRepos)
|
||||
{
|
||||
deployments.Should().ContainKey(expectedRunner.Key);
|
||||
|
||||
var container = deployments[expectedRunner.Key].ContainerMappings().Should().ContainSingle().Subject;
|
||||
EnvValue(container, "REPO_URL").Should().Be(expectedRunner.Value);
|
||||
EnvValue(container, "EPHEMERAL").Should().Be("true");
|
||||
EnvValue(container, "LABELS").Should().Be("self-hosted,linux,fc-build-linux");
|
||||
EnvValue(container, "RUN_AS_ROOT").Should().Be("false");
|
||||
EnvValue(container, "ACCESS_TOKEN").Should().BeNull("ACCESS_TOKEN must come from github-runner-token Secret, not a literal");
|
||||
EnvSecretName(container, "ACCESS_TOKEN").Should().Be("github-runner-token");
|
||||
EnvSecretKey(container, "ACCESS_TOKEN").Should().Be("credential");
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GitHubRunnerFleet_MustSetWritableNonRootDotnetAndCachePaths()
|
||||
{
|
||||
foreach (var deployment in GitHubRunnerDeployments().Values)
|
||||
{
|
||||
var container = deployment.ContainerMappings().Should().ContainSingle().Subject;
|
||||
|
||||
foreach (var expectedEnv in WritableRunnerEnv)
|
||||
{
|
||||
EnvValue(container, expectedEnv.Key).Should().Be(expectedEnv.Value, $"{deployment.Name} must keep .NET paths writable for uid 1001");
|
||||
}
|
||||
|
||||
var mounts = ManifestNodeExtensions.MappingSequence(container, "volumeMounts")
|
||||
.ToDictionary(
|
||||
mount => ManifestNodeExtensions.Scalar(mount, "name") ?? string.Empty,
|
||||
mount => ManifestNodeExtensions.Scalar(mount, "mountPath") ?? string.Empty,
|
||||
StringComparer.Ordinal);
|
||||
|
||||
mounts.Should().Contain("runner-home", "/home/runner");
|
||||
mounts.Should().Contain("nuget-cache", "/home/runner/.nuget/packages");
|
||||
mounts.Should().Contain("tmp", "/tmp");
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GitHubRunnerFleet_MustAvoidRwoMultiAttachForScaledDeployments()
|
||||
{
|
||||
var deployments = GitHubRunnerDeployments();
|
||||
|
||||
foreach (var deploymentName in ScaledLinuxRunnerDeployments)
|
||||
{
|
||||
var deployment = deployments[deploymentName];
|
||||
ReplicaCount(deployment).Should().Be(2);
|
||||
|
||||
var volumes = deployment.MappingSequence("spec", "template", "spec", "volumes");
|
||||
var claimNames = volumes
|
||||
.Select(volume => ManifestNodeExtensions.Scalar(volume, "persistentVolumeClaim", "claimName"))
|
||||
.Where(value => !string.IsNullOrWhiteSpace(value))
|
||||
.ToList();
|
||||
|
||||
claimNames.Should().BeEmpty($"{deploymentName} is scaled and must not share a RWO PVC");
|
||||
volumes.Should().Contain(volume =>
|
||||
string.Equals(ManifestNodeExtensions.Scalar(volume, "name"), "nuget-cache", StringComparison.Ordinal)
|
||||
&& ManifestNodeExtensions.Mapping(volume, "emptyDir") != null);
|
||||
}
|
||||
|
||||
var common = deployments["github-runner"];
|
||||
ReplicaCount(common).Should().Be(1);
|
||||
common.MappingSequence("spec", "template", "spec", "volumes")
|
||||
.Select(volume => ManifestNodeExtensions.Scalar(volume, "persistentVolumeClaim", "claimName"))
|
||||
.Where(value => !string.IsNullOrWhiteSpace(value))
|
||||
.Should()
|
||||
.ContainSingle()
|
||||
.Which
|
||||
.Should()
|
||||
.Be("github-runner-nuget-cache");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Monitoring_MustAlertWhenLinuxRunnerDeploymentIsUnavailable()
|
||||
{
|
||||
var monitoring = File.ReadAllText(Path.Combine(Inventory.BluejayRoot, "apps", "monitoring", "noc-monitoring.yaml"));
|
||||
|
||||
monitoring.Should().Contain("MacMiniRunnerOffline");
|
||||
monitoring.Should().Contain("LinuxRunnerOffline");
|
||||
monitoring.Should().Contain("kube_deployment_status_replicas_ready");
|
||||
monitoring.Should().Contain("github-runner(|-(sharedpos|puppet|signage|dms|telephony|print-web|chat|mysql|kiosk-linux))");
|
||||
monitoring.Should().Contain("folder: CI Alerts");
|
||||
monitoring.Should().Contain("uid: linux-runner-offline");
|
||||
monitoring.Should().Contain("alert_channel: irc");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void StatefulSets_WithVolumeClaimTemplates_MustDeclareFilesystemDefaults()
|
||||
{
|
||||
@@ -314,6 +443,44 @@ public sealed class FleetManifestLintTests
|
||||
$"{document.Descriptor} container '{containerName}' still uses {probeKey}.httpGet on /health.",
|
||||
};
|
||||
}
|
||||
|
||||
private static IReadOnlyDictionary<string, ManifestDocument> GitHubRunnerDeployments()
|
||||
{
|
||||
return Inventory.Documents
|
||||
.Where(document => document.Kind == "Deployment")
|
||||
.Where(document => document.Namespace == "github-runner")
|
||||
.ToDictionary(document => document.Name, StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
private static int ReplicaCount(ManifestDocument document)
|
||||
{
|
||||
return int.TryParse(document.Scalar("spec", "replicas"), out var replicas) ? replicas : 1;
|
||||
}
|
||||
|
||||
private static string? EnvValue(YamlMappingNode container, string name)
|
||||
{
|
||||
return EnvMapping(container, name) is { } env ? ManifestNodeExtensions.Scalar(env, "value") : null;
|
||||
}
|
||||
|
||||
private static string? EnvSecretName(YamlMappingNode container, string name)
|
||||
{
|
||||
return EnvMapping(container, name) is { } env
|
||||
? ManifestNodeExtensions.Scalar(env, "valueFrom", "secretKeyRef", "name")
|
||||
: null;
|
||||
}
|
||||
|
||||
private static string? EnvSecretKey(YamlMappingNode container, string name)
|
||||
{
|
||||
return EnvMapping(container, name) is { } env
|
||||
? ManifestNodeExtensions.Scalar(env, "valueFrom", "secretKeyRef", "key")
|
||||
: null;
|
||||
}
|
||||
|
||||
private static YamlMappingNode? EnvMapping(YamlMappingNode container, string name)
|
||||
{
|
||||
return ManifestNodeExtensions.MappingSequence(container, "env")
|
||||
.SingleOrDefault(env => string.Equals(ManifestNodeExtensions.Scalar(env, "name"), name, StringComparison.Ordinal));
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class ManifestInventory
|
||||
|
||||
Reference in New Issue
Block a user