Compare commits

..

2 Commits

Author SHA1 Message Date
Andrew Stoltz
2c8968f5d0 Wire SignalControl platform observability 2026-06-02 03:02:13 -05:00
Andrew Stoltz
0307ae16ae monitoring(probe): signage/mysql/php blackbox probe / -> /healthz (K8s-target mirror)
Mirrors the live noc1 podman fix + Notes scripts/monitoring/prometheus.yml.
These services enforce OIDC bearer auth (FlowerCore__Auth__Enabled=true), so an
anonymous probe of / returns 401 -> false TraefikServiceDown. All three expose
anonymous /healthz=200. This noc-monitoring.yaml is the forward K8s-migration
target (not live); brings it in sync with the live config.
2026-06-02 01:09:57 -05:00
5 changed files with 134 additions and 59 deletions

View File

@@ -46,7 +46,7 @@ spec:
spec:
containers:
- name: signalcontrol-web
image: localhost/fc-signalcontrol-web:latest
image: localhost/fc-signalcontrol-web:s50cx12-20260602-1d26c58
imagePullPolicy: Never
ports:
- containerPort: 5000
@@ -65,6 +65,48 @@ spec:
secretKeyRef:
name: signalcontrol-auth
key: Auth__ApiKey
- name: Auth__AdminApiKey
valueFrom:
secretKeyRef:
name: signalcontrol-auth
key: Auth__AdminApiKey
optional: true
- name: Auth__Enabled
value: "false"
- name: FlowerCore__Auth__Enabled
value: "false"
- name: FlowerCore__Auth__Oidc__Enabled
value: "true"
- name: FlowerCore__Auth__Oidc__Authority
valueFrom:
secretKeyRef:
name: signalcontrol-oidc-client
key: issuer_url
optional: true
- name: FlowerCore__Auth__Oidc__ClientId
valueFrom:
secretKeyRef:
name: signalcontrol-oidc-client
key: client_id
optional: true
- name: FlowerCore__Auth__Oidc__ClientSecret
valueFrom:
secretKeyRef:
name: signalcontrol-oidc-client
key: client_secret
optional: true
- name: TrafficSignal__RelayBridge__Enabled
value: "true"
- name: TrafficSignal__RelayBridge__BaseUrl
value: https://pirelay.iamworkin.lan
- name: TrafficSignal__RelayBridge__ApiKey
valueFrom:
secretKeyRef:
name: signalcontrol-pirelay
key: ApiKey
optional: true
- name: LiveStatus__TrafficSignal__BaseAddress
value: https://signalcontrol.iamworkin.lan
volumeMounts:
- name: data
mountPath: /data

View File

@@ -149,8 +149,8 @@ data:
"targets": [
{
"editorMode": "code",
"expr": "sum by (source, to_phase) (rate(signal_transitions_total{job=\"signalcontrol-pi-app\",instance=\"pirelay\"}[$__rate_interval]))",
"legendFormat": "{{source}} -> {{to_phase}}",
"expr": "sum by (source, to) (rate(signal_transitions_total{job=\"signalcontrol-pi-app\",instance=\"pirelay\"}[$__rate_interval]))",
"legendFormat": "{{source}} -> {{to}}",
"range": true,
"refId": "A"
}
@@ -167,8 +167,8 @@ data:
"targets": [
{
"editorMode": "code",
"expr": "sum by (action) (increase(signal_schedule_fires_total{job=\"signalcontrol-pi-app\",instance=\"pirelay\"}[24h]))",
"legendFormat": "{{action}}",
"expr": "sum by (action, outcome) (increase(signal_schedule_fires_total{job=\"signalcontrol-pi-app\",instance=\"pirelay\"}[24h]))",
"legendFormat": "{{action}} {{outcome}}",
"range": true,
"refId": "A"
},

View File

@@ -492,11 +492,11 @@ data:
- "https://gitea.iamworkin.lan/"
- "https://argocd.iamworkin.lan/"
- "https://intranet.iamworkin.lan/"
- "https://signage.iamworkin.lan/"
- "https://signage.iamworkin.lan/healthz" # root 401 auth-gated 2026-06-01; /healthz anon 200
- "https://kiosk.iamworkin.lan/"
- "https://media.iamworkin.lan/"
- "https://mysql.iamworkin.lan/"
- "https://php.iamworkin.lan/"
- "https://mysql.iamworkin.lan/healthz" # root 401 auth-gated 2026-06-01; /healthz anon 200
- "https://php.iamworkin.lan/healthz" # root 401 auth-gated 2026-06-01; /healthz anon 200
- "https://zabbix.iamworkin.lan/"
- "https://desktop.iamworkin.lan/"
- "https://print.iamworkin.lan/"

View File

@@ -227,6 +227,50 @@ public sealed class FleetManifestLintTests
violations.Should().BeEmpty();
}
[Fact]
public void SignalControlDeployment_MustKeepAuthOffAndStageOidcSecret()
{
var deployment = Inventory.Documents.Single(document =>
document.Kind == "Deployment"
&& document.Namespace == "fc-signalcontrol"
&& document.Name == "signalcontrol-web"
&& document.RelativePath == "fc-signalcontrol/fc-signalcontrol.yaml");
var container = deployment.MainContainerMappings().Single(container =>
ManifestNodeExtensions.Scalar(container, "name") == "signalcontrol-web");
EnvValue(container, "Auth__Enabled").Should().Be("false");
EnvValue(container, "FlowerCore__Auth__Enabled").Should().Be("false");
EnvValue(container, "FlowerCore__Auth__Oidc__Enabled").Should().Be("true");
EnvSecretName(container, "FlowerCore__Auth__Oidc__Authority").Should().Be("signalcontrol-oidc-client");
EnvSecretKey(container, "FlowerCore__Auth__Oidc__Authority").Should().Be("issuer_url");
EnvSecretName(container, "FlowerCore__Auth__Oidc__ClientId").Should().Be("signalcontrol-oidc-client");
EnvSecretKey(container, "FlowerCore__Auth__Oidc__ClientId").Should().Be("client_id");
EnvSecretName(container, "FlowerCore__Auth__Oidc__ClientSecret").Should().Be("signalcontrol-oidc-client");
EnvSecretKey(container, "FlowerCore__Auth__Oidc__ClientSecret").Should().Be("client_secret");
EnvSecretOptional(container, "FlowerCore__Auth__Oidc__Authority").Should().BeTrue();
EnvSecretOptional(container, "FlowerCore__Auth__Oidc__ClientId").Should().BeTrue();
EnvSecretOptional(container, "FlowerCore__Auth__Oidc__ClientSecret").Should().BeTrue();
}
[Fact]
public void SignalControlDeployment_MustWirePirelayRelayBridgeSecret()
{
var deployment = Inventory.Documents.Single(document =>
document.Kind == "Deployment"
&& document.Namespace == "fc-signalcontrol"
&& document.Name == "signalcontrol-web"
&& document.RelativePath == "fc-signalcontrol/fc-signalcontrol.yaml");
var container = deployment.MainContainerMappings().Single(container =>
ManifestNodeExtensions.Scalar(container, "name") == "signalcontrol-web");
EnvValue(container, "TrafficSignal__RelayBridge__Enabled").Should().Be("true");
EnvValue(container, "TrafficSignal__RelayBridge__BaseUrl").Should().Be("https://pirelay.iamworkin.lan");
EnvSecretName(container, "TrafficSignal__RelayBridge__ApiKey").Should().Be("signalcontrol-pirelay");
EnvSecretKey(container, "TrafficSignal__RelayBridge__ApiKey").Should().Be("ApiKey");
EnvSecretOptional(container, "TrafficSignal__RelayBridge__ApiKey").Should().BeTrue();
EnvValue(container, "LiveStatus__TrafficSignal__BaseAddress").Should().Be("https://signalcontrol.iamworkin.lan");
}
[Fact]
public void GitHubRunnerFleet_MustRegisterRequiredReposAsRepoScopedDeployments()
{
@@ -424,6 +468,36 @@ public sealed class FleetManifestLintTests
monitoring.Should().Contain("alert_channel: irc");
}
[Fact]
public void Monitoring_MustScrapeSignalControlPiAppAndMountDashboard()
{
var monitoring = File.ReadAllText(Path.Combine(Inventory.BluejayRoot, "apps", "monitoring", "noc-monitoring.yaml"));
monitoring.Should().Contain("job_name: \"signalcontrol-pi-app\"");
monitoring.Should().Contain("metrics_path: /metrics/prometheus");
monitoring.Should().Contain("10.0.58.113:5200");
monitoring.Should().Contain("host: \"signal-a.iamworkin.lan\"");
monitoring.Should().Contain("mountPath: /var/lib/grafana/dashboards/signalcontrol");
monitoring.Should().Contain("name: grafana-dashboard-signalcontrol");
}
[Fact]
public void SignalControlGrafanaDashboard_MustCoverAppNodeAndPhysicalControlMetrics()
{
var dashboard = File.ReadAllText(Path.Combine(
Inventory.BluejayRoot,
"apps",
"monitoring",
"grafana-dashboard-signalcontrol.yaml"));
dashboard.Should().Contain("uid\": \"flowercore-signalcontrol\"");
dashboard.Should().Contain("up{job=\\\"signalcontrol-pi-app\\\",instance=\\\"pirelay\\\"}");
dashboard.Should().Contain("up{job=\\\"edge-nodes\\\",instance=\\\"pirelay\\\"}");
dashboard.Should().Contain("signal_relay_writes_total");
dashboard.Should().Contain("signal_schedule_fires_total");
dashboard.Should().Contain("signalcontrol_screen_saver_enabled");
}
[Fact]
public void StatefulSets_WithVolumeClaimTemplates_MustDeclareFilesystemDefaults()
{
@@ -762,6 +836,16 @@ public sealed class FleetManifestLintTests
: null;
}
private static bool EnvSecretOptional(YamlMappingNode container, string name)
{
return string.Equals(
EnvMapping(container, name) is { } env
? ManifestNodeExtensions.Scalar(env, "valueFrom", "secretKeyRef", "optional")
: null,
"true",
StringComparison.Ordinal);
}
private static YamlMappingNode? EnvMapping(YamlMappingNode container, string name)
{
return ManifestNodeExtensions.MappingSequence(container, "env")

View File

@@ -1,51 +0,0 @@
using FluentAssertions;
using Xunit;
namespace BluejayInfraLint.Tests;
[Trait("Category", "Unit")]
public sealed class SignalControlPlatformManifestTests
{
private static readonly string Root = ManifestInventory.Load().BluejayRoot;
[Fact]
public void Monitoring_PrometheusScrapesSignalControlPiAppAndPirelayNodeExporter()
{
var monitoring = File.ReadAllText(Path.Combine(Root, "apps", "monitoring", "noc-monitoring.yaml"));
monitoring.Should().Contain("job_name: \"signalcontrol-pi-app\"");
monitoring.Should().Contain("metrics_path: /metrics/prometheus");
monitoring.Should().Contain("targets: [\"10.0.58.113:5200\"]");
monitoring.Should().Contain("host: \"signal-a.iamworkin.lan\"");
monitoring.Should().Contain("targets: [\"10.0.58.113:9100\"]");
monitoring.Should().Contain("instance: \"pirelay\"");
}
[Fact]
public void Monitoring_GrafanaMountsSignalControlDashboard()
{
var monitoring = File.ReadAllText(Path.Combine(Root, "apps", "monitoring", "noc-monitoring.yaml"));
var dashboard = File.ReadAllText(Path.Combine(Root, "apps", "monitoring", "grafana-dashboard-signalcontrol.yaml"));
monitoring.Should().Contain("name: dashboards-signalcontrol");
monitoring.Should().Contain("mountPath: /var/lib/grafana/dashboards/signalcontrol");
monitoring.Should().Contain("name: grafana-dashboard-signalcontrol");
dashboard.Should().Contain("\"uid\": \"flowercore-signalcontrol\"");
dashboard.Should().Contain("signalcontrol_active_pattern");
dashboard.Should().Contain("signal_relay_writes_total");
dashboard.Should().Contain("node_cpu_seconds_total");
}
[Fact]
public void FcSignalControlReadme_DocumentsMtlsTelemetryAndDefaultOffAudit()
{
var readme = File.ReadAllText(Path.Combine(Root, "apps", "fc-signalcontrol", "README.md"));
readme.Should().Contain("step-ca-agent");
readme.Should().Contain("10.0.58.113:5200");
readme.Should().Contain("10.0.58.113:9100");
readme.Should().Contain("PhysicalAudit:Enabled=false");
readme.Should().Contain("ForwardingEnabled=false");
readme.Should().Contain("Secrets, enrollment codes, private keys");
}
}