Compare commits
45 Commits
sprint37/c
...
codex/s49-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
62f6d8e7d5 | ||
|
|
6c18f69cf2 | ||
|
|
47e2256556 | ||
|
|
9d77f8ba0e | ||
|
|
2f4be19c85 | ||
|
|
2a62c40990 | ||
|
|
7be98e5efc | ||
|
|
a65b356c9d | ||
|
|
08c17ef1b4 | ||
|
|
06f2f002b7 | ||
|
|
7ac4a8b4b7 | ||
|
|
90f2a86819 | ||
|
|
cbdefb2b23 | ||
|
|
1c36fe3a0a | ||
|
|
2b420ce8a4 | ||
|
|
5cbc1a06b1 | ||
|
|
9e7ee39b3a | ||
|
|
ae030a5f33 | ||
| bc8c35896f | |||
|
|
2cc91b6df0 | ||
| 0d2090fe81 | |||
|
|
bc3548e715 | ||
| 74333cc26b | |||
|
|
7310fb88c2 | ||
| 148bc87b9a | |||
|
|
2a1e842100 | ||
| bc28430d24 | |||
|
|
cc92272217 | ||
| d6f4468a9c | |||
|
|
2f796a2ebd | ||
| 1f1f6823db | |||
|
|
b92f74b63a | ||
|
|
cb7f7dbc4d | ||
|
|
03126d5584 | ||
|
|
495e884c41 | ||
|
|
65aa1e6104 | ||
|
|
7f2a3b76b4 | ||
| ea73f00461 | |||
|
|
25ace30a03 | ||
|
|
ca574c2280 | ||
|
|
09387f90e1 | ||
|
|
e641ceab48 | ||
|
|
c263426ea5 | ||
|
|
bacac067cf | ||
| 914fed08d8 |
448
apps/authentik/authentik.yaml
Normal file
448
apps/authentik/authentik.yaml
Normal file
@@ -0,0 +1,448 @@
|
||||
# Authentik OIDC backend
|
||||
# ArgoCD-managed. BlueJay Lab.
|
||||
#
|
||||
# Stack:
|
||||
# - PostgreSQL 16 StatefulSet (single replica, Longhorn RWO 5Gi)
|
||||
# - Redis 7 Deployment (no persistence — session/cache only)
|
||||
# - Authentik server + worker Deployments (image ghcr.io/goauthentik/server:2024.12.3)
|
||||
# - Media PVC shared between server + worker (Longhorn RWO 2Gi)
|
||||
# - Certificate via step-ca-acme ClusterIssuer
|
||||
# - Traefik IngressRoute at id.iamworkin.lan
|
||||
#
|
||||
# Secrets come from 1Password item "authentik-credentials" (IAmWorkin vault, id y6i74ch22q5wvm7znquq4nhhcu)
|
||||
# via the OnePasswordItem CRD, materialized into k8s Secret authentik/authentik-credentials.
|
||||
#
|
||||
# Why the discovery URL is /application/o/pimanager/ : Authentik issues per-application OIDC providers.
|
||||
# The pimanager OIDC application/provider is created after the cluster pods are healthy (manual or
|
||||
# via API once the bootstrap token is available — see Notes substrate).
|
||||
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: authentik
|
||||
labels:
|
||||
app.kubernetes.io/part-of: bluejay-infra
|
||||
|
||||
---
|
||||
# 1Password operator pulls the authentik-credentials item into a k8s Secret of the same name.
|
||||
# Field labels in 1P become Secret keys: AUTHENTIK_SECRET_KEY, POSTGRES_PASSWORD, REDIS_PASSWORD,
|
||||
# BOOTSTRAP_ADMIN_PASSWORD, BOOTSTRAP_ADMIN_TOKEN, BOOTSTRAP_ADMIN_EMAIL.
|
||||
apiVersion: onepassword.com/v1
|
||||
kind: OnePasswordItem
|
||||
metadata:
|
||||
name: authentik-credentials
|
||||
namespace: authentik
|
||||
spec:
|
||||
itemPath: "vaults/IAmWorkin/items/authentik-credentials"
|
||||
|
||||
---
|
||||
# Shared media volume for server + worker pods.
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: authentik-media
|
||||
namespace: authentik
|
||||
spec:
|
||||
storageClassName: longhorn
|
||||
accessModes: [ReadWriteOnce]
|
||||
resources:
|
||||
requests:
|
||||
storage: 2Gi
|
||||
|
||||
---
|
||||
# PostgreSQL 16 StatefulSet — Authentik's primary store.
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: authentik-postgres
|
||||
namespace: authentik
|
||||
labels:
|
||||
app: authentik-postgres
|
||||
argocd.argoproj.io/instance: infra-authentik
|
||||
spec:
|
||||
persistentVolumeClaimRetentionPolicy:
|
||||
whenDeleted: Retain
|
||||
whenScaled: Retain
|
||||
podManagementPolicy: OrderedReady
|
||||
serviceName: authentik-postgres
|
||||
replicas: 1
|
||||
revisionHistoryLimit: 10
|
||||
selector:
|
||||
matchLabels:
|
||||
app: authentik-postgres
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: authentik-postgres
|
||||
spec:
|
||||
containers:
|
||||
- name: postgres
|
||||
image: postgres:16-alpine
|
||||
ports:
|
||||
- containerPort: 5432
|
||||
name: postgres
|
||||
env:
|
||||
- name: POSTGRES_USER
|
||||
value: authentik
|
||||
- name: POSTGRES_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: authentik-credentials
|
||||
key: POSTGRES_PASSWORD
|
||||
- name: POSTGRES_DB
|
||||
value: authentik
|
||||
- name: POSTGRES_INITDB_ARGS
|
||||
value: "--encoding=UTF-8 --lc-collate=C --lc-ctype=C"
|
||||
- name: PGDATA
|
||||
value: /var/lib/postgresql/data/pgdata
|
||||
readinessProbe:
|
||||
exec:
|
||||
command: ["pg_isready", "-U", "authentik"]
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 5
|
||||
livenessProbe:
|
||||
exec:
|
||||
command: ["pg_isready", "-U", "authentik"]
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
resources:
|
||||
requests: { cpu: 100m, memory: 256Mi }
|
||||
limits: { cpu: 1000m, memory: 1Gi }
|
||||
volumeMounts:
|
||||
- name: pgdata
|
||||
mountPath: /var/lib/postgresql/data
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
name: pgdata
|
||||
spec:
|
||||
storageClassName: longhorn
|
||||
accessModes: [ReadWriteOnce]
|
||||
volumeMode: Filesystem
|
||||
resources:
|
||||
requests:
|
||||
storage: 5Gi
|
||||
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: authentik-postgres
|
||||
namespace: authentik
|
||||
spec:
|
||||
clusterIP: None
|
||||
selector:
|
||||
app: authentik-postgres
|
||||
ports:
|
||||
- name: postgres
|
||||
port: 5432
|
||||
targetPort: 5432
|
||||
|
||||
---
|
||||
# Redis 7 — session storage + Celery broker. No persistence needed (cache).
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: authentik-redis
|
||||
namespace: authentik
|
||||
labels:
|
||||
app: authentik-redis
|
||||
argocd.argoproj.io/instance: infra-authentik
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app: authentik-redis
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: authentik-redis
|
||||
spec:
|
||||
containers:
|
||||
- name: redis
|
||||
image: redis:7-alpine
|
||||
args:
|
||||
- "--save"
|
||||
- ""
|
||||
- "--appendonly"
|
||||
- "no"
|
||||
- "--requirepass"
|
||||
- "$(REDIS_PASSWORD)"
|
||||
env:
|
||||
- name: REDIS_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: authentik-credentials
|
||||
key: REDIS_PASSWORD
|
||||
ports:
|
||||
- containerPort: 6379
|
||||
name: redis
|
||||
readinessProbe:
|
||||
tcpSocket: { port: 6379 }
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 5
|
||||
livenessProbe:
|
||||
tcpSocket: { port: 6379 }
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
resources:
|
||||
requests: { cpu: 50m, memory: 64Mi }
|
||||
limits: { cpu: 500m, memory: 256Mi }
|
||||
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: authentik-redis
|
||||
namespace: authentik
|
||||
spec:
|
||||
selector:
|
||||
app: authentik-redis
|
||||
ports:
|
||||
- name: redis
|
||||
port: 6379
|
||||
targetPort: 6379
|
||||
|
||||
---
|
||||
# Authentik server Deployment — HTTP frontend on :9000.
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: authentik-server
|
||||
namespace: authentik
|
||||
labels:
|
||||
app: authentik-server
|
||||
argocd.argoproj.io/instance: infra-authentik
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate # shares /media RWO PVC with worker
|
||||
selector:
|
||||
matchLabels:
|
||||
app: authentik-server
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: authentik-server
|
||||
spec:
|
||||
securityContext:
|
||||
# Authentik image runs as uid 1000 "authentik" but the Longhorn PVC mounts
|
||||
# root:root by default. fsGroup recursively chgrp + chmod g+rwx so the
|
||||
# non-root container can mkdir /media/public during the tenant_files migration.
|
||||
fsGroup: 1000
|
||||
containers:
|
||||
- name: server
|
||||
image: ghcr.io/goauthentik/server:2024.12.3
|
||||
args: ["server"]
|
||||
ports:
|
||||
- containerPort: 9000
|
||||
name: http
|
||||
- containerPort: 9443
|
||||
name: https
|
||||
env:
|
||||
- name: AUTHENTIK_SECRET_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: authentik-credentials
|
||||
key: AUTHENTIK_SECRET_KEY
|
||||
- name: AUTHENTIK_REDIS__HOST
|
||||
value: authentik-redis
|
||||
- name: AUTHENTIK_REDIS__PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: authentik-credentials
|
||||
key: REDIS_PASSWORD
|
||||
- name: AUTHENTIK_POSTGRESQL__HOST
|
||||
value: authentik-postgres
|
||||
- name: AUTHENTIK_POSTGRESQL__NAME
|
||||
value: authentik
|
||||
- name: AUTHENTIK_POSTGRESQL__USER
|
||||
value: authentik
|
||||
- name: AUTHENTIK_POSTGRESQL__PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: authentik-credentials
|
||||
key: POSTGRES_PASSWORD
|
||||
- name: AUTHENTIK_BOOTSTRAP_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: authentik-credentials
|
||||
key: BOOTSTRAP_ADMIN_PASSWORD
|
||||
- name: AUTHENTIK_BOOTSTRAP_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: authentik-credentials
|
||||
key: BOOTSTRAP_ADMIN_TOKEN
|
||||
- name: AUTHENTIK_BOOTSTRAP_EMAIL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: authentik-credentials
|
||||
key: BOOTSTRAP_ADMIN_EMAIL
|
||||
- name: AUTHENTIK_DISABLE_UPDATE_CHECK
|
||||
value: "true"
|
||||
- name: AUTHENTIK_ERROR_REPORTING__ENABLED
|
||||
value: "false"
|
||||
- name: AUTHENTIK_LOG_LEVEL
|
||||
value: info
|
||||
# First-boot Authentik can take 3+ min on the migration phase
|
||||
# (waiting on DB lock while worker also runs migrations). Initial
|
||||
# delays are generous so kubelet doesn't kill the pod mid-migration;
|
||||
# periodSeconds keeps post-startup probing responsive.
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /-/health/ready/
|
||||
port: 9000
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 12
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /-/health/live/
|
||||
port: 9000
|
||||
initialDelaySeconds: 300
|
||||
periodSeconds: 30
|
||||
timeoutSeconds: 10
|
||||
failureThreshold: 3
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /-/health/live/
|
||||
port: 9000
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 15
|
||||
timeoutSeconds: 10
|
||||
failureThreshold: 40 # 30s + 40*15s = 10.5 min budget
|
||||
resources:
|
||||
requests: { cpu: 150m, memory: 512Mi }
|
||||
limits: { cpu: 1500m, memory: 1Gi }
|
||||
volumeMounts:
|
||||
- name: media
|
||||
mountPath: /media
|
||||
volumes:
|
||||
- name: media
|
||||
persistentVolumeClaim:
|
||||
claimName: authentik-media
|
||||
|
||||
---
|
||||
# Authentik worker Deployment — runs Celery background tasks.
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: authentik-worker
|
||||
namespace: authentik
|
||||
labels:
|
||||
app: authentik-worker
|
||||
argocd.argoproj.io/instance: infra-authentik
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate # shares /media RWO PVC with server
|
||||
selector:
|
||||
matchLabels:
|
||||
app: authentik-worker
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: authentik-worker
|
||||
spec:
|
||||
securityContext:
|
||||
# Same as server pod — non-root uid 1000 needs PVC group write.
|
||||
fsGroup: 1000
|
||||
containers:
|
||||
- name: worker
|
||||
image: ghcr.io/goauthentik/server:2024.12.3
|
||||
args: ["worker"]
|
||||
env:
|
||||
- name: AUTHENTIK_SECRET_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: authentik-credentials
|
||||
key: AUTHENTIK_SECRET_KEY
|
||||
- name: AUTHENTIK_REDIS__HOST
|
||||
value: authentik-redis
|
||||
- name: AUTHENTIK_REDIS__PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: authentik-credentials
|
||||
key: REDIS_PASSWORD
|
||||
- name: AUTHENTIK_POSTGRESQL__HOST
|
||||
value: authentik-postgres
|
||||
- name: AUTHENTIK_POSTGRESQL__NAME
|
||||
value: authentik
|
||||
- name: AUTHENTIK_POSTGRESQL__USER
|
||||
value: authentik
|
||||
- name: AUTHENTIK_POSTGRESQL__PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: authentik-credentials
|
||||
key: POSTGRES_PASSWORD
|
||||
- name: AUTHENTIK_DISABLE_UPDATE_CHECK
|
||||
value: "true"
|
||||
- name: AUTHENTIK_ERROR_REPORTING__ENABLED
|
||||
value: "false"
|
||||
- name: AUTHENTIK_LOG_LEVEL
|
||||
value: info
|
||||
resources:
|
||||
requests: { cpu: 100m, memory: 256Mi }
|
||||
limits: { cpu: 1000m, memory: 768Mi }
|
||||
volumeMounts:
|
||||
- name: media
|
||||
mountPath: /media
|
||||
volumes:
|
||||
- name: media
|
||||
persistentVolumeClaim:
|
||||
claimName: authentik-media
|
||||
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: authentik-server
|
||||
namespace: authentik
|
||||
spec:
|
||||
selector:
|
||||
app: authentik-server
|
||||
ports:
|
||||
- name: http
|
||||
port: 9000
|
||||
targetPort: 9000
|
||||
- name: https
|
||||
port: 9443
|
||||
targetPort: 9443
|
||||
|
||||
---
|
||||
# step-ca leaf certificate for id.iamworkin.lan.
|
||||
# step-ca container resolver uses pfSense Unbound, so the public A record for id.iamworkin.lan
|
||||
# MUST exist before this Certificate is applied (cert-manager HTTP-01 will silently 2h-backoff
|
||||
# otherwise). Added 2026-05-25 via scripts/pfsense-add-id-host.py.
|
||||
apiVersion: cert-manager.io/v1
|
||||
kind: Certificate
|
||||
metadata:
|
||||
name: authentik-tls
|
||||
namespace: authentik
|
||||
spec:
|
||||
secretName: authentik-tls
|
||||
dnsNames:
|
||||
- id.iamworkin.lan
|
||||
issuerRef:
|
||||
name: step-ca-acme
|
||||
kind: ClusterIssuer
|
||||
|
||||
---
|
||||
apiVersion: traefik.io/v1alpha1
|
||||
kind: IngressRoute
|
||||
metadata:
|
||||
name: authentik
|
||||
namespace: authentik
|
||||
spec:
|
||||
entryPoints: [websecure]
|
||||
routes:
|
||||
- match: Host(`id.iamworkin.lan`)
|
||||
kind: Rule
|
||||
services:
|
||||
- name: authentik-server
|
||||
port: 9000
|
||||
tls:
|
||||
secretName: authentik-tls
|
||||
@@ -1,27 +0,0 @@
|
||||
# FlowerCore Brochure
|
||||
|
||||
`apps/brochure` hosts the public brochure split from `FlowerCore.Intranet.Web`.
|
||||
ArgoCD's `apps/*` ApplicationSet will create `infra-brochure` after this
|
||||
directory lands on `main`.
|
||||
|
||||
## Runtime
|
||||
|
||||
- Host: `https://brochure.flowercore.io`
|
||||
- Namespace: `brochure`
|
||||
- Deployment: `brochure-web`
|
||||
- Image: `localhost/fc-brochure-web:v20260524-sprint32`
|
||||
- Port: `8080`
|
||||
- Public route method allowlist: `GET` and `HEAD`
|
||||
|
||||
## Operator Actions
|
||||
|
||||
1. Publish and import `localhost/fc-brochure-web:v20260524-sprint32` to every
|
||||
RKE2 node before sync, using the same podman save + `ctr images import`
|
||||
flow as the Intranet deployment.
|
||||
2. Create the Cloudflare DNS record for `brochure.flowercore.io` pointing at
|
||||
the FlowerCore public edge.
|
||||
3. Verify `infra-brochure` appears in ArgoCD, the certificate becomes Ready,
|
||||
and `GET https://brochure.flowercore.io/` returns `200`.
|
||||
|
||||
The route intentionally does not expose `/ops/*` or `/admin/*`; the Brochure
|
||||
web app returns `404` for those paths and Traefik only forwards read methods.
|
||||
@@ -1,131 +0,0 @@
|
||||
# FlowerCore Brochure public host
|
||||
#
|
||||
# Thin Blazor host for public What's New, walkthrough, and gallery content
|
||||
# carved out of FlowerCore.Intranet.Web. The ApplicationSet creates
|
||||
# infra-brochure from this directory after merge.
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: brochure
|
||||
labels:
|
||||
app.kubernetes.io/part-of: flowercore
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: brochure-web
|
||||
namespace: brochure
|
||||
labels:
|
||||
app: brochure-web
|
||||
app.kubernetes.io/name: brochure-web
|
||||
app.kubernetes.io/part-of: flowercore
|
||||
spec:
|
||||
replicas: 1
|
||||
revisionHistoryLimit: 3
|
||||
selector:
|
||||
matchLabels:
|
||||
app: brochure-web
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: brochure-web
|
||||
app.kubernetes.io/name: brochure-web
|
||||
app.kubernetes.io/part-of: flowercore
|
||||
spec:
|
||||
containers:
|
||||
- name: brochure-web
|
||||
image: localhost/fc-brochure-web:v20260524-sprint32
|
||||
imagePullPolicy: Never
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
name: http
|
||||
env:
|
||||
- name: ASPNETCORE_ENVIRONMENT
|
||||
value: Production
|
||||
- name: ASPNETCORE_URLS
|
||||
value: "http://+:8080"
|
||||
resources:
|
||||
requests:
|
||||
cpu: "25m"
|
||||
memory: "128Mi"
|
||||
limits:
|
||||
cpu: "500m"
|
||||
memory: "512Mi"
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1654
|
||||
runAsGroup: 1654
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
volumeMounts:
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
volumes:
|
||||
- name: tmp
|
||||
emptyDir: {}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: brochure-web
|
||||
namespace: brochure
|
||||
labels:
|
||||
app: brochure-web
|
||||
app.kubernetes.io/name: brochure-web
|
||||
app.kubernetes.io/part-of: flowercore
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: brochure-web
|
||||
ports:
|
||||
- name: http
|
||||
port: 8080
|
||||
targetPort: http
|
||||
---
|
||||
apiVersion: cert-manager.io/v1
|
||||
kind: Certificate
|
||||
metadata:
|
||||
name: brochure-web-tls
|
||||
namespace: brochure
|
||||
spec:
|
||||
secretName: brochure-web-tls
|
||||
issuerRef:
|
||||
name: step-ca-acme
|
||||
kind: ClusterIssuer
|
||||
dnsNames:
|
||||
- brochure.flowercore.io
|
||||
duration: 720h
|
||||
renewBefore: 240h
|
||||
---
|
||||
apiVersion: traefik.io/v1alpha1
|
||||
kind: IngressRoute
|
||||
metadata:
|
||||
name: brochure-web-public
|
||||
namespace: brochure
|
||||
spec:
|
||||
entryPoints:
|
||||
- websecure
|
||||
routes:
|
||||
- match: Host(`brochure.flowercore.io`) && (Method(`GET`) || Method(`HEAD`))
|
||||
kind: Rule
|
||||
services:
|
||||
- name: brochure-web
|
||||
port: 8080
|
||||
tls:
|
||||
secretName: brochure-web-tls
|
||||
@@ -1,33 +0,0 @@
|
||||
# Explicit ArgoCD Application shape for bootstrap/review.
|
||||
#
|
||||
# The live bluejay-infra ApplicationSet already discovers apps/* directories
|
||||
# and creates this same Application name (`infra-fc-devicemgmt`) automatically.
|
||||
# Keep repoURL on the internal Gitea ClusterIP URL; ArgoCD does not trust the
|
||||
# external step-ca HTTPS endpoint.
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: infra-fc-devicemgmt
|
||||
namespace: argocd
|
||||
labels:
|
||||
app.kubernetes.io/name: fc-devicemgmt
|
||||
app.kubernetes.io/part-of: flowercore
|
||||
app.kubernetes.io/managed-by: argocd
|
||||
flowercore.io/tenant-id: system
|
||||
flowercore.io/created-by: bluejay-infra
|
||||
spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: http://gitea-clusterip.gitea.svc.cluster.local:3000/bluejay/bluejay-infra.git
|
||||
targetRevision: main
|
||||
path: apps/fc-devicemgmt
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: fc-devicemgmt
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
- ServerSideApply=true
|
||||
@@ -47,7 +47,7 @@ spec:
|
||||
fsGroupChangePolicy: OnRootMismatch
|
||||
containers:
|
||||
- name: operator
|
||||
image: localhost/fc-devicemgmt-operator:v20260512-cx5
|
||||
image: localhost/fc-devicemgmt-operator:v20260519-sp34cl3-fix
|
||||
imagePullPolicy: Never
|
||||
ports:
|
||||
- name: metrics
|
||||
|
||||
@@ -4,6 +4,22 @@
|
||||
# Sprint 9+ lane. This manifest is static-valid without requiring the image to
|
||||
# exist yet; import localhost/fc-devicemgmt-web:<tag> to all schedulable RKE2
|
||||
# nodes before letting ArgoCD sync a live rollout.
|
||||
#
|
||||
# SCALED TO 0 — 2026-05-19 morning-routine cleanup.
|
||||
# The Web pod cannot start until TWO upstream gaps close:
|
||||
# 1. MySQL DB instance `flowercore_devicemgmt` (user `fc_devicemgmt`) is
|
||||
# provisioned via fc-mysql Manager. The cluster currently has ZERO
|
||||
# MySqlInstanceCrds and no `mysql.fc-mysql.svc:3306` Service, so the
|
||||
# deployment-web container env `FlowerCore__Database__Host=mysql.fc-mysql.svc`
|
||||
# points at nothing. Provision via the fc-mysql Manager UI/REST/MCP.
|
||||
# 2. 1Password vault item `IAmWorkin/FlowerCore DeviceManagement Runtime`
|
||||
# with 5 fields (DB-Password, mtls-ca.pem, mtls-client.crt, mtls-client.key,
|
||||
# mtls-chain.pem) — see apps/fc-devicemgmt/1password-item.yaml. Mint mTLS
|
||||
# from step-ca-agent ClusterIssuer per ADR-126; DB-Password must match the
|
||||
# password configured for the MySQL user.
|
||||
# Re-enable: change replicas back to 2 after both gaps close. The image tag
|
||||
# in this file (v20260512-cx5) MAY also need a refresh — it predates the
|
||||
# Sprint 34 Cl-3 operator fix; Web may have an analogous bug.
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
@@ -20,7 +36,7 @@ metadata:
|
||||
annotations:
|
||||
flowercore.io/traceability-standard: k8s-pod-ownership-and-traceability-standard
|
||||
spec:
|
||||
replicas: 2
|
||||
replicas: 0
|
||||
revisionHistoryLimit: 3
|
||||
selector:
|
||||
matchLabels:
|
||||
|
||||
33
apps/fc-signalcontrol/README.md
Normal file
33
apps/fc-signalcontrol/README.md
Normal file
@@ -0,0 +1,33 @@
|
||||
# FlowerCore SignalControl platform notes
|
||||
|
||||
This app owns the cluster web manager at `signalcontrol.iamworkin.lan` and documents the physical Pi pilot at `signal-a.iamworkin.lan` / `pirelay`.
|
||||
|
||||
## mTLS enrollment pattern
|
||||
|
||||
Do not install or restart anything from this repo. The intended pirelay pattern is the Pi-signage step-ca-agent shape:
|
||||
|
||||
- stable node identity: `pirelay`
|
||||
- local private key and CSR generated on the node
|
||||
- CSR submitted through the approved DeviceManagement/step-ca enrollment path
|
||||
- client certificate and chain stored node-local under `/etc/flowercore/signalcontrol/mtls/`
|
||||
- daily renewal timer, renewing only when fewer than 30 days remain
|
||||
- certificate used for DM-agent to DM-web traffic and future SignalControl inter-service calls
|
||||
|
||||
Secrets, enrollment codes, private keys, p12 passphrases, and OIDC client secrets stay out of Git.
|
||||
|
||||
## Telemetry
|
||||
|
||||
Monitoring manifests add a dedicated Prometheus job:
|
||||
|
||||
- `signalcontrol-pi-app`
|
||||
- target `10.0.58.113:5200`
|
||||
- path `/metrics/prometheus`
|
||||
- labels `instance="pirelay"`, `host="signal-a.iamworkin.lan"`, `service="signalcontrol-pi"`
|
||||
|
||||
Host metrics continue through the `edge-nodes` node_exporter target at `10.0.58.113:9100`.
|
||||
|
||||
## Physical-control audit
|
||||
|
||||
The app ships with `FlowerCore:SignalControl:PhysicalAudit:Enabled=false` and `ForwardingEnabled=false`. Enabling local audit creates a SHA-256 hash chain for physical-control mutations. Forwarding to `https://audit.iamworkin.lan/api/v1/audit/signalcontrol` requires flipping the forwarding gate separately.
|
||||
|
||||
Telemetry reads and `/metrics` scrapes are not audited.
|
||||
@@ -532,7 +532,7 @@ spec:
|
||||
fsGroupChangePolicy: OnRootMismatch
|
||||
containers:
|
||||
- name: web
|
||||
image: localhost/fc-ttsreader-web:v20260518-sprint36-demo-finish-b132cbf
|
||||
image: localhost/fc-ttsreader-web:v20260531-tts-corrections-r2
|
||||
imagePullPolicy: Never
|
||||
ports:
|
||||
- containerPort: 5217
|
||||
@@ -554,6 +554,8 @@ spec:
|
||||
value: "/data/chapter-context.db"
|
||||
- name: TtsReader__Jobs__Root
|
||||
value: "/data/jobs"
|
||||
- name: TtsReader__Export__LocalCasRoot
|
||||
value: "/data/bundles/cas"
|
||||
- name: TtsReader__Piper__Host
|
||||
value: "10.0.57.17"
|
||||
- name: TtsReader__Piper__Port
|
||||
|
||||
@@ -58,7 +58,7 @@ spec:
|
||||
nodeName: rke2-server
|
||||
containers:
|
||||
- name: web
|
||||
image: localhost/fc-updater-web:v20260509-4162dca-authgate
|
||||
image: localhost/fc-updater-web:v202605310029-7974fc4
|
||||
imagePullPolicy: Never
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
@@ -88,6 +88,8 @@ spec:
|
||||
value: Faith AI Mike Edition
|
||||
- name: FlowerCore__Updater__PublicShares__Links__0__Description
|
||||
value: Private release link for Mike's Faith AI bundle.
|
||||
- name: FlowerCore__Audit__Sinks__Loki__Enabled
|
||||
value: "false"
|
||||
- name: FlowerCore__Updater__Auth__Bootstrap__Enabled
|
||||
value: "true"
|
||||
- name: FlowerCore__Updater__Auth__Bootstrap__Username
|
||||
|
||||
2
apps/github-runner/.gitattributes
vendored
Normal file
2
apps/github-runner/.gitattributes
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
*.sh text eol=lf
|
||||
Dockerfile text eol=lf
|
||||
54
apps/github-runner/Dockerfile
Normal file
54
apps/github-runner/Dockerfile
Normal file
@@ -0,0 +1,54 @@
|
||||
FROM myoung34/github-runner:latest
|
||||
|
||||
ARG RUBY_VERSION=3.3.11
|
||||
ARG RUBY_MINOR=3.3
|
||||
ARG RUBY_BUILD_VERSION=v20260326
|
||||
ARG RUNNER_UID=1001
|
||||
ARG RUNNER_GID=1001
|
||||
|
||||
ENV RUNNER_TOOL_CACHE=/home/runner/_tool
|
||||
ENV RUNNER_RUBY_TOOLCACHE=/opt/runner-toolcache
|
||||
ENV PATH="/home/runner/_tool/Ruby/${RUBY_MINOR}/x64/bin:/opt/runner-toolcache/Ruby/${RUBY_MINOR}/x64/bin:${PATH}"
|
||||
|
||||
USER root
|
||||
|
||||
# Bake the IAmWorkin step-ca root CA into the system trust store. Without
|
||||
# this, .NET HttpClient calls from CI tests against *.iamworkin.lan
|
||||
# (e.g. https://selenium.iamworkin.lan/session) fail with `PartialChain`
|
||||
# because the runner image's default Ubuntu trust bundle doesn't include
|
||||
# our internal Root CA. update-ca-certificates regenerates
|
||||
# /etc/ssl/certs/ca-certificates.crt, which OpenSSL + .NET on Linux read
|
||||
# automatically — no SSL_CERT_FILE env var needed.
|
||||
COPY step-ca-root.crt /usr/local/share/ca-certificates/iamworkin-step-ca-root.crt
|
||||
|
||||
RUN apt-get update \
|
||||
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
|
||||
autoconf \
|
||||
bison \
|
||||
build-essential \
|
||||
ca-certificates \
|
||||
curl \
|
||||
libdb-dev \
|
||||
libffi-dev \
|
||||
libgdbm-dev \
|
||||
libgmp-dev \
|
||||
libncurses-dev \
|
||||
libreadline-dev \
|
||||
libssl-dev \
|
||||
libyaml-dev \
|
||||
patch \
|
||||
pkg-config \
|
||||
uuid-dev \
|
||||
zlib1g-dev \
|
||||
&& update-ca-certificates \
|
||||
&& curl -fsSL "https://github.com/rbenv/ruby-build/archive/refs/tags/${RUBY_BUILD_VERSION}.tar.gz" -o /tmp/ruby-build.tar.gz \
|
||||
&& mkdir -p /tmp/ruby-build \
|
||||
&& tar -xzf /tmp/ruby-build.tar.gz --strip-components=1 -C /tmp/ruby-build \
|
||||
&& /tmp/ruby-build/install.sh \
|
||||
&& rm -rf /tmp/ruby-build /tmp/ruby-build.tar.gz /var/lib/apt/lists/*
|
||||
|
||||
COPY install-ruby-toolcache.sh /usr/local/bin/install-ruby-toolcache.sh
|
||||
|
||||
RUN chmod +x /usr/local/bin/install-ruby-toolcache.sh \
|
||||
&& RUBY_VERSION="${RUBY_VERSION}" RUBY_MINOR="${RUBY_MINOR}" TOOLCACHE_ROOT="${RUNNER_RUBY_TOOLCACHE}" RUNNER_UID="${RUNNER_UID}" RUNNER_GID="${RUNNER_GID}" /usr/local/bin/install-ruby-toolcache.sh \
|
||||
&& ruby -v
|
||||
@@ -7,12 +7,17 @@ Deployments with `kubectl`; update this manifest and let ArgoCD reconcile.
|
||||
|
||||
All repo-scoped Linux runners use:
|
||||
|
||||
- `localhost/fc-github-runner:v20260525-ruby3.3.11-stepca`, derived from
|
||||
`myoung34/github-runner:latest`
|
||||
- `ACCESS_TOKEN` from the `github-runner-token` Secret
|
||||
- `RUN_AS_ROOT=false`
|
||||
- `EPHEMERAL=true`
|
||||
- `LABELS=self-hosted,linux,fc-build-linux`
|
||||
- writable non-root paths under `/home/runner` for .NET, NuGet, XDG cache, and
|
||||
Actions tool cache
|
||||
- Ruby 3.3.11 seeded into `/home/runner/_tool/Ruby/3.3/x64` from the baked
|
||||
`/opt/runner-toolcache` copy so `ruby/setup-ruby@v1` can discover it on
|
||||
self-hosted `ubuntu-20.04-x64` runners
|
||||
|
||||
`github-runner` for `FlowerCore.Common` is single-replica because it retains the
|
||||
original Longhorn ReadWriteOnce NuGet PVC. Every other repo-scoped runner uses
|
||||
@@ -28,6 +33,46 @@ Sprint 32 final long-tail wave adds 16 two-replica Deployments:
|
||||
`FlowerCore.Provisioning`, `FlowerCore.Redis`, `FlowerCore.MessageBoard`, and
|
||||
`FlowerCore.MenuBoard`.
|
||||
|
||||
## Image Build
|
||||
|
||||
Ruby is baked with a pinned `ruby-build` release and Ruby patch version. The pod
|
||||
still mounts an `emptyDir` over `/home/runner`, so the `setup-runner-home` init
|
||||
container copies the baked toolcache from `/opt/runner-toolcache/Ruby` into
|
||||
`/home/runner/_tool/Ruby` before the runner container starts.
|
||||
|
||||
The IAmWorkin step-ca root CA is also baked into the system trust store
|
||||
(`/usr/local/share/ca-certificates/iamworkin-step-ca-root.crt`, registered by
|
||||
`update-ca-certificates`). Without it, .NET HttpClient calls from CI tests
|
||||
against `*.iamworkin.lan` (e.g. `https://selenium.iamworkin.lan/session`)
|
||||
fail with `PartialChain`. To refresh the bundled cert when the root rotates,
|
||||
re-extract from the cluster and overwrite `step-ca-root.crt`:
|
||||
|
||||
```bash
|
||||
kubectl get secret -n cert-manager step-ca-root \
|
||||
-o jsonpath='{.data.ca\.crt}' | base64 -d > step-ca-root.crt
|
||||
```
|
||||
|
||||
```bash
|
||||
cd apps/github-runner
|
||||
podman build -t localhost/fc-github-runner:v20260525-ruby3.3.11-stepca .
|
||||
podman run --rm localhost/fc-github-runner:v20260525-ruby3.3.11-stepca ruby -v
|
||||
podman run --rm localhost/fc-github-runner:v20260525-ruby3.3.11-stepca \
|
||||
test -f /opt/runner-toolcache/Ruby/3.3/x64.complete
|
||||
podman save localhost/fc-github-runner:v20260525-ruby3.3.11-stepca \
|
||||
-o fc-github-runner-v20260525-ruby3.3.11-stepca.tar
|
||||
```
|
||||
|
||||
Import the saved image on every schedulable RKE2 node before ArgoCD rolls the
|
||||
Deployments:
|
||||
|
||||
```bash
|
||||
for node in rke2-server rke2-agent1 rke2-agent2; do
|
||||
scp fc-github-runner-v20260525-ruby3.3.11-stepca.tar "$node:/tmp/"
|
||||
ssh "$node" 'sudo ctr -a /run/k3s/containerd/containerd.sock -n k8s.io images rm localhost/fc-github-runner:v20260525-ruby3.3.11-stepca || true'
|
||||
ssh "$node" 'sudo ctr -a /run/k3s/containerd/containerd.sock -n k8s.io images import /tmp/fc-github-runner-v20260525-ruby3.3.11-stepca.tar'
|
||||
done
|
||||
```
|
||||
|
||||
## Post-Merge Proof
|
||||
|
||||
After the PR is merged and ArgoCD syncs, verify the runner fleet:
|
||||
@@ -36,6 +81,14 @@ After the PR is merged and ArgoCD syncs, verify the runner fleet:
|
||||
kubectl -n github-runner get deploy,pods,pvc
|
||||
```
|
||||
|
||||
Verify the Ruby toolcache in a fresh pod:
|
||||
|
||||
```bash
|
||||
kubectl -n github-runner exec deploy/github-runner-puppet -c runner -- ruby -v
|
||||
kubectl -n github-runner exec deploy/github-runner-puppet -c runner -- sh -c \
|
||||
'echo "$RUNNER_TOOL_CACHE" && test -f "$RUNNER_TOOL_CACHE/Ruby/3.3/x64.complete"'
|
||||
```
|
||||
|
||||
Verify GitHub registration for the repo-scoped runners:
|
||||
|
||||
```bash
|
||||
@@ -69,6 +122,10 @@ from GitHub Actions and verify it lands on an `rke2-linux-*` runner.
|
||||
- `actions/setup-dotnet` permission error at `/usr/share/dotnet`: check that
|
||||
`DOTNET_INSTALL_DIR=/home/runner/.dotnet` and related cache env vars are
|
||||
present on the runner pod.
|
||||
- `ruby/setup-ruby@v1` says self-hosted runners must install Ruby in
|
||||
`$RUNNER_TOOL_CACHE`: check that the init container copied
|
||||
`/opt/runner-toolcache/Ruby` into `/home/runner/_tool/Ruby` and that
|
||||
`/home/runner/_tool/Ruby/3.3/x64.complete` exists.
|
||||
- `404` during runner registration: the fine-grained PAT is valid but missing
|
||||
repository access for that repo. Add the repo to the PAT access list; the PAT
|
||||
value does not change.
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
19
apps/github-runner/install-ruby-toolcache.sh
Normal file
19
apps/github-runner/install-ruby-toolcache.sh
Normal file
@@ -0,0 +1,19 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
RUBY_VERSION="${RUBY_VERSION:-3.3.11}"
|
||||
RUBY_MINOR="${RUBY_MINOR:-3.3}"
|
||||
TOOLCACHE_ROOT="${TOOLCACHE_ROOT:-/opt/runner-toolcache}"
|
||||
RUNNER_UID="${RUNNER_UID:-1001}"
|
||||
RUNNER_GID="${RUNNER_GID:-1001}"
|
||||
RUBY_PREFIX="${TOOLCACHE_ROOT}/Ruby/${RUBY_VERSION}/x64"
|
||||
|
||||
mkdir -p "${TOOLCACHE_ROOT}/Ruby"
|
||||
RUBY_CONFIGURE_OPTS="${RUBY_CONFIGURE_OPTS:---disable-install-doc --disable-yjit}" ruby-build "${RUBY_VERSION}" "${RUBY_PREFIX}"
|
||||
|
||||
touch "${TOOLCACHE_ROOT}/Ruby/${RUBY_VERSION}/x64.complete"
|
||||
ln -sfn "${RUBY_VERSION}" "${TOOLCACHE_ROOT}/Ruby/${RUBY_MINOR}"
|
||||
|
||||
"${RUBY_PREFIX}/bin/ruby" -v
|
||||
chown -R "${RUNNER_UID}:${RUNNER_GID}" "${TOOLCACHE_ROOT}"
|
||||
chmod -R a+rX "${TOOLCACHE_ROOT}"
|
||||
12
apps/github-runner/step-ca-root.crt
Normal file
12
apps/github-runner/step-ca-root.crt
Normal file
@@ -0,0 +1,12 @@
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIIBxDCCAWqgAwIBAgIRAPY357G6ow6zMAL5+4bS2kkwCgYIKoZIzj0EAwIwQDEa
|
||||
MBgGA1UEChMRSUFtV29ya2luIEFDTUUgQ0ExIjAgBgNVBAMTGUlBbVdvcmtpbiBB
|
||||
Q01FIENBIFJvb3QgQ0EwHhcNMjYwMzA4MTgwNzExWhcNMzYwMzA1MTgwNzExWjBA
|
||||
MRowGAYDVQQKExFJQW1Xb3JraW4gQUNNRSBDQTEiMCAGA1UEAxMZSUFtV29ya2lu
|
||||
IEFDTUUgQ0EgUm9vdCBDQTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABJ2n04X1
|
||||
JZo5Zdq/i1Idv8+fqwZyAzBh7whbqj0SWsJL8UWRabCMqYCs7+dXO0xRSzqkwFDL
|
||||
x+vooOai8RgRNhajRTBDMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAGAQH/
|
||||
AgEBMB0GA1UdDgQWBBRnuPPQR6iM/H6vOluiU3Sygayz8jAKBggqhkjOPQQDAgNI
|
||||
ADBFAiEArQK9dYPGmAZsdYnjziuFVVE5NKZUcceYvGfGC+tLXUsCIAudF2zJrCRq
|
||||
3mK50ZZET/fwTkJwiEF4824mjP8p1CKM
|
||||
-----END CERTIFICATE-----
|
||||
@@ -46,7 +46,7 @@ spec:
|
||||
spec:
|
||||
containers:
|
||||
- name: intranet-web
|
||||
image: localhost/fc-intranet-web:v20260508-brochure-w1
|
||||
image: localhost/fc-intranet-web:v20260531-ttsreader-bridge
|
||||
imagePullPolicy: Never
|
||||
ports:
|
||||
- containerPort: 5300
|
||||
|
||||
@@ -25,7 +25,7 @@ metadata:
|
||||
role: github-actions-runner
|
||||
flowercore.io/managed-by: bluejay-infra
|
||||
spec:
|
||||
runStrategy: Always
|
||||
runStrategy: Halted
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
|
||||
@@ -207,20 +207,13 @@ spec:
|
||||
- port: 993
|
||||
targetPort: 993
|
||||
name: imaps
|
||||
---
|
||||
# TLS Certificate via cert-manager
|
||||
apiVersion: cert-manager.io/v1
|
||||
kind: Certificate
|
||||
metadata:
|
||||
name: mail-tls
|
||||
namespace: mail
|
||||
spec:
|
||||
secretName: mail-tls
|
||||
issuerRef:
|
||||
name: step-ca-acme
|
||||
kind: ClusterIssuer
|
||||
dnsNames:
|
||||
- mail.iamworkin.lan
|
||||
# --- mail-tls Certificate REMOVED 2026-06-01 ---
|
||||
# mail-tls is now managed OUTSIDE cert-manager: issued from step-ca's JWK 'admin'
|
||||
# provisioner and auto-renewed by a systemd timer on noc1 (step ca renew), which
|
||||
# writes the mail-tls secret directly. step-ca-acme only has an HTTP-01 (Traefik)
|
||||
# solver, but mail.iamworkin.lan must resolve to the dedicated MetalLB IP 10.0.56.202
|
||||
# (SMTP/IMAP), so HTTP-01 cannot validate. Do NOT re-add a cert-manager Certificate
|
||||
# here unless a DNS-01 solver is deployed for step-ca-acme.
|
||||
---
|
||||
# Traefik IngressRoute - Webmail placeholder
|
||||
apiVersion: traefik.io/v1alpha1
|
||||
|
||||
260
apps/monitoring/grafana-dashboard-signalcontrol.yaml
Normal file
260
apps/monitoring/grafana-dashboard-signalcontrol.yaml
Normal file
@@ -0,0 +1,260 @@
|
||||
# Grafana dashboard ConfigMap for FlowerCore.SignalControl on pirelay.
|
||||
#
|
||||
# The Grafana Deployment in noc-monitoring.yaml mounts this ConfigMap at
|
||||
# /var/lib/grafana/dashboards/signalcontrol. The paired Prometheus jobs are:
|
||||
# - signalcontrol-pi-app: 10.0.58.113:5200 /metrics/prometheus
|
||||
# - edge-nodes: 10.0.58.113:9100 with instance="pirelay"
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: grafana-dashboard-signalcontrol
|
||||
namespace: monitoring
|
||||
data:
|
||||
signalcontrol.json: |
|
||||
{
|
||||
"annotations": { "list": [] },
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 0,
|
||||
"id": null,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "red", "value": null },
|
||||
{ "color": "green", "value": 1 }
|
||||
]
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": { "h": 5, "w": 6, "x": 0, "y": 0 },
|
||||
"id": 1,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false },
|
||||
"textMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{ "editorMode": "code", "expr": "up{job=\"signalcontrol-pi-app\",instance=\"pirelay\"}", "range": true, "refId": "A" }
|
||||
],
|
||||
"title": "SignalControl App Up",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "red", "value": null },
|
||||
{ "color": "green", "value": 1 }
|
||||
]
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": { "h": 5, "w": 6, "x": 6, "y": 0 },
|
||||
"id": 2,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false },
|
||||
"textMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{ "editorMode": "code", "expr": "up{job=\"edge-nodes\",instance=\"pirelay\"}", "range": true, "refId": "A" }
|
||||
],
|
||||
"title": "pirelay node_exporter Up",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
|
||||
"gridPos": { "h": 5, "w": 6, "x": 12, "y": 0 },
|
||||
"id": 3,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false },
|
||||
"textMode": "name"
|
||||
},
|
||||
"targets": [
|
||||
{ "editorMode": "code", "expr": "signalcontrol_active_pattern{job=\"signalcontrol-pi-app\",instance=\"pirelay\"}", "legendFormat": "{{pattern}}", "range": true, "refId": "A" }
|
||||
],
|
||||
"title": "Active Pattern",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
|
||||
"gridPos": { "h": 5, "w": 6, "x": 18, "y": 0 },
|
||||
"id": 4,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false },
|
||||
"textMode": "name"
|
||||
},
|
||||
"targets": [
|
||||
{ "editorMode": "code", "expr": "signalcontrol_phase{job=\"signalcontrol-pi-app\",instance=\"pirelay\"}", "legendFormat": "{{phase}}", "range": true, "refId": "A" }
|
||||
],
|
||||
"title": "Current Phase",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 5 },
|
||||
"id": 5,
|
||||
"options": { "legend": { "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "single" } },
|
||||
"targets": [
|
||||
{
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (channel, state) (rate(signal_relay_writes_total{job=\"signalcontrol-pi-app\",instance=\"pirelay\"}[$__rate_interval]))",
|
||||
"legendFormat": "channel {{channel}} {{state}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Relay Activations",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 5 },
|
||||
"id": 6,
|
||||
"options": { "legend": { "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "single" } },
|
||||
"targets": [
|
||||
{
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (source, to_phase) (rate(signal_transitions_total{job=\"signalcontrol-pi-app\",instance=\"pirelay\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{source}} -> {{to_phase}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Phase Dwell / Transitions",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 13 },
|
||||
"id": 7,
|
||||
"options": { "legend": { "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "single" } },
|
||||
"targets": [
|
||||
{
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (action) (increase(signal_schedule_fires_total{job=\"signalcontrol-pi-app\",instance=\"pirelay\"}[24h]))",
|
||||
"legendFormat": "{{action}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (from_pattern, to_pattern) (increase(flowercore_signalcontrol_pattern_switches_total{job=\"signalcontrol-pi-app\",instance=\"pirelay\"}[24h]))",
|
||||
"legendFormat": "{{from_pattern}} -> {{to_pattern}}",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"title": "Schedule Fires and Pattern Switches",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "percentunit" }, "overrides": [] },
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 13 },
|
||||
"id": 8,
|
||||
"options": { "legend": { "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "single" } },
|
||||
"targets": [
|
||||
{
|
||||
"editorMode": "code",
|
||||
"expr": "1 - avg by (instance) (rate(node_cpu_seconds_total{job=\"edge-nodes\",instance=\"pirelay\",mode=\"idle\"}[$__rate_interval]))",
|
||||
"legendFormat": "CPU",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"editorMode": "code",
|
||||
"expr": "1 - (node_memory_MemAvailable_bytes{job=\"edge-nodes\",instance=\"pirelay\"} / node_memory_MemTotal_bytes{job=\"edge-nodes\",instance=\"pirelay\"})",
|
||||
"legendFormat": "Memory",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"title": "pirelay Host Utilization",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
|
||||
"gridPos": { "h": 6, "w": 12, "x": 0, "y": 21 },
|
||||
"id": 9,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false },
|
||||
"textMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{ "editorMode": "code", "expr": "signalcontrol_screen_saver_enabled{job=\"signalcontrol-pi-app\",instance=\"pirelay\"}", "range": true, "refId": "A" }
|
||||
],
|
||||
"title": "Screen-saver Enabled",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
|
||||
"gridPos": { "h": 6, "w": 12, "x": 12, "y": 21 },
|
||||
"id": 10,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false },
|
||||
"textMode": "name"
|
||||
},
|
||||
"targets": [
|
||||
{ "editorMode": "code", "expr": "signalcontrol_animation_active{job=\"signalcontrol-pi-app\",instance=\"pirelay\"}", "legendFormat": "{{planner}}", "range": true, "refId": "A" }
|
||||
],
|
||||
"title": "Screen-saver / Animation Engaged",
|
||||
"type": "stat"
|
||||
}
|
||||
],
|
||||
"refresh": "30s",
|
||||
"schemaVersion": 39,
|
||||
"style": "dark",
|
||||
"tags": [ "flowercore", "signalcontrol", "pirelay" ],
|
||||
"templating": { "list": [] },
|
||||
"time": { "from": "now-24h", "to": "now" },
|
||||
"timezone": "browser",
|
||||
"title": "FlowerCore SignalControl",
|
||||
"uid": "flowercore-signalcontrol",
|
||||
"version": 1
|
||||
}
|
||||
@@ -230,6 +230,19 @@ data:
|
||||
vlan: "home"
|
||||
device: "pi3-ks0212"
|
||||
|
||||
# SignalControl Pi-edition app metrics (pirelay / signal-a)
|
||||
- job_name: "signalcontrol-pi-app"
|
||||
scrape_interval: 15s
|
||||
metrics_path: /metrics/prometheus
|
||||
static_configs:
|
||||
- targets: ["10.0.58.113:5200"]
|
||||
labels:
|
||||
instance: "pirelay"
|
||||
host: "signal-a.iamworkin.lan"
|
||||
service: "signalcontrol-pi"
|
||||
vlan: "home"
|
||||
device: "pi3-ks0212"
|
||||
|
||||
# Epson ET-3750 EcoTank Printer SNMP
|
||||
- job_name: "snmp-printer"
|
||||
scrape_interval: 5m
|
||||
@@ -280,13 +293,14 @@ data:
|
||||
printer_model: "NuPrint 210"
|
||||
|
||||
# Print.Web health (Blazor app on edge2:5200)
|
||||
# Target `/health` (anonymous) — root path requires API key auth and returns 401.
|
||||
- job_name: "probe-printweb"
|
||||
metrics_path: /probe
|
||||
params:
|
||||
module: [http_2xx]
|
||||
scrape_interval: 30s
|
||||
static_configs:
|
||||
- targets: ["http://10.0.57.16:5200/"]
|
||||
- targets: ["http://10.0.57.16:5200/health"]
|
||||
labels:
|
||||
instance: "print-web"
|
||||
service: "print-web"
|
||||
@@ -1273,24 +1287,55 @@ metadata:
|
||||
data:
|
||||
notify.py: |
|
||||
#!/usr/bin/env python3
|
||||
"""HTTP->IRC alert relay with thermal printer forwarding for Grafana webhooks.
|
||||
Listens on :9119, posts to #alerts on UnrealIRCd via raw IRC protocol.
|
||||
Alerts tagged alert_channel=thermal_print also POST to Print.Web /api/print/alert.
|
||||
"""HTTP->IRC alert relay with thermal-printer DIGEST forwarding.
|
||||
|
||||
Listens on :9119, posts to #alerts on UnrealIRCd, forwards to Print.Web
|
||||
/api/print/alert. Thermal printing is BATCHED into hourly digests by
|
||||
default so the printer no longer spam-fires per Grafana webhook.
|
||||
|
||||
Routing (per Grafana webhook alert):
|
||||
- IRC: always per-event (operator likes the stream)
|
||||
- Thermal printer:
|
||||
* severity in {critical,disaster,page} OR
|
||||
label alert_channel=thermal_print_immediate -> print NOW
|
||||
* label alert_channel=thermal_print -> enqueue into hourly digest
|
||||
* everything else -> IRC only
|
||||
- RESOLVED webhooks remove the alert from the digest buffer
|
||||
|
||||
Env vars (defaults preserve old behavior on first deploy):
|
||||
THERMAL_PRINT_ENABLED default "true" - master kill switch
|
||||
BATCH_INTERVAL_MIN default "60" - minutes between digest prints
|
||||
BATCH_MAX_PENDING default "50" - force-flush threshold
|
||||
|
||||
HTTP surface:
|
||||
POST / - Grafana webhook entry
|
||||
POST /flush - manual digest flush (idempotent)
|
||||
GET / - status + config + buffer depth + stats
|
||||
"""
|
||||
import json, socket, sys, time
|
||||
import json, os, socket, sys, threading, time
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from http.server import HTTPServer, BaseHTTPRequestHandler
|
||||
from urllib.request import Request, urlopen
|
||||
from urllib.error import URLError
|
||||
|
||||
IRC_HOST = "unrealircd.irc.svc" # short name: CoreDNS ndots:5 + iamworkin.lan template hijacks full .cluster.local (see memory)
|
||||
IRC_PORT = 6667
|
||||
IRC_NICK = "grafana-bot"
|
||||
IRC_CHANNEL = "#alerts"
|
||||
PRINT_WEB_URL = "http://10.0.57.16:5200/api/print/alert"
|
||||
PRINT_ENABLED = True
|
||||
THERMAL_PRINT_ENABLED = os.environ.get("THERMAL_PRINT_ENABLED", "true").lower() == "true"
|
||||
BATCH_INTERVAL_MIN = int(os.environ.get("BATCH_INTERVAL_MIN", "60"))
|
||||
BATCH_MAX_PENDING = int(os.environ.get("BATCH_MAX_PENDING", "50"))
|
||||
|
||||
IRC_HOST = os.environ.get("IRC_HOST", "unrealircd.irc.svc")
|
||||
IRC_PORT = int(os.environ.get("IRC_PORT", "6667"))
|
||||
IRC_NICK = os.environ.get("IRC_NICK", "grafana-bot")
|
||||
IRC_CHANNEL = os.environ.get("IRC_CHANNEL", "#alerts")
|
||||
PRINT_WEB_URL = os.environ.get("PRINT_WEB_URL", "http://10.0.57.16:5200/api/print/alert")
|
||||
|
||||
_buffer_lock = threading.Lock()
|
||||
_buffer = {} # fingerprint -> {"alert": dict, "first_seen": float, "last_seen": float}
|
||||
_last_flush_time = time.time()
|
||||
_stats = {"webhooks_received": 0, "irc_sent": 0, "print_immediate": 0,
|
||||
"digest_flushed": 0, "buffer_dedup": 0, "buffer_added": 0,
|
||||
"buffer_resolved": 0, "started_at": time.time()}
|
||||
|
||||
def send_irc(message):
|
||||
"""Connect, handle PING, join, send, quit."""
|
||||
try:
|
||||
sock = socket.create_connection((IRC_HOST, IRC_PORT), timeout=15)
|
||||
sock.sendall(f"NICK {IRC_NICK}\r\n".encode())
|
||||
@@ -1323,52 +1368,137 @@ data:
|
||||
time.sleep(0.5)
|
||||
sock.sendall(b"QUIT :alert delivered\r\n")
|
||||
sock.close()
|
||||
_stats["irc_sent"] += 1
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"[irc-notify] IRC send failed: {e}", file=sys.stderr)
|
||||
return False
|
||||
|
||||
def send_thermal_print(alert):
|
||||
if not PRINT_ENABLED: return
|
||||
labels = alert.get("labels", {})
|
||||
annotations = alert.get("annotations", {})
|
||||
status = alert.get("status", "firing").upper()
|
||||
summary = annotations.get("summary", "")
|
||||
description = annotations.get("description", "")
|
||||
runbook = annotations.get("runbook", "")
|
||||
# Build a useful message: summary + description + runbook steps
|
||||
parts = []
|
||||
if summary: parts.append(summary)
|
||||
if description and description != summary: parts.append(description)
|
||||
if runbook: parts.append("STEPS: " + runbook)
|
||||
message = " | ".join(parts) if parts else labels.get("alertname", "Unknown alert")
|
||||
payload = {
|
||||
"title": labels.get("alertname", "Unknown"),
|
||||
"severity": labels.get("severity", "warning").capitalize(),
|
||||
"host": labels.get("instance", labels.get("host", "unknown")),
|
||||
"message": message,
|
||||
"eventId": alert.get("fingerprint", ""),
|
||||
"source": "Grafana",
|
||||
"status": "RESOLVED" if status == "RESOLVED" else "PROBLEM",
|
||||
"acknowledged": False
|
||||
}
|
||||
def post_thermal(payload, kind):
|
||||
if not THERMAL_PRINT_ENABLED:
|
||||
print(f"[irc-notify] thermal disabled; skip {kind} ({payload.get('title','?')[:40]})", file=sys.stderr)
|
||||
return False
|
||||
try:
|
||||
req = Request(PRINT_WEB_URL, data=json.dumps(payload).encode("utf-8"),
|
||||
headers={"Content-Type": "application/json"}, method="POST")
|
||||
resp = urlopen(req, timeout=10)
|
||||
print(f"[irc-notify] Thermal print sent: {resp.read().decode()}", file=sys.stderr)
|
||||
if kind == "immediate": _stats["print_immediate"] += 1
|
||||
print(f"[irc-notify] thermal {kind} sent: {payload.get('title','?')[:50]}", file=sys.stderr)
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"[irc-notify] Thermal print failed: {e}", file=sys.stderr)
|
||||
|
||||
def should_print(alert):
|
||||
labels = alert.get("labels", {})
|
||||
if labels.get("alert_channel") == "thermal_print": return True
|
||||
if labels.get("severity", "").lower() in ("critical", "disaster"): return True
|
||||
if alert.get("status", "").upper() == "RESOLVED": return False
|
||||
print(f"[irc-notify] thermal {kind} failed: {e}", file=sys.stderr)
|
||||
return False
|
||||
|
||||
def fingerprint_of(alert):
|
||||
fp = alert.get("fingerprint", "")
|
||||
if fp: return fp
|
||||
labels = alert.get("labels", {})
|
||||
target = labels.get("pod") or labels.get("instance") or labels.get("deployment") or labels.get("statefulset") or labels.get("namespace") or ""
|
||||
return f"{labels.get('alertname','?')}/{labels.get('namespace','')}/{target}"
|
||||
|
||||
def is_critical(alert):
|
||||
return alert.get("labels", {}).get("severity", "").lower() in ("critical", "disaster", "page")
|
||||
|
||||
def is_immediate_label(alert):
|
||||
return alert.get("labels", {}).get("alert_channel") == "thermal_print_immediate"
|
||||
|
||||
def is_batched_label(alert):
|
||||
return alert.get("labels", {}).get("alert_channel") == "thermal_print"
|
||||
|
||||
def add_to_digest(alert):
|
||||
"""Add an alert to the digest buffer. Returns True if the buffer GREW
|
||||
(new fingerprint), False if it was a dedup, resolution, or no-op.
|
||||
"""
|
||||
if not THERMAL_PRINT_ENABLED: return False
|
||||
fp = fingerprint_of(alert)
|
||||
status = alert.get("status", "firing").lower()
|
||||
with _buffer_lock:
|
||||
if status == "resolved":
|
||||
if fp in _buffer:
|
||||
del _buffer[fp]
|
||||
_stats["buffer_resolved"] += 1
|
||||
return False
|
||||
if fp in _buffer:
|
||||
_buffer[fp]["last_seen"] = time.time()
|
||||
_buffer[fp]["alert"] = alert
|
||||
_stats["buffer_dedup"] += 1
|
||||
return False
|
||||
_buffer[fp] = {"alert": alert, "first_seen": time.time(), "last_seen": time.time()}
|
||||
_stats["buffer_added"] += 1
|
||||
return True
|
||||
|
||||
def build_digest_payload():
|
||||
with _buffer_lock:
|
||||
items = list(_buffer.values())
|
||||
if not items: return None
|
||||
by_name = defaultdict(list)
|
||||
for item in items:
|
||||
labels = item["alert"].get("labels", {})
|
||||
by_name[labels.get("alertname", "Unknown")].append(item)
|
||||
lines = []
|
||||
for name, group in sorted(by_name.items()):
|
||||
targets = []
|
||||
for it in group[:5]:
|
||||
labels = it["alert"].get("labels", {})
|
||||
t = (labels.get("pod") or labels.get("instance") or labels.get("deployment")
|
||||
or labels.get("statefulset") or labels.get("namespace") or "?")
|
||||
targets.append(t)
|
||||
more = f" (+{len(group)-5})" if len(group) > 5 else ""
|
||||
sevs = sorted({it["alert"].get("labels", {}).get("severity", "warning") for it in group})
|
||||
lines.append(f"[{'/'.join(sevs)}] {name} x{len(group)}: {', '.join(targets)}{more}")
|
||||
now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
|
||||
title = f"Alert digest: {len(items)} firing"
|
||||
body = "\n".join([
|
||||
f"=== {title} ===",
|
||||
f"as of {now}",
|
||||
"",
|
||||
*lines,
|
||||
"",
|
||||
"Stream: #alerts (IRC) | Triage: grafana-noc1.iamworkin.lan",
|
||||
"Force-flush: POST irc-notify.monitoring.svc:9119/flush",
|
||||
])
|
||||
return {"title": title, "severity": "Warning", "host": "monitoring",
|
||||
"message": body, "eventId": f"digest-{int(time.time())}",
|
||||
"source": "Grafana digest", "status": "PROBLEM", "acknowledged": False}
|
||||
|
||||
def flush_digest():
|
||||
payload = build_digest_payload()
|
||||
if payload is None:
|
||||
print("[irc-notify] flush: buffer empty, no digest sent", file=sys.stderr)
|
||||
return False
|
||||
sent = post_thermal(payload, "digest")
|
||||
with _buffer_lock:
|
||||
_buffer.clear()
|
||||
if sent: _stats["digest_flushed"] += 1
|
||||
return sent
|
||||
|
||||
def digest_loop():
|
||||
global _last_flush_time
|
||||
while True:
|
||||
try:
|
||||
now = time.time()
|
||||
elapsed = now - _last_flush_time
|
||||
if elapsed >= BATCH_INTERVAL_MIN * 60:
|
||||
print(f"[irc-notify] digest tick: interval reached ({BATCH_INTERVAL_MIN}m); buffer={len(_buffer)}", file=sys.stderr)
|
||||
flush_digest()
|
||||
_last_flush_time = now
|
||||
elif len(_buffer) >= BATCH_MAX_PENDING:
|
||||
print(f"[irc-notify] digest tick: buffer full ({len(_buffer)}); force flush", file=sys.stderr)
|
||||
flush_digest()
|
||||
_last_flush_time = now
|
||||
time.sleep(15)
|
||||
except Exception as e:
|
||||
print(f"[irc-notify] digest loop error: {e}", file=sys.stderr)
|
||||
time.sleep(60)
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def do_POST(self):
|
||||
if self.path == "/flush":
|
||||
ok = flush_digest()
|
||||
self.send_response(200); self.send_header("Content-Type", "application/json"); self.end_headers()
|
||||
self.wfile.write(json.dumps({"flushed": ok, "buffer_after": len(_buffer)}).encode())
|
||||
return
|
||||
_stats["webhooks_received"] += 1
|
||||
length = int(self.headers.get("Content-Length", 0))
|
||||
body = json.loads(self.rfile.read(length)) if length else {}
|
||||
for alert in body.get("alerts", []):
|
||||
@@ -1383,22 +1513,56 @@ data:
|
||||
msg = f"{icon}{sev_tag} {name}: {summary}"
|
||||
if desc: msg += f"\n {desc}"
|
||||
send_irc(msg)
|
||||
if should_print(alert): send_thermal_print(alert)
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.end_headers()
|
||||
# Thermal routing — EVERYTHING (including criticals) goes into
|
||||
# the hourly digest. Only the explicit `alert_channel=thermal_print_immediate`
|
||||
# label bypasses, and even that flushes-the-current-digest rather
|
||||
# than printing a standalone job, so the same fingerprint can't
|
||||
# spam the printer per webhook cycle.
|
||||
if status == "RESOLVED":
|
||||
add_to_digest(alert) # removes from buffer
|
||||
continue
|
||||
if is_immediate_label(alert):
|
||||
# Explicit opt-in for "paper this NOW" — first arrival of a
|
||||
# new fingerprint triggers an immediate digest flush; repeat
|
||||
# webhooks for the same fingerprint dedupe in the buffer
|
||||
# until the next interval or until the alert resolves.
|
||||
new_in_buffer = add_to_digest(alert)
|
||||
if new_in_buffer:
|
||||
global _last_flush_time
|
||||
flush_digest()
|
||||
_last_flush_time = time.time()
|
||||
elif is_critical(alert) or is_batched_label(alert):
|
||||
add_to_digest(alert)
|
||||
# else: IRC-only (warnings without thermal_print label)
|
||||
self.send_response(200); self.send_header("Content-Type", "application/json"); self.end_headers()
|
||||
self.wfile.write(b'{"status":"ok"}')
|
||||
|
||||
def do_GET(self):
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.end_headers()
|
||||
self.wfile.write(json.dumps({"service":"irc-notify","thermal_print":PRINT_ENABLED}).encode())
|
||||
self.send_response(200); self.send_header("Content-Type", "application/json"); self.end_headers()
|
||||
with _buffer_lock:
|
||||
alertnames = sorted({it["alert"].get("labels", {}).get("alertname", "?") for it in _buffer.values()})
|
||||
depth = len(_buffer)
|
||||
info = {
|
||||
"service": "irc-notify",
|
||||
"config": {"thermal_print_enabled": THERMAL_PRINT_ENABLED,
|
||||
"batch_interval_min": BATCH_INTERVAL_MIN,
|
||||
"batch_max_pending": BATCH_MAX_PENDING,
|
||||
"irc_target": f"{IRC_HOST}:{IRC_PORT} {IRC_CHANNEL}",
|
||||
"print_web_url": PRINT_WEB_URL},
|
||||
"buffer": {"depth": depth, "alertnames": alertnames,
|
||||
"seconds_since_last_flush": int(time.time() - _last_flush_time),
|
||||
"seconds_until_next_flush": max(0, int(BATCH_INTERVAL_MIN*60 - (time.time() - _last_flush_time)))},
|
||||
"stats": _stats,
|
||||
}
|
||||
self.wfile.write(json.dumps(info, indent=2).encode())
|
||||
|
||||
def log_message(self, format, *args):
|
||||
print(f"[irc-notify] {args[0]}", file=sys.stderr)
|
||||
|
||||
if __name__ == "__main__":
|
||||
threading.Thread(target=digest_loop, daemon=True).start()
|
||||
server = HTTPServer(("0.0.0.0", 9119), Handler)
|
||||
print(f"IRC alert relay :9119 -> {IRC_HOST}:{IRC_PORT} {IRC_CHANNEL} (thermal: {PRINT_ENABLED})")
|
||||
print(f"[irc-notify] :9119 -> IRC {IRC_HOST}:{IRC_PORT} {IRC_CHANNEL} | thermal={'ON' if THERMAL_PRINT_ENABLED else 'OFF'} | digest={BATCH_INTERVAL_MIN}m max={BATCH_MAX_PENDING}", file=sys.stderr)
|
||||
server.serve_forever()
|
||||
|
||||
# =============================================================================
|
||||
@@ -3900,6 +4064,9 @@ spec:
|
||||
- name: dashboards-remotedesktop
|
||||
mountPath: /var/lib/grafana/dashboards/remotedesktop
|
||||
readOnly: true
|
||||
- name: dashboards-signalcontrol
|
||||
mountPath: /var/lib/grafana/dashboards/signalcontrol
|
||||
readOnly: true
|
||||
- name: datasource-provisioning
|
||||
mountPath: /etc/grafana/provisioning/datasources
|
||||
readOnly: true
|
||||
@@ -3953,6 +4120,9 @@ spec:
|
||||
- name: dashboards-remotedesktop
|
||||
configMap:
|
||||
name: grafana-dashboard-remotedesktop
|
||||
- name: dashboards-signalcontrol
|
||||
configMap:
|
||||
name: grafana-dashboard-signalcontrol
|
||||
- name: datasource-provisioning
|
||||
configMap:
|
||||
name: grafana-datasource-provisioning
|
||||
|
||||
@@ -24,7 +24,16 @@
|
||||
# (10.0.57.16:5200), public internet 80/443 (excluding RFC1918), and
|
||||
# fc-signage:5190 for the signage AAT lane.
|
||||
# - Ingress: Traefik (4444 + 8089 ACME-solver-style), intra-pod,
|
||||
# telephony / gitea / fc-system / fc-signage namespaces on 4444.
|
||||
# telephony / gitea / fc-system / fc-signage / github-runner namespaces
|
||||
# on 4444.
|
||||
#
|
||||
# 2026-05-25: added github-runner ingress on 4444 so CI jobs running in
|
||||
# self-hosted runner pods (e.g. FlowerCore.Print.Web `help-screenshots`)
|
||||
# can reach the grid. Without this allow, the session POST to
|
||||
# `selenium-hub.selenium.svc.cluster.local:4444` was DNAT'd to the hub
|
||||
# pod IP and then dropped at the Calico ingress hook — Selenium UI showed
|
||||
# 0/4 sessions while the .NET HTTP client timed out at 60s. Same family
|
||||
# as `feedback_netpol_dnat_backend_port`, wrong-source-namespace flavor.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
@@ -203,6 +212,13 @@ spec:
|
||||
ports:
|
||||
- port: 4444
|
||||
protocol: TCP
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: github-runner
|
||||
ports:
|
||||
- port: 4444
|
||||
protocol: TCP
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
|
||||
427
apps/selenium/selenium-grid.yaml
Normal file
427
apps/selenium/selenium-grid.yaml
Normal file
@@ -0,0 +1,427 @@
|
||||
# Selenium Grid 4 — RKE2 deployment
|
||||
#
|
||||
# Hub + chrome + firefox + edge browser nodes serving fleet-wide AAT runs from
|
||||
# the GitHub Actions self-hosted runners. ArgoCD owns this namespace from
|
||||
# 2026-05-25 (`infra-selenium` Application; previously these resources were
|
||||
# orphan kubectl-applied since 2026-03-15).
|
||||
#
|
||||
# Endpoints:
|
||||
# - Internal cluster: http://selenium-hub.selenium.svc.cluster.local:4444
|
||||
# - LAN LoadBalancer (MetalLB): http://10.0.56.208:4444
|
||||
# - Traefik public: https://selenium.iamworkin.lan
|
||||
#
|
||||
# Browser maxSessions:
|
||||
# - chrome 2 (bumped from 1 on 2026-05-25 morning-routine — AAT-heavy
|
||||
# Print.Web help-screenshots was the global bottleneck;
|
||||
# see commit history for ops/runner-replica-rightsize)
|
||||
# - firefox 1
|
||||
# - edge 1
|
||||
#
|
||||
# Screenshots + video recording write to NFS via the chrome video sidecar.
|
||||
# See: CLAUDE.md "Selenium Grid & Visual AAT Testing" + bluejay-infra ADR notes.
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
labels:
|
||||
app: selenium-hub
|
||||
app.kubernetes.io/name: selenium-hub
|
||||
app.kubernetes.io/part-of: selenium-grid
|
||||
name: selenium-hub
|
||||
namespace: selenium
|
||||
spec:
|
||||
ports:
|
||||
- name: web
|
||||
port: 4444
|
||||
targetPort: 4444
|
||||
- name: publish
|
||||
port: 4442
|
||||
targetPort: 4442
|
||||
- name: subscribe
|
||||
port: 4443
|
||||
targetPort: 4443
|
||||
selector:
|
||||
app: selenium-hub
|
||||
type: ClusterIP
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
annotations:
|
||||
metallb.io/ip-allocated-from-pool: bluejay-pool
|
||||
metallb.universe.tf/loadBalancerIPs: 10.0.56.208
|
||||
labels:
|
||||
app: selenium-hub
|
||||
component: external-access
|
||||
name: selenium-hub-external
|
||||
namespace: selenium
|
||||
spec:
|
||||
clusterIP: 10.43.90.147
|
||||
clusterIPs:
|
||||
- 10.43.90.147
|
||||
externalTrafficPolicy: Local
|
||||
healthCheckNodePort: 32213
|
||||
ports:
|
||||
- name: web
|
||||
nodePort: 32411
|
||||
port: 4444
|
||||
targetPort: 4444
|
||||
- name: publish
|
||||
nodePort: 32068
|
||||
port: 4442
|
||||
targetPort: 4442
|
||||
- name: subscribe
|
||||
nodePort: 31000
|
||||
port: 4443
|
||||
targetPort: 4443
|
||||
selector:
|
||||
app: selenium-hub
|
||||
type: LoadBalancer
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
labels:
|
||||
app: selenium-hub
|
||||
app.kubernetes.io/name: selenium-hub
|
||||
app.kubernetes.io/part-of: selenium-grid
|
||||
name: selenium-hub
|
||||
namespace: selenium
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: selenium-hub
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: selenium-hub
|
||||
app.kubernetes.io/name: selenium-hub
|
||||
app.kubernetes.io/part-of: selenium-grid
|
||||
spec:
|
||||
containers:
|
||||
- env:
|
||||
- name: SE_NODE_SESSION_TIMEOUT
|
||||
value: '300'
|
||||
- name: SE_SESSION_REQUEST_TIMEOUT
|
||||
value: '300'
|
||||
- name: SE_SESSION_RETRY_INTERVAL
|
||||
value: '5'
|
||||
- name: JAVA_OPTS
|
||||
value: -Xmx512m
|
||||
image: selenium/hub:4.27.0
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /wd/hub/status
|
||||
port: 4444
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 15
|
||||
timeoutSeconds: 5
|
||||
name: selenium-hub
|
||||
ports:
|
||||
- containerPort: 4444
|
||||
name: web
|
||||
- containerPort: 4442
|
||||
name: publish
|
||||
- containerPort: 4443
|
||||
name: subscribe
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /wd/hub/status
|
||||
port: 4444
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 5
|
||||
# Hub baseline working set ~766Mi on 2026-05-25 (75% of prior 1Gi
|
||||
# limit). Bump to 1.5Gi / 1Gi to keep ~50% headroom; matches the
|
||||
# stampede-buffer pattern documented for multus
|
||||
# (feedback_k8s_cni_multus_sizing). CPU left alone — observed 54m
|
||||
# against a 500m limit, no contention.
|
||||
resources:
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 1536Mi
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 1Gi
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
labels:
|
||||
app: selenium-node-chrome
|
||||
app.kubernetes.io/name: selenium-node-chrome
|
||||
app.kubernetes.io/part-of: selenium-grid
|
||||
name: selenium-node-chrome
|
||||
namespace: selenium
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: selenium-node-chrome
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: selenium-node-chrome
|
||||
app.kubernetes.io/name: selenium-node-chrome
|
||||
app.kubernetes.io/part-of: selenium-grid
|
||||
spec:
|
||||
containers:
|
||||
- env:
|
||||
- name: SE_EVENT_BUS_HOST
|
||||
value: selenium-hub
|
||||
- name: SE_EVENT_BUS_PUBLISH_PORT
|
||||
value: '4442'
|
||||
- name: SE_EVENT_BUS_SUBSCRIBE_PORT
|
||||
value: '4443'
|
||||
- name: SE_NODE_MAX_SESSIONS
|
||||
value: '2'
|
||||
- name: SE_NODE_OVERRIDE_MAX_SESSIONS
|
||||
value: 'false'
|
||||
- name: SE_VNC_NO_PASSWORD
|
||||
value: '1'
|
||||
- name: SE_SCREEN_WIDTH
|
||||
value: '1920'
|
||||
- name: SE_SCREEN_HEIGHT
|
||||
value: '1080'
|
||||
- name: SE_NODE_SESSION_TIMEOUT
|
||||
value: '300'
|
||||
image: selenium/node-chrome:4.27.0
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /status
|
||||
port: 5555
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 15
|
||||
name: selenium-chrome
|
||||
ports:
|
||||
- containerPort: 5555
|
||||
name: node
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /status
|
||||
port: 5555
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 5
|
||||
# Chromium-based browser node. Bumped from 1Gi -> 2Gi (req 512Mi
|
||||
# -> 1Gi) on 2026-05-25 — Edge had 51 OOMKills in 5d on the
|
||||
# original 1Gi cap (~1 OOM every 2.4h), and Chrome at maxSessions=2
|
||||
# was running 684Mi idle on the same cap. Matches the Firefox node's
|
||||
# tested-stable 2Gi limit. CPU unchanged.
|
||||
resources:
|
||||
limits:
|
||||
cpu: '1'
|
||||
memory: 2Gi
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 1Gi
|
||||
volumeMounts:
|
||||
- mountPath: /dev/shm
|
||||
name: dshm
|
||||
- env:
|
||||
- name: DISPLAY_CONTAINER_NAME
|
||||
value: localhost
|
||||
- name: SE_SCREEN_WIDTH
|
||||
value: '1920'
|
||||
- name: SE_SCREEN_HEIGHT
|
||||
value: '1080'
|
||||
- name: SE_VIDEO_FILE_NAME
|
||||
value: auto
|
||||
- name: SE_VIDEO_UPLOAD_ENABLED
|
||||
value: 'false'
|
||||
image: selenium/video:ffmpeg-7.1-20250101
|
||||
name: video
|
||||
resources:
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 768Mi
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 384Mi
|
||||
volumeMounts:
|
||||
- mountPath: /videos
|
||||
name: selenium-videos
|
||||
volumes:
|
||||
- emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 2Gi
|
||||
name: dshm
|
||||
- emptyDir:
|
||||
sizeLimit: 5Gi
|
||||
name: selenium-videos
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
labels:
|
||||
app: selenium-node-firefox
|
||||
app.kubernetes.io/name: selenium-node-firefox
|
||||
app.kubernetes.io/part-of: selenium-grid
|
||||
name: selenium-node-firefox
|
||||
namespace: selenium
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: selenium-node-firefox
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: selenium-node-firefox
|
||||
app.kubernetes.io/name: selenium-node-firefox
|
||||
app.kubernetes.io/part-of: selenium-grid
|
||||
spec:
|
||||
containers:
|
||||
- env:
|
||||
- name: SE_EVENT_BUS_HOST
|
||||
value: selenium-hub
|
||||
- name: SE_EVENT_BUS_PUBLISH_PORT
|
||||
value: '4442'
|
||||
- name: SE_EVENT_BUS_SUBSCRIBE_PORT
|
||||
value: '4443'
|
||||
- name: SE_NODE_MAX_SESSIONS
|
||||
value: '1'
|
||||
- name: SE_NODE_OVERRIDE_MAX_SESSIONS
|
||||
value: 'true'
|
||||
- name: SE_VNC_NO_PASSWORD
|
||||
value: '1'
|
||||
- name: SE_START_VNC
|
||||
value: 'false'
|
||||
- name: SE_SCREEN_WIDTH
|
||||
value: '1920'
|
||||
- name: SE_SCREEN_HEIGHT
|
||||
value: '1080'
|
||||
- name: SE_NODE_SESSION_TIMEOUT
|
||||
value: '300'
|
||||
image: selenium/node-firefox:4.27.0
|
||||
livenessProbe:
|
||||
failureThreshold: 5
|
||||
httpGet:
|
||||
path: /status
|
||||
port: 5555
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 15
|
||||
timeoutSeconds: 5
|
||||
name: selenium-firefox
|
||||
ports:
|
||||
- containerPort: 5555
|
||||
name: node
|
||||
readinessProbe:
|
||||
failureThreshold: 5
|
||||
httpGet:
|
||||
path: /status
|
||||
port: 5555
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 5
|
||||
resources:
|
||||
limits:
|
||||
cpu: '1'
|
||||
memory: 2Gi
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 1Gi
|
||||
volumeMounts:
|
||||
- mountPath: /dev/shm
|
||||
name: dshm
|
||||
volumes:
|
||||
- emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 2Gi
|
||||
name: dshm
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
labels:
|
||||
app: selenium-node-edge
|
||||
app.kubernetes.io/name: selenium-node-edge
|
||||
app.kubernetes.io/part-of: selenium-grid
|
||||
name: selenium-node-edge
|
||||
namespace: selenium
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: selenium-node-edge
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: selenium-node-edge
|
||||
app.kubernetes.io/name: selenium-node-edge
|
||||
app.kubernetes.io/part-of: selenium-grid
|
||||
spec:
|
||||
containers:
|
||||
- env:
|
||||
- name: SE_EVENT_BUS_HOST
|
||||
value: selenium-hub
|
||||
- name: SE_EVENT_BUS_PUBLISH_PORT
|
||||
value: '4442'
|
||||
- name: SE_EVENT_BUS_SUBSCRIBE_PORT
|
||||
value: '4443'
|
||||
- name: SE_NODE_MAX_SESSIONS
|
||||
value: '1'
|
||||
- name: SE_NODE_OVERRIDE_MAX_SESSIONS
|
||||
value: 'true'
|
||||
- name: SE_VNC_NO_PASSWORD
|
||||
value: '1'
|
||||
- name: SE_SCREEN_WIDTH
|
||||
value: '1920'
|
||||
- name: SE_SCREEN_HEIGHT
|
||||
value: '1080'
|
||||
- name: SE_NODE_SESSION_TIMEOUT
|
||||
value: '300'
|
||||
image: selenium/node-edge:4.27.0
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /status
|
||||
port: 5555
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 15
|
||||
name: selenium-edge
|
||||
ports:
|
||||
- containerPort: 5555
|
||||
name: node
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /status
|
||||
port: 5555
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 5
|
||||
# Chromium-based browser node. Bumped from 1Gi -> 2Gi (req 512Mi
|
||||
# -> 1Gi) on 2026-05-25 — Edge had 51 OOMKills in 5d on the
|
||||
# original 1Gi cap (~1 OOM every 2.4h), and Chrome at maxSessions=2
|
||||
# was running 684Mi idle on the same cap. Matches the Firefox node's
|
||||
# tested-stable 2Gi limit. CPU unchanged.
|
||||
resources:
|
||||
limits:
|
||||
cpu: '1'
|
||||
memory: 2Gi
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 1Gi
|
||||
volumeMounts:
|
||||
- mountPath: /dev/shm
|
||||
name: dshm
|
||||
volumes:
|
||||
- emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 2Gi
|
||||
name: dshm
|
||||
---
|
||||
apiVersion: traefik.io/v1alpha1
|
||||
kind: IngressRoute
|
||||
metadata:
|
||||
name: selenium-hub
|
||||
namespace: selenium
|
||||
spec:
|
||||
entryPoints:
|
||||
- websecure
|
||||
routes:
|
||||
- kind: Rule
|
||||
match: Host(`selenium.iamworkin.lan`)
|
||||
services:
|
||||
- name: selenium-hub
|
||||
port: 4444
|
||||
tls:
|
||||
secretName: selenium-tls
|
||||
@@ -67,6 +67,7 @@ public sealed class FleetManifestLintTests
|
||||
["github-runner-chat"] = "https://github.com/astoltz/FlowerCore.Chat",
|
||||
["github-runner-mysql"] = "https://github.com/astoltz/FlowerCore.MySQL",
|
||||
["github-runner-kiosk-linux"] = "https://github.com/astoltz/FlowerCore.Kiosk.Linux",
|
||||
["github-runner-updater"] = "https://github.com/astoltz/FlowerCore.Updater",
|
||||
};
|
||||
|
||||
private static readonly HashSet<string> ScaledLinuxRunnerDeployments = new(StringComparer.Ordinal)
|
||||
@@ -80,6 +81,7 @@ public sealed class FleetManifestLintTests
|
||||
"github-runner-chat",
|
||||
"github-runner-mysql",
|
||||
"github-runner-kiosk-linux",
|
||||
"github-runner-updater",
|
||||
};
|
||||
|
||||
private static readonly IReadOnlyDictionary<string, string> WritableRunnerEnv = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
@@ -234,7 +236,7 @@ public sealed class FleetManifestLintTests
|
||||
{
|
||||
deployments.Should().ContainKey(expectedRunner.Key);
|
||||
|
||||
var container = deployments[expectedRunner.Key].ContainerMappings().Should().ContainSingle().Subject;
|
||||
var container = deployments[expectedRunner.Key].MainContainerMappings().Should().ContainSingle().Subject;
|
||||
EnvValue(container, "REPO_URL").Should().Be(expectedRunner.Value);
|
||||
EnvValue(container, "EPHEMERAL").Should().Be("true");
|
||||
EnvValue(container, "LABELS").Should().Be("self-hosted,linux,fc-build-linux");
|
||||
@@ -250,7 +252,7 @@ public sealed class FleetManifestLintTests
|
||||
{
|
||||
foreach (var deployment in GitHubRunnerDeployments().Values)
|
||||
{
|
||||
var container = deployment.ContainerMappings().Should().ContainSingle().Subject;
|
||||
var container = deployment.MainContainerMappings().Should().ContainSingle().Subject;
|
||||
|
||||
foreach (var expectedEnv in WritableRunnerEnv)
|
||||
{
|
||||
@@ -277,7 +279,10 @@ public sealed class FleetManifestLintTests
|
||||
foreach (var deploymentName in ScaledLinuxRunnerDeployments)
|
||||
{
|
||||
var deployment = deployments[deploymentName];
|
||||
ReplicaCount(deployment).Should().Be(2);
|
||||
// Scaled runners must have >= 2 replicas (avoid single-pod bottleneck).
|
||||
// Individual deployments may be tuned upward per CI activity — see
|
||||
// "runners: right-size replica counts per 14d CI activity (#24)".
|
||||
ReplicaCount(deployment).Should().BeGreaterOrEqualTo(2, $"{deploymentName} is in the scaled set and must run with at least 2 replicas");
|
||||
|
||||
var volumes = deployment.MappingSequence("spec", "template", "spec", "volumes");
|
||||
var claimNames = volumes
|
||||
@@ -303,6 +308,108 @@ public sealed class FleetManifestLintTests
|
||||
.Be("github-runner-nuget-cache");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Runners_MustNotPinToOperatorWorkstationHosts()
|
||||
{
|
||||
// CRITICAL SAFETY (operator directive 2026-05-26): BLUEJAY-WS is the
|
||||
// operator's primary workstation — host of the 1Password Connect
|
||||
// bearer token, fcadmin SSH keys to noc1, signing CA private keys,
|
||||
// and source for every FC repo. A self-hosted GitHub Actions runner
|
||||
// there would execute arbitrary PR code with that local access.
|
||||
// Build-side analog of the Sprint 9 NEW safe-account exclusion gate
|
||||
// (Puppet GPO/AppLocker/WDAC/audit-forwarder modules refuse to apply
|
||||
// on BLUEJAY-WS). This lint asserts no GitHub-runner Deployment in
|
||||
// apps/github-runner/ pins to a forbidden operator-workstation host
|
||||
// via nodeName, nodeSelector, nodeAffinity, or tolerations.
|
||||
// Existing legacy `bluejay-ws-sandbox-1` GitHub-registered runner is
|
||||
// out of scope here (it's a runtime registration, not a K8s
|
||||
// Deployment) — see CLAUDE.md "Common Mistakes" entry and
|
||||
// feedback_bluejay_ws_never_public_runner.md.
|
||||
var forbiddenHostPatterns = new[]
|
||||
{
|
||||
"bluejay-ws",
|
||||
"BLUEJAY-WS",
|
||||
"bluejay-ws.iamworkin.lan",
|
||||
"iamworkin-ws",
|
||||
};
|
||||
|
||||
bool ContainsForbidden(string? value) =>
|
||||
!string.IsNullOrWhiteSpace(value)
|
||||
&& forbiddenHostPatterns.Any(pattern => value!.Contains(pattern, StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
var violations = GitHubRunnerDeployments().Values.SelectMany(deployment =>
|
||||
{
|
||||
var local = new List<string>();
|
||||
var podSpec = ManifestNodeExtensions.Mapping(deployment.Root, "spec", "template", "spec");
|
||||
if (podSpec is null)
|
||||
{
|
||||
return local;
|
||||
}
|
||||
|
||||
// nodeName: pins the pod to a specific node by name.
|
||||
var nodeName = ManifestNodeExtensions.Scalar(podSpec, "nodeName");
|
||||
if (ContainsForbidden(nodeName))
|
||||
{
|
||||
local.Add($"{deployment.Name} sets nodeName='{nodeName}' which targets a forbidden operator-workstation host.");
|
||||
}
|
||||
|
||||
// nodeSelector: dict of label → value pinning the pod to nodes
|
||||
// carrying matching labels. Examples that would trip this:
|
||||
// kubernetes.io/hostname: bluejay-ws
|
||||
// flowercore.io/host: bluejay-ws.iamworkin.lan
|
||||
var nodeSelector = ManifestNodeExtensions.Mapping(podSpec, "nodeSelector");
|
||||
if (nodeSelector is not null)
|
||||
{
|
||||
foreach (var entry in nodeSelector.Children)
|
||||
{
|
||||
var key = entry.Key is YamlScalarNode keyScalar ? keyScalar.Value : null;
|
||||
var value = entry.Value is YamlScalarNode valueScalar ? valueScalar.Value : null;
|
||||
if (ContainsForbidden(value))
|
||||
{
|
||||
local.Add($"{deployment.Name} has nodeSelector entry '{key}: {value}' which targets a forbidden operator-workstation host.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// nodeAffinity: matchExpressions over node labels.
|
||||
foreach (var term in ManifestNodeExtensions.MappingSequence(podSpec, "affinity", "nodeAffinity", "requiredDuringSchedulingIgnoredDuringExecution", "nodeSelectorTerms"))
|
||||
{
|
||||
foreach (var expr in ManifestNodeExtensions.MappingSequence(term, "matchExpressions"))
|
||||
{
|
||||
var key = ManifestNodeExtensions.Scalar(expr, "key");
|
||||
foreach (var valueNode in ManifestNodeExtensions.ScalarSequence(expr, "values"))
|
||||
{
|
||||
if (ContainsForbidden(valueNode))
|
||||
{
|
||||
local.Add($"{deployment.Name} has nodeAffinity matchExpression '{key}' value '{valueNode}' which targets a forbidden operator-workstation host.");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// tolerations: scheduling onto a tainted operator-workstation
|
||||
// node would let the runner run there. Forbid any toleration
|
||||
// value that names the workstation.
|
||||
foreach (var toleration in ManifestNodeExtensions.MappingSequence(podSpec, "tolerations"))
|
||||
{
|
||||
var key = ManifestNodeExtensions.Scalar(toleration, "key");
|
||||
var value = ManifestNodeExtensions.Scalar(toleration, "value");
|
||||
if (ContainsForbidden(key))
|
||||
{
|
||||
local.Add($"{deployment.Name} has toleration key '{key}' which targets a forbidden operator-workstation host.");
|
||||
}
|
||||
if (ContainsForbidden(value))
|
||||
{
|
||||
local.Add($"{deployment.Name} has toleration value '{value}' which targets a forbidden operator-workstation host.");
|
||||
}
|
||||
}
|
||||
|
||||
return local;
|
||||
}).ToList();
|
||||
|
||||
violations.Should().BeEmpty("BLUEJAY-WS / iamworkin-ws must never host a fleet GitHub Actions runner; see CLAUDE.md 'Registering BLUEJAY-WS as a fleet GitHub Actions runner' and feedback_bluejay_ws_never_public_runner.md");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Monitoring_MustAlertWhenLinuxRunnerDeploymentIsUnavailable()
|
||||
{
|
||||
@@ -890,6 +997,22 @@ internal sealed record ManifestDocument(
|
||||
.ToList();
|
||||
}
|
||||
|
||||
// MainContainerMappings excludes initContainers. Use this when asserting
|
||||
// properties of the primary container (env, image, volumeMounts) where an
|
||||
// initContainer would be a false-positive match — e.g. the GitHub runner
|
||||
// image's `setup-runner-home` initContainer should not count toward the
|
||||
// single-container assertions on the runner deployments.
|
||||
public IReadOnlyList<YamlMappingNode> MainContainerMappings()
|
||||
{
|
||||
var podSpec = PodSpec();
|
||||
if (podSpec is null)
|
||||
{
|
||||
return Array.Empty<YamlMappingNode>();
|
||||
}
|
||||
|
||||
return ManifestNodeExtensions.MappingSequence(podSpec, "containers").ToList();
|
||||
}
|
||||
|
||||
public IReadOnlyList<ContainerSpec> ContainerSpecs()
|
||||
{
|
||||
return ContainerMappings()
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
using FluentAssertions;
|
||||
using Xunit;
|
||||
|
||||
namespace BluejayInfraLint.Tests;
|
||||
|
||||
[Trait("Category", "Unit")]
|
||||
public sealed class SignalControlPlatformManifestTests
|
||||
{
|
||||
private static readonly string Root = ManifestInventory.Load().BluejayRoot;
|
||||
|
||||
[Fact]
|
||||
public void Monitoring_PrometheusScrapesSignalControlPiAppAndPirelayNodeExporter()
|
||||
{
|
||||
var monitoring = File.ReadAllText(Path.Combine(Root, "apps", "monitoring", "noc-monitoring.yaml"));
|
||||
|
||||
monitoring.Should().Contain("job_name: \"signalcontrol-pi-app\"");
|
||||
monitoring.Should().Contain("metrics_path: /metrics/prometheus");
|
||||
monitoring.Should().Contain("targets: [\"10.0.58.113:5200\"]");
|
||||
monitoring.Should().Contain("host: \"signal-a.iamworkin.lan\"");
|
||||
monitoring.Should().Contain("targets: [\"10.0.58.113:9100\"]");
|
||||
monitoring.Should().Contain("instance: \"pirelay\"");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Monitoring_GrafanaMountsSignalControlDashboard()
|
||||
{
|
||||
var monitoring = File.ReadAllText(Path.Combine(Root, "apps", "monitoring", "noc-monitoring.yaml"));
|
||||
var dashboard = File.ReadAllText(Path.Combine(Root, "apps", "monitoring", "grafana-dashboard-signalcontrol.yaml"));
|
||||
|
||||
monitoring.Should().Contain("name: dashboards-signalcontrol");
|
||||
monitoring.Should().Contain("mountPath: /var/lib/grafana/dashboards/signalcontrol");
|
||||
monitoring.Should().Contain("name: grafana-dashboard-signalcontrol");
|
||||
dashboard.Should().Contain("\"uid\": \"flowercore-signalcontrol\"");
|
||||
dashboard.Should().Contain("signalcontrol_active_pattern");
|
||||
dashboard.Should().Contain("signal_relay_writes_total");
|
||||
dashboard.Should().Contain("node_cpu_seconds_total");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FcSignalControlReadme_DocumentsMtlsTelemetryAndDefaultOffAudit()
|
||||
{
|
||||
var readme = File.ReadAllText(Path.Combine(Root, "apps", "fc-signalcontrol", "README.md"));
|
||||
|
||||
readme.Should().Contain("step-ca-agent");
|
||||
readme.Should().Contain("10.0.58.113:5200");
|
||||
readme.Should().Contain("10.0.58.113:9100");
|
||||
readme.Should().Contain("PhysicalAudit:Enabled=false");
|
||||
readme.Should().Contain("ForwardingEnabled=false");
|
||||
readme.Should().Contain("Secrets, enrollment codes, private keys");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user