Compare commits
1 Commits
sprint37/c
...
783720cc58
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
783720cc58 |
@@ -118,7 +118,6 @@ That test project sweeps `bluejay-infra/apps/**` plus the canonical sibling `Flo
|
||||
|
||||
## References
|
||||
|
||||
- OpenVox noc1 durability runbook: `docs/runbooks/openvoxserver-quadlet-durability.md`
|
||||
- Cert-manager recovery playbook: `FlowerCore.Notes/memory/project_cert_manager_recovery_2026_04_22.md`
|
||||
- Why pfSense DNS is required: `FlowerCore.Notes/memory/feedback_pfsense_dns_required_for_acme.md`
|
||||
- Public DNS operator host: `https://dns.iamworkin.lan`
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
# FlowerCore Brochure
|
||||
|
||||
`apps/brochure` hosts the public brochure split from `FlowerCore.Intranet.Web`.
|
||||
ArgoCD's `apps/*` ApplicationSet will create `infra-brochure` after this
|
||||
directory lands on `main`.
|
||||
|
||||
## Runtime
|
||||
|
||||
- Host: `https://brochure.flowercore.io`
|
||||
- Namespace: `brochure`
|
||||
- Deployment: `brochure-web`
|
||||
- Image: `localhost/fc-brochure-web:v20260524-sprint32`
|
||||
- Port: `8080`
|
||||
- Public route method allowlist: `GET` and `HEAD`
|
||||
|
||||
## Operator Actions
|
||||
|
||||
1. Publish and import `localhost/fc-brochure-web:v20260524-sprint32` to every
|
||||
RKE2 node before sync, using the same podman save + `ctr images import`
|
||||
flow as the Intranet deployment.
|
||||
2. Create the Cloudflare DNS record for `brochure.flowercore.io` pointing at
|
||||
the FlowerCore public edge.
|
||||
3. Verify `infra-brochure` appears in ArgoCD, the certificate becomes Ready,
|
||||
and `GET https://brochure.flowercore.io/` returns `200`.
|
||||
|
||||
The route intentionally does not expose `/ops/*` or `/admin/*`; the Brochure
|
||||
web app returns `404` for those paths and Traefik only forwards read methods.
|
||||
@@ -1,131 +0,0 @@
|
||||
# FlowerCore Brochure public host
|
||||
#
|
||||
# Thin Blazor host for public What's New, walkthrough, and gallery content
|
||||
# carved out of FlowerCore.Intranet.Web. The ApplicationSet creates
|
||||
# infra-brochure from this directory after merge.
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: brochure
|
||||
labels:
|
||||
app.kubernetes.io/part-of: flowercore
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: brochure-web
|
||||
namespace: brochure
|
||||
labels:
|
||||
app: brochure-web
|
||||
app.kubernetes.io/name: brochure-web
|
||||
app.kubernetes.io/part-of: flowercore
|
||||
spec:
|
||||
replicas: 1
|
||||
revisionHistoryLimit: 3
|
||||
selector:
|
||||
matchLabels:
|
||||
app: brochure-web
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: brochure-web
|
||||
app.kubernetes.io/name: brochure-web
|
||||
app.kubernetes.io/part-of: flowercore
|
||||
spec:
|
||||
containers:
|
||||
- name: brochure-web
|
||||
image: localhost/fc-brochure-web:v20260524-sprint32
|
||||
imagePullPolicy: Never
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
name: http
|
||||
env:
|
||||
- name: ASPNETCORE_ENVIRONMENT
|
||||
value: Production
|
||||
- name: ASPNETCORE_URLS
|
||||
value: "http://+:8080"
|
||||
resources:
|
||||
requests:
|
||||
cpu: "25m"
|
||||
memory: "128Mi"
|
||||
limits:
|
||||
cpu: "500m"
|
||||
memory: "512Mi"
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1654
|
||||
runAsGroup: 1654
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
volumeMounts:
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
volumes:
|
||||
- name: tmp
|
||||
emptyDir: {}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: brochure-web
|
||||
namespace: brochure
|
||||
labels:
|
||||
app: brochure-web
|
||||
app.kubernetes.io/name: brochure-web
|
||||
app.kubernetes.io/part-of: flowercore
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: brochure-web
|
||||
ports:
|
||||
- name: http
|
||||
port: 8080
|
||||
targetPort: http
|
||||
---
|
||||
apiVersion: cert-manager.io/v1
|
||||
kind: Certificate
|
||||
metadata:
|
||||
name: brochure-web-tls
|
||||
namespace: brochure
|
||||
spec:
|
||||
secretName: brochure-web-tls
|
||||
issuerRef:
|
||||
name: step-ca-acme
|
||||
kind: ClusterIssuer
|
||||
dnsNames:
|
||||
- brochure.flowercore.io
|
||||
duration: 720h
|
||||
renewBefore: 240h
|
||||
---
|
||||
apiVersion: traefik.io/v1alpha1
|
||||
kind: IngressRoute
|
||||
metadata:
|
||||
name: brochure-web-public
|
||||
namespace: brochure
|
||||
spec:
|
||||
entryPoints:
|
||||
- websecure
|
||||
routes:
|
||||
- match: Host(`brochure.flowercore.io`) && (Method(`GET`) || Method(`HEAD`))
|
||||
kind: Rule
|
||||
services:
|
||||
- name: brochure-web
|
||||
port: 8080
|
||||
tls:
|
||||
secretName: brochure-web-tls
|
||||
@@ -30,41 +30,3 @@ spec:
|
||||
port: 80
|
||||
tls:
|
||||
secretName: chat-web-tls
|
||||
---
|
||||
# Public host profile marker. The app treats this header as authoritative for
|
||||
# the public twin, while the internal chat.iamworkin.lan route does not attach
|
||||
# it and keeps the operator-oriented UI.
|
||||
apiVersion: traefik.io/v1alpha1
|
||||
kind: Middleware
|
||||
metadata:
|
||||
name: chat-public-profile-header
|
||||
namespace: fc-chat
|
||||
spec:
|
||||
headers:
|
||||
customRequestHeaders:
|
||||
X-FC-Chat-Host-Profile: "public"
|
||||
---
|
||||
# Public Cloudflare-fronted twin for the anonymous chat surface. Operator
|
||||
# paths are intentionally absent from the allowlist below, so /admin,
|
||||
# /operator, /console, /ops, /api/operator, and /operatorhub miss this route
|
||||
# and return Traefik 404 before reaching the pod. Operator action still needed:
|
||||
# create/verify Cloudflare DNS chat.flowercore.io -> public Traefik endpoint
|
||||
# and mirror the cf-origin-flowercore-io TLS secret into namespace fc-chat.
|
||||
apiVersion: traefik.io/v1alpha1
|
||||
kind: IngressRoute
|
||||
metadata:
|
||||
name: chat-web-public
|
||||
namespace: fc-chat
|
||||
spec:
|
||||
entryPoints:
|
||||
- websecure
|
||||
routes:
|
||||
- match: Host(`chat.flowercore.io`) && (Path(`/`) || Path(`/chat`) || PathPrefix(`/_blazor`) || PathPrefix(`/_framework`) || PathPrefix(`/_content`) || PathPrefix(`/avatars`) || PathPrefix(`/css`) || PathPrefix(`/js`) || PathPrefix(`/favicon`) || PathPrefix(`/chathub`)) && (Method(`GET`) || Method(`HEAD`) || Method(`POST`) || Method(`OPTIONS`))
|
||||
kind: Rule
|
||||
middlewares:
|
||||
- name: chat-public-profile-header
|
||||
services:
|
||||
- name: chat-web
|
||||
port: 80
|
||||
tls:
|
||||
secretName: cf-origin-flowercore-io
|
||||
|
||||
@@ -532,7 +532,7 @@ spec:
|
||||
fsGroupChangePolicy: OnRootMismatch
|
||||
containers:
|
||||
- name: web
|
||||
image: localhost/fc-ttsreader-web:v20260518-sprint36-demo-finish-b132cbf
|
||||
image: localhost/fc-ttsreader-web:v20260506-phase6
|
||||
imagePullPolicy: Never
|
||||
ports:
|
||||
- containerPort: 5217
|
||||
@@ -555,13 +555,9 @@ spec:
|
||||
- name: TtsReader__Jobs__Root
|
||||
value: "/data/jobs"
|
||||
- name: TtsReader__Piper__Host
|
||||
value: "10.0.57.17"
|
||||
value: "ttsreader-piper.fc-ttsreader.svc.cluster.local."
|
||||
- name: TtsReader__Piper__Port
|
||||
value: "8500"
|
||||
- name: TtsReader__Piper__Transport
|
||||
value: "http"
|
||||
- name: TtsReader__Piper__HttpPath
|
||||
value: "/tts"
|
||||
value: "10200"
|
||||
- name: TtsReader__Kokoro__Enabled
|
||||
value: "true"
|
||||
- name: TtsReader__Kokoro__BaseUrl
|
||||
|
||||
@@ -15,22 +15,9 @@ All repo-scoped Linux runners use:
|
||||
Actions tool cache
|
||||
|
||||
`github-runner` for `FlowerCore.Common` is single-replica because it retains the
|
||||
original Longhorn ReadWriteOnce NuGet PVC. Every other repo-scoped runner uses
|
||||
two replicas with per-pod `emptyDir` caches. That is the safe backlog-drain
|
||||
strategy: no two pods share one RWO PVC.
|
||||
|
||||
Sprint 32 final long-tail wave adds 16 two-replica Deployments:
|
||||
`FlowerCore.Knowledge`, `FlowerCore.LlmBridge`, `FlowerCore.Media`,
|
||||
`FlowerCore.Presentations`, `FlowerCore.RemoteDesktop`, `FlowerCore.DNS`,
|
||||
`FlowerCore.Distribution`, `FlowerCore.Scoreboard`,
|
||||
`FlowerCore.SegmentDisplay`, `FlowerCore.Signage.Contracts`,
|
||||
`FlowerCore.SignalControl`, `FlowerCore.Intranet.Web`,
|
||||
`FlowerCore.Provisioning`, `FlowerCore.Redis`, `FlowerCore.MessageBoard`, and
|
||||
`FlowerCore.MenuBoard`.
|
||||
|
||||
Sprint 37 Cx-2 closes the audited Linux runner gaps for
|
||||
`FlowerCore.DeviceManagement` and `FlowerCore.WorldBuilder` with the same
|
||||
two-replica `emptyDir` pattern.
|
||||
original Longhorn ReadWriteOnce NuGet PVC. `github-runner-sharedpos` and the top
|
||||
Linux-cost repo runners use two replicas with per-pod `emptyDir` caches. That is
|
||||
the safe backlog-drain strategy: no two pods share one RWO PVC.
|
||||
|
||||
## Post-Merge Proof
|
||||
|
||||
@@ -45,13 +32,7 @@ Verify GitHub registration for the repo-scoped runners:
|
||||
```bash
|
||||
for repo in FlowerCore.Common FlowerCore.Shared.Pos FlowerCore.Puppet FlowerCore.Signage \
|
||||
FlowerCore.DMS FlowerCore.Telephony FlowerCore.Print.Web FlowerCore.Chat \
|
||||
FlowerCore.MySQL FlowerCore.Kiosk.Linux FlowerCore.Marquee FlowerCore.TtsReader \
|
||||
FlowerCore.Knowledge FlowerCore.LlmBridge FlowerCore.Media \
|
||||
FlowerCore.Presentations FlowerCore.RemoteDesktop FlowerCore.DNS \
|
||||
FlowerCore.Distribution FlowerCore.Scoreboard FlowerCore.SegmentDisplay \
|
||||
FlowerCore.Signage.Contracts FlowerCore.SignalControl FlowerCore.Intranet.Web \
|
||||
FlowerCore.Provisioning FlowerCore.Redis FlowerCore.MessageBoard \
|
||||
FlowerCore.MenuBoard FlowerCore.DeviceManagement FlowerCore.WorldBuilder; do
|
||||
FlowerCore.MySQL FlowerCore.Kiosk.Linux; do
|
||||
echo "=== $repo ==="
|
||||
gh api "/repos/astoltz/$repo/actions/runners" \
|
||||
--jq '.runners[] | select(.labels[].name == "fc-build-linux") | {name,status,busy,labels:[.labels[].name]}'
|
||||
@@ -68,20 +49,6 @@ gh run list --repo astoltz/FlowerCore.Shared.Pos \
|
||||
If the latest run is still queued after runner registration, rerun the workflow
|
||||
from GitHub Actions and verify it lands on an `rke2-linux-*` runner.
|
||||
|
||||
## Sprint 37 Cx-2 Gap Audit
|
||||
|
||||
The 2026-05-18 GitHub workflow scan found these remaining repos with
|
||||
`runs-on: [self-hosted, linux, fc-build-linux]` but no K8s runner Deployment:
|
||||
`FlowerCore.AiStation.Linux`, `FlowerCore.PHP`, `FlowerCore.PiManager`,
|
||||
`FlowerCore.Shared.Barcodes`, `FlowerCore.Shared.Lookup`,
|
||||
`FlowerCore.Shared.Nodes`, `FlowerCore.Shared.PrintClient`,
|
||||
`FlowerCore.Shared.Relay`, `FlowerCore.Shared.ShowRunner`, and
|
||||
`FlowerCore.Shared.Storage`.
|
||||
|
||||
Mixed/platform repos also have Linux workflow legs but need owner review before
|
||||
adding Linux runner Deployments: `FlowerCore.Library.Mac`,
|
||||
`FlowerCore.Signage.Agent.AppleTv`, and `FlowerCore.Signage.Player.Wpf`.
|
||||
|
||||
## Failure Notes
|
||||
|
||||
- `actions/setup-dotnet` permission error at `/usr/share/dotnet`: check that
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -729,7 +729,7 @@ data:
|
||||
expr: |
|
||||
kube_deployment_status_replicas_ready{
|
||||
namespace="github-runner",
|
||||
deployment=~"github-runner(|-.+)"
|
||||
deployment=~"github-runner(|-(sharedpos|puppet|signage|dms|telephony|print-web|chat|mysql|kiosk-linux))"
|
||||
} == 0
|
||||
for: 5m
|
||||
labels:
|
||||
@@ -966,52 +966,6 @@ data:
|
||||
annotations:
|
||||
summary: "Disk usage high on {{ $labels.instance }} ({{ $value | printf \"%.1f\" }}%)"
|
||||
|
||||
# Puppet agent + service alerts.
|
||||
# Mirror of FlowerCore.Notes/scripts/monitoring/alerts.yml `puppet` group
|
||||
# so a future migration to in-cluster Prometheus inherits the ruleset.
|
||||
# Source-of-truth for the live Podman Prometheus on noc1 is the Notes file.
|
||||
# See feedback_monitoring_k8s_target_vs_live_podman.
|
||||
- name: puppet
|
||||
rules:
|
||||
- alert: PuppetAgentReportStale
|
||||
expr: puppet_last_run_age_seconds > 7200
|
||||
for: 30m
|
||||
labels:
|
||||
severity: warning
|
||||
alert_channel: irc
|
||||
annotations:
|
||||
summary: "Puppet agent {{ $labels.instance }} hasn't reported in over 2h"
|
||||
description: "Last run age: {{ $value | humanizeDuration }}. The puppet agent on {{ $labels.instance }} may be stopped, the node may be powered off, or noc1 may be unreachable from this node."
|
||||
runbook: "1. SSH to node (via noc1 jumpbox if needed) 2. sudo systemctl status puppet 3. sudo puppet agent -t --noop to force a run 4. Check r10k: ssh fcadmin@10.0.56.10 'sudo podman logs openvoxserver --tail 50' 5. Verify noc1 reachability: ping puppet.iamworkin.lan"
|
||||
|
||||
- alert: PuppetAgentReportCritical
|
||||
expr: puppet_last_run_age_seconds > 86400
|
||||
for: 1h
|
||||
labels:
|
||||
severity: critical
|
||||
alert_channel: irc
|
||||
annotations:
|
||||
summary: "Puppet agent {{ $labels.instance }} silent for over 24h — node is unmanaged"
|
||||
description: "Last run age: {{ $value | humanizeDuration }}. Node {{ $labels.instance }} has not submitted a Puppet report in over 24 hours. Config drift is accumulating — investigate immediately. If intentional (maintenance), add to the exclusion filter or silence in Grafana."
|
||||
runbook: "URGENT: 1. Check node power state 2. SSH via noc1 jumpbox: ssh fcadmin@10.0.56.10 then ssh <node> 3. sudo systemctl status puppet 4. sudo systemctl start puppet + sudo puppet agent -t 5. Check for network partitions (VLAN connectivity to 10.0.56.10) 6. If node was recently reimaged: sudo puppet agent -t to re-register with new SSL cert"
|
||||
|
||||
# Sprint 33 Cx-7 Phase B (2026-05-25 postmortem follow-up):
|
||||
# Detects puppet.service in failed state — distinct from PuppetAgentReportStale
|
||||
# which catches "agent hasn't run." This catches "systemd gave up restarting it"
|
||||
# (CA-verify loop or other fatal exit). Requires node-exporter systemd collector
|
||||
# enabled with --collector.systemd. If `node_systemd_unit_state` has no series
|
||||
# for a node, the collector is disabled there — flag in postmortem follow-up.
|
||||
- alert: PuppetServiceFailed
|
||||
expr: node_systemd_unit_state{name="puppet.service",state="failed"} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
alert_channel: irc
|
||||
annotations:
|
||||
summary: "Puppet service failed on {{ $labels.instance }}"
|
||||
description: "puppet.service on {{ $labels.instance }} has been in failed state for 5+ minutes. systemd has stopped auto-restarting (CA-verify-loop or other exit). Manual `systemctl status puppet` confirms. Run `sudo systemctl start puppet` to recover; investigate journal for root cause."
|
||||
runbook_url: "https://github.com/astoltz/FlowerCore.Notes/blob/master/memory/feedback_puppet_service_dead_after_ca_loop_alert_misreads.md"
|
||||
|
||||
# K8s pod-state alerts. Require kube-state-metrics scrape (added
|
||||
# 2026-04-26 — see scrape_configs above). Would have surfaced the
|
||||
# agent-zero ollama-proxy 172x crash-loop instead of letting it
|
||||
@@ -3509,7 +3463,7 @@ data:
|
||||
- refId: A
|
||||
relativeTimeRange: {from: 300, to: 0}
|
||||
datasourceUid: prometheus
|
||||
model: {expr: 'kube_deployment_status_replicas_ready{namespace="github-runner",deployment=~"github-runner(|-.+)"} == 0', instant: true, refId: A}
|
||||
model: {expr: 'kube_deployment_status_replicas_ready{namespace="github-runner",deployment=~"github-runner(|-(sharedpos|puppet|signage|dms|telephony|print-web|chat|mysql|kiosk-linux))"} == 0', instant: true, refId: A}
|
||||
- refId: B
|
||||
relativeTimeRange: {from: 300, to: 0}
|
||||
datasourceUid: __expr__
|
||||
|
||||
@@ -28,12 +28,9 @@ Source: `D:\git\FlowerCore\FlowerCore.WorldBuilder` (master)
|
||||
Memory: `feedback_rke2_image_import_per_node_scp`.
|
||||
3. **Bump image tag** in `worldbuilder.yaml` and git push.
|
||||
ArgoCD ApplicationSet picks up within ~3 minutes.
|
||||
4. **First production render** — open
|
||||
`https://worldbuilder.iamworkin.lan/studio/c32e0000-0000-4000-8000-000000000004`
|
||||
and confirm the Cyberpunk Blue Jay demo prompt loads with five seeded fake
|
||||
generated images. This Sprint 32 visitor-safe profile uses
|
||||
`ClientMode=fake`; switch the image-generation env vars back to ComfyUI only
|
||||
for an operator-owned GPU render lane.
|
||||
4. **First production render** — open `https://worldbuilder.iamworkin.lan`,
|
||||
create World → Character → Storyboard → ExportJob, confirm artifact
|
||||
downloads. ComfyUI lives on BLUEJAY-WS at `http://10.0.56.20:8188`.
|
||||
|
||||
## Health probes
|
||||
|
||||
@@ -56,13 +53,8 @@ Source: `D:\git\FlowerCore\FlowerCore.WorldBuilder` (master)
|
||||
|
||||
## Image generation backend
|
||||
|
||||
Sprint 32 pins the Kubernetes profile to
|
||||
`FlowerCore:WorldBuilder:ImageGeneration:ClientMode=fake` with
|
||||
`BaseUrl=http://127.0.0.1:1`. That keeps the public/internal visitor demo
|
||||
deterministic, avoids GPU exposure, and still exercises the studio/gallery
|
||||
surface with persisted generated-image metadata.
|
||||
|
||||
The previous ComfyUI backend target was `http://10.0.56.20:8188` on
|
||||
BLUEJAY-WS (R9700 / gfx1201 / ROCm 7.2.1). Re-enable it only in an
|
||||
operator-owned follow-up that also verifies workstation reachability and image
|
||||
import freshness.
|
||||
`FlowerCore:WorldBuilder:ImageGeneration:BaseUrl=http://10.0.56.20:8188` —
|
||||
ComfyUI runs on BLUEJAY-WS Windows (R9700 / gfx1201 / ROCm 7.2.1). Pod reaches
|
||||
the workstation directly across the 10.0.56.0/24 VLAN (no Podman-style host-
|
||||
filter issues — K8s pods route via Calico, which is L3-routed across the
|
||||
VLAN).
|
||||
|
||||
@@ -16,11 +16,7 @@ kind: Namespace
|
||||
metadata:
|
||||
name: fc-worldbuilder
|
||||
labels:
|
||||
app.kubernetes.io/name: fc-worldbuilder
|
||||
app.kubernetes.io/part-of: flowercore
|
||||
app.kubernetes.io/managed-by: argocd
|
||||
flowercore.io/tenant-id: system
|
||||
flowercore.io/created-by: bluejay-infra
|
||||
---
|
||||
# SQLite DB + generated image gallery + PDF/PNG exports.
|
||||
# Longhorn RWO — single replica with `Recreate` rollout strategy keeps it safe.
|
||||
@@ -29,13 +25,6 @@ kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: worldbuilder-data
|
||||
namespace: fc-worldbuilder
|
||||
labels:
|
||||
app.kubernetes.io/name: worldbuilder-data
|
||||
app.kubernetes.io/component: storage
|
||||
app.kubernetes.io/part-of: flowercore
|
||||
app.kubernetes.io/managed-by: argocd
|
||||
flowercore.io/tenant-id: system
|
||||
flowercore.io/created-by: bluejay-infra
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
@@ -51,13 +40,7 @@ metadata:
|
||||
namespace: fc-worldbuilder
|
||||
labels:
|
||||
app.kubernetes.io/name: worldbuilder-web
|
||||
app.kubernetes.io/component: web
|
||||
app.kubernetes.io/part-of: flowercore
|
||||
app.kubernetes.io/managed-by: argocd
|
||||
flowercore.io/tenant-id: system
|
||||
flowercore.io/created-by: bluejay-infra
|
||||
annotations:
|
||||
flowercore.io/traceability-standard: k8s-pod-ownership-and-traceability-standard
|
||||
spec:
|
||||
replicas: 1
|
||||
revisionHistoryLimit: 3
|
||||
@@ -71,16 +54,11 @@ spec:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: worldbuilder-web
|
||||
app.kubernetes.io/component: web
|
||||
app.kubernetes.io/part-of: flowercore
|
||||
app.kubernetes.io/managed-by: argocd
|
||||
flowercore.io/tenant-id: system
|
||||
flowercore.io/created-by: bluejay-infra
|
||||
annotations:
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "8080"
|
||||
prometheus.io/path: "/metrics/prometheus"
|
||||
flowercore.io/audit-trace-id: "worldbuilder-runtime-demo"
|
||||
spec:
|
||||
securityContext:
|
||||
fsGroup: 1654
|
||||
@@ -114,14 +92,11 @@ spec:
|
||||
value: "/data/gallery"
|
||||
- name: FlowerCore__WorldBuilder__Export__RootPath
|
||||
value: "/data/exports"
|
||||
# Visitor-safe Sprint 32 profile: fake backend keeps public demo
|
||||
# rendering deterministic and avoids exposing BLUEJAY-WS GPU.
|
||||
# ComfyUI on BLUEJAY-WS (R9700 / gfx1201 / ROCm 7.2.1).
|
||||
- name: FlowerCore__WorldBuilder__ImageGeneration__BaseUrl
|
||||
value: "http://127.0.0.1:1"
|
||||
value: "http://10.0.56.20:8188"
|
||||
- name: FlowerCore__WorldBuilder__ImageGeneration__ClientMode
|
||||
value: "fake"
|
||||
- name: FlowerCore__WorldBuilder__ImageGeneration__BackendId
|
||||
value: "fake"
|
||||
value: "comfyui"
|
||||
resources:
|
||||
# Cluster CPU-request budget runs hot (99% on all 3 nodes at deploy
|
||||
# time) while actual CPU usage is well below capacity. Idle Blazor
|
||||
@@ -190,11 +165,7 @@ metadata:
|
||||
namespace: fc-worldbuilder
|
||||
labels:
|
||||
app.kubernetes.io/name: worldbuilder-web
|
||||
app.kubernetes.io/component: web
|
||||
app.kubernetes.io/part-of: flowercore
|
||||
app.kubernetes.io/managed-by: argocd
|
||||
flowercore.io/tenant-id: system
|
||||
flowercore.io/created-by: bluejay-infra
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
@@ -209,13 +180,6 @@ kind: Certificate
|
||||
metadata:
|
||||
name: worldbuilder-web-tls
|
||||
namespace: fc-worldbuilder
|
||||
labels:
|
||||
app.kubernetes.io/name: worldbuilder-web-tls
|
||||
app.kubernetes.io/component: ingress
|
||||
app.kubernetes.io/part-of: flowercore
|
||||
app.kubernetes.io/managed-by: argocd
|
||||
flowercore.io/tenant-id: system
|
||||
flowercore.io/created-by: bluejay-infra
|
||||
spec:
|
||||
secretName: worldbuilder-web-tls
|
||||
issuerRef:
|
||||
@@ -236,13 +200,6 @@ kind: IngressRoute
|
||||
metadata:
|
||||
name: worldbuilder-web
|
||||
namespace: fc-worldbuilder
|
||||
labels:
|
||||
app.kubernetes.io/name: worldbuilder-web
|
||||
app.kubernetes.io/component: ingress
|
||||
app.kubernetes.io/part-of: flowercore
|
||||
app.kubernetes.io/managed-by: argocd
|
||||
flowercore.io/tenant-id: system
|
||||
flowercore.io/created-by: bluejay-infra
|
||||
spec:
|
||||
entryPoints:
|
||||
- websecure
|
||||
|
||||
@@ -1,84 +0,0 @@
|
||||
# openvoxserver Quadlet Durability
|
||||
|
||||
This runbook documents the noc1 `openvoxserver` durability fix for the Puppet control-repo deploy path. The service is a noc1 host artifact, not an ArgoCD application, so discovery always starts on noc1 rather than in `apps/*`.
|
||||
|
||||
## Current State
|
||||
|
||||
As of the Sprint 32 Cx-12 apply on 2026-05-17:
|
||||
|
||||
- `/etc/containers/systemd/openvoxserver.container` has a `GIT_SSH_COMMAND` environment entry that points at the persisted serverdata deploy key.
|
||||
- `/etc/systemd/system/openvoxserver-safeconfig.service` is enabled and active, and reapplies `git config --global --add safe.directory *` inside the running container.
|
||||
- `/opt/puppet/r10k-deploy.sh` self-heals before each fetch by setting `safe.directory`, the repo-local `core.sshCommand`, and the persisted `known_hosts` file when needed.
|
||||
- `puppet-deploy.service` exits `0/SUCCESS` after the apply and the control repo reports `HEAD == origin/master`.
|
||||
- `systemctl cat openvoxserver` does not currently resolve to a generated unit on noc1. The container is running through Podman with `restart=always`, so destructive recreate smoke must not run until the generated unit is present.
|
||||
|
||||
## Discovery
|
||||
|
||||
Run every command through noc1 as `fcadmin`; do not assume BLUEJAY-WS can reach container-local surfaces directly.
|
||||
|
||||
```bash
|
||||
ssh -i ~/.ssh/fcadmin_ed25519 fcadmin@10.0.56.10 "hostname && sudo -n true"
|
||||
ssh -i ~/.ssh/fcadmin_ed25519 fcadmin@10.0.56.10 "sudo find /etc/containers/systemd /usr/share/containers/systemd /etc/systemd/system -name 'openvoxserver*' 2>/dev/null"
|
||||
ssh -i ~/.ssh/fcadmin_ed25519 fcadmin@10.0.56.10 "sudo sed -n '1,220p' /etc/containers/systemd/openvoxserver.container"
|
||||
ssh -i ~/.ssh/fcadmin_ed25519 fcadmin@10.0.56.10 "sudo systemctl cat puppet-deploy.service"
|
||||
```
|
||||
|
||||
If a future noc1 profile manages these files, update the Puppet control repo and let `puppet-deploy.service` apply the change. On 2026-05-17, host `puppet` was not installed, so Cx-12 used a direct noc1 host edit.
|
||||
|
||||
## Durable Fix Shape
|
||||
|
||||
The Quadlet keeps the deploy key as a path reference only:
|
||||
|
||||
```ini
|
||||
Environment=GIT_SSH_COMMAND=ssh -i /opt/puppetlabs/server/data/puppetserver/.puppet-deploy-key -o StrictHostKeyChecking=yes -o IdentitiesOnly=yes -o UserKnownHostsFile=/opt/puppetlabs/server/data/puppetserver/.known_hosts
|
||||
```
|
||||
|
||||
The safeconfig service is intentionally independent of `openvoxserver.service` until the generated unit exists. It waits for the `openvoxserver` container name and then runs:
|
||||
|
||||
```bash
|
||||
/usr/bin/podman exec openvoxserver git config --global --add safe.directory *
|
||||
```
|
||||
|
||||
The deploy script self-heals inside the container before it fetches the control repo:
|
||||
|
||||
```bash
|
||||
git config --global --add safe.directory "*" 2>/dev/null || true
|
||||
DEPLOY_KEY="/opt/puppetlabs/server/data/puppetserver/.puppet-deploy-key"
|
||||
KNOWN_HOSTS="/opt/puppetlabs/server/data/puppetserver/.known_hosts"
|
||||
REPO="/etc/puppetlabs/code/environments/production"
|
||||
export GIT_SSH_COMMAND="ssh -i $DEPLOY_KEY -o StrictHostKeyChecking=yes -o IdentitiesOnly=yes -o UserKnownHostsFile=$KNOWN_HOSTS"
|
||||
git -C "$REPO" config core.sshCommand "$GIT_SSH_COMMAND" 2>/dev/null || true
|
||||
```
|
||||
|
||||
## Validation
|
||||
|
||||
Non-destructive validation:
|
||||
|
||||
```bash
|
||||
ssh -i ~/.ssh/fcadmin_ed25519 fcadmin@10.0.56.10 "sudo grep -n 'GIT_SSH_COMMAND' /etc/containers/systemd/openvoxserver.container"
|
||||
ssh -i ~/.ssh/fcadmin_ed25519 fcadmin@10.0.56.10 "sudo systemctl status openvoxserver-safeconfig.service --no-pager -l"
|
||||
ssh -i ~/.ssh/fcadmin_ed25519 fcadmin@10.0.56.10 "sudo systemctl start puppet-deploy.service && sudo systemctl status puppet-deploy.service --no-pager -l"
|
||||
ssh -i ~/.ssh/fcadmin_ed25519 fcadmin@10.0.56.10 "sudo podman exec openvoxserver git -C /etc/puppetlabs/code/environments/production config --get core.sshCommand"
|
||||
```
|
||||
|
||||
Destructive recreate smoke is opt-in only:
|
||||
|
||||
```bash
|
||||
scp scripts/monitoring/openvox-recreate-smoke.sh fcadmin@10.0.56.10:/tmp/openvox-recreate-smoke.sh
|
||||
ssh -i ~/.ssh/fcadmin_ed25519 fcadmin@10.0.56.10 "chmod +x /tmp/openvox-recreate-smoke.sh && sudo OPENVOX_RECREATE_SMOKE=1 /tmp/openvox-recreate-smoke.sh"
|
||||
```
|
||||
|
||||
Do not run the smoke during normal sprint work. It stops and removes the production container before starting it again through systemd, and it now refuses to continue unless `systemctl cat openvoxserver` succeeds.
|
||||
|
||||
## Credential Rotation Note
|
||||
|
||||
When rotating the Puppet deploy key, update the persisted serverdata copy on noc1:
|
||||
|
||||
```bash
|
||||
sudo install -m 0600 -o root -g root <new-deploy-key> /opt/puppet/serverdata/.puppet-deploy-key
|
||||
sudo podman exec openvoxserver sh -c "ssh-keyscan github.com > /opt/puppetlabs/server/data/puppetserver/.known_hosts"
|
||||
sudo systemctl start openvoxserver-safeconfig.service
|
||||
sudo systemctl start puppet-deploy.service
|
||||
```
|
||||
|
||||
Never commit the deploy key or print it in logs.
|
||||
@@ -1,48 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if [ "${OPENVOX_RECREATE_SMOKE:-}" != "1" ]; then
|
||||
echo "SKIP: set OPENVOX_RECREATE_SMOKE=1 to run the destructive openvoxserver recreate smoke." >&2
|
||||
exit 64
|
||||
fi
|
||||
|
||||
SUDO="${SUDO:-sudo}"
|
||||
REPO="/etc/puppetlabs/code/environments/production"
|
||||
CORE_SSH_COMMAND_FRAGMENT=".puppet-deploy-key"
|
||||
|
||||
if ! $SUDO systemctl cat openvoxserver >/dev/null 2>&1; then
|
||||
echo "SKIP: systemctl cat openvoxserver failed; refusing to remove a container without a verified systemd recreate path." >&2
|
||||
exit 65
|
||||
fi
|
||||
|
||||
before="$($SUDO podman exec openvoxserver git -C "$REPO" rev-parse --short HEAD)"
|
||||
echo "Before recreate: $before"
|
||||
|
||||
$SUDO systemctl stop openvoxserver
|
||||
$SUDO podman rm openvoxserver 2>/dev/null || true
|
||||
$SUDO systemctl start openvoxserver
|
||||
|
||||
sleep 50
|
||||
|
||||
$SUDO systemctl start puppet-deploy.service
|
||||
sleep 5
|
||||
|
||||
$SUDO systemctl status puppet-deploy.service --no-pager -l
|
||||
|
||||
after="$($SUDO podman exec openvoxserver git -C "$REPO" rev-parse --short origin/master)"
|
||||
echo "After recreate origin/master: $after"
|
||||
|
||||
$SUDO test -d /opt/puppet/code/environments/production/site-modules/profile/manifests
|
||||
|
||||
core_ssh="$($SUDO podman exec openvoxserver git -C "$REPO" config --get core.sshCommand)"
|
||||
case "$core_ssh" in
|
||||
*"$CORE_SSH_COMMAND_FRAGMENT"*) ;;
|
||||
*)
|
||||
echo "FAIL: core.sshCommand does not reference the persisted deploy key." >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
$SUDO podman exec openvoxserver git -C "$REPO" status --short --branch
|
||||
|
||||
echo "PASS: openvoxserver recreate smoke completed without git safety or deploy-key failure."
|
||||
@@ -13,7 +13,6 @@ public sealed class FleetManifestLintTests
|
||||
|
||||
private static readonly HashSet<string> PublicReadOnlyHosts = new(StringComparer.Ordinal)
|
||||
{
|
||||
"brochure.flowercore.io",
|
||||
"dist.flowercore.io",
|
||||
"dns.iamworkin.lan",
|
||||
};
|
||||
@@ -67,8 +66,6 @@ public sealed class FleetManifestLintTests
|
||||
["github-runner-chat"] = "https://github.com/astoltz/FlowerCore.Chat",
|
||||
["github-runner-mysql"] = "https://github.com/astoltz/FlowerCore.MySQL",
|
||||
["github-runner-kiosk-linux"] = "https://github.com/astoltz/FlowerCore.Kiosk.Linux",
|
||||
["github-runner-devicemgmt"] = "https://github.com/astoltz/FlowerCore.DeviceManagement",
|
||||
["github-runner-worldbuilder"] = "https://github.com/astoltz/FlowerCore.WorldBuilder",
|
||||
};
|
||||
|
||||
private static readonly HashSet<string> ScaledLinuxRunnerDeployments = new(StringComparer.Ordinal)
|
||||
@@ -82,8 +79,6 @@ public sealed class FleetManifestLintTests
|
||||
"github-runner-chat",
|
||||
"github-runner-mysql",
|
||||
"github-runner-kiosk-linux",
|
||||
"github-runner-devicemgmt",
|
||||
"github-runner-worldbuilder",
|
||||
};
|
||||
|
||||
private static readonly IReadOnlyDictionary<string, string> WritableRunnerEnv = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
@@ -238,7 +233,7 @@ public sealed class FleetManifestLintTests
|
||||
{
|
||||
deployments.Should().ContainKey(expectedRunner.Key);
|
||||
|
||||
var container = RunnerContainer(deployments[expectedRunner.Key]);
|
||||
var container = deployments[expectedRunner.Key].ContainerMappings().Should().ContainSingle().Subject;
|
||||
EnvValue(container, "REPO_URL").Should().Be(expectedRunner.Value);
|
||||
EnvValue(container, "EPHEMERAL").Should().Be("true");
|
||||
EnvValue(container, "LABELS").Should().Be("self-hosted,linux,fc-build-linux");
|
||||
@@ -254,7 +249,7 @@ public sealed class FleetManifestLintTests
|
||||
{
|
||||
foreach (var deployment in GitHubRunnerDeployments().Values)
|
||||
{
|
||||
var container = RunnerContainer(deployment);
|
||||
var container = deployment.ContainerMappings().Should().ContainSingle().Subject;
|
||||
|
||||
foreach (var expectedEnv in WritableRunnerEnv)
|
||||
{
|
||||
@@ -315,7 +310,7 @@ public sealed class FleetManifestLintTests
|
||||
monitoring.Should().Contain("MacMiniRunnerOffline");
|
||||
monitoring.Should().Contain("LinuxRunnerOffline");
|
||||
monitoring.Should().Contain("kube_deployment_status_replicas_ready");
|
||||
monitoring.Should().Contain("github-runner(|-.+)");
|
||||
monitoring.Should().Contain("github-runner(|-(sharedpos|puppet|signage|dms|telephony|print-web|chat|mysql|kiosk-linux))");
|
||||
monitoring.Should().Contain("folder: CI Alerts");
|
||||
monitoring.Should().Contain("uid: linux-runner-offline");
|
||||
monitoring.Should().Contain("alert_channel: irc");
|
||||
@@ -645,15 +640,6 @@ public sealed class FleetManifestLintTests
|
||||
return EnvMapping(container, name) is { } env ? ManifestNodeExtensions.Scalar(env, "value") : null;
|
||||
}
|
||||
|
||||
private static YamlMappingNode RunnerContainer(ManifestDocument deployment)
|
||||
{
|
||||
return deployment.ContainerMappings()
|
||||
.Where(container => string.Equals(ManifestNodeExtensions.Scalar(container, "name"), "runner", StringComparison.Ordinal))
|
||||
.Should()
|
||||
.ContainSingle($"{deployment.Name} must keep exactly one main runner container")
|
||||
.Subject;
|
||||
}
|
||||
|
||||
private static string? EnvSecretName(YamlMappingNode container, string name)
|
||||
{
|
||||
return EnvMapping(container, name) is { } env
|
||||
|
||||
@@ -1,99 +0,0 @@
|
||||
using FluentAssertions;
|
||||
using Xunit;
|
||||
|
||||
namespace BluejayInfraLint.Tests;
|
||||
|
||||
[Trait("Category", "Unit")]
|
||||
public sealed class OpenVoxServerDurabilityTests
|
||||
{
|
||||
private static readonly string Root = FindRepoRoot();
|
||||
private static readonly string RunbookPath = Path.Combine(Root, "docs", "runbooks", "openvoxserver-quadlet-durability.md");
|
||||
private static readonly string SmokePath = Path.Combine(Root, "scripts", "monitoring", "openvox-recreate-smoke.sh");
|
||||
|
||||
[Fact]
|
||||
public void Runbook_DocumentsHostArtifactAndNonArgoPath()
|
||||
{
|
||||
var runbook = File.ReadAllText(RunbookPath);
|
||||
|
||||
runbook.Should().Contain("noc1 host artifact");
|
||||
runbook.Should().Contain("not an ArgoCD application");
|
||||
runbook.Should().Contain("systemctl cat openvoxserver");
|
||||
runbook.Should().Contain("/etc/containers/systemd/openvoxserver.container");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Runbook_DocumentsCx12LiveApplyState()
|
||||
{
|
||||
var runbook = File.ReadAllText(RunbookPath);
|
||||
|
||||
runbook.Should().Contain("Sprint 32 Cx-12");
|
||||
runbook.Should().Contain("openvoxserver-safeconfig.service");
|
||||
runbook.Should().Contain("/opt/puppet/r10k-deploy.sh");
|
||||
runbook.Should().Contain("HEAD == origin/master");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SmokeScript_IsExplicitlyOptIn()
|
||||
{
|
||||
var smoke = File.ReadAllText(SmokePath);
|
||||
|
||||
smoke.Should().Contain("OPENVOX_RECREATE_SMOKE");
|
||||
smoke.Should().Contain("exit 64");
|
||||
smoke.IndexOf("OPENVOX_RECREATE_SMOKE", StringComparison.Ordinal)
|
||||
.Should().BeLessThan(smoke.IndexOf("systemctl stop openvoxserver", StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SmokeScript_RequiresGeneratedSystemdUnitBeforeRemovingContainer()
|
||||
{
|
||||
var smoke = File.ReadAllText(SmokePath);
|
||||
|
||||
smoke.Should().Contain("systemctl cat openvoxserver");
|
||||
smoke.Should().Contain("refusing to remove a container without a verified systemd recreate path");
|
||||
smoke.IndexOf("systemctl cat openvoxserver", StringComparison.Ordinal)
|
||||
.Should().BeLessThan(smoke.IndexOf("podman rm openvoxserver", StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Artifacts_DoNotStoreSecretsOrPaidRunnerLabels()
|
||||
{
|
||||
var forbidden = new[]
|
||||
{
|
||||
"BEGIN OPENSSH PRIVATE KEY",
|
||||
"BEGIN RSA PRIVATE KEY",
|
||||
"ubuntu-latest",
|
||||
"windows-latest",
|
||||
"macos-latest",
|
||||
};
|
||||
|
||||
var violations = new[] { RunbookPath, SmokePath }
|
||||
.SelectMany(path =>
|
||||
{
|
||||
var text = File.ReadAllText(path);
|
||||
return forbidden
|
||||
.Where(token => text.Contains(token, StringComparison.OrdinalIgnoreCase))
|
||||
.Select(token => $"{Path.GetRelativePath(Root, path)} contains forbidden token {token}");
|
||||
})
|
||||
.ToList();
|
||||
|
||||
violations.Should().BeEmpty();
|
||||
}
|
||||
|
||||
private static string FindRepoRoot()
|
||||
{
|
||||
var current = new DirectoryInfo(AppContext.BaseDirectory);
|
||||
while (current is not null)
|
||||
{
|
||||
if (Directory.Exists(Path.Combine(current.FullName, "apps"))
|
||||
&& Directory.Exists(Path.Combine(current.FullName, "scripts"))
|
||||
&& File.Exists(Path.Combine(current.FullName, "README.md")))
|
||||
{
|
||||
return current.FullName;
|
||||
}
|
||||
|
||||
current = current.Parent;
|
||||
}
|
||||
|
||||
throw new DirectoryNotFoundException("Could not find bluejay-infra root.");
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
package bluejayinfra.public_method_allowlist
|
||||
|
||||
public_hosts := {"brochure.flowercore.io", "dist.flowercore.io", "dns.iamworkin.lan"}
|
||||
public_hosts := {"dist.flowercore.io", "dns.iamworkin.lan"}
|
||||
|
||||
deny[msg] {
|
||||
input.kind == "IngressRoute"
|
||||
|
||||
Reference in New Issue
Block a user