Compare commits
1 Commits
sprint40/c
...
783720cc58
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
783720cc58 |
@@ -118,7 +118,6 @@ That test project sweeps `bluejay-infra/apps/**` plus the canonical sibling `Flo
|
|||||||
|
|
||||||
## References
|
## References
|
||||||
|
|
||||||
- OpenVox noc1 durability runbook: `docs/runbooks/openvoxserver-quadlet-durability.md`
|
|
||||||
- Cert-manager recovery playbook: `FlowerCore.Notes/memory/project_cert_manager_recovery_2026_04_22.md`
|
- Cert-manager recovery playbook: `FlowerCore.Notes/memory/project_cert_manager_recovery_2026_04_22.md`
|
||||||
- Why pfSense DNS is required: `FlowerCore.Notes/memory/feedback_pfsense_dns_required_for_acme.md`
|
- Why pfSense DNS is required: `FlowerCore.Notes/memory/feedback_pfsense_dns_required_for_acme.md`
|
||||||
- Public DNS operator host: `https://dns.iamworkin.lan`
|
- Public DNS operator host: `https://dns.iamworkin.lan`
|
||||||
|
|||||||
@@ -30,41 +30,3 @@ spec:
|
|||||||
port: 80
|
port: 80
|
||||||
tls:
|
tls:
|
||||||
secretName: chat-web-tls
|
secretName: chat-web-tls
|
||||||
---
|
|
||||||
# Public host profile marker. The app treats this header as authoritative for
|
|
||||||
# the public twin, while the internal chat.iamworkin.lan route does not attach
|
|
||||||
# it and keeps the operator-oriented UI.
|
|
||||||
apiVersion: traefik.io/v1alpha1
|
|
||||||
kind: Middleware
|
|
||||||
metadata:
|
|
||||||
name: chat-public-profile-header
|
|
||||||
namespace: fc-chat
|
|
||||||
spec:
|
|
||||||
headers:
|
|
||||||
customRequestHeaders:
|
|
||||||
X-FC-Chat-Host-Profile: "public"
|
|
||||||
---
|
|
||||||
# Public Cloudflare-fronted twin for the anonymous chat surface. Operator
|
|
||||||
# paths are intentionally absent from the allowlist below, so /admin,
|
|
||||||
# /operator, /console, /ops, /api/operator, and /operatorhub miss this route
|
|
||||||
# and return Traefik 404 before reaching the pod. Operator action still needed:
|
|
||||||
# create/verify Cloudflare DNS chat.flowercore.io -> public Traefik endpoint
|
|
||||||
# and mirror the cf-origin-flowercore-io TLS secret into namespace fc-chat.
|
|
||||||
apiVersion: traefik.io/v1alpha1
|
|
||||||
kind: IngressRoute
|
|
||||||
metadata:
|
|
||||||
name: chat-web-public
|
|
||||||
namespace: fc-chat
|
|
||||||
spec:
|
|
||||||
entryPoints:
|
|
||||||
- websecure
|
|
||||||
routes:
|
|
||||||
- match: Host(`chat.flowercore.io`) && (Path(`/`) || Path(`/chat`) || PathPrefix(`/_blazor`) || PathPrefix(`/_framework`) || PathPrefix(`/_content`) || PathPrefix(`/avatars`) || PathPrefix(`/css`) || PathPrefix(`/js`) || PathPrefix(`/favicon`) || PathPrefix(`/chathub`)) && (Method(`GET`) || Method(`HEAD`) || Method(`POST`) || Method(`OPTIONS`))
|
|
||||||
kind: Rule
|
|
||||||
middlewares:
|
|
||||||
- name: chat-public-profile-header
|
|
||||||
services:
|
|
||||||
- name: chat-web
|
|
||||||
port: 80
|
|
||||||
tls:
|
|
||||||
secretName: cf-origin-flowercore-io
|
|
||||||
|
|||||||
@@ -1,24 +0,0 @@
|
|||||||
# FlowerCore Remote Desktop - Namespace ResourceQuota (GitOps-managed)
|
|
||||||
#
|
|
||||||
# Codifies the live cap applied on 2026-05-19 after disabling automatic
|
|
||||||
# desktop pool prewarm: no more than 15 pods and no more than 8 CPU / 16Gi
|
|
||||||
# memory requested across the fc-desktop namespace.
|
|
||||||
#
|
|
||||||
# ArgoCD adoption note: this manifest uses the same kind/name/namespace as
|
|
||||||
# the live ResourceQuota and avoids hook, force, or replace annotations, so
|
|
||||||
# automated sync should patch/adopt the existing object in place instead of
|
|
||||||
# recreating it.
|
|
||||||
apiVersion: v1
|
|
||||||
kind: ResourceQuota
|
|
||||||
metadata:
|
|
||||||
name: fc-desktop-cap
|
|
||||||
namespace: fc-desktop
|
|
||||||
labels:
|
|
||||||
app.kubernetes.io/part-of: remotedesktop
|
|
||||||
app.kubernetes.io/component: capacity-guard
|
|
||||||
app.kubernetes.io/managed-by: argocd
|
|
||||||
spec:
|
|
||||||
hard:
|
|
||||||
count/pods: "15"
|
|
||||||
cpu: "8"
|
|
||||||
memory: 16Gi
|
|
||||||
@@ -47,7 +47,7 @@ spec:
|
|||||||
fsGroupChangePolicy: OnRootMismatch
|
fsGroupChangePolicy: OnRootMismatch
|
||||||
containers:
|
containers:
|
||||||
- name: operator
|
- name: operator
|
||||||
image: localhost/fc-devicemgmt-operator:v20260519-sp34cl3-fix
|
image: localhost/fc-devicemgmt-operator:v20260512-cx5
|
||||||
imagePullPolicy: Never
|
imagePullPolicy: Never
|
||||||
ports:
|
ports:
|
||||||
- name: metrics
|
- name: metrics
|
||||||
|
|||||||
@@ -4,22 +4,6 @@
|
|||||||
# Sprint 9+ lane. This manifest is static-valid without requiring the image to
|
# Sprint 9+ lane. This manifest is static-valid without requiring the image to
|
||||||
# exist yet; import localhost/fc-devicemgmt-web:<tag> to all schedulable RKE2
|
# exist yet; import localhost/fc-devicemgmt-web:<tag> to all schedulable RKE2
|
||||||
# nodes before letting ArgoCD sync a live rollout.
|
# nodes before letting ArgoCD sync a live rollout.
|
||||||
#
|
|
||||||
# SCALED TO 0 — 2026-05-19 morning-routine cleanup.
|
|
||||||
# The Web pod cannot start until TWO upstream gaps close:
|
|
||||||
# 1. MySQL DB instance `flowercore_devicemgmt` (user `fc_devicemgmt`) is
|
|
||||||
# provisioned via fc-mysql Manager. The cluster currently has ZERO
|
|
||||||
# MySqlInstanceCrds and no `mysql.fc-mysql.svc:3306` Service, so the
|
|
||||||
# deployment-web container env `FlowerCore__Database__Host=mysql.fc-mysql.svc`
|
|
||||||
# points at nothing. Provision via the fc-mysql Manager UI/REST/MCP.
|
|
||||||
# 2. 1Password vault item `IAmWorkin/FlowerCore DeviceManagement Runtime`
|
|
||||||
# with 5 fields (DB-Password, mtls-ca.pem, mtls-client.crt, mtls-client.key,
|
|
||||||
# mtls-chain.pem) — see apps/fc-devicemgmt/1password-item.yaml. Mint mTLS
|
|
||||||
# from step-ca-agent ClusterIssuer per ADR-126; DB-Password must match the
|
|
||||||
# password configured for the MySQL user.
|
|
||||||
# Re-enable: change replicas back to 2 after both gaps close. The image tag
|
|
||||||
# in this file (v20260512-cx5) MAY also need a refresh — it predates the
|
|
||||||
# Sprint 34 Cl-3 operator fix; Web may have an analogous bug.
|
|
||||||
apiVersion: apps/v1
|
apiVersion: apps/v1
|
||||||
kind: Deployment
|
kind: Deployment
|
||||||
metadata:
|
metadata:
|
||||||
@@ -36,7 +20,7 @@ metadata:
|
|||||||
annotations:
|
annotations:
|
||||||
flowercore.io/traceability-standard: k8s-pod-ownership-and-traceability-standard
|
flowercore.io/traceability-standard: k8s-pod-ownership-and-traceability-standard
|
||||||
spec:
|
spec:
|
||||||
replicas: 0
|
replicas: 2
|
||||||
revisionHistoryLimit: 3
|
revisionHistoryLimit: 3
|
||||||
selector:
|
selector:
|
||||||
matchLabels:
|
matchLabels:
|
||||||
|
|||||||
@@ -532,7 +532,7 @@ spec:
|
|||||||
fsGroupChangePolicy: OnRootMismatch
|
fsGroupChangePolicy: OnRootMismatch
|
||||||
containers:
|
containers:
|
||||||
- name: web
|
- name: web
|
||||||
image: localhost/fc-ttsreader-web:v20260518-sprint36-demo-finish-b132cbf
|
image: localhost/fc-ttsreader-web:v20260506-phase6
|
||||||
imagePullPolicy: Never
|
imagePullPolicy: Never
|
||||||
ports:
|
ports:
|
||||||
- containerPort: 5217
|
- containerPort: 5217
|
||||||
@@ -555,13 +555,9 @@ spec:
|
|||||||
- name: TtsReader__Jobs__Root
|
- name: TtsReader__Jobs__Root
|
||||||
value: "/data/jobs"
|
value: "/data/jobs"
|
||||||
- name: TtsReader__Piper__Host
|
- name: TtsReader__Piper__Host
|
||||||
value: "10.0.57.17"
|
value: "ttsreader-piper.fc-ttsreader.svc.cluster.local."
|
||||||
- name: TtsReader__Piper__Port
|
- name: TtsReader__Piper__Port
|
||||||
value: "8500"
|
value: "10200"
|
||||||
- name: TtsReader__Piper__Transport
|
|
||||||
value: "http"
|
|
||||||
- name: TtsReader__Piper__HttpPath
|
|
||||||
value: "/tts"
|
|
||||||
- name: TtsReader__Kokoro__Enabled
|
- name: TtsReader__Kokoro__Enabled
|
||||||
value: "true"
|
value: "true"
|
||||||
- name: TtsReader__Kokoro__BaseUrl
|
- name: TtsReader__Kokoro__BaseUrl
|
||||||
|
|||||||
@@ -15,18 +15,9 @@ All repo-scoped Linux runners use:
|
|||||||
Actions tool cache
|
Actions tool cache
|
||||||
|
|
||||||
`github-runner` for `FlowerCore.Common` is single-replica because it retains the
|
`github-runner` for `FlowerCore.Common` is single-replica because it retains the
|
||||||
original Longhorn ReadWriteOnce NuGet PVC. Every other repo-scoped runner uses
|
original Longhorn ReadWriteOnce NuGet PVC. `github-runner-sharedpos` and the top
|
||||||
two replicas with per-pod `emptyDir` caches. That is the safe backlog-drain
|
Linux-cost repo runners use two replicas with per-pod `emptyDir` caches. That is
|
||||||
strategy: no two pods share one RWO PVC.
|
the safe backlog-drain strategy: no two pods share one RWO PVC.
|
||||||
|
|
||||||
Sprint 32 final long-tail wave adds 16 two-replica Deployments:
|
|
||||||
`FlowerCore.Knowledge`, `FlowerCore.LlmBridge`, `FlowerCore.Media`,
|
|
||||||
`FlowerCore.Presentations`, `FlowerCore.RemoteDesktop`, `FlowerCore.DNS`,
|
|
||||||
`FlowerCore.Distribution`, `FlowerCore.Scoreboard`,
|
|
||||||
`FlowerCore.SegmentDisplay`, `FlowerCore.Signage.Contracts`,
|
|
||||||
`FlowerCore.SignalControl`, `FlowerCore.Intranet.Web`,
|
|
||||||
`FlowerCore.Provisioning`, `FlowerCore.Redis`, `FlowerCore.MessageBoard`, and
|
|
||||||
`FlowerCore.MenuBoard`.
|
|
||||||
|
|
||||||
## Post-Merge Proof
|
## Post-Merge Proof
|
||||||
|
|
||||||
@@ -41,13 +32,7 @@ Verify GitHub registration for the repo-scoped runners:
|
|||||||
```bash
|
```bash
|
||||||
for repo in FlowerCore.Common FlowerCore.Shared.Pos FlowerCore.Puppet FlowerCore.Signage \
|
for repo in FlowerCore.Common FlowerCore.Shared.Pos FlowerCore.Puppet FlowerCore.Signage \
|
||||||
FlowerCore.DMS FlowerCore.Telephony FlowerCore.Print.Web FlowerCore.Chat \
|
FlowerCore.DMS FlowerCore.Telephony FlowerCore.Print.Web FlowerCore.Chat \
|
||||||
FlowerCore.MySQL FlowerCore.Kiosk.Linux FlowerCore.Marquee FlowerCore.TtsReader \
|
FlowerCore.MySQL FlowerCore.Kiosk.Linux; do
|
||||||
FlowerCore.Knowledge FlowerCore.LlmBridge FlowerCore.Media \
|
|
||||||
FlowerCore.Presentations FlowerCore.RemoteDesktop FlowerCore.DNS \
|
|
||||||
FlowerCore.Distribution FlowerCore.Scoreboard FlowerCore.SegmentDisplay \
|
|
||||||
FlowerCore.Signage.Contracts FlowerCore.SignalControl FlowerCore.Intranet.Web \
|
|
||||||
FlowerCore.Provisioning FlowerCore.Redis FlowerCore.MessageBoard \
|
|
||||||
FlowerCore.MenuBoard; do
|
|
||||||
echo "=== $repo ==="
|
echo "=== $repo ==="
|
||||||
gh api "/repos/astoltz/$repo/actions/runners" \
|
gh api "/repos/astoltz/$repo/actions/runners" \
|
||||||
--jq '.runners[] | select(.labels[].name == "fc-build-linux") | {name,status,busy,labels:[.labels[].name]}'
|
--jq '.runners[] | select(.labels[].name == "fc-build-linux") | {name,status,busy,labels:[.labels[].name]}'
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -966,52 +966,6 @@ data:
|
|||||||
annotations:
|
annotations:
|
||||||
summary: "Disk usage high on {{ $labels.instance }} ({{ $value | printf \"%.1f\" }}%)"
|
summary: "Disk usage high on {{ $labels.instance }} ({{ $value | printf \"%.1f\" }}%)"
|
||||||
|
|
||||||
# Puppet agent + service alerts.
|
|
||||||
# Mirror of FlowerCore.Notes/scripts/monitoring/alerts.yml `puppet` group
|
|
||||||
# so a future migration to in-cluster Prometheus inherits the ruleset.
|
|
||||||
# Source-of-truth for the live Podman Prometheus on noc1 is the Notes file.
|
|
||||||
# See feedback_monitoring_k8s_target_vs_live_podman.
|
|
||||||
- name: puppet
|
|
||||||
rules:
|
|
||||||
- alert: PuppetAgentReportStale
|
|
||||||
expr: puppet_last_run_age_seconds > 7200
|
|
||||||
for: 30m
|
|
||||||
labels:
|
|
||||||
severity: warning
|
|
||||||
alert_channel: irc
|
|
||||||
annotations:
|
|
||||||
summary: "Puppet agent {{ $labels.instance }} hasn't reported in over 2h"
|
|
||||||
description: "Last run age: {{ $value | humanizeDuration }}. The puppet agent on {{ $labels.instance }} may be stopped, the node may be powered off, or noc1 may be unreachable from this node."
|
|
||||||
runbook: "1. SSH to node (via noc1 jumpbox if needed) 2. sudo systemctl status puppet 3. sudo puppet agent -t --noop to force a run 4. Check r10k: ssh fcadmin@10.0.56.10 'sudo podman logs openvoxserver --tail 50' 5. Verify noc1 reachability: ping puppet.iamworkin.lan"
|
|
||||||
|
|
||||||
- alert: PuppetAgentReportCritical
|
|
||||||
expr: puppet_last_run_age_seconds > 86400
|
|
||||||
for: 1h
|
|
||||||
labels:
|
|
||||||
severity: critical
|
|
||||||
alert_channel: irc
|
|
||||||
annotations:
|
|
||||||
summary: "Puppet agent {{ $labels.instance }} silent for over 24h — node is unmanaged"
|
|
||||||
description: "Last run age: {{ $value | humanizeDuration }}. Node {{ $labels.instance }} has not submitted a Puppet report in over 24 hours. Config drift is accumulating — investigate immediately. If intentional (maintenance), add to the exclusion filter or silence in Grafana."
|
|
||||||
runbook: "URGENT: 1. Check node power state 2. SSH via noc1 jumpbox: ssh fcadmin@10.0.56.10 then ssh <node> 3. sudo systemctl status puppet 4. sudo systemctl start puppet + sudo puppet agent -t 5. Check for network partitions (VLAN connectivity to 10.0.56.10) 6. If node was recently reimaged: sudo puppet agent -t to re-register with new SSL cert"
|
|
||||||
|
|
||||||
# Sprint 33 Cx-7 Phase B (2026-05-25 postmortem follow-up):
|
|
||||||
# Detects puppet.service in failed state — distinct from PuppetAgentReportStale
|
|
||||||
# which catches "agent hasn't run." This catches "systemd gave up restarting it"
|
|
||||||
# (CA-verify loop or other fatal exit). Requires node-exporter systemd collector
|
|
||||||
# enabled with --collector.systemd. If `node_systemd_unit_state` has no series
|
|
||||||
# for a node, the collector is disabled there — flag in postmortem follow-up.
|
|
||||||
- alert: PuppetServiceFailed
|
|
||||||
expr: node_systemd_unit_state{name="puppet.service",state="failed"} == 1
|
|
||||||
for: 5m
|
|
||||||
labels:
|
|
||||||
severity: warning
|
|
||||||
alert_channel: irc
|
|
||||||
annotations:
|
|
||||||
summary: "Puppet service failed on {{ $labels.instance }}"
|
|
||||||
description: "puppet.service on {{ $labels.instance }} has been in failed state for 5+ minutes. systemd has stopped auto-restarting (CA-verify-loop or other exit). Manual `systemctl status puppet` confirms. Run `sudo systemctl start puppet` to recover; investigate journal for root cause."
|
|
||||||
runbook_url: "https://github.com/astoltz/FlowerCore.Notes/blob/master/memory/feedback_puppet_service_dead_after_ca_loop_alert_misreads.md"
|
|
||||||
|
|
||||||
# K8s pod-state alerts. Require kube-state-metrics scrape (added
|
# K8s pod-state alerts. Require kube-state-metrics scrape (added
|
||||||
# 2026-04-26 — see scrape_configs above). Would have surfaced the
|
# 2026-04-26 — see scrape_configs above). Would have surfaced the
|
||||||
# agent-zero ollama-proxy 172x crash-loop instead of letting it
|
# agent-zero ollama-proxy 172x crash-loop instead of letting it
|
||||||
@@ -1273,55 +1227,24 @@ metadata:
|
|||||||
data:
|
data:
|
||||||
notify.py: |
|
notify.py: |
|
||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
"""HTTP->IRC alert relay with thermal-printer DIGEST forwarding.
|
"""HTTP->IRC alert relay with thermal printer forwarding for Grafana webhooks.
|
||||||
|
Listens on :9119, posts to #alerts on UnrealIRCd via raw IRC protocol.
|
||||||
Listens on :9119, posts to #alerts on UnrealIRCd, forwards to Print.Web
|
Alerts tagged alert_channel=thermal_print also POST to Print.Web /api/print/alert.
|
||||||
/api/print/alert. Thermal printing is BATCHED into hourly digests by
|
|
||||||
default so the printer no longer spam-fires per Grafana webhook.
|
|
||||||
|
|
||||||
Routing (per Grafana webhook alert):
|
|
||||||
- IRC: always per-event (operator likes the stream)
|
|
||||||
- Thermal printer:
|
|
||||||
* severity in {critical,disaster,page} OR
|
|
||||||
label alert_channel=thermal_print_immediate -> print NOW
|
|
||||||
* label alert_channel=thermal_print -> enqueue into hourly digest
|
|
||||||
* everything else -> IRC only
|
|
||||||
- RESOLVED webhooks remove the alert from the digest buffer
|
|
||||||
|
|
||||||
Env vars (defaults preserve old behavior on first deploy):
|
|
||||||
THERMAL_PRINT_ENABLED default "true" - master kill switch
|
|
||||||
BATCH_INTERVAL_MIN default "60" - minutes between digest prints
|
|
||||||
BATCH_MAX_PENDING default "50" - force-flush threshold
|
|
||||||
|
|
||||||
HTTP surface:
|
|
||||||
POST / - Grafana webhook entry
|
|
||||||
POST /flush - manual digest flush (idempotent)
|
|
||||||
GET / - status + config + buffer depth + stats
|
|
||||||
"""
|
"""
|
||||||
import json, os, socket, sys, threading, time
|
import json, socket, sys, time
|
||||||
from collections import defaultdict
|
|
||||||
from datetime import datetime, timezone
|
|
||||||
from http.server import HTTPServer, BaseHTTPRequestHandler
|
from http.server import HTTPServer, BaseHTTPRequestHandler
|
||||||
from urllib.request import Request, urlopen
|
from urllib.request import Request, urlopen
|
||||||
|
from urllib.error import URLError
|
||||||
|
|
||||||
THERMAL_PRINT_ENABLED = os.environ.get("THERMAL_PRINT_ENABLED", "true").lower() == "true"
|
IRC_HOST = "unrealircd.irc.svc" # short name: CoreDNS ndots:5 + iamworkin.lan template hijacks full .cluster.local (see memory)
|
||||||
BATCH_INTERVAL_MIN = int(os.environ.get("BATCH_INTERVAL_MIN", "60"))
|
IRC_PORT = 6667
|
||||||
BATCH_MAX_PENDING = int(os.environ.get("BATCH_MAX_PENDING", "50"))
|
IRC_NICK = "grafana-bot"
|
||||||
|
IRC_CHANNEL = "#alerts"
|
||||||
IRC_HOST = os.environ.get("IRC_HOST", "unrealircd.irc.svc")
|
PRINT_WEB_URL = "http://10.0.57.16:5200/api/print/alert"
|
||||||
IRC_PORT = int(os.environ.get("IRC_PORT", "6667"))
|
PRINT_ENABLED = True
|
||||||
IRC_NICK = os.environ.get("IRC_NICK", "grafana-bot")
|
|
||||||
IRC_CHANNEL = os.environ.get("IRC_CHANNEL", "#alerts")
|
|
||||||
PRINT_WEB_URL = os.environ.get("PRINT_WEB_URL", "http://10.0.57.16:5200/api/print/alert")
|
|
||||||
|
|
||||||
_buffer_lock = threading.Lock()
|
|
||||||
_buffer = {} # fingerprint -> {"alert": dict, "first_seen": float, "last_seen": float}
|
|
||||||
_last_flush_time = time.time()
|
|
||||||
_stats = {"webhooks_received": 0, "irc_sent": 0, "print_immediate": 0,
|
|
||||||
"digest_flushed": 0, "buffer_dedup": 0, "buffer_added": 0,
|
|
||||||
"buffer_resolved": 0, "started_at": time.time()}
|
|
||||||
|
|
||||||
def send_irc(message):
|
def send_irc(message):
|
||||||
|
"""Connect, handle PING, join, send, quit."""
|
||||||
try:
|
try:
|
||||||
sock = socket.create_connection((IRC_HOST, IRC_PORT), timeout=15)
|
sock = socket.create_connection((IRC_HOST, IRC_PORT), timeout=15)
|
||||||
sock.sendall(f"NICK {IRC_NICK}\r\n".encode())
|
sock.sendall(f"NICK {IRC_NICK}\r\n".encode())
|
||||||
@@ -1354,137 +1277,52 @@ data:
|
|||||||
time.sleep(0.5)
|
time.sleep(0.5)
|
||||||
sock.sendall(b"QUIT :alert delivered\r\n")
|
sock.sendall(b"QUIT :alert delivered\r\n")
|
||||||
sock.close()
|
sock.close()
|
||||||
_stats["irc_sent"] += 1
|
|
||||||
return True
|
return True
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"[irc-notify] IRC send failed: {e}", file=sys.stderr)
|
print(f"[irc-notify] IRC send failed: {e}", file=sys.stderr)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def post_thermal(payload, kind):
|
def send_thermal_print(alert):
|
||||||
if not THERMAL_PRINT_ENABLED:
|
if not PRINT_ENABLED: return
|
||||||
print(f"[irc-notify] thermal disabled; skip {kind} ({payload.get('title','?')[:40]})", file=sys.stderr)
|
labels = alert.get("labels", {})
|
||||||
return False
|
annotations = alert.get("annotations", {})
|
||||||
|
status = alert.get("status", "firing").upper()
|
||||||
|
summary = annotations.get("summary", "")
|
||||||
|
description = annotations.get("description", "")
|
||||||
|
runbook = annotations.get("runbook", "")
|
||||||
|
# Build a useful message: summary + description + runbook steps
|
||||||
|
parts = []
|
||||||
|
if summary: parts.append(summary)
|
||||||
|
if description and description != summary: parts.append(description)
|
||||||
|
if runbook: parts.append("STEPS: " + runbook)
|
||||||
|
message = " | ".join(parts) if parts else labels.get("alertname", "Unknown alert")
|
||||||
|
payload = {
|
||||||
|
"title": labels.get("alertname", "Unknown"),
|
||||||
|
"severity": labels.get("severity", "warning").capitalize(),
|
||||||
|
"host": labels.get("instance", labels.get("host", "unknown")),
|
||||||
|
"message": message,
|
||||||
|
"eventId": alert.get("fingerprint", ""),
|
||||||
|
"source": "Grafana",
|
||||||
|
"status": "RESOLVED" if status == "RESOLVED" else "PROBLEM",
|
||||||
|
"acknowledged": False
|
||||||
|
}
|
||||||
try:
|
try:
|
||||||
req = Request(PRINT_WEB_URL, data=json.dumps(payload).encode("utf-8"),
|
req = Request(PRINT_WEB_URL, data=json.dumps(payload).encode("utf-8"),
|
||||||
headers={"Content-Type": "application/json"}, method="POST")
|
headers={"Content-Type": "application/json"}, method="POST")
|
||||||
resp = urlopen(req, timeout=10)
|
resp = urlopen(req, timeout=10)
|
||||||
if kind == "immediate": _stats["print_immediate"] += 1
|
print(f"[irc-notify] Thermal print sent: {resp.read().decode()}", file=sys.stderr)
|
||||||
print(f"[irc-notify] thermal {kind} sent: {payload.get('title','?')[:50]}", file=sys.stderr)
|
|
||||||
return True
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"[irc-notify] thermal {kind} failed: {e}", file=sys.stderr)
|
print(f"[irc-notify] Thermal print failed: {e}", file=sys.stderr)
|
||||||
return False
|
|
||||||
|
|
||||||
def fingerprint_of(alert):
|
def should_print(alert):
|
||||||
fp = alert.get("fingerprint", "")
|
|
||||||
if fp: return fp
|
|
||||||
labels = alert.get("labels", {})
|
labels = alert.get("labels", {})
|
||||||
target = labels.get("pod") or labels.get("instance") or labels.get("deployment") or labels.get("statefulset") or labels.get("namespace") or ""
|
if labels.get("alert_channel") == "thermal_print": return True
|
||||||
return f"{labels.get('alertname','?')}/{labels.get('namespace','')}/{target}"
|
if labels.get("severity", "").lower() in ("critical", "disaster"): return True
|
||||||
|
if alert.get("status", "").upper() == "RESOLVED": return False
|
||||||
def is_critical(alert):
|
|
||||||
return alert.get("labels", {}).get("severity", "").lower() in ("critical", "disaster", "page")
|
|
||||||
|
|
||||||
def is_immediate_label(alert):
|
|
||||||
return alert.get("labels", {}).get("alert_channel") == "thermal_print_immediate"
|
|
||||||
|
|
||||||
def is_batched_label(alert):
|
|
||||||
return alert.get("labels", {}).get("alert_channel") == "thermal_print"
|
|
||||||
|
|
||||||
def add_to_digest(alert):
|
|
||||||
"""Add an alert to the digest buffer. Returns True if the buffer GREW
|
|
||||||
(new fingerprint), False if it was a dedup, resolution, or no-op.
|
|
||||||
"""
|
|
||||||
if not THERMAL_PRINT_ENABLED: return False
|
|
||||||
fp = fingerprint_of(alert)
|
|
||||||
status = alert.get("status", "firing").lower()
|
|
||||||
with _buffer_lock:
|
|
||||||
if status == "resolved":
|
|
||||||
if fp in _buffer:
|
|
||||||
del _buffer[fp]
|
|
||||||
_stats["buffer_resolved"] += 1
|
|
||||||
return False
|
return False
|
||||||
if fp in _buffer:
|
|
||||||
_buffer[fp]["last_seen"] = time.time()
|
|
||||||
_buffer[fp]["alert"] = alert
|
|
||||||
_stats["buffer_dedup"] += 1
|
|
||||||
return False
|
|
||||||
_buffer[fp] = {"alert": alert, "first_seen": time.time(), "last_seen": time.time()}
|
|
||||||
_stats["buffer_added"] += 1
|
|
||||||
return True
|
|
||||||
|
|
||||||
def build_digest_payload():
|
|
||||||
with _buffer_lock:
|
|
||||||
items = list(_buffer.values())
|
|
||||||
if not items: return None
|
|
||||||
by_name = defaultdict(list)
|
|
||||||
for item in items:
|
|
||||||
labels = item["alert"].get("labels", {})
|
|
||||||
by_name[labels.get("alertname", "Unknown")].append(item)
|
|
||||||
lines = []
|
|
||||||
for name, group in sorted(by_name.items()):
|
|
||||||
targets = []
|
|
||||||
for it in group[:5]:
|
|
||||||
labels = it["alert"].get("labels", {})
|
|
||||||
t = (labels.get("pod") or labels.get("instance") or labels.get("deployment")
|
|
||||||
or labels.get("statefulset") or labels.get("namespace") or "?")
|
|
||||||
targets.append(t)
|
|
||||||
more = f" (+{len(group)-5})" if len(group) > 5 else ""
|
|
||||||
sevs = sorted({it["alert"].get("labels", {}).get("severity", "warning") for it in group})
|
|
||||||
lines.append(f"[{'/'.join(sevs)}] {name} x{len(group)}: {', '.join(targets)}{more}")
|
|
||||||
now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
|
|
||||||
title = f"Alert digest: {len(items)} firing"
|
|
||||||
body = "\n".join([
|
|
||||||
f"=== {title} ===",
|
|
||||||
f"as of {now}",
|
|
||||||
"",
|
|
||||||
*lines,
|
|
||||||
"",
|
|
||||||
"Stream: #alerts (IRC) | Triage: grafana-noc1.iamworkin.lan",
|
|
||||||
"Force-flush: POST irc-notify.monitoring.svc:9119/flush",
|
|
||||||
])
|
|
||||||
return {"title": title, "severity": "Warning", "host": "monitoring",
|
|
||||||
"message": body, "eventId": f"digest-{int(time.time())}",
|
|
||||||
"source": "Grafana digest", "status": "PROBLEM", "acknowledged": False}
|
|
||||||
|
|
||||||
def flush_digest():
|
|
||||||
payload = build_digest_payload()
|
|
||||||
if payload is None:
|
|
||||||
print("[irc-notify] flush: buffer empty, no digest sent", file=sys.stderr)
|
|
||||||
return False
|
|
||||||
sent = post_thermal(payload, "digest")
|
|
||||||
with _buffer_lock:
|
|
||||||
_buffer.clear()
|
|
||||||
if sent: _stats["digest_flushed"] += 1
|
|
||||||
return sent
|
|
||||||
|
|
||||||
def digest_loop():
|
|
||||||
global _last_flush_time
|
|
||||||
while True:
|
|
||||||
try:
|
|
||||||
now = time.time()
|
|
||||||
elapsed = now - _last_flush_time
|
|
||||||
if elapsed >= BATCH_INTERVAL_MIN * 60:
|
|
||||||
print(f"[irc-notify] digest tick: interval reached ({BATCH_INTERVAL_MIN}m); buffer={len(_buffer)}", file=sys.stderr)
|
|
||||||
flush_digest()
|
|
||||||
_last_flush_time = now
|
|
||||||
elif len(_buffer) >= BATCH_MAX_PENDING:
|
|
||||||
print(f"[irc-notify] digest tick: buffer full ({len(_buffer)}); force flush", file=sys.stderr)
|
|
||||||
flush_digest()
|
|
||||||
_last_flush_time = now
|
|
||||||
time.sleep(15)
|
|
||||||
except Exception as e:
|
|
||||||
print(f"[irc-notify] digest loop error: {e}", file=sys.stderr)
|
|
||||||
time.sleep(60)
|
|
||||||
|
|
||||||
class Handler(BaseHTTPRequestHandler):
|
class Handler(BaseHTTPRequestHandler):
|
||||||
def do_POST(self):
|
def do_POST(self):
|
||||||
if self.path == "/flush":
|
|
||||||
ok = flush_digest()
|
|
||||||
self.send_response(200); self.send_header("Content-Type", "application/json"); self.end_headers()
|
|
||||||
self.wfile.write(json.dumps({"flushed": ok, "buffer_after": len(_buffer)}).encode())
|
|
||||||
return
|
|
||||||
_stats["webhooks_received"] += 1
|
|
||||||
length = int(self.headers.get("Content-Length", 0))
|
length = int(self.headers.get("Content-Length", 0))
|
||||||
body = json.loads(self.rfile.read(length)) if length else {}
|
body = json.loads(self.rfile.read(length)) if length else {}
|
||||||
for alert in body.get("alerts", []):
|
for alert in body.get("alerts", []):
|
||||||
@@ -1499,56 +1337,22 @@ data:
|
|||||||
msg = f"{icon}{sev_tag} {name}: {summary}"
|
msg = f"{icon}{sev_tag} {name}: {summary}"
|
||||||
if desc: msg += f"\n {desc}"
|
if desc: msg += f"\n {desc}"
|
||||||
send_irc(msg)
|
send_irc(msg)
|
||||||
# Thermal routing — EVERYTHING (including criticals) goes into
|
if should_print(alert): send_thermal_print(alert)
|
||||||
# the hourly digest. Only the explicit `alert_channel=thermal_print_immediate`
|
self.send_response(200)
|
||||||
# label bypasses, and even that flushes-the-current-digest rather
|
self.send_header("Content-Type", "application/json")
|
||||||
# than printing a standalone job, so the same fingerprint can't
|
self.end_headers()
|
||||||
# spam the printer per webhook cycle.
|
|
||||||
if status == "RESOLVED":
|
|
||||||
add_to_digest(alert) # removes from buffer
|
|
||||||
continue
|
|
||||||
if is_immediate_label(alert):
|
|
||||||
# Explicit opt-in for "paper this NOW" — first arrival of a
|
|
||||||
# new fingerprint triggers an immediate digest flush; repeat
|
|
||||||
# webhooks for the same fingerprint dedupe in the buffer
|
|
||||||
# until the next interval or until the alert resolves.
|
|
||||||
new_in_buffer = add_to_digest(alert)
|
|
||||||
if new_in_buffer:
|
|
||||||
global _last_flush_time
|
|
||||||
flush_digest()
|
|
||||||
_last_flush_time = time.time()
|
|
||||||
elif is_critical(alert) or is_batched_label(alert):
|
|
||||||
add_to_digest(alert)
|
|
||||||
# else: IRC-only (warnings without thermal_print label)
|
|
||||||
self.send_response(200); self.send_header("Content-Type", "application/json"); self.end_headers()
|
|
||||||
self.wfile.write(b'{"status":"ok"}')
|
self.wfile.write(b'{"status":"ok"}')
|
||||||
|
|
||||||
def do_GET(self):
|
def do_GET(self):
|
||||||
self.send_response(200); self.send_header("Content-Type", "application/json"); self.end_headers()
|
self.send_response(200)
|
||||||
with _buffer_lock:
|
self.send_header("Content-Type", "application/json")
|
||||||
alertnames = sorted({it["alert"].get("labels", {}).get("alertname", "?") for it in _buffer.values()})
|
self.end_headers()
|
||||||
depth = len(_buffer)
|
self.wfile.write(json.dumps({"service":"irc-notify","thermal_print":PRINT_ENABLED}).encode())
|
||||||
info = {
|
|
||||||
"service": "irc-notify",
|
|
||||||
"config": {"thermal_print_enabled": THERMAL_PRINT_ENABLED,
|
|
||||||
"batch_interval_min": BATCH_INTERVAL_MIN,
|
|
||||||
"batch_max_pending": BATCH_MAX_PENDING,
|
|
||||||
"irc_target": f"{IRC_HOST}:{IRC_PORT} {IRC_CHANNEL}",
|
|
||||||
"print_web_url": PRINT_WEB_URL},
|
|
||||||
"buffer": {"depth": depth, "alertnames": alertnames,
|
|
||||||
"seconds_since_last_flush": int(time.time() - _last_flush_time),
|
|
||||||
"seconds_until_next_flush": max(0, int(BATCH_INTERVAL_MIN*60 - (time.time() - _last_flush_time)))},
|
|
||||||
"stats": _stats,
|
|
||||||
}
|
|
||||||
self.wfile.write(json.dumps(info, indent=2).encode())
|
|
||||||
|
|
||||||
def log_message(self, format, *args):
|
def log_message(self, format, *args):
|
||||||
print(f"[irc-notify] {args[0]}", file=sys.stderr)
|
print(f"[irc-notify] {args[0]}", file=sys.stderr)
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
threading.Thread(target=digest_loop, daemon=True).start()
|
|
||||||
server = HTTPServer(("0.0.0.0", 9119), Handler)
|
server = HTTPServer(("0.0.0.0", 9119), Handler)
|
||||||
print(f"[irc-notify] :9119 -> IRC {IRC_HOST}:{IRC_PORT} {IRC_CHANNEL} | thermal={'ON' if THERMAL_PRINT_ENABLED else 'OFF'} | digest={BATCH_INTERVAL_MIN}m max={BATCH_MAX_PENDING}", file=sys.stderr)
|
print(f"IRC alert relay :9119 -> {IRC_HOST}:{IRC_PORT} {IRC_CHANNEL} (thermal: {PRINT_ENABLED})")
|
||||||
server.serve_forever()
|
server.serve_forever()
|
||||||
|
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
|
|||||||
@@ -28,12 +28,9 @@ Source: `D:\git\FlowerCore\FlowerCore.WorldBuilder` (master)
|
|||||||
Memory: `feedback_rke2_image_import_per_node_scp`.
|
Memory: `feedback_rke2_image_import_per_node_scp`.
|
||||||
3. **Bump image tag** in `worldbuilder.yaml` and git push.
|
3. **Bump image tag** in `worldbuilder.yaml` and git push.
|
||||||
ArgoCD ApplicationSet picks up within ~3 minutes.
|
ArgoCD ApplicationSet picks up within ~3 minutes.
|
||||||
4. **First production render** — open
|
4. **First production render** — open `https://worldbuilder.iamworkin.lan`,
|
||||||
`https://worldbuilder.iamworkin.lan/studio/c32e0000-0000-4000-8000-000000000004`
|
create World → Character → Storyboard → ExportJob, confirm artifact
|
||||||
and confirm the Cyberpunk Blue Jay demo prompt loads with five seeded fake
|
downloads. ComfyUI lives on BLUEJAY-WS at `http://10.0.56.20:8188`.
|
||||||
generated images. This Sprint 32 visitor-safe profile uses
|
|
||||||
`ClientMode=fake`; switch the image-generation env vars back to ComfyUI only
|
|
||||||
for an operator-owned GPU render lane.
|
|
||||||
|
|
||||||
## Health probes
|
## Health probes
|
||||||
|
|
||||||
@@ -56,13 +53,8 @@ Source: `D:\git\FlowerCore\FlowerCore.WorldBuilder` (master)
|
|||||||
|
|
||||||
## Image generation backend
|
## Image generation backend
|
||||||
|
|
||||||
Sprint 32 pins the Kubernetes profile to
|
`FlowerCore:WorldBuilder:ImageGeneration:BaseUrl=http://10.0.56.20:8188` —
|
||||||
`FlowerCore:WorldBuilder:ImageGeneration:ClientMode=fake` with
|
ComfyUI runs on BLUEJAY-WS Windows (R9700 / gfx1201 / ROCm 7.2.1). Pod reaches
|
||||||
`BaseUrl=http://127.0.0.1:1`. That keeps the public/internal visitor demo
|
the workstation directly across the 10.0.56.0/24 VLAN (no Podman-style host-
|
||||||
deterministic, avoids GPU exposure, and still exercises the studio/gallery
|
filter issues — K8s pods route via Calico, which is L3-routed across the
|
||||||
surface with persisted generated-image metadata.
|
VLAN).
|
||||||
|
|
||||||
The previous ComfyUI backend target was `http://10.0.56.20:8188` on
|
|
||||||
BLUEJAY-WS (R9700 / gfx1201 / ROCm 7.2.1). Re-enable it only in an
|
|
||||||
operator-owned follow-up that also verifies workstation reachability and image
|
|
||||||
import freshness.
|
|
||||||
|
|||||||
@@ -16,11 +16,7 @@ kind: Namespace
|
|||||||
metadata:
|
metadata:
|
||||||
name: fc-worldbuilder
|
name: fc-worldbuilder
|
||||||
labels:
|
labels:
|
||||||
app.kubernetes.io/name: fc-worldbuilder
|
|
||||||
app.kubernetes.io/part-of: flowercore
|
app.kubernetes.io/part-of: flowercore
|
||||||
app.kubernetes.io/managed-by: argocd
|
|
||||||
flowercore.io/tenant-id: system
|
|
||||||
flowercore.io/created-by: bluejay-infra
|
|
||||||
---
|
---
|
||||||
# SQLite DB + generated image gallery + PDF/PNG exports.
|
# SQLite DB + generated image gallery + PDF/PNG exports.
|
||||||
# Longhorn RWO — single replica with `Recreate` rollout strategy keeps it safe.
|
# Longhorn RWO — single replica with `Recreate` rollout strategy keeps it safe.
|
||||||
@@ -29,13 +25,6 @@ kind: PersistentVolumeClaim
|
|||||||
metadata:
|
metadata:
|
||||||
name: worldbuilder-data
|
name: worldbuilder-data
|
||||||
namespace: fc-worldbuilder
|
namespace: fc-worldbuilder
|
||||||
labels:
|
|
||||||
app.kubernetes.io/name: worldbuilder-data
|
|
||||||
app.kubernetes.io/component: storage
|
|
||||||
app.kubernetes.io/part-of: flowercore
|
|
||||||
app.kubernetes.io/managed-by: argocd
|
|
||||||
flowercore.io/tenant-id: system
|
|
||||||
flowercore.io/created-by: bluejay-infra
|
|
||||||
spec:
|
spec:
|
||||||
accessModes:
|
accessModes:
|
||||||
- ReadWriteOnce
|
- ReadWriteOnce
|
||||||
@@ -51,13 +40,7 @@ metadata:
|
|||||||
namespace: fc-worldbuilder
|
namespace: fc-worldbuilder
|
||||||
labels:
|
labels:
|
||||||
app.kubernetes.io/name: worldbuilder-web
|
app.kubernetes.io/name: worldbuilder-web
|
||||||
app.kubernetes.io/component: web
|
|
||||||
app.kubernetes.io/part-of: flowercore
|
app.kubernetes.io/part-of: flowercore
|
||||||
app.kubernetes.io/managed-by: argocd
|
|
||||||
flowercore.io/tenant-id: system
|
|
||||||
flowercore.io/created-by: bluejay-infra
|
|
||||||
annotations:
|
|
||||||
flowercore.io/traceability-standard: k8s-pod-ownership-and-traceability-standard
|
|
||||||
spec:
|
spec:
|
||||||
replicas: 1
|
replicas: 1
|
||||||
revisionHistoryLimit: 3
|
revisionHistoryLimit: 3
|
||||||
@@ -71,16 +54,11 @@ spec:
|
|||||||
metadata:
|
metadata:
|
||||||
labels:
|
labels:
|
||||||
app.kubernetes.io/name: worldbuilder-web
|
app.kubernetes.io/name: worldbuilder-web
|
||||||
app.kubernetes.io/component: web
|
|
||||||
app.kubernetes.io/part-of: flowercore
|
app.kubernetes.io/part-of: flowercore
|
||||||
app.kubernetes.io/managed-by: argocd
|
|
||||||
flowercore.io/tenant-id: system
|
|
||||||
flowercore.io/created-by: bluejay-infra
|
|
||||||
annotations:
|
annotations:
|
||||||
prometheus.io/scrape: "true"
|
prometheus.io/scrape: "true"
|
||||||
prometheus.io/port: "8080"
|
prometheus.io/port: "8080"
|
||||||
prometheus.io/path: "/metrics/prometheus"
|
prometheus.io/path: "/metrics/prometheus"
|
||||||
flowercore.io/audit-trace-id: "worldbuilder-runtime-demo"
|
|
||||||
spec:
|
spec:
|
||||||
securityContext:
|
securityContext:
|
||||||
fsGroup: 1654
|
fsGroup: 1654
|
||||||
@@ -114,14 +92,11 @@ spec:
|
|||||||
value: "/data/gallery"
|
value: "/data/gallery"
|
||||||
- name: FlowerCore__WorldBuilder__Export__RootPath
|
- name: FlowerCore__WorldBuilder__Export__RootPath
|
||||||
value: "/data/exports"
|
value: "/data/exports"
|
||||||
# Visitor-safe Sprint 32 profile: fake backend keeps public demo
|
# ComfyUI on BLUEJAY-WS (R9700 / gfx1201 / ROCm 7.2.1).
|
||||||
# rendering deterministic and avoids exposing BLUEJAY-WS GPU.
|
|
||||||
- name: FlowerCore__WorldBuilder__ImageGeneration__BaseUrl
|
- name: FlowerCore__WorldBuilder__ImageGeneration__BaseUrl
|
||||||
value: "http://127.0.0.1:1"
|
value: "http://10.0.56.20:8188"
|
||||||
- name: FlowerCore__WorldBuilder__ImageGeneration__ClientMode
|
- name: FlowerCore__WorldBuilder__ImageGeneration__ClientMode
|
||||||
value: "fake"
|
value: "comfyui"
|
||||||
- name: FlowerCore__WorldBuilder__ImageGeneration__BackendId
|
|
||||||
value: "fake"
|
|
||||||
resources:
|
resources:
|
||||||
# Cluster CPU-request budget runs hot (99% on all 3 nodes at deploy
|
# Cluster CPU-request budget runs hot (99% on all 3 nodes at deploy
|
||||||
# time) while actual CPU usage is well below capacity. Idle Blazor
|
# time) while actual CPU usage is well below capacity. Idle Blazor
|
||||||
@@ -190,11 +165,7 @@ metadata:
|
|||||||
namespace: fc-worldbuilder
|
namespace: fc-worldbuilder
|
||||||
labels:
|
labels:
|
||||||
app.kubernetes.io/name: worldbuilder-web
|
app.kubernetes.io/name: worldbuilder-web
|
||||||
app.kubernetes.io/component: web
|
|
||||||
app.kubernetes.io/part-of: flowercore
|
app.kubernetes.io/part-of: flowercore
|
||||||
app.kubernetes.io/managed-by: argocd
|
|
||||||
flowercore.io/tenant-id: system
|
|
||||||
flowercore.io/created-by: bluejay-infra
|
|
||||||
spec:
|
spec:
|
||||||
type: ClusterIP
|
type: ClusterIP
|
||||||
selector:
|
selector:
|
||||||
@@ -209,13 +180,6 @@ kind: Certificate
|
|||||||
metadata:
|
metadata:
|
||||||
name: worldbuilder-web-tls
|
name: worldbuilder-web-tls
|
||||||
namespace: fc-worldbuilder
|
namespace: fc-worldbuilder
|
||||||
labels:
|
|
||||||
app.kubernetes.io/name: worldbuilder-web-tls
|
|
||||||
app.kubernetes.io/component: ingress
|
|
||||||
app.kubernetes.io/part-of: flowercore
|
|
||||||
app.kubernetes.io/managed-by: argocd
|
|
||||||
flowercore.io/tenant-id: system
|
|
||||||
flowercore.io/created-by: bluejay-infra
|
|
||||||
spec:
|
spec:
|
||||||
secretName: worldbuilder-web-tls
|
secretName: worldbuilder-web-tls
|
||||||
issuerRef:
|
issuerRef:
|
||||||
@@ -236,13 +200,6 @@ kind: IngressRoute
|
|||||||
metadata:
|
metadata:
|
||||||
name: worldbuilder-web
|
name: worldbuilder-web
|
||||||
namespace: fc-worldbuilder
|
namespace: fc-worldbuilder
|
||||||
labels:
|
|
||||||
app.kubernetes.io/name: worldbuilder-web
|
|
||||||
app.kubernetes.io/component: ingress
|
|
||||||
app.kubernetes.io/part-of: flowercore
|
|
||||||
app.kubernetes.io/managed-by: argocd
|
|
||||||
flowercore.io/tenant-id: system
|
|
||||||
flowercore.io/created-by: bluejay-infra
|
|
||||||
spec:
|
spec:
|
||||||
entryPoints:
|
entryPoints:
|
||||||
- websecure
|
- websecure
|
||||||
|
|||||||
@@ -1,84 +0,0 @@
|
|||||||
# openvoxserver Quadlet Durability
|
|
||||||
|
|
||||||
This runbook documents the noc1 `openvoxserver` durability fix for the Puppet control-repo deploy path. The service is a noc1 host artifact, not an ArgoCD application, so discovery always starts on noc1 rather than in `apps/*`.
|
|
||||||
|
|
||||||
## Current State
|
|
||||||
|
|
||||||
As of the Sprint 32 Cx-12 apply on 2026-05-17:
|
|
||||||
|
|
||||||
- `/etc/containers/systemd/openvoxserver.container` has a `GIT_SSH_COMMAND` environment entry that points at the persisted serverdata deploy key.
|
|
||||||
- `/etc/systemd/system/openvoxserver-safeconfig.service` is enabled and active, and reapplies `git config --global --add safe.directory *` inside the running container.
|
|
||||||
- `/opt/puppet/r10k-deploy.sh` self-heals before each fetch by setting `safe.directory`, the repo-local `core.sshCommand`, and the persisted `known_hosts` file when needed.
|
|
||||||
- `puppet-deploy.service` exits `0/SUCCESS` after the apply and the control repo reports `HEAD == origin/master`.
|
|
||||||
- `systemctl cat openvoxserver` does not currently resolve to a generated unit on noc1. The container is running through Podman with `restart=always`, so destructive recreate smoke must not run until the generated unit is present.
|
|
||||||
|
|
||||||
## Discovery
|
|
||||||
|
|
||||||
Run every command through noc1 as `fcadmin`; do not assume BLUEJAY-WS can reach container-local surfaces directly.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
ssh -i ~/.ssh/fcadmin_ed25519 fcadmin@10.0.56.10 "hostname && sudo -n true"
|
|
||||||
ssh -i ~/.ssh/fcadmin_ed25519 fcadmin@10.0.56.10 "sudo find /etc/containers/systemd /usr/share/containers/systemd /etc/systemd/system -name 'openvoxserver*' 2>/dev/null"
|
|
||||||
ssh -i ~/.ssh/fcadmin_ed25519 fcadmin@10.0.56.10 "sudo sed -n '1,220p' /etc/containers/systemd/openvoxserver.container"
|
|
||||||
ssh -i ~/.ssh/fcadmin_ed25519 fcadmin@10.0.56.10 "sudo systemctl cat puppet-deploy.service"
|
|
||||||
```
|
|
||||||
|
|
||||||
If a future noc1 profile manages these files, update the Puppet control repo and let `puppet-deploy.service` apply the change. On 2026-05-17, host `puppet` was not installed, so Cx-12 used a direct noc1 host edit.
|
|
||||||
|
|
||||||
## Durable Fix Shape
|
|
||||||
|
|
||||||
The Quadlet keeps the deploy key as a path reference only:
|
|
||||||
|
|
||||||
```ini
|
|
||||||
Environment=GIT_SSH_COMMAND=ssh -i /opt/puppetlabs/server/data/puppetserver/.puppet-deploy-key -o StrictHostKeyChecking=yes -o IdentitiesOnly=yes -o UserKnownHostsFile=/opt/puppetlabs/server/data/puppetserver/.known_hosts
|
|
||||||
```
|
|
||||||
|
|
||||||
The safeconfig service is intentionally independent of `openvoxserver.service` until the generated unit exists. It waits for the `openvoxserver` container name and then runs:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
/usr/bin/podman exec openvoxserver git config --global --add safe.directory *
|
|
||||||
```
|
|
||||||
|
|
||||||
The deploy script self-heals inside the container before it fetches the control repo:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
git config --global --add safe.directory "*" 2>/dev/null || true
|
|
||||||
DEPLOY_KEY="/opt/puppetlabs/server/data/puppetserver/.puppet-deploy-key"
|
|
||||||
KNOWN_HOSTS="/opt/puppetlabs/server/data/puppetserver/.known_hosts"
|
|
||||||
REPO="/etc/puppetlabs/code/environments/production"
|
|
||||||
export GIT_SSH_COMMAND="ssh -i $DEPLOY_KEY -o StrictHostKeyChecking=yes -o IdentitiesOnly=yes -o UserKnownHostsFile=$KNOWN_HOSTS"
|
|
||||||
git -C "$REPO" config core.sshCommand "$GIT_SSH_COMMAND" 2>/dev/null || true
|
|
||||||
```
|
|
||||||
|
|
||||||
## Validation
|
|
||||||
|
|
||||||
Non-destructive validation:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
ssh -i ~/.ssh/fcadmin_ed25519 fcadmin@10.0.56.10 "sudo grep -n 'GIT_SSH_COMMAND' /etc/containers/systemd/openvoxserver.container"
|
|
||||||
ssh -i ~/.ssh/fcadmin_ed25519 fcadmin@10.0.56.10 "sudo systemctl status openvoxserver-safeconfig.service --no-pager -l"
|
|
||||||
ssh -i ~/.ssh/fcadmin_ed25519 fcadmin@10.0.56.10 "sudo systemctl start puppet-deploy.service && sudo systemctl status puppet-deploy.service --no-pager -l"
|
|
||||||
ssh -i ~/.ssh/fcadmin_ed25519 fcadmin@10.0.56.10 "sudo podman exec openvoxserver git -C /etc/puppetlabs/code/environments/production config --get core.sshCommand"
|
|
||||||
```
|
|
||||||
|
|
||||||
Destructive recreate smoke is opt-in only:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
scp scripts/monitoring/openvox-recreate-smoke.sh fcadmin@10.0.56.10:/tmp/openvox-recreate-smoke.sh
|
|
||||||
ssh -i ~/.ssh/fcadmin_ed25519 fcadmin@10.0.56.10 "chmod +x /tmp/openvox-recreate-smoke.sh && sudo OPENVOX_RECREATE_SMOKE=1 /tmp/openvox-recreate-smoke.sh"
|
|
||||||
```
|
|
||||||
|
|
||||||
Do not run the smoke during normal sprint work. It stops and removes the production container before starting it again through systemd, and it now refuses to continue unless `systemctl cat openvoxserver` succeeds.
|
|
||||||
|
|
||||||
## Credential Rotation Note
|
|
||||||
|
|
||||||
When rotating the Puppet deploy key, update the persisted serverdata copy on noc1:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
sudo install -m 0600 -o root -g root <new-deploy-key> /opt/puppet/serverdata/.puppet-deploy-key
|
|
||||||
sudo podman exec openvoxserver sh -c "ssh-keyscan github.com > /opt/puppetlabs/server/data/puppetserver/.known_hosts"
|
|
||||||
sudo systemctl start openvoxserver-safeconfig.service
|
|
||||||
sudo systemctl start puppet-deploy.service
|
|
||||||
```
|
|
||||||
|
|
||||||
Never commit the deploy key or print it in logs.
|
|
||||||
@@ -1,48 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
if [ "${OPENVOX_RECREATE_SMOKE:-}" != "1" ]; then
|
|
||||||
echo "SKIP: set OPENVOX_RECREATE_SMOKE=1 to run the destructive openvoxserver recreate smoke." >&2
|
|
||||||
exit 64
|
|
||||||
fi
|
|
||||||
|
|
||||||
SUDO="${SUDO:-sudo}"
|
|
||||||
REPO="/etc/puppetlabs/code/environments/production"
|
|
||||||
CORE_SSH_COMMAND_FRAGMENT=".puppet-deploy-key"
|
|
||||||
|
|
||||||
if ! $SUDO systemctl cat openvoxserver >/dev/null 2>&1; then
|
|
||||||
echo "SKIP: systemctl cat openvoxserver failed; refusing to remove a container without a verified systemd recreate path." >&2
|
|
||||||
exit 65
|
|
||||||
fi
|
|
||||||
|
|
||||||
before="$($SUDO podman exec openvoxserver git -C "$REPO" rev-parse --short HEAD)"
|
|
||||||
echo "Before recreate: $before"
|
|
||||||
|
|
||||||
$SUDO systemctl stop openvoxserver
|
|
||||||
$SUDO podman rm openvoxserver 2>/dev/null || true
|
|
||||||
$SUDO systemctl start openvoxserver
|
|
||||||
|
|
||||||
sleep 50
|
|
||||||
|
|
||||||
$SUDO systemctl start puppet-deploy.service
|
|
||||||
sleep 5
|
|
||||||
|
|
||||||
$SUDO systemctl status puppet-deploy.service --no-pager -l
|
|
||||||
|
|
||||||
after="$($SUDO podman exec openvoxserver git -C "$REPO" rev-parse --short origin/master)"
|
|
||||||
echo "After recreate origin/master: $after"
|
|
||||||
|
|
||||||
$SUDO test -d /opt/puppet/code/environments/production/site-modules/profile/manifests
|
|
||||||
|
|
||||||
core_ssh="$($SUDO podman exec openvoxserver git -C "$REPO" config --get core.sshCommand)"
|
|
||||||
case "$core_ssh" in
|
|
||||||
*"$CORE_SSH_COMMAND_FRAGMENT"*) ;;
|
|
||||||
*)
|
|
||||||
echo "FAIL: core.sshCommand does not reference the persisted deploy key." >&2
|
|
||||||
exit 1
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
$SUDO podman exec openvoxserver git -C "$REPO" status --short --branch
|
|
||||||
|
|
||||||
echo "PASS: openvoxserver recreate smoke completed without git safety or deploy-key failure."
|
|
||||||
@@ -13,7 +13,6 @@ public sealed class FleetManifestLintTests
|
|||||||
|
|
||||||
private static readonly HashSet<string> PublicReadOnlyHosts = new(StringComparer.Ordinal)
|
private static readonly HashSet<string> PublicReadOnlyHosts = new(StringComparer.Ordinal)
|
||||||
{
|
{
|
||||||
"brochure.flowercore.io",
|
|
||||||
"dist.flowercore.io",
|
"dist.flowercore.io",
|
||||||
"dns.iamworkin.lan",
|
"dns.iamworkin.lan",
|
||||||
};
|
};
|
||||||
@@ -421,35 +420,6 @@ public sealed class FleetManifestLintTests
|
|||||||
violations.Should().BeEmpty();
|
violations.Should().BeEmpty();
|
||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
|
||||||
[Trait("Question", "Q-RD-DESKTOP-CAP-4")]
|
|
||||||
public void FcDesktop_ResourceQuotaMustCodifyLivePodCpuMemoryCap()
|
|
||||||
{
|
|
||||||
var quota = FcDesktopDocuments()
|
|
||||||
.Single(document => document.Kind == "ResourceQuota" && document.Name == "fc-desktop-cap");
|
|
||||||
|
|
||||||
quota.RelativePath.Should().Be("fc-desktop/resourcequota.yaml");
|
|
||||||
quota.Namespace.Should().Be("fc-desktop");
|
|
||||||
quota.Scalar("spec", "hard", "count/pods").Should().Be("15");
|
|
||||||
quota.Scalar("spec", "hard", "cpu").Should().Be("8");
|
|
||||||
quota.Scalar("spec", "hard", "memory").Should().Be("16Gi");
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
[Trait("Question", "Q-RD-DESKTOP-CAP-5")]
|
|
||||||
public void FcDesktop_ResourceQuotaMustBeArgoCdAdoptableInPlace()
|
|
||||||
{
|
|
||||||
var quota = FcDesktopDocuments()
|
|
||||||
.Single(document => document.Kind == "ResourceQuota" && document.Name == "fc-desktop-cap");
|
|
||||||
|
|
||||||
quota.RelativePath.Should().StartWith("fc-desktop/");
|
|
||||||
quota.Scalar("metadata", "annotations", "argocd.argoproj.io/hook").Should().BeNull();
|
|
||||||
|
|
||||||
var syncOptions = quota.Scalar("metadata", "annotations", "argocd.argoproj.io/sync-options") ?? string.Empty;
|
|
||||||
syncOptions.Should().NotContain("Force=true");
|
|
||||||
syncOptions.Should().NotContain("Replace=true");
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public void FcDeviceManagement_MustShipExpectedManifestSet()
|
public void FcDeviceManagement_MustShipExpectedManifestSet()
|
||||||
{
|
{
|
||||||
@@ -696,13 +666,6 @@ public sealed class FleetManifestLintTests
|
|||||||
.Where(document => document.RelativePath.StartsWith("fc-devicemgmt/", StringComparison.Ordinal))
|
.Where(document => document.RelativePath.StartsWith("fc-devicemgmt/", StringComparison.Ordinal))
|
||||||
.ToList();
|
.ToList();
|
||||||
}
|
}
|
||||||
|
|
||||||
private static IReadOnlyList<ManifestDocument> FcDesktopDocuments()
|
|
||||||
{
|
|
||||||
return Inventory.Documents
|
|
||||||
.Where(document => document.RelativePath.StartsWith("fc-desktop/", StringComparison.Ordinal))
|
|
||||||
.ToList();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
internal sealed class ManifestInventory
|
internal sealed class ManifestInventory
|
||||||
|
|||||||
@@ -1,99 +0,0 @@
|
|||||||
using FluentAssertions;
|
|
||||||
using Xunit;
|
|
||||||
|
|
||||||
namespace BluejayInfraLint.Tests;
|
|
||||||
|
|
||||||
[Trait("Category", "Unit")]
|
|
||||||
public sealed class OpenVoxServerDurabilityTests
|
|
||||||
{
|
|
||||||
private static readonly string Root = FindRepoRoot();
|
|
||||||
private static readonly string RunbookPath = Path.Combine(Root, "docs", "runbooks", "openvoxserver-quadlet-durability.md");
|
|
||||||
private static readonly string SmokePath = Path.Combine(Root, "scripts", "monitoring", "openvox-recreate-smoke.sh");
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public void Runbook_DocumentsHostArtifactAndNonArgoPath()
|
|
||||||
{
|
|
||||||
var runbook = File.ReadAllText(RunbookPath);
|
|
||||||
|
|
||||||
runbook.Should().Contain("noc1 host artifact");
|
|
||||||
runbook.Should().Contain("not an ArgoCD application");
|
|
||||||
runbook.Should().Contain("systemctl cat openvoxserver");
|
|
||||||
runbook.Should().Contain("/etc/containers/systemd/openvoxserver.container");
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public void Runbook_DocumentsCx12LiveApplyState()
|
|
||||||
{
|
|
||||||
var runbook = File.ReadAllText(RunbookPath);
|
|
||||||
|
|
||||||
runbook.Should().Contain("Sprint 32 Cx-12");
|
|
||||||
runbook.Should().Contain("openvoxserver-safeconfig.service");
|
|
||||||
runbook.Should().Contain("/opt/puppet/r10k-deploy.sh");
|
|
||||||
runbook.Should().Contain("HEAD == origin/master");
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public void SmokeScript_IsExplicitlyOptIn()
|
|
||||||
{
|
|
||||||
var smoke = File.ReadAllText(SmokePath);
|
|
||||||
|
|
||||||
smoke.Should().Contain("OPENVOX_RECREATE_SMOKE");
|
|
||||||
smoke.Should().Contain("exit 64");
|
|
||||||
smoke.IndexOf("OPENVOX_RECREATE_SMOKE", StringComparison.Ordinal)
|
|
||||||
.Should().BeLessThan(smoke.IndexOf("systemctl stop openvoxserver", StringComparison.Ordinal));
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public void SmokeScript_RequiresGeneratedSystemdUnitBeforeRemovingContainer()
|
|
||||||
{
|
|
||||||
var smoke = File.ReadAllText(SmokePath);
|
|
||||||
|
|
||||||
smoke.Should().Contain("systemctl cat openvoxserver");
|
|
||||||
smoke.Should().Contain("refusing to remove a container without a verified systemd recreate path");
|
|
||||||
smoke.IndexOf("systemctl cat openvoxserver", StringComparison.Ordinal)
|
|
||||||
.Should().BeLessThan(smoke.IndexOf("podman rm openvoxserver", StringComparison.Ordinal));
|
|
||||||
}
|
|
||||||
|
|
||||||
[Fact]
|
|
||||||
public void Artifacts_DoNotStoreSecretsOrPaidRunnerLabels()
|
|
||||||
{
|
|
||||||
var forbidden = new[]
|
|
||||||
{
|
|
||||||
"BEGIN OPENSSH PRIVATE KEY",
|
|
||||||
"BEGIN RSA PRIVATE KEY",
|
|
||||||
"ubuntu-latest",
|
|
||||||
"windows-latest",
|
|
||||||
"macos-latest",
|
|
||||||
};
|
|
||||||
|
|
||||||
var violations = new[] { RunbookPath, SmokePath }
|
|
||||||
.SelectMany(path =>
|
|
||||||
{
|
|
||||||
var text = File.ReadAllText(path);
|
|
||||||
return forbidden
|
|
||||||
.Where(token => text.Contains(token, StringComparison.OrdinalIgnoreCase))
|
|
||||||
.Select(token => $"{Path.GetRelativePath(Root, path)} contains forbidden token {token}");
|
|
||||||
})
|
|
||||||
.ToList();
|
|
||||||
|
|
||||||
violations.Should().BeEmpty();
|
|
||||||
}
|
|
||||||
|
|
||||||
private static string FindRepoRoot()
|
|
||||||
{
|
|
||||||
var current = new DirectoryInfo(AppContext.BaseDirectory);
|
|
||||||
while (current is not null)
|
|
||||||
{
|
|
||||||
if (Directory.Exists(Path.Combine(current.FullName, "apps"))
|
|
||||||
&& Directory.Exists(Path.Combine(current.FullName, "scripts"))
|
|
||||||
&& File.Exists(Path.Combine(current.FullName, "README.md")))
|
|
||||||
{
|
|
||||||
return current.FullName;
|
|
||||||
}
|
|
||||||
|
|
||||||
current = current.Parent;
|
|
||||||
}
|
|
||||||
|
|
||||||
throw new DirectoryNotFoundException("Could not find bluejay-infra root.");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
package bluejayinfra.public_method_allowlist
|
package bluejayinfra.public_method_allowlist
|
||||||
|
|
||||||
public_hosts := {"brochure.flowercore.io", "dist.flowercore.io", "dns.iamworkin.lan"}
|
public_hosts := {"dist.flowercore.io", "dns.iamworkin.lan"}
|
||||||
|
|
||||||
deny[msg] {
|
deny[msg] {
|
||||||
input.kind == "IngressRoute"
|
input.kind == "IngressRoute"
|
||||||
|
|||||||
Reference in New Issue
Block a user