Compare commits
7 Commits
sprint37/c
...
sprint40/c
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
40b5027ee3 | ||
|
|
ca574c2280 | ||
|
|
09387f90e1 | ||
|
|
e641ceab48 | ||
|
|
c263426ea5 | ||
|
|
bacac067cf | ||
| 914fed08d8 |
@@ -1,27 +0,0 @@
|
||||
# FlowerCore Brochure
|
||||
|
||||
`apps/brochure` hosts the public brochure split from `FlowerCore.Intranet.Web`.
|
||||
ArgoCD's `apps/*` ApplicationSet will create `infra-brochure` after this
|
||||
directory lands on `main`.
|
||||
|
||||
## Runtime
|
||||
|
||||
- Host: `https://brochure.flowercore.io`
|
||||
- Namespace: `brochure`
|
||||
- Deployment: `brochure-web`
|
||||
- Image: `localhost/fc-brochure-web:v20260524-sprint32`
|
||||
- Port: `8080`
|
||||
- Public route method allowlist: `GET` and `HEAD`
|
||||
|
||||
## Operator Actions
|
||||
|
||||
1. Publish and import `localhost/fc-brochure-web:v20260524-sprint32` to every
|
||||
RKE2 node before sync, using the same podman save + `ctr images import`
|
||||
flow as the Intranet deployment.
|
||||
2. Create the Cloudflare DNS record for `brochure.flowercore.io` pointing at
|
||||
the FlowerCore public edge.
|
||||
3. Verify `infra-brochure` appears in ArgoCD, the certificate becomes Ready,
|
||||
and `GET https://brochure.flowercore.io/` returns `200`.
|
||||
|
||||
The route intentionally does not expose `/ops/*` or `/admin/*`; the Brochure
|
||||
web app returns `404` for those paths and Traefik only forwards read methods.
|
||||
@@ -1,131 +0,0 @@
|
||||
# FlowerCore Brochure public host
|
||||
#
|
||||
# Thin Blazor host for public What's New, walkthrough, and gallery content
|
||||
# carved out of FlowerCore.Intranet.Web. The ApplicationSet creates
|
||||
# infra-brochure from this directory after merge.
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: brochure
|
||||
labels:
|
||||
app.kubernetes.io/part-of: flowercore
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: brochure-web
|
||||
namespace: brochure
|
||||
labels:
|
||||
app: brochure-web
|
||||
app.kubernetes.io/name: brochure-web
|
||||
app.kubernetes.io/part-of: flowercore
|
||||
spec:
|
||||
replicas: 1
|
||||
revisionHistoryLimit: 3
|
||||
selector:
|
||||
matchLabels:
|
||||
app: brochure-web
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: brochure-web
|
||||
app.kubernetes.io/name: brochure-web
|
||||
app.kubernetes.io/part-of: flowercore
|
||||
spec:
|
||||
containers:
|
||||
- name: brochure-web
|
||||
image: localhost/fc-brochure-web:v20260524-sprint32
|
||||
imagePullPolicy: Never
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
name: http
|
||||
env:
|
||||
- name: ASPNETCORE_ENVIRONMENT
|
||||
value: Production
|
||||
- name: ASPNETCORE_URLS
|
||||
value: "http://+:8080"
|
||||
resources:
|
||||
requests:
|
||||
cpu: "25m"
|
||||
memory: "128Mi"
|
||||
limits:
|
||||
cpu: "500m"
|
||||
memory: "512Mi"
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1654
|
||||
runAsGroup: 1654
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
volumeMounts:
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
volumes:
|
||||
- name: tmp
|
||||
emptyDir: {}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: brochure-web
|
||||
namespace: brochure
|
||||
labels:
|
||||
app: brochure-web
|
||||
app.kubernetes.io/name: brochure-web
|
||||
app.kubernetes.io/part-of: flowercore
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: brochure-web
|
||||
ports:
|
||||
- name: http
|
||||
port: 8080
|
||||
targetPort: http
|
||||
---
|
||||
apiVersion: cert-manager.io/v1
|
||||
kind: Certificate
|
||||
metadata:
|
||||
name: brochure-web-tls
|
||||
namespace: brochure
|
||||
spec:
|
||||
secretName: brochure-web-tls
|
||||
issuerRef:
|
||||
name: step-ca-acme
|
||||
kind: ClusterIssuer
|
||||
dnsNames:
|
||||
- brochure.flowercore.io
|
||||
duration: 720h
|
||||
renewBefore: 240h
|
||||
---
|
||||
apiVersion: traefik.io/v1alpha1
|
||||
kind: IngressRoute
|
||||
metadata:
|
||||
name: brochure-web-public
|
||||
namespace: brochure
|
||||
spec:
|
||||
entryPoints:
|
||||
- websecure
|
||||
routes:
|
||||
- match: Host(`brochure.flowercore.io`) && (Method(`GET`) || Method(`HEAD`))
|
||||
kind: Rule
|
||||
services:
|
||||
- name: brochure-web
|
||||
port: 8080
|
||||
tls:
|
||||
secretName: brochure-web-tls
|
||||
@@ -47,7 +47,7 @@ spec:
|
||||
fsGroupChangePolicy: OnRootMismatch
|
||||
containers:
|
||||
- name: operator
|
||||
image: localhost/fc-devicemgmt-operator:v20260512-cx5
|
||||
image: localhost/fc-devicemgmt-operator:v20260519-sp34cl3-fix
|
||||
imagePullPolicy: Never
|
||||
ports:
|
||||
- name: metrics
|
||||
|
||||
@@ -4,6 +4,22 @@
|
||||
# Sprint 9+ lane. This manifest is static-valid without requiring the image to
|
||||
# exist yet; import localhost/fc-devicemgmt-web:<tag> to all schedulable RKE2
|
||||
# nodes before letting ArgoCD sync a live rollout.
|
||||
#
|
||||
# SCALED TO 0 — 2026-05-19 morning-routine cleanup.
|
||||
# The Web pod cannot start until TWO upstream gaps close:
|
||||
# 1. MySQL DB instance `flowercore_devicemgmt` (user `fc_devicemgmt`) is
|
||||
# provisioned via fc-mysql Manager. The cluster currently has ZERO
|
||||
# MySqlInstanceCrds and no `mysql.fc-mysql.svc:3306` Service, so the
|
||||
# deployment-web container env `FlowerCore__Database__Host=mysql.fc-mysql.svc`
|
||||
# points at nothing. Provision via the fc-mysql Manager UI/REST/MCP.
|
||||
# 2. 1Password vault item `IAmWorkin/FlowerCore DeviceManagement Runtime`
|
||||
# with 5 fields (DB-Password, mtls-ca.pem, mtls-client.crt, mtls-client.key,
|
||||
# mtls-chain.pem) — see apps/fc-devicemgmt/1password-item.yaml. Mint mTLS
|
||||
# from step-ca-agent ClusterIssuer per ADR-126; DB-Password must match the
|
||||
# password configured for the MySQL user.
|
||||
# Re-enable: change replicas back to 2 after both gaps close. The image tag
|
||||
# in this file (v20260512-cx5) MAY also need a refresh — it predates the
|
||||
# Sprint 34 Cl-3 operator fix; Web may have an analogous bug.
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
@@ -20,7 +36,7 @@ metadata:
|
||||
annotations:
|
||||
flowercore.io/traceability-standard: k8s-pod-ownership-and-traceability-standard
|
||||
spec:
|
||||
replicas: 2
|
||||
replicas: 0
|
||||
revisionHistoryLimit: 3
|
||||
selector:
|
||||
matchLabels:
|
||||
|
||||
@@ -656,14 +656,15 @@ data:
|
||||
summary: "Print queue backlog on edge2 ({{ $value }} active jobs)"
|
||||
description: "CUPS has {{ $value }} active jobs queued. Possible printer jam, USB disconnect, or paper out."
|
||||
|
||||
# Paper roll lifecycle alerts (XL Track I, 2026-04-26).
|
||||
# Source-of-truth gauge: print_paper_remaining_percent (Print.Web OTEL,
|
||||
# hydrated on startup from the active PaperRoll row).
|
||||
# alert_channel=thermal_print routes through irc-notify -> Print.Web
|
||||
# /api/print/alert so the printer announces its own paper-out warning
|
||||
# on its remaining paper. Self-referential humor + operator nudge.
|
||||
# Printer hardware and paper-roll lifecycle alerts.
|
||||
# print_printer_online: 1 when the transport is reachable/selected.
|
||||
# print_printer_state enum: 0 unknown, 1 online, 2 offline,
|
||||
# 3 paper_depleted, 4 jam, 5 head_error, 6 cover_open.
|
||||
# Offline/jam/cover alerts stay IRC-only. Paper depleted and head
|
||||
# error may route to the thermal digest only when the printer is
|
||||
# online enough to make that useful.
|
||||
- alert: PrintPaperRollLow
|
||||
expr: print_paper_remaining_percent{job="printweb-otel"} < 10 and print_paper_remaining_percent{job="printweb-otel"} > 5
|
||||
expr: (print_paper_remaining_percent{job="printweb-otel"} < 10 and print_paper_remaining_percent{job="printweb-otel"} > 5) and print_printer_online{job="printweb-otel"} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
@@ -672,15 +673,59 @@ data:
|
||||
summary: "Print roll low on edge2 ({{ $value | printf \"%.1f\" }}% remaining)"
|
||||
description: "NuPrint 210 paper roll has {{ $value | printf \"%.1f\" }}% remaining. Operator should load a fresh roll soon. Run /api/paper/status for the precise mm + estimated jobs left."
|
||||
|
||||
- alert: PrinterOfflineWarning
|
||||
expr: print_printer_state{job="printweb-otel"} == 2
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
service: print-web
|
||||
alert_channel: irc
|
||||
annotations:
|
||||
summary: "Print.Web printer offline on edge2"
|
||||
description: "Print.Web reports the NuPrint 210 transport is offline or unreachable. IRC-only by design: do not thermal-print an alert when the thermal printer itself is offline."
|
||||
|
||||
- alert: PrintPaperRollCritical
|
||||
expr: print_paper_remaining_percent{job="printweb-otel"} <= 5
|
||||
expr: print_printer_state{job="printweb-otel"} == 3 and print_printer_online{job="printweb-otel"} == 1
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
alert_channel: thermal_print
|
||||
annotations:
|
||||
summary: "Print roll critical on edge2 ({{ $value | printf \"%.1f\" }}% remaining)"
|
||||
description: "NuPrint 210 paper roll at {{ $value | printf \"%.1f\" }}% — load a new roll NOW. The 50ft roll has a ~12% red-stripe zone; once paper passes that, the printer can run dry mid-job."
|
||||
summary: "Print paper depleted on edge2"
|
||||
description: "NuPrint 210 reports paper depleted while the printer is still online. Load a new roll, drain the hardware buffer if needed, then replay DeadLetter jobs from /print-log."
|
||||
|
||||
- alert: PrinterJamWarning
|
||||
expr: print_printer_state{job="printweb-otel"} == 4
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
service: print-web
|
||||
alert_channel: irc
|
||||
annotations:
|
||||
summary: "Print.Web printer jam on edge2"
|
||||
description: "Print.Web reports a paper/cutter jam state. IRC-only: clear the jam, drain the hardware buffer if bytes were queued, then retry affected jobs."
|
||||
|
||||
- alert: PrinterHeadErrorCritical
|
||||
expr: print_printer_state{job="printweb-otel"} == 5
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
service: print-web
|
||||
alert_channel: thermal_print
|
||||
annotations:
|
||||
summary: "Print.Web printer head error on edge2"
|
||||
description: "Print.Web reports a thermal head or unrecoverable printer error. Critical routing may enter the thermal digest per existing policy; IRC remains the primary triage stream."
|
||||
|
||||
- alert: PrinterCoverOpenWarning
|
||||
expr: print_printer_state{job="printweb-otel"} == 6
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
service: print-web
|
||||
alert_channel: irc
|
||||
annotations:
|
||||
summary: "Print.Web printer cover open on edge2"
|
||||
description: "Print.Web reports the printer cover/lid is open. IRC-only: close the cover and verify /api/print/status before retrying jobs."
|
||||
|
||||
- alert: PrintJobDeadLetter
|
||||
expr: increase(print_jobs_dead_letter_total[15m]) > 0
|
||||
@@ -1273,24 +1318,55 @@ metadata:
|
||||
data:
|
||||
notify.py: |
|
||||
#!/usr/bin/env python3
|
||||
"""HTTP->IRC alert relay with thermal printer forwarding for Grafana webhooks.
|
||||
Listens on :9119, posts to #alerts on UnrealIRCd via raw IRC protocol.
|
||||
Alerts tagged alert_channel=thermal_print also POST to Print.Web /api/print/alert.
|
||||
"""HTTP->IRC alert relay with thermal-printer DIGEST forwarding.
|
||||
|
||||
Listens on :9119, posts to #alerts on UnrealIRCd, forwards to Print.Web
|
||||
/api/print/alert. Thermal printing is BATCHED into hourly digests by
|
||||
default so the printer no longer spam-fires per Grafana webhook.
|
||||
|
||||
Routing (per Grafana webhook alert):
|
||||
- IRC: always per-event (operator likes the stream)
|
||||
- Thermal printer:
|
||||
* severity in {critical,disaster,page} OR
|
||||
label alert_channel=thermal_print_immediate -> print NOW
|
||||
* label alert_channel=thermal_print -> enqueue into hourly digest
|
||||
* everything else -> IRC only
|
||||
- RESOLVED webhooks remove the alert from the digest buffer
|
||||
|
||||
Env vars (defaults preserve old behavior on first deploy):
|
||||
THERMAL_PRINT_ENABLED default "true" - master kill switch
|
||||
BATCH_INTERVAL_MIN default "60" - minutes between digest prints
|
||||
BATCH_MAX_PENDING default "50" - force-flush threshold
|
||||
|
||||
HTTP surface:
|
||||
POST / - Grafana webhook entry
|
||||
POST /flush - manual digest flush (idempotent)
|
||||
GET / - status + config + buffer depth + stats
|
||||
"""
|
||||
import json, socket, sys, time
|
||||
import json, os, socket, sys, threading, time
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from http.server import HTTPServer, BaseHTTPRequestHandler
|
||||
from urllib.request import Request, urlopen
|
||||
from urllib.error import URLError
|
||||
|
||||
IRC_HOST = "unrealircd.irc.svc" # short name: CoreDNS ndots:5 + iamworkin.lan template hijacks full .cluster.local (see memory)
|
||||
IRC_PORT = 6667
|
||||
IRC_NICK = "grafana-bot"
|
||||
IRC_CHANNEL = "#alerts"
|
||||
PRINT_WEB_URL = "http://10.0.57.16:5200/api/print/alert"
|
||||
PRINT_ENABLED = True
|
||||
THERMAL_PRINT_ENABLED = os.environ.get("THERMAL_PRINT_ENABLED", "true").lower() == "true"
|
||||
BATCH_INTERVAL_MIN = int(os.environ.get("BATCH_INTERVAL_MIN", "60"))
|
||||
BATCH_MAX_PENDING = int(os.environ.get("BATCH_MAX_PENDING", "50"))
|
||||
|
||||
IRC_HOST = os.environ.get("IRC_HOST", "unrealircd.irc.svc")
|
||||
IRC_PORT = int(os.environ.get("IRC_PORT", "6667"))
|
||||
IRC_NICK = os.environ.get("IRC_NICK", "grafana-bot")
|
||||
IRC_CHANNEL = os.environ.get("IRC_CHANNEL", "#alerts")
|
||||
PRINT_WEB_URL = os.environ.get("PRINT_WEB_URL", "http://10.0.57.16:5200/api/print/alert")
|
||||
|
||||
_buffer_lock = threading.Lock()
|
||||
_buffer = {} # fingerprint -> {"alert": dict, "first_seen": float, "last_seen": float}
|
||||
_last_flush_time = time.time()
|
||||
_stats = {"webhooks_received": 0, "irc_sent": 0, "print_immediate": 0,
|
||||
"digest_flushed": 0, "buffer_dedup": 0, "buffer_added": 0,
|
||||
"buffer_resolved": 0, "started_at": time.time()}
|
||||
|
||||
def send_irc(message):
|
||||
"""Connect, handle PING, join, send, quit."""
|
||||
try:
|
||||
sock = socket.create_connection((IRC_HOST, IRC_PORT), timeout=15)
|
||||
sock.sendall(f"NICK {IRC_NICK}\r\n".encode())
|
||||
@@ -1323,52 +1399,137 @@ data:
|
||||
time.sleep(0.5)
|
||||
sock.sendall(b"QUIT :alert delivered\r\n")
|
||||
sock.close()
|
||||
_stats["irc_sent"] += 1
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"[irc-notify] IRC send failed: {e}", file=sys.stderr)
|
||||
return False
|
||||
|
||||
def send_thermal_print(alert):
|
||||
if not PRINT_ENABLED: return
|
||||
labels = alert.get("labels", {})
|
||||
annotations = alert.get("annotations", {})
|
||||
status = alert.get("status", "firing").upper()
|
||||
summary = annotations.get("summary", "")
|
||||
description = annotations.get("description", "")
|
||||
runbook = annotations.get("runbook", "")
|
||||
# Build a useful message: summary + description + runbook steps
|
||||
parts = []
|
||||
if summary: parts.append(summary)
|
||||
if description and description != summary: parts.append(description)
|
||||
if runbook: parts.append("STEPS: " + runbook)
|
||||
message = " | ".join(parts) if parts else labels.get("alertname", "Unknown alert")
|
||||
payload = {
|
||||
"title": labels.get("alertname", "Unknown"),
|
||||
"severity": labels.get("severity", "warning").capitalize(),
|
||||
"host": labels.get("instance", labels.get("host", "unknown")),
|
||||
"message": message,
|
||||
"eventId": alert.get("fingerprint", ""),
|
||||
"source": "Grafana",
|
||||
"status": "RESOLVED" if status == "RESOLVED" else "PROBLEM",
|
||||
"acknowledged": False
|
||||
}
|
||||
def post_thermal(payload, kind):
|
||||
if not THERMAL_PRINT_ENABLED:
|
||||
print(f"[irc-notify] thermal disabled; skip {kind} ({payload.get('title','?')[:40]})", file=sys.stderr)
|
||||
return False
|
||||
try:
|
||||
req = Request(PRINT_WEB_URL, data=json.dumps(payload).encode("utf-8"),
|
||||
headers={"Content-Type": "application/json"}, method="POST")
|
||||
resp = urlopen(req, timeout=10)
|
||||
print(f"[irc-notify] Thermal print sent: {resp.read().decode()}", file=sys.stderr)
|
||||
if kind == "immediate": _stats["print_immediate"] += 1
|
||||
print(f"[irc-notify] thermal {kind} sent: {payload.get('title','?')[:50]}", file=sys.stderr)
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"[irc-notify] Thermal print failed: {e}", file=sys.stderr)
|
||||
|
||||
def should_print(alert):
|
||||
labels = alert.get("labels", {})
|
||||
if labels.get("alert_channel") == "thermal_print": return True
|
||||
if labels.get("severity", "").lower() in ("critical", "disaster"): return True
|
||||
if alert.get("status", "").upper() == "RESOLVED": return False
|
||||
print(f"[irc-notify] thermal {kind} failed: {e}", file=sys.stderr)
|
||||
return False
|
||||
|
||||
def fingerprint_of(alert):
|
||||
fp = alert.get("fingerprint", "")
|
||||
if fp: return fp
|
||||
labels = alert.get("labels", {})
|
||||
target = labels.get("pod") or labels.get("instance") or labels.get("deployment") or labels.get("statefulset") or labels.get("namespace") or ""
|
||||
return f"{labels.get('alertname','?')}/{labels.get('namespace','')}/{target}"
|
||||
|
||||
def is_critical(alert):
|
||||
return alert.get("labels", {}).get("severity", "").lower() in ("critical", "disaster", "page")
|
||||
|
||||
def is_immediate_label(alert):
|
||||
return alert.get("labels", {}).get("alert_channel") == "thermal_print_immediate"
|
||||
|
||||
def is_batched_label(alert):
|
||||
return alert.get("labels", {}).get("alert_channel") == "thermal_print"
|
||||
|
||||
def add_to_digest(alert):
|
||||
"""Add an alert to the digest buffer. Returns True if the buffer GREW
|
||||
(new fingerprint), False if it was a dedup, resolution, or no-op.
|
||||
"""
|
||||
if not THERMAL_PRINT_ENABLED: return False
|
||||
fp = fingerprint_of(alert)
|
||||
status = alert.get("status", "firing").lower()
|
||||
with _buffer_lock:
|
||||
if status == "resolved":
|
||||
if fp in _buffer:
|
||||
del _buffer[fp]
|
||||
_stats["buffer_resolved"] += 1
|
||||
return False
|
||||
if fp in _buffer:
|
||||
_buffer[fp]["last_seen"] = time.time()
|
||||
_buffer[fp]["alert"] = alert
|
||||
_stats["buffer_dedup"] += 1
|
||||
return False
|
||||
_buffer[fp] = {"alert": alert, "first_seen": time.time(), "last_seen": time.time()}
|
||||
_stats["buffer_added"] += 1
|
||||
return True
|
||||
|
||||
def build_digest_payload():
|
||||
with _buffer_lock:
|
||||
items = list(_buffer.values())
|
||||
if not items: return None
|
||||
by_name = defaultdict(list)
|
||||
for item in items:
|
||||
labels = item["alert"].get("labels", {})
|
||||
by_name[labels.get("alertname", "Unknown")].append(item)
|
||||
lines = []
|
||||
for name, group in sorted(by_name.items()):
|
||||
targets = []
|
||||
for it in group[:5]:
|
||||
labels = it["alert"].get("labels", {})
|
||||
t = (labels.get("pod") or labels.get("instance") or labels.get("deployment")
|
||||
or labels.get("statefulset") or labels.get("namespace") or "?")
|
||||
targets.append(t)
|
||||
more = f" (+{len(group)-5})" if len(group) > 5 else ""
|
||||
sevs = sorted({it["alert"].get("labels", {}).get("severity", "warning") for it in group})
|
||||
lines.append(f"[{'/'.join(sevs)}] {name} x{len(group)}: {', '.join(targets)}{more}")
|
||||
now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
|
||||
title = f"Alert digest: {len(items)} firing"
|
||||
body = "\n".join([
|
||||
f"=== {title} ===",
|
||||
f"as of {now}",
|
||||
"",
|
||||
*lines,
|
||||
"",
|
||||
"Stream: #alerts (IRC) | Triage: grafana-noc1.iamworkin.lan",
|
||||
"Force-flush: POST irc-notify.monitoring.svc:9119/flush",
|
||||
])
|
||||
return {"title": title, "severity": "Warning", "host": "monitoring",
|
||||
"message": body, "eventId": f"digest-{int(time.time())}",
|
||||
"source": "Grafana digest", "status": "PROBLEM", "acknowledged": False}
|
||||
|
||||
def flush_digest():
|
||||
payload = build_digest_payload()
|
||||
if payload is None:
|
||||
print("[irc-notify] flush: buffer empty, no digest sent", file=sys.stderr)
|
||||
return False
|
||||
sent = post_thermal(payload, "digest")
|
||||
with _buffer_lock:
|
||||
_buffer.clear()
|
||||
if sent: _stats["digest_flushed"] += 1
|
||||
return sent
|
||||
|
||||
def digest_loop():
|
||||
global _last_flush_time
|
||||
while True:
|
||||
try:
|
||||
now = time.time()
|
||||
elapsed = now - _last_flush_time
|
||||
if elapsed >= BATCH_INTERVAL_MIN * 60:
|
||||
print(f"[irc-notify] digest tick: interval reached ({BATCH_INTERVAL_MIN}m); buffer={len(_buffer)}", file=sys.stderr)
|
||||
flush_digest()
|
||||
_last_flush_time = now
|
||||
elif len(_buffer) >= BATCH_MAX_PENDING:
|
||||
print(f"[irc-notify] digest tick: buffer full ({len(_buffer)}); force flush", file=sys.stderr)
|
||||
flush_digest()
|
||||
_last_flush_time = now
|
||||
time.sleep(15)
|
||||
except Exception as e:
|
||||
print(f"[irc-notify] digest loop error: {e}", file=sys.stderr)
|
||||
time.sleep(60)
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def do_POST(self):
|
||||
if self.path == "/flush":
|
||||
ok = flush_digest()
|
||||
self.send_response(200); self.send_header("Content-Type", "application/json"); self.end_headers()
|
||||
self.wfile.write(json.dumps({"flushed": ok, "buffer_after": len(_buffer)}).encode())
|
||||
return
|
||||
_stats["webhooks_received"] += 1
|
||||
length = int(self.headers.get("Content-Length", 0))
|
||||
body = json.loads(self.rfile.read(length)) if length else {}
|
||||
for alert in body.get("alerts", []):
|
||||
@@ -1383,22 +1544,56 @@ data:
|
||||
msg = f"{icon}{sev_tag} {name}: {summary}"
|
||||
if desc: msg += f"\n {desc}"
|
||||
send_irc(msg)
|
||||
if should_print(alert): send_thermal_print(alert)
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.end_headers()
|
||||
# Thermal routing — EVERYTHING (including criticals) goes into
|
||||
# the hourly digest. Only the explicit `alert_channel=thermal_print_immediate`
|
||||
# label bypasses, and even that flushes-the-current-digest rather
|
||||
# than printing a standalone job, so the same fingerprint can't
|
||||
# spam the printer per webhook cycle.
|
||||
if status == "RESOLVED":
|
||||
add_to_digest(alert) # removes from buffer
|
||||
continue
|
||||
if is_immediate_label(alert):
|
||||
# Explicit opt-in for "paper this NOW" — first arrival of a
|
||||
# new fingerprint triggers an immediate digest flush; repeat
|
||||
# webhooks for the same fingerprint dedupe in the buffer
|
||||
# until the next interval or until the alert resolves.
|
||||
new_in_buffer = add_to_digest(alert)
|
||||
if new_in_buffer:
|
||||
global _last_flush_time
|
||||
flush_digest()
|
||||
_last_flush_time = time.time()
|
||||
elif is_critical(alert) or is_batched_label(alert):
|
||||
add_to_digest(alert)
|
||||
# else: IRC-only (warnings without thermal_print label)
|
||||
self.send_response(200); self.send_header("Content-Type", "application/json"); self.end_headers()
|
||||
self.wfile.write(b'{"status":"ok"}')
|
||||
|
||||
def do_GET(self):
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.end_headers()
|
||||
self.wfile.write(json.dumps({"service":"irc-notify","thermal_print":PRINT_ENABLED}).encode())
|
||||
self.send_response(200); self.send_header("Content-Type", "application/json"); self.end_headers()
|
||||
with _buffer_lock:
|
||||
alertnames = sorted({it["alert"].get("labels", {}).get("alertname", "?") for it in _buffer.values()})
|
||||
depth = len(_buffer)
|
||||
info = {
|
||||
"service": "irc-notify",
|
||||
"config": {"thermal_print_enabled": THERMAL_PRINT_ENABLED,
|
||||
"batch_interval_min": BATCH_INTERVAL_MIN,
|
||||
"batch_max_pending": BATCH_MAX_PENDING,
|
||||
"irc_target": f"{IRC_HOST}:{IRC_PORT} {IRC_CHANNEL}",
|
||||
"print_web_url": PRINT_WEB_URL},
|
||||
"buffer": {"depth": depth, "alertnames": alertnames,
|
||||
"seconds_since_last_flush": int(time.time() - _last_flush_time),
|
||||
"seconds_until_next_flush": max(0, int(BATCH_INTERVAL_MIN*60 - (time.time() - _last_flush_time)))},
|
||||
"stats": _stats,
|
||||
}
|
||||
self.wfile.write(json.dumps(info, indent=2).encode())
|
||||
|
||||
def log_message(self, format, *args):
|
||||
print(f"[irc-notify] {args[0]}", file=sys.stderr)
|
||||
|
||||
if __name__ == "__main__":
|
||||
threading.Thread(target=digest_loop, daemon=True).start()
|
||||
server = HTTPServer(("0.0.0.0", 9119), Handler)
|
||||
print(f"IRC alert relay :9119 -> {IRC_HOST}:{IRC_PORT} {IRC_CHANNEL} (thermal: {PRINT_ENABLED})")
|
||||
print(f"[irc-notify] :9119 -> IRC {IRC_HOST}:{IRC_PORT} {IRC_CHANNEL} | thermal={'ON' if THERMAL_PRINT_ENABLED else 'OFF'} | digest={BATCH_INTERVAL_MIN}m max={BATCH_MAX_PENDING}", file=sys.stderr)
|
||||
server.serve_forever()
|
||||
|
||||
# =============================================================================
|
||||
@@ -3485,6 +3680,146 @@ data:
|
||||
relativeTimeRange: {from: 120, to: 0}
|
||||
datasourceUid: __expr__
|
||||
model: {type: threshold, expression: B, conditions: [{evaluator: {params: [600], type: gt}}], refId: C}
|
||||
- orgId: 1
|
||||
name: Print Services
|
||||
folder: Print Alerts
|
||||
interval: 1m
|
||||
rules:
|
||||
- uid: printer-offline-warning
|
||||
title: PrinterOfflineWarning
|
||||
condition: C
|
||||
for: 2m
|
||||
noDataState: OK
|
||||
execErrState: OK
|
||||
annotations:
|
||||
summary: "Print.Web printer offline on edge2"
|
||||
description: "Print.Web reports the NuPrint 210 transport is offline or unreachable. IRC-only by design: do not thermal-print an alert when the thermal printer itself is offline."
|
||||
runbook: "1. Check edge2 power/network 2. Check USB/CUPS queue 3. Open https://print.iamworkin.lan/admin 4. Do not force thermal routing for offline alerts."
|
||||
labels:
|
||||
severity: warning
|
||||
service: print-web
|
||||
alert_channel: irc
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: {from: 120, to: 0}
|
||||
datasourceUid: prometheus
|
||||
model: {expr: 'print_printer_state{job="printweb-otel"} == 2', instant: true, refId: A}
|
||||
- refId: B
|
||||
relativeTimeRange: {from: 120, to: 0}
|
||||
datasourceUid: __expr__
|
||||
model: {type: reduce, expression: A, reducer: last, refId: B}
|
||||
- refId: C
|
||||
relativeTimeRange: {from: 120, to: 0}
|
||||
datasourceUid: __expr__
|
||||
model: {type: threshold, expression: B, conditions: [{evaluator: {params: [0], type: gt}}], refId: C}
|
||||
- uid: print-paper-roll-critical
|
||||
title: PrintPaperRollCritical
|
||||
condition: C
|
||||
for: 2m
|
||||
noDataState: OK
|
||||
execErrState: OK
|
||||
annotations:
|
||||
summary: "Print paper depleted on edge2"
|
||||
description: "NuPrint 210 reports paper depleted while the printer is still online. Load a new roll, drain the hardware buffer if needed, then replay DeadLetter jobs from /print-log."
|
||||
runbook: "1. Load a fresh roll 2. Drain the hardware buffer if paper-out happened mid-job 3. Open https://print.iamworkin.lan/print-log 4. Retry DeadLetter jobs after the state clears."
|
||||
labels:
|
||||
severity: critical
|
||||
service: print-web
|
||||
alert_channel: thermal_print
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: {from: 120, to: 0}
|
||||
datasourceUid: prometheus
|
||||
model: {expr: 'print_printer_state{job="printweb-otel"} == 3 and print_printer_online{job="printweb-otel"} == 1', instant: true, refId: A}
|
||||
- refId: B
|
||||
relativeTimeRange: {from: 120, to: 0}
|
||||
datasourceUid: __expr__
|
||||
model: {type: reduce, expression: A, reducer: last, refId: B}
|
||||
- refId: C
|
||||
relativeTimeRange: {from: 120, to: 0}
|
||||
datasourceUid: __expr__
|
||||
model: {type: threshold, expression: B, conditions: [{evaluator: {params: [0], type: gt}}], refId: C}
|
||||
- uid: printer-jam-warning
|
||||
title: PrinterJamWarning
|
||||
condition: C
|
||||
for: 2m
|
||||
noDataState: OK
|
||||
execErrState: OK
|
||||
annotations:
|
||||
summary: "Print.Web printer jam on edge2"
|
||||
description: "Print.Web reports a paper/cutter jam state. IRC-only: clear the jam, drain the hardware buffer if bytes were queued, then retry affected jobs."
|
||||
runbook: "1. Clear paper/cutter path 2. Drain hardware buffer if CUPS queued bytes 3. Verify /api/print/status 4. Retry affected jobs."
|
||||
labels:
|
||||
severity: warning
|
||||
service: print-web
|
||||
alert_channel: irc
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: {from: 120, to: 0}
|
||||
datasourceUid: prometheus
|
||||
model: {expr: 'print_printer_state{job="printweb-otel"} == 4', instant: true, refId: A}
|
||||
- refId: B
|
||||
relativeTimeRange: {from: 120, to: 0}
|
||||
datasourceUid: __expr__
|
||||
model: {type: reduce, expression: A, reducer: last, refId: B}
|
||||
- refId: C
|
||||
relativeTimeRange: {from: 120, to: 0}
|
||||
datasourceUid: __expr__
|
||||
model: {type: threshold, expression: B, conditions: [{evaluator: {params: [0], type: gt}}], refId: C}
|
||||
- uid: printer-head-error-critical
|
||||
title: PrinterHeadErrorCritical
|
||||
condition: C
|
||||
for: 2m
|
||||
noDataState: OK
|
||||
execErrState: OK
|
||||
annotations:
|
||||
summary: "Print.Web printer head error on edge2"
|
||||
description: "Print.Web reports a thermal head or unrecoverable printer error. Critical routing may enter the thermal digest per existing policy; IRC remains the primary triage stream."
|
||||
runbook: "1. Let the printer cool if overheated 2. Power-cycle only after checking queued jobs 3. Verify /api/print/status 4. Retry jobs after the state clears."
|
||||
labels:
|
||||
severity: critical
|
||||
service: print-web
|
||||
alert_channel: thermal_print
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: {from: 120, to: 0}
|
||||
datasourceUid: prometheus
|
||||
model: {expr: 'print_printer_state{job="printweb-otel"} == 5', instant: true, refId: A}
|
||||
- refId: B
|
||||
relativeTimeRange: {from: 120, to: 0}
|
||||
datasourceUid: __expr__
|
||||
model: {type: reduce, expression: A, reducer: last, refId: B}
|
||||
- refId: C
|
||||
relativeTimeRange: {from: 120, to: 0}
|
||||
datasourceUid: __expr__
|
||||
model: {type: threshold, expression: B, conditions: [{evaluator: {params: [0], type: gt}}], refId: C}
|
||||
- uid: printer-cover-open-warning
|
||||
title: PrinterCoverOpenWarning
|
||||
condition: C
|
||||
for: 2m
|
||||
noDataState: OK
|
||||
execErrState: OK
|
||||
annotations:
|
||||
summary: "Print.Web printer cover open on edge2"
|
||||
description: "Print.Web reports the printer cover/lid is open. IRC-only: close the cover and verify /api/print/status before retrying jobs."
|
||||
runbook: "1. Close the printer cover 2. Verify /api/print/status returns online 3. Retry affected jobs only after the state clears."
|
||||
labels:
|
||||
severity: warning
|
||||
service: print-web
|
||||
alert_channel: irc
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: {from: 120, to: 0}
|
||||
datasourceUid: prometheus
|
||||
model: {expr: 'print_printer_state{job="printweb-otel"} == 6', instant: true, refId: A}
|
||||
- refId: B
|
||||
relativeTimeRange: {from: 120, to: 0}
|
||||
datasourceUid: __expr__
|
||||
model: {type: reduce, expression: A, reducer: last, refId: B}
|
||||
- refId: C
|
||||
relativeTimeRange: {from: 120, to: 0}
|
||||
datasourceUid: __expr__
|
||||
model: {type: threshold, expression: B, conditions: [{evaluator: {params: [0], type: gt}}], refId: C}
|
||||
- orgId: 1
|
||||
name: CI Runners
|
||||
folder: CI Alerts
|
||||
|
||||
@@ -304,7 +304,7 @@ public sealed class FleetManifestLintTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Monitoring_MustAlertWhenLinuxRunnerDeploymentIsUnavailable()
|
||||
public void Monitoring_MustIncludeRequiredAlertRoutingGuards()
|
||||
{
|
||||
var monitoring = File.ReadAllText(Path.Combine(Inventory.BluejayRoot, "apps", "monitoring", "noc-monitoring.yaml"));
|
||||
|
||||
@@ -315,6 +315,15 @@ public sealed class FleetManifestLintTests
|
||||
monitoring.Should().Contain("folder: CI Alerts");
|
||||
monitoring.Should().Contain("uid: linux-runner-offline");
|
||||
monitoring.Should().Contain("alert_channel: irc");
|
||||
|
||||
monitoring.Should().Contain("PrinterOfflineWarning");
|
||||
monitoring.Should().Contain("expr: print_printer_state{job=\"printweb-otel\"} == 2");
|
||||
monitoring.Should().Contain("IRC-only by design: do not thermal-print an alert when the thermal printer itself is offline.");
|
||||
monitoring.Should().Contain("PrintPaperRollCritical");
|
||||
monitoring.Should().Contain("expr: print_printer_state{job=\"printweb-otel\"} == 3 and print_printer_online{job=\"printweb-otel\"} == 1");
|
||||
monitoring.Should().Contain("PrinterJamWarning");
|
||||
monitoring.Should().Contain("PrinterHeadErrorCritical");
|
||||
monitoring.Should().Contain("PrinterCoverOpenWarning");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
||||
Reference in New Issue
Block a user