Compare commits
10 Commits
claude/ci1
...
codex/sign
| Author | SHA1 | Date | |
|---|---|---|---|
|
5cf665e77d | ||
|
|
fc2aca0e9e | ||
|
|
ba18c52130 | ||
|
|
9f6dc1a9d5 | ||
|
|
0bf47dfa33 | ||
|
|
87a7d7c70a | ||
|
|
1c4145a581 | ||
|
|
c50a403f74 | ||
|
|
fb7bd10528 | ||
|
|
6c21d14a98 |
@@ -58,7 +58,7 @@ spec:
|
||||
nodeName: rke2-server
|
||||
containers:
|
||||
- name: web
|
||||
image: localhost/fc-updater-web:v20260507-public-privacy
|
||||
image: localhost/fc-updater-web:v20260508-pub3-deepening-2bdf108
|
||||
imagePullPolicy: Never
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
|
||||
@@ -6,14 +6,29 @@
|
||||
# `bluejay-ws-sandbox-1` runner placeholder. Andrew explicitly does NOT want
|
||||
# BLUEJAY-WS registered as a runner (workstation has personal/operator state).
|
||||
#
|
||||
# Status (2026-05-08): STAGED ONLY — DO NOT APPLY without operator review.
|
||||
# See docs/infrastructure/windows-server-build-runner-plan.md "Phase 1 readiness gate".
|
||||
# Storage layout (2026-05-08):
|
||||
# * ISO is now sourced from Synology NFS (Path B) — see
|
||||
# win2025-iso-nfs-pv.yaml. The Longhorn Filesystem PVC
|
||||
# `windows-server-2025-iso` below is RETAINED but UNUSED so the prior
|
||||
# CDI upload state is preserved as a fallback (and so ArgoCD doesn't
|
||||
# prune it on this commit). It can be deleted in a follow-up commit
|
||||
# after the NFS path is proven on a successful Windows install.
|
||||
#
|
||||
# Prerequisites that MUST be satisfied first:
|
||||
# 1. Windows Server 2025 ISO populated into the `windows-server-2025-iso` PVC
|
||||
# (operator interactive step — Microsoft Evaluation Center download).
|
||||
# 2. Either Multus + PROD VLAN NAD (preferred) OR pod-network only (this YAML).
|
||||
# 3. KubeVirt CR feature gates: none required for non-persistent vTPM.
|
||||
# Status (2026-05-08): LIVE — Phase 1 prereqs satisfied:
|
||||
# * Multus CNI v4.2.2 thick-plugin DaemonSet running on all 3 RKE2 nodes
|
||||
# (apps/multus/multus.yaml; ApplicationSet `infra-multus` Synced/Healthy)
|
||||
# * CDI v1.65.0 operator + CR Deployed (apps/cdi/; ApplicationSet
|
||||
# `infra-cdi` Synced/Healthy; uploadproxy reachable via kubectl port-forward)
|
||||
# * Windows Server 2025 ISO uploaded via CDI virtctl image-upload to
|
||||
# PVC windows-server-2025-iso (7.7 GiB → 10Gi PVC, Bound, Upload Complete)
|
||||
# * Local Administrator password generated, stored in 1Password vault
|
||||
# IAmWorkin (qaphopopkryhbg353ukzhhuqoq) item id h3ix4mgfk65gmkcmvh6ly3d3hu
|
||||
# * NetworkAttachmentDefinition prod-vlan57 registered (apps/kubevirt-vms/
|
||||
# prod-vlan57-nad.yaml). VM still uses pod-network masquerade until Phase 1.5
|
||||
# host bridge work lands (Puppet br-prod + enp86s0.57); switching is a
|
||||
# one-line YAML edit + git push.
|
||||
#
|
||||
# See docs/infrastructure/windows-server-build-runner-plan.md "Phase 1 readiness gate".
|
||||
#
|
||||
# Network choice in this draft: **pod-network fallback** (Calico default).
|
||||
# Outbound-only is fine for the Updater Sandbox E2E runner workload (the runner
|
||||
@@ -42,21 +57,49 @@ metadata:
|
||||
pod-security.kubernetes.io/enforce: privileged
|
||||
|
||||
---
|
||||
# ISO PVC — operator must populate this before applying the VM manifest.
|
||||
# Population paths (see plan doc "Phase 1 readiness gate", section 2):
|
||||
# Path A — manual upload via helper pod + kubectl cp
|
||||
# Path B — install CDI, then DataVolume HTTP import
|
||||
# ISO PVC — populated via CDI virtctl image-upload (CDI is now installed).
|
||||
#
|
||||
# **Volume mode (2026-05-08 status):** Filesystem-mode PVC. A migration to
|
||||
# `volumeMode: Block` via DataVolume was attempted to address an OVMF SATA
|
||||
# CDROM read timeout, but CDI v1.65.0's upload-target pod runs as uid 107
|
||||
# with `capabilities.drop: [ALL]` and cannot open the underlying block
|
||||
# device (`blockdev: cannot open /dev/cdi-block-volume: Permission denied`).
|
||||
# Reverted to Filesystem PVC pending one of:
|
||||
# - CDI deployment override granting CAP_SYS_RAWIO to upload pod
|
||||
# - Pre-populated PVC via privileged init pod that dd's the ISO directly
|
||||
# - Migration to a different storage class that exposes block devices
|
||||
# differently (e.g. iSCSI, where Longhorn's CSI mount path may behave
|
||||
# differently)
|
||||
#
|
||||
# Population workflow (this PVC, Filesystem mode):
|
||||
# 1. virtctl --kubeconfig $env:USERPROFILE\.kube\rke2.yaml image-upload pvc \
|
||||
# windows-server-2025-iso -n kubevirt-vms \
|
||||
# --image-path "$env:USERPROFILE\Downloads\en-us_windows_server_2025_updated_march_2026_x64_dvd_8e06425a.iso" \
|
||||
# --size 10Gi --storage-class longhorn --access-mode ReadWriteOnce \
|
||||
# --uploadproxy-url https://localhost:8443 --insecure
|
||||
# (--uploadproxy-url uses port-forward in practice: `kubectl port-forward
|
||||
# -n cdi service/cdi-uploadproxy 8443:443 &` first.)
|
||||
#
|
||||
# **Open boot issue:** even with the ISO at bootOrder:1, OVMF console showed:
|
||||
# BdsDxe: starting Boot0001 "UEFI QEMU DVD-ROM QM00001 " from ... Sata(...)
|
||||
# BdsDxe: failed to start Boot0001 ... Time out
|
||||
# Diagnosis confirmed PVC content IS a valid bootable ISO9660 image — the
|
||||
# timeout is in OVMF reading from the SATA-CDROM-backed-by-filesystem-PVC.
|
||||
# Block mode would likely fix it; see CDI permission issue above.
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: windows-server-2025-iso
|
||||
namespace: kubevirt-vms
|
||||
labels:
|
||||
app: ci-runner
|
||||
flowercore.io/managed-by: bluejay-infra
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce # Bump to ReadOnlyMany after population for multi-VM use
|
||||
resources:
|
||||
requests:
|
||||
storage: 6Gi
|
||||
storage: 10Gi # Server 2025 ISO is 7.7GB; 10Gi for headroom
|
||||
storageClassName: longhorn
|
||||
|
||||
---
|
||||
@@ -220,10 +263,16 @@ data:
|
||||
</OOBE>
|
||||
<UserAccounts>
|
||||
<AdministratorPassword>
|
||||
<!-- IMPORTANT: replace the Value below with a real password BEFORE applying.
|
||||
Generate via: $pw = "YourPasswordHere" + "AdministratorPassword";
|
||||
[Convert]::ToBase64String([Text.Encoding]::Unicode.GetBytes($pw)) -->
|
||||
<Value>UABMAEEAQwBFAEgATwBMAEQARQBSAEEAZABtAGkAbgBpAHMAdAByAGEAdABvAHIAUABhAHMAcwB3AG8AcgBkAA==</Value>
|
||||
<!-- Real password is in 1Password — vault qaphopopkryhbg353ukzhhuqoq,
|
||||
item id h3ix4mgfk65gmkcmvh6ly3d3hu, title:
|
||||
"ci1 Administrator (Windows Server 2025 KubeVirt VM)".
|
||||
Field "autounattend AdministratorPassword Value (UTF-16-LE base64)"
|
||||
matches the Value below.
|
||||
To rotate: regenerate, recompute base64
|
||||
$combined = $pw + "AdministratorPassword"
|
||||
[Convert]::ToBase64String([Text.Encoding]::Unicode.GetBytes($combined))
|
||||
then update both 1P item AND this Value field, recreate VM. -->
|
||||
<Value>bAA3AGsANABOAHcAcgBMAG4AeQBTAHUAYgBBAHQAaQBzAFUAcAB6AEMAWQAhADkAYQBCAEEAZABtAGkAbgBpAHMAdAByAGEAdABvAHIAUABhAHMAcwB3AG8AcgBkAA==</Value>
|
||||
<PlainText>false</PlainText>
|
||||
</AdministratorPassword>
|
||||
</UserAccounts>
|
||||
@@ -260,7 +309,33 @@ metadata:
|
||||
role: github-actions-runner
|
||||
flowercore.io/managed-by: bluejay-infra
|
||||
spec:
|
||||
running: false # Set to true after operator approves + ISO loaded
|
||||
# `running: true` is deprecated in favor of `runStrategy`. They are mutually
|
||||
# exclusive — KubeVirt's validating webhook rejects any VM that sets both:
|
||||
# admission webhook "virtualmachine-validator.kubevirt.io" denied the request:
|
||||
# Running and RunStrategy are mutually exclusive.
|
||||
# `Always` keeps a VMI running and restarts it if it crashes/exits — same
|
||||
# semantics as the old `running: true`.
|
||||
#
|
||||
# **2026-05-08 status: VM cannot start due to a stale QEMU flock on the
|
||||
# rootdisk PVC** (qemu reports `Failed to get "write" lock` on
|
||||
# `/var/run/kubevirt-private/vmi-disks/rootdisk/disk.img`). The flock was
|
||||
# left by a previous QEMU process during a force-deleted launcher pod
|
||||
# cycle. Recovery requires either (a) a Longhorn engine restart on
|
||||
# rke2-agent2, (b) a Longhorn volume detach via the longhorn-manager API
|
||||
# (kubectl patch on `volume.longhorn.io/<pvc-name>` does not work — the
|
||||
# spec.nodeID is reconciled back), or (c) a node reboot of rke2-agent2.
|
||||
#
|
||||
# **Confirmed working:** the bootOrder swap (windows-iso=1, rootdisk=2)
|
||||
# and the runStrategy migration (above). The ISO PVC was successfully
|
||||
# repopulated via virtctl image-upload pvc on the Filesystem-mode PVC.
|
||||
#
|
||||
# **Open: SATA CDROM read timeout** — even with bootOrder=1, OVMF reported
|
||||
# `BdsDxe: failed to start Boot0001 ... Time out` reading the SATA CDROM
|
||||
# backed by the Filesystem-mode PVC. A switch to Block-mode DataVolume
|
||||
# was attempted but blocked by a CDI v1.65.0 upload-pod permission issue
|
||||
# (capability drop prevents writing to the underlying block device).
|
||||
# See header docstring on the ISO PVC.
|
||||
runStrategy: Always # LIVE — ISO uploaded 2026-05-08, password in 1P
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
@@ -306,14 +381,29 @@ spec:
|
||||
devices:
|
||||
tpm: {} # Non-persistent vTPM — sufficient for runner; no BitLocker
|
||||
disks:
|
||||
- name: rootdisk
|
||||
bootOrder: 1
|
||||
disk:
|
||||
bus: virtio
|
||||
# bootOrder: ISO must be 1 for first-boot install (the rootdisk has no
|
||||
# EFI bootloader yet). After Windows installs, it writes its own UEFI
|
||||
# Boot#### entries pointing at the rootdisk's EFI partition; UEFI then
|
||||
# boots from rootdisk going forward and the ISO at bootOrder:2 acts as
|
||||
# a fallback for re-install scenarios.
|
||||
#
|
||||
# Original (broken) order had rootdisk=1, windows-iso=2 — UEFI tried
|
||||
# the empty virtio disk first, got nothing, fell back to the SATA
|
||||
# CDROM at Boot0001 with a short timeout, and timed out before the
|
||||
# CDROM enumerated. Console showed:
|
||||
# BdsDxe: failed to start Boot0001 ... Time out
|
||||
# BdsDxe: No bootable option or device was found.
|
||||
# Confirmed via debug pod: PVC content IS a real bootable ISO9660
|
||||
# (file: "ISO 9660 CD-ROM filesystem data ... (bootable)"), so the
|
||||
# only bug was boot priority.
|
||||
- name: windows-iso
|
||||
bootOrder: 2
|
||||
bootOrder: 1
|
||||
cdrom:
|
||||
bus: sata
|
||||
- name: rootdisk
|
||||
bootOrder: 2
|
||||
disk:
|
||||
bus: virtio
|
||||
- name: virtio-drivers
|
||||
cdrom:
|
||||
bus: sata
|
||||
@@ -340,11 +430,27 @@ spec:
|
||||
persistentVolumeClaim:
|
||||
claimName: ci1-rootdisk
|
||||
- name: windows-iso
|
||||
# Path B (2026-05-08): mount ISO from Synology NFS instead of
|
||||
# Longhorn Filesystem PVC. The Filesystem-PVC path was confirmed to
|
||||
# contain a valid bootable ISO9660 image but caused OVMF's
|
||||
# SATA-CDROM read window to time out:
|
||||
# BdsDxe: failed to start Boot0001 ... Time out
|
||||
# Block-mode DataVolume was attempted as Path A but blocked by CDI
|
||||
# v1.65.0's upload pod capability drop. NFS-mounted ISO bypasses
|
||||
# both issues. See win2025-iso-nfs-pv.yaml header for full rationale
|
||||
# and Synology layout.
|
||||
persistentVolumeClaim:
|
||||
claimName: windows-server-2025-iso
|
||||
claimName: windows-server-2025-iso-nfs
|
||||
- name: virtio-drivers
|
||||
containerDisk:
|
||||
image: quay.io/kubevirt/virtio-container-disk
|
||||
# Pinned to v1.8.2 (latest stable as of 2026-05-08).
|
||||
# The :latest tag uses Docker manifest v1 schema which containerd
|
||||
# 2.1 (RKE2 v1.34.5) refuses to pull with:
|
||||
# "media type application/vnd.docker.distribution.manifest.v1+prettyjws
|
||||
# is no longer supported since containerd v2.1"
|
||||
# v1.8.2 is rebuilt with manifest v2/OCI and works on containerd 2.1.
|
||||
# Bump available: https://quay.io/repository/kubevirt/virtio-container-disk?tab=tags
|
||||
image: quay.io/kubevirt/virtio-container-disk:v1.8.2
|
||||
- name: sysprep
|
||||
sysprep:
|
||||
configMap:
|
||||
|
||||
99
apps/kubevirt-vms/win2025-iso-nfs-pv.yaml
Normal file
99
apps/kubevirt-vms/win2025-iso-nfs-pv.yaml
Normal file
@@ -0,0 +1,99 @@
|
||||
# =============================================================================
|
||||
# Windows Server 2025 ISO — Static NFS PV (Path B for SATA-CDROM timeout)
|
||||
# =============================================================================
|
||||
# Purpose: Mount the ISO from Synology NAS via NFS instead of from a Longhorn-
|
||||
# backed Filesystem PVC.
|
||||
#
|
||||
# Why: SATA-CDROM emulation reading from a Longhorn-backed Filesystem PVC is
|
||||
# too slow for OVMF's boot read window — the DVD-ROM enumeration times out
|
||||
# before the bootloader can be read. Symptom on the serial console:
|
||||
# BdsDxe: failed to start Boot0001 "UEFI QEMU DVD-ROM QM00001 " from ...
|
||||
# BdsDxe: failed to start Boot0001 ... Time out
|
||||
# BdsDxe: No bootable option or device was found
|
||||
# Diagnosis confirmed the ISO content is a perfectly valid bootable ISO9660
|
||||
# image — the bug is in the timing path between OVMF and Longhorn-backed
|
||||
# storage, not in the ISO itself.
|
||||
#
|
||||
# Block-mode PVC was tried (`volumeMode: Block` via DataVolume) and would
|
||||
# likely fix the timing, but CDI v1.65.0's upload-target pod cannot open the
|
||||
# block device due to runAsUser:107 + capabilities.drop:[ALL] and we got:
|
||||
# blockdev: cannot open /dev/cdi-block-volume: Permission denied
|
||||
#
|
||||
# NFS-mounted ISO bypasses both issues: no Longhorn slowness, no CDI upload
|
||||
# pod permission concerns. The ISO is read directly from the NAS over a
|
||||
# native NFSv4.1 mount that QEMU's SATA emulator can read at full LAN speed.
|
||||
#
|
||||
# Layout on Synology:
|
||||
# /volume1/ISOs/ (existing export, RKE2 ACL)
|
||||
# en-us_windows_server_2025_updated_march_2026_x64_dvd_8e06425a.iso
|
||||
# win2025-iso-disk/ (new subdir, 2026-05-08)
|
||||
# disk.img -> hardlink to ../en-us_windows_server_2025_..._8e06425a.iso
|
||||
#
|
||||
# KubeVirt's launcher pod expects a PVC mounted at
|
||||
# /var/run/kubevirt-private/vmi-disks/<diskName>/disk.img — by mounting the
|
||||
# `win2025-iso-disk/` subdir as the NFS PV root, `disk.img` lives at the PV's
|
||||
# root and KubeVirt's CDROM emulator finds it without any path manipulation.
|
||||
#
|
||||
# A symlink would NOT work for sub-path NFS mounts (the relative target
|
||||
# `../...iso` falls outside the sub-mount root). A hardlink works because it
|
||||
# references the same inode regardless of mount point.
|
||||
#
|
||||
# Memory references:
|
||||
# - feedback_synology_nfs_volume1_kubernetes_export_scoped (Synology export
|
||||
# scoping pattern — but /volume1/ISOs export, unlike /volume1/kubernetes,
|
||||
# does support sub-path mounts because Synology NFS is configured with
|
||||
# pseudo-fs in NFSv4.1)
|
||||
# - feedback_kubevirt_iso_first_install_bootorder_and_runstrategy (boot
|
||||
# order / runStrategy gotchas, separate from the storage timing issue)
|
||||
#
|
||||
# Validation (2026-05-08, from rke2-server / rke2-agent1 / rke2-agent2):
|
||||
# mount -t nfs -o nfsvers=4.1,ro 10.0.58.3:/volume1/ISOs/win2025-iso-disk /tmp/m
|
||||
# file /tmp/m/disk.img
|
||||
# -> ISO 9660 CD-ROM filesystem data 'SSS_X64FRE_EN-US_DV9' (bootable)
|
||||
# All 3 RKE2 nodes can mount and read.
|
||||
# =============================================================================
|
||||
|
||||
apiVersion: v1
|
||||
kind: PersistentVolume
|
||||
metadata:
|
||||
name: windows-server-2025-iso-nfs
|
||||
labels:
|
||||
flowercore.io/iso: windows-server-2025
|
||||
flowercore.io/managed-by: bluejay-infra
|
||||
spec:
|
||||
capacity:
|
||||
storage: 8Gi
|
||||
accessModes:
|
||||
- ReadOnlyMany
|
||||
volumeMode: Filesystem
|
||||
persistentVolumeReclaimPolicy: Retain
|
||||
storageClassName: "" # static, no provisioner
|
||||
mountOptions:
|
||||
- nfsvers=4.1
|
||||
- ro
|
||||
- hard
|
||||
- timeo=600
|
||||
- retrans=3
|
||||
nfs:
|
||||
server: 10.0.58.3 # BlueJayNAS Synology DS1621+ on HOME VLAN 58
|
||||
path: /volume1/ISOs/win2025-iso-disk
|
||||
readOnly: true
|
||||
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: windows-server-2025-iso-nfs
|
||||
namespace: kubevirt-vms
|
||||
labels:
|
||||
app: ci-runner
|
||||
flowercore.io/managed-by: bluejay-infra
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadOnlyMany
|
||||
volumeMode: Filesystem
|
||||
resources:
|
||||
requests:
|
||||
storage: 8Gi
|
||||
storageClassName: ""
|
||||
volumeName: windows-server-2025-iso-nfs
|
||||
@@ -3362,6 +3362,92 @@ data:
|
||||
relativeTimeRange: {from: 120, to: 0}
|
||||
datasourceUid: __expr__
|
||||
model: {type: threshold, expression: B, conditions: [{evaluator: {params: [600], type: gt}}], refId: C}
|
||||
- orgId: 1
|
||||
name: Signage Marquee
|
||||
folder: AI Stack Alerts
|
||||
interval: 1m
|
||||
rules:
|
||||
- uid: marquee-dropped-frames-high
|
||||
title: MarqueeDroppedFramesHigh
|
||||
condition: C
|
||||
for: 5m
|
||||
noDataState: OK
|
||||
execErrState: OK
|
||||
annotations:
|
||||
summary: Marquee dropped-frame rate above 5%
|
||||
description: "Dropped frames exceeded the IR-21 budget for a renderer/phase/node tuple. Grafana owns alert delivery to IRC #alerts; Prometheus rules remain only the visibility source."
|
||||
runbook: "1. Open /d/fc-marquee-perf/marquee-animation-performance 2. Filter renderer/node/phase 3. Compare latest AAT baseline diff 4. Restart only the affected player if the issue is node-local"
|
||||
labels:
|
||||
severity: warning
|
||||
service: signage
|
||||
alert_channel: irc
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: {from: 300, to: 0}
|
||||
datasourceUid: prometheus
|
||||
model: {expr: '(sum by (renderer, node_id, phase) (rate(marquee_dropped_frames_total[5m])) / sum by (renderer, node_id, phase) (rate(marquee_render_latency_ms_count[5m]))) * 100', instant: true, refId: A}
|
||||
- refId: B
|
||||
relativeTimeRange: {from: 300, to: 0}
|
||||
datasourceUid: __expr__
|
||||
model: {type: reduce, expression: A, reducer: last, refId: B}
|
||||
- refId: C
|
||||
relativeTimeRange: {from: 300, to: 0}
|
||||
datasourceUid: __expr__
|
||||
model: {type: threshold, expression: B, conditions: [{evaluator: {params: [5], type: gt}}], refId: C}
|
||||
- uid: marquee-render-latency-p99-high
|
||||
title: MarqueeRenderLatencyP99High
|
||||
condition: C
|
||||
for: 5m
|
||||
noDataState: OK
|
||||
execErrState: OK
|
||||
annotations:
|
||||
summary: Marquee render latency p99 above 16ms
|
||||
description: "Renderer p99 latency exceeded the Pi-class 16ms budget. Grafana delivers this alert to IRC #alerts."
|
||||
runbook: "1. Open /d/fc-marquee-perf/marquee-animation-performance 2. Check render latency p99 by renderer/node/phase 3. Compare with dropped frames and node CPU 4. If isolated to WPF, capture current Player.Wpf frame set before restart"
|
||||
labels:
|
||||
severity: warning
|
||||
service: signage
|
||||
alert_channel: irc
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: {from: 300, to: 0}
|
||||
datasourceUid: prometheus
|
||||
model: {expr: 'histogram_quantile(0.99, sum by (renderer, node_id, phase, le) (rate(marquee_render_latency_ms_bucket[5m])))', instant: true, refId: A}
|
||||
- refId: B
|
||||
relativeTimeRange: {from: 300, to: 0}
|
||||
datasourceUid: __expr__
|
||||
model: {type: reduce, expression: A, reducer: last, refId: B}
|
||||
- refId: C
|
||||
relativeTimeRange: {from: 300, to: 0}
|
||||
datasourceUid: __expr__
|
||||
model: {type: threshold, expression: B, conditions: [{evaluator: {params: [16], type: gt}}], refId: C}
|
||||
- uid: marquee-animation-duration-drift
|
||||
title: MarqueeAnimationDurationDrift
|
||||
condition: C
|
||||
for: 10m
|
||||
noDataState: OK
|
||||
execErrState: OK
|
||||
annotations:
|
||||
summary: Marquee animation duration drift above 10%
|
||||
description: "Observed cycle duration has drifted more than 10% from target for a renderer/phase pair. Grafana delivers this alert to IRC #alerts."
|
||||
runbook: "1. Open /d/fc-marquee-perf/marquee-animation-performance 2. Compare observed vs target duration 3. Check recent theme/preset changes 4. Re-run MarqueeHolidayBrandTrajectoryTests before promoting a baseline"
|
||||
labels:
|
||||
severity: warning
|
||||
service: signage
|
||||
alert_channel: irc
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: {from: 900, to: 0}
|
||||
datasourceUid: prometheus
|
||||
model: {expr: 'abs((histogram_quantile(0.5, sum by (renderer, phase, le) (rate(marquee_animation_duration_ms_bucket[15m]))) - avg by (renderer, phase) (marquee_animation_duration_target_ms)) / avg by (renderer, phase) (marquee_animation_duration_target_ms))', instant: true, refId: A}
|
||||
- refId: B
|
||||
relativeTimeRange: {from: 900, to: 0}
|
||||
datasourceUid: __expr__
|
||||
model: {type: reduce, expression: A, reducer: last, refId: B}
|
||||
- refId: C
|
||||
relativeTimeRange: {from: 900, to: 0}
|
||||
datasourceUid: __expr__
|
||||
model: {type: threshold, expression: B, conditions: [{evaluator: {params: [0.1], type: gt}}], refId: C}
|
||||
- orgId: 1
|
||||
name: Infrastructure
|
||||
folder: AI Stack Alerts
|
||||
|
||||
Reference in New Issue
Block a user