agent-zero: drop BLUEJAY-WS upstream, edge1 Pi is sole Ollama backend

Workstation (BLUEJAY-WS) is private dev hardware and should not be in the
cluster path. Repointing the nginx ollama-proxy sidecar so cluster Agent Zero
talks ONLY to edge1 Pi 5 + AI HAT+ (10.0.57.17:11434):

- nginx upstream: edge1 sole server, no workstation entry
- wait-for-ollama init container: only checks edge1
- NetworkPolicy egress: drop 10.0.56.20/32, keep 10.0.57.17/32
- Comments updated throughout to flag workstation as off-limits to cluster
- Annotation rewritten to document the architectural intent

Pulled qwen2.5:1.5b on edge1 first so Agent Zero's utility_model survives
the cutover (existing models on edge1: qwen3:4b, gemma3:4b, qwen2.5-coder:7b,
nomic-embed-text). Model count on edge1: 4 → 5.

Lets BLUEJAY-WS lock down its Ollama port to localhost without breaking
the cluster Agent Zero.

apps/agent-zero/agent-zero.yaml

@@ -2,14 +2,15 @@
 # Agent Zero AI Stack — NUC Deployment (RKE2 Bare-Metal)
 # =============================================================================
 # Deploys: AgentZero (agent UI) on RKE2 cluster with Blue Jay profile
-# Ollama: workstation-first via BLUEJAY-WS (10.0.56.20:11434) with edge1 Pi 5
-# fallback (10.0.57.17:11434)
+# Ollama: edge1 Pi 5 + AI HAT+ ONLY (10.0.57.17:11434).
+# Workstation Ollama (BLUEJAY-WS) is intentionally NOT in the upstream —
+# the workstation is private dev hardware, not a cluster dependency.
 # Target: RKE2 bare-metal cluster, namespace: agent-zero
 # Profile: Blue Jay (21 tools, 3 prompts, 4 extensions, theme)
 #
 # Differences from LOCAL (WSL K3s):
 #   - Uses Longhorn StorageClass (not local-path)
-#   - Prefers workstation Ollama on the R9700, falls back to edge1 Pi 5
+#   - Cluster-only Ollama path (edge1) — keeps workstation private
 #   - NO Anthropic API key (free/local models only)
 #   - NO Piper TTS or Kiwix (edge1 handles TTS, no Wikipedia needed)
 #   - NO hostPath volumes — profile/tools/extensions loaded via ConfigMaps
@@ -91,7 +92,7 @@ subjects:
 # =============================================================================
 # Agent Zero — AI Agent Web UI (NUC Edition, Blue Jay Profile)
 # =============================================================================
-# Connects to a local proxy that routes to workstation Ollama first and edge1 second
+# Connects to a local nginx proxy that routes to edge1 Pi 5 + AI HAT+ Ollama only
 # Blue Jay profile with 21 tools, 3 prompts, 4 extensions
 
 ---
@@ -118,7 +119,7 @@ metadata:
   annotations:
     agent-zero/deployment: "nuc"
     agent-zero/profile: "bluejay"
-    agent-zero/ollama: "BLUEJAY-WS primary (10.0.56.20:11434), edge1 fallback (10.0.57.17:11434)"
+    agent-zero/ollama: "edge1 Pi 5 + AI HAT+ only (10.0.57.17:11434) — workstation Ollama is private dev hardware, not a cluster dependency"
 spec:
   replicas: 1
   selector:
@@ -133,19 +134,19 @@ spec:
     spec:
       serviceAccountName: agent-zero
       initContainers:
-        # Wait for either workstation or edge1 Ollama to be reachable before starting Agent Zero.
+        # Wait for edge1 Ollama to be reachable before starting Agent Zero.
+        # (Workstation Ollama is intentionally NOT in the cluster path.)
         - name: wait-for-ollama
           image: busybox:1.37
           command: ["sh", "-c"]
           args:
             - |
-              echo "Waiting for Ollama at BLUEJAY-WS or edge1..."
-              until wget -qO- --timeout=2 http://10.0.56.20:11434/api/tags >/dev/null 2>&1 || \
-                    wget -qO- --timeout=2 http://10.0.57.17:11434/api/tags >/dev/null 2>&1; do
-                echo "No Ollama endpoint ready yet, retrying in 5s..."
+              echo "Waiting for edge1 Ollama (10.0.57.17:11434)..."
+              until wget -qO- --timeout=2 http://10.0.57.17:11434/api/tags >/dev/null 2>&1; do
+                echo "edge1 Ollama not ready yet, retrying in 5s..."
                 sleep 5
               done
-              echo "At least one Ollama endpoint is reachable."
+              echo "edge1 Ollama is reachable."
         # Assemble the Blue Jay profile directory structure from ConfigMaps.
         # ConfigMaps can't create nested dirs, so we copy into the workspace PVC.
         - name: setup-bluejay
@@ -202,16 +203,18 @@ spec:
               events { worker_connections 1024; }
               http {
                 upstream ollama_upstream {
-                  server 10.0.56.20:11434 max_fails=2 fail_timeout=10s;
-                  server 10.0.57.17:11434 backup;
+                  # edge1 Pi 5 + AI HAT+ is the SOLE upstream.
+                  # Workstation Ollama (BLUEJAY-WS) is private dev hardware and
+                  # MUST NOT be added back here without explicit operator decision —
+                  # adding it would expose the workstation to cluster traffic.
+                  server 10.0.57.17:11434 max_fails=2 fail_timeout=10s;
                   keepalive 16;
                 }
                 server {
                   listen 11434;
                   # Local healthcheck — proves nginx itself is alive.
                   # Must NOT depend on upstream so liveness doesn't restart
-                  # the container when BLUEJAY-WS Ollama is slow/offline
-                  # and nginx is mid-failover to the edge1 backup.
+                  # the container when edge1 is slow/offline.
                   location = /healthz {
                     access_log off;
                     return 200 'ok\n';
@@ -234,10 +237,10 @@ spec:
           ports:
             - containerPort: 11434
           # Readiness probe DOES check upstream so K8s only routes traffic
-          # when at least one Ollama backend is reachable. timeoutSeconds=5
-          # allows nginx to fail over from BLUEJAY-WS primary to edge1
-          # backup before the probe fails (was timeoutSeconds=1 default →
-          # 172 historic restarts when workstation Ollama was down).
+          # when edge1 Ollama is reachable. timeoutSeconds=5 absorbs the Pi's
+          # slower TCP handshake under load (was timeoutSeconds=1 default →
+          # 172 historic restarts when the workstation primary path went down,
+          # before the cluster was repointed to edge1-only on 2026-04-27).
           readinessProbe:
             httpGet:
               path: /api/tags
@@ -277,8 +280,9 @@ spec:
               # chat_model: FlowerCore LLM Bridge (ADR-088) — OpenAI-compat,
               # spend-tracked, tier-aliased (fc:balanced → Claude Sonnet).
               # api_key comes from A0_SET_chat_model_api_key env var (overrides
-              # config.json). util + embedding stay on local 127.0.0.1 Ollama
-              # proxy (workstation primary, edge1 fallback).
+              # config.json). util + embedding go to local 127.0.0.1 nginx
+              # proxy which routes to edge1 Pi 5 + AI HAT+ ONLY (workstation
+              # is private dev hardware, intentionally not in the cluster path).
               mkdir -p /a0/usr/plugins/_model_config
               cat > /a0/usr/plugins/_model_config/config.json << 'MODELCFG'
               {"allow_chat_override":true,"chat_model":{"provider":"openai","name":"fc:balanced","api_base":"http://fc-llm-bridge.fc-llm-bridge.svc:8080/v1","ctx_length":8192,"ctx_history":0.7,"vision":false,"kwargs":{"temperature":0,"num_ctx":8192}},"utility_model":{"provider":"ollama","name":"qwen2.5:1.5b","api_base":"http://127.0.0.1:11434","ctx_length":8192,"ctx_input":0.7,"kwargs":{"num_ctx":8192}},"embedding_model":{"provider":"ollama","name":"nomic-embed-text","api_base":"http://127.0.0.1:11434","kwargs":{}}}
@@ -554,13 +558,8 @@ spec:
           protocol: UDP
         - port: 53
           protocol: TCP
-    # Ollama on BLUEJAY-WS
-    - to:
-        - ipBlock:
-            cidr: 10.0.56.20/32
-      ports:
-        - port: 11434
-    # Ollama on edge1 fallback
+    # Ollama on edge1 Pi 5 + AI HAT+ (sole upstream — workstation
+    # is private dev hardware and intentionally not allowlisted)
     - to:
         - ipBlock:
             cidr: 10.0.57.17/32

apps/knowledge/README.md

@@ -0,0 +1,153 @@
+# knowledge — FlowerCore.Knowledge.Web (Phase 2.4 K8s deploy)
+
+**Status:** manifests staged, **NOT YET APPLIED**. Image must be built +
+imported AND DNS record provisioned before `git push`.
+
+- Plan: [`../../../FlowerCore.Notes/docs/ai-agents/flowercore-knowledge-service-plan.md`](../../../FlowerCore.Notes/docs/ai-agents/flowercore-knowledge-service-plan.md)
+- Sprint: [`../../../FlowerCore.Notes/docs/ai-station/sprint-e-xxl-plan.md`](../../../FlowerCore.Notes/docs/ai-station/sprint-e-xxl-plan.md) (Track B)
+- Repo: `D:\git\FlowerCore\FlowerCore.Knowledge\` (private GitHub repo,
+  bootstrapped Sprint D batch 35)
+
+`FlowerCore.Knowledge.Web` is the fleet-wide vector-indexing & RAG hub —
+a REST + MCP service that scans `*.db` files under
+`/data/vector-stores` and exposes per-edition reachability + corpus
+search to the rest of the FC ecosystem (Agent Zero, Chat.Web persona
+memory, AiStation embeddings explorer, TtsReader chapter context, BMO
+bot, Pi nodes via `fc-index sync`).
+
+## Deployment order (do NOT skip / reorder)
+
+### 1. FlowerCore.DNS public A record — knowledge.iamworkin.lan -> 10.0.56.200
+
+Required BEFORE the Certificate resource is created, or cert-manager
+HTTP-01 silently backs off ~2h. Memory: `feedback_pfsense_dns_required_for_acme`.
+
+The canonical path is FlowerCore.DNS:
+
+```bash
+curl -sk https://dns.iamworkin.lan/api/v1/servers
+# Find the pfSense serverId, then create the record using the host label only.
+
+curl -sk -X POST https://dns.iamworkin.lan/api/v1/servers/<serverId>/zones/iamworkin.lan/records \
+  -H "Content-Type: application/json" \
+  -d '{"name":"knowledge","type":"A","data":"10.0.56.200","ttl":300}'
+```
+
+If FlowerCore.DNS provider writes are failing 502 with "pfSense
+diag_command.php response did not contain a `<pre>` block" (status as of
+Sprint E Track B authoring 2026-04-27), add the override manually via
+the pfSense web UI:
+
+1. Log in to `https://10.0.56.1` as admin
+2. Services → DNS Resolver → General Settings → Host Overrides
+3. Add: Host=`knowledge`, Domain=`iamworkin.lan`, IP Address=`10.0.56.200`
+4. Save + Apply Changes
+
+Verify resolution from anywhere on LAN:
+
+```bash
+nslookup knowledge.iamworkin.lan 10.0.56.1
+# Expect: 10.0.56.200
+```
+
+Or against FlowerCore.DNS once the provider is fixed:
+
+```bash
+curl -sk "https://dns.iamworkin.lan/api/v1/zones/iamworkin.lan/resolve-preflight?hostname=knowledge.iamworkin.lan"
+# Expect: "resolvable": true
+```
+
+### 2. Build + import the image to ALL RKE2 nodes
+
+Pods may schedule on any RKE2 worker (server, agent1, agent2). The
+Longhorn PVC accepts mounts from any node, so the image must be
+imported to all three. Memory:
+`feedback_rke2_image_import_targets_all_nodes` +
+`feedback_rke2_localhost_imagepullpolicy`.
+
+```bash
+# From BLUEJAY-WS, in D:\git\FlowerCore\FlowerCore.Knowledge
+TAG="v$(date +%Y%m%d%H%M)"
+dotnet.exe publish -c Release -o deploy/app \
+  src/FlowerCore.Knowledge.Web/FlowerCore.Knowledge.Web.csproj
+podman build -t localhost/fc-knowledge-web:$TAG -f deploy/Dockerfile.deploy deploy
+podman save localhost/fc-knowledge-web:$TAG -o /tmp/fc-knowledge-web.tar
+
+# Import to all three RKE2 nodes
+for node in rke2-server rke2-agent1 rke2-agent2; do
+  scp /tmp/fc-knowledge-web.tar $node:/tmp/
+  ssh $node "sudo /var/lib/rancher/rke2/bin/ctr -a /run/k3s/containerd/containerd.sock -n k8s.io images import /tmp/fc-knowledge-web.tar"
+done
+```
+
+The repo's `scripts/deploy-knowledge.sh` automates this loop.
+
+### 3. Bump the image tag + push
+
+Edit `knowledge.yaml`, replace `localhost/fc-knowledge-web:v202604272200`
+with the tag from step 2, then:
+
+```bash
+cd D:/git/FlowerCore/bluejay-infra
+python scripts/check-pfsense-dns.py     # confirms the DNS preflight
+git add apps/knowledge/
+git commit -m "feat(knowledge): deploy Phase 2.4 K8s manifest"
+git push
+```
+
+ArgoCD picks up within ~3 minutes and creates `infra-knowledge`.
+
+### 4. Verify
+
+```bash
+fcadmin_ssh noc1 '
+  kubectl -n argocd get application infra-knowledge
+  kubectl -n knowledge get certificate,pod,pvc
+  curl -sk -m 8 -o /dev/null -w "HTTP %{http_code}\n" https://knowledge.iamworkin.lan/healthz
+  curl -sk -m 8 https://knowledge.iamworkin.lan/api/v1/editions | jq
+'
+```
+
+Expect: Certificate `Ready: True` within ~60s, `/healthz` HTTP 200,
+`/api/v1/editions` returns an empty array (no DBs in the PVC yet) on
+first deploy.
+
+## Initial-deploy state and Phase 2.5 follow-up
+
+The Longhorn PVC is empty on first deploy. Knowledge.Web's filesystem
+catalog will report zero editions until vector-store `*.db` files are
+pushed into `/data/vector-stores`. Initial population is a follow-up
+step (Phase 2.5+, Blazor admin UI's "Rebuild" button); for the first
+deploy the goal is just to prove the pod boots, `/healthz` returns 200,
+and the Traefik IngressRoute serves the Scalar UI.
+
+To copy an existing local DB into the PVC (one-time, manual until
+Phase 2.5 admin UI lands):
+
+```bash
+fcadmin_ssh noc1 '
+  POD=$(kubectl -n knowledge get pod -l app=knowledge-web -o jsonpath="{.items[0].metadata.name}")
+  kubectl -n knowledge cp /var/lib/flowercore/vector-stores/bluejay-ai.db $POD:/data/vector-stores/bluejay-ai.db
+'
+```
+
+## Probes + middleware notes
+
+- `/healthz` is mapped by `Controllers/HealthController.cs` (controller-based
+  attribute route). Cheap — no DB, no dependencies.
+- Liveness uses `tcpSocket` as a defensive fallback in case future
+  middleware accidentally gates `/healthz` behind auth (memory:
+  `feedback_k8s_probes_behind_auth_middleware`).
+- `/openapi/v1.json` and `/scalar/v1` are wired by `UseFlowerCoreApi`.
+  Per memory `feedback_k8s_probes_must_not_hit_openapi`, probes must NOT
+  point at OpenAPI documents — the `MapOpenApi` call can be slow during
+  cold startup.
+
+## Resource sizing
+
+- 256Mi memory request / 1Gi limit.
+- 100m CPU request / 1000m limit.
+- 20Gi Longhorn PVC initial — sufficient for the bluejay-ai 1.94Gi DB +
+  fleet-pi-edge 352Mi + fleet-bmo-bot 141Mi + headroom. Resize via
+  `kubectl -n knowledge edit pvc knowledge-vector-store` if growing
+  past 15Gi.

apps/knowledge/knowledge.yaml

@@ -0,0 +1,227 @@
+# FlowerCore.Knowledge.Web — fleet vector indexing & RAG hub.
+#
+# Phase 2.4 of the Knowledge service plan. REST + MCP service that scans
+# *.db files under /data/vector-stores and exposes:
+#   - REST: /api/v1/editions, /api/v1/corpus/search, /healthz
+#   - MCP:  list_editions, describe_edition, corpus_search
+#   - Static OpenAPI/Scalar via UseFlowerCoreApi
+#
+# Architecture:
+#   Plan:    FlowerCore.Notes/docs/ai-agents/flowercore-knowledge-service-plan.md
+#   Sprint:  FlowerCore.Notes/docs/ai-station/sprint-e-xxl-plan.md (Track B)
+#   Repo:    D:\git\FlowerCore\FlowerCore.Knowledge\
+#   Shared:  FlowerCore.Common -> FlowerCore.Shared.Indexing (chunkers, vector
+#            stores, edition profiles, ICorpusSearchService facade)
+#
+# Deployment order (see apps/knowledge/README.md and the bluejay-infra/README.md
+# top-level checklist):
+#   1. FlowerCore.DNS public A record knowledge.iamworkin.lan -> 10.0.56.200
+#      MUST exist BEFORE the Certificate is created, or cert-manager HTTP-01
+#      backs off ~2h. Memory: feedback_pfsense_dns_required_for_acme.
+#   2. Build + import the image to ALL RKE2 nodes (server + both agents) since
+#      the Pod uses a Longhorn PVC and may schedule anywhere.
+#      Memory: feedback_rke2_localhost_imagepullpolicy.
+#   3. Bump the image tag in this file, git push.
+#   4. ArgoCD ApplicationSet picks up within ~3 minutes and creates
+#      infra-knowledge.
+#
+# Initial-deploy state:
+#   The Longhorn PVC is empty on first deploy. Knowledge.Web's filesystem
+#   catalog will report zero editions until vector-store *.db files are
+#   pushed into /data/vector-stores. Initial population is a follow-up step
+#   (Phase 2.5+, Blazor admin UI's "Rebuild" button); for the first deploy
+#   the goal is just to prove the pod boots, /healthz returns 200, and the
+#   Traefik IngressRoute serves the Scalar UI.
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: knowledge
+  labels:
+    app.kubernetes.io/part-of: bluejay-infra
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: knowledge-vector-store
+  namespace: knowledge
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: longhorn
+  resources:
+    requests:
+      storage: 20Gi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: knowledge-web
+  namespace: knowledge
+  labels:
+    app: knowledge-web
+    app.kubernetes.io/name: knowledge-web
+    app.kubernetes.io/part-of: bluejay-infra
+spec:
+  replicas: 1
+  revisionHistoryLimit: 3
+  # RWO Longhorn PVC blocks rolling updates (multi-attach error). Recreate
+  # is the canonical pattern (memory: feedback_rwo_pvc_blocks_rolling).
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: knowledge-web
+  template:
+    metadata:
+      labels:
+        app: knowledge-web
+        app.kubernetes.io/name: knowledge-web
+        app.kubernetes.io/part-of: bluejay-infra
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "8080"
+        prometheus.io/path: "/metrics"
+    spec:
+      securityContext:
+        runAsNonRoot: true
+        fsGroup: 1654
+        fsGroupChangePolicy: OnRootMismatch
+      containers:
+        - name: web
+          # Placeholder tag — bump to the image you built + imported to ALL
+          # RKE2 nodes via scripts/deploy-knowledge.sh before applying.
+          image: localhost/fc-knowledge-web:v202604272200
+          imagePullPolicy: Never
+          ports:
+            - containerPort: 8080
+              name: http
+          env:
+            - name: ASPNETCORE_URLS
+              value: "http://+:8080"
+            - name: ASPNETCORE_ENVIRONMENT
+              value: "Production"
+            - name: DOTNET_SYSTEM_GLOBALIZATION_INVARIANT
+              value: "false"
+            # Vector-store directory + embedding model + edition profile dir.
+            # Profile JSON is baked into the image at /app/editions via the
+            # csproj Content-link from FlowerCore.Common/editions/.
+            - name: Knowledge__VectorStoresDirectory
+              value: "/data/vector-stores"
+            - name: Knowledge__EmbeddingModel
+              value: "nomic-embed-text"
+            - name: Knowledge__DefaultLimit
+              value: "5"
+            - name: Knowledge__MaxLimit
+              value: "50"
+            - name: FlowerCore__Editions__ProfileDirectory
+              value: "/app/editions"
+            # Embed via BLUEJAY-WS GPU (R9700, 32GB VRAM). Pi5 Ollama is
+            # ~4-5x slower; use the workstation while we have it.
+            # Memory: feedback_pi5_nomic_embed_slow.
+            - name: FlowerCore__Ollama__BaseUrl
+              value: "http://10.0.56.20:11434"
+          resources:
+            requests:
+              cpu: 100m
+              memory: 256Mi
+            limits:
+              cpu: 1000m
+              memory: 1Gi
+          # /healthz is mapped by HealthController (controller-based route).
+          # tcpSocket liveness is the defensive fallback in case middleware
+          # later gates /healthz behind auth (memory:
+          # feedback_k8s_probes_behind_auth_middleware).
+          startupProbe:
+            httpGet:
+              path: /healthz
+              port: 8080
+            initialDelaySeconds: 5
+            periodSeconds: 5
+            failureThreshold: 30
+          readinessProbe:
+            httpGet:
+              path: /healthz
+              port: 8080
+            periodSeconds: 10
+            failureThreshold: 3
+          livenessProbe:
+            tcpSocket:
+              port: 8080
+            initialDelaySeconds: 30
+            periodSeconds: 30
+            failureThreshold: 3
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 1654
+            runAsGroup: 1654
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+          volumeMounts:
+            - name: vector-store
+              mountPath: /data/vector-stores
+            - name: tmp
+              mountPath: /tmp
+            - name: logs
+              mountPath: /app/logs
+      volumes:
+        - name: vector-store
+          persistentVolumeClaim:
+            claimName: knowledge-vector-store
+        - name: tmp
+          emptyDir: {}
+        - name: logs
+          emptyDir: {}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: knowledge-web
+  namespace: knowledge
+  labels:
+    app: knowledge-web
+    app.kubernetes.io/name: knowledge-web
+    app.kubernetes.io/part-of: bluejay-infra
+spec:
+  type: ClusterIP
+  selector:
+    app: knowledge-web
+  ports:
+    - name: http
+      port: 80
+      targetPort: 8080
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: knowledge-tls
+  namespace: knowledge
+spec:
+  secretName: knowledge-tls
+  issuerRef:
+    name: step-ca-acme
+    kind: ClusterIssuer
+  dnsNames:
+    - knowledge.iamworkin.lan
+  duration: 2160h    # 90d
+  renewBefore: 720h  # 30d
+---
+apiVersion: traefik.io/v1alpha1
+kind: IngressRoute
+metadata:
+  name: knowledge
+  namespace: knowledge
+spec:
+  entryPoints:
+    - websecure
+  routes:
+    - match: Host(`knowledge.iamworkin.lan`)
+      kind: Rule
+      services:
+        - name: knowledge-web
+          port: 80
+  tls:
+    secretName: knowledge-tls

apps/knowledge/kustomization.yaml

@@ -0,0 +1,7 @@
+# ArgoCD's bluejay-infra ApplicationSet uses a directory generator and does
+# not require kustomization.yaml. Mirrors the fc-distribution shape so
+# `kubectl kustomize` previews work from a working copy.
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - knowledge.yaml