eb8cfaa787
Both found running it on lin001 straight after the Caddy blocks went in. The public-endpoint check called lf, cube and api unreachable. They are not: the host resolves yokogawa.tech through the DC, which holds pinpoint records only, and those three have none. Nothing is wrong with them and the check cannot tell from here - so it now says so, and says to check from outside the VNet, instead of printing a failure that means nothing. 403 now has its own message. It is what Caddy plus Authelia return for a hostname with a site block and no access_control rule, which is the state between applying the Caddy blocks and applying the Authelia rule - the one case an operator is most likely to hit and least likely to recognise. And the log check ran after the agent_ro test, which deliberately attempts a forbidden INSERT that pg-ai logs. verify.sh was flagging an error line it had just written itself. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
152 lines
7 KiB
Bash
152 lines
7 KiB
Bash
#!/usr/bin/env bash
|
||
# Verify the AI stack on lin001. Read-only: it starts nothing and changes nothing.
|
||
#
|
||
# ./scripts/verify.sh
|
||
#
|
||
# "docker ps showing Up" is not proof of anything. What this checks instead:
|
||
# a 302 to the auth portal on every public hostname, clean logs, pg-ai
|
||
# unreachable from outside its own network, agent_ro genuinely read-only, and
|
||
# no host ports published by anything we added.
|
||
#
|
||
# Exit code is the number of failed checks, so it is usable in CI.
|
||
|
||
set -uo pipefail
|
||
|
||
PASS=0
|
||
FAIL=0
|
||
ok() { printf ' \033[32mok\033[0m %s\n' "$*"; PASS=$((PASS+1)); }
|
||
bad() { printf ' \033[31mFAIL\033[0m %s\n' "$*"; FAIL=$((FAIL+1)); }
|
||
head_() { printf '\n\033[1m%s\033[0m\n' "$*"; }
|
||
|
||
head_ "Containers"
|
||
for name in pg-ai cube ai-api ai-web langfuse lf-db; do
|
||
if docker ps --format '{{.Names}}' | grep -qx "$name"; then
|
||
state=$(docker inspect -f '{{.State.Status}}' "$name")
|
||
health=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$name")
|
||
if [ "$state" = "running" ] && [ "$health" != "unhealthy" ]; then
|
||
ok "$name running (health: $health)"
|
||
else
|
||
bad "$name state=$state health=$health"
|
||
fi
|
||
else
|
||
bad "$name is not running"
|
||
fi
|
||
done
|
||
|
||
head_ "No published host ports on anything we added"
|
||
# openplc-runtime publishing 502 is the one deliberate exception on this host,
|
||
# and it is not ours. Everything in the AI stack must publish nothing.
|
||
for name in pg-ai cube ai-api ai-web langfuse lf-db; do
|
||
ports=$(docker port "$name" 2>/dev/null || true)
|
||
if [ -z "$ports" ]; then
|
||
ok "$name publishes no host port"
|
||
else
|
||
bad "$name publishes: $ports"
|
||
fi
|
||
done
|
||
|
||
head_ "pg-ai network isolation"
|
||
if docker inspect pg-ai -f '{{range $k,$v := .NetworkSettings.Networks}}{{$k}} {{end}}' 2>/dev/null | grep -qw proxy; then
|
||
bad "pg-ai is attached to the proxy network - it must be ai-internal only"
|
||
else
|
||
ok "pg-ai is not on the proxy network"
|
||
fi
|
||
|
||
head_ "Public endpoints - expect 302 to the auth portal"
|
||
# Run this from OUTSIDE the VNet as well. lin001 resolves yokogawa.tech through
|
||
# the DC, which holds pinpoint records only - ai and influx have one, lf, api
|
||
# and cube do not. So on this host those three do not resolve at all, and that
|
||
# says nothing about whether they work from a browser. The check reports the
|
||
# two cases separately rather than calling both a failure.
|
||
for host in lf.yokogawa.tech cube.yokogawa.tech api.yokogawa.tech ai.yokogawa.tech; do
|
||
if ! getent hosts "$host" >/dev/null 2>&1; then
|
||
printf ' [33m??[0m %s does not resolve FROM THIS HOST (no DC pinpoint record) - check it from outside the VNet
|
||
' "$host"
|
||
continue
|
||
fi
|
||
code=$(curl -s -o /dev/null -w '%{http_code}' -I "https://$host" --max-time 10 || echo "000")
|
||
case "$code" in
|
||
302|303) ok "$host -> $code (auth portal)" ;;
|
||
200) bad "$host -> 200 WITHOUT AUTH - check 'import authelia' in ~/Caddyfile" ;;
|
||
403) bad "$host -> 403 - Caddy is serving it but Authelia has no access_control rule, so default_policy: deny applies. Add the hostname (authelia/access-rules.md)" ;;
|
||
000) bad "$host unreachable - Caddy has no certificate, or nothing is listening" ;;
|
||
*) bad "$host -> $code" ;;
|
||
esac
|
||
done
|
||
|
||
head_ "The operator page and the API share an origin"
|
||
# The browser posts to /ask on ai.yokogawa.tech. If the route is missing, ai-web
|
||
# serves the SPA's 404 and every question fails - with the UI itself looking
|
||
# perfectly healthy. Unauthenticated this must be the auth portal redirect, NOT
|
||
# a 404 from ai-web.
|
||
code=$(curl -s -o /dev/null -w '%{http_code}' -X POST "https://ai.yokogawa.tech/ask" -H 'Content-Type: application/json' -d '{}' --max-time 10 || echo "000")
|
||
case "$code" in
|
||
302|303) ok "ai.yokogawa.tech/ask -> $code (routed to ai-api, gated)" ;;
|
||
403) bad "ai.yokogawa.tech/ask -> 403 - Authelia has no rule for this hostname yet; the route cannot be proved until it does" ;;
|
||
404) bad "ai.yokogawa.tech/ask -> 404 - the /ask route is missing from the Caddy block; ai-web is answering" ;;
|
||
000) bad "ai.yokogawa.tech/ask unreachable" ;;
|
||
*) bad "ai.yokogawa.tech/ask -> $code" ;;
|
||
esac
|
||
# api.yokogawa.tech has no pinpoint DNS record, so it is unresolvable from
|
||
# inside the VNet. A bundle that hard-codes it loads fine here and fails on a
|
||
# control-room PC. Check what was actually built into the image.
|
||
if docker exec ai-web sh -c 'grep -rqs "api\.yokogawa\.tech" /usr/share/nginx/html' 2>/dev/null; then
|
||
bad "the ai-web bundle hard-codes api.yokogawa.tech - cicore1 cannot resolve it; rebuild with VITE_API_BASE empty"
|
||
else
|
||
ok "ai-web bundle carries no cross-origin API hostname"
|
||
fi
|
||
|
||
# Runs BEFORE the agent_ro test on purpose. That test deliberately attempts an
|
||
# INSERT it is not allowed to make, which pg-ai logs as "permission denied for
|
||
# table equipment" - in the other order verify.sh flags, every single run, an
|
||
# error line it created itself.
|
||
head_ "Recent errors in the logs"
|
||
for name in pg-ai cube ai-api ai-web; do
|
||
errors=$(docker logs --tail 200 "$name" 2>&1 | grep -icE 'error|fatal|panic' || true)
|
||
[ "${errors:-0}" -eq 0 ] && ok "$name logs clean (last 200 lines)" \
|
||
|| bad "$name has $errors error lines - docker logs --tail 200 $name"
|
||
done
|
||
|
||
head_ "agent_ro is read-only"
|
||
if docker exec pg-ai psql -U agent_ro -d plant -tAc 'SELECT count(*) FROM equipment' >/dev/null 2>&1; then
|
||
ok "agent_ro can SELECT"
|
||
else
|
||
bad "agent_ro cannot SELECT"
|
||
fi
|
||
if docker exec pg-ai psql -U agent_ro -d plant -tAc \
|
||
"INSERT INTO equipment (equipment_id) VALUES ('VERIFY-DELETE-ME')" >/dev/null 2>&1; then
|
||
bad "agent_ro CAN INSERT - this is a Phase 1 failure, fix db/003_roles.sql now"
|
||
docker exec pg-ai psql -U postgres -d plant -c \
|
||
"DELETE FROM equipment WHERE equipment_id='VERIFY-DELETE-ME'" >/dev/null 2>&1
|
||
else
|
||
ok "agent_ro INSERT is rejected"
|
||
fi
|
||
|
||
head_ "Reference data"
|
||
missing_eq=$(docker exec pg-ai psql -U postgres -d plant -tAc \
|
||
"SELECT count(*) FROM equipment WHERE coalesce(array_length(aliases,1),0)=0" 2>/dev/null || echo "?")
|
||
missing_tag=$(docker exec pg-ai psql -U postgres -d plant -tAc \
|
||
"SELECT count(*) FROM tags WHERE coalesce(array_length(aliases,1),0)=0" 2>/dev/null || echo "?")
|
||
[ "$missing_eq" = "0" ] && ok "every equipment item has an alias" || bad "$missing_eq equipment items have no alias"
|
||
[ "$missing_tag" = "0" ] && ok "every tag has an alias" || bad "$missing_tag tags have no alias"
|
||
|
||
if docker exec pg-ai psql -U postgres -d plant -tAc \
|
||
"SELECT 1 FROM pg_extension WHERE extname='vector'" 2>/dev/null | grep -q 1; then
|
||
ok "pgvector extension present"
|
||
else
|
||
bad "pgvector extension missing"
|
||
fi
|
||
|
||
head_ "Fixture data"
|
||
if docker exec pg-ai psql -U postgres -d plant -tAc \
|
||
"SELECT 1 FROM information_schema.schemata WHERE schema_name='fixture'" 2>/dev/null | grep -q 1; then
|
||
rows=$(docker exec pg-ai psql -U postgres -d plant -tAc \
|
||
"SELECT count(*) FROM fixture.alarm_history" 2>/dev/null)
|
||
printf ' \033[33m!!\033[0m fixture schema present (%s alarm rows) - answers are TEST DATA, not plant history\n' "$rows"
|
||
fi
|
||
|
||
head_ "Disk"
|
||
df -h / /datadisk | sed 's/^/ /'
|
||
|
||
printf '\n%s passed, %s failed\n' "$PASS" "$FAIL"
|
||
exit "$FAIL"
|