yau-plant-assistant/scripts/verify.sh

#!/usr/bin/env bash
# Verify the AI stack on lin001. Read-only: it starts nothing and changes nothing.
#
#   ./scripts/verify.sh
#
# "docker ps showing Up" is not proof of anything. What this checks instead:
# a 302 to the auth portal on every public hostname, clean logs, pg-ai
# unreachable from outside its own network, agent_ro genuinely read-only, and
# no host ports published by anything we added.
#
# Exit code is the number of failed checks, so it is usable in CI.

set -uo pipefail

PASS=0
FAIL=0
ok()   { printf '  \033[32mok\033[0m   %s\n' "$*"; PASS=$((PASS+1)); }
bad()  { printf '  \033[31mFAIL\033[0m %s\n' "$*"; FAIL=$((FAIL+1)); }
head_() { printf '\n\033[1m%s\033[0m\n' "$*"; }

head_ "Containers"
for name in pg-ai cube ai-api ai-web langfuse lf-db; do
  if docker ps --format '{{.Names}}' | grep -qx "$name"; then
    state=$(docker inspect -f '{{.State.Status}}' "$name")
    health=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$name")
    if [ "$state" = "running" ] && [ "$health" != "unhealthy" ]; then
      ok "$name running (health: $health)"
    else
      bad "$name state=$state health=$health"
    fi
  else
    bad "$name is not running"
  fi
done

head_ "No published host ports on anything we added"
# openplc-runtime publishing 502 is the one deliberate exception on this host,
# and it is not ours. Everything in the AI stack must publish nothing.
for name in pg-ai cube ai-api ai-web langfuse lf-db; do
  ports=$(docker port "$name" 2>/dev/null || true)
  if [ -z "$ports" ]; then
    ok "$name publishes no host port"
  else
    bad "$name publishes: $ports"
  fi
done

head_ "pg-ai network isolation"
if docker inspect pg-ai -f '{{range $k,$v := .NetworkSettings.Networks}}{{$k}} {{end}}' 2>/dev/null | grep -qw proxy; then
  bad "pg-ai is attached to the proxy network - it must be ai-internal only"
else
  ok "pg-ai is not on the proxy network"
fi

head_ "Public endpoints - expect 302 to the auth portal"
# Run this from OUTSIDE the VNet as well. lin001 resolves yokogawa.tech through
# the DC, which holds pinpoint records only - ai and influx have one, lf, api
# and cube do not. So on this host those three do not resolve at all, and that
# says nothing about whether they work from a browser. The check reports the
# two cases separately rather than calling both a failure.
# auth.yokogawa.tech is in this list because every one of the others redirects
# to it: a hostname that resolves and answers is still unusable if the portal it
# sends the browser to does not resolve. That is the case on the LAN today, and
# it is reported as information, NOT a failure - it blocks an operator demo from
# a control-room PC and nothing else. Development and the eval run from outside
# the VNet, where auth resolves normally. Promote it to a failure when a demo
# from cicore1 is actually being prepared.
for host in lf.yokogawa.tech cube.yokogawa.tech api.yokogawa.tech ai.yokogawa.tech auth.yokogawa.tech; do
  if ! getent hosts "$host" >/dev/null 2>&1; then
    if [ "$host" = "auth.yokogawa.tech" ]; then
      printf '  ??   auth.yokogawa.tech does not resolve from inside the VNet. Every gated hostname redirects here, so nobody on the LAN can sign in until the DC gets a pinpoint record -> 10.0.0.17. Not a blocker for anything but a demo from a control-room PC.
'
    else
      printf '  ??   %s does not resolve FROM THIS HOST (no DC pinpoint record) - check it from outside the VNet
' "$host"
    fi
    continue
  fi
  code=$(curl -s -o /dev/null -w '%{http_code}' -I "https://$host" --max-time 10 || echo "000")
  case "$code" in
    302|303) ok "$host -> $code (auth portal)" ;;
    200)     bad "$host -> 200 WITHOUT AUTH - check 'import authelia' in ~/Caddyfile" ;;
    403)     bad "$host -> 403 - Caddy is serving it but Authelia has no access_control rule, so default_policy: deny applies. Add the hostname (authelia/access-rules.md)" ;;
    000)     bad "$host unreachable - Caddy has no certificate, or nothing is listening" ;;
    *)       bad "$host -> $code" ;;
  esac
done

head_ "The operator page and the API share an origin"
# The browser posts to /ask on ai.yokogawa.tech. If the route is missing, ai-web
# serves the SPA's 404 and every question fails - with the UI itself looking
# perfectly healthy. Unauthenticated this must be the auth portal redirect, NOT
# a 404 from ai-web.
code=$(curl -s -o /dev/null -w '%{http_code}' -X POST "https://ai.yokogawa.tech/ask"        -H 'Content-Type: application/json' -d '{}' --max-time 10 || echo "000")
case "$code" in
  302|303) ok "ai.yokogawa.tech/ask -> $code (routed to ai-api, gated)" ;;
  403)     bad "ai.yokogawa.tech/ask -> 403 - Authelia has no rule for this hostname yet; the route cannot be proved until it does" ;;
  404)     bad "ai.yokogawa.tech/ask -> 404 - the /ask route is missing from the Caddy block; ai-web is answering" ;;
  000)     bad "ai.yokogawa.tech/ask unreachable" ;;
  *)       bad "ai.yokogawa.tech/ask -> $code" ;;
esac
# api.yokogawa.tech has no pinpoint DNS record, so it is unresolvable from
# inside the VNet. A bundle that hard-codes it loads fine here and fails on a
# control-room PC. Check what was actually built into the image.
if docker exec ai-web sh -c 'grep -rqs "api\.yokogawa\.tech" /usr/share/nginx/html' 2>/dev/null; then
  bad "the ai-web bundle hard-codes api.yokogawa.tech - cicore1 cannot resolve it; rebuild with VITE_API_BASE empty"
else
  ok "ai-web bundle carries no cross-origin API hostname"
fi

# Runs BEFORE the agent_ro test on purpose. That test deliberately attempts an
# INSERT it is not allowed to make, which pg-ai logs as "permission denied for
# table equipment" - in the other order verify.sh flags, every single run, an
# error line it created itself.
head_ "Recent errors in the logs"
for name in pg-ai cube ai-api ai-web; do
  errors=$(docker logs --tail 200 "$name" 2>&1 | grep -icE 'error|fatal|panic' || true)
  [ "${errors:-0}" -eq 0 ] && ok "$name logs clean (last 200 lines)" \
                           || bad "$name has $errors error lines - docker logs --tail 200 $name"
done

head_ "agent_ro is read-only"
if docker exec pg-ai psql -U agent_ro -d plant -tAc 'SELECT count(*) FROM equipment' >/dev/null 2>&1; then
  ok "agent_ro can SELECT"
else
  bad "agent_ro cannot SELECT"
fi
if docker exec pg-ai psql -U agent_ro -d plant -tAc \
     "INSERT INTO equipment (equipment_id) VALUES ('VERIFY-DELETE-ME')" >/dev/null 2>&1; then
  bad "agent_ro CAN INSERT - this is a Phase 1 failure, fix db/003_roles.sql now"
  docker exec pg-ai psql -U postgres -d plant -c \
    "DELETE FROM equipment WHERE equipment_id='VERIFY-DELETE-ME'" >/dev/null 2>&1
else
  ok "agent_ro INSERT is rejected"
fi

head_ "Reference data"
missing_eq=$(docker exec pg-ai psql -U postgres -d plant -tAc \
  "SELECT count(*) FROM equipment WHERE coalesce(array_length(aliases,1),0)=0" 2>/dev/null || echo "?")
missing_tag=$(docker exec pg-ai psql -U postgres -d plant -tAc \
  "SELECT count(*) FROM tags WHERE coalesce(array_length(aliases,1),0)=0" 2>/dev/null || echo "?")
[ "$missing_eq" = "0" ]  && ok "every equipment item has an alias" || bad "$missing_eq equipment items have no alias"
[ "$missing_tag" = "0" ] && ok "every tag has an alias"            || bad "$missing_tag tags have no alias"

if docker exec pg-ai psql -U postgres -d plant -tAc \
     "SELECT 1 FROM pg_extension WHERE extname='vector'" 2>/dev/null | grep -q 1; then
  ok "pgvector extension present"
else
  bad "pgvector extension missing"
fi

head_ "Fixture data"
if docker exec pg-ai psql -U postgres -d plant -tAc \
     "SELECT 1 FROM information_schema.schemata WHERE schema_name='fixture'" 2>/dev/null | grep -q 1; then
  rows=$(docker exec pg-ai psql -U postgres -d plant -tAc \
    "SELECT count(*) FROM fixture.alarm_history" 2>/dev/null)
  printf '  \033[33m!!\033[0m   fixture schema present (%s alarm rows) - answers are TEST DATA, not plant history\n' "$rows"
fi

head_ "Disk"
df -h / /datadisk | sed 's/^/  /'

printf '\n%s passed, %s failed\n' "$PASS" "$FAIL"
exit "$FAIL"