yau-plant-assistant/scripts/verify.sh

#!/usr/bin/env bash
# Verify the AI stack on lin001. Read-only: it starts nothing and changes nothing.
#
#   ./scripts/verify.sh
#
# "docker ps showing Up" is not proof of anything. What this checks instead:
# a 302 to the auth portal on every public hostname, clean logs, pg-ai
# unreachable from outside its own network, agent_ro genuinely read-only, and
# no host ports published by anything we added.
#
# Exit code is the number of failed checks, so it is usable in CI.

set -uo pipefail

PASS=0
FAIL=0
ok()   { printf '  \033[32mok\033[0m   %s\n' "$*"; PASS=$((PASS+1)); }
bad()  { printf '  \033[31mFAIL\033[0m %s\n' "$*"; FAIL=$((FAIL+1)); }
head_() { printf '\n\033[1m%s\033[0m\n' "$*"; }

head_ "Containers"
for name in pg-ai cube ai-api ai-web langfuse lf-db; do
  if docker ps --format '{{.Names}}' | grep -qx "$name"; then
    state=$(docker inspect -f '{{.State.Status}}' "$name")
    health=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$name")
    if [ "$state" = "running" ] && [ "$health" != "unhealthy" ]; then
      ok "$name running (health: $health)"
    else
      bad "$name state=$state health=$health"
    fi
  else
    bad "$name is not running"
  fi
done

head_ "No published host ports on anything we added"
# openplc-runtime publishing 502 is the one deliberate exception on this host,
# and it is not ours. Everything in the AI stack must publish nothing.
for name in pg-ai cube ai-api ai-web langfuse lf-db; do
  ports=$(docker port "$name" 2>/dev/null || true)
  if [ -z "$ports" ]; then
    ok "$name publishes no host port"
  else
    bad "$name publishes: $ports"
  fi
done

head_ "pg-ai network isolation"
if docker inspect pg-ai -f '{{range $k,$v := .NetworkSettings.Networks}}{{$k}} {{end}}' 2>/dev/null | grep -qw proxy; then
  bad "pg-ai is attached to the proxy network - it must be ai-internal only"
else
  ok "pg-ai is not on the proxy network"
fi

head_ "Public endpoints - expect 302 to the auth portal"
# Run this from OUTSIDE the VNet as well. lin001 resolves yokogawa.tech through
# the DC, which holds pinpoint records only - ai and influx have one, lf, api
# and cube do not. So on this host those three do not resolve at all, and that
# says nothing about whether they work from a browser. The check reports the
# two cases separately rather than calling both a failure.
for host in lf.yokogawa.tech cube.yokogawa.tech api.yokogawa.tech; do
  if ! getent hosts "$host" >/dev/null 2>&1; then
    printf '  ??   %s does not resolve FROM THIS HOST (no DC pinpoint record) - check it from outside the VNet
' "$host"
    continue
  fi
  code=$(curl -s -o /dev/null -w '%{http_code}' -I "https://$host" --max-time 10 || echo "000")
  case "$code" in
    302|303) ok "$host -> $code (auth portal)" ;;
    200)     bad "$host -> 200 WITHOUT AUTH - check 'import authelia' in ~/Caddyfile" ;;
    403)     bad "$host -> 403 - Caddy is serving it but Authelia has no access_control rule, so default_policy: deny applies. Add the hostname (authelia/access-rules.md)" ;;
    000)     bad "$host unreachable - Caddy has no certificate, or nothing is listening" ;;
    *)       bad "$host -> $code" ;;
  esac
done

head_ "ai.yokogawa.tech is closed to everything except the SCADA console"
# Applied 2026-08-28: the block admits remote_ip 10.0.0.21 (cicore1) only and
# 403s everything else. lin001 is NOT 10.0.0.21 as Caddy sees it, so from here
# 403 is the PASS - it proves the deny arm works and that this host, the VPN and
# the internet are all shut out.
#
# A CURL FROM HERE CANNOT PROVE THE ALLOW ARM: a typo in the matcher gives 403
# to cicore1 too and looks identical from this side. But the evidence does exist
# on this host - ai-web's nginx log records the REAL client address forwarded by
# Caddy, so a page load from 10.0.0.21 proves the matcher admits the console.
# That check is below, and it replaces the old "somebody go and look" step.
#
# PROVEN 2026-08-28: 10.0.0.21 loaded the page the same day the rule was applied,
# and asked three questions on 31 August, each answered 200. It sat unnoticed for
# three days because Caddy keeps no access log of its own.
for u in "https://ai.yokogawa.tech" "https://ai.yokogawa.tech/ask"; do
  code=$(curl -s -o /dev/null -w '%{http_code}' -X POST "$u" \
           -H 'Content-Type: application/json' -d '{}' --max-time 10 || echo "000")
  case "$code" in
    403) ok "$u -> 403 from this host (deny arm working)" ;;
    302|303) bad "$u -> $code - still going through Authelia; the SCADA-only block was not applied or was reverted" ;;
    200) bad "$u -> 200 FROM THIS HOST - the matcher is not restricting anything. The assistant is open to the LAN, the VPN and, via the public A record, the internet" ;;
    404) bad "$u -> 404 - the /ask route is missing from the Caddy block; ai-web is answering" ;;
    000) bad "$u unreachable - Caddy has no certificate, or nothing is listening" ;;
    *)   bad "$u -> $code" ;;
  esac
done
# The allow arm, proved from evidence rather than asserted. Caddy forwards the
# real client address and ai-web logs it as the last field; 10.0.0.21 there means
# the console was admitted. NOTE docker logs are EPHEMERAL - recreating ai-web
# destroys this evidence, so a clean miss is a warning, not a failure.
scada_hits=$(docker logs ai-web 2>&1 | grep -c '"10\.0\.0\.21"' || true)
if [ "${scada_hits:-0}" -gt 0 ]; then
  ok "allow arm PROVEN: $scada_hits requests from cicore1 (10.0.0.21) in the ai-web log"
  last_hit=$(docker logs ai-web 2>&1 | grep '"10\.0\.0\.21"' | tail -1 | grep -oE '\[[^]]+\]' || true)
  ok "most recent console request: ${last_hit:-unknown}"
else
  printf '  \033[33m>>\033[0m   allow arm NOT evidenced in the current ai-web log. If the container was\n'
  printf '       recreated this proves nothing - the log starts empty. Otherwise open\n'
  printf '       https://ai.yokogawa.tech on cicore1 (10.0.0.21) - expect the UI, no login.\n'
fi

# The deny arm from the other direction: anything in the log that is NOT the
# console got past a matcher that should have refused it. Entries dated before
# 2026-08-28 are legitimate - the hostname was Authelia-gated and open then.
foreign=$(docker logs ai-web 2>&1 | grep -oE '"[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+"$' | grep -v '"10\.0\.0\.21"' | sort -u || true)
if [ -n "$foreign" ]; then
  printf '  \033[33m>>\033[0m   non-console clients in the ai-web log. CHECK THE DATES - anything before\n'
  printf '       2026-08-28 predates the SCADA-only rule and was correctly gated by Authelia:\n'
  printf '       %s\n' $foreign
else
  ok "no non-console client has ever reached ai-web"
fi
# api.yokogawa.tech has no pinpoint DNS record, so it is unresolvable from
# inside the VNet. A bundle that hard-codes it loads fine here and fails on a
# control-room PC. Check what was actually built into the image.
if docker exec ai-web sh -c 'grep -rqs "api\.yokogawa\.tech" /usr/share/nginx/html' 2>/dev/null; then
  bad "the ai-web bundle hard-codes api.yokogawa.tech - cicore1 cannot resolve it; rebuild with VITE_API_BASE empty"
else
  ok "ai-web bundle carries no cross-origin API hostname"
fi

# Runs BEFORE the agent_ro test on purpose. That test deliberately attempts an
# INSERT it is not allowed to make, which pg-ai logs as "permission denied for
# table equipment" - in the other order verify.sh flags, every single run, an
# error line it created itself.
head_ "Recent errors in the logs"
for name in pg-ai cube ai-api ai-web; do
  errors=$(docker logs --tail 200 "$name" 2>&1 | grep -icE 'error|fatal|panic' || true)
  [ "${errors:-0}" -eq 0 ] && ok "$name logs clean (last 200 lines)" \
                           || bad "$name has $errors error lines - docker logs --tail 200 $name"
done

head_ "agent_ro is read-only"
if docker exec pg-ai psql -U agent_ro -d plant -tAc 'SELECT count(*) FROM equipment' >/dev/null 2>&1; then
  ok "agent_ro can SELECT"
else
  bad "agent_ro cannot SELECT"
fi
if docker exec pg-ai psql -U agent_ro -d plant -tAc \
     "INSERT INTO equipment (equipment_id) VALUES ('VERIFY-DELETE-ME')" >/dev/null 2>&1; then
  bad "agent_ro CAN INSERT - this is a Phase 1 failure, fix db/003_roles.sql now"
  docker exec pg-ai psql -U postgres -d plant -c \
    "DELETE FROM equipment WHERE equipment_id='VERIFY-DELETE-ME'" >/dev/null 2>&1
else
  ok "agent_ro INSERT is rejected"
fi

head_ "Reference data"
missing_eq=$(docker exec pg-ai psql -U postgres -d plant -tAc \
  "SELECT count(*) FROM equipment WHERE coalesce(array_length(aliases,1),0)=0" 2>/dev/null || echo "?")
missing_tag=$(docker exec pg-ai psql -U postgres -d plant -tAc \
  "SELECT count(*) FROM tags WHERE coalesce(array_length(aliases,1),0)=0" 2>/dev/null || echo "?")
[ "$missing_eq" = "0" ]  && ok "every equipment item has an alias" || bad "$missing_eq equipment items have no alias"
[ "$missing_tag" = "0" ] && ok "every tag has an alias"            || bad "$missing_tag tags have no alias"

if docker exec pg-ai psql -U postgres -d plant -tAc \
     "SELECT 1 FROM pg_extension WHERE extname='vector'" 2>/dev/null | grep -q 1; then
  ok "pgvector extension present"
else
  bad "pgvector extension missing"
fi

head_ "Historian item mapping"
# The check that stops Phase 5 finding (a) coming back. A historised item with
# no tag and no written reason is unreachable from a tag-level lookup, and it
# fails as "no records found" - which an operator cannot tell apart from an
# absence of data. It is enforced at generation and at load; this confirms what
# is actually in the running database.
orphans=$(docker exec pg-ai psql -U postgres -d plant -tAc \
  "SELECT count(*) FROM historian_items
    WHERE his_group IS NOT NULL AND tag_id IS NULL AND exclusion_reason IS NULL" 2>/dev/null || echo "?")
[ "$orphans" = "0" ] && ok "every historised item resolves to a tag or a stated reason" \
                     || bad "$orphans historised items resolve to nothing"

# Equipment must be asserted in exactly one place. Nothing in the history may
# carry an equipment column - that denormalisation is what made the station's
# most obvious question unanswerable in finding (c).
stray=$(docker exec pg-ai psql -U postgres -d plant -tAc \
  "SELECT count(*) FROM information_schema.columns
    WHERE table_schema='fixture' AND column_name='equipment_id'" 2>/dev/null || echo "?")
[ "$stray" = "0" ] && ok "no equipment column in the history (equipment is asserted once, in tags)" \
                   || bad "$stray history columns named equipment_id - equipment is asserted twice"

head_ "Timezone"
# alarms.yml converts first_alarm/last_alarm inside the measure and names the
# zone as a literal, because Jinja env_var support could not be tested against
# the pinned Cube v1.1.7. That literal must not drift from api.env.
env_tz=$(grep -E '^SITE_TIMEZONE=' "$HOME/ai/api.env" 2>/dev/null | cut -d= -f2 | tr -d ' ')
model_tz=$(grep -oE "AT TIME ZONE '[^']+'" "$HOME/ai/cube/model/alarms.yml" 2>/dev/null \
             | head -1 | sed "s/.*'\(.*\)'/\1/")
if [ -z "$env_tz" ] || [ -z "$model_tz" ]; then
  bad "could not read SITE_TIMEZONE from api.env ($env_tz) or alarms.yml ($model_tz)"
elif [ "$env_tz" = "$model_tz" ]; then
  ok "alarms.yml converts to $model_tz, matching SITE_TIMEZONE"
else
  bad "alarms.yml converts to '$model_tz' but SITE_TIMEZONE is '$env_tz' - first_alarm will be reported in the wrong zone"
fi

head_ "Fixture data"
if docker exec pg-ai psql -U postgres -d plant -tAc \
     "SELECT 1 FROM information_schema.schemata WHERE schema_name='fixture'" 2>/dev/null | grep -q 1; then
  rows=$(docker exec pg-ai psql -U postgres -d plant -tAc \
    "SELECT count(*) FROM fixture.alarm_history WHERE state='ACTIVE'" 2>/dev/null)
  span=$(docker exec pg-ai psql -U postgres -d plant -tAc \
    "SELECT horizon_days || ' days from ' || origin::date FROM fixture.build_meta" 2>/dev/null)
  printf '  \033[33m!!\033[0m   fixture schema present (%s alarm activations over %s) - answers are TEST DATA, not plant history\n' "$rows" "$span"
  printf '  \033[33m!!\033[0m   history is retained for 7 days, matching CI Server. Questions older than that return nothing, correctly.\n'
fi

head_ "Disk"
df -h / /datadisk | sed 's/^/  /'

printf '\n%s passed, %s failed\n' "$PASS" "$FAIL"
exit "$FAIL"