yau-plant-assistant/scripts/deploy.sh

#!/usr/bin/env bash
# Deploy the AI stack to yau-sls-poc-lin001.
#
#   ./scripts/deploy.sh [phase1|phase2|api|web|all]
#
# Run this ON lin001, from a checkout at ~/ai. It is deliberately additive and
# deliberately noisy: this host is shared and live, it runs customer-facing
# demos, and openplc-runtime on it is the PLC for the demo plant.
#
# WHAT THIS SCRIPT WILL NOT DO, because these interrupt other people or need a
# human decision:
#   - restart Caddy or Authelia (it prints the command and stops)
#   - edit ~/authelia/configuration.yml (root-owned; see authelia/access-rules.md)
#   - touch openplc-runtime, cicore1 or imh
#   - create DNS records (ask Dan)

set -euo pipefail

TARGET="${1:-all}"
REPO="$HOME/ai"
COMPOSE="$HOME/ai-compose.yml"
LANGFUSE_COMPOSE="$HOME/langfuse-compose.yml"
STAMP="$(date +%Y%m%d)"

say()  { printf '\n\033[1m==> %s\033[0m\n' "$*"; }
warn() { printf '\033[33m!!  %s\033[0m\n' "$*"; }
die()  { printf '\033[31mxx  %s\033[0m\n' "$*" >&2; exit 1; }

# --- Preflight: the checks that have actually caught problems here ----------
preflight() {
  say "Preflight"

  # Root has hit 100% before and killed Grafana. Growing data goes on /datadisk.
  local root_used datadisk_used
  root_used=$(df --output=pcent / | tail -1 | tr -dc '0-9')
  datadisk_used=$(df --output=pcent /datadisk | tail -1 | tr -dc '0-9')
  echo "  /          ${root_used}% used"
  echo "  /datadisk  ${datadisk_used}% used"
  [ "$root_used" -lt 85 ] || die "/ is ${root_used}% full - stop and clear space first"
  [ "$datadisk_used" -lt 85 ] || warn "/datadisk is ${datadisk_used}% full - InfluxDB is the usual cause"

  docker network inspect proxy >/dev/null 2>&1 || die "the external 'proxy' network is missing"

  # The one deliberate published-port exception on this host. If it is not
  # running, the demo plant is down and that is more urgent than this deploy.
  docker ps --format '{{.Names}}' | grep -qx openplc-runtime \
    || warn "openplc-runtime is NOT running - the demo plant is down"

  for envfile in "$HOME/ai/pg-ai.env" "$HOME/ai/api.env"; do
    [ -f "$envfile" ] || die "missing $envfile - create it 0600, see .env.example"
    local mode
    mode=$(stat -c '%a' "$envfile")
    [ "$mode" = "600" ] || die "$envfile is mode $mode, must be 600"
  done

  mkdir -p /datadisk/pg-ai /datadisk/ai-docs /datadisk/langfuse/db
}

# --- Sync the repo into place ------------------------------------------------
sync_files() {
  say "Syncing compose files and application code"
  # Compose files live in ~ by house convention; the repo is the source of them.
  cp -v "$REPO/compose/ai-compose.yml"       "$COMPOSE"
  cp -v "$REPO/compose/langfuse-compose.yml" "$LANGFUSE_COMPOSE"
  mkdir -p "$HOME/ai/cube"
  rsync -a --delete "$REPO/cube/model/" "$HOME/ai/cube/model/"
}

# --- Phase 1: pg-ai, schema, roles, seed, fixtures --------------------------
phase1() {
  say "Phase 1 - pg-ai"
  docker compose -f "$COMPOSE" up -d pg-ai

  echo "  waiting for pg-ai to report healthy"
  for _ in $(seq 1 30); do
    [ "$(docker inspect -f '{{.State.Health.Status}}' pg-ai)" = "healthy" ] && break
    sleep 2
  done
  [ "$(docker inspect -f '{{.State.Health.Status}}' pg-ai)" = "healthy" ] \
    || die "pg-ai did not become healthy - check docker logs pg-ai"

  say "Applying schema, roles and seed data"
  docker cp "$REPO/db" pg-ai:/tmp/db
  docker exec -e PGPASSWORD_FILE=/dev/null pg-ai \
    psql -U postgres -d plant -v ON_ERROR_STOP=1 -f /tmp/db/001_schema.sql
  docker exec pg-ai \
    psql -U postgres -d plant -v ON_ERROR_STOP=1 -f /tmp/db/003_roles.sql

  # Aliases are pipe-separated in the CSVs; split them on load.
  docker exec -i pg-ai psql -U postgres -d plant -v ON_ERROR_STOP=1 <<'PSQL'
CREATE TEMP TABLE eq_stage (equipment_id TEXT, display_name TEXT, aliases TEXT,
                            equipment_type TEXT, unit_name TEXT, description TEXT);
\copy eq_stage FROM '/tmp/db/seed/equipment.csv' WITH (FORMAT csv, HEADER true)
INSERT INTO equipment
SELECT equipment_id, display_name, string_to_array(aliases,'|'),
       equipment_type, unit_name, description FROM eq_stage
ON CONFLICT (equipment_id) DO UPDATE SET
  display_name=EXCLUDED.display_name, aliases=EXCLUDED.aliases,
  equipment_type=EXCLUDED.equipment_type, unit_name=EXCLUDED.unit_name,
  description=EXCLUDED.description;

CREATE TEMP TABLE tag_stage (tag_id TEXT, equipment_id TEXT, display_name TEXT,
  aliases TEXT, signal_type TEXT, engineering_unit TEXT, range_low DOUBLE PRECISION,
  range_high DOUBLE PRECISION, alarm_setpoint_hi DOUBLE PRECISION,
  alarm_setpoint_lo DOUBLE PRECISION, trip_setpoint DOUBLE PRECISION, description TEXT);
\copy tag_stage FROM '/tmp/db/seed/tags.csv' WITH (FORMAT csv, HEADER true)
INSERT INTO tags
SELECT tag_id, equipment_id, display_name, string_to_array(aliases,'|'),
       signal_type, engineering_unit, range_low, range_high, alarm_setpoint_hi,
       alarm_setpoint_lo, trip_setpoint, description FROM tag_stage
ON CONFLICT (tag_id) DO UPDATE SET
  equipment_id=EXCLUDED.equipment_id, display_name=EXCLUDED.display_name,
  aliases=EXCLUDED.aliases, signal_type=EXCLUDED.signal_type,
  engineering_unit=EXCLUDED.engineering_unit, range_low=EXCLUDED.range_low,
  range_high=EXCLUDED.range_high, alarm_setpoint_hi=EXCLUDED.alarm_setpoint_hi,
  alarm_setpoint_lo=EXCLUDED.alarm_setpoint_lo, trip_setpoint=EXCLUDED.trip_setpoint,
  description=EXCLUDED.description;

-- The CI Server item dictionary. This is what the historian is keyed on, and
-- the only place an item name is joined to a tag - so it must load AFTER tags
-- (it references them) and BEFORE the fixtures (they are driven by it).
-- Regenerate from the WRPS repo with scripts/gen_historian_items.py.
-- Columns are NAMED, not positional. hi_stage is LIKE historian_items, so it
-- inherits the LIVE table's column order - and ALTER TABLE appends, so a
-- migrated database orders its columns differently from a freshly created one.
-- A positional copy then loads the wrong column into the wrong place, or fails
-- with a type error if you are lucky. Migration 008 made that concrete.
-- The \copy line is long because psql meta-commands cannot be wrapped.
CREATE TEMP TABLE hi_stage (LIKE historian_items EXCLUDING CONSTRAINTS);
\copy hi_stage (item_name,tag_id,exclusion_reason,section_path,section,attribute,section_description,description,eng_unit,value_format,conv_type,has_sign,phys_low,phys_high,eng_gain,raw_to_eng,his_group,scan_interval_seconds,life_time,ci_station,ci_point,poll_group,iec_address,modbus_kind,modbus_address,data_type,point_time_zone) FROM '/tmp/db/seed/historian_items.csv' WITH (FORMAT csv, HEADER true, NULL '')
INSERT INTO historian_items (item_name,tag_id,exclusion_reason,section_path,section,attribute,section_description,description,eng_unit,value_format,conv_type,has_sign,phys_low,phys_high,eng_gain,raw_to_eng,his_group,scan_interval_seconds,life_time,ci_station,ci_point,poll_group,iec_address,modbus_kind,modbus_address,data_type,point_time_zone)
SELECT item_name,tag_id,exclusion_reason,section_path,section,attribute,section_description,description,eng_unit,value_format,conv_type,has_sign,phys_low,phys_high,eng_gain,raw_to_eng,his_group,scan_interval_seconds,life_time,ci_station,ci_point,poll_group,iec_address,modbus_kind,modbus_address,data_type,point_time_zone FROM hi_stage
ON CONFLICT (item_name) DO UPDATE SET
  tag_id=EXCLUDED.tag_id, exclusion_reason=EXCLUDED.exclusion_reason,
  section_path=EXCLUDED.section_path, section=EXCLUDED.section,
  attribute=EXCLUDED.attribute, section_description=EXCLUDED.section_description,
  description=EXCLUDED.description, eng_unit=EXCLUDED.eng_unit,
  value_format=EXCLUDED.value_format, conv_type=EXCLUDED.conv_type,
  has_sign=EXCLUDED.has_sign, phys_low=EXCLUDED.phys_low,
  phys_high=EXCLUDED.phys_high, eng_gain=EXCLUDED.eng_gain,
  raw_to_eng=EXCLUDED.raw_to_eng, his_group=EXCLUDED.his_group,
  scan_interval_seconds=EXCLUDED.scan_interval_seconds, life_time=EXCLUDED.life_time,
  ci_station=EXCLUDED.ci_station, ci_point=EXCLUDED.ci_point,
  poll_group=EXCLUDED.poll_group, iec_address=EXCLUDED.iec_address,
  modbus_kind=EXCLUDED.modbus_kind, modbus_address=EXCLUDED.modbus_address,
  data_type=EXCLUDED.data_type, point_time_zone=EXCLUDED.point_time_zone;

-- How the PLC alarm word decomposes. Reference data, not fixtures: it is a
-- property of the PLC program and survives the cutover to imh unchanged.
CREATE TEMP TABLE ab_stage (LIKE alarm_bits EXCLUDING CONSTRAINTS);
\copy ab_stage FROM '/tmp/db/seed/alarm_bits.csv' WITH (FORMAT csv, HEADER true)
INSERT INTO alarm_bits SELECT * FROM ab_stage
ON CONFLICT (bit) DO UPDATE SET
  alarm_type=EXCLUDED.alarm_type, priority=EXCLUDED.priority,
  tag_id=EXCLUDED.tag_id, alarm_text=EXCLUDED.alarm_text,
  description=EXCLUDED.description;

-- Phase 1 gate, and the check that finding (a) cannot come back: every
-- historised item resolves to a tag, or says in writing why it does not.
DO $$
DECLARE orphan TEXT;
BEGIN
    SELECT string_agg(item_name, ', ') INTO orphan
      FROM historian_items
     WHERE his_group IS NOT NULL AND tag_id IS NULL AND exclusion_reason IS NULL;
    IF orphan IS NOT NULL THEN
        RAISE EXCEPTION 'historised items with no tag and no stated reason: %', orphan;
    END IF;
END $$;
PSQL

  # Fixtures last, and only while imh is pending.
  if grep -q '^USE_FIXTURES=true' "$HOME/ai/api.env"; then
    warn "USE_FIXTURES=true - loading GENERATED fixture data, not plant history"
    docker exec pg-ai psql -U postgres -d plant -v ON_ERROR_STOP=1 -f /tmp/db/002_fixtures.sql
  else
    say "USE_FIXTURES is not true - skipping fixtures, Cube should point at imh"
  fi

  docker exec pg-ai rm -rf /tmp/db
}

# --- Phase 2: Langfuse -------------------------------------------------------
phase2() {
  say "Phase 2 - Langfuse"
  [ -f "$HOME/ai/langfuse.env" ] || die "missing ~/ai/langfuse.env (0600)"
  docker compose -f "$LANGFUSE_COMPOSE" up -d
  manual_steps "lf.yokogawa.tech"
}

# --- Application containers --------------------------------------------------
deploy_api() {
  say "Building and starting cube and ai-api"
  docker compose -f "$COMPOSE" up -d --build cube ai-api
  manual_steps "cube.yokogawa.tech and api.yokogawa.tech"
}

deploy_web() {
  say "Building and starting ai-web"
  docker compose -f "$COMPOSE" up -d --build ai-web
  manual_steps "ai.yokogawa.tech"
  warn "The ai.yokogawa.tech Caddy block routes /ask to ai-api so the page is"
  warn "same-origin: api.yokogawa.tech has NO pinpoint record and does not"
  warn "resolve inside the VNet. Do not widen that route past /ask - the"
  warn "Phase 9 publisher rule is scoped to api.yokogawa.tech. See"
  warn "caddy/ai-routes.caddy."
  warn ""
  warn "SCADA-ONLY, applied 2026-08-28: the ai.yokogawa.tech block does NOT"
  warn "import authelia. It admits remote_ip 10.0.0.21 (cicore1) and 403s"
  warn "everything else - so step 3 below does not apply to this hostname."
  warn "Leave the ai.yokogawa.tech entry in the Authelia rule in place anyway;"
  warn "it is inert, and it keeps rollback to a Caddy reload instead of an"
  warn "Authelia restart. Verifying from this host gives 403, which is the PASS."
  warn "The allow arm cannot be proved by curl from here, but it IS proved:"
  warn "ai-web logs the real client address Caddy forwards, and 10.0.0.21"
  warn "appears there from 28 August (page) and 31 August (questions, 200)."
  warn "verify.sh checks that log rather than asking someone to go and look."
  # A bundle built with VITE_API_BASE set to a public hostname works from
  # outside and fails on every control-room PC. Catch it here, not on the day.
  if docker exec ai-web sh -c 'grep -rqs "api\.yokogawa\.tech" /usr/share/nginx/html' 2>/dev/null; then
    warn "THIS BUILD hard-codes api.yokogawa.tech. Rebuild with VITE_API_BASE empty."
  fi
}

# --- The parts a human must do -----------------------------------------------
manual_steps() {
  local hostnames="$1"
  cat <<EOM

  ------------------------------------------------------------------
  MANUAL STEPS for ${hostnames} - this script stops here on purpose.

  1. DNS A record -> 20.211.144.151. Ask Dan; DNS is not managed here.
     Caddy cannot issue a certificate without it.
     DONE 2026-08-27 for ai, api and cube, plus the DC pinpoint record
     ai.yokogawa.tech -> 10.0.0.17. Nothing to do for those three.

  2. Append the block from caddy/ai-routes.caddy to ~/Caddyfile.
     Keep 'import authelia'. Omitting it silently makes the service public.
       cp ~/Caddyfile ~/Caddyfile.bak-ai-${STAMP}
       docker exec caddy caddy reload --config /etc/caddy/Caddyfile

  3. Add the hostname to the HTTPS_UserAccess two_factor rule.
     See authelia/access-rules.md. Root-owned - use sudo, back up first.
       sudo cp ~/authelia/configuration.yml ~/authelia/configuration.yml.bak-ai-${STAMP}

  4. ANNOUNCE, then restart Authelia. It logs out every active user on the
     host, including anyone mid-demo.
       docker compose -f ~/authelia-compose.yml restart authelia

  5. Verify. 'Up' is not proof.
       ./scripts/verify.sh
  ------------------------------------------------------------------

EOM
}

case "$TARGET" in
  phase1) preflight; sync_files; phase1 ;;
  phase2) preflight; sync_files; phase2 ;;
  api)    preflight; sync_files; deploy_api ;;
  web)    preflight; sync_files; deploy_web ;;
  all)    preflight; sync_files; phase1; phase2; deploy_api; deploy_web ;;
  *)      die "unknown target '$TARGET' - use phase1|phase2|api|web|all" ;;
esac

say "Done. Now run ./scripts/verify.sh - docker ps showing Up is not proof."