yau-plant-assistant/scripts/deploy.sh

#!/usr/bin/env bash
# Deploy the AI stack to yau-sls-poc-lin001.
#
#   ./scripts/deploy.sh [phase1|phase2|api|web|all]
#
# Run this ON lin001, from a checkout at ~/ai. It is deliberately additive and
# deliberately noisy: this host is shared and live, it runs customer-facing
# demos, and openplc-runtime on it is the PLC for the demo plant.
#
# WHAT THIS SCRIPT WILL NOT DO, because these interrupt other people or need a
# human decision:
#   - restart Caddy or Authelia (it prints the command and stops)
#   - edit ~/authelia/configuration.yml (root-owned; see authelia/access-rules.md)
#   - touch openplc-runtime, cicore1 or imh
#   - create DNS records (ask Dan)

set -euo pipefail

TARGET="${1:-all}"
REPO="$HOME/ai"
COMPOSE="$HOME/ai-compose.yml"
LANGFUSE_COMPOSE="$HOME/langfuse-compose.yml"
STAMP="$(date +%Y%m%d)"

say()  { printf '\n\033[1m==> %s\033[0m\n' "$*"; }
warn() { printf '\033[33m!!  %s\033[0m\n' "$*"; }
die()  { printf '\033[31mxx  %s\033[0m\n' "$*" >&2; exit 1; }

# --- Preflight: the checks that have actually caught problems here ----------
preflight() {
  say "Preflight"

  # Root has hit 100% before and killed Grafana. Growing data goes on /datadisk.
  local root_used datadisk_used
  root_used=$(df --output=pcent / | tail -1 | tr -dc '0-9')
  datadisk_used=$(df --output=pcent /datadisk | tail -1 | tr -dc '0-9')
  echo "  /          ${root_used}% used"
  echo "  /datadisk  ${datadisk_used}% used"
  [ "$root_used" -lt 85 ] || die "/ is ${root_used}% full - stop and clear space first"
  [ "$datadisk_used" -lt 85 ] || warn "/datadisk is ${datadisk_used}% full - InfluxDB is the usual cause"

  docker network inspect proxy >/dev/null 2>&1 || die "the external 'proxy' network is missing"

  # The one deliberate published-port exception on this host. If it is not
  # running, the demo plant is down and that is more urgent than this deploy.
  docker ps --format '{{.Names}}' | grep -qx openplc-runtime \
    || warn "openplc-runtime is NOT running - the demo plant is down"

  for envfile in "$HOME/ai/pg-ai.env" "$HOME/ai/api.env"; do
    [ -f "$envfile" ] || die "missing $envfile - create it 0600, see .env.example"
    local mode
    mode=$(stat -c '%a' "$envfile")
    [ "$mode" = "600" ] || die "$envfile is mode $mode, must be 600"
  done

  mkdir -p /datadisk/pg-ai /datadisk/ai-docs /datadisk/langfuse/db
}

# --- Sync the repo into place ------------------------------------------------
sync_files() {
  say "Syncing compose files and application code"
  # Compose files live in ~ by house convention; the repo is the source of them.
  cp -v "$REPO/compose/ai-compose.yml"       "$COMPOSE"
  cp -v "$REPO/compose/langfuse-compose.yml" "$LANGFUSE_COMPOSE"
  mkdir -p "$HOME/ai/cube"
  rsync -a --delete "$REPO/cube/model/" "$HOME/ai/cube/model/"
}

# --- Phase 1: pg-ai, schema, roles, seed, fixtures --------------------------
phase1() {
  say "Phase 1 - pg-ai"
  docker compose -f "$COMPOSE" up -d pg-ai

  echo "  waiting for pg-ai to report healthy"
  for _ in $(seq 1 30); do
    [ "$(docker inspect -f '{{.State.Health.Status}}' pg-ai)" = "healthy" ] && break
    sleep 2
  done
  [ "$(docker inspect -f '{{.State.Health.Status}}' pg-ai)" = "healthy" ] \
    || die "pg-ai did not become healthy - check docker logs pg-ai"

  say "Applying schema, roles and seed data"
  docker cp "$REPO/db" pg-ai:/tmp/db
  docker exec -e PGPASSWORD_FILE=/dev/null pg-ai \
    psql -U postgres -d plant -v ON_ERROR_STOP=1 -f /tmp/db/001_schema.sql
  docker exec pg-ai \
    psql -U postgres -d plant -v ON_ERROR_STOP=1 -f /tmp/db/003_roles.sql

  # Aliases are pipe-separated in the CSVs; split them on load.
  docker exec -i pg-ai psql -U postgres -d plant -v ON_ERROR_STOP=1 <<'PSQL'
CREATE TEMP TABLE eq_stage (equipment_id TEXT, display_name TEXT, aliases TEXT,
                            equipment_type TEXT, unit_name TEXT, description TEXT);
\copy eq_stage FROM '/tmp/db/seed/equipment.csv' WITH (FORMAT csv, HEADER true)
INSERT INTO equipment
SELECT equipment_id, display_name, string_to_array(aliases,'|'),
       equipment_type, unit_name, description FROM eq_stage
ON CONFLICT (equipment_id) DO UPDATE SET
  display_name=EXCLUDED.display_name, aliases=EXCLUDED.aliases,
  equipment_type=EXCLUDED.equipment_type, unit_name=EXCLUDED.unit_name,
  description=EXCLUDED.description;

CREATE TEMP TABLE tag_stage (tag_id TEXT, equipment_id TEXT, display_name TEXT,
  aliases TEXT, signal_type TEXT, engineering_unit TEXT, range_low DOUBLE PRECISION,
  range_high DOUBLE PRECISION, alarm_setpoint_hi DOUBLE PRECISION,
  alarm_setpoint_lo DOUBLE PRECISION, trip_setpoint DOUBLE PRECISION, description TEXT);
\copy tag_stage FROM '/tmp/db/seed/tags.csv' WITH (FORMAT csv, HEADER true)
INSERT INTO tags
SELECT tag_id, equipment_id, display_name, string_to_array(aliases,'|'),
       signal_type, engineering_unit, range_low, range_high, alarm_setpoint_hi,
       alarm_setpoint_lo, trip_setpoint, description FROM tag_stage
ON CONFLICT (tag_id) DO UPDATE SET
  equipment_id=EXCLUDED.equipment_id, display_name=EXCLUDED.display_name,
  aliases=EXCLUDED.aliases, signal_type=EXCLUDED.signal_type,
  engineering_unit=EXCLUDED.engineering_unit, range_low=EXCLUDED.range_low,
  range_high=EXCLUDED.range_high, alarm_setpoint_hi=EXCLUDED.alarm_setpoint_hi,
  alarm_setpoint_lo=EXCLUDED.alarm_setpoint_lo, trip_setpoint=EXCLUDED.trip_setpoint,
  description=EXCLUDED.description;

-- The CI Server item dictionary. This is what the historian is keyed on, and
-- the only place an item name is joined to a tag - so it must load AFTER tags
-- (it references them) and BEFORE the fixtures (they are driven by it).
-- Regenerate from the WRPS repo with scripts/gen_historian_items.py.
CREATE TEMP TABLE hi_stage (LIKE historian_items EXCLUDING CONSTRAINTS);
\copy hi_stage FROM '/tmp/db/seed/historian_items.csv' WITH (FORMAT csv, HEADER true, NULL '')
INSERT INTO historian_items SELECT * FROM hi_stage
ON CONFLICT (item_name) DO UPDATE SET
  tag_id=EXCLUDED.tag_id, exclusion_reason=EXCLUDED.exclusion_reason,
  section_path=EXCLUDED.section_path, section=EXCLUDED.section,
  attribute=EXCLUDED.attribute, section_description=EXCLUDED.section_description,
  description=EXCLUDED.description, eng_unit=EXCLUDED.eng_unit,
  value_format=EXCLUDED.value_format, conv_type=EXCLUDED.conv_type,
  has_sign=EXCLUDED.has_sign, phys_low=EXCLUDED.phys_low,
  phys_high=EXCLUDED.phys_high, eng_gain=EXCLUDED.eng_gain,
  raw_to_eng=EXCLUDED.raw_to_eng, his_group=EXCLUDED.his_group,
  scan_interval_seconds=EXCLUDED.scan_interval_seconds, life_time=EXCLUDED.life_time,
  scada_point=EXCLUDED.scada_point, iec_address=EXCLUDED.iec_address,
  modbus_kind=EXCLUDED.modbus_kind, modbus_address=EXCLUDED.modbus_address,
  data_type=EXCLUDED.data_type, point_time_zone=EXCLUDED.point_time_zone;

-- How the PLC alarm word decomposes. Reference data, not fixtures: it is a
-- property of the PLC program and survives the cutover to imh unchanged.
CREATE TEMP TABLE ab_stage (LIKE alarm_bits EXCLUDING CONSTRAINTS);
\copy ab_stage FROM '/tmp/db/seed/alarm_bits.csv' WITH (FORMAT csv, HEADER true)
INSERT INTO alarm_bits SELECT * FROM ab_stage
ON CONFLICT (bit) DO UPDATE SET
  alarm_type=EXCLUDED.alarm_type, priority=EXCLUDED.priority,
  tag_id=EXCLUDED.tag_id, alarm_text=EXCLUDED.alarm_text,
  description=EXCLUDED.description;

-- Phase 1 gate, and the check that finding (a) cannot come back: every
-- historised item resolves to a tag, or says in writing why it does not.
DO $$
DECLARE orphan TEXT;
BEGIN
    SELECT string_agg(item_name, ', ') INTO orphan
      FROM historian_items
     WHERE his_group IS NOT NULL AND tag_id IS NULL AND exclusion_reason IS NULL;
    IF orphan IS NOT NULL THEN
        RAISE EXCEPTION 'historised items with no tag and no stated reason: %', orphan;
    END IF;
END $$;
PSQL

  # Fixtures last, and only while imh is pending.
  if grep -q '^USE_FIXTURES=true' "$HOME/ai/api.env"; then
    warn "USE_FIXTURES=true - loading GENERATED fixture data, not plant history"
    docker exec pg-ai psql -U postgres -d plant -v ON_ERROR_STOP=1 -f /tmp/db/002_fixtures.sql
  else
    say "USE_FIXTURES is not true - skipping fixtures, Cube should point at imh"
  fi

  docker exec pg-ai rm -rf /tmp/db
}

# --- Phase 2: Langfuse -------------------------------------------------------
phase2() {
  say "Phase 2 - Langfuse"
  [ -f "$HOME/ai/langfuse.env" ] || die "missing ~/ai/langfuse.env (0600)"
  docker compose -f "$LANGFUSE_COMPOSE" up -d
  manual_steps "lf.yokogawa.tech"
}

# --- Application containers --------------------------------------------------
deploy_api() {
  say "Building and starting cube and ai-api"
  docker compose -f "$COMPOSE" up -d --build cube ai-api
  manual_steps "cube.yokogawa.tech and api.yokogawa.tech"
}

deploy_web() {
  say "Building and starting ai-web"
  docker compose -f "$COMPOSE" up -d --build ai-web
  manual_steps "ai.yokogawa.tech"
  warn "The ai.yokogawa.tech Caddy block routes /ask to ai-api so the page is"
  warn "same-origin: api.yokogawa.tech has NO pinpoint record and does not"
  warn "resolve inside the VNet. Do not widen that route past /ask - the"
  warn "Phase 9 publisher rule is scoped to api.yokogawa.tech. See"
  warn "caddy/ai-routes.caddy."
  warn ""
  warn "SCADA-ONLY, applied 2026-08-28: the ai.yokogawa.tech block does NOT"
  warn "import authelia. It admits remote_ip 10.0.0.21 (cicore1) and 403s"
  warn "everything else - so step 3 below does not apply to this hostname."
  warn "Leave the ai.yokogawa.tech entry in the Authelia rule in place anyway;"
  warn "it is inert, and it keeps rollback to a Caddy reload instead of an"
  warn "Authelia restart. Verifying from this host gives 403, which is the PASS."
  warn "The allow arm can only be proved from cicore1."
  # A bundle built with VITE_API_BASE set to a public hostname works from
  # outside and fails on every control-room PC. Catch it here, not on the day.
  if docker exec ai-web sh -c 'grep -rqs "api\.yokogawa\.tech" /usr/share/nginx/html' 2>/dev/null; then
    warn "THIS BUILD hard-codes api.yokogawa.tech. Rebuild with VITE_API_BASE empty."
  fi
}

# --- The parts a human must do -----------------------------------------------
manual_steps() {
  local hostnames="$1"
  cat <<EOM

  ------------------------------------------------------------------
  MANUAL STEPS for ${hostnames} - this script stops here on purpose.

  1. DNS A record -> 20.211.144.151. Ask Dan; DNS is not managed here.
     Caddy cannot issue a certificate without it.
     DONE 2026-08-27 for ai, api and cube, plus the DC pinpoint record
     ai.yokogawa.tech -> 10.0.0.17. Nothing to do for those three.

  2. Append the block from caddy/ai-routes.caddy to ~/Caddyfile.
     Keep 'import authelia'. Omitting it silently makes the service public.
       cp ~/Caddyfile ~/Caddyfile.bak-ai-${STAMP}
       docker exec caddy caddy reload --config /etc/caddy/Caddyfile

  3. Add the hostname to the HTTPS_UserAccess two_factor rule.
     See authelia/access-rules.md. Root-owned - use sudo, back up first.
       sudo cp ~/authelia/configuration.yml ~/authelia/configuration.yml.bak-ai-${STAMP}

  4. ANNOUNCE, then restart Authelia. It logs out every active user on the
     host, including anyone mid-demo.
       docker compose -f ~/authelia-compose.yml restart authelia

  5. Verify. 'Up' is not proof.
       ./scripts/verify.sh
  ------------------------------------------------------------------

EOM
}

case "$TARGET" in
  phase1) preflight; sync_files; phase1 ;;
  phase2) preflight; sync_files; phase2 ;;
  api)    preflight; sync_files; deploy_api ;;
  web)    preflight; sync_files; deploy_web ;;
  all)    preflight; sync_files; phase1; phase2; deploy_api; deploy_web ;;
  *)      die "unknown target '$TARGET' - use phase1|phase2|api|web|all" ;;
esac

say "Done. Now run ./scripts/verify.sh - docker ps showing Up is not proof."