yau-plant-assistant/db/004_doc_uploads.sql

-- =============================================================================
-- 004_doc_uploads.sql — operator document upload, review and publication.
--
--   psql -h pg-ai -U postgres -d plant -f 004_doc_uploads.sql
--
-- Applied at Phase 9. Additive: it does not alter doc_chunks, and the answer
-- path behaves identically whether this table exists or not.
--
-- WHAT THIS TABLE IS. The queue and the audit trail for documents that arrive
-- through the UI instead of by SSH. One row per uploaded file, from the moment
-- it lands in the inbox to the moment its chunks are citable — including who
-- uploaded it, who confirmed its identity, and exactly what they confirmed.
--
-- WHAT IT IS NOT. It is not document control. It records what a named person
-- asserted about a file on a date; it does not know what the current revision
-- of WRPS-OPS-014 actually is. See section 16 of BUILD-AI-CONTAINERS.md.
--
-- THE RULE THIS TABLE EXISTS TO PRESERVE: a wrong revision on a procedure is a
-- safety issue, not a data-quality one. ingest.py asks a human at a terminal.
-- Nothing about a web form removes that requirement — it moves the question
-- from a terminal prompt to a review screen, and records the answer instead of
-- discarding it.
-- =============================================================================

-- -----------------------------------------------------------------------------
-- The state machine. A CHECK rather than an enum, because adding a value to an
-- enum needs an ALTER TYPE that will not run inside the rest of a migration.
--
--   uploaded          file written to the inbox, nothing parsed yet
--        |            (ai-docs-worker picks it up)
--        v
--   scanning          Docling parsing; header proposal being extracted
--        |
--        v
--   awaiting_review   proposal ready; a human must confirm or correct it
--        |     \
--        |      \--> rejected     reviewer refused it; file stays in the inbox
--        v
--   approved          header CONFIRMED BY A NAMED PERSON; supersede decided
--        |            (worker claims it)
--        v
--   ingesting         file moved into /docs/<folder>/, chunked and embedded
--        |     \
--        |      \--> failed       error recorded; retryable, file is in place
--        v
--   published         chunks live in doc_chunks and citable
--
-- Only `approved` causes chunks to be written. There is no transition from
-- `uploaded` to `ingesting`. That is the design, not an omission.
-- -----------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS doc_uploads (
    upload_id        UUID PRIMARY KEY,
    status           TEXT NOT NULL DEFAULT 'uploaded',

    -- --- the file -----------------------------------------------------------
    original_filename TEXT NOT NULL,     -- as the operator's browser sent it
    stored_path       TEXT NOT NULL,     -- /inbox/<upload_id>/<safe_filename>
    content_type      TEXT,
    size_bytes        BIGINT NOT NULL,
    sha256            TEXT NOT NULL,     -- dedupe, and proof the bytes are unchanged
    page_count        INT,
    preview_text      TEXT,              -- first page, for the review screen

    -- --- who ----------------------------------------------------------------
    -- From Authelia's Remote-User / Remote-Name, forwarded by Caddy. NEVER from
    -- the request body: a browser must not be able to name its own uploader.
    uploaded_by       TEXT NOT NULL,
    uploaded_by_name  TEXT,
    uploaded_at       TIMESTAMPTZ NOT NULL DEFAULT now(),
    uploader_note     TEXT,              -- "assist pump start logic changed, rev C"

    -- --- what the uploader said it is (a proposal, never authoritative) ------
    proposed_doc_type TEXT,
    detected_doc_number     TEXT,
    detected_revision       TEXT,
    detected_effective_date DATE,

    -- --- what a human confirmed ---------------------------------------------
    -- These are the values that reach doc_chunks. They stay NULL until somebody
    -- in the publisher group types or accepts them.
    confirmed_doc_type       TEXT,
    confirmed_doc_number     TEXT,
    confirmed_revision       TEXT,
    confirmed_effective_date DATE,
    supersede_previous       BOOLEAN,    -- withdraw other revisions of this doc_number
    reference_data_checked   BOOLEAN,    -- see the note below — not cosmetic
    reviewed_by       TEXT,
    reviewed_by_name  TEXT,
    reviewed_at       TIMESTAMPTZ,
    review_note       TEXT,              -- required when rejecting

    -- --- outcome ------------------------------------------------------------
    published_source_file TEXT,          -- the doc_chunks.source_file key
    chunk_count      INT,
    superseded_count INT,
    error            TEXT,               -- operator-readable; never a stack trace
    attempts         INT NOT NULL DEFAULT 0,
    claimed_at       TIMESTAMPTZ,        -- worker lease, for stuck-job detection
    updated_at       TIMESTAMPTZ NOT NULL DEFAULT now(),

    CONSTRAINT doc_uploads_status_ck CHECK (status IN (
        'uploaded','scanning','awaiting_review','rejected',
        'approved','ingesting','published','failed')),

    -- doc_type still comes from a fixed list, and on approval the file is moved
    -- into the folder that matches it. The folder stays the on-disk truth.
    CONSTRAINT doc_uploads_proposed_type_ck CHECK (
        proposed_doc_type IS NULL OR proposed_doc_type IN
        ('procedure','manual','rationalisation','design')),
    CONSTRAINT doc_uploads_confirmed_type_ck CHECK (
        confirmed_doc_type IS NULL OR confirmed_doc_type IN
        ('procedure','manual','rationalisation','design')),

    -- An approved row with an incomplete confirmed header is the exact defect
    -- this design exists to prevent. Refuse it in the database, not only in the
    -- API — the API is one bad deployment away from being bypassed.
    CONSTRAINT doc_uploads_approved_needs_header_ck CHECK (
        status NOT IN ('approved','ingesting','published')
        OR (confirmed_doc_type IS NOT NULL
            AND confirmed_doc_number IS NOT NULL
            AND confirmed_revision IS NOT NULL
            AND confirmed_effective_date IS NOT NULL
            AND supersede_previous IS NOT NULL
            AND reviewed_by IS NOT NULL)),

    CONSTRAINT doc_uploads_rejected_needs_reason_ck CHECK (
        status <> 'rejected'
        OR (reviewed_by IS NOT NULL AND review_note IS NOT NULL))
);

-- The scan the worker runs every few seconds. Partial: the interesting rows are
-- a handful at a time, the published ones accumulate forever.
CREATE INDEX IF NOT EXISTS doc_uploads_pending
    ON doc_uploads (uploaded_at)
    WHERE status IN ('uploaded','approved');

CREATE INDEX IF NOT EXISTS doc_uploads_status_recent
    ON doc_uploads (status, uploaded_at DESC);

-- The same bytes uploaded twice is almost always a double-click or a re-send,
-- not a second document. Not UNIQUE: re-uploading after a rejection is
-- legitimate, and the API decides, having seen the earlier row.
CREATE INDEX IF NOT EXISTS doc_uploads_sha256 ON doc_uploads (sha256);

CREATE OR REPLACE FUNCTION doc_uploads_touch() RETURNS TRIGGER AS $fn$
BEGIN
    NEW.updated_at := now();
    RETURN NEW;
END;
$fn$ LANGUAGE plpgsql;

DROP TRIGGER IF EXISTS doc_uploads_touch_tr ON doc_uploads;
CREATE TRIGGER doc_uploads_touch_tr BEFORE UPDATE ON doc_uploads
    FOR EACH ROW EXECUTE FUNCTION doc_uploads_touch();

-- -----------------------------------------------------------------------------
-- reference_data_checked — why a boolean on this table earns its place.
--
-- This upload path exists because PLC logic and SCADA programs change and a new
-- document is issued. Ingesting that document changes what the assistant can
-- CITE. It does not change db/seed/tags.csv, the Cube models, the alarm
-- setpoints in `tags`, or anything else the numeric answers are computed from.
--
-- So a design document describing a new interlock can go live while every
-- Historical and Advisory answer is still built on the old tag metadata — and
-- both look equally confident on screen.
--
-- The review screen asks the reviewer to confirm they have considered that. It
-- is an acknowledgement, not a check; nothing here can verify it. It exists so
-- the gap is visible at the one moment somebody can still act on it, and so the
-- audit trail shows who was asked.
-- -----------------------------------------------------------------------------

-- -----------------------------------------------------------------------------
-- live_documents — what is currently citable, one row per document revision.
--
-- The review screen needs this to answer "what am I about to supersede?" before
-- the reviewer ticks the box, not after.
-- -----------------------------------------------------------------------------
CREATE OR REPLACE VIEW live_documents AS
SELECT doc_number,
       revision,
       doc_type,
       max(effective_date)     AS effective_date,
       min(source_file)        AS source_file,
       count(*)                AS chunk_count,
       max(created_at)         AS ingested_at
  FROM doc_chunks
 WHERE superseded = FALSE
 GROUP BY doc_number, revision, doc_type;

-- =============================================================================
-- Roles. Three, not one, and the split is the point.
--
--   agent_ro    the answer path. SELECT only, everywhere. UNCHANGED here — it
--               gains no write anywhere, including on this table.
--   uploads_rw  ai-api. Writes the QUEUE and nothing else. It cannot write
--               doc_chunks, so no defect in an HTTP endpoint can put a chunk in
--               front of an operator without a human approval in between.
--   ingest_rw   ai-docs-worker. Writes doc_chunks. Has no HTTP surface at all
--               and is not on the proxy network.
--
-- ai-api therefore holds two connections: the existing agent_ro one for
-- answering, and a uploads_rw one for the document screens. Do not collapse
-- them into one role that can do both.
-- =============================================================================

DO $$
BEGIN
    IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'uploads_rw') THEN
        CREATE ROLE uploads_rw LOGIN;
    END IF;
    IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'ingest_rw') THEN
        CREATE ROLE ingest_rw LOGIN;
    END IF;
END
$$;

-- Passwords are NOT in this file. Set them from the 0600 env files, as in
-- 003_roles.sql.

-- --- uploads_rw — the queue, and read-only on everything else ----------------
GRANT CONNECT ON DATABASE plant TO uploads_rw;
GRANT USAGE ON SCHEMA public TO uploads_rw;
GRANT SELECT ON ALL TABLES IN SCHEMA public TO uploads_rw;
GRANT INSERT, UPDATE ON doc_uploads TO uploads_rw;
-- No DELETE: a rejected upload is history, not a mistake to erase.
REVOKE DELETE ON doc_uploads FROM uploads_rw;
-- Explicit, and re-checked at the Phase 9 gate: no write on doc_chunks.
REVOKE INSERT, UPDATE, DELETE ON doc_chunks FROM uploads_rw;
REVOKE CREATE ON SCHEMA public FROM uploads_rw;
REVOKE TEMPORARY ON DATABASE plant FROM uploads_rw;

-- --- ingest_rw — the worker -------------------------------------------------
GRANT CONNECT ON DATABASE plant TO ingest_rw;
GRANT USAGE ON SCHEMA public TO ingest_rw;
GRANT SELECT ON ALL TABLES IN SCHEMA public TO ingest_rw;
GRANT INSERT, UPDATE, DELETE ON doc_chunks TO ingest_rw;
GRANT USAGE, SELECT ON SEQUENCE doc_chunks_id_seq TO ingest_rw;
GRANT INSERT, UPDATE ON doc_uploads TO ingest_rw;
REVOKE CREATE ON SCHEMA public FROM ingest_rw;
REVOKE TEMPORARY ON DATABASE plant FROM ingest_rw;

-- The answer path reads the queue — so the UI can tell an operator a document
-- is pending rather than silently not finding it — and writes nothing.
GRANT SELECT ON doc_uploads TO agent_ro;
GRANT SELECT ON live_documents TO agent_ro, uploads_rw, ingest_rw;

-- =============================================================================
-- Phase 9 gate — verify, do not assume. Prove each role separately.
--
--   As agent_ro:
--     SELECT count(*) FROM doc_uploads;                  -- must work
--     UPDATE doc_uploads SET status = 'approved';        -- must be REJECTED
--
--   As uploads_rw:
--     INSERT INTO doc_uploads (...) VALUES (...);        -- must work
--     INSERT INTO doc_chunks (source_file, doc_type, chunk_text)
--          VALUES ('x','manual','x');                    -- must be REJECTED
--     DELETE FROM doc_uploads;                           -- must be REJECTED
--
--   As anyone — the constraint that carries the safety rule:
--     UPDATE doc_uploads SET status = 'approved'
--      WHERE upload_id = '<a row with no confirmed header>';
--                                                        -- must be REJECTED
--
-- An approved row with a NULL confirmed_revision is a Phase 9 failure, not a
-- detail to fix later.
-- =============================================================================