yau-plant-assistant/db/006_doc_pool.sql

-- =============================================================================
-- 006_doc_pool.sql — which documents are in the retrieval pool.
--
--   psql -h pg-ai -U postgres -d plant -f 006_doc_pool.sql
--
-- Applied at Phase 9, after 005. Additive: `pool_enabled` defaults TRUE, so
-- every document already ingested stays exactly where it was.
--
-- TWO REASONS A CHUNK IS NOT RETRIEVABLE, AND THEY MUST NOT BE CONFLATED:
--
--   superseded = TRUE     A statement ABOUT THE DOCUMENT. It is withdrawn, or
--                         a newer revision replaced it. Document-control state.
--                         Safety-meaningful. Changing it is an operational act
--                         with a reason and an audit row (005).
--
--   pool_enabled = FALSE  A statement ABOUT THE CORPUS. This document is not
--                         part of the set we are running with. It says nothing
--                         about whether the document is valid or current.
--                         Curation, and — see section 16.13 — demonstration.
--
-- Retrieval requires BOTH: superseded = FALSE AND pool_enabled. That is what
-- makes them orthogonal and safe to expose separately. Re-enabling a withdrawn
-- document in the pool does NOT make it citable again; only a restore does.
-- Somebody curating the pool cannot accidentally resurrect a withdrawn
-- procedure, which is the mistake this separation is here to make impossible.
--
-- If these two ever collapse into one flag, a demo that trimmed the corpus
-- becomes indistinguishable from a document that was withdrawn on purpose.
-- =============================================================================

ALTER TABLE doc_chunks
    ADD COLUMN IF NOT EXISTS pool_enabled BOOLEAN NOT NULL DEFAULT TRUE;

-- The retrieval predicate, as an index. Replaces doc_chunks_live_type as the
-- filter that matches what tools/retrieval.py actually asks for.
CREATE INDEX IF NOT EXISTS doc_chunks_retrievable
    ON doc_chunks (doc_type)
    WHERE superseded = FALSE AND pool_enabled;

-- -----------------------------------------------------------------------------
-- READ THIS BEFORE DISABLING MOST OF THE CORPUS.
--
-- The HNSW index is built over EVERY embedding, superseded and disabled rows
-- included. An approximate scan finds the k nearest vectors and the WHERE
-- clause filters afterwards, so if 90% of the pool is disabled — exactly what a
-- coverage demo does — the scan can come back with almost nothing even though
-- relevant enabled documents exist. The failure looks like "retrieval got
-- worse", which in a demo about corpus size is the single most misleading
-- result available.
--
-- With a corpus this small (thousands of chunks, not millions) the fix is
-- cheap. Either raise the candidate list for the query:
--
--     SET LOCAL hnsw.ef_search = 200;
--
-- or, when the enabled fraction is low, drop to an exact scan for that query:
--
--     SET LOCAL enable_indexscan = off;   -- sequential + exact, milliseconds here
--
-- tools/retrieval.py should do the second automatically below a threshold
-- (POOL_EXACT_SCAN_BELOW_PCT). Measure it before the demo, not during.
-- -----------------------------------------------------------------------------

-- -----------------------------------------------------------------------------
-- doc_pool_profiles — a named set of documents, applied and reverted as one.
--
-- "Show them the answer with three documents, then with all forty-seven" is not
-- forty-four checkbox clicks, and it must be revertible in one action, in front
-- of an audience, without anyone wondering afterwards whether the pool was left
-- trimmed. A profile is a saved selection, not a copy of the documents.
--
-- Profiles are also how a demo avoids touching global state at all: the
-- intended path is a PER-REQUEST override naming a profile (section 16.14),
-- which changes nothing stored and leaks nothing to the operator asking a real
-- question on the same shared host at the same time.
-- -----------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS doc_pool_profiles (
    profile_id   UUID PRIMARY KEY,
    name         TEXT NOT NULL UNIQUE,   -- 'full', 'procedures-only', 'minimal-3'
    description  TEXT,
    is_builtin   BOOLEAN NOT NULL DEFAULT FALSE,
    created_by   TEXT,
    created_at   TIMESTAMPTZ NOT NULL DEFAULT now()
);

CREATE TABLE IF NOT EXISTS doc_pool_profile_members (
    profile_id   UUID NOT NULL REFERENCES doc_pool_profiles(profile_id) ON DELETE CASCADE,
    source_file  TEXT NOT NULL,
    PRIMARY KEY (profile_id, source_file)
);

CREATE INDEX IF NOT EXISTS doc_pool_members_file
    ON doc_pool_profile_members (source_file);

-- 'full' is the only profile with no members: it means "no restriction", not
-- "no documents". Special-cased in one place in the API and nowhere else.
INSERT INTO doc_pool_profiles (profile_id, name, description, is_builtin)
VALUES ('00000000-0000-0000-0000-000000000001',
        'full',
        'Every enabled, live document. The operational default.',
        TRUE)
ON CONFLICT (name) DO NOTHING;

-- -----------------------------------------------------------------------------
-- Curating the pool is an audited act, like withdrawing.
-- 005 already holds the trail; it gains two more verbs.
-- -----------------------------------------------------------------------------
ALTER TABLE doc_actions DROP CONSTRAINT IF EXISTS doc_actions_action_ck;
ALTER TABLE doc_actions ADD CONSTRAINT doc_actions_action_ck
    CHECK (action IN ('withdraw','restore','purge','pool_disable','pool_enable'));

-- -----------------------------------------------------------------------------
-- pool_status — what the assistant is actually running with right now.
--
-- The number in the corner of the screen. An operator should be able to see
-- that the pool is not whole without asking anybody, and a demo should not be
-- able to hide it.
-- -----------------------------------------------------------------------------
CREATE OR REPLACE VIEW pool_status AS
SELECT count(DISTINCT source_file) FILTER (
           WHERE superseded = FALSE AND pool_enabled)          AS documents_in_pool,
       count(DISTINCT source_file) FILTER (WHERE superseded = FALSE) AS documents_live,
       count(*) FILTER (WHERE superseded = FALSE AND pool_enabled)   AS chunks_in_pool,
       count(*) FILTER (WHERE superseded = FALSE)              AS chunks_live
  FROM doc_chunks;

-- -----------------------------------------------------------------------------
-- pool_documents — the curation screen's list. Every live document, in or out.
-- -----------------------------------------------------------------------------
CREATE OR REPLACE VIEW pool_documents AS
SELECT source_file,
       max(doc_number)       AS doc_number,
       max(revision)         AS revision,
       max(doc_type)         AS doc_type,
       max(effective_date)   AS effective_date,
       count(*)              AS chunk_count,
       bool_and(pool_enabled) AS pool_enabled,
       max(created_at)       AS ingested_at
  FROM doc_chunks
 WHERE superseded = FALSE
 GROUP BY source_file;

-- =============================================================================
-- Grants.
--
-- pool_enabled is NOT a safety flag, so uploads_rw may set it in BOTH
-- directions - unlike `superseded`, where the trigger in 005 allows only
-- withdrawal. Being in the pool is not a claim that a document is current;
-- `superseded` is still the only thing that says that, and it is still the only
-- thing the web-facing role cannot undo.
-- =============================================================================

GRANT UPDATE (pool_enabled) ON doc_chunks TO uploads_rw;
GRANT SELECT, INSERT, UPDATE, DELETE ON doc_pool_profiles TO uploads_rw;
GRANT SELECT, INSERT, UPDATE, DELETE ON doc_pool_profile_members TO uploads_rw;
GRANT SELECT ON doc_pool_profiles, doc_pool_profile_members TO agent_ro, ingest_rw;
GRANT SELECT ON pool_status, pool_documents TO agent_ro, uploads_rw, ingest_rw;

-- A profile is a selection, so deleting one destroys no documents. `full` is
-- protected because a demo that deletes the way back to the operational pool is
-- not a recoverable position in front of an audience.
CREATE OR REPLACE FUNCTION doc_pool_profiles_protect() RETURNS TRIGGER AS $fn$
BEGIN
    IF OLD.is_builtin THEN
        RAISE EXCEPTION 'the % profile is built in and cannot be % ',
            OLD.name, TG_OP;
    END IF;
    RETURN OLD;
END;
$fn$ LANGUAGE plpgsql;

DROP TRIGGER IF EXISTS doc_pool_profiles_protect_tr ON doc_pool_profiles;
CREATE TRIGGER doc_pool_profiles_protect_tr BEFORE DELETE OR UPDATE
    ON doc_pool_profiles
    FOR EACH ROW EXECUTE FUNCTION doc_pool_profiles_protect();

-- =============================================================================
-- Phase 9 gate — the pool half.
--
--   Orthogonality, which is the property the whole file exists for:
--     UPDATE doc_chunks SET pool_enabled = TRUE
--      WHERE superseded = TRUE;              -- allowed, and changes NOTHING:
--     -- the withdrawn document must still not be retrievable. Ask the question
--     -- that used to cite it and confirm it is not cited.
--
--   As uploads_rw:
--     UPDATE doc_chunks SET pool_enabled = FALSE WHERE ...;   -- must work
--     UPDATE doc_chunks SET pool_enabled = TRUE  WHERE ...;   -- must work
--     UPDATE doc_chunks SET superseded  = FALSE WHERE ...;    -- must be REJECTED
--     DELETE FROM doc_pool_profiles WHERE name = 'full';      -- must be REJECTED
--
--   Retrieval, with most of the corpus disabled - the demo case:
--     Disable 90% of documents, then ask a question whose answer is in one of
--     the remaining 10%. It MUST still be found. If it is not, the HNSW note at
--     the top of this file is why, and the fix is there, not in the prompt.
--
--   The banner, which is the part that protects everyone else on this host:
--     With any document out of the pool, EVERY answer must carry the reduced-
--     pool banner, exactly as fixture data does. An answer from a trimmed
--     corpus that looks like an answer from the whole corpus is the failure
--     this feature introduces, and the banner is the whole mitigation.
-- =============================================================================