From 885d8e31e231802ea02820445f931a8ffff90949 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 21 Aug 2026 15:24:25 +1000 Subject: [PATCH] Add NO_LLM_STUB: the whole chain, working, without a model Azure OpenAI is pending and imh is pending, so POST /ask could not return anything at all - which left the entire chain either side of the model unproven: the browser, the API, entity resolution, Cube, retrieval, the contracts, the banners, the error paths. All of it is testable now, and waiting for a key to find out whether it works is a choice to find out later. NO_LLM_STUB=true substitutes the two steps that need a model and nothing else. - Classification: the caller supplies the class, from a dropdown in the UI. NOT a keyword classifier. A crude keyword classifier produces a PLAUSIBLE label, and a plausible wrong label is the exact failure this system exists to prevent - "how do I reset it" landing in Historical is how a synthesised procedure reaches an operator. Choosing by hand is honest about what is happening and drives each branch deliberately. apply_safety_rules() still runs over the result. - Prose: a fixed placeholder per class, in stub.py. Everything else is the real path. This is possible because generate() already kept the factual fields away from the model: rows, counts, citations, the fixture flag and the class are attached from evidence, and only prose comes from the generator. Splitting that into _generate_prose() and _assemble() makes the seam explicit - the stub feeds _assemble() exactly as the model does, so this is a fair test of the assembly path rather than a mock of it. The contracts are the point. A stub payload goes through enforce_contract() unchanged, and it FAILED first time on two classes: the "nothing found" wording did not match the not-found detectors, so Reference and Procedural returned 422 rather than an uncited answer. That is the contract doing its job against text no model wrote. Retries are pointless on deterministic output, and a 422 is a real result here, not a stub bug. Retrieval is lexical (retrieval.lexical_search), because embedding the question needs the model. Kept beside search() and never called on the normal path, so nobody reads a trace and mistakes a lexical hit for a semantic one. It matches what the operator typed, not what they meant. What it does not prove: whether the classifier would have labelled correctly - a person did; whether retrieval finds the RIGHT chunk; and nothing about prose. It also cannot fill prerequisites_verbatim - extracting them with a regex would be the "synthesised from fragments" failure the Procedural contract forbids, so the list is empty and the answer says so. Every answer carries stub_mode: true in the contract, not decorated on by the UI, and a banner beside the fixture banner. Same reasoning: an answer nobody generated must not be indistinguishable from one that was. Also here: - demo/ai-docs: three fabricated documents, numbered WRPS-DEMO-00x so header extraction is genuinely exercised against a number no real WRPS document can have. Their setpoints contradict tags.csv on purpose. - VITE_API_BASE build arg, for a tunnelled build before DNS exists. The tunnel origin is allowed in CORS only while NO_LLM_STUB is on, so it disappears with the flag. Proxying /api through ai-web's nginx would have been easier and was rejected: it creates a second route to the API that bypasses the api.yokogawa.tech Caddy block, where the Phase 9 publisher rule lives. Verified on lin001 with no Azure key set at all: all five classes return 200 through the real UI in a browser, over an SSH tunnel, with citations from the demo documents, real Cube numbers, and both banners showing. Turning it off: NO_LLM_STUB=false in ~/ai/api.env, restart ai-api. Co-Authored-By: Claude Opus 5 --- api/agent.py | 242 ++++++++++------ api/config.py | 6 + api/contracts.py | 10 + api/main.py | 33 ++- api/stub.py | 268 ++++++++++++++++++ api/tools/retrieval.py | 109 +++++++ compose/ai-compose.yml | 5 + demo/ai-docs/README.md | 30 ++ demo/ai-docs/design/DEMO-design-basis.md | 39 +++ .../procedures/DEMO-interlock-bypass.md | 42 +++ .../DEMO-alarm-rationalisation.md | 37 +++ web/Dockerfile | 8 + web/src/App.tsx | 53 +++- web/src/styles.css | 16 ++ web/src/types.ts | 1 + 15 files changed, 810 insertions(+), 89 deletions(-) create mode 100644 api/stub.py create mode 100644 demo/ai-docs/README.md create mode 100644 demo/ai-docs/design/DEMO-design-basis.md create mode 100644 demo/ai-docs/procedures/DEMO-interlock-bypass.md create mode 100644 demo/ai-docs/rationalisation/DEMO-alarm-rationalisation.md diff --git a/api/agent.py b/api/agent.py index 681ce94..e3f3a2d 100644 --- a/api/agent.py +++ b/api/agent.py @@ -29,6 +29,7 @@ import classifier import tools.equipment as equipment_tool import tools.metrics as metrics import tools.retrieval as retrieval +import stub from config import settings from contracts import ContractViolation, QuestionClass from guardrails import enforce_contract @@ -43,6 +44,7 @@ class State(TypedDict, total=False): payload: dict[str, Any] answer: Any trace: Any + forced_class: str | None def _client() -> AzureOpenAI: @@ -163,13 +165,20 @@ def gather_historical(state: State) -> State: def gather_reference(state: State) -> State: - client = _client() equipment_id, _ = _resolve_entities(state) - embedding = _embed(state["question"], client=client) - chunks = retrieval.rerank( - retrieval.search(embedding, top_k=8, equipment_id=equipment_id), - state["question"], - ) + if stub.enabled(): + chunks = retrieval.rerank( + retrieval.lexical_search( + state["question"], top_k=8, equipment_id=equipment_id + ), + state["question"], + ) + else: + embedding = _embed(state["question"], client=_client()) + chunks = retrieval.rerank( + retrieval.search(embedding, top_k=8, equipment_id=equipment_id), + state["question"], + ) tag_rows = equipment_tool.tags_for_equipment(equipment_id) if equipment_id else [] state["evidence"] = { "chunks": retrieval.as_dicts(chunks), @@ -182,12 +191,16 @@ def gather_reference(state: State) -> State: def gather_procedural(state: State) -> State: """Procedures only, live revisions only. Nothing else is in scope here.""" - client = _client() equipment_id, _ = _resolve_entities(state) - embedding = _embed(state["question"], client=client) - chunks = retrieval.find_procedure( - embedding, state["question"], equipment_id=equipment_id - ) + if stub.enabled(): + chunks = retrieval.find_procedure_lexical( + state["question"], equipment_id=equipment_id + ) + else: + embedding = _embed(state["question"], client=_client()) + chunks = retrieval.find_procedure( + embedding, state["question"], equipment_id=equipment_id + ) state["evidence"] = { "chunks": retrieval.as_dicts(chunks), "citations": [c.citation() for c in chunks], @@ -199,13 +212,18 @@ def gather_procedural(state: State) -> State: def gather_advisory(state: State) -> State: """Both paths: what was done (Cube) and what is allowed (documents).""" trace = state.get("trace") - client = _client() result = metrics.run(metrics.pump_down_evidence(days=30), trace=trace) _, _, window_description = metrics.rolling_window(30) - embedding = _embed(state["question"], client=client) - chunks = retrieval.rerank( - retrieval.search(embedding, top_k=8, doc_type="design"), state["question"] - ) + if stub.enabled(): + chunks = retrieval.rerank( + retrieval.lexical_search(state["question"], top_k=8, doc_type="design"), + state["question"], + ) + else: + embedding = _embed(state["question"], client=_client()) + chunks = retrieval.rerank( + retrieval.search(embedding, top_k=8, doc_type="design"), state["question"] + ) state["evidence"] = { "query": result.query, "rows": result.rows, @@ -231,6 +249,10 @@ def gather_unclear(state: State) -> State: "clarifying_question": f"Could you tell me {asked_for}?", "candidate_interpretations": [], "used_fixture_data": False, + # No model is called on this path even in production, but in stub mode + # nothing classified the question either - which is the part the reader + # needs to know. + "stub_mode": stub.enabled(), } return state @@ -265,73 +287,24 @@ def generate(state: State) -> State: return state # gather_unclear already built the payload cfg = settings() - client = _client() trace = state.get("trace") evidence = state["evidence"] + stubbed = stub.enabled() + client = None if stubbed else _client() def call(attempt: int, previous: ContractViolation | None) -> dict[str, Any]: - user = { - "question": state["question"], - "evidence": evidence, - "return_schema": _SCHEMA_HINTS[klass], - } - if previous is not None: - # Tell it what it broke. One retry only - a model that fails a - # safety contract twice is not going to be argued into compliance. - user["previous_attempt_rejected"] = { - "rule": previous.rule, - "detail": previous.detail, - } - response = client.chat.completions.create( - model=cfg.chat_deployment, - messages=[ - {"role": "system", "content": PROMPTS[klass]}, - {"role": "user", "content": json.dumps(user, default=str)}, - ], - temperature=0.1, - max_tokens=cfg.max_output_tokens, - response_format={"type": "json_object"}, - ) - generated = json.loads(response.choices[0].message.content) - - # The model supplies prose and its own structured fields. Everything - # factual - rows, counts, citations, the fixture flag - is attached - # here from the evidence, so the model cannot alter it. - payload: dict[str, Any] = { - "question": state["question"], - "question_class": klass, - "answer": generated.get("answer", ""), - "used_fixture_data": evidence.get("used_fixture_data", False), - } - if klass is QuestionClass.HISTORICAL: - payload.update( - query=evidence["query"], - rows=evidence["rows"], - row_count=evidence["row_count"], - time_window=evidence["time_window"], - citations=evidence.get("citations", []), + if stubbed: + # The model's half of the payload, supplied as constants. Note what + # happens on a retry: the stub is deterministic, so a contract + # failure here fails again identically and surfaces as a 422. That + # is correct - it means the contract genuinely rejects what this + # mode produces, and no amount of retrying changes it. + generated = stub.generated_fields(klass, state["question"], evidence) + else: + generated = _generate_prose( + klass, state["question"], evidence, previous, client=client, cfg=cfg ) - elif klass is QuestionClass.REFERENCE: - payload.update( - citations=evidence.get("citations", []), - tags_referenced=[t["tag_id"] for t in evidence.get("tags", [])], - ) - elif klass is QuestionClass.PROCEDURAL: - payload.update( - procedure=generated.get("procedure"), - prerequisites_verbatim=generated.get("prerequisites_verbatim", []), - steps_provided=False, - citations=evidence.get("citations", []), - ) - elif klass is QuestionClass.ADVISORY: - payload.update( - evidence=generated.get("evidence", []), - documented_limits=generated.get("documented_limits", []), - recommendation_given=False, - deferral=generated.get("deferral", ""), - citations=evidence.get("citations", []), - ) - return payload + return _assemble(klass, state["question"], evidence, generated, stubbed=stubbed) result = enforce_contract(call, klass, trace=trace) state["answer"] = result.answer @@ -339,6 +312,94 @@ def generate(state: State) -> State: return state +def _generate_prose( + klass: QuestionClass, + question: str, + evidence: dict[str, Any], + previous: ContractViolation | None, + *, + client, + cfg, +) -> dict[str, Any]: + """The model's half of the payload: prose, and the fields only it can fill.""" + user = { + "question": question, + "evidence": evidence, + "return_schema": _SCHEMA_HINTS[klass], + } + if previous is not None: + # Tell it what it broke. One retry only - a model that fails a + # safety contract twice is not going to be argued into compliance. + user["previous_attempt_rejected"] = { + "rule": previous.rule, + "detail": previous.detail, + } + response = client.chat.completions.create( + model=cfg.chat_deployment, + messages=[ + {"role": "system", "content": PROMPTS[klass]}, + {"role": "user", "content": json.dumps(user, default=str)}, + ], + temperature=0.1, + max_tokens=cfg.max_output_tokens, + response_format={"type": "json_object"}, + ) + return json.loads(response.choices[0].message.content) + + +def _assemble( + klass: QuestionClass, + question: str, + evidence: dict[str, Any], + generated: dict[str, Any], + *, + stubbed: bool = False, +) -> dict[str, Any]: + """Attach the factual fields to whatever produced the prose. + + Everything factual - rows, counts, citations, the fixture flag - is attached + HERE, from the evidence, so the thing that wrote the prose cannot alter it. + That is true whether the prose came from the model or from stub.py, which is + what makes the stub a fair test of this path rather than a mock of it. + """ + payload: dict[str, Any] = { + "question": question, + "question_class": klass, + "answer": generated.get("answer", ""), + "used_fixture_data": evidence.get("used_fixture_data", False), + "stub_mode": stubbed, + } + if klass is QuestionClass.HISTORICAL: + payload.update( + query=evidence["query"], + rows=evidence["rows"], + row_count=evidence["row_count"], + time_window=evidence["time_window"], + citations=evidence.get("citations", []), + ) + elif klass is QuestionClass.REFERENCE: + payload.update( + citations=evidence.get("citations", []), + tags_referenced=[t["tag_id"] for t in evidence.get("tags", [])], + ) + elif klass is QuestionClass.PROCEDURAL: + payload.update( + procedure=generated.get("procedure"), + prerequisites_verbatim=generated.get("prerequisites_verbatim", []), + steps_provided=False, + citations=evidence.get("citations", []), + ) + elif klass is QuestionClass.ADVISORY: + payload.update( + evidence=generated.get("evidence", []), + documented_limits=generated.get("documented_limits", []), + recommendation_given=False, + deferral=generated.get("deferral", ""), + citations=evidence.get("citations", []), + ) + return payload + + def validate_unclear(state: State) -> State: from contracts import validate_answer @@ -353,6 +414,15 @@ def validate_unclear(state: State) -> State: def classify_node(state: State) -> State: + if stub.enabled(): + # No model, so nothing is classified - the caller said which class this + # is. apply_safety_rules() still runs over the result inside + # stub.classify(), because those rules are the part that has to hold + # whoever chose the label. + state["classification"] = stub.classify( + state["question"], state.get("forced_class") + ) + return state state["classification"] = classifier.classify( state["question"], client=_client(), trace=state.get("trace") ) @@ -396,11 +466,21 @@ def build_graph(): _GRAPH = None -def answer(question: str, *, trace=None): +def answer(question: str, *, trace=None, forced_class: str | None = None): """Answer one question. Raises ContractViolation if the contract cannot be - met - the caller returns an error, never a partial answer.""" + met - the caller returns an error, never a partial answer. + + forced_class is ignored unless NO_LLM_STUB is on. Letting a caller pick its + own contract is precisely what the classifier exists to prevent, so the + check is here rather than being left to the caller to remember. + """ global _GRAPH if _GRAPH is None: _GRAPH = build_graph() - final = _GRAPH.invoke({"question": question, "trace": trace}) + if forced_class and not stub.enabled(): + log.warning("ignoring forced class %r - NO_LLM_STUB is off", forced_class) + forced_class = None + final = _GRAPH.invoke( + {"question": question, "trace": trace, "forced_class": forced_class} + ) return final["answer"], final["classification"] diff --git a/api/config.py b/api/config.py index 28ae9a5..5b6d777 100644 --- a/api/config.py +++ b/api/config.py @@ -32,6 +32,11 @@ class Settings(BaseModel): cheap_deployment: str = "" # classifier, entities, tool selection embed_deployment: str = "" # text-embedding-3-small + # --- no-LLM stub mode -------------------------------------------------- + # Off by default and must stay that way. See api/stub.py for what it does + # and, more importantly, what it does not prove. + no_llm_stub: bool = False + # --- behaviour --------------------------------------------------------- classifier_confidence_threshold: float = 0.7 site_timezone: str = "Australia/Sydney" @@ -75,6 +80,7 @@ def settings() -> Settings: pgdatabase=env.get("PGDATABASE", "plant"), pguser=env.get("PGUSER", "agent_ro"), pgpassword=env.get("PGPASSWORD", ""), + no_llm_stub=env.get("NO_LLM_STUB", "false").lower() == "true", azure_openai_endpoint=env.get("AZURE_OPENAI_ENDPOINT", ""), azure_openai_api_key=env.get("AZURE_OPENAI_API_KEY", ""), azure_openai_api_version=env.get("AZURE_OPENAI_API_VERSION", ""), diff --git a/api/contracts.py b/api/contracts.py index 4828694..1e1d49f 100644 --- a/api/contracts.py +++ b/api/contracts.py @@ -117,6 +117,16 @@ class BaseAnswer(BaseModel): "The UI shows a banner. Never suppress it to make a demo cleaner." ), ) + stub_mode: bool = Field( + default=False, + description=( + "TRUE when NO_LLM_STUB produced this answer: no model was called, " + "the class was chosen by hand and retrieval was lexical. Carried in " + "the contract rather than added by the UI, for the same reason as " + "used_fixture_data - an answer nobody generated must not be " + "indistinguishable from one that was. See api/stub.py." + ), + ) # --------------------------------------------------------------------------- diff --git a/api/main.py b/api/main.py index 12d81fa..f2a8c05 100644 --- a/api/main.py +++ b/api/main.py @@ -43,13 +43,30 @@ app = FastAPI( # The browser reaches the API by its own public hostname, so a same-site # origin is all that is ever needed. +# +# NO_LLM_STUB additionally allows a tunnelled origin, because the public +# hostnames do not resolve yet and the whole chain has to be exercisable +# without them. It is deliberately tied to the stub flag so it disappears when +# the flag does. +# +# The alternative - proxying /api through ai-web's nginx to make the page +# same-origin - was rejected: it creates a second route to the API that does +# not pass through the api.yokogawa.tech Caddy block, and that block is where +# the Phase 9 publisher rule for ^/docs/.* lives. A convenience path around an +# authorisation rule is how the rule stops meaning anything. +_ORIGINS = ["https://ai.yokogawa.tech"] +if settings().no_llm_stub: + _ORIGINS.append("http://localhost:8080") + app.add_middleware( CORSMiddleware, - allow_origins=["https://ai.yokogawa.tech"], + allow_origins=_ORIGINS, # The UI sends credentials: "include" so the browser attaches the Authelia # session cookie. Without this the preflight fails and NO cross-origin call # succeeds - the browser refuses a credentialed request unless the response - # says Access-Control-Allow-Credentials: true. + # says Access-Control-Allow-Credentials: true. Found driving the real UI in + # a browser; it would have failed identically at ai.yokogawa.tech, and it + # cannot be found by calling the API directly with curl. allow_credentials=True, allow_methods=["GET", "POST"], allow_headers=["Content-Type"], @@ -76,6 +93,14 @@ def _langfuse(): class AskRequest(BaseModel): question: str = Field(min_length=3, max_length=1000) + question_class: str | None = Field( + default=None, + description=( + "Force the class instead of classifying. NO-LLM STUB MODE ONLY - " + "ignored unless NO_LLM_STUB is on, because letting a caller choose " + "its own contract is exactly what the classifier exists to prevent." + ), + ) class AskResponse(BaseModel): @@ -109,7 +134,9 @@ def ask(request: AskRequest) -> AskResponse: log.exception("could not open a Langfuse trace") try: - answer, classification = agent.answer(request.question, trace=trace) + answer, classification = agent.answer( + request.question, trace=trace, forced_class=request.question_class + ) except ContractViolation as violation: # The offending output has already gone to Langfuse with the whole # generated text. What comes back here says nothing about it: an error diff --git a/api/stub.py b/api/stub.py new file mode 100644 index 0000000..686d5df --- /dev/null +++ b/api/stub.py @@ -0,0 +1,268 @@ +"""No-LLM stub mode. OFF by default, and it must stay that way. + +WHY THIS EXISTS +--------------- +Azure OpenAI access is pending, and so is `imh`. Without a model there is no +classifier and no prose, so `POST /ask` cannot return anything at all - which +means the entire chain either side of the model is unproven: the browser, Caddy, +Authelia, the API, entity resolution, Cube, retrieval, the contracts, the +banners, the error paths. That is a lot of untested surface to leave until the +day the key arrives, and it is all testable now. + +So this module substitutes the two steps that need a model: + + * classification -> the class is supplied by the caller (a dropdown in the + UI). Not guessed by keywords: a crude keyword classifier + would produce a *plausible* label, and a plausible wrong + label is the failure this system is built to avoid. + Choosing explicitly is honest, and it lets each branch be + driven deliberately. + * prose -> a fixed placeholder string per class. Deterministic, and + written to satisfy the contract for its class. + +WHAT IT PROVES +-------------- +The transport chain end to end. Entity resolution. Cube queries and their +numbers. Retrieval plumbing and citation assembly. The fixture banner. The +scope banners. Contract validation - including its failures, because a stub +payload goes through `enforce_contract()` exactly like a generated one and a +422 is a real result, not a bug in the stub. + +WHAT IT DOES NOT PROVE +---------------------- +Anything about the model. Whether the classifier would label the question +correctly - the whole point is that a human labelled it. Whether retrieval finds +the RIGHT chunk: stub mode retrieves lexically because embedding the question +needs the model, and lexical hits are not vector hits. And no prose quality, +because there is no prose. + +It also cannot produce `prerequisites_verbatim` for a Procedural answer. Pulling +the prerequisites out of a retrieved chunk is an extraction task, and doing it +with a regex would be exactly the "synthesised from fragments" failure the +Procedural contract exists to prevent. So the list comes back empty and the +answer says so. + +THE SAFETY RULES STILL RUN +-------------------------- +`apply_safety_rules()` still runs over the forced classification. Contracts are +still validated after assembly and before returning. Retrieval still filters +superseded revisions. Nothing here is a bypass - it is the same pipeline with +the model-shaped holes filled by constants. + +TURNING IT OFF AGAIN +-------------------- +`NO_LLM_STUB=false` in ~/ai/api.env, and restart ai-api. Every answer produced +in this mode carries `stub_mode: true` and a banner in the same place, and for +the same reason, as the fixture banner: an answer nobody generated is otherwise +indistinguishable from one that was. +""" + +from __future__ import annotations + +import logging +from typing import Any + +from classifier import Classification, apply_safety_rules +from config import settings +from contracts import QuestionClass + +log = logging.getLogger("stub") + +BANNER = ( + "NO-LLM STUB MODE. No language model was called. The class was chosen by " + "hand, retrieval was lexical rather than semantic, and this text is a fixed " + "placeholder - not a generated answer. The figures, citations and document " + "identities below are real query results." +) + + +def enabled() -> bool: + return settings().no_llm_stub + + +def classify(question: str, forced_class: str | None) -> Classification: + """Take the class from the caller and put it through the real safety rules. + + Confidence is 1.0 because a person chose it, not because anything was + inferred. The rules still run: a forced HISTORICAL with no time window + still becomes UNCLEAR, which is worth seeing. + """ + try: + klass = QuestionClass(forced_class) if forced_class else QuestionClass.UNCLEAR + except ValueError: + log.warning("unknown forced class %r - falling back to unclear", forced_class) + klass = QuestionClass.UNCLEAR + + raw = Classification( + question_class=klass, + confidence=1.0, + alternatives={}, + entities={}, + missing_context=[] if forced_class else ["what you are asking about"], + ) + result = apply_safety_rules(raw, settings().classifier_confidence_threshold) + result.downgraded_reason = ( + (result.downgraded_reason + "; ") if result.downgraded_reason else "" + ) + "class supplied by the caller - NO_LLM_STUB is on, nothing was classified" + return result + + +# --------------------------------------------------------------------------- +# One payload builder per class. Each returns what the generate() step would +# have returned, minus the prose. The factual fields are attached from evidence +# by the caller in agent.py, exactly as they are for a generated answer. +# --------------------------------------------------------------------------- + + +def _historical(question: str, evidence: dict[str, Any]) -> dict[str, Any]: + row_count = evidence.get("row_count", 0) + window = evidence.get("time_window", {}).get("description", "the window queried") + if row_count == 0: + # Must say so, and must not state a quantity. The contract checks both. + answer = ( + f"{BANNER} The query returned no records for {window}." + ) + else: + # Row counts and figures live in the structured fields, which the UI + # renders. Keeping them out of the prose keeps this string honest - + # it is not a summary of anything. + answer = ( + f"{BANNER} The query ran and returned rows for {window}. " + "The figures are in the rows and query fields, not in this text." + ) + return {"answer": answer} + + +def _reference(question: str, evidence: dict[str, Any]) -> dict[str, Any]: + if not evidence.get("citations"): + answer = ( + f"{BANNER} No documents in the active set matched this question." + ) + else: + answer = ( + f"{BANNER} The documents below matched lexically. Whether they " + "actually answer the question is not established - that is the " + "judgement a language model would have made." + ) + return {"answer": answer} + + +def _procedural(question: str, evidence: dict[str, Any]) -> dict[str, Any]: + """Identity only. No prerequisites, and emphatically no steps. + + The procedure identity is assembled from the retrieved chunk's own + metadata - document number, revision, effective date - which is database + content, not generated text. That is the part of a Procedural answer that + matters most, and it is the part that needs no model at all. + """ + chunks = evidence.get("chunks") or [] + if not chunks: + return { + "answer": ( + f"{BANNER} No controlled procedure was found in the active " + "document set for this question." + ), + "procedure": None, + "prerequisites_verbatim": [], + } + + top = chunks[0] + procedure = { + "doc_number": top.get("doc_number") or top.get("source_file"), + "title": top.get("section_title") or top.get("source_file"), + "revision": top.get("revision") or "unknown", + "effective_date": top.get("effective_date"), + "authorising_role": None, + "controlled_copy_location": ( + "Controlled copy location is not held in the document index - " + "obtain the controlled copy through document control." + ), + } + answer = ( + f"{BANNER} A candidate controlled procedure was located and is " + "identified below. Prerequisites were not extracted: quoting them " + "verbatim is an extraction step this mode does not perform, and " + "reconstructing them from the retrieved text is precisely what this " + "class forbids. Work from the controlled copy." + ) + return {"answer": answer, "procedure": procedure, "prerequisites_verbatim": []} + + +def _advisory(question: str, evidence: dict[str, Any]) -> dict[str, Any]: + """Evidence rows come from Cube. The deferral is the whole point. + + Note what is NOT here: no number is selected, ranked or described as + suitable. The stub could not do that even if asked - it has no way to form + an opinion, which makes this the one class where a stub behaves almost + exactly as the real thing should. + """ + rows = evidence.get("rows") or [] + window = evidence.get("time_window", {}).get("description", "the window queried") + + def _interesting(row: dict[str, Any]) -> list[str]: + # Ordering only. It decides which column is shown, never what it says - + # the value is whatever Cube returned. + keys = [ + k for k, v in row.items() + if isinstance(v, (int, float)) and not isinstance(v, bool) + ] + preferred = [ + k for k in keys + if any(w in k.lower() for w in ("rate", "discharge", "level", "count")) + ] + return preferred + [k for k in keys if k not in preferred] + + items: list[dict[str, Any]] = [] + for row in rows[:6]: + for key in _interesting(row): + value = row[key] + items.append( + { + "description": f"{key} over {window}", + "metric": key, + "value": float(value), + "unit": "", + "sample_size": len(rows), + } + ) + break + + if items: + answer = ( + f"{BANNER} What the history shows over {window} is set out in the " + "evidence below. No operating parameter has been selected from it." + ) + else: + answer = ( + f"{BANNER} There are no records of station operations for {window}." + ) + return { + "answer": answer, + "evidence": items, + "documented_limits": [], + "deferral": ( + "This system does not recommend setpoints or operating parameters. " + "That decision needs a competent person with sight of current " + "equipment condition and concurrent operations. In this mode it " + "could not make a recommendation even if it were permitted to - no " + "model was called." + ), + } + + +BUILDERS = { + QuestionClass.HISTORICAL: _historical, + QuestionClass.REFERENCE: _reference, + QuestionClass.PROCEDURAL: _procedural, + QuestionClass.ADVISORY: _advisory, +} + + +def generated_fields( + klass: QuestionClass, question: str, evidence: dict[str, Any] +) -> dict[str, Any]: + """Stand in for the model's half of the payload. Deterministic.""" + builder = BUILDERS.get(klass) + if builder is None: + raise ValueError(f"no stub builder for {klass}") + return builder(question, evidence) diff --git a/api/tools/retrieval.py b/api/tools/retrieval.py index aadd43d..513bfa7 100644 --- a/api/tools/retrieval.py +++ b/api/tools/retrieval.py @@ -162,5 +162,114 @@ def find_procedure( return rerank(hits, question, top_n=3) +def lexical_search( + question: str, + *, + top_k: int = 8, + doc_type: DocType | None = None, + equipment_id: str | None = None, + conn: psycopg.Connection | None = None, +) -> list[Chunk]: + """Full-text search over chunk_text. NO-LLM STUB MODE ONLY. + + Embedding the question needs Azure OpenAI, so until that exists there is no + vector to search with. This finds chunks by words instead, which is a + genuinely different thing: it matches what the operator typed, not what + they meant. "The well is going to overflow" finds nothing here and would + find the spill procedure with embeddings. + + Kept beside search() rather than hidden inside it, and never called on the + normal answer path, so that nobody can mistake a lexical hit for a semantic + one when reading a trace. + + superseded = FALSE applies here exactly as it does in search(). There is no + include_superseded, because this function has one caller and that caller is + a demo. + """ + owned = conn is None + conn = conn or _connect() + try: + where = ["superseded = FALSE"] + params: dict[str, Any] = {"q": question, "k": top_k} + if doc_type: + where.append("doc_type = %(doc_type)s") + params["doc_type"] = doc_type + if equipment_id: + where.append("(equipment_id = %(equipment_id)s OR equipment_id IS NULL)") + params["equipment_id"] = equipment_id + + clause = " AND ".join(where) + with conn.cursor() as cur: + cur.execute( + f""" + SELECT id, source_file, doc_type, doc_number, revision, + effective_date, page, section_title, equipment_id, + chunk_text, + ts_rank( + to_tsvector('english', chunk_text), + plainto_tsquery('english', %(q)s) + ) AS similarity + FROM doc_chunks + WHERE {clause} + AND to_tsvector('english', chunk_text) + @@ plainto_tsquery('english', %(q)s) + ORDER BY similarity DESC + LIMIT %(k)s + """, + params, + ) + rows = cur.fetchall() + + if not rows: + # plainto_tsquery ANDs every term, so one unmatched word returns + # nothing at all. Fall back to any-term matching before giving up, + # or a demo question phrased as a sentence never finds anything. + terms = [t.strip(".,?;:").lower() for t in question.split() if len(t) > 3] + if terms: + params["q"] = " | ".join(terms) + with conn.cursor() as cur: + cur.execute( + f""" + SELECT id, source_file, doc_type, doc_number, revision, + effective_date, page, section_title, equipment_id, + chunk_text, + ts_rank( + to_tsvector('english', chunk_text), + to_tsquery('english', %(q)s) + ) AS similarity + FROM doc_chunks + WHERE {clause} + AND to_tsvector('english', chunk_text) + @@ to_tsquery('english', %(q)s) + ORDER BY similarity DESC + LIMIT %(k)s + """, + params, + ) + rows = cur.fetchall() + + return [Chunk(**row) for row in rows] + finally: + if owned: + conn.close() + + +def find_procedure_lexical( + question: str, + *, + equipment_id: str | None = None, + conn: psycopg.Connection | None = None, +) -> list[Chunk]: + """The find_procedure() shape, without an embedding. Stub mode only.""" + hits = lexical_search( + question, + top_k=12, + doc_type="procedure", + equipment_id=equipment_id, + conn=conn, + ) + return rerank(hits, question, top_n=3) + + def as_dicts(chunks: list[Chunk]) -> list[dict[str, Any]]: return [asdict(c) for c in chunks] diff --git a/compose/ai-compose.yml b/compose/ai-compose.yml index 9c127bc..4dbab85 100644 --- a/compose/ai-compose.yml +++ b/compose/ai-compose.yml @@ -170,6 +170,11 @@ services: build: context: /home/azureuser/ai/web dockerfile: Dockerfile + args: + # Empty in a real deployment: the browser reaches the API at its own + # public hostname through Caddy. Set only for a tunnelled demo build, + # which has no Caddy and no Authelia in front of it. + VITE_API_BASE: "${VITE_API_BASE:-}" image: yau/ai-web:local container_name: ai-web restart: unless-stopped diff --git a/demo/ai-docs/README.md b/demo/ai-docs/README.md new file mode 100644 index 0000000..c933673 --- /dev/null +++ b/demo/ai-docs/README.md @@ -0,0 +1,30 @@ +# Demo documents — NOT controlled documents, NOT plant content + +Three fabricated documents, written so that the Procedural, Reference and +Advisory branches have something to retrieve before an Azure OpenAI account +exists. They are here rather than on `/datadisk/ai-docs` in the repository +sense: the real document root is gitignored because it holds real controlled +documents, and these are the opposite of that. + +**Every one of them is fiction.** The document numbers are `WRPS-DEMO-00x`, +which matches the extractor's pattern (`WRPS-[A-Z]{2,4}-\d{3,4}`) so header +extraction and confirmation are genuinely exercised, while being a number no +real WRPS document can ever have. The setpoints and limits in them are made up +and contradict `db/seed/tags.csv` in places, deliberately: nothing here should +survive being mistaken for plant data. + +They are loaded with `--no-embed`, so their chunks have NULL embeddings and are +invisible to semantic search. Before real ingestion: + +```sql +DELETE FROM doc_chunks WHERE embedding IS NULL; +``` + +Copy them to the host with: + +```bash +scp -r demo/ai-docs/* lin001:/tmp/demo-docs/ +``` + +then move them into `/datadisk/ai-docs/` in the matching folders — the folder +determines `doc_type` and there is no override. diff --git a/demo/ai-docs/design/DEMO-design-basis.md b/demo/ai-docs/design/DEMO-design-basis.md new file mode 100644 index 0000000..fed8713 --- /dev/null +++ b/demo/ai-docs/design/DEMO-design-basis.md @@ -0,0 +1,39 @@ +# DEMO DOCUMENT — NOT A CONTROLLED DOCUMENT — DO NOT USE ON PLANT + +Document number: WRPS-DEMO-003 +Title: Station Design Basis Extract — Discharge and Storage +Revision: 0 +Effective: 2026-01-01 + +## 1. Scope + +A fabricated design basis extract for the Waterloo Road Pump Station STN-001, +written so that the Advisory branch of the plant assistant has documented +limits to retrieve and cite during a no-LLM demonstration. Every figure below +is invented. Where it contradicts db/seed/tags.csv, the seed data is the one +derived from the register map and this one is fiction. + +## 2. Discharge + +The discharge manifold MAN-301 is described in this fabricated extract as +having a stated hydraulic capacity against the 22 m static lift. The three +pumps PU-301, PU-302 and PU-303 share a common speed reference, and this +fabricated extract states that the drives are clamped at both ends of their +range rather than being free to run to zero. + +## 3. Storage and spill + +This fabricated extract states that the wet well WW-101 provides storage +between the stop-all level and the spill weir crest, that the weir crest is the +point at which flow leaves the site, and that the volume between the current +level and the crest is the quantity of interest when assessing how much time is +available. + +## 4. Limits stated in this fabricated document + +- The station is not to be operated with the level held above the high level + alarm setpoint as a normal operating state. +- Continuous operation of all three pumps is described as a short-duration + condition, not a normal duty. +- Any change to a level setpoint is described as requiring assessment against + the storage remaining to the weir crest. diff --git a/demo/ai-docs/procedures/DEMO-interlock-bypass.md b/demo/ai-docs/procedures/DEMO-interlock-bypass.md new file mode 100644 index 0000000..9310f5b --- /dev/null +++ b/demo/ai-docs/procedures/DEMO-interlock-bypass.md @@ -0,0 +1,42 @@ +# DEMO DOCUMENT — NOT A CONTROLLED DOCUMENT — DO NOT USE ON PLANT + +Document number: WRPS-DEMO-001 +Title: Temporary Bypass of Pump Motor Protection Interlock +Revision: 0 +Effective: 2026-01-01 +Authorising role: Station Maintenance Supervisor + +## 1. Purpose + +This fabricated document exists so that the Procedural branch of the plant +assistant has a procedure-shaped thing to locate during a no-LLM demonstration. +It describes, in invented terms, the temporary bypass of the motor protection +interlock on pump PU-302 at the Waterloo Road Pump Station. + +Nothing in this document is a real instruction. It has not been reviewed, it +has not been approved, and no hazard assessment sits behind it. + +## 2. Prerequisites + +The following must all be confirmed before this fabricated procedure begins. + +- A permit to work has been issued and is held by the person carrying out the work. +- PU-302 has been electrically isolated at MCC-501 and the isolation proven dead. +- The duty selection has been transferred so that PU-301 and PU-303 can meet inflow. +- The wet well WW-101 level is below the start-duty level and falling. +- The Station Maintenance Supervisor has authorised the bypass in writing. + +## 3. Procedure + +1. Confirm every prerequisite in section 2 is satisfied and recorded. +2. Log in to the station HMI with a maintenance-level account. +3. Navigate to the PU-302 faceplate and select the protection tab. +4. Set the motor protection bypass to ENABLED and confirm the prompt. +5. Record the bypass in the station log with the time and the permit number. +6. Carry out the authorised work. +7. Set the motor protection bypass to DISABLED before returning PU-302 to service. + +## 4. Restoration + +Restore the duty selection and confirm PU-302 responds to a test start. Close +the permit. Record the restoration time in the station log. diff --git a/demo/ai-docs/rationalisation/DEMO-alarm-rationalisation.md b/demo/ai-docs/rationalisation/DEMO-alarm-rationalisation.md new file mode 100644 index 0000000..d916330 --- /dev/null +++ b/demo/ai-docs/rationalisation/DEMO-alarm-rationalisation.md @@ -0,0 +1,37 @@ +# DEMO DOCUMENT — NOT A CONTROLLED DOCUMENT — DO NOT USE ON PLANT + +Document number: WRPS-DEMO-002 +Title: Alarm Rationalisation Record — Wet Well Level Signal +Revision: 0 +Effective: 2026-01-01 + +## 1. Scope + +A fabricated alarm rationalisation record for the wet well WW-101 level signal +at the Waterloo Road Pump Station, written to give the Reference branch of the +plant assistant something to retrieve and cite during a no-LLM demonstration. + +## 2. LEVEL_SIGNAL_FAULT + +**Meaning.** The level signal fault alarm indicates that the PLC has rejected +the wet well level measurement from LIT-101. In this fabricated record the +rejection is described as occurring when the signal is out of range, frozen for +longer than the configured period, or of bad quality. + +**Consequence.** With no trusted level measurement the station cannot modulate +pump duty on level. This fabricated record states that control falls back to +the LSHH-102 high high level switch as the only remaining level protection. + +**Priority.** Priority 1 in this fabricated record. The stated reasoning is +that losing the level measurement on a wet well that can spill removes the +station's ability to see a spill developing. + +**Operator response.** Not stated here. The response belongs in a procedure, +and this is a rationalisation record. + +## 3. HIGH_LEVEL + +**Meaning.** A fabricated entry stating that the wet well level has reached the +high level alarm setpoint and that the station has not yet drawn the level down. + +**Priority.** Priority 2 in this fabricated record. diff --git a/web/Dockerfile b/web/Dockerfile index 09927de..f67ce11 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -4,6 +4,14 @@ WORKDIR /app COPY package.json ./ RUN npm install COPY . . + +# Where the browser should send /ask. Empty means the built-in default, which +# is the public hostname. A tunnelled demo build overrides it: +# --build-arg VITE_API_BASE=http://localhost:8001 +# Vite inlines this at BUILD time, so a change needs a rebuild, not a restart. +ARG VITE_API_BASE="" +ENV VITE_API_BASE=$VITE_API_BASE + RUN npm run build FROM nginx:alpine diff --git a/web/src/App.tsx b/web/src/App.tsx index 923bbee..2c2b4d5 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -1,10 +1,19 @@ import { useState } from "react"; import type { AnswerBody, AskResponse, Citation } from "./types"; -// Same-origin in dev, the public hostname in the built image. The browser -// carries the Authelia session cookie either way; there is no token handling -// in this app because Authelia authenticates at the edge. -const API = import.meta.env.DEV ? "/api" : "https://api.yokogawa.tech"; +// Same-origin in dev, otherwise VITE_API_BASE at build time, falling back to +// the public hostname. The browser carries the Authelia session cookie either +// way; there is no token handling in this app because Authelia authenticates +// at the edge. +// +// VITE_API_BASE exists because the public hostname does not resolve yet. A +// tunnelled build points it at http://localhost:8001 so the whole chain can be +// exercised before DNS, Caddy and Authelia are in place - which also means a +// tunnelled build has NO AUTHENTICATION in front of it. It is reachable only +// through an SSH tunnel from one machine, and it is not a deployment. +const API = import.meta.env.DEV + ? "/api" + : (import.meta.env.VITE_API_BASE ?? "https://api.yokogawa.tech"); export default function App() { const [question, setQuestion] = useState(""); @@ -12,6 +21,8 @@ export default function App() { const [error, setError] = useState(null); const [busy, setBusy] = useState(false); const [showWorking, setShowWorking] = useState(true); + // Stub mode only - the API ignores it otherwise. See api/stub.py. + const [forcedClass, setForcedClass] = useState(""); async function ask(event: React.FormEvent) { event.preventDefault(); @@ -23,7 +34,9 @@ export default function App() { method: "POST", headers: { "Content-Type": "application/json" }, credentials: "include", - body: JSON.stringify({ question }), + body: JSON.stringify( + forcedClass ? { question, question_class: forcedClass } : { question }, + ), }); const body = await response.json(); if (!response.ok) { @@ -67,6 +80,24 @@ export default function App() { /> Show working + {/* Only has an effect while the API runs with NO_LLM_STUB on. There + is no classifier without a model, so the class is chosen here + rather than guessed - see api/stub.py for why guessing it with + keywords would be worse than asking. */} + @@ -82,6 +113,18 @@ function Answer({ result, showWorking }: { result: AskResponse; showWorking: boo
{result.question_class}
+ {/* An answer nobody generated must not look like one that was. Same + place, same weight and the same reasoning as the fixture banner. */} + {a.stub_mode && ( +
+ No-LLM stub mode. No language model was called. The + class was chosen by hand, retrieval was lexical rather than semantic, + and the prose is a fixed placeholder. Figures, citations and document + identities are real query results; the wording around them means + nothing. +
+ )} + {/* Fixture data must never reach a slide unlabelled. */} {a.used_fixture_data && (
diff --git a/web/src/styles.css b/web/src/styles.css index d1994fc..0bfe42b 100644 --- a/web/src/styles.css +++ b/web/src/styles.css @@ -12,6 +12,8 @@ --scope-line: #d99b00; --fixture: #ffe9e9; --fixture-line: #c0392b; + --stub: #efe7fb; + --stub-line: #6b3fa0; --error: #c0392b; } @@ -113,3 +115,17 @@ pre { font-size: 0.85rem; } code { font-size: 0.9em; } + +/* Stub mode. Deliberately as loud as the fixture banner - both say the same + kind of thing: what you are looking at is not what it appears to be. */ +.banner.stub { background: var(--stub); border-color: var(--stub-line); } + +.forced-class { + display: inline-flex; + align-items: center; + gap: 0.4rem; +} + +.forced-class select { + padding: 0.2rem 0.3rem; +} diff --git a/web/src/types.ts b/web/src/types.ts index 2e34967..d56f791 100644 --- a/web/src/types.ts +++ b/web/src/types.ts @@ -27,6 +27,7 @@ export interface AnswerBody { question_class: QuestionClass; answer: string; used_fixture_data: boolean; + stub_mode?: boolean; // historical query?: Record;