diff --git a/api/agent.py b/api/agent.py index 681ce94..e3f3a2d 100644 --- a/api/agent.py +++ b/api/agent.py @@ -29,6 +29,7 @@ import classifier import tools.equipment as equipment_tool import tools.metrics as metrics import tools.retrieval as retrieval +import stub from config import settings from contracts import ContractViolation, QuestionClass from guardrails import enforce_contract @@ -43,6 +44,7 @@ class State(TypedDict, total=False): payload: dict[str, Any] answer: Any trace: Any + forced_class: str | None def _client() -> AzureOpenAI: @@ -163,13 +165,20 @@ def gather_historical(state: State) -> State: def gather_reference(state: State) -> State: - client = _client() equipment_id, _ = _resolve_entities(state) - embedding = _embed(state["question"], client=client) - chunks = retrieval.rerank( - retrieval.search(embedding, top_k=8, equipment_id=equipment_id), - state["question"], - ) + if stub.enabled(): + chunks = retrieval.rerank( + retrieval.lexical_search( + state["question"], top_k=8, equipment_id=equipment_id + ), + state["question"], + ) + else: + embedding = _embed(state["question"], client=_client()) + chunks = retrieval.rerank( + retrieval.search(embedding, top_k=8, equipment_id=equipment_id), + state["question"], + ) tag_rows = equipment_tool.tags_for_equipment(equipment_id) if equipment_id else [] state["evidence"] = { "chunks": retrieval.as_dicts(chunks), @@ -182,12 +191,16 @@ def gather_reference(state: State) -> State: def gather_procedural(state: State) -> State: """Procedures only, live revisions only. Nothing else is in scope here.""" - client = _client() equipment_id, _ = _resolve_entities(state) - embedding = _embed(state["question"], client=client) - chunks = retrieval.find_procedure( - embedding, state["question"], equipment_id=equipment_id - ) + if stub.enabled(): + chunks = retrieval.find_procedure_lexical( + state["question"], equipment_id=equipment_id + ) + else: + embedding = _embed(state["question"], client=_client()) + chunks = retrieval.find_procedure( + embedding, state["question"], equipment_id=equipment_id + ) state["evidence"] = { "chunks": retrieval.as_dicts(chunks), "citations": [c.citation() for c in chunks], @@ -199,13 +212,18 @@ def gather_procedural(state: State) -> State: def gather_advisory(state: State) -> State: """Both paths: what was done (Cube) and what is allowed (documents).""" trace = state.get("trace") - client = _client() result = metrics.run(metrics.pump_down_evidence(days=30), trace=trace) _, _, window_description = metrics.rolling_window(30) - embedding = _embed(state["question"], client=client) - chunks = retrieval.rerank( - retrieval.search(embedding, top_k=8, doc_type="design"), state["question"] - ) + if stub.enabled(): + chunks = retrieval.rerank( + retrieval.lexical_search(state["question"], top_k=8, doc_type="design"), + state["question"], + ) + else: + embedding = _embed(state["question"], client=_client()) + chunks = retrieval.rerank( + retrieval.search(embedding, top_k=8, doc_type="design"), state["question"] + ) state["evidence"] = { "query": result.query, "rows": result.rows, @@ -231,6 +249,10 @@ def gather_unclear(state: State) -> State: "clarifying_question": f"Could you tell me {asked_for}?", "candidate_interpretations": [], "used_fixture_data": False, + # No model is called on this path even in production, but in stub mode + # nothing classified the question either - which is the part the reader + # needs to know. + "stub_mode": stub.enabled(), } return state @@ -265,73 +287,24 @@ def generate(state: State) -> State: return state # gather_unclear already built the payload cfg = settings() - client = _client() trace = state.get("trace") evidence = state["evidence"] + stubbed = stub.enabled() + client = None if stubbed else _client() def call(attempt: int, previous: ContractViolation | None) -> dict[str, Any]: - user = { - "question": state["question"], - "evidence": evidence, - "return_schema": _SCHEMA_HINTS[klass], - } - if previous is not None: - # Tell it what it broke. One retry only - a model that fails a - # safety contract twice is not going to be argued into compliance. - user["previous_attempt_rejected"] = { - "rule": previous.rule, - "detail": previous.detail, - } - response = client.chat.completions.create( - model=cfg.chat_deployment, - messages=[ - {"role": "system", "content": PROMPTS[klass]}, - {"role": "user", "content": json.dumps(user, default=str)}, - ], - temperature=0.1, - max_tokens=cfg.max_output_tokens, - response_format={"type": "json_object"}, - ) - generated = json.loads(response.choices[0].message.content) - - # The model supplies prose and its own structured fields. Everything - # factual - rows, counts, citations, the fixture flag - is attached - # here from the evidence, so the model cannot alter it. - payload: dict[str, Any] = { - "question": state["question"], - "question_class": klass, - "answer": generated.get("answer", ""), - "used_fixture_data": evidence.get("used_fixture_data", False), - } - if klass is QuestionClass.HISTORICAL: - payload.update( - query=evidence["query"], - rows=evidence["rows"], - row_count=evidence["row_count"], - time_window=evidence["time_window"], - citations=evidence.get("citations", []), + if stubbed: + # The model's half of the payload, supplied as constants. Note what + # happens on a retry: the stub is deterministic, so a contract + # failure here fails again identically and surfaces as a 422. That + # is correct - it means the contract genuinely rejects what this + # mode produces, and no amount of retrying changes it. + generated = stub.generated_fields(klass, state["question"], evidence) + else: + generated = _generate_prose( + klass, state["question"], evidence, previous, client=client, cfg=cfg ) - elif klass is QuestionClass.REFERENCE: - payload.update( - citations=evidence.get("citations", []), - tags_referenced=[t["tag_id"] for t in evidence.get("tags", [])], - ) - elif klass is QuestionClass.PROCEDURAL: - payload.update( - procedure=generated.get("procedure"), - prerequisites_verbatim=generated.get("prerequisites_verbatim", []), - steps_provided=False, - citations=evidence.get("citations", []), - ) - elif klass is QuestionClass.ADVISORY: - payload.update( - evidence=generated.get("evidence", []), - documented_limits=generated.get("documented_limits", []), - recommendation_given=False, - deferral=generated.get("deferral", ""), - citations=evidence.get("citations", []), - ) - return payload + return _assemble(klass, state["question"], evidence, generated, stubbed=stubbed) result = enforce_contract(call, klass, trace=trace) state["answer"] = result.answer @@ -339,6 +312,94 @@ def generate(state: State) -> State: return state +def _generate_prose( + klass: QuestionClass, + question: str, + evidence: dict[str, Any], + previous: ContractViolation | None, + *, + client, + cfg, +) -> dict[str, Any]: + """The model's half of the payload: prose, and the fields only it can fill.""" + user = { + "question": question, + "evidence": evidence, + "return_schema": _SCHEMA_HINTS[klass], + } + if previous is not None: + # Tell it what it broke. One retry only - a model that fails a + # safety contract twice is not going to be argued into compliance. + user["previous_attempt_rejected"] = { + "rule": previous.rule, + "detail": previous.detail, + } + response = client.chat.completions.create( + model=cfg.chat_deployment, + messages=[ + {"role": "system", "content": PROMPTS[klass]}, + {"role": "user", "content": json.dumps(user, default=str)}, + ], + temperature=0.1, + max_tokens=cfg.max_output_tokens, + response_format={"type": "json_object"}, + ) + return json.loads(response.choices[0].message.content) + + +def _assemble( + klass: QuestionClass, + question: str, + evidence: dict[str, Any], + generated: dict[str, Any], + *, + stubbed: bool = False, +) -> dict[str, Any]: + """Attach the factual fields to whatever produced the prose. + + Everything factual - rows, counts, citations, the fixture flag - is attached + HERE, from the evidence, so the thing that wrote the prose cannot alter it. + That is true whether the prose came from the model or from stub.py, which is + what makes the stub a fair test of this path rather than a mock of it. + """ + payload: dict[str, Any] = { + "question": question, + "question_class": klass, + "answer": generated.get("answer", ""), + "used_fixture_data": evidence.get("used_fixture_data", False), + "stub_mode": stubbed, + } + if klass is QuestionClass.HISTORICAL: + payload.update( + query=evidence["query"], + rows=evidence["rows"], + row_count=evidence["row_count"], + time_window=evidence["time_window"], + citations=evidence.get("citations", []), + ) + elif klass is QuestionClass.REFERENCE: + payload.update( + citations=evidence.get("citations", []), + tags_referenced=[t["tag_id"] for t in evidence.get("tags", [])], + ) + elif klass is QuestionClass.PROCEDURAL: + payload.update( + procedure=generated.get("procedure"), + prerequisites_verbatim=generated.get("prerequisites_verbatim", []), + steps_provided=False, + citations=evidence.get("citations", []), + ) + elif klass is QuestionClass.ADVISORY: + payload.update( + evidence=generated.get("evidence", []), + documented_limits=generated.get("documented_limits", []), + recommendation_given=False, + deferral=generated.get("deferral", ""), + citations=evidence.get("citations", []), + ) + return payload + + def validate_unclear(state: State) -> State: from contracts import validate_answer @@ -353,6 +414,15 @@ def validate_unclear(state: State) -> State: def classify_node(state: State) -> State: + if stub.enabled(): + # No model, so nothing is classified - the caller said which class this + # is. apply_safety_rules() still runs over the result inside + # stub.classify(), because those rules are the part that has to hold + # whoever chose the label. + state["classification"] = stub.classify( + state["question"], state.get("forced_class") + ) + return state state["classification"] = classifier.classify( state["question"], client=_client(), trace=state.get("trace") ) @@ -396,11 +466,21 @@ def build_graph(): _GRAPH = None -def answer(question: str, *, trace=None): +def answer(question: str, *, trace=None, forced_class: str | None = None): """Answer one question. Raises ContractViolation if the contract cannot be - met - the caller returns an error, never a partial answer.""" + met - the caller returns an error, never a partial answer. + + forced_class is ignored unless NO_LLM_STUB is on. Letting a caller pick its + own contract is precisely what the classifier exists to prevent, so the + check is here rather than being left to the caller to remember. + """ global _GRAPH if _GRAPH is None: _GRAPH = build_graph() - final = _GRAPH.invoke({"question": question, "trace": trace}) + if forced_class and not stub.enabled(): + log.warning("ignoring forced class %r - NO_LLM_STUB is off", forced_class) + forced_class = None + final = _GRAPH.invoke( + {"question": question, "trace": trace, "forced_class": forced_class} + ) return final["answer"], final["classification"] diff --git a/api/config.py b/api/config.py index 28ae9a5..5b6d777 100644 --- a/api/config.py +++ b/api/config.py @@ -32,6 +32,11 @@ class Settings(BaseModel): cheap_deployment: str = "" # classifier, entities, tool selection embed_deployment: str = "" # text-embedding-3-small + # --- no-LLM stub mode -------------------------------------------------- + # Off by default and must stay that way. See api/stub.py for what it does + # and, more importantly, what it does not prove. + no_llm_stub: bool = False + # --- behaviour --------------------------------------------------------- classifier_confidence_threshold: float = 0.7 site_timezone: str = "Australia/Sydney" @@ -75,6 +80,7 @@ def settings() -> Settings: pgdatabase=env.get("PGDATABASE", "plant"), pguser=env.get("PGUSER", "agent_ro"), pgpassword=env.get("PGPASSWORD", ""), + no_llm_stub=env.get("NO_LLM_STUB", "false").lower() == "true", azure_openai_endpoint=env.get("AZURE_OPENAI_ENDPOINT", ""), azure_openai_api_key=env.get("AZURE_OPENAI_API_KEY", ""), azure_openai_api_version=env.get("AZURE_OPENAI_API_VERSION", ""), diff --git a/api/contracts.py b/api/contracts.py index 4828694..1e1d49f 100644 --- a/api/contracts.py +++ b/api/contracts.py @@ -117,6 +117,16 @@ class BaseAnswer(BaseModel): "The UI shows a banner. Never suppress it to make a demo cleaner." ), ) + stub_mode: bool = Field( + default=False, + description=( + "TRUE when NO_LLM_STUB produced this answer: no model was called, " + "the class was chosen by hand and retrieval was lexical. Carried in " + "the contract rather than added by the UI, for the same reason as " + "used_fixture_data - an answer nobody generated must not be " + "indistinguishable from one that was. See api/stub.py." + ), + ) # --------------------------------------------------------------------------- diff --git a/api/main.py b/api/main.py index 12d81fa..f2a8c05 100644 --- a/api/main.py +++ b/api/main.py @@ -43,13 +43,30 @@ app = FastAPI( # The browser reaches the API by its own public hostname, so a same-site # origin is all that is ever needed. +# +# NO_LLM_STUB additionally allows a tunnelled origin, because the public +# hostnames do not resolve yet and the whole chain has to be exercisable +# without them. It is deliberately tied to the stub flag so it disappears when +# the flag does. +# +# The alternative - proxying /api through ai-web's nginx to make the page +# same-origin - was rejected: it creates a second route to the API that does +# not pass through the api.yokogawa.tech Caddy block, and that block is where +# the Phase 9 publisher rule for ^/docs/.* lives. A convenience path around an +# authorisation rule is how the rule stops meaning anything. +_ORIGINS = ["https://ai.yokogawa.tech"] +if settings().no_llm_stub: + _ORIGINS.append("http://localhost:8080") + app.add_middleware( CORSMiddleware, - allow_origins=["https://ai.yokogawa.tech"], + allow_origins=_ORIGINS, # The UI sends credentials: "include" so the browser attaches the Authelia # session cookie. Without this the preflight fails and NO cross-origin call # succeeds - the browser refuses a credentialed request unless the response - # says Access-Control-Allow-Credentials: true. + # says Access-Control-Allow-Credentials: true. Found driving the real UI in + # a browser; it would have failed identically at ai.yokogawa.tech, and it + # cannot be found by calling the API directly with curl. allow_credentials=True, allow_methods=["GET", "POST"], allow_headers=["Content-Type"], @@ -76,6 +93,14 @@ def _langfuse(): class AskRequest(BaseModel): question: str = Field(min_length=3, max_length=1000) + question_class: str | None = Field( + default=None, + description=( + "Force the class instead of classifying. NO-LLM STUB MODE ONLY - " + "ignored unless NO_LLM_STUB is on, because letting a caller choose " + "its own contract is exactly what the classifier exists to prevent." + ), + ) class AskResponse(BaseModel): @@ -109,7 +134,9 @@ def ask(request: AskRequest) -> AskResponse: log.exception("could not open a Langfuse trace") try: - answer, classification = agent.answer(request.question, trace=trace) + answer, classification = agent.answer( + request.question, trace=trace, forced_class=request.question_class + ) except ContractViolation as violation: # The offending output has already gone to Langfuse with the whole # generated text. What comes back here says nothing about it: an error diff --git a/api/stub.py b/api/stub.py new file mode 100644 index 0000000..686d5df --- /dev/null +++ b/api/stub.py @@ -0,0 +1,268 @@ +"""No-LLM stub mode. OFF by default, and it must stay that way. + +WHY THIS EXISTS +--------------- +Azure OpenAI access is pending, and so is `imh`. Without a model there is no +classifier and no prose, so `POST /ask` cannot return anything at all - which +means the entire chain either side of the model is unproven: the browser, Caddy, +Authelia, the API, entity resolution, Cube, retrieval, the contracts, the +banners, the error paths. That is a lot of untested surface to leave until the +day the key arrives, and it is all testable now. + +So this module substitutes the two steps that need a model: + + * classification -> the class is supplied by the caller (a dropdown in the + UI). Not guessed by keywords: a crude keyword classifier + would produce a *plausible* label, and a plausible wrong + label is the failure this system is built to avoid. + Choosing explicitly is honest, and it lets each branch be + driven deliberately. + * prose -> a fixed placeholder string per class. Deterministic, and + written to satisfy the contract for its class. + +WHAT IT PROVES +-------------- +The transport chain end to end. Entity resolution. Cube queries and their +numbers. Retrieval plumbing and citation assembly. The fixture banner. The +scope banners. Contract validation - including its failures, because a stub +payload goes through `enforce_contract()` exactly like a generated one and a +422 is a real result, not a bug in the stub. + +WHAT IT DOES NOT PROVE +---------------------- +Anything about the model. Whether the classifier would label the question +correctly - the whole point is that a human labelled it. Whether retrieval finds +the RIGHT chunk: stub mode retrieves lexically because embedding the question +needs the model, and lexical hits are not vector hits. And no prose quality, +because there is no prose. + +It also cannot produce `prerequisites_verbatim` for a Procedural answer. Pulling +the prerequisites out of a retrieved chunk is an extraction task, and doing it +with a regex would be exactly the "synthesised from fragments" failure the +Procedural contract exists to prevent. So the list comes back empty and the +answer says so. + +THE SAFETY RULES STILL RUN +-------------------------- +`apply_safety_rules()` still runs over the forced classification. Contracts are +still validated after assembly and before returning. Retrieval still filters +superseded revisions. Nothing here is a bypass - it is the same pipeline with +the model-shaped holes filled by constants. + +TURNING IT OFF AGAIN +-------------------- +`NO_LLM_STUB=false` in ~/ai/api.env, and restart ai-api. Every answer produced +in this mode carries `stub_mode: true` and a banner in the same place, and for +the same reason, as the fixture banner: an answer nobody generated is otherwise +indistinguishable from one that was. +""" + +from __future__ import annotations + +import logging +from typing import Any + +from classifier import Classification, apply_safety_rules +from config import settings +from contracts import QuestionClass + +log = logging.getLogger("stub") + +BANNER = ( + "NO-LLM STUB MODE. No language model was called. The class was chosen by " + "hand, retrieval was lexical rather than semantic, and this text is a fixed " + "placeholder - not a generated answer. The figures, citations and document " + "identities below are real query results." +) + + +def enabled() -> bool: + return settings().no_llm_stub + + +def classify(question: str, forced_class: str | None) -> Classification: + """Take the class from the caller and put it through the real safety rules. + + Confidence is 1.0 because a person chose it, not because anything was + inferred. The rules still run: a forced HISTORICAL with no time window + still becomes UNCLEAR, which is worth seeing. + """ + try: + klass = QuestionClass(forced_class) if forced_class else QuestionClass.UNCLEAR + except ValueError: + log.warning("unknown forced class %r - falling back to unclear", forced_class) + klass = QuestionClass.UNCLEAR + + raw = Classification( + question_class=klass, + confidence=1.0, + alternatives={}, + entities={}, + missing_context=[] if forced_class else ["what you are asking about"], + ) + result = apply_safety_rules(raw, settings().classifier_confidence_threshold) + result.downgraded_reason = ( + (result.downgraded_reason + "; ") if result.downgraded_reason else "" + ) + "class supplied by the caller - NO_LLM_STUB is on, nothing was classified" + return result + + +# --------------------------------------------------------------------------- +# One payload builder per class. Each returns what the generate() step would +# have returned, minus the prose. The factual fields are attached from evidence +# by the caller in agent.py, exactly as they are for a generated answer. +# --------------------------------------------------------------------------- + + +def _historical(question: str, evidence: dict[str, Any]) -> dict[str, Any]: + row_count = evidence.get("row_count", 0) + window = evidence.get("time_window", {}).get("description", "the window queried") + if row_count == 0: + # Must say so, and must not state a quantity. The contract checks both. + answer = ( + f"{BANNER} The query returned no records for {window}." + ) + else: + # Row counts and figures live in the structured fields, which the UI + # renders. Keeping them out of the prose keeps this string honest - + # it is not a summary of anything. + answer = ( + f"{BANNER} The query ran and returned rows for {window}. " + "The figures are in the rows and query fields, not in this text." + ) + return {"answer": answer} + + +def _reference(question: str, evidence: dict[str, Any]) -> dict[str, Any]: + if not evidence.get("citations"): + answer = ( + f"{BANNER} No documents in the active set matched this question." + ) + else: + answer = ( + f"{BANNER} The documents below matched lexically. Whether they " + "actually answer the question is not established - that is the " + "judgement a language model would have made." + ) + return {"answer": answer} + + +def _procedural(question: str, evidence: dict[str, Any]) -> dict[str, Any]: + """Identity only. No prerequisites, and emphatically no steps. + + The procedure identity is assembled from the retrieved chunk's own + metadata - document number, revision, effective date - which is database + content, not generated text. That is the part of a Procedural answer that + matters most, and it is the part that needs no model at all. + """ + chunks = evidence.get("chunks") or [] + if not chunks: + return { + "answer": ( + f"{BANNER} No controlled procedure was found in the active " + "document set for this question." + ), + "procedure": None, + "prerequisites_verbatim": [], + } + + top = chunks[0] + procedure = { + "doc_number": top.get("doc_number") or top.get("source_file"), + "title": top.get("section_title") or top.get("source_file"), + "revision": top.get("revision") or "unknown", + "effective_date": top.get("effective_date"), + "authorising_role": None, + "controlled_copy_location": ( + "Controlled copy location is not held in the document index - " + "obtain the controlled copy through document control." + ), + } + answer = ( + f"{BANNER} A candidate controlled procedure was located and is " + "identified below. Prerequisites were not extracted: quoting them " + "verbatim is an extraction step this mode does not perform, and " + "reconstructing them from the retrieved text is precisely what this " + "class forbids. Work from the controlled copy." + ) + return {"answer": answer, "procedure": procedure, "prerequisites_verbatim": []} + + +def _advisory(question: str, evidence: dict[str, Any]) -> dict[str, Any]: + """Evidence rows come from Cube. The deferral is the whole point. + + Note what is NOT here: no number is selected, ranked or described as + suitable. The stub could not do that even if asked - it has no way to form + an opinion, which makes this the one class where a stub behaves almost + exactly as the real thing should. + """ + rows = evidence.get("rows") or [] + window = evidence.get("time_window", {}).get("description", "the window queried") + + def _interesting(row: dict[str, Any]) -> list[str]: + # Ordering only. It decides which column is shown, never what it says - + # the value is whatever Cube returned. + keys = [ + k for k, v in row.items() + if isinstance(v, (int, float)) and not isinstance(v, bool) + ] + preferred = [ + k for k in keys + if any(w in k.lower() for w in ("rate", "discharge", "level", "count")) + ] + return preferred + [k for k in keys if k not in preferred] + + items: list[dict[str, Any]] = [] + for row in rows[:6]: + for key in _interesting(row): + value = row[key] + items.append( + { + "description": f"{key} over {window}", + "metric": key, + "value": float(value), + "unit": "", + "sample_size": len(rows), + } + ) + break + + if items: + answer = ( + f"{BANNER} What the history shows over {window} is set out in the " + "evidence below. No operating parameter has been selected from it." + ) + else: + answer = ( + f"{BANNER} There are no records of station operations for {window}." + ) + return { + "answer": answer, + "evidence": items, + "documented_limits": [], + "deferral": ( + "This system does not recommend setpoints or operating parameters. " + "That decision needs a competent person with sight of current " + "equipment condition and concurrent operations. In this mode it " + "could not make a recommendation even if it were permitted to - no " + "model was called." + ), + } + + +BUILDERS = { + QuestionClass.HISTORICAL: _historical, + QuestionClass.REFERENCE: _reference, + QuestionClass.PROCEDURAL: _procedural, + QuestionClass.ADVISORY: _advisory, +} + + +def generated_fields( + klass: QuestionClass, question: str, evidence: dict[str, Any] +) -> dict[str, Any]: + """Stand in for the model's half of the payload. Deterministic.""" + builder = BUILDERS.get(klass) + if builder is None: + raise ValueError(f"no stub builder for {klass}") + return builder(question, evidence) diff --git a/api/tools/retrieval.py b/api/tools/retrieval.py index aadd43d..513bfa7 100644 --- a/api/tools/retrieval.py +++ b/api/tools/retrieval.py @@ -162,5 +162,114 @@ def find_procedure( return rerank(hits, question, top_n=3) +def lexical_search( + question: str, + *, + top_k: int = 8, + doc_type: DocType | None = None, + equipment_id: str | None = None, + conn: psycopg.Connection | None = None, +) -> list[Chunk]: + """Full-text search over chunk_text. NO-LLM STUB MODE ONLY. + + Embedding the question needs Azure OpenAI, so until that exists there is no + vector to search with. This finds chunks by words instead, which is a + genuinely different thing: it matches what the operator typed, not what + they meant. "The well is going to overflow" finds nothing here and would + find the spill procedure with embeddings. + + Kept beside search() rather than hidden inside it, and never called on the + normal answer path, so that nobody can mistake a lexical hit for a semantic + one when reading a trace. + + superseded = FALSE applies here exactly as it does in search(). There is no + include_superseded, because this function has one caller and that caller is + a demo. + """ + owned = conn is None + conn = conn or _connect() + try: + where = ["superseded = FALSE"] + params: dict[str, Any] = {"q": question, "k": top_k} + if doc_type: + where.append("doc_type = %(doc_type)s") + params["doc_type"] = doc_type + if equipment_id: + where.append("(equipment_id = %(equipment_id)s OR equipment_id IS NULL)") + params["equipment_id"] = equipment_id + + clause = " AND ".join(where) + with conn.cursor() as cur: + cur.execute( + f""" + SELECT id, source_file, doc_type, doc_number, revision, + effective_date, page, section_title, equipment_id, + chunk_text, + ts_rank( + to_tsvector('english', chunk_text), + plainto_tsquery('english', %(q)s) + ) AS similarity + FROM doc_chunks + WHERE {clause} + AND to_tsvector('english', chunk_text) + @@ plainto_tsquery('english', %(q)s) + ORDER BY similarity DESC + LIMIT %(k)s + """, + params, + ) + rows = cur.fetchall() + + if not rows: + # plainto_tsquery ANDs every term, so one unmatched word returns + # nothing at all. Fall back to any-term matching before giving up, + # or a demo question phrased as a sentence never finds anything. + terms = [t.strip(".,?;:").lower() for t in question.split() if len(t) > 3] + if terms: + params["q"] = " | ".join(terms) + with conn.cursor() as cur: + cur.execute( + f""" + SELECT id, source_file, doc_type, doc_number, revision, + effective_date, page, section_title, equipment_id, + chunk_text, + ts_rank( + to_tsvector('english', chunk_text), + to_tsquery('english', %(q)s) + ) AS similarity + FROM doc_chunks + WHERE {clause} + AND to_tsvector('english', chunk_text) + @@ to_tsquery('english', %(q)s) + ORDER BY similarity DESC + LIMIT %(k)s + """, + params, + ) + rows = cur.fetchall() + + return [Chunk(**row) for row in rows] + finally: + if owned: + conn.close() + + +def find_procedure_lexical( + question: str, + *, + equipment_id: str | None = None, + conn: psycopg.Connection | None = None, +) -> list[Chunk]: + """The find_procedure() shape, without an embedding. Stub mode only.""" + hits = lexical_search( + question, + top_k=12, + doc_type="procedure", + equipment_id=equipment_id, + conn=conn, + ) + return rerank(hits, question, top_n=3) + + def as_dicts(chunks: list[Chunk]) -> list[dict[str, Any]]: return [asdict(c) for c in chunks] diff --git a/compose/ai-compose.yml b/compose/ai-compose.yml index 9c127bc..4dbab85 100644 --- a/compose/ai-compose.yml +++ b/compose/ai-compose.yml @@ -170,6 +170,11 @@ services: build: context: /home/azureuser/ai/web dockerfile: Dockerfile + args: + # Empty in a real deployment: the browser reaches the API at its own + # public hostname through Caddy. Set only for a tunnelled demo build, + # which has no Caddy and no Authelia in front of it. + VITE_API_BASE: "${VITE_API_BASE:-}" image: yau/ai-web:local container_name: ai-web restart: unless-stopped diff --git a/demo/ai-docs/README.md b/demo/ai-docs/README.md new file mode 100644 index 0000000..c933673 --- /dev/null +++ b/demo/ai-docs/README.md @@ -0,0 +1,30 @@ +# Demo documents — NOT controlled documents, NOT plant content + +Three fabricated documents, written so that the Procedural, Reference and +Advisory branches have something to retrieve before an Azure OpenAI account +exists. They are here rather than on `/datadisk/ai-docs` in the repository +sense: the real document root is gitignored because it holds real controlled +documents, and these are the opposite of that. + +**Every one of them is fiction.** The document numbers are `WRPS-DEMO-00x`, +which matches the extractor's pattern (`WRPS-[A-Z]{2,4}-\d{3,4}`) so header +extraction and confirmation are genuinely exercised, while being a number no +real WRPS document can ever have. The setpoints and limits in them are made up +and contradict `db/seed/tags.csv` in places, deliberately: nothing here should +survive being mistaken for plant data. + +They are loaded with `--no-embed`, so their chunks have NULL embeddings and are +invisible to semantic search. Before real ingestion: + +```sql +DELETE FROM doc_chunks WHERE embedding IS NULL; +``` + +Copy them to the host with: + +```bash +scp -r demo/ai-docs/* lin001:/tmp/demo-docs/ +``` + +then move them into `/datadisk/ai-docs/` in the matching folders — the folder +determines `doc_type` and there is no override. diff --git a/demo/ai-docs/design/DEMO-design-basis.md b/demo/ai-docs/design/DEMO-design-basis.md new file mode 100644 index 0000000..fed8713 --- /dev/null +++ b/demo/ai-docs/design/DEMO-design-basis.md @@ -0,0 +1,39 @@ +# DEMO DOCUMENT — NOT A CONTROLLED DOCUMENT — DO NOT USE ON PLANT + +Document number: WRPS-DEMO-003 +Title: Station Design Basis Extract — Discharge and Storage +Revision: 0 +Effective: 2026-01-01 + +## 1. Scope + +A fabricated design basis extract for the Waterloo Road Pump Station STN-001, +written so that the Advisory branch of the plant assistant has documented +limits to retrieve and cite during a no-LLM demonstration. Every figure below +is invented. Where it contradicts db/seed/tags.csv, the seed data is the one +derived from the register map and this one is fiction. + +## 2. Discharge + +The discharge manifold MAN-301 is described in this fabricated extract as +having a stated hydraulic capacity against the 22 m static lift. The three +pumps PU-301, PU-302 and PU-303 share a common speed reference, and this +fabricated extract states that the drives are clamped at both ends of their +range rather than being free to run to zero. + +## 3. Storage and spill + +This fabricated extract states that the wet well WW-101 provides storage +between the stop-all level and the spill weir crest, that the weir crest is the +point at which flow leaves the site, and that the volume between the current +level and the crest is the quantity of interest when assessing how much time is +available. + +## 4. Limits stated in this fabricated document + +- The station is not to be operated with the level held above the high level + alarm setpoint as a normal operating state. +- Continuous operation of all three pumps is described as a short-duration + condition, not a normal duty. +- Any change to a level setpoint is described as requiring assessment against + the storage remaining to the weir crest. diff --git a/demo/ai-docs/procedures/DEMO-interlock-bypass.md b/demo/ai-docs/procedures/DEMO-interlock-bypass.md new file mode 100644 index 0000000..9310f5b --- /dev/null +++ b/demo/ai-docs/procedures/DEMO-interlock-bypass.md @@ -0,0 +1,42 @@ +# DEMO DOCUMENT — NOT A CONTROLLED DOCUMENT — DO NOT USE ON PLANT + +Document number: WRPS-DEMO-001 +Title: Temporary Bypass of Pump Motor Protection Interlock +Revision: 0 +Effective: 2026-01-01 +Authorising role: Station Maintenance Supervisor + +## 1. Purpose + +This fabricated document exists so that the Procedural branch of the plant +assistant has a procedure-shaped thing to locate during a no-LLM demonstration. +It describes, in invented terms, the temporary bypass of the motor protection +interlock on pump PU-302 at the Waterloo Road Pump Station. + +Nothing in this document is a real instruction. It has not been reviewed, it +has not been approved, and no hazard assessment sits behind it. + +## 2. Prerequisites + +The following must all be confirmed before this fabricated procedure begins. + +- A permit to work has been issued and is held by the person carrying out the work. +- PU-302 has been electrically isolated at MCC-501 and the isolation proven dead. +- The duty selection has been transferred so that PU-301 and PU-303 can meet inflow. +- The wet well WW-101 level is below the start-duty level and falling. +- The Station Maintenance Supervisor has authorised the bypass in writing. + +## 3. Procedure + +1. Confirm every prerequisite in section 2 is satisfied and recorded. +2. Log in to the station HMI with a maintenance-level account. +3. Navigate to the PU-302 faceplate and select the protection tab. +4. Set the motor protection bypass to ENABLED and confirm the prompt. +5. Record the bypass in the station log with the time and the permit number. +6. Carry out the authorised work. +7. Set the motor protection bypass to DISABLED before returning PU-302 to service. + +## 4. Restoration + +Restore the duty selection and confirm PU-302 responds to a test start. Close +the permit. Record the restoration time in the station log. diff --git a/demo/ai-docs/rationalisation/DEMO-alarm-rationalisation.md b/demo/ai-docs/rationalisation/DEMO-alarm-rationalisation.md new file mode 100644 index 0000000..d916330 --- /dev/null +++ b/demo/ai-docs/rationalisation/DEMO-alarm-rationalisation.md @@ -0,0 +1,37 @@ +# DEMO DOCUMENT — NOT A CONTROLLED DOCUMENT — DO NOT USE ON PLANT + +Document number: WRPS-DEMO-002 +Title: Alarm Rationalisation Record — Wet Well Level Signal +Revision: 0 +Effective: 2026-01-01 + +## 1. Scope + +A fabricated alarm rationalisation record for the wet well WW-101 level signal +at the Waterloo Road Pump Station, written to give the Reference branch of the +plant assistant something to retrieve and cite during a no-LLM demonstration. + +## 2. LEVEL_SIGNAL_FAULT + +**Meaning.** The level signal fault alarm indicates that the PLC has rejected +the wet well level measurement from LIT-101. In this fabricated record the +rejection is described as occurring when the signal is out of range, frozen for +longer than the configured period, or of bad quality. + +**Consequence.** With no trusted level measurement the station cannot modulate +pump duty on level. This fabricated record states that control falls back to +the LSHH-102 high high level switch as the only remaining level protection. + +**Priority.** Priority 1 in this fabricated record. The stated reasoning is +that losing the level measurement on a wet well that can spill removes the +station's ability to see a spill developing. + +**Operator response.** Not stated here. The response belongs in a procedure, +and this is a rationalisation record. + +## 3. HIGH_LEVEL + +**Meaning.** A fabricated entry stating that the wet well level has reached the +high level alarm setpoint and that the station has not yet drawn the level down. + +**Priority.** Priority 2 in this fabricated record. diff --git a/web/Dockerfile b/web/Dockerfile index 09927de..f67ce11 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -4,6 +4,14 @@ WORKDIR /app COPY package.json ./ RUN npm install COPY . . + +# Where the browser should send /ask. Empty means the built-in default, which +# is the public hostname. A tunnelled demo build overrides it: +# --build-arg VITE_API_BASE=http://localhost:8001 +# Vite inlines this at BUILD time, so a change needs a rebuild, not a restart. +ARG VITE_API_BASE="" +ENV VITE_API_BASE=$VITE_API_BASE + RUN npm run build FROM nginx:alpine diff --git a/web/src/App.tsx b/web/src/App.tsx index 923bbee..2c2b4d5 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -1,10 +1,19 @@ import { useState } from "react"; import type { AnswerBody, AskResponse, Citation } from "./types"; -// Same-origin in dev, the public hostname in the built image. The browser -// carries the Authelia session cookie either way; there is no token handling -// in this app because Authelia authenticates at the edge. -const API = import.meta.env.DEV ? "/api" : "https://api.yokogawa.tech"; +// Same-origin in dev, otherwise VITE_API_BASE at build time, falling back to +// the public hostname. The browser carries the Authelia session cookie either +// way; there is no token handling in this app because Authelia authenticates +// at the edge. +// +// VITE_API_BASE exists because the public hostname does not resolve yet. A +// tunnelled build points it at http://localhost:8001 so the whole chain can be +// exercised before DNS, Caddy and Authelia are in place - which also means a +// tunnelled build has NO AUTHENTICATION in front of it. It is reachable only +// through an SSH tunnel from one machine, and it is not a deployment. +const API = import.meta.env.DEV + ? "/api" + : (import.meta.env.VITE_API_BASE ?? "https://api.yokogawa.tech"); export default function App() { const [question, setQuestion] = useState(""); @@ -12,6 +21,8 @@ export default function App() { const [error, setError] = useState(null); const [busy, setBusy] = useState(false); const [showWorking, setShowWorking] = useState(true); + // Stub mode only - the API ignores it otherwise. See api/stub.py. + const [forcedClass, setForcedClass] = useState(""); async function ask(event: React.FormEvent) { event.preventDefault(); @@ -23,7 +34,9 @@ export default function App() { method: "POST", headers: { "Content-Type": "application/json" }, credentials: "include", - body: JSON.stringify({ question }), + body: JSON.stringify( + forcedClass ? { question, question_class: forcedClass } : { question }, + ), }); const body = await response.json(); if (!response.ok) { @@ -67,6 +80,24 @@ export default function App() { /> Show working + {/* Only has an effect while the API runs with NO_LLM_STUB on. There + is no classifier without a model, so the class is chosen here + rather than guessed - see api/stub.py for why guessing it with + keywords would be worse than asking. */} + @@ -82,6 +113,18 @@ function Answer({ result, showWorking }: { result: AskResponse; showWorking: boo
{result.question_class}
+ {/* An answer nobody generated must not look like one that was. Same + place, same weight and the same reasoning as the fixture banner. */} + {a.stub_mode && ( +
+ No-LLM stub mode. No language model was called. The + class was chosen by hand, retrieval was lexical rather than semantic, + and the prose is a fixed placeholder. Figures, citations and document + identities are real query results; the wording around them means + nothing. +
+ )} + {/* Fixture data must never reach a slide unlabelled. */} {a.used_fixture_data && (
diff --git a/web/src/styles.css b/web/src/styles.css index d1994fc..0bfe42b 100644 --- a/web/src/styles.css +++ b/web/src/styles.css @@ -12,6 +12,8 @@ --scope-line: #d99b00; --fixture: #ffe9e9; --fixture-line: #c0392b; + --stub: #efe7fb; + --stub-line: #6b3fa0; --error: #c0392b; } @@ -113,3 +115,17 @@ pre { font-size: 0.85rem; } code { font-size: 0.9em; } + +/* Stub mode. Deliberately as loud as the fixture banner - both say the same + kind of thing: what you are looking at is not what it appears to be. */ +.banner.stub { background: var(--stub); border-color: var(--stub-line); } + +.forced-class { + display: inline-flex; + align-items: center; + gap: 0.4rem; +} + +.forced-class select { + padding: 0.2rem 0.3rem; +} diff --git a/web/src/types.ts b/web/src/types.ts index 2e34967..d56f791 100644 --- a/web/src/types.ts +++ b/web/src/types.ts @@ -27,6 +27,7 @@ export interface AnswerBody { question_class: QuestionClass; answer: string; used_fixture_data: boolean; + stub_mode?: boolean; // historical query?: Record;