yau-plant-assistant/api/tests/test_model_contract_shapes.py

"""Three defects that only appear once a real model runs, pinned in Python.

All three were invisible under NO_LLM_STUB and surfaced within minutes of the
first live Azure OpenAI call on 2026-08-27. None of them needs an API key or a
database to test, which is the whole argument for putting them here.
"""

import json

import pytest

from classifier import FEW_SHOT
from config import settings
from contracts import QuestionClass, documented_limits, procedure_identity

# ---------------------------------------------------------------------------
# 1. The classifier few-shot must model the WHOLE reply shape
# ---------------------------------------------------------------------------

# Read by classify(); an absent key silently becomes a default, and for
# `confidence` that default is 0.0 - below every sane threshold.
REQUIRED_KEYS = {"question_class", "confidence", "alternatives",
                 "entities", "missing_context"}


def test_every_few_shot_reply_carries_the_full_payload():
    """The bug: replies were {"question_class": ...} alone.

    A few-shot reply is a shape the model copies. Omitting `confidence` taught
    it to omit `confidence`, so every question scored 0.0 and downgraded to
    UNCLEAR.
    """
    for question, reply in FEW_SHOT:
        missing = REQUIRED_KEYS - reply.keys()
        assert not missing, f"{question!r} omits {sorted(missing)}"


def test_few_shot_confidences_are_usable_and_not_constant():
    confidences = [r["confidence"] for _, r in FEW_SHOT]
    assert all(0.0 < c <= 1.0 for c in confidences)
    # A constant teaches the model to emit that constant.
    assert len(set(confidences)) > 1


def test_few_shot_replies_are_json_serialisable():
    for _, reply in FEW_SHOT:
        json.loads(json.dumps(reply))


def test_few_shot_classes_are_real_classes():
    for _, reply in FEW_SHOT:
        QuestionClass(reply["question_class"])
        for name in reply["alternatives"]:
            QuestionClass(name)


# ---------------------------------------------------------------------------
# 3. Procedure identity comes from evidence, never from the model
# ---------------------------------------------------------------------------

CITATION = {
    "doc_number": "WRPS-DEMO-001",
    "title": "Temporary Bypass of Pump Motor Protection Interlock",
    "revision": "0",
    "effective_date": "2026-01-01",
    "page": 1,
    "section_title": "2. Prerequisites",
    "source_file": "procedures/DEMO-interlock-bypass.md",
    "superseded": False,
}

# authorising_role is denormalised onto every chunk of the document by ingest.
CHUNKS = [{"authorising_role": "Station Maintenance Supervisor"}]


def test_identity_ignores_the_model_entirely():
    """The wrong-revision hazard. procedure_identity takes no `generated`
    argument at all now: there is no path by which a model can name a revision
    an operator does not hold."""
    identity = procedure_identity({"citations": [CITATION], "chunks": CHUNKS})
    assert identity["doc_number"] == "WRPS-DEMO-001"
    assert identity["revision"] == "0"
    # The bug: the model returned "", which is neither a date nor None, so
    # ProceduralAnswer rejected it twice and /ask returned 422.
    assert identity["effective_date"] == "2026-01-01"


def test_title_comes_from_the_confirmed_header():
    """Was "" whenever the header chunk was not retrieved - which under the old
    similarity-ranked find_procedure() was most of the time."""
    identity = procedure_identity({"citations": [CITATION], "chunks": CHUNKS})
    assert identity["title"] == "Temporary Bypass of Pump Motor Protection Interlock"


def test_authorising_role_comes_from_the_chunks():
    identity = procedure_identity({"citations": [CITATION], "chunks": CHUNKS})
    assert identity["authorising_role"] == "Station Maintenance Supervisor"


def test_controlled_copy_location_comes_from_settings():
    """Never the model's: an invented location sends somebody to a place that
    does not exist."""
    identity = procedure_identity({"citations": [CITATION], "chunks": CHUNKS})
    assert identity["controlled_copy_location"] == settings().controlled_copy_location
    assert identity["controlled_copy_location"]


def test_title_falls_back_to_source_file_on_a_pre_007_row():
    stale = {**CITATION, "title": None}
    identity = procedure_identity({"citations": [stale], "chunks": [{}]})
    assert identity["title"] == "procedures/DEMO-interlock-bypass.md"


def test_no_evidence_means_no_procedure():
    """Never assert a procedure with nothing retrieved behind it."""
    assert procedure_identity({"citations": [], "chunks": []}) is None


# ---------------------------------------------------------------------------
# 4. Documented limits are cited from evidence, never by the model
# ---------------------------------------------------------------------------

DESIGN_CITATION = {
    "doc_number": "WRPS-DEMO-003",
    "title": "Design Basis",
    "revision": "0",
    "effective_date": "2026-01-01",
    "page": 1,
    "section_title": "4. Hydraulics",
    "source_file": "design/DEMO-design-basis.md",
    "superseded": False,
}


def test_limit_gets_the_real_citation_object():
    """The bug: the model returned citation as the string
    "WRPS-DEMO-003, Section 4" where a Citation was required, so every advisory
    answer failed its contract twice and /ask returned 422."""
    generated = {"documented_limits": [
        {"description": "Spill weir crest", "value": 6000, "unit": "mm",
         "source_file": "design/DEMO-design-basis.md",
         "citation": "WRPS-DEMO-003, Section 4"},
    ]}
    limits = documented_limits({"citations": [DESIGN_CITATION]}, generated)
    assert limits[0]["citation"] == DESIGN_CITATION
    assert limits[0]["value"] == 6000
    assert limits[0]["unit"] == "mm"


def test_limit_with_an_unretrieved_source_is_dropped():
    """Never attach a limit to the nearest citation. A limit carries the
    authority of the document behind it."""
    generated = {"documented_limits": [
        {"description": "Invented limit", "value": 42,
         "source_file": "design/NOT-RETRIEVED.md"},
    ]}
    assert documented_limits({"citations": [DESIGN_CITATION]}, generated) == []


def test_no_limits_is_an_empty_list_not_none():
    assert documented_limits({"citations": [DESIGN_CITATION]}, {}) == []