yau-plant-assistant/scripts/gen_historian_items.py

#!/usr/bin/env python3
"""Regenerate db/seed/historian_items.csv from the WRPS SCADA configuration.

    python scripts/gen_historian_items.py --wrps /c/Claude/WRPS

WHY THIS EXISTS
---------------
The historian is keyed on CI Server ITEM names - `AID.WRPS.STN.LEVEL` - and
nothing else. Four namespaces describe the same measurement and only the last
one is what `imh` actually stores:

    LIT-101                    instrument tag      WRPS/01-design-doc
    %QW0                       PLC symbol address  WRPS/04-plc/register-map.csv
    PS_STN_WET_WELL_LEVEL      CI Server point     WRPS/05-scada/modbus/scada-points.csv
    AID.WRPS.STN.LEVEL         CI Server item      WRPS/05-scada/modbus/wrps_item_df.qli

db/002_fixtures.sql was written against the third of those, which is why a
tag-level lookup matched zero history rows. This script pins the fourth into
the AID repository so the stand-in historian and the real one agree by
construction rather than by review.

Following the WRPS house rule for 05-scada/modbus: nothing downstream is
hand-edited. If a register changes, re-run the WRPS generators, then re-run
this one. The output CSV is checked in so that a deploy on lin001 does not
need the WRPS repository present.

WHAT IT READS
    <wrps>/05-scada/modbus/wrps_item_df.qli         49 items, units, formats
    <wrps>/05-scada/modbus/wrps_section_df.qli      the six AID.WRPS sections
    <wrps>/05-scada/modbus/wrps_modbus_point_df.qli point scaling, IO address
    <wrps>/05-scada/modbus/item_his.qli             which items are historised
    <wrps>/05-scada/modbus/export_his_group.qli     the LIVE historisation rates
    <wrps>/05-scada/modbus/scada-points.csv         engineering gain per point

ON THE HISTORISATION GROUPS
    The repository's own `his_group.qli` (the intended import) and the live
    system's `export_his_group.qli` DISAGREE: the file says WRPS_ONE_SEC is a
    1 second group and pairs it with a 60 second WRPS_ONE_MIN; the live server
    runs WRPS_ONE_SEC at 5 seconds and has WRPS_THIRTY_SEC at 30 seconds
    instead. The live server wins here - the stand-in exists to behave like
    the thing it stands in for. `--groups-from-repo` selects the other
    reading. Either way the chosen rates are written into the CSV, so
    db/002_fixtures.sql never hardcodes a sample interval.
"""

from __future__ import annotations

import argparse
import csv
import io
import re
import sys
from pathlib import Path

BS = chr(92)  # backslash; kept out of literals so shell heredocs cannot mangle it

# item_his.qli names the group each item belongs to. Those names are the
# repository's; map them onto whatever the live server actually calls the
# equivalent group. Identity for WRPS_EVENT - both agree it is on-change.
LIVE_GROUP_FOR = {
    "WRPS_ONE_SEC": "WRPS_ONE_SEC",
    "WRPS_ONE_MIN": "WRPS_THIRTY_SEC",
    "WRPS_EVENT": "WRPS_EVENT",
}

# An item is normally matched to a db/seed/tags.csv row by its CI Server POINT
# name, which the AID seed uses verbatim as its tag_id. Two items cannot be:
# scada-points.csv reuses the name PS_STN_HIGH_LEVEL_ALARM for BOTH the coil 10
# status bit and the holding register 1032 setpoint, so the point name is not
# unique and the item layer is the first place the two are distinguishable
# (STN.HIGH_LEVEL versus SP.HIGH_ALARM). The AID seed keeps them apart the way
# it always has, with an _SP suffix on the setpoint.
TAG_FOR_ITEM = {
    "AID.WRPS.SP.HIGH_ALARM": "PS_STN_HIGH_LEVEL_ALARM_SP",
}

# Items deliberately NOT answerable by the assistant. They are historised, so
# they exist in the stand-in historian and in public.historian_items, but they
# carry no tag and no equipment and the agent cannot resolve a question onto
# them. Being listed here with a reason is the whole point: an item that is
# simply missing from the tag seed fails as "no records found", which an
# operator cannot tell apart from an absence of data. That was finding (a).
EXCLUDED_ITEMS = {
    "AID.WRPS.SIM.INFLOW":
        "simulation control, not a plant measurement - answering from it would "
        "report the scenario driver as though it were the real inflow",
    "AID.WRPS.SIM.SCENARIO":
        "simulation control, not a plant measurement",
    "AID.WRPS.SIM.RESET":
        "simulation control, not a plant measurement",
    "AID.WRPS.SIM.TIME_SCALE":
        "simulation control - a non-unity time scale means wall-clock durations "
        "in the history are compressed and must not be quoted as real durations",
}


def load_qli(path: Path, tag: str) -> list[dict[str, str]]:
    """Parse a CI Server .qli export into a list of records.

    The format is a @FIELDS block naming the columns, then a @<TAG> block of
    records. Records are comma-separated, quoted, and continued across lines
    with a trailing backslash.
    """
    text = path.read_text(encoding="utf-8", errors="replace").replace("\r\n", "\n")
    header = text.split("@FIELDS", 1)[1].split(tag, 1)[0].replace(BS, "")
    fields = [
        f.strip()
        for f in re.split(r"[,\n]", header)
        if f.strip() and not f.strip().startswith("!")
    ]
    body = text.split(tag, 1)[1].replace(BS + "\n", "")
    records = [line.strip() for line in body.split("\n") if line.strip().startswith('"')]
    return [
        dict(zip(fields, next(csv.reader(io.StringIO(r)))))
        for r in records
    ]


def io_address_parts(io_address: str) -> tuple[str, int]:
    """'RO:1025' -> ('holding', 1024).

    CI Server's IO_ADDRESS numbering is 1-based - confirmed on the WRPS system
    2026-08-14 by cross-checking STN.LEVEL against a direct pymodbus read - so
    holding register 0 is RO:01. DO is a coil, read with FC01.
    """
    kind, _, number = io_address.partition(":")
    zero_based = int(number) - 1
    return ("coil" if kind == "DO" else "holding"), zero_based


def scan_interval(groups: list[dict[str, str]], name: str) -> str:
    """Seconds between samples for a Scan/Time group; empty for event groups."""
    for g in groups:
        if g["NAME"] == name:
            return "" if g["COL_STOR_TYPE"] == "Event/Item" else g["SCAN_INTERVAL"]
    raise SystemExit(f"historisation group {name!r} is not defined on the server")


def life_time(groups: list[dict[str, str]], name: str) -> str:
    for g in groups:
        if g["NAME"] == name:
            return g["LIFE_TIME"]
    return ""


def build(wrps: Path, groups_from_repo: bool, tags_csv: Path) -> list[dict[str, str]]:
    modbus = wrps / "05-scada" / "modbus"

    with tags_csv.open(encoding="utf-8") as fh:
        known_tags = {row["tag_id"] for row in csv.DictReader(fh)}

    items = load_qli(modbus / "wrps_item_df.qli", "@ITEM_DF")
    sections = {s["NAME"]: s for s in load_qli(modbus / "wrps_section_df.qli", "@SECTION_DF")}
    points = {p["NAME"]: p for p in load_qli(modbus / "wrps_modbus_point_df.qli", "@MODBUS_POINT_DF")}
    item_his = {h["ITEM_NAME"]: h for h in load_qli(modbus / "item_his.qli", "@ITEM_HIS_DF")}

    group_file = "his_group.qli" if groups_from_repo else "export_his_group.qli"
    groups = load_qli(modbus / group_file, "@HIS_GROUP_DF")

    # scada-points.csv carries the engineering gain, which is a SCADA-side
    # presentation choice and lives nowhere in the .qli point definitions.
    # Key it the same way the point definitions are keyed, by resolved address.
    gain_by_address: dict[tuple[str, int], dict[str, str]] = {}
    with (modbus / "scada-points.csv").open(encoding="utf-8") as fh:
        for row in csv.DictReader(fh):
            kind = "coil" if row["function_code"] == "FC01" else "holding"
            gain_by_address[(kind, int(row["modbus_address"]))] = row

    out: list[dict[str, str]] = []
    for item in items:
        name = item["NAME"]
        point = points[item["POINT_NAME"]]
        kind, address = io_address_parts(point["IO_ADDRESS"])
        scada = gain_by_address.get((kind, address), {})

        his = item_his.get(name)
        repo_group = his["GROUP_NAME"] if his else ""
        group = "" if not repo_group else (
            repo_group if groups_from_repo else LIVE_GROUP_FOR.get(repo_group, repo_group)
        )

        # Resolve the item onto a tag, or onto a stated reason for having none.
        # Anything else is a build failure - see check_mapping below.
        tag_id = TAG_FOR_ITEM.get(name, scada.get("scada_tag", ""))
        if tag_id not in known_tags:
            tag_id = ""

        section_path = item["SECTION_PATH"]
        out.append(
            {
                "item_name": name,
                "tag_id": tag_id,
                "exclusion_reason": "" if tag_id else EXCLUDED_ITEMS.get(name, ""),
                "section_path": section_path,
                "section": item["UNIT"],
                "attribute": item["TAG"],
                "section_description": sections.get(section_path, {}).get("DESCRIPTION", ""),
                "description": item["DESCRIPTION"],
                "eng_unit": item["ENG_UNIT"],
                "value_format": item["VALUE_FORMAT"],
                "conv_type": point["CONV_TYPE"],
                "has_sign": point["HAS_SIGN"],
                "phys_low": point["PHYS_LOW"],
                "phys_high": point["PHYS_HIGH"],
                "eng_gain": scada.get("eng_gain", ""),
                "raw_to_eng": scada.get("raw_to_eng", ""),
                "his_group": group,
                # Empty for the on-change group. db/002_fixtures.sql reads this
                # rather than assuming a rate - the old fixtures hardcoded 60
                # seconds in two Cube measures and were wrong by 12x.
                "scan_interval_seconds": scan_interval(groups, group) if group else "",
                "life_time": life_time(groups, group) if group else "",
                "scada_point": scada.get("scada_tag", ""),
                "iec_address": scada.get("iec_address", ""),
                "modbus_kind": kind,
                "modbus_address": str(address),
                "data_type": scada.get("data_type", ""),
                # CI Server stamps every one of these points "Date+time GMT"
                # (TIME_ZONE in wrps_modbus_point_df.qli), and the WRPS history
                # groups all carry CORRECT_DAYLIGHT=0. Storage is UTC. Carried
                # per row so the claim is checkable at the point of use.
                "point_time_zone": point["TIME_ZONE"],
            }
        )
    return out


def check_mapping(rows: list[dict[str, str]]) -> list[str]:
    """Every historised item must resolve to a tag or to a stated reason.

    This is the check that stops finding (a) recurring. The old failure was an
    item the history was keyed on with no matching row in the tag seed: the
    join silently matched nothing and the assistant reported "no records
    found". Here that is a build error instead, and the only way past it is to
    add the tag or to write down why the item is deliberately unanswerable.
    """
    problems = []
    for r in rows:
        if not r["his_group"]:
            continue
        if not r["tag_id"] and not r["exclusion_reason"]:
            problems.append(
                f"  {r['item_name']} (point {r['scada_point'] or '?'}) has no row in "
                f"db/seed/tags.csv and no entry in EXCLUDED_ITEMS"
            )
    return problems


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__,
                                     formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument("--wrps", type=Path, default=Path("C:/Claude/WRPS"),
                        help="path to the WRPS repository")
    parser.add_argument("--out", type=Path,
                        default=Path(__file__).parent.parent / "db" / "seed" / "historian_items.csv")
    parser.add_argument("--groups-from-repo", action="store_true",
                        help="take historisation rates from his_group.qli (the intended "
                             "import) rather than export_his_group.qli (what the server runs)")
    args = parser.parse_args()

    if not (args.wrps / "05-scada" / "modbus").is_dir():
        print(f"not a WRPS repository: {args.wrps}", file=sys.stderr)
        return 1

    tags_csv = args.out.parent / "tags.csv"
    rows = build(args.wrps, args.groups_from_repo, tags_csv)

    problems = check_mapping(rows)
    if problems:
        print("historised items that resolve to nothing:", file=sys.stderr)
        print("\n".join(problems), file=sys.stderr)
        print("\nAdd the tag to db/seed/tags.csv, or add the item to "
              "EXCLUDED_ITEMS with a reason. Nothing is written.", file=sys.stderr)
        return 1

    args.out.parent.mkdir(parents=True, exist_ok=True)
    with args.out.open("w", encoding="utf-8", newline="") as fh:
        writer = csv.DictWriter(fh, fieldnames=list(rows[0].keys()), lineterminator="\n")
        writer.writeheader()
        writer.writerows(rows)

    historised = [r for r in rows if r["his_group"]]
    print(f"wrote {len(rows)} items to {args.out}")
    print(f"  historised: {len(historised)}"
          f"   answerable: {sum(1 for r in historised if r['tag_id'])}"
          f"   deliberately excluded: {sum(1 for r in historised if r['exclusion_reason'])}")
    for group in sorted({r['his_group'] for r in historised}):
        members = [r for r in historised if r["his_group"] == group]
        interval = members[0]["scan_interval_seconds"] or "on change"
        print(f"    {group:16} {len(members):2} items   every {interval}"
              f"   life {members[0]['life_time']}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())