#!/usr/bin/env python3 """Regenerate db/seed/historian_items.csv from the WRPS SCADA configuration. python scripts/gen_historian_items.py --wrps /c/Claude/WRPS WHY THIS EXISTS --------------- The historian is keyed on CI Server ITEM names - `AID.WRPS.STN.LEVEL` - and nothing else. Four namespaces describe the same measurement and only the last one is what `imh` actually stores: LIT-101 instrument tag WRPS/01-design-doc %QW0 PLC symbol address WRPS/04-plc/register-map.csv PS_STN_WET_WELL_LEVEL CI Server point WRPS/05-scada/modbus/scada-points.csv AID.WRPS.STN.LEVEL CI Server item WRPS/05-scada/modbus/wrps_item_df.qli db/002_fixtures.sql was written against the third of those, which is why a tag-level lookup matched zero history rows. This script pins the fourth into the AID repository so the stand-in historian and the real one agree by construction rather than by review. Following the WRPS house rule for 05-scada/modbus: nothing downstream is hand-edited. If a register changes, re-run the WRPS generators, then re-run this one. The output CSV is checked in so that a deploy on lin001 does not need the WRPS repository present. WHAT IT READS /05-scada/modbus/wrps_item_df.qli 49 items, units, formats /05-scada/modbus/wrps_section_df.qli the six AID.WRPS sections /05-scada/modbus/wrps_modbus_point_df.qli point scaling, IO address /05-scada/modbus/item_his.qli which items are historised /05-scada/modbus/export_his_group.qli the LIVE historisation rates /05-scada/modbus/scada-points.csv engineering gain per point ON THE HISTORISATION GROUPS The repository's own `his_group.qli` (the intended import) and the live system's `export_his_group.qli` DISAGREE: the file says WRPS_ONE_SEC is a 1 second group and pairs it with a 60 second WRPS_ONE_MIN; the live server runs WRPS_ONE_SEC at 5 seconds and has WRPS_THIRTY_SEC at 30 seconds instead. The live server wins here - the stand-in exists to behave like the thing it stands in for. `--groups-from-repo` selects the other reading. Either way the chosen rates are written into the CSV, so db/002_fixtures.sql never hardcodes a sample interval. """ from __future__ import annotations import argparse import csv import io import re import sys from pathlib import Path BS = chr(92) # backslash; kept out of literals so shell heredocs cannot mangle it # item_his.qli names the group each item belongs to. Those names are the # repository's; map them onto whatever the live server actually calls the # equivalent group. Identity for WRPS_EVENT - both agree it is on-change. LIVE_GROUP_FOR = { "WRPS_ONE_SEC": "WRPS_ONE_SEC", "WRPS_ONE_MIN": "WRPS_THIRTY_SEC", "WRPS_EVENT": "WRPS_EVENT", } # An item is normally matched to a db/seed/tags.csv row by its CI Server POINT # name, which the AID seed uses verbatim as its tag_id. Two items cannot be: # scada-points.csv reuses the name PS_STN_HIGH_LEVEL_ALARM for BOTH the coil 10 # status bit and the holding register 1032 setpoint, so the point name is not # unique and the item layer is the first place the two are distinguishable # (STN.HIGH_LEVEL versus SP.HIGH_ALARM). The AID seed keeps them apart the way # it always has, with an _SP suffix on the setpoint. TAG_FOR_ITEM = { "AID.WRPS.SP.HIGH_ALARM": "PS_STN_HIGH_LEVEL_ALARM_SP", } # Items deliberately NOT answerable by the assistant. They are historised, so # they exist in the stand-in historian and in public.historian_items, but they # carry no tag and no equipment and the agent cannot resolve a question onto # them. Being listed here with a reason is the whole point: an item that is # simply missing from the tag seed fails as "no records found", which an # operator cannot tell apart from an absence of data. That was finding (a). EXCLUDED_ITEMS = { "AID.WRPS.SIM.INFLOW": "simulation control, not a plant measurement - answering from it would " "report the scenario driver as though it were the real inflow", "AID.WRPS.SIM.SCENARIO": "simulation control, not a plant measurement", "AID.WRPS.SIM.RESET": "simulation control, not a plant measurement", "AID.WRPS.SIM.TIME_SCALE": "simulation control - a non-unity time scale means wall-clock durations " "in the history are compressed and must not be quoted as real durations", } def load_qli(path: Path, tag: str) -> list[dict[str, str]]: """Parse a CI Server .qli export into a list of records. The format is a @FIELDS block naming the columns, then a @ block of records. Records are comma-separated, quoted, and continued across lines with a trailing backslash. """ text = path.read_text(encoding="utf-8", errors="replace").replace("\r\n", "\n") header = text.split("@FIELDS", 1)[1].split(tag, 1)[0].replace(BS, "") fields = [ f.strip() for f in re.split(r"[,\n]", header) if f.strip() and not f.strip().startswith("!") ] body = text.split(tag, 1)[1].replace(BS + "\n", "") records = [line.strip() for line in body.split("\n") if line.strip().startswith('"')] return [ dict(zip(fields, next(csv.reader(io.StringIO(r))))) for r in records ] def io_address_parts(io_address: str) -> tuple[str, int]: """'RO:1025' -> ('holding', 1024). CI Server's IO_ADDRESS numbering is 1-based - confirmed on the WRPS system 2026-08-14 by cross-checking STN.LEVEL against a direct pymodbus read - so holding register 0 is RO:01. DO is a coil, read with FC01. """ kind, _, number = io_address.partition(":") zero_based = int(number) - 1 return ("coil" if kind == "DO" else "holding"), zero_based def scan_interval(groups: list[dict[str, str]], name: str) -> str: """Seconds between samples for a Scan/Time group; empty for event groups.""" for g in groups: if g["NAME"] == name: return "" if g["COL_STOR_TYPE"] == "Event/Item" else g["SCAN_INTERVAL"] raise SystemExit(f"historisation group {name!r} is not defined on the server") def life_time(groups: list[dict[str, str]], name: str) -> str: for g in groups: if g["NAME"] == name: return g["LIFE_TIME"] return "" def build(wrps: Path, groups_from_repo: bool, tags_csv: Path) -> list[dict[str, str]]: modbus = wrps / "05-scada" / "modbus" with tags_csv.open(encoding="utf-8") as fh: known_tags = {row["tag_id"] for row in csv.DictReader(fh)} items = load_qli(modbus / "wrps_item_df.qli", "@ITEM_DF") sections = {s["NAME"]: s for s in load_qli(modbus / "wrps_section_df.qli", "@SECTION_DF")} points = {p["NAME"]: p for p in load_qli(modbus / "wrps_modbus_point_df.qli", "@MODBUS_POINT_DF")} item_his = {h["ITEM_NAME"]: h for h in load_qli(modbus / "item_his.qli", "@ITEM_HIS_DF")} group_file = "his_group.qli" if groups_from_repo else "export_his_group.qli" groups = load_qli(modbus / group_file, "@HIS_GROUP_DF") # scada-points.csv carries the engineering gain, which is a SCADA-side # presentation choice and lives nowhere in the .qli point definitions. # Key it the same way the point definitions are keyed, by resolved address. gain_by_address: dict[tuple[str, int], dict[str, str]] = {} with (modbus / "scada-points.csv").open(encoding="utf-8") as fh: for row in csv.DictReader(fh): kind = "coil" if row["function_code"] == "FC01" else "holding" gain_by_address[(kind, int(row["modbus_address"]))] = row out: list[dict[str, str]] = [] for item in items: name = item["NAME"] point = points[item["POINT_NAME"]] kind, address = io_address_parts(point["IO_ADDRESS"]) scada = gain_by_address.get((kind, address), {}) his = item_his.get(name) repo_group = his["GROUP_NAME"] if his else "" group = "" if not repo_group else ( repo_group if groups_from_repo else LIVE_GROUP_FOR.get(repo_group, repo_group) ) # Resolve the item onto a tag, or onto a stated reason for having none. # Anything else is a build failure - see check_mapping below. tag_id = TAG_FOR_ITEM.get(name, scada.get("scada_tag", "")) if tag_id not in known_tags: tag_id = "" section_path = item["SECTION_PATH"] out.append( { "item_name": name, "tag_id": tag_id, "exclusion_reason": "" if tag_id else EXCLUDED_ITEMS.get(name, ""), "section_path": section_path, "section": item["UNIT"], "attribute": item["TAG"], "section_description": sections.get(section_path, {}).get("DESCRIPTION", ""), "description": item["DESCRIPTION"], "eng_unit": item["ENG_UNIT"], "value_format": item["VALUE_FORMAT"], "conv_type": point["CONV_TYPE"], "has_sign": point["HAS_SIGN"], "phys_low": point["PHYS_LOW"], "phys_high": point["PHYS_HIGH"], "eng_gain": scada.get("eng_gain", ""), "raw_to_eng": scada.get("raw_to_eng", ""), "his_group": group, # Empty for the on-change group. db/002_fixtures.sql reads this # rather than assuming a rate - the old fixtures hardcoded 60 # seconds in two Cube measures and were wrong by 12x. "scan_interval_seconds": scan_interval(groups, group) if group else "", "life_time": life_time(groups, group) if group else "", "scada_point": scada.get("scada_tag", ""), "iec_address": scada.get("iec_address", ""), "modbus_kind": kind, "modbus_address": str(address), "data_type": scada.get("data_type", ""), # CI Server stamps every one of these points "Date+time GMT" # (TIME_ZONE in wrps_modbus_point_df.qli), and the WRPS history # groups all carry CORRECT_DAYLIGHT=0. Storage is UTC. Carried # per row so the claim is checkable at the point of use. "point_time_zone": point["TIME_ZONE"], } ) return out def check_mapping(rows: list[dict[str, str]]) -> list[str]: """Every historised item must resolve to a tag or to a stated reason. This is the check that stops finding (a) recurring. The old failure was an item the history was keyed on with no matching row in the tag seed: the join silently matched nothing and the assistant reported "no records found". Here that is a build error instead, and the only way past it is to add the tag or to write down why the item is deliberately unanswerable. """ problems = [] for r in rows: if not r["his_group"]: continue if not r["tag_id"] and not r["exclusion_reason"]: problems.append( f" {r['item_name']} (point {r['scada_point'] or '?'}) has no row in " f"db/seed/tags.csv and no entry in EXCLUDED_ITEMS" ) return problems def main() -> int: parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("--wrps", type=Path, default=Path("C:/Claude/WRPS"), help="path to the WRPS repository") parser.add_argument("--out", type=Path, default=Path(__file__).parent.parent / "db" / "seed" / "historian_items.csv") parser.add_argument("--groups-from-repo", action="store_true", help="take historisation rates from his_group.qli (the intended " "import) rather than export_his_group.qli (what the server runs)") args = parser.parse_args() if not (args.wrps / "05-scada" / "modbus").is_dir(): print(f"not a WRPS repository: {args.wrps}", file=sys.stderr) return 1 tags_csv = args.out.parent / "tags.csv" rows = build(args.wrps, args.groups_from_repo, tags_csv) problems = check_mapping(rows) if problems: print("historised items that resolve to nothing:", file=sys.stderr) print("\n".join(problems), file=sys.stderr) print("\nAdd the tag to db/seed/tags.csv, or add the item to " "EXCLUDED_ITEMS with a reason. Nothing is written.", file=sys.stderr) return 1 args.out.parent.mkdir(parents=True, exist_ok=True) with args.out.open("w", encoding="utf-8", newline="") as fh: writer = csv.DictWriter(fh, fieldnames=list(rows[0].keys()), lineterminator="\n") writer.writeheader() writer.writerows(rows) historised = [r for r in rows if r["his_group"]] print(f"wrote {len(rows)} items to {args.out}") print(f" historised: {len(historised)}" f" answerable: {sum(1 for r in historised if r['tag_id'])}" f" deliberately excluded: {sum(1 for r in historised if r['exclusion_reason'])}") for group in sorted({r['his_group'] for r in historised}): members = [r for r in historised if r["his_group"] == group] interval = members[0]["scan_interval_seconds"] or "on change" print(f" {group:16} {len(members):2} items every {interval}" f" life {members[0]['life_time']}") return 0 if __name__ == "__main__": raise SystemExit(main())