# ============================================================================= # process_values.yml — sampled analogue history. # # REMOVING THE STAND-IN: db/README-standin-historian.md. The view below is # CONTRACT - repoint it at imh, do not edit this file to absorb a difference. # # SOURCE: fixture.process_value_history while USE_FIXTURES=true; the same view # repointed at imh from Phase 4. It is keyed on CI SERVER ITEM NAMES # (AID.WRPS.STN.LEVEL), which is what the historian is keyed on, and it # resolves each item onto its tag through public.historian_items. # # PHASE 5 FINDING (a), FIXED — AND HOW. # The history used to be keyed AID.WRPS.STN.LEVEL, a CI Server POINT name, # while db/seed/tags.csv carried that string only as an ALIAS of LIT-101. So # public.tags had no row with that tag_id, a tag-level lookup for the wet well # matched ZERO of 43,201 level rows, and it surfaced as "no records found" — # which an operator cannot tell apart from an absence of data. # # The names were two layers apart, not one: # # LIT-101 instrument tag WRPS/01-design-doc # %QW0 PLC symbol WRPS/04-plc/register-map.csv # AID.WRPS.STN.LEVEL CI Server point WRPS/05-scada/modbus/scada-points.csv # AID.WRPS.STN.LEVEL CI Server ITEM <- what the historian stores # # Nothing is aliased across that gap any more. public.historian_items holds the # item-to-tag mapping, it is generated from the SCADA configuration by # scripts/gen_historian_items.py, and both that script and scripts/deploy.sh # refuse to proceed if a historised item resolves to neither a tag nor a # written reason for having none. LIT-101 is now correctly marked NOT # HISTORISED: it is a field input on %IW0 and never reaches SCADA. # # SAMPLE RATES ARE DECLARED, NOT ASSUMED. # scan_interval_seconds rides on every row, from the historisation group the # item belongs to: 5 s for WRPS_ONE_SEC, 30 s for WRPS_THIRTY_SEC. Two measures # below convert sample counts into durations and USED TO HARDCODE 60 SECONDS. # Against a 5-second group that is wrong by a factor of twelve, and it would # have read as plausible. # # ON DEADBAND COMPRESSION — the previous note here was WRONG and it mattered. # It warned that real CI Server history is deadband-compressed and therefore # irregular, so a plain average would be biased and only a time-weighted one # could be trusted. The WRPS configuration says otherwise: every history group # has DATA_COMP = 0, every item STORE_DEADBAND = 0, and the analogue groups are # COL_STOR_TYPE "Scan/Time". These samples are regular. The warning is true # only of WRPS_EVENT, which is Event/Item and genuinely on-change, and which # this cube does not read. # # THE CHECK THAT MUST STILL RUN AT THE PHASE 4 GATE: confirm against real imh # data that the gap between consecutive samples of AID.WRPS.STN.LEVEL really is # the declared 5 seconds. If it is not — if someone enables compression, or the # link drops — every duration measure here is wrong, and the fix is to compute # gaps with a LEAD window rather than trusting the declaration. # # SENTINELS: AID.WRPS.STN.TIME_TO_SPILL and .TIME_TO_LSHH use 32767 to mean # "drawing down or holding" - it is not a duration. Every measure here excludes # it. Do not remove that filter to make a number look tidier. # # QUALITY: rows with quality other than GOOD are excluded from every measure. # A BAD sample from a failed transmitter is not a low reading. The fixtures # exercise this - the level transmitter is frozen for an hour and those samples # are flagged BAD. # # UNITS: whatever the historian stores, which is not always what the PLC works # in. Wet well level is historised as percent of the spill weir crest (raw mm # divided by 60): 100.0 % = 6000 mm. See db/seed/historian_items.csv for every # gain, taken from the SCADA point list rather than restated here. # ============================================================================= cubes: - name: process_values sql_table: fixture.process_value_history # -> imh-backed view at Phase 4 description: > Sampled analogue history - wet well level, inflow, discharge flow, drive speed, net accumulation, and the 30-second station items. This is what makes an advisory question answerable with evidence; you cannot answer a flow question from alarms. joins: - name: tags sql: "{CUBE}.tag_id = {tags}.tag_id" relationship: many_to_one dimensions: - name: id sql: "{CUBE}.item_name || '@' || {CUBE}.sample_time" type: string primary_key: true - name: sample_time sql: sample_time type: time description: > Stored UTC - every WRPS Modbus point carries TIME_ZONE "Date+time GMT" - and presented in SITE_TIMEZONE. Converted once, here. - name: item_name sql: item_name type: string description: > The CI Server item. This is the historian's own key; filter on it when you know exactly which point you want. - name: tag_id sql: tag_id type: string description: > The tag the item corresponds to, resolved through public.historian_items. The join to equipment goes through here. - name: his_group sql: his_group type: string description: WRPS_ONE_SEC (5 s) or WRPS_THIRTY_SEC (30 s). - name: scan_interval_seconds sql: scan_interval_seconds type: number description: > Seconds between samples, from the item's historisation group. Any measure turning a count of samples into a duration must use this and never a literal. - name: engineering_unit sql: engineering_unit type: string description: > Always report this with the number. A level of 86.7 is a percentage of the weir crest, not a metre reading. - name: quality sql: quality type: string - name: is_fixture sql: is_fixture type: boolean measures: - name: sample_count type: count filters: - sql: "{CUBE}.quality = 'GOOD' AND {CUBE}.value <> 32767" - name: avg_value sql: value type: avg filters: - sql: "{CUBE}.quality = 'GOOD' AND {CUBE}.value <> 32767" description: > Plain average of samples. Correct on Scan/Time history, which is regular - CI Server has compression and deadband switched off on every WRPS group. Prefer time_weighted_avg only if that ever changes. - name: time_weighted_avg sql: > SUM(CASE WHEN {CUBE}.quality = 'GOOD' AND {CUBE}.value <> 32767 THEN {CUBE}.value * {CUBE}.scan_interval_seconds END) / NULLIF(SUM(CASE WHEN {CUBE}.quality = 'GOOD' AND {CUBE}.value <> 32767 THEN {CUBE}.scan_interval_seconds END), 0) type: number description: > Each sample weighted by how long it stood, using the item's declared scan interval. On regular Scan/Time history this agrees with avg_value exactly, which is why the Phase 4 gate has to confirm the real gaps ARE the declared interval - if they are not, this is the measure that stays honest and avg_value is the one that quietly stops being. Bad and sentinel samples are excluded in the CASE rather than by a measure filter, for the same reason as p95_value below. - name: max_value sql: value type: max filters: - sql: "{CUBE}.quality = 'GOOD' AND {CUBE}.value <> 32767" - name: min_value sql: value type: min filters: - sql: "{CUBE}.quality = 'GOOD' AND {CUBE}.value <> 32767" - name: p95_value # The quality filter is INSIDE the ordered-set aggregate on purpose. # A Cube measure `filters:` block cannot be applied to PERCENTILE_CONT # - it lands outside the aggregate and Postgres rejects the query with # "column process_values.quality must appear in the GROUP BY clause". # PERCENTILE_CONT ignores the NULLs the CASE produces. sql: > PERCENTILE_CONT(0.95) WITHIN GROUP ( ORDER BY CASE WHEN {CUBE}.quality = 'GOOD' AND {CUBE}.value <> 32767 THEN {CUBE}.value END) type: number description: > 95th percentile. More useful than max for "how high does it normally get", because max is one sample and often a transient. - name: bad_sample_count type: count filters: - sql: "{CUBE}.quality <> 'GOOD'" description: > Samples excluded for quality. Report this whenever it is non-zero: an average over a window in which the transmitter was frozen is an average over less data than the operator thinks, and the level signal fault (alarm word bit 13) is the reason to look. - name: seconds_above_high_level_alarm # Sums the DECLARED scan interval per qualifying sample rather than a # literal. The previous version summed 60 per sample against what is # now a 5-second group - a twelvefold overstatement that would have # read as entirely plausible. sql: > SUM(CASE WHEN {CUBE}.item_name = 'AID.WRPS.STN.LEVEL' AND {CUBE}.quality = 'GOOD' AND {CUBE}.value >= 86.7 THEN {CUBE}.scan_interval_seconds ELSE 0 END) type: number description: > Seconds the wet well spent above the high level alarm setpoint (86.7 % = 5200 mm, the %MW8 default). If the setpoint itself was changed during the window - AID.WRPS.SP.HIGH_ALARM - this measure is wrong and the answer must say so. - name: seconds_above_lshh sql: > SUM(CASE WHEN {CUBE}.item_name = 'AID.WRPS.STN.LEVEL' AND {CUBE}.quality = 'GOOD' AND {CUBE}.value >= 91.7 THEN {CUBE}.scan_interval_seconds ELSE 0 END) type: number description: > Seconds above LSHH (91.7 % = 5500 mm). Any non-zero value here is worth reporting explicitly. pre_aggregations: - name: pv_by_hour measures: [avg_value, max_value, min_value, sample_count, bad_sample_count] dimensions: [item_name, tag_id, engineering_unit, his_group] time_dimension: sample_time granularity: hour partition_granularity: month refresh_key: # 24h, not minutes: the fixtures are static, so a shorter # interval rebuilds a byte-identical result on a 2-vCPU host # and was a standing CPU load for no gain. Tune this back # deliberately when imh makes the data genuinely live. every: 24 hours build_range_start: # Eight days, not 180. The historian retains one week; partitions # older than that would be built empty, every refresh, forever. sql: "SELECT now() - interval '8 days'" build_range_end: sql: "SELECT now()" views: - name: process_history description: > Analogue history joined to the tag and equipment it belongs to, so a level question about "the wet well" resolves without the caller knowing the item is called AID.WRPS.STN.LEVEL. cubes: - join_path: process_values includes: - sample_time - item_name - tag_id - his_group - scan_interval_seconds - engineering_unit - quality - is_fixture - sample_count - avg_value - time_weighted_avg - max_value - min_value - p95_value - bad_sample_count - seconds_above_high_level_alarm - seconds_above_lshh - join_path: process_values.tags prefix: true includes: - display_name - signal_type - join_path: process_values.tags.equipment prefix: true includes: - equipment_id - display_name - equipment_type