"""Runnable check for the sign-disagreement watchdog. `python3 test_watchdog.py` Every case here is either a fault this must catch or a piece of NORMAL behaviour it must not mistake for one. The second list is the longer one on purpose: an alarm that fires on ordinary operation is worse than no alarm, because the next person switches it off. The headline validation is not here - it is `sim/replay_sign.py` in the infra repo, which drives this module over the two captured bus days. This file is the table-driven half: boundaries, timers, the latch, and the arming gate. ⚠️ MUTATING THE MODULE THIS FILE GRADES? PURGE `__pycache__` FIRST - or run `tools/mutation_run.sh`, which does. `python3 -B` alone does NOT save you: it stops bytecode being WRITTEN, not a stale one being READ, and a size-preserving edit reverted inside one second is invisible to CPython's staleness test. That is TEST-15, and it has already turned one live mutation into a false survivor. Read MUTATION.md before you report a survivor. """ import sys from datetime import datetime, timedelta, timezone from app.watchdog import ( SignWatchdog, SignWatchdogConfig, loop_is_following, ) fails = [] T0 = datetime(2026, 8, 20, 15, 0, 0, tzinfo=timezone.utc) def check(name, cond): print(f" {'ok ' if cond else 'FAIL'} {name}") if not cond: fails.append(name) def drive(samples, cfg=None, following=True): """Feed (offset_s, commanded_w, grid_w) tuples. Returns (wd, trip_offset).""" wd = SignWatchdog(cfg) trip_at = None for item in samples: offset, cmd, grid = item[0], item[1], item[2] f = item[3] if len(item) > 3 else following if wd.evaluate(cmd, grid, f, T0 + timedelta(seconds=offset)) and trip_at is None: trip_at = offset return wd, trip_at def opposed(duration_s, step=5, cmd=-1000, grid=1500, following=True, start=0): return [(start + t, cmd, grid, following) for t in range(0, int(duration_s) + 1, step)] print("the fault it exists to catch") wd, at = drive(opposed(300)) check("charging into an importing house trips", wd.tripped) check("and trips at 180 s, not before", at == 180) check("the reason names both signals", "-1000" in wd.reason and "+1500" in wd.reason) print(f" e.g. {wd.reason!r}") wd, at = drive(opposed(300, cmd=2000, grid=-2000)) check("discharging into an exporting house trips too", wd.tripped and at == 180) print() print("the boundaries") wd, _ = drive(opposed(600, cmd=-300, grid=1500)) check("300 W exactly does NOT qualify (strictly greater)", not wd.tripped) wd, _ = drive(opposed(600, cmd=-301, grid=1500)) check("301 W does", wd.tripped) wd, _ = drive(opposed(600, cmd=-1000, grid=300)) check("a 300 W meter does not qualify either", not wd.tripped) wd, _ = drive(opposed(175)) check("175 s of disagreement does not trip", not wd.tripped) wd, _ = drive(opposed(180)) check("180 s does (>= hold_s)", wd.tripped) print() print("continuity - the rule that produced zero false positives on real data") broken = opposed(175) + [(180, -1000, -1500)] + opposed(175, start=185) wd, _ = drive(broken) check("179 s, one agreeing sample, then 175 s more: no trip", not wd.tripped) check("and the window really was reset", wd.disagree_since is not None) long_enough = broken + opposed(200, start=365) wd, _ = drive(long_enough) check("but a full 180 s after the reset does trip", wd.tripped) print() print("things that are NOT faults") # The load-step transient. Measured on the healthy capture: longest 30 s. wd, _ = drive(opposed(30) + [(35 + t, 800, 900) for t in range(0, 300, 5)]) check("a 30 s load-step crossover does not trip", not wd.tripped) # SoC 10 %, Standby, declining a 1350 W discharge. The battery does nothing, so # the meter stays where it was and the command agrees with it. wd, _ = drive([(t, 1350, 1350) for t in range(0, 900, 5)]) check("a battery refusing a discharge command cannot trip it", not wd.tripped) # A frozen meter drives command and reading into agreement, not disagreement. wd, _ = drive([(t, 800, 800) for t in range(0, 900, 5)]) check("a frozen meter cannot trip it", not wd.tripped) # Maintenance grid charge: 2500 W of charge against an importing house, for # hours. Opposed, both huge - and entirely correct. wd, _ = drive(opposed(7200, cmd=-2500, grid=3000, following=False)) check("two hours of maintenance grid charge does not trip", not wd.tripped) check("and the timer never even started", wd.disagree_since is None) wd, _ = drive([(t, -1000, None) for t in range(0, 900, 5)]) check("a missing meter reading does not trip", not wd.tripped) wd, _ = drive(opposed(175) + [(180, -1000, None)] + opposed(600, start=185)) check("a missing reading mid-window RESETS it rather than accumulating", wd.tripped and wd.disagree_since is not None) wd, at = drive(opposed(175) + [(180, -1000, None)] + opposed(600, start=185)) check("...so the trip lands 180 s after the gap, not after 180 s total", at == 365) wd, _ = drive([(t, 0.0, 3000) for t in range(0, 900, 5)]) check("a 0 W command is not a wrong-direction command", not wd.tripped) print() print("the latch") wd, _ = drive(opposed(200)) check("tripped after the fault", wd.tripped) # The trip commands 0 W, which removes the command side of the comparison. for t in range(200, 900, 5): wd.evaluate(0.0, 3000, True, T0 + timedelta(seconds=t)) check("stays tripped once the command is zeroed", wd.tripped) for t in range(900, 1500, 5): wd.evaluate(1000, 1000, True, T0 + timedelta(seconds=t)) check("stays tripped even once everything looks healthy again", wd.tripped) wd.reset() check("reset() is the only way out", not wd.tripped and wd.disagree_since is None) check("and the reason is cleared with it", wd.reason == "") print() print("the thresholds are configuration, not constants") cfg = SignWatchdogConfig(min_w=800.0, hold_s=60.0) wd, at = drive(opposed(300, cmd=-1000, grid=1500), cfg=cfg) check("a 60 s hold trips at 60 s", wd.tripped and at == 60) wd, _ = drive(opposed(600, cmd=-700, grid=1500), cfg=cfg) check("an 800 W gate ignores a 700 W command", not wd.tripped) print() print("the configuration that would disarm it is refused") from app.validate import validate_startup_config # noqa: E402 base = {"setpoint_entity": "number.sp", "meter_source": "off", "meter_phases": 1, "meter_max_age_s": 30} p = validate_startup_config({**base, "max_w": 2000, "sign_disagree_w": 300}, -5000, 5000) check("300 W under a 2000 W ceiling is accepted", p == []) p = validate_startup_config({**base, "max_w": 2000, "sign_disagree_w": 2000}, -5000, 5000) check("a threshold AT max_w is refused", len(p) == 1 and "never fire" in p[0]) # ⚠️ maintenance_charge_w is named here only to keep this case about the SIGN # rule. MAINT-04 added a second rule against the same ceiling, and a 500 W # max_w is also below the 2000 W shipped maintenance charge - so without this # the dict is refused twice, for two correct but different reasons, and the # `len(p) == 1` below would be measuring the interaction rather than this rule. p = validate_startup_config({**base, "max_w": 500, "sign_disagree_w": 3000, "maintenance_charge_w": 500}, -5000, 5000) check("a threshold above max_w is refused", len(p) == 1 and "never fire" in p[0]) p = validate_startup_config({**base, "max_w": 2000, "sign_disagree_w": 1999}, -5000, 5000) check("one watt under the ceiling is reachable, so accepted", p == []) print() print("the arming gate, derived from the REAL arbiter") # ⚠️ THIS IS THE DECISION THE WHOLE TICKET TURNS ON, and until now it was # asserted in three comment blocks and verified by nothing: every other case in # this file passes `following` in as a literal. What has to hold is that the # expression in main.py - loop won AND the emitted value points the way the loop # asked AND is no larger - is never True for a resolution the loop did not # actually get. So run real claims through the real resolve() and derive it. from app.arbiter import ( # noqa: E402 P_LOOP, P_MAINT_SHAPE, P_MAINTENANCE, P_PEAK, P_SAFETY, Claim, resolve, ) def following_from(claims, asked): """THE SHIPPED gate, not a copy of it. ⚠️ This used to transcribe main.py's expression with a comment saying "keep the two in step", which is exactly the arrangement that does not keep two things in step. Measured: editing the shipped gate alone to `winner in ("loop", "peak")` left this suite - including the P_PEAK row below, whose entire job is to catch that edit - completely green, while the watchdog went blind for the duration of every peak event. Import it or the P_PEAK assertion is decoration. """ r = resolve(claims) return r, loop_is_following(r, asked) MAXW = Claim.limit("supervised", P_SAFETY, -2000, 2000, "max_w") DEV = Claim.limit("device", P_SAFETY, -5000, 5000, "rating") # name, claims, what the loop asked, expected `following` TABLE = [ ("the loop alone, unopposed", [Claim.set("loop", P_LOOP, 900)], 900, True), ("the loop bound by max_w - reduced, not redirected", [Claim.set("loop", P_LOOP, 4000), MAXW], 4000, True), ("charging, bound by max_w on the other side", [Claim.set("loop", P_LOOP, -4000), MAXW], -4000, True), ("maintenance owns the setpoint outright", [Claim.set("loop", P_LOOP, 900), Claim.set("maintenance", P_MAINTENANCE, -2500, "charge"), MAXW], 900, False), ("maintenance charge-only clamps the loop to 0 W", [Claim.set("loop", P_LOOP, 900), Claim.limit("maintenance", P_MAINT_SHAPE, hi=0.0, reason="charge-only"), MAXW], 900, False), # ⚠️ THE CASE `winner == \"loop\"` ALONE WOULD HAVE GOT WRONG. 1500 W, not # 2500 W: at max_w 2000 a -2500 hi against a -2000 lo is an arbiter # contradiction, not a charge. That pair was the SHIPPED default until # MAINT-04 refused it and CFG-03 gave the default one source of truth; it is # kept below as a case rather than as a configuration. ("cheap-window charge REDIRECTS the loop while it still 'wins'", [Claim.set("loop", P_LOOP, 500), Claim.limit("maintenance", P_MAINT_SHAPE, hi=-1500, reason="cheap-window"), MAXW], 500, False), ("...and the contradiction when charge_w exceeds max_w is not following either", [Claim.set("loop", P_LOOP, 500), Claim.limit("maintenance", P_MAINT_SHAPE, hi=-2500, reason="cheap-window"), MAXW], 500, False), ("a lo limit RAISES the magnitude without flipping the sign", [Claim.set("loop", P_LOOP, 200), Claim.limit("floor", P_SAFETY, lo=800), MAXW], 200, False), ("a safety stop wins outright", [Claim.set("loop", P_LOOP, 900), Claim.set("safety", P_SAFETY, 0.0, "stopped"), MAXW], 900, False), ("no claims at all - the arbiter's own failsafe", [MAXW, DEV], None, False), ("the loop asked for zero", [Claim.set("loop", P_LOOP, 0.0), MAXW], 0.0, False), # The forward hazard, asserted as it stands TODAY rather than as it should # be. When peak shaving lands, these expectations are what have to change. ("peak shaving at P_PEAK disarms the gate (loop_is_following's forward hazard)", [Claim.set("loop", P_LOOP, 900), Claim.set("peak", P_PEAK, 1800, "shaving"), MAXW], 900, False), # ⚠️ AND THE ROW ABOVE CANNOT DETECT A WIDENED GATE ON ITS OWN, which was # found by trying it: with the shipped gate edited to # `winner in ("loop", "peak")` that case STILL comes out False, because peak # shaving asked for MORE than the loop and the magnitude term rejects it # anyway. The winner term is masked. This row isolates it - a peak claim # smaller than the loop's ask and pointing the same way, so `winner` is the # only term that can decide - and it is what actually turns red if the gate # is widened without anyone revisiting the hazard. ("...and with peak BELOW the loop's ask, only the winner term separates them", [Claim.set("loop", P_LOOP, 1800), Claim.set("peak", P_PEAK, 900, "shaving"), MAXW], 1800, False), ] for name, claims, asked, expect in TABLE: r, got = following_from(claims, asked) check(f"{name} -> following={expect}", got is expect) if got is not expect: print(f" resolution was {r.explain()!r}") # And the gate is never True when the loop did not win, whatever else is going on. check("no configuration above arms the gate without the loop winning", all(not following_from(c, a)[1] for _, c, a, _ in TABLE if following_from(c, a)[0].winner != "loop")) print() print("end to end: the trip is COMMAND ZERO, not alarm-only") # ⚠️ THE FAULT INJECTED HERE IS A NEGATIVE GAIN, and that is not a contrivance - # it is the minimal faithful model of what this watchdog is for. The check # compares the loop's OUTPUT DIRECTION against the INPUT STATE it was regulating, # so what it detects is a loop whose two ends have become decoupled in sign: a # gain, a claim or a strategy with its polarity inverted, or an integrator so far # wound up that unwinding takes minutes. (Illusmart's was the last one: it # reached -14 768 W against a +-5000 W rail while writing once every 20-60 s, and # that arithmetic alone is 5-15 minutes of wrong-direction command.) # # `gain` is float(0.05,1.0) in the schema, so this is not a reachable # configuration - it is a stand-in for a code regression, which is the case a # unit test can actually hold the line on. import asyncio # noqa: E402 from app.main import Controller # noqa: E402 # ⚠️ DEV-02. THE KEY SPELLING IS IMPORTED, NEVER TRANSCRIBED. `sign_tripped` and # `sign_reason` are now NAMESPACED PER INVERTER - flat, a trip caused by B # latched A across every restart and one operator press cleared both. A check # that spelled `"unit/inverter-a/sign_tripped"` by hand here would go on passing # against a main.py that had started writing somewhere else, which is this # repository's own named failure mode: the copy that drifts is always the # grading one. `unit_key()` is the one definition and `Inverter.key()` calls it. from app.inverter import UNIT_A # noqa: E402 from app.store import unit_key # noqa: E402 TRIPPED_KEY = unit_key(UNIT_A, "sign_tripped") REASON_KEY = unit_key(UNIT_A, "sign_reason") class _Hass: def __init__(self): self.writes = [] self.calls = [] # ⚠️ ADDED FOR SAFETY-03, AND THE REASON IS WORTH READING. The e2e below # runs a DELIBERATELY INVERTED loop (gain -0.6) into a steady 2000 W import, # which is a loop that winds straight into the rail - so the output-clamp # alarm fires here, and the stub not having `call` is how that was # discovered. This scenario is now a second, independent witness that the # clamp alarm sees a real winding loop; test_clamp.py asserts it explicitly. async def call(self, domain, service, data): self.calls.append((domain, service, data)) async def limits(self, entity): return (-5000.0, 5000.0, 1.0) async def number(self, entity, invert=False): # A house importing 2000 W, steadily, whatever the battery does. return {"sensor.grid": 2000.0, "sensor.soc": 50.0, "sensor.batt": 0.0}.get(entity) async def set_number(self, entity, value): self.writes.append(float(value)) return True class _S: data = {} def set(self, *a): pass def get_time(self, *a): return None class _P: enabled = False def publish(self, *a, **k): pass def close(self): pass async def _e2e(): hass = _Hass() opts = {"setpoint_entity": "number.sp", "meter_entity": "sensor.grid", "soc_entity": "sensor.soc", "batt_entity": "sensor.batt", "meter_source": "off", "max_w": 2000, "gain": -0.6, "sign_disagree_w": 300, "sign_disagree_s": 1, "auto_start": True} c = Controller(opts, hass, _S(), _P(), stop=asyncio.Event()) c.auto = True c.dev_min, c.dev_max = -5000.0, 5000.0 seen_wrong_way = False for _ in range(12): await c.read_inputs() await c.cycle() if c.target < -300 and not c.sign_wd.tripped: seen_wrong_way = True await asyncio.sleep(0.25) return c, hass, seen_wrong_way ctrl, hass, wrong_way = asyncio.run(_e2e()) check("the broken loop really did command charge into an import first", wrong_way) check("the watchdog tripped", ctrl.sign_wd.tripped) check("the commanded value is 0 W", ctrl.target == 0.0) check("0 W actually reached the device", 0.0 in hass.writes) check("and it got there through the failsafe claim, not by accident", "failsafe" in ctrl.reason) print(f" status line: {ctrl.reason!r}") check("the status page says so in words", any("SIGN-DISAGREEMENT FAILSAFE" in c["text"] for c in ctrl.checks())) check("the banner leads with it", "FAILSAFE" in ctrl.status_payload()["banner"]) check("it stays at 0 W on later cycles too", hass.writes[-1] == 0.0 and hass.writes.count(0.0) >= 1) # ⚠️ THIS USED TO BE `check("MQTT reports the failsafe state", ctrl.sign_wd.tripped)` # which restates the assertion three lines above and never touches the payload. # The `status: "failsafe"` value was shipped untested. Capture what publish() # actually hands the broker. published = {} ctrl.mqtt = type("P", (), {"enabled": True, "publish": lambda _s, v: published.update(v), "close": lambda _s: None})() ctrl.publish() check("MQTT's controller-status sensor reports 'failsafe'", published.get("status") == "failsafe") ctrl.sign_wd.reset() ctrl.publish() check("...and goes back to 'running' once cleared", published.get("status") == "running") print() print("the latch survives a restart - it is the only thing between a tripped") print("controller and the first thing an operator will try") # ⚠️ THE BLOCKER THIS FILE MISSED FIRST TIME ROUND. `auto` is persisted and # `tripped` was not, so reconstructing a Controller against the same Store - # which is exactly what a container restart does - came back armed, into the # fault, with no operator involved: tripped=False, banner NOT READY, and it # resumed commanding -1000 then -2000 W before re-tripping 180 s later. class _MemStore: """A Store that keeps its dict across "restarts", like the real file does.""" def __init__(self): self.data = {"phase": "idle", "sign_tripped": False, "sign_reason": ""} def set(self, k, v): self.data[k] = v def get_time(self, *a): return None async def _restart(): shared = _MemStore() hass1 = _Hass() opts = {"setpoint_entity": "number.sp", "meter_entity": "sensor.grid", "soc_entity": "sensor.soc", "batt_entity": "sensor.batt", "meter_source": "off", "max_w": 2000, "gain": -0.6, "sign_disagree_w": 300, "sign_disagree_s": 1, "auto_start": True} a = Controller(opts, hass1, shared, _P(), stop=asyncio.Event()) a.auto = True a.dev_min, a.dev_max = -5000.0, 5000.0 for _ in range(12): await a.read_inputs() await a.cycle() await asyncio.sleep(0.25) # The restart. Same options, same store, brand new Controller and Hass. hass2 = _Hass() b = Controller(opts, hass2, shared, _P(), stop=asyncio.Event()) b.auto = True b.dev_min, b.dev_max = -5000.0, 5000.0 for _ in range(6): await b.read_inputs() await b.cycle() await asyncio.sleep(0.05) return a, b, shared, hass2 pre, post, store, hass2 = asyncio.run(_restart()) check("it tripped before the restart", pre.sign_wd.tripped) check("the trip reached the store, UNDER THE TRIPPED INVERTER'S OWN KEY", store.data[TRIPPED_KEY] is True) check("...with the reason beside it, so the operator is told why", "SIGN_DISAGREEMENT" in store.data[REASON_KEY]) check("⚠️ ...AND THE FLAT PRE-DEV-02 KEY IS MIRRORED, AS THE ROLLBACK VIEW. " "This assertion used to read the other way - that the flat key was NOT " "written - on the argument that writing it would latch every other " "inverter on the site. That argument was right about the READ and wrong " "about the WRITE: nothing at runtime reads the flat key, and the one " "thing that does (`_grade_store_migration()`, on every start) ignores it " "whenever ANY namespaced latch is set, which a sibling's trip always " "leaves. What the absence actually cost was the ROLLBACK: store.py's " "rule 2 keeps this key so a downgrade finds the latch, and a trip taken " "after the migration wrote nothing here - so an operator who downgraded " "came up DRIVING. `test_inverter.py` part 3 is where 'B's trip does not " "latch A across a restart' is graded, on a real roster of two", store.data.get("sign_tripped") is True and "SIGN_DISAGREEMENT" in str(store.data.get("sign_reason", ""))) check("the new Controller comes back TRIPPED", post.sign_wd.tripped) check("...and says why, restored rather than re-detected", "SIGN_DISAGREEMENT" in post.sign_wd.reason) check("...commands 0 W from its very first cycle", post.target == 0.0) check("...and never commanded anything else after the restart", hass2.writes != [] and set(hass2.writes) == {0.0}) check("the banner still leads with the failsafe", "FAILSAFE" in post.status_payload()["banner"]) # ⚠️ TWO PRESSES, and the first one must NOT clear it. `auto` is True while # tripped, so the first press stops control; only the transition back to True # clears. That is the documented behaviour and it is worth an assertion, because # a one-press clear would be reachable by an operator who never read the banner. asyncio.run(post.handle_action("auto_toggle")) # -> stopped check("pressing stop does not clear the failsafe", post.sign_wd.tripped and store.data[TRIPPED_KEY] is True) asyncio.run(post.handle_action("auto_toggle")) # -> started check("pressing start DOES clear it, and clears the store with it - BOTH " "spellings, which is what stops the every-start latch guard re-arming a " "failsafe the operator has already dealt with", not post.sign_wd.tripped and store.data[TRIPPED_KEY] is False and store.data[REASON_KEY] == "" and store.data.get("sign_tripped") is False) # One more restart from the cleared store, to prove the clear persisted too. fresh = Controller({"setpoint_entity": "number.sp", "meter_source": "off"}, _Hass(), store, _P(), stop=asyncio.Event()) check("...so the next restart comes back armed and untripped, as it should", not fresh.sign_wd.tripped) # ⚠️ THE WRITE ORDERING IS AN INVARIANT, SO IT GETS AN ASSERTION. main.py writes # `sign_reason` and THEN `sign_tripped`, each one its own `save()`, so that a # process that dies between the two leaves a reason nothing reads - harmless - # rather than a latch with no explanation for the operator who has to decide # whether to clear it. Documented reasoning is not enforcement: swapping those two # lines used to pass every check in this file. # # The death is modelled as the second write raising. In the field it is a power # cut or a container kill (Store.save() swallows OSError rather than raising), so # what is asserted here is the PROPERTY - what survives on disk - not the # mechanism. class _CrashOnFlag(_MemStore): def set(self, k, v): if k == TRIPPED_KEY and v: raise RuntimeError("power cut, mid-trip") super().set(k, v) async def _crash_mid_write(): store = _CrashOnFlag() hass = _Hass() opts = {"setpoint_entity": "number.sp", "meter_entity": "sensor.grid", "soc_entity": "sensor.soc", "batt_entity": "sensor.batt", "meter_source": "off", "max_w": 2000, "gain": -0.6, "sign_disagree_w": 300, "sign_disagree_s": 1, "auto_start": True} c = Controller(opts, hass, store, _P(), stop=asyncio.Event()) c.auto = True c.dev_min, c.dev_max = -5000.0, 5000.0 died = False for _ in range(12): try: await c.read_inputs() await c.cycle() except RuntimeError: died = True break await asyncio.sleep(0.25) return store, died crashed_store, died = asyncio.run(_crash_mid_write()) check("the second write is the one that sets the flag", died) check("a crash between the two writes leaves the REASON on disk", "SIGN_DISAGREEMENT" in crashed_store.data.get(REASON_KEY, "")) check("...and never a latch with nothing to explain it", not crashed_store.data.get(TRIPPED_KEY)) print() if fails: print(f"{len(fails)} FAILED: {', '.join(fails)}") sys.exit(1) print("all checks passed")