goodwe-addon/goodwe_controller/test_watchdog.py

"""Runnable check for the sign-disagreement watchdog.  `python3 test_watchdog.py`

Every case here is either a fault this must catch or a piece of NORMAL behaviour
it must not mistake for one. The second list is the longer one on purpose: an
alarm that fires on ordinary operation is worse than no alarm, because the next
person switches it off.

The headline validation is not here - it is `sim/replay_sign.py` in the infra
repo, which drives this module over the two captured bus days. This file is the
table-driven half: boundaries, timers, the latch, and the arming gate.

⚠️ MUTATING THE MODULE THIS FILE GRADES? PURGE `__pycache__` FIRST - or run
`tools/mutation_run.sh`, which does. `python3 -B` alone does NOT save you: it
stops bytecode being WRITTEN, not a stale one being READ, and a size-preserving
edit reverted inside one second is invisible to CPython's staleness test. That
is TEST-15, and it has already turned one live mutation into a false survivor.
Read MUTATION.md before you report a survivor.
"""

import sys
from datetime import datetime, timedelta, timezone

from app.watchdog import (
    SignWatchdog, SignWatchdogConfig, loop_is_following,
)

fails = []
T0 = datetime(2026, 8, 20, 15, 0, 0, tzinfo=timezone.utc)


def check(name, cond):
    print(f"  {'ok  ' if cond else 'FAIL'} {name}")
    if not cond:
        fails.append(name)


def drive(samples, cfg=None, following=True):
    """Feed (offset_s, commanded_w, grid_w) tuples. Returns (wd, trip_offset)."""
    wd = SignWatchdog(cfg)
    trip_at = None
    for item in samples:
        offset, cmd, grid = item[0], item[1], item[2]
        f = item[3] if len(item) > 3 else following
        if wd.evaluate(cmd, grid, f, T0 + timedelta(seconds=offset)) and trip_at is None:
            trip_at = offset
    return wd, trip_at


def opposed(duration_s, step=5, cmd=-1000, grid=1500, following=True, start=0):
    return [(start + t, cmd, grid, following)
            for t in range(0, int(duration_s) + 1, step)]


print("the fault it exists to catch")
wd, at = drive(opposed(300))
check("charging into an importing house trips", wd.tripped)
check("and trips at 180 s, not before", at == 180)
check("the reason names both signals", "-1000" in wd.reason and "+1500" in wd.reason)
print(f"       e.g. {wd.reason!r}")

wd, at = drive(opposed(300, cmd=2000, grid=-2000))
check("discharging into an exporting house trips too", wd.tripped and at == 180)

print()
print("the boundaries")
wd, _ = drive(opposed(600, cmd=-300, grid=1500))
check("300 W exactly does NOT qualify (strictly greater)", not wd.tripped)
wd, _ = drive(opposed(600, cmd=-301, grid=1500))
check("301 W does", wd.tripped)
wd, _ = drive(opposed(600, cmd=-1000, grid=300))
check("a 300 W meter does not qualify either", not wd.tripped)
wd, _ = drive(opposed(175))
check("175 s of disagreement does not trip", not wd.tripped)
wd, _ = drive(opposed(180))
check("180 s does (>= hold_s)", wd.tripped)

print()
print("continuity - the rule that produced zero false positives on real data")
broken = opposed(175) + [(180, -1000, -1500)] + opposed(175, start=185)
wd, _ = drive(broken)
check("179 s, one agreeing sample, then 175 s more: no trip", not wd.tripped)
check("and the window really was reset", wd.disagree_since is not None)
long_enough = broken + opposed(200, start=365)
wd, _ = drive(long_enough)
check("but a full 180 s after the reset does trip", wd.tripped)

print()
print("things that are NOT faults")
# The load-step transient. Measured on the healthy capture: longest 30 s.
wd, _ = drive(opposed(30) + [(35 + t, 800, 900) for t in range(0, 300, 5)])
check("a 30 s load-step crossover does not trip", not wd.tripped)

# SoC 10 %, Standby, declining a 1350 W discharge. The battery does nothing, so
# the meter stays where it was and the command agrees with it.
wd, _ = drive([(t, 1350, 1350) for t in range(0, 900, 5)])
check("a battery refusing a discharge command cannot trip it", not wd.tripped)

# A frozen meter drives command and reading into agreement, not disagreement.
wd, _ = drive([(t, 800, 800) for t in range(0, 900, 5)])
check("a frozen meter cannot trip it", not wd.tripped)

# Maintenance grid charge: 2500 W of charge against an importing house, for
# hours. Opposed, both huge - and entirely correct.
wd, _ = drive(opposed(7200, cmd=-2500, grid=3000, following=False))
check("two hours of maintenance grid charge does not trip", not wd.tripped)
check("and the timer never even started", wd.disagree_since is None)

wd, _ = drive([(t, -1000, None) for t in range(0, 900, 5)])
check("a missing meter reading does not trip", not wd.tripped)
wd, _ = drive(opposed(175) + [(180, -1000, None)] + opposed(600, start=185))
check("a missing reading mid-window RESETS it rather than accumulating",
      wd.tripped and wd.disagree_since is not None)
wd, at = drive(opposed(175) + [(180, -1000, None)] + opposed(600, start=185))
check("...so the trip lands 180 s after the gap, not after 180 s total",
      at == 365)

wd, _ = drive([(t, 0.0, 3000) for t in range(0, 900, 5)])
check("a 0 W command is not a wrong-direction command", not wd.tripped)

print()
print("the latch")
wd, _ = drive(opposed(200))
check("tripped after the fault", wd.tripped)
# The trip commands 0 W, which removes the command side of the comparison.
for t in range(200, 900, 5):
    wd.evaluate(0.0, 3000, True, T0 + timedelta(seconds=t))
check("stays tripped once the command is zeroed", wd.tripped)
for t in range(900, 1500, 5):
    wd.evaluate(1000, 1000, True, T0 + timedelta(seconds=t))
check("stays tripped even once everything looks healthy again", wd.tripped)
wd.reset()
check("reset() is the only way out", not wd.tripped and wd.disagree_since is None)
check("and the reason is cleared with it", wd.reason == "")

print()
print("the thresholds are configuration, not constants")
cfg = SignWatchdogConfig(min_w=800.0, hold_s=60.0)
wd, at = drive(opposed(300, cmd=-1000, grid=1500), cfg=cfg)
check("a 60 s hold trips at 60 s", wd.tripped and at == 60)
wd, _ = drive(opposed(600, cmd=-700, grid=1500), cfg=cfg)
check("an 800 W gate ignores a 700 W command", not wd.tripped)

print()
print("the configuration that would disarm it is refused")
from app.validate import validate_startup_config  # noqa: E402

base = {"setpoint_entity": "number.sp", "meter_source": "off",
        "meter_phases": 1, "meter_max_age_s": 30}
p = validate_startup_config({**base, "max_w": 2000, "sign_disagree_w": 300},
                            -5000, 5000)
check("300 W under a 2000 W ceiling is accepted", p == [])
p = validate_startup_config({**base, "max_w": 2000, "sign_disagree_w": 2000},
                            -5000, 5000)
check("a threshold AT max_w is refused", len(p) == 1 and "never fire" in p[0])
# ⚠️ maintenance_charge_w is named here only to keep this case about the SIGN
# rule. MAINT-04 added a second rule against the same ceiling, and a 500 W
# max_w is also below the 2000 W shipped maintenance charge - so without this
# the dict is refused twice, for two correct but different reasons, and the
# `len(p) == 1` below would be measuring the interaction rather than this rule.
p = validate_startup_config({**base, "max_w": 500, "sign_disagree_w": 3000,
                             "maintenance_charge_w": 500}, -5000, 5000)
check("a threshold above max_w is refused", len(p) == 1 and "never fire" in p[0])
p = validate_startup_config({**base, "max_w": 2000, "sign_disagree_w": 1999},
                            -5000, 5000)
check("one watt under the ceiling is reachable, so accepted", p == [])

print()
print("the arming gate, derived from the REAL arbiter")
# ⚠️ THIS IS THE DECISION THE WHOLE TICKET TURNS ON, and until now it was
# asserted in three comment blocks and verified by nothing: every other case in
# this file passes `following` in as a literal. What has to hold is that the
# expression in main.py - loop won AND the emitted value points the way the loop
# asked AND is no larger - is never True for a resolution the loop did not
# actually get. So run real claims through the real resolve() and derive it.
from app.arbiter import (  # noqa: E402
    P_LOOP, P_MAINT_SHAPE, P_MAINTENANCE, P_PEAK, P_SAFETY, Claim, resolve,
)


def following_from(claims, asked):
    """THE SHIPPED gate, not a copy of it.

    ⚠️ This used to transcribe main.py's expression with a comment saying
    "keep the two in step", which is exactly the arrangement that does not keep
    two things in step. Measured: editing the shipped gate alone to
    `winner in ("loop", "peak")` left this suite - including the P_PEAK row below,
    whose entire job is to catch that edit - completely green, while the watchdog
    went blind for the duration of every peak event. Import it or the P_PEAK
    assertion is decoration.
    """
    r = resolve(claims)
    return r, loop_is_following(r, asked)


MAXW = Claim.limit("supervised", P_SAFETY, -2000, 2000, "max_w")
DEV = Claim.limit("device", P_SAFETY, -5000, 5000, "rating")

# name, claims, what the loop asked, expected `following`
TABLE = [
    ("the loop alone, unopposed",
     [Claim.set("loop", P_LOOP, 900)], 900, True),
    ("the loop bound by max_w - reduced, not redirected",
     [Claim.set("loop", P_LOOP, 4000), MAXW], 4000, True),
    ("charging, bound by max_w on the other side",
     [Claim.set("loop", P_LOOP, -4000), MAXW], -4000, True),
    ("maintenance owns the setpoint outright",
     [Claim.set("loop", P_LOOP, 900),
      Claim.set("maintenance", P_MAINTENANCE, -2500, "charge"), MAXW], 900, False),
    ("maintenance charge-only clamps the loop to 0 W",
     [Claim.set("loop", P_LOOP, 900),
      Claim.limit("maintenance", P_MAINT_SHAPE, hi=0.0, reason="charge-only"),
      MAXW], 900, False),
    # ⚠️ THE CASE `winner == \"loop\"` ALONE WOULD HAVE GOT WRONG. 1500 W, not
    # 2500 W: at max_w 2000 a -2500 hi against a -2000 lo is an arbiter
    # contradiction, not a charge. That pair was the SHIPPED default until
    # MAINT-04 refused it and CFG-03 gave the default one source of truth; it is
    # kept below as a case rather than as a configuration.
    ("cheap-window charge REDIRECTS the loop while it still 'wins'",
     [Claim.set("loop", P_LOOP, 500),
      Claim.limit("maintenance", P_MAINT_SHAPE, hi=-1500, reason="cheap-window"),
      MAXW], 500, False),
    ("...and the contradiction when charge_w exceeds max_w is not following either",
     [Claim.set("loop", P_LOOP, 500),
      Claim.limit("maintenance", P_MAINT_SHAPE, hi=-2500, reason="cheap-window"),
      MAXW], 500, False),
    ("a lo limit RAISES the magnitude without flipping the sign",
     [Claim.set("loop", P_LOOP, 200),
      Claim.limit("floor", P_SAFETY, lo=800), MAXW], 200, False),
    ("a safety stop wins outright",
     [Claim.set("loop", P_LOOP, 900),
      Claim.set("safety", P_SAFETY, 0.0, "stopped"), MAXW], 900, False),
    ("no claims at all - the arbiter's own failsafe",
     [MAXW, DEV], None, False),
    ("the loop asked for zero",
     [Claim.set("loop", P_LOOP, 0.0), MAXW], 0.0, False),
    # The forward hazard, asserted as it stands TODAY rather than as it should
    # be. When peak shaving lands, these expectations are what have to change.
    ("peak shaving at P_PEAK disarms the gate (loop_is_following's forward hazard)",
     [Claim.set("loop", P_LOOP, 900),
      Claim.set("peak", P_PEAK, 1800, "shaving"), MAXW], 900, False),
    # ⚠️ AND THE ROW ABOVE CANNOT DETECT A WIDENED GATE ON ITS OWN, which was
    # found by trying it: with the shipped gate edited to
    # `winner in ("loop", "peak")` that case STILL comes out False, because peak
    # shaving asked for MORE than the loop and the magnitude term rejects it
    # anyway. The winner term is masked. This row isolates it - a peak claim
    # smaller than the loop's ask and pointing the same way, so `winner` is the
    # only term that can decide - and it is what actually turns red if the gate
    # is widened without anyone revisiting the hazard.
    ("...and with peak BELOW the loop's ask, only the winner term separates them",
     [Claim.set("loop", P_LOOP, 1800),
      Claim.set("peak", P_PEAK, 900, "shaving"), MAXW], 1800, False),
]
for name, claims, asked, expect in TABLE:
    r, got = following_from(claims, asked)
    check(f"{name} -> following={expect}", got is expect)
    if got is not expect:
        print(f"       resolution was {r.explain()!r}")

# And the gate is never True when the loop did not win, whatever else is going on.
check("no configuration above arms the gate without the loop winning",
      all(not following_from(c, a)[1]
          for _, c, a, _ in TABLE if following_from(c, a)[0].winner != "loop"))

print()
print("end to end: the trip is COMMAND ZERO, not alarm-only")
# ⚠️ THE FAULT INJECTED HERE IS A NEGATIVE GAIN, and that is not a contrivance -
# it is the minimal faithful model of what this watchdog is for. The check
# compares the loop's OUTPUT DIRECTION against the INPUT STATE it was regulating,
# so what it detects is a loop whose two ends have become decoupled in sign: a
# gain, a claim or a strategy with its polarity inverted, or an integrator so far
# wound up that unwinding takes minutes. (Illusmart's was the last one: it
# reached -14 768 W against a +-5000 W rail while writing once every 20-60 s, and
# that arithmetic alone is 5-15 minutes of wrong-direction command.)
#
# `gain` is float(0.05,1.0) in the schema, so this is not a reachable
# configuration - it is a stand-in for a code regression, which is the case a
# unit test can actually hold the line on.
import asyncio  # noqa: E402

from app.main import Controller  # noqa: E402
# ⚠️ DEV-02. THE KEY SPELLING IS IMPORTED, NEVER TRANSCRIBED. `sign_tripped` and
# `sign_reason` are now NAMESPACED PER INVERTER - flat, a trip caused by B
# latched A across every restart and one operator press cleared both. A check
# that spelled `"unit/inverter-a/sign_tripped"` by hand here would go on passing
# against a main.py that had started writing somewhere else, which is this
# repository's own named failure mode: the copy that drifts is always the
# grading one. `unit_key()` is the one definition and `Inverter.key()` calls it.
from app.inverter import UNIT_A  # noqa: E402
from app.store import unit_key  # noqa: E402

TRIPPED_KEY = unit_key(UNIT_A, "sign_tripped")
REASON_KEY = unit_key(UNIT_A, "sign_reason")


class _Hass:
    def __init__(self):
        self.writes = []
        self.calls = []

    # ⚠️ ADDED FOR SAFETY-03, AND THE REASON IS WORTH READING. The e2e below
    # runs a DELIBERATELY INVERTED loop (gain -0.6) into a steady 2000 W import,
    # which is a loop that winds straight into the rail - so the output-clamp
    # alarm fires here, and the stub not having `call` is how that was
    # discovered. This scenario is now a second, independent witness that the
    # clamp alarm sees a real winding loop; test_clamp.py asserts it explicitly.
    async def call(self, domain, service, data):
        self.calls.append((domain, service, data))

    async def limits(self, entity):
        return (-5000.0, 5000.0, 1.0)

    async def number(self, entity, invert=False):
        # A house importing 2000 W, steadily, whatever the battery does.
        return {"sensor.grid": 2000.0, "sensor.soc": 50.0,
                "sensor.batt": 0.0}.get(entity)

    async def set_number(self, entity, value):
        self.writes.append(float(value))
        return True


class _S:
    data = {}

    def set(self, *a):
        pass

    def get_time(self, *a):
        return None


class _P:
    enabled = False

    def publish(self, *a, **k):
        pass

    def close(self):
        pass


async def _e2e():
    hass = _Hass()
    opts = {"setpoint_entity": "number.sp", "meter_entity": "sensor.grid",
            "soc_entity": "sensor.soc", "batt_entity": "sensor.batt",
            "meter_source": "off", "max_w": 2000, "gain": -0.6,
            "sign_disagree_w": 300, "sign_disagree_s": 1, "auto_start": True}
    c = Controller(opts, hass, _S(), _P(), stop=asyncio.Event())
    c.auto = True
    c.dev_min, c.dev_max = -5000.0, 5000.0
    seen_wrong_way = False
    for _ in range(12):
        await c.read_inputs()
        await c.cycle()
        if c.target < -300 and not c.sign_wd.tripped:
            seen_wrong_way = True
        await asyncio.sleep(0.25)
    return c, hass, seen_wrong_way


ctrl, hass, wrong_way = asyncio.run(_e2e())
check("the broken loop really did command charge into an import first", wrong_way)
check("the watchdog tripped", ctrl.sign_wd.tripped)
check("the commanded value is 0 W", ctrl.target == 0.0)
check("0 W actually reached the device", 0.0 in hass.writes)
check("and it got there through the failsafe claim, not by accident",
      "failsafe" in ctrl.reason)
print(f"       status line: {ctrl.reason!r}")
check("the status page says so in words",
      any("SIGN-DISAGREEMENT FAILSAFE" in c["text"] for c in ctrl.checks()))
check("the banner leads with it",
      "FAILSAFE" in ctrl.status_payload()["banner"])
check("it stays at 0 W on later cycles too",
      hass.writes[-1] == 0.0 and hass.writes.count(0.0) >= 1)

# ⚠️ THIS USED TO BE `check("MQTT reports the failsafe state", ctrl.sign_wd.tripped)`
# which restates the assertion three lines above and never touches the payload.
# The `status: "failsafe"` value was shipped untested. Capture what publish()
# actually hands the broker.
published = {}
ctrl.mqtt = type("P", (), {"enabled": True,
                           "publish": lambda _s, v: published.update(v),
                           "close": lambda _s: None})()
ctrl.publish()
check("MQTT's controller-status sensor reports 'failsafe'",
      published.get("status") == "failsafe")
ctrl.sign_wd.reset()
ctrl.publish()
check("...and goes back to 'running' once cleared",
      published.get("status") == "running")


print()
print("the latch survives a restart - it is the only thing between a tripped")
print("controller and the first thing an operator will try")
# ⚠️ THE BLOCKER THIS FILE MISSED FIRST TIME ROUND. `auto` is persisted and
# `tripped` was not, so reconstructing a Controller against the same Store -
# which is exactly what a container restart does - came back armed, into the
# fault, with no operator involved: tripped=False, banner NOT READY, and it
# resumed commanding -1000 then -2000 W before re-tripping 180 s later.


class _MemStore:
    """A Store that keeps its dict across "restarts", like the real file does."""

    def __init__(self):
        self.data = {"phase": "idle", "sign_tripped": False, "sign_reason": ""}

    def set(self, k, v):
        self.data[k] = v

    def get_time(self, *a):
        return None


async def _restart():
    shared = _MemStore()
    hass1 = _Hass()
    opts = {"setpoint_entity": "number.sp", "meter_entity": "sensor.grid",
            "soc_entity": "sensor.soc", "batt_entity": "sensor.batt",
            "meter_source": "off", "max_w": 2000, "gain": -0.6,
            "sign_disagree_w": 300, "sign_disagree_s": 1, "auto_start": True}
    a = Controller(opts, hass1, shared, _P(), stop=asyncio.Event())
    a.auto = True
    a.dev_min, a.dev_max = -5000.0, 5000.0
    for _ in range(12):
        await a.read_inputs()
        await a.cycle()
        await asyncio.sleep(0.25)

    # The restart. Same options, same store, brand new Controller and Hass.
    hass2 = _Hass()
    b = Controller(opts, hass2, shared, _P(), stop=asyncio.Event())
    b.auto = True
    b.dev_min, b.dev_max = -5000.0, 5000.0
    for _ in range(6):
        await b.read_inputs()
        await b.cycle()
        await asyncio.sleep(0.05)
    return a, b, shared, hass2


pre, post, store, hass2 = asyncio.run(_restart())
check("it tripped before the restart", pre.sign_wd.tripped)
check("the trip reached the store, UNDER THE TRIPPED INVERTER'S OWN KEY",
      store.data[TRIPPED_KEY] is True)
check("...with the reason beside it, so the operator is told why",
      "SIGN_DISAGREEMENT" in store.data[REASON_KEY])
check("⚠️ ...AND THE FLAT PRE-DEV-02 KEY IS MIRRORED, AS THE ROLLBACK VIEW. "
      "This assertion used to read the other way - that the flat key was NOT "
      "written - on the argument that writing it would latch every other "
      "inverter on the site. That argument was right about the READ and wrong "
      "about the WRITE: nothing at runtime reads the flat key, and the one "
      "thing that does (`_grade_store_migration()`, on every start) ignores it "
      "whenever ANY namespaced latch is set, which a sibling's trip always "
      "leaves. What the absence actually cost was the ROLLBACK: store.py's "
      "rule 2 keeps this key so a downgrade finds the latch, and a trip taken "
      "after the migration wrote nothing here - so an operator who downgraded "
      "came up DRIVING. `test_inverter.py` part 3 is where 'B's trip does not "
      "latch A across a restart' is graded, on a real roster of two",
      store.data.get("sign_tripped") is True
      and "SIGN_DISAGREEMENT" in str(store.data.get("sign_reason", "")))
check("the new Controller comes back TRIPPED", post.sign_wd.tripped)
check("...and says why, restored rather than re-detected",
      "SIGN_DISAGREEMENT" in post.sign_wd.reason)
check("...commands 0 W from its very first cycle", post.target == 0.0)
check("...and never commanded anything else after the restart",
      hass2.writes != [] and set(hass2.writes) == {0.0})
check("the banner still leads with the failsafe",
      "FAILSAFE" in post.status_payload()["banner"])
# ⚠️ TWO PRESSES, and the first one must NOT clear it. `auto` is True while
# tripped, so the first press stops control; only the transition back to True
# clears. That is the documented behaviour and it is worth an assertion, because
# a one-press clear would be reachable by an operator who never read the banner.
asyncio.run(post.handle_action("auto_toggle"))          # -> stopped
check("pressing stop does not clear the failsafe",
      post.sign_wd.tripped and store.data[TRIPPED_KEY] is True)
asyncio.run(post.handle_action("auto_toggle"))          # -> started
check("pressing start DOES clear it, and clears the store with it - BOTH "
      "spellings, which is what stops the every-start latch guard re-arming a "
      "failsafe the operator has already dealt with",
      not post.sign_wd.tripped and store.data[TRIPPED_KEY] is False
      and store.data[REASON_KEY] == ""
      and store.data.get("sign_tripped") is False)
# One more restart from the cleared store, to prove the clear persisted too.
fresh = Controller({"setpoint_entity": "number.sp", "meter_source": "off"},
                   _Hass(), store, _P(), stop=asyncio.Event())
check("...so the next restart comes back armed and untripped, as it should",
      not fresh.sign_wd.tripped)

# ⚠️ THE WRITE ORDERING IS AN INVARIANT, SO IT GETS AN ASSERTION. main.py writes
# `sign_reason` and THEN `sign_tripped`, each one its own `save()`, so that a
# process that dies between the two leaves a reason nothing reads - harmless -
# rather than a latch with no explanation for the operator who has to decide
# whether to clear it. Documented reasoning is not enforcement: swapping those two
# lines used to pass every check in this file.
#
# The death is modelled as the second write raising. In the field it is a power
# cut or a container kill (Store.save() swallows OSError rather than raising), so
# what is asserted here is the PROPERTY - what survives on disk - not the
# mechanism.


class _CrashOnFlag(_MemStore):
    def set(self, k, v):
        if k == TRIPPED_KEY and v:
            raise RuntimeError("power cut, mid-trip")
        super().set(k, v)


async def _crash_mid_write():
    store = _CrashOnFlag()
    hass = _Hass()
    opts = {"setpoint_entity": "number.sp", "meter_entity": "sensor.grid",
            "soc_entity": "sensor.soc", "batt_entity": "sensor.batt",
            "meter_source": "off", "max_w": 2000, "gain": -0.6,
            "sign_disagree_w": 300, "sign_disagree_s": 1, "auto_start": True}
    c = Controller(opts, hass, store, _P(), stop=asyncio.Event())
    c.auto = True
    c.dev_min, c.dev_max = -5000.0, 5000.0
    died = False
    for _ in range(12):
        try:
            await c.read_inputs()
            await c.cycle()
        except RuntimeError:
            died = True
            break
        await asyncio.sleep(0.25)
    return store, died


crashed_store, died = asyncio.run(_crash_mid_write())
check("the second write is the one that sets the flag", died)
check("a crash between the two writes leaves the REASON on disk",
      "SIGN_DISAGREEMENT" in crashed_store.data.get(REASON_KEY, ""))
check("...and never a latch with nothing to explain it",
      not crashed_store.data.get(TRIPPED_KEY))

print()
if fails:
    print(f"{len(fails)} FAILED: {', '.join(fails)}")
    sys.exit(1)
print("all checks passed")