SAFETY-34: re-assert latched alarms, because a Core restart keeps the add-on

A Core-only restart wipes persistent_notification and does NOT restart the
add-on (measured on TEST-23: the add-on's SAFETY-31 card was gone 4 min after
`docker restart test23-ha`, add-on StartedAt unchanged). The latched SAFETY-31
and SAFETY-42 cards were re-raised only on add-on start, so they stayed gone.

They are now re-asserted every REANNOUNCE_S = 300 s while tripped (same
notification_id, so it replaces). test_watchdog part 6 and test_delivery part
8b grade it with the same process across the "restart"; red on the old code.
The docstrings that said the add-on restarts with HA are corrected.
Record: goodwe docs/DECISIONS.md, SAFETY-34 D4.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LyR88rXETvwhTgx6yHE39p

.gitea/infra-pin

@@ -1,2 +1,2 @@
 # The admin/goodwe commit CI tests against. Bump: set to the full SHA of admin/goodwe release/1.0 HEAD (README.md, "CI and the infra pin").
-2d026dfa7c425f462c6969b2f7cdfddc17b17a81
+a1d0618eb339b1d4841cd31d6399b67a22b9dfee

.gitea/workflows/checks.yml

@@ -286,8 +286,9 @@ jobs:
         run: |
           set -eu
           rm -rf "$WS" && mkdir -p "$WS/goodwe-addon" && cd "$WS/goodwe-addon"
-          git init -q .
+          git init -q . && git remote add origin "$URL"
-          git remote add origin "$(echo "$URL" | sed "s#://#://x-access-token:${TOKEN}@#")"
+          # Token as an env-set header: not in argv or .git/config (TOOL-9).
+          export GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=http.extraHeader GIT_CONFIG_VALUE_0="Authorization: token $TOKEN"
           if [ -n "$PRNUM" ]; then
             # Two fetches, not one, so that a failure to reach either ref fails
             # HERE with the ref in the message, rather than inside the merge
@@ -498,7 +499,8 @@ jobs:
           TOKEN: ${{ secrets.INFRA_READ_TOKEN }}
         run: |
           set -eu
-          PIN=$(grep -v '^#' "$WS/goodwe-addon/.gitea/infra-pin" | tr -d '[:space:]')
+          # `|| true`: under pipefail a missing or comment-only pin would skip the FAIL line (TOOL-9).
+          PIN=$(grep -v '^#' "$WS/goodwe-addon/.gitea/infra-pin" | tr -d '[:space:]') || true
           echo "$PIN" | grep -Eq '^[0-9a-f]{40}$' || { echo "FAIL: .gitea/infra-pin holds '$PIN', not a full 40-hex SHA."; exit 1; }
           test -n "$TOKEN" || { echo "FAIL: secret INFRA_READ_TOKEN is not available to this run (a fork PR?)."; exit 1; }
           rm -rf "$WS/goodwe" && mkdir -p "$WS/goodwe" && cd "$WS/goodwe"

.gitea/workflows/infra-pin.yml

@@ -23,16 +23,20 @@ jobs:
           INFRA_TOKEN: ${{ secrets.INFRA_READ_TOKEN }}   # admin/goodwe is private
         run: |
           set -eu
-          D=$(mktemp -d) && cd "$D"
+          D=$(mktemp -d) && trap 'cd / && rm -rf "$D"' EXIT && cd "$D"
-          git init -q . && git remote add origin "$(echo "$SELF" | sed "s#://#://x-access-token:${TOKEN}@#")"
+          git init -q .
           # The pin of the commit this event is about (the PR head, or the push).
-          git fetch -q --depth 1 origin "$GITHUB_SHA"
+          # Token as an env-set header: not in argv or .git/config (TOOL-9).
-          PIN=$(git show FETCH_HEAD:.gitea/infra-pin | grep -v '^#' | tr -d '[:space:]')
+          GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=http.extraHeader GIT_CONFIG_VALUE_0="Authorization: token $TOKEN" \
+            git fetch -q --depth 1 "$SELF" "$GITHUB_SHA" || { echo "FAIL: could not fetch $GITHUB_SHA here."; exit 1; }
+          # `|| true`: under the runner's pipefail a missing or comment-only pin
+          # would stop here on git's error instead of the FAIL line below (TOOL-9).
+          PIN=$(git show FETCH_HEAD:.gitea/infra-pin | grep -v '^#' | tr -d '[:space:]') || true
+          test -n "$PIN" || { echo "FAIL: .gitea/infra-pin is missing at $GITHUB_SHA, or holds only comments."; exit 1; }
           test -n "$INFRA_TOKEN" || { echo "FAIL: secret INFRA_READ_TOKEN is not available to this run (a fork PR?)."; exit 1; }
           # Header via env, as in checks.yml: the token stays out of argv.
           HEAD=$(GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=http.extraHeader GIT_CONFIG_VALUE_0="Authorization: token $INFRA_TOKEN" \
-            git ls-remote "$INFRA" refs/heads/release/1.0 | cut -f1)
+            git ls-remote "$INFRA" refs/heads/release/1.0 | cut -f1) || true
-          cd / && rm -rf "$D"
           test -n "$HEAD" || { echo "FAIL: could not read admin/goodwe release/1.0."; exit 1; }
           if [ "$PIN" != "$HEAD" ]; then
             echo "STALE: .gitea/infra-pin is $PIN but admin/goodwe release/1.0 is $HEAD."

goodwe_controller/app/main.py

@@ -113,6 +113,12 @@ __all__ = ["UNIT_A", "Controller", "amain", "main"]
 # the re-issue happens by itself) than hold the control loop for ten seconds.
 WRITE_BUDGET_S = 4.0
 
+# SAFETY-34: a Core-only restart wipes persistent_notification and does NOT
+# restart this add-on (measured on TEST-23), so a latched alarm is re-asserted
+# this often. Why: goodwe docs/DECISIONS.md, SAFETY-34.
+REANNOUNCE_S = 300.0
+_mono = time.monotonic          # patched by the checks; asyncio keeps the real one
+
 # --------------------------------------------------------------------------- #
 # OPT-04: THE BACKGROUND TARIFF REFRESH, AND WHY IT IS NOT OPTIONAL
 #
@@ -691,6 +697,9 @@ class Controller:
         # does, so the latch alone is not the alarm - re-raising it is.
         # `run_control()` does that, once, before the first cycle.
         self._circ_announced = False
+        # SAFETY-34: when each latched alarm was last put in front of HA, by
+        # notification_id - see REANNOUNCE_S.
+        self._announced_mono = {}
         if self.circ_wd.tripped:
             _LOG.error(
                 "SAFETY-31 was TRIPPED before this restart and stays tripped: "
@@ -3943,15 +3952,17 @@ class Controller:
         The latch says "this site is tripped"; the flag says "this process has
         put that in front of a human". A restart clears the flag and keeps the
         latch, so the notification is RE-RAISED on the first cycle after every
-        add-on start - which is also the first cycle after a Home Assistant
+        add-on start. A Core-only restart does NOT restart the add-on, so it is
-        restart, because the add-on is restarted with it.
+        also re-asserted every REANNOUNCE_S (SAFETY-34).
 
         ⚠️ ONE CODE PATH FOR BOTH, ON PURPOSE. A fresh trip and a restart into
         an existing trip raise the alarm through these same lines. Two paths
         would be two chances for the restart one to rot unnoticed, and the
         restart one is the case nobody exercises by hand.
         """
-        if not self.circ_wd.tripped or self._circ_announced:
+        if not self.circ_wd.tripped:
+            return
+        if self._circ_announced and not self._reannounce_due("goodwe_circulating_power"):
             return
         # Set BEFORE the await. `hass.call()` can raise or hang, and a retry
         # every cycle for a broker that is down would be an alarm storm on the
@@ -3959,6 +3970,7 @@ class Controller:
         # already durable and the log line above already fired; this surface is
         # best-effort by design.
         self._circ_announced = True
+        self._announced_mono["goodwe_circulating_power"] = _mono()
         names = self.units
         pos, neg = opposing([inv.batt for inv in self.inverters],
                             min_w=self.circ_wd.cfg.min_w / 2.0)
@@ -3993,6 +4005,11 @@ class Controller:
                 f"know why two inverters were opposing."),
         })
 
+    def _reannounce_due(self, notification_id: str) -> bool:
+        """SAFETY-34: has this latched alarm gone REANNOUNCE_S without being re-asserted?"""
+        t = self._announced_mono.get(notification_id)
+        return t is None or _mono() - t >= REANNOUNCE_S
+
     async def check_not_delivering(self) -> None:
         """SAFETY-42 + SAFETY-34. The alarm, and it must survive a restart.
 
@@ -4012,8 +4029,8 @@ class Controller:
         not persisted. The latch says "this unit is tripped"; membership says
         "this process has put that in front of a human". A restart empties the
         set and keeps the latch, so the notification is RE-RAISED on the first
-        cycle after every add-on start - which is also the first cycle after a
+        cycle after every add-on start, and re-asserted every REANNOUNCE_S
-        Home Assistant restart, because the add-on is restarted with it.
+        because a Core-only restart does not restart the add-on (SAFETY-34).
 
         ⚠️ ONE CODE PATH FOR BOTH, ON PURPOSE - `check_circulating()`'s
         argument. A fresh trip and a restart into an existing trip raise the
@@ -4031,7 +4048,10 @@ class Controller:
         names = self.units
         slugs = unit_slugs(names)
         for inv, slug in zip(self.inverters, slugs):
-            if not inv.delivery_wd.tripped or inv.name in self._delivery_announced:
+            nid = f"goodwe_not_delivering_{slug}"
+            if not inv.delivery_wd.tripped:
+                continue
+            if inv.name in self._delivery_announced and not self._reannounce_due(nid):
                 continue
             # Added BEFORE the await, for `check_circulating()`'s reason:
             # `hass.call()` can raise or hang, and a retry every cycle for a
@@ -4039,9 +4059,10 @@ class Controller:
             # latch is already durable and the log line already fired; this
             # surface is best-effort by design.
             self._delivery_announced.add(inv.name)
+            self._announced_mono[nid] = _mono()
             peers = ", ".join(n for n in names if n != inv.name) or "no sibling"
             await self.hass.call("persistent_notification", "create", {
-                "notification_id": f"goodwe_not_delivering_{slug}",
+                "notification_id": nid,
                 "title": f"GoodWe: {inv.name} is not delivering its share",
                 "message": (
                     f"{inv.delivery_wd.reason}\n\nWHY NOTHING ELSE SAID SO. A "

goodwe_controller/test_ci_gate.py

@@ -62,10 +62,9 @@ the step it wants: if checks.yml is restructured so that the step cannot be
 found, this file goes RED rather than quietly grading nothing.
 
 ⚠️ NEEDS `git`, which is why .gitea/ci/Dockerfile installs it. No network: the
-fixtures are local repositories built in a temp directory, and the workflow's
+fixtures are local repositories built in a temp directory; the workflow's
-`git remote add origin "$(echo "$URL" | sed "s#://#://x-access-token:...@#")"`
+token header (TOOL-9) is ignored for a local path, which is what lets the
-is a no-op substitution on a path with no `://` in it, which is what lets the
+shipped lines be used verbatim.
-shipped line be used verbatim.
 
 ⚠️ This file writes only under a temporary directory. It must stay that way:
 CI mounts the checkout READ-ONLY.

goodwe_controller/test_delivery.py

@@ -1195,8 +1195,8 @@ async def _restart():
     c1 = _ctrl(h1, store=shared)
     await _drive(c1)
     # A NEW process, a NEW HomeAssistant object (so the in-memory notification
-    # is gone with it), the SAME durable file. That is what a Home Assistant
+    # is gone with it), the SAME durable file: an ADD-ON restart. A Core-only
-    # restart is from this add-on's point of view: the add-on goes with it.
+    # restart keeps the process; graded in part 8b (SAFETY-34).
     h2 = _DeliveryHass(b_soc=50.0)
     c2 = _ctrl(h2, store=shared)
     await c2.check_not_delivering()
@@ -1250,6 +1250,40 @@ check("the operator's START press is the only way out, and it clears the "
       and _rs.data.get(unit_key("inverter-b", "delivery_reason")) == ""
       and _r2._delivery_announced == set())
 
+print()
+print("8b. SAFETY-34: a Core-only restart - same add-on process, HA's memory wiped")
+import app.main as _main_mod  # noqa: E402
+
+
+async def _core_restart():
+    shared = _MemStore()
+    c = _ctrl(_DeliveryHass(b_soc=50.0), store=shared)
+    await _drive(c)
+    return c
+
+
+_c = asyncio.run(_core_restart())
+_real_mono, _clock = _main_mod._mono, [_main_mod._mono()]
+_main_mod._mono = lambda: _clock[0]
+try:
+    _c.hass = _DeliveryHass(b_soc=50.0)
+    asyncio.run(_c.check_not_delivering())
+    _early = [x for x in _c.hass.calls
+              if "not_delivering" in str(x[2].get("notification_id", ""))]
+    _clock[0] += _main_mod.REANNOUNCE_S
+    asyncio.run(_c.check_not_delivering())
+    _late = [x for x in _c.hass.calls
+             if "not_delivering" in str(x[2].get("notification_id", ""))]
+finally:
+    _main_mod._mono = _real_mono
+check("SAFETY-34: the latch tripped in this process",
+      _c.inverters[1].delivery_wd.tripped)
+check("SAFETY-34: inside REANNOUNCE_S the same process does not re-raise",
+      len(_early) == 0)
+check("SAFETY-34: after REANNOUNCE_S the SAME process re-asserts the card, "
+      "because a Core-only restart wiped it and did not restart the add-on",
+      len(_late) == 1 and "inverter_b" in _late[0][2]["notification_id"])
+
 
 print()
 if fails:

goodwe_controller/test_watchdog.py

@@ -1055,8 +1055,8 @@ async def _circ_restart():
         await a.read_inputs()
         await a.cycle()
         await asyncio.sleep(0.25)
-    # THE RESTART. Same store, brand new Controller and Hass - which is exactly
+    # THE RESTART. Same store, brand new Controller and Hass - an ADD-ON
-    # what a Home Assistant restart does to this add-on. The house has stopped
+    # restart. A Core-only restart keeps the process; graded below (SAFETY-34). The house has stopped
     # circulating in the meantime, so nothing in the new process's own
     # measurements could re-derive the trip.
     h2 = _CircHass(a_w=0.0, b_w=0.0)
@@ -1116,6 +1116,28 @@ check("...and it does not raise the same card on every cycle - a per-PROCESS "
       "this' a different question from 'is this site tripped'",
       len(_notes2) == 1)
 
+# SAFETY-34: a Core-ONLY restart. The add-on process lives on (measured on
+# TEST-23), HA's memory is wiped, so the SAME process must re-assert the card.
+import app.main as _main_mod  # noqa: E402
+_real_mono, _clock = _main_mod._mono, [_main_mod._mono()]
+_main_mod._mono = lambda: _clock[0]
+try:
+    _h3 = _CircHass(a_w=0.0, b_w=0.0)
+    _b.hass = _h3
+    asyncio.run(_b.check_circulating())
+    _early = [c for c in _h3.calls if c[0] == "persistent_notification"]
+    _clock[0] += _main_mod.REANNOUNCE_S
+    asyncio.run(_b.check_circulating())
+    _late = [c for c in _h3.calls if c[0] == "persistent_notification"]
+finally:
+    _main_mod._mono = _real_mono
+check("SAFETY-34: inside REANNOUNCE_S the same process does not re-raise - no "
+      "storm on the event bus", len(_early) == 0)
+check("SAFETY-34: a Core-only restart wipes the card and the add-on process "
+      "survives it, so after REANNOUNCE_S the SAME process re-asserts it",
+      len(_late) == 1
+      and _late[0][2]["notification_id"] == "goodwe_circulating_power")
+
 
 print()
 print("6a. AC 4: THE ALARM NAMES WHICH PACKS, AND WHICH WAY ROUND")