SAFETY-61: the control task's death ends the process, after the wind-down

A done-callback on run_control()'s task logs the traceback at the death and
sets `stop`; amain() runs the ordinary wind-down (shutdown() ->
command_site_zero()) and main() exits 70 (EX_SOFTWARE). Normal stops and
run_control()'s refusal path are ignored by the callback (still 0 / 78).

hass.py: a malformed JSON body, a JSON non-object, and an undecodable error
body are an absence / a failed write, not an exception into the loop.

checks() gains a first "control loop" row, red past FIRMWARE_INGEST_VETO_S.

Why exit rather than restart in-process, and what the Supervisor actually
does on exit (restarts only with the per-install Watchdog toggle on, default
off): DECISIONS/SAFETY-61-control-task-death.md.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BwRLw7eKYj2byVdwB6kbjU

goodwe_controller/app/hass.py

@@ -61,10 +61,17 @@ class HomeAssistant:
                 if resp.status == 404:
                     return None
                 resp.raise_for_status()
-                return await resp.json()
-        except (aiohttp.ClientError, TimeoutError) as err:
+                obj = await resp.json()
+        # SAFETY-61: an undecodable body is an absence like any other failed
+        # read, never an exception into the control loop.
+        except (aiohttp.ClientError, TimeoutError, ValueError) as err:
             self._moan(f"read:{entity_id}", "read %s failed: %s", entity_id, err)
             return None
+        if not isinstance(obj, dict):
+            self._moan(f"shape:{entity_id}", "read %s: not a state object: %.80r",
+                       entity_id, obj)
+            return None
+        return obj
 
     async def number(self, entity_id: str, invert: bool = False) -> float | None:
         """Numeric state, or None. Never substitutes a default."""
@@ -117,7 +124,8 @@ class HomeAssistant:
                                domain, service, resp.status, body[:200])
                     return False
                 return True
-        except (aiohttp.ClientError, TimeoutError) as err:
+        # ValueError: an undecodable error body (SAFETY-61) is a failed write.
+        except (aiohttp.ClientError, TimeoutError, ValueError) as err:
             self._moan(f"svcerr:{domain}.{service}", "service %s.%s failed: %s",
                        domain, service, err)
             return False

goodwe_controller/app/main.py

@@ -25,9 +25,11 @@ battery to zero on its own. Never "hold the last value to be safe".
 
 import asyncio
 import contextlib
+import functools
 import json
 import logging
 import signal
+import time
 from datetime import datetime, timedelta, timezone
 
 import aiohttp
@@ -74,8 +76,8 @@ from .price_history import PriceHistory
 from .static_tariff import resolve_tariff, static_status
 from .tariff import PeakStore, PeakTracker
 from .store import migrate_maint_keys, migrate_unit_keys
-from .validate import (ConfigRefused, refuse, validate_startup_config,
-                       validate_unit_ranges)
+from .validate import (FIRMWARE_INGEST_VETO_S, ConfigRefused, refuse,
+                       validate_startup_config, validate_unit_ranges)
 from .watchdog import (
     AGAINST_METER, AGAINST_OWN_POWER, SIGN_OPTION_DEFAULTS,
     SignWatchdog, SignWatchdogConfig, loop_is_following,
@@ -985,6 +987,8 @@ class Controller:
         # the value a clamp bit is a value written to ONE device.
         self.loop_clamped: tuple[bool, str] = (False, "")
         self.stopping = False
+        # SAFETY-61. time.monotonic() at the end of the last run_control() pass.
+        self.loop_pass_mono = None
 
     # -- helpers -------------------------------------------------------------
     def log_event(self, msg: str) -> None:
@@ -4397,6 +4401,7 @@ class Controller:
             await self.send_heartbeats(asyncio.get_running_loop().time(),
                                        heartbeat)
             self.publish()
+            self.loop_pass_mono = time.monotonic()
             await asyncio.sleep(1)
 
     async def send_heartbeats(self, loop_now: float, heartbeat_s: float) -> None:
@@ -4596,6 +4601,22 @@ class Controller:
         """
         o = self.o
         out = []
+        # SAFETY-61. First, so a stalled loop is the headline: the heartbeat is
+        # sent from this loop, so past layer 1's veto the inverter is at 0 W.
+        if self.loop_pass_mono is None:
+            out.append({"ok": True, "warn": False,
+                        "text": "control loop: starting"})
+        else:
+            _pass_age = time.monotonic() - self.loop_pass_mono
+            if _pass_age > FIRMWARE_INGEST_VETO_S:
+                out.append({"ok": False, "warn": False,
+                            "text": f"control loop: NO PASS FOR {_pass_age:.0f} s"
+                                    f" (layer 1 zeroes the inverter after "
+                                    f"{FIRMWARE_INGEST_VETO_S:.0f} s without a "
+                                    f"heartbeat)"})
+            else:
+                out.append({"ok": True, "warn": False,
+                            "text": f"control loop: last pass {_pass_age:.0f} s ago"})
         if self.p1_enabled:
             age = self.p1.published_age_s
             if self.p1.stale:
@@ -5449,6 +5470,30 @@ class Controller:
         return None
 
 
+# SAFETY-61. Exit status when the control task dies; 70 is sysexits(3)
+# EX_SOFTWARE. See DECISIONS/SAFETY-61-control-task-death.md.
+EXIT_CONTROL_DIED = 70
+
+
+class ControlTaskDied(RuntimeError):
+    """The control task ended while nothing had asked it to stop."""
+
+
+def control_task_done(task: asyncio.Task, *, controller, stop: asyncio.Event,
+                      died: list) -> None:
+    """Done-callback on the control task: log its death now and wake amain()
+    to wind down and exit (SAFETY-61, see its DECISIONS record)."""
+    if stop.is_set() or controller.stopping:
+        return          # the normal wind-down, or run_control()'s refusal path
+    exc = None if task.cancelled() else task.exception()
+    _LOG.critical("CONTROL LOOP DIED (%s) - commanding 0 W and exiting with "
+                  "status %d", "cancelled" if task.cancelled() else
+                  (repr(exc) if exc else "returned"), EXIT_CONTROL_DIED,
+                  exc_info=exc)
+    died.append(exc)
+    stop.set()
+
+
 async def amain() -> None:
     opts = load_options()
     logging.basicConfig(
@@ -5546,7 +5591,13 @@ async def amain() -> None:
             with contextlib.suppress(NotImplementedError):
                 loop.add_signal_handler(sig, stop.set)
 
-        tasks = [asyncio.create_task(controller.run_control())]
+        # SAFETY-61: the control task's death ends the process, after the
+        # wind-down below. See DECISIONS/SAFETY-61-control-task-death.md.
+        died: list = []
+        control = asyncio.create_task(controller.run_control())
+        control.add_done_callback(functools.partial(
+            control_task_done, controller=controller, stop=stop, died=died))
+        tasks = [control]
 
         # OPT-04. The tariff refresh runs as its OWN long-lived task, for the
         # reason P1 ingestion does and then some: `resolve_tariff()` may open an
@@ -5581,10 +5632,14 @@ async def amain() -> None:
         for task in tasks:
             task.cancel()
         for task in tasks:
+            if task is control and died:
+                continue            # already logged by control_task_done()
             with contextlib.suppress(asyncio.CancelledError):
                 await task
         await runner.cleanup()
         _LOG.info("stopped")
+        if died:
+            raise ControlTaskDied(died[0])
         # The late-arriving-device refusal from run_control(). Re-raised only
         # now, so the wind-down above still ran and the inverter is at 0 W.
         if controller.refused:
@@ -5603,6 +5658,9 @@ def main() -> None:
         # EX_CONFIG, so even a script reading the exit status can tell a bad
         # config from a bug.
         raise SystemExit(78)
+    except ControlTaskDied:
+        # No second traceback: control_task_done() logged it at the death.
+        raise SystemExit(EXIT_CONTROL_DIED)
 
 
 if __name__ == "__main__":