diff --git a/goodwe_controller/DECISIONS/SAFETY-123-hour-slices-step-in-utc.md b/goodwe_controller/DECISIONS/SAFETY-123-hour-slices-step-in-utc.md new file mode 100644 index 0000000..a3a8a18 --- /dev/null +++ b/goodwe_controller/DECISIONS/SAFETY-123-hour-slices-step-in-utc.md @@ -0,0 +1,35 @@ +# SAFETY-123 - the day plan's hour loop steps in UTC + +**Status: for review.** Tests: `test_safety123.py`. No option, no text change. + +`dayplan._hour_slices()` stepped to the next local hour by adding an hour to +the local wall clock and converting back. Where the zone has no tzdata +(`tariff._EUTimezone`: the CI image, a dev host, an image that loses tzdata) +the missing 02:00 of the spring change reads as summer time, which is +00:00 UTC again. At 28 March 2027 00:00 UTC `t` did not move and the list +grew until the process died. Control and the web page share one event loop, +so the whole add-on froze. `dayplan.plan()` calls it on every plan, before it +looks at a battery's mode, so a site with all batteries on Solar first reaches +it as well, if the prices give a plan. + +The shipped image has tzdata (`amd64-base:3.20`), where this zone gives the +right answer, so a running add-on is not known to be hit. The fix is for any +host without it. (AUDIT-01 F4, card SAFETY-120, is the same defect.) + +**Decision.** Each step adds the time left in the local hour to the UTC +instant. It is at least a microsecond, so `t` always advances and the loop +ends, in any zone, with or without tzdata. A day of 23 hours gives 23 slices +and a day of 25 hours gives 25, each at most an hour, none across a local +hour. On ordinary days the slices are the old ones. + +**One change on the autumn day.** The old function (with tzdata) made one +two-hour slice for the doubled hour 02:00-03:00; it now makes two of an hour, +both labelled 2. `plan()` sums per slice, so only a forecast with sun in that +hour could tell the difference. There is none. + +**Other loops.** The other code that walks local hours or days steps in UTC +or on the calendar date (`static_tariff`, `price_forecast`, `price_history`, +`price`, `maintenance`, `homeview.local_day`, `solarfc._bounds`), or has a +fixed bound. None can stall. One known inexactness stays, outside this fix: +`solarfc` spreads a state-only forecast over hours since local midnight, so on +the two change days its sun curve is an hour off the clock. diff --git a/goodwe_controller/app/dayplan.py b/goodwe_controller/app/dayplan.py index 4bd8ecc..a64fc1c 100644 --- a/goodwe_controller/app/dayplan.py +++ b/goodwe_controller/app/dayplan.py @@ -157,13 +157,15 @@ class Plan: def _hour_slices(t0, t1): - """Local-hour slices of [t0, t1): (start, end, local hour).""" + """Local-hour slices of [t0, t1): (start, end, local hour). + + Steps in UTC, so `t` always advances (SAFETY-123, DECISIONS).""" out, t = [], t0 while t < t1: lt = _local(t) - nxt = (lt.replace(minute=0, second=0, microsecond=0) - + timedelta(hours=1)).astimezone(timezone.utc) - e = min(nxt, t1) + left = timedelta(minutes=60 - lt.minute) - timedelta( + seconds=lt.second, microseconds=lt.microsecond) + e = min(t.astimezone(timezone.utc) + left, t1) out.append((t, e, lt.hour)) t = e return out diff --git a/goodwe_controller/test_safety123.py b/goodwe_controller/test_safety123.py new file mode 100644 index 0000000..2708594 --- /dev/null +++ b/goodwe_controller/test_safety123.py @@ -0,0 +1,523 @@ +"""SAFETY-123: the day plan's local-hour loop ends on the clock-change days. +`python3 test_safety123.py` + +On a host with no tzdata (`tariff._EUTimezone`) `dayplan._hour_slices()` never +returned on the spring clock change (28 March 2027, 00:00 UTC): the control +tick froze. Graded here, each call under a hard time limit (SIGALRM): + +1. `_hour_slices()` over 27-29 March and 30 Oct - 1 Nov 2027, and the US + change days, at every quarter, in the no-tzdata zone, a US-rule zone built + the same way, and (where the host has tzdata) Europe/Brussels, + America/New_York and zones with half-hour changes: it ends, covers the + span, never crosses a local hour, labels the local hour, and gives 23 + slices on the spring day and 25 on the autumn day. +2. Normal days: the same slices as release/1.0 a1d0703 (a copy of its + function is kept below as the reference). +3. `dayplan.plan()` at every quarter of those days with each pack mode, and + its answer on normal days equal to the old function's. +4. `Controller._mode_roles()` at every quarter of those days with modes on, + for Solar first, Leftover solar and Cheap hours, and one full control + cycle through the forecast read at the hang's instants. +See DECISIONS/SAFETY-123-hour-slices-step-in-utc.md. + +Mutating what this file grades? Read MUTATION.md first (stale bytecode). +""" + +import asyncio +import logging +import os +import resource +import signal +import sys +from datetime import date, datetime, timedelta, timezone, tzinfo + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +sys.path.insert(0, os.path.join(HERE, "tools")) +logging.disable(logging.WARNING) + +import opt11_identity as rig # noqa: E402 +import app.main as M # noqa: E402 +from app import dayplan, modetrade, packmode, priceruns, solarfc # noqa: E402 +from app.price import PricePoint, PriceProduct, PriceRead, brussels # noqa +from app.static_tariff import TariffPlan # noqa: E402 +from app.store import unit_key # noqa: E402 +from app.tariff import BILLING_TZ, _EUTimezone # noqa: E402 + +UTC = timezone.utc +fails = [] +LIMIT_S = 3.0 # one call; the old loop never returned +try: # a loop that appends without end must not take the host + resource.setrlimit(resource.RLIMIT_AS, (3 << 29, resource.getrlimit( + resource.RLIMIT_AS)[1])) +except (ValueError, OSError): + pass + + +def check(name, cond): + print(f" {'ok ' if cond else 'FAIL'} {name}") + if not cond: + fails.append(name) + + +class Slow(Exception): + pass + + +def _alarm(*_a): + raise Slow() + + +def limited(fn, *a, **k): + """`fn(*a, **k)` or `Slow` when it takes more than LIMIT_S seconds.""" + signal.signal(signal.SIGALRM, _alarm) + signal.setitimer(signal.ITIMER_REAL, LIMIT_S) + try: + return fn(*a, **k) + except MemoryError: + raise Slow() from None + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + + +# --------------------------------------------------------------------------- # +# The reference: release/1.0 a1d0703's function, verbatim, for normal days +# --------------------------------------------------------------------------- # +def old_slices(t0, t1): + out, t = [], t0 + while t < t1: + lt = dayplan._local(t) + nxt = (lt.replace(minute=0, second=0, microsecond=0) + + timedelta(hours=1)).astimezone(timezone.utc) + e = min(nxt, t1) + out.append((t, e, lt.hour)) + t = e + return out + + +# --------------------------------------------------------------------------- # +# Zones +# --------------------------------------------------------------------------- # +def _nth_sunday(year, month, n): + d = date(year, month, 1) + d += timedelta(days=(6 - d.weekday()) % 7 + 7 * (n - 1)) + return datetime(d.year, d.month, d.day) + + +class USZone(tzinfo): + """America/New_York by the US rule (DST from the second Sunday of March + 02:00 to the first Sunday of November 02:00, local), built like + tariff._EUTimezone: the offset is read from the local wall clock, so the + missing 02:00 reads as DST. Its change days are not the EU's.""" + + def _dst_local(self, naive): + y = naive.year + return (_nth_sunday(y, 3, 2).replace(hour=2) <= naive + < _nth_sunday(y, 11, 1).replace(hour=2)) + + def _dst_utc(self, naive_utc): + y = naive_utc.year + return (_nth_sunday(y, 3, 2).replace(hour=7) <= naive_utc + < _nth_sunday(y, 11, 1).replace(hour=6)) + + def fromutc(self, dt): + return dt + timedelta( + hours=-4 if self._dst_utc(dt.replace(tzinfo=None)) else -5) + + def utcoffset(self, dt): + if dt is None: + return timedelta(hours=-5) + return timedelta(hours=-4 if self._dst_local( + dt.replace(tzinfo=None)) else -5) + + def dst(self, dt): + return timedelta(hours=1) if dt is not None and self._dst_local( + dt.replace(tzinfo=None)) else timedelta(0) + + +def _q(y, mo, d, h=0): + return datetime(y, mo, d, h, tzinfo=UTC) + + +# (name, tzinfo, whole-hour offsets?, windows [from, to) at every quarter) +BRU_SPRING = (_q(2027, 3, 27), _q(2027, 3, 30)) +BRU_AUTUMN = (_q(2027, 10, 30), _q(2027, 11, 2)) +US_SPRING = (_q(2027, 3, 13), _q(2027, 3, 16)) +US_AUTUMN = (_q(2027, 11, 6), _q(2027, 11, 9)) +COMMON = [BRU_SPRING, BRU_AUTUMN, US_SPRING, US_AUTUMN] +ZONES = [("no-tzdata EU rule", _EUTimezone(), True, COMMON), + ("US rule", USZone(), True, COMMON)] +try: + from zoneinfo import ZoneInfo + for _n, _w, _extra in ( + ("Europe/Brussels", True, []), + ("America/New_York", True, []), + # DST of half an hour, and clocks that jump at midnight + ("Australia/Lord_Howe", False, + [(_q(2027, 4, 3), _q(2027, 4, 6)), + (_q(2027, 10, 2), _q(2027, 10, 5))]), + ("America/Santiago", True, + [(_q(2027, 4, 2), _q(2027, 4, 5)), + (_q(2027, 9, 4), _q(2027, 9, 7))]), + ("Asia/Kathmandu", False, [])): + try: + ZONES.append((_n, ZoneInfo(_n), _w, COMMON + _extra)) + except Exception: # noqa: BLE001 + print(f" no tzdata for {_n} here: not graded") +except ImportError: + print(" no zoneinfo here") +print(" zones graded:", ", ".join(z[0] for z in ZONES)) + + +class In: + """`dayplan.BILLING_TZ` is `tz` inside the with-block.""" + + def __init__(self, tz): + self.tz = tz + + def __enter__(self): + self.was = dayplan.BILLING_TZ + dayplan.BILLING_TZ = self.tz + + def __exit__(self, *_a): + dayplan.BILLING_TZ = self.was + + +def quarters(a, b): + t = a + while t < b: + yield t + t += timedelta(minutes=15) + + +# ============================================================================ # +print("1. _hour_slices(): ends, covers, local hours, 23 and 25 hours") +HOUR = timedelta(hours=1) +for zname, tz, whole, windows in ZONES: + bad, slow, n = [], None, 0 + with In(tz): + for a, b in windows: + for t0 in quarters(a, b): + for span in (HOUR, 7 * HOUR, 30 * HOUR): + t1 = t0 + span + n += 1 + try: + sl = limited(dayplan._hour_slices, t0, t1) + except Slow: + slow = (t0, t1) + break + ok = bool(sl) and sl[0][0] == t0 and sl[-1][1] == t1 + ok = ok and all(x[1] == y[0] for x, y in zip(sl, sl[1:])) + for s, e, h in sl: + ok = ok and timedelta(0) < e - s <= HOUR + lt = s.astimezone(tz) + ok = ok and h == lt.hour + last = (e - timedelta(microseconds=1)).astimezone(tz) + # no local hour boundary inside the slice + ok = ok and (lt.year, lt.month, lt.day, lt.hour) == ( + last.year, last.month, last.day, last.hour) + # it ends on a local hour, the span's end, or where + # the clock itself jumps (half-hour changes) + el = e.astimezone(tz) + ok = ok and (e == t1 or el.minute == 0 + or el.utcoffset() != lt.utcoffset()) + if whole: + # whole-hour zones: local hours are UTC hours + ok = ok and (e.minute == 0 or e == t1) \ + and (s.minute == 0 or s == t0) + if not ok: + bad.append((t0, t1)) + if slow: + break + if slow: + break + check(f"{zname}: every quarter x 1, 7, 30 h ({n} spans) returns within " + f"{LIMIT_S:.0f} s" + ("" if not slow else f" - HUNG at {slow[0]}"), + slow is None) + check(f"{zname}: ...contiguous from t0 to t1, each slice at most an " + f"hour, labelled with its local hour, none across a local hour", + not bad) + +# The change days: 23 and 25 local hours, from local midnight to midnight. +# (spring day, autumn day, the local hour that happens twice in autumn) +BRU_DAYS = ((2027, 3, 28), (2027, 10, 31), 2) +US_DAYS = ((2027, 3, 14), (2027, 11, 7), 1) +DAYS = {"no-tzdata EU rule": BRU_DAYS, "Europe/Brussels": BRU_DAYS, + "US rule": US_DAYS, "America/New_York": US_DAYS} + + +def local_day(tz, y, m, d): + d0 = datetime(y, m, d, tzinfo=tz).astimezone(UTC) + n = date(y, m, d) + timedelta(days=1) + return d0, datetime(n.year, n.month, n.day, tzinfo=tz).astimezone(UTC) + + +for zname, tz, _w, _win in ZONES: + if zname not in DAYS: + continue + spring, autumn, twice = DAYS[zname] + with In(tz): + for what, (y, m, d), want, want_hours in ( + ("spring", spring, 23, [h for h in range(24) if h != 2]), + ("autumn", autumn, 25, + [h for h in range(24) for _ in range(2 if h == twice else 1)])): + d0, d1 = local_day(tz, y, m, d) + try: + sl = limited(dayplan._hour_slices, d0, d1) + except Slow: + sl = None + check(f"{zname}, {what} day {y}-{m:02d}-{d:02d}: {want} one-hour " + f"slices from local midnight to midnight", + sl is not None and len(sl) == want + and all(e - s == HOUR for s, e, _h in sl) + and d1 - d0 == want * HOUR and sl[0][0] == d0 + and sl[-1][1] == d1) + check(f"{zname}, {what} day: the local hours are " + f"{'0-23 without 2' if what == 'spring' else f'0-23 with {twice} twice'}", + sl is not None and [h for _s, _e, h in sl] == want_hours) + +# ============================================================================ # +print("2. normal days: the same slices as release/1.0 a1d0703") +NORMAL = [(_q(2027, 1, 9), _q(2027, 1, 12)), (_q(2027, 6, 18), _q(2027, 6, 21)), + (_q(2027, 8, 30), _q(2027, 9, 2)), (_q(2027, 12, 18), _q(2027, 12, 21))] +for zname, tz, _w, _win in ZONES: + diff, n = [], 0 + with In(tz): + for a, b in NORMAL: + for t0 in quarters(a, b): + for off in (timedelta(0), timedelta(seconds=7, microseconds=5), + timedelta(minutes=14, seconds=59, microseconds=999999), + timedelta(minutes=37, seconds=1)): + for span in (timedelta(minutes=1), timedelta(minutes=50), + 3 * HOUR, 26 * HOUR): + s0 = t0 + off + n += 1 + if dayplan._hour_slices(s0, s0 + span) != old_slices( + s0, s0 + span): + diff.append(s0) + check(f"{zname}: {n} spans on ordinary days equal the old slices " + f"(including starts off the quarter)", n > 1000 and not diff) +with In(ZONES[0][1]): + s0 = _q(2027, 1, 10, 9) + eq = dayplan._hour_slices(s0, s0) == [] == old_slices(s0, s0) \ + and dayplan._hour_slices(s0, s0 - HOUR) == [] + check("an empty or backwards span gives no slices, as before", eq) + plus2 = timezone(timedelta(hours=2)) + t0 = datetime(2027, 6, 1, 10, 20, tzinfo=plus2) + check("a start given in another zone still gives the old slices", + dayplan._hour_slices(t0, t0 + 3 * HOUR) == + old_slices(t0, t0 + 3 * HOUR)) + +# ============================================================================ # +print("3. dayplan.plan() at every quarter of the change days") + + +def st(kwh, at): + return {"state": str(kwh), "attributes": {"unit_of_measurement": "kWh"}, + "last_updated": at.isoformat(), "last_reported": at.isoformat()} + + +def cheap_dear(h): + return 0.08 if 1.0 <= h < 5.0 else 0.34 if 17.0 <= h < 21.0 else 0.20 + + +def price_plan(now, f=cheap_dear, days=4): + """15-minute firm prices from a day before `now` for `days` days.""" + start = (now - timedelta(days=1)).replace(minute=0, second=0, + microsecond=0) + pts = [] + for i in range(days * 96 + 96): + b = brussels(start + timedelta(minutes=15 * i)) + pts.append(PricePoint(start=b, eur_per_kwh=f(b.hour + b.minute / 60), + provisional=False)) + return TariffPlan(PriceRead.series(pts, PriceProduct.DAY_AHEAD, + "generic_dayahead", now), None, + "provider") + + +_PLANS = {} + + +def plan_for(start): + """One price plan covering `start` - 1 day .. + 5 days, kept: building it + for every quarter is most of the run time.""" + if start not in _PLANS: + _PLANS[start] = price_plan(start, days=6) + return _PLANS[start] + + +HOUSE = dayplan.HouseProfile({"2027-03-20": { + str(h): (3000.0 if 17 <= h < 22 else 600.0) for h in range(24)}}) +MODES = {"Solar first": (packmode.SOLAR_FIRST, packmode.SOLAR_FIRST), + "Leftover solar": (packmode.SOLAR_FIRST, packmode.LEFTOVER_SOLAR), + "Cheap hours": (packmode.CHEAP_HOURS, packmode.CHEAP_HOURS)} + + +def packs_of(modes, soc=(55.0, 40.0)): + return [modetrade.Pack(f"inverter-{u}", True, mode, False, 2500.0, s, 10.0) + for u, mode, s in zip("ab", modes, soc)] + + +_INPUTS = {} + + +def plan_at(now, modes, start=None): + """plan() on the price runs and forecast of `now` (made once per quarter: + the runs do not depend on the pack modes).""" + if now not in _INPUTS: + pts = plan_for(start or now.replace(hour=0, minute=0)).read.points + _INPUTS[now] = ( + priceruns.plan(pts, now, required_eur_kwh=0.1, charge_kwh=8.0, + charge_kw=5.0, sell_kwh=0.0, sell_kw=0.0), + solarfc.read(st(2.0, now), st(8.0, now), now)[0]) + runs, fc = _INPUTS[now] + return dayplan.plan(fc, HOUSE.estimator(), runs, now, packs_of(modes), + trust_pct=70, floor_pct=20, ceiling_pct=95) + + +spy = [] +_slices = dayplan._hour_slices +dayplan._hour_slices = lambda a, b: (spy.append((a, b)), _slices(a, b))[1] +try: + for mname, modes in MODES.items(): + for wname, (a, b) in (("27-29 March", BRU_SPRING), + ("30 Oct-1 Nov", BRU_AUTUMN)): + made, slow, n = 0, None, 0 + spy.clear() + for now in quarters(a, b): + n += 1 + try: + p = limited(plan_at, now, modes, a) + except Slow: + slow = now + break + made += p is not None + check(f"plan(), {mname}, {wname}: {n} quarters return within " + f"{LIMIT_S:.0f} s" + ("" if not slow else + f" - HUNG at {slow}") + + f"; a plan is made at {made} of them", slow is None + and made > 20) + if wname == "27-29 March" and not slow: + cross = [s for s in spy if s[0] <= _q(2027, 3, 28) < s[1]] + check(f"plan(), {mname}: {len(cross)} spans ran across the " + "change (28 March 00:00 UTC): the test reaches the " + "loop", len(cross) > 20) +finally: + dayplan._hour_slices = _slices + +# The answer on normal days is the old function's. +for mname, modes in MODES.items(): + same, n = True, 0 + for now in quarters(_q(2027, 1, 11), _q(2027, 1, 13)): + new = plan_at(now, modes) + dayplan._hour_slices = old_slices + try: + was = plan_at(now, modes) + finally: + dayplan._hour_slices = _slices + n += 1 + same = same and new == was + check(f"plan(), {mname}: {n} quarters on ordinary days equal the old " + "answer", same) + +# ============================================================================ # +print("4. Controller._mode_roles() at every quarter, modes on") + + +def controller(modes, now): + extra = {"battery_modes_enabled": True, "arbitrage_enabled": True, + "cycle_cost_threshold_eur": 0.05, + "pv_forecast_entities": ["sensor.t1", "sensor.t2", "sensor.t3"], + "pv_forecast_tomorrow_entities": ["sensor.m1", "sensor.m2", + "sensor.m3"]} + opts = rig._options(2, extra) + for i in opts["inverters"]: + i["capacity_kwh"] = 10.0 + store = rig._Store() + for u, mode in zip("ab", modes): + store.data[unit_key(f"inverter-{u}", "pack_mode")] = mode + store.data["house_profile"] = {"2027-03-20": { + str(h): (3000.0 if 17 <= h < 22 else 600.0) for h in range(24)}} + + class Hass(rig._Hass): + async def state(self, e): + if e.startswith(("sensor.t", "sensor.m")): + return st(2.0 if e.startswith("sensor.t") else 8.0, self.at) + return await super().state(e) + hass = Hass(("a", "b"), lambda u, t: 55.0 if u == "a" else 40.0, + lambda u, t: False) + hass.at = now + c = M.Controller(opts, hass, store, rig._Pub(), stop=asyncio.Event()) + for i in c.inverters: + i.dev_min, i.dev_max = -5000.0, 5000.0 + i.soc = 55.0 if i.name.endswith("a") else 40.0 + c.auto = True + return c, hass + + +for mname, modes in MODES.items(): + for wname, (a, b) in (("27-29 March", BRU_SPRING), + ("30 Oct-1 Nov", BRU_AUTUMN)): + c, hass = controller(modes, a) + slow, n, made, roles = None, 0, 0, set() + spy.clear() + dayplan._hour_slices = lambda x, y: (spy.append((x, y)), + _slices(x, y))[1] + try: + for now in quarters(a, b): + n += 1 + c.tariff_plan, c.tariff_at = plan_for(a), now + c.solar_fc, _w, _g = solarfc.read_sum( + [("sensor.t1", st(2.0, now))], + [("sensor.m1", st(8.0, now))], now) + try: + t = limited(c._mode_roles, now, None) + except Slow: + slow = now + break + made += c.day_plan is not None + roles |= set(t.roles.values()) + finally: + dayplan._hour_slices = _slices + check(f"_mode_roles(), {mname}, {wname}: {n} quarters return within " + f"{LIMIT_S:.0f} s" + ("" if not slow else f" - HUNG at {slow}"), + slow is None) + check(f"...a day plan is made at {made} of them (the loop is " + f"reached), roles seen {sorted(roles)}", slow is None + and made > 20 and len(spy) > 20) + +# One whole control cycle through the real forecast read (three sensors a +# day) at the instants the old loop hung: the page and control share the loop. +for iso in ("2027-03-27T21:00:05+00:00", "2027-03-27T23:45:05+00:00", + "2027-03-28T00:00:05+00:00"): + now = datetime.fromisoformat(iso) + c, hass = controller(MODES["Leftover solar"], now) + real_dt = M.datetime + + class _DT(real_dt): + @classmethod + def now(cls, tz=None, _n=now): + return _n if tz is None else _n.astimezone(tz) + M.datetime = _DT + try: + async def one(): + await c.read_inputs(now_mono=0.0) + c.tariff_plan, c.tariff_at = price_plan(now), now + await c.cycle() + try: + limited(lambda: asyncio.run(one())) + ok = c.solar_fc is not None and c.day_plan is not None + except Slow: + ok = False + finally: + M.datetime = real_dt + check(f"a whole control cycle at {iso[:16]}Z (three forecast sensors a " + "day) returns with a day plan", ok) + +print() +if fails: + print(f"{len(fails)} FAILED:") + for f in fails: + print(" -", f) + sys.exit(1) +print("all ok")