diff --git a/.gitea/ci/run-tests.sh b/.gitea/ci/run-tests.sh new file mode 100755 index 0000000..aed601e --- /dev/null +++ b/.gitea/ci/run-tests.sh @@ -0,0 +1,125 @@ +#!/bin/sh +# CI-15: run the add-on's test_*.py files, in parallel, inside the CI container. +# Called by the step "the add-on's own tests" of .gitea/workflows/checks.yml as +# sh -eu /w/goodwe-addon/.gitea/ci/run-tests.sh +# and runnable the same way from a clone (see TESTS_DIR below). +# +# WHY: the serial loop ran past the runner's 60-minute limit (SAFETY-125: 85 files, +# 3 557 s of test time, run 1430 cut off at file 84). The host has 12 cores. +# +# SHAPE +# 1. A parallel batch: JOBS files at a time (xargs -P), longest first. +# 2. THEN the serial group, one at a time, after the batch has finished, so +# each runs on a machine with nothing else of ours on it. +# Every file writes to its own $RUN_DIR/.out (there is no shared output +# file) and prints its result as ONE block under a lock, so the PASS/FAIL +# lines and the `CI: ` / tail -40 text of two files never interleave. +# +# THE FILE LIST IS STILL DERIVED FROM THE TREE (see checks.yml). The two typed +# lists below only SORT or SET ASIDE files that exist; a new test_*.py that is +# in neither list is run, in the parallel batch. A name that no longer exists +# in a list is harmless. Nothing can make a file silently not run. +# +# SERIAL GROUP: files whose assertions depend on the wall clock, so CPU load +# from siblings could flip them. Name and reason, one each: +# test_loop.py drives the real run_control() against a real wall-clock +# floor (~93 s on purpose); worst inter-cycle gap 3.0 s +# against a 4.5 s bound (TEST-30, CI-06). +# test_freeze.py part 3 runs the real sim/hwsim.py over a transport with a +# 16 s window, a 10 s hold and an alarm measured at 11.0 s: +# one poll of margin, and "failed 1 run in 3" on the rig +# transport (TEST-30, CI-06). +# test_safety87.py builds a controller (CH65) whose meter reading is stamped +# on the REAL clock, and section 8 asks its status page, +# which says "Not ready - Meter: no reading for N seconds, +# limit 30 seconds" once N passes 30. Measured by CI-15: N is +# about half the file's run time (N = 45 s in a 94 s run, 48 s +# in a 103 s run). The file takes ~25 s idle and took 45 s in +# CI's first parallel run, so N was ~20 s of the 30. It FAILED +# alone under --cpus 0.15 (94 s) and in a loaded parallel run +# (103 s). No test file is changed here, so it runs alone. +# Add a file here only with a measured reason, and say it in this list. +SERIAL="test_loop.py test_freeze.py test_safety87.py" +# +# LONGEST FIRST (SAFETY-125's CI run, job 2580; then a local parallel run, seconds: +# modes_p4 758, safety126 634 (SAFETY-126, local, one core), safety125 508 on hotfix-0.19.2 only, modes_p3 364, safety119 328, +# price_run_fixes 184, safety90 160, safety103 134, safety94 114, peakguard 113, +# opt20 102, safety97 101, safety81 100). With JOBS workers pulling from one +# queue, starting the longest files first keeps the tail short: the batch takes +# about as long as the longest file. A sort order, not a selection. +HEAVY="test_modes_p4.py test_safety126.py test_safety125.py test_modes_p3.py test_safety119.py test_price_run_fixes.py test_safety90.py test_safety103.py test_safety94.py test_peakguard.py test_opt20.py test_safety97.py test_safety81.py" +# +JOBS=${CI_JOBS:-6} +cd "${TESTS_DIR:-/w/goodwe-addon/goodwe_controller}" + +# one FILE: run it, keep its output in its own file, print its block whole. +if [ "${1:-}" = "--one" ]; then + f=$2 + o="$RUN_DIR/$f.out" + start=$(date +%s) + if python3 "$f" > "$o" 2>&1; then rc=0; else rc=1; fi + secs=$(( $(date +%s) - start )) + { + if [ $rc = 0 ]; then + echo "PASS $f (${secs}s)" + grep "^CI: " "$o" || true + else + echo "FAIL $f (${secs}s)" + tail -40 "$o" + fi + } > "$RUN_DIR/$f.blk" + echo $rc > "$RUN_DIR/$f.rc" + flock "$RUN_DIR/print.lock" cat "$RUN_DIR/$f.blk" + exit 0 +fi + +set -- test_*.py +if [ "$1" = "test_*.py" ]; then + echo "NO test_*.py FOUND - the mount is empty or the layout moved. NOTHING has been checked." + exit 1 +fi +echo "$# test file(s) to run: $JOBS at a time, then the serial group" + +RUN_DIR=$(mktemp -d /tmp/ci-tests.XXXXXX) +export RUN_DIR +: > "$RUN_DIR/par" +: > "$RUN_DIR/ser" +for f in $HEAVY; do + case " $SERIAL " in *" $f "*) continue ;; esac + if [ -f "$f" ]; then echo "$f" >> "$RUN_DIR/par"; fi +done +for f in "$@"; do + case " $SERIAL $HEAVY " in *" $f "*) continue ;; esac + echo "$f" >> "$RUN_DIR/par" +done +for f in $SERIAL; do + if [ -f "$f" ]; then echo "$f" >> "$RUN_DIR/ser"; fi +done +echo "parallel: $(wc -l < "$RUN_DIR/par") file(s); serial: $(wc -l < "$RUN_DIR/ser") file(s)" + +t0=$(date +%s) +rc=0 +# `exit 0` in the worker: a failing FILE is recorded in its .rc, so a non-zero +# from xargs itself means the machinery broke, which is a failure too. +xargs -n 1 -P "$JOBS" sh -eu "$0" --one < "$RUN_DIR/par" || rc=1 +echo "parallel batch done in $(( $(date +%s) - t0 ))s; serial group next" +while read -r f; do + sh -eu "$0" --one "$f" || rc=1 +done < "$RUN_DIR/ser" + +# Every file that was meant to run has a verdict, and every verdict is PASS. A +# worker that died before writing its .rc leaves no file, which counts as a fail. +ran=0 +bad=0 +for f in "$@"; do + ran=$((ran + 1)) + if [ ! -f "$RUN_DIR/$f.rc" ]; then + echo "FAIL $f (no result: the run never reported)" + bad=$((bad + 1)) + elif [ "$(cat "$RUN_DIR/$f.rc")" != 0 ]; then + bad=$((bad + 1)) + fi +done +echo "$ran file(s), $bad failed, $(( $(date +%s) - t0 ))s wall" +if [ "$bad" = 0 ] && [ "$rc" = 0 ]; then exit 0; fi +exit 1 diff --git a/.gitea/workflows/checks.yml b/.gitea/workflows/checks.yml index 4cffd72..749be2d 100644 --- a/.gitea/workflows/checks.yml +++ b/.gitea/workflows/checks.yml @@ -209,7 +209,7 @@ on: jobs: unit: runs-on: [self-hosted, goodwe-rig] - timeout-minutes: 70 # ~4 to run; the rest is queueing for the single slot + timeout-minutes: 40 # ~15 to run (CI-15: 6 files at a time; the serial suite was ~60 and hit the runner cap); the rest is margin for a busy host env: # ⚠️ UNDER /home/homelab, NOT $GITHUB_WORKSPACE (ENV-01, and re-measured # during CI-01). act's workspace is `/root/.cache/act//hostexecutor`, a @@ -534,6 +534,19 @@ jobs: # shorter timeout - a 60 s limit was tried during CI-01 and reported # exactly the false hang the docstring predicts. # + # ⚠️ CI-15: THE FILES RUN SIX AT A TIME, NOT ONE AFTER ANOTHER. The serial + # loop summed 3 557 s on SAFETY-125's run (job 2580) and ran past the + # runner's 60-minute cap. The loop now lives in .gitea/ci/run-tests.sh (it + # comes from the merged tree, like the Dockerfile), so the same script runs + # locally and here. It runs the longest files first, then the files whose + # assertions depend on the wall clock - test_loop.py and test_freeze.py + # (TEST-30 / CI-06) and test_safety87.py (CI-15) - ALONE, after the + # parallel batch, so no sibling can steal their CPU. The reasons are in + # the script. Its output stays one + # PASS/FAIL block per file, in a file per file (never one shared /tmp/out). + # A file that is not in the script's two typed lists still runs, in the + # parallel batch: the list is a sort order, not a selection. + # # test_freeze.py part 3 runs against the real `sim/hwsim.py` from the # checkout above; GOODWE_REQUIRE_HWSIM=1 makes it fail rather than use its # stub (CI-12). Lines a test prints with a `CI: ` prefix are echoed on PASS. @@ -552,28 +565,7 @@ jobs: - name: the add-on's own tests run: | set -eu - docker run --rm -e PYTHONDONTWRITEBYTECODE=1 -e GOODWE_REQUIRE_HWSIM=1 -v "$WS:/w:ro" goodwe-addon-checks:1 sh -eu -c ' - cd /w/goodwe-addon/goodwe_controller - set -- test_*.py - if [ "$1" = "test_*.py" ]; then - echo "NO test_*.py FOUND - the mount is empty or the layout moved. NOTHING has been checked." - exit 1 - fi - echo "$# test file(s) to run" - rc=0 - for f in "$@"; do - start=$(date +%s) - if python3 "$f" > /tmp/out 2>&1; then - echo "PASS $f ($(( $(date +%s) - start ))s)" - grep "^CI: " /tmp/out || true - else - echo "FAIL $f ($(( $(date +%s) - start ))s)" - tail -40 /tmp/out - rc=1 - fi - done - exit $rc - ' + docker run --rm -e PYTHONDONTWRITEBYTECODE=1 -e GOODWE_REQUIRE_HWSIM=1 -v "$WS:/w:ro" goodwe-addon-checks:1 sh -eu /w/goodwe-addon/.gitea/ci/run-tests.sh - name: clean up if: always() diff --git a/README.md b/README.md index d9711d4..c25d5c7 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ on real hardware, and the comments in `control.py` say which. ### CI and the infra pin -CI (`.gitea/workflows/checks.yml`) runs every `test_*.py`, with `admin/goodwe` checked out next to this repo at the commit in `.gitea/infra-pin`, so `test_freeze.py` grades against the real `sim/hwsim.py`. If that checkout fails, the job fails. The local stub is used only in local runs, and it prints `STUB USED` when it is. +CI (`.gitea/workflows/checks.yml`) runs every `test_*.py` (six at a time, the wall-clock ones alone afterwards: `.gitea/ci/run-tests.sh`), with `admin/goodwe` checked out next to this repo at the commit in `.gitea/infra-pin`, so `test_freeze.py` grades against the real `sim/hwsim.py`. If that checkout fails, the job fails. The local stub is used only in local runs, and it prints `STUB USED` when it is. The pin tracks `admin/goodwe` **release/1.0**, not master. Two non-required checks go red when release/1.0 has moved past the pin: `infra-pin / fresh` here, on any add-on PR or push to release/1.0, and `addon-pin / fresh` in `admin/goodwe`, on every push to its release/1.0. Nothing re-runs `addon-pin` after a bump, so it stays red on that admin/goodwe commit; `infra-pin / fresh` green on the bump is the all-clear. **Bump the pin** whenever either is red. Set `.gitea/infra-pin` to the new release/1.0 SHA (`git ls-remote /admin/goodwe.git refs/heads/release/1.0`) in a PR, and let `checks / unit` grade the tests against it. CI reads `admin/goodwe` with the Actions secret `INFRA_READ_TOKEN`. diff --git a/goodwe_controller/CHANGELOG.md b/goodwe_controller/CHANGELOG.md index 1e965c1..08eb53f 100644 --- a/goodwe_controller/CHANGELOG.md +++ b/goodwe_controller/CHANGELOG.md @@ -1,5 +1,114 @@ # Changelog +## 0.20.0 + +**A minor bump: a Forecast tab, a night grid charge sized to what the sun will not give, a kept evening forecast, and no option changes.** +No option was added, removed or renamed, and no default changed (86 options). It is a minor bump because Leftover solar now buys different amounts from the grid and the night charge keeps its level when the forecast goes quiet; every saved setting still starts. +**Existing installs need no option change:** options that start 0.19.2 also +start 0.20.0. The changes below apply with one inverter or two. + +**A Forecast tab.** A new tab next to Home and Batteries shows today and tomorrow hour by hour: the solar forecast, the sun measured so far, the house's expected use, and each battery's level with its planned charge and give blocks. It shows what the add-on plans and changes no command. + +**Leftover solar buys what the sun will not give.** On a day/night tariff the night grid charge now covers the expensive hours' use less the sun counted for them, where it used to buy for the whole day. A forecast with only a daily total, such as Forecast.Solar, is no longer counted several times over by day, so Leftover solar no longer charges too little because of it. + +**The evening forecast is kept for the night.** Forecast.Solar-type sensors stop updating in the evening. The last fresh reading is now kept until 06:00, so the night cheap run is judged with the forecast's price gap and Leftover solar still plans its charge. A newer reading replaces it at once, and a charge that is running keeps its planned level if the figure ends. + +**No false peak alarm during the night grid charge.** "Battery stopped limiting the peak", and its notification, no longer appear while the 15-minute period stays within 10 W of the aim. + +Contents of this release, in merge order since 0.19.2 (`f1c9733`): + +| | | +|---|---| +| `a1d0703` | Home now draws a grid charge only where the add-on will take it, with battery modes on or off; before, it could show the rest of a refused cheap run, or a charge after the last dear period. The price texts say "lowest price left" and "highest price left", and no command changes ([[UX-53]], #209) | +| `f5afb42` | The day planner's hour loop now always ends on the night the clocks go forward. Without time-zone data it could go round forever and freeze the add-on; the add-on's own image has that data ([[SAFETY-123]], #211) | +| `4a1f6cc` | A new Forecast tab shows today and tomorrow hour by hour: the sun, the house and each battery's level, with the charge and give blocks Home would draw. It shows the plan and changes no command; notes say when a part is not known yet ([[UX-54]], #210) | +| `6be7e41` | The add-on's test files now run six at a time in the checks, so a check takes about 15 to 25 minutes, where one by one it ran past an hour. No change in behaviour ([[CI-15]], #215) | +| `b6be312` | The day plan counts the sun right: a forecast with only a daily total is no longer counted several times over. On a day/night tariff the night charge for Leftover solar covers the expensive hours' use less the sun counted for them, shared between batteries by capacity, and the screens say when another battery or the grid covers the rest ([[OPT-23]], [[OPT-28]], #218) | +| `cc9bbeb` | When a solar forecast sensor stops updating in the evening, the night plan uses its last fresh reading, for the day it is for only. One missed reading no longer drops the forecast at once, and the pages say when a figure is kept ([[OPT-27]], #219) | +| `deee98d` | The night grid charge under the peak limit no longer raises a false "Battery stopped limiting the peak" alarm when the 15-minute period stays within 10 W of the aim. A real overrun still alarms; a steadily rising load alarms a little later, and with a peak margin under 10 W a period can end up to 10 W over the limit without it ([[SAFETY-126]], #221) | +| `8afadbd`, `cdafa2c` | The kept evening figure now counts until 06:00, not until 03:00, and a charge that is running keeps its planned level when the figure ends. A newer reading replaces the evening figure at once and for good, so it does not come back later in the night ([[OPT-34]], #222, with its fix #225) | +| `d4c01da` | Comments and docstrings in the peak guard and the price files were cut to a line and a pointer to the decision notes. No change in behaviour: a check shows the code itself is the same ([[MAINT-20]] part 1, #223) | + +The merge that brought 0.19.1 and 0.19.2 into this branch (`f1b7244`, #220) adds nothing else; the Forecast tab uses the same price gap as control since it. + +### Known issues + +- A false "Battery is not keeping up" can still show when the peak guard pauses a maintenance charge ([[SAFETY-92]]). +- A battery capped at a low limit with a perfectly flat reading can still trip the stale check ([[SAFETY-74]]). +- A battery whose reading at rest is beyond about ±175 W is still not caught when it refuses ([[SAFETY-97]]). +- The peak guard can still back off near the end of a 15-minute period when a meter reading is late, by less than 30 s. This is about the guard's discharge only; a grid charge is cut back since 0.19.2 ([[SAFETY-99]]). +- Two inverters: the maintenance charge can switch on and off while the house is above the peak aim ([[SAFETY-88]]). +- Two inverters: the "circulating" figure counts a battery's reading at rest ([[SAFETY-98]]). +- Two inverters: a drain that starts during a big step in house use can send the other battery to full power, and about 1 kW goes to the grid for about 10 s ([[SAFETY-104]]). +- Two inverters: when a battery comes back during a maintenance charge, the grid can step by several kW at once ([[SAFETY-101]]). +- Two inverters: when a battery drops out during a solar surplus, the others can swing by up to 1.5 kW in a simulation ([[SAFETY-105]]). +- Two inverters: after a drain ends, a change in house use can go unanswered for about 9 s, because some readings are old ([[SAFETY-106]]). +- Two inverters: when one battery's level is missing, both batteries are held at 0 W, and for about 15 s the last command stays on ([[SAFETY-108]]). +- Two inverters: if one battery's level stays missing for 12 h, the other battery's cycle can be marked failed, with a 7-day wait and a notice, though it is healthy. This comes from reading the code; it has not been seen on a site ([[MAINT-17]]). +- When the 15-minute demand forecast sensor counts the battery's own charge, the maintenance charge runs at about half the power the peak limit leaves, so it takes about twice as long. Seen on a site: about 1.2 kW instead of 2.5 kW, and the peak limit was kept ([[SAFETY-111]]). +- One inverter: after a link drop, the battery's own safety stop to 0 W can be taken as a refusal, and the site stays at 0 W for up to 15 min ([[SAFETY-102]]). +- After a Home Assistant restart, control still waits until Home Assistant has finished starting, which can take several minutes, though the readings are already there ([[SAFETY-115]]). +- While Home Assistant is not answering, the page can list inverters as refusing commands and sensors as missing, and send you to Developer Tools, though they are fine ([[UX-37]]). +- If Home Assistant hangs or stops, the page keeps saying "Waiting for Home Assistant" and never turns into an alarm ([[SAFETY-113]]). +- Right after Home Assistant has started, the page can say for about 4 s that an inverter cannot be controlled, and the first command can come late ([[SAFETY-114]]). +- After a Home Assistant restart, the controller status sensor can still say "running" while the add-on cannot control. It was seen once, and the cause is not known ([[SAFETY-116]]). +- Battery modes on: after a planned top-up from the grid, the Batteries screen can say "Enough sun is forecast" ([[UX-32]]). +- Battery modes on: the forecast line "we count on" can show more sun than the plan counts, because the plan takes house use off first ([[UX-33]]). +- Battery modes on: before a planned top-up, the Today list shows a smaller "gives" amount than the plan will use; it grows as the charge happens ([[UX-34]]). +- Two inverters, battery modes on: while the other battery is in its maintenance charge, a battery's card can say it charges from the grid while it stays at 0 W ([[OPT-18]]). +- Two inverters, battery modes on: while a battery refuses to charge in a cheap run, the action log on the page gets about 8 lines an hour for its retries, and the page hint says "waiting for a better price" ([[UX-38]]). +- A battery in a safety stop, or one the add-on cannot reach, can still be started for a cycle, and its drain then waits about 12 h and ends in a failed-cycle notice. This comes from reading the code; it has not been seen on a site ([[MAINT-19]]). +- On the Settings page, the rows of the solar forecast lists are labelled "Phase 1", "Phase 2" instead of "Sensor 1", "Sensor 2" ([[UX-27]]). +- In the solar forecast list, a blank or repeated row makes "source 2 of 2" count differently from the rows on the Settings page ([[UX-28]]). +- The solar forecast message says "has no reading" also when the sensor shows a number that is too old or in the wrong unit ([[UX-29]]). +- If you use only the forecast lists, two Settings notes sit on the single-sensor field, and one of them tells you to set that field ([[UX-30]]). +- After a restart during a price outage, the "No prices" message can say "yet" when prices did come in earlier ([[UX-39]]). +- On the new settings page, you cannot see whether a price service token is set. Check the Configuration tab ([[UX-41]]). +- On the classic settings page, the first click on Save after editing a field can do nothing. Click again ([[UX-42]]). +- A steady overload on a 1 s meter can warn, and send the Home Assistant notification, 2 or 3 times at the start of each 15-minute period instead of once. The peak limit is still kept ([[SAFETY-122]]). +- A fixed tariff is priced once per hour, so a price window that starts or ends at :15, :30 or :45 gets the wrong price for part of that hour. Windows that start and end on the hour are right ([[TARIFF-23]]). +- Step 6 of the setup wizard stays empty because of an error in the page, in English and in Dutch ([[UX-51]]). +- On Dutch pages, refusal texts, the last-event line under the Home buttons, the installer details and the trading status sentences are still English, and a few Dutch texts read badly, such as "1 metingen geweigerd" ([[UX-52]]). +- Some price texts say "hour" where the add-on acts on a 15-minute period, such as "cheapest hour" ([[UX-47]]). +- After the add-on restarts, the Home price chart shows no price for the earlier part of today, and the setup check can say "12 of 24 hours", though the prices are known ([[UX-48]]). +- With battery modes on and only batteries on "Leftover solar", the Home price chart can mark a cheap period where no battery charges ([[UX-49]]). +- On a phone, the bars of the Home price chart are too thin to tap one 15-minute period. The arrow keys work ([[UX-50]]). +- Control acts on Home Assistant readings that arrive late. With readings 10 s behind, the give run can swing and send up to 1.9 kW to the grid for about 50 s in a rig test; with readings 30 s behind, a battery can charge from the grid in an expensive period. Also in 0.19.2 ([[SAFETY-128]]). +- In the last second of a 15-minute period, being a few watts over the aim can make the peak guard ask a battery for full power for one second. When a charge ends on a period edge, the last-second give can fade out over about 10 s, and a rig test exported up to 7.4 kW for about 8 s ([[SAFETY-124]]). +- During the night grid charge, a 15-minute period can end 6 to 14 W over the peak aim, because the battery charges harder than asked. This was seen in simulation; a rig test saw 2 to 8 W ([[OPT-35]]). +- The test rig could not bring back the false peak alarm that [[SAFETY-126]] fixes, so that fix is shown in simulation only. Watch the notifications for a few nights ([[TEST-35]]). +- The night cheap run is sized for the batteries' full charging power, not for what the peak limit leaves. Under a tight limit the run can end before the batteries reach their level, and Home can show a level they do not reach (79.9 % against a shown 95 % in a rig test) ([[OPT-25]]). +- By day, a Leftover solar plan that has only a daily forecast figure counts the sun from now, not from when the figure was read. It counts about 22 % too much for a figure one hour old; the night plan is not affected ([[OPT-30]]). +- The night charge can buy too much in a house that uses more at night than by day, such as one with a car charger or heat pump, and on a sunny day, because spare sun in one expensive hour does not count for the later ones ([[OPT-31]]). +- A cheap run that has already started can overlap a new give run after a re-plan, so a battery can charge from the grid at the dearest price for part of an hour. This was seen in a test case, not on a site. Also in 0.19.2 ([[OPT-32]]). +- If the solar forecast sensors stop updating before 17:00, as in winter or further north, there is no forecast from three hours after their last update until 20:00 ([[OPT-33]]). +- With a forecast that gives only a daily total, the sun is spread an hour off on the two clock-change days after the change. The day's total is right ([[OPT-24]]). +- The add-on learns the house's use only once a solar forecast is set, so a site that adds a forecast later waits one more day for its first plan ([[OPT-26]]). +- Home says "No trading on price yet: no prices have arrived" when every battery is on Solar first, though the prices are shown ([[UX-55]]). +- The Home plan header can say "Charges in the cheapest hour, gives power in the dearest" when Home draws no charge, and it says "hour" for 15-minute prices ([[UX-56]]). +- While the peak guard holds back charging, Home hides the rest of the cheap run and gives no reason. The charge starts again when the guard lets go ([[UX-57]]). +- With battery modes on and selling on, Home can draw a "Sells up to" block when the add-on judges the trade not worth it, and then it will not sell ([[UX-58]]). +- Home, the plan and the Forecast tab stop drawing a battery's give at the trading floor, though control goes on giving down to the battery's cut-out ([[UX-59]]). +- During a price outage, the Batteries screen says "Prices differ too little" and Home says "The price gap is too small" ([[UX-60]]). +- With battery modes on and no meter, or a meter that is a Home Assistant entity, a grid charge is not possible and nothing says so ([[UX-61]]). +- On the Batteries screen, a charge the add-on refused in a cheap run reads as "Saves its power for ...", for up to 15 minutes ([[UX-62]]). +- A Leftover solar battery with no capacity set never charges from the grid when a solar forecast is set, and nothing warns about it ([[UX-63]]). +- The "How the modes work" box on the Batteries screen does not describe Cheap hours only ([[UX-64]]). +- With battery modes on, Settings still shows "Trade on the price spread", which the modes ignore. A site with modes on and no price source starts, and the Batteries screen says "No prices, so it follows the house" without pointing to Settings ([[UX-65]]). +- With battery modes off and several equal prices in a row, Home and the Forecast tab draw one price quarter, while the add-on trades every quarter of the run ([[UX-67]]). +- Late in the day, the Forecast tab's "now" line and last bars sit over the axis labels at the right edge ([[UX-68]]). +- The Forecast tab's level curve charges at each battery's full rating. Under a peak limit the levels it draws are too high and a charge block can end early (54.6 % shown against 39.7 % in a rig test) ([[UX-70]]). +- The peak forecast check says maintenance charging is not kept under the peak limit when no peak forecast is set, but since 0.19.2 it is ([[UX-73]]). +- The peak guard row can show the same figure twice, such as "at this rate 3 500 W, over 3 500 W". Also in 0.19.2 ([[UX-74]]). +- If the inverter's setpoint entity appears in Home Assistant after the add-on has started, the page keeps saying "Inverter control: not found in Home Assistant" and the batteries say "An alarm stopped control" until a restart, though control runs ([[UX-75]]). + +### If you are crossing from a version older than 0.10.0 + +`## 0.10.0` below can refuse an old config: `slew_w` above its limit, and the +withdrawn `price_provider` ids `engie_dynamic`, `totalenergies_dynamic` and +`eneco_eddy`. Read it before you update. Run `tools/predeploy_check.py` with the +bare options (`jq '.data.options'`) first. + ## 0.19.2 **A patch bump: one fix for charging from the grid under the peak limit, and no option changes.** @@ -7,7 +116,7 @@ No option was added, removed or renamed, and no default changed (86 options). **Existing installs need no option change:** options that start 0.19.1 also start 0.19.2. The change below applies with one inverter or two. -**Charging from the grid now stays under this month's peak.** A late meter reading near a period's end could push a grid charge to full power. That could take the 15-minute period over the peak. Now the charge is cut back, with one inverter too. The peak guard line no longer says "under" when the period is over. +**Charging from the grid now stays under this month's peak.** A late meter reading near a period's end could push a grid charge to full power. That could take the 15-minute period over the peak. Now the charge is cut back, with one inverter too. The peak guard line no longer says "under" when the period would go over the limit. Contents of this release, in merge order since 0.19.1 (`2932f3a`): diff --git a/goodwe_controller/DECISIONS/FCST-03-the-hours-not-yet-published.md b/goodwe_controller/DECISIONS/FCST-03-the-hours-not-yet-published.md new file mode 100644 index 0000000..90b0ad7 --- /dev/null +++ b/goodwe_controller/DECISIONS/FCST-03-the-hours-not-yet-published.md @@ -0,0 +1,109 @@ +# FCST-03 - the hours the feed has not published, and what may not rest on them + +**Status: SHIPPED.** Tests: `test_price_forecast.py`, `test_price_history.py`. +Written down by MAINT-20 from the long module comment of `app/price_forecast.py`. +Follows specs.md 8.3: "Day-ahead prices arrive once daily. Until they do, unknown +hours use yesterday's profile and are flagged `provisional: true`, and no +irreversible action may be taken on a provisional price. Real-time / imbalance +products are treated as high-volatility: arbitrage thresholds widen." This module +owns the first two; the widening is in price.py (`Uncertainty`, `widen_factor`, +`required_spread`), spent by `static_tariff.TariffPlan.arbitrage()` as one more +refusal on its existing ladder. Persistence of yesterday: [[FCST-04]]. + +## The flag is never passed through + +A carried point and a published one are the same three floats and a bool. +`PricePoint(..., provisional=y.provisional)` copies yesterday's `False` onto +today's guess: stale data presented as current (SAFETY-18's defect one layer up). +So: + +- `_carried()` has no parameter for the flag; `provisional=True` is a literal in + one place. +- A published point is passed through by identity (`filled.points[i] is + fetched.points[j]`, checked by the test), never copied. +- `_audit()` re-derives the expected flag for every emitted point from the hours + the feed published and raises on a mismatch. It never corrects (a module that + quietly fixes its own provenance cannot be trusted). +- test_price_forecast.py walks this file's AST and fails if any `PricePoint(...)` + passes anything but a literal for `provisional`. + +"Published" does not mean "firm": 9.3 carries `provisional` per point because a +real-time product's current-interval print is published AND provisional, so the +audit's rule for a published point is "whatever the feed said". No shipped +plugin emits one since [[TARIFF-07]]; the rule is a property of the format and +weakening it would let the first imbalance feed arrive mispriced and flagged firm. + +## The audit is a provenance check, not a completeness check (FCST-05) + +Rules: a PUBLISHED point must be one of `published[u]` by identity and be +emitted once, with the feed's own flag; a CARRIED point's hour must not be in +`published` at all, flag exactly True, and no second point in that hour. This +replaced FCST-03's "at most one point per hour" count, which cannot express a +quarter-hourly feed. FCST-04 claimed two strengthenings; both were false: a +differential over every published mask on 3 hours crossed with every output of +length 0-3 found zero divergence either way (OLD raised/NEW accepted 0, OLD +accepted/NEW raised 0). The one real change is the intended relaxation (several +distinct published points per hour). + +The audit does not check completeness: fed the decimation defect (four points in +an hour, only the last emitted), two of four, or an empty output, it accepts. +What stops decimation is the point COUNT and IDENTITY checks in the two test +files. It cannot be made complete from its arguments: `published` is over the +whole unwindowed read while the fill emits `[since, until)`, so a published point +outside the window is legitimately absent. FCST-05 built "every published point +is emitted" (+162 B) and it took the 1 h, 12 h and 48 h horizon checks red on a +healthy feed (a real change: the differential scored it 49 cases stronger), +so the rule as stated is wrong, and +asserting a completeness the instrument lacks is MUTATION.md habit 16's defect. + +## The fill keys on the wall clock + +For each whole hour of the horizon the feed has not published, take yesterday's +value for the SAME BRUSSELS WALL-CLOCK HOUR, not `t - 86400 s`: a 23-hour day has +no 02:00 and a 25-hour day has two, so a fixed 24 h shifts every carried hour +across a transition. On a 25-hour day both 02:00s are filled from yesterday's +single 02:00 (both guesses, both flagged); if yesterday was a 25-hour day, the +first of its two 02:00 points is used. An hour with no published price and no +yesterday value is left out, not filled with a neighbour, mean or zero: 11.4 +applies (`Absence`, then the static profile). If nothing can be produced the +result is `no_data`, not an empty series. + +A sub-hourly feed is not decimated: the hourly walk decides which hours need +filling, never what a published hour may hold. Belgian day-ahead settles per +quarter since 2025-10-01 (96 points a day), so every published point in the +window is passed through and a carried one is added only for an hour with no +published coverage. `{_utc_hour(p.start): p for p in r.points}` reads ordinary +and keeps the last point of each hour; FCST-04's first caller shipped it, the +series came out 15 points long, the spread fell from 0.143 to 0.014 EUR/kWh and +the cheapest "hour" became a `:45` point. `_published_by_hour()` holds a tuple +per hour for that reason. + +The window is whole hours at both ends: `since` floored (the current hour is +still tradeable) and `until` floored too; with `until` compared raw a 48 h horizon +asked at 08:17 emitted 49 hours. `yesterday` has no default (a missing yesterday +and a useless one must not look alike). The fill is idempotent and re-runnable +(5.4 late publication, 14-16 h CET): published hours win, and the series is +rebuilt, not patched. TARIFF-11 log line: carried HOURS, total hours and +published POINTS are three different quantities (a quarter-hourly feed made it +read "33 of 141 hours" for 48 h). + +## AC2: the irreversible-action gate + +An irreversible action is a value, `Irreversible`, carrying the hours it commits. +`price_permits()` looks at exactly those hours and answers with a `Gate` carrying +evidence on the permitting arm and a reason on the blocking one (the `_Outcome` +shape). It blocks when the read observed nothing (11.4), an hour has no price, or +an hour is provisional, and names the hour. The maintenance drain's window is +derived from `maintenance.DRAIN_TIMEOUT_MIN` / `CHARGE_TIMEOUT_MIN`, so the gate +and the phase machine cannot drift. `_window_hours()` is separate so the gate and +its caller agree on "the hours it commits". + +What is NOT gated matters more: the maintenance cycle is a calendar obligation +(12.2: a cycle that silently never fires looks like one not yet due), and a site +with `price_provider: off` (the default) has no series. Routing every start +through a gate that blocks on `no_data` would stop the monthly cycle on every +default install. So `begin_phase()` is the entry point for a start JUSTIFIED BY A +PRICE; main.py's calendar path is untouched, and OPT-04 / MAINT-05 own which a +site runs. Reversible actions (self-consumption modulation, the loop) are not +gated: 6 says absent machinery must not disable the loop. Read this before wiring +`begin_phase()` in. diff --git a/goodwe_controller/DECISIONS/FCST-04-the-day-that-is-carried.md b/goodwe_controller/DECISIONS/FCST-04-the-day-that-is-carried.md index d0e85bd..2576c03 100644 --- a/goodwe_controller/DECISIONS/FCST-04-the-day-that-is-carried.md +++ b/goodwe_controller/DECISIONS/FCST-04-the-day-that-is-carried.md @@ -188,3 +188,64 @@ still raises; so does the same published point emitted twice; and a carried point sharing an hour with a real print now raises too, which the count never looked at. Each of those three is driven by its own mutation in `test_price_forecast.py`. + +## 5. Store details (moved here from `app/price_history.py` by MAINT-20) + +**Why a file, and which one.** FCST-03 shipped the fill with no caller, so no +series ever contained a carried point. Without a yesterday that survives a +restart, the mornings after an operator touched something spend the whole +window on the truncated series. The cache is `/data/price_history.json`, not +`/data/state.json`: `Store.set()` rewrites the whole of state.json on each +write, and the refresh runs every `TARIFF_REFRESH_S` (1800 s), so 48 extra +rewrites a day would go through the file whose corruption reads as "no +maintenance ever ran". Separate file, separate blast radius. The directory is +not created (`tariff.PeakStore`'s rule): `/data` is always mounted, so a +`makedirs` could only succeed where the durable volume is missing and build a +cache on the ephemeral layer. "Cannot persist" is one question asked after the +attempt: see [[FCST-06]]; `self._days` never holds a point the file does not. + +**Brussels date, not UTC.** 23:30 UTC in July is already tomorrow in Belgium. +`_yesterday_of()` subtracts on the local `date`, not the instant: the old +`brussels(stamp) - timedelta(days=1)` took 24 absolute hours and, at +2026-03-30T00:30+02:00, returned 2026-03-28 (a two-day-old profile flagged as +yesterday's). Rows are keyed by instant, not local hour, so the two 02:00s of +the 25-hour October day both survive and `fill_from_yesterday()`'s own rule is +the only collapsing rule. The merge key is the validated instant; the old +string key merged by spelling and sorted the `+01:00` 02:00 before the `+02:00`. +`KEEP_DAYS` is a sort-order cap, not a horizon: nothing is deleted by arithmetic +against the clock (a jumped clock, WITNESS-03, would otherwise delete everything +on the one morning the fill is for), and the two dates `yesterday()` can be +asked for are pinned on top of the cap, which can only retain. + +**One validator (FCST-05, MEDIUM).** The read path was defensive about every +field and the write path trusted `q["start"]` to be hashable and sortable. An +int, null, dict or list `start` raised `TypeError` out of `carry()` on the day +being written; `run_tariff_refresh` caught it, so the add-on stayed up and every +refresh failed for good (the plan retires after `TARIFF_MAX_AGE_S`, arbitrage and +the cheap window go off, and `_remember()` raised before `_save()`, so no +self-heal). `_point_of()` is now the one standard for both paths and returns +`None` rather than raising. + +**A broken cache is a short horizon, never a controller that will not start** +(6: absent machinery must not disable the loop). `PriceHistory(...)` and `.words` +are unguarded in `Controller.__init__`. Shipped defects, all fixed: + +- `_load()` caught `json.JSONDecodeError` and `OSError` only; one non-UTF-8 byte + raised `UnicodeDecodeError` (a `ValueError`, not an `OSError`) and the add-on + did not start (FCST-05, HIGH). The handler names the `ValueError` family. The + schema version is compared with its type (`1.0 == 1` in Python). +- `{"days": {"2026-01-14": "oops"}}` loaded as healthy and raised + `AttributeError` out of `words`, `yesterday()` and `resolve_tariff()`. + `_usable_days()` drops such records, and counts (not drops) malformed rows + inside a kept record so the store stops reporting health it lacks. +- `words` printed `f"{d} ({_n_points(rec)} h)"`: points under the hour's name + (`'2026-01-13 (96 h)'` for a 24-hour quarter-hourly day, FCST-05 LOW). Both + numbers are printed now; only rows that are points are counted. `words` must + never raise, `_held()` is total. +- A failed merge is undone from a deep copy (a one-level copy shares the day + records the merge rebinds). + +`NO_HISTORY` is a name, not `None`, so `history=` can be required: `None` +meaning "skip the fill" is how a caller silently gets the truncated horizon. +`carry()` records first, then fills, so the filled series exists only as a +return value that the store never sees again (rule 1). diff --git a/goodwe_controller/DECISIONS/OPT-23-28-the-night-charge-covers-the-dear-hours.md b/goodwe_controller/DECISIONS/OPT-23-28-the-night-charge-covers-the-dear-hours.md new file mode 100644 index 0000000..04dac7e --- /dev/null +++ b/goodwe_controller/DECISIONS/OPT-23-28-the-night-charge-covers-the-dear-hours.md @@ -0,0 +1,11 @@ +# OPT-23 and OPT-28 - the day plan counts the sun right, and the night charge covers the expensive-hour shortfall + +**Status: for review.** Tests: `test_opt23_28.py`. + +`dayplan.plan()` now buys, for Leftover solar batteries, what the house uses in the expensive hours less the sun counted for each hour, shared by capacity, and reads a state-only forecast from now for every span (OPT-23). + +- **Shortfall.** `sum_h max(0, house_h - trust x sun_h)` over every expensive hour, `/ EFF`, from the floor. It replaces the use over the run with no sun counted, which on a day and night tariff bought for the whole day (G1, `reports/UX-69-design-report.md`). The sun before the run is unchanged. A run with its dear hours after sunset gives the old number. +- **House use.** An hour with sun is read at no less than the night base, as the sun sum reads it (the profile is net of the sun). An hour without is read as measured. An hour with no forecast counts no sun. +- **Several batteries.** One total, shared by capacity with the ceiling passed on (`dayplan._spread()`). With the same floor and ceiling for all, they reach the ceiling together, so a shortfall that one battery cannot hold fills both to the same level, not one first. More than all hold: all go to the ceiling, `Plan.uncovered_kwh` is the rest, carried by another battery (Cheap hours, Solar first) or the grid, as the screen says. +- **Cheap hours** still fills up and is not counted in the share; **Solar first** still never charges from the grid. The mode help says so. +- **Not done:** the run length (OPT-25); netting what a Cheap hours or Solar first battery will hold against the shortfall. diff --git a/goodwe_controller/DECISIONS/OPT-27-last-fresh-forecast-for-the-night.md b/goodwe_controller/DECISIONS/OPT-27-last-fresh-forecast-for-the-night.md new file mode 100644 index 0000000..ae9d68a --- /dev/null +++ b/goodwe_controller/DECISIONS/OPT-27-last-fresh-forecast-for-the-night.md @@ -0,0 +1,46 @@ +# OPT-27 - the night plan keeps the last fresh forecast + +**Status: done on `release/1.0`, not released.** Tests: `test_opt27.py`. + +**The defect.** A today sensor is stale after 3 h (`FRESH_S`) or when it is from +before local midnight. Forecast.Solar stops updating in the evening, so at night +every Forecast.Solar site had no forecast: a price mode's night cheap run needed +1.5x the cycle cost (TARIFF-24 round 2) and Leftover solar planned nothing. +Separately, one missed poll (`unavailable`) dropped the forecast at once. + +**Where: `solarfc`, not the plan.** The last fresh reading of each sensor is +kept in `solarfc.Hold` and `read_sum()` uses it when a sensor has none. A hold +at the plan (`_wx_track`/`weather_held`) can only keep the 1.5x gap off. The +day plan, Leftover solar's levels, Home and the Forecast tab all read +`Controller.solar_fc`, so one kept forecast makes control and every screen agree +(UX-53) with no second rule to keep in step. `weather_held()` is unchanged: it +still keeps a run under way when a read is lost. + +**The rule** (`Hold.stand_in()`). A kept reading is for one day. It covers a +sensor that is missing or stale, never one with a bad value or unit, and: +- for up to `FRESH_S` when the sensor is missing: a short outage, any time of day; +- from the end of the bell (20:00 local, `BELL_TO_H`) until local midnight, if the + reading is at most `KEPT_MAX_AGE_S` (12 h) old when used; after midnight the + tomorrow figure is today's until the bell starts (06:00), up to 15 h old + (OPT-34 changed this from midnight + 3 h); +- the day it is for: a today figure only on that day, a tomorrow figure as + tomorrow until midnight and, if the today sensors are still stale, as today's + until 06:00 (all sensors of the sum are needed, as for any sum); +- until the next fresh reading, which replaces it at once. + +So a figure is never used for another day and never older than 12 hours (`KEPT_MAX_AGE_S`), except the morning case above: a 17:05 reading serves until 06:00 (12 h 55 min), a 09:05 reading only until 21:05. Until OPT-34 the figure ended at 03:00, and a charge under way lost its level there; OPT-34 closes that limit. +Daytime is as before: a sensor that stopped is stale after 3 h, and no forecast +all day is still no forecast (the night run still needs 1.5x). + +**Where it is stored.** The controller's store, key `solar_fc_hold`, so an +evening restart keeps it. It is written when a reading's figure changes or +every 15 min (`SAVE_EVERY_S`), not on every poll. A damaged entry is dropped. + +**The screens.** `Forecast.kept` is true when a reading older than `FRESH_S` is +in use; the forecast line then reads "Solar forecast read 17:05, kept for +tonight." (`home_words.solar.kept`, EN and NL). The event log says +"read, kept from 17:05". + +**Not changed.** Control with modes off: it does not use the forecast. The +screens still show the line, so it says "kept", not "used for the plan". A +forecast that stays fresh. The hold does not look at the weather, only at the clock and the day. diff --git a/goodwe_controller/DECISIONS/OPT-34-the-night-charge-keeps-its-level.md b/goodwe_controller/DECISIONS/OPT-34-the-night-charge-keeps-its-level.md new file mode 100644 index 0000000..97e895c --- /dev/null +++ b/goodwe_controller/DECISIONS/OPT-34-the-night-charge-keeps-its-level.md @@ -0,0 +1,69 @@ +# OPT-34 - the night charge keeps its level when the kept forecast ends + +**Status: done on `release/1.0`, not released.** Tests: `test_opt34.py`. + +**The defect** (OPT-27 rig test, F-1). The kept evening figure ended at midnight + 3 h. +With the sensors silent from 17:05 the Leftover solar battery's level went with +it in the middle of the charge: A ended at 49.8 % against 79.9 % with a fresh +figure. Whether Forecast.Solar refreshes after midnight on a live site is not +known, so the add-on does not depend on it. + +**The rule.** +1. *A charge under way keeps its level* (`Controller._latch_charge()`). While + there is a forecast (fresh or kept), each battery that charges from the grid in the + cheap run takes its planned level into a latch (store key `night_charge_latch`). + With no forecast, `plan_targets()` gives the latched level, and the wide + no-forecast gap does not refuse the next slice of the run (`weather_held()`, + as TARIFF-24 does for the run under way). While a + forecast is there the latch follows the plan, up or down as a fresh figure moves + it. It lasts through every cheap run on and one price slot after the last, and it + ends at once when its end time passes, whatever the run state (it is also + ignored at a restart if it has ended). A battery that is switched off or + changed to another mode leaves it. + *Night bound (manager's call, 2026-10-31):* a run carries the latch on only if + it starts by the bell start (06:00 local) after the latch began; that run + finishes, plus one slot. Chosen over a fixed 12 h because the night tariff + (22:00-07:00) needs the 06:00-07:00 slice and 12 h from a late start would + reach into the day. With dynamic prices a chain of cheap slots cannot carry a + night's level into the next day. +2. *The evening's tomorrow figure is today's until the bell starts*, 06:00 local + (`BELL_FROM_H`, `solarfc._bell_start()`), not until midnight + 3 h. + Nothing has been produced yet, so it is still the whole day. +3. *Up to 15 h old for that case only* (`KEPT_MORNING_MAX_AGE_S`): a 17:05 + reading is 12 h 55 min old at 06:00. It must have been at most 12 h old at + midnight, so a 09:05 reading, unusable since 21:05, does not come back for five + minutes after midnight. Same-day figures and the daytime rule keep 12 h and 3 h. +4. A fresh figure replaces the kept one at once, as before. + *Rig F-1 fix:* once every today sensor has given a fresh reading after + midnight, the evening figure is no stand-in for that day any more + (`Hold.replaced()`). When that reading goes stale (3 h), it is kept or not + as any today reading (OPT-27), so it is "no forecast" at night and the latch + keeps the level it set; the older 17:05 figure never comes back. Rig: A + stopped at 69.5 % after a brighter 02:00 figure, then charged back to 94.4 % + from 05:15 when the 17:05 figure returned. Test: `test_opt34.py` F-1. + +**Step 1 alone is enough for the owner's shape, steps 2-3 are still needed.** +With the window put back to midnight + 3 h the owner's night still ends as a +fresh figure ends it (A 86.9 %, B 95.0 %), because the charge had started at +22:00. But a charge that has not started by 03:00 (the price dips at 04:00, or +the headroom is taken) has no level to latch: A ends at 38 % against 78 %. The +plan, the screens and a restart also need the figure itself. + +**Screens.** "Solar forecast read {at}, kept for tonight." is still true until +06:00 and unchanged. After the figure ends, with a charge still going on: "No +solar forecast now. The charge under way goes on to its planned level." / +"Nu geen zonneprognose. De lopende lading gaat door tot het geplande niveau." +(`home_words.solar.latched`, only while a cheap run is on and a latched battery is below its level). + +**Not used.** A figure over 15 h old; a morning figure at night (12 h); a figure +for another day; a bad value or unit; and no latch for a charge that has not +started, for a battery that is not charging, or after the cheap runs have ended +(plus one slot). + +**Not changed.** Charge sizing, the peak guard, Cheap hours, modes off, arbitrage. +A fresh figure that LOWERS the level during a charge still lowers it, as before. + +**Mutants** (all red in `test_opt34.py`): no latch; window back to midnight + 3 h; +15 h without a limit; no "usable at midnight" rule; the latch without the narrow +gap, without its save, or not kept through the next run; the window 6 h after +midnight instead of 06:00 local (DST); 15 h cut to 12 h; the new screen line off or shown without a cheap run; no expiry check; no pruning of a changed mode; no night bound; a left-the-plan battery keeping its level; no `replaced()` check (F-1); `replaced()` with one of several today sensors (any, not all). diff --git a/goodwe_controller/DECISIONS/SAFETY-07-peak-guard.md b/goodwe_controller/DECISIONS/SAFETY-07-peak-guard.md new file mode 100644 index 0000000..1d4ef8f --- /dev/null +++ b/goodwe_controller/DECISIONS/SAFETY-07-peak-guard.md @@ -0,0 +1,166 @@ +# SAFETY-07 - the quarter-peak projection guard (with SAFETY-24 and CFG-03) + +**Status: SHIPPED.** Tests: `test_peakguard.py`. Written down by MAINT-20 from +the long module comment in `app/peakguard.py`, so the module can stay short. +Later changes have their own notes: [[SAFETY-25]], [[SAFETY-32]], [[SAFETY-45]], +[[SAFETY-79]], [[SAFETY-81]], [[SAFETY-87]], [[SAFETY-90]], [[SAFETY-125]], +[[SAFETY-126]]. + +## The hazard + +The capacity tariff bills the month's single worst 15-minute average offtake, +with no averaging and no forgiveness. specs.md:583 records one 4998 W quarter +that moved a monthly peak from 4832 W to 5013 W and billed the whole month at +it. SAFETY-18 alarms on a frozen meter (`meter_freeze_s`, 300 s) but a frozen +meter can rail the command in 75 s (freeze.py's table, `max_w: 5000`, meter at +3000 W or more), against a 900 s quarter. The guard is the cap on that. It +reads freeze.py's `looks_frozen()`; it is not a second freeze detector. + +## The mechanism: one `lo` limit at `P_PEAK` + +specs.md 10.1 row 2 wants pre-emptive discharge near the limit and a hard clamp +on charging above it; 9.1 says discharge and export are never capped by the +peak guard. A lower bound on the setpoint is the only shape that does both: a +floor cannot stop the loop discharging harder (so 9.1 holds by structure), it +clamps charging when above the standing charge, and it forces discharge when +above zero. + +- `P_PEAK` (50) is above `P_LOOP` (10) and below `P_MAINTENANCE` (60) and + `P_SAFETY` (100). arbiter clause 2 applies only limits at or above the + winning set's priority, so a P_SAFETY 0 W stop is not pulled off zero. +- A limit does not out-vote another limit: clause 2 takes the most restrictive + with no priority between limits. The floor (`lo >= 0`) and the maintenance + charge-only shaping (`hi <= 0`) therefore resolve to `lo > hi`, a + contradiction, which the arbiter answers with 0 W (battery parked in the + worst quarter). The rule for two crossing limits is suppression, not + priority: main.py drops the shaping while the guard is engaged + (`shaping_suspended()`). The floor is clamped to `achievable_w` to close the + only other crossing pair (the P_SAFETY rating bounds). +- Arbitrage arrives as a SET below `P_PEAK`, so clause 2 binds it with this + floor without any edit here (9.1: peak shaving outranks arbitrage). + +## The floor is derived + +With `projected = (acc + held * remaining) / 900`, the watts that must come off +the meter are `excess_w = 900 * (projected - ceiling) / remaining`, and the +floor is `floor_w = batt_w + excess_w`. + +1. The claim is filed while the floor is positive, not while the excess is. + `excess` falls to zero as soon as the battery is already removing it, and a + guard that released there handed the actuator back and re-engaged next + cycle. The first cut did that on 2026-08-20's billed quarter: 17 + engage/release cycles in 900 s, and the quarter still ended 49 W over. + Thresholding `batt_w + excess` (the offtake with the battery at zero, less + the level the quarter can afford) decays continuously, so there is no + hysteresis constant. 10.2: an EMS that oscillates is worse than none. +2. The gain 900/remaining is 1.0 at the start of a quarter and grows. The guard + is gentlest when its evidence is weakest. No minimum-evidence threshold was + added: it would delay the guard on a sustained rail from second zero. +3. `batt_w` is the measured battery power, not the command. The bill is what + flowed, and a floor built on the command winds up to the rail when the + inverter cannot follow; one built on the measurement stops rising when + delivery does. + +## The ceiling + +`ceiling_w = max(peak_cap_w, the month's banked peak)`. Once a month's peak +stands at 5013 W, holding later quarters to 3500 W buys nothing. Only an +`OBSERVED` reading may raise it; `NO_RECORD` and `NOT_OBSERVED` leave it at +`peak_cap_w`, so a peak store that cannot be read makes the guard stricter, +never looser. That is also why the guard does not use `headroom_w()`: a peak +read as 0 W returns full headroom, which is a guard that is not there. +SAFETY-79 aims `peak_margin_w` under the ceiling; the ceiling stays the real +limit for reports and the bill. + +## Blindness comes first + +The projection is graded only if the quantity under it was measured. Three +ways it was not, checked before grading: + +- No open block: a fresh averager answers an all-zero `Projection`, which sits + under every ceiling. That is "no quarter", not "the quarter is fine". +- Coverage: a meter gap longer than `max_hold_s` leaves seconds the averager + does not accumulate (holding a dead meter's value forward is how a fabricated + peak gets onto a permanent record), so the projection reads low. The + threshold is `max_hold_s`, not a new number. SAFETY-90 fills gaps once + readings return. +- Frozen: samples keep arriving, coverage is perfect and the projection is a + confident zero while the real meter is at the rail. + +The blind action is `floor_w = 0.0`: no importing while the billed quantity +cannot be observed. It is one-directional (forbidding charge only lowers +offtake), so a false blindness costs a charge and never pack energy. + +## Why it trips where SAFETY-18 only notifies + +The healthy-day false-engagement rate is zero (2026-08-23, 95 quarters; +test_peakguard.py re-measures it from the committed capture). The action is +level-triggered and unlatched: it is re-derived each cycle and releases when the +projection drops, so a false engagement costs one cycle, not a parked battery. +It is a limit, so every safety stop outranks it. It is the only guard whose +trigger is the billed quantity itself. + +## Saturation is reported, not alarmed as a bug + +10.3 treats an out-of-range setpoint as a bug (SAFETY-03). A peak floor above +the inverter rating is not: it is a 5900 W house on a 5000 W inverter (2026-08-20 +15:45). The floor is clamped to `achievable_w`, the decision carries +`saturated`, and main.py raises one notification. + +## SAFETY-24: the energy backstop the guard shipped without + +The graded branch can force discharge (878 Wh in a day on the 2026-08-20 replay) +and nothing below it consulted the state of charge; the BMS was the only +backstop, which protects the pack and not the operator. `decide()` now takes +`soc` and `soc_floor` and `discharge_permitted()` answers. + +1. Not a trip. When it binds, the guard stops asking for more discharge and + commands nothing; every stop in 11.2 still outranks it. +2. The action is `floor_w = 0.0` (forbids charging, forbids nothing else). A + `lo` limit cannot cap discharge, and a negative floor would destroy that. + The loop is not bounded: self-consumption may still drain the pack past the + floor. [[SAFETY-25]] measured it (2026-08-20, 19:00-22:00 UTC, 830 W mean: + the pack ends at the BMS floor, 10.0 %, whatever `peak_soc_floor` is) and the + owner decided on 2026-09-02 that the BMS stays the only backstop for + demand-driven discharge, a loop-side reserve being deferred to ALLOC-01. +3. Graded only after `wanted > 0`, where a forced discharge is asked for. +4. An unreadable SoC is not a full battery. `soc=None` and `soc_floor=None` are + floor-limited. Both arguments are required keyword-only, no default (TEST-14). +5. Failing to shave is not silent: `floor_limited=True` and one notification. + It excludes `saturated`, since it asks for nothing. + +The action log says "the peak reserve", not "the SoC floor": `peak_soc_floor` +(“Peak reserve”) bounds this guard only, and the wider words would contradict +a loop that is draining the pack at that moment. + +## CFG-03: two options, mirrored once + +- `peak_soc_floor` is the EMS floor, not `maintenance_soc_floor` (the monthly + recalibration's drain target); sharing a key would couple features whose + right values differ. Default 20: the reference pack is 300 Ah at 51.2 V, + DoD 90, 13.8 kWh usable; the replay's 878 Wh is 6.4 %; a floor 10 points above + the 10 % cut-out leaves about 1.6 worst days of room. +- `bms_soc_floor` is the pack's own cut-out (register 0x0560, 10 on the + reference unit). It exists so validate.py rule 8 can refuse a + `peak_soc_floor` at or below it. +- Both defaults live once in `PEAK_OPTION_DEFAULTS` (TARIFF-02's pattern), + imported by validate.py and main.py and pinned to config.yaml by the test. + The module reads no option itself. +- `maintenance_soc_floor` is the ambiguous name of the two. Decision: keep it, + carry the distinction in DOCS.md; revisit if a third floor appears. +- SAFETY-07 itself added no option: `peak_cap_w` was already validated + `int(500,15000)`, and 14's `capacity_peak_limit_kw` stays declined (a kW key + beside a W key is the mismatch trap). + +## Per-pack floors (SAFETY-32) and phases + +`bms_soc_floor` became a roster key; `pack_floor_pct()` conjoins it with the +site's EMS floor, and rule 8 names the unit it refuses. `peak_soc_floor` stayed +one number for the site, so with cut-outs of 10 and 18 the operator sets it +above 18; the per-pack EMS floor is the open question in [[SAFETY-32]]. The +`min()` over measured SoC (`Controller.pack_floor_soc`) is still right for the +site verdict; `pack_permissions()` is the per-pack answer. + +Nothing here reads a per-phase quantity: the sum/max/net choice is an open +owner question, defaulted to NET inside `billed_w()`, so this file does not +change whichever way it is answered. diff --git a/goodwe_controller/DECISIONS/SAFETY-123-hour-slices-step-in-utc.md b/goodwe_controller/DECISIONS/SAFETY-123-hour-slices-step-in-utc.md new file mode 100644 index 0000000..a3a8a18 --- /dev/null +++ b/goodwe_controller/DECISIONS/SAFETY-123-hour-slices-step-in-utc.md @@ -0,0 +1,35 @@ +# SAFETY-123 - the day plan's hour loop steps in UTC + +**Status: for review.** Tests: `test_safety123.py`. No option, no text change. + +`dayplan._hour_slices()` stepped to the next local hour by adding an hour to +the local wall clock and converting back. Where the zone has no tzdata +(`tariff._EUTimezone`: the CI image, a dev host, an image that loses tzdata) +the missing 02:00 of the spring change reads as summer time, which is +00:00 UTC again. At 28 March 2027 00:00 UTC `t` did not move and the list +grew until the process died. Control and the web page share one event loop, +so the whole add-on froze. `dayplan.plan()` calls it on every plan, before it +looks at a battery's mode, so a site with all batteries on Solar first reaches +it as well, if the prices give a plan. + +The shipped image has tzdata (`amd64-base:3.20`), where this zone gives the +right answer, so a running add-on is not known to be hit. The fix is for any +host without it. (AUDIT-01 F4, card SAFETY-120, is the same defect.) + +**Decision.** Each step adds the time left in the local hour to the UTC +instant. It is at least a microsecond, so `t` always advances and the loop +ends, in any zone, with or without tzdata. A day of 23 hours gives 23 slices +and a day of 25 hours gives 25, each at most an hour, none across a local +hour. On ordinary days the slices are the old ones. + +**One change on the autumn day.** The old function (with tzdata) made one +two-hour slice for the doubled hour 02:00-03:00; it now makes two of an hour, +both labelled 2. `plan()` sums per slice, so only a forecast with sun in that +hour could tell the difference. There is none. + +**Other loops.** The other code that walks local hours or days steps in UTC +or on the calendar date (`static_tariff`, `price_forecast`, `price_history`, +`price`, `maintenance`, `homeview.local_day`, `solarfc._bounds`), or has a +fixed bound. None can stall. One known inexactness stays, outside this fix: +`solarfc` spreads a state-only forecast over hours since local midnight, so on +the two change days its sun curve is an hour off the clock. diff --git a/goodwe_controller/DECISIONS/SAFETY-126-landing-error-is-not-an-overrun.md b/goodwe_controller/DECISIONS/SAFETY-126-landing-error-is-not-an-overrun.md new file mode 100644 index 0000000..feda5a4 --- /dev/null +++ b/goodwe_controller/DECISIONS/SAFETY-126-landing-error-is-not-an-overrun.md @@ -0,0 +1,47 @@ +# SAFETY-126 - a landing error on the aim is not an overrun + +**Status: FIXED on release/1.0 (after 0.19.2).** Tests: `test_safety126.py`. + +**Cause.** A grid charge under the peak limit is sized to land the 15-minute +period on the aim (SAFETY-125), so the period ends 0-7 W over it. In the last +second `peakguard.excess_w()` multiplies that by 900 / remaining (3 000 at +0.3 s left, 90 000 at 0.01 s), so `decide()` wants tens of kW +(`wanted > 0`, peakguard.py `decide()`). With every battery under the peak +reserve that is floor-limited at 0 W, and `check_peak_floor_limited()` +alarmed on it at once: SAFETY-87 graded only the case where another battery +could give. With one battery able, the guard asked it for its rating for that +cycle and the Batteries "Why" said "The peak limit needs power". The +add-on's own charge is not the cause: `wanted` already counts it as +stoppable. + +**Rule.** Each cycle the guard wants power, `Controller._peak_landing()` +asks `peakguard.misses_aim()`: with the batteries at 0 W from now on, does +the period end more than `ALARM_MISS_W` (10 W) over the aim? This is +`load_too_high()` with nothing to give, without `decide()`'s gain. If not, and +no real miss was seen since the guard took hold, the decision is +`within_aim`. Such a decision raises no reserve alarm (ERROR, +notification, red row), no red "Load too high" row, no "Why" peak and +no "Battery stopped limiting the peak" event hint. The claim, the floor and the +commands are unchanged. + +**Still alarms.** House load the batteries cannot cover under the reserve: +the period would end over the aim with the batteries at 0 W. A step load +alarms on the same cycle as before. SAFETY-87's case and SAFETY-81's 'Load too +high' notification are unchanged. + +**What the 10 W tolerance costs** (review, owner accepted 2026-10-31): +- A steadily rising load alarms a little later, once the projected miss passes + 10 W (3 000 -> 4 000 W over 45 min: 56 s later). With the default 50 W + margin the alarm still comes before the limit. +- With `peak_margin_w` 0, a period can end up to 10 W over the limit without + this alarm, the tolerance 'Load too high' already accepts (SAFETY-81). + +`misses_aim()` takes the house as meter plus batteries, so an export nets +off (`decide()`'s projection counts offtake only). A re-alarm while the loop's +draw on held batteries covers the house is gone too (test_safety126.py part 2, +modes off: 2 notifications, was 4). + +**Not changed.** The one-cycle floor itself: at a period's last instant the +guard still asks an able battery for its rating, or stops a charge, for a +landing error. That is control, outside the alarm path. Raise a card if it +should go. diff --git a/goodwe_controller/DECISIONS/TARIFF-02-price-provider-plugins.md b/goodwe_controller/DECISIONS/TARIFF-02-price-provider-plugins.md new file mode 100644 index 0000000..d032440 --- /dev/null +++ b/goodwe_controller/DECISIONS/TARIFF-02-price-provider-plugins.md @@ -0,0 +1,273 @@ +# TARIFF-02 - price provider plugins, and the canonical shape (with later additions) + +**Status: SHIPPED.** Tests: `test_price.py`, `test_price_service.py`. Written +down by MAINT-20 from the long comments of `app/price.py` so the module can stay +short. Related: [[TARIFF-07]], [[TARIFF-09]], [[TARIFF-10]], [[TARIFF-14]], +[[TARIFF-15]], [[PRICE-01]]. + +## Where it lives + +specs.md 2.2 puts this at `ems/ingest/prices/`; that layout is not being built. +Owner decision 2026-08-30: everything ships in `goodwe_controller/app/`. The +behaviour in specs.md 9.3 / 8.3 / 11.4 is unchanged and authoritative. + +## The canonical format (9.3) + +`{"prices": [{"start": "2026-01-15T00:00:00+01:00", "eur_per_kwh": 0.1834, +"provisional": false}]}`. `start` is tz-aware at the Brussels offset for that +instant, the price is EUR/kWh and all-in, `provisional` is per point (8.3: no +irreversible action on a provisional price). Three fields; a fourth would change +the wire format every store, fixture and captured body is written against. + +## Three rules the module is shaped by + +1. **Double-counting markup is refused, given a correct declaration.** A plugin + returns `RawPoint`s with the figure its source published and declares in + `capabilities().sources` which of {ENERGY, MARKUP, TAXES} it already holds. + `assemble()` (framework only) adds the complement from `price_markup_eur_kwh` + / `price_taxes_eur_kwh`. A plugin cannot reach those two numbers by any route: + `PluginOptions` raises `FrameworkOption`, and review of PR #16 wrote a + double-counting plugin three more ways (`opts._o`, `price_option()`, and the + public `PRICE_OPTION_DEFAULTS`), so `register()` also refuses any plugin + whose own code mentions a framework key (`code_leaks()`). The guarantee is + exactly this: the complement is added once *given a correct `sources`*. A + plugin declaring `{ENERGY}` for an all-in feed comes out one markup high and + no code can catch that. test_price.py checks that a plugin's docstring lines + (`SOURCES:` / `The framework ADDS:`) agree with its declaration, not that + either is true. `code_leaks()` strips docstrings before searching (they must + name the components) and fails open on missing source (a frozen build must + still start; CI asserts source is found for every shipped plugin). +2. **The product type is declared, not inferred from the id.** `capabilities(). + product` is a `PriceProduct`, rides on the series, and `volatility_regime()` + reads it (8.3: real-time products widen thresholds). A plugin id appears only + in `PROVIDERS`, `RETIRED_PROVIDERS` and its own class (AC2). No shipped plugin + declares `REALTIME` since [[TARIFF-07]]; `_WIDE`, `Uncertainty.REALTIME_PRODUCT` + and `WIDEN_FACTOR`'s 2.5 are kept, and test_price.py section 10 declares + REALTIME from a class outside this file to keep them live. +3. **"No price data" is a type, not an empty list.** 11.4: no data means static + profile, provisional plan, no arbitrage; all-zero prices mean arbitrage is + merely pointless. `PriceRead.series()` refuses an empty list; + `PriceRead.no_data(reason)` carries `.reason` and nothing else, so + `min(read.points)`, `for p in read.points` and `.get("points", [])` raise + `Absence`; `.match(*, series, no_data)` takes both handlers as required + keywords. Same shape as `sim/riglock.py` after TEST-04 (read its `Absence` + block before changing this). Two ways past it survive and are not worth code: + `vars(read).get("points", [])`, and `bool(read)` being True for both + variants (a falsy `no_data` would turn `if read:` into a guard by accident; + use `.match`). + +## Timezone without `zoneinfo` + +The Alpine image has no `tzdata`, so `ZoneInfo("Europe/Brussels")` would raise +on the client's Pi. Every instant is UTC and rendered at its fixed offset, so +the October 02:00-03:00 hour that happens twice is two instants (`02:00+02:00` +and `02:00+01:00`); a naive-local-hour key or a 24-slot array is wrong on both +transition days (test_price.py counts points on both 2026 transitions). The EU +rule: CEST from 01:00 UTC on the last Sunday of March to 01:00 UTC on the last +Sunday of October. Review of PR #16 matched it against `zoneinfo` over 149 040 +hourly instants 2020-2036 with zero mismatches. Residual risk is political (EU +abolishes the change): one edit in `brussels_offset()`. + +## The feed range (TARIFF-09) + +`generic_dayahead.fetch()` used to ignore `since`/`until` and request the bare +URL, so the feed answered ONE Brussels day and 8.3 dropped everything past today +as provisional: at 09:00 over a 48 h horizon 15 firm and 33 carried, at 23:00 1 +firm and 47 carried. `feed_days()` returns the Brussels calendar days (first, +last, both inclusive; the API takes dates) that the half-open `[since, until)` +touches: Brussels, never UTC (22:30 UTC in summer is already tomorrow); `until` +exclusive, so local midnight belongs to the day before; an empty or inverted +window collapses to one day. `api.energy-charts.info` capture: +`scenarios/energy-charts-be-price-2026-09-09_2026-09-10-twoday.json`. +`dated_feed_url()` refuses a base that already has a query string (appending +would give `?bzn=BE?bzn=BE&start=...`). An operator's `price_feed_url` override +is used verbatim and loses the range, on purpose: rewriting an operator's URL +reorders and re-escapes it, and both providers declare a second host +(`web-api.tp.entsoe.eu`) whose parameters are not `start`/`end`. The cost is +logged at fetch time, naming the horizon consequence. The URL rule is shared by +`generic_dayahead` and `custom_formula` (TARIFF-15), taking `plugin_id` as an +argument so AC2 holds. The "range is used" promise is not mechanically enforced +(a rule would be satisfied by an invented parameter name); test_price.py pins +which plugins consume the range as a literal roster. + +## Absence and the covering window + +See [[TARIFF-14]] for the cover table: a point covers from its own start to the +next point's start, never more than an hour; the last point gets the span of the +one before; sorted at construction. `covers()` delegates to `at()` (a second copy +of the rule is a second place to be wrong); `window_points()` grades only the +`start` against `[since, until)` and is not a coverage query. `at()` is a linear +scan over about 192 entries on the 30 s path; a bisect is not worth a second +way to be wrong. A duplicated start (not reachable through `load_prices()`) +gives the earlier entry a zero-width window. + +## Taxes components (TARIFF-18) + +`price_taxes_eur_kwh`'s default (0.06 before) is the sum of +`FLEMISH_TAXES_COMPONENTS_EUR_KWH`, each row one line of the Eneco residential +card for Flanders, September 2026, including 6 % VAT, in EUR/kWh; +test_price.py section 1b re-adds them. It is per DSO: the distribution row runs +5,28 c (Midden-Vlaanderen) to 6,70 c (West), so the all-in Flemish total runs +about 0.117 to 0.131; a site outside Antwerpen types its own. The default sits +near the low end on purpose: the scalar cancels in `max - min`, and its one +decision is the 17 negative-price guard in arbitrage.py, which tests the +absolute all-in sell price. Too high carries a genuinely negative hour above +zero (export into paid-for disposal); too low only declines a trade that would +have paid. `energiebijdrage` is a row of zero: from 1 August 2026 it is merged +into the bijzondere accijns, and the zero row makes the double-count a visible +decision. Moving a shipped default changes which configs start: TARIFF-17's rung +in validate.py refuses `price_taxes_profile` together with a non-default scalar, +comparing against the default (the Supervisor merges every schema key into the +stored blob, 70 of 71 keys on the reference install, CHANGELOG 0.9.0), so a +superseded shipped default (0.06) must still read as untouched, and the rung's +sentence names both figures. + +## One in-code mirror of option defaults (CFG-03) + +Option defaults live in config.yaml's `options:` and `schema:` and main.py's +`.get()` fallbacks. `PRICE_OPTION_DEFAULTS` is the single in-code mirror for the +price options: validate.py imports it, main.py has no `o.get("price_...", X)` +fallbacks, no dataclass carries a copy, and test_price.py section 1 reads +config.yaml and asserts equality key for key. `static_profile` (TARIFF-03), +`price_taxes_profile` (TARIFF-17), `price_formula` (TARIFF-15), +`price_export_formula` (TARIFF-16) and the OPT-04 keys live there for the same +reason, not in their own modules (a second dict would be a fourth location). +Details: + +- The tables / formulas are in `FRAMEWORK_KEYS`: a plugin reading one could add + taxes a second time per hour, or fold an injection curve into a consumption + price. `BANNED_IN_PLUGIN_CODE` derives from that set. +- `arbitrage_soc_floor` / `arbitrage_soc_ceiling` are not the maintenance keys + (a balancing cycle is not a trading reserve). +- `cycle_cost_threshold_eur` is 0.10 because 10.1's name says so, not because it + makes anything trade: on Belgian day/night numbers the static spread is about + 0.08, so a static-tariff site does not trade at this default (test_arbitrage.py + section 7). Whether 0.10, or FCST-03's 2.5 / 1.5, is right is an owner + decision. +- `price_service_token` is a secret (`SECRET_OPTIONS`); `scrub()` removes secret + values from any text built from an error. + +## Widening (FCST-03) + +One path, a set of active uncertainty sources (real-time product 8.3, no weather +data 11.4). The combination rule is `max`, not a product (2.5 x 1.5 would give a +threshold nothing on a Belgian tariff clears). The factors are policy defaults, +not derived (no arbitrage ever ran); they differ so `max`, `sum` and `product` +are distinguishable in a check. `widen_factor()` refuses a string (a silent 1.0 +would report a widening that is not applied). + +## Injection price (AC4, TARIFF-16) + +`export_price_eur_kwh()` is a separate scalar. It was said to be so because +nothing dynamic exists to route; that was false (Eneco publishes `0,1 x BELPEX-H +- 1,188`, ENGIE `(B x eSpot_15) - A`). The true reason: an injection curve has +its own coefficients and VAT treatment (injection is VAT-exempt, consumption is +6 %). TARIFF-16 added `price_export_formula`, read by `export_formula()` through +the same parser as the consumption leg (`parse_export_formula()` is +`parse_price_formula()` with its option name moved). `export_price_at()` falls +through to the scalar and ignores the index, correctly. A negative answer is a +real answer (Eneco's formula is below zero for any index under 11,88 EUR/MWh); +nothing floors it. Nothing under `app/` calls either reader to take a decision: +arbitrage prices the sell side as displaced import. A malformed leg is refused at +startup by validate.py; `taxes_profile()` and `export_formula()` still handle it +loudly at run time. `static_tariff` is imported lazily in `taxes_profile()` +because it imports this module at module scope. + +## Taxes at each point's own instant (TARIFF-17) + +`taxes_at()` takes a scalar or a period table (duck-typed on `price_at`, because +price.py must not import static_tariff.py). A non-flat component matters: ORES +bills `dag 13,27 / nacht 7,39` c, a 5,88 c gap coinciding with the cheap spot +hours, and flattening it understates the spread. `assemble()` must evaluate it +per point; hoisting the lookup out of the loop collapses the feature back into a +level (test_price.py section 24 grades which hour is cheapest, since a +spread-magnitude assertion passes on the hoisted version). On a scalar, `0.0 + +markup` is exactly `markup` in IEEE 754, so scalar sites get identical bytes. +The table wins over the scalar; validate.py refuses configs where both speak. + +## Retired providers (TARIFF-07) + +`RETIRED_PROVIDERS` exists so `eneco_eddy` and friends are not refused as typos: +they were shipped, the operator chose them, and every failed fetch fell back +silently to `static_profile`. No alias to `generic_dayahead`: they published +retail prices (`B x spot + A`) and it reads wholesale; an alias would invent the +site's two coefficients (behind the klantenzone login; CREG tariff-card pages +404, VREG unreachable). Each value is the id-specific half of the refusal (the +endpoint, what it answered on 2026-09-13, why no replacement can be built); +validate.py owns the framing. `register()` refuses at import to register a +retired id again, because `pid in PROVIDERS` going true silences the named +refusal; re-shipping is a decision recorded by removing it from +`RETIRED_PROVIDERS` in the same commit. See [[TARIFF-07]]. + +## Parsers (TARIFF-08) + +The only parser is columnar: `https://api.energy-charts.info/price?bzn=BE` gives +parallel `unix_seconds` / `price` arrays plus `unit`. The earlier row parser read +`doc["data"]`, a shape no feed ever published, graded by fixtures written from +the same guess, so every fetch ended `SourceUnavailable` and 11.4 fell back to +static silently. Two shapes, two functions: a parser that sniffs can decide +wrong, and a wrong-but-parseable response becomes a price. +`_points_from_columnar_json()` refuses: + +- unequal arrays (`zip()` would truncate the evening peak, and a series missing + the expensive hours is traded on while no-data switches arbitrage off); +- a unit other than the declared one (EUR/kWh under the same keys would be 1000 + times too small and `MAX_PLAUSIBLE_EUR_KWH` bounds only the large side); +- naive time: `unix_seconds` are UTC (`fromtimestamp(t, timezone.utc)`). + Rendering at the Brussels offset is `assemble()`'s, once, for every plugin. + +A `null` price is dropped and logged (the pairing is reliable, the source said +it published none; `_Series.at()` answers the hole with `Absence`); +`scenarios/fetch_epex_be.py` dropped 0 nulls in 12 months to 2026-08-31. The +feed is quarter-hourly since Belgium's 15-minute MTU (2025-10-01); nothing +assumes either granularity (test_price.py section 2b replays both). +`_validate_raw()` accepts negative prices (windy-Sunday hours are the year's best +charging; 17 keeps the regression) and rejects NaN, infinity, an implausible +magnitude, naive datetimes, and duplicate or out-of-order starts. + +## Transport and plugin details + +- `check_host()` splits the scheme before parsing the host. The first version + stripped `len("https://")`, which on `http://` ate two characters of the host + (`pi.energy-charts.info`), so a non-TLS feed failed the allow-list and the check + "plain http is refused" passed even with the TLS rule deleted. Found by + mutation in review of PR #16. +- The default transport imports aiohttp lazily so `manual_csv` works with no HTTP + stack. With `headers` (price_service only) it answers in the safe words of + `status_words()` and follows no redirects (the host is fixed; a redirect would + take the token somewhere the allow-list never saw). +- `manual_csv`: the CSV is all-in (the plugin cannot know what a typed number + contains); `load_prices()` warns when markup / taxes are set non-zero under a + plugin that sources them. No transport, aiohttp, network or tz database: it is + the offline escape hatch and the TEST-01 replay ingest path. +- `custom_formula` (TARIFF-15) declares `{ENERGY, MARKUP}`, not `ALL_IN`: a + card's `Tariefformule` is the energy component only (Eneco tabulates network + tariffs separately; ENGIE excludes network, transport, levies, excise), so + `ALL_IN` would drop about 12 c/kWh. The risk left is an operator folding + network charges into `offset`; the card self-check in custom_formula.py refuses + that, and grades the coefficients against the card, not the card against + reality. No expression evaluation: the string is settable over HTTP through + the ingress config page, so `eval` would be RCE in a root container holding + `SUPERVISOR_TOKEN`. It reuses `generic_dayahead`'s feed, URL and parser. It + keeps `scale=1.0`, so its `RawPoint`s carry EUR/MWh in a field called + `eur_per_kwh`: contained (destructured on the next line, never leaves the + method; `_validate_raw()` would refuse an escape). +- `startup_preview()` is a hook (TARIFF-15), not a line in `load_prices()`: only + the plugin knows what went into its arithmetic, and the index beside the + all-in figure makes a factor of 1000 obvious. It must not raise into the control + path; the id is recorded before the hook runs so a failing preview is not + retried every 30 minutes. +- `load_prices()` returns, never raises, except `FrameworkOption` (a plugin trying + to read the markup; turning it into "no price data" would hide the defect). +- `price_provider_id()` is one spelling of the selected id because the same + expression was written at three call sites (TARIFF-10). validate.py keeps its + own reader on purpose: it validates the raw blob. +- `price_provider_selected()` (TARIFF-10): the background refresh used to start + only when `arbitrage_enabled` (ships false) was on, so a configured site + fetched nothing (40 h soak: zero feed requests, `present: false` on all 802 + samples). It is not a trading switch: `arbitrage.enabled()` answers that. +- `price_service` (PRICE-01): `FALLBACKS` and `KEEP_LAST` are empty/False for + every earlier plugin, so `price_chain.py` is not used for them. + `TOKEN_REFUSED` is the one phrase the fallback chain reads to tell "the owner + must act" from "the server is down". `covers` lists complete days only; an hour + missing inside a day is a hole, not a free hour. diff --git a/goodwe_controller/DECISIONS/TARIFF-16-the-injection-leg.md b/goodwe_controller/DECISIONS/TARIFF-16-the-injection-leg.md new file mode 100644 index 0000000..9dee873 --- /dev/null +++ b/goodwe_controller/DECISIONS/TARIFF-16-the-injection-leg.md @@ -0,0 +1,102 @@ +# TARIFF-16 - the injection leg of a dynamic contract (`price_export_formula`) + +**Status: SHIPPED.** Tests: `test_price.py` (17 series). Written down by +MAINT-20 from `app/custom_formula.py` and `app/price.py`. Follows [[TARIFF-15]], +whose parser it reuses. + +## Why a formula, not the scalar + +`price.py` modelled injection as one scalar. The market publishes a line a +scalar cannot hold: + + Eneco Zon & Wind Dynamisch 0,1 x BELPEX-H - 1,188 c EUR/kWh + ENGIE Dynamic (B x eSpot_15) - A + Aspiravi (non-dynamic) 0,07 x Belpex - 2 + +1. Its own factor (Eneco 0,102 on consumption, 0,100 on injection; Aspiravi 0,07 + against about 0,10): not the consumption `B` with a sign flip. +2. A negative offset (`- 1,188`, `- 2`); `price_markup_eur_kwh: float(0,1)` + cannot express one. +3. VAT-exempt ("Injectietarieven zijn vrijgesteld van btw") against 6 % on + consumption: representable only because `vat_pct` is per spec. +4. It goes negative with no floor: Eneco's is below zero for any BELPEX-H under + 11,88 EUR/MWh, the hours a PV household exports in. You can be charged to + export. + +`price_export_formula` is `custom_formula`'s parser with one argument moved +(`option`, a row of `_LEGS`), as TARIFF-17 did for `parse_static_profile()`. A +second parser would be a second `_finite()`, unknown-key rule, unit table and +`to_resolution()`, diverging first on the 23- and 25-hour days. Every earlier +call site passes no `option` and reads byte-identical sentences. The flat scalar +`export_price_eur_kwh()` is untouched and still answers when the option is "". + +## What it does not affect + +`arbitrage.py` does not read the injection price at all (9.3: near-zero Belgian +injection; it prices the sell side as displaced import), so no trade today turns +on this number and it is not a wrong-trade fix. It changes what an operator is +told and what a future export-aware strategy (OPT-06) rests on. + +## The card cannot always grade it + +Eneco's injection formula is printed with no price beside it. The `card` block +is required on the consumption leg and optional here (`FormulaSpec.card_graded` +is how callers ask; `card_discrepancy()` raises on an ungraded spec rather than +returning 0.0). The self-check runs wherever a printed estimate exists, and +validate.py says at startup that a leg is ungraded. Inventing an estimate would +grade the arithmetic against itself (MUTATION.md habit 4). + +## `implausible_injection()`: bounds on the coefficients, not the price + +The consumption band (`implausible_all_in()`) cannot be reused: it adds +`price_taxes_eur_kwh`, a consumption component, and its floor `energy >= 0.02` +would refuse the real market's negative hours. But that floor catches a whole +card read 100x too small, which the card check cannot see (it cancels). A band +spanning zero cannot catch 0.00132 EUR/kWh, so the bound moves onto the +coefficients, index-free and sign-free: + +- pass-through `factor x UNITS[units] / 0.001` (one EUR/MWh is 0.001 EUR/kWh): + 1.00 for Eneco's injection leg, 0.70 for Aspiravi's; 100x off reads 0.01 or 100; +- offset magnitude, on the absolute value (1,188 c = 0.01188; a hundredfold error + is 1.188 EUR/kWh). + +Two rungs can both answer one stimulus, so they are graded on their reasons and +mutated together (MUTATION.md, TEST-16); test_price.py 17d builds fixtures so +each is the only one firing. It bounds the shape of the coefficients, never +their truth. + +## The consumption band's asymmetry (`implausible_all_in()`) + +The low end is graded on the energy component and the high end on the all-in +figure. `price_taxes_eur_kwh` ships at 0.11736 (TARIFF-18; 0.06 before), nearly +six times the low bound, so a card read 100x too small (0.0016 EUR/kWh) is +carried over the floor by the flat tax addend alone and would pass a band applied +to the all-in total. Raising the default made the trap wider. The ceiling is on +the all-in figure because that is what the site is billed. It grades the card's +own point, not every hour: live EPEX hours may fall outside (a negative hour is +the best charging opportunity of the year, 17 keeps a regression case). It is +called from validate.py only, since it reads the framework-owned taxes number. + +## Smaller rules + +- `CARD_TOLERANCE` is 0.01 in the card's units, the last digit both real cards + print (Eneco 15,55 against 15.5471; ENGIE 7,522 against 7.5218). Every error + class it exists for is at least two orders of magnitude bigger. +- The card age warning (about 6 months) is never a refusal: ENGIE freezes a + contract's coefficients, so an old card can be exactly right. +- `units` is required with no default (a default is the factor-1000 class). + A zero factor is a fixed tariff in a dynamic one's clothes (every spread 0, + arbitrage never fires; 14's warning box), so the operator is pointed at the + first-class way to say one price. A negative factor inverts the market (charge + into the evening peak). `vat_pct: 0` is legitimate (injection is exempt; some + cards are quoted VAT-inclusive); the bound exists for the 106 / 1.06 / 6 + confusion. An unknown key is refused, not ignored, so a user who types "expr" + is not priced on the fields they also typed. A non-finite factor would make + every price NaN; `True` is an `int` and would price the day at the index. +- `to_resolution()`: the bucket is a UTC hour, exact for Belgium (CET and CEST + are whole-hour offsets, so keying by UTC cannot collide the two 02:00s). It + decides how many points are emitted, not how long one covers ([[TARIFF-14]]). An + incomplete hour is averaged over what is there, not dropped or zero-filled. +- The startup preview prints the index that went in beside the price that came + out (`61.96 -> 0.13522`): every error this module refuses is invisible on the + status page, and a factor of 1000 is one glance there. diff --git a/goodwe_controller/DECISIONS/TARIFF-24-modes-need-the-tariff-task.md b/goodwe_controller/DECISIONS/TARIFF-24-modes-need-the-tariff-task.md index 322049a..f2c3d10 100644 --- a/goodwe_controller/DECISIONS/TARIFF-24-modes-need-the-tariff-task.md +++ b/goodwe_controller/DECISIONS/TARIFF-24-modes-need-the-tariff-task.md @@ -50,14 +50,18 @@ was one input judged twice. **The rule** (`arbitrage.weather_uncertainty()`). With battery modes on and a usable forecast (`solar_fc` not None, the day plan's own test) the doubt is not added: the gap needed is the cycle cost, widened by the price product only. -No forecast, or a stale or unusable one: 1.5 x, as before. It is read in three +No forecast, or a stale or unusable one: 1.5 x, as before. It is read in four places that must agree: the run plan (`_price_runs()`), the charge gate -(`_mode_roles()`) and Home's mirror (`homeview.trade_verdict()`). The forecast +(`_mode_roles()`), Home's mirror (`homeview.trade_verdict()`) and the Forecast +tab's longer cheap run (`dayview.py`, from release/1.0). The forecast is read at most every 60 s and the plan key already holds "forecast or not", so the gap follows in the cycle the read changes, both ways, with one exception: a run planned with a forecast that is under way keeps that gap to its end (`Controller.weather_held()`), so a sensor that goes unavailable does not stop -or restart a charge. The wider gap applies from the next run. +or restart a Cheap hours charge (a Leftover solar battery still stops: without +a forecast it has no planned level). The wider gap applies from the next run. +A run that has started runs to its end: a give run that starts during the hold +is kept after it (`keep_started()`), with no grid charge or sale. **Not changed.** Modes off (the site trade) keeps the doubt always. The shipped cost 0.10 still needs a gap of 0.10 even with a forecast: a 0.08 diff --git a/goodwe_controller/DECISIONS/UX-53-home-follows-the-gate.md b/goodwe_controller/DECISIONS/UX-53-home-follows-the-gate.md new file mode 100644 index 0000000..998cf53 --- /dev/null +++ b/goodwe_controller/DECISIONS/UX-53-home-follows-the-gate.md @@ -0,0 +1,40 @@ +# UX-53 - Home draws a grid charge only where the add-on will take it + +**Status: DONE 2026-10-26 (owner go 2026-10-25, from the PRICE-02 rig test F-1 and F-2).** + +## Decision + +Home asks the add-on's own gate, and does not decide for itself. + +- **Modes on.** `Controller._mode_roles()` files the gate it just asked + (`TariffPlan.arbitrage()` with the measured peak guard) as `mode_gate`: None + when it allowed a charge, else its reason (`peak`, `not_worth`, + `not_covered`). While it is set, `homeview.mode_runs()` draws no grid charge + and `price_windows()` marks no cheap quarter. Control never reads it. A + refusal that is not the gate (no measured headroom, a battery cycling) is + not filed, so those pages are as before. +- **Modes off.** A charge the verdict puts after its last dear period is + refused when its time comes: the quarters left are all equal. `leg_taken()` + asks `trade_verdict()` (the control cycle's own call) at the leg's start and + keeps the leg only if the gate allows it there. A leg in progress is the + verdict's own answer now. The sale, "Tomorrow:" lines and chart marks use + the same answer. +- **Words.** "Lowest/Highest published price" becomes "Lowest/Highest price + left" (the verdict reads only the periods not yet over). The flat-tariff + refusal calls a site "a fully fixed product" only when its product is fixed. + +## What does not change + +No control decision, command or option. Day/night and hourly sites with a +second day of prices keep both legs at every half hour of a day +(`test_ux53.py` part 4), except where every price left is equal: with an +all-night weekend, Friday's 22:00 charge is no longer drawn, as the add-on +does not take it. Price-off sites draw nothing, as before. + +## Limits + +The leg's gate is asked on today's plan, as the control cycle will find it at +that time. A plan that gets new prices in between is drawn again at the next +refresh. While the peak guard holds, a modes-on page shows no grid charge at +all, also not the rest of a run the add-on charges in once the guard clears, +as the modes-off page already did. diff --git a/goodwe_controller/DECISIONS/UX-54-forecast-tab.md b/goodwe_controller/DECISIONS/UX-54-forecast-tab.md new file mode 100644 index 0000000..e0ff423 --- /dev/null +++ b/goodwe_controller/DECISIONS/UX-54-forecast-tab.md @@ -0,0 +1,146 @@ +# UX-54 - a Forecast tab: the sun, the house and each battery's level, per hour + +**Status: DONE 2026-10-26 (owner go 2026-10-26: "Day curve", "New tab").** + +## Decision + +A tab beside Home and Batteries shows today and tomorrow, per hour: the solar +forecast, the solar measured so far, the house's expected use and each +battery's expected level with its blocks. It follows the add-on, as Home does +(UX-53), and decides nothing. + +- **Blocks are Home's.** Today's are the `runs` of the same plan Home draws + (grid, give, monthly cycle). Tomorrow's come from the same builders + (`homeview.runs_for()`). The words are the plan lines'. A block is on the tab + exactly where Home has one, so the UX-53 gate is followed too. +- **Tomorrow is read ahead.** Home sizes a block from the battery's level now + and draws none when there is no room or energy (`room()` and `plan_need()` + in `homeview.mode_runs()`, the leg sizing in `trade_runs()`). For a day that + has not started that level says nothing, so tomorrow's blocks are built with + `ahead` (Home never passes it): `mode_runs()` reads each block at the level + the curve has when it starts, with its own filter and `MIN_KWH`, so there is + no give block where the battery is at the floor and no grid block where it + is at its planned level. Where the curve has no level yet, a charge is read + from the floor and a give from the ceiling. The blocks and the curve follow + each other and are taken to a fixed point. The gate (`mode_gate`, UX-53) is + not bypassed. +- **Modes off has no curve** (no forecast is read, no house profile kept), so + `trade_runs(ahead=f)` reads each leg at Home's own count + (`dayview._counted()`): the level now, plus what the blocks before it put in, + less what they take out. No sun is read and the house only takes, so no give + block where the emptiest battery is at the floor by that count, and control + does not give there. A charge is read from the floor: a full battery covers + the house before the cheap hour, and control charges it back then. Walked + every 15 minutes on a cloudy day, the packs moved by control and a 400 W + house (`test_ux54.py` part 7, cases A-D, one price quarter per leg), + control charges and gives in exactly the quarters the tab has a block. +- **A block lasts as long as control acts.** A cheap run is sized for all the + batteries charging together, so a slower one is still below + `modetrade.charge_to()` when it ends, and control plans the next cheap slots + (`Controller._price_runs()`). `dayview._levels()` asks the same functions, + `modetrade.energy()` and `priceruns.plan()`, with the packs at the curve's + level, and draws the grid block to where control stops. Each battery's block + then ends where the curve has it at its own level, and a give block where + it reaches the floor: the curve puts a point at both (`daycurve.curve()` + cuts the step there), and the block ends at the first point where the level + the tab shows (0.1 %) is there. What is proven: control walked every 15 + minutes through tomorrow with its own model of the packs, not the curve's + levels (`_mode_roles()` each quarter; the house from the profile, the sun + at 100 % trust, a GIVE role counted only where control's own discharge + envelope `packshare.envelope()` is open, the cut-out at the floor), charges + and gives in exactly the quarters the tab has a block in 13 cases + (`test_ux54.py` part 7, `own_walk()`), e.g. 25/45 % with the peak guard: + control gives 17:00-20:15, the block is 17:00-20:06; 30/80 % with a 7 kW + evening house, B held to its rating: control 17:00-19:45, the block + 17:00-19:32. The floor point is placed in one cut, where the battery runs + out at the rate it gives (`_floor_cut()`, checked to the second against + times worked out by hand), so a build stays small: at most 14 floor cuts + and 28 level points per battery and day in those cases. With a cut-out under + the floor, or a house or sun that differs from the forecast and profile, + control differs (Limits). +- **The sun** is what `solarfc` reads. An hourly split (Solcast + `detailedForecast`, Open-Meteo `wh_period`) is used as given. A daily total + (Forecast.Solar, and any other state-only sensor, the owner's three "today + remaining" and three "tomorrow") is spread over the 06:00-20:00 bell, and the + tab says so. +- **The house** is `HouseProfile.estimator()`, never below the night median, as + the plan reads it. +- **The level did not exist** (the plan holds targets and run blocks only), so + `daycurve.curve()` derives it: each battery has the role `modetrade.decide()` + gives (follow the house, hold, give, charge from the grid, rest), spare solar + goes to Solar first batteries first, a shortfall is met by those that may + give, shared as control shares it (by what each holds above the floor, + `packshare.weights()`, each at most its rating, the rest passed on), both + at `dayplan.EFF`, inside the plan's floor and ceiling, from the + level now. Spare solar is `trust x max(0, sun - house)` and what is stored + loses `EFF`, the plan's own `trust x EFF x max(0, sun - house)` (10 kWh of + sun, 2 kWh of house, 70 %: 5.32 kWh stored). A shortfall is not scaled by + trust. The tab says it counts on `solar_trust_pct`. Grid charging goes to + `modetrade.charge_to()`. It is an estimate. +- **Measured solar** is kept per local hour in memory (`SolarToday`), from the + reading the control cycle already takes; nothing is read back. +- **On request.** `GET forecast` builds the tab (`Controller.forecast_payload()` + reads `/status`' plan first, so today's blocks are the very ones Home has). No + Home Assistant call, nothing cached, nothing in the cycle but one in-memory + add. The page asks every 30 s while the tab shows. +- **Not known yet is a note, not an empty chart:** modes off, no forecast set, + forecast missing or old, learning, no tomorrow forecast, a monthly cycle, no + level reading, control stopped, shadow. + +## What does not change + +No decision, command, timing or option key. The one write in the control path +is `SolarToday.add()` after the solar read, in a `try`, in memory. + +## Limits + +- Prices are planned about a day ahead (`priceruns.HORIZON`), so a block of + tomorrow later than 24 h from now, or one that needs prices not yet + published, appears as the day comes near. The tab says so. +- Today's blocks are Home's, and are not extended or shortened: control may + plan more once they end, and Home draws it then. The curve stops charging + at its level (`charge_to()`) and giving at the floor. +- Today, a give run in the plan on a battery that is empty now has no block on + Home (no energy to give). The tab equals Home, and says so: "The add-on plans + to give power. No block shows while a battery has no energy to give." The + curve gives in that run if the sun fills the battery first. +- With no forecast for tomorrow the curve stops at the first hour of the sun, + `solarfc.BELL_FROM_H` (06:00): `Forecast.between()` has no answer past it. The + note names that hour from the constant. +- Tomorrow shows price blocks only. A monthly cycle is drawn today, where the + curve then stops for that battery. +- The curve uses the arbitrage floor and ceiling, as the plan does, not the + battery's own cut-out level. Control stops a pack at its cut-out, so with a + cut-out under the floor a battery in a give run covers the house below the + floor; Home draws no block there either. The house profile is a median per + hour, net of the solar the house used. +- Modes off: the count knows no sun and no house. A battery at the floor that + the sun fills before a give gets no block until then (the tab says "The + add-on plans to give power. No block shows while a battery has no energy to + give."), and a small charge may be used by the house before the give. The + other way round, a charge is drawn even when the sun fills the batteries + before it: at the ceiling control then takes none (a cheap quarter at 13:00 + on a sunny day: the tab draws the charge, control charges nothing at 95 % + or 100 %). + A leg is the verdict's as it stands now, one price period, as on Home; with + several equal prices in a row control takes the next one as each passes. +- Measured solar starts when the add-on starts; the tab says from when. +- Clock changes: the tab walks local hours with `dayplan._hour_slices()`, + which steps in UTC (SAFETY-123), and keys measured solar by the start of + the local hour taken back from the UTC instant, so the spring day has 23 + hours, the autumn day 25 (two of them 02:00), and a request always returns + (`test_ux54_dst.py`, every quarter of both change days, with and without + tzdata). The solar bell is spread over hours from local midnight, so on + those two days it is an hour off the clock (SAFETY-123, not changed here). + +## Found, not changed + +`dayplan.plan()` reads the forecast hour by hour with `Forecast.between(s, e)`. +For a state-only forecast `between()` takes its start as the moment the "left +today" figure was read, so each later hour is spread over what is left of the +bell from that hour: on the owner's shape 14.4 kWh left gives 57 kWh over the +hours to midnight (`test_ux54.py` part 1). The plan's sun is overcounted for +those sites. It does not show on a site whose batteries are all Solar first +(no level is planned for them). The tab measures both ends from now +(`dayview.sun_kwh()`), which keeps the total and gives the same hours for an +hourly split. A card should fix the plan. diff --git a/goodwe_controller/DOCS.md b/goodwe_controller/DOCS.md index c4e7287..f7ca6f9 100644 --- a/goodwe_controller/DOCS.md +++ b/goodwe_controller/DOCS.md @@ -530,8 +530,8 @@ Off by default. Turn it on, restart the add-on, and the Batteries screen (and Se - **On or off.** Off asks the battery for 0 W and starts no monthly cycle on it. A cycle that is running stops, and the add-on log says so. The peak limit may still use it, but only for what the other batteries cannot cover. - **Mode.** - *Solar first* charges from spare solar before the other batteries and covers the house at any hour. It never charges from the grid and never sells. - - *Leftover solar* charges from the solar the others cannot take. With a solar forecast it may also charge from the grid, but only what the sun will not give (see below). It saves its power for the expensive hours and covers the house then. - - *Cheap hours only* charges from the grid in the cheapest hours, saves its power for the expensive hours and covers the house then. + - *Leftover solar* charges from the solar the others cannot take. With a solar forecast it may also charge from the grid, but it buys only what the sun will not give in the expensive hours (see below). It saves its power for those hours and covers the house then. + - *Cheap hours only* fills up from the grid in the cheapest hours, saves its power for the expensive hours and covers the house then. It does not look at the forecast. - **Selling** is off until it can be judged on the price the grid pays (owner decision 2026-09-29, option A). With battery modes on, these choices replace the *Charging and selling on price* switch (`arbitrage_enabled`); that switch is only read with modes off. @@ -548,13 +548,16 @@ With battery modes on, these choices replace the *Charging and selling on price* - If one of them has no reading, is stale or is not in an energy unit, that day's forecast is not used at all. A part of the sun is not counted as all of it. The status page names the source by its place in the list. - The hourly shape is kept only if every sensor has one. Otherwise the total is spread over the day as for Forecast.Solar. - Before each cheap run the add-on asks: will the sun fill the batteries before the expensive hours? It counts on `solar_trust_pct` of the forecast (70 %), less what the house uses. Solar first batteries get that sun first. -- *Leftover solar* then charges from the grid only the shortfall: what the house will draw from it in the expensive hours, less its share of the sun. Not up to the top level. +- *Leftover solar* then charges from the grid only the shortfall: for each expensive hour, what the house uses less the sun counted for that hour (never below 0), added up over all the expensive hours, early morning and evening. With a day and night tariff that is 07:00 to 22:00. Not up to the top level. +- All *Leftover solar* batteries hold that shortfall together, shared by their size. A battery that reaches the top level passes the rest to the others. If all of them together cannot hold it, they all fill to the top level. In the expensive hours the rest comes from another battery (*Cheap hours only* or *Solar first*) or from the grid. The Batteries screen says so. The sun before the expensive hours still fills first, and is taken off what is bought. +- A forecast that gives only one figure for the day (Forecast.Solar) is spread over the hours from now, so the hours add up to that figure. - *Cheap hours only* ignores the forecast and fills to the top level. - *Solar first* never charges from the grid. - The house use is the add-on's own measurement (grid + battery power) per hour, over the last 7 days. It needs one day of it before the first plan. - The plan is checked every 15 minutes. A planned level moves only by 1 kWh or 10 %, so a wobbling forecast does not flip it. - No forecast, one older than 3 hours or from before midnight, or no house use yet: *Leftover solar* takes solar only and *Cheap hours only* fills to the top level. The Batteries screen says so. - A battery needs its capacity (`inverters.capacity_kwh`) for a plan. +- **At night.** Some forecasts stop updating in the evening (Forecast.Solar does). The add-on keeps the last reading of each sensor and uses it for that night's plan: from 20:00 until 06:00, or until a newer reading comes. After midnight the evening's figure for tomorrow is today's, whole, because nothing has been produced yet. A sensor that is missing for up to 3 hours is covered the same way. A figure is never used for another day, and never when it is more than 12 hours old (15 hours after midnight, for a figure that was still in use at midnight). A charge from the grid that has started keeps its level when the forecast ends, and goes on to it; the Batteries screen says so. The Batteries screen says "Solar forecast read 17:05, kept for tonight." With no reading all day, the night run still needs the wider gap. **What always comes first.** - The peak limit, the safety stops and the monthly cycle come before a battery's mode. @@ -3455,7 +3458,7 @@ Charging from the grid stays blocked meanwhile. With several inverters the notification names the battery at its limit (SAFETY-32), each graded against its own cut-out level. That battery is held (weight 0 while the site gives) and the guard's need goes to the others, up to their rating; the alarm fires -only if they miss the limit at full power for `ALARM_HOLD_S` (SAFETY-87). ⚠️ **The reserve binds the peak guard only +only if they miss the limit at full power for `ALARM_HOLD_S` (SAFETY-87). With no battery able to give it fires at once, but only if the period would end more than 10 W over the aim with the batteries at 0 W. A grid charge the add-on runs counts as one it can stop, and the last second's landing error on the aim is not an alarm. A steadily rising load therefore alarms a little later. With `peak_margin_w` 0 a period can end up to 10 W over the limit without this alarm, as for “Load too high” (SAFETY-126). ⚠️ **The reserve binds the peak guard only (SAFETY-25).** Self-consumption has no such floor. See *What `peak_soc_floor` does not protect*. @@ -3532,6 +3535,21 @@ the peak limit, the batteries' switches and modes, and shadow mode, each marked "Applies right away" or "Applies after a restart". **Installer** shows every setting. +### The Forecast screen + +The **Forecast** tab shows today and tomorrow, hour by hour: the solar forecast, +the solar measured so far today, the house's expected use, and each battery's +expected level. Under each battery's chart are its planned blocks, in the same +words as Home. A block is there only where the add-on plans one. + +The level is an estimate. It counts on `solar_trust_pct` of the forecast, as the +plan does, and stays between the battery's low and high levels. A forecast that +gives one total for the day (Forecast.Solar) is spread over the hours, and the +screen says so. Where a part is not known yet (battery modes off, no forecast, +still learning your house, no forecast for tomorrow) the screen says that +instead of drawing an empty chart. It changes nothing in control, and it asks +Home Assistant for nothing: it is built when you open it. + ### Setup, for a new install The **Setup** tab walks an installer through six steps: the meter, the diff --git a/goodwe_controller/app/custom_formula.py b/goodwe_controller/app/custom_formula.py index b233295..638be6c 100644 --- a/goodwe_controller/app/custom_formula.py +++ b/goodwe_controller/app/custom_formula.py @@ -1,174 +1,18 @@ """TARIFF-15 - the customer's own tariff card, as two fixed coefficients. -WHAT THIS IS FOR, in one sentence: a Belgian dynamic contract prices energy as -`B x index + A` with coefficients that exist in exactly one place - the tariff -card in the customer's klantenzone - so the customer types those two numbers -and the add-on does every unit conversion and the VAT multiplication itself. +A Belgian dynamic contract prices energy as `B x index + A` with coefficients +that exist only on the customer's card. The customer types those two numbers; +the add-on does every unit conversion and the VAT multiplication. Fixed +coefficients, no expression evaluation of any kind (the string is settable over +HTTP, and an expression cannot be graded against the card). An unknown key is +refused, never ignored. validate.py refuses to start unless the formula +reproduces the card's printed estimate within CARD_TOLERANCE: that grades the +coefficients against the card, not the card against reality. -WHY A FORMULA AND NOT FOURTEEN MORE PLUGINS (§9.2, TARIFF-07) -============================================================= -There is NO machine-readable source of Belgian tariff coefficients anywhere. -CREG's consumer tariff-card pages 404, CREG Scan is a JS-only wizard, the -annual report carries the supplier list and no per-product coefficients, and -VREG was unreachable. Every coefficient found in research came off a supplier's -own PDF, one of them image-only. So there is nothing to scrape and a -per-supplier plugin would have no source for its numbers - fourteen plugins -would be fourteen copies of the same invention, which is the defect TARIFF-07 -withdrew three plugins for. THE CUSTOMER IS THE ONLY PARTY WHO CAN SEE THEIR -OWN COEFFICIENTS, and the card they read them from is the only artefact that -can grade them. - -THE SHAPE IS THE MARKET'S PUBLISHED ARITHMETIC, NOT OUR APPROXIMATION -==================================================================== -Both suppliers whose 2026 cards were readable print the formula, the reference -index they assumed AND the resulting price, so the arithmetic below is -reproducible against something they published rather than against our own -guess: - - Eneco (0.102 x 133.9905 + 1.00) x 1.06 = 15.5471 card prints 15,55 - ENGIE (0.100 x 61.96 + 0.90) x 1.06 = 7.5218 card prints 7,522 - -Two independent suppliers, reproduced to the last printed digit, in -c EUR/kWh. test_price.py section 13 runs both through the SHIPPED code. - -⚠️ THE DOUBLE-COUNTING MECHANISM, AND WHY THIS DESIGN IS SAFE AT ALL -==================================================================== -This provider declares `sources = {ENERGY, MARKUP}` and NOT `ALL_IN`. A Belgian -card's `Tariefformule` line is THE ENERGY COMPONENT ONLY: Eneco puts network -tariffs in one table and levies in a second, and ENGIE states in so many words -that the formula "does not include fixed network/distribution costs, transport, -grid levies, or excise duties". Declaring ALL_IN would lie on behalf of every -user who typed what their card says, and would silently drop ~12 c EUR/kWh. - -price.py already mechanises that a PLUGIN cannot double-count (`PluginOptions`, -`FRAMEWORK_KEYS`, `code_leaks()`). This provider introduces a risk none of those -can see, because it is not in the code: THE OPERATOR double-counting, by folding -their network charges into `offset` while `price_taxes_eur_kwh` carries them as -well. `code_leaks()` scans code. This is config. - -So the card is the grader: - - At startup validate.py computes - `(factor x card.index_eur_per_mwh + offset) x (1 + vat_pct/100)` and - REFUSES TO START unless it equals `card.expect_incl_vat` to within - CARD_TOLERANCE, naming the discrepancy. - -An `offset` inflated by network charges CANNOT PASS: the card's printed estimate -is by construction the energy component alone, so adding ~12 c EUR/kWh moves -15.55 to ~27 and the add-on refuses. The same check catches VAT omitted, VAT -applied twice, a transposed digit, and a UNIT MISMATCH BETWEEN THE TWO SIDES - -coefficients read off the formula line while the expected price is typed in -another convention, which is the factor-10/100/1000 class as it actually -happens. - -⚠️ THE ONE UNIT ERROR THE CARD CHECK CANNOT SEE IS THE ONE APPLIED TO BOTH -SIDES AT ONCE, and saying so is the difference between a guard and a slogan: -scaling the coefficients AND the expected price by the same 100 cancels out of -a comparison between them. That is what `implausible_all_in()` is for, and it -is a SEPARATE refusal on a SEPARATE quantity - the card's own point, all-in, -against the band a Belgian retail tariff can occupy. - -⚠️ AND THE LIMIT, IN THE SAME BREATH AS THE CLAIM, exactly as price.py's trap 1 -states its own: THE CHECK GRADES THE COEFFICIENTS AGAINST THE CARD, NOT THE CARD -AGAINST REALITY. A consistent-but-wrong pair - both numbers copied off the wrong -product, or off last year's card - reproduces its own printed price perfectly -and passes. Nothing here or anywhere else can catch that; what is mechanised is -that the two coefficients and the printed estimate AGREE, which is the whole of -the unit/VAT/decimal-point class and none of the "wrong contract" class. - -⚠️ FIXED COEFFICIENTS. NO EXPRESSION EVALUATION OF ANY KIND. -============================================================ -All 24 surveyed Belgian products, both injection legs and aWATTar AT fit -`B x index + A`, so an expression buys nothing the market needs. Against that: - - * the add-on serves `POST config/save` -> `POST /addons/self/options` on its - own ingress page, so this string is SETTABLE OVER HTTP. `eval` on it is - remote code execution in a root container holding SUPERVISOR_TOKEN, with - `homeassistant_api` and `hassio_api`, driving an inverter over RS485. - (`eval(s, {"__builtins__": {}})` is not a sandbox and never was.) - * ⚠️ AND THE DECIDING ARGUMENT IS NOT SECURITY - AN EXPRESSION IS - UNVERIFIABLE. Two coefficients can be graded against the card's printed - estimate. An arbitrary expression cannot be graded against anything, so it - makes the invisible-wrong-coefficient problem STRICTLY WORSE, which is the - exact failure this provider exists to prevent. - -Future shapes become NAMED FIELDS - `clamp_lo`, `factor_negative`, -`offset_bands` - each individually refusable and each individually gradeable -against a card. Never a grammar. `parse_price_formula()` therefore REFUSES AN -UNKNOWN KEY rather than ignoring it, so a user who types `"expr"` is told it -does not exist instead of being silently priced on the fields they also typed. -test_price.py section 13f walks this module's AST and fails if `eval`, `exec`, -`compile`, `__import__` or `literal_eval` ever appears in it. - -WHERE THE NUMBERS COME FROM AND WHERE THEY GO -============================================= -`index` names the reference series. `epex_be` is the EPEX BE day-ahead curve in -EUR/MWh, which is what `generic_dayahead` already fetches - same feed, same -`brussels_date_range()` request, same columnar parser, same declared egress -hosts. No new parser and no new failure mode. - -`resolution` says which series the contract is written against: - - * `quarter_hourly` - the 96 points the feed publishes since Belgium's - 15-minute MTU (2025-10-01), emitted as they are; - * `hourly` - the four quarters of each hour averaged into one point. - -⚠️ NEITHER IS AN HOURLY ASSUMPTION IN `_Series.at()`'s SENSE (TARIFF-14). The -hourly option changes how many POINTS are emitted and their spacing; the window -each point covers is still derived from its neighbours by `_cover_table()`, and -nothing here tells anything downstream how long a point lasts. A feed that is -already hourly averages one member per hour, which is the identity - so the -option cannot silently assume the granularity it was handed. - -`units` is NOT bureaucracy. Two suppliers publish the same shape in different -units and the market aggregator prints both conventions in one column, so the -add-on has to be told which one the card is in - and the unit conversion is the -add-on's job precisely because it is one of the two steps that silently costs a -factor of 10, 100 or 1000. The other is VAT, which is also the add-on's. - -TARIFF-16 - THE SECOND LEG, AND WHY IT IS THIS PARSER AND NOT A SECOND ONE -========================================================================== -A Belgian dynamic contract has TWO published lines, not one, and the second is -what the site is PAID for export: - - Eneco Zon & Wind Dynamisch 0,1 x BELPEX-H - 1,188 c EUR/kWh - ENGIE Dynamic (B x eSpot_15) - A - Aspiravi (non-dynamic) 0,07 x Belpex - 2 - -Same `B x index + A` shape - which is why `price_export_formula` is this -module's parser with ONE argument moved (`option`), exactly as TARIFF-17 made -`parse_static_profile()`'s option name a parameter rather than forking the -period grammar. What differs between the two legs is FOUR numbers and a -handful of sentences, and all of them are rows of `_LEGS`: - - * its OWN factor - Eneco 0,102 on consumption against 0,100 on injection, - Aspiravi 0,07 against ~0,10. NOT the consumption `B` with a sign flip; - * a NEGATIVE offset, which `factor`/`offset` already allow and which - `price_markup_eur_kwh: float(0,1)` cannot express at all; - * `vat_pct: 0` - "Injectietarieven zijn vrijgesteld van btw" - against 6 on - consumption, which is a SIX PERCENT ASYMMETRY between the buy leg and the - sell leg and is representable only because `vat_pct` is PER SPEC; - * a price that goes BELOW ZERO with no floor, in the hours a PV household - exports. `implausible_injection()` is the band that accepts that and still - refuses an implausible one; `implausible_all_in()` is NOT it, and its - docstring says why. - -⚠️ AND THE CARD CANNOT ALWAYS GRADE IT. Eneco's injection formula is printed -with NO price beside it, so on that card there is nothing to reproduce. The -`card` block is therefore OPTIONAL on this leg and REQUIRED on the other, the -self-check runs wherever there is a printed estimate, and validate.py says at -startup when a leg is ungraded. Inventing an estimate to grade against would -be grading arithmetic against itself, which is the defect MUTATION.md habit 4 -is about; TARIFF-15 demonstrated two tariffs end to end rather than three for -the same reason. - -⚠️ NOTHING TRADES ON THIS TODAY, AND SAYING SO IS PART OF SHIPPING IT. -`arbitrage.py` does not read `export_price_eur_kwh` at all - its own comment -says so, citing §9.3's near-zero Belgian injection price - and both hours it -compares are all-in CONSUMPTION prices. So this is not a wrong-trade defect -and must not be described as one. What it changes is what an operator is told, -and what a future export-aware strategy would rest on. -""" +The provider declares `{ENERGY, MARKUP}`, never `ALL_IN` (a card's formula is +the energy component only). TARIFF-16 added the injection leg through this same +parser. DECISIONS/TARIFF-15-the-card-is-the-grader.md and +DECISIONS/TARIFF-16-the-injection-leg.md.""" import json import math @@ -183,13 +27,11 @@ class FormulaInvalid(Exception): validate.py at startup - never at 03:00 on a fetch.""" -# --------------------------------------------------------------------------- # -# The enumerations. Every one of these is a CLOSED set, and that is the point: -# an unknown value is refused by name rather than guessed at. -# --------------------------------------------------------------------------- # +# The enumerations. Each is a CLOSED set: an unknown value is refused by name, +# not guessed. -# unit -> how many EUR/kWh one of it is. ⚠️ THE WHOLE FACTOR-1000 CLASS LIVES -# HERE, so the table is tiny, explicit, and has no "default". +# unit -> EUR/kWh per one of it. The whole factor-1000 class lives here: tiny, +# explicit, no default. UNITS: dict = { "c_eur_per_kwh": 0.01, # eurocent per kWh - Eneco and ENGIE print this "eur_per_kwh": 1.0, # some aggregators print this @@ -198,54 +40,41 @@ UNITS: dict = { RESOLUTIONS: tuple = ("hourly", "quarter_hourly") -# index id -> the human name of the series it refers to. One entry today; a -# second index is a new row here plus a fetch path, not a new grammar. +# index id -> the series it names. A second index is a new row here plus a +# fetch path. INDEXES: dict = { "epex_be": "Belgian day-ahead price, EUR/MWh", } -# ⚠️ 0.01 IN THE CARD'S OWN UNITS, WHICH IS THE LAST DIGIT BOTH REAL CARDS -# PRINT. Eneco prints 15,55 against a computed 15.5471 (0.0029 out) and ENGIE -# prints 7,522 against a computed 7.5218 (0.0002 out), so the tolerance has to -# absorb the card's own rounding and nothing more. Every error class this check -# exists for - a factor of 10, of 100, of 1000, a missing or doubled 1.06, a -# transposed digit, ~12 c EUR/kWh of network charge folded into `offset` - is -# at least two orders of magnitude bigger than this. +# 0.01 in the card's own units, the last digit both real cards print (Eneco +# 15,55 vs 15.5471; ENGIE 7,522 vs 7.5218): the card's rounding and nothing +# more. DECISIONS/TARIFF-16-the-injection-leg.md. CARD_TOLERANCE = 0.01 -# The all-in plausibility band, EUR/kWh. `MAX_PLAUSIBLE_EUR_KWH` in price.py -# bounds only prices that are too LARGE; this is the other end, and it is the -# one the factor-1000 error falls off. +# The all-in plausibility band, EUR/kWh: the low end, where a factor-1000 error +# falls off. PLAUSIBLE_LO_EUR_KWH = 0.02 PLAUSIBLE_HI_EUR_KWH = 1.00 -# ~6 months. ⚠️ A WARNING, NEVER A REFUSAL: ENGIE freezes a contract's -# coefficients for its duration, so a card two years old can be exactly right -# and a site refused over one would be a site that cannot start. +# ~6 months. A WARNING, never a refusal: ENGIE freezes a contract's +# coefficients. CARD_STALE_DAYS = 183 _CARD_KEYS = ("month", "index_eur_per_mwh", "expect_incl_vat") _TOP_KEYS = ("index", "resolution", "factor", "offset", "units", "vat_pct", "card") -# --------------------------------------------------------------------------- # # TARIFF-16 - the SECOND leg, and the one parameter that carries it -# --------------------------------------------------------------------------- # EXPORT_FORMULA_OPTION = "price_export_formula" -# ⚠️ THE PASS-THROUGH FRACTION, WHICH IS WHY THE NEXT TWO BOUNDS CAN BE -# INDEX-FREE. One EUR/MWh IS 0.001 EUR/kWh, so `factor x UNITS[units] / 0.001` -# is the dimensionless share of the wholesale price the contract passes -# through - 1.00 for Eneco's injection leg (0,100 c EUR/kWh per EUR/MWh), 0.70 -# for Aspiravi's (0,07). A coefficient typed a hundred times too small or too -# large moves this by a hundred, whatever the index and whatever the sign of -# the price. +# The pass-through fraction, `factor x UNITS[units] / 0.001`: index-free and +# sign-free, so a coefficient typed 100x off moves it by 100 whatever the +# price. EUR_KWH_PER_EUR_MWH = 0.001 -# The two plausibility bounds for an INJECTION leg. ⚠️ NEITHER LOOKS AT THE -# SIGN OF THE PRICE, AND THAT IS THE WHOLE POINT - see -# `implausible_injection()`. +# The two plausibility bounds for an INJECTION leg; neither looks at the sign +# of the price. PASSTHROUGH_LO = 0.10 # 10% of the wholesale price PASSTHROUGH_HI = 10.0 # 1000% of it INJECTION_OFFSET_MAX_EUR_KWH = 0.20 @@ -254,18 +83,8 @@ INJECTION_OFFSET_MAX_EUR_KWH = 0.20 @dataclass(frozen=True) class _Leg: """What ONE option's coefficients price, as sentences its refusals read. - - ⚠️ TARIFF-16 MADE `option` A PARAMETER RATHER THAN FORKING THE PARSER, - exactly as TARIFF-17 made `option`/`means` parameters of - `parse_static_profile()` rather than forking the period grammar. A second - copy of this parser would be a second `_finite()`, a second unknown-key - rule, a second unit table and a second `to_resolution()` - and the two - would diverge first on the 23- and 25-hour days, where nobody looks. - Everything that differs between the consumption leg and the injection leg - is a STRING in this table; the arithmetic, the enumerations, the card - comparison and the resolution rule are ONE implementation with one set of - checks over it. - """ + `option` is a parameter, not a fork of the parser (TARIFF-16, as TARIFF-17 did + for `parse_static_profile()`).""" option: str what: str # "consumption" / "injection", for a sentence @@ -316,12 +135,8 @@ _LEGS = { @dataclass(frozen=True) class FormulaSpec: - """One contract's published arithmetic. SEVEN numbers and no code. - - ⚠️ EVERY FIELD IS A FIXED, NAMED SCALAR. There is deliberately no field - that holds an expression, a callable, a lambda or a template, and there is - no place to put one: `parse_price_formula()` refuses an unrecognised key. - """ + """One contract's published arithmetic: SEVEN numbers and no code. Every field is + a fixed named scalar; `parse_price_formula()` refuses an unrecognised key.""" index: str resolution: str @@ -332,8 +147,8 @@ class FormulaSpec: card_month: str = "" # "YYYY-MM", as the card is dated card_index_eur_per_mwh: float = None card_expect_incl_vat: float = None # in `units`, as the card prints it - # TARIFF-16. Which OPTION this spec was read from, so a refusal, a log line - # and `card_graded` name the leg the operator actually typed. + # TARIFF-16. Which OPTION this spec was read from, so refusals name the leg + # typed. option: str = "price_formula" @property @@ -342,24 +157,14 @@ class FormulaSpec: @property def card_graded(self) -> bool: - """Is there a printed estimate to grade these coefficients against? - - ⚠️ FALSE IS A REAL, SHIPPED STATE ON THE INJECTION LEG and is reported - rather than defaulted: Eneco's injection formula is printed with no - price beside it. Never False on `price_formula`, where `_card()` - refuses an absent block. - """ + """Is there a printed estimate to grade these coefficients against? False is a + real state on the injection leg (Eneco prints none); never False on `price_formula`.""" return self.card_expect_incl_vat is not None @property def passthrough(self) -> float: - """The dimensionless share of the wholesale price this passes through. - - `factor x UNITS[units] / 0.001`. 1.00 for Eneco's injection leg, 0.70 - for Aspiravi's. ⚠️ INDEX-FREE AND SIGN-FREE, which is what lets - `implausible_injection()` bound a leg whose price legitimately goes - below zero. - """ + """The dimensionless share of the wholesale price this passes through. Index-free + and sign-free, which lets `implausible_injection()` bound a leg that goes negative.""" return self.factor * self.unit_to_eur_kwh / EUR_KWH_PER_EUR_MWH @property @@ -368,46 +173,27 @@ class FormulaSpec: return self.offset * self.unit_to_eur_kwh -# --------------------------------------------------------------------------- # # The arithmetic. Two functions, and the SECOND one is the contract. -# --------------------------------------------------------------------------- # def in_card_units(spec: FormulaSpec, index_eur_per_mwh: float) -> float: - """`(factor x index + offset) x (1 + vat_pct/100)`, in the card's units. - - This is the line the card prints, spelled exactly as the card spells it, - and it is the only place the coefficients are combined. The card - self-check and the price path both go through here, so a mutation to it - cannot move the price without also moving the self-check. - """ + """`(factor x index + offset) x (1 + vat_pct/100)`, in the card's units: the line + the card prints. The only place the coefficients are combined, so the card + check and the price path cannot diverge.""" return ((spec.factor * float(index_eur_per_mwh) + spec.offset) * (1.0 + spec.vat_pct / 100.0)) def price_eur_kwh(spec: FormulaSpec, index_eur_per_mwh: float) -> float: - """The energy+markup price for one index value, in EUR/kWh. - - ⚠️ THE UNIT CONVERSION IS THE ADD-ON'S JOB AND IT HAPPENS HERE, ONCE. The - user types what their card prints, in the card's own units; every step - from there to EUR/kWh is code, because those steps are exactly the ones - that silently cost a factor of 10, 100 or 1000 when a human does them. - """ + """The energy+markup price for one index value, in EUR/kWh. The unit conversion + happens here, once: those steps are the ones that cost a factor of 10, 100 or 1000.""" return in_card_units(spec, index_eur_per_mwh) * spec.unit_to_eur_kwh def card_discrepancy(spec: FormulaSpec) -> float: - """How far the coefficients land from the card's own printed estimate. - - In the card's units, signed: positive means the formula computes MORE than - the card prints, which is the direction an `offset` carrying network - charges moves it. - - ⚠️ RAISES ON A SPEC WITH NO CARD RATHER THAN RETURNING 0.0 (TARIFF-16). An - ungraded injection leg has no printed estimate, and a discrepancy of zero - is exactly what "the coefficients reproduce the card" looks like - so a - silent zero here would report the strongest possible result for the one - state in which nothing was compared at all. - """ + """How far the coefficients land from the card's printed estimate, in the card's + units, signed: positive means MORE than the card prints (the direction an + `offset` carrying network charges moves it). Raises on a spec with no card + rather than returning 0.0 (TARIFF-16).""" if not spec.card_graded: raise FormulaInvalid( f"{spec.option} has no card block, so there is no printed " @@ -417,18 +203,10 @@ def card_discrepancy(spec: FormulaSpec) -> float: - spec.card_expect_incl_vat -# --------------------------------------------------------------------------- # # Reading the option -# --------------------------------------------------------------------------- # def _finite(value, field: str, option: str = "price_formula") -> float: - """A real, finite number - or a refusal naming the field. - - ⚠️ SAME CLASS AS `_validate_raw()`'s NaN RULE, one layer earlier. A NaN - factor makes every price NaN, every comparison False, and an arbitrage - strategy that silently never fires; `True` is an `int` in Python and would - price the whole day at the index. - """ + """A real, finite number, or a refusal naming the field (NaN would price every hour NaN; `True` is an `int`).""" if isinstance(value, bool) or not isinstance(value, (int, float)): raise FormulaInvalid( f"{option}.{field} is {value!r}, which is not a number. Write a " @@ -441,15 +219,9 @@ def _finite(value, field: str, option: str = "price_formula") -> float: def _card(raw, leg: _Leg = None) -> tuple: - """The `card` block: the estimate the coefficients are graded against. - - ⚠️ REQUIRED ON THE CONSUMPTION LEG AND OPTIONAL ON THE INJECTION ONE, and - the asymmetry is a fact about the CARDS rather than a relaxed rule. See - `_LEGS[EXPORT_FORMULA_OPTION].card_note`: Eneco's injection formula is - printed with no price beside it, so on that card there is nothing to grade - against. `("", None, None)` is the ungraded answer and - `FormulaSpec.card_graded` is how every caller asks. - """ + """The `card` block: the estimate the coefficients are graded against. Required on + the consumption leg, optional on the injection one because Eneco prints none + (`("", None, None)`; `FormulaSpec.card_graded`).""" if leg is None: leg = _LEGS["price_formula"] if raw is None: @@ -491,31 +263,8 @@ def _month_date(month: str): def parse_price_formula(value, *, option: str = "price_formula"): - """`price_formula` -> a `FormulaSpec`, or None when unset. - - ⚠️ `option` IS TARIFF-16'S WHOLE REUSE, and it is TARIFF-17's move. It - selects a row of `_LEGS` and nothing else: the enumerations, the - arithmetic, `to_resolution()`, the unknown-key rule and the card - comparison are shared, and every pre-existing call site - which passes no - `option` at all - lands on the consumption row and reads byte-identical - sentences. `parse_export_formula()` is a two-line wrapper, not a fork. - - Accepts what the option can actually arrive as, exactly as - `parse_static_profile()` does: - - * `None` / `""` / `"null"` - unset. - * a JSON string - what config.yaml's flat `str?` option - carries on a Supervisor install. - * a dict - already parsed, from a future nested - schema, from sim/inject.py, or a check. - - Raises `FormulaInvalid`; never returns a half-built spec. - - ⚠️ THE CARD SELF-CHECK IS PART OF PARSING, not a separate step a caller - may forget. Both callers - validate.py's startup gate and the plugin's own - fetch - go through this one function, so there is no route to a price that - has not been graded against the card. - """ + """`price_formula` -> a `FormulaSpec`, or None when unset. `option` selects a row of + `_LEGS`; everything else is shared, so `parse_export_formula()` is a wrapper.""" leg = _LEGS[option] if value is None: return None @@ -534,12 +283,8 @@ def parse_price_formula(value, *, option: str = "price_formula"): f"{option} is {type(value).__name__}, not a JSON object with " f"{', '.join(_TOP_KEYS)}.") - # ⚠️ AN UNKNOWN KEY IS REFUSED, NOT IGNORED, AND THIS IS THE HALF THAT - # KEEPS THE "NEVER A GRAMMAR" PROMISE HONEST. A future shape is a NAMED - # field - clamp_lo, factor_negative, offset_bands - added here and - # refusable on its own. Ignoring unknown keys would mean a user who typed - # "expr" or "formula" gets priced on the fields they also typed, with the - # one they believed in dropped in silence. + # An unknown key is refused, not ignored: a future shape is a NAMED field, + # never a grammar. unknown = sorted(set(value) - set(_TOP_KEYS)) if unknown: raise FormulaInvalid( @@ -559,10 +304,8 @@ def parse_price_formula(value, *, option: str = "price_formula"): f"{option}.resolution is {value.get('resolution')!r}. Choose one " f"of {list(RESOLUTIONS)}, as your tariff card says.") - # ⚠️ `units` IS REQUIRED AND HAS NO DEFAULT. A default here would be the - # factor-1000 class with a friendly face: the add-on would silently assume - # a convention the card never stated, and the card self-check would then - # be graded in units nobody chose. + # `units` is required with no default: a default is the factor-1000 class + # with a friendly face. units = str(value.get("units", "") or "").strip() if units not in UNITS: raise FormulaInvalid( @@ -573,29 +316,22 @@ def parse_price_formula(value, *, option: str = "price_formula"): offset = _finite(value.get("offset"), "offset", option) vat_pct = _finite(value.get("vat_pct"), "vat_pct", option) - # ⚠️ ZERO IS NOT "NO INDEXATION", IT IS A FIXED TARIFF WEARING A DYNAMIC - # ONE'S CLOTHES. Every hour comes out at the same price, so every spread is - # 0, arbitrage can never clear any threshold, and a strategy that never - # fires is indistinguishable from one that is switched off (§14's warning - # box). The add-on already has a first-class way to say "one price": point - # the operator at it rather than let them build it by accident. + # Zero is a fixed tariff in a dynamic one's clothes (arbitrage never + # fires); point the operator at the first-class fixed price. if factor == 0.0: raise FormulaInvalid( f"{option}.factor is 0, so every hour gets the same {leg.what} " f"price{leg.zero_factor}. For one flat price, {leg.flat}.") - # ⚠️ A NEGATIVE FACTOR INVERTS THE MARKET, and the direction is the danger: - # the dearest hour becomes the cheapest one, so the battery is told to - # charge into the evening peak and to discharge at 03:00. That is not a - # mispriced site, it is a site actively trading against itself. + # A negative factor inverts the market: charge into the evening peak, + # discharge at 03:00. if factor < 0.0: raise FormulaInvalid( f"{option}.factor is {factor:g}, which turns the market upside " f"down. {leg.neg_factor}.") - # ⚠️ 0 IS LEGITIMATE AND MUST STAY SO: injection is VAT-exempt in Belgium, - # and some cards are already quoted VAT-inclusive. The bound exists for the - # 106 / 1.06 / 6 confusion, not to make VAT compulsory. + # 0 is legitimate (injection is VAT-exempt; some cards are VAT-inclusive); + # the bound is for the 106 / 1.06 / 6 confusion. if not 0.0 <= vat_pct <= 30.0: raise FormulaInvalid( f"{option}.vat_pct is {vat_pct:g}. Write a percentage between 0 " @@ -607,13 +343,9 @@ def parse_price_formula(value, *, option: str = "price_formula"): card_month=month, card_index_eur_per_mwh=card_index, card_expect_incl_vat=card_expect, option=option) - # ⚠️ THE CARD SELF-CHECK STILL RUNS WHEREVER THERE IS A CARD TO RUN IT - # AGAINST, on either leg, with the same tolerance. What TARIFF-16 adds is - # the honest answer for the leg where the card prints no estimate: NOT a - # weakened comparison and NOT an estimate computed here, but two bounds on - # the COEFFICIENTS that need no printed price - see - # `implausible_injection()` - and a startup line saying the leg is - # ungraded. + # The card self-check still runs wherever there is a card, with the same + # tolerance. Where the card prints no estimate, `implausible_injection()` + # bounds the coefficients and a startup line says the leg is ungraded. if spec.card_graded: delta = card_discrepancy(spec) if abs(delta) > CARD_TOLERANCE: @@ -626,60 +358,15 @@ def parse_price_formula(value, *, option: str = "price_formula"): def parse_export_formula(value): - """`price_export_formula` -> the INJECTION leg's `FormulaSpec`, or None. - - TARIFF-16. THE DEFECT THIS EXISTS TO CLOSE: `price.py` modelled injection - as ONE scalar, and the market publishes a line with four properties a - scalar cannot hold. Verbatim off the cards: - - Eneco Zon & Wind Dynamisch 0,1 x BELPEX-H - 1,188 c EUR/kWh - "Het injectietarief is dynamisch en kan per uur wijzigen" - - identical on the January and the September 2026 cards - ENGIE Dynamic (B x eSpot_15) - A c EUR/kWh - Aspiravi (non-dynamic) 0,07 x Belpex - 2 - - 1. ITS OWN FACTOR. Eneco uses 0,102 on consumption and 0,100 on - injection; Aspiravi 0,07 against ~0,10 on its consumption leg. It is - NOT the consumption `B` with a sign flip. - 2. A NEGATIVE OFFSET - `- 1,188`, `- 2`. Note that - `price_markup_eur_kwh: float(0,1)` cannot express a negative markup - at all, so the offset has no home on the consumption side either. - 3. VAT-EXEMPT. "Injectietarieven zijn vrijgesteld van btw", while - consumption is at 6% - a SIX PERCENT ASYMMETRY between the buy leg - and the sell leg, which a single scalar cannot even state. - 4. IT GOES NEGATIVE, WITH NO FLOOR. Eneco's injection price is negative - for any BELPEX-H below 11,88 EUR/MWh, and a PV household exports - precisely in the hours most likely to be negative. You can be charged - to export. - - ⚠️ AND WHAT THIS DOES **NOT** AFFECT TODAY, SAID HERE SO NOBODY READS THE - ABOVE AS A TRADING BUG. `arbitrage.py` does not read - `export_price_eur_kwh` at all - its own comment says so, and gives §9.3's - near-zero Belgian injection price as the reason the economically sound - trade on this market is charge-cheap-to-displace-expensive-IMPORT. Both - hours it compares are all-in CONSUMPTION prices. So no trade the add-on - takes today is decided by this number, and this card is not a correction - of a wrong trade. What it changes is what an operator is told, and what - any future export-aware strategy would rest on: OPT-06 and any export - logic would otherwise inherit a scalar that cannot represent the market, - and the 6% VAT asymmetry would be invisible in the arithmetic. - - ⚠️ THE SCALAR IS NOT REPLACED. `export_price_eur_kwh()` is untouched and - still answers for every site with a flat injection price, which is every - site that exists today; this option ships "". That is `price.py`'s own - prediction coming true rather than a new design - "when it does land, an - injection curve gets a `PriceProvider` of its own and this scalar becomes - its FIXED-product default; the seam is already the right shape for that." - """ + """`price_export_formula` -> the INJECTION leg's `FormulaSpec`, or None (TARIFF-16). + Same parser with `option` moved; the flat `export_price_eur_kwh()` still answers + when unset. Nothing trades on it today. DECISIONS/TARIFF-16-the-injection-leg.md.""" return parse_price_formula(value, option=EXPORT_FORMULA_OPTION) def card_refusal(spec: FormulaSpec, delta: float) -> str: - """The sentence an operator reads when their formula misses its own card. - - It names both numbers and what to check. The usual causes are in DOCS.md, - because each of these errors looks like a working tariff on the status page. - """ + """The sentence an operator reads when their formula misses its own card: both + numbers and what to check (causes are in DOCS.md).""" got = in_card_units(spec, spec.card_index_eur_per_mwh) leg = _LEGS[spec.option] return ( @@ -688,53 +375,14 @@ def card_refusal(spec: FormulaSpec, delta: float) -> str: f"{spec.card_expect_incl_vat:g}. Check {leg.card_causes}.") -# --------------------------------------------------------------------------- # # The two questions the PLUGIN cannot ask, because they need framework numbers -# --------------------------------------------------------------------------- # def implausible_all_in(spec: FormulaSpec, taxes_eur_kwh: float): - """The card's own point, if its LEVEL cannot be a Belgian retail price. - - Returns a refusal sentence or None. ⚠️ CALLED FROM validate.py AND FROM - NOWHERE ELSE, because it reads a FRAMEWORK-owned number: a plugin that - could see `taxes` could add it a second time, which is price.py's trap 1. - - ⚠️ THE LOW END IS GRADED ON THE ENERGY COMPONENT AND THE HIGH END ON THE - ALL-IN FIGURE, AND THAT ASYMMETRY IS THE CHECK RATHER THAN AN OVERSIGHT. - `price_taxes_eur_kwh` ships at 0.11736 (TARIFF-18; it was 0.06 before, and - the argument was already decisive at that figure), which is nearly SIX - times the low bound - so a card read a HUNDRED times too small comes out at - 0.0016 EUR/kWh of energy, is carried over the floor by the flat tax addend - alone, and passes a band applied to the all-in total. The hundredfold error - is precisely what this exists to catch, so the floor is applied where it - can still see it. ⚠️ Raising the default made this trap WIDER, not - narrower: the addend that does the carrying got bigger. The ceiling is - applied to the all-in figure because that is the number the site is really - billed at, and taxes only push it up. - - ⚠️ AND THE LIMIT, WITH THE CLAIM. This grades the CARD'S OWN POINT - one - price, at the monthly index the card assumed - and not every hour of every - day. A live EPEX hour can and should fall outside this band (a negative - hour is the best charging opportunity of the year, and §17 keeps a - regression case for it), so the band is deliberately NOT applied to the - fetched series. What it catches is a formula whose whole LEVEL is wrong, - which is the factor-of-1000 class arriving from the end - `MAX_PLAUSIBLE_EUR_KWH` does not bound. - - ⚠️ AND IT IS THE CONSUMPTION LEG'S BAND ONLY - TARIFF-16 DELIBERATELY DID - NOT POINT IT AT THE INJECTION LEG. Two reasons, and the second is an - acceptance criterion rather than a preference: - - * it adds `taxes_eur_kwh`, and `price_taxes_eur_kwh` is a CONSUMPTION - component. Nothing is added to what a site is PAID for export, so the - `all_in` half would be adding a number the injection leg never carries; - * its floor is `energy >= 0.02 EUR/kWh`, and a REAL Eneco injection price - is negative for any BELPEX-H below 11,88 EUR/MWh - the hours a PV - household actually exports in. Applied to injection this floor would - refuse the market, and refuse it hardest in the hours where knowing the - number matters most. `implausible_injection()` is the injection band - and it is deliberately NOT this function with a lower floor. - """ + """The card's own point, if its LEVEL cannot be a Belgian retail price. Refusal + sentence or None. Called from validate.py only (it reads the framework-owned + taxes). The low end is graded on the energy component and the high end on the + all-in figure; the consumption leg's band only. It grades the card's own point, + not the fetched series. DECISIONS/TARIFF-16-the-injection-leg.md.""" energy = price_eur_kwh(spec, spec.card_index_eur_per_mwh) all_in = energy + float(taxes_eur_kwh) if energy >= PLAUSIBLE_LO_EUR_KWH and all_in <= PLAUSIBLE_HI_EUR_KWH: @@ -747,49 +395,10 @@ def implausible_all_in(spec: FormulaSpec, taxes_eur_kwh: float): def implausible_injection(spec: FormulaSpec): - """The INJECTION leg's plausibility band. Returns a refusal or None. - - ⚠️ A NEGATIVE INJECTION PRICE IS ACCEPTED, NOT REFUSED, AND NEITHER RUNG - BELOW LOOKS AT THE SIGN OR THE LEVEL OF A PRICE. Eneco's published line - `0,1 x BELPEX-H - 1,188` is negative for any BELPEX-H under 11,88 EUR/MWh; - those are the midday hours a PV household exports in, so a band with a - floor above zero - which is what `implausible_all_in()` has, for the - consumption leg's own good reasons - would refuse the real market and - would hide it in exactly the hours it costs money. You CAN be charged to - export. The add-on says so rather than declining to represent it. - - THE PROBLEM THAT LEAVES, STATED BEFORE THE SOLUTION. The consumption - floor was not decoration: it is what catches a whole card read a hundred - times too small, the one unit error the card self-check cannot see because - it cancels out of a comparison between the two sides. A band that spans - zero cannot catch it - 0.00132 EUR/kWh is inside any interval that - contains -0.03. So the bound moves OFF the price and ONTO the - coefficients, where it is index-free and sign-free: - - * PASS-THROUGH. `factor x UNITS[units] / 0.001` is the dimensionless - share of the wholesale price the contract passes through, because one - EUR/MWh IS 0.001 EUR/kWh. Eneco's injection leg is 1.00 and Aspiravi's - is 0.70. A factor typed 100x too small reads 0.01 and 100x too large - reads 100, whatever the index and whatever the sign of the result. - * OFFSET MAGNITUDE. Eneco's is 1,188 c EUR/kWh = 0.01188 EUR/kWh and - Aspiravi's is 2 c = 0.02. A hundredfold error is 1.188 EUR/kWh, which - is not an injection tariff in any direction. The bound is on the - ABSOLUTE value, so a negative offset of a believable size passes and - that is the normal case on this leg. - - ⚠️ TWO RUNGS THAT CAN BOTH ANSWER ONE STIMULUS, SO THEY ARE GRADED ON - THEIR REASONS AND MUTATED TOGETHER (MUTATION.md, TEST-16). A whole card - read 100x too large trips both. The fixtures in test_price.py 17d are - built so that each rung is the ONLY one that fires on its own stimulus. - - ⚠️ AND WHAT THIS CANNOT SEE, WITH THE CLAIM (habit 13). It bounds the - SHAPE of the coefficients, never their truth: a contract really written - against a different index, or last year's coefficients, passes both rungs - and reproduces any card typed beside them. Where a card DOES print an - estimate the self-check grades the pair against it; where it does not - - which is Eneco's injection line - these two bounds are the whole of what - is mechanised, and validate.py says so at startup. - """ + """The INJECTION leg's plausibility band. Refusal or None. A negative price is + accepted; the bounds are on the coefficients (pass-through and offset + magnitude), since a band spanning zero cannot catch a card 100x too small. It + bounds the shape, not the truth. DECISIONS/TARIFF-16-the-injection-leg.md.""" if not PASSTHROUGH_LO <= spec.passthrough <= PASSTHROUGH_HI: return ( f"{spec.option}.factor passes through {spec.passthrough:.4g}x the " @@ -806,10 +415,7 @@ def implausible_injection(spec: FormulaSpec): def stale_card(spec: FormulaSpec, today=None): - """A warning when the card is older than ~6 months, or None. NEVER a - refusal: ENGIE freezes a contract's coefficients for its duration, so an - old card can be exactly right and refusing one would stop the add-on on a - correctly configured site.""" + """A warning when the card is older than ~6 months, or None. Never a refusal (ENGIE freezes coefficients).""" dated = _month_date(spec.card_month) now = today or datetime.now(timezone.utc).date() if dated is None or (now - dated).days <= CARD_STALE_DAYS: @@ -820,36 +426,14 @@ def stale_card(spec: FormulaSpec, today=None): f"contract changed since, copy the new figures from your supplier.") -# --------------------------------------------------------------------------- # # Resolution -# --------------------------------------------------------------------------- # def to_resolution(pairs, resolution: str) -> list: """`[(instant, index EUR/MWh)]` -> the series the CONTRACT is written on. - - `quarter_hourly` hands the feed's own points back, ascending. - `hourly` averages each hour's members into one point AT THE TOP OF THE - HOUR. - - ⚠️ THE BUCKET IS A UTC HOUR AND THAT IS EXACT FOR BELGIUM, not an - approximation: CET and CEST are both WHOLE-hour offsets, so a Brussels hour - and a UTC hour start at the same instant on every day of the year including - both transition days. Keying by a naive local hour is what collides the two - 02:00s in October; keying by the UTC instant cannot. - - ⚠️ AND THIS IS NOT `_Series.at()`'s HOURLY ASSUMPTION COMING BACK - (TARIFF-14). It decides how many points are EMITTED, not how long one - covers - the coverage window is still derived from the neighbours in - `_cover_table()`. A feed that already publishes hourly points averages one - member per hour, i.e. the identity, so this cannot silently assume the - granularity it was handed. - - ⚠️ AN INCOMPLETE HOUR IS AVERAGED OVER WHAT IS THERE rather than dropped or - zero-filled. A horizon that starts at 09:20 has two quarters in its first - hour; their mean is the best statement available about that hour, and - dropping it would leave a hole `at()` answers with `Absence` for an hour - the feed did publish. - """ + `quarter_hourly` returns the feed's points, `hourly` averages each UTC hour + (exact for Belgium: whole-hour offsets) at the top of the hour. An incomplete + hour is averaged over what is there. It sets how many points are emitted, not + how long one covers (TARIFF-14).""" ordered = sorted(pairs, key=lambda r: r[0].astimezone(timezone.utc)) if resolution == "quarter_hourly": return list(ordered) @@ -863,28 +447,16 @@ def to_resolution(pairs, resolution: str) -> list: return [(k, sum(v) / len(v)) for k, v in sorted(buckets.items())] -# --------------------------------------------------------------------------- # # The startup preview -# --------------------------------------------------------------------------- # PREVIEW_HOURS = 24 PREVIEW_MAX_ROWS = 24 def preview_lines(spec: FormulaSpec, rows) -> list: - """`[(instant, index EUR/MWh, all-in EUR/kWh)]` -> lines for the log. - - THE DEFECT THIS EXISTS TO MAKE VISIBLE: every error this module refuses is - one the operator cannot see from the status page, and the two numbers that - would have shown it - the index that went IN and the price that came OUT - - are never printed beside each other anywhere else. A factor of 1000 is - obvious in one line of `61.96 -> 0.13522` and invisible in a plan. - - ⚠️ IT PRINTS A BOUNDED NUMBER OF ROWS AND SAYS SO. At `quarter_hourly` the - next 24 h is 96 points; the cap is a row COUNT rather than "every fourth - point", because sampling every fourth point would assume the granularity - TARIFF-14 spent a card removing the assumption of. - """ + """`[(instant, index EUR/MWh, all-in EUR/kWh)]` -> lines for the log: the index + that went IN beside the price that came OUT. Prints a bounded row COUNT, not + every fourth point (that would assume the granularity).""" if not rows: return [] shown = rows[:PREVIEW_MAX_ROWS] diff --git a/goodwe_controller/app/daycurve.py b/goodwe_controller/app/daycurve.py new file mode 100644 index 0000000..b662fb1 --- /dev/null +++ b/goodwe_controller/app/daycurve.py @@ -0,0 +1,293 @@ +"""UX-54: the expected battery level through the day, for the Forecast tab. + +Pure: batteries, the sun and the house's use in; a level per battery over +time out. Display only: nothing here is read by control, and it never +decides a block. The blocks are the plan's (`homeview.mode_runs()`); this +only says where the level goes, hour by hour, if the add-on does what the +plan says. + +Each step gives every battery a role, as `modetrade.decide()` does (the +names are modetrade's): in a grid block it charges (CHARGE) and the others +rest (REST, no charge); else it follows the house (HOUSE) if it is Solar +first or no price plan is on; inside a give run it covers the house (GIVE); +else it holds (HOLD, may charge from spare solar, gives nothing). Spare +solar (sun less the house's use) goes to Solar first batteries first, then +the rest, by room. A shortfall comes from the batteries that may give, +those in a give run first, by what they hold above the floor, each at most +its rating, the rest passed on (as control shares it, `packshare.weights()` +and the allocator's fill). A battery that empties inside a step gets a +point there (`_floor_cut()`), as one that reaches its grid level does. +Charging and giving lose `dayplan.EFF`, as the plan counts. A level stays +inside the floor and ceiling the plan uses, and a battery already outside +them does not move further out. + +`SolarToday` keeps the site's measured solar per hour, in memory, from the +reading the control cycle already has. It is shown and never read back +(TEL-11 keeps site solar out of control). +""" + +import math +from datetime import datetime, timedelta, timezone +from dataclasses import dataclass + +from .dayplan import EFF, _hour_slices, _local +from .modetrade import CHARGE, GIVE, HOLD, HOUSE, REST + +EPS = 1e-9 +MIN_CUT = timedelta(seconds=1) # no step is cut closer to its ends +MAX_FLOOR_CUTS = 400 # a bound on the floor cuts of one curve +GAP_S = 60.0 # a longer gap between readings is not counted +MIN_COVER = 0.5 # share of the elapsed hour a reading must cover + + +@dataclass(frozen=True) +class Batt: + """One battery that is on, with a level and a capacity.""" + name: str + capacity_kwh: float + rating_kw: float + soc: float # % now + solar_first: bool # Solar first: follows the house all day + priced: bool # a price plan is on and its mode is a price mode + grid: tuple = () # ((start, end, to_pct), ...) grid blocks + give: tuple = () # ((start, end), ...) give runs + stop_at: datetime | None = None # a monthly cycle starts: no level after + + +def _role(b, t, charging, own): + """The battery's role at instant `t`; `own`: it charges from the grid; + `charging`: some battery does. Returns `(role, may_charge, may_give)`.""" + in_give = any(s <= t < e for s, e in b.give) + if own: + return CHARGE, False, False + if charging: + return REST, False, b.solar_first or not b.priced or in_give + if b.solar_first or not b.priced: + return HOUSE, True, True + if in_give: + return GIVE, True, True + return HOLD, True, False + + +def _spread(amount, caps, weights): + """Split `amount` over the keys of `caps` by `weights`, none over its + cap. Returns `(taken, left)`: what each took and what no one could.""" + got = {k: 0.0 for k in caps} + left = float(amount) + live = {k for k, c in caps.items() if c > EPS and weights.get(k, 0.0) > 0} + while left > EPS and live: + total = sum(weights[k] for k in live) + used, full = 0.0, set() + for k in live: + part = left * weights[k] / total + room = caps[k] - got[k] + if part >= room - EPS: + part = room + full.add(k) + got[k] += part + used += part + left -= used + if not full: + break + live -= full + return got, max(0.0, left) + + +def _give_caps(roles, level, by, floor, hrs): + """`(first, rest, held)`: what each battery that may give can give in a + step of `hrs` hours (its rating, or what it holds above the floor), those + in a give run first, and what each holds above the floor (as given).""" + held = {n: max(0.0, level[n] - floor) / 100.0 * by[n].capacity_kwh * EFF + for n, r in roles.items() if r[2]} + caps = {n: min(by[n].rating_kw * hrs, h) for n, h in held.items()} + first = {n: c for n, c in caps.items() if roles[n][0] == GIVE} + return first, {n: c for n, c in caps.items() if n not in first}, held + + +def _floor_cut(a, z, net, roles, level, by, floor, hrs): + """Where in the step `[a, z)` the first battery that gives reaches the + floor, or None. The shortfall comes at a steady rate, `-net / hrs` kW. + It is shared as control shares it (`packshare.weights()`: by the energy + each holds above its cut-out, here the floor), each at most its rating, + what one cannot take passed to the others, those in a give run first. + A battery giving `v` kW that holds `h` kWh is empty at `h / v` hours; a + cut there makes the step come out exact. A battery already at the floor + holds nothing, gives nothing and never sets a cut.""" + rate, cut = -net / hrs, None + *groups, held = _give_caps(roles, level, by, floor, hrs) + for grp in groups: + rates, rate = _spread(rate, {n: by[n].rating_kw for n in grp}, + {n: held[n] for n in grp}) + for n, v in rates.items(): + if held[n] > EPS and v > EPS: + t = a + timedelta(hours=held[n] / v) + if a + MIN_CUT < t < z - MIN_CUT and (cut is None + or t < cut): + cut = t + return cut + + +def curve(batts, start, end, sun_kwh, use_kwh, *, floor_pct, ceiling_pct, + trust=1.0, stats=None): + """`{name: [(t, pct), ...]}` from `start` (each battery's level now) to + `end`, a point at every hour edge, every block edge and where a battery + reaches its grid level or, giving, the floor. A battery's list + stops where `sun_kwh(a, b)` is None (the sun is not known) or its + `stop_at` comes. `sun_kwh` and `use_kwh` give kWh for the span `[a, b)`. + Spare solar is `trust x max(0, sun - use)`, and what is stored loses + `EFF`: `dayplan.plan()`'s own `trust x EFF x max(0, sun - house)`. A + shortfall is the house's use less the whole sun. `stats`, a dict or + None: gets `floor_cuts` (the floor points placed) and `capped` (True when + `MAX_FLOOR_CUTS` stopped one).""" + floor, ceil = float(floor_pct), float(ceiling_pct) + cuts = {end} + for a, _b, _h in _hour_slices(start, end): + cuts.add(a) + for b in batts: + for s, e, *_ in b.grid + b.give: + cuts.update(x for x in (s, e) if start < x < end) + cuts.add(start) + ts = sorted(cuts) + level = {b.name: float(b.soc) for b in batts} + out = {b.name: [(start, level[b.name])] for b in batts} + alive = {b.name for b in batts} + floor_cuts, capped = 0, False + i = 0 + while i < len(ts) - 1: + a, z = ts[i], ts[i + 1] + if not alive: + break + sun = sun_kwh(a, z) + if sun is None: + break + for b in batts: + if b.name in alive and b.stop_at is not None and a >= b.stop_at: + alive.discard(b.name) + net = sun - use_kwh(a, z) + if net > 0.0: + net *= trust + hrs = (z - a).total_seconds() / 3600.0 + want = {} # name -> grid target % + for b in batts: + if b.name in alive: + for s, e, to in b.grid: + if s <= a < e and level[b.name] < to - EPS: + want[b.name] = float(to) + by = {b.name: b for b in batts} + # A battery that reaches its level inside the step gets a point + # there: the step is cut and read again, so the line is exact. + cut = None + for n, to in want.items(): + b = by[n] + room = (to - level[n]) / 100.0 * b.capacity_kwh + rate = b.rating_kw * hrs * EFF + if EPS < room < rate * (1.0 - 1e-9): + t_r = a + (z - a) * (room / rate) + if a < t_r < z and (cut is None or t_r < cut): + cut = t_r + if cut is not None: + ts.insert(i + 1, cut) + continue + roles = {b.name: _role(b, a, bool(want), b.name in want) + for b in batts if b.name in alive} + # A battery that gives all it holds inside the step gets a point + # where it reaches the floor, the same way. + cut = _floor_cut(a, z, net, roles, level, by, floor, hrs) \ + if net < -EPS else None + if cut is not None and floor_cuts >= MAX_FLOOR_CUTS: + capped, cut = True, None + if cut is not None: + floor_cuts += 1 + ts.insert(i + 1, cut) + continue + for n, to in want.items(): # charging from the grid + b = by[n] + room = (to - level[n]) / 100.0 * b.capacity_kwh + level[n] += min(b.rating_kw * hrs * EFF, room) \ + / b.capacity_kwh * 100.0 + if net > EPS: + caps = {n: min(by[n].rating_kw * hrs, + max(0.0, ceil - level[n]) / 100.0 + * by[n].capacity_kwh / EFF) + for n, r in roles.items() if r[1]} + first = {n: c for n, c in caps.items() if by[n].solar_first} + rest = {n: c for n, c in caps.items() if n not in first} + left = net + for grp in (first, rest): + got, left = _spread(left, grp, dict(grp)) + for n, v in got.items(): + level[n] += v * EFF / by[n].capacity_kwh * 100.0 + elif net < -EPS: + first, rest, held = _give_caps(roles, level, by, floor, hrs) + left = -net + for grp in (first, rest): + got, left = _spread(left, grp, {n: held[n] for n in grp}) + for n, v in got.items(): + level[n] -= v / EFF / by[n].capacity_kwh * 100.0 + for b in batts: + if b.name in alive: + out[b.name].append((z, level[b.name])) + i += 1 + if stats is not None: + stats["floor_cuts"] = stats.get("floor_cuts", 0) + floor_cuts + stats["capped"] = stats.get("capped", False) or capped + return out + + +# --------------------------------------------------------------------------- # +# Measured solar, per local hour +# --------------------------------------------------------------------------- # +class SolarToday: + """The site's solar power, averaged per local hour of one local day. In + memory only: a restart starts it again, and `first` says from when it + has counted. Display only.""" + + def __init__(self): + self.day = None + self.first = None + self._hours = {} # hour start (UTC) -> [W x s, seconds] + self._last = None # (time, W) + + def add(self, now: datetime, w) -> None: + """One reading (`None`: no reading, which ends the interval).""" + day = _local(now).date() + if day != self.day: + self.day, self.first, self._hours, self._last = day, None, {}, None + try: + ok = w is not None and math.isfinite(float(w)) + except (TypeError, ValueError): + ok = False + if not ok: + self._last = None + return + w = max(0.0, float(w)) + if self.first is None: + self.first = now + if self._last is not None and now > self._last[0]: + dt = min(GAP_S, (now - self._last[0]).total_seconds()) + key = _hour_start(self._last[0]) + row = self._hours.setdefault(key, [0.0, 0.0]) + row[0] += (self._last[1] + w) / 2.0 * dt + row[1] += dt + self._last = (now, w) + + def kwh(self, start: datetime, end: datetime, now: datetime): + """kWh for the hour `[start, end)` up to `now`: the mean power it was + read at over the part of the hour that has passed, or None when the + readings cover under half of that part (or none has passed).""" + if self.day != _local(now).date(): + return None + over = (min(end, now) - start).total_seconds() + row = self._hours.get(start) + if over <= 0 or row is None or row[1] < MIN_COVER * over: + return None + return row[0] / row[1] * over / 3.6e6 + + +def _hour_start(t: datetime) -> datetime: + """The start of `t`'s local hour, in UTC. Taken back from the UTC instant, + never from the wall clock, so each of the two 02:00 hours of the autumn + change has its own (`dayplan._hour_slices()`, SAFETY-123).""" + lt = _local(t) + return t.astimezone(timezone.utc) - timedelta( + minutes=lt.minute, seconds=lt.second, microseconds=lt.microsecond) diff --git a/goodwe_controller/app/dayplan.py b/goodwe_controller/app/dayplan.py index 4bd8ecc..09a1021 100644 --- a/goodwe_controller/app/dayplan.py +++ b/goodwe_controller/app/dayplan.py @@ -7,20 +7,31 @@ turns the level into permission to charge from the grid; main.py rebuilds the plan with the price runs (at most every 15 minutes) and keeps a level unless it moves by `HYST_KWH` or `HYST_FRAC` (`hold()`). -The sums (OPT-11-design section 1, "Excess solar", with the owner's answers): +The sums (OPT-11-design section 1, "Excess solar", with the owner's answers; +OPT-28 for the second): * `sun`: trust x the solar the house will not use, between now and the expensive run the cheap run is for, x the charge efficiency: `trust x sum_h max(0, forecast_h - house_h) x EFF`. * Solar first batteries take it first, up to the ceiling. What is left is shared by the Leftover solar batteries by their room. +* `short` (OPT-28): what the house draws in the expensive run that the sun + will not give, `sum_h max(0, house_h - trust x forecast_h)` over every + hour of the run, early morning and evening. The Leftover solar batteries + hold it together: `short / EFF` above the floor, shared by capacity, and + a battery that reaches the ceiling passes the rest to the others. What + all of them cannot hold (`Plan.uncovered_kwh`) is not bought: another + battery (Cheap hours, Solar first) or the grid carries it. * Cheap hours only ignores the plan and fills to the ceiling, as in P3 and the design (section 1). Skipping the sun's share cost money in the replay (PR #156 review: 29.72 against 29.41 EUR on real 2026 prices). -* A Leftover solar battery charges only up to what the house will draw from - it in the expensive runs before the next cheap run, less its share: - `min(ceiling, floor + house use / EFF) - share`. So it buys only the +* A Leftover solar battery charges from the grid up to the level it needs + for its part of `short`, less its share of the sun before the run: + `min(ceiling, floor + its part / EFF) - share`. So it buys only the shortfall, not up to the ceiling. * Solar first never charges from the grid (owner, 2026-09-29). +* A state-only forecast is read from now for every span (`Forecast.between` + `since=`, OPT-23), so the hours add up to the day's figure. +DECISIONS/OPT-23-28-the-night-charge-covers-the-dear-hours.md has the record. The house profile is the add-on's own measurement, grid + battery power, per local hour, over the last `PROFILE_DAYS` days; the estimate for an hour is the @@ -31,7 +42,7 @@ control decision. """ import statistics -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone from .packmode import LEFTOVER_SOLAR, SOLAR_FIRST @@ -154,21 +165,53 @@ class Plan: targets: dict # name -> level % (Leftover solar only) need_kwh: dict # name -> kWh from the grid (0: none) share_kwh: dict # name -> kWh of solar counted for it + # OPT-28: the expensive hours' shortfall, the part of it each battery + # covers, and what no battery can hold. All kWh on the house's side. + short_kwh: float = 0.0 + cover_kwh: dict = field(default_factory=dict) + uncovered_kwh: float = 0.0 def _hour_slices(t0, t1): - """Local-hour slices of [t0, t1): (start, end, local hour).""" + """Local-hour slices of [t0, t1): (start, end, local hour). + + Steps in UTC, so `t` always advances (SAFETY-123, DECISIONS).""" out, t = [], t0 while t < t1: lt = _local(t) - nxt = (lt.replace(minute=0, second=0, microsecond=0) - + timedelta(hours=1)).astimezone(timezone.utc) - e = min(nxt, t1) + left = timedelta(minutes=60 - lt.minute) - timedelta( + seconds=lt.second, microseconds=lt.microsecond) + e = min(t.astimezone(timezone.utc) + left, t1) out.append((t, e, lt.hour)) t = e return out +def _spread(amount, room, weight): + """Share `amount` over the keys of `room` by `weight`, none past its + room, what one cannot take passed to the others. Returns `(taken, + left)`: what each took and what none could.""" + got = {k: 0.0 for k in room} + left = float(amount) + live = {k for k, r in room.items() if r > 1e-9 and weight.get(k, 0) > 0} + while left > 1e-9 and live: + total = sum(weight[k] for k in live) + used, full = 0.0, set() + for k in sorted(live): + part = left * weight[k] / total + free = room[k] - got[k] + if part >= free - 1e-9: + part = free + full.add(k) + got[k] += part + used += part + left -= used + if not full: + break + live -= full + return got, max(0.0, left) + + def _room(p, to_pct): if p.capacity_kwh is None or p.soc is None: return 0.0 @@ -192,7 +235,7 @@ def plan(forecast, house, runs, now, packs, *, trust_pct, floor_pct, trust = max(0.0, min(1.0, float(trust_pct) / 100.0)) sun = fc_sum = 0.0 for s, e, h in _hour_slices(now, until.start): - pv = forecast.between(s, e) + pv = forecast.between(s, e, since=now) if pv is None: return None # the span reaches a day with no forecast hrs = (e - s).total_seconds() / 3600.0 @@ -210,30 +253,40 @@ def plan(forecast, house, runs, now, packs, *, trust_pct, floor_pct, rooms = {p.name: _room(p, ceiling_pct) for p in price} total_room = sum(rooms.values()) - # What the house draws in the expensive runs from the one this plan is - # for until the next cheap run (or a day), from the profile. + # OPT-28: what the house draws in the expensive runs, from the one this + # plan is for until the next cheap run (or a day), that the sun will not + # give. An hour with sun is read at no less than the night base, as the + # sun sum above reads it (the profile is net of the sun the house used); + # an hour without is read as measured. A run hour with no forecast counts + # no sun. horizon = until.start + timedelta(hours=24) - use_kwh = 0.0 + short = 0.0 for g in runs.give: if g.start < until.start or g.start >= horizon: continue for s, e, h in _hour_slices(g.start, g.end): - use_kwh += max(0.0, house_w(h)) * ( - e - s).total_seconds() / 3600.0 / 1000.0 + pv = forecast.between(s, e, since=now) or 0.0 + w = max(0.0, house_w(h)) + if pv > 0.0: + w = max(w, base_w) + short += max(0.0, w * (e - s).total_seconds() / 3.6e6 + - trust * pv) caps = {p.name: float(p.capacity_kwh) for p in price} - cap_sum = sum(caps.values()) or 1.0 + span = max(0.0, float(ceiling_pct) - float(floor_pct)) + held, left = _spread(short / EFF, + {n: span / 100.0 * c for n, c in caps.items()}, + caps) - targets, need, share = {}, {}, {} + targets, need, share, cover = {}, {}, {}, {} for p in price: sh = excess * rooms[p.name] / total_room if total_room > 0 else 0.0 sh_pct = sh / caps[p.name] * 100.0 - want = use_kwh * caps[p.name] / cap_sum / EFF - top = min(float(ceiling_pct), - float(floor_pct) + want / caps[p.name] * 100.0) + top = float(floor_pct) + held[p.name] / caps[p.name] * 100.0 level = max(0.0, top - sh_pct) targets[p.name] = round(level, 2) need[p.name] = round(_room(p, level), 3) share[p.name] = round(sh, 3) + cover[p.name] = round(held[p.name] * EFF, 3) if sun < sf_room: verdict = DARK elif any(v > 0.05 for v in need.values()): @@ -241,7 +294,8 @@ def plan(forecast, house, runs, now, packs, *, trust_pct, floor_pct, else: verdict = SUNNY return Plan(verdict, round(sun, 3), round(fc_sum, 3), until.start, - targets, need, share) + targets, need, share, round(short, 3), cover, + round(left * EFF if price else 0.0, 3)) def hold(old: dict | None, new: dict, capacity: dict) -> dict: diff --git a/goodwe_controller/app/dayview.py b/goodwe_controller/app/dayview.py new file mode 100644 index 0000000..1664a13 --- /dev/null +++ b/goodwe_controller/app/dayview.py @@ -0,0 +1,452 @@ +"""UX-54: the Forecast tab, built on request (`GET forecast`). Reads only. + +Two days, today and tomorrow, per local hour: the solar forecast, the solar +measured so far, the house's expected use, and each battery's expected level +with its blocks. + +It follows the add-on, as Home does (UX-53), and decides nothing: +* the blocks are Home's own: today's are the `runs` of `/status`'s plan, and + tomorrow's come from the same builders (`homeview.runs_for()`), so a block + is on the tab exactly where the plan has one, with the plan's words; +* the sun is `solarfc.Forecast.between()` and the house's use is + `dayplan.HouseProfile.estimator()` read as `dayplan.plan()` reads it; +* the level is `daycurve.curve()`, fed the blocks, the give runs, the packs + and the plan's levels (`modetrade.charge_to()`), floor and ceiling. +A part that is not known yet is a plain note (`home_words.day`), never an +empty chart. DECISIONS/UX-54-forecast-tab.md has the record. +""" + +import dataclasses +from datetime import datetime, timedelta + +from . import arbitrage, dayplan, homeview as hv, modetrade, priceruns +from . import solarfc +from .daycurve import Batt, curve +from .packmode import SOLAR_FIRST +from .price import brussels, required_spread, uncertainty_of + +BLOCKS = ("grid", "give", "cycle") # the kinds a battery's blocks carry +LEVEL_DP = 1 # the tab shows a level to 0.1 % + + +def _hours_between(a, b) -> float: + return (b - a).total_seconds() / 3600.0 + + +def build(c, status: dict, now: datetime, lang: str = "en") -> dict: + """The tab's JSON. `status` is the `/status` payload just built, whose + plan holds today's blocks.""" + with hv.in_lang(lang): + return _build(c, status, now) + + +def sun_kwh(fc, now, a, b): + """kWh the forecast gives in `[a, b)`, `a` at or after `now`, or None. + Every span is measured from `now` (`Forecast.between(since=)`, OPT-23), + as `dayplan.plan()` measures them: a state-only "left today" figure is + the sun from the moment it was taken, so a span that starts later must + not be spread over what is left of the bell from its own start + (DECISIONS/UX-54-forecast-tab.md).""" + return fc.between(a, b, since=now) + + +def _times(run): + return (datetime.fromisoformat(run["start"]), + datetime.fromisoformat(run["end"])) + + +def _build(c, status, now): + plan, rows = status.get("plan"), status.get("batteries") + if not plan or rows is None: + return {"ok": False} + d0, d1 = hv.local_day(now) + n0, n1 = hv.local_day(d1 + timedelta(hours=2)) + shadow = bool(c.shadow) + key = plan.get("verdict_key") + running = key not in ("held", "stopped") + modes_on = bool(getattr(c, "modes_on", False)) + floor, ceil = arbitrage.soc_bounds(c.o) + fc = getattr(c, "solar_fc", None) + hp = getattr(c, "house_profile", None) + est = hp.estimator() if hp is not None and fc is not None else None + trust = max(0.0, min(1.0, float(getattr(c, "solar_trust_pct", 70)) + / 100.0)) + + # Why a part is not known (None: nothing is missing). + missing = ("modes_off" if not modes_on + else "not_set" if c.fc_entity is None + else "no_forecast" if fc is None + else "learning" if est is None else None) + + # The blocks of each day: today's from Home, tomorrow's by its builders. + # Tomorrow has not started, so the level now says nothing about it: the + # builders read each block at the level it starts at (`ahead`), with + # their own room and energy filter. With modes on that level is the + # curve's (`_levels()`); with modes off there is no curve, and it is + # Home's own count (`_counted()`). + runs = [plan["runs"], []] + if running and not modes_on: + runs[1] = _off_tomorrow(c, runs[0], now, shadow, n0, n1) + elif running: + runs[1] = hv.runs_for(c, n0, n1, now, shadow, ahead=True) + + # The level curve of every battery that is on and known. It also makes + # tomorrow's blocks again at the level each has when its run starts, and + # draws a grid block as long as control keeps charging (`_levels()`). + notes_b = {} + levels = {} + if running and missing is None: + levels, runs = _levels(c, est, fc, trust, now, n1, runs, floor, ceil, + notes_b, shadow, n0, n1) + blocks = [[[r for r in rs if r["battery"] == i and r["kind"] in BLOCKS] + for i in range(len(c.inverters))] for rs in runs] + # Prices are planned about a day ahead (`priceruns.HORIZON`), so tomorrow's + # blocks may be missing now and appear later. + priced = bool(plan.get("has_prices")) and ( + any(c.pack_mode(i).use and c.pack_mode(i).mode in hv.PRICE_MODES + for i in c.inverters) if modes_on else arbitrage.enabled(c.o)) + days = [] + for k, (a, z) in enumerate(((d0, d1), (n0, n1))): + days.append(_day(c, k, a, z, now, fc, est, trust, levels, blocks[k], + rows, shadow, running, key, missing, notes_b, status, + priced)) + return {"ok": True, "running": running, "shadow": shadow, + "now_h": round(_hours_between(d0, now), 3), + "updated": None if fc is None else hv.hhmm(fc.at), + "days": days} + + +def _levels(c, est, fc, trust, now, end, runs, floor, ceil, notes_b, shadow, + n0, n1): + """`({name: [(t, pct), ...]}, runs)`: `daycurve.curve()` for the batteries + that are on, have a level and are not in their monthly cycle, and `runs` + with tomorrow's blocks made as control will meet them. + + Tomorrow's blocks are `homeview.mode_runs()`'s, read at the level the + curve has when each block starts (its `ahead` function), so its own room + and energy filter and `MIN_KWH` decide: no give block where the battery + is at the floor when the run starts, no grid block where it is at its + planned level. The curve in turn follows the blocks, so the two are taken + to a fixed point. + + A cheap run is sized for all the batteries charging together, so a slower + one is still below `modetrade.charge_to()` when it ends, and control then + plans the next cheap slots (`Controller._price_runs()`). That is asked of + the same functions, `modetrade.energy()` and `priceruns.plan()`, with the + packs at the curve's level, and the block is drawn to where control + stops charging. Today's blocks are Home's, as they are.""" + packs = {p.name: p for p in c._mode_packs()} + targets = c.plan_targets() + rp = getattr(c, "mode_runs", None) + plan_on = rp is not None and rp.on + house_w, base_w = est + + def make(rs): + batts = [] + for i, inv in enumerate(c.inverters): + pm = c.pack_mode(inv) + if not pm.use: + continue + if inv.soc is None or not inv.capacity_kwh: + notes_b[inv.name] = hv.day_note("no_level", name=inv.name) + continue + mine = [r for day in rs for r in day + if r["battery"] == i and r["kind"] in BLOCKS] + cyc = [_times(r)[0] for r in mine if r["kind"] == "cycle"] + if cyc and min(cyc) <= now: + notes_b[inv.name] = hv.day_note("cycle", name=inv.name) + continue + if cyc: + notes_b[inv.name] = hv.day_note("cycle", name=inv.name) + to = modetrade.charge_to(packs[inv.name], ceil, targets) + grid = tuple((*_times(r), to) for r in mine + if r["kind"] == "grid" and to is not None) + priced = plan_on and pm.mode != SOLAR_FIRST + give = tuple((g.start, g.end) for g in rp.give) if priced else () + batts.append(Batt(inv.name, float(inv.capacity_kwh), + inv.max_w / 1000.0, float(inv.soc), + pm.mode == SOLAR_FIRST, priced, grid, give, + min(cyc) if cyc else None)) + return batts + + def sun(a, z): + return sun_kwh(fc, now, a, z) + + def use(a, z): + return sum(max(house_w(h), base_w) * _hours_between(s, e) / 1000.0 + for s, e, h in dayplan._hour_slices(a, z)) + + def run(rs): + batts = make(rs) + return curve(batts, now, end, sun, use, floor_pct=floor, + ceiling_pct=ceil, trust=trust) if batts else {} + + def settle(ext, lv): + """Tomorrow's blocks read at the levels of `lv`, and the curve they + make, again until the blocks stop changing.""" + seen = None + for _ in range(4): + def at(name, _kind, start, lv=lv): + return _level_at(lv.get(name, ()), start) + rt = hv.runs_for(c, n0, n1, now, shadow, ahead=at, rp=ext) + lv = run([runs[0], rt]) + sig = [(r["battery"], r["kind"], r["start"], r["end"]) + for r in rt] + if sig == seen: + break + seen = sig + return rt, lv + + lv0 = run([runs[0], []]) + ext = rp + plan = c.fresh_tariff_plan(now) + cheap = None if rp is None else rp.cheap + end_t = None + if plan_on and cheap is not None and rp.charge_useful() \ + and n0 <= cheap.start < n1 and plan is not None: + # The gap control's run plan needs (TARIFF-24): no x1.5 with a + # forecast read. + required = required_spread( + arbitrage.threshold_eur_kwh(c.o), + uncertainty_of(plan.read.product) + | arbitrage.weather_uncertainty(True, c.solar_fc)) + end_t = cheap.end + for _ in range(24): + ext = dataclasses.replace(rp, cheap=priceruns.Run( + cheap.start, end_t, cheap.mean_eur_kwh)) + rt, lv = settle(ext, lv0) + at_end = [dataclasses.replace(packs[n], soc=v) + for n, v in ((n, _level_at(lv.get(n, ()), end_t)) + for n in lv) if v is not None] + e = modetrade.energy(at_end, floor, ceil, targets=targets) + if e[0] <= 1e-6: + break + new = priceruns.plan( + plan.read.points, end_t, required_eur_kwh=required, + charge_kwh=e[0], charge_kw=e[1], sell_kwh=0.0, sell_kw=0.0) + nxt = new.cheap + if nxt is None or not new.charge_useful() or nxt.start > end_t \ + or nxt.end <= end_t: + break + end_t = nxt.end + ext = dataclasses.replace(rp, cheap=priceruns.Run( + cheap.start, end_t, cheap.mean_eur_kwh)) + rt, lv = settle(ext, lv0) + # A run is for all the batteries charging together; each stops when it is + # at its own level (control's `chargers()`), so its block ends there. + names = [inv.name for inv in c.inverters] + rt = [_stops(r, names, lv, packs, targets, floor, ceil, n0) for r in rt] + return lv, [runs[0], rt] + + +def _off_tomorrow(c, today, now, shadow, n0, n1): + """Modes off: tomorrow's blocks by `homeview.trade_runs()`, each read at + the level `_counted()` gives when it starts. A charge may come before a + give, so this repeats until the blocks stop changing.""" + sig, rt = None, [] + for _ in range(4): + rt = hv.runs_for(c, n0, n1, now, shadow, + ahead=_counted(c, [today, rt])) + new = [(r["battery"], r["kind"], r["start"], r["end"]) for r in rt] + if new == sig: + break + sig = new + return rt + + +def _counted(c, rs): + """Modes off reads no forecast and keeps no house profile, so there is no + curve. A give is read at Home's own count (`trade_runs()`): the level now, + plus what the blocks in `rs` that start before it put in, less what they + take out. The house can only take from that and no sun is read, so a + battery at the floor by the count is at the floor then. A charge is read + from the floor (None): the house uses some of a full battery before the + cheap hour, and control charges it back then.""" + _floor, ceil = arbitrage.soc_bounds(c.o) + + def at(name, kind, start): + i = next(k for k, inv in enumerate(c.inverters) if inv.name == name) + inv = c.inverters[i] + if kind == "grid" or inv.soc is None or not inv.capacity_kwh: + return None + v = float(inv.soc) + mine = sorted((r for day in rs for r in day if r["battery"] == i + and r["kind"] in ("grid", "give") + and r.get("kwh") is not None + and _times(r)[0] < start), + key=lambda r: _times(r)[0]) + for r in mine: + d = float(r["kwh"]) / float(inv.capacity_kwh) * 100.0 + v = min(max(v, ceil), v + d) if r["kind"] == "grid" else v - d + return v + return at + + +def _stops(r, names, lv, packs, targets, floor, ceil, day0): + """`r` ended where its battery stops, if that is before the run ends: a + grid block where the curve has it at the level `modetrade.charge_to()` + allows (the curve has a point there), a give block at the first point + where the level the tab shows (`LEVEL_DP`) is at the floor. A run is for + all the batteries together; each leaves it on its own.""" + if r["kind"] not in ("grid", "give"): + return r + name = names[r["battery"]] + start, end = _times(r) + if r["kind"] == "grid": + to = modetrade.charge_to(packs[name], ceil, targets) + if to is None: + return r + hit = lambda v: v >= to - 1e-6 # noqa: E731 + else: + hit = lambda v: round(v, LEVEL_DP) <= floor # noqa: E731 + done = next((t for t, v in lv.get(name, ()) if start < t <= end + and hit(v)), None) + if done is None or done >= end: + return r + return dict(r, end=done.isoformat(), to=hv.hhmm(done), + end_h=round(_hours_between(day0, done), 3)) + + +def _level_at(pts, t): + """The curve's level at `t` (between two points, straight), or None when + the curve does not reach `t`.""" + for (t0, v0), (t1, v1) in zip(pts, pts[1:]): + if t0 <= t <= t1: + span = (t1 - t0).total_seconds() + return v0 if span <= 0 else v0 + (v1 - v0) * ( + (t - t0).total_seconds() / span) + return None + + +def _day(c, k, d0, d1, now, fc, house, trust, levels, blocks, rows, shadow, + running, key, missing, notes_b, status, priced): + day = (fc.today, fc.tomorrow)[k] if fc is not None else None + solar = getattr(c, "solar_today", None) + names = [inv.name for inv in c.inverters] + hours = [] + for s, e, h in dayplan._hour_slices(d0, d1): + f = None + if fc is not None and e > now and (k == 0 or fc.tomorrow is not None): + f = sun_kwh(fc, now, max(s, now), e) + got = solar.kwh(s, e, now) if k == 0 and solar is not None \ + and s < now else None + use = None if house is None else max(house[0](h), house[1]) \ + * _hours_between(s, e) / 1000.0 + row = {"start_h": round(_hours_between(d0, s), 3), + "end_h": round(_hours_between(d0, e), 3), "label": hv.hhmm(s), + "forecast_kwh": None if f is None else round(f, 3), + "measured_kwh": None if got is None else round(got, 3), + "house_kwh": None if use is None else round(use, 3), + "levels": []} + hours.append((s, row)) + batts = [] + for i, inv in enumerate(c.inverters): + row = rows[i] if i < len(rows) else {} + pts = [(t, v) for t, v in levels.get(inv.name, ()) if d0 <= t <= d1] + mine = blocks[i] + batts.append({ + "name": inv.name, "capacity_kwh": inv.capacity_kwh, + "off": row.get("use") is False, + "levels": [[round(_hours_between(d0, t), 3), round(v, LEVEL_DP)] + for t, v in pts], + "blocks": [{x: r[x] for x in ("kind", "from", "to", "start_h", + "end_h", "what", "why", "amount")} + for r in mine], + "line": _line(c, inv, row, mine, running, key, shadow, d0, d1, + now)}) + at = {t: v for t, v in pts} + for s, hr in hours: + hr["levels"].append(None if s not in at + else round(at[s], LEVEL_DP)) + has_levels = any(b["levels"] for b in batts) + fcs = [h["forecast_kwh"] for _s, h in hours] + notes = [] + if missing is not None: + notes.append(hv.day_note(missing)) + if not running: + notes.append(hv.why_words(key)) + if shadow: + notes.append(hv.why_words("shadow")) + line = "" + if fc is not None: + if k == 0: + line = hv.line_solar(fc.today.kwh, fc.today.kwh * trust, + hv.hhmm(fc.at), + kept=getattr(fc, "kept", False)) + elif fc.tomorrow is not None: + line = hv.day_note("tomorrow_total", + kwh=hv._kwh(fc.tomorrow.kwh), + count=hv._kwh(fc.tomorrow.kwh * trust)) + if day is not None and not day.periods: + notes.append(hv.day_note("estimate")) + if k == 1 and fc.tomorrow is None: + notes.append(hv.day_note("no_tomorrow", at=_sun_from())) + if k == 0 and solar is not None and solar.first is not None \ + and solar.day == brussels(now).date() \ + and _hours_between(d0, solar.first) > 0.5: + notes.append(hv.day_note("measured_from", at=hv.hhmm(solar.first))) + if has_levels: + notes.append(hv.day_note("trust", pct=round(trust * 100))) + if k == 1 and _tomorrow_line(status): + notes.append(_tomorrow_line(status)) + if k == 1 and running and priced: + notes.append(hv.day_note("ahead")) + notes += [notes_b[n] for n in names if n in notes_b] + return {"key": ("today", "tomorrow")[k], "date": brussels(d0).date() + .isoformat(), + "length_h": int(round(_hours_between(d0, d1))), + "line": line, "notes": list(dict.fromkeys(notes)), + "sun": {"forecast": any(x is not None for x in fcs), + "estimate": day is not None and not day.periods, + "measured": any(h["measured_kwh"] is not None + for _s, h in hours), + "house": house is not None}, + "hours": [h for _s, h in hours], "batteries": batts} + + +def _sun_from() -> str: + """When the sun of a day starts for `solarfc` (its bell's first hour): + with no forecast for tomorrow the sun is unknown from then, and so the + curve stops there (`Forecast.between()` answers None past it).""" + return f"{int(solarfc.BELL_FROM_H):02d}:00" + + +def _tomorrow_line(status) -> str: + """Home's own line about tomorrow's prices, for the tomorrow tab.""" + return (status.get("plan") or {}).get("tomorrow_line") or "" + + +def _give_planned(c, d0, d1, now) -> bool: + """Does the add-on plan a give in `[d0, d1)` not yet over? Modes on: a + give run of the run plan. Modes off: a give leg of the verdict that the + gate takes (`homeview.trade_legs()`), at a price that is not negative.""" + if not getattr(c, "modes_on", False): + plan = c.fresh_tariff_plan(now) + verdict, _why = hv.trade_verdict(c, plan, now) + return verdict is not None and any( + k == "give" and pt.eur_per_kwh >= 0 and pt.start < d1 + and hv._point_end(plan, pt) > max(now, d0) + for k, pt in hv.trade_legs(c, plan, verdict, now)) + rp = getattr(c, "mode_runs", None) + return rp is not None and rp.on and any( + g.end > max(now, d0) and g.start < d1 for g in rp.give) + + +def _line(c, inv, row, mine, running, key, shadow, d0, d1, now) -> str: + """What to say for a battery with no block: the plan's own words. Home + draws no give block for a battery with no energy to give, so a give that + the add-on plans with no block gets its own sentence.""" + if not running: + return hv.why_words(key) + if row.get("use") is False: + return hv.why_words("off") + modes_on = getattr(c, "modes_on", False) + gives = not modes_on or row.get("mode_choice") in hv.PRICE_MODES + if gives and not any(r["kind"] == "give" for r in mine) \ + and _give_planned(c, d0, d1, now): + return hv.day_note("give_later") + if mine: + return "" + if modes_on and gives: + return hv.day_note("no_blocks") + return hv.run_house_what(shadow) + " " + hv.run_house_why(shadow) diff --git a/goodwe_controller/app/homeview.py b/goodwe_controller/app/homeview.py index c416a0e..9acf705 100644 --- a/goodwe_controller/app/homeview.py +++ b/goodwe_controller/app/homeview.py @@ -16,9 +16,11 @@ them, so nothing here invents a plan. Every sentence is in the words section at the bottom and is held to STYLE.md by test_messages.py. """ +import contextlib from datetime import datetime, timedelta, timezone from . import arbitrage, messages, peakguard, uitext +from .dayplan import EFF from .maintenance import CHARGE as M_CHARGE, DRAIN as M_DRAIN, HOLD as M_HOLD from .maintenance import IDLE as M_IDLE, next_slot_start from .packmode import CHEAP_HOURS, GRID_CHARGE_MODES, LEFTOVER_SOLAR, SOLAR_FIRST @@ -212,25 +214,26 @@ def _price_mode_in_use(c): return False -def price_windows(c, plan, verdict): +def price_windows(c, plan, verdict, now=None): """`[(kind, start, end)]`: where the add-on itself buys ("cheap") and gives ("dear"). With battery modes on that is the kept `priceruns.RunPlan` - (`c.mode_runs`): the cheap run when charging in it pays, and every give - run. With them off it is the arbitrage verdict's two points, the ones - `trade_runs()` draws. Nothing is classified here.""" + (`c.mode_runs`): the cheap run when charging in it pays and the trade gate + allows it (UX-53), and every give run. With them off it is the arbitrage + verdict's legs the add-on will take, the ones `trade_runs()` draws + (`trade_legs()`, given `now`). Nothing is classified here.""" if getattr(c, "modes_on", False): rp = getattr(c, "mode_runs", None) if rp is None or not rp.on or not _price_mode_in_use(c): return [] out = [("cheap", rp.cheap.start, rp.cheap.end)] \ - if rp.charge_useful() else [] + if rp.charge_useful() and getattr(c, "mode_gate", None) is None \ + else [] return out + [("dear", g.start, g.end) for g in rp.give] if verdict is None: return [] - return [("cheap", verdict.cheapest.start, - _point_end(plan, verdict.cheapest)), - ("dear", verdict.dearest.start, - _point_end(plan, verdict.dearest))] + return [("cheap" if kind == "grid" else "dear", pt.start, + _point_end(plan, pt)) + for kind, pt in trade_legs(c, plan, verdict, now)] def day_range(points): @@ -310,32 +313,65 @@ def _point_end(plan, point): return point.start + timedelta(hours=1) +def leg_taken(c, plan, pt, now): + """UX-53: will the add-on act on this point of the verdict? One in + progress is the verdict's own answer now. One still ahead is asked of the + same gate (`trade_verdict()`, the call the control cycle makes) at the + point's own start. A charge that comes after the last dear period meets a + flat remainder there and is refused, so it is not announced. When the gate + allows it, the point is still the cheapest (dearest) one: the points + before it are gone and it was the first extreme of the points from now.""" + if now is None or pt.start <= now: + return True + return trade_verdict(c, plan, pt.start)[0] is not None + + +def trade_legs(c, plan, verdict, now): + """`[(kind, point)]` by start: the verdict's charge ("grid") and give + ("give") legs that the add-on will take (`leg_taken()`).""" + legs = sorted((("grid", verdict.cheapest), ("give", verdict.dearest)), + key=lambda kp: kp[1].start) + return [(k, pt) for k, pt in legs if leg_taken(c, plan, pt, now)] + + # Under this many kWh a block is not drawn: the cycle would not move it. MIN_KWH = 0.05 -def trade_runs(c, plan, verdict, day0, day1, now, shadow): +def trade_runs(c, plan, verdict, day0, day1, now, shadow, ahead=False): """A charge block and a give block on each battery, where the verdict put them. The site trades as one; each battery's share is capped by its own rating and by the room its level leaves (arbitrage's own floor/ceiling). Only what `arbitrage.decide()` would still do is drawn: no block on a battery with no room, no sale at a negative price, and no sale while the emptiest battery is at the floor (the cycle's `soc` is the emptiest). - With a level unknown the cycle does not trade, so nothing is drawn.""" + With a level unknown the cycle does not trade, so nothing is drawn. + UX-54 `ahead`: the window has not started, so the level now says nothing + about it. The same rules then read each battery at the level a leg starts + at: `ahead(name, kind, start)`, a level in % or None, as in `mode_runs()`; + for None (and for `ahead=True`) a charge is read from the floor and a give + from the ceiling (the most it could be). The leg itself is still the + gate's (`trade_legs()`). Home never passes it.""" floor, ceiling = arbitrage.soc_bounds(c.o) - if any(inv.soc is None for inv in c.inverters): + if not ahead and any(inv.soc is None for inv in c.inverters): return [], [], "no_level" - level = [float(inv.soc) for inv in c.inverters] - legs = sorted((("grid", verdict.cheapest), ("give", verdict.dearest)), - key=lambda kp: kp[1].start) + level = [] if ahead else [float(inv.soc) for inv in c.inverters] + + def start_pct(kind, start): + got = [ahead(inv.name, kind, start) if callable(ahead) else None + for inv in c.inverters] + return [(floor if kind == "grid" else ceiling) if v is None + else float(v) for v in got] runs, later = [], [] - for kind, pt in legs: + for kind, pt in trade_legs(c, plan, verdict, now): end = _point_end(plan, pt) if end <= now: continue if not (day0 <= pt.start < day1): later.append((kind, pt)) continue + if ahead: + level = start_pct(kind, pt.start) if kind == "give" and (pt.eur_per_kwh < 0 or min(level) <= floor): continue hrs = (end - pt.start).total_seconds() / 3600.0 @@ -350,7 +386,8 @@ def trade_runs(c, plan, verdict, day0, day1, now, shadow): kwh = min(kwh, max(0.0, room) / 100.0 * cap) # A later give may use what this charge put in: an upper # bound, like every amount here ("up to"). - level[i] += (kwh if kind == "grid" else -kwh) / cap * 100.0 + if not ahead: + level[i] += (kwh if kind == "grid" else -kwh) / cap * 100.0 if kwh < MIN_KWH: continue eur = kwh * pt.eur_per_kwh @@ -370,14 +407,22 @@ def trade_runs(c, plan, verdict, day0, day1, now, shadow): PRICE_MODES = (LEFTOVER_SOLAR, CHEAP_HOURS) -def mode_runs(c, day0, day1, now, shadow): +def mode_runs(c, day0, day1, now, shadow, ahead=False, rp=None): """OPT-11 P3: each battery's price runs today, as the control cycle keeps them (`c.mode_runs`): the cheap run on the batteries that may charge from the grid, the give runs on Leftover solar and Cheap hours, and the sell run on those with selling on. Amounts are upper bounds ("up to"), - from the battery's own room, rating and the run's length. Returns - `(runs, no_trade_key)`.""" - rp = getattr(c, "mode_runs", None) + from the battery's own room, rating and the run's length. UX-53: no grid + charge is drawn while the trade gate refuses one (`c.mode_gate`, what the + control cycle found). Returns `(runs, no_trade_key)`. + UX-54 `ahead`: the window has not started, so the level now says nothing + about it. The room and energy filter (`room()` and `plan_need()` below) + then reads the level a block starts at: `ahead(name, kind, start)`, a + function giving the expected level in % or None, or for None (and for + `ahead=True`) a charge from the floor and a give from the ceiling. The + same filter, the same `MIN_KWH`. `rp`: a run plan to read instead of + `c.mode_runs`. Home never passes either.""" + rp = rp if rp is not None else getattr(c, "mode_runs", None) if getattr(c, "mode_tick", None) is None: return [], None # no cycle yet: nothing is known if rp is None: @@ -385,6 +430,7 @@ def mode_runs(c, day0, day1, now, shadow): if not rp.on: return [], "not_worth" floor, ceiling = arbitrage.soc_bounds(c.o) + gate = getattr(c, "mode_gate", None) out = [] for i, inv in enumerate(c.inverters): pm = c.pack_mode(inv) @@ -392,17 +438,26 @@ def mode_runs(c, day0, day1, now, shadow): continue soc, cap = inv.soc, inv.capacity_kwh - def room(kind, hrs): + def start_pct(kind, start): + """The level a block of `kind` is read at when `ahead`.""" + got = ahead(inv.name, kind, start) if callable(ahead) else None + return got if got is not None else ( + floor if kind == "grid" else ceiling) + + def room(kind, hrs, start): kwh = inv.max_w / 1000.0 * hrs - if cap and soc is not None: - kwh = min(kwh, max(0.0, (ceiling - soc) if kind == "grid" - else (soc - floor)) / 100.0 * cap) + lvl = start_pct(kind, start) if ahead else soc + if cap and lvl is not None: + kwh = min(kwh, max(0.0, (ceiling - lvl) if kind == "grid" + else (lvl - floor)) / 100.0 * cap) return kwh blocks = [] # P4: with a day plan, the grid block is the planned shortfall. - need = plan_need(c, inv.name) - if rp.charge_useful() and (need > 0.0 if need is not None - else pm.mode in GRID_CHARGE_MODES): + need = plan_need(c, inv.name, start_pct("grid", rp.cheap.start) + if ahead and rp.cheap is not None else None) + if gate is None and rp.charge_useful() and ( + need > 0.0 if need is not None + else pm.mode in GRID_CHARGE_MODES): blocks.append(("grid", rp.cheap)) sell = rp.sell if pm.sell and pm.can_sell else None for g in rp.give: @@ -414,16 +469,22 @@ def mode_runs(c, day0, day1, now, shadow): if run.end <= now or not (run.start < day1 and run.end > day0): continue hrs = (run.end - run.start).total_seconds() / 3600.0 - kwh = room(kind, hrs) + kwh = room(kind, hrs, run.start) if kind == "grid" and need is not None: kwh = min(kwh, need) if kwh < MIN_KWH: continue price = run.mean_eur_kwh if kind == "grid": + cov = plan_cover(c, inv.name) what, why, amount = (run_grid_what(kwh, shadow), run_cheap_why(price), run_grid_amount(kwh * price)) + if cov is not None: + # OPT-28: with a day plan the block says how far it + # charges and what that covers. + what = run_grid_what_to(kwh, cov[0], shadow) + why += " " + run_covers(cov[1]) elif kind == "sell": # No money line: the price is what the house pays, and a sale # earns the lower injection price (OPT-11 P3 review). @@ -441,28 +502,62 @@ def mode_runs(c, day0, day1, now, shadow): return out, None -def plan_need(c, name): +def plan_need(c, name, soc=None): """P4: kWh this battery buys from the grid by the day plan, or None when - there is no plan for it (then P3 option A applies).""" + there is no plan for it (then P3 option A applies). `soc`: UX-54, a level + to read it from instead of the one now (`mode_runs(ahead=True)`).""" targets = c.plan_targets() if hasattr(c, "plan_targets") else None if not targets or name not in targets: return None inv = next((i for i in c.inverters if i.name == name), None) - if inv is None or inv.soc is None or not inv.capacity_kwh: + at = None if inv is None else (inv.soc if soc is None else soc) + if inv is None or at is None or not inv.capacity_kwh: return None - return max(0.0, targets[name] - float(inv.soc)) / 100.0 * float( + return max(0.0, targets[name] - float(at)) / 100.0 * float( inv.capacity_kwh) +def plan_cover(c, name): + """OPT-28: `(level %, kWh)` for a battery the day plan charges from the + grid: the level it charges to, and the expensive-hour use of the house + that charge covers and the sun will not. Read from the level control + holds (`plan_targets()`), so Home says what control does. None when the + plan buys nothing for it.""" + dp = getattr(c, "day_plan", None) + targets = c.plan_targets() if hasattr(c, "plan_targets") else None + inv = next((i for i in c.inverters if i.name == name), None) + if dp is None or not targets or name not in targets or inv is None \ + or not inv.capacity_kwh or targets[name] <= 0.0: + return None + floor, _ceil = arbitrage.soc_bounds(c.o) + cap = float(inv.capacity_kwh) + top = targets[name] + dp.share_kwh.get(name, 0.0) / cap * 100.0 + return targets[name], max(0.0, top - floor) / 100.0 * cap * EFF + + +def _latched_below(c) -> bool: + """OPT-34: no forecast, but a cheap run is on and a battery is still below + the level its night charge keeps (`Controller.plan_targets()`), so control + goes on charging.""" + rp = getattr(c, "mode_runs", None) + if rp is None or rp.cheap is None or not rp.cheap.holds( + datetime.now(timezone.utc)): + return False + targets = getattr(c, "plan_targets", lambda: None)() or {} + return any(inv.soc is not None and inv.name in targets + and inv.soc < targets[inv.name] for inv in c.inverters) + + def plan_state(c): """P4: `"plan"` (a day plan is made), `"learning"` (no house profile - yet), `"no_forecast"` (set but missing or stale), or None (not set, or - battery modes off).""" + yet), `"no_forecast"` (set but missing or stale), `"latched"` (OPT-34: + the same, but a night charge under way is still going on to its level), + or None (not set, or battery modes off).""" if not getattr(c, "modes_on", False) or \ getattr(c, "fc_entity", None) is None: return None if getattr(c, "solar_fc", None) is None: - return "no_forecast" + return "latched" if _latched_below(c) else "no_forecast" hp = getattr(c, "house_profile", None) if hp is not None and not hp.ready(): return "learning" @@ -483,11 +578,15 @@ def solar_block(c, now): out.update(fc_kwh=round(fc.today.kwh, 1), count_kwh=round(fc.today.kwh * trust, 1), updated=hhmm(fc.at)) - if st in ("no_forecast", "learning"): + if st in ("no_forecast", "learning", "latched"): out["line"] = _t("solar." + st) elif fc is not None: out["line"] = line_solar(fc.today.kwh, out["count_kwh"], - out["updated"]) + out["updated"], + kept=getattr(fc, "kept", False)) + dp = getattr(c, "day_plan", None) + if st == "plan" and dp is not None and dp.uncovered_kwh >= MIN_KWH: + out["line"] += " " + line_uncovered(dp.uncovered_kwh) return out @@ -593,6 +692,20 @@ def region_of(opts): if rid in REGION_PRESETS else None} +def runs_for(c, day0, day1, now, shadow, ahead=False, rp=None): + """UX-54: the price blocks of the window `[day0, day1)`, by the same + builders `build_plan()` uses for today: the batteries' price runs with + modes on, the verdict's legs with them off. Nothing is decided here. + `ahead`: see `mode_runs()`; for a day that has not started.""" + if getattr(c, "modes_on", False): + return mode_runs(c, day0, day1, now, shadow, ahead, rp)[0] + fresh = c.fresh_tariff_plan(now) + verdict, _why = trade_verdict(c, fresh, now) + if verdict is None: + return [] + return trade_runs(c, fresh, verdict, day0, day1, now, shadow, ahead)[0] + + def build_plan(c, now, shadow, running, held=False, settled=True): day0, day1 = local_day(now) fresh = c.fresh_tariff_plan(now) @@ -612,7 +725,7 @@ def build_plan(c, now, shadow, running, held=False, settled=True): runs, lines = [], [] # The control cycle trades on the fresh plan only, so this does too. verdict, trade_why = trade_verdict(c, fresh, now) - bars = price_bars(seen, day0, day1, price_windows(c, fresh, verdict), + bars = price_bars(seen, day0, day1, price_windows(c, fresh, verdict, now), now, days.kinds) modes_on = bool(getattr(c, "modes_on", False)) if running: @@ -719,9 +832,11 @@ def refusing(c, inv) -> int: def peak_engaged(c) -> bool: """The peak guard asks the batteries for power this cycle. Not when it is blind or floor-limited: those are engaged with a 0 W floor (no charging - only), so no battery gives power for it.""" + only), so no battery gives power for it. Nor when the quarter ends on + the aim anyway (SAFETY-126, `within_aim`).""" pg = getattr(c, "peak_guard", None) - return bool(pg is not None and pg.engaged and (pg.floor_w or 0.0) > 0.0) + return bool(pg is not None and pg.engaged and (pg.floor_w or 0.0) > 0.0 + and not pg.within_aim) def battery_rows(c, shadow, running, held=False): @@ -797,7 +912,8 @@ def battery_rows(c, shadow, running, held=False): # knows this cycle. Price and forecast reasons are OPT-11 P3/P4. "why_key": why, "why": why_words(why, name=cycling if why == "other_cycle" - else other, at=at), + else other, at=at, + **plan_words(c, inv.name, why)), }) if modes_on: # OPT-11 P1: the battery's switch and mode, for the Batteries @@ -983,6 +1099,16 @@ def state_of(c, level, shadow, waiting=False): return "running" +@contextlib.contextmanager +def in_lang(lang): + """UX-54: build sentences in `lang` for the length of a `with` block.""" + _LANG[0] = uitext.lang_of(lang) + try: + yield + finally: + _LANG[0] = "en" + + def build(c, *, level: str, checks: list, now: datetime | None = None, lang: str = "en", held: bool | None = None, waiting: bool = False) -> dict: @@ -1085,6 +1211,10 @@ def _kwh(v) -> str: return _dec(f"{float(v):.1f}") + " kWh" +def _pct(v) -> str: + return _dec(f"{float(v):.0f}") + " %" + + # Row id -> the title on Home. The row's own sentence is the body, in the # page's language (UX-46, `messages.pick`). Five titles are the notification's # own title, which test_ui_text.py checks. @@ -1217,11 +1347,18 @@ def battery_now(soc, batt_w, *, phase, shadow, running, held=False, return _t("now.resting") -def why_words(key, name=None, at="") -> str: +def why_words(key, name=None, at="", pct="", kwh="") -> str: """One battery's "why" line (UX-13 P3), from `battery_why()`'s key. `name`: the battery that refuses, for `other_refuses`; `at`: the - give run's time, for `saves`.""" - return _t("why." + key, name=name or "", at=at) + give run's time, for `saves`; `pct` and `kwh`: the planned level and + what it covers, for `tops_up` (OPT-28).""" + return _t("why." + key, name=name or "", at=at, pct=pct, kwh=kwh) + + +def plan_words(c, name, why) -> dict: + """OPT-28: the figures of a `tops_up` line, or nothing.""" + cov = plan_cover(c, name) if why == "tops_up" else None + return {} if cov is None else {"pct": _pct(cov[0]), "kwh": _kwh(cov[1])} def today_words(key) -> str: @@ -1243,6 +1380,16 @@ def run_grid_what(kwh, shadow) -> str: return _w(shadow, "run.grid_what", kwh=_kwh(kwh)) +def run_grid_what_to(kwh, pct, shadow) -> str: + """OPT-28: a grid block of the day plan, with the level it charges to.""" + return _w(shadow, "run.grid_what_to", kwh=_kwh(kwh), pct=_pct(pct)) + + +def run_covers(kwh) -> str: + """OPT-28: what a planned grid charge covers.""" + return _t("run.covers", kwh=_kwh(kwh)) + + # The trade legs and the price line use the feed's own points (a 15-minute # period on a quarter-hourly feed), so both name the same price and time. def run_grid_why(price) -> str: @@ -1311,10 +1458,25 @@ def run_house_amount() -> str: return _t("run.house_amount") -def line_solar(fc_kwh, count_kwh, at) -> str: +def day_note(key, **values) -> str: + """UX-54: one sentence of the Forecast tab (`home_words.day`).""" + return _t("day." + key, **values) + + +def line_solar(fc_kwh, count_kwh, at, kept=False) -> str: + """The forecast line. OPT-27: a reading kept past three hours (the + evening's, used at night) says so instead of "Updated".""" + if kept: + return _t("solar.kept", at=at) return _t("solar.line", kwh=_kwh(fc_kwh), count=_kwh(count_kwh), at=at) +def line_uncovered(kwh) -> str: + """OPT-28: the Leftover solar batteries cannot hold the expensive hours' + shortfall. The rest comes from another battery or the grid.""" + return _t("solar.uncovered", kwh=_kwh(kwh)) + + def line_no_trade(key) -> str: return "" if key == "off" else _t("no_trade." + key) @@ -1442,6 +1604,7 @@ def _samples() -> list: for k in VERDICT: out.append(("plan", plan_head(k, sh))) out += [("plan", run_grid_what(12.34, sh)), + ("plan", run_grid_what_to(12.34, 82, sh)), ("plan", run_give_what(9.6, sh)), ("plan", run_cycle_what(M_IDLE, sh)), ("plan", run_cycle_what(M_DRAIN, sh)), @@ -1458,6 +1621,7 @@ def _samples() -> list: ("plan", control_text(True, sh)), ("plan", control_text(False, sh))] out += [("plan", run_grid_why(0.1412)), ("plan", run_grid_amount(1.23)), + ("plan", run_covers(9.4)), ("plan", line_uncovered(2.5)), ("plan", run_give_why(0.3612)), ("plan", run_give_amount(3.47)), ("plan", run_cycle_why()), ("plan", run_cycle_why(True)), ("plan", run_cycle_amount(None)), @@ -1488,10 +1652,15 @@ def _samples() -> list: out.append(("plan", battery_now(62, w, phase=M_IDLE, shadow=False, running=True, off=off, peak=peak))) - out += [("plan", why_words(k, name="Battery A")) for k in words["why"]] + out += [("plan", why_words(k, name="Battery A", pct="82 %", + kwh="9.4 kWh")) for k in words["why"]] out += [("plan", today_words(k)) for k in words["today"]] out += [("plan", line_solar(9.2, 6.4, "10:15")), - ("plan", _t("solar.no_forecast")), ("plan", _t("solar.learning"))] + ("plan", _t("solar.no_forecast")), ("plan", _t("solar.learning")), + ("plan", _t("solar.latched"))] + out += [("plan", day_note(k, pct=70, name="Battery A", at="08:15", + kwh="9.2 kWh", count="6.4 kWh")) + for k in words["day"]] out += [("plan", line_no_trade(k)) for k in ("not_covered", "no_prices", "peak", "not_firm", "not_worth", "no_level")] diff --git a/goodwe_controller/app/main.py b/goodwe_controller/app/main.py index 702a592..35fced1 100644 --- a/goodwe_controller/app/main.py +++ b/goodwe_controller/app/main.py @@ -64,7 +64,8 @@ from .clamp import ( shortfall_note, write_clamp_note, ) from . import freshness -from . import homeview +from . import dayview, homeview +from .daycurve import SolarToday from . import messages from .freeze import ( FreezeWatchdog, FreezeWatchdogConfig, looks_frozen, rail_unreachable, rail_w, @@ -207,9 +208,44 @@ EDGE_HOLD_S = _ctl.PLANT_SETTLE_S # OPT-11 P4: the store key of the house profile (dayplan.HouseProfile), and # how often the forecast sensors are read (they change hourly at most). HOUSE_PROFILE_KEY = "house_profile" +# OPT-27: the forecast sensors' last fresh readings (`solarfc.Hold`), so an +# evening restart keeps the figure the night plan uses. +FC_HOLD_KEY = "solar_fc_hold" +# OPT-34: the level of a night grid charge that has started, so it does not +# drop when the forecast goes (`Controller._latch_charge()`). +CHARGE_LATCH_KEY = "night_charge_latch" +# OPT-34: a latch outlives the cheap run it was last used in by one price +# slot, because the next run is planned at the slot edge, in the cycle after +# the old one ended. +CHARGE_LATCH_GRACE_S = 900.0 FORECAST_READ_S = 60.0 +def _load_latch(data, now=None): + """`(until, {name: level %}, born)` from the store, or None if it is not a + well-formed latch or has ended (OPT-34).""" + try: + until = datetime.fromisoformat(data["until"]) + born = datetime.fromisoformat(data.get("born") or data["until"]) + levels = {str(n): float(v) for n, v in data["levels"].items()} + except (TypeError, KeyError, ValueError, AttributeError): + return None + if until.tzinfo is None or born.tzinfo is None or not levels or not all( + math.isfinite(v) and 0.0 <= v <= 100.0 for v in levels.values()): + return None + if now is not None and now >= until: + return None + return until, levels, born + + +def _night_limit(born): + """OPT-34: a run that starts after this cannot carry the latch on: the + first bell start (`solarfc.BELL_FROM_H`, local) after `born`.""" + d0, d1 = solarfc._bounds(born) + bell = solarfc._bell_start(d0) + return bell if bell > born else solarfc._bell_start(d1) + + def why_raised(resolution, claims) -> bool: """F2 (OPT-11 P1 test): did the limit that bound the site target push it ABOVE what the winner asked (the peak floor asking for more power)? Then @@ -946,6 +982,9 @@ class Controller: # SAFETY-87: the reserve alarm's hold, as SAFETY-81's above. self._reserve_batt_prev = None self._reserve_high_since = None + # SAFETY-126: `_peak_landing()`'s last battery reading and latch. + self._land_batt_prev = None + self._peak_needed = False self._maint_shape_suspended = False # OPT-11. Battery modes, off by default so an install that has not # turned them on behaves exactly as before (SAFETY-10 section 1). Each @@ -964,6 +1003,10 @@ class Controller: # was read; until then losing the forecast does not widen the gap. self._wx_free_to = None self.mode_tick = None + # UX-53: why the trade gate refused a grid charge this tick (`peak`, + # `not_worth`, `not_covered`), or None when it allowed one. Home reads + # it; nothing in control does. + self.mode_gate = None self._run_done = {} self._mode_trade_was = False self._charge_blocked_q = None @@ -990,8 +1033,14 @@ class Controller: self.solar_fc_why = (solarfc.NOT_SET if self.fc_entity is None else solarfc.MISSING) self._fc_read_mono = None + self.fc_hold = solarfc.Hold( + (getattr(store, "data", None) or {}).get(FC_HOLD_KEY)) self.day_plan = None self._plan_targets = None + # OPT-34: `(until, {name: level %})` of a night charge under way. + self._charge_latch = _load_latch( + (getattr(store, "data", None) or {}).get(CHARGE_LATCH_KEY), + datetime.now(timezone.utc)) self.house_profile = None if self.fc_entity is not None: self.house_profile = dayplan.HouseProfile( @@ -1222,6 +1271,9 @@ class Controller: # UX-13 P1: today's prices as the Home chart has seen them. Display # only; nothing reads it back into a decision. self.home_prices = homeview.DayPrices() + # UX-54: the site's measured solar per hour, for the Forecast tab. + # In memory, display only; nothing reads it back into a decision. + self.solar_today = SolarToday() # TARIFF-11. When the most recent cycle() STARTED. `arb_at < cycle_at` # is the exact, derived statement of "the latest cycle did not get as # far as the arbitrage step", which is the fact the *Arbitrage* row @@ -2249,9 +2301,13 @@ class Controller: def weather_held(self, now) -> bool: """TARIFF-24: no forecast now, but a run planned with one is under way: it keeps its gap to its end. The wider gap applies from the next - run. Pure; Home asks the same (`homeview.trade_verdict()`).""" - return (self.solar_fc is None and self._wx_free_to is not None - and now < self._wx_free_to) + run. OPT-34: a night charge that keeps its level (`_latch_charge()`) + keeps the gap too, run after run, or the next slice of it would be + refused. Pure; Home asks the same (`homeview.trade_verdict()`).""" + return self.solar_fc is None and ( + (self._wx_free_to is not None and now < self._wx_free_to) + or (self._charge_latch is not None + and now < self._charge_latch[0])) def _price_runs(self, now, packs, floor_pct, ceiling_pct): """The kept run plan, rebuilt at most once per price slot (15 min) or @@ -2336,11 +2392,73 @@ class Controller: def plan_targets(self): """P4: each battery's planned level, or None (no day plan). Dropped - at once when the forecast goes (stale or missing).""" + at once when the forecast goes (stale or missing), except the level + of a night charge that has started (OPT-34): it keeps that level.""" if self.solar_fc is None: - return None + lt = self._charge_latch + if lt is None or datetime.now(timezone.utc) >= lt[0]: + return None + return dict(lt[1]) return self._plan_targets + def _latch_charge(self, now, runs, targets, packs=None) -> None: + """OPT-34, once per cycle after `modetrade.decide()`. A night grid + charge that has started keeps its planned level when the forecast goes + mid-charge (the kept figure ends and the sensors are still silent). + + While there is a forecast, fresh or kept, the latch is rebuilt each + cycle from the plan: a battery that charges from the grid in the cheap + run comes in with its planned level, and stays in it at whatever level + the plan gives it, up or down as a fresh figure moves it (also when it + has stopped, at a lowered level), until it leaves the plan. While there + is none, `plan_targets()` gives the latched levels, so the charge goes + on to them, and `weather_held()` keeps the narrow price gap. The latch + lasts through every cheap run on and one price slot after the last + (`CHARGE_LATCH_GRACE_S`), but a run is carried on only if it starts + by the bell start of the night the latch began in (`_night_limit()`; + the run under way then finishes). It is dropped when `now` passes its + end whatever the run state, and a battery that is off or no longer in + a mode with a planned level (`packs`) leaves it. A charge that never + started has none.""" + old = lt = self._charge_latch + tick = self.mode_tick + run = None if runs is None else runs.cheap + in_run = run is not None and run.holds(now) + grace = timedelta(seconds=CHARGE_LATCH_GRACE_S) + if lt is not None and now >= lt[0]: + lt = None + if lt is not None and packs is not None: + ok = {p.name for p in packs + if p.use and p.mode in packmode.PLAN_CHARGE_MODES} + lv = {n: v for n, v in lt[1].items() if n in ok} + lt = (lt[0], lv, lt[2]) if lv else None + # A run may carry the latch on only if it starts by the night's bell. + carry = in_run and (lt is None or run.start <= _night_limit(lt[2])) + if in_run and self.solar_fc is not None: + # Rebuilt from the plan: a battery already latched follows its + # level up or down whatever its role this tick (a fresh figure may + # lower it below the SoC), one that left the plan is dropped, and + # a new one comes in only while it charges from the grid. + tg = targets or {} + lv = {n: float(tg[n]) for n in (lt[1] if lt else {}) if n in tg} + if tick is not None and tick.action == modetrade.CHARGE: + lv.update({n: float(tg[n]) for n, r in tick.roles.items() + if r == modetrade.CHARGE and n in tg}) + if not lv: + lt = None + elif lt is None: + lt = (run.end + grace, lv, now) + else: + lt = (max(run.end + grace, lt[0]) if carry else lt[0], lv, + lt[2]) + elif lt is not None and carry: + lt = (max(lt[0], run.end + grace), lt[1], lt[2]) + if lt != old: + self._charge_latch = lt + self.store.set(CHARGE_LATCH_KEY, None if lt is None else { + "until": lt[0].isoformat(), "levels": lt[1], + "born": lt[2].isoformat()}) + def _mode_roles(self, now, maint_results): """OPT-11 P3: each battery's role and range this tick. Grid charging and selling need a fresh plan that `TariffPlan.arbitrage()` allows @@ -2371,6 +2489,7 @@ class Controller: charge_ok = sell_ok and allowed(guard) trade_why = (None if charge_ok else "peak" if sell_ok else "not_worth") + self.mode_gate = None if charge_ok else trade_why price_now = None if plan is not None: try: @@ -2412,6 +2531,7 @@ class Controller: blocked=self._charge_blocked_q == quarter, done_charge=done_c, done_sell=done_s, targets=targets, draining=any(r.phase == DRAIN for r in maint_results or ())) + self._latch_charge(now, runs, targets, packs) # A charge the guard or the headroom stopped stays off this period. # OPT-15: only a MEASURED stop latches; the first unmeasured headroom # (None) in a quarter stops this tick only, so a gap cannot cost the @@ -2975,6 +3095,27 @@ class Controller: soc=min(float(p.soc) for p in able), soc_floor=peakguard.binding_soc_floor(able, floor)) + def _peak_landing(self, decision): + """SAFETY-126: `decision` marked `within_aim` while the guard wants + power but the batteries at 0 W would end the quarter within + `ALARM_MISS_W` of the aim (`peakguard.misses_aim()`). Once a real miss + is seen, the need stands until the guard lets go. The claim and the + commands are unchanged; only the alarms and the "Why" read it. + DECISIONS/SAFETY-126-landing-error-is-not-an-overrun.md.""" + batt, prev = self.site_batt_w, self._land_batt_prev + self._land_batt_prev = batt + if not (decision.engaged and decision.blind is None + and (decision.wanted_w or 0.0) > 0.0): + self._peak_needed = False + return decision + if batt is not None and prev is not None: + batt = min(float(batt), float(prev)) + self._peak_needed = self._peak_needed or ( + self._peak_proj is not None and peakguard.misses_aim( + self._peak_proj, batt_w=batt, target_w=decision.target_w)) + return (decision if self._peak_needed + else dataclasses.replace(decision, within_aim=True)) + def _peak_reserve(self) -> frozenset: """SAFETY-87: the batteries that give nothing for the peak this cycle: under their reserve while the others carry a peak floor.""" @@ -3369,6 +3510,11 @@ class Controller: *(self.hass.state(entity) for entity in roster)) self.pv_w, self.pv_why, self.pv_parts = pv_read.total( list(zip(roster, states)), pv_read.inverted(self.o)) + # UX-54: kept for the Forecast tab only. It must never cost a cycle. + try: + self.solar_today.add(datetime.now(timezone.utc), self.pv_w) + except Exception: # noqa: BLE001 + _LOG.debug("measured solar not kept", exc_info=True) def _peak_fc_unit_log(self, fault) -> None: """SAFETY-118. One WARNING when the peak forecast's unit cannot be read @@ -3396,17 +3542,25 @@ class Controller: now = datetime.now(timezone.utc) fc, why, gaps = solarfc.read_sum( list(zip(self.fc_entities, states[:n])), - list(zip(self.fc_tomorrow_entities, states[n:])), now) + list(zip(self.fc_tomorrow_entities, states[n:])), now, + hold=self.fc_hold) + if self.fc_hold.changed: + self.fc_hold.changed = False + self.store.set(FC_HOLD_KEY, self.fc_hold.dump()) # A member is named only in a sum of several; one entity reads as # before. Logged once per episode: on a change, never every read. gaps = tuple(g for g in gaps if len( self.fc_entities if g[0] == "today" else self.fc_tomorrow_entities) > 1) + was_kept = bool(self.solar_fc is not None and self.solar_fc.kept) if (fc is None) != (self.solar_fc is None) \ - or why != self.solar_fc_why or gaps != self.solar_fc_gaps: + or why != self.solar_fc_why or gaps != self.solar_fc_gaps \ + or bool(fc is not None and fc.kept) != was_kept: self.log_event( - "solar forecast: " + ("read" if fc is not None else - f"none ({why})") + "solar forecast: " + ( + "none (" + str(why) + ")" if fc is None else + f"read, kept from {brussels(fc.at):%H:%M}" if fc.kept + else "read") + "".join(f", {d} {e} not usable ({w})" for d, _i, e, w in gaps), hint=messages.hint( @@ -4818,6 +4972,7 @@ class Controller: if self.peak_guard is not None: self.peak_guard = self._peak_on_able(self.peak_guard, maint_results) + self.peak_guard = self._peak_landing(self.peak_guard) # SITE, and peakguard.claim() says so on the object it builds. The # quantity is the 15-minute average OFFTAKE AT THE GRID CONNECTION, # which is what the capacity tariff bills; there is one quarter-hour @@ -4842,6 +4997,7 @@ class Controller: self.log_event(_pg.explain(), hint=messages.hint( "peak_off" if not _pg.engaged else "peak_blind" if _pg.blind else + "peak_on" if _pg.within_aim else "peak_reserve" if _pg.floor_limited else "peak_saturated" if _pg.saturated else "peak_on")) @@ -6139,6 +6295,9 @@ class Controller: limited = self._reserve_short(g) else: self._reserve_batt_prev = self._reserve_high_since = None + # SAFETY-126: nothing can give, so it alarms at once, but only + # when the quarter really misses the aim (`_peak_landing()`). + limited = limited and not g.within_aim was, self._peak_floor_limited = self._peak_floor_limited, limited if not limited or was: return @@ -7016,12 +7175,13 @@ class Controller: if _pg is None: out.append({"ok": True, "warn": True, "id": "peak_guard", "text": messages.row_peak_guard_waiting()}) - elif _pg.saturated: + elif _pg.saturated and not _pg.within_aim: out.append({"ok": False, "warn": False, "id": "peak_guard", "runs": True, "text": messages.row_peak_saturated(_pg.ceiling_w)}) - elif _pg.floor_limited and (not (_pg.floor_w or 0.0) > 0.0 - or self._peak_floor_limited): + elif _pg.floor_limited and ( + not ((_pg.floor_w or 0.0) > 0.0 or _pg.within_aim) + or self._peak_floor_limited): # SAFETY-24. NOT ok, like saturation: in both states the money # is going to be spent and a human is the only actuator left. A # separate row from saturation because the two ask for opposite @@ -7638,6 +7798,21 @@ class Controller: out["home_error"] = True return out + def forecast_payload(self, lang: str = "en") -> dict: + """UX-54: the Forecast tab's data. Built on request, from what the + control cycle already holds; no Home Assistant call. Today's blocks are + Home's own, so `status_payload()` is built first and read.""" + try: + return dayview.build(self, self.status_payload(lang=lang), + datetime.now(timezone.utc), lang) + except Exception: # noqa: BLE001 + if not getattr(self, "_forecast_failed", False): + self._forecast_failed = True + _LOG.exception("The Forecast tab could not be built") + else: + _LOG.debug("The Forecast tab failed again", exc_info=True) + return {"ok": False} + async def handle_action(self, what: str, *, unit=None, value=None): now = datetime.now(timezone.utc) if what in ("pack_use", "pack_mode", "pack_sell"): diff --git a/goodwe_controller/app/peakguard.py b/goodwe_controller/app/peakguard.py index ddf9bad..b189970 100644 --- a/goodwe_controller/app/peakguard.py +++ b/goodwe_controller/app/peakguard.py @@ -1,403 +1,7 @@ -"""SAFETY-07 - the quarter-peak projection guard, and the only thing in this -add-on that is allowed to spend the loop's freedom to save money. - -WHAT IT IS. TARIFF-01 built the INSTRUMENT: `tariff.py` measures the number the -Belgian capaciteitstarief bills on - the highest 15-minute average offtake of the -calendar month - and banks it where a restart cannot reach. It files no claim and -commands nothing, and says so in its own first paragraph. This module is the -CONSUMER it named. It turns that measurement into one arbiter claim and nothing -else: no writes, no I/O, no clock, no state between calls. Given a projection and -two measured numbers it returns a decision; `main.py` files it. - ------------------------------------------------------------------------------- -THE HAZARD, WITH THE EURO FIGURE - -specs.md:583 records a measured case: ONE 4998 W quarter moved a monthly peak -from 4832 W to 5013 W and billed the whole month at it. The capacity tariff has -no averaging and no forgiveness - the month's single worst quarter is the bill. - -SAFETY-18 shipped the ALARM for the way this system produces such a quarter by -itself: a meter that answers 200 OK forever with a document that never changes, -which every freshness mechanism in this stack reads as healthy while the loop -integrates a constant error to the rail and holds it there. freeze.py's own -table: at the live install's `max_w: 5000`, a meter frozen at >= 3000 W rails the -command in 75 s. SAFETY-18 notifies at `meter_freeze_s` (300 s default), against -a 900 s billing quarter. That is about nine minutes of human reaction time and -nothing else - freeze.py says so in the words "THIS CUT DOES NOT CAP THAT." - -This is the cap. It is deliberately NOT a second freeze detector: it consumes -freeze.py's shipped predicate rather than transcribing it. - ------------------------------------------------------------------------------- -THE MECHANISM: ONE `lo` LIMIT AT `P_PEAK`, AND WHY THAT SHAPE - -specs.md 10.1 row 2 asks for exactly two behaviours - "pre-emptive discharge as -the projected quarter average approaches the soft limit; hard clamp on charging -above it" - and 9.1 constrains the lever: "Discharge and export are never capped -by the peak guard; only charging and import are." - -A LOWER BOUND on the setpoint is both of those in one object, and it is the only -shape that is: - - * incapable of capping discharge - a floor never stops the loop discharging - HARDER, so 9.1's asymmetry is structural here rather than remembered; - * incapable of capping export, for the same reason; - * a hard clamp on charging whenever the floor is above the standing charge; - * pre-emptive discharge whenever the floor is above zero. - -It is a LIMIT, not a SET, and that matters at both ends of the priority table. -`P_PEAK` (50) is above `P_LOOP` (10), so it binds the grid-following loop. It is -BELOW `P_MAINTENANCE` (60) and `P_SAFETY` (100), so a maintenance phase that owns -the actuator and every layer-1 stop still outrank it: arbiter.py's clause 2 only -applies limits whose priority is >= the WINNING SET's, so a P_SAFETY 0 W stop is -not dragged off zero by this floor. That is the correct order - a controller -pushing power the wrong way is wrong however much money it is trying to save. - -⚠️ AND A LIMIT DOES NOT OUT-VOTE ANOTHER LIMIT. This was assumed, and it is -false: arbiter.py clause 2 applies EVERY limit at or above the winner's priority -and takes the MOST RESTRICTIVE, with no notion of priority between them - which -is exactly what makes the safety bounds unconditional and must not be changed. -So this floor (`lo >= 0`) and the maintenance charge-only shaping (`hi <= 0`) do -not resolve in P_PEAK's favour when both are filed. They resolve to `lo > hi`, a -CONTRADICTION, which arbiter.py answers with 0 W - the battery parked in the one -quarter of the month it should be shaving hardest. The rule for two limits that -can cross is therefore SUPPRESSION, not priority: main.py drops the maintenance -shaping for as long as this guard is engaged, which is the same thing that branch -already did for `peak_forecast_entity` and is why 9.1's "money outranks the -maintenance schedule" reads as a suppression there rather than as a priority -here. The floor's clamp to `achievable_w` closes the only other pair that could -cross - the P_SAFETY rating bounds. - -AND IT IS WHY OPT-04 NEEDS NO CHANGE HERE. 9.1 requires peak shaving to outrank -arbitrage. Arbitrage does not exist yet, but it will arrive as a SET below -P_PEAK, and arbiter.py clause 2 will bind it with this floor on the day it lands, -with no edit to this file. Expressing the priority as a claim rather than as an -if/else in main.py is the whole reason that holds. - ------------------------------------------------------------------------------- -THE FLOOR IS DERIVED, AND EVERY TERM OF IT IS MEASURED - -Let `ceiling_w` be the quarter average that must not be exceeded and `remaining` -the seconds left in the open block. `project()` gives - - projected = (acc + held * remaining) / 900 - -so the held level that would land the quarter exactly on the ceiling is -`held + 900 * (ceiling - projected) / remaining`, and the watts that must come -off the meter are the difference between the two: - - excess_w = 900 * (projected - ceiling) / remaining - -That single expression carries the whole control law, and two properties of it -are the argument for this design: - - 1. THE CLAIM IS FILED WHILE THE FLOOR IS POSITIVE, NOT WHILE THE EXCESS IS. - `excess` is what must come off the METER, so it falls to zero the moment the - battery is already taking it - and a guard that disengaged there would hand - the actuator back, watch the offtake spring straight to the raw load, and - engage again next cycle. That is not a hypothetical: it is what the first - cut of this file did to 2026-08-20's billed quarter on replay - seventeen - engage/release cycles in 900 s, and the quarter still finished 49 W over the - limit it had spent a quarter of an hour holding. 10.2 names the cost - exactly: "an EMS that oscillates is worse than no EMS, both for hardware - wear and - per 9.1 - for the peak it is trying to avoid." - - `batt_w + excess` IS the quantity to threshold: the offtake there would be - with the battery at zero, less the level this quarter can still afford. Read - as a LEVEL rather than as a switch, the same arithmetic gives a claim that - is filed for as long as a positive floor is needed and decays continuously - to nothing as the load falls. So there is still no hysteresis constant in - this file - not because oscillation was ruled out by argument, but because - the thresholded quantity is the one that does not oscillate. - - 2. THE GAIN IS 900/remaining, WHICH IS 1.0 AT THE START OF THE QUARTER AND - GROWS. This is the answer to the degeneracy at the top of a block, where the - projection is an extrapolation from almost no evidence: the guard is at its - GENTLEST exactly when its evidence is weakest, and becomes aggressive only - once most of the quarter is behind it and the projection is nearly a - measurement. The formula supplies that; no minimum-evidence threshold was - invented, because inventing one would delay the guard in the case it exists - for - a sustained rail from second zero. - -The floor itself is then - - floor_w = batt_w + excess_w - -`batt_w` IS THE MEASURED BATTERY POWER, NOT THE STANDING COMMAND. Both were -available. The measured one is right for two independent reasons. First, 9.1's -settled constraint is that the utility bills what FLOWED - the arithmetic -`grid = load - batt` is true of delivered power and false of intent. Second, and -concretely: when the inverter cannot follow (the saturation case control.py -already freezes the integrator for), a floor built on the command asks for -`command + excess` every cycle while the command never arrives, and winds itself -to the rail; a floor built on the measurement asks for `delivered + excess`, -which stops rising the moment delivery stops rising. The delivered form cannot -wind up. The intent form can. - ------------------------------------------------------------------------------- -THE CEILING IS `peak_cap_w`, RAISED ONLY BY AN OBSERVATION - -11.2's row is "Quarter peak | Projected quarter average approaches THE MONTHLY -PEAK | Clamp the command", and 9.1's soft limit is `peak_cap_w`. Both matter, -because the money is spent on `max(month's peak so far, this quarter)`: - - ceiling_w = max(peak_cap_w, the month's banked peak) - -SAFETY-79: the guard AIMS `peak_margin_w` under that ceiling -(`quarter_target_w()`), so a quarter that lands a little high still stays under -it. `ceiling_w` stays the real limit for reports and the bill. - -Once a month's peak already stands at 5013 W, holding every later quarter to -3500 W buys nothing - the bill for that month is already 5013 W - while the -holding itself costs real charging the site wanted. Raising the ceiling to a peak -that has already been paid for is what stops one bad quarter from parking the -battery's charging for three weeks. - -AND THAT RELAXATION IS GRANTED ONLY BY AN `OBSERVED` READING. `NO_RECORD` and -`NOT_OBSERVED` both leave the ceiling at `peak_cap_w`, the tighter of the two, so -a peak store that cannot be read makes this guard STRICTER and never looser. That -direction is the whole point: tariff.py's `PeakReading` exists because a database -that did not answer must not become a number, and the number it would have become -here - a zero peak - is exactly the one that reads as "no peak recorded, full -freedom". It is also why this module does NOT consume `headroom_w()`, whose -disarming failure it would otherwise inherit: a peak read as 0 W returns FULL -headroom, and full headroom is a guard that is not there. - ------------------------------------------------------------------------------- -BLINDNESS COMES FIRST, AND IT IS A DIFFERENT ACTION - -A projection is evidence only if the quantity under it was actually measured. -Three ways it is not, all of them checked BEFORE the projection is graded, -because grading a stimulus where the quantity it grades is degenerate is how a -guard passes its own tests while being switched off: - - * NO OPEN BLOCK. `project()` answers a fresh averager with - `Projection(0.0, 0.0, 0.0, 0.0, False, None, 0.0)` - a ZERO projection, which - compares as comfortably under any ceiling. That is not "the quarter is fine", - it is "there is no quarter". Graded as a number it silently disarms the guard - on exactly the startup path where nothing else is watching. - - * COVERAGE. A meter gap longer than one `max_hold_s` window leaves seconds that - `QuarterAverager` deliberately does not accumulate (p1.py: holding a dead - meter's last value forward is how a fabricated peak gets onto a permanent - record). Those seconds make the projection read LOW, which is the - under-guarding direction. The threshold is `max_hold_s` itself - the point - p1.py already stops trusting a held sample - not a new number. - SAFETY-90: once readings are back, main.py counts those seconds - (`fill_gaps()`), so this holds only while the gap lasts. - - * FROZEN. SAFETY-18's conjunction, evaluated through `freeze.looks_frozen()` - rather than re-written here. A frozen meter is the worst case of all: the - samples keep ARRIVING, so coverage stays perfect and the projection is - confident - and if the stuck value is an export, the projection is a - confident ZERO while the real meter runs at the rail. This is the case - SAFETY-07 exists to backstop, and it is the case a projection cannot see. - -THE BLIND ACTION IS `floor_w = 0.0`: the battery may not IMPORT while the billed -quantity is unobservable. It is the minimal action that closes the hazard, and it -is one-directional - forbidding charging can only ever lower delivered offtake, -never raise it - so a false blindness costs a charge the site wanted and cannot -cost energy the pack did not want to give. Contrast the graded branch, which can -force discharge; blindness deliberately does not earn that, because there is by -definition no measurement to size it from. - ------------------------------------------------------------------------------- -WHY THIS ONE TRIPS WHERE SAFETY-18 ONLY NOTIFIES - -SAFETY-18 shipped notification-first, and config.yaml says why in one line: "A -false trip parks the battery; a false alarm costs an alert." That reasoning is -right for a DETECTOR whose negative case had to be established on a rig run. It -does not carry to this guard, and the two committed captures say so: - - * MEASURED FALSE-ENGAGEMENT RATE ON THE HEALTHY DAY: zero. 2026-08-23's worst - quarter is a few hundred watts against a 3500 W ceiling and nothing in its 95 - quarters comes near engaging. test_peakguard.py re-measures that every run - from the committed capture rather than quoting it here. - * THE ACTION IS LEVEL-TRIGGERED AND UNLATCHED. It is re-derived from a fresh - projection every cycle and releases the instant the projection drops back - under the ceiling. There is no state to clear and no operator to wake. A - false engagement costs one cycle of discharge, not a parked battery - which - is precisely the cost SAFETY-18 could not bound and this one can. - * THE ACTION IS A LIMIT, NOT A SET. Every safety stop in 11.2's table outranks - it, so the worst thing a bug in this file can do is fail to relax a floor - - and a floor is bounded above by `achievable_w`, which main.py takes from the - same `max_w` and device rating the arbiter already enforces. - -So it trips. It is also the only guard here whose trigger IS the billed quantity -itself, measured from parsed P1 telegrams, rather than a proxy for it. - ------------------------------------------------------------------------------- -SATURATION IS REPORTED, NOT ALARMED AS A BUG - -10.3's doctrine - "treat an out-of-range computed setpoint as a bug signal, clamp -it AND alarm" - is SAFETY-03's, and it is right there: a loop computing 14 768 W -is a winding integrator, i.e. a defect. A peak floor above the inverter's rating -is NOT a defect. It is the ordinary physics of a 5900 W house load on a 5000 W -inverter, which is literally what 2026-08-20 15:45 was. So the floor is clamped -to `achievable_w` and the decision carries `saturated`, and main.py raises ONE -notification on that - not because the controller is broken, but because it is -the one case where the money WILL be spent and the controller has already done -everything it can. An engagement that is not saturated is an ordinary control -action and goes to the action log, like the clamp's engagements do. - ------------------------------------------------------------------------------- -NO NEW OPTION. CFG-03 STILL TRACKS THREE PLACES. - -`peak_cap_w` already ships, is already validated `int(500,15000)`, and is already -read by `control.peak_at_risk()`. `max_hold_s` comes off the averager the caller -already owns; `achievable_w` off `max_w` and the device rating the arbiter -already enforces; the freeze predicate off `meter_freeze_s`. config.yaml and -validate.py are untouched by this ticket, exactly as TARIFF-01 left them, and -14's `capacity_peak_limit_kw` stays declined - a kW key beside a W key is the -mismatch trap 14's own warning box describes. - ------------------------------------------------------------------------------- -SAFETY-24: THE ENERGY BACKSTOP THIS GUARD SHIPPED WITHOUT - -SAFETY-07 left this file saying, in its own words, that the graded branch can -FORCE DISCHARGE and that nothing below it consults the state of charge - "the -BMS is the sole energy backstop" (freeze.py). On the 2026-08-20 replay the guard -asks for 878 Wh in a day, and it keeps asking for as long as the meter says the -quarter is over the ceiling. That is the behaviour working as designed, and it -is unbounded from below by anything in the EMS. - -⚠️ STATE THE RISK FAIRLY, because overstating it is how a real one gets ignored. -Relying on the BMS is not absurd: it is a genuine floor and it works. But it is -THE PACK's protection, not the OPERATOR's. A month of aggressive quarters can -leave the battery repeatedly parked at that floor with no EMS-side record of -why, and the first symptom - a pack that is never available when it is wanted - -is indistinguishable from the SoC drift MAINT-01 exists to correct. - -So `decide()` now takes `soc` and `soc_floor`, and `discharge_permitted()` is -the one place the question is answered. Four things about the shape of it: - - 1. IT IS NOT A TRIP, AND IT MUST NOT BECOME ONE. SAFETY-07's claim is a LIMIT - and every stop in 11.2's table still outranks it. This floor is about - ENERGY - how far down the guard may spend the pack for money - not about - last-resort safety. The action when it binds is therefore to stop ASKING - for more discharge, not to command anything. - - 2. THE ACTION IS `floor_w = 0.0`, WHICH IS THE BLIND BRANCH'S ACTION FOR THE - BLIND BRANCH'S REASON. Zero is one-directional: it forbids CHARGING, which - can only ever lower delivered offtake, and it forbids nothing else. It - cannot pull the loop off a discharge it is already doing, because a `lo` - limit structurally cannot cap discharge - the property this module's whole - shape rests on, and one a NEGATIVE floor would destroy. Nothing here can - produce a negative floor: the only two values `floor_w` ever takes are 0.0 - and `min(wanted, achievable)` with `wanted > 0` already established. - - ⚠️ AND IT DOES NOT BOUND THE LOOP. Self-consumption may still drain the - pack past this floor, exactly as SAFETY-18 recorded. That is not fixed - here and must not be claimed: this card bounds THE PEAK GUARD's OWN forced - discharge, which is the thing SAFETY-07 introduced. A loop-side SoC guard - is a different lever with a different right value. - - ⚠️ SAFETY-25 MEASURED HOW FAR PAST, so this is no longer a caveat that has - to be taken on trust. On the committed 2026-08-20 capture, 19:00-22:00 UTC - - three hours of house load after the sun has gone, 830 W mean, zero export - - the 13.8 kWh reference pack started one point ABOVE `peak_soc_floor` - ends the evening at 10.0 %, i.e. at `bms_soc_floor`, having delivered - 1525 Wh; 343 of 587 control cycles deliver discharge with the pack already - under this floor, and the guard files no floor at all on any of them. The - same replay at `peak_soc_floor` = 5, 11, 20 and 60 ends at the SAME 10.0 %, - which is the finding stated as sharply as it can be: THE EXPOSURE IS A - PROPERTY OF THE CONTROL LAW, NOT A VALUE ANYONE CAN CONFIGURE. And on the - same day's high-load block the two states coexist - 49 cycles reporting - PEAK GUARD FLOOR-LIMITED while the arbiter grants the loop up to 2000 W of - discharge in every one of them. test_peakguard.py part 10 is the replay; - DECISIONS/SAFETY-25-loop-soc-floor.md carries the options and the - recommendation, and the product owner DECIDED it on 2026-09-02: Option A, - the BMS stays the only backstop for demand-driven discharge, with a - loop-side reserve DEFERRED to ALLOC-01. This floor's scope is therefore - settled rather than provisional - it bounds the peak guard, on purpose. - - 3. IT IS GRADED ONLY WHERE IT IS NOT DEGENERATE. The SoC question is asked - AFTER `wanted > 0` is established - i.e. only where a forced discharge is - actually being asked for. Asked earlier it would grade a quantity that - governs nothing, and would report "floor-limited" on a quiet afternoon - where the guard was never going to discharge anyway. - - 4. AN UNREADABLE SoC IS NOT A FULL BATTERY. `soc=None` is floor-limited, not - permitted. This is `PeakReading`'s lesson in the one place it bites - hardest: a full battery is exactly the value that would disarm this floor, - so a failed read defaulting to one would silently restore the unbounded - discharge this card exists to remove. Both new arguments are REQUIRED - keyword-only and neither has a default, which is TEST-14 read as a rule - rather than as a story: a required field that gains a default is how three - separate files in this repo shipped a quiet fallback. - -AND FAILING TO SHAVE MUST NOT BE SILENT. The whole point of the guard is the -euro figure, so a floor-limited cycle carries `floor_limited=True` and main.py -raises ONE notification on the edge, saying the quarter will be exceeded - the -same shape, and the same argument, as `saturated`. The two are mutually -exclusive by construction: a floor-limited decision asks for nothing, so it -cannot have run out of inverter. - ------------------------------------------------------------------------------- -CFG-03: TWO NEW OPTIONS, MIRRORED ONCE, AND WHY BOTH ARE NEEDED - -SAFETY-07 added none and said so. This card cannot: an EMS-side floor is a -number only the site knows, and refusing one set below the pack's own cut-out -means knowing that cut-out too. - - * `peak_soc_floor` - the EMS floor. ⚠️ NOT `maintenance_soc_floor`. That is - the maintenance cycle's DRAIN TARGET: the depth a monthly recalibration - deliberately goes to, once, under supervision. This is the depth a - money-saving discharge may never reach, on any day of the month. Sharing - one key would couple two features whose right values differ and would widen - CFG-03 rather than narrow it. - * `bms_soc_floor` - the pack's own cut-out (register 0x0560 on the reference - unit, 10 %). It is here so validate.py rule 8 can REFUSE a `peak_soc_floor` - at or below it. That configuration is CFG-01 rule (a): the operator states - a floor, the BMS enforces a different one, and the status page reports an - energy backstop that can never bind. - -Both defaults are mirrored ONCE, in `PEAK_OPTION_DEFAULTS` below, which is -TARIFF-02's `PRICE_OPTION_DEFAULTS` pattern - imported by validate.py and by -main.py, pinned to config.yaml by test_peakguard.py rather than transcribed. -This module still reads no option itself: `decide()` takes both as ARGUMENTS, -makes no `.get()` call, and the mirror is a default table, not a read. - -⚠️ A DECISION RECORDED, NOT A RENAME DONE. Now that a second SoC floor exists, -`maintenance_soc_floor` is the ambiguous name of the two - it reads like "the -SoC floor" and is not. Renaming it is an option migration on a live install for -a clarity gain, so the decision recorded here is: KEEP IT, and carry the -distinction in DOCS.md and in this comment. Revisit if a THIRD floor appears - -a loop-side guard would be the third - at which point the migration buys enough -to be worth its cost. - ------------------------------------------------------------------------------- -SAFETY-32: ONE `bms_soc_floor` FOR TWO PACKS GRADES ONE AGAINST THE OTHER'S BMS - -`bms_soc_floor` above describes A PACK, not a site, and it was a flat option -because there was one pack. At two, two batteries of different models have -different cut-outs, and one number for both is wrong in the direction that -empties a battery: the SHALLOWER pack (the higher cut-out) is protected by a -floor that belongs to the DEEPER one. So it became a roster key - `Pack` carries -each unit's own value, `pack_floor_pct()` conjoins it with the site's EMS floor, -and validate.py rule 8 grades EVERY unit and NAMES the one it refuses. - -⚠️ `peak_soc_floor` DID NOT MOVE, AND THE RESIDUAL IS RECORDED RATHER THAN -HIDDEN. It is one number for the site, so with cut-outs of 10 % and 18 % the -operator must set it above 18 and the deeper pack is then withheld above its own -need. That half - a per-pack EMS floor - is NOT this card and must not be -smuggled in with it; DECISIONS/SAFETY-32-per-pack-bms-floor.md states it as the -open question it is. - -⚠️ AND THE MIN() OVER MEASURED SoC IS STILL RIGHT. `Controller.pack_floor_soc` -reduces to the emptiest pack and this card does not touch it. What changed is -the FLOOR it is graded against, and the addition of a per-pack answer beside the -site's one verdict - see `pack_permissions()`. - ------------------------------------------------------------------------------- -AND IT READS NO PER-PHASE QUANTITY, ON PURPOSE - -The sum/max/net reduction is an open product question with the owner. TARIFF-01 -put it behind `billed_w()` and defaulted to NET. Everything this module consumes -- `PeakReading.peak_w`, `Projection.projected_w` - is already reduced by that one -function, so whichever way the owner answers, this file does not change and does -not need re-testing. Nothing here indexes a phase. -""" +"""SAFETY-07 - the quarter-peak projection guard: one arbiter `lo` limit at +`P_PEAK` from tariff.py's measured month peak. Pure: no I/O, clock or state. +Why a limit, the floor's derivation, blindness, the SoC reserve and the +options: DECISIONS/SAFETY-07-peak-guard.md.""" from dataclasses import dataclass, replace @@ -406,77 +10,45 @@ from .arbiter import ACTUATOR_SITE, P_PEAK, Claim from .control import peak_at_risk from .tariff import QUARTER_S -# Why the projection may not be graded. Reasons, not booleans: the status page -# and the action log both have to say WHICH, and "blind" alone is unactionable. +# Why the projection may not be graded; the status page and log say which. BLIND_NO_BLOCK = "no open 15-minute period, the meter has given no reading yet" BLIND_COVERAGE = "meter gap" BLIND_FROZEN = "the meter is answering without changing" -# Where the ceiling came from. Only the second is a relaxation, and only an -# OBSERVED reading may produce it. +# Where the ceiling came from. Only an OBSERVED reading may relax it. CEILING_CAP = "peak_cap_w" CEILING_MONTH_PEAK = "this month's peak" -# SAFETY-24. Why the forced discharge may not be sized. Reasons, not booleans, -# for BLIND_*'s reason: the status page, the action log and the operator -# notification all have to say WHICH, and "floor-limited" alone is unactionable. +# SAFETY-24. Why a forced discharge may not be sized. FLOOR_SOC_UNREADABLE = ("the battery level was not read this cycle, and " "unread is not full") -# ⚠️ SAFETY-25. "the peak reserve", not "the SoC floor", and the name is -# load-bearing: this string is what the action log and the add-on log print, -# and `peak_soc_floor` (“Peak reserve”) bounds THIS GUARD ONLY. Widening it to -# "the SoC floor" would make the one sentence an operator reads at the moment -# the battery stops shaving into a claim about the pack that the -# grid-following loop is at that instant disproving. +# The words are "the peak reserve", not "the SoC floor": `peak_soc_floor` +# bounds this guard only (SAFETY-25). FLOOR_AT_SOC_FLOOR = "the battery level is at or below the peak reserve" SOURCE = "peak" -# ⚠️ CFG-03. THE ONE IN-CODE MIRROR OF config.yaml FOR THIS CARD'S OPTIONS, and -# it is TARIFF-02's `PRICE_OPTION_DEFAULTS` pattern for TARIFF-02's reason: the -# add-on already carries option defaults in three places (config.yaml's -# `options:`, config.yaml's `schema:`, and main.py's literal `.get()` -# fallbacks), and a fourth is the open ticket. validate.py and main.py both -# import THIS dict rather than typing a literal, and test_peakguard.py pins it -# to config.yaml by READING the file - so a drift goes red instead of shipping. -# -# ⚠️ NOT A READ. `decide()` takes both values as arguments and this module makes -# no `.get()` call at all; the dict is a default table for its two callers. +# Mirror of config.yaml's peak options, pinned to the file by +# test_peakguard.py. Not a read: `decide()` takes them as arguments. PEAK_OPTION_DEFAULTS = { - # The EMS-side floor on SAFETY-07's forced discharge, in percent. - # - # 20 is DERIVED from the two numbers this repository already has, not - # chosen. The reference pack is 300 Ah at 51.2 V with DoD 90 - 13.8 kWh - # usable, the figure validate.py's rule 6 note already quotes - and the - # 2026-08-20 replay models 878 Wh of guard discharge in a day, which is - # 6.4 % of that. A floor 10 points above the pack's own 10 % cut-out - # therefore leaves the guard roughly 1.6 of its worst observed days of room - # before it stops, which is the smallest margin that lets the guard do its - # job on a bad day and still never reach the BMS. + # Percent a forced discharge may not go below. 20 is derived from the + # reference pack, 300 Ah at 51.2 V with DoD 90 - 13.8 kWh usable (a test + # pins this wording). See DECISIONS/SAFETY-07-peak-guard.md. "peak_soc_floor": 20, - # The pack's OWN cut-out, in percent: what the BMS enforces regardless of - # anything here. On the reference unit this is register 0x0560, read as 10, - # and it is consistent with DoD 90 on a full pack. It exists so validate.py - # rule 8 can refuse a `peak_soc_floor` at or below it - see that rule. + # The pack's own cut-out in percent; rule 8 of validate.py refuses a + # `peak_soc_floor` at or below it. "bms_soc_floor": 10, - # SAFETY-79. Watts the guard aims under `peak_cap_w`, so a quarter that - # lands a little high still stays under the cap. Owner decision 2026-09-28. + # SAFETY-79. Watts the guard aims under `peak_cap_w`. "peak_margin_w": 50, } -# SAFETY-79. config.yaml's schema range for `peak_margin_w`, mirrored so -# validate.py can refuse it in words; test_peakguard.py pins it to the file. +# SAFETY-79. Range of `peak_margin_w` in config.yaml; test_peakguard.py pins it. PEAK_MARGIN_RANGE_W = (0, 500) @dataclass(frozen=True) class PeakDecision: - """What the guard concluded this cycle, with everything it concluded it from. - - Every field is derived from a measurement taken in the same cycle. There is - no `floor_w_or(default)` and there must not be one, for `PeakReading`'s - reason: an accessor that turns "not measured" into a number re-creates the - defect the type exists to remove. - """ + """What the guard concluded this cycle. No `floor_w_or(default)`: turning "not + measured" into a number is the defect `PeakReading` removes.""" engaged: bool floor_w: float | None # the `lo` the claim carries; None = no claim @@ -488,22 +60,22 @@ class PeakDecision: wanted_w: float | None = None # the floor before `achievable_w` clamped it target_w: float | None = None # SAFETY-79: what the guard aims at saturated: bool = False - # SAFETY-24. True when a forced discharge WAS wanted and the EMS-side SoC - # floor refused to size it. Mutually exclusive with `saturated` by - # construction: this branch asks for nothing, so it cannot have run out of - # inverter. `soc_floor_reason` says WHICH of the two ways it bound. - # SAFETY-87: at N>=2 it may ask the batteries above their reserve for - # `able_w`, all they have, when that is not enough (`place_on_able()`). + # SAFETY-24: a forced discharge was wanted and the peak reserve refused it; + # excludes `saturated`. SAFETY-87: at N>=2 the others may be asked up to + # `able_w`. floor_limited: bool = False soc: float | None = None soc_floor: float | None = None soc_floor_reason: str | None = None reason: str = "" - # SAFETY-87, N>=2 only: the batteries at or under their own reserve, - # which give nothing for the peak, and what the others can give (None - # when they are not asked). See `place_on_able()`. + # SAFETY-87, N>=2: batteries at or under their reserve, and what the others + # can give. held: tuple = () able_w: float | None = None + # SAFETY-126: power is wanted but the quarter still ends within + # ALARM_MISS_W of the aim with the batteries at 0 W (`misses_aim()`). Set + # by main.py. + within_aim: bool = False def explain(self) -> str: """Idle: the status row's plain words (UX-08). Engaged: `reason`, the @@ -530,13 +102,8 @@ def blind_kind(blind: str) -> str: def quarter_ceiling_w(cap_w, reading) -> tuple[float, str]: - """The average this quarter may not exceed, and where the number came from. - - ONLY an OBSERVED reading may raise it. `NO_RECORD` (the store answered, this - month has nothing yet) and `NOT_OBSERVED` (the store did not answer) both - return the cap, which is the tighter answer. A failed read therefore makes - the guard stricter, never looser - see the module note. - """ + """The average this quarter may not exceed, and where it came from. + Only an OBSERVED reading raises it; a failed read keeps the cap (stricter).""" cap = float(cap_w) if (reading is not None and getattr(reading, "is_observed", False) and reading.peak_w is not None and float(reading.peak_w) > cap): @@ -546,25 +113,13 @@ def quarter_ceiling_w(cap_w, reading) -> tuple[float, str]: def quarter_target_w(cap_w, margin_w, reading) -> float: """SAFETY-79. The average the guard AIMS at: `margin_w` under - `quarter_ceiling_w()`, so under a month peak already banked as well as - under the cap. Aiming AT a banked peak can bank a few W over it, which - raises the bill as surely as going over the cap. Reports and the bill keep - the real limit (`ceiling_w`).""" + `quarter_ceiling_w()`. Reports and the bill keep the real limit (`ceiling_w`).""" return quarter_ceiling_w(cap_w, reading)[0] - float(margin_w) def excess_w(projection, ceiling_w) -> float: - """Watts that must come off the meter NOW to land the quarter on the ceiling. - - The whole control law, and the only place it is written down: - - excess = 900 * (projected - ceiling) / remaining - - Positive means the quarter is projected over; zero or negative means it is - not. `remaining <= 0` is not a large number, it is an undefined one - a - quarter with no seconds left cannot be influenced - so it answers 0.0 rather - than dividing. - """ + """Watts that must come off the meter now to land the quarter on the ceiling: + `900 * (projected - ceiling) / remaining`; 0.0 when `remaining <= 0`.""" remaining = QUARTER_S - float(projection.elapsed_s) if remaining <= 0.0: return 0.0 @@ -572,23 +127,14 @@ def excess_w(projection, ceiling_w) -> float: def uncovered_s(projection) -> float: - """Seconds of the open block the meter did not actually cover. - - Derived from what `Projection` already carries rather than re-measured: - `covered_frac` is `elapsed / elapsed_wall`, and `elapsed_s` IS that wall. - """ + """Seconds of the open block the meter did not cover (`covered_frac` of `elapsed_s`).""" return max(0.0, float(projection.elapsed_s) * (1.0 - float(projection.covered_frac))) def blindness(projection, max_hold_s, frozen: bool) -> str | None: - """Why this projection may not be graded, or None if it may be. - - Evaluated BEFORE the projection is compared to anything. Each of the three is - a way for the projected number to be confident and wrong, and the first - a - fresh averager's all-zero `Projection` - is confident, wrong AND under every - ceiling there is. - """ + """Why this projection may not be graded, or None. Checked first: a fresh + averager's all-zero `Projection` sits under every ceiling.""" if frozen: return BLIND_FROZEN if projection.block_start is None: @@ -601,8 +147,7 @@ def blindness(projection, max_hold_s, frozen: bool) -> str | None: return None -# SAFETY-90. Seconds of uncounted time below which nothing is filled: float -# noise between the averager's sums and the wall clock, never a real gap. +# SAFETY-90. Uncounted seconds below this are float noise, never a gap. GAP_TOL_S = 1e-3 @@ -616,17 +161,11 @@ def gap_w(ceiling_w, *seen_w) -> float: def fill_gaps(projection, *, tail_s, max_hold_s, unseen_s, gap_w, unseen_w, held_ws=0.0): - """SAFETY-90: `projection` with the seconds the averager did not count - (a meter gap past `max_hold_s`) counted at `gap_w`, and those from before - this run's first reading (`unseen_s`) at `unseen_w`. `held_ws` is added - too: what the gap's first `max_hold_s`, credited at the reading before - it, lacks to reach `gap_w`. Returns - `(projection, gap seconds filled, unseen seconds filled)`. - - `tail_s` is the age of the held reading: up to `max_hold_s` of it is left - as before (the averager credits it once the next reading lands). Past - that the gap is still open, nothing is filled and `blindness()` answers - as before. DECISIONS/SAFETY-90-peak-guard-after-a-gap.md.""" + """SAFETY-90: `projection` with the meter-gap seconds the averager did not + count filled at `gap_w` (those before the first reading, `unseen_s`, at + `unseen_w`), plus `held_ws`. Returns `(projection, gap s, unseen s)`. + `tail_s`: up to `max_hold_s` of the held reading is left as before; past + that nothing is filled. DECISIONS/SAFETY-90-peak-guard-after-a-gap.md.""" if projection.block_start is None or max_hold_s is None: return projection, 0.0, 0.0 wall = float(projection.elapsed_s) @@ -653,32 +192,9 @@ def fill_gaps(projection, *, tail_s, max_hold_s, unseen_s, gap_w, unseen_w, def discharge_permitted(soc, soc_floor) -> str | None: - """SAFETY-24. May the guard size a forced discharge, or why may it not? - - Returns None when it may, and otherwise the REASON it may not - which the - decision, the status page, the action log and the operator notification all - carry, because "floor-limited" on its own tells nobody what to do. - - THE ONE PLACE THIS QUESTION IS ANSWERED, and it is public for the reason - `freeze.looks_frozen()`, `watchdog.loop_is_following()` and - `shaping_suspended()` are: a safety predicate transcribed into its own check - is one copy plus a future disagreement about what the check means, and the - copy that drifts is always the grading one. - - ⚠️ `soc is None` IS NOT PERMISSION. A failed read is not an observation, and - a FULL BATTERY is exactly the value that would disarm this floor - so a - `float(soc or 100)` here would restore the unbounded discharge this card - removes, silently, on the one install whose SoC sensor is misnamed. There is - deliberately no default argument on either parameter. - - ⚠️ AND `soc_floor is None` IS NOT PERMISSION EITHER, for the same reason one - step further out: a caller that has no floor configured has not thereby been - granted an unbounded one. - - Boundary: `<=`, not `<`. A pack sitting exactly ON the floor has already - spent everything this guard was allowed to spend; permitting one more cycle - there would make the floor mean "one cycle below the floor". - """ + """SAFETY-24. May the guard size a forced discharge? None if so, else the reason. + `None` for `soc` or `soc_floor` is NOT permission, and neither has a default. + `<=`: a pack exactly on the floor has spent what the guard may spend.""" if soc is None: return FLOOR_SOC_UNREADABLE if soc_floor is None: @@ -691,25 +207,8 @@ def discharge_permitted(soc, soc_floor) -> str | None: @dataclass(frozen=True) class Pack: - """SAFETY-32. ONE pack's three numbers for the floor question, NAMED. - - `name` the roster name, because a refusal that does not say WHICH pack - stopped the site sends an operator to the wrong device - the - same defect CFG-04's `unit_caps_indexed()` was written for. - `soc` THIS pack's measured state of charge. None when its sensor did - not answer, and None is not a full battery. - `bms_floor` THIS pack's OWN cut-out percent (`bms_soc_floor`), which is a - property of the hardware in the cupboard and NOT of the site. - None when the roster entry did not state one - see - `pack_floor_pct()` for why that is withheld and not 0 %. - - ⚠️ IT CARRIES THE CUT-OUT, NOT THE EMS FLOOR. `peak_soc_floor` is one number - for the whole site (the depth a money-saving discharge may never reach) and - stays a flat option; `bms_soc_floor` describes a battery and became a roster - key here, because two packs of different models have different cut-outs and - grading both against one of them is wrong in the direction that empties the - shallower one. - """ + """SAFETY-32. One pack's numbers for the floor question. + `bms_floor` is its own cut-out percent; None when the roster states none.""" name: str soc: float | None @@ -717,74 +216,26 @@ class Pack: def pack_floor_pct(pack, soc_floor): - """The percent THIS pack may not be spent below, or None if that is unknown. - - `max(the site's EMS floor, this pack's own cut-out)` - the CONJUNCTION, and - the direction matters: the EMS floor is what the operator asked for and the - cut-out is what the hardware will do anyway, so a pack is protected by - whichever is higher. Taking the site floor alone is the shipped defect this - card removes (the shallower pack graded against the deeper one's cut-out); - taking the cut-out alone would throw SAFETY-24's margin away. - - ⚠️ AN UNSTATED CUT-OUT IS None, NOT ZERO, AND NOT THE FLAT OPTION. 0 % would - be a floor that permits everything - the most permissive value there is, - invented from an absence - and borrowing the flat `bms_soc_floor` is CFG-04's - "nothing is inherited" broken in one line, which is this card's whole subject - restated. `discharge_permitted()` already treats a None floor as NO - permission, so an entry that forgot the key costs a shave rather than a pack. - - ⚠️ AND UNDER A rule-8-VALID CONFIG THIS IS `soc_floor`, FOR EVERY PACK. - validate.py rule 8 refuses `peak_soc_floor <= bms_soc_floor` on EVERY unit, - so `max()` selects the EMS floor everywhere the add-on will actually start - - which is exactly why the reduction is written here rather than assumed at the - call site. "Another rule refuses that config" is a claim about today's rule - ordering, not a property of this function, and `validate_startup_config()` is - also `sim/inject.py`'s entry point. `unit_caps_indexed()`'s docstring records - the same argument, and CFG-04's index desynchronisation is what happened the - last time it was left implicit. - """ + """The percent THIS pack may not be spent below, or None if unknown: + `max(site EMS floor, its own cut-out)`. An unstated cut-out is None, not 0 + (`discharge_permitted()` reads None as no permission). The reduction is + written out because it must not depend on validate.py rule 8.""" if soc_floor is None or pack.bms_floor is None: return None return max(float(soc_floor), float(pack.bms_floor)) def pack_permissions(packs, soc_floor) -> list[tuple[str, str | None]]: - """SAFETY-32. Per pack: None if it may be spent, else the reason it may not. - - ONE `(name, reason)` pair per pack, in roster order, from the SAME - `discharge_permitted()` the site verdict uses - a second transcription of - "may this pack be spent" is a second answer to it, which is the argument - that made that predicate public in the first place. - - ⚠️ THIS IS THE OBSERVABLE THE MIN() CANNOT PRODUCE. `Controller.pack_floor_soc` - reduces the measured SoC to the emptiest pack BEFORE anything is graded, and - that reduction is right for the site's one verdict - but it has thrown away - which pack it came from and how far that pack was from ITS OWN floor by the - time the answer exists. Two packs at 15 % against floors of 10 % and 20 % - have opposite answers and are indistinguishable after the `min()`. - """ + """SAFETY-32. Per pack, in roster order: None if it may be spent, else why not. + Unlike the `min()` over SoC, it keeps which pack and which floor.""" return [(p.name, discharge_permitted(p.soc, pack_floor_pct(p, soc_floor))) for p in packs] def binding_soc_floor(packs, soc_floor): - """The ONE floor `decide()` may grade `pack_floor_soc` against, at a roster. - - MAXIMUM over the packs' own floors, and it is the conservative reduction that - pairs with the MINIMUM `pack_floor_soc` reduction: `min(soc) <= max(floor)` - is implied by "some pack is at or below its own floor", so this verdict is - never more permissive than grading each pack separately would be. The other - direction is not true and is not claimed - a site can be refused here while - every pack individually has headroom, which costs a shave and never a pack. - - None when ANY pack's floor is unknown, for `Controller.site_dev_bound()`'s - reason and in the same direction: `discharge_permitted()` treats a None floor - as no permission, so an unknown floor withholds rather than defaults. - - ⚠️ AT N=1 THIS IS `soc_floor` ITSELF under any config rule 8 accepts, and - test_peakguard.py part 12 asserts that over the whole schema range rather - than assuming it. - """ + """The one floor `decide()` grades `pack_floor_soc` against: the MAXIMUM of the + packs' floors, so never more permissive than grading each pack. None if any + floor is unknown. At N=1 it is `soc_floor` (test_peakguard.py part 12).""" if not packs: return None if soc_floor is None else float(soc_floor) floors = [pack_floor_pct(p, soc_floor) for p in packs] @@ -834,25 +285,10 @@ def decide(projection, *, batt_w, cap_w, margin_w, reading, achievable_w, soc, soc_floor, max_hold_s=None, frozen: bool = False) -> PeakDecision: """The whole guard. Pure: no clock, no I/O, no state carried between calls. - `batt_w` MEASURED battery power, positive = discharge. None when the - sensor for it is missing - and then there is nothing to size a - floor from, so no floor is invented. - `achievable_w` the largest discharge the arbiter could actually grant - (`min(max_w, device max)`). The floor is clamped to it so this - claim can NEVER produce `lo > hi` in the resolver - a - contradiction there resolves to 0 W, which would park the - battery in the exact minute it should be discharging hardest. - `soc` SAFETY-24. MEASURED state of charge, percent. None when the - sensor for it did not answer - which is floor-limited, not - permission. - `soc_floor` SAFETY-24. The percent below which this guard may not spend - the pack for money. REQUIRED, and deliberately without a - default: a default here would be CFG-03's fourth defaults - site and TEST-14's quiet fallback in one line. - `margin_w` SAFETY-79. Watts to aim under `cap_w` (`peak_margin_w`). - REQUIRED for the same reason. `ceiling_w` stays the real - limit for every report; only `excess_w` uses the target. - """ + `batt_w` measured battery power, positive = discharge; None invents no floor. + `achievable_w` largest discharge the arbiter could grant; the floor is + clamped to it so the claim never produces `lo > hi` (resolves to 0 W). + `soc` percent; None is floor-limited. `soc_floor`, `margin_w`: required.""" ceiling, from_where = quarter_ceiling_w(cap_w, reading) target = quarter_target_w(cap_w, margin_w, reading) aim = "" if target == ceiling else f", aiming at {target:.0f} W" @@ -860,9 +296,8 @@ def decide(projection, *, batt_w, cap_w, margin_w, reading, achievable_w, soc, blind = blindness(projection, max_hold_s, frozen) if blind is not None: - # 0.0, not a sized floor. There is no trustworthy measurement to size one - # from; the claim that remains is "do not add to the offtake you cannot - # see", which is one-directional and cannot cost pack energy. + # 0.0: nothing trustworthy to size a floor from; forbid charging, which + # costs no pack energy. return PeakDecision( engaged=True, floor_w=0.0, ceiling_w=ceiling, ceiling_from=from_where, target_w=target, blind=blind, projected_w=None, @@ -872,17 +307,14 @@ def decide(projection, *, batt_w, cap_w, margin_w, reading, achievable_w, soc, projected = float(projection.projected_w) if batt_w is None: - # main.py's P_SAFETY stop already owns a cycle with a missing input, at - # 0 W. Filing nothing is the honest answer: a floor sized from a - # measurement that is not there is a made-up number. + # main.py's P_SAFETY stop already owns a missing input; file nothing. return PeakDecision( engaged=False, floor_w=None, ceiling_w=ceiling, ceiling_from=from_where, target_w=target, projected_w=projected, reason="no battery measurement to size a floor from") if QUARTER_S - float(projection.elapsed_s) <= 0.0: - # A block with no seconds left cannot be influenced. This is degenerate - # rather than urgent - `excess_w` answers 0.0 there and a floor equal to - # the battery's own current output would be a claim that says nothing. + # A block with no seconds left cannot be influenced (`excess_w` answers + # 0.0). return PeakDecision( engaged=False, floor_w=None, ceiling_w=ceiling, ceiling_from=from_where, target_w=target, projected_w=projected, @@ -891,22 +323,10 @@ def decide(projection, *, batt_w, cap_w, margin_w, reading, achievable_w, soc, ex = excess_w(projection, target) wanted = float(batt_w) + ex if wanted <= 0.0: - # ⚠️ THE TEST IS ON THE FLOOR, NOT ON `excess > 0`, AND THE REPLAY IS WHY. - # `excess` is the watts that must come off the METER, so it goes to zero - # as soon as the battery is already removing them - and a guard that - # disengaged there handed the actuator straight back, the offtake sprang - # back to the raw load, and the next cycle engaged again. On 2026-08-20's - # billed quarter that bang-bang cycled 17 times in 900 s and left the - # quarter 49 W over the limit it had just spent a quarter of an hour - # holding. §10.2: "an EMS that oscillates is worse than no EMS, both for - # hardware wear and - per §9.1 - for the peak it is trying to avoid." - # - # `batt_w + excess` is the offtake that would exist with the battery at - # zero, minus the level the quarter can afford. Testing THAT is the same - # arithmetic read as a level rather than as a switch: the claim is filed - # for as long as a positive floor is needed and decays continuously to - # nothing as the load falls, so there is still no hysteresis constant - - # the quantity being thresholded is simply the right one. + # The test is on the floor (`batt_w + excess`), not on `excess > 0`: + # excess falls to zero once the battery is removing it, and releasing + # there cycled the guard 17 times in one quarter (2026-08-20). + # DECISIONS/SAFETY-07-peak-guard.md. return PeakDecision( engaged=False, floor_w=None, ceiling_w=ceiling, ceiling_from=from_where, target_w=target, projected_w=projected, @@ -914,26 +334,18 @@ def decide(projection, *, batt_w, cap_w, margin_w, reading, achievable_w, soc, wanted_w=wanted, reason="the load is under the level this quarter can afford") - # SAFETY-24 - AND ONLY HERE. `wanted > 0` is now established, so a forced - # discharge really is being asked for and the SoC is the quantity that - # decides whether it may be sized. Asked one line earlier it would grade a - # number that governs nothing on every quiet cycle of the day, and would - # report "floor-limited" over a guard that was never going to discharge. + # SAFETY-24, only here: `wanted > 0`, so a forced discharge is really asked + # for. denied = discharge_permitted(soc, soc_floor) if denied is not None: - # 0.0, not `wanted`, and not a negative number. Zero forbids CHARGING - - # one-directional, cannot cost pack energy - and forbids nothing else; - # a `lo` limit structurally cannot cap the discharge the loop is already - # doing. See the module note: this bounds THE GUARD's forced discharge, - # not the grid-following loop, which still has no SoC guard. + # 0.0, never negative: forbids charging only. It bounds the guard, not + # the loop. return PeakDecision( engaged=True, floor_w=0.0, ceiling_w=ceiling, ceiling_from=from_where, target_w=target, projected_w=projected, excess_w=ex, wanted_w=wanted, floor_limited=True, soc=soc, soc_floor=soc_floor, soc_floor_reason=denied, - # UX-10: plain sentences, every figure kept (test_messages.py - # part 6). The code is one token, so it is not read as shouting. reason=( f"PEAK_GUARD_FLOOR_LIMITED: this 15-minute period heads for " f"{projected:.0f} W against a {ceiling:.0f} W limit " @@ -957,36 +369,21 @@ def decide(projection, *, batt_w, cap_w, margin_w, reading, achievable_w, soc, f"{QUARTER_S - projection.elapsed_s:.0f} s left, so {ex:.0f} W must " f"come off the meter. The battery measured {float(batt_w):+.0f} W " f"and is asked for {floor:+.0f} W" - # Saturated means floor == ceil_top: the ask IS the most the - # inverters give, so one clause says both. + (f", the most the inverters give, but {wanted:+.0f} W was needed" if saturated else ""))) -# SAFETY-81. A quarter must end this many W over the real limit before -# "Load too high for battery" fires. Above the guard's own landing error at -# margin 0 (up to +5.7 W, SAFETY-79 test check 5), which margin 0 accepts. +# SAFETY-81. A quarter must end this many W over the real limit (and for +# ALARM_HOLD_S, so a hob cycling does not count) before "Load too high" fires. ALARM_MISS_W = 10.0 -# ...and for this long without a break, so a load that peaks for a few seconds -# (a hob or heater cycling) is not read as the rest of the quarter. ALARM_HOLD_S = 10.0 def load_too_high(projection, *, batt_w, ceiling_w, achievable_w) -> bool: - """SAFETY-81. May "Load too high for battery" fire? Only if, with the - battery at full power from now on, the quarter still ends more than - `ALARM_MISS_W` over the real limit (`ceiling_w`, not the aim). - - This is the miss itself, in W of the quarter average, so it does not have - `decide()`'s gain (900 / remaining): near a quarter's end that gain turns - a watt over the aim into thousands of W wanted. - - The load is the held meter reading plus `batt_w`. The caller passes the - lower of this cycle's and the last cycle's battery reading, so a meter - sample taken before the battery's last increase is not counted twice - (DECISIONS/SAFETY-81-load-too-high.md). A real overload shows on the meter - before the battery moves, so that filter costs it no cycle. - """ + """SAFETY-81. May "Load too high for battery" fire? Only if, with the battery at + full power, the quarter still ends over the real limit by `ALARM_MISS_W`. + No `decide()` gain; `batt_w` is the lower of two cycles' readings. + DECISIONS/SAFETY-81-load-too-high.md.""" remaining = QUARTER_S - float(projection.elapsed_s) if batt_w is None or remaining <= 0.0: return False @@ -997,136 +394,22 @@ def load_too_high(projection, *, batt_w, ceiling_w, achievable_w) -> bool: return ends_w - float(ceiling_w) > ALARM_MISS_W +def misses_aim(projection, *, batt_w, target_w) -> bool: + """SAFETY-126. With the batteries at 0 W from now on, does the quarter end more + than `ALARM_MISS_W` over the aim? + DECISIONS/SAFETY-126-landing-error-is-not-an-overrun.md.""" + return load_too_high(projection, batt_w=batt_w, ceiling_w=target_w, + achievable_w=0.0) + + def charge_headroom_w(decision) -> float | None: """SAFETY-45. The hardest this SITE may charge without putting the billed - quarter over the ceiling, in W, positive - or None when nothing measured it. + quarter over the ceiling, in W; None when nothing measured it (not unlimited). - THE ONE PLACE THIS QUESTION IS ANSWERED, and it is public for the reason - `discharge_permitted()`, `shaping_suspended()`, `guard_state()` and - `looks_frozen()` are: the check that grades it has to assert on THE SHIPPED - EXPRESSION, and a predicate transcribed into its own test is one copy plus a - future disagreement about what the check means. - - ------------------------------------------------------------------------ - THE DEFECT IT PREVENTS, AND IT IS NOT A NEW IDEA IN THIS REPOSITORY - - `control.maintenance_charge_floor()` already caps MAINTENANCE charging at - "the headroom left under the site's cap", and its docstring says why in - words this card only has to widen: - - "A maintenance charge is the only thing this system does that is big - enough and long enough to set that peak, so it is capped by the - headroom left under the site's cap." - - ⚠️ THAT SENTENCE STOPPED BEING TRUE WHEN OPT-04 LANDED. Price arbitrage is a - SECOND thing that is big enough and long enough - it commands `max_w` for a - whole tariff interval, deliberately, against the meter - and it was given no - such cap. Measured on TEST-23 stage 3's closed rig at the shipped - `peak_cap_w: 3500` / `max_w: 2000`: 28 charge onsets in one hour, mean period - 131 s, grid swinging -456 <-> +3200 W, on any house drawing over 1500 W in - the cheapest quarter. §10.2: "an EMS that oscillates is worse than no EMS, - both for hardware wear and - per §9.1 - for the peak it is trying to avoid." - - ⚠️ AND IT IS THE **MEASURED** GUARD, NOT `peak_forecast_entity`. - `maintenance_charge_floor()` sizes itself from the forecast sensor, which is - OPTIONAL and empty on a default install - the exact hole SAFETY-25 had to - close in `shaping_suspended()`, where a suspension gated on the forecast - alone could not fire on the very site that needed it. This number comes off - `decide()`, which runs wherever P1 does. - - ------------------------------------------------------------------------ - WHERE THE NUMBER COMES FROM: IT IS `wanted_w`, READ IN THE OTHER DIRECTION - - `decide()` already computes - - wanted = batt_w + excess = (the offtake there would be with the battery - at zero) - (the level this quarter can still afford) - - as the battery DISCHARGE that would land the quarter exactly on the ceiling. - A negative `wanted` is not "nothing to say" - it is a permitted CHARGE of - `-wanted` watts, and throwing it away is SAFETY-45's whole defect. So this - function invents no arithmetic; it reads the sign the guard already had. - - ⚠️ AND THE QUANTITY IS INVARIANT IN THE BATTERY, WHICH IS WHY SIZING A TRADE - TO IT CANNOT OSCILLATE. Raise the charge by dW and the meter rises by dW, so - `excess` rises by exactly dW while `batt_w` falls by dW: `wanted` does not - move. It is a FEED-FORWARD number - a function of the house, the ceiling and - the seconds already banked - so consuming it adds no feedback path around the - plant. The fixed point is `charge = ceiling - load`, with the meter sitting - ON the ceiling and `wanted` parked at `load - ceiling`, a whole `ceiling - - load` clear of the engagement boundary rather than chattering across it. - - ⚠️ AND A QUARTER BOUNDARY DOES NOT HAND IT BACK AT FULL POWER, WHICH IS THE - OTHER HALF OF SAFETY-45 AND IS NOT THE SAME CLAIM AS "IT DOES NOT CHANGE". - It DOES change - it is `(900 / remaining) * (ceiling - projected)`, so it is - largest in the last seconds of a block and collapses at the boundary. What - matters is WHAT IT COLLAPSES TO: at `elapsed = 0` the projection IS the held - level, so the answer is `ceiling - (load - pv)` - the house and the cap, and - nothing else. A trade refused at 14:59 therefore resumes at 15:00 at the rate - the new quarter can afford, not at `max_w`, and §10.1's answer survives the - boundary because the arithmetic does: no timer, no backoff constant, no state - carried between calls. - - ⚠️ AND THE AMPLIFICATION IS BOUNDED IN ITS EFFECT, NOT MERELY LARGE. Hold a - charge of exactly this size for the rest of the block and the block's average - lands on `ceiling` EXACTLY, wherever in the block it started: - - (acc + (held + headroom) * remaining) / 900 - = projected + (900/remaining) * (ceiling - projected) * remaining/900 - = ceiling - - so 162 kW of headroom with 5 s left and 900 W of it with the whole quarter to - go are the same statement about the bill. `test_peakguard.py` part 13 sweeps - that identity across the block rather than arguing it here. - - ------------------------------------------------------------------------ - THE THREE ANSWERS - - `None` nothing measured it: the guard never ran (`meter_source: off`), or - it ran with no battery reading to size a floor from, or the block - had no seconds left to influence. ⚠️ None IS NOT UNLIMITED. A caller - that treats it as "charge away" is `arbitrage.guard_state()`'s - collapse of None into False, one layer down - a strategy creating - the billed quantity while nothing measures the quarter it lands in. - `0.0` measured, and the answer is no charging: the guard is BLIND (its - claim already says so at `floor 0 W` and this agrees with it rather - than restating it), or it is holding and `wanted >= 0`. - `> 0` the watts a charge may pull this cycle. - - ⚠️ HOLDING IMPLIES ZERO, BY CONSTRUCTION AND NOT BY COINCIDENCE. Every state - that files a claim with `engaged=True` has `wanted > 0`, `wanted is None` or - `blind`, so this answers 0.0 or None there and can never hand a strategy - permission the guard's own `lo` is at that instant refusing. - - ------------------------------------------------------------------------ - ⚠️ WHY THIS IS A SIZING AND NOT A CLAIM, AND DO NOT "SIMPLIFY" IT INTO ONE - - The obvious tidier answer is to delete `decide()`'s `wanted <= 0` gate and - let it file `lo = wanted` unconditionally - a NEGATIVE floor, meaning "you - may charge, but no harder than this". The arithmetic is identical, it is - continuous, and it would bind every claimant rather than only arbitrage. It - was measured, considered and REJECTED, and the reason is not style: - - * `main.py`'s cheap-window maintenance charge files - `Claim.limit("maintenance", P_MAINT_SHAPE, hi=-floor)` - a NEGATIVE `hi`. - arbiter.py clause 2 applies EVERY limit at or above the winner's priority - and takes the most restrictive, with no notion of one limit out-ranking - another. A peak `lo = -300 W` against a shaping `hi = -1000 W` is - `lo > hi`: a CONTRADICTION, which resolves to 0 W - the battery PARKED, - in the one quarter of the month that sets the bill. That is the exact - failure `shaping_suspended()` exists to prevent, and the only way to - close it would be to key that suppression on "a claim exists", which on - every site this card is about suspends the maintenance charge-only - shaping PERMANENTLY. - * and it would change what `engaged` MEANS at five call sites outside this - module - `arbitrage.guard_state()`, `shaping_suspended()`, `claim()`, - main.py's edge-logged action line and main.py's status row - each of - which today reads it as "the guard is HOLDING". - - `control.maintenance_charge_floor()` is a sizing and not a limit for the same - reason, which is why this is the shape that already fits. - DECISIONS/SAFETY-45-sizing-the-trade.md §2b carries the full argument. - """ + It is `wanted_w` read the other way, independent of the battery, so sizing a + trade to it cannot oscillate. 0.0 when blind or holding. A SIZING, NOT A + CLAIM: a negative `lo` would cross the maintenance shaping's negative `hi` + (0 W, battery parked). DECISIONS/SAFETY-45-sizing-the-trade.md section 2b.""" if decision is None: return None if decision.blind: @@ -1137,9 +420,8 @@ def charge_headroom_w(decision) -> float | None: def charge_room_w(decision, projection) -> float | None: - """SAFETY-125: `charge_headroom_w()` with the seconds since the last - meter reading counted at that reading, so a late reading near a - quarter's end is not read as room for a grid charge (SAFETY-99). + """SAFETY-125: `charge_headroom_w()` less the unread seconds counted at the + last reading, so a late reading is not room for a grid charge. DECISIONS/SAFETY-125-grid-charge-under-the-month-peak.md.""" room = charge_headroom_w(decision) if not room or projection is None or projection.block_start is None: @@ -1153,59 +435,24 @@ def charge_room_w(decision, projection) -> float | None: def shaping_suspended(decision, peak_forecast_w, cap_w) -> bool: """Must the maintenance charge-only shaping stand down this cycle? - - THE ONE PLACE THIS CONJUNCTION IS WRITTEN, and it is public for the reason - `watchdog.loop_is_following()` and `freeze.looks_frozen()` are: the check - that grades it has to assert on THE SHIPPED EXPRESSION. A safety predicate - transcribed into its own test is one copy plus a future disagreement about - what the check means, and the copy that drifts is always the grading one. - It was inline in `main.py` and nothing could reach it; a reviewer mutated it - back to its historical broken form and all twelve suites stayed green. - - ⚠️ WHY SUPPRESSION AND NOT PRIORITY. The shaping is a LIMIT (`hi <= 0`) and - the peak guard's claim is a LIMIT (`lo >= 0`). arbiter.py clause 2 applies - EVERY limit at or above the winner's priority and takes the MOST RESTRICTIVE, - with no notion of one limit outranking another - which is precisely what - makes the safety bounds unconditional and must not be changed. Two crossing - limits are therefore a CONTRADICTION, which resolves to 0 W: the battery - PARKED, in the one quarter of the month that sets the bill. P_PEAK 50 > - P_MAINT_SHAPE 40 does not save it and never could. - - ⚠️ AND THE FIRST TERM IS NOT ENOUGH ON ITS OWN. `peak_forecast_entity` is - OPTIONAL and is empty on a default install, so `peak_forecast_w` is None - there and `peak_at_risk()` answers False forever. Before this card the - suspension therefore could not fire on the very site that needed it. The - second term is the MEASURED guard, which is always present wherever P1 is. - Deleting either term - or re-gating the whole thing on the forecast - parks - a default install at 0 W, silently, in the billed quarter. test_peakguard.py - part 7 drives the real cycle with each of those mutations in place. - """ + Crossing limits resolve to 0 W, so the rule is suppression, not priority. + Both terms are needed: the forecast entity is empty on a default install, + where the measured guard fires. Dropping either parks the battery in the + billed quarter (test_peakguard.py part 7). The one place this conjunction is + written, public so the test grades the shipped expression: do not inline it. + DECISIONS/SAFETY-07-peak-guard.md.""" if peak_at_risk(peak_forecast_w, float(cap_w)): return True return decision is not None and decision.engaged def claim(decision: PeakDecision) -> Claim | None: - """The decision as one arbiter claim, or None when the guard is idle. - - A LIMIT with only `lo` set. `hi` stays `+inf`, which is 9.1's "discharge and - export are never capped by the peak guard" expressed as a fact about the - object rather than as a rule somebody has to remember. - """ + """The decision as one arbiter claim (`lo` only, `hi` stays `+inf`), or None.""" if not decision.engaged or decision.floor_w is None: return None - # Three reasons, not two. A `lo = 0 W` claim reaches the arbiter from THREE - # different states now - blind, floor-limited, and a genuine floor that - # happens to be zero - and the arbiter's explain() is what an operator reads - # when the battery stops charging. Spelling them the same way would make the - # SAFETY-24 case indistinguishable from the SAFETY-18 one on the one line - # that gets read. - # ⚠️ ON THE SITE ACTUATOR, EXPLICITLY. The billed quantity is the quarter's - # average offtake AT THE GRID CONNECTION - one per site, whatever the site - # has behind it - so this floor bounds the site target and is divided by the - # allocator along with it. It is never a bound on one unit's share: SAFETY-25 - # §0c's table has this `lo` at the SITE stage precisely so the deferred loop - # reserve's per-inverter `hi` cannot become its counterpart in a `lo > hi`. + # Blind, floor-limited and a genuine zero floor all file `lo = 0 W`; each + # gets its own reason. On the SITE actuator: the billed quantity is one per + # site. return Claim.limit(SOURCE, P_PEAK, lo=float(decision.floor_w), reason=("blind" if decision.blind else "SoC floor" if decision.floor_limited diff --git a/goodwe_controller/app/price.py b/goodwe_controller/app/price.py index 40d8982..b2f6d0a 100644 --- a/goodwe_controller/app/price.py +++ b/goodwe_controller/app/price.py @@ -1,140 +1,22 @@ """TARIFF-02 - price provider plugins, and the one canonical shape they all emit. -WHERE THIS LIVES, AND WHY IT IS NOT WHERE THE SPEC SAYS. specs.md §2.2 puts this -at `ems/ingest/prices/`. That module layout does not exist and is not being -built; the product owner's decision of 2026-08-30 is that everything ships in -`goodwe_controller/app/`, next to arbiter.py, clamp.py, freeze.py and -validate.py. The BEHAVIOUR specs.md §9.3 / §8.3 / §11.4 describes is unchanged -and still authoritative - only the path moved. +A new Belgian supplier is a new class and one line in `PROVIDERS`; the +optimiser never learns a supplier's name. Lives in `goodwe_controller/app/`, +not `ems/ingest/prices/` (owner, 2026-08-30). -WHAT THIS IS FOR, in one sentence: so that a new Belgian supplier is a new class -and one line in `PROVIDERS`, and the optimiser that eventually consumes prices -never learns a supplier's name. - -THE CANONICAL FORMAT (§9.3), exactly three fields and no more: +Canonical format (specs.md 9.3), three fields and no more: { "prices": [ { "start": "2026-01-15T00:00:00+01:00", "eur_per_kwh": 0.1834, "provisional": false } ] } `start` is tz-aware at the Brussels offset for that instant, the price is EUR -per kWh and ALL-IN (energy + markup + taxes), and `provisional` is per point, -not per series (§8.3: unknown hours are flagged, and no irreversible action may -be taken on a provisional price). +per kWh and ALL-IN, `provisional` is per point. - -THE THREE THINGS THIS MODULE IS SHAPED BY -========================================= - -1. DOUBLE-COUNTING MARKUP IS REFUSED, NOT DISCOURAGED - GIVEN A CORRECT - DECLARATION. - `generic_dayahead` reads a raw EPEX figure and the user's markup and taxes - have to be added to it. Engie and TotalEnergies publish a figure that already - HAS the markup in it. Add the configured markup to that one too and every - arbitrage decision on that site is made against a price that is 3 c/kWh - wrong, in a direction nothing reports. - - A comment saying "do not add markup here" is not a mechanism. So: - - * A plugin does not return a price. It returns `RawPoint`s carrying the - figure ITS SOURCE PUBLISHED, and declares in `capabilities().sources` - which of {ENERGY, MARKUP, TAXES} that figure already contains. - * `assemble()` - the framework, never a plugin - adds exactly the - components the plugin declared it did NOT source, from - `price_markup_eur_kwh` / `price_taxes_eur_kwh`. - * ⚠️ A PLUGIN CANNOT REACH THOSE TWO NUMBERS BY ANY ROUTE, and the word - "any" was earned the hard way. `PluginOptions` raises `FrameworkOption` - on them, which closes the documented path - but review of PR #16 wrote a - double-counting plugin THREE more ways, one of them touching nothing but - `PRICE_OPTION_DEFAULTS`, a public dict in the module every plugin must - already import for `PriceProvider` and `register`. One closed door in a - room with three is not a guarantee. So `register()` now also REFUSES ANY - PLUGIN WHOSE OWN CODE MENTIONS A FRAMEWORK KEY - see `code_leaks()`. - - ⚠️ THE PRECISE SHAPE OF THE GUARANTEE, because an overclaim here is worse - than no claim: **the framework adds the complement exactly once, GIVEN A - CORRECT `sources` DECLARATION.** A plugin that declares `{ENERGY}` while its - feed is really all-in still comes out one markup high, and NO CODE CAN CATCH - THAT - the declaration is the plugin's unverifiable claim about somebody - else's tariff. What is mechanised is that a plugin cannot obtain the numbers - and cannot apply them; what remains human is whether it described its feed - truthfully. The docstring/declaration check in test_price.py verifies that - the plugin's two claims AGREE, not that either is true, and it is worth - knowing which of those two things you are reading. - -2. THE PRODUCT TYPE IS DECLARED, NOT INFERRED FROM THE PLUGIN'S NAME. - §9.2 classes real-time / imbalance products separately from day-ahead ones - and §8.3 says those are treated as high-volatility with widened arbitrage - thresholds. If the optimiser had to write `if provider_id == ""` - to know that, then adding a supplier would mean editing the optimiser - - which is precisely what AC2 forbids. So `capabilities().product` is a - `PriceProduct`, it rides along on the series, and `volatility_regime()` - reads the enum. A plugin id appears nowhere in this file except in - `PROVIDERS`, in `RETIRED_PROVIDERS`, and in the class that answers to it. - - ⚠️ NO SHIPPED PLUGIN DECLARES `PriceProduct.REALTIME` (TARIFF-07). All - three shipped plugins are day-ahead, so `_WIDE`, - `Uncertainty.REALTIME_PRODUCT` and `WIDEN_FACTOR`'s 2.5 currently have no - shipped declarer. They are KEPT - deliberately - see DECISIONS/TARIFF-07-withdrawing-three-plugins.md - and - test_price.py section 10 keeps them live by declaring REALTIME from a class - defined outside this file, which is AC2's own stimulus. - -3. "NO PRICE DATA" IS A TYPE, NOT AN EMPTY LIST. - §11.4: no price data -> fall back to the static profile, mark the plan - provisional, NO ARBITRAGE. That is a different instruction from "prices - exist and they are all zero", under which arbitrage is merely pointless - rather than forbidden - and the two are indistinguishable if a failed fetch - returns `[]`, or `None`, or a dict a caller `.get()`s a default out of. - - ⚠️ THIS REPO SPENT 2026-08-30 REMOVING EXACTLY THAT DEFECT. `sim/riglock.py` - in the infra repo used to return a partial dict from `holder()`; TEST-04 - rewrote it so a look at the lock is `observed / not_observed / gone` and only - the observed variant carries fields. `_Outcome` below is that shape, - deliberately, down to the fatal `.get()`. Read that file's `class Absence` - block before changing anything here. - - Concretely: - - * `PriceRead.series(...)` REFUSES AN EMPTY POINT LIST. A series with no - prices in it cannot be constructed, so "I have no prices" cannot arrive - wearing the shape of an answer. - * `PriceRead.no_data(reason)` carries `.reason` AND NOTHING ELSE. It has no - `.points` to iterate, no `.product` to switch on, no `.at()` to call, and - `.get("points", [])` raises. `min(read.points)` - the line that would - silently make "no data" mean "everything is free, charge now" - does not - run against it. - * `.match(*, series, no_data)` takes both handlers as REQUIRED keyword - arguments, so a caller who forgets the no-data case gets a TypeError at - the call site rather than a fall-through. - - -TIMEZONE, AND WHY THERE IS NO `zoneinfo` HERE -============================================= -§9.3's example is `+01:00`, and Belgium runs CET/CEST. Two constraints: - - * The shipped image is Alpine with `py3-aiohttp` from apk and NO `tzdata` - package (goodwe_controller/Dockerfile, and .gitea/ci/Dockerfile is the same - base by design). `ZoneInfo("Europe/Brussels")` raises - `ZoneInfoNotFoundError` there. A price module that only works on a machine - with a tz database is a price module that fails on the client's Pi. - * Fixed offsets have no fold. Every instant here is carried as UTC and - RENDERED at the offset that applies to it, so the 02:00-03:00 hour that - happens twice in October is two different instants that render as - `02:00+02:00` and `02:00+01:00`. A naive-local-hour key collides those two - and misprices an hour of a 25-hour day; an implementation on a fixed - 24-slot array is wrong on both transition days. test_price.py drives both - 2026 transitions and counts the points rather than asserting 24. - -The EU rule is fixed in law and needs no database: CEST from 01:00 UTC on the -last Sunday of March until 01:00 UTC on the last Sunday of October. Review of -PR #16 cross-checked this against `zoneinfo`'s Europe/Brussels over 149,040 -hourly instants, 2020-2036 inclusive, and found ZERO mismatches - so the -residual risk here is not arithmetic, it is political: if the EU ever abolishes -the seasonal change, `brussels_offset()` needs editing. That is the same -exposure as a machine whose `tzdata` has gone stale, and it is one edit in one -function rather than a package upgrade. -""" +Three things shape the module (trap 1 markup is added once by the framework, +2 the product is declared, 3 "no data" is a type), and there is no `zoneinfo` +(the image has no tzdata). The reasoning and the later additions: +DECISIONS/TARIFF-02-price-provider-plugins.md.""" import ast import csv @@ -158,9 +40,8 @@ from .custom_formula import ( _LOG = logging.getLogger("goodwe.price") -# --------------------------------------------------------------------------- # -# Brussels wall time, without a tz database. See the module docstring. -# --------------------------------------------------------------------------- # +# Brussels wall time, without a tz database. +# DECISIONS/TARIFF-02-price-provider-plugins.md. CET = timezone(timedelta(hours=1)) CEST = timezone(timedelta(hours=2)) @@ -185,11 +66,7 @@ def brussels_offset(instant: datetime) -> timezone: def brussels(instant: datetime) -> datetime: """Render a UTC instant at the Brussels offset for that instant. - - ⚠️ Takes an INSTANT, and refuses a naive one. A naive datetime is not a - moment in time, it is a wall clock reading plus an assumption, and the - assumption is wrong for one hour twice a year. - """ + Takes an INSTANT and refuses a naive datetime (wrong for one hour twice a year).""" if instant.tzinfo is None or instant.utcoffset() is None: raise ValueError( f"brussels({instant!r}): a naive datetime is not an instant. Parse " @@ -201,50 +78,11 @@ def brussels(instant: datetime) -> datetime: def brussels_date_range(since: datetime, until: datetime) -> tuple: """The Brussels calendar days a half-open window [since, until) touches. - Returns `(first, last)`, two `date`s, BOTH INCLUSIVE - which is the shape - `api.energy-charts.info` takes: `?bzn=BE&start=2026-09-09&end=2026-09-10` - answered 192 quarter-hours across exactly those two local days, measured - 2026-09-10T07:45:49Z and committed as - `scenarios/energy-charts-be-price-2026-09-09_2026-09-10-twoday.json`. - - THE DEFECT THIS EXISTS TO PREVENT (TARIFF-09). `generic_dayahead.fetch()` - accepted `since` and `until` and requested the bare default URL, so the - feed answered ONE Brussels day. Everything past today was then carried - forward from yesterday's profile, flagged provisional, and §8.3 dropped all - of it from the trade set - so the optimiser never saw tomorrow as firm, at - any hour. Measured on the merged tree at 2026-09-10T09:00+02:00 over a 48 h - horizon: 15 firm whole hours and 33 carried, and both ends of the only - permitted cycle inside ONE calendar day. The ceiling was structural: at - hour H of the local day a one-day window gives at most 24-H firm hours and - 24+H carried - 1/47 at 23:00, where the firm spread no longer even clears - the round-trip cost. That is not what FCST-04's fill is for; it covers the - pre-publication window until the ~13:00-14:00 CET auction, after which a - healthy feed hands over tomorrow in full. - - ⚠️ THE CALENDAR IS BRUSSELS, NEVER UTC, AND THIS IS `_bdate()`'s LESSON ONE - LAYER UP. `since`/`until` are INSTANTS; the API takes DATES, and the two are - not the same question. 22:30 UTC in summer is already tomorrow in Belgium, - so a range derived from `since.date()` asks for the wrong FIRST day for the - last one or two hours of every single day - and the day it drops is the one - the controller is standing in. price_history.py made this mistake at the - cache layer and documents the same arithmetic; here it decides what is - fetched at all, so getting it wrong loses the hours rather than misfiling - them. - - ⚠️ `until` IS EXCLUSIVE, SO LOCAL MIDNIGHT BELONGS TO THE DAY BEFORE IT. - A 48 h horizon from local midnight ends at local midnight two days later, - and that instant is the first tick of a day the caller did not ask for. - Taking `brussels(until).date()` unconditionally would request a THIRD day - on every whole-hour horizon. This feed answers such a request without - complaint, so nothing would report it - which is the reason to get it right - here rather than to rely on the server's tolerance: the range is a claim - about what the caller asked for, and it should be true. - - ⚠️ AND AN EMPTY OR INVERTED WINDOW COLLAPSES TO ONE DAY RATHER THAN TO A - BACKWARDS RANGE. `start` after `end` is a request no feed has a defined - answer for; some return everything, some return an error, and neither is a - thing to hand a battery controller. - """ + Returns `(first, last)`, both inclusive: the shape `api.energy-charts.info` + takes. TARIFF-09: a one-day request left everything past today provisional. + The calendar is Brussels, never UTC; `until` is exclusive (local midnight + belongs to the day before); an empty or inverted window is one day. + DECISIONS/TARIFF-02-price-provider-plugins.md.""" first = brussels(since).date() edge = brussels(until) last = edge.date() @@ -256,20 +94,9 @@ def brussels_date_range(since: datetime, until: datetime) -> tuple: def dated_feed_url(base: str, **params) -> str: - """`base` plus a query string. Refuses a `base` that already carries one. - - ⚠️ THE REFUSAL IS THE POINT, NOT THE FORMATTING. Appending `&start=...` to - a URL that may already hold its own query string is how a plugin produces a - request nobody intended - `?bzn=BE?bzn=BE&start=...`, or a duplicated - parameter whose winner is the server's choice rather than ours. So the base - is required to be parameterless and the parameters are added exactly once, - by `urlencode`, which is also what keeps a date out of the escaping - question. A future edit that folds the query back into the constant is a - refused fetch with a sentence, not a silently malformed URL. - - ⚠️ IT IS NOT REACHED BY AN OPERATOR'S `price_feed_url`. See - `GenericDayAhead.fetch()`: an override is used exactly as given. - """ + """`base` plus a query string. Refuses a `base` that already carries one + (appending would build a malformed or duplicated query). Not reached by an + operator's `price_feed_url`, which is used exactly as given.""" if "?" in base or "#" in base: raise SourceUnavailable( f"the feed base {base!r} already carries a query string or " @@ -279,9 +106,7 @@ def dated_feed_url(base: str, **params) -> str: return f"{base}?{urlencode(params)}" if params else base -# --------------------------------------------------------------------------- # -# Absence, borrowed wholesale from sim/riglock.py (TEST-04). -# --------------------------------------------------------------------------- # +# Absence, borrowed from sim/riglock.py (TEST-04). class Absence(Exception): """A value was asked of a read that never observed one.""" @@ -292,14 +117,9 @@ class FrameworkOption(Exception): class SourceUnavailable(Exception): - """A plugin could not get its data. Raised by plugins, caught by the - framework, and turned into `PriceRead.no_data(...)`. - - ⚠️ It must never escape into the control tick. §11.4's row is a returned - condition, not a traceback: the loop keeps regulating on P1 whatever the - price situation is, and a price fetch that could stop it would be a price - fetch that can stop the battery. - """ + """A plugin could not get its data. Raised by plugins, caught by the framework, + turned into `PriceRead.no_data(...)`. It must never escape into the control + tick: the loop keeps regulating on P1 whatever the price situation is.""" class _Outcome: @@ -324,8 +144,8 @@ class _Outcome: f"both outcomes.") def __iter__(self): - # ⚠️ DELIBERATE. `for p in read:` and `list(read)` are the other two - # idioms that turn an absent read into an empty, harmless-looking loop. + # DELIBERATE: `for p in read:` and `list(read)` would also turn an + # absent read into an empty loop. raise Absence( f"{type(self).__name__} is not iterable: an empty loop body is how " f"'we have no prices' becomes 'we checked every hour and none were " @@ -335,9 +155,7 @@ class _Outcome: return f"{type(self).__name__}({self.words!r})" -# --------------------------------------------------------------------------- # # The vocabulary -# --------------------------------------------------------------------------- # class Component(Enum): """The three parts of an all-in Belgian consumption price (§9.3).""" @@ -357,51 +175,26 @@ class PriceProduct(Enum): FIXED = "fixed" -# ⚠️ §8.3: "Real-time / imbalance products are treated as high-volatility: -# arbitrage thresholds widen." This maps product -> regime, and it is the ONLY -# place in the shipped code that says which products are volatile. OPT-04 reads -# it; it does not re-derive it, and it certainly does not check plugin ids. +# specs.md 8.3: real-time / imbalance products are high-volatility and widen +# the arbitrage thresholds. The ONLY place that says which products are +# volatile. OPT-04 reads it; nothing checks plugin ids. _WIDE = frozenset({PriceProduct.REALTIME}) def volatility_regime(product: PriceProduct) -> str: - """"wide" | "normal" - the arbitrage threshold regime for a product (§8.3). - - Deliberately a plain string and deliberately tiny: OPT-04 owns what the - widened numbers ARE. This owns only which products get them, because that - is a property of the product and belongs with the product enum. - """ + """"wide" | "normal" - the arbitrage threshold regime for a product (8.3). + OPT-04 owns what the widened numbers are; this owns which products get them.""" if not isinstance(product, PriceProduct): raise TypeError(f"volatility_regime() takes a PriceProduct, not " f"{product!r} - a name is exactly what AC2 forbids.") return "wide" if product in _WIDE else "normal" -# --------------------------------------------------------------------------- # -# FCST-03: ONE widening path, driven by a SET of active uncertainty sources. -# -# Two spec rows widen the same number by two different triggers: -# -# §9.2 / §8.3 real-time / imbalance product -> "arbitrage thresholds widen" -# §11.4 no weather data -> "widen arbitrage thresholds" -# -# They are one mechanism reached twice, not two mechanisms, so there is one -# function and a set of reasons. ⚠️ THE COMBINATION RULE IS `max`, NOT A -# PRODUCT: multiplying 2.5 by 1.5 would make two independent doubts compound -# into a threshold nothing on a Belgian tariff can ever clear, and an arbitrage -# strategy that can never fire is indistinguishable from one that is switched -# off. The widest single reason wins. -# -# ⚠️ HONEST ABOUT WHERE THE NUMBERS COME FROM. specs.md says thresholds widen -# and does not say by how much; there is no capture to derive a factor from, -# because no arbitrage strategy has ever run here. These are POLICY DEFAULTS, -# chosen so that (a) a real-time product needs a materially larger spread than -# a settled day-ahead one before a cycle is worth its wear, and (b) the two -# factors DIFFER, so `max`, `sum` and `product` are three distinguishable -# answers rather than the same number - a check that grades a combination rule -# where all the candidates agree grades nothing. When OPT-04 lands -# `cycle_cost_threshold_eur` (§10.1/§14) these become the natural second option -# beside it, in PRICE_OPTION_DEFAULTS and nowhere else (CFG-03). +# FCST-03: ONE widening path, driven by a SET of active uncertainty sources +# (real-time product 8.3, no weather data 11.4). The combination rule is `max`, +# not a product. The factors are policy defaults, not derived, and they differ +# so the combination rules stay distinguishable. +# DECISIONS/TARIFF-02-price-provider-plugins.md. class Uncertainty(Enum): """Why an arbitrage threshold is currently wider than the tariff alone says.""" REALTIME_PRODUCT = "realtime_product" # §9.2 row 2, §8.3 @@ -415,12 +208,8 @@ WIDEN_FACTOR = { def uncertainty_of(product: PriceProduct) -> frozenset: - """The uncertainty a PRODUCT contributes on its own (§9.2 row 2). - - Derived from the declared product, never from a plugin id - the same rule - `volatility_regime()` exists for. A sixth supplier declaring - `PriceProduct.REALTIME` is widened without anyone editing this file. - """ + """The uncertainty a PRODUCT contributes on its own (9.2 row 2), derived from the + declared product, never from a plugin id.""" if not isinstance(product, PriceProduct): raise TypeError(f"uncertainty_of() takes a PriceProduct, not " f"{product!r} - a plugin name is what §9.3 forbids.") @@ -430,11 +219,7 @@ def uncertainty_of(product: PriceProduct) -> frozenset: def widen_factor(sources) -> float: """The multiplier the widest active source asks for. 1.0 when there are none. - - ⚠️ A STRING IS REFUSED. `widen_factor({"realtime"})` would otherwise be a - silent 1.0 - a widening that reports itself as applied and is not, which is - the shape of defect this repo keeps finding. - """ + A string is refused: it would be a silent 1.0.""" srcs = frozenset(sources) bad = [s for s in srcs if not isinstance(s, Uncertainty)] if bad: @@ -444,12 +229,8 @@ def widen_factor(sources) -> float: def required_spread(base_eur_kwh: float, sources) -> float: - """The spread an arbitrage cycle must clear, given what is uncertain (§8.3). - - `base_eur_kwh` is the caller's own round-trip cost threshold - OPT-04 owns - that number (§10.1's `cycle_cost_threshold_eur`), this owns only how much - wider it gets and why. - """ + """The spread an arbitrage cycle must clear, given what is uncertain (8.3). + `base_eur_kwh` is OPT-04's own round-trip cost threshold.""" if isinstance(base_eur_kwh, bool) or not isinstance(base_eur_kwh, (int, float)) or not math.isfinite(base_eur_kwh) or base_eur_kwh < 0: raise ValueError( f"required_spread() takes a non-negative, finite base threshold in " @@ -460,7 +241,7 @@ def required_spread(base_eur_kwh: float, sources) -> float: @dataclass(frozen=True) class PricePoint: - """One canonical point. THREE FIELDS. §9.3 lists three; this has three.""" + """One canonical point. THREE FIELDS, as 9.3 lists them.""" start: datetime # tz-aware, rendered at the Brussels offset eur_per_kwh: float # ALL-IN: energy + markup + taxes @@ -474,12 +255,8 @@ class PricePoint: @dataclass(frozen=True) class RawPoint: - """What a plugin returns: the figure its source published, unmodified. - - ⚠️ NOT a PricePoint, and the type difference is the guard. A plugin cannot - hand back something already claiming to be all-in, because the only - constructor of `PricePoint` that `load_prices()` uses is `assemble()`. - """ + """What a plugin returns: the figure its source published, unmodified. Not a + PricePoint: only `assemble()` builds those, so a plugin cannot claim all-in.""" start: datetime # tz-aware instant eur_per_kwh: float # as published, containing exactly caps.sources @@ -513,30 +290,16 @@ class ProviderCapabilities: return frozenset(ALL_IN - self.sources) -# --------------------------------------------------------------------------- # # The read: a series, or no data. Nothing in between. -# --------------------------------------------------------------------------- # class PriceRead(_Outcome): - """What one attempt to get prices said. Two outcomes, and only one answers. - - * `series` - we have prices. Carries `.points` (NON-EMPTY, by - construction), `.product`, `.provider`, `.fetched_at`. - * `no_data` - we do not. Carries `.reason` and NOTHING ELSE. §11.4: static - profile, plan provisional, no arbitrage. - - ⚠️ A SERIES OF ALL-ZERO PRICES IS A `series`, NOT `no_data`, and that - distinction is the whole point. Zero prices mean arbitrage is pointless; - no data means arbitrage is FORBIDDEN. An implementation that returns `[]` - on a failed fetch cannot tell a caller which of those two it is in. - """ + """What one attempt to get prices said: `series` (prices, NON-EMPTY by + construction) or `no_data` (a `.reason` and nothing else; 11.4: static profile, + plan provisional, no arbitrage). A series of all-zero prices is still a + `series`: zero prices make arbitrage pointless, no data forbids it.""" def match(self, *, series, no_data): - """Fold to a value. Both handlers required and keyword-only. - - Handlers take one argument: this read. Required-ness is what makes the - no-data case impossible to forget rather than merely easy to remember. - """ + """Fold to a value. Both handlers are required keywords, so the no-data case cannot be forgotten.""" return {"series": series, "no_data": no_data}[self.kind](self) @staticmethod @@ -553,104 +316,25 @@ class PriceRead(_Outcome): return _NoData(reason) -# --------------------------------------------------------------------------- # # How long ONE point covers - TARIFF-14 -# --------------------------------------------------------------------------- # -# ⚠️ A POINT DOES NOT CARRY ITS OWN DURATION AND MUST NOT LEARN TO. §9.3 lists -# THREE fields and `PricePoint` has three; a fourth would change the wire -# format every store, replay, fixture and captured body in this repository is -# written against. The span is DERIVED from the neighbours instead - here, -# once, at construction. +# A point does not carry its own duration (9.3 has three fields): the span is +# derived from the neighbours once, at construction. MAX_POINT_SPAN = timedelta(hours=1) def _cover_table(points) -> tuple: """`(start_utc, end_utc, point)` per point, ascending. The `at()` index. - THE DEFECT THIS PREVENTS (TARIFF-14, HIGH). `at()` asked - `s <= u < s + 1 h` of every point IN LIST ORDER and returned the FIRST - match. On an hourly series - the only resolution any fixture carried - before TARIFF-08 - that is right by accident. Belgium moved to a - 15-minute MTU on 2025-10-01 and the live day-ahead feed returns 96 points - at 900 s, where it is wrong twice over. Measured on shipped `5059e28`: - - series: 02:00=0.10 02:15=0.11 02:30=0.12 02:45=0.13 03:00=0.14 - - at(02:15) -> 02:00 the :00 point's HOUR has not run out yet - at(03:15) -> 02:30 and from the second hour on it is not even the - top of the hour: the first point still inside - its own hour wins, so the error COMPOUNDS with - depth into the series - - `arbitrage._act()` compares `here.start` against `v.cheapest.start` and - `v.dearest.start` to decide whether this interval is the buy or the sell - one. That identity test almost never matched the extremum, so the - strategy idled through exactly the interval it exists to trade - and read - `here.provisional` and §17's `price < 0` guard off the wrong point while - doing it. `static_tariff.py` emits hourly points, so static sites never - saw it; this is reachable only down the dynamic path TARIFF-08/09 made - work. - - THE RULE, in one sentence: - - A POINT COVERS FROM ITS OWN START UNTIL THE NEXT POINT'S START, AND - NEVER MORE THAN ONE HOUR. - - * 900 s -> each point covers its own quarter, so `at(02:15)` is the - 02:15 point; - * 3600 s -> each point covers its own hour, unchanged from shipped; - * MIXED -> EACH POINT TAKES ITS OWN GAP. FCST-04's carry-forward emits - HOURLY carried points beside QUARTER-HOURLY firm ones, and the soak - measured spacing `[900, 3600]` in 70 % of samples - so mixed is the - normal steady state, and a window derived from one global spacing - would be wrong THERE rather than in an edge case. - - ⚠️ WHY THE ONE-HOUR CAP, WHICH IS NOT DECORATION. FCST-04 LEAVES HOLES ON - PURPOSE: an hour with no published price AND no yesterday value is left - OUT of the series, and `price_forecast.py`'s module note names `at()` - raising `Absence` for it as the behaviour that sends the caller to the - static profile (§11.4 for that hour). Uncapped, "until the next point's - start" would stretch the point BEFORE a three-hour hole across the whole - hole and answer every instant in it with a stale price. Filling holes is - the defect this module exists to refuse, and it would have been - introduced by the fix for a different one. - - ⚠️ THE LAST POINT'S WINDOW IS DECIDED, NOT INHERITED. There is no next - start and nothing in §9.3 says how wide the final interval is, so THE - LAST POINT IS GIVEN THE SAME SPAN AS THE POINT BEFORE IT - one hour when - it is the only point. Three consequences, all intended: - - * on a 96-point 900 s day the 23:45 point covers 23:45 .. 00:00, so - `at(23:50)` ANSWERS and `at(00:00)` raises `Absence`. A flat one-hour - tail would have answered 00:30 TOMORROW with today's last price, - which is reaching for a neighbour across a day boundary; - * on an hourly series the tail is an hour, exactly as it shipped, so no - static or hourly site changes behaviour; - * on a MIXED series whose last point is a carried HOURLY one behind - quarter-hourly firm ones, the tail is 900 s and the last 45 minutes - of the horizon raise `Absence`. That is the SAFE direction and it is - accepted rather than tolerated: the caller falls back to the static - profile (§11.4) instead of trading on a span the series never - demonstrated, and a carried point is provisional, so §8.3 forbids any - irreversible action resting on it regardless. - - ⚠️ ORDER IS ESTABLISHED HERE, NOT ASSUMED - and that is the difference - between this and the rule it replaces. `at()` now reads neighbours, so it - WOULD depend on sortedness; the table is therefore sorted by UTC start at - construction, and no caller's list order can reach it. `_validate_raw()` - rejects out-of-order and duplicate starts on the `load_prices()` path, - `price_history._parse_points()` re-sorts what it reads off disk, and - FCST-04's fill emits ascending - but `PriceRead.series()` itself has - never enforced any of that, and an assumption that holds today and is - unstated is how this class of defect recurs. - - ⚠️ NOTHING HERE RAISES ON A MALFORMED SERIES. A duplicated start - not - reachable through `load_prices()`, which refuses it - gives the earlier - list entry a ZERO-width window and the later one the real span, so one of - the pair becomes unreachable through `at()` and neither the 30 s control - path nor construction raises. - """ + TARIFF-14: a point covers from its own start until the next point's start, and + never more than one hour. Each point takes its own gap (firm quarter-hours and + carried hours are mixed in steady state). The cap keeps FCST-04's deliberate + holes from being filled by stretching the point before them. The last point + gets the span of the one before it (one hour if alone), so on a 96-point day + `at(00:00)` raises `Absence` rather than answering with today's last price. + Sorted here, so no caller's list order matters. Nothing here raises on a + malformed series: a duplicated start (refused by `load_prices()`) gives the + earlier entry a zero-width window. DECISIONS/TARIFF-14-the-window-a-point-covers.md.""" rows = sorted(((p.start.astimezone(timezone.utc), i, p) for i, p in enumerate(points)), key=lambda r: (r[0], r[1])) @@ -673,9 +357,8 @@ class _Series(PriceRead): def __init__(self, points, product, provider, fetched_at): points = list(points) - # ⚠️ THE REFUSAL THAT MAKES THE TYPE WORTH HAVING. An empty series is - # how "the fetch failed" gets to wear the shape of "we looked and there - # is nothing expensive today". It is not constructible. + # An empty series is not constructible: "the fetch failed" must not + # look like "nothing is dear today". if not points: raise ValueError( "PriceRead.series() refuses an empty point list. A fetch that " @@ -691,9 +374,8 @@ class _Series(PriceRead): self.provider = provider self.fetched_at = fetched_at self.regime = volatility_regime(product) - # ⚠️ TARIFF-14. Built ONCE, here, so that `at()` scans an index whose - # order it established itself rather than one it hopes it was handed. - # See `_cover_table()` for the window rule and for what the tail does. + # TARIFF-14. Built once here so `at()` scans an index whose order it + # established itself. self._cover = _cover_table(self.points) self.words = (f"{len(self.points)} points from {provider} " f"({product.value}, {self.regime} thresholds)") @@ -703,25 +385,10 @@ class _Series(PriceRead): return {"prices": [p.canonical() for p in self.points]} def at(self, instant: datetime) -> PricePoint: - """The point whose OWN interval covers `instant`, at ANY resolution. - - ⚠️ RETURNS A POINT OR RAISES; it never returns None. `series.at(t) or - fallback` is the same defaulting bug one level down, so the falsy - answer does not exist. Ask `covers()` first if you mean to branch. - - ⚠️ TARIFF-14: THE WINDOW IS NOT ONE HOUR WIDE ANY MORE, AND WAS NEVER - ENTITLED TO BE. It runs to the next point's start, capped at an hour - - `_cover_table()` above holds the rule, the tail decision and the - reproduction on a quarter-hourly series. A fixed `s + 1 h` here - returned the FIRST point still inside its own hour, which on the live - 900 s feed is neither the covering point nor, past the first hour, - even the top of the hour. - - A linear scan over the index, which is 192 entries on the shipped 48 h - horizon. This runs on the 30 s control path through - `arbitrage._point_at()`; a bisect would be correct and is not worth - the second way to be wrong at this size. - """ + """The point whose OWN interval covers `instant`, at ANY resolution. Returns a + point or raises; there is no falsy answer, so `at(t) or fallback` cannot happen. + Ask `covers()` first to branch. Linear scan (about 192 entries, 30 s path). + The window rule is in `_cover_table()`.""" u = instant.astimezone(timezone.utc) for start, end, point in self._cover: if start <= u < end: @@ -735,15 +402,8 @@ class _Series(PriceRead): f"§11.4 for it, not a reason to reach for a neighbouring one.") def covers(self, instant: datetime) -> bool: - """Whether `at(instant)` would answer. - - ⚠️ SHARED TARIFF-14 EXACTLY, AND IS FIXED BY THE SAME CHANGE. It has - no window rule of its own - it asks `at()` - so on the 900 s feed it - answered True for instants no point covered and False for none of - them, because the hour-wide window is never SHORTER than the truth. - It is kept as a delegation for that reason: a second copy of the rule - is a second place for it to be wrong. - """ + """Whether `at(instant)` would answer. Delegates to `at()`: a second copy of the + window rule is a second place for it to be wrong.""" try: self.at(instant) return True @@ -751,29 +411,15 @@ class _Series(PriceRead): return False def last_end_before(self, instant: datetime): - """When the last price that has run out by `instant` stopped covering, - or None if no price has (PRICE-01: "no prices since 00:00"). - - Each point's end is the one `_cover_table()` worked out, so a hole in - the series is reported from where the hole starts and not from the end - of the day. - """ + """When the last price that has run out by `instant` stopped covering, or None + (PRICE-01: "no prices since 00:00"). A hole is reported from where it starts.""" u = instant.astimezone(timezone.utc) ends = [end for _start, end, _p in self._cover if end <= u] return max(ends) if ends else None def window(self, since: datetime, until: datetime) -> tuple: - """Points whose START falls in [since, until). May legitimately be empty. - - ⚠️ DID NOT SHARE TARIFF-14, AND THE REASON IS WORTH STATING RATHER - THAN REDISCOVERING. This grades a point's `start` against a half-open - range and never asks how long a point lasts, so it carries no - resolution assumption to be wrong about - the same call answers the - same set at 900 s, at 3600 s and mixed. It is therefore NOT a - coverage query: a point that starts before `since` and runs into it - is absent from this tuple by design. `at()`/`covers()` are the - coverage questions. - """ + """Points whose START falls in [since, until). May be empty. Not a coverage + query (it never asks how long a point lasts); `at()`/`covers()` are.""" a = since.astimezone(timezone.utc) b = until.astimezone(timezone.utc) return tuple(p for p in self.points @@ -781,27 +427,10 @@ class _Series(PriceRead): class _NoData(PriceRead): - """We have no prices, and the §11.4 row applies. - - ⚠️ THIS CLASS HAS NO `.points` AND THAT IS THE POINT. `min(read.points)`, - `for p in read.points`, `read.get("points", [])` and `len(read.points)` all - raise `Absence` here rather than quietly reporting that the cheapest hour of - the day costs nothing. - - ⚠️ TWO WAYS PAST IT SURVIVE, BOTH FOUND IN REVIEW OF PR #16, AND NEITHER IS - WORTH CODE: - - * `vars(read).get("points", [])` returns `[]`. `vars()` bypasses - `__getattr__` entirely; nothing short of `__slots__` plus a metaclass - would stop it, and anyone writing `vars()` on a variant type has left - the road on purpose. - * `bool(read)` is `True` for BOTH variants, so `if read:` looks like a - check and is not one. Defining `__bool__` to be False here would make - `if read:` "work" - and that is exactly the wrong fix: it would turn a - line that reads like a guard into one that IS a guard by accident, - which is how `.get(k, default)` got into this codebase in the first - place. Use `.match(...)`. There is no falsy variant on purpose. - """ + """We have no prices, and the 11.4 row applies. There is no `.points`, so + `min(read.points)` and `.get("points", [])` raise `Absence` rather than report + the cheapest hour as free. `bool(read)` is True for both variants and has no + falsy form on purpose: use `.match(...)`.""" kind = "no_data" @@ -811,62 +440,28 @@ class _NoData(PriceRead): def arbitrage_allowed(read: PriceRead) -> bool: - """§11.4, stated once, as a function so no caller has to remember it. - - Not the arbitrage strategy - that is OPT-04. This is only the veto, which - belongs with the type that expresses the condition. - """ + """11.4 as a function, so no caller has to remember it. Only the veto; the strategy is OPT-04.""" return read.match(series=lambda _r: True, no_data=lambda _r: False) def plan_is_provisional(read: PriceRead) -> bool: - """§8.3 / §11.4: provisional if there is no data at all, or if ANY point in - the series is itself provisional. Derived from the points, never asserted.""" + """8.3 / 11.4: provisional if there is no data at all, or if ANY point is itself + provisional. Derived from the points, never asserted.""" return read.match( series=lambda r: any(p.provisional for p in r.points), no_data=lambda _r: True) -# --------------------------------------------------------------------------- # # Options: what a plugin may see, and what it may not -# --------------------------------------------------------------------------- # -# --------------------------------------------------------------------------- # # TARIFF-18: what `price_taxes_eur_kwh` IS MADE OF -# --------------------------------------------------------------------------- # -# ⚠️ THE DEFECT THIS TABLE PREVENTS IS THE ONE THE SHIPPED 0.06 WAS: A NUMBER -# NOBODY COULD SOURCE. 0.06 EUR/kWh is roughly HALF the real Flemish non-energy -# figure, and because no reader could see what it was made of there was nothing -# to argue with. Every row below is one line of the Eneco residential card for -# Flanders, September 2026, INCLUDING 6 % VAT, converted from the c EUR/kWh the -# card prints to the EUR/kWh this option takes. `price_taxes_eur_kwh`'s default -# is their sum, and test_price.py section 1b re-adds them and refuses a default -# that is not exactly that. -# -# ⚠️ THIS IS A PER-DSO FIGURE, AND IT IS THE DISTRIBUTION ROW THAT MOVES. Across -# the eight Fluvius DSOs that row runs 5,28 c EUR/kWh (Midden-Vlaanderen) to -# 6,70 (West), so the all-in Flemish total runs about 0.117 to 0.131 EUR/kWh. -# A site outside Antwerpen should type its own total; the other two rows are -# Flanders-wide and do not move with the DSO. -# -# ⚠️ AND THE DEFAULT SITS NEAR THE LOW END OF THAT BAND ON PURPOSE, BECAUSE THE -# TWO ERROR DIRECTIONS ARE NOT SYMMETRIC. This scalar is a LEVEL component: it -# cancels exactly in `max - min` and cannot move which hour is cheapest -# (test_price.py 16b measures that as a family). The one DECISION it reaches is -# §17's negative-price guard in arbitrage.py, which tests the ABSOLUTE all-in -# price of the sell hour. A figure set too HIGH carries a genuinely negative -# hour up above zero, and the add-on exports into paid-for disposal - the exact -# case §17 exists to refuse. A figure set too LOW only declines a trade that -# would have paid. So where a default must stand in for eight different DSOs, -# it belongs at the bottom of the band and not in the middle of it. -# -# ⚠️ `energiebijdrage` IS ZERO, AND IT IS A ROW HERE RATHER THAN AN ABSENCE. -# "Vanaf 1 augustus 2026 bedraagt de energiebijdrage 0 EUR/MWh" - it was merged -# into the bijzondere accijns. Anyone re-deriving this total from a guide -# written before that date will find a separate energiebijdrage line and add it -# on top of an accijns that already contains it. The zero row is here so that -# the double-count is visible as a decision rather than as an omission. +# Each row is one line of the Eneco residential card for Flanders, September +# 2026, including 6 % VAT, in EUR/kWh. The default of `price_taxes_eur_kwh` is +# their sum (test_price.py section 1b re-adds them). Per DSO, and set near the +# low end of the band on purpose; `energiebijdrage` is a zero row since +# 2026-08-01. Why: DECISIONS/TARIFF-02-price-provider-plugins.md, "Taxes +# components". FLEMISH_TAXES_COMPONENTS_EUR_KWH = ( ("Fluvius distribution, Antwerpen", 0.0535), ("Bijzondere accijns, 'Alle verbruik' - no volume bands", 0.04876), @@ -874,152 +469,79 @@ FLEMISH_TAXES_COMPONENTS_EUR_KWH = ( ("Energiebijdrage - 0 EUR/MWh since 2026-08-01, inside the accijns", 0.0), ) -# ⚠️ TARIFF-18, AND IT IS NOT HOUSEKEEPING - IT DECIDES WHICH CONFIGS START. -# TARIFF-17's rung in validate.py refuses a site that sets `price_taxes_profile` -# AND leaves `price_taxes_eur_kwh` at something other than the shipped default, -# because the table wins and the scalar would decide nothing. It compares -# against the DEFAULT rather than against presence, because the Supervisor sets -# `App.options = deepmerge(config.yaml defaults, stored options)` and a saved -# blob therefore carries every schema key whether the operator typed it or not -# (measured: the reference install's export is 70 of 71 keys - CHANGELOG 0.9.0, -# `tools/supervisor_selfapi_probe.py` §D). -# -# So moving the default turns a previously ACCEPTED configuration into -# `SystemExit(78)`. A site that stored 0.06 - which is every install that has -# ever saved its configuration - and that also carries a taxes table would stop -# starting on the upgrade, over a number it never typed and that decides -# nothing. That refusal would be TRUE and USELESS: the rung's subject is a -# number the OPERATOR typed, and a superseded shipped default is not one. It -# still reads as "untouched" here, and the rung's sentence names both figures. +# TARIFF-18: moving this default changes which configs start. validate.py +# (TARIFF-17 rung) compares `price_taxes_eur_kwh` to the default, so a +# superseded default (0.06) must still read as untouched. Do not change a +# default without that check. DECISIONS/TARIFF-02-price-provider-plugins.md. SUPERSEDED_TAXES_DEFAULTS_EUR_KWH = (0.06,) # TARIFF-02 .. TARIFF-17 -# ⚠️ CFG-03, AND HOW THIS TICKET AVOIDS WIDENING IT. Option defaults in this -# add-on already live in three places - config.yaml, main.py's `o.get(k, X)` -# fallbacks, and a dataclass default - held apart only by a tripwire in -# test_validate.py. This module adds ONE more location, not another three: -# -# * every price option's fallback is here, and only here; -# * validate.py imports this dict rather than restating any number; -# * main.py gets no `o.get("price_...", X)` fallbacks at all; -# * there is no dataclass carrying a default copy of any of these; -# * test_price.py READS config.yaml and asserts the shipped default equals -# this dict, key for key, so the two cannot drift silently. -# -# config.yaml remains the source of truth. This is the single in-code mirror, -# pinned to it by a check, which is the `validate.py` pattern rather than a -# fourth freehand copy. +# CFG-03. The one in-code mirror of config.yaml for the price options: +# validate.py imports it, main.py has no price `.get()` fallbacks, and +# test_price.py section 1 pins it to config.yaml key for key. PRICE_OPTION_DEFAULTS = { "price_provider": "off", "price_markup_eur_kwh": 0.03, # §14 tariffs.energy_price.markup_eur_kwh - # TARIFF-18: the sum of FLEMISH_TAXES_COMPONENTS_EUR_KWH above, to the - # card's own last digit. §14 tariffs.energy_price.taxes_eur_kwh. + # TARIFF-18: the sum of FLEMISH_TAXES_COMPONENTS_EUR_KWH, to the card's + # last digit. "price_taxes_eur_kwh": 0.11736, "export_price_eur_kwh": 0.04, # §14 tariffs.export_price_eur_kwh (AC4) "price_csv_path": "/share/goodwe_prices.csv", "price_feed_url": "", - # PRICE-01. The bearer token of the owner's price server. A secret: it is - # in SECRET_OPTIONS below, so no text built from an error reaches a log, - # the status page or a notification with it in. + # PRICE-01. The bearer token (a secret: in SECRET_OPTIONS, so `scrub()` + # keeps it out of text). "price_service_token": "", "price_horizon_h": 48, "price_timeout_s": 10, - # TARIFF-03. §14 `tariffs.static_profile: null`. A JSON object describing a - # static day/night or fully-fixed tariff; "" is §14's `null`. Parsed and - # validated by app/static_tariff.py, refused at startup by validate.py. - # - # ⚠️ IT LIVES HERE RATHER THAN IN static_tariff.py FOR CFG-03's SAKE. This - # dict is the ONE in-code mirror of config.yaml, pinned to it by - # test_price.py section 1. A second defaults dict in the new module would be - # a fourth place option defaults live, which is exactly the open ticket. + # TARIFF-03. A JSON static day/night or fixed tariff; "" is unset. Parsed + # by app/static_tariff.py, refused at startup by validate.py. Here, not in + # static_tariff.py, so there is one defaults mirror (CFG-03). "static_profile": "", - # TARIFF-17. A JSON object in `static_profile`'s OWN period grammar whose - # per-period `eur_per_kwh` is the TAXES COMPONENT for that band, not an - # all-in price; "" is unset and the scalar above decides. Parsed and - # validated by app/static_tariff.py (`parse_taxes_profile`), refused at - # startup by validate.py - the `static_profile` precedent exactly, and here - # for the same CFG-03 reason: this dict is the ONE in-code mirror of - # config.yaml. - # - # ⚠️ IT IS IN FRAMEWORK_KEYS. A plugin that could read this could add the - # taxes a second time, per hour, which is the double-counting defect trap 1 - # exists to refuse - wearing a period table instead of a scalar. + # TARIFF-17. A period table in `static_profile`'s grammar whose + # `eur_per_kwh` is the TAXES component per band; "" is unset and the scalar + # decides. In FRAMEWORK_KEYS, so a plugin cannot add taxes a second time + # per hour. "price_taxes_profile": "", - # TARIFF-15. A JSON object carrying ONE contract's two coefficients, the - # units they are printed in, its VAT rate and the card figures the add-on - # grades them against; "" is unset. Parsed and validated by - # app/custom_formula.py, refused at startup by validate.py - the - # `static_profile` precedent exactly, and here for the same CFG-03 reason: - # this dict is the ONE in-code mirror of config.yaml. + # TARIFF-15. One contract's two coefficients, units, VAT rate and the card + # figures they are graded against; "" is unset. Parsed by + # app/custom_formula.py. "price_formula": "", - # TARIFF-16. The INJECTION leg, in `price_formula`'s OWN grammar and read - # by `custom_formula.parse_export_formula()` - the same parser with its - # option name as an argument, so a second unit table and a second unknown- - # key rule do not exist to diverge. "" is unset and the scalar - # `export_price_eur_kwh` above decides, which is every site today. - # - # ⚠️ IT IS IN FRAMEWORK_KEYS, for the reason the scalar is: a CONSUMPTION - # plugin that could read an injection curve could fold it into a - # consumption price, and nothing downstream could tell. - # - # ⚠️ AND IT LIVES HERE RATHER THAN IN custom_formula.py FOR CFG-03's SAKE, - # exactly as `price_formula` and `price_taxes_profile` do: this dict is the - # ONE in-code mirror of config.yaml. + # TARIFF-16. The INJECTION leg in `price_formula`'s own grammar, read by + # `custom_formula.parse_export_formula()`; "" is unset and + # `export_price_eur_kwh` decides. In FRAMEWORK_KEYS, so a consumption + # plugin cannot fold it in. "price_export_formula": "", - # OPT-04. §10.1 priority 3, and §14's `optimizer.cycle_cost_threshold_eur`. - # - # ⚠️ THEY LIVE HERE FOR CFG-03's SAKE, exactly as `static_profile` does, and - # FCST-03's note above `WIDEN_FACTOR` names this as the place they would - # land. `arbitrage_soc_floor` / `arbitrage_soc_ceiling` are the reserve - # arbitrage may not spend and the headroom it may not overfill; they are - # NOT the maintenance SoC keys, which describe a balancing cycle rather - # than a trading reserve and would silently couple two unrelated policies. - # - # ⚠️ AND `cycle_cost_threshold_eur` IS 0.10 BECAUSE §10.1's NAME FOR IT SAYS - # SO, NOT BECAUSE IT MAKES ANYTHING TRADE. On realistic Belgian day/night - # numbers the static spread is ~0.08 EUR/kWh, so a static-tariff site does - # NOT trade at this default. That is a reported result (test_arbitrage.py - # section 7), and whether 0.10 - or FCST-03's 2.5/1.5 - is right is one - # product decision that belongs to the owner. No number here was chosen to - # make arbitrage happen. + # OPT-04, 10.1 priority 3, and `optimizer.cycle_cost_threshold_eur`. The + # two SoC keys are not the maintenance keys. 0.10 is the 10.1 name, not a + # value that makes anything trade (static spread is about 0.08): an owner + # decision. DECISIONS/TARIFF-02-price-provider-plugins.md. "arbitrage_enabled": False, "cycle_cost_threshold_eur": 0.10, "arbitrage_soc_floor": 20, "arbitrage_soc_ceiling": 95, } -# ⚠️ The keys the FRAMEWORK owns. A plugin that reads one of these is a plugin -# that can double-count, so `PluginOptions` refuses them by name. Trap 1. +# The keys the FRAMEWORK owns. A plugin that reads one can double-count, so +# `PluginOptions` refuses them by name (trap 1). FRAMEWORK_KEYS = frozenset({ "price_markup_eur_kwh", "price_taxes_eur_kwh", "export_price_eur_kwh", - # TARIFF-17. The hour-varying spelling of `price_taxes_eur_kwh`, and it - # belongs here for the same reason the scalar does: it is a component the - # FRAMEWORK adds exactly once, by `caps.adds`. `BANNED_IN_PLUGIN_CODE` is - # derived from this set, so `code_leaks()` refuses a plugin that names it - # without anything else being edited. + # TARIFF-17. The hour-varying `price_taxes_eur_kwh`, added once by + # `caps.adds`. `code_leaks()` derives from this set. "price_taxes_profile", - # TARIFF-16. The hour-varying spelling of `export_price_eur_kwh`, and it - # belongs here for the same reason the scalar does: a plugin that could - # read an INJECTION curve could fold it into the CONSUMPTION series it - # emits, and `assemble()` has no way to notice a component that arrived - # inside a RawPoint. `BANNED_IN_PLUGIN_CODE` is derived from this set, so - # `code_leaks()` refuses a plugin that names it with nothing else edited. + # TARIFF-16. The hour-varying `export_price_eur_kwh`; same reason, same + # derivation. EXPORT_FORMULA_OPTION, }) -# PRICE-01. Options whose VALUE must never be printed. `scrub()` removes them -# from any text the price code builds out of an error. +# PRICE-01. Options whose VALUE must never be printed; `scrub()` removes them +# from error text. SECRET_OPTIONS = frozenset({"price_service_token"}) def scrub(text, opts) -> str: - """`text` with every secret option's value replaced by `***`. - - The price code builds reasons out of library errors, and a library may put - a request header into its error text. Taking the value out here is the - last line of defence; the plugin also never puts it in a message itself. - """ + """`text` with every secret option's value replaced by `***`. The last line of + defence: a library may put a request header into its error text.""" out = str(text) for key in SECRET_OPTIONS: secret = str((opts or {}).get(key) or "").strip() @@ -1029,17 +551,9 @@ def scrub(text, opts) -> str: class PluginOptions: - """The options dict, minus the keys only the framework may read. - - Reading `price_markup_eur_kwh` here raises `FrameworkOption` rather than - returning 0.03. - - ⚠️ THIS IS ONE OF THE TWO DOORS, NOT THE ONLY ONE. `_o` is reachable, and - `price_option()` and `PRICE_OPTION_DEFAULTS` are public names in this - module. This class alone is a convention; it is `register()`'s - `code_leaks()` scan that makes it an inability. Do not weaken either half - on the strength of the other. - """ + """The options dict, minus the keys only the framework may read: reading + `price_markup_eur_kwh` raises `FrameworkOption` rather than returning 0.03. One of + two doors; `register()`'s `code_leaks()` scan is the other. Do not weaken either.""" def __init__(self, opts: dict): self._o = dict(opts or {}) @@ -1078,136 +592,38 @@ def price_option(opts: dict, key: str): def provider_id(opts: dict) -> str: """The selected plugin id, normalised. `"off"` when none is selected. - - ⚠️ ONE SPELLING OF THIS EXPRESSION, AND TARIFF-10 IS WHY IT IS A FUNCTION. - `str(price_option(opts, "price_provider") or "off").strip()` was written out - twice - in `load_prices()` below and in `static_tariff.resolve_tariff()` - - and main.py's startup gate would have been a third. "Is a provider - configured" is now a DECISION the add-on takes (it is what starts the - background refresh), and a decision spelled out at three call sites is three - places to disagree about whether `" off "` or `None` counts. - ⚠️ `validate.py` deliberately keeps its own reader: it validates the RAW - options blob, including one the Supervisor never saw, so it must not route - through `price_option()`'s defaults. Note what this does NOT do: it does not - decide - whether the id names a shipped plugin - `load_prices()` and validate.py's - rule 7 own that, and answering "unknown" here would turn a typo into - "no provider" instead of into a startup refusal. - """ + One spelling for three call sites (TARIFF-10). validate.py keeps its own reader + (it validates the raw blob). It does not decide whether the id is shipped.""" return str(price_option(opts, "price_provider") or "off").strip() def provider_configured(opts: dict) -> bool: - """Has the operator selected a price feed at all? (TARIFF-10) - - THE DEFECT THIS PREVENTS: the background tariff refresh used to start only - when `arbitrage_enabled` was on, which ships `false` - so a site with a - provider configured fetched NOTHING, ever. `/data/price_history.json` was - never created, the *Price history* status row read `empty` forever, and - TARIFF-09's short-horizon warning could not fire because no fetch happened - to warn about. Measured over a 40 h two-instance soak: zero feed requests - and `present: false` across all 802 samples on the shipped default, against - a resolved plan on the same feed with arbitrage armed. - - ⚠️ THIS IS NOT A TRADING SWITCH AND MUST NEVER BECOME ONE. It answers - "fetch prices?"; `arbitrage.enabled()` answers "move energy on them?", and - §10.1's priority 3 is gated on the second alone. See - `main.tariff_refresh_wanted()` for the one place the two meet. - """ + """Has the operator selected a price feed at all? (TARIFF-10) Starts the + background refresh, which used to wait for `arbitrage_enabled` (ships false). + NOT a trading switch: `arbitrage.enabled()` answers that (`main.tariff_refresh_wanted()`).""" return provider_id(opts) not in ("", "off") def export_price_eur_kwh(opts: dict) -> float: - """AC4 / §9.3: the injection price is a SEPARATE SCALAR, not a second series. + """AC4 / 9.3: the injection price is a SEPARATE SCALAR, not a second series. - In Belgium it is often near zero, which by itself favours self-consumption - over export. It is deliberately not routed through the plugin framework - - but ⚠️ NOT because nothing dynamic exists to route. That claim stood here - until TARIFF-07 and it was FALSE: - - * Eneco publishes an injection formula on the same tariff card as its - consumption formula - `0,1 x BELPEX-H - 1,188` - identical on the - January and the September 2026 cards; - * ENGIE publishes one in the same two-coefficient shape, - `(B x eSpot_15) - A`. - - So a supplier DOES publish a dynamic injection curve. What is true is - narrower and is the actual reason this is a scalar: an injection curve is - the SAME `B x spot - A` shape as the consumption curve but carries ITS OWN - coefficients and its own VAT treatment - injection is VAT-exempt where - consumption is at 6% - so it is a second per-contract formula, not a second - copy of the first. - - ⚠️ TARIFF-16 LANDED THE INJECTION LEG, AND THIS FUNCTION IS UNCHANGED. The - curve lives in `price_export_formula` and is read by `export_formula()` - below; this reader still answers the FLAT injection price, still from one - option, still with no formula anywhere in its arithmetic. That is the - arrangement this docstring predicted before either existed - "an injection - curve gets a `PriceProvider` of its own and this scalar becomes its - FIXED-product default" - and it is why a site with a flat injection price - sees nothing change: `price_export_formula` ships "", `export_price_at()` - falls through to this function, and this function's body is byte-identical - to the one TARIFF-02 shipped. - - ⚠️ It is NOT reachable from a plugin: `export_price_eur_kwh` is in - FRAMEWORK_KEYS, so a consumption plugin cannot quietly fold an injection - price into a consumption price. `price_export_formula` is in the same set - for the same reason. - - ⚠️ AND NOTHING IN THIS ADD-ON READS EITHER ONE TO TAKE A DECISION. - `arbitrage.py` prices the sell side as DISPLACED IMPORT and says so in its - own comment; both hours it compares are all-in consumption prices. This - reader has no caller under `app/` and neither does `export_price_at()`. - They are what an operator is shown and what a future export-aware strategy - would call - not a number any trade turns on today. - """ + Not reachable from a plugin (`export_price_eur_kwh` is in FRAMEWORK_KEYS). The + formula leg is `export_formula()`; this stays the flat default. Nothing under + `app/` takes a decision on it. DECISIONS/TARIFF-02-price-provider-plugins.md.""" return float(price_option(opts, "export_price_eur_kwh")) def export_formula(opts: dict): - """The configured `price_export_formula` as a `FormulaSpec`, or None. - - TARIFF-16. Raises `custom_formula.FormulaInvalid` on a malformed leg - and - ⚠️ THAT EXCEPTION IS NOT MEANT TO REACH A RUNNING SITE, by the rule - `taxes_profile()` states above and `static_profile` established: - validate.py parses the same option at STARTUP and refuses to start on it, - because a formula that only fails when something reads it is a defect - discovered at the worst possible moment and §11.4 makes every price - defect quiet. - - ⚠️ THE PARSE IS THE SAME ONE THE CONSUMPTION LEG GETS. There is no second - parser, no second unit table and no second resolution rule - - `parse_export_formula()` is `parse_price_formula()` with its `option` - argument moved, so every refusal an operator can hit on this leg is a - refusal that already had checks over it. - """ + """The configured `price_export_formula` as a `FormulaSpec`, or None (TARIFF-16). + `custom_formula.FormulaInvalid` is refused at startup by validate.py, so it + should not reach a running site. Same parser as the consumption leg.""" return parse_export_formula(price_option(opts, EXPORT_FORMULA_OPTION)) def export_price_at(opts: dict, index_eur_per_mwh: float) -> float: """What the site is PAID for one exported kWh at one index value, EUR/kWh. - - THE SEAM, AND ITS HONEST SCOPE. This is the one function a future - export-aware strategy calls, and it has NO CALLER UNDER `app/` today - - stated here rather than implied, because the add-on takes no decision on an - injection price (see `export_price_eur_kwh()` above and `arbitrage.py`'s - accounting-basis comment). What it buys now is that OPT-06 and anything - after it inherit a shape the market fits, instead of a scalar that cannot - hold its own factor, a negative offset, or a VAT rate different from the - consumption leg's. - - ⚠️ THE FLAT PATH IS THE FALLTHROUGH AND IT IGNORES THE INDEX, which is - CORRECT rather than a simplification: a site with a flat injection price is - paid the same for every kWh whatever the market did, and that is what the - scalar means. With `price_export_formula` empty - every site today - this - returns exactly `export_price_eur_kwh(opts)` for every index. - - ⚠️ A NEGATIVE ANSWER IS A REAL ANSWER. Eneco's `0,1 x BELPEX-H - 1,188` is - below zero for any BELPEX-H under 11,88 EUR/MWh, which is the middle of a - sunny day and precisely when a PV household exports. Nothing here floors - it: a caller that silently clamped it to zero would hide the hours in which - exporting COSTS money, which is the one thing an injection price is for. - """ + No caller under `app/` yet. A flat price ignores the index (correct); a + negative answer is real and is never floored.""" spec = export_formula(opts) if spec is None: return export_price_eur_kwh(opts) @@ -1215,47 +631,28 @@ def export_price_at(opts: dict, index_eur_per_mwh: float) -> float: def taxes_profile(opts: dict): - """The configured `price_taxes_profile` as a period table, or None. - - TARIFF-17. Raises `static_tariff.ProfileInvalid` on a malformed table - and - ⚠️ THAT EXCEPTION IS NOT MEANT TO REACH A RUNNING SITE. validate.py parses - the same option at STARTUP and refuses to start on it, for the reason - `static_profile` is refused there: a table that only fails when a fetch - happens is a site that stops pricing at 03:00, and §11.4 turns that into a - quiet fallback nothing on the status page distinguishes from a healthy - fixed-tariff day. `load_prices()` still handles it, loudly, because a guard - whose only proof is "the other guard ran first" is not a guard. - - ⚠️ THE IMPORT IS LAZY AND THE CYCLE IS THE REASON. `static_tariff.py` - imports this module at module scope; importing it back at module scope here - would make which file is imported first decide whether the add-on starts. - `sys.modules` makes the repeat cost a dict lookup. - """ + """The configured `price_taxes_profile` as a period table, or None (TARIFF-17). + `static_tariff.ProfileInvalid` is refused at startup by validate.py; + `load_prices()` still handles it loudly. The import is lazy: static_tariff.py + imports this module at module scope.""" from .static_tariff import parse_taxes_profile return parse_taxes_profile(price_option(opts, "price_taxes_profile")) -# --------------------------------------------------------------------------- # # The plugin interface -# --------------------------------------------------------------------------- # MAX_PLAUSIBLE_EUR_KWH = 10.0 # see _validate_raw class PriceProvider(ABC): """One supplier. Declares what it is, then returns what its source said. - - ⚠️ `fetch()` DOES NOT RETURN A PRICE. It returns `RawPoint`s carrying the - published figure. The all-in arithmetic happens in `assemble()`, once, in - the framework, using `capabilities().sources`. See trap 1. - """ + `fetch()` returns `RawPoint`s, not prices; `assemble()` does the arithmetic.""" id = "?" - # PRICE-01. The ids to try, in order, when this source cannot answer, and - # whether the last answer is kept for a day without any source. Empty and - # False for every plugin that existed before, which is what keeps them - # exactly as they were: `price_chain.py` is not used for them at all. + # PRICE-01. Ids to try in order when this source cannot answer, and whether + # the last answer is kept for a day. Empty / False for every earlier + # plugin. FALLBACKS: tuple = () KEEPS_LAST_GOOD: bool = False @@ -1269,91 +666,30 @@ class PriceProvider(ABC): transport) -> list: """Return `RawPoint`s, or raise `SourceUnavailable(reason)`. - `transport(url, timeout_s) -> bytes` is injected so the checks never - touch a network and `manual_csv` never needs one. `opts` is a - `PluginOptions`, so the markup and taxes are not reachable from here. - - ⚠️ `since`/`until` ARE THE RANGE THE CALLER WANTS, AND A PLUGIN THAT - IGNORES THEM GETS WHATEVER ITS FEED DEFAULTS TO (TARIFF-09). That is - not a small difference: `generic_dayahead` ignored both, the public - EPEX feed answered ONE Brussels day, and §8.3 then dropped every hour - past today from the trade set as provisional - so the optimiser never - saw tomorrow as firm, at any hour of any day, and nothing reported it. - - This promise is NOT mechanically enforced. A rule obliging every - plugin to pass a range would be satisfied by an INVENTED parameter - name - the parser-graded-against-its-own-guess defect, moved from the - response to the request, and the defect TARIFF-08 found on the - response side. What exists instead is a roster pin in test_price.py: - which plugins consume the range and which do not is asserted as a - literal, so a third supplier forces the decision rather than - inheriting the silence. - """ + `transport(url, timeout_s) -> bytes` is injected. `opts` is a `PluginOptions`. + `since`/`until` are the range the caller wants; a plugin that ignores them gets + its feed's default (TARIFF-09). Not mechanically enforced: test_price.py pins + which plugins consume the range as a literal roster.""" def startup_preview(self, points) -> list: - """Lines to log ONCE, the first time this provider answers (TARIFF-15). - - `points` are the ASSEMBLED, all-in `PricePoint`s - what the site is - really priced on - and the default is to say nothing, so no existing - plugin changes behaviour. - - ⚠️ IT IS A HOOK RATHER THAN A LINE IN `load_prices()` BECAUSE THE - FRAMEWORK CANNOT KNOW WHAT IS WORTH PREVIEWING. Only the plugin knows - what went IN to its own arithmetic - for `custom_formula` the index in - EUR/MWh - and printing that beside the all-in figure is the whole - point: a factor of 1000 is obvious in one line of - `61.96 -> 0.13522` and invisible in a plan. The framework supplies the - one number a plugin may never read, by handing back the assembled - result rather than the tax rate. - - ⚠️ IT MUST NOT RAISE INTO THE CONTROL PATH and it is not trusted not - to: `_emit_startup_preview()` catches everything. A log line is not a - reason to stop regulating a battery. - """ + """Lines to log ONCE, the first time this provider answers (TARIFF-15). `points` + are the assembled all-in `PricePoint`s; the default says nothing. A hook, since + only the plugin knows what went in. Must not raise into the control path + (`_emit_startup_preview()` catches everything).""" return [] -# ⚠️ THE NAMES A PLUGIN'S OWN CODE MAY NOT MENTION. FRAMEWORK_KEYS is the -# obvious half; the other three are the routes review of PR #16 actually used to -# write a double-counting plugin after `PluginOptions` was in place: -# -# `_o` - PluginOptions' private dict, reachable as -# `opts._o["price_markup_eur_kwh"]`. -# `price_option` - a PUBLIC framework helper that takes the raw -# options dict and answers with the markup. -# `PRICE_OPTION_DEFAULTS` - a PUBLIC module constant. This is the one that -# decided the review: no private access, no options -# dict, no helper - just a dict in the module every -# plugin already imports for `register`. -# -# Any of them turns +0.088 into +0.119 on a wholesale feed with nothing -# anywhere reporting it. +# Names a plugin's own CODE may not mention: FRAMEWORK_KEYS plus `_o`, +# `price_option` and `PRICE_OPTION_DEFAULTS`, the other routes PR #16 review +# used to double-count. BANNED_IN_PLUGIN_CODE = tuple(sorted(FRAMEWORK_KEYS)) + ( "PRICE_OPTION_DEFAULTS", "price_option", "_o") def code_leaks(cls) -> list: """Framework-owned names appearing in a plugin's own CODE. Empty is clean. - - ⚠️ DOCSTRINGS ARE STRIPPED BEFORE THE SEARCH, AND THAT IS NOT COSMETIC. - Every shipped plugin's docstring is REQUIRED to name the components it - sources, so `generic_dayahead` says "markup" in prose and would - false-positive against `price_markup_eur_kwh` on a raw text scan. The - check that fires on the documentation it also mandates is the check that - gets deleted. So the - class is parsed, every docstring node is dropped, and the search runs on - `ast.unparse` of what is left - i.e. on code, which is the only thing that - can actually add a number twice. - - ⚠️ FAIL-OPEN ON MISSING SOURCE, DELIBERATELY. `inspect.getsource` needs the - .py file, which the add-on image has (Dockerfile COPYs `app/`) but a frozen - or zipapp build would not. Refusing to start a battery controller because a - STATIC ANALYSIS could not run would trade a real outage for a hypothetical - accounting error, so an unreadable source returns `[]` and logs. CI proves - the scan really runs: test_price.py asserts source was found for every - shipped plugin, so a build that silently lost it is a red check, not a - quietly disarmed guard. - """ + Docstrings are stripped first (they must name the components). Fails open on + missing source; test_price.py asserts source is found for every shipped plugin.""" try: src = dedent(inspect.getsource(cls)) tree = ast.parse(src) @@ -1370,36 +706,10 @@ def code_leaks(cls) -> list: return sorted({name for name in BANNED_IN_PLUGIN_CODE if name in code}) -# --------------------------------------------------------------------------- # -# TARIFF-07: the ids that USED to resolve here, and what an operator on one of -# them has actually been running. -# -# ⚠️ THE DEFECT THIS EXISTS TO PREVENT: A RETIRED ID READ AS A TYPO. Without -# this table `price_provider: "eneco_eddy"` reaches validate.py's unknown-id -# branch and is refused as "not a shipped plugin" - true, and useless. The -# operator did not mistype it; it WAS a shipped plugin, they chose it on -# purpose, and the add-on has accepted it every startup since. What they need -# told is not "that id is wrong" but "that id never fetched a price, and here -# is what you have been running instead" - because §11.4 turned every failed -# fetch into a silent fall back to `static_profile`, so NOTHING they can see -# has ever differed from a working provider that finds no cheap hour. -# -# ⚠️ AND WHY THERE IS NO SILENT ALIAS. Mapping these to `generic_dayahead` -# would be the friendlier-looking option and it is the wrong one: these three -# published RETAIL prices (`B x spot + A`, per contract) and `generic_dayahead` -# reads WHOLESALE. Aliasing needs the site's own two coefficients, which nobody -# has - they live behind the customer's klantenzone login, and there is no -# regulator index to read them from (CREG's tariff-card pages 404; VREG was -# unreachable; every coefficient found in research came from a supplier's own -# PDF). An alias would therefore have to INVENT the numbers it prices a battery -# against, which is the same class of defect as the parser graded against its -# own guess - just one layer further from anything that reports. -# -# Each value is the ID-SPECIFIC half of the refusal: the endpoint, what it was -# observed to answer on 2026-09-13, and why no replacement plugin can be built. -# validate.py owns the common framing. Verified by hand before deletion, not -# taken on report - see DECISIONS/TARIFF-07-withdrawing-three-plugins.md. -# --------------------------------------------------------------------------- # +# TARIFF-07: the ids that USED to resolve here. A retired id is not a typo and +# has no silent alias (it published retail prices, `generic_dayahead` reads +# wholesale). Each value is the id-specific half of the refusal; validate.py +# owns the framing. DECISIONS/TARIFF-07-withdrawing-three-plugins.md. RETIRED_PROVIDERS: dict = { "engie_dynamic": ( "its endpoint https://api.engie.be/dynamic/be does not exist - the " @@ -1426,24 +736,13 @@ PROVIDERS: dict = {} def register(cls): - """Register a provider by its own declared id. Decorator. - - ⚠️ THE ONLY NAME->CLASS MAPPING IN THE SYSTEM. AC2 is "adding a plugin needs - no optimiser changes", and this is where that is bought: config resolves an - id here, the optimiser is handed a `_Series` carrying a `PriceProduct`, and - nothing downstream ever sees the id except to print it. - """ + """Register a provider by its own declared id. Decorator. The only name->class + mapping in the system (AC2: adding a plugin needs no optimiser changes).""" if cls.id in PROVIDERS: raise ValueError(f"duplicate provider id {cls.id!r}") if cls.id in RETIRED_PROVIDERS: - # ⚠️ AT IMPORT, NOT AT FETCH, and for the same reason the leak check is. - # A retired id that is registered again is the one state in which - # validate.py's retired-id refusal goes quiet WITHOUT anyone deciding - # it should: `pid in PROVIDERS` becomes true, the named message is - # never reached, and the operator is back to a provider that looks - # like it works. Re-shipping an id is legitimate - the endpoint could - # become real - but it is a DECISION, and taking the id out of - # RETIRED_PROVIDERS in the same commit is what records it. + # At import: re-registering a retired id silences validate.py's + # refusal. Take the id out of RETIRED_PROVIDERS in the same commit. raise ValueError( f"{cls.id!r} is in RETIRED_PROVIDERS and cannot also be a shipped " f"plugin. A site configured on it is refused at startup with a " @@ -1454,9 +753,8 @@ def register(cls): cls.capabilities() # fail at import, not at 15:00 on fetch day leaks = code_leaks(cls) if leaks: - # ⚠️ AT IMPORT, NOT AT FETCH. A plugin that can reach the markup is a - # plugin that can bill wrong for a month before anyone notices, so it - # does not get to exist - the add-on fails to start and names the line. + # At import, not at fetch: a plugin that can reach the markup must not + # exist. raise FrameworkOption( f"{cls.id!r} mentions {', '.join(leaks)} in its own code. A plugin " f"returns what its SOURCE published and declares " @@ -1471,13 +769,8 @@ def register(cls): def _validate_raw(points, caps): """Reject at the boundary, so a malformed feed cannot become a bad decision. - - ⚠️ NEGATIVE PRICES ARE VALID AND ARE NOT CLAMPED (§17 keeps a negative-price - regression case; Belgian day-ahead goes negative on windy Sundays and those - hours are the single best charging opportunity of the year). What is - rejected is what cannot be a price at all: NaN, infinity, an implausible - magnitude, a naive datetime, and duplicate or out-of-order starts. - """ + Negative prices are valid and not clamped (9.3, 17). Rejected: NaN, infinity, + an implausible magnitude, a naive datetime, duplicate or out-of-order starts.""" if not isinstance(caps, ProviderCapabilities): raise SourceUnavailable("the plugin did not declare capabilities") if not points: @@ -1509,29 +802,10 @@ def _validate_raw(points, caps): def taxes_at(taxes, instant: datetime) -> float: - """The TAXES component at ONE instant. A scalar, or a period table. - - TARIFF-17. THE DEFECT THIS EXISTS TO CLOSE, and it is a MISSING CAPABILITY - rather than a wrong constant: `price_taxes_eur_kwh` is additive and - identical every hour, so it cancels exactly out of `max - min` and cannot - move `min(firm, key=...)`. It is not a distortion - it is provably inert on - both quantities the optimiser reads. What was missing is anywhere for an - HOUR-VARYING non-energy component to live, and the Walloon and Brussels - DSOs bill one: ORES `dag 13,27 / nacht 7,39` c EUR/kWh, a 5,88 c gap whose - cheap band coincides with the cheap spot hours. Flattening it does not - shift the curve, it UNDERSTATES the spread and declines profitable cycles. - See DOCS.md "The three buckets". - - ⚠️ DUCK-TYPED ON `price_at`, NOT ON `isinstance`. price.py must not import - static_tariff.py - static_tariff.py imports THIS module, and a module-level - cycle would make the import order load-bearing. `taxes_profile()` below is - the one place the real parser is reached, and it imports it lazily. - - ⚠️ THE LOOKUP IS ON THE ~30 s PLANNING PATH through `assemble()`, so it - stays a linear scan over the two-to-four periods a Belgian tariff has - (`StaticProfile.at()`), with no parsing, no allocation and no I/O. A - malformed table never reaches here: validate.py refuses it at STARTUP. - """ + """The TAXES component at ONE instant. A scalar, or a period table (TARIFF-17). + Duck-typed on `price_at` (price.py must not import static_tariff.py). A linear + scan over two to four periods on the 30 s path; validate.py refuses a malformed + table at startup. DECISIONS/TARIFF-02-price-provider-plugins.md.""" at = getattr(taxes, "price_at", None) if at is None: return float(taxes) @@ -1542,24 +816,10 @@ def assemble(caps: ProviderCapabilities, raw, markup: float, taxes) -> list: """RawPoints -> canonical all-in PricePoints. THE ONLY PLACE MARKUP IS ADDED. - Adds exactly `caps.adds` - the complement of what the plugin declared its - source already contains. A plugin that publishes an all-in figure gets - nothing added; a raw EPEX figure gets both. - - ⚠️ `taxes` IS A SCALAR *OR* A PERIOD TABLE (TARIFF-17), AND IT IS EVALUATED - AT EACH POINT'S OWN INSTANT. Hoisting the lookup out of the loop - reading - it once and adding the same number to every point - is the mutation that - collapses the whole feature back into the LEVEL bucket: the assembled - series would still differ from the untaxed one by the right MAGNITUDE, and - the cheapest hour would be wrong. `test_price.py` section 24 is the check - that goes red for it, and it grades WHICH HOUR IS CHEAPEST rather than the - spread, because a spread-magnitude assertion passes on the hoisted version. - - ⚠️ THE SCALAR PATH IS UNCHANGED, ARITHMETICALLY AND NOT ONLY IN INTENT. - `taxes_at()` on a float returns that float at every instant, and - `0.0 + markup` is exactly `markup` in IEEE 754 for every finite markup, so - a site on a scalar gets the identical bytes it got before this landed. - """ + Adds exactly `caps.adds`, the complement of what the plugin declared. `taxes` + is evaluated at EACH point's own instant: hoisting it out of the loop collapses + the feature (test_price.py section 24). On a scalar the result is byte-identical + to before TARIFF-17.""" add_markup = float(markup) if Component.MARKUP in caps.adds else 0.0 wants_taxes = Component.TAXES in caps.adds out = [] @@ -1572,23 +832,13 @@ def assemble(caps: ProviderCapabilities, raw, markup: float, return out -# --------------------------------------------------------------------------- # # Transport - one place, HTTPS only, host checked against the declaration -# --------------------------------------------------------------------------- # async def https_get(url: str, timeout_s: float, headers=None) -> bytes: - """The default transport. §16: outbound is allow-listed and TLS'd. - - Imported lazily so `manual_csv` - the offline fallback - never needs aiohttp - to be importable at all. AC3 says it works fully offline; that includes a - machine where the HTTP stack is missing. - - PRICE-01. `headers` is None for every plugin but `price_service`, and with - None this is the code it has always been, byte for byte. With headers it - answers in the SAFE words of `status_words()` and never lets a library - error through: an error from a request that carried a token is the one - place a token could be printed by somebody else's code. - """ + """The default transport. 16: outbound is allow-listed and TLS'd. aiohttp is + imported lazily so `manual_csv` works offline. PRICE-01: with `headers` it + answers in the safe words of `status_words()` and never lets a library error + through (it could print a token).""" import aiohttp t = aiohttp.ClientTimeout(total=float(timeout_s)) if headers is None: @@ -1598,9 +848,8 @@ async def https_get(url: str, timeout_s: float, headers=None) -> bytes: return await r.read() try: async with aiohttp.ClientSession(timeout=t) as s: - # No redirects: the host is fixed in code, and a redirect would - # take the request (and the token) somewhere the allow-list never - # saw. A 3xx is reported like any other status. + # No redirects: the host is fixed in code and a redirect would take + # the token past the allow-list. async with s.get(url, headers=headers, allow_redirects=False) as r: if r.status != 200: @@ -1613,9 +862,8 @@ async def https_get(url: str, timeout_s: float, headers=None) -> bytes: f"the price service did not answer ({type(e).__name__})") -# PRICE-01. The one phrase a refused token is reported with. The fallback chain -# reads it to tell "the owner must act" from "the server is down", so it is a -# constant both sides import. +# PRICE-01. The phrase a refused token is reported with; the fallback chain +# reads it. AUTH_REFUSED = "the price service refused the token" @@ -1631,24 +879,9 @@ def status_words(status: int) -> str: def check_egress(url: str, caps: ProviderCapabilities) -> str: - """§16: a plugin may only reach a host it DECLARED. Enforced, not documented. - - Returns the host, so a caller that forgets to call this is not the same - shape as one that did. - - ⚠️ THE SCHEME IS SPLIT OFF BEFORE THE HOST IS PARSED, AND THE ORDER MATTERS - FOR THE TEST MORE THAN FOR THE CODE. The first version stripped a fixed - `len("https://")` from the front, which on an `http://` URL ate two - characters of the hostname instead - `http://api.energy-charts.info/p` - became the host `pi.energy-charts.info`, which then failed the ALLOW-LIST. - So a non-TLS feed was refused, the check named "plain http is refused" - passed, and deleting the TLS rule entirely left it passing. Review of - PR #16 found that by mutation: an assertion that does not test what it - names, inside a security check - this project's own recurring defect. Now - the scheme is parsed as a scheme, the host of an `http://` URL comes out - correct and DECLARED, and the TLS rule is the only thing that can refuse - it - which is what makes the check able to fail. - """ + """16: a plugin may only reach a host it DECLARED. Returns the host. The scheme + is split off before the host is parsed (a fixed-length strip once made the TLS + rule untestable; see DECISIONS/TARIFF-02-price-provider-plugins.md).""" if not url: raise SourceUnavailable("no feed URL configured (price_feed_url)") scheme, sep, rest = url.partition("://") @@ -1671,48 +904,14 @@ def check_egress(url: str, caps: ProviderCapabilities) -> str: return host -# --------------------------------------------------------------------------- # -# The shipped plugins. -# -# ⚠️ EACH DOCSTRING STATES WHICH COMPONENTS IT SOURCES AND WHICH THE FRAMEWORK -# ADDS, in the two fixed lines `SOURCES: ...` / `The framework ADDS: ...`. That -# is not decoration - it is the human-readable half of `capabilities().sources`, -# and test_price.py PARSES those lines out of the docstring and compares them -# with the declaration, so a plugin whose prose and whose code disagree about -# what its feed contains is a red check rather than a 3 c/kWh error nobody sees. -# --------------------------------------------------------------------------- # +# The shipped plugins. Each docstring has the two fixed lines `SOURCES: ...` / +# `The framework ADDS: ...`; test_price.py parses them and compares with +# `capabilities().sources`. -# TARIFF-08: the COLUMNAR document, which is what the public EPEX feed really -# publishes. -# -# ⚠️ THIS IS THE ONLY PARSER LEFT, AND THE SHAPE IT DOES **NOT** READ IS WHY. -# Until TARIFF-07 a second function, `_points_from_json()`, read a row-per-point -# document: `{"data": [{...}, {...}]}`. NO FEED ANYBODY REACHED EVER PUBLISHED -# THAT SHAPE - it was invented alongside the three withdrawn plugins, and the -# only bodies ever fed to it were the fixtures written from the same guess. It -# went with them. `https://api.energy-charts.info/price?bzn=BE` publishes -# PARALLEL ARRAYS: -# -# {"unix_seconds": [1788991200, 1788992100, ...], <- UTC epoch seconds -# "price": [216.08, 209.90, ...], <- one per instant -# "unit": "EUR / MWh", "license_info": "...", "deprecated": false} -# -# The row parser was pointed at that URL and asked for `doc["data"]`, so -# every fetch ended `SourceUnavailable: the feed carried no 'data' list` -> -# §11.4 -> static profile, arbitrage off, and nothing in the status page said -# why. Verified live 2026-09-10 against both fixtures under `scenarios/`. -# -# ⚠️ AND IT IS NOT ONE FUNCTION THAT SNIFFS FOR BOTH. A parser that decides -# which document it got is a parser that can decide wrong, and a wrong-but- -# parseable response then becomes a price. Two shapes, two functions, and each -# plugin names the one its own feed publishes. -# -# The columnar shape was already known in this repository: DATA-02's -# `scenarios/fetch_epex_be.py` has read `zip(d["unix_seconds"], d["price"])` -# since 2026-08-31. Only the SHIPPED path never learned it, because every -# fixture in the suite was built from the invented row shape - the parser was -# graded against its own assumption. See test_price.py section 2b. -# --------------------------------------------------------------------------- # +# TARIFF-08: the COLUMNAR document the public EPEX feed publishes (parallel +# `unix_seconds` / `price` arrays plus `unit`). The only parser; the row parser +# it replaced read a shape no feed published. Two shapes, two functions, no +# sniffing. DECISIONS/TARIFF-02-price-provider-plugins.md. COLUMNAR_TIME_KEY = "unix_seconds" COLUMNAR_PRICE_KEY = "price" @@ -1728,46 +927,10 @@ def _points_from_columnar_json(body, expect_unit, scale, provisional_default): """Parse a COLUMNAR price document: parallel `unix_seconds` / `price`. - Defects this refuses, each of which has a way of becoming a number rather - than an error: - - * ⚠️ UNEQUAL ARRAYS ARE `SourceUnavailable`, NOT A SHORT SERIES. `zip()` - stops at the shorter array and says nothing, so a `price` list that - arrived truncated would produce a REAL price series with its tail - missing - and the tail of a Belgian day is the evening peak. A series - that is missing the expensive hours is worse than no series: §11.4's - no-data row switches arbitrage OFF, whereas a truncated series is - traded on. Nothing downstream can tell the two apart, so the refusal - has to happen here. - * ⚠️ A UNIT THAT IS NOT THE DECLARED ONE IS REFUSED. `scale` is the - caller's standing assumption about the unit; if the feed ever - publishes EUR/kWh under the same keys, every price silently comes out - a THOUSAND times too small. `_validate_raw()` cannot catch that - - `MAX_PLAUSIBLE_EUR_KWH` only bounds prices that are too LARGE. - DATA-02's fetcher refuses on the same field for the same reason. - * ⚠️ `unix_seconds` ARE UTC INSTANTS, SAID SO EXPLICITLY. - `datetime.fromtimestamp(t, timezone.utc)`. Without that second - argument CPython returns a NAIVE datetime in the HOST's zone, and a - naive datetime is a wall clock reading plus an assumption about whose - clock it was. `brussels()` and `_validate_raw()` both refuse one, so - the mistake surfaces as a refused fetch rather than as a mispriced - day - but they refuse ALL naive datetimes and can say nothing about - whether this one was the right instant. That part is decided here. - - §9.3's rendering AT THE BRUSSELS OFFSET is `assemble()`'s, once, for - every plugin - `brussels(p.start)` there is the single line that does - it, and calling it here as well would be a second copy of a rule that - already has an owner, indistinguishable from a no-op under mutation. - A plugin's job is the right INSTANT; the framework's is the rendering. - - A `null` price is DROPPED and logged, not fatal, and that is a different - judgement from the one above: an unequal pair of arrays means the pairing - of instant to price is unreliable, whereas a null means the source stated - it published no price for that one instant. A dropped instant leaves a - hole, and `_Series.at()` already answers a hole with `Absence` - §11.4 for - that hour. `scenarios/fetch_epex_be.py` dropped 0 nulls over the 12 months - to 2026-08-31, so this is a shape the source allows rather than one it uses. - """ + Refuses what could become a number: unequal arrays (a truncated evening peak), + a unit other than the declared one (a factor of 1000), and anything but UTC + instants. A `null` price is dropped and logged. The Brussels rendering is + `assemble()`'s. DECISIONS/TARIFF-02-price-provider-plugins.md.""" try: text = body.decode("utf-8") if isinstance(body, (bytes, bytearray)) \ else str(body) @@ -1798,9 +961,8 @@ def _points_from_columnar_json(body, expect_unit, scale, f"the feed reports {doc.get(COLUMNAR_UNIT_KEY)!r}, not " f"{expect_unit!r}, so its prices cannot be converted") if doc.get("deprecated"): - # Not a refusal: today's prices are still prices. But §11.4 makes a - # dead feed look exactly like a quiet one, so the warning that the - # source is going away has to be emitted while it still works. + # Not a refusal, but 11.4 makes a dead feed look quiet, so warn while + # the source still works. _LOG.warning("The price feed says it will stop (deprecated=%r), " "though it still answers for now.", doc.get("deprecated")) out = [] @@ -1836,38 +998,10 @@ def _points_from_columnar_json(body, expect_unit, scale, def dayahead_feed_url(plugin_id, opts, since, until, base, bzn) -> str: """The day-ahead URL for a window, or the operator's override, verbatim. - ⚠️ SHARED BY `generic_dayahead` AND `custom_formula`, ON PURPOSE - (TARIFF-15). They read the SAME feed for the same range; two copies of this - rule would be two places for the Brussels-calendar derivation and the - override decision to disagree, and the one that is wrong would be wrong - silently - §11.4 turns a short horizon into provisional hours, not into an - error. It takes `plugin_id` rather than reading one, so the framework's - AC2 rule (no plugin id in shared code) still holds here. - - ⚠️ AN OVERRIDE IS THE EXACT URL AND NOTHING IS APPENDED TO IT, WHICH COSTS - THE OVERRIDING SITE THE RANGE. That is a decision, not an oversight, and - the sentence below is why it is not a footnote: - - * `price_feed_url` says "override the plugin's default feed", and an - override that is silently rewritten is not one. Merging parameters - into an operator's query string means parsing and re-serialising a - URL they wrote - which reorders, de-duplicates and re-escapes it, - and can hand the server a request nobody typed. - * `start`/`end` are ENERGY-CHARTS' spelling, and both providers DECLARE - A SECOND HOST: `web-api.tp.entsoe.eu`. Nothing here has verified that - host's day-ahead parameters and nothing here should guess them - but - they are not `start`/`end`, so an override to the other DECLARED host - is exactly the case where appending our vocabulary adds two parameters - that host ignores and yields the short horizon anyway, with nothing - saying so. - * and blind appending is refused outright: `dated_feed_url()` will not - add a query string to a base that has one. - - So the cost is real and it is REPORTED, at fetch time, on the shipped - logger, naming the horizon consequence rather than the substitution. - §11.4 is what makes a price defect quiet; an operator who has taken the - short horizon has to be able to find out from the log that they did. - """ + Shared by `generic_dayahead` and `custom_formula` (TARIFF-15); takes + `plugin_id` as an argument (AC2). An override is used exactly as given and so + loses the range; that is logged at fetch time with the horizon consequence. + DECISIONS/TARIFF-02-price-provider-plugins.md.""" override = str(opts.get("price_feed_url") or "").strip() if override: _LOG.warning( @@ -1894,25 +1028,9 @@ class GenericDayAhead(PriceProvider): SOURCES: energy The framework ADDS: markup, taxes - This is the plugin the double-counting trap is about. The published figure - is the wholesale clearing price and nothing else, so both user-entered - components have to be added to it - and because this plugin cannot read - either number, it cannot be the one that adds them twice. - - ⚠️ IT READS THE COLUMNAR SHAPE, AND SAYS SO IN ITS OWN `fetch()`. Until - TARIFF-08 it called the row-per-point parser (withdrawn by TARIFF-07) and - asked - `api.energy-charts.info` for a `"data"` list it has never published, so - every fetch this provider ever made ended in `SourceUnavailable` and §11.4 - turned that into a silent fall back to the static profile. See - `_points_from_columnar_json()` for the shape and for why there are two - parsers rather than one that guesses. - - ⚠️ THE FEED IS QUARTER-HOURLY SINCE BELGIUM'S 15-MINUTE MTU (2025-10-01): - 96 points a day at 900 s, not 24 at 3600 s. Nothing here assumes either - - the instants come from the document - and both granularities are replayed - from real captured bodies in test_price.py section 2b. - """ + The published figure is the wholesale clearing price, so both user-entered + components are added by the framework. Quarter-hourly since 2025-10-01; nothing + here assumes a granularity.""" id = "generic_dayahead" @@ -1927,22 +1045,8 @@ class GenericDayAhead(PriceProvider): BZN = "BE" async def fetch(self, opts, since, until, transport): - """Request the range the CALLER asked for, on the Brussels calendar. - - ⚠️ THE ARGUMENTS ARE USED, and `dayahead_feed_url()` above is where - they are turned into a URL. Until TARIFF-09 this method took `since` - and `until` and asked for the bare `?bzn=BE`, which answers ONE - Brussels day - so a 48 h horizon was 15 firm hours and 33 carried at - 09:00, and 1 firm and 47 carried at 23:00, with the whole trade set - inside a single calendar day at every hour of every day. - - ⚠️ THE URL DECISION MOVED OUT OF THIS METHOD IN TARIFF-15 AND DID NOT - CHANGE. `custom_formula` reads the same feed with the same range and - the same parser, and a second copy of the override rule would be a - second place for it to be wrong - which is the reason `covers()` - delegates to `at()` rather than restating its window. The reasoning, - the warning and the exact URL are all in `dayahead_feed_url()`. - """ + """Request the range the CALLER asked for, on the Brussels calendar + (TARIFF-09; `dayahead_feed_url()` builds the URL).""" url = dayahead_feed_url(self.id, opts, since, until, self.FEED_BASE, self.BZN) check_egress(url, self.capabilities()) @@ -1959,18 +1063,10 @@ class PriceService(PriceProvider): SOURCES: energy The framework ADDS: markup, taxes - The server answers the energy-charts layout (`unix_seconds`, `price`, - `unit`) plus `covers`, the complete Brussels days it holds. The price is - the wholesale energy price and nothing else, so markup and taxes are added - by the framework exactly as for `generic_dayahead`. - - The host is fixed here, HTTPS only, and a bearer token (the option - `price_service_token`) goes in the `Authorization` header. The token is not - put into any message this class builds. - - `FALLBACKS` names the next source down. `app/price_chain.py` walks it and - then the stored prices; this class only fetches. - """ + Answers the energy-charts layout plus `covers`, the complete Brussels days it + holds. The host is fixed here, HTTPS only; the bearer token (option + `price_service_token`) goes in the `Authorization` header and into no message. + `FALLBACKS` names the next source; `app/price_chain.py` walks it.""" id = "price_service" @@ -2018,11 +1114,7 @@ class PriceService(PriceProvider): def _covered_days(body): - """The `covers` list of a price service answer as a set, or None if absent. - - A day the server does not list as complete is not used: an hour missing - inside a day is a hole, and a hole is not a free hour. - """ + """The `covers` list of a price service answer as a set, or None. A day not listed as complete is not used.""" try: text = body.decode("utf-8") if isinstance(body, (bytes, bytearray)) \ else str(body) @@ -2044,22 +1136,13 @@ class ManualCsv(PriceProvider): SOURCES: energy, markup, taxes The framework ADDS: nothing - ⚠️ THE CSV IS ALL-IN, AND DOCS.md says so. It has to be: this plugin has no - way to know what the numbers a human typed already contain, and guessing - would be the double-counting defect with a friendlier face. `load_prices()` - warns when markup/taxes are configured non-zero under a plugin that sources - them, so the ignored numbers are visible rather than silently discarded. - - ⚠️ NO TRANSPORT, NO aiohttp, NO NETWORK, NO tz DATABASE. It is the escape - hatch for a site with no internet and the ingest path the TEST-01 replay - harness will share, so it must not acquire a dependency on anything that - needs one. `transport` is accepted and deliberately unused. + The CSV is all-in (the plugin cannot know what a typed number contains), and + DOCS.md says so. No transport, aiohttp, network or tz database. Format - header required, two columns or three: start,eur_per_kwh,provisional - 2026-01-15T00:00:00+01:00,0.1834,false - """ + 2026-01-15T00:00:00+01:00,0.1834,false""" id = "manual_csv" @@ -2113,35 +1196,10 @@ class CustomFormula(PriceProvider): SOURCES: energy, markup The framework ADDS: taxes - ⚠️ IT DECLARES `{ENERGY, MARKUP}` AND NOT `ALL_IN`, AND THAT IS THE WHOLE - SAFETY ARGUMENT. A Belgian card's `Tariefformule` line is the ENERGY - component only - Eneco tabulates network tariffs and levies separately, - ENGIE states that the formula excludes network, distribution, transport, - grid levies and excise - so declaring ALL_IN would lie on behalf of every - operator who typed exactly what their card says, and would silently drop - ~12 c EUR/kWh from every hour. The framework adds `price_taxes_eur_kwh` - on top, once, and this class cannot see or apply it (trap 1). - - The risk that introduces is one no guard in this file can see, because it - is not in code: THE OPERATOR folding their network charges into `offset` - while the tax option carries them too. `app/custom_formula.py`'s card - self-check is what refuses that - the card's printed estimate is the - energy component alone, so an inflated offset moves 15.55 to ~27 and the - add-on will not start. Its limit is stated in the same breath there: it - grades the coefficients against the card, NOT the card against reality. - - ⚠️ NO EXPRESSION EVALUATION OF ANY KIND, AND THE COEFFICIENTS ARE NUMBERS - BECAUSE THE PARSER REFUSES ANYTHING ELSE. This string is settable over - HTTP through the add-on's own ingress config page, so `eval` on it would - be RCE in a root container holding SUPERVISOR_TOKEN - and, the deciding - argument, an arbitrary expression could not be graded against the card at - all. See app/custom_formula.py. - - ⚠️ THE INDEX PATH IS `generic_dayahead`'s, NOT A SECOND ONE. Same feed, - same `dayahead_feed_url()` request on the Brussels calendar, same columnar - parser, same declared egress hosts. A second parser would be a second - failure mode for a document that already has one that works. - """ + It declares `{ENERGY, MARKUP}`, not `ALL_IN`: a card's formula is the energy + component only. No expression evaluation: the coefficients are numbers because + the parser refuses anything else (the string is settable over HTTP). The index + path is `generic_dayahead`'s. DECISIONS/TARIFF-02-price-provider-plugins.md.""" id = "custom_formula" @@ -2160,11 +1218,8 @@ class CustomFormula(PriceProvider): egress_hosts=("api.energy-charts.info", "web-api.tp.entsoe.eu")) async def fetch(self, opts, since, until, transport): - """Fetch the index, apply the card's arithmetic, return RawPoints. - - ⚠️ `since`/`until` ARE USED, through the shared URL builder - see - `dayahead_feed_url()` and TARIFF-09 for what ignoring them costs. - """ + """Fetch the index, apply the card's arithmetic, return RawPoints. `since`/`until` + are used through the shared URL builder (TARIFF-09).""" try: spec = parse_price_formula(opts.get("price_formula")) except FormulaInvalid as e: @@ -2178,15 +1233,9 @@ class CustomFormula(PriceProvider): self.FEED_BASE, self.BZN) check_egress(url, self.capabilities()) body = await transport(url, float(opts.get("price_timeout_s"))) - # ⚠️ `scale=1.0`, SO THESE RawPoints CARRY EUR/**MWh** IN A FIELD - # SPELLED `eur_per_kwh`. That is a trap and it is contained on purpose: - # the list is destructured on the very next line and never leaves this - # method, so nothing downstream can read the wrong unit off it. The - # alternative - scaling to EUR/kWh and multiplying by 1000 again to - # feed the formula - is two conversions where the contract needs none, - # and every extra conversion here is another place to lose a factor of - # 1000. If this list ever DID escape, `_validate_raw()` would refuse it - # at `MAX_PLAUSIBLE_EUR_KWH` rather than price a day on it. + # `scale=1.0`, so these RawPoints carry EUR/**MWh** in a field spelled + # `eur_per_kwh`. Contained: destructured on the next line and never + # leaves this method (`_validate_raw()` would refuse an escape). index = _points_from_columnar_json(body, expect_unit="EUR / MWh", scale=1.0, provisional_default=False) @@ -2195,22 +1244,14 @@ class CustomFormula(PriceProvider): self.spec = spec self.index_by_instant = {t.astimezone(timezone.utc): v for t, v in pairs} - # ⚠️ `provisional=False` FOR THE SAME REASON `generic_dayahead` SAYS - # SO: the day-ahead auction is settled once published, and the columnar - # document carries no per-point flag to read one from. FCST-04's - # carry-forward is what marks the hours nobody published yet. + # `provisional=False` as for `generic_dayahead`: the auction is settled + # once published. return [RawPoint(start=t, eur_per_kwh=price_eur_kwh(spec, v), provisional=False) for t, v in pairs] def startup_preview(self, points) -> list: - """`index EUR/MWh -> all-in EUR/kWh` for the first 24 h, once. - - ⚠️ THE INDEX COLUMN IS WHY THIS HOOK EXISTS. The all-in price alone - says nothing about which of the two numbers is wrong; printed beside - the index it came from, a factor of 1000 is one glance. See - `preview_lines()` for the rows and the min/median/max/spread line. - """ + """`index EUR/MWh -> all-in EUR/kWh` for the first 24 h, once. The index beside the price makes a factor of 1000 obvious.""" if self.spec is None or not points: return [] cut = points[0].start + timedelta(hours=PREVIEW_HOURS) @@ -2225,24 +1266,16 @@ class CustomFormula(PriceProvider): return preview_lines(self.spec, rows) -# --------------------------------------------------------------------------- # # The one entry point -# --------------------------------------------------------------------------- # -# Provider ids whose startup preview has already been emitted in this process. -# ⚠️ NOT A PLUGIN NAME IN THE FRAMEWORK: it is filled at runtime from whatever -# id was resolved, and starts empty. AC2 is intact. +# Provider ids whose startup preview was already emitted in this process; +# filled at runtime. _PREVIEWED: set = set() def _emit_startup_preview(plugin, pid: str, points) -> None: - """Log a plugin's preview once per process, and never fail because of it. - - ⚠️ THE ID IS RECORDED BEFORE THE HOOK RUNS, DELIBERATELY. A preview that - raises must not be retried every 30 minutes for the life of the add-on - - that would turn a cosmetic defect into a log flood on the one path §11.4 - already makes hard to read. - """ + """Log a plugin's preview once per process, and never fail because of it. The id + is recorded before the hook runs, so a raising preview is not retried every 30 minutes.""" if pid in _PREVIEWED: return _PREVIEWED.add(pid) @@ -2262,15 +1295,9 @@ async def load_prices(opts: dict, since: datetime, until: datetime, transport=None) -> PriceRead: """Resolve the configured provider, fetch, assemble all-in, return a read. - ⚠️ RETURNS, NEVER RAISES, for every failure a site can actually have. §11.4 - makes "no price data" a degradation row, and a degradation row that arrives - as a traceback into the 30 s planning tick is a controller that stops - regulating a battery because a supplier's TLS certificate expired. - - The one exception is `FrameworkOption`, which is re-raised: that is not a - site condition, it is a plugin trying to read the markup, and turning it - into "no price data" would hide the exact defect trap 1 exists to catch. - """ + RETURNS, NEVER RAISES, for any failure a site can have (11.4 is a degradation + row, not a traceback into the 30 s tick). `FrameworkOption` is re-raised: it is + a plugin reading the markup, not a site condition.""" pid = provider_id(opts) if not provider_configured(opts): return PriceRead.no_data( @@ -2285,18 +1312,14 @@ async def load_prices(opts: dict, since: datetime, until: datetime, caps = cls.capabilities() markup = float(price_option(opts, "price_markup_eur_kwh")) - # ⚠️ TARIFF-17: THE TABLE WINS OVER THE SCALAR, AND validate.py REFUSES THE - # CASE WHERE BOTH SAY SOMETHING. Two writers of one quantity, one of which - # silently loses, is the shape validate.py exists to refuse - so the - # precedence here is only ever exercised on a configuration that has - # already been accepted, and it is spelled out rather than left implicit. + # TARIFF-17: the table wins over the scalar; validate.py refuses configs + # where both speak. taxes_key = "price_taxes_eur_kwh" try: table = taxes_profile(opts) except Exception as e: # noqa: BLE001 - # Not silent, and not a traceback into the 30 s tick. validate.py - # should have refused this at startup; if the add-on is running with it - # anyway, the operator is told which key and what is wrong with it. + # Not silent: validate.py should have refused this at startup, so say + # which key is wrong. _LOG.error("price_taxes_profile cannot be used (%s: %s), so no taxes " "are added to this price update. Fix the table or clear it.", type(e).__name__, e) @@ -2311,9 +1334,8 @@ async def load_prices(opts: dict, since: datetime, until: datetime, else: taxes = float(price_option(opts, "price_taxes_eur_kwh")) - # ⚠️ Not a refusal, but not silent either. A plugin whose feed is already - # all-in ignores these two numbers, and an operator who typed them has every - # reason to believe they took effect. + # Not a refusal, but not silent: a plugin whose feed is already all-in + # ignores these two numbers. ignored = [n for c, n, v in ((Component.MARKUP, "price_markup_eur_kwh", markup), (Component.TAXES, taxes_key, taxes)) if c not in caps.adds and v] diff --git a/goodwe_controller/app/price_chain.py b/goodwe_controller/app/price_chain.py index bdf93e2..57b1d41 100644 --- a/goodwe_controller/app/price_chain.py +++ b/goodwe_controller/app/price_chain.py @@ -1,27 +1,15 @@ """PRICE-01 - where the prices come from when the first source does not answer. -A provider that names `FALLBACKS` (app/price.py) is read through this module -instead of straight from `load_prices()`. The order is fixed: +A provider that names `FALLBACKS` (app/price.py) is read through here: first the +provider (the owner's price server), then each id in `FALLBACKS`, then the +prices the history store kept from the last fetch that worked. A step counts +only if its answer holds a price for NOW (a server answering 200 with +yesterday's prices is as useless as one that is down). Other providers never +come here (`uses_chain()` is False). - 1. the provider itself (the owner's price server) - 2. each id in its `FALLBACKS` (energy-charts.info) - 3. the prices the history store kept from the last fetch that worked - -A step counts only if its answer holds a price for NOW. A server that answers -200 with yesterday's prices is as useless to a battery as one that is down, so -it is skipped, with the same log line. - -Every other provider never comes here (`uses_chain()` is False for it), so -`generic_dayahead`, `manual_csv`, `custom_formula` and `off` take the road they -always took. - -THE GATE. `gate_for()` is the maintenance charge rule for a site on such a -provider: the current 15-minute price is at or below today's average. It is the -rule the Home Assistant pair (`price_now_entity`, `price_avg_entity`) gave, read -from the provider's own prices, so moving a site from the pair to the provider -does not change WHEN the charge runs. DECISIONS/PRICE-01-own-price-server.md -says how equal, and where it is not. -""" +`gate_for()` is the maintenance charge rule for such a site: the current +15-minute price is at or below today's average, as the Home Assistant pair gave. +DECISIONS/PRICE-01-own-price-server.md says how equal, and where it is not.""" import logging from dataclasses import dataclass @@ -74,13 +62,8 @@ class Resolved: class PriceWatch: - """Which step the prices come from, and one log line per change of it. - - The line is written when the step CHANGES and not before. Nothing repeats - while a source stays down, and nothing is written for a refresh that - changes nothing. The reasons (a status code, a timeout) go on a DEBUG line - beside it, with the token taken out. - """ + """Which step the prices come from, and one log line per change of it (nothing + repeats while a source stays down). Reasons go on a DEBUG line, token removed.""" def __init__(self): self.step = None @@ -123,20 +106,14 @@ async def resolve(opts: dict, since: datetime, until: datetime, transport, history, stamp: datetime, watch: PriceWatch, load) -> Resolved: """Walk the chain and return the first step that has a price for `stamp`. - - `load` is the caller's `load_prices`, passed in so that `static_tariff.py` - stays the one module that fetches prices. Never raises - for a source condition: a read that gets nowhere is a `no_data` whose - reason names every step, so the §11.4 row applies exactly as for a single - provider that failed. - """ + `load` is the caller's `load_prices`, so static_tariff.py stays the one module + that fetches. Never raises for a source condition: `no_data` names every step.""" pid = provider_id(opts) cls = PROVIDERS[pid] reasons = [] for step, sid in enumerate((pid,) + tuple(cls.FALLBACKS)): - # A fallback is asked for ITS OWN feed. The operator's feed override is - # for the first provider and would send energy-charts' request to a - # URL it was never meant for. + # A fallback is asked for ITS OWN feed; the operator's override is for + # the first provider. o = opts if step == 0 else dict(opts, price_provider=sid, price_feed_url="") read = await load(o, since, until, transport) @@ -158,8 +135,8 @@ async def resolve(opts: dict, since: datetime, until: datetime, transport, series=lambda r: "price_history: it has no price for now", no_data=lambda r: r.reason)) watch.note(STEP_NONE, reasons, opts) - # The reason is short on purpose: it is printed on the status page and in - # a WARNING every refresh. What each step said is on the DEBUG line. + # Short on purpose: it is on the status page and in a WARNING every + # refresh. return Resolved(PriceRead.no_data( "no price source has a price for now"), STEP_NONE, tuple(scrub(r, opts) for r in reasons)) @@ -171,13 +148,9 @@ async def resolve(opts: dict, since: datetime, until: datetime, transport, @dataclass(frozen=True) class Gate: - """What the 30 s tick takes from the standing plan. - - `covered` is whether a published price covers now. When it is False, - `cheap` is False and the charge waits: no price is not a cheap price. - `since` is when the cover ended, for the status line, or None when nothing - was ever published. - """ + """What the 30 s tick takes from the standing plan. `covered` False means `cheap` + is False and the charge waits (no price is not a cheap price). `since` is when + the cover ended, or None.""" cheap: bool covered: bool @@ -189,24 +162,16 @@ NO_PLAN = Gate(False, False, None, "No prices yet") def _round5(value: float) -> float: - # The Home Assistant template rounded the average to 5 decimals, and a - # price to 5 as well. Both sides are compared as it did, so a quarter that - # sits within 0.00001 EUR/kWh of the average gets the same answer. + # The Home Assistant template rounded both sides to 5 decimals; so do we, + # for the same answer. return round(float(value), 5) def gate_for(published, now: datetime) -> Gate: - """The charge gate for `now`, from the series the chain published. - - THE RULE (owner's decision, PRICE-01): the current 15-minute price is at or - below the average of today's prices, today being the Brussels day. This is - what `now_p <= avg_p` was on the Home Assistant pair. It is NOT - `TariffPlan.cheap_now()`, which is strictly below the mean of the firm - hours of the whole plan, and it is not meant to be. - - `published` is a series or None. With no price for now there is no gate - answer: the charge waits. - """ + """The charge gate for `now`, from the series the chain published: the current + 15-minute price is at or below the average of today's (Brussels day) prices, + as `now_p <= avg_p` was on the HA pair (owner, PRICE-01). Not + `TariffPlan.cheap_now()`. With no price for now the charge waits.""" if published is None: return NO_PLAN if not published.covers(now): diff --git a/goodwe_controller/app/price_forecast.py b/goodwe_controller/app/price_forecast.py index 916a8fc..85911cc 100644 --- a/goodwe_controller/app/price_forecast.py +++ b/goodwe_controller/app/price_forecast.py @@ -1,163 +1,16 @@ """FCST-03 - the hours the feed has not published yet, and what may not rest on them. -WHERE THIS LIVES. specs.md §2.2 puts price ingest under `ems/ingest/prices/`. -That layout does not exist and is not being built; the product owner's decision -of 2026-08-30 is that everything ships in `goodwe_controller/app/`. The -BEHAVIOUR §8.3 describes is unchanged and still authoritative - only the path -moved. Read `ems/...` as `app/...`. - -§8.3, in full, because every line below is one clause of it: - - "Day-ahead prices arrive once daily. Until they do, unknown hours use - yesterday's profile and are flagged `provisional: true` - and NO - IRREVERSIBLE ACTION MAY BE TAKEN ON A PROVISIONAL PRICE. Real-time / - imbalance products are treated as high-volatility: arbitrage thresholds - widen." - -Three separate things, and this module owns the first two. The third - the -widening arithmetic - lives in price.py beside the product enum it is derived -from (`Uncertainty`, `widen_factor`, `required_spread`), and is spent by -`static_tariff.TariffPlan.arbitrage()`, because that is where the verdict this -repo already refuses to hand out casually is produced. FCST-03 added a refusal -to that existing ladder rather than a second mechanism beside it. - - -THE DEFECT THIS MODULE IS SHAPED BY: A COPY THAT KEEPS THE WRONG FLAG -===================================================================== -A point filled from yesterday and a point published this morning are the SAME -SHAPE. Three floats and a bool; only the bool tells them apart. And the natural -way to write a carry-forward fill is: - - PricePoint(start=t, eur_per_kwh=y.eur_per_kwh, provisional=y.provisional) - ^^^^^^^^^^^^^ - -which copies yesterday's `False` onto today's guess and produces exactly the -thing the ticket exists to prevent: STALE DATA PRESENTED AS CURRENT. It reads -fine, it type-checks, it round-trips through §9.3's JSON, and every price on -the status page looks like a price. This is the meter-side SAFETY-18 defect one -layer up: the reading arrives on time, looks fresh, and is wrong - and a -freshness check that only asks "did something arrive" cannot see it. - -So the flag is not passed through anywhere in this file: - - * A CARRIED POINT IS BUILT BY `_carried()`, WHICH HAS NO PARAMETER FOR THE - FLAG. `provisional=True` is a literal in one place. There is no argument a - caller can pass, no keyword to forget, and no variable that could hold - yesterday's value at the moment the point is constructed. - * A PUBLISHED POINT IS NEVER COPIED AT ALL. It is passed through BY IDENTITY - - the object the provider produced is the object in the output series - so - there is no copy that could lose or gain a flag. `filled.points[i] is - fetched.points[j]` is a checkable property, and test_price_forecast.py - checks it. EVERY published point in the window is passed through, not one - per hour: see the sub-hourly note below. - * `_audit()` RE-DERIVES the expected flag for every output point from the set - of hours the feed actually published, and RAISES if any point disagrees. - It is a post-condition, not a filter: it never fixes a flag, because a - module that quietly corrects its own provenance is one whose provenance - cannot be trusted. - - ⚠️ IT GRADES THE POINTS THAT ARE EMITTED AND NOTHING ABOUT THE ONES THAT - ARE NOT (FCST-05). `_audit()` is a provenance check, NOT a completeness - check: fed FCST-04's own decimation defect - four published points in an - hour, only the last emitted - it ACCEPTS, and it accepts an empty output - too. Measured, and the FCST-03 audit it replaced was equally blind, so - this is a limit rather than a regression. What actually guards against a - decimated series is the point COUNT and the pass-through IDENTITY checks - in `test_price_forecast.py` (section on the quarter-hourly feed) and - `test_price_history.py`; see `_audit()`'s own docstring for why the - completeness rule is not expressible from its two arguments. - * and test_price_forecast.py walks THIS FILE'S AST and fails if any - `PricePoint(...)` in it passes anything but a literal for `provisional`, so - the pass-through idiom cannot come back in a later edit and pass review by - looking ordinary. - -⚠️ "PUBLISHED" DOES NOT MEAN "FIRM". §9.3's canonical format carries -`provisional` PER POINT, not per series, precisely because a real-time / -imbalance product's current-interval print is settled after the fact and is -genuinely provisional while it is published. Such a point is published AND -provisional, and the audit's rule for a published point is "the flag is -whatever the feed said", never "False". A provenance check that forced -published points to False would be a bug that looks like a guarantee. - -⚠️ NO SHIPPED PLUGIN EMITS ONE TODAY (TARIFF-07 withdrew the only declarer of -`PriceProduct.REALTIME`), and that is exactly why this paragraph stays. The -rule is a property of the FORMAT, not of any supplier; weakening it to -"published implies False" while nothing exercises it is how the first real -imbalance feed would arrive mispriced and flagged firm. - - -THE FILL, AND WHY IT KEYS ON THE WALL CLOCK -=========================================== -For every whole hour in the requested horizon that the feed has not published, -take yesterday's value for THE SAME BRUSSELS WALL-CLOCK HOUR - not `t - 86400 -s`. On the two days a year that matter those are different instants, and the -difference is not academic: a 23-hour day has no 02:00 at all and a 25-hour day -has two, so a fill that subtracts a fixed 24 h shifts every carried hour by one -across a transition and misprices the whole day in a way nothing reports. - -On the 25-hour day the two 02:00 hours - `02:00+02:00` and `02:00+01:00` - both -key to local hour 2 and are BOTH filled from yesterday's single 02:00 value. -That is deliberate: they are both guesses, they are both flagged, and nothing -irreversible may rest on either. If yesterday itself was a 25-hour day, the -first of its two 02:00 points is the one used, deterministically. - -An hour with no published price AND no yesterday value is LEFT OUT of the -series. It is not filled with a neighbour, a mean, or a zero. §11.4 applies to -that hour - `_Series.at()` raises `Absence` for it - and the caller that wanted -a tariff for the whole horizon falls back to the static profile (TARIFF-03), -which is the row the spec actually writes for "no price data". If NOTHING can -be produced, the result is `PriceRead.no_data(reason)` and not an empty series, -because an empty series is not constructible on purpose (price.py trap 3). - -⚠️ A SUB-HOURLY FEED IS NOT DECIMATED TO ONE POINT AN HOUR, and the hourly walk -below is about WHICH HOURS NEED FILLING - never about what a published hour is -allowed to contain. BELGIAN DAY-AHEAD ITSELF SETTLES PER QUARTER since the -15-minute MTU took effect on 2025-10-01 - `generic_dayahead` reads 96 points a -day, not 24 - `_validate_raw()` accepts any strictly-increasing spacing, and a -published hour may therefore hold four points. So EVERY published point inside -the window is passed through, in order, and a carried point is added only for a -whole hour with NO published coverage at all. - -The alternative was measured rather than reasoned about, because FCST-04's -first caller shipped it: keying the published set as `{_utc_hour(p.start): p -for p in r.points}` reads perfectly ordinary and keeps the LAST point of each -hour, dropping the other three. On a 96-point quarter-hourly feed - all firm, -fully published, nothing to carry - the series came out 15 points long, the -arbitrage spread collapsed from 0.143 to 0.014 EUR/kWh, and the cheapest -"hour" became a `:45` point whose `at()` window runs into the next hour. A feed -losing 84 % of its own prints is not a windowing effect and is not something a -provisional flag would ever have reported. - - -AC2: THE IRREVERSIBLE-ACTION GATE -================================= -"The distinction should be an explicit property of the action, not a judgement -call at each call site" - so an irreversible action is a value, `Irreversible`, -carrying the number of hours it COMMITS. `price_permits()` looks at exactly -those hours and answers with a `Gate` variant that carries the evidence on the -permitting arm and a reason on the blocking one, the `_Outcome` shape price.py -borrowed from `sim/riglock.py`. - -The shipped example is the maintenance drain (§12): starting one commits hours -of forced discharge that cannot be undone if the price turns out to have been -yesterday's. Its commitment window is DERIVED from `maintenance.DRAIN_TIMEOUT_MIN` -rather than typed here, so the gate and the phase machine cannot drift apart. - -⚠️ WHAT DOES NOT GO THROUGH THIS GATE, AND WHY THAT MATTERS MORE THAN WHAT -DOES. The maintenance cycle is a CALENDAR obligation (§12.2: a cycle that -silently never fires is indistinguishable from one that is not due yet), and a -site with `price_provider: off` - the shipped default - has no price series at -all. Routing every maintenance start through a gate that blocks on `no_data` -would stop the monthly cycle on every default install, permanently and -silently: the exact failure §12 opens by naming. So `begin_phase()` is the -entry point for a start that is being JUSTIFIED BY A PRICE, main.py's existing -calendar path is untouched by this ticket, and OPT-04/MAINT-05 own the decision -of which one a site runs. Nothing here disables anything that runs today. - -Reversible actions - ordinary self-consumption modulation, the loop that -actually holds the grid at zero - are not gated at all and cannot be: §6 says -absent machinery must not disable the loop. -""" +specs.md 8.3: unknown hours use yesterday's profile and are flagged +`provisional: true`, and no irreversible action may be taken on a provisional +price. This module owns the fill and the gate; the widening arithmetic is in +price.py. The flag is never passed through: a carried point is built only by +`_carried()` (no flag parameter), a published point is passed by identity, and +`_audit()` raises on a mismatch. The fill keys on the Brussels wall-clock hour, +and a sub-hourly feed is never decimated. An hour with neither a published +price nor a yesterday value is left out, never filled (`at()` raises `Absence`, +11.4 applies). `begin_phase()` is the price-justified entry point only; the +calendar path is untouched. All of it: +DECISIONS/FCST-03-the-hours-not-yet-published.md.""" import logging from dataclasses import dataclass @@ -172,13 +25,8 @@ _LOG = logging.getLogger("goodwe.price.forecast") class Provenance(Enum): - """Where a point in a filled series came from. INTERNAL - §9.3's wire format - carries the boolean and nothing else, so this never leaves the module. - - It exists so `_audit()` has a vocabulary to state its post-condition in, and - so the two cases have names in the log rather than being "the ones with the - flag" and "the others". - """ + """Where a point in a filled series came from. INTERNAL: the wire format carries + only the boolean. It gives `_audit()` and the log names for the two cases.""" PUBLISHED = "published" # the provider printed this hour. Flag: the # feed's own - a real-time print may be # published AND provisional (§9.2). @@ -186,9 +34,7 @@ class Provenance(Enum): # Flag: ALWAYS True, by construction. -# --------------------------------------------------------------------------- # # The fill -# --------------------------------------------------------------------------- # def _utc_hour(instant: datetime) -> datetime: u = instant.astimezone(timezone.utc) @@ -196,16 +42,9 @@ def _utc_hour(instant: datetime) -> datetime: def _published_by_hour(read: PriceRead) -> dict: - """UTC hour -> the provider's OWN points that start inside it, in order. - - ⚠️ A TUPLE PER HOUR, NOT A POINT, AND THAT IS THE WHOLE OF THE SUB-HOURLY - RULE. `{_utc_hour(p.start): p for p in r.points}` reads identically, is one - character shorter, and silently keeps one point per hour - the last one - written. The mapping answers two DIFFERENT questions here: "does this hour - need filling at all?" is `u in published`, and "what does the output carry - for it?" is all of `published[u]`. Collapsing them is how a quarter-hourly - feed loses three quarters of its prints. - """ + """UTC hour -> the provider's OWN points that start inside it, in order. A tuple + per hour, not a point: a dict of `{hour: p}` silently keeps the last one and + decimates a quarter-hourly feed.""" out: dict = {} read.match( series=lambda r: [out.setdefault(_utc_hour(p.start), []).append(p) @@ -215,13 +54,8 @@ def _published_by_hour(read: PriceRead) -> dict: def _carried(start_utc: datetime, eur_per_kwh: float) -> PricePoint: - """A point filled from yesterday's profile. §8.3: ALWAYS provisional. - - ⚠️ THERE IS NO `provisional` PARAMETER, AND THAT IS THE MECHANISM. This is - the only constructor of a carried point, the flag is a literal, and no - caller - including a future one editing in a hurry - has anything to pass, - default, or copy from the point being carried. - """ + """A point filled from yesterday's profile. 8.3: ALWAYS provisional. There is no + `provisional` parameter: the flag is a literal, nothing to pass or copy.""" return PricePoint(start=brussels(start_utc), eur_per_kwh=round(float(eur_per_kwh), 6), provisional=True) @@ -230,88 +64,15 @@ def _carried(start_utc: datetime, eur_per_kwh: float) -> PricePoint: def _audit(points, published) -> None: """Post-condition: every flag matches the point's TRUE provenance (AC1). - `published` maps a UTC hour to the TUPLE of points the provider produced - for it (`_published_by_hour()`). An hour ABSENT from that mapping is an hour - the feed did not print, and a carried point is the only thing that may - stand in it. + `published` maps a UTC hour to the TUPLE of points the provider produced for it. + PUBLISHED: the point must be one of `published[u]` by identity, emitted once, + flag whatever the feed said (never forced to False). CARRIED: the hour must not + be in `published`, the flag exactly True, no second point in that hour. - The rule, in the two forms an output point can take: - - * PUBLISHED - the point must BE one of `published[u]`, by identity, and - may be emitted only once. Its flag is whatever the feed said, never - forced to False: an imbalance print is published AND provisional (§9.2, - TARIFF-02), and a check that "corrected" it would be a bug wearing a - guarantee. - * CARRIED - the hour must not appear in `published` AT ALL, the flag must - be exactly True, and that hour may hold no second point. - - ⚠️ WHAT REPLACED "AT MOST ONE POINT PER HOUR", AND WHAT THAT DID AND DID - NOT CHANGE. This audit began as FCST-03's `seen` set: any repeated - `_utc_hour` raised, beside an identity rule `p is not src`. A bare hour - count cannot express a quarter-hourly feed, where four distinct published - points share one hour legitimately - so the count was replaced by the two - rules above, which subsume it. - - ⚠️ AND IT IS NOT STRICTER. FCST-04's version of this paragraph claimed two - strengthenings and BOTH WERE FALSE; FCST-05 measured it. A differential - over the whole domain this audit can express - every published mask over 3 - hours crossed with every output series of length 0-3 drawn from - {published, published-provisional, carried, carried-unflagged, - copy-of-published} - found ZERO divergence in EITHER direction: - - OLD raised, NEW accepted (WEAKER): 0 - OLD accepted, NEW raised (STRONGER): 0 - - The two specific claims, one at a time: - - * "so does the same published point emitted twice" - the OLD `seen` - count already raised on that, because emitting one point twice is one - HOUR twice. Preserved by the identity/`emitted` rules, not added. - * "a carried point may no longer share an hour with a published one, - which the count never checked at all" - true of the COUNT, and the old - IDENTITY rule raised on it anyway: a carried point in a published hour - took the `else` branch and failed `p is not src`. The message was about - a copy rather than about a carry, which is the only thing that moved. - - The one real behavioural change is the intended RELAXATION - several - distinct published points in one hour, which the old audit could not - express at all and refused. That relaxation is why this was rewritten, and - it stays. A right decision with a fabricated reason is still a defect here. - - ⚠️ AND THE REAL LIMIT, WHICH IS NOT A REGRESSION AND IS NOT SMALL: THIS - AUDIT DOES NOT CHECK COMPLETENESS. It grades every point that IS emitted - and says nothing about the ones that are not. Fed the H1 decimation - directly - four published points in an hour, output emitting only the last - - it ACCEPTS; so does two of four, and so does an EMPTY output. So `_audit` - is NOT what stops H1 recurring. What stops it is the point COUNT and the - IDENTITY checks in `test_price_history.py` and `test_price_forecast.py`, - which compare the emitted series against the feed's own prints. - - It is not made complete here, and the reason is structural rather than - effort: `published` is `_published_by_hour(read)` over the WHOLE read, - unwindowed, while `fill_from_yesterday()` emits only [since, until). A - published point outside the window is legitimately absent, so "every - published point appears" is not a true post-condition of these two - arguments - it would need the window passed in, which is a different - signature and a different check. - - ⚠️ AND THAT IS MEASURED, NOT ASSUMED. FCST-05 built the rule - raise unless - every point of every `published[u]` is emitted, +162 B - and ran it. It - takes the 1 h, 12 h and 48 h HORIZON checks in test_price_forecast.py RED, - on the shipped fill and a healthy feed: at an off-the-hour instant the - feed's already-elapsed prints are outside [since, until) and are correctly - dropped, and the rule calls that a provenance failure. (The differential - beside those checks scores the same rule as 49 STRONGER cases, so it is a - real change of behaviour and not a no-op.) The rule as stated is therefore - not merely unimplemented, it is WRONG - and stating a false post-condition - is MUTATION.md habit 16's defect: an instrument that asserts a completeness - it does not have is worse than no instrument. - - ⚠️ RAISES, NEVER REPAIRS. A module that silently corrects its own - provenance is one whose provenance means nothing; the only safe response to - "this point's flag disagrees with where it came from" is to refuse to emit - the series at all. - """ + Provenance, NOT completeness: it accepts a decimated or empty output, because + `published` is unwindowed while the fill emits [since, until). The point count + and identity checks in the tests guard that. Raises, never repairs. + DECISIONS/FCST-03-the-hours-not-yet-published.md.""" carried_hours = set() emitted = set() for p in points: @@ -351,37 +112,12 @@ def _audit(points, published) -> None: def fill_from_yesterday(read: PriceRead, *, yesterday: PriceRead, since: datetime, until: datetime, stamp: datetime) -> PriceRead: - """§8.3: fill the hours the feed has not published with yesterday's profile. + """8.3: fill the hours the feed has not published with yesterday's profile. - Every argument is required and keyword-only past the first. In particular - `yesterday` has NO DEFAULT: a fill with no yesterday to fill from is a - different situation from a fill that found nothing useful in one, and a - default `None` would make the two arrive looking identical. - - Returns a `PriceRead`: - - * a series, carrying the provider's own objects - ALL of them, which on - a quarter-hourly feed is four inside one hour - for the hours it - published, and one `_carried()` point for each remaining whole hour, - ordered by instant; - * `no_data` when neither source can produce a single hour - never an - empty series, which is not constructible (price.py trap 3). - - ⚠️ THE WINDOW IS WHOLE HOURS AT BOTH ENDS. `since` is floored to the hour - that CONTAINS it, because the current hour is still a tradeable hour; and - so is `until`, which is the half FCST-04's first draft left out. With - `since` floored and `until` compared raw, a 48 h horizon asked for at 08:17 - emitted 49 hours - the hour of `until` was still `< until` - and the count - was one too many for every `now` that is not exactly on the hour, which is - every `now` a running controller has. - - ⚠️ IDEMPOTENT AND RE-RUNNABLE, WHICH IS THE §5.4 LATE-PUBLICATION CONTRACT. - Day-ahead publication at 14-16 h CET can be late or fail, so the fetcher - retries and this runs again against the fresher read. Published hours always - win over carried ones, so a re-run REPLACES a carried point with the real - print and the hour becomes firm - the flag and the number move together - because the whole series is rebuilt rather than patched in place. - """ + `yesterday` has no default. Returns a series (every published point, by + identity, plus one `_carried()` point per remaining whole hour) or `no_data`, + never an empty series. The window is whole hours at both ends. Idempotent and + re-runnable (5.4): published hours win and the series is rebuilt.""" for name, value in (("since", since), ("until", until), ("stamp", stamp)): if value.tzinfo is None or value.utcoffset() is None: raise ValueError(f"{name} must be tz-aware; a naive instant is " @@ -390,7 +126,7 @@ def fill_from_yesterday(read: PriceRead, *, yesterday: PriceRead, published = _published_by_hour(read) # Yesterday's profile, keyed by BRUSSELS WALL-CLOCK HOUR. First point wins - # when yesterday itself was a 25-hour day and printed local hour 2 twice. + # when yesterday itself was a 25-hour day. by_local_hour = {} yesterday.match( series=lambda r: [by_local_hour.setdefault(brussels(p.start).hour, p) @@ -424,14 +160,8 @@ def fill_from_yesterday(read: PriceRead, *, yesterday: PriceRead, _audit(out, published) carried = sum(1 for p in out if _utc_hour(p.start) not in published) if carried: - # ⚠️ THREE QUANTITIES, THREE NAMES (TARIFF-11). This line read - # "%d of %d hours ... ; %d published" with `carried` (a count of - # carried HOURS - the fill emits exactly one point per unpublished - # hour) over `len(out)` (a count of POINTS, four per hour wherever the - # feed printed) and `len(out) - carried` (POINTS again) under the same - # noun. On the Belgian quarter-hourly feed that reads "33 of 141 - # hours" about a 48-hour horizon. The fill's unit is the HOUR and the - # feed's is the POINT; nothing derived from one answers for the other. + # TARIFF-11: carried hours, total hours and published points are three + # different quantities. total_h = len({_utc_hour(p.start) for p in out}) _LOG.info("%d of %s use yesterday's prices, marked as estimated. " "The feed published %s over the other %s.", @@ -446,17 +176,12 @@ def fill_from_yesterday(read: PriceRead, *, yesterday: PriceRead, return PriceRead.series(out, product, provider, stamp) -# --------------------------------------------------------------------------- # # AC2: irreversibility as a property of the ACTION, and the gate that reads it -# --------------------------------------------------------------------------- # @dataclass(frozen=True) class Irreversible: - """An action that cannot be undone once started, and how far it commits. - - `commits_h` is the horizon the action ties up - the hours whose prices must - therefore be firm before it may be justified by a price at all. - """ + """An action that cannot be undone once started. `commits_h` is the horizon it + ties up: those hours must be firm before a price may justify it.""" name: str # the §10.2 reason code commits_h: float @@ -473,25 +198,16 @@ class Irreversible: f"reversible one and does not belong here.") -# ⚠️ DERIVED FROM THE PHASE MACHINE'S OWN BOUNDS, NOT TYPED HERE. maintenance.py -# already states how long each phase may run before it is a fault; a second copy -# of those numbers here would be a place for them to drift apart, and the drift -# would show up as a gate that checked the wrong hours. +# Derived from the phase machine's own bounds, not typed here, so the gate +# cannot drift from it. MAINTENANCE_DRAIN = Irreversible("MAINTENANCE_DRAIN", DRAIN_TIMEOUT_MIN / 60.0) MAINTENANCE_CHARGE = Irreversible("MAINTENANCE_CHARGE", CHARGE_TIMEOUT_MIN / 60.0) class Gate(_Outcome): """May an irreversible action proceed on the price evidence available? - - * `permitted` - carries `.action` and `.hours`, the firm points that were - actually checked. The evidence exists only on this arm. - * `blocked` - carries `.reason`. Nothing else; `.hours` raises `Absence`. - - No `__bool__`, both `match()` arms required and keyword-only, for the reason - price.py's `_Outcome` gives: `if gate:` must not look like a guard while - being none. - """ + `permitted` carries `.action` and `.hours` (the firm points checked); `blocked` + carries `.reason` only. No `__bool__`; both `match()` arms are required.""" def match(self, *, permitted, blocked): return {"permitted": permitted, "blocked": blocked}[self.kind](self) @@ -541,12 +257,7 @@ class _Blocked(Gate): def committed_hours(now: datetime, action: Irreversible) -> tuple: - """The whole hours `action` started at `now` would run through. - - Split out because the gate and its caller must agree on what "the hours it - commits" means, and because a window that quietly excluded the last hour - would be a gate that permits an action on an hour it never looked at. - """ + """The whole hours `action` started at `now` would run through (the gate and its caller must agree).""" if not isinstance(action, Irreversible): raise TypeError(f"action must be an Irreversible, not {action!r}") if now.tzinfo is None or now.utcoffset() is None: @@ -562,13 +273,9 @@ def committed_hours(now: datetime, action: Irreversible) -> tuple: def price_permits(read: PriceRead, now: datetime, action: Irreversible) -> Gate: - """§8.3's rule, as a value: may this irreversible action rest on this price? - - Blocks when the read observed nothing (§11.4 - an action justified by a - price nobody has is an action justified by nothing), when any hour it - commits has no price at all, and when any of those hours is provisional. - Names the offending hour, so an operator reading the log knows which one. - """ + """8.3's rule as a value: may this irreversible action rest on this price? + Blocks on no data (11.4), an hour with no price, or a provisional hour, and + names the offending hour.""" hours = committed_hours(now, action) return read.match( series=lambda r: _walk(r, hours, action), @@ -600,18 +307,10 @@ def _walk(series, hours, action) -> Gate: def begin_phase(maint, now: datetime, read: PriceRead, action: Irreversible) -> Gate: - """Start a maintenance phase BECAUSE OF A PRICE, or refuse and say why. - - The AC2 example made real: `maint.force_start(now)` - the call that moves - the phase machine out of `idle` and commits the pack to hours of forced - discharge - happens on the permitting arm and cannot happen on the other - one, because the blocking arm has no code path to it. - - ⚠️ READ THE MODULE DOCSTRING'S LAST WARNING BEFORE WIRING THIS IN. This is - the price-justified entry point, not the calendar one; a site with no price - provider must keep starting its monthly cycle the way it does today, or - §12's opening sentence comes true - a cycle that silently never fires. - """ + """Start a maintenance phase BECAUSE OF A PRICE, or refuse and say why. The + price-justified entry point, not the calendar one: a site with no provider must + keep starting its monthly cycle as today (12). The blocking arm has no path to + `maint.force_start(now)`.""" gate = price_permits(read, now, action) return gate.match(permitted=lambda g: _go(maint, now, g), blocked=lambda g: _no(g)) diff --git a/goodwe_controller/app/price_history.py b/goodwe_controller/app/price_history.py index 804f48d..c3fb6c3 100644 --- a/goodwe_controller/app/price_history.py +++ b/goodwe_controller/app/price_history.py @@ -1,185 +1,24 @@ -"""FCST-04 - the day the carry-forward carries, kept somewhere a restart cannot reach. +"""FCST-04 - the day the carry-forward carries, kept where a restart cannot reach. -WHAT WAS MISSING, AND WHY IT WAS INVISIBLE. FCST-03 shipped -`price_forecast.fill_from_yesterday()` complete and checked, and shipped the -consumers that were written to expect it - `TariffPlan.firm_points()`, -`cheap_now()`'s refusal of a provisional hour, and `arbitrage()`'s refusal 3, -which drops provisional hours from the trade set instead of refusing the whole -series. Every one of those was live. The FILL ITSELF HAD NO CALLER: nothing in -`app/` referenced `fill_from_yesterday`, so no series ever contained a carried -point and every one of those consumers was exercising its all-firm branch. +`price_forecast.fill_from_yesterday()` needs a yesterday that survives a +restart; this module is that store and the one caller of the fill. Four rules: -The defect that produced is not an error anywhere. `load_prices()` returns -whatever the feed published and nothing widens it to the requested horizon, so -on a dynamic site between midnight and the 14-16 h CET day-ahead publication -the series covers the rest of TODAY and nothing more - typically 10-24 of the -48 hours `price_horizon_h` asked for. `TariffPlan.arbitrage()` then computes -`max - min` over that truncated window and reports a spread. The number is -real, the window is half the one the operator configured, and NOTHING SAYS SO: -tomorrow evening's peak - the hour a morning charge would be sold into - is -simply not in the series, so the spread it would have created cannot clear a -threshold it was never compared against. A missing hour is not a wrong price; -it is an hour the optimiser never learns exists. +1. It records what the PROVIDER published, never what the fill produced + (feeding a fill back in rots the numbers behind a true `provisional` flag). + `carry()` is the only public entry point and `_remember()` has exactly one + caller; test_price_history.py parses every shipped module for the name and + grades the order by driving it. +2. "Yesterday" is the Brussels day before the instant asked about, not the + newest record; any other date is `no_data` naming both dates. +3. A store that cannot answer says so in the type: `yesterday()` returns a + `PriceRead`, never an empty series. +4. Hours accumulate within a day (merge by instant, the newer price wins), so a + feed that serves only the remaining hours cannot overwrite a full day. -So this module is the half FCST-03 left out, and it is deliberately the SMALL -half: the fill's arithmetic, its DST rule and its provenance audit all stay in -`price_forecast.py`, untouched. What lives here is the one thing that -arithmetic needs and could not have: A YESTERDAY THAT SURVIVES A RESTART. - - -WHY PERSISTENCE, AND NOT A VARIABLE ON THE REFRESH TASK -====================================================== -An in-memory "the last full day I saw" costs nothing and is wrong exactly when -it matters. The window the fill exists for is midnight -> publication; an -add-on restarted at 08:00 - a config change, an HA update, a power cut - would -come up with no yesterday at all and spend that whole window on the truncated -series, which is the pre-FCST-03 behaviour arriving silently on the mornings -after an operator touched something. The fill's own value is a function of -surviving a restart, so the day it carries has to. - -⚠️ AND NOT IN `/data/state.json`. That file holds the maintenance history and -SAFETY-02's sign latch, and `Store.set()` rewrites the whole of it on every -write. The tariff refresh runs every `TARIFF_REFRESH_S` (1800 s), so folding a -price cache into it would put 48 extra whole-file rewrites a day through the -file whose corruption reads as "no maintenance ever ran" and "never tripped". -The cache is worth having; it is not worth putting anywhere near that. Separate -file, separate blast radius. - -⚠️ AND THE DIRECTORY IS NOT CREATED, which is `tariff.PeakStore`'s rule and its -reason applies unchanged: `/data` is mounted by the Supervisor and always -exists on a real install, so `makedirs` here could only ever succeed where the -durable volume is MISSING - manufacturing a cache on the container's ephemeral -layer that writes fine, reads fine, and is thrown away on the next restart. -That is a yesterday that silently forgets, which is worse than one that says it -cannot answer. - -⚠️ AND THAT RULE IS CARRIED OUT IN `_remember()`, NOT ONLY IN `_save()`. FCST-04 -first shipped it half-done: `_save()` short-circuited on `not self.ok` while -`_remember()` filled `self._days` anyway, so a store with no durable volume -answered `words` = "unavailable" and `yesterday()` = A SERIES - an in-memory -yesterday that works all day and is gone on the next restart, which is exactly -the silently-forgetting cache this rule exists to refuse. A store that cannot -persist now records nothing, and both of its answers say the same thing. - -⚠️ AND "CANNOT PERSIST" IS ONE QUESTION, ASKED ONCE, ABOUT THE OUTCOME - -FCST-06. The paragraph above enforced that rule against `not self.ok`, which is -one SYMPTOM of not persisting: the durable volume is missing. There is a second, -and FCST-06 measured it. With `ok=True` and the directory present, `_save()` can -still fail - a read-only remount (`[Errno 30] Read-only file system`), a full -volume (`[Errno 28] No space left on device`) - and it logs, records -`last_error` and returns. `_remember()` had already populated `self._days`, so -`words` read `'2026-01-14 (24 points over 24 h); 2026-01-15 (24 points over -24 h) [could not write (...)]'` while the file on disk held `{}`, and -`yesterday()` answered 24 REAL POINTS for the rest of the process's life. Gone -on the next restart, and nothing said it would be - the same silently-forgetting -cache, reached by the condition the rule did not name. - -⚠️ SO IT IS NOT TWO RULES, AND IT IS DELIBERATELY NOT TWO TESTS. "Can this -store persist?" CANNOT BE ANSWERED BEFORE THE ATTEMPT - a directory that existed -and was writable a moment ago is not a write that completed - so it is answered -AFTER it, once, by the only thing holding the evidence: `_save()` returns -whether the file now holds what memory holds. A missing volume is one of its -False cases and an `OSError` from the write is another, and `_remember()` KEEPS -ONLY WHAT THAT ONE ANSWER SAYS WAS KEPT. There is no `not self.ok` test in -`_remember()` any more, because there is nothing left for a second test to add - -and a third symptom nobody has thought of is how this defect arrived both times. - -⚠️ AND WHAT IS KEPT IS THE LAST STATE THAT IS ON DISK, NOT AN EMPTY ONE. The -MERGE is undone, not the store. A site whose volume fills at 14:00 goes on -answering the yesterday it wrote at 13:30 - that record survives a restart, so -answering it is not a promise the store cannot keep. What it stops doing is -adding to it. The invariant the rule was always describing, stated once: -`self._days` NEVER HOLDS A POINT THE FILE DOES NOT. (Not "mirrors the file" - -`_usable_days()` drops records the file still holds. The only direction that can -invent a yesterday is the one named.) - - -THE FOUR RULES THIS FILE IS SHAPED BY -===================================== - -1. IT RECORDS WHAT THE PROVIDER PUBLISHED, NEVER WHAT THE FILL PRODUCED. - Feeding a filled series back in is the defect that makes the whole mechanism - rot: day 2 carries day 1's guesses, day 3 carries day 2's carry of them, and - after a week of a dead feed the "yesterday" being carried forward is a - fossil of the last real publication with nothing anywhere reporting its age. - Every point still carries `provisional=True`, so the flag never lies - what - decays is the NUMBER, silently, behind a flag that is telling the truth. - - ⚠️ NO VALUE-LEVEL CHECK CAN CATCH THAT, and saying so is more useful than a - guarantee that would not hold. A carried point and a published-provisional - point (a real-time product's current interval, §9.2) are the same three - fields with - the same flag; the difference is provenance, and provenance is not in the - value. So the mechanism is ORDERING, not validation: `carry()` is the only - public entry point, it remembers the argument and returns the fill, and the - filled series exists only as a return value that `carry()` itself never - sees again. `_remember()` is private, is called from exactly one place, and - `test_price_history.py` PARSES every shipped module under `app/`, `tools/`, - `scenarios/` and `estop/` and fails if any of them uses the NAME - an - attribute, a bare name, a def, or the string constant a `getattr` needs. By - name rather than by substring, so a comment about the rule does not fail it - and a call from `tools/` cannot pass unseen. - - ⚠️ AND THE ORDER IS GRADED BY DRIVING IT, NOT BY READING THE SOURCE. The - first version of that check compared positions in `ast.walk`, which is - breadth-first: `self._remember(...)` is a statement and `self.yesterday(...)` - is an ARGUMENT of the fill, one level deeper, so `_remember` came out first - whichever statement was written first - shipped and reversed scored the - same. What grades it now is the behaviour: a feed carrying yesterday - alongside today, into an EMPTY store, must fill the whole horizon, and - reversed it fills nothing. - -2. "YESTERDAY" MEANS YESTERDAY, NOT "THE NEWEST DAY I HAVE". - `newest()` is the natural accessor and it is the one that decays: an add-on - down from Friday to Monday would answer Friday's prices to a fill that flags - them as one day old. The lookup is BY DATE - the Brussels day before the - Brussels day of the instant asked about - and a record for any other date is - NO_RECORD with a reason naming both dates. A three-day-old profile is not a - worse yesterday; it is not a yesterday. - -3. A STORE THAT CANNOT ANSWER SAYS SO, IN THE TYPE. - `yesterday()` returns a `PriceRead`: a `series` when a full record exists, - `no_data(reason)` for every other outcome - no file, no record for that - date, an unreadable file, a record that parses to nothing. There is no - empty-series answer and no `.get(..., [])`-shaped accessor, for - `price.py`'s reason: an empty series is how "the cache is broken" gets to - wear the shape of "yesterday was free". `fill_from_yesterday()` takes a - `PriceRead` and handles `no_data` on its own, so the degraded case needs no - branch at the call site - it fills nothing and the horizon stays short, - which is exactly the behaviour that shipped before this ticket. - -4. HOURS ACCUMULATE WITHIN A DAY; THEY DO NOT REPLACE IT. - A feed that serves only the REMAINING hours of the day (a legitimate shape - - nothing in the plugin contract promises a whole day) would, under - replace-the-record, overwrite a complete 24-hour profile with the two points - left at 22:00 and hand the next morning a yesterday with two hours in it. So - a record is MERGED by instant: a later read updates the hours it carries and - leaves the rest, and a revised price for an hour already held wins because - it is the newer statement about the same hour. - -⚠️ KEYED BY INSTANT, NOT BY LOCAL HOUR, AND THAT IS THE DST DECISION. Storing -`hour -> price` would collapse the two 02:00s of a 25-hour October day into -one, inventing a second and different collapsing rule beside the one -`fill_from_yesterday()` already documents ("the first of its two 02:00 points -is the one used, deterministically"). Storing the instants keeps both, in -order, and lets the fill's own rule be the only one in the system. - -WHAT THIS DOES NOT DO, ON PURPOSE -================================= -It does not extend the fill to a feed that published NOTHING. `resolve_tariff()` -applies `carry()` only on a `series` read; a `no_data` read still falls to -`_degrade()` and §11.4's row - "No price data -> fall back to the static -profile, mark the plan provisional, no arbitrage" - which is the fallback the -operator configured `static_profile` FOR. Preferring yesterday's feed over it -would change a documented degradation without being asked, and it would buy -nothing measurable: a fully-carried series is 100 % provisional, so -`arbitrage()` finds fewer than two firm hours and refuses, and `cheap_now()` is -False, exactly as on the §11.4 fallback. The two differ only in which numbers -the status page shows. `fill_from_yesterday()` handles a `no_data` read anyway, -and that branch is therefore not reached from production - it stays checked, -and turning it on is a product decision, not a wiring one. See -DECISIONS/FCST-04-the-day-that-is-carried.md. -""" +Rows are keyed by instant, not local hour (the DST rule stays in the fill). A +store that cannot persist records nothing (FCST-06). It does not extend the fill +to a feed that published nothing: that stays 11.4's static fallback. +DECISIONS/FCST-04-the-day-that-is-carried.md (section 5 has the store details).""" import copy import json @@ -194,89 +33,31 @@ from .price_forecast import fill_from_yesterday _LOG = logging.getLogger("goodwe.price.history") -# How many Brussels days of published prices the SORT-ORDER cap keeps. The fill -# needs exactly one - yesterday - and rule 2 refuses anything older, so this is -# not a horizon: it is the number of days a record may sit unused before -# pruning removes it, and it is small because a bigger cache would only hold -# days no lookup can ask for. Three covers today, yesterday, and the day a -# fetch spanning midnight writes ahead of itself. -# -# ⚠️ IT IS A CAP, NOT THE RETENTION RULE, AND THREE HAS NO SLACK. The steady -# state is already {yesterday, today, tomorrow}; a feed that publishes to D+2, -# or ONE stray point in the wrong year - which sorts LAST and is therefore -# immortal under a newest-N cap - pushes yesterday out of a three-slot store -# and keeps it out, permanently and silently. So `_prune()` PINS the two days a -# lookup can actually ask for on top of the cap; see its docstring. The store -# is then bounded at KEEP_DAYS + 2 records rather than at KEEP_DAYS. +# Brussels days kept: a sort-order cap, not a horizon (the fill needs only +# yesterday). KEEP_DAYS = 3 SCHEMA = 1 def _bdate(instant: datetime) -> str: - """The Brussels calendar date of an instant, as `YYYY-MM-DD`. - - ⚠️ BRUSSELS, NOT UTC. 23:30 UTC in July is already tomorrow in Belgium, and - a cache keyed on the UTC date would file the last 90-120 minutes of every - day under the wrong one - so a lookup on the following morning would find a - "yesterday" missing its own evening hours, which are the expensive ones. - """ + """The Brussels calendar date of an instant, as `YYYY-MM-DD` (not UTC: 23:30 UTC in July is tomorrow).""" return brussels(instant).strftime("%Y-%m-%d") def _yesterday_of(instant: datetime) -> str: """The Brussels date of the day BEFORE `instant`'s Brussels day. - - ⚠️ THE ARITHMETIC IS ON THE LOCAL DATE, NOT ON THE INSTANT, AND THAT IS A - DST DEFECT THIS FILE SHIPPED. `brussels()` renders at the FIXED offset in - force at that instant, so `brussels(stamp) - timedelta(days=1)` subtracts 24 - ABSOLUTE hours from a wall clock - and the Brussels day before a - spring-forward is 23 hours long. Measured at 2026-03-30T00:30+02:00, the - first hour of the morning after the March transition: it returned - 2026-03-28, so the fill carried a TWO-DAY-OLD profile flagged as yesterday's - (or, with only the last three days held, answered `no_data` naming a date - that is not yesterday). - - A calendar day minus one day is `date` arithmetic, and a `date` has no - offset to lose. The 25-hour October day is right for the same reason. - """ + Arithmetic on the local `date`, not the instant: subtracting 24 absolute hours + was wrong after the spring-forward. DECISIONS/FCST-04-the-day-that-is-carried.md.""" return (brussels(instant).date() - timedelta(days=1)).strftime("%Y-%m-%d") def _point_of(row): """One stored row -> a `PricePoint`, or `None` if the row is not one. - THE ONE VALIDATOR, AND THE ASYMMETRY IT CLOSES (FCST-05's MEDIUM). This - file used to hold TWO standards for what a stored point is. The READ path - (`_parse_points()`) was properly defensive - every field checked, a - malformed row dropped and reported. The WRITE path was - `{q["start"]: q for q in raw if isinstance(q, dict) and "start" in q}` - followed by `sorted(..., key=lambda q: q["start"])`, which trusts - `q["start"]` to be a hashable, sortable string it never checked. Measured, - with the malformed row under the day the feed is currently writing to: - - start is an int -> TypeError: '<' not supported between instances - of 'str' and 'int' - start is a dict -> TypeError: cannot use 'dict' as a dict key - start is null -> TypeError: '<' not supported between 'str' and - 'NoneType' - start is a list -> TypeError: cannot use 'list' as a dict key - - ⚠️ AND THE BLAST RADIUS IS PERMANENT, WHICH IS WHY IT IS NOT MERELY UNTIDY. - `_remember()` is reached through `carry()` -> `resolve_tariff()` -> - `run_tariff_refresh`, which catches `Exception` - so the add-on stays UP - and EVERY refresh raises forever. The plan retires after - `TARIFF_MAX_AGE_S`, and arbitrage and the cheap window are off from then - on. There is no self-heal, because `_remember()` raises before it reaches - `_save()`. `_usable_days()` only required `rec["points"]` to be a LIST; the - elements were unchecked on the write path, and the existing fixture put its - malformed row under a day the feed was NOT writing to, so `_remember()` - never touched it. - - So both paths call this, and neither of them has a standard of its own. - Returning `None` rather than raising is the contract: a caller decides - whether a bad row is dropped (both of them do) and how it is reported. - """ + The one validator for both the read and the write path (FCST-05): a malformed + row used to raise out of `carry()` and fail every refresh for good. Returns + `None` rather than raising; the caller decides how a bad row is reported.""" if not isinstance(row, dict): return None try: @@ -293,33 +74,8 @@ def _point_of(row): def _held(rec) -> tuple: """`(points, whole hours they cover)` for a day record. `(0, 0)` otherwise. - - ⚠️ POINTS AND HOURS ARE TWO NUMBERS, AND FCST-04's STATUS ROW PRINTED THE - FIRST UNDER THE SECOND'S NAME. The row was `f"{d} ({_n_points(rec)} h)"`, - which counts POINTS: on the quarter-hourly feed this add-on already - supports it read `'2026-01-13 (96 h)'` - 96 "h" for a 24-hour day. That is - the points/hours conflation H1 was raised for, on the row H1's own ticket - shipped. Both numbers are reported now, so an operator can see a day that - is 24 points over 24 h and a day that is 96 points over 24 h and tell them - apart, and neither number is wearing the other's unit. - - ⚠️ TOLERANT, AND IT IS THE SECOND LINE OF DEFENCE, NOT THE FIRST. `_load()` - drops a malformed day record and says so, so this should never see one. It - is written this way because `words` is read from `Controller.__init__` and - from every status render: a status LINE that raises takes the add-on down, - and "the cache is broken" must never be able to mean "the controller does - not start". That is exactly what FCST-04 shipped - `{"days": {"2026-01-14": - "oops"}}` gave `ok=True`, `last_error=None`, and an `AttributeError` out of - `words`, `yesterday()` and `resolve_tariff()` alike, with no self-heal - because `_remember()` raised before it could rewrite the file. - - ⚠️ AND IT COUNTS ONLY ROWS THAT ARE POINTS. A row `_point_of()` refuses is - not held prices, and a row count that included it would report a store as - healthier than it is - which is the second half of FCST-05's MEDIUM: the - status row said `'2026-01-15 (1 h)'` about a record whose single row was - killing every tariff refresh. What is dropped is named in `last_error`, and - `words` prints that beside the counts. - """ + Two numbers, each in its own unit (96 points over 24 h on a quarter-hourly + feed). Total, and counts only rows that are points.""" if not isinstance(rec, dict): return (0, 0) raw = rec.get("points") @@ -337,15 +93,7 @@ def _held(rec) -> tuple: def _row_of(day: str, points: int, hours: int) -> str: - """One day's cell of the status row: what is held, in BOTH of its units. - - ⚠️ THE UNIT IS SPELT OUT ON EACH NUMBER. FCST-04's cell was - `f"{day} ({points} h)"` - the point count wearing the hour's name, which on - a quarter-hourly feed reads `'2026-01-13 (96 h)'` for a 24-hour day. What - an operator needs from this row is whether tomorrow morning's fill has a - whole day to carry, and that is the HOUR figure; the point count says how - finely it is priced. Neither answers for the other. - """ + """One day's cell of the status row, with the unit spelt out on each number.""" # UX-10: the day as the page writes dates ("26 Sep"), not ISO. return (f"{points} price{'' if points == 1 else 's'} over {hours} " f"hour{'' if hours == 1 else 's'} for {day_words(day)}") @@ -373,21 +121,8 @@ def _bad_rows(rec) -> int: class PriceHistory: """The published price series of the last few Brussels days, in /data. - - ⚠️ `ok` IS ABOUT THE FILE, NOT ABOUT THE DATA. A healthy store with no - record for yesterday is `ok` and answers `no_data` - a normal state on a - site's first day, and a different thing from a store that could not be - opened. Both are visible: `ok`/`last_error` describe the file, and - `yesterday()`'s reason describes the answer. - - ⚠️ AND `ok` IS NOT "CAN PERSIST" (FCST-06). It says the directory was - there when the store opened, which a full volume and a read-only remount - both leave true. The one thing that answers "can this store persist" is - `_save()`'s return value, and `self._days` NEVER HOLDS A POINT THE FILE DOES - NOT, because `_remember()` undoes a merge that answer refused. That is what - makes `words` and `yesterday()` describe the same store in EVERY degraded - state rather than only in the ones somebody enumerated. - """ + `ok` is about the file, not the data: a healthy store with no record for + yesterday is `ok` and answers `no_data`. `ok`/`last_error` describe the file.""" def __init__(self, path: str = "/data/price_history.json"): self.path = path @@ -396,15 +131,14 @@ class PriceHistory: self._days: dict = {} self._open() - # -- the file ------------------------------------------------------------ + # the file def _open(self) -> None: parent = os.path.dirname(self.path) - # See the module note: NOT created. A cache on the ephemeral layer is - # one that forgets without saying so. + # NOT created (see FCST-04 section 5): a cache on the ephemeral layer + # forgets silently. if parent and not os.path.isdir(parent): self.last_error = f"no durable volume at {parent}" - # INFO, not ERROR: on a dev host or in a test container there is no - # /data and nothing is wrong. + # INFO, not ERROR: a dev host or test container has no /data. _LOG.info("No price history is kept (%s), so missing hours are " "not filled from yesterday's prices.", self.last_error) return @@ -421,32 +155,11 @@ class PriceHistory: "after that", self.path) return except (ValueError, OSError) as err: - # A corrupt cache must not stop the controller, and must not be - # silently re-created either: the record is discarded, the reason - # is kept, and `yesterday()` reports it rather than answering - # "no record", which would read as a normal first day. - # - # ⚠️ `ValueError`, NOT `json.JSONDecodeError`, AND THAT IS - # FCST-05's HIGH. Reading this file has two failure families: the - # BYTES (`OSError`) and the CONTENT - and the content family has at - # least two members, because `open(..., encoding="utf-8")` decodes - # before `json` ever sees a character. `UnicodeDecodeError` is - # `ValueError -> UnicodeError -> UnicodeDecodeError`; it is NOT a - # `json.JSONDecodeError` and it is NOT an `OSError` - # (`issubclass(UnicodeDecodeError, OSError)` is False). So ONE - # non-UTF-8 byte in /data/price_history.json escaped `_load()` -> - # `_open()` -> `PriceHistory.__init__`, which is unguarded at - # main.py's `Controller.__init__`, AND THE ADD-ON DID NOT START. - # Measured: `UnicodeDecodeError: 'utf-8' codec can't decode byte - # 0xff in position 39`. Valid-UTF-8-invalid-JSON, a UTF-8 BOM and a - # non-dict day record all degraded correctly - it was the one byte - # class left uncovered, which is the same class as the defect the - # `_usable_days()` rework was raised for. - # - # The tuple names the FAMILY rather than the two members known - # today, because "the file's content is not readable as JSON" is - # exactly what this handler says, and a third `ValueError` subclass - # nobody has thought of is how this defect arrived the first time. + # A corrupt cache must not stop the controller nor be silently re- + # created: the record is discarded and the reason kept, so + # `yesterday()` reports it. `ValueError` is the family caught, not + # `json.JSONDecodeError` (one non-UTF-8 byte raised + # `UnicodeDecodeError` and the add-on did not start, FCST-05). self.last_error = f"unreadable ({err})" _LOG.warning("Price history at %s is %s, so it starts empty. " "Missing hours cannot be filled in until a full day " @@ -454,10 +167,7 @@ class PriceHistory: self.path, self.last_error) return found = blob.get("version") if isinstance(blob, dict) else None - # ⚠️ THE TYPE IS PART OF THE VERSION. `1.0 == 1` in Python, so a float - # written by some other tool would pass a bare `!=` and be read as - # schema 1 - a version check that cannot tell 1 from 1.0 is one that - # would not notice a writer this build has never met. + # The type is part of the version: `1.0 == 1` in Python. if not isinstance(blob, dict) or isinstance(found, bool) \ or not isinstance(found, int) or found != SCHEMA: self.last_error = f"version {found!r}, not {SCHEMA}" @@ -474,35 +184,8 @@ class PriceHistory: def _usable_days(self, days: dict) -> dict: """The day records that are shaped like day records. The rest are dropped. - - ⚠️ THE ASYMMETRY THAT PRODUCED THE DEFECT: `_parse_points()` below is - carefully defensive about every field of a POINT, and `_load()` was not - defensive about the record holding them at all. `{"version": 1, "days": - {"2026-01-14": "oops"}}` therefore loaded as `ok=True`, - `last_error=None` - and then `rec.get(...)` raised `AttributeError` out - of `words`, `yesterday()` and `resolve_tariff()`. `PriceHistory(...)` - and `.words` are both unguarded in `Controller.__init__`, so THE ADD-ON - DID NOT START; and it never self-healed, because `_remember()` raised on - the same record before it could reach `_save()`. - - A broken cache is a short horizon. It is not a controller that will not - regulate a battery, and §6's rule - absent machinery must not disable - the loop - does not stop applying because the machinery is a JSON file. - - Dropped, not repaired: an unusable record has nothing in it to keep, and - the next `_remember()` writes a fresh one under that date. - - ⚠️ AND THE ROWS INSIDE A KEPT RECORD ARE COUNTED, NOT ONLY THE RECORD - (FCST-05). This checked that `rec["points"]` is a LIST and stopped - there, so a record holding one row with a `start` of `7` loaded as - healthy: `last_error=None`, and `words` said `'2026-01-15 (1 h)'` about - a store whose next `_remember()` would raise on that row and go on - raising forever. The rows are NOT dropped here - `_parse_points()` is - the read path's one filter and stays so - but they are COUNTED, so the - store stops reporting health it does not have from the moment it is - opened. `_remember()` drops them on the next refresh and `_save()` - rewrites the file, after which a restart clears the note. - """ + A broken cache is a short horizon, never a controller that will not start. + Rows inside a kept record are counted, not dropped (`_parse_points()` filters).""" kept, dropped, malformed = {}, [], {} for key, rec in days.items(): if not isinstance(key, str) or not isinstance(rec, dict) \ @@ -524,9 +207,8 @@ class PriceHistory: + ", ".join(f"{day_words(d)} ({n})" for d, n in sorted(malformed.items()))) if notes: - # Recorded, not silent: `yesterday()` and `words` both report - # `last_error`, and a cache that quietly discards half a file reads - # to an operator exactly like a normal first day. + # Recorded, not silent: a cache that quietly discards half a file + # reads like a first day. self.last_error = " and ".join(notes) _LOG.warning("Price history at %s: %s. The add-on starts anyway, " "and fills in only what the rest holds.", @@ -535,27 +217,7 @@ class PriceHistory: def _save(self) -> bool: """Write `self._days` out. True only if the file now holds it. - - ⚠️ THE RETURN VALUE IS THE "CAN THIS STORE PERSIST?" PREDICATE, AND - IT IS THE ONLY ONE (FCST-06). Both ways of not persisting end here and - are the same answer: no durable volume returns False without touching - the disk, an `OSError` out of the write returns False after recording - why. `_remember()` acts on that one answer, so the second place that - used to test the first symptom on its own - and missed the second - has - nothing left to test. See the module docstring's fifth warning. - - ⚠️ AND IT REPORTS, IT DOES NOT RAISE. §6 and this module's opening: a - broken cache must not be a broken controller. An add-on that will not - start because /data filled up is a worse defect than the one FCST-06 - fixed, so a failed write is logged, kept in `last_error` - where `words` - and `yesterday()` both read it - and RETURNED, never thrown. - - ⚠️ THE TEMP FILE IS REMOVED ON EVERY FAILING PATH. A write that dies - between `mkstemp` and `os.replace` used to leave its temp behind, and - the refresh task runs 48 times a day: a persistent write failure - a - full volume being the likeliest cause - leaked 48 inodes a day onto the - volume that was already the problem. - """ + The one "can this store persist?" answer (FCST-06), asked after the attempt.""" if not self.ok: return False tmp = None @@ -563,8 +225,7 @@ class PriceHistory: fd, tmp = tempfile.mkstemp(dir=os.path.dirname(self.path)) with os.fdopen(fd, "w", encoding="utf-8") as fh: json.dump({"version": SCHEMA, "days": self._days}, fh, indent=1) - # Atomic: a power cut mid-write must not leave a truncated file - # that parses as an empty day. + # Atomic: a power cut mid-write must not leave a truncated file. os.replace(tmp, self.path) tmp = None # it IS the file now; do not unlink except OSError as err: @@ -579,45 +240,24 @@ class PriceHistory: pass # failed cleanup is not a fault return True - # -- the one public entry point ----------------------------------------- + # the one public entry point def carry(self, read: PriceRead, *, since: datetime, until: datetime, stamp: datetime, remember: bool = True) -> PriceRead: - """Remember what this read published, then §8.3-fill the missing hours. - - THE ORDER IS DELIBERATE AND IS THE POINT OF THE METHOD. The read is - recorded FIRST, so a feed that hands back yesterday alongside today - refreshes yesterday's record from live data before it is consumed as - the profile; and the filled series - the one thing that must never be - recorded - exists only as this method's return value, which this object - never sees again. See rule 1, including how that order is graded. - - Returns whatever `fill_from_yesterday()` returns: a series over the - whole hours of [since, until), carrying EVERY point the provider - published inside it - by identity, and four of them in one hour on a - quarter-hourly feed - plus one carried point for each remaining whole - hour FOR WHICH YESTERDAY HELD THAT WALL-CLOCK HOUR. An hour neither - source can price is left out (§11.4 for that hour), and `no_data` comes - back when that is every hour. - """ + """Remember what this read published, then 8.3-fill the missing hours. + Recorded FIRST, so the filled series exists only as the return value (rule 1). + Returns `fill_from_yesterday()`'s series, or `no_data` when no hour can be priced.""" if remember: - # PRICE-01: False only for a read that CAME FROM this store - # (`stored()`), which is already filed; filing it again would - # rewrite the same file for nothing, and nothing else may pass it. + # PRICE-01: False only for a read that came from this store, + # already filed. self._remember(read, stamp) return fill_from_yesterday(read, yesterday=self.yesterday(stamp), since=since, until=until, stamp=stamp) - # -- the halves, private because rule 1 is an ordering ------------------ + # the halves, private because rule 1 is an ordering def _remember(self, read: PriceRead, stamp: datetime) -> None: """File a provider read's points under their own Brussels days. - - ⚠️ PRIVATE, AND CALLED FROM EXACTLY ONE PLACE. A caller that can reach - this can record a filled series, which is the decay in rule 1. There is - no value-level defence against that - a carried point and a published - provisional point are the same three fields - so the defence is that - nothing outside this file names it, and `test_price_history.py` checks - the tree for that rather than trusting the underscore. - """ + Private, called from exactly one place: no value-level defence can tell a + carried point from a published-provisional one, so the defence is the ordering.""" points = read.match(series=lambda r: r.points, no_data=lambda _r: ()) if not points: return @@ -626,23 +266,9 @@ class PriceHistory: provider = read.match(series=lambda r: r.provider, no_data=lambda _r: None) - # ⚠️ THE STATE THIS READ STARTED FROM, KEPT SO IT CAN BE PUT BACK - # (FCST-06). Everything below merges into `self._days` IN PLACE, and - # whether any of it may be KEPT is not knowable until `_save()` has - # tried - so the merge runs, and it is undone if the file did not take - # it. There is no `not self.ok` branch above any more: a store with no - # durable volume is one of the answers `_save()` gives, not a separate - # symptom with a test of its own. Module docstring, fifth warning. - # - # ⚠️ DEEP, NOT `dict(self._days)`. A one-level copy shares the day - # RECORDS, and the loop below rebinds keys on the ones it touches - so - # the "previous" state would hand back records the merge had already - # edited, and a store with a good yesterday on disk would go on - # answering the merge of it that never reached the file. It would also - # be right only for as long as nobody makes this loop mutate a `points` - # list or a row in place. A copy that is right by construction outlives - # that, and the store holds at most KEEP_DAYS + 2 records, 48 times a - # day. + # The state this read started from, kept so it can be put back + # (FCST-06): the merge runs in place and is undone if `_save()` did not + # take it. Deep copy, since the loop rebinds the day records. before = copy.deepcopy(self._days) touched = set() @@ -651,29 +277,11 @@ class PriceHistory: day = _bdate(p.start) rec = self._days.setdefault( day, {"product": product, "provider": provider, "points": []}) - # Rule 4: MERGE by instant. A later read updates the hours it - # carries and leaves the rest; a revised price for an hour already - # held wins, because it is the newer statement about that hour. - # - # ⚠️ EVERY HELD ROW GOES THROUGH `_point_of()` (FCST-05's MEDIUM). - # This was `{q["start"]: q for q in raw if isinstance(q, dict) and - # "start" in q}` sorted by `q["start"]` - a key and a sort key that - # trusted a value read off disk to be a hashable, sortable string. - # An `int`, `null`, a `dict` or a `list` there raised `TypeError` - # out of `carry()` on the day the feed was writing to, and - # `run_tariff_refresh` caught it, so the add-on stayed up while - # EVERY refresh failed for good. A row that is not a point is now - # dropped and reported here, exactly as the read path drops it, and - # `_save()` below writes the record back clean - which is the - # self-heal the raising version could never reach. - # - # ⚠️ AND THE MERGE KEY IS THE VALIDATED INSTANT, NOT ITS SPELLING. - # Aware datetimes hash and compare by instant, so this is rule 4 as - # rule 4 is written. The old string key merged by SPELLING, and the - # old lexicographic sort put the second 02:00 of the 25-hour - # October day (`+01:00`) BEFORE the first (`+02:00`). Only the - # stored order was affected - `_parse_points()` re-sorts by instant - # on the way out - but the store now holds what it says it holds. + # Rule 4: MERGE by instant; a later read updates the hours it + # carries. Every held row goes through `_point_of()` and the key is + # the validated instant, not its spelling (FCST-05). A row that is + # not a point is dropped and reported, and `_save()` writes the + # record back clean. raw = rec.get("points") held = {} for q in (raw if isinstance(raw, list) else ()): @@ -689,13 +297,8 @@ class PriceHistory: touched.add(day) if junked: - # Said, not swallowed. `words` prints `last_error`, so a store that - # has just discarded rows off disk stops reporting plain health - - # and after `_save()` below the file is clean, so a restart clears - # this. `_save()` overwrites it if the write itself fails, which is - # the more urgent fact of the two - and on that path the drop is - # undone with the rest of the merge, so those rows are still in the - # file and the next write that completes drops them again. + # Said, not swallowed: `words` prints `last_error`. A failed + # `_save()` overwrites it and undoes the drop. self.last_error = ( f"{messages.count(sum(junked.values()), 'bad price')} dropped " f"from " + ", ".join(f"{day_words(d)} ({n})" @@ -705,48 +308,19 @@ class PriceHistory: self.path, self.last_error) self._prune(stamp) if not self._save(): - # ⚠️ A STORE THAT CANNOT PERSIST RECORDS NOTHING - the rule, asked - # ONCE as a question about the outcome instead of tested twice - # against two symptoms of it. Keeping the merge here is exactly the - # cache the module docstring refuses: `yesterday()` would answer a - # series the file does not contain, all day, until a restart - # dropped it without a word - and `words` and `yesterday()` would - # be describing two different stores. The reason is already in - # `last_error` (from `_save()`, or from `_open()` when there is no - # volume at all), so both answers now describe what is on disk. - # - # It does NOT raise, and §6 is why: a broken cache must not be a - # broken controller. `carry()` still returns the fill, the horizon - # is as short as the feed makes it, and the add-on regulates. + # A store that cannot persist records nothing (FCST-06): + # `_save()`'s answer decides what is kept, so `yesterday()` never + # answers a series the file does not hold. self._days = before return _LOG.debug("price history: %d point(s) filed under %s; %d day(s) held", len(points), ", ".join(sorted(touched)), len(self._days)) def _prune(self, stamp: datetime) -> None: - """Drop records no lookup can reach. A sort-order cap, plus two pins. - - ⚠️ NOTHING IS DELETED BY ARITHMETIC AGAINST THE CLOCK. A clock that - jumped - the failure mode WITNESS-03 spent a session on - would - otherwise delete every record in one pass and leave the fill with - nothing on the morning after, which is the one morning it is for. - Deleting on a bad clock is unrecoverable; holding a few days too many - is not, so the cap is `sorted(...)[-KEEP_DAYS:]` and stays that way. - - ⚠️ AND A CAP ALONE IS DEFEATED BY ONE POINT IN THE WRONG YEAR. Sorting - is lexicographic on the date string, so a stray far-future record sorts - LAST and is immortal - it consumes a slot for good, and with the steady - state already {yesterday, today, tomorrow} filling all three, yesterday - is what falls out. A feed that publishes to D+2 does the same thing - without any stray at all. So the two days `yesterday()` can ACTUALLY be - asked for are pinned on top of the cap. - - `stamp` is READ here, which it was not before: the pin is the only use - of it, and it is additive. Pinning can only RETAIN a record, never - delete one, so a jumped `stamp` pins two dates that are not held and - changes nothing - the WITNESS-03 property is preserved by construction - rather than by remembering to check it. - """ + """Drop records no lookup can reach: a sort-order cap plus two pins. Nothing is + deleted by arithmetic against the clock (a jumped clock would delete everything, + WITNESS-03). The pin of the two dates `yesterday()` can be asked for only + retains, so a far-future record cannot push yesterday out.""" if len(self._days) <= KEEP_DAYS: return keep = set(sorted(self._days)[-KEEP_DAYS:]) @@ -756,11 +330,7 @@ class PriceHistory: def yesterday(self, stamp: datetime) -> PriceRead: """The published series of the Brussels day BEFORE `stamp`'s, or no_data. - - ⚠️ THE DATE IS COMPUTED, NOT LOOKED UP. See rule 2: `newest()` would - answer Friday's prices to a Monday fill, and the fill would flag them as - one day old because that is the only thing its flag can say. - """ + Computed, not looked up (rule 2).""" if stamp.tzinfo is None or stamp.utcoffset() is None: raise ValueError( f"yesterday({stamp!r}): a naive datetime is not an instant, " @@ -803,19 +373,9 @@ class PriceHistory: stamp) def stored(self, stamp: datetime) -> PriceRead: - """PRICE-01: every price this store holds, as one series, or no_data. - - This is the LAST step of the price server's fallback (price_chain.py): - the answer of the last fetch that worked, kept on disk, so a site can - plan for as long as the days it holds last. It is a read of what was - PUBLISHED and filed, in the order they happened, and it is never the - carried fill (rule 1 holds: `carry(..., remember=False)` is how this - read is filled, and it is not filed again). - - The prices are the all-in ones stored when they were fetched. They are - not worked out again, so a change of markup or taxes shows from the - next fetch that works, not in these. - """ + """PRICE-01: every price this store holds, as one series, or no_data. The last + step of the price server's fallback (price_chain.py): what was PUBLISHED and + filed, never what a fill produced.""" if stamp.tzinfo is None or stamp.utcoffset() is None: raise ValueError(f"stored({stamp!r}): a naive datetime is not an " f"instant") @@ -833,25 +393,12 @@ class PriceHistory: return PriceRead.series(points, PriceProduct.DAY_AHEAD, "price_history", stamp) - # -- observability ------------------------------------------------------- + # observability @property def words(self) -> str: """One line for the status page. Says what is held, or why nothing is. - - ⚠️ THIS MUST NOT RAISE, EVER. It is read once from `Controller.__init__` - and again on every status render, and neither call site guards it - so - an exception here is an add-on that does not start. It went out that - way: a day record that was a string instead of a dict raised - `AttributeError` from the `len(...)` below. `_load()` now drops such a - record and `_held()` is total, so this has two independent reasons not - to fail on a broken file. - - ⚠️ AND IT COUNTED POINTS AND CALLED THEM HOURS (FCST-05's LOW). The row - was `f"{d} ({_n_points(rec)} h)"`; on the quarter-hourly feed this - add-on already supports that reads `'2026-01-13 (96 h)'` for a 24-hour - day. Both numbers are printed now - see `_held()` - because they are - two different facts and only one of them was ever being shown. - """ + Must not raise, ever: it runs from `Controller.__init__` and every status + render, unguarded. Prints points and hours separately (FCST-05).""" if not self.ok: return f"unavailable - {self.last_error}" if not self._days: @@ -859,26 +406,20 @@ class PriceHistory: return f"empty{why} - nothing to carry forward yet" held = ", ".join(_row_of(d, *_held(self._days[d])) for d in sorted(self._days, key=str)) - # A file that PARTLY loaded says so, or the dropped days read as days - # that were simply never written. + # A file that partly loaded says so, or dropped days read as never + # written. return held + (f" [{self.last_error}]" if self.last_error else "") def _parse_points(raw) -> tuple: - """Canonical dicts -> PricePoints, and a phrase naming what was dropped. - - ⚠️ A MALFORMED POINT IS DROPPED, NOT DEFAULTED, and the drop is reported. - `float(row.get("eur_per_kwh", 0))` is the shape of the whole family of - defects this repo has been removing: a cache that lost a field would answer - "that hour was free" to a fill that flags it as an ordinary guess. - """ + """Canonical dicts -> PricePoints, and a phrase naming what was dropped. A + malformed point is dropped, not defaulted: a lost field must not read as a free hour.""" if not isinstance(raw, list): return (), "the `points` field is not a list" out, dropped = [], 0 for row in raw: - # ⚠️ `_point_of()` IS THE ONE VALIDATOR, and this loop used to BE it. - # The write path had its own, weaker standard; see `_point_of()` for - # what that cost. One standard, two callers, no asymmetry to re-open. + # `_point_of()` is the one validator; this loop used to be a weaker + # one. pt = _point_of(row) if pt is None: dropped += 1 @@ -889,20 +430,8 @@ def _parse_points(raw) -> tuple: class NoHistory: - """The explicit "this caller keeps no history" - a NAME, not a `None`. - - ⚠️ IT EXISTS SO THE PARAMETER CAN BE REQUIRED. `history=None` meaning "skip - the fill" is the shape this repo keeps removing: a caller who forgets the - argument gets the truncated-horizon behaviour of the bug this ticket is - about, silently, and the code reads as though the fill is happening. With - no default there is nothing to forget, and a caller with no durable volume - - a check, a replay harness - has to say `NoHistory()` and be visible in - the diff. - - It carries out the ordering faithfully: it records nothing and carries - nothing, so `carry()` returns the provider's read unchanged and the plan is - exactly what it was before FCST-04. - """ + """The explicit "this caller keeps no history": a NAME, not `None`, so the + parameter can be required. It records nothing and carries nothing.""" ok = False last_error = "no history is kept by this caller" diff --git a/goodwe_controller/app/solarfc.py b/goodwe_controller/app/solarfc.py index 4ebbf6f..3928d71 100644 --- a/goodwe_controller/app/solarfc.py +++ b/goodwe_controller/app/solarfc.py @@ -28,6 +28,21 @@ A reading older than `FRESH_S`, or from before local midnight, is stale, and a stale or missing forecast is no forecast: the day plan is off and every mode acts as in P3 option A. +OPT-27: but the last fresh reading of each sensor is kept (`Hold`), so the +night plan is not made blind. Forecast.Solar's sensors stop changing in the +evening and api.forecast.solar is sometimes out for a poll. A kept reading +stands in for its sensor (a) for up to `FRESH_S` when the sensor is missing +or unavailable, as a sensor that merely stopped updating is already +tolerated, and (b) from the end of the bell (`BELL_TO_H`) until local +midnight, if it is at most `KEPT_MAX_AGE_S` old when used. +Never beyond that, and never for another day: a today figure is only today's, +a tomorrow figure is tomorrow's until midnight and then, until the bell starts +(`BELL_FROM_H`, 06:00 local), today's (OPT-34: at most `KEPT_MORNING_MAX_AGE_S` +old, it must have been usable at midnight, and no fresh today reading may +have replaced it since). +DECISIONS/OPT-27-last-fresh-forecast-for-the-night.md, +DECISIONS/OPT-34-the-night-charge-keeps-its-level.md. + OPT-17: each day may instead name a list (`pv_forecast_entities`, `pv_forecast_tomorrow_entities`), summed by `read_sum()`. One member that is not usable makes that day's sum unusable, never a partial sum. @@ -50,6 +65,19 @@ FRESH_S = 3 * 3600.0 # The bell a state-only forecast is spread over, in local hours. BELL_FROM_H, BELL_TO_H = 6.0, 20.0 +# OPT-27: a kept reading serves the night plan from the end of the bell until +# local midnight, and a reading older than this when used is not kept (a +# morning figure must not drive the night). +KEPT_MAX_AGE_S = 12 * 3600.0 +# OPT-34: the evening's "tomorrow" figure, once midnight has passed, is today's +# until the bell starts (`BELL_FROM_H`): nothing has been produced yet, so it +# is still the whole day. A 17:05 reading is 12 h 55 min old at 06:00, so this +# case may be older than `KEPT_MAX_AGE_S`, up to this, but only a figure that +# was usable at midnight (at most `KEPT_MAX_AGE_S` old then), so one that has +# not been used all evening does not come back at 00:00. +KEPT_MORNING_MAX_AGE_S = 15 * 3600.0 +SAVE_EVERY_S = 900.0 + # Why there is no forecast (the page words are in ui_text `home_words`). NOT_SET, MISSING, UNIT, STALE, BAD = ( "not_set", "missing", "unit", "stale", "bad") @@ -75,13 +103,25 @@ class Forecast: today: Day tomorrow: Day | None at: datetime # when the sensor last reported + # OPT-27: a reading kept past `FRESH_S` is in it (the evening's figure at + # night). The screens say so. Last, so `Forecast(today, tomorrow, at)` is + # as before. + kept: bool = False - def between(self, t0: datetime, t1: datetime) -> float | None: + def between(self, t0: datetime, t1: datetime, + since: datetime | None = None) -> float | None: """kWh the sun should give between `t0` (now or later) and `t1`. None when the span reaches the sunny hours of a day with no - forecast (tomorrow, when its sensor is not set).""" + forecast (tomorrow, when its sensor is not set). + + `since` is the moment a state-only "left today" figure counts from + (OPT-23): `t0` itself when not given, so a span that starts later + than now would be spread over what is left of the bell from its own + start. Pass now for every span of a plan, and the spans add up to + the figure. An hourly split ignores it.""" if t1 <= t0: return 0.0 + since = t0 if since is None else since last = self.tomorrow or self.today if t1 > last.day0 + timedelta(days=1, hours=BELL_FROM_H): return None @@ -92,7 +132,7 @@ class Forecast: a = max(t0, day.day0) b = min(t1, day.day0 + timedelta(days=1)) if b > a: - total += _day_between(day, a, b, t0) + total += _day_between(day, a, b, since) return total @@ -124,8 +164,12 @@ def _day_between(day: Day, a: datetime, b: datetime, now: datetime) -> float: out += kwh * (hi - lo).total_seconds() / (e - s).total_seconds() return out # State only: `kwh` is what is left from `max(now, day0)` to the day's - # end, spread over the bell. - h_from = _hours_since(day.day0, max(now, day.day0)) + # end, spread over the bell. Nothing of it is before that moment. + start = max(now, day.day0) + a = max(a, start) + if b <= a: + return 0.0 + h_from = _hours_since(day.day0, start) left = 1.0 - _bell(h_from) if left <= 1e-9: return 0.0 @@ -234,6 +278,119 @@ def _reported(state): return max(ts) if ts else None +class Hold: + """OPT-27: the last fresh reading of each forecast sensor, per day it is + for. `read_sum()` remembers every fresh reading here and, when a sensor + has none now, asks it (`stand_in()`). + + Keyed `(kind, entity, day0)`: `kind` is "today" (energy still to come on + `day0`) or "tomorrow" (all of `day0`), and `day0` is the day the figure is + FOR. A today figure therefore never answers for another day, and a + tomorrow figure answers for its own day only: tomorrow until midnight, + then today until the bell starts (OPT-34). Only the newest reading of a + key is kept, so the next fresh reading ends the hold at once. + + `dump()` / `load()` are what the controller keeps in its store, so an + evening restart does not lose it (`load()` drops what is bad). + `changed` is set when `dump()` would differ.""" + + def __init__(self, data=None): + self._r = {} + self._saved = {} # key -> the `at` last flagged for saving + self.changed = False + if isinstance(data, dict): + self.load(data) + + def remember(self, kind, entity, day: "Day", at: datetime) -> None: + key = (kind, entity, day.day0) + old = self._r.get(key) + if old is not None and at <= old[1]: + return + self._r[key] = (day, at) + # A sensor that reports every minute with the same figure must not + # rewrite the store every minute: the saved age may lag by + # `SAVE_EVERY_S` (the bounds are hours). + mark = self._saved.get(key) + if old is None or old[0].kwh != day.kwh or mark is None \ + or (at - mark).total_seconds() >= SAVE_EVERY_S: + self._saved[key] = at + self.changed = True + + def forget_before(self, day0: datetime) -> None: + """Drop what is for a day before `day0` (the local midnight now).""" + gone = [k for k in self._r if k[2] < day0] + for k in gone: + del self._r[k] + self._saved.pop(k, None) + if gone: + self.changed = True + + def stand_in(self, kind, entity, day0, now, *, as_today=False): + """`(Day, at, kept)` for a sensor that has no fresh reading, or None. + + The kept reading must be for `day0`, and either (a) at most `FRESH_S` + old, or (b) used at night: from the end of the bell to local midnight, + at most `KEPT_MAX_AGE_S` old. `kept` is true when (b) was needed, + or `as_today` (OPT-34: a tomorrow figure standing in for today, after + midnight and until the bell starts, `KEPT_MORNING_MAX_AGE_S`).""" + rec = self._r.get((kind, entity, day0)) + if rec is None: + return None + day, at = rec + age = (now - at).total_seconds() + if age <= FRESH_S and not as_today: + return day, at, False + if as_today: + ok = (day0 <= now < _bell_start(day0) + and age <= KEPT_MORNING_MAX_AGE_S + and (day0 - at).total_seconds() <= KEPT_MAX_AGE_S) + return (day, at, True) if ok else None + if age > KEPT_MAX_AGE_S: + return None + # A tomorrow sensor is asked with day0 = the next midnight, which is + # in the future all day, so the evening ends at that day's end. + evening = _local_hour(now) >= BELL_TO_H and now < _bounds(day0)[1] + return (day, at, True) if evening else None + + def replaced(self, entities, day0) -> bool: + """OPT-34 F-1: every today sensor has given a fresh reading for + `day0` (so after midnight, newer than any evening figure). The + evening's tomorrow figure is then no stand-in for today any more, + also once those readings are stale.""" + return all(("today", e, day0) in self._r for e in entities) + + def dump(self) -> dict: + return {"v": 1, "r": [ + [kind, ent, d.day0.isoformat(), d.kwh, at.isoformat(), + [[s.isoformat(), e.isoformat(), k] for s, e, k in d.periods]] + for (kind, ent, _d0), (d, at) in sorted( + self._r.items(), key=lambda kv: (kv[0][0], kv[0][1], + kv[0][2]))]} + + def load(self, data: dict) -> None: + rows = data.get("r") if isinstance(data, dict) else None + if not isinstance(rows, list): + return + for row in rows: + try: + kind, ent, d0, kwh, at, per = row + day0, at = _parse_time(d0), _parse_time(at) + kwh = float(kwh) + periods = tuple((_parse_time(s), _parse_time(e), float(k)) + for s, e, k in per) + ok = (kind in ("today", "tomorrow") and isinstance(ent, str) + and day0 is not None and at is not None + and math.isfinite(kwh) and kwh >= 0 + and all(s is not None and e is not None and e > s + and math.isfinite(k) and k >= 0 + for s, e, k in periods)) + except (TypeError, ValueError): + continue + if ok: + self._r[(kind, ent, day0)] = (Day(day0, kwh, periods), at) + self._saved[(kind, ent, day0)] = at + + def entities(opts: dict, *, tomorrow=False) -> tuple: """The forecast entities one day names, in configured order: the list if it has any entry, else the single key, else (). A repeated id is @@ -255,6 +412,15 @@ def _bounds(now): return d0.astimezone(timezone.utc), d1.astimezone(timezone.utc) +def _bell_start(day0): + """The moment the bell starts on the local day that begins at `day0` + (`BELL_FROM_H` local, so 06:00 on a 23 h or 25 h day too).""" + here = day0.astimezone(BILLING_TZ) + t = datetime(here.year, here.month, here.day, tzinfo=BILLING_TZ) \ + + timedelta(hours=BELL_FROM_H) + return t.astimezone(timezone.utc) + + def _today(state, day0, now): """`(Day, at, None)` or `(None, None, why)` for one today sensor.""" today, why = _day(state, day0, now, today=True) @@ -270,14 +436,14 @@ def _today(state, day0, now): def _tomorrow(state, day0, day1, now): - """`(Day, None)` or `(None, why)` for one tomorrow sensor.""" + """`(Day, at, None)` or `(None, None, why)` for one tomorrow sensor.""" t, why = _day(state, day1, now, today=False) if t is None: - return None, why + return None, None, why t_at = _reported(state) if t_at is None or t_at < day0 or (now - t_at).total_seconds() > FRESH_S: - return None, STALE - return t, None + return None, None, STALE + return t, t_at, None def read(today_state, tomorrow_state, now: datetime, *, @@ -293,7 +459,7 @@ def read(today_state, tomorrow_state, now: datetime, *, return None, why tomorrow = None if tomorrow_state is not None: - tomorrow, _w = _tomorrow(tomorrow_state, day0, day1, now) + tomorrow, _at, _w = _tomorrow(tomorrow_state, day0, day1, now) return Forecast(today, tomorrow, at), None @@ -309,7 +475,12 @@ def _sum(days): return Day(days[0].day0, sum(d.kwh for d in days), periods) -def read_sum(today, tomorrow, now: datetime): +# Why a sensor may be covered by a kept reading: it has none now, or it is +# too old. A bad value or a wrong unit is a set-up fault and is never covered. +_COVERABLE = (MISSING, STALE) + + +def read_sum(today, tomorrow, now: datetime, hold: Hold | None = None): """OPT-17: `(Forecast or None, why, gaps)` over lists of `(entity, state)`, one per configured member, in configured order. @@ -322,29 +493,69 @@ def read_sum(today, tomorrow, now: datetime): `gaps` is `((day, position, entity, why), ...)` for every unusable member (`day` "today" or "tomorrow", `position` 1-based), so the page and the log can name it. With one member per day the forecast is what - `read()` gives (test_opt17.py proves it).""" + `read()` gives (test_opt17.py proves it). + + OPT-27: with a `hold`, every fresh reading is remembered in it, and a + member that is missing or stale is covered by its last fresh reading + where `Hold.stand_in()` allows it (the forecast then has `kept` set when + that took a reading older than `FRESH_S`). After midnight, if the today + side is still short, the tomorrow figures read before midnight are + today's, until the bell starts (`stand_in()`), unless fresh today + readings have replaced them (`Hold.replaced()`). Without a `hold` this is + exactly as before.""" if not today: return None, NOT_SET, () day0, day1 = _bounds(now) - gaps, days, ats, why0 = [], [], [], None + if hold is not None: + hold.forget_before(day0) + gaps, days, ats, why0, kept = [], [], [], None, False for i, (ent, st) in enumerate(today, 1): d, at, why = _today(st, day0, now) + if d is not None and hold is not None: + hold.remember("today", ent, d, at) + elif hold is not None and why in _COVERABLE: + r = hold.stand_in("today", ent, day0, now) + if r is not None: + d, at, k = r + kept = kept or k if d is None: gaps.append(("today", i, ent, why)) why0 = why0 or why else: days.append(d) ats.append(at) - tdays = [] + tdays, tats, tkept = [], [], False for i, (ent, st) in enumerate(tomorrow, 1): - t, why = _tomorrow(st, day0, day1, now) + t, t_at, why = _tomorrow(st, day0, day1, now) + if t is not None and hold is not None: + hold.remember("tomorrow", ent, t, t_at) + elif hold is not None and why in _COVERABLE: + r = hold.stand_in("tomorrow", ent, day1, now) + if r is not None: + t, t_at, k = r + tkept = tkept or k if t is None: gaps.append(("tomorrow", i, ent, why)) else: tdays.append(t) + tats.append(t_at) + if why0 is not None and hold is not None and tomorrow and all( + g[3] in _COVERABLE for g in gaps if g[0] == "today"): + # Past midnight the sensors may still show yesterday's "left today". + # The figure each tomorrow sensor gave before midnight is for this + # day; all of them are needed, as for any sum. + subs = [hold.stand_in("tomorrow", ent, day0, now, as_today=True) + for ent, _st in tomorrow] + if all(r is not None for r in subs) and not hold.replaced( + [ent for ent, _st in today], day0): + days = [r[0] for r in subs] + ats = [r[1] for r in subs] + gaps = [g for g in gaps if g[0] != "today"] + why0, kept = None, True gaps = tuple(gaps) if why0 is not None: return None, why0, gaps tday = _sum(tdays) if tomorrow and len(tdays) == len(tomorrow) else None + kept = kept or (tday is not None and tkept) # The oldest member's report is when the sum was last fully known. - return Forecast(_sum(days), tday, min(ats)), None, gaps + return Forecast(_sum(days), tday, min(ats), kept), None, gaps diff --git a/goodwe_controller/app/static/app.css b/goodwe_controller/app/static/app.css index 2a38a93..faf35e5 100644 --- a/goodwe_controller/app/static/app.css +++ b/goodwe_controller/app/static/app.css @@ -246,3 +246,29 @@ select.sel:disabled{opacity:.6} .simple .card .lbl{font-weight:600;font-size:15px} .simple .inv .hd{border-bottom:0} .simple .inv .f{border-top:0;padding-top:0} +/* UX-54: the Forecast screen */ +.fcline{text-align:right} +.fcnote{background:var(--info-bg);border-radius:10px;padding:8px 10px;margin:0 0 8px} +.fcbars{display:flex;align-items:flex-end;gap:2px;height:110px} +.fcol{flex:1;position:relative;height:100%;min-width:0} +.fcol i{position:absolute;display:block} +.fcol .sun{bottom:0;left:0;right:0;background:var(--solar);border-radius:3px 3px 0 0;opacity:.85} +.fcol .sun.est{opacity:.5} +.fcol .got{bottom:0;left:30%;right:30%;background:repeating-linear-gradient(135deg,var(--ink) 0 3px,var(--card) 3px 5px);border:1px solid var(--ink);border-bottom:0} +.fcol .use{left:-1px;right:-1px;height:3px;background:var(--house)} +.fcols{position:absolute;inset:0;display:flex;gap:0} +.fcols>i{flex:1;min-width:0;display:block} +.lvlchart{position:relative;height:110px;margin:18px 0 4px} +.lvlchart svg{position:absolute;inset:0;width:100%;height:100%;display:block} +.lvlchart .yl{position:absolute;right:2px;font-size:10px;color:var(--faint);pointer-events:none} +.fcnow{position:absolute;top:-4px;bottom:0;width:2px;background:var(--ink);opacity:.7;pointer-events:none} +.fcnow span{position:absolute;top:-15px;left:-10px;font-size:10px} +.legend i.sun{background:var(--solar)} +.legend i.got{background:repeating-linear-gradient(135deg,var(--ink) 0 2px,var(--card) 2px 4px);border:1px solid var(--ink)} +.legend i.use{height:3px;background:var(--house)} +.legend i.lv{height:3px;background:var(--batt)} +table.fct{width:100%;border-collapse:collapse;font-size:12px;font-variant-numeric:tabular-nums} +table.fct th,table.fct td{padding:4px 3px;text-align:right;border-bottom:1px solid var(--line)} +table.fct th:first-child,table.fct td:first-child{text-align:left} +table.fct th{color:var(--muted);font-weight:600} +.ymax{position:absolute;top:-2px;left:2px;font-size:10px;color:var(--faint);pointer-events:none} diff --git a/goodwe_controller/app/static/app.js b/goodwe_controller/app/static/app.js index faa8b1b..529b878 100644 --- a/goodwe_controller/app/static/app.js +++ b/goodwe_controller/app/static/app.js @@ -448,6 +448,175 @@ }); } + /* ---------------- Forecast (UX-54) ---------------- */ + // Today and tomorrow, per hour: the solar forecast and what was measured, + // the house's expected use, and each battery's expected level with its + // blocks. The data is built by app/dayview.py when the tab asks (GET + // forecast), the sentences in the page's language. A part that is not + // known yet comes as a note, so no chart is ever drawn empty. + // The SVG namespace, read off the page's own icons: the page names no URL. + const SVGNS = document.querySelector("svg").namespaceURI; + function sv(tag, attrs) { + const e = document.createElementNS(SVGNS, tag); + for (const k in attrs || {}) e.setAttribute(k, attrs[k]); + return e; + } + let FC = null; + let fcDay = 0; + let fcBusy = false; + let fcTimer = null; + const fmtKwh = (x) => dec(Number(x).toFixed(1)); + const hoursOf = (d) => d.length_h || d.hours.length || 24; + function endLabel(d, i) { return i + 1 < d.hours.length ? d.hours[i + 1].label : "24:00"; } + function hourTip(d, i) { + const x = d.hours[i]; + const lines = []; + if (x.forecast_kwh != null) lines.push(t("forecast.forecast") + ": " + fmtKwh(x.forecast_kwh) + " kWh"); + if (x.measured_kwh != null) lines.push(t("forecast.measured") + ": " + fmtKwh(x.measured_kwh) + " kWh"); + if (x.house_kwh != null) lines.push(t("forecast.house") + ": " + fmtKwh(x.house_kwh) + " kWh"); + d.batteries.forEach((b, j) => { if (x.levels[j] != null) lines.push(b.name + ": " + dec(Math.round(x.levels[j]).toString()) + " %"); }); + return [x.label + " " + t("plan.time_to") + " " + endLabel(d, i)].concat(lines); + } + function hourItem(tag, cls, d, i, kids) { + const tip = hourTip(d, i); + const e = h(tag, { class: cls, tabindex: "-1", role: "img", "aria-label": tip.join(". "), "data-tip": JSON.stringify(tip) }, kids); + return e; + } + function renderSun(d) { + const card = $("#fc-sun"); + const any = d.sun.forecast || d.sun.measured || d.sun.house; + card.hidden = !any; + if (!any) return; + const mx = Math.max(0.5, ...d.hours.map((x) => Math.max(x.forecast_kwh || 0, x.measured_kwh || 0, x.house_kwh || 0))); + const pct = (v) => Math.max(0, Math.min(100, v / mx * 100)) + "%"; + const bars = $("#fc-bars"); + bars.setAttribute("aria-label", t("forecast.sun")); + bars.replaceChildren(...d.hours.map((x, i) => hourItem("div", "fcol", d, i, [ + x.forecast_kwh != null ? h("i", { class: "sun" + (d.sun.estimate ? " est" : ""), style: "height:" + pct(x.forecast_kwh) }) : null, + x.measured_kwh != null ? h("i", { class: "got", style: "height:" + pct(x.measured_kwh) }) : null, + x.house_kwh != null ? h("i", { class: "use", style: "bottom:calc(" + pct(x.house_kwh) + " - 1px)" }) : null]))); + roving(bars); + const chart = $("#fc-sunchart"); + chart.querySelectorAll(".fcnow, .ymax").forEach((e) => e.remove()); + chart.append(h("span", { class: "ymax", "aria-hidden": "true" }, fmtKwh(mx))); + if (fcDay === 0) chart.append(nowMark(d)); + const used = [["sun", d.sun.forecast, "forecast"], ["got", d.sun.measured, "measured"], ["use", d.sun.house, "house"]]; + $("#fc-legend").replaceChildren(...used.filter((u) => u[1]).map((u) => h("span", {}, h("i", { class: u[0], "aria-hidden": "true" }), t("forecast." + u[2])))); + } + function nowMark(d) { + return h("div", { class: "fcnow", "aria-hidden": "true", style: "left:" + Math.max(0, Math.min(100, FC.now_h / hoursOf(d) * 100)) + "%" }, h("span", {}, t("plan.now"))); + } + function levelChart(d, b, bi) { + const L = hoursOf(d), W = L * 10; + const x = (hr) => hr * 10; + const box = h("div", { class: "chart" }); + const area = h("div", { class: "lvlchart" }); + const svg = sv("svg", { viewBox: "0 0 " + W + " 100", preserveAspectRatio: "none", "aria-hidden": "true", focusable: "false" }); + [0, 50, 100].forEach((p) => svg.append(sv("line", { x1: 0, x2: W, y1: 100 - p, y2: 100 - p, stroke: "var(--line)", "stroke-width": 1, "vector-effect": "non-scaling-stroke" }))); + const fill = { grid: "var(--accent)", give: "var(--house)", cycle: "var(--warn)" }; + for (const r of b.blocks) { + if (r.end_h <= r.start_h) continue; + svg.append(sv("rect", { x: x(r.start_h), y: 0, width: x(r.end_h) - x(r.start_h), height: 100, fill: fill[r.kind] || "var(--mid)", opacity: 0.18 })); + } + const pts = b.levels.map((p) => x(p[0]) + "," + (100 - p[1])); + svg.append(sv("polygon", { points: x(b.levels[0][0]) + ",100 " + pts.join(" ") + " " + x(b.levels[b.levels.length - 1][0]) + ",100", fill: "var(--batt)", opacity: 0.12 })); + svg.append(sv("polyline", { points: pts.join(" "), fill: "none", stroke: "var(--batt)", "stroke-width": 2.5, "stroke-linejoin": "round", "vector-effect": "non-scaling-stroke" })); + area.append(svg); + [["100", 0], ["50", 50], ["0", 100]].forEach((y) => area.append(h("span", { class: "yl", "aria-hidden": "true", style: "top:" + y[1] + "%;transform:translateY(" + (y[1] === 100 ? "-100%" : y[1] === 0 ? "0" : "-50%") + ")" }, y[0] + " %"))); + area.append(h("div", { class: "fcols", role: "group", "aria-label": b.name }, ...d.hours.map((hx, i) => hourItem("i", "", d, i, null)))); + roving(area.querySelector(".fcols")); + if (fcDay === 0) area.append(nowMark(d)); + box.append(area); + // The blocks under the chart, drawn as Home draws them. + if (b.blocks.length) { + const lane = h("div", { class: "lane", role: "group", "aria-label": b.name }); + for (const r of b.blocks) { + if (r.end_h <= r.start_h) continue; + lane.append(chartItem("b", r.kind, "left:" + (r.start_h / L * 100) + "%;width:" + ((r.end_h - r.start_h) / L * 100) + "%", + b.name + ", " + r.from + " " + t("plan.time_to") + " " + r.to, [r.what, r.why, r.amount])); + } + box.append(lane); + roving(lane); + } + box.append(h("div", { class: "axis", "aria-hidden": "true" }, h("span", {}, "00:00"), h("span", {}, "06:00"), h("span", {}, "12:00"), h("span", {}, "18:00"), h("span", {}, "24:00"))); + return box; + } + function batteryCardFc(d, b, bi) { + const drawn = b.levels.length > 1; + const list = b.blocks.map((r) => h("li", {}, h("strong", {}, r.from + " " + t("plan.time_to") + " " + r.to + ": "), tipText([r.what, r.why, r.amount]))); + return h("div", { class: "card" }, + h("div", { class: "row spread" }, h("h2", {}, b.name), + h("span", { class: "tiny muted" }, (b.capacity_kwh ? fmtKwh(b.capacity_kwh) + " kWh · " : "") + t("forecast.levels"))), + drawn ? levelChart(d, b, bi) : null, + list.length ? h("ul", { class: "plain small" }, ...list) : null, + b.line ? h("p", { class: "small muted", style: "margin:6px 0 0" }, b.line) : null); + } + function renderTable(d) { + const more = $("#fc-more"); + const any = d.sun.forecast || d.sun.measured || d.sun.house; + more.hidden = !any; + if (!any) return; + const lv = d.batteries.filter((b) => b.levels.length > 1); + const head = h("tr", {}, h("th", { scope: "col" }, t("forecast.hour")), + d.sun.forecast ? h("th", { scope: "col" }, t("forecast.solar")) : null, + d.sun.measured ? h("th", { scope: "col" }, t("forecast.got")) : null, + d.sun.house ? h("th", { scope: "col" }, t("forecast.use")) : null, + ...lv.map((b) => h("th", { scope: "col" }, b.name + " %"))); + const cell = (v, f) => h("td", {}, v == null ? "–" : f(v)); + const rows = d.hours.map((x) => h("tr", {}, + h("td", {}, x.label), + d.sun.forecast ? cell(x.forecast_kwh, fmtKwh) : null, + d.sun.measured ? cell(x.measured_kwh, fmtKwh) : null, + d.sun.house ? cell(x.house_kwh, fmtKwh) : null, + ...lv.map((b) => cell(x.levels[d.batteries.indexOf(b)], (v) => dec(Math.round(v).toString()))))); + $("#fc-table").replaceChildren(h("thead", {}, head), h("tbody", {}, ...rows)); + } + function renderForecast() { + const loading = $("#fc-loading"); + if (!FC || FC.ok !== true) { + loading.hidden = false; loading.textContent = FC ? t("offline.name") : t("forecast.loading"); + $("#fc-notes").replaceChildren(); $("#fc-sun").hidden = true; $("#fc-batts").replaceChildren(); $("#fc-more").hidden = true; $("#fc-line").textContent = ""; + return; + } + loading.hidden = true; + const d = FC.days[fcDay]; + $("#fc-line").textContent = d.line || ""; + $("#fc-notes").replaceChildren(...d.notes.map((n) => h("p", { class: "fcnote small" }, n))); + renderSun(d); + $("#fc-batts").replaceChildren(...d.batteries.map((b, i) => batteryCardFc(d, b, i))); + renderTable(d); + document.querySelectorAll("#fc-day button").forEach((b) => { + const on = Number(b.dataset.d) === fcDay; + b.setAttribute("aria-checked", String(on)); b.tabIndex = on ? 0 : -1; + }); + } + async function loadForecast() { + if (fcBusy) return; + fcBusy = true; + try { + const r = await fetch("forecast?lang=" + LANG, { cache: "no-store" }); + if (!r.ok) throw new Error("forecast " + r.status); + FC = await r.json(); + } catch (e) { FC = { ok: false }; } + finally { fcBusy = false; } + renderForecast(); + } + document.querySelectorAll("#fc-day button").forEach((b, i, all) => { + b.addEventListener("click", () => { fcDay = Number(b.dataset.d); renderForecast(); }); + b.addEventListener("keydown", (e) => { + const j = { ArrowRight: i + 1, ArrowLeft: i - 1 + all.length }[e.key]; + if (j === undefined) return; + e.preventDefault(); fcDay = j % all.length; renderForecast(); all[fcDay].focus(); + }); + }); + // Every 30 s while the screen shows: the data moves slowly, and each load + // is built on request. + function scheduleForecast() { + const on = !document.hidden && $("#t-fc").getAttribute("aria-selected") === "true"; + if (on && !fcTimer) { loadForecast(); fcTimer = setInterval(loadForecast, 30000); } + if (!on && fcTimer) { clearInterval(fcTimer); fcTimer = null; } + } + /* ---------------- control buttons ---------------- */ // The action is a toggle, so the page reads the live state first and sends // nothing when control is already where the person asked it to be. @@ -511,11 +680,13 @@ } addEventListener("hashchange", fromHash); - /* ---------------- polling: every 3 s while Home or Batteries shows ---------------- */ + /* ---------------- polling: every 3 s while Home, Batteries or Forecast shows ---------------- */ let timer = null; function schedule() { + scheduleForecast(); const live = !document.hidden && ($("#t-home").getAttribute("aria-selected") === "true" || - $("#t-batt").getAttribute("aria-selected") === "true"); + $("#t-batt").getAttribute("aria-selected") === "true" || + $("#t-fc").getAttribute("aria-selected") === "true"); if (live && !timer) { refresh(); timer = setInterval(refresh, 3000); } if (!live && timer) { clearInterval(timer); timer = null; } } diff --git a/goodwe_controller/app/static/index.html b/goodwe_controller/app/static/index.html index 24a34eb..0566911 100644 --- a/goodwe_controller/app/static/index.html +++ b/goodwe_controller/app/static/index.html @@ -18,6 +18,7 @@ @@ -114,6 +115,32 @@ + +