diff --git a/.gitea/ci/run-tests.sh b/.gitea/ci/run-tests.sh new file mode 100755 index 0000000..e1be1bb --- /dev/null +++ b/.gitea/ci/run-tests.sh @@ -0,0 +1,125 @@ +#!/bin/sh +# CI-15: run the add-on's test_*.py files, in parallel, inside the CI container. +# Called by the step "the add-on's own tests" of .gitea/workflows/checks.yml as +# sh -eu /w/goodwe-addon/.gitea/ci/run-tests.sh +# and runnable the same way from a clone (see TESTS_DIR below). +# +# WHY: the serial loop ran past the runner's 60-minute limit (SAFETY-125: 85 files, +# 3 557 s of test time, run 1430 cut off at file 84). The host has 12 cores. +# +# SHAPE +# 1. A parallel batch: JOBS files at a time (xargs -P), longest first. +# 2. THEN the serial group, one at a time, after the batch has finished, so +# each runs on a machine with nothing else of ours on it. +# Every file writes to its own $RUN_DIR/.out (there is no shared output +# file) and prints its result as ONE block under a lock, so the PASS/FAIL +# lines and the `CI: ` / tail -40 text of two files never interleave. +# +# THE FILE LIST IS STILL DERIVED FROM THE TREE (see checks.yml). The two typed +# lists below only SORT or SET ASIDE files that exist; a new test_*.py that is +# in neither list is run, in the parallel batch. A name that no longer exists +# in a list is harmless. Nothing can make a file silently not run. +# +# SERIAL GROUP: files whose assertions depend on the wall clock, so CPU load +# from siblings could flip them. Name and reason, one each: +# test_loop.py drives the real run_control() against a real wall-clock +# floor (~93 s on purpose); worst inter-cycle gap 3.0 s +# against a 4.5 s bound (TEST-30, CI-06). +# test_freeze.py part 3 runs the real sim/hwsim.py over a transport with a +# 16 s window, a 10 s hold and an alarm measured at 11.0 s: +# one poll of margin, and "failed 1 run in 3" on the rig +# transport (TEST-30, CI-06). +# test_safety87.py builds a controller (CH65) whose meter reading is stamped +# on the REAL clock, and section 8 asks its status page, +# which says "Not ready - Meter: no reading for N seconds, +# limit 30 seconds" once N passes 30. Measured by CI-15: N is +# about half the file's run time (N = 45 s in a 94 s run, 48 s +# in a 103 s run). The file takes ~25 s idle and took 45 s in +# CI's first parallel run, so N was ~20 s of the 30. It FAILED +# alone under --cpus 0.15 (94 s) and in a loaded parallel run +# (103 s). No test file is changed here, so it runs alone. +# Add a file here only with a measured reason, and say it in this list. +SERIAL="test_loop.py test_freeze.py test_safety87.py" +# +# LONGEST FIRST (SAFETY-125's CI run, job 2580; then a local parallel run, seconds: +# modes_p4 758, safety125 508 on hotfix-0.19.2 only, modes_p3 364, safety119 328, +# price_run_fixes 184, safety90 160, safety103 134, safety94 114, peakguard 113, +# opt20 102, safety97 101, safety81 100). With JOBS workers pulling from one +# queue, starting the longest files first keeps the tail short: the batch takes +# about as long as the longest file. A sort order, not a selection. +HEAVY="test_modes_p4.py test_safety125.py test_modes_p3.py test_safety119.py test_price_run_fixes.py test_safety90.py test_safety103.py test_safety94.py test_peakguard.py test_opt20.py test_safety97.py test_safety81.py" +# +JOBS=${CI_JOBS:-6} +cd "${TESTS_DIR:-/w/goodwe-addon/goodwe_controller}" + +# one FILE: run it, keep its output in its own file, print its block whole. +if [ "${1:-}" = "--one" ]; then + f=$2 + o="$RUN_DIR/$f.out" + start=$(date +%s) + if python3 "$f" > "$o" 2>&1; then rc=0; else rc=1; fi + secs=$(( $(date +%s) - start )) + { + if [ $rc = 0 ]; then + echo "PASS $f (${secs}s)" + grep "^CI: " "$o" || true + else + echo "FAIL $f (${secs}s)" + tail -40 "$o" + fi + } > "$RUN_DIR/$f.blk" + echo $rc > "$RUN_DIR/$f.rc" + flock "$RUN_DIR/print.lock" cat "$RUN_DIR/$f.blk" + exit 0 +fi + +set -- test_*.py +if [ "$1" = "test_*.py" ]; then + echo "NO test_*.py FOUND - the mount is empty or the layout moved. NOTHING has been checked." + exit 1 +fi +echo "$# test file(s) to run: $JOBS at a time, then the serial group" + +RUN_DIR=$(mktemp -d /tmp/ci-tests.XXXXXX) +export RUN_DIR +: > "$RUN_DIR/par" +: > "$RUN_DIR/ser" +for f in $HEAVY; do + case " $SERIAL " in *" $f "*) continue ;; esac + if [ -f "$f" ]; then echo "$f" >> "$RUN_DIR/par"; fi +done +for f in "$@"; do + case " $SERIAL $HEAVY " in *" $f "*) continue ;; esac + echo "$f" >> "$RUN_DIR/par" +done +for f in $SERIAL; do + if [ -f "$f" ]; then echo "$f" >> "$RUN_DIR/ser"; fi +done +echo "parallel: $(wc -l < "$RUN_DIR/par") file(s); serial: $(wc -l < "$RUN_DIR/ser") file(s)" + +t0=$(date +%s) +rc=0 +# `exit 0` in the worker: a failing FILE is recorded in its .rc, so a non-zero +# from xargs itself means the machinery broke, which is a failure too. +xargs -n 1 -P "$JOBS" sh -eu "$0" --one < "$RUN_DIR/par" || rc=1 +echo "parallel batch done in $(( $(date +%s) - t0 ))s; serial group next" +while read -r f; do + sh -eu "$0" --one "$f" || rc=1 +done < "$RUN_DIR/ser" + +# Every file that was meant to run has a verdict, and every verdict is PASS. A +# worker that died before writing its .rc leaves no file, which counts as a fail. +ran=0 +bad=0 +for f in "$@"; do + ran=$((ran + 1)) + if [ ! -f "$RUN_DIR/$f.rc" ]; then + echo "FAIL $f (no result: the run never reported)" + bad=$((bad + 1)) + elif [ "$(cat "$RUN_DIR/$f.rc")" != 0 ]; then + bad=$((bad + 1)) + fi +done +echo "$ran file(s), $bad failed, $(( $(date +%s) - t0 ))s wall" +if [ "$bad" = 0 ] && [ "$rc" = 0 ]; then exit 0; fi +exit 1 diff --git a/.gitea/workflows/checks.yml b/.gitea/workflows/checks.yml index 4cffd72..749be2d 100644 --- a/.gitea/workflows/checks.yml +++ b/.gitea/workflows/checks.yml @@ -209,7 +209,7 @@ on: jobs: unit: runs-on: [self-hosted, goodwe-rig] - timeout-minutes: 70 # ~4 to run; the rest is queueing for the single slot + timeout-minutes: 40 # ~15 to run (CI-15: 6 files at a time; the serial suite was ~60 and hit the runner cap); the rest is margin for a busy host env: # ⚠️ UNDER /home/homelab, NOT $GITHUB_WORKSPACE (ENV-01, and re-measured # during CI-01). act's workspace is `/root/.cache/act//hostexecutor`, a @@ -534,6 +534,19 @@ jobs: # shorter timeout - a 60 s limit was tried during CI-01 and reported # exactly the false hang the docstring predicts. # + # ⚠️ CI-15: THE FILES RUN SIX AT A TIME, NOT ONE AFTER ANOTHER. The serial + # loop summed 3 557 s on SAFETY-125's run (job 2580) and ran past the + # runner's 60-minute cap. The loop now lives in .gitea/ci/run-tests.sh (it + # comes from the merged tree, like the Dockerfile), so the same script runs + # locally and here. It runs the longest files first, then the files whose + # assertions depend on the wall clock - test_loop.py and test_freeze.py + # (TEST-30 / CI-06) and test_safety87.py (CI-15) - ALONE, after the + # parallel batch, so no sibling can steal their CPU. The reasons are in + # the script. Its output stays one + # PASS/FAIL block per file, in a file per file (never one shared /tmp/out). + # A file that is not in the script's two typed lists still runs, in the + # parallel batch: the list is a sort order, not a selection. + # # test_freeze.py part 3 runs against the real `sim/hwsim.py` from the # checkout above; GOODWE_REQUIRE_HWSIM=1 makes it fail rather than use its # stub (CI-12). Lines a test prints with a `CI: ` prefix are echoed on PASS. @@ -552,28 +565,7 @@ jobs: - name: the add-on's own tests run: | set -eu - docker run --rm -e PYTHONDONTWRITEBYTECODE=1 -e GOODWE_REQUIRE_HWSIM=1 -v "$WS:/w:ro" goodwe-addon-checks:1 sh -eu -c ' - cd /w/goodwe-addon/goodwe_controller - set -- test_*.py - if [ "$1" = "test_*.py" ]; then - echo "NO test_*.py FOUND - the mount is empty or the layout moved. NOTHING has been checked." - exit 1 - fi - echo "$# test file(s) to run" - rc=0 - for f in "$@"; do - start=$(date +%s) - if python3 "$f" > /tmp/out 2>&1; then - echo "PASS $f ($(( $(date +%s) - start ))s)" - grep "^CI: " /tmp/out || true - else - echo "FAIL $f ($(( $(date +%s) - start ))s)" - tail -40 /tmp/out - rc=1 - fi - done - exit $rc - ' + docker run --rm -e PYTHONDONTWRITEBYTECODE=1 -e GOODWE_REQUIRE_HWSIM=1 -v "$WS:/w:ro" goodwe-addon-checks:1 sh -eu /w/goodwe-addon/.gitea/ci/run-tests.sh - name: clean up if: always() diff --git a/README.md b/README.md index d9711d4..c25d5c7 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ on real hardware, and the comments in `control.py` say which. ### CI and the infra pin -CI (`.gitea/workflows/checks.yml`) runs every `test_*.py`, with `admin/goodwe` checked out next to this repo at the commit in `.gitea/infra-pin`, so `test_freeze.py` grades against the real `sim/hwsim.py`. If that checkout fails, the job fails. The local stub is used only in local runs, and it prints `STUB USED` when it is. +CI (`.gitea/workflows/checks.yml`) runs every `test_*.py` (six at a time, the wall-clock ones alone afterwards: `.gitea/ci/run-tests.sh`), with `admin/goodwe` checked out next to this repo at the commit in `.gitea/infra-pin`, so `test_freeze.py` grades against the real `sim/hwsim.py`. If that checkout fails, the job fails. The local stub is used only in local runs, and it prints `STUB USED` when it is. The pin tracks `admin/goodwe` **release/1.0**, not master. Two non-required checks go red when release/1.0 has moved past the pin: `infra-pin / fresh` here, on any add-on PR or push to release/1.0, and `addon-pin / fresh` in `admin/goodwe`, on every push to its release/1.0. Nothing re-runs `addon-pin` after a bump, so it stays red on that admin/goodwe commit; `infra-pin / fresh` green on the bump is the all-clear. **Bump the pin** whenever either is red. Set `.gitea/infra-pin` to the new release/1.0 SHA (`git ls-remote /admin/goodwe.git refs/heads/release/1.0`) in a PR, and let `checks / unit` grade the tests against it. CI reads `admin/goodwe` with the Actions secret `INFRA_READ_TOKEN`.