#!/bin/sh
# CI-15: run the add-on's test_*.py files, in parallel, inside the CI container.
# Called by the step "the add-on's own tests" of .gitea/workflows/checks.yml as
#   sh -eu /w/goodwe-addon/.gitea/ci/run-tests.sh
# and runnable the same way from a clone (see TESTS_DIR below).
#
# WHY: the serial loop ran past the runner's 60-minute limit (SAFETY-125: 85 files,
# 3 557 s of test time, run 1430 cut off at file 84). The host has 12 cores.
#
# SHAPE
#   1. A parallel batch: JOBS files at a time (xargs -P), longest first.
#   2. THEN the serial group, one at a time, after the batch has finished, so
#      each runs on a machine with nothing else of ours on it.
#   Every file writes to its own $RUN_DIR/<file>.out (there is no shared output
#   file) and prints its result as ONE block under a lock, so the PASS/FAIL
#   lines and the `CI: ` / tail -40 text of two files never interleave.
#
# THE FILE LIST IS STILL DERIVED FROM THE TREE (see checks.yml). The two typed
# lists below only SORT or SET ASIDE files that exist; a new test_*.py that is
# in neither list is run, in the parallel batch. A name that no longer exists
# in a list is harmless. Nothing can make a file silently not run.
#
# SERIAL GROUP: files whose assertions depend on the wall clock, so CPU load
# from siblings could flip them. Name and reason, one each:
#   test_loop.py    drives the real run_control() against a real wall-clock
#                   floor (~93 s on purpose); worst inter-cycle gap 3.0 s
#                   against a 4.5 s bound (TEST-30, CI-06).
#   test_freeze.py  part 3 runs the real sim/hwsim.py over a transport with a
#                   16 s window, a 10 s hold and an alarm measured at 11.0 s:
#                   one poll of margin, and "failed 1 run in 3" on the rig
#                   transport (TEST-30, CI-06).
#   test_safety87.py  builds a controller (CH65) whose meter reading is stamped
#                   on the REAL clock, and section 8 asks its status page,
#                   which says "Not ready - Meter: no reading for N seconds,
#                   limit 30 seconds" once N passes 30. Measured by CI-15: N is
#                   about half the file's run time (N = 45 s in a 94 s run, 48 s
#                   in a 103 s run). The file takes ~25 s idle and took 45 s in
#                   CI's first parallel run, so N was ~20 s of the 30. It FAILED
#                   alone under --cpus 0.15 (94 s) and in a loaded parallel run
#                   (103 s). No test file is changed here, so it runs alone.
# Add a file here only with a measured reason, and say it in this list.
SERIAL="test_loop.py test_freeze.py test_safety87.py"
#
# LONGEST FIRST (SAFETY-125's CI run, job 2580; then a local parallel run, seconds:
# modes_p4 758, safety126 634 (SAFETY-126, local, one core), safety125 508 on hotfix-0.19.2 only, modes_p3 364, safety119 328,
# price_run_fixes 184, safety90 160, safety103 134, safety94 114, peakguard 113,
# opt20 102, safety97 101, safety81 100). With JOBS workers pulling from one
# queue, starting the longest files first keeps the tail short: the batch takes
# about as long as the longest file. A sort order, not a selection.
HEAVY="test_modes_p4.py test_safety126.py test_safety125.py test_modes_p3.py test_safety119.py test_price_run_fixes.py test_safety90.py test_safety103.py test_safety94.py test_peakguard.py test_opt20.py test_safety97.py test_safety81.py"
#
JOBS=${CI_JOBS:-6}
cd "${TESTS_DIR:-/w/goodwe-addon/goodwe_controller}"

# one FILE: run it, keep its output in its own file, print its block whole.
if [ "${1:-}" = "--one" ]; then
  f=$2
  o="$RUN_DIR/$f.out"
  start=$(date +%s)
  if python3 "$f" > "$o" 2>&1; then rc=0; else rc=1; fi
  secs=$(( $(date +%s) - start ))
  {
    if [ $rc = 0 ]; then
      echo "PASS  $f  (${secs}s)"
      grep "^CI: " "$o" || true
    else
      echo "FAIL  $f  (${secs}s)"
      tail -40 "$o"
    fi
  } > "$RUN_DIR/$f.blk"
  echo $rc > "$RUN_DIR/$f.rc"
  flock "$RUN_DIR/print.lock" cat "$RUN_DIR/$f.blk"
  exit 0
fi

set -- test_*.py
if [ "$1" = "test_*.py" ]; then
  echo "NO test_*.py FOUND - the mount is empty or the layout moved. NOTHING has been checked."
  exit 1
fi
echo "$# test file(s) to run: $JOBS at a time, then the serial group"

RUN_DIR=$(mktemp -d /tmp/ci-tests.XXXXXX)
export RUN_DIR
: > "$RUN_DIR/par"
: > "$RUN_DIR/ser"
for f in $HEAVY; do
  case " $SERIAL " in *" $f "*) continue ;; esac
  if [ -f "$f" ]; then echo "$f" >> "$RUN_DIR/par"; fi
done
for f in "$@"; do
  case " $SERIAL $HEAVY " in *" $f "*) continue ;; esac
  echo "$f" >> "$RUN_DIR/par"
done
for f in $SERIAL; do
  if [ -f "$f" ]; then echo "$f" >> "$RUN_DIR/ser"; fi
done
echo "parallel: $(wc -l < "$RUN_DIR/par") file(s); serial: $(wc -l < "$RUN_DIR/ser") file(s)"

t0=$(date +%s)
rc=0
# `exit 0` in the worker: a failing FILE is recorded in its .rc, so a non-zero
# from xargs itself means the machinery broke, which is a failure too.
xargs -n 1 -P "$JOBS" sh -eu "$0" --one < "$RUN_DIR/par" || rc=1
echo "parallel batch done in $(( $(date +%s) - t0 ))s; serial group next"
while read -r f; do
  sh -eu "$0" --one "$f" || rc=1
done < "$RUN_DIR/ser"

# Every file that was meant to run has a verdict, and every verdict is PASS. A
# worker that died before writing its .rc leaves no file, which counts as a fail.
ran=0
bad=0
for f in "$@"; do
  ran=$((ran + 1))
  if [ ! -f "$RUN_DIR/$f.rc" ]; then
    echo "FAIL  $f  (no result: the run never reported)"
    bad=$((bad + 1))
  elif [ "$(cat "$RUN_DIR/$f.rc")" != 0 ]; then
    bad=$((bad + 1))
  fi
done
echo "$ran file(s), $bad failed, $(( $(date +%s) - t0 ))s wall"
if [ "$bad" = 0 ] && [ "$rc" = 0 ]; then exit 0; fi
exit 1