96 lines
3.9 KiB
Bash
Executable File
96 lines
3.9 KiB
Bash
Executable File
#!/bin/sh
|
|
# The house mutation harness (TEST-15). `tools/mutation_run.sh [TREE] [test_*.py ...]`
|
|
#
|
|
# WHY THIS EXISTS. Nearly every safety claim in this repository was verified by
|
|
# mutation: change the shipped module, run the suite, watch it go red, revert,
|
|
# watch it go green. FCST-03's developer found that the middle two steps can be
|
|
# a lie. CPython decides a `.pyc` is still valid from (source size, source mtime
|
|
# TRUNCATED TO WHOLE SECONDS). Revert `self.provisional` to `here.provisional`
|
|
# and neither term moves - same length, same second - so the interpreter loads
|
|
# the bytecode it already has. Confirmed by disassembly: the source read `here`,
|
|
# the running code loaded `self`. The mutation "survived" because it never ran.
|
|
#
|
|
# ⚠️ `python3 -B` IS NOT THE FIX, AND NEITHER IS PYTHONDONTWRITEBYTECODE=1.
|
|
# Both stop bytecode being WRITTEN. Neither stops a stale `.pyc` that is already
|
|
# on disk from being READ - measured, and graded by test_bytecode.py part 2.
|
|
# The load-bearing half is the PURGE. `-B` is here so the run leaves no cache
|
|
# behind for the NEXT mutation to inherit.
|
|
#
|
|
# ⚠️ NOT A SLEEP. One-second mtime resolution is the mechanism, not the enemy;
|
|
# the answer is to stop trusting mtime, not to make every edit wait a second.
|
|
# A sleep also only covers the edit you remembered to slow down.
|
|
#
|
|
# It prints a fingerprint of the tree it is about to grade, because the other
|
|
# half of a mutation report is WHICH SOURCE produced it: a report that cannot
|
|
# say what was on disk is the same unverifiable claim in a different dress.
|
|
# Diff two fingerprints and the mutated file is the line that changed.
|
|
set -eu
|
|
|
|
TREE=""
|
|
case "${1-}" in
|
|
"" ) : ;;
|
|
-h|--help ) sed -n '2,30p' "$0"; exit 0 ;;
|
|
* ) if [ -d "$1" ]; then TREE=$1; shift; fi ;;
|
|
esac
|
|
if [ -z "$TREE" ]; then
|
|
# Default: the add-on directory this script ships in, so `tools/mutation_run.sh`
|
|
# works from anywhere. An explicit TREE is what lets the infra repo's replay
|
|
# harnesses purge $GOODWE_ADDON_REPO before they grade it - see MUTATION.md.
|
|
TREE=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
|
fi
|
|
PY=${PY:-python3}
|
|
|
|
cd "$TREE"
|
|
|
|
# CI-01's rule, for the same reason: a loop over an empty glob exits 0, and a
|
|
# green run that examined nothing is worse than a red one.
|
|
if [ $# -eq 0 ]; then set -- test_*.py; fi
|
|
if [ ! -f "$1" ]; then
|
|
echo "mutation_run: no test_*.py under $TREE - NOTHING has been graded." >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo "== purge"
|
|
find . -name '__pycache__' -type d -prune -exec rm -rf {} + 2>/dev/null || true
|
|
find . -name '*.pyc' -type f -exec rm -f {} + 2>/dev/null || true
|
|
left=$(find . -name '__pycache__' -type d -o -name '*.pyc' -type f | head -5)
|
|
if [ -n "$left" ]; then
|
|
# A read-only tree cannot hold a cache written during this run either, so this
|
|
# is only reachable when the purge itself failed on a tree that HAS one - the
|
|
# one state where continuing would grade unknown bytecode.
|
|
echo "mutation_run: could not purge, and a cache remains:" >&2
|
|
echo "$left" >&2
|
|
exit 1
|
|
fi
|
|
echo " no bytecode cache under $TREE"
|
|
|
|
echo "== tree graded (sha256, bytes)"
|
|
find app -name '*.py' 2>/dev/null | sort | while read -r f; do
|
|
printf ' %s %8d %s\n' "$(sha256sum "$f" | cut -c1-16)" "$(wc -c < "$f")" "$f"
|
|
done
|
|
|
|
echo "== run ($PY -B)"
|
|
rc=0
|
|
for f in "$@"; do
|
|
start=$(date +%s)
|
|
if "$PY" -B "$f" > /tmp/mutation_run.out 2>&1; then
|
|
echo " PASS $f ($(( $(date +%s) - start ))s)"
|
|
else
|
|
echo " FAIL $f ($(( $(date +%s) - start ))s)"
|
|
tail -40 /tmp/mutation_run.out
|
|
rc=1
|
|
fi
|
|
done
|
|
|
|
# -B was asked for; check it was honoured rather than assuming. A cache here
|
|
# would be inherited by the next mutation in the batch, which is exactly the
|
|
# state this script exists to prevent.
|
|
after=$(find . -name '__pycache__' -type d -o -name '*.pyc' -type f | head -5)
|
|
if [ -n "$after" ]; then
|
|
echo "mutation_run: a bytecode cache appeared DURING the run - the next" >&2
|
|
echo "mutation in this batch would inherit it:" >&2
|
|
echo "$after" >&2
|
|
rc=1
|
|
fi
|
|
exit $rc
|