Files
glenn schrooyenandClaude Opus 5.5 22ef89bf10
checks / unit (pull_request) Successful in 20m22s
infra-pin / fresh (pull_request) Successful in 1s
CI-18 review: --head needs B checked out; guard the CI image against CI_ONLY
The selector reads the graph, and run-affected.sh runs the tests, from the
working tree, so a --base/--head range is only right with the head checked
out. The CI image is built from the merged tree, so an ENV CI_ONLY in its
Dockerfile would make the gate partial; test_affected_tests.py now fails on it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-15 23:29:34 +01:00

274 lines
12 KiB
Python

"""Runnable check for CI-18's selector. `python3 test_affected_tests.py`
`.gitea/ci/affected_tests.py` says which test_*.py files a diff affects, so a
developer's round can skip the rest. It is used ONLY for local rounds; the merge
gate (`checks / unit (pull_request)`) runs every file. So the properties graded
here are the ones whose failure would be a missed test:
1. A synthetic tree: transitive imports, a tool launched by path, a module
named in a string, and what must NOT be chosen.
2. FAIL OPEN: a non-Python file, a deletion, a .py nothing references, a
file that does not parse, a base that does not exist, all select ALL.
3. The real tree: the safety set names files that exist, config.yaml and
run-tests.sh select ALL, a changed test file is chosen.
4. THE MUTANT: break a real module in a private copy of the tree; the
selector must choose a test that goes red, and the unbroken copy is green.
5. The merge gate cannot be partial: checks.yml and the CI image never
mention CI_ONLY, and checks.yml still runs the unfiltered run-tests.sh line.
Read MUTATION.md before you report a survivor.
"""
import os
import shutil
import subprocess
import sys
import tempfile
HERE = os.path.dirname(os.path.abspath(__file__))
REPO = os.path.dirname(HERE)
SEL = os.path.join(REPO, ".gitea", "ci", "affected_tests.py")
sys.path.insert(0, os.path.dirname(SEL))
import affected_tests as sel # noqa: E402
fails = []
tmps = []
def check(name, cond):
print(f" {'ok ' if cond else 'FAIL'} {name}")
if not cond:
fails.append(name)
def mk(root, files):
for rel, text in files.items():
p = os.path.join(root, rel)
os.makedirs(os.path.dirname(p), exist_ok=True)
with open(p, "w", encoding="utf-8") as fh:
fh.write(text)
def tmp():
d = tempfile.mkdtemp(prefix="test_affected.")
tmps.append(d)
return d
def choose(root, paths, status="M"):
names, _r, why = sel.select(root, [(status, p) for p in paths])
return ([os.path.basename(n) for n in names] if names is not None else None,
why)
# ---------------------------------------------------------------- 1. synthetic
print("1. a synthetic tree")
R = tmp()
P = "goodwe_controller/"
mk(R, {
P + "app/__init__.py": "",
P + "app/low.py": "def f():\n return 1\n",
P + "app/mid.py": "from app import low\n\ndef g():\n return low.f()\n",
P + "app/top.py": "from app.mid import g\n",
P + "app/lonely.py": "X = 1\n",
P + "app/viastr.py": "Y = 2\n",
P + "tools/launched.py": "print('hi')\n",
P + "tools/reader.py": "print('r')\n",
P + "test_a.py": "from app import top\n",
P + "test_b.py": "from app import lonely\n",
P + "test_c.py": "import subprocess\nsubprocess.run(['python3', 'tools/launched.py'])\n",
P + "test_d.py": "import importlib\nm = importlib.import_module('app.viastr')\n",
# reads every test_*.py as text: affected by a changed test, but it is a
# reader, not a dependency of anything that imports a module
P + "test_reader.py": "import glob\nfor f in glob.glob('test_*.py'):\n open(f).read()\n",
P + "test_unrelated.py": "print('x')\n",
# a test imported by another test carries the change on (an import runs it)
P + "test_base.py": "from app import lonely\n",
P + "test_chain.py": "import test_base\n",
})
n, why = choose(R, [P + "app/low.py"])
check("a change two imports down chooses the test that imports the top (transitive)",
n is not None and "test_a.py" in n)
check("...and not tests that do not reach it",
n is not None and not {"test_b.py", "test_c.py", "test_unrelated.py"} & set(n))
n, why = choose(R, [P + "app/lonely.py"])
check("a direct importer is chosen", n is not None and "test_b.py" in n)
check("a test that imports another test is chosen with it, even though "
"test_reader (a pattern reader) also reaches test_base as text",
n is not None and "test_chain.py" in n and "test_base.py" in n)
n, why = choose(R, [P + "tools/launched.py"])
check("a tool launched by path in a string chooses the launching test",
n is not None and "test_c.py" in n)
n, why = choose(R, [P + "app/viastr.py"])
check("a module imported dynamically by name chooses its test",
n is not None and "test_d.py" in n)
n, why = choose(R, [P + "test_unrelated.py"])
check("a changed test file is chosen", n is not None and "test_unrelated.py" in n)
check("a test that only READS test files by pattern is chosen with it",
n is not None and "test_reader.py" in n)
check("...but a reader does not drag in the importers of an unrelated module",
n is not None and "test_a.py" not in n and "test_b.py" not in n)
n, why = choose(R, [P + "app/low.py", P + "tools/launched.py"])
check("two changes choose the union",
n is not None and {"test_a.py", "test_c.py"} <= set(n))
# the safety set is always in, whatever changed
R2 = tmp()
files = {P + "app/__init__.py": "", P + "app/m.py": "A = 1\n",
P + "test_m.py": "from app import m\n", P + "test_zzz.py": "print(1)\n"}
for name in sel.SAFETY_SET:
files[P + name] = "print('safety')\n"
mk(R2, files)
n, why = choose(R2, [P + "app/m.py"])
check("every safety-set file is chosen on a one-module change",
n is not None and set(sel.SAFETY_SET) <= set(n))
check("...with a reason printed for each", all(sel.SAFETY_SET.values()))
check("...and a test outside the set that does not reach it stays out",
n is not None and "test_zzz.py" not in n)
# ------------------------------------------------------------ 2. fails open
print("\n2. it fails open")
for label, paths, status in (
("config.yaml (not Python)", [P + "config.yaml"], "M"),
("a translation file", [P + "translations/en.yaml"], "M"),
("run-tests.sh itself", [".gitea/ci/run-tests.sh"], "M"),
("a Dockerfile", [P + "Dockerfile"], "M"),
("a markdown file", ["README.md"], "M"),
("a deleted module", [P + "app/low.py"], "D"),
("a renamed module", [P + "app/low.py"], "R"),
("a .py nothing references", [P + "app/never_imported.py"], "M")):
if label.startswith("a .py"):
mk(R, {P + "app/never_imported.py": "Z = 1\n"})
n, why = choose(R, paths, status)
check(f"{label}: SELECT ALL", n is None and why)
mk(R, {P + "app/broken.py": "def (:\n"})
n, why = choose(R, [P + "app/low.py"])
check("a file in the tree that does not parse: SELECT ALL", n is None and why)
os.remove(os.path.join(R, P + "app/broken.py"))
n, why = choose(tmp(), [P + "app/low.py"])
check("a tree with no test files: SELECT ALL", n is None and why)
# the CLI: --names prints every test on SELECT ALL, and a bad base is ALL
g = tmp()
mk(g, {P + "app/__init__.py": "", P + "app/m.py": "A = 1\n",
P + "test_m.py": "from app import m\n"})
env = {**os.environ, "GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t",
"GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@t"}
def git(*a):
return subprocess.run(["git", "-C", g] + list(a), env=env,
capture_output=True, text=True)
git("init", "-q", "-b", "base")
git("add", "-A")
git("commit", "-q", "-m", "base")
git("checkout", "-q", "-b", "work")
mk(g, {P + "app/m.py": "A = 2\n"})
git("commit", "-q", "-am", "work")
r = subprocess.run([sys.executable, SEL, "--root", g, "--base", "base",
"--head", "work", "--names"], capture_output=True, text=True)
check("a committed range via git: the importer is chosen",
r.returncode == 0 and r.stdout.split() == ["test_m.py"])
r = subprocess.run([sys.executable, SEL, "--root", g, "--base", "nosuchref",
"--head", "work"], capture_output=True, text=True)
check("a base that does not exist: SELECT ALL, not an error and not nothing",
r.returncode == 0 and "SELECT ALL" in r.stdout)
mk(g, {P + "app/m.py": "A = 3\n"})
r = subprocess.run([sys.executable, SEL, "--root", g, "--base", "base",
"--names"], capture_output=True, text=True)
check("uncommitted work is part of the diff (no --head)",
r.returncode == 0 and r.stdout.split() == ["test_m.py"])
mk(g, {P + "app/newmod.py": "B = 1\n", P + "test_new.py": "from app import newmod\n"})
r = subprocess.run([sys.executable, SEL, "--root", g, "--base", "base",
"--names"], capture_output=True, text=True)
check("an untracked new module and its new test are in the diff",
r.returncode == 0 and set(r.stdout.split()) == {"test_m.py", "test_new.py"})
# ---------------------------------------------------------------- 3. real tree
print("\n3. the real tree")
tests = sorted(f for f in os.listdir(HERE)
if f.startswith("test_") and f.endswith(".py"))
check("every safety-set name is a test file that exists",
all(f in tests for f in sel.SAFETY_SET))
for p in ("goodwe_controller/config.yaml", ".gitea/ci/run-tests.sh",
".gitea/workflows/checks.yml", "goodwe_controller/DOCS.md",
"goodwe_controller/translations/en.yaml"):
n, why = choose(REPO, [p])
check(f"{p}: SELECT ALL", n is None and why)
n, why = choose(REPO, ["goodwe_controller/app/maintenance.py"])
check("app/maintenance.py chooses test_maint25.py (it imports it through main)",
n is not None and "test_maint25.py" in n)
n, why = choose(REPO, ["goodwe_controller/test_maint25.py"])
check("a changed test chooses itself and the safety set",
n is not None and "test_maint25.py" in n and set(sel.SAFETY_SET) <= set(n))
check("...and far fewer than all",
n is not None and len(n) < len(tests) // 2)
n, why = choose(REPO, [".gitea/ci/affected_tests.py"])
check("the selector's own file chooses this file",
n is not None and "test_affected_tests.py" in n)
# ------------------------------------------------------------------ 4. mutant
print("\n4. the mutant: break a module, the selector must choose a red test")
M = tmp()
ign = shutil.ignore_patterns(".git", "scenarios", "__pycache__", "*.pyc")
shutil.copytree(REPO, os.path.join(M, "repo"), ignore=ign, symlinks=True)
mroot = os.path.join(M, "repo")
cwd = os.path.join(mroot, "goodwe_controller")
env2 = {**os.environ, "PYTHONDONTWRITEBYTECODE": "1"}
def run_test(name):
r = subprocess.run([sys.executable, "-B", name], cwd=cwd, env=env2,
capture_output=True, text=True, timeout=600)
return r.returncode
check("control: test_maint25.py is green on the unbroken copy",
run_test("test_maint25.py") == 0)
path = os.path.join(cwd, "app", "maintenance.py")
with open(path, encoding="utf-8") as fh:
src = fh.read()
old = " return until if now < until else None\n"
check("the line to break is where this test expects it", src.count(old) == 1)
with open(path, "w", encoding="utf-8") as fh:
fh.write(src.replace(old, " return None\n"))
n, why = choose(mroot, ["goodwe_controller/app/maintenance.py"])
check("the selector chooses test_maint25.py for the broken module",
n is not None and "test_maint25.py" in n)
check("...and running the chosen file is RED", run_test("test_maint25.py") != 0)
with open(path, "w", encoding="utf-8") as fh:
fh.write(src)
check("put back: GREEN again", run_test("test_maint25.py") == 0)
# --------------------------------------------- 5. the gate cannot be partial
print("\n5. the merge gate runs everything")
with open(os.path.join(REPO, ".gitea", "workflows", "checks.yml"),
encoding="utf-8") as fh:
wf = fh.read()
check("checks.yml never mentions CI_ONLY", "CI_ONLY" not in wf)
check("checks.yml runs run-tests.sh with no file list",
"sh -eu /w/goodwe-addon/.gitea/ci/run-tests.sh\n" in wf)
check("checks.yml never calls the selector or the local wrapper",
"affected_tests" not in wf and "run-affected" not in wf)
with open(os.path.join(REPO, ".gitea", "ci", "run-tests.sh"),
encoding="utf-8") as fh:
rt = fh.read()
check("run-tests.sh takes CI_ONLY from the environment only and says PARTIAL",
"PARTIAL RUN" in rt and 'set -- test_*.py' in rt)
with open(os.path.join(REPO, ".gitea", "ci", "Dockerfile"),
encoding="utf-8") as fh:
df = fh.read()
check("the CI image (built from the merged tree) never sets CI_ONLY",
"CI_ONLY" not in df)
for d in tmps:
shutil.rmtree(d, ignore_errors=True)
print()
if fails:
print(f"{len(fails)} FAILED:")
for f in fails:
print(f" - {f}")
sys.exit(1)
print("all checks passed")