The selector reads the graph, and run-affected.sh runs the tests, from the working tree, so a --base/--head range is only right with the head checked out. The CI image is built from the merged tree, so an ENV CI_ONLY in its Dockerfile would make the gate partial; test_affected_tests.py now fails on it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
274 lines
12 KiB
Python
274 lines
12 KiB
Python
"""Runnable check for CI-18's selector. `python3 test_affected_tests.py`
|
|
|
|
`.gitea/ci/affected_tests.py` says which test_*.py files a diff affects, so a
|
|
developer's round can skip the rest. It is used ONLY for local rounds; the merge
|
|
gate (`checks / unit (pull_request)`) runs every file. So the properties graded
|
|
here are the ones whose failure would be a missed test:
|
|
|
|
1. A synthetic tree: transitive imports, a tool launched by path, a module
|
|
named in a string, and what must NOT be chosen.
|
|
2. FAIL OPEN: a non-Python file, a deletion, a .py nothing references, a
|
|
file that does not parse, a base that does not exist, all select ALL.
|
|
3. The real tree: the safety set names files that exist, config.yaml and
|
|
run-tests.sh select ALL, a changed test file is chosen.
|
|
4. THE MUTANT: break a real module in a private copy of the tree; the
|
|
selector must choose a test that goes red, and the unbroken copy is green.
|
|
5. The merge gate cannot be partial: checks.yml and the CI image never
|
|
mention CI_ONLY, and checks.yml still runs the unfiltered run-tests.sh line.
|
|
Read MUTATION.md before you report a survivor.
|
|
"""
|
|
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
REPO = os.path.dirname(HERE)
|
|
SEL = os.path.join(REPO, ".gitea", "ci", "affected_tests.py")
|
|
sys.path.insert(0, os.path.dirname(SEL))
|
|
import affected_tests as sel # noqa: E402
|
|
|
|
fails = []
|
|
tmps = []
|
|
|
|
|
|
def check(name, cond):
|
|
print(f" {'ok ' if cond else 'FAIL'} {name}")
|
|
if not cond:
|
|
fails.append(name)
|
|
|
|
|
|
def mk(root, files):
|
|
for rel, text in files.items():
|
|
p = os.path.join(root, rel)
|
|
os.makedirs(os.path.dirname(p), exist_ok=True)
|
|
with open(p, "w", encoding="utf-8") as fh:
|
|
fh.write(text)
|
|
|
|
|
|
def tmp():
|
|
d = tempfile.mkdtemp(prefix="test_affected.")
|
|
tmps.append(d)
|
|
return d
|
|
|
|
|
|
def choose(root, paths, status="M"):
|
|
names, _r, why = sel.select(root, [(status, p) for p in paths])
|
|
return ([os.path.basename(n) for n in names] if names is not None else None,
|
|
why)
|
|
|
|
|
|
# ---------------------------------------------------------------- 1. synthetic
|
|
print("1. a synthetic tree")
|
|
R = tmp()
|
|
P = "goodwe_controller/"
|
|
mk(R, {
|
|
P + "app/__init__.py": "",
|
|
P + "app/low.py": "def f():\n return 1\n",
|
|
P + "app/mid.py": "from app import low\n\ndef g():\n return low.f()\n",
|
|
P + "app/top.py": "from app.mid import g\n",
|
|
P + "app/lonely.py": "X = 1\n",
|
|
P + "app/viastr.py": "Y = 2\n",
|
|
P + "tools/launched.py": "print('hi')\n",
|
|
P + "tools/reader.py": "print('r')\n",
|
|
P + "test_a.py": "from app import top\n",
|
|
P + "test_b.py": "from app import lonely\n",
|
|
P + "test_c.py": "import subprocess\nsubprocess.run(['python3', 'tools/launched.py'])\n",
|
|
P + "test_d.py": "import importlib\nm = importlib.import_module('app.viastr')\n",
|
|
# reads every test_*.py as text: affected by a changed test, but it is a
|
|
# reader, not a dependency of anything that imports a module
|
|
P + "test_reader.py": "import glob\nfor f in glob.glob('test_*.py'):\n open(f).read()\n",
|
|
P + "test_unrelated.py": "print('x')\n",
|
|
# a test imported by another test carries the change on (an import runs it)
|
|
P + "test_base.py": "from app import lonely\n",
|
|
P + "test_chain.py": "import test_base\n",
|
|
})
|
|
n, why = choose(R, [P + "app/low.py"])
|
|
check("a change two imports down chooses the test that imports the top (transitive)",
|
|
n is not None and "test_a.py" in n)
|
|
check("...and not tests that do not reach it",
|
|
n is not None and not {"test_b.py", "test_c.py", "test_unrelated.py"} & set(n))
|
|
n, why = choose(R, [P + "app/lonely.py"])
|
|
check("a direct importer is chosen", n is not None and "test_b.py" in n)
|
|
check("a test that imports another test is chosen with it, even though "
|
|
"test_reader (a pattern reader) also reaches test_base as text",
|
|
n is not None and "test_chain.py" in n and "test_base.py" in n)
|
|
n, why = choose(R, [P + "tools/launched.py"])
|
|
check("a tool launched by path in a string chooses the launching test",
|
|
n is not None and "test_c.py" in n)
|
|
n, why = choose(R, [P + "app/viastr.py"])
|
|
check("a module imported dynamically by name chooses its test",
|
|
n is not None and "test_d.py" in n)
|
|
n, why = choose(R, [P + "test_unrelated.py"])
|
|
check("a changed test file is chosen", n is not None and "test_unrelated.py" in n)
|
|
check("a test that only READS test files by pattern is chosen with it",
|
|
n is not None and "test_reader.py" in n)
|
|
check("...but a reader does not drag in the importers of an unrelated module",
|
|
n is not None and "test_a.py" not in n and "test_b.py" not in n)
|
|
n, why = choose(R, [P + "app/low.py", P + "tools/launched.py"])
|
|
check("two changes choose the union",
|
|
n is not None and {"test_a.py", "test_c.py"} <= set(n))
|
|
|
|
# the safety set is always in, whatever changed
|
|
R2 = tmp()
|
|
files = {P + "app/__init__.py": "", P + "app/m.py": "A = 1\n",
|
|
P + "test_m.py": "from app import m\n", P + "test_zzz.py": "print(1)\n"}
|
|
for name in sel.SAFETY_SET:
|
|
files[P + name] = "print('safety')\n"
|
|
mk(R2, files)
|
|
n, why = choose(R2, [P + "app/m.py"])
|
|
check("every safety-set file is chosen on a one-module change",
|
|
n is not None and set(sel.SAFETY_SET) <= set(n))
|
|
check("...with a reason printed for each", all(sel.SAFETY_SET.values()))
|
|
check("...and a test outside the set that does not reach it stays out",
|
|
n is not None and "test_zzz.py" not in n)
|
|
|
|
# ------------------------------------------------------------ 2. fails open
|
|
print("\n2. it fails open")
|
|
for label, paths, status in (
|
|
("config.yaml (not Python)", [P + "config.yaml"], "M"),
|
|
("a translation file", [P + "translations/en.yaml"], "M"),
|
|
("run-tests.sh itself", [".gitea/ci/run-tests.sh"], "M"),
|
|
("a Dockerfile", [P + "Dockerfile"], "M"),
|
|
("a markdown file", ["README.md"], "M"),
|
|
("a deleted module", [P + "app/low.py"], "D"),
|
|
("a renamed module", [P + "app/low.py"], "R"),
|
|
("a .py nothing references", [P + "app/never_imported.py"], "M")):
|
|
if label.startswith("a .py"):
|
|
mk(R, {P + "app/never_imported.py": "Z = 1\n"})
|
|
n, why = choose(R, paths, status)
|
|
check(f"{label}: SELECT ALL", n is None and why)
|
|
mk(R, {P + "app/broken.py": "def (:\n"})
|
|
n, why = choose(R, [P + "app/low.py"])
|
|
check("a file in the tree that does not parse: SELECT ALL", n is None and why)
|
|
os.remove(os.path.join(R, P + "app/broken.py"))
|
|
n, why = choose(tmp(), [P + "app/low.py"])
|
|
check("a tree with no test files: SELECT ALL", n is None and why)
|
|
|
|
# the CLI: --names prints every test on SELECT ALL, and a bad base is ALL
|
|
g = tmp()
|
|
mk(g, {P + "app/__init__.py": "", P + "app/m.py": "A = 1\n",
|
|
P + "test_m.py": "from app import m\n"})
|
|
env = {**os.environ, "GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t",
|
|
"GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@t"}
|
|
|
|
|
|
def git(*a):
|
|
return subprocess.run(["git", "-C", g] + list(a), env=env,
|
|
capture_output=True, text=True)
|
|
|
|
|
|
git("init", "-q", "-b", "base")
|
|
git("add", "-A")
|
|
git("commit", "-q", "-m", "base")
|
|
git("checkout", "-q", "-b", "work")
|
|
mk(g, {P + "app/m.py": "A = 2\n"})
|
|
git("commit", "-q", "-am", "work")
|
|
r = subprocess.run([sys.executable, SEL, "--root", g, "--base", "base",
|
|
"--head", "work", "--names"], capture_output=True, text=True)
|
|
check("a committed range via git: the importer is chosen",
|
|
r.returncode == 0 and r.stdout.split() == ["test_m.py"])
|
|
r = subprocess.run([sys.executable, SEL, "--root", g, "--base", "nosuchref",
|
|
"--head", "work"], capture_output=True, text=True)
|
|
check("a base that does not exist: SELECT ALL, not an error and not nothing",
|
|
r.returncode == 0 and "SELECT ALL" in r.stdout)
|
|
mk(g, {P + "app/m.py": "A = 3\n"})
|
|
r = subprocess.run([sys.executable, SEL, "--root", g, "--base", "base",
|
|
"--names"], capture_output=True, text=True)
|
|
check("uncommitted work is part of the diff (no --head)",
|
|
r.returncode == 0 and r.stdout.split() == ["test_m.py"])
|
|
mk(g, {P + "app/newmod.py": "B = 1\n", P + "test_new.py": "from app import newmod\n"})
|
|
r = subprocess.run([sys.executable, SEL, "--root", g, "--base", "base",
|
|
"--names"], capture_output=True, text=True)
|
|
check("an untracked new module and its new test are in the diff",
|
|
r.returncode == 0 and set(r.stdout.split()) == {"test_m.py", "test_new.py"})
|
|
|
|
# ---------------------------------------------------------------- 3. real tree
|
|
print("\n3. the real tree")
|
|
tests = sorted(f for f in os.listdir(HERE)
|
|
if f.startswith("test_") and f.endswith(".py"))
|
|
check("every safety-set name is a test file that exists",
|
|
all(f in tests for f in sel.SAFETY_SET))
|
|
for p in ("goodwe_controller/config.yaml", ".gitea/ci/run-tests.sh",
|
|
".gitea/workflows/checks.yml", "goodwe_controller/DOCS.md",
|
|
"goodwe_controller/translations/en.yaml"):
|
|
n, why = choose(REPO, [p])
|
|
check(f"{p}: SELECT ALL", n is None and why)
|
|
n, why = choose(REPO, ["goodwe_controller/app/maintenance.py"])
|
|
check("app/maintenance.py chooses test_maint25.py (it imports it through main)",
|
|
n is not None and "test_maint25.py" in n)
|
|
n, why = choose(REPO, ["goodwe_controller/test_maint25.py"])
|
|
check("a changed test chooses itself and the safety set",
|
|
n is not None and "test_maint25.py" in n and set(sel.SAFETY_SET) <= set(n))
|
|
check("...and far fewer than all",
|
|
n is not None and len(n) < len(tests) // 2)
|
|
n, why = choose(REPO, [".gitea/ci/affected_tests.py"])
|
|
check("the selector's own file chooses this file",
|
|
n is not None and "test_affected_tests.py" in n)
|
|
|
|
# ------------------------------------------------------------------ 4. mutant
|
|
print("\n4. the mutant: break a module, the selector must choose a red test")
|
|
M = tmp()
|
|
ign = shutil.ignore_patterns(".git", "scenarios", "__pycache__", "*.pyc")
|
|
shutil.copytree(REPO, os.path.join(M, "repo"), ignore=ign, symlinks=True)
|
|
mroot = os.path.join(M, "repo")
|
|
cwd = os.path.join(mroot, "goodwe_controller")
|
|
env2 = {**os.environ, "PYTHONDONTWRITEBYTECODE": "1"}
|
|
|
|
|
|
def run_test(name):
|
|
r = subprocess.run([sys.executable, "-B", name], cwd=cwd, env=env2,
|
|
capture_output=True, text=True, timeout=600)
|
|
return r.returncode
|
|
|
|
|
|
check("control: test_maint25.py is green on the unbroken copy",
|
|
run_test("test_maint25.py") == 0)
|
|
path = os.path.join(cwd, "app", "maintenance.py")
|
|
with open(path, encoding="utf-8") as fh:
|
|
src = fh.read()
|
|
old = " return until if now < until else None\n"
|
|
check("the line to break is where this test expects it", src.count(old) == 1)
|
|
with open(path, "w", encoding="utf-8") as fh:
|
|
fh.write(src.replace(old, " return None\n"))
|
|
n, why = choose(mroot, ["goodwe_controller/app/maintenance.py"])
|
|
check("the selector chooses test_maint25.py for the broken module",
|
|
n is not None and "test_maint25.py" in n)
|
|
check("...and running the chosen file is RED", run_test("test_maint25.py") != 0)
|
|
with open(path, "w", encoding="utf-8") as fh:
|
|
fh.write(src)
|
|
check("put back: GREEN again", run_test("test_maint25.py") == 0)
|
|
|
|
# --------------------------------------------- 5. the gate cannot be partial
|
|
print("\n5. the merge gate runs everything")
|
|
with open(os.path.join(REPO, ".gitea", "workflows", "checks.yml"),
|
|
encoding="utf-8") as fh:
|
|
wf = fh.read()
|
|
check("checks.yml never mentions CI_ONLY", "CI_ONLY" not in wf)
|
|
check("checks.yml runs run-tests.sh with no file list",
|
|
"sh -eu /w/goodwe-addon/.gitea/ci/run-tests.sh\n" in wf)
|
|
check("checks.yml never calls the selector or the local wrapper",
|
|
"affected_tests" not in wf and "run-affected" not in wf)
|
|
with open(os.path.join(REPO, ".gitea", "ci", "run-tests.sh"),
|
|
encoding="utf-8") as fh:
|
|
rt = fh.read()
|
|
check("run-tests.sh takes CI_ONLY from the environment only and says PARTIAL",
|
|
"PARTIAL RUN" in rt and 'set -- test_*.py' in rt)
|
|
with open(os.path.join(REPO, ".gitea", "ci", "Dockerfile"),
|
|
encoding="utf-8") as fh:
|
|
df = fh.read()
|
|
check("the CI image (built from the merged tree) never sets CI_ONLY",
|
|
"CI_ONLY" not in df)
|
|
|
|
for d in tmps:
|
|
shutil.rmtree(d, ignore_errors=True)
|
|
print()
|
|
if fails:
|
|
print(f"{len(fails)} FAILED:")
|
|
for f in fails:
|
|
print(f" - {f}")
|
|
sys.exit(1)
|
|
print("all checks passed")
|