"""Runnable check for CI-18's selector. `python3 test_affected_tests.py` `.gitea/ci/affected_tests.py` says which test_*.py files a diff affects, so a developer's round can skip the rest. It is used ONLY for local rounds; the merge gate (`checks / unit (pull_request)`) runs every file. So the properties graded here are the ones whose failure would be a missed test: 1. A synthetic tree: transitive imports, a tool launched by path, a module named in a string, and what must NOT be chosen. 2. FAIL OPEN: a non-Python file, a deletion, a .py nothing references, a file that does not parse, a base that does not exist, all select ALL. 3. The real tree: the safety set names files that exist, config.yaml and run-tests.sh select ALL, a changed test file is chosen. 4. THE MUTANT: break a real module in a private copy of the tree; the selector must choose a test that goes red, and the unbroken copy is green. 5. The merge gate cannot be partial: checks.yml and the CI image never mention CI_ONLY, and checks.yml still runs the unfiltered run-tests.sh line. Read MUTATION.md before you report a survivor. """ import os import shutil import subprocess import sys import tempfile HERE = os.path.dirname(os.path.abspath(__file__)) REPO = os.path.dirname(HERE) SEL = os.path.join(REPO, ".gitea", "ci", "affected_tests.py") sys.path.insert(0, os.path.dirname(SEL)) import affected_tests as sel # noqa: E402 fails = [] tmps = [] def check(name, cond): print(f" {'ok ' if cond else 'FAIL'} {name}") if not cond: fails.append(name) def mk(root, files): for rel, text in files.items(): p = os.path.join(root, rel) os.makedirs(os.path.dirname(p), exist_ok=True) with open(p, "w", encoding="utf-8") as fh: fh.write(text) def tmp(): d = tempfile.mkdtemp(prefix="test_affected.") tmps.append(d) return d def choose(root, paths, status="M"): names, _r, why = sel.select(root, [(status, p) for p in paths]) return ([os.path.basename(n) for n in names] if names is not None else None, why) # ---------------------------------------------------------------- 1. synthetic print("1. a synthetic tree") R = tmp() P = "goodwe_controller/" mk(R, { P + "app/__init__.py": "", P + "app/low.py": "def f():\n return 1\n", P + "app/mid.py": "from app import low\n\ndef g():\n return low.f()\n", P + "app/top.py": "from app.mid import g\n", P + "app/lonely.py": "X = 1\n", P + "app/viastr.py": "Y = 2\n", P + "tools/launched.py": "print('hi')\n", P + "tools/reader.py": "print('r')\n", P + "test_a.py": "from app import top\n", P + "test_b.py": "from app import lonely\n", P + "test_c.py": "import subprocess\nsubprocess.run(['python3', 'tools/launched.py'])\n", P + "test_d.py": "import importlib\nm = importlib.import_module('app.viastr')\n", # reads every test_*.py as text: affected by a changed test, but it is a # reader, not a dependency of anything that imports a module P + "test_reader.py": "import glob\nfor f in glob.glob('test_*.py'):\n open(f).read()\n", P + "test_unrelated.py": "print('x')\n", # a test imported by another test carries the change on (an import runs it) P + "test_base.py": "from app import lonely\n", P + "test_chain.py": "import test_base\n", }) n, why = choose(R, [P + "app/low.py"]) check("a change two imports down chooses the test that imports the top (transitive)", n is not None and "test_a.py" in n) check("...and not tests that do not reach it", n is not None and not {"test_b.py", "test_c.py", "test_unrelated.py"} & set(n)) n, why = choose(R, [P + "app/lonely.py"]) check("a direct importer is chosen", n is not None and "test_b.py" in n) check("a test that imports another test is chosen with it, even though " "test_reader (a pattern reader) also reaches test_base as text", n is not None and "test_chain.py" in n and "test_base.py" in n) n, why = choose(R, [P + "tools/launched.py"]) check("a tool launched by path in a string chooses the launching test", n is not None and "test_c.py" in n) n, why = choose(R, [P + "app/viastr.py"]) check("a module imported dynamically by name chooses its test", n is not None and "test_d.py" in n) n, why = choose(R, [P + "test_unrelated.py"]) check("a changed test file is chosen", n is not None and "test_unrelated.py" in n) check("a test that only READS test files by pattern is chosen with it", n is not None and "test_reader.py" in n) check("...but a reader does not drag in the importers of an unrelated module", n is not None and "test_a.py" not in n and "test_b.py" not in n) n, why = choose(R, [P + "app/low.py", P + "tools/launched.py"]) check("two changes choose the union", n is not None and {"test_a.py", "test_c.py"} <= set(n)) # the safety set is always in, whatever changed R2 = tmp() files = {P + "app/__init__.py": "", P + "app/m.py": "A = 1\n", P + "test_m.py": "from app import m\n", P + "test_zzz.py": "print(1)\n"} for name in sel.SAFETY_SET: files[P + name] = "print('safety')\n" mk(R2, files) n, why = choose(R2, [P + "app/m.py"]) check("every safety-set file is chosen on a one-module change", n is not None and set(sel.SAFETY_SET) <= set(n)) check("...with a reason printed for each", all(sel.SAFETY_SET.values())) check("...and a test outside the set that does not reach it stays out", n is not None and "test_zzz.py" not in n) # ------------------------------------------------------------ 2. fails open print("\n2. it fails open") for label, paths, status in ( ("config.yaml (not Python)", [P + "config.yaml"], "M"), ("a translation file", [P + "translations/en.yaml"], "M"), ("run-tests.sh itself", [".gitea/ci/run-tests.sh"], "M"), ("a Dockerfile", [P + "Dockerfile"], "M"), ("a markdown file", ["README.md"], "M"), ("a deleted module", [P + "app/low.py"], "D"), ("a renamed module", [P + "app/low.py"], "R"), ("a .py nothing references", [P + "app/never_imported.py"], "M")): if label.startswith("a .py"): mk(R, {P + "app/never_imported.py": "Z = 1\n"}) n, why = choose(R, paths, status) check(f"{label}: SELECT ALL", n is None and why) mk(R, {P + "app/broken.py": "def (:\n"}) n, why = choose(R, [P + "app/low.py"]) check("a file in the tree that does not parse: SELECT ALL", n is None and why) os.remove(os.path.join(R, P + "app/broken.py")) n, why = choose(tmp(), [P + "app/low.py"]) check("a tree with no test files: SELECT ALL", n is None and why) # the CLI: --names prints every test on SELECT ALL, and a bad base is ALL g = tmp() mk(g, {P + "app/__init__.py": "", P + "app/m.py": "A = 1\n", P + "test_m.py": "from app import m\n"}) env = {**os.environ, "GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t", "GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@t"} def git(*a): return subprocess.run(["git", "-C", g] + list(a), env=env, capture_output=True, text=True) git("init", "-q", "-b", "base") git("add", "-A") git("commit", "-q", "-m", "base") git("checkout", "-q", "-b", "work") mk(g, {P + "app/m.py": "A = 2\n"}) git("commit", "-q", "-am", "work") r = subprocess.run([sys.executable, SEL, "--root", g, "--base", "base", "--head", "work", "--names"], capture_output=True, text=True) check("a committed range via git: the importer is chosen", r.returncode == 0 and r.stdout.split() == ["test_m.py"]) r = subprocess.run([sys.executable, SEL, "--root", g, "--base", "nosuchref", "--head", "work"], capture_output=True, text=True) check("a base that does not exist: SELECT ALL, not an error and not nothing", r.returncode == 0 and "SELECT ALL" in r.stdout) mk(g, {P + "app/m.py": "A = 3\n"}) r = subprocess.run([sys.executable, SEL, "--root", g, "--base", "base", "--names"], capture_output=True, text=True) check("uncommitted work is part of the diff (no --head)", r.returncode == 0 and r.stdout.split() == ["test_m.py"]) mk(g, {P + "app/newmod.py": "B = 1\n", P + "test_new.py": "from app import newmod\n"}) r = subprocess.run([sys.executable, SEL, "--root", g, "--base", "base", "--names"], capture_output=True, text=True) check("an untracked new module and its new test are in the diff", r.returncode == 0 and set(r.stdout.split()) == {"test_m.py", "test_new.py"}) # ---------------------------------------------------------------- 3. real tree print("\n3. the real tree") tests = sorted(f for f in os.listdir(HERE) if f.startswith("test_") and f.endswith(".py")) check("every safety-set name is a test file that exists", all(f in tests for f in sel.SAFETY_SET)) for p in ("goodwe_controller/config.yaml", ".gitea/ci/run-tests.sh", ".gitea/workflows/checks.yml", "goodwe_controller/DOCS.md", "goodwe_controller/translations/en.yaml"): n, why = choose(REPO, [p]) check(f"{p}: SELECT ALL", n is None and why) n, why = choose(REPO, ["goodwe_controller/app/maintenance.py"]) check("app/maintenance.py chooses test_maint25.py (it imports it through main)", n is not None and "test_maint25.py" in n) n, why = choose(REPO, ["goodwe_controller/test_maint25.py"]) check("a changed test chooses itself and the safety set", n is not None and "test_maint25.py" in n and set(sel.SAFETY_SET) <= set(n)) check("...and far fewer than all", n is not None and len(n) < len(tests) // 2) n, why = choose(REPO, [".gitea/ci/affected_tests.py"]) check("the selector's own file chooses this file", n is not None and "test_affected_tests.py" in n) # ------------------------------------------------------------------ 4. mutant print("\n4. the mutant: break a module, the selector must choose a red test") M = tmp() ign = shutil.ignore_patterns(".git", "scenarios", "__pycache__", "*.pyc") shutil.copytree(REPO, os.path.join(M, "repo"), ignore=ign, symlinks=True) mroot = os.path.join(M, "repo") cwd = os.path.join(mroot, "goodwe_controller") env2 = {**os.environ, "PYTHONDONTWRITEBYTECODE": "1"} def run_test(name): r = subprocess.run([sys.executable, "-B", name], cwd=cwd, env=env2, capture_output=True, text=True, timeout=600) return r.returncode check("control: test_maint25.py is green on the unbroken copy", run_test("test_maint25.py") == 0) path = os.path.join(cwd, "app", "maintenance.py") with open(path, encoding="utf-8") as fh: src = fh.read() old = " return until if now < until else None\n" check("the line to break is where this test expects it", src.count(old) == 1) with open(path, "w", encoding="utf-8") as fh: fh.write(src.replace(old, " return None\n")) n, why = choose(mroot, ["goodwe_controller/app/maintenance.py"]) check("the selector chooses test_maint25.py for the broken module", n is not None and "test_maint25.py" in n) check("...and running the chosen file is RED", run_test("test_maint25.py") != 0) with open(path, "w", encoding="utf-8") as fh: fh.write(src) check("put back: GREEN again", run_test("test_maint25.py") == 0) # --------------------------------------------- 5. the gate cannot be partial print("\n5. the merge gate runs everything") with open(os.path.join(REPO, ".gitea", "workflows", "checks.yml"), encoding="utf-8") as fh: wf = fh.read() check("checks.yml never mentions CI_ONLY", "CI_ONLY" not in wf) check("checks.yml runs run-tests.sh with no file list", "sh -eu /w/goodwe-addon/.gitea/ci/run-tests.sh\n" in wf) check("checks.yml never calls the selector or the local wrapper", "affected_tests" not in wf and "run-affected" not in wf) with open(os.path.join(REPO, ".gitea", "ci", "run-tests.sh"), encoding="utf-8") as fh: rt = fh.read() check("run-tests.sh takes CI_ONLY from the environment only and says PARTIAL", "PARTIAL RUN" in rt and 'set -- test_*.py' in rt) with open(os.path.join(REPO, ".gitea", "ci", "Dockerfile"), encoding="utf-8") as fh: df = fh.read() check("the CI image (built from the merged tree) never sets CI_ONLY", "CI_ONLY" not in df) for d in tmps: shutil.rmtree(d, ignore_errors=True) print() if fails: print(f"{len(fails)} FAILED:") for f in fails: print(f" - {f}") sys.exit(1) print("all checks passed")