diff --git a/.gitea/ci/Dockerfile b/.gitea/ci/Dockerfile index 2e8c778..d87adf1 100644 --- a/.gitea/ci/Dockerfile +++ b/.gitea/ci/Dockerfile @@ -18,5 +18,17 @@ # python3, no aiohttp, no pip (`No module named pip`; checked on the box, not # assumed). An image is not gold-plating here, it is the only option that does # not mutate a shared long-lived runner. +# ⚠️ `git` IS A TEST DEPENDENCY, NOT A CONVENIENCE (CI-02). test_ci_gate.py +# grades the gate's own checkout by EXTRACTING the shell out of +# .gitea/workflows/checks.yml and RUNNING it - against two throwaway local +# repositories it builds itself, one whose pull request merges and one whose +# does not. Reading those commands as a string would be the thing this +# repository keeps finding: a check that never observed its subject. Running +# them needs git in the image the checks run in. +# +# It costs one apk package and no network at test time: the fixtures are local +# paths, and the token substitution in the workflow's `git remote add` is a +# no-op on a path with no `://` in it, which is exactly why the test can use +# the shipped line verbatim instead of a paraphrase of it. FROM alpine:3 -RUN apk add --no-cache python3 py3-aiohttp py3-paho-mqtt +RUN apk add --no-cache python3 py3-aiohttp py3-paho-mqtt git diff --git a/.gitea/workflows/checks.yml b/.gitea/workflows/checks.yml index 46fb54f..bbaeea7 100644 --- a/.gitea/workflows/checks.yml +++ b/.gitea/workflows/checks.yml @@ -18,7 +18,9 @@ # so it QUEUES behind a rig campaign - a delayed verdict, never a destroyed one. # # NO JS ACTIONS: the runner image has no node, so `actions/checkout` cannot run. -# Checkout is seven lines of git below. +# Checkout is plain git below - and since CI-02 it is a checkout of the MERGE of +# the pull request into its base, not of the branch tip. See the long note on +# that step; a green from this workflow is a claim about content that can land. name: checks on: @@ -26,15 +28,53 @@ on: branches: [release/1.0, master] workflow_dispatch: -concurrency: - # ⚠️ PER PULL REQUEST, NOT A BARE NAME. A shared group holds one running and - # one pending run, so a bare name lets a second PR supersede a first PR's run - # before any runner has picked it up - which is exactly what happened to the - # other repo's run 521 (four jobs, `started_at` at the Unix epoch, - # `runner_id: 0`, concluded `cancelled`, and the PR merged anyway). Do not - # replace this with a constant. - group: addon-checks-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true +# ⚠️ NO `concurrency:` BLOCK, AND ITS ABSENCE IS AS LOAD-BEARING AS THE MISSING +# `paths:` FILTER ABOVE. CI-01 put one here - `group: addon-checks-`, +# `cancel-in-progress: true` - to stop a second pull request superseding a +# first one's run. It did stop that. It also did this, twice, on PR #14: +# +# run 531 created 17:08:19 `pending / Waiting to run` +# cancelled 17:10:44 <- the exact second run 532 was created +# run 533 created 17:45:11 `pending / Waiting to run` +# cancelled 18:11:22 <- the exact second run 534 was created +# +# Both cancelled runs have `started_at` at the Unix epoch: they never executed. +# A cancelled run posts `failure` on the required context, described +# `Canceled after 0s`. Branch protection compares STATE, not description, so +# "nothing ran" and "your tests are broken" arrive at the gate as the same red, +# and the correct operator response to each is the opposite of the other. +# +# CI-02 MEASURED THE THREE CANDIDATE SETTINGS ON A THROWAWAY PR (#29) RATHER +# THAN READING THE DOCS, because the docs and this Gitea disagree about which +# knob does it. In each trial one run was made to hold the single slot and two +# further pushes followed: +# +# trial 1, NO concurrency block - runs 561 and 562 both sat queued together +# and both executed. Nothing was cancelled. +# trial 2, group kept, cancel-in-progress: FALSE +# - run 564 sat `pending / Blocked by required +# conditions` and was cancelled at 0s the +# second run 565 was created. +# +# So `cancel-in-progress` IS NOT THE KNOB. A shared group cancels a run that has +# not started no matter what that flag says - GitHub documents this ("any +# previously pending run in the group will be cancelled") and Gitea 1.27 +# implements it. The only configuration that leaves a queued run alive is having +# no group at all, which is what is written here. +# +# ⚠️ DO NOT REPLACE THIS WITH A SHA-KEYED GROUP as a compromise. It looks +# tempting - two runs on different commits could never share a group, so nothing +# would be cancelled - but the only cancellations it still permits are the +# same-sha ones, and those are precisely the dangerous half: a red for a run +# that never started, landing on a commit that IS still the head. It +# concentrates the failure instead of removing it. +# +# WHAT THIS COSTS, stated honestly: the runner is `capacity: 1` and shared with +# the other repo's rig, so a developer who pushes four times in a minute now +# occupies it four times instead of once. That is CI-01's own trade written the +# other way round - it accepted "a delayed verdict, never a destroyed one" for +# the rig, and this accepts the same for pushes. A destroyed verdict here is +# worse than a slow one, because it is indistinguishable from a real failure. jobs: unit: @@ -58,19 +98,132 @@ jobs: # exist, so the test dies on `IndexError: 1` before its first assertion. # Measured during CI-01; the flat layout was tried first and did exactly # that. $WS/goodwe-addon/goodwe_controller gives parents[1] == $WS. - - name: checkout into a path the host shares (git; no node for actions/checkout) + # + # ⚠️ WHAT THIS CHECKS OUT IS THE MERGE, NOT THE BRANCH, AND THAT IS THE + # WHOLE OF CI-02. What was here before was one line: + # + # git fetch -q --depth 1 origin "$GITHUB_REF" \ + # || git fetch -q --depth 1 origin "$GITHUB_SHA" + # + # CI-02 measured what those two names actually are on this Gitea, on a + # deliberately conflicted throwaway PR (#29, runs 558 and 559), by asking + # one question per STEP - the Actions log endpoints 404 here, so a step's + # own conclusion is the only channel out of a job. The answers: + # + # GITHUB_REF == refs/pull//head (step P01 passed) + # GITHUB_REF != refs/pull//merge (step P02 failed) + # GITHUB_SHA == the pull request head sha (step P03 passed) + # fetching GITHUB_REF succeeds (step P04 passed) + # ...and lands on the bare head (step P05 passed) + # refs/pull//merge does not exist (step P06 failed) + # + # ⚠️ SO THE FALLBACK NEVER FIRED, AND THE DEFECT IS WORSE THAN THE ONE + # REPORTED. The ticket says a conflicted PR loses its merge ref and the + # `||` quietly grades the bare head instead. It does not: this Gitea has + # no merge refs at all - `git ls-remote origin 'refs/pull/*'` advertises + # only `/head`, for open and closed pull requests alike - so the FIRST + # half always succeeds, and both halves name the same commit anyway + # (GITHUB_REF resolves to GITHUB_SHA). The gate has never once graded + # merged content, on any pull request in this repository's history. Green + # has meant "this branch passed on its own", which on a base that has + # moved - and `block_on_outdated_branch` is off here, so it usually has - + # is not the content that lands. PR #14 reporting success while + # `mergeable: false` was not the fallback misfiring; it was the gate + # working exactly as written. + # + # A fallback that cannot fire cannot be "kept for the case it was for", so + # it is gone from this path and kept, ANNOUNCED, on the one path that has + # no pull request to merge (workflow_dispatch, below). + # + # ⚠️ NOT `--depth 1` ANY MORE. A merge needs the merge base, and a shallow + # pair does not have one. Measured before changing it: a full fetch of + # this repository is 6.8 MB and 1.1 s against this server, over 95 + # commits. The depth flag was buying nothing worth a wrong verdict. + # + # ⚠️ THE WORKFLOW AND THE CONTENT NOW COME FROM DIFFERENT TREES. Gitea + # picks the workflow file from refs/pull//head, so THIS file is always + # the branch's; the tree graded below - including .gitea/ci/Dockerfile - + # is the merge. That is the right way round (a pull request must not be + # able to grade itself with a gate the base has not seen) but it means a + # change to this file is tested by the branch it is on, which is how you + # are reading this. + - name: check out the base branch and the pull request head (git; no node for actions/checkout) env: URL: ${{ github.server_url }}/${{ github.repository }}.git TOKEN: ${{ secrets.GITHUB_TOKEN }} + PRNUM: ${{ github.event.pull_request.number }} + BASEREF: ${{ github.base_ref }} run: | set -eu rm -rf "$WS" && mkdir -p "$WS/goodwe-addon" && cd "$WS/goodwe-addon" git init -q . git remote add origin "$(echo "$URL" | sed "s#://#://x-access-token:${TOKEN}@#")" - git fetch -q --depth 1 origin "$GITHUB_REF" || git fetch -q --depth 1 origin "$GITHUB_SHA" - git checkout -qf FETCH_HEAD + if [ -n "$PRNUM" ]; then + # Two fetches, not one, so that a failure to reach either ref fails + # HERE with the ref in the message, rather than inside the merge + # step where it would read as a conflict. `github.base_ref` being + # populated at all was checked on the probe PR too (step Q01). + git fetch -q origin "+refs/heads/$BASEREF:refs/remotes/origin/base" + git fetch -q origin "+refs/pull/$PRNUM/head:refs/remotes/origin/pr" + git checkout -qf refs/remotes/origin/base + echo "base $BASEREF $(git rev-parse refs/remotes/origin/base)" + echo "head refs/pull/$PRNUM/head $(git rev-parse refs/remotes/origin/pr)" + else + # NO PULL REQUEST (workflow_dispatch). There is nothing to merge + # into, so this path grades a bare ref and SAYS SO - which is all + # the old `||` ever needed to do. It is kept here because a branch + # can be deleted or renamed between the dispatch and the fetch, and + # unlike the pull_request path that is a case nobody has ruled out. + if git fetch -q origin "$GITHUB_REF"; then + echo "GRADED: the bare ref $GITHUB_REF - NOT merged content (no pull request in this event)." + else + git fetch -q origin "$GITHUB_SHA" + echo "GRADED: the bare commit $GITHUB_SHA - NOT merged content, and NOT $GITHUB_REF either, which would not fetch." + fi + git checkout -qf FETCH_HEAD + fi echo "checked out $(git log --oneline -1)" + # ⚠️ THIS STEP IS THE ONE THAT MAKES A GREEN MEAN SOMETHING. It exists as + # its OWN step rather than as three lines of the checkout because the + # Actions log endpoints 404 on this Gitea and + # /api/v1/repos/.../actions/runs//jobs - which carries a per-step + # conclusion - is the only machine-readable account of what a run did. A + # reader who sees this step red knows the pull request does not apply and + # that NO TEST RAN; a reader who sees the next two red knows the tests ran + # and spoke. Collapse them back together and that distinction is gone + # again, which is the shape of defect this whole ticket is about. + # + # The refusal is a FAILURE rather than a skip on purpose. "Reports + # nothing" would leave the required context pending forever, which this + # file's own header explains is unmergeable-forever; and a conflicted pull + # request cannot be merged by Gitea anyway, so the red blocks nothing that + # was not already blocked. What it does is stop a green being posted about + # content that cannot land. + - name: grade the merge, not the branch - refuse a pull request that does not apply + env: + PRNUM: ${{ github.event.pull_request.number }} + BASEREF: ${{ github.base_ref }} + run: | + set -eu + cd "$WS/goodwe-addon" + if [ -z "$PRNUM" ]; then + echo "no pull request in this event - nothing to merge, and the previous step said what it graded." + exit 0 + fi + if git -c user.email=checks@localhost -c user.name=checks \ + merge --no-edit refs/remotes/origin/pr; then + echo "GRADED: refs/pull/$PRNUM/head merged into $BASEREF" + echo " result $(git rev-parse HEAD)" + echo " tree $(git rev-parse 'HEAD^{tree}')" + else + git merge --abort 2>/dev/null || true + echo "REFUSED: refs/pull/$PRNUM/head does not merge into $BASEREF." + echo " NOTHING HAS BEEN TESTED. This is not a test failure - no test ran." + echo " Rebase or merge $BASEREF into the branch and push again." + exit 1 + fi + - name: build the test image (shipped deps, from apk - see .gitea/ci/Dockerfile) run: | set -eu diff --git a/goodwe_controller/CHANGELOG.md b/goodwe_controller/CHANGELOG.md index 5a6a68e..7579e31 100644 --- a/goodwe_controller/CHANGELOG.md +++ b/goodwe_controller/CHANGELOG.md @@ -2,6 +2,71 @@ ## Unreleased +**The gate has never once graded the content a pull request would land, and a +run that never started reported as a test failure (CI-02).** Both halves were +measured on a deliberately conflicted throwaway pull request rather than read +off the workflow, because the Actions log endpoints 404 on this Gitea and a +step's own conclusion is the only channel out of a job. + +**Case 1, and the ticket's mechanism was wrong in a way that made it worse.** +CI-02 was filed saying a conflicted pull request loses its merge ref, so that +`git fetch $GITHUB_REF || git fetch $GITHUB_SHA` falls through and grades the +bare branch head. The symptom was real — PR #14 reported `checks / unit -> +success` while `mergeable: false` — but the mechanism is not. On this server +`GITHUB_REF` **is** `refs/pull//head`, `GITHUB_SHA` is the same commit, and +`git ls-remote origin 'refs/pull/*'` advertises no `/merge` refs at all, for +open and closed pull requests alike. The first fetch has always succeeded; **the +fallback has never fired once.** So the gate did not lose merged content on +conflicted pull requests — it has never had merged content, on any pull request +in this repository's history. Green has meant "this branch passed on its own", +which on a base that has moved is not what lands, and `block_on_outdated_branch` +is off on release/1.0 so the base usually has moved. + +**The checkout now fetches the base tip and the pull request head, merges them, +and refuses if the merge does not apply** — as its own named step, so +`/actions/runs//jobs` distinguishes "nothing was tested" from "the tests +spoke". Verified by producing it: with the fix on a conflicted pull request the +required context went `failure` in 2 s with the two test steps `skipped`. The +`--depth 1` is gone (a merge needs a merge base; a full fetch of this repository +is 6.8 MB and 1.1 s), and the fallback survives only on the `workflow_dispatch` +path, which has no pull request to merge and now says so in its output. + +**Case 2: `concurrency:` is gone, and `cancel-in-progress` was never the knob.** +Runs 531 and 533 on PR #14 were cancelled at 0 s each at the exact second the +next run was created, having never started; a cancelled run posts `failure` on +the required context, described `Canceled after 0s`, and branch protection +compares state and not description — so "nothing ran" and "your tests are +broken" arrive at the gate as the same red, wanting opposite responses. Three +settings were measured with one run holding the single runner slot and two +pushes behind it: with **no block at all** both queued runs survived and +executed; with the group kept and **`cancel-in-progress: false`** the queued run +was still cancelled at 0 s. A shared group cancels a run that has not started +regardless of that flag. Having no group is the only configuration measured to +leave a queued run alive, and it is what ships. The cost — a developer who +pushes four times occupies the shared runner four times — is CI-01's own trade +written the other way round: a delayed verdict, never a destroyed one. + +**`test_ci_gate.py` grades the gate by running it, not by reading it.** It +extracts the checkout and merge steps out of `checks.yml` and executes the +shipped commands against two throwaway local repositories it builds — one whose +pull request merges, one whose does not — and pins that the old one-liner exits +0 on the unmergeable one while the new steps refuse. It also pins the two tokens +that assemble `checks / unit (pull_request)`: renaming `name:` or the `unit:` +job key would leave every open and future pull request waiting forever on a +context that never arrives, with no error anywhere in that sequence. Seven +mutations of `checks.yml` were each confirmed red. `.gitea/ci/Dockerfile` gains +`git` for it. + +**What was NOT established, stated so nobody reads more into this.** The +original report that "no run was ever created for `0942228d`" was already +corrected on the card — run 533 was created and cancelled — and this work +confirms that from the runs list and the per-commit status history. A commit +that never had a run carries an empty `/statuses/` list; a queued one +carries `pending / Waiting to run` within seconds, and a concurrency-held one +`pending / Blocked by required conditions`. Those are distinguishable at the +API. `/actions/tasks`, which is what was polled originally, lists only tasks +that started, and cannot answer the question at all. + **A setpoint device smaller than `max_w` silently disarmed the frozen-meter detector, and the status page reported it as armed (SAFETY-20).** SAFETY-18's conjunction needs `abs(commanded_w) >= 0.95 * max_w`, but the emitted command is diff --git a/goodwe_controller/test_ci_gate.py b/goodwe_controller/test_ci_gate.py new file mode 100644 index 0000000..03c9d1a --- /dev/null +++ b/goodwe_controller/test_ci_gate.py @@ -0,0 +1,407 @@ +"""Runnable check for CI-02 - what a green from `checks / unit` is a claim about. +`python3 test_ci_gate.py` + +test_bytecode.py grades the thing that grades the add-on. This file grades the +thing that decides whether anything was graded at all. + +THE DEFECT, and the correction the measurement forced. CI-02 was filed saying +that a CONFLICTED pull request loses its merge ref, so that + + git fetch -q --depth 1 origin "$GITHUB_REF" \ + || git fetch -q --depth 1 origin "$GITHUB_SHA" + +falls through to the second half and quietly grades the bare branch head. PR #14 +did report `checks / unit -> success` while `mergeable: false`, so the symptom +was real. The mechanism was not. Measured on a deliberately conflicted throwaway +pull request (#29, runs 558 and 559 - one question per STEP, because this +Gitea's Actions log endpoints 404 and a step's conclusion is the only channel +out of a job): + + GITHUB_REF == refs/pull//head P01 passed + GITHUB_REF != refs/pull//merge P02 failed + GITHUB_SHA == the pull request head sha P03 passed + fetching GITHUB_REF succeeds P04 passed <- even conflicted + ...and it lands on the bare head P05 passed + refs/pull//merge does not exist P06 failed + +`git ls-remote origin 'refs/pull/*'` on this server advertises only `/head`, for +open and closed pull requests alike. THERE ARE NO MERGE REFS HERE. So the first +half of the `||` always succeeded, the fallback never fired once, and both halves +named the same commit anyway. The gate has never graded merged content on any +pull request in this repository's history. Green has meant "this branch passed +on its own" - which, on a base branch that has moved underneath it, is not the +content that lands. `block_on_outdated_branch` is off on release/1.0, so the +base usually has moved. + +WHAT THIS FILE PINS. + + Part 1 The context string branch protection requires is assembled from two + tokens in checks.yml. Renaming either makes every open and future + pull request unmergeable forever, silently, because Gitea waits on a + context it will now never receive. This part is the tripwire on that. + Part 2 The three absences in checks.yml that are load-bearing, each graded + on the command lines rather than on the prose around them. + Part 3 THE DEFECT, REPRODUCED. The old one-line checkout is run verbatim + against a real conflicted pull request built here, and is shown to + exit 0 having checked out content that cannot land. + Part 4 THE FIX, EXECUTED. The checkout and merge steps are EXTRACTED FROM + checks.yml AND RUN - the shipped commands, not a paraphrase - against + a pull request that merges. The tree they leave behind is the merge, + and is shown to differ from the head the old line would have graded. + Part 5 The same two steps against a pull request that does NOT merge: the + merge step exits non-zero, says nothing was tested, and never prints + a GRADED line. + Part 6 The gate announces which content it graded, on both paths, so that a + reader can tell merged content from a bare ref (CI-02 AC2). + +⚠️ THIS FILE RUNS THE WORKFLOW'S OWN SHELL. That is the point: a check that +read those commands as a string and matched them against an expectation would +be exactly the failure this repository keeps finding - a reading that never +observed its subject. The extractor below is deliberately brittle about finding +the step it wants: if checks.yml is restructured so that the step cannot be +found, this file goes RED rather than quietly grading nothing. + +⚠️ NEEDS `git`, which is why .gitea/ci/Dockerfile installs it. No network: the +fixtures are local repositories built in a temp directory, and the workflow's +`git remote add origin "$(echo "$URL" | sed "s#://#://x-access-token:...@#")"` +is a no-op substitution on a path with no `://` in it, which is what lets the +shipped line be used verbatim. + +⚠️ This file writes only under a temporary directory. It must stay that way: +CI mounts the checkout READ-ONLY. + +⚠️ MUTATING THE MODULE THIS FILE GRADES? PURGE `__pycache__` FIRST - or use +`tools/mutation_run.sh`, which does. See MUTATION.md. (There is no module here +to mutate; the subject is checks.yml. Break a line of it and re-run this file +expecting red - if it stays green, it was never watching.) +""" + +import os +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +HERE = Path(__file__).resolve().parent +WF = HERE.parent / ".gitea" / "workflows" / "checks.yml" +fails = [] + + +def check(name, cond): + if cond: + print(f" ok {name}") + else: + print(f" FAIL {name}") + fails.append(name) + + +# --------------------------------------------------------------------------- +# Reading checks.yml. There is no yaml module in the image the checks run in +# (alpine py3-aiohttp py3-paho-mqtt, and adding a parser to grade five lines +# would be a worse trade than a small parser that fails loudly), so the two +# things this file needs are pulled out by hand: +# +# * the workflow name and the job keys, for part 1 +# * the `run:` block of a named step, verbatim, for parts 3 to 6 +# +# COMMAND LINES ONLY WHERE IT MATTERS. test_bytecode.py's part 6 learned this +# the hard way: an `in y` search over a whole workflow passes on the PROSE that +# explains a flag, and stayed green after the flag itself was deleted. Every +# absence checked in part 2 is checked against the file with its comment lines +# stripped. +# --------------------------------------------------------------------------- +if not WF.is_file(): + print(f"cannot find the workflow at {WF} - nothing to grade") + raise SystemExit(1) +YAML = WF.read_text() +CODE = "\n".join(l for l in YAML.splitlines() if not l.lstrip().startswith("#")) + + +def workflow_name(): + for line in YAML.splitlines(): + if line.startswith("name:"): + return line.split(":", 1)[1].strip() + return None + + +def job_keys(): + out, seen = [], False + for line in YAML.splitlines(): + if line.startswith("jobs:"): + seen = True + continue + if seen: + if line and not line[0].isspace(): + break + if line.startswith(" ") and not line.startswith(" ") and line.rstrip().endswith(":"): + out.append(line.strip()[:-1]) + return out + + +def step_script(name_prefix): + """Return the `run: |` body of the step whose name starts with name_prefix.""" + lines = YAML.splitlines() + i = 0 + while i < len(lines): + if lines[i].strip().startswith("- name:") and name_prefix in lines[i]: + break + i += 1 + else: + return None + if i >= len(lines): + return None + while i < len(lines) and lines[i].strip() != "run: |": + # stop at the next step so a missing block cannot borrow the next one's + if i > 0 and lines[i].strip().startswith("- name:") and name_prefix not in lines[i]: + return None + i += 1 + if i >= len(lines): + return None + body, i = [], i + 1 + while i < len(lines): + l = lines[i] + if l.strip() == "": + body.append("") + elif l.startswith(" " * 10): + body.append(l[10:]) + else: + break + i += 1 + return "\n".join(body).strip("\n") + + +# --------------------------------------------------------------------------- +# The fixture: a real bare repository with a base branch and a refs/pull/N/head, +# built to order. `conflict=True` makes the pull request touch the same lines +# the base moved, which is how PR #29 was made unmergeable on the server. +# +# The shapes matter. In BOTH cases the pull request adds `from_pr.txt` and the +# base adds `from_base.txt` AFTER the branch point, so the branch head is never +# the content that lands even when it merges perfectly - that difference is what +# part 4 measures, and it is the ordinary case, not the pathological one. +# --------------------------------------------------------------------------- +GIT = ["git", "-c", "user.email=fixture@localhost", "-c", "user.name=fixture", + "-c", "init.defaultBranch=release/1.0", "-c", "commit.gpgsign=false"] +PRNUM = "7" +BASEREF = "release/1.0" + + +def git(cwd, *args, check_rc=True): + r = subprocess.run(GIT + list(args), cwd=cwd, capture_output=True, text=True) + if check_rc and r.returncode != 0: + raise RuntimeError(f"git {' '.join(args)} failed in {cwd}:\n{r.stderr}") + return r.stdout.strip() + + +def build_origin(stack, conflict): + """A bare repo carrying refs/heads/release/1.0 and refs/pull/7/head.""" + root = Path(tempfile.mkdtemp(prefix="test_ci_gate.")) + stack.append(str(root)) + work, bare = root / "work", root / "origin.git" + work.mkdir() + git(work, "init", "-q", ".") + (work / "shared.txt").write_text("one\ntwo\nthree\n") + git(work, "add", "-A") + git(work, "commit", "-qm", "branch point") + fork = git(work, "rev-parse", "HEAD") + + # the base moves on + (work / "from_base.txt").write_text("only the base has this\n") + if conflict: + (work / "shared.txt").write_text("one\nTWO FROM BASE\nthree\n") + git(work, "add", "-A") + git(work, "commit", "-qm", "the base branch moves on") + base_sha = git(work, "rev-parse", "HEAD") + + # the pull request branches from the fork point + git(work, "checkout", "-q", "-b", "pr", fork) + (work / "from_pr.txt").write_text("only the pull request has this\n") + if conflict: + (work / "shared.txt").write_text("one\nTWO FROM THE PULL REQUEST\nthree\n") + git(work, "add", "-A") + git(work, "commit", "-qm", "the pull request") + pr_sha = git(work, "rev-parse", "HEAD") + + git(root, "init", "-q", "--bare", str(bare)) + git(work, "remote", "add", "origin", str(bare)) + git(work, "push", "-q", "origin", f"{base_sha}:refs/heads/{BASEREF}") + git(work, "push", "-q", "origin", f"{pr_sha}:refs/pull/{PRNUM}/head") + return str(bare), base_sha, pr_sha, str(root) + + +def run_script(script, ws, origin, pr_sha, extra=None): + env = dict(os.environ) + env.pop("PYTHONDONTWRITEBYTECODE", None) + env.update( + WS=ws, PRNUM=PRNUM, BASEREF=BASEREF, URL=origin, TOKEN="", + GITHUB_REF=f"refs/pull/{PRNUM}/head", GITHUB_SHA=pr_sha, + GIT_TERMINAL_PROMPT="0", + ) + if extra: + env.update(extra) + return subprocess.run(["sh", "-c", script], env=env, capture_output=True, text=True) + + +tmpdirs = [] +print("the gate's own honesty (CI-02)") + +# --------------------------------------------------------------------------- +print("\n1. the required context string, from the two tokens that build it") +# --------------------------------------------------------------------------- +# ⚠️ READ BEFORE RENAMING ANYTHING IN checks.yml. Branch protection on +# release/1.0 has `enable_status_check: true` and +# `status_check_contexts: ['checks / unit (pull_request)']`. Gitea assembles +# that string from `name:` at the top of the workflow and the job KEY under +# `jobs:`. Change either and Gitea starts posting a context nobody is waiting +# for, while the one branch protection requires never arrives - so it stays +# `pending` forever, on every open pull request and every future one, and +# admin override is blocked. There is no error message anywhere in that +# sequence. This check is the only thing between a tidy-up and that. +REQUIRED = "checks / unit (pull_request)" +name, jobs = workflow_name(), job_keys() +check("the workflow still has a name", name is not None) +check("the workflow declares exactly one job", len(jobs) == 1) +built = f"{name} / {jobs[0]} (pull_request)" if name and len(jobs) == 1 else "" +check(f"the context it builds is exactly {REQUIRED!r}", built == REQUIRED) +if built != REQUIRED: + print(f" it now builds {built!r} - branch protection will wait forever for {REQUIRED!r}") +check("the workflow still triggers on pull_request", "pull_request:" in CODE) + +# --------------------------------------------------------------------------- +print("\n2. the three absences that are load-bearing") +# --------------------------------------------------------------------------- +# `paths:` is CI-01's: a workflow that learns to sit a pull request out leaves +# the required context pending forever, which is unmergeable forever. +check("no paths: filter", not any(l.strip().startswith("paths:") or l.strip().startswith("paths-ignore:") + for l in CODE.splitlines())) +# `concurrency:` is CI-02's, and it was MEASURED rather than reasoned about, on +# throwaway PR #29 with one run holding the single runner slot and two more +# pushes behind it: +# no block at all -> runs 561 and 562 both queued, both executed +# group + cancel-in-progress: false +# -> run 564 sat `Blocked by required conditions` +# and was cancelled at 0s the moment 565 existed +# A shared group cancels a run that has not started regardless of that flag, and +# a cancelled run posts `failure` on the required context ("Canceled after 0s"), +# which branch protection cannot tell from a real test failure. Runs 531 and 533 +# on PR #14 died exactly this way, each at the same second the next run was +# created. Having no group is the only setting measured to leave a queued run +# alive. +check("no concurrency: group", not any(l.startswith("concurrency:") for l in CODE.splitlines())) +check("no cancel-in-progress anywhere in the commands", "cancel-in-progress" not in CODE) +# The fallback itself. Graded on the command lines: the prose above the step +# quotes the old line on purpose, and an `in YAML` search would match that. +check("the || $GITHUB_SHA fallback is gone from the commands", + not any("||" in l and "GITHUB_SHA" in l for l in CODE.splitlines())) +check("the checkout is no longer shallow (a merge needs the merge base)", + "--depth 1" not in CODE) + +# --------------------------------------------------------------------------- +print("\n3. the defect, reproduced: the old line grades a conflicted head, happily") +# --------------------------------------------------------------------------- +# NON-VACUITY FOR EVERYTHING BELOW. If this part were already red - if the old +# line refused a conflicted pull request on its own - then parts 4 and 5 would +# be measuring nothing, and the ticket would have no subject. +origin, base_sha, pr_sha, _root = build_origin(tmpdirs, conflict=True) +ws = tempfile.mkdtemp(prefix="test_ci_gate.ws.") +tmpdirs.append(ws) +OLD = ''' +set -eu +rm -rf "$WS" && mkdir -p "$WS/goodwe-addon" && cd "$WS/goodwe-addon" +git init -q . +git remote add origin "$(echo "$URL" | sed "s#://#://x-access-token:${TOKEN}@#")" +git fetch -q --depth 1 origin "$GITHUB_REF" || git fetch -q --depth 1 origin "$GITHUB_SHA" +git checkout -qf FETCH_HEAD +echo "checked out $(git log --oneline -1)" +''' +old = run_script(OLD, ws, origin, pr_sha) +check("the pull request genuinely does not merge (the fixture is not lying)", + subprocess.run(GIT + ["merge-tree", "--write-tree", base_sha, pr_sha], + cwd=origin, capture_output=True, text=True).returncode != 0) +check("...and the old checkout exits 0 on it anyway", old.returncode == 0) +tree = Path(ws) / "goodwe-addon" +check("...having checked out the bare head", (tree / "from_pr.txt").is_file()) +check("...which is missing what the base branch added", not (tree / "from_base.txt").is_file()) +check("...and it says nothing about which of the two refs it used", + "GRADED" not in old.stdout and "REFUSED" not in old.stdout) +if old.returncode != 0: + print(old.stdout[-800:], old.stderr[-800:]) + +# --------------------------------------------------------------------------- +print("\n4. the shipped steps, executed, on a pull request that MERGES") +# --------------------------------------------------------------------------- +CO = step_script("check out the base branch and the pull request head") +MERGE = step_script("grade the merge, not the branch") +check("the checkout step can be found in checks.yml", CO is not None) +check("the merge/refuse step can be found in checks.yml", MERGE is not None) +if CO is None or MERGE is None: + print(" checks.yml has been restructured; this file grades nothing until it is updated") + print() + print(f"{len(fails)} FAILED") + raise SystemExit(1) + +origin2, base2, pr2, _r2 = build_origin(tmpdirs, conflict=False) +ws2 = tempfile.mkdtemp(prefix="test_ci_gate.ws2.") +tmpdirs.append(ws2) +co = run_script(CO, ws2, origin2, pr2) +mg = run_script(MERGE, ws2, origin2, pr2) +check("the checkout step succeeds", co.returncode == 0) +check("the merge step succeeds on a mergeable pull request", mg.returncode == 0) +if co.returncode or mg.returncode: + print(co.stdout[-600:], co.stderr[-600:], mg.stdout[-600:], mg.stderr[-600:]) +t2 = Path(ws2) / "goodwe-addon" +check("what is left on disk carries the pull request's file", (t2 / "from_pr.txt").is_file()) +check("...AND the file the base branch added after the branch point", (t2 / "from_base.txt").is_file()) +# That second one is the whole difference. The old line graded a tree WITHOUT +# from_base.txt - green about content that is not what lands - and nothing in +# the run said so. This is the ordinary case: no conflict anywhere in it. +check("the graded tree is exactly the tree of a real merge", + git(t2, "rev-parse", "HEAD^{tree}") == + subprocess.run(GIT + ["merge-tree", "--write-tree", base2, pr2], + cwd=origin2, capture_output=True, text=True).stdout.strip()) + +# --------------------------------------------------------------------------- +print("\n5. the same two steps on a pull request that does NOT merge") +# --------------------------------------------------------------------------- +ws3 = tempfile.mkdtemp(prefix="test_ci_gate.ws3.") +tmpdirs.append(ws3) +co3 = run_script(CO, ws3, origin, pr_sha) +mg3 = run_script(MERGE, ws3, origin, pr_sha) +check("the checkout step still succeeds - a conflict does not stop a fetch", + co3.returncode == 0) +# Stated as its own check because it is why the defect was invisible: nothing +# about a conflicted pull request makes any git fetch fail on this server. +check("the merge step REFUSES (non-zero)", mg3.returncode != 0) +check("...and says nothing was tested, not that a test failed", + "REFUSED" in mg3.stdout and "NOTHING HAS BEEN TESTED" in mg3.stdout) +check("...and never claims to have graded anything", "GRADED:" not in mg3.stdout) +check("...and leaves no half-merged tree behind for a later step to grade", + not (Path(ws3) / "goodwe-addon" / "from_pr.txt").is_file() + or git(Path(ws3) / "goodwe-addon", "rev-parse", "HEAD") == base_sha) + +# --------------------------------------------------------------------------- +print("\n6. the run says which content it graded (CI-02 AC2)") +# --------------------------------------------------------------------------- +check("the mergeable run announces the merge it graded", "GRADED:" in mg.stdout) +check("...naming the base branch it merged into", BASEREF in mg.stdout) +check("...and the tree, which is the only unambiguous name for content", + "tree" in mg.stdout and git(t2, "rev-parse", "HEAD^{tree}")[:12] in mg.stdout) +# The no-pull-request path (workflow_dispatch) keeps a fallback, because a +# branch really can vanish between the dispatch and the fetch. It has to say so. +disp = run_script(CO, tempfile.mkdtemp(prefix="test_ci_gate.ws4."), origin2, pr2, + extra={"PRNUM": "", "GITHUB_REF": f"refs/heads/{BASEREF}"}) +check("the no-pull-request path still checks something out", disp.returncode == 0) +check("...and announces that it is NOT merged content", + "GRADED:" in disp.stdout and "NOT merged content" in disp.stdout) +if disp.returncode != 0: + print(disp.stdout[-600:], disp.stderr[-600:]) + +for d in tmpdirs: + shutil.rmtree(d, ignore_errors=True) + +print() +if fails: + print(f"{len(fails)} FAILED: " + "; ".join(fails)) + sys.exit(1) +print("all checks passed")