shater/scripts/run-tests.sh

#!/usr/bin/env bash
#
# run-tests.sh — THE test gate of the release tract.
#
# WHY THIS EXISTS (2026-07-26)
#   Until now the release tract ran almost no tests. The only `go test` calls in
#   the whole publishing path were scripts/build-shaterd.sh's one-package
#   buildtags check and the three named tests scripts/check-router-tags.sh runs.
#   The upstream .github/workflows/test.yml triggers on `stable`/`testing`/
#   `unstable` — branches this fork does not have — and Gitea does not read
#   .github/workflows at all once .gitea/workflows exists. Net effect: 115 of the
#   116 test files under shater/** had never executed in CI, and
#   TestDNSFilterRemoteBlocklistHTTPClient shipped red through two releases
#   before anyone ran it by hand.
#
# WHAT IT GUARANTEES
#   1. The suite runs under the SHIPPED build tags (scripts/router-tags.sh), not
#      under some CI-local tag set. This is not cosmetic: the AmneziaWG tests in
#      transport/wireguard are `//go:build with_awg` — 1 test file compiles
#      without the tag set, 7 with it. The 2026-07-25 WireGuard outage was
#      exactly a "built with X, verified with Y" gap.
#   2. It runs on linux. shater/generate has 44 test files on linux against 32 on
#      windows/darwin; the linux-only half is where the routing, ruleset, DNS and
#      health tests live.
#   3. Nothing is skipped SILENTLY. Two machine checks:
#        - the tag set may only ADD test files, never hide them (a test behind
#          `//go:build !with_awg` would vanish from the gate — this fails first);
#        - every package that has tests must report `ok` by name; a suite that
#          compiles down to "no test files" fails the gate instead of passing it.
#      A guard that silently runs nothing is worse than no guard (same rule as
#      scripts/check-router-tags.sh).
#
# Usage:
#   scripts/run-tests.sh            # full gate (~3 min warm on the runner)
#   scripts/run-tests.sh --no-race  # skip the -race pass (faster; local loop)
#
# Env:
#   SHATER_GO_IMAGE      docker image used to reach linux from a non-linux host
#                        (default golang:1.26 — keep it >= go.mod's toolchain).
#   SHATER_NO_DOCKER=1   fail instead of falling back to docker.
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO="$(cd "$SCRIPT_DIR/.." && pwd)"
cd "$REPO"

RACE=1
for a in "$@"; do
	case "$a" in
		--no-race) RACE=0 ;;
		-h|--help) sed -n '2,41p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
		*) echo "run-tests: unknown flag: $a" >&2; exit 2 ;;
	esac
done

# shellcheck source=router-tags.sh
. "$SCRIPT_DIR/router-tags.sh"

# The fork's own trees plus the upstream trees the fork edits (adapter/, route/,
# option/, dns/ all carry shater changes). ROOTS, not a hand-kept package list: a
# new package with tests joins the gate the moment it is created, which is the
# whole point.
ROOTS=(./shater/... ./protocol/... ./transport/... ./adapter/... ./route/... ./option/... ./dns/...)

# common/ is mostly upstream, but common/tls, common/tlsfragment, common/sniff
# and common/urltest carry fork behaviour (D13 DPI-bypass, urltest health), so it
# is in — minus the privileged integration tests below.
ROOTS_COMMON=(./common/...)
#   SKIP, WITH REASON: common/tlsspoof's TestIntegration* enter TCP_REPAIR and
#   need CAP_NET_ADMIN. The act_runner job container runs as root but WITHOUT
#   that capability, so they do not skip — they FAIL. Excluded by name so the
#   rest of common/ can be a real gate instead of a permanently red one. (On
#   linux every tlsspoof test is a TestIntegration*, so that package is
#   effectively uncovered here; it is covered by the VM runs.)
SKIP_COMMON='^TestIntegration'

#   SKIP, WITH REASON: the first -race run over this tree (2026-07-26 — nobody
#   had ever run one) turned up two failures. One was a REAL product race:
#   ClientBind.connect() touched its fields from both the Send() path and
#   RoutineReceiveIncoming() with no lock, caught by
#   transport/wireguard.TestAwgDetourClientBindDelivers; that one has since been
#   fixed in client_bind.go and is NOT skipped — it is exactly what this pass is
#   for. What was left:
#     - shater/alert.TestExpiryDedupWithinDay — the test's own closure read a
#       variable the test body wrote while Notifier.dispatch's goroutine was
#       still delivering. FIXED 2026-07-26 (the simulated clock now has a mutex),
#       so the entry is gone and the -race pass covers the whole tree again.
#   Nothing is skipped under -race any more. Keep it that way: an entry here is a
#   hole in the gate, so add one only with a named reason and delete it the moment
#   the race is fixed.
RACE_SKIP='^$'

echo "== shater test gate =="
echo "  tags   : $SHATER_ROUTER_TAGS"
echo "  ldflags: $SHATER_ROUTER_LDFLAGS"
echo "  race   : $([ "$RACE" -eq 1 ] && echo yes || echo no)"
echo

# --- linux, or re-exec on linux ---------------------------------------------
# The linux-only half of the suite is the half worth running (see header). From a
# non-linux host, re-exec inside a golang container rather than quietly testing
# 32 of shater/generate's 44 files — a partial gate reads exactly like a passing
# one.
if [ "$(go env GOOS)" != "linux" ] && [ "${SHATER_TESTS_IN_DOCKER:-0}" != "1" ]; then
	if [ "${SHATER_NO_DOCKER:-0}" = "1" ] || ! command -v docker >/dev/null 2>&1; then
		echo "  ERROR: the gate needs linux (GOOS=$(go env GOOS)) and docker is unavailable/disabled." >&2
		echo "         Run it on the linux CI runner or the OpenWrt VM." >&2
		exit 1
	fi
	image="${SHATER_GO_IMAGE:-golang:1.26}"
	echo "== re-exec on linux via docker ($image) =="
	host_repo="$REPO"
	command -v cygpath >/dev/null 2>&1 && host_repo="$(cygpath -w "$REPO")"
	MSYS2_ARG_CONV_EXCL='*' MSYS_NO_PATHCONV=1 docker run --rm \
		-v "$host_repo":/src \
		-v shater-tagcheck-gomod:/go/pkg/mod \
		-v shater-tagcheck-gocache:/root/.cache/go-build \
		-w /src \
		-e SHATER_TESTS_IN_DOCKER=1 \
		"$image" bash scripts/run-tests.sh "$@"
	exit $?
fi

ALL_ROOTS=("${ROOTS[@]}" "${ROOTS_COMMON[@]}")

# --- [1/4] the tag set may only ADD test files, never hide them --------------
# `go list` counts the test files the compiler would actually take. If adding the
# shipped tags REMOVES a test file from any package, that test exists but the
# gate would never see it — which is the failure mode this whole script is about,
# just pointed the other way.
echo "== [1/4] no test file is hidden by the shipped tag set =="
LISTFMT='{{.ImportPath}} {{len .TestGoFiles}} {{len .XTestGoFiles}}'
plain="$(go list -f "$LISTFMT" "${ALL_ROOTS[@]}")"
tagged="$(go list -tags "$SHATER_ROUTER_TAGS" -f "$LISTFMT" "${ALL_ROOTS[@]}")"
hidden=0
while read -r pkg t x; do
	[ -n "${pkg:-}" ] || continue
	n_plain=$((t + x))
	[ "$n_plain" -gt 0 ] || continue
	line="$(awk -v p="$pkg" '$1 == p { print; exit }' <<<"$tagged")"
	if [ -z "$line" ]; then
		echo "  HIDDEN: $pkg has $n_plain test file(s) untagged but no package at all under the shipped tags" >&2
		hidden=1
		continue
	fi
	read -r _ tt tx <<<"$line"
	n_tagged=$((tt + tx))
	if [ "$n_tagged" -lt "$n_plain" ]; then
		echo "  HIDDEN: $pkg — $n_plain test file(s) untagged, only $n_tagged under the shipped tags" >&2
		hidden=1
	elif [ "$n_tagged" -gt "$n_plain" ]; then
		echo "  +$((n_tagged - n_plain)) tag-gated test file(s): $pkg ($n_plain -> $n_tagged)"
	fi
done <<<"$plain"
if [ "$hidden" -ne 0 ]; then
	echo >&2
	echo "  FAILED: a test file is invisible to the tag set we ship. Either the" >&2
	echo "          constraint is wrong or the tag set is — do not paper over it" >&2
	echo "          by testing with different tags than we build with." >&2
	exit 1
fi
echo

# --- the runner --------------------------------------------------------------
# Runs one suite and then PROVES it ran: every package `go list` says has tests
# must appear as `ok <pkg>` in the output. `go test` over a package whose tests
# all vanished behind a build constraint prints "[no test files]" and exits 0 —
# a green run that verified nothing.
LOG="$(mktemp)"
trap 'rm -f "$LOG"' EXIT
FAILED=0

run_suite() { # $1=label  $2=extra go-test flags (may be empty)  $3..=packages
	local label="$1" extra="$2"
	shift 2
	local pkgs=("$@") rc=0 expect missing=0 pkg

	expect="$(go list -tags "$SHATER_ROUTER_TAGS" \
		-f '{{if or .TestGoFiles .XTestGoFiles}}{{.ImportPath}}{{end}}' \
		"${pkgs[@]}" | grep -v '^$' || true)"
	if [ -z "$expect" ]; then
		echo "  FAILED [$label]: go list reports no package with tests here — the gate" >&2
		echo "          would have run nothing and passed." >&2
		FAILED=1
		return
	fi
	echo "  packages with tests: $(wc -l <<<"$expect" | tr -d ' ')"

	set +e
	# shellcheck disable=SC2086  # $extra is a deliberate word-split flag list
	go test -count=1 $extra \
		-tags "$SHATER_ROUTER_TAGS" -ldflags "$SHATER_ROUTER_LDFLAGS" \
		"${pkgs[@]}" >"$LOG" 2>&1
	rc=$?
	set -e
	sed 's/^/    /' "$LOG"

	if [ "$rc" -ne 0 ]; then
		echo "  FAILED [$label]: go test exited $rc" >&2
		FAILED=1
		return
	fi

	while read -r pkg; do
		[ -n "$pkg" ] || continue
		grep -qE "^ok[[:space:]]+$pkg([[:space:]]|\$)" "$LOG" || {
			echo "  DID NOT RUN [$label]: $pkg" >&2
			missing=1
		}
	done <<<"$expect"
	if [ "$missing" -ne 0 ]; then
		echo "  FAILED [$label]: package(s) above have test files but produced no 'ok'" >&2
		echo "          line. Build-constraint or file-name drift emptied them." >&2
		FAILED=1
		return
	fi
	echo "  OK [$label]"
}

# --- [2/4] the fork's trees, shipped tags, linux -----------------------------
echo "== [2/4] go test — the fork's trees (shipped tags, linux) =="
run_suite main "" "${ROOTS[@]}"
echo

# --- [3/4] common/, minus the tests that need CAP_NET_ADMIN ------------------
echo "== [3/4] go test — common/ (minus the CAP_NET_ADMIN integration tests) =="
run_suite common "-skip $SKIP_COMMON" "${ROOTS_COMMON[@]}"
echo

# --- [4/4] -race over the same trees -----------------------------------------
# Everything, not a subset: shater/netplane alone is ~110 s under -race and it is
# the single most concurrency-critical package we own (the nft data plane), so
# once it is in, adding the rest costs ~40 s more. common/ is left out — it is
# upstream code exercised by upstream CI.
if [ "$RACE" -eq 1 ]; then
	echo "== [4/4] go test -race — the fork's trees =="
	echo "  nothing is skipped under -race"
	
	run_suite race "-race -skip $RACE_SKIP" "${ROOTS[@]}"
else
	echo "== [4/4] -race pass skipped (--no-race) =="
fi
echo

if [ "$FAILED" -ne 0 ]; then
	echo "== TEST GATE FAILED — nothing may be published from this run. ==" >&2
	exit 1
fi
echo "== OK: the shipped tag set, on linux, passes every test we own. =="