shater/shater/engine/grouphealth_test.go

package engine

import (
	"context"
	"fmt"
	"net"
	"testing"
	"time"

	"github.com/sagernet/sing-box/adapter"
	"github.com/sagernet/sing-box/common/urltest"
	C "github.com/sagernet/sing-box/constant"
	M "github.com/sagernet/sing/common/metadata"
)

// fakeGroup is a minimal adapter.OutboundGroup: exactly the three things
// groupHealthOf reads (Tag/All/Now) plus the Outbound surface needed to satisfy the
// interface. It never dials — group health is a pure projection of stored history.
type fakeGroup struct {
	tag  string
	all  []string
	now  string
	kind string
}

func (f *fakeGroup) Type() string           { return f.kind }
func (f *fakeGroup) Tag() string            { return f.tag }
func (f *fakeGroup) Network() []string      { return []string{"tcp"} }
func (f *fakeGroup) Dependencies() []string { return nil }
func (f *fakeGroup) Now() string            { return f.now }
func (f *fakeGroup) All() []string          { return f.all }
func (f *fakeGroup) DialContext(context.Context, string, M.Socksaddr) (net.Conn, error) {
	return nil, fmt.Errorf("fakeGroup does not dial")
}
func (f *fakeGroup) ListenPacket(context.Context, M.Socksaddr) (net.PacketConn, error) {
	return nil, fmt.Errorf("fakeGroup does not dial")
}

var _ adapter.OutboundGroup = (*fakeGroup)(nil)

// copyTag mirrors generate.groupCopyTag EXACTLY. Duplicated here (rather than
// imported) on purpose: shater/engine must not depend on shater/generate, and this
// test is precisely the tripwire for the two drifting apart — if the generator ever
// changes its tag format, parseGroupCopyTag stops recovering node names and the panel
// starts showing "group-auto-m3-…" instead of a node name.
func copyTag(group string, idx int, member string) string {
	return fmt.Sprintf("group-%s-m%d-%s", group, idx, member)
}

// TestParseGroupCopyTag round-trips the generator's copy-tag format and pins the
// cases that must NOT be read as a copy.
func TestParseGroupCopyTag(t *testing.T) {
	// Round-trip: whatever the generator emits, we recover the member name — including
	// names with dashes, spaces and non-ASCII, and a sparse (non-dense) index, which is
	// what a group with skipped members produces.
	for _, member := range []string{"node-a", "🇩🇪 Frankfurt-01", "a-b-c", "m5-x"} {
		for _, idx := range []int{0, 3, 17, 330} {
			tag := copyTag("auto", idx, member)
			got, ok := parseGroupCopyTag("auto", tag)
			if !ok || got != member {
				t.Errorf("parseGroupCopyTag(auto, %q) = (%q,%v), want (%q,true)", tag, got, ok, member)
			}
		}
	}

	// A group whose NAME itself looks like a copy tag is still matched exactly.
	weird := copyTag("grp-m1-x", 2, "node")
	if got, ok := parseGroupCopyTag("grp-m1-x", weird); !ok || got != "node" {
		t.Errorf("group name containing -m<digits>-: got (%q,%v), want (node,true)", got, ok)
	}

	// Non-copies: a plain node tag, another group's copy, a chain hop, a missing
	// index, a non-numeric index and an empty member name.
	for _, tag := range []string{
		"node-a",
		copyTag("other", 1, "node-a"),
		"chain-office-h1",
		"group-auto-mnode",
		"group-auto-mx-node",
		"group-auto-m1-",
	} {
		if got, ok := parseGroupCopyTag("auto", tag); ok {
			t.Errorf("parseGroupCopyTag(auto, %q) = (%q,true), want not-a-copy", tag, got)
		}
	}
}

// TestGroupHealthUnbound: a group WITHOUT an egress binding balances the nodes' base
// tags, so its health comes straight off the history the engine already has, and the
// member rows carry the node names verbatim.
func TestGroupHealthUnbound(t *testing.T) {
	hist := urltest.NewHistoryStorage()
	now := time.Now()
	hist.StoreURLTestHistory("n1", &adapter.URLTestHistory{LastOK: now.Add(-time.Minute), Delay: 100})
	// n2 is DEAD: a failed probe/dial marks LastFail on the board record — nothing
	// is deleted, so a dead member stays distinguishable from a never-measured one.
	hist.StoreURLTestHistory("n2", &adapter.URLTestHistory{LastFail: now.Add(-10 * time.Second)})
	// n3: nothing at all — never probed (outside every used path). Untested.

	g := &fakeGroup{tag: "auto", kind: C.TypeURLTest, all: []string{"n1", "n2", "n3"}, now: "n1"}
	gh := groupHealthOf(g, C.TypeURLTest, newHealthView(hist, healthTTLFloor, now), nil, true)
	if !gh.Used {
		t.Error("a nil used-set must report Used=true (no badge is better than a wrong one)")
	}

	if gh.Bound {
		t.Error("unbound group reported bound=true")
	}
	if gh.Selected != "n1" {
		t.Errorf("Selected = %q, want n1", gh.Selected)
	}
	assertCounters(t, gh, 3 /*total*/, 1 /*alive*/, 1 /*dead*/, 1 /*untested*/)
	if gh.FreshestSeconds != 10 {
		t.Errorf("FreshestSeconds = %d, want 10 (the newest measurement)", gh.FreshestSeconds)
	}
	if len(gh.Members) != 3 {
		t.Fatalf("Members = %d rows, want 3", len(gh.Members))
	}
	for _, m := range gh.Members {
		if m.IsBound || m.Tag != m.Node {
			t.Errorf("unbound member %+v: want tag == node and bound=false", m)
		}
	}
	if gh.Members[0].DelayMs != 100 {
		t.Errorf("n1 delay = %d, want 100", gh.Members[0].DelayMs)
	}
	if gh.Members[1].State != HealthDead || gh.Members[1].DelayMs != 0 {
		t.Errorf("n2 = %+v, want dead with delay 0", gh.Members[1])
	}
	if gh.Members[2].State != HealthUntested || gh.Members[2].AgeSeconds != -1 {
		t.Errorf("n3 = %+v, want untested with age -1", gh.Members[2])
	}
}

// TestGroupHealthBound is the whole point of the feature: a group bound to an egress
// balances per-group COPIES, so its health must be read off the copy tags — and shown
// under the node names. The same node's BASE health is deliberately ignored, because
// it measures a different network path.
func TestGroupHealthBound(t *testing.T) {
	hist := urltest.NewHistoryStorage()
	now := time.Now()
	// Base tags say everything is fine (the nodes answer on the plain WAN)...
	hist.StoreURLTestHistory("n1", &adapter.URLTestHistory{LastOK: now, Delay: 50})
	hist.StoreURLTestHistory("n2", &adapter.URLTestHistory{LastOK: now, Delay: 60})
	hist.StoreURLTestHistory("n3", &adapter.URLTestHistory{LastOK: now, Delay: 70})
	// ...but THROUGH the group's egress only n1 answers, n2 is a confirmed failure and
	// n3 was never measured. Note the sparse indices: generate/group.go skips members
	// whose copy cannot be built, so the index is not a dense 0..n-1 and must not be
	// used to look members up positionally.
	hist.StoreURLTestHistory(copyTag("vpn", 0, "n1"),
		&adapter.URLTestHistory{LastOK: now.Add(-2 * time.Second), Delay: 480})
	hist.StoreURLTestHistory(copyTag("vpn", 4, "n2"),
		&adapter.URLTestHistory{LastFail: now.Add(-time.Second)})

	g := &fakeGroup{
		tag:  "vpn",
		kind: C.TypeSelector,
		all: []string{
			copyTag("vpn", 0, "n1"),
			copyTag("vpn", 4, "n2"),
			copyTag("vpn", 9, "n3"),
		},
		now: copyTag("vpn", 0, "n1"),
	}
	// The used-set names this group: Used=true, and an unrelated name changes nothing.
	used := map[string]bool{"vpn": true}
	gh := groupHealthOf(g, C.TypeSelector, newHealthView(hist, healthTTLFloor, now), used, true)
	if !gh.Used {
		t.Error("group named in the used-set reported Used=false")
	}

	if !gh.Bound {
		t.Error("egress-bound group reported bound=false")
	}
	if gh.Selected != "n1" {
		t.Errorf("Selected = %q, want the NODE NAME n1 (not the copy tag)", gh.Selected)
	}
	// The base tags all read alive; the group's own view is 1 alive / 1 dead / 1
	// untested. If this ever reports 3 alive, the per-node history has leaked back in.
	assertCounters(t, gh, 3, 1, 1, 1)
	if gh.Members[0].Node != "n1" || gh.Members[0].DelayMs != 480 {
		t.Errorf("member 0 = %+v, want node n1 with the COPY's delay 480 (not the base 50)", gh.Members[0])
	}
	for i, m := range gh.Members {
		if !m.IsBound {
			t.Errorf("member %d (%+v): want bound=true", i, m)
		}
		if m.Tag == m.Node {
			t.Errorf("member %d (%+v): copy tag must differ from the node name", i, m)
		}
	}
}

// TestGroupHealthSummaryOmitsMembers: withMembers=false yields counts only, so a
// periodic poll never carries a member row per node of a 376-node subscription.
func TestGroupHealthSummaryOmitsMembers(t *testing.T) {
	hist := urltest.NewHistoryStorage()
	g := &fakeGroup{tag: "auto", kind: C.TypeURLTest, all: []string{"n1", "n2"}}
	gh := groupHealthOf(g, C.TypeURLTest, newHealthView(hist, 0, time.Now()), nil, false)
	if gh.Members != nil {
		t.Errorf("summary carried %d member rows, want none", len(gh.Members))
	}
	assertCounters(t, gh, 2, 0, 0, 2)
	if gh.FreshestSeconds != -1 {
		t.Errorf("FreshestSeconds = %d, want -1 with no measurements at all", gh.FreshestSeconds)
	}
}

// TestGroupHealthUnused: a group the observatory's used-set does NOT name is
// reported Used=false — the panel renders it "unused" instead of health counters
// (its members are deliberately never probed and stay untested by design).
func TestGroupHealthUnused(t *testing.T) {
	hist := urltest.NewHistoryStorage()
	g := &fakeGroup{tag: "idle", kind: C.TypeSelector, all: []string{"n1", "n2"}}
	used := map[string]bool{"someOtherGroup": true}
	gh := groupHealthOf(g, C.TypeSelector, newHealthView(hist, 0, time.Now()), used, true)
	if gh.Used {
		t.Fatal("group absent from the used-set reported Used=true")
	}
	// The counters are still honest: everything untested, nothing invented.
	assertCounters(t, gh, 2, 0, 0, 2)
}

// TestGroupHealthNilEngine: the read is nil-safe end to end and always yields a
// non-nil slice (the panel maps over it unconditionally).
func TestGroupHealthNilEngine(t *testing.T) {
	e := New()
	gs := e.GroupHealth(true)
	if gs == nil {
		t.Fatal("GroupHealth must never be nil (want empty slice for JSON [])")
	}
	if len(gs) != 0 {
		t.Fatalf("GroupHealth = %+v, want empty with no running box", gs)
	}
	if _, ok := e.GroupHealthOne("auto"); ok {
		t.Error("GroupHealthOne found a group in a stopped engine")
	}
	if _, ok := e.GroupHealthOne(""); ok {
		t.Error("GroupHealthOne accepted an empty name")
	}
}

// assertCounters pins the two invariants the panel relies on — tested == alive+dead
// and alive+dead+untested == total — alongside the expected values.
func assertCounters(t *testing.T, gh GroupHealth, total, alive, dead, untested int) {
	t.Helper()
	if gh.Total != total || gh.Alive != alive || gh.Dead != dead || gh.Untested != untested {
		t.Errorf("counters = total %d / alive %d / dead %d / untested %d, want %d/%d/%d/%d",
			gh.Total, gh.Alive, gh.Dead, gh.Untested, total, alive, dead, untested)
	}
	if gh.Tested != gh.Alive+gh.Dead {
		t.Errorf("Tested = %d, want alive+dead = %d", gh.Tested, gh.Alive+gh.Dead)
	}
	if gh.Alive+gh.Dead+gh.Untested != gh.Total {
		t.Errorf("alive+dead+untested = %d, want total = %d",
			gh.Alive+gh.Dead+gh.Untested, gh.Total)
	}
}


// TestUsedChainNames pins the chain-name recovery from the observatory's tag-keyed
// used-set: only exit-wrapper tags "chain-<name>-h<digits>" yield a name, and every
// other tag the set can hold (group tags, per-group egress copies, chain member
// copies, plain node tags) is left out. nil ⇒ nil (the caller's "everything used").
func TestUsedChainNames(t *testing.T) {
	if got := usedChainNames(nil); got != nil {
		t.Fatalf("usedChainNames(nil) = %v, want nil (everything used sentinel)", got)
	}
	used := map[string]bool{
		"chain-relay-h2":        true, // a 2-hop chain "relay"'s exit wrapper
		"chain-relay-h1":        true, // its group-hop wrapper (used, not an exit tag — still parses to "relay")
		"chain-a-h2-h1":         true, // chain literally named "a-h2" (parseChainExitTag takes the last -h)
		"chain-relay-h1-nodeA":  false, // a member copy — must NOT parse as an exit, so no false "relay"/"nodeA"
		"auto":                  true, // a group tag — not a chain
		"group-vpn-m0-n1":       true, // a per-group egress copy — not a chain
		"n1":                    true, // a plain node tag — not a chain
		"chain-c":               true, // no hop suffix — not an exit tag
		"chain--h1":             true, // empty name — not an exit tag
	}
	got := usedChainNames(used)
	// Only the three real exit/group wrapper tags parse to a chain name. A group-hop
	// wrapper ("chain-relay-h1") is NOT an exit, but parseChainExitTag recovers the
	// same name from it (it only checks the -h<digits> suffix, not topological
	// exits-ness), so "relay" is covered either way — which is what the badge needs.
	want := map[string]bool{"relay": true, "a-h2": true}
	if len(got) != len(want) {
		t.Fatalf("usedChainNames = %v, want %v", got, want)
	}
	for name := range want {
		if !got[name] {
			t.Errorf("usedChainNames missing chain %q (got %v)", name, got)
		}
	}
}

// TestChainHealthNilUsedSet: a stopped engine (no observatory configured) yields a
// nil used-set, and ChainHealth reports every named chain Used=true — no badge is
// better than a wrong one, the same convention as GroupHealth.Used. Names are
// echoed back in order, blanks dropped.
func TestChainHealthNilUsedSet(t *testing.T) {
	e := New()
	got := e.ChainHealth([]string{"relay", "", "  unused  ", "single"})
	want := []ChainHealth{
		{Name: "relay", Used: true},
		{Name: "unused", Used: true},
		{Name: "single", Used: true},
	}
	if len(got) != len(want) {
		t.Fatalf("ChainHealth = %+v, want %+v (nil used-set ⇒ all used, blanks dropped)", got, want)
	}
	for i, w := range want {
		if got[i].Name != w.Name || got[i].Used != w.Used {
			t.Errorf("ChainHealth[%d] = %+v, want %+v", i, got[i], w)
		}
		// A stopped engine materialised nothing: hops must be empty, and the
		// contract says empty means "nothing materialised", not "no hops".
		if len(got[i].Hops) != 0 {
			t.Errorf("ChainHealth[%d].Hops = %+v, want empty on a stopped engine", i, got[i].Hops)
		}
	}
}

// TestChainHopHealthProjection drives the per-hop readout over a 3-hop chain —
// node hop L1, group hop L2, group hop L3 (the exit) — with exactly ONE hop's
// members dead. The dead state must land on THAT hop's index and nowhere else,
// the counters must obey the GroupHealth invariants on every hop, and the exit
// flag must sit on the largest index. This is the "which hop died" question the
// whole hop surface exists to answer.
//
// L3 additionally pins the ordered walk's half of the contract: it sits BEHIND
// the dead L2, so whatever its stale board records say, it is reported untested
// with L2 named — never as a hop of its own with a verdict nobody took.
func TestChainHopHealthProjection(t *testing.T) {
	hist := urltest.NewHistoryStorage()
	now := time.Now()

	// L1 (node hop wrapper): alive — the prefix up to hop 1 works.
	hist.StoreURLTestHistory("chain-c-h1", &adapter.URLTestHistory{LastOK: now.Add(-5 * time.Second), Delay: 40})
	// L2 (group hop): BOTH member copies dead — this is the hop that died.
	hist.StoreURLTestHistory("chain-c-h2-x", &adapter.URLTestHistory{LastFail: now.Add(-3 * time.Second)})
	hist.StoreURLTestHistory("chain-c-h2-y", &adapter.URLTestHistory{LastFail: now.Add(-2 * time.Second)})
	// L3 (group hop, exit): one member alive, one never measured.
	hist.StoreURLTestHistory("chain-c-h3-a", &adapter.URLTestHistory{LastOK: now.Add(-7 * time.Second), Delay: 200})
	// chain-c-h3-b: nothing at all.

	pool := []adapter.Outbound{
		&depOutbound{failingOutbound{tag: "chain-c-h1"}, nil},
		&depGroup{fakeGroup{
			tag: "chain-c-h2", kind: C.TypeSelector,
			all: []string{"chain-c-h2-x", "chain-c-h2-y"},
			now: "chain-c-h2-x",
		}, []string{"chain-c-h2-x", "chain-c-h2-y"}},
		&depGroup{fakeGroup{
			tag: "chain-c-h3", kind: C.TypeSelector,
			all: []string{"chain-c-h3-a", "chain-c-h3-b"},
			now: "chain-c-h3-a",
		}, []string{"chain-c-h3-a", "chain-c-h3-b"}},
		// Noise the projection must ignore: a member copy is not a hop, another
		// chain's wrapper is not this chain's.
		&depOutbound{failingOutbound{tag: "chain-c-h2-x"}, nil},
		&depOutbound{failingOutbound{tag: "chain-other-h1"}, nil},
	}

	hops := chainHopHealthOf(pool, "c", newHealthView(hist, healthTTLFloor, now))
	if len(hops) != 3 {
		t.Fatalf("got %d hops, want 3: %+v", len(hops), hops)
	}

	// Ordered by index, exit on the largest.
	for i, wantIdx := range []int{1, 2, 3} {
		if hops[i].Index != wantIdx {
			t.Fatalf("hops out of order: %+v", hops)
		}
		if got, want := hops[i].Exit, wantIdx == 3; got != want {
			t.Errorf("hop %d Exit = %v, want %v", wantIdx, got, want)
		}
	}

	h1, h2, h3 := hops[0], hops[1], hops[2]

	// L1: one measurement, its own numbers, no selection.
	if h1.Kind != "node" || h1.State != HealthAlive || h1.Total != 1 || h1.Alive != 1 ||
		h1.DelayMs != 40 || h1.AgeSeconds != 5 || h1.Selected != "" {
		t.Errorf("h1 = %+v, want an alive node hop with its own 40ms/5s and no selection", h1)
	}

	// L2: the dead hop. Both members tested, none alive => a POSITIVE dead
	// finding on exactly this index. The selected member (x) is dead, and a
	// dead observation carries delay 0 with the failure's age.
	if h2.Kind != "group" || h2.State != HealthDead {
		t.Fatalf("h2 = %+v, want the DEAD group hop — this is the answer to 'which hop died'", h2)
	}
	if h2.Total != 2 || h2.Tested != 2 || h2.Alive != 0 || h2.Dead != 2 || h2.Untested != 0 {
		t.Errorf("h2 counters = %+v, want 2 tested / 2 dead", h2)
	}
	if h2.Selected != "x" {
		t.Errorf("h2 Selected = %q, want the node NAME x", h2.Selected)
	}
	if h2.DelayMs != 0 || h2.AgeSeconds != 3 {
		t.Errorf("h2 delay/age = %d/%d, want 0/3 (the selected member's failure observation)", h2.DelayMs, h2.AgeSeconds)
	}

	// L3: BEHIND the break. Its board still holds an alive record for member "a"
	// (probed before L2 died), but the prober has not dialled this hop since, so
	// publishing that record would be a verdict nobody currently holds — and the
	// symmetric case, a stale FAILURE, would send the operator to fix a hop that
	// is fine. It reads untested, counters collapsed into Untested, with L2 named.
	if h3.Kind != "group" || h3.State != HealthUntested {
		t.Fatalf("h3 = %+v, want an UNTESTED hop: the walk stopped at L2 and never dialled it", h3)
	}
	if h3.BlockedBy == nil {
		t.Fatalf("h3 = %+v, want BlockedBy naming L2 — 'why is this empty' must be answered on the row", h3)
	}
	if h3.BlockedBy.Index != 2 || h3.BlockedBy.Tag != "chain-c-h2" {
		t.Errorf("h3.BlockedBy = %+v, want the dead hop 2 / chain-c-h2", *h3.BlockedBy)
	}
	if h3.Total != 2 || h3.Tested != 0 || h3.Alive != 0 || h3.Dead != 0 || h3.Untested != 2 {
		t.Errorf("h3 counters = %+v, want every member untested (nothing was measured at this hop)", h3)
	}
	if h3.DelayMs != 0 || h3.AgeSeconds != -1 {
		t.Errorf("h3 delay/age = %d/%d, want 0/-1 — there is no observation to age", h3.DelayMs, h3.AgeSeconds)
	}
	// The selection is not a measurement: the wrapper really would route through
	// "a", and that stays true whether or not anything reached this hop.
	if h3.Selected != "a" {
		t.Errorf("h3 Selected = %q, want a — a block hides measurements, not configuration", h3.Selected)
	}
	// Nothing in FRONT of the break may be touched by it.
	if h1.BlockedBy != nil || h2.BlockedBy != nil {
		t.Errorf("BlockedBy leaked onto a measured hop: h1=%+v h2=%+v", h1.BlockedBy, h2.BlockedBy)
	}

	// The invariants, on every hop, exactly as GroupHealth promises.
	for _, h := range hops {
		if h.Tested != h.Alive+h.Dead {
			t.Errorf("hop %d: Tested = %d, want alive+dead = %d", h.Index, h.Tested, h.Alive+h.Dead)
		}
		if h.Alive+h.Dead+h.Untested != h.Total {
			t.Errorf("hop %d: alive+dead+untested = %d, want total = %d",
				h.Index, h.Alive+h.Dead+h.Untested, h.Total)
		}
	}
}

// A group hop whose SELECTION has no measurement shows the freshest ALIVE
// member's numbers instead — the shown number must always be an observation
// somebody took.
func TestChainHopSelectionWithoutMeasurementFallsBack(t *testing.T) {
	hist := urltest.NewHistoryStorage()
	now := time.Now()
	hist.StoreURLTestHistory("chain-c-h1-a", &adapter.URLTestHistory{LastOK: now.Add(-30 * time.Second), Delay: 90})
	hist.StoreURLTestHistory("chain-c-h1-b", &adapter.URLTestHistory{LastOK: now.Add(-4 * time.Second), Delay: 150})
	// The selection points at c, which has no observation at all.
	pool := []adapter.Outbound{
		&depGroup{fakeGroup{
			tag: "chain-c-h1", kind: C.TypeSelector,
			all: []string{"chain-c-h1-a", "chain-c-h1-b", "chain-c-h1-c"},
			now: "chain-c-h1-c",
		}, nil},
	}
	hops := chainHopHealthOf(pool, "c", newHealthView(hist, healthTTLFloor, now))
	if len(hops) != 1 {
		t.Fatalf("got %d hops, want 1", len(hops))
	}
	h := hops[0]
	if h.Selected != "c" {
		t.Errorf("Selected = %q, want c (the selection is reported even unmeasured)", h.Selected)
	}
	if h.DelayMs != 150 || h.AgeSeconds != 4 {
		t.Errorf("delay/age = %d/%d, want the freshest ALIVE member's 150/4", h.DelayMs, h.AgeSeconds)
	}
	if h.State != HealthAlive || h.Alive != 2 || h.Untested != 1 {
		t.Errorf("hop = %+v, want alive with 2 alive / 1 untested", h)
	}
}