shater/protocol/group/urltest_health_lx_test.go

package group

// lx: health board §5.B tests — verdict-driven selection, slot liveness read-through,
// and the dial-retry path, per mode × {alive, dead, untested, stale}.

import (
	"context"
	"errors"
	"net"
	"testing"
	"time"

	"github.com/sagernet/sing-box/adapter"
	"github.com/sagernet/sing-box/common/interrupt"
	"github.com/sagernet/sing-box/common/urltest"
	"github.com/sagernet/sing-box/log"
	M "github.com/sagernet/sing/common/metadata"
	N "github.com/sagernet/sing/common/network"
)

// --- fixtures -----------------------------------------------------------------------

// healthNode is a dialable fake outbound: DialContext/ListenPacket succeed or fail per
// the fail flag and record the dial order into dialed.
type healthNode struct {
	adapter.Outbound
	tag    string
	fail   bool
	dialed *[]string
}

func (n *healthNode) Tag() string       { return n.tag }
func (n *healthNode) Network() []string { return []string{N.NetworkTCP, N.NetworkUDP} }

func (n *healthNode) DialContext(ctx context.Context, network string, destination M.Socksaddr) (net.Conn, error) {
	if n.dialed != nil {
		*n.dialed = append(*n.dialed, n.tag)
	}
	if n.fail {
		return nil, errors.New("dial refused")
	}
	left, right := net.Pipe()
	_ = right.Close()
	return left, nil
}

func (n *healthNode) ListenPacket(ctx context.Context, destination M.Socksaddr) (net.PacketConn, error) {
	if n.dialed != nil {
		*n.dialed = append(*n.dialed, n.tag)
	}
	if n.fail {
		return nil, errors.New("listen refused")
	}
	return net.ListenPacket("udp", "127.0.0.1:0")
}

// fakeOutboundManager resolves tags over a fixed node set (only Outbound is used).
type fakeOutboundManager struct {
	adapter.OutboundManager
	nodes map[string]adapter.Outbound
}

func (m *fakeOutboundManager) Outbound(tag string) (adapter.Outbound, bool) {
	node, ok := m.nodes[tag]
	return node, ok
}

func managerOf(nodes ...adapter.Outbound) *fakeOutboundManager {
	m := &fakeOutboundManager{nodes: make(map[string]adapter.Outbound, len(nodes))}
	for _, node := range nodes {
		m.nodes[node.Tag()] = node
	}
	return m
}

// healthTestGroup builds a minimal URLTestGroup (not started: no ticker, no checker).
// interval 3m → healthTTL = 10m floor.
func healthTestGroup(hist *urltest.HistoryStorage, manager adapter.OutboundManager, nodes ...adapter.Outbound) *URLTestGroup {
	return &URLTestGroup{
		ctx:            context.Background(),
		outbound:       manager,
		outbounds:      nodes,
		history:        hist,
		interval:       3 * time.Minute,
		tolerance:      50,
		logger:         log.NewNOPFactory().Logger(),
		interruptGroup: interrupt.NewGroup(),
	}
}

// healthURLTest wraps a group into a dialable *URLTest, wiring the balancer the same way
// Start() does (slot liveness through the board verdict).
func healthURLTest(g *URLTestGroup, bal *balancer, manager adapter.OutboundManager) *URLTest {
	g.balancer = bal
	if bal != nil {
		bal.verdict = g.slotVerdict
	}
	return &URLTest{
		group:    g,
		balancer: bal,
		outbound: manager,
		logger:   log.NewNOPFactory().Logger(),
	}
}

// storeAlive records a fresh success 1s in the past: still well inside the TTL, but
// strictly older than any failure the test triggers afterwards (Windows' coarse
// monotonic clock can otherwise produce LastOK == LastFail ties within one tick).
func storeAlive(hist *urltest.HistoryStorage, tag string, delay uint16) {
	hist.StoreURLTestHistory(tag, &adapter.URLTestHistory{LastOK: time.Now().Add(-time.Second), Delay: delay})
}

// storeStale records a success far older than the TTL: verdict must degrade to untested.
func storeStale(hist *urltest.HistoryStorage, tag string, delay uint16) {
	hist.StoreURLTestHistory(tag, &adapter.URLTestHistory{LastOK: time.Now().Add(-time.Hour), Delay: delay})
}

// --- healthTTL ----------------------------------------------------------------------

func TestHealthTTLFloor(t *testing.T) {
	g := &URLTestGroup{interval: 3 * time.Minute}
	if ttl := g.healthTTL(); ttl != 10*time.Minute {
		t.Fatalf("healthTTL(3m) = %v, want the 10m floor", ttl)
	}
	g.interval = 5 * time.Minute
	if ttl := g.healthTTL(); ttl != 15*time.Minute {
		t.Fatalf("healthTTL(5m) = %v, want 3×interval = 15m", ttl)
	}
}

// --- Select (least_test): alive > untested > config order ---------------------------

func TestSelectPrefersAliveOverDeadAndUntested(t *testing.T) {
	hist := urltest.NewHistoryStorage()
	a, b, c := &balNode{tag: "a"}, &balNode{tag: "b"}, &balNode{tag: "c"}
	g := healthTestGroup(hist, nil, a, b, c)
	// a: dead but with a LOWER recorded delay than b (the old "has an entry" logic
	// would have picked it); b: alive; c: untested.
	storeAlive(hist, "a", 10)
	hist.MarkFailed("a")
	storeAlive(hist, "b", 100)
	selected, exists := g.Select(N.NetworkTCP)
	if !exists || selected != adapter.Outbound(b) {
		t.Fatalf("Select = %v (exists %v), want alive b", selected, exists)
	}
}

func TestSelectRanksAliveByDelay(t *testing.T) {
	hist := urltest.NewHistoryStorage()
	a, b := &balNode{tag: "a"}, &balNode{tag: "b"}
	g := healthTestGroup(hist, nil, a, b)
	storeAlive(hist, "a", 200)
	storeAlive(hist, "b", 50) // 200 > 50+tolerance(50) → b wins
	selected, exists := g.Select(N.NetworkTCP)
	if !exists || selected != adapter.Outbound(b) {
		t.Fatalf("Select = %v (exists %v), want faster alive b", selected, exists)
	}
}

func TestSelectHysteresisKeepsAliveCurrent(t *testing.T) {
	hist := urltest.NewHistoryStorage()
	a, b := &balNode{tag: "a"}, &balNode{tag: "b"}
	g := healthTestGroup(hist, nil, a, b)
	g.selectedOutboundTCP = a
	storeAlive(hist, "a", 100)
	storeAlive(hist, "b", 60) // within tolerance (100 ≤ 60+50) → keep a
	if selected, _ := g.Select(N.NetworkTCP); selected != adapter.Outbound(a) {
		t.Fatalf("Select = %v, want current a kept within tolerance", selected)
	}
	storeAlive(hist, "b", 40) // beyond tolerance (100 > 40+50) → switch to b
	if selected, _ := g.Select(N.NetworkTCP); selected != adapter.Outbound(b) {
		t.Fatalf("Select = %v, want b beyond tolerance", selected)
	}
}

// A dead current selection must NOT seed the hysteresis: any alive member wins.
func TestSelectDeadCurrentLosesToAlive(t *testing.T) {
	hist := urltest.NewHistoryStorage()
	a, b := &balNode{tag: "a"}, &balNode{tag: "b"}
	g := healthTestGroup(hist, nil, a, b)
	g.selectedOutboundTCP = a
	storeAlive(hist, "a", 10)
	hist.MarkFailed("a")
	storeAlive(hist, "b", 500)
	selected, exists := g.Select(N.NetworkTCP)
	if !exists || selected != adapter.Outbound(b) {
		t.Fatalf("Select = %v (exists %v), want alive b over dead current a", selected, exists)
	}
}

func TestSelectFallsBackToUntestedInConfigOrder(t *testing.T) {
	hist := urltest.NewHistoryStorage()
	a, b, c, d := &balNode{tag: "a"}, &balNode{tag: "b"}, &balNode{tag: "c"}, &balNode{tag: "d"}
	g := healthTestGroup(hist, nil, a, b, c, d)
	hist.MarkFailed("a")
	hist.MarkFailed("b")
	// c and d untested → first by config order (c), not exists.
	selected, exists := g.Select(N.NetworkTCP)
	if exists || selected != adapter.Outbound(c) {
		t.Fatalf("Select = %v (exists %v), want first untested c", selected, exists)
	}
}

func TestSelectAllDeadFallsBackToConfigOrder(t *testing.T) {
	hist := urltest.NewHistoryStorage()
	a, b := &balNode{tag: "a"}, &balNode{tag: "b"}
	g := healthTestGroup(hist, nil, a, b)
	hist.MarkFailed("a")
	hist.MarkFailed("b")
	selected, exists := g.Select(N.NetworkTCP)
	if exists || selected != adapter.Outbound(a) {
		t.Fatalf("Select = %v (exists %v), want first-by-config a", selected, exists)
	}
}

func TestSelectStaleSuccessCountsAsUntested(t *testing.T) {
	hist := urltest.NewHistoryStorage()
	a, b := &balNode{tag: "a"}, &balNode{tag: "b"}
	g := healthTestGroup(hist, nil, a, b)
	storeStale(hist, "a", 20) // success older than TTL → untested tier
	hist.MarkFailed("b")
	selected, exists := g.Select(N.NetworkTCP)
	if exists || selected != adapter.Outbound(a) {
		t.Fatalf("Select = %v (exists %v), want stale a as untested over dead b", selected, exists)
	}
	// A fresh-alive member still beats the stale one.
	c := &balNode{tag: "c"}
	g.outbounds = append(g.outbounds, c)
	storeAlive(hist, "c", 300)
	selected, exists = g.Select(N.NetworkTCP)
	if !exists || selected != adapter.Outbound(c) {
		t.Fatalf("Select = %v (exists %v), want fresh alive c over stale a", selected, exists)
	}
}

func TestSelectExcludingSkipsTriedMembers(t *testing.T) {
	hist := urltest.NewHistoryStorage()
	a, b := &balNode{tag: "a"}, &balNode{tag: "b"}
	g := healthTestGroup(hist, nil, a, b)
	storeAlive(hist, "a", 10)
	storeAlive(hist, "b", 20)
	selected, _ := g.selectExcluding(N.NetworkTCP, map[string]bool{"a": true})
	if selected != adapter.Outbound(b) {
		t.Fatalf("selectExcluding = %v, want b with a excluded", selected)
	}
	// Every member excluded → nothing to dial.
	selected, _ = g.selectExcluding(N.NetworkTCP, map[string]bool{"a": true, "b": true})
	if selected != nil {
		t.Fatalf("selectExcluding = %v, want nil with all excluded", selected)
	}
}

// --- slot liveness reads the board verdict (RR / random / failover) -----------------

func TestSlotLivenessVerdictOverridesFlag(t *testing.T) {
	hist := urltest.NewHistoryStorage()
	b := rrBalancer(t, 2, []string{"none"})
	b.verdict = func(tag string) urltest.HealthVerdict { return hist.Verdict(tag, 10*time.Minute) }
	_, resolve := resolveFrom("a", "b")
	// Both slots flagged live by the last check, but the board learned "a" died since
	// (observatory or a failed dial) — pick must skip it the very next connection.
	b.setSlots([]string{"a", "b"}, allLive("a", "b"))
	hist.MarkFailed("a")
	storeAlive(hist, "b", 30)
	for range 10 {
		picked := b.pick(context.Background(), destDomain("example.com"), nil, resolve)
		if picked == nil || picked.Tag() != "b" {
			t.Fatalf("pick = %v, want b (a is board-dead despite live flag)", picked)
		}
	}
}

func TestSlotLivenessVerdictAliveRevives(t *testing.T) {
	hist := urltest.NewHistoryStorage()
	b := rrBalancer(t, 1, []string{"none"})
	b.verdict = func(tag string) urltest.HealthVerdict { return hist.Verdict(tag, 10*time.Minute) }
	_, resolve := resolveFrom("a")
	// Slot flagged dead by the last check, but the board holds a fresh success now:
	// the verdict wins and the slot is selectable again before the next tick.
	b.setSlots([]string{"a"}, nil)
	storeAlive(hist, "a", 20)
	picked := b.pick(context.Background(), destDomain("example.com"), nil, resolve)
	if picked == nil || picked.Tag() != "a" {
		t.Fatalf("pick = %v, want board-alive a despite dead flag", picked)
	}
}

func TestRandomPickSkipsBoardDeadSlot(t *testing.T) {
	hist := urltest.NewHistoryStorage()
	b := randomBalancer(t, 3)
	b.verdict = func(tag string) urltest.HealthVerdict { return hist.Verdict(tag, 10*time.Minute) }
	_, resolve := resolveFrom("a", "b", "c")
	b.setSlots([]string{"a", "b", "c"}, allLive("a", "b", "c"))
	hist.MarkFailed("a")
	for range 50 {
		picked := b.pick(context.Background(), destDomain("example.com"), nil, resolve)
		if picked == nil || picked.Tag() == "a" {
			t.Fatalf("random pick returned board-dead a")
		}
	}
}

// failover (priority balancer, pool 1): the slot occupant went board-dead → the group
// falls back to Select, which routes to a live member, never the dead occupant.
func TestFailoverDeadSlotFallsBackToAliveMember(t *testing.T) {
	hist := urltest.NewHistoryStorage()
	dialed := []string{}
	a := &healthNode{tag: "a", fail: true, dialed: &dialed}
	nodeB := &healthNode{tag: "b", dialed: &dialed}
	manager := managerOf(a, nodeB)
	g := healthTestGroup(hist, manager, a, nodeB)
	bal := &balancer{poolSize: 1, priority: true}
	s := healthURLTest(g, bal, manager)
	bal.setSlots([]string{"a"}, allLive("a"))
	hist.MarkFailed("a")
	storeAlive(hist, "b", 40)
	selected := s.dialSelect(context.Background(), N.NetworkTCP, destDomain("example.com"), nil)
	if selected == nil || selected.Tag() != "b" {
		t.Fatalf("dialSelect = %v, want alive fallback b for a dead failover slot", selected)
	}
}

// --- dial retry: mark-fail + re-pick, ≤ 3 candidates, within deadline ---------------

func TestDialContextRetriesThroughNextAlive(t *testing.T) {
	hist := urltest.NewHistoryStorage()
	dialed := []string{}
	a := &healthNode{tag: "a", fail: true, dialed: &dialed}
	nodeB := &healthNode{tag: "b", dialed: &dialed}
	g := healthTestGroup(hist, nil, a, nodeB)
	s := healthURLTest(g, nil, nil)
	storeAlive(hist, "a", 10)
	storeAlive(hist, "b", 100)
	g.selectedOutboundTCP = a // the checker had picked a; it dies between ticks
	conn, err := s.DialContext(context.Background(), N.NetworkTCP, destDomain("example.com"))
	if err != nil {
		t.Fatalf("DialContext failed despite live member b: %v", err)
	}
	_ = conn.Close()
	if len(dialed) != 2 || dialed[0] != "a" || dialed[1] != "b" {
		t.Fatalf("dial order = %v, want [a b]", dialed)
	}
	if v := hist.Verdict("a", 10*time.Minute); v != urltest.VerdictDead {
		t.Fatalf("verdict(a) = %v, want dead after failed dial", v)
	}
	if entry := hist.LoadURLTestHistory("a"); entry == nil {
		t.Fatalf("history entry for a was deleted; mark-fail must keep it")
	}
}

func TestDialContextStopsAfterThreeCandidates(t *testing.T) {
	hist := urltest.NewHistoryStorage()
	dialed := []string{}
	nodes := make([]adapter.Outbound, 0, 4)
	for i, tag := range []string{"a", "b", "c", "d"} {
		nodes = append(nodes, &healthNode{tag: tag, fail: true, dialed: &dialed})
		storeAlive(hist, tag, uint16(10+80*i)) // spread beyond tolerance so order is a,b,c
	}
	g := healthTestGroup(hist, nil, nodes...)
	s := healthURLTest(g, nil, nil)
	_, err := s.DialContext(context.Background(), N.NetworkTCP, destDomain("example.com"))
	if err == nil {
		t.Fatalf("DialContext succeeded with every member failing")
	}
	if len(dialed) != dialAttemptsMax {
		t.Fatalf("dialed %d members (%v), want at most %d", len(dialed), dialed, dialAttemptsMax)
	}
}

func TestDialContextStopsWhenContextDone(t *testing.T) {
	hist := urltest.NewHistoryStorage()
	dialed := []string{}
	a := &healthNode{tag: "a", fail: true, dialed: &dialed}
	nodeB := &healthNode{tag: "b", dialed: &dialed}
	g := healthTestGroup(hist, nil, a, nodeB)
	s := healthURLTest(g, nil, nil)
	storeAlive(hist, "a", 10)
	storeAlive(hist, "b", 100)
	ctx, cancel := context.WithCancel(context.Background())
	cancel()
	if _, err := s.DialContext(ctx, N.NetworkTCP, destDomain("example.com")); err == nil {
		t.Fatalf("DialContext succeeded past a done context")
	}
	if len(dialed) != 1 {
		t.Fatalf("dialed %v, want a single attempt within a done context", dialed)
	}
}

func TestDialContextBalancedDemotesAndRetries(t *testing.T) {
	hist := urltest.NewHistoryStorage()
	dialed := []string{}
	a := &healthNode{tag: "a", fail: true, dialed: &dialed}
	nodeB := &healthNode{tag: "b", dialed: &dialed}
	manager := managerOf(a, nodeB)
	g := healthTestGroup(hist, manager, a, nodeB)
	bal := rrBalancer(t, 2, []string{"none"})
	s := healthURLTest(g, bal, manager)
	bal.setSlots([]string{"a", "b"}, allLive("a", "b"))
	conn, err := s.DialContext(context.Background(), N.NetworkTCP, destDomain("example.com"))
	if err != nil {
		t.Fatalf("balanced DialContext failed despite live member b: %v", err)
	}
	_ = conn.Close()
	if len(dialed) != 2 || dialed[0] != "a" || dialed[1] != "b" {
		t.Fatalf("dial order = %v, want [a b] (a demoted, b retried)", dialed)
	}
	if v := hist.Verdict("a", 10*time.Minute); v != urltest.VerdictDead {
		t.Fatalf("verdict(a) = %v, want dead after failed dial", v)
	}
	// The demotion is visible to the NEXT connection immediately: a is never dialed again.
	conn, err = s.DialContext(context.Background(), N.NetworkTCP, destDomain("example.com"))
	if err != nil {
		t.Fatalf("second balanced DialContext failed: %v", err)
	}
	_ = conn.Close()
	if dialed[len(dialed)-1] != "b" {
		t.Fatalf("dial order = %v, want the second connection to go straight to b", dialed)
	}
}

func TestListenPacketRetriesBeforeFirstSend(t *testing.T) {
	hist := urltest.NewHistoryStorage()
	dialed := []string{}
	a := &healthNode{tag: "a", fail: true, dialed: &dialed}
	nodeB := &healthNode{tag: "b", dialed: &dialed}
	g := healthTestGroup(hist, nil, a, nodeB)
	s := healthURLTest(g, nil, nil)
	storeAlive(hist, "a", 10)
	storeAlive(hist, "b", 100)
	g.selectedOutboundUDP = a
	conn, err := s.ListenPacket(context.Background(), destDomain("example.com"))
	if err != nil {
		t.Fatalf("ListenPacket failed despite live member b: %v", err)
	}
	_ = conn.Close()
	if len(dialed) != 2 || dialed[0] != "a" || dialed[1] != "b" {
		t.Fatalf("listen order = %v, want [a b]", dialed)
	}
	if v := hist.Verdict("a", 10*time.Minute); v != urltest.VerdictDead {
		t.Fatalf("verdict(a) = %v, want dead after failed listen", v)
	}
}

// --- check failures mark the board, never delete ------------------------------------

func TestTestNodesMarksFailedKeepsEntry(t *testing.T) {
	hist := urltest.NewHistoryStorage()
	a := &healthNode{tag: "a", fail: true}
	manager := managerOf(a)
	g := healthTestGroup(hist, manager, a)
	storeAlive(hist, "a", 42)
	result := g.testNodes(context.Background(), []adapter.Outbound{a}, true)
	if len(result) != 0 {
		t.Fatalf("testNodes result = %v, want empty for a failing node", result)
	}
	entry := hist.LoadURLTestHistory("a")
	if entry == nil {
		t.Fatalf("history entry deleted on check failure; must be marked instead")
	}
	if entry.Delay != 42 {
		t.Fatalf("entry.Delay = %d, want the last-known 42 preserved", entry.Delay)
	}
	if v := hist.Verdict("a", 10*time.Minute); v != urltest.VerdictDead {
		t.Fatalf("verdict(a) = %v, want dead after failed check", v)
	}
}

// --- Pool / seedPool read the verdict, not entry presence ---------------------------

func TestPoolDelayRequiresAliveVerdict(t *testing.T) {
	hist := urltest.NewHistoryStorage()
	a := &healthNode{tag: "a"}
	manager := managerOf(a)
	g := healthTestGroup(hist, manager, a)
	bal := rrBalancer(t, 1, []string{"none"})
	s := healthURLTest(g, bal, manager)
	bal.setSlots([]string{"a"}, allLive("a"))
	storeAlive(hist, "a", 0) // live sub-ms → clamped to 1
	pool := s.Pool()
	if len(pool) != 1 || pool[0].Delay != 1 {
		t.Fatalf("Pool = %v, want alive slot a with clamped delay 1", pool)
	}
	hist.MarkFailed("a") // entry still present, but dead → delay must read 0
	pool = s.Pool()
	if len(pool) != 1 || pool[0].Delay != 0 {
		t.Fatalf("Pool = %v, want dead slot a reported with delay 0", pool)
	}
}

func TestSeedPoolSkipsBoardDeadNodes(t *testing.T) {
	hist := urltest.NewHistoryStorage()
	a, b, c := &balNode{tag: "a"}, &balNode{tag: "b"}, &balNode{tag: "c"}
	g := healthTestGroup(hist, nil, a, b, c)
	bal := rrBalancer(t, 2, []string{"none"})
	g.balancer = bal
	// a has an entry (fast, but DEAD); c is fresh-alive. The warm seed must be c, not a.
	storeAlive(hist, "a", 10)
	hist.MarkFailed("a")
	storeAlive(hist, "c", 30)
	g.seedPool()
	tags := bal.poolTags()
	if len(tags) != 2 || tags[0] != "c" {
		t.Fatalf("seeded pool = %v, want the alive c seeded first", tags)
	}
}