Four maps had no bound on a box with 512 MB that runs for months. The health board only ever inserted — the delete exists but no path in this fork calls it — and it lives on the engine context, so it outlives every generation. Its keys are node tags, and providers rename nodes on each subscription refresh: about 440k keys a year, some 88 MB. Alert dedup keyed on MAC with no delete at all. The stats aggregator's server and outbound counters were the only ones with no cap, no prune and no top-N, and one of them was handed to the panel whole on every poll. They are bounded now, evicting least-recently-seen, with numbers argued from this box rather than round: the board holds 4096 against a live generation of about 1200 tags, so a rename day cannot evict a tag still in use. Nothing is dropped silently — the same rule the log sink already follows — and a new Dropped section in the snapshot reports all six bounded aggregates, including the three that had been evicting without saying so. Snapshot did O(devices × domains) under the aggregator lock, sorting five thousand entries to show fifteen, and could read the DHCP lease file from inside it. Meanwhile the event subscribers have 64-slot buffers that drop without a counter, so an open Overview page cost the query log real rows. Selection is top-K now — proven byte-identical to the old sort over 200 random trials — and both the lease read and the row ordering happen outside the lock. The panel server had one timeout, on headers. An unauthenticated client could hold a goroutine, a socket and a descriptor forever by sending its body one byte at a time; a stopped reader on the log stream held the handler, the pipe and a child process that outlived the request. Every phase is bounded now, with the unauthenticated route on a tighter budget than the rest, and the log stream renewing its deadline per chunk so a slow-but-reading client is never truncated. And the last of the detour transports: each call built a fresh one, and the alert delivery path dropped it, pinning keep-alive sessions through the engine's own outbounds for 90 seconds — eighteen times the budget a retiring generation gets. The race skip is gone from the gate. The test it existed for raced in its own clock, not in the product; that is fixed, so nothing is excluded under -race any more. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
142 lines
4.5 KiB
Go
142 lines
4.5 KiB
Go
// lx:begin health-board
|
|
|
|
package urltest
|
|
|
|
import (
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/sagernet/sing-box/adapter"
|
|
)
|
|
|
|
// captureEvictions swaps the eviction notice sink for the duration of a test and
|
|
// returns a func that reads back everything reported.
|
|
func captureEvictions(t *testing.T) func() []string {
|
|
t.Helper()
|
|
var (
|
|
mu sync.Mutex
|
|
msgs []string
|
|
)
|
|
orig := boardEvictionLog
|
|
boardEvictionLog = func(m string) {
|
|
mu.Lock()
|
|
msgs = append(msgs, m)
|
|
mu.Unlock()
|
|
}
|
|
t.Cleanup(func() { boardEvictionLog = orig })
|
|
return func() []string {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
return append([]string(nil), msgs...)
|
|
}
|
|
}
|
|
|
|
// TestBoardHoldsAGenerationWithoutEvicting is the "what it holds" half of the
|
|
// bound. A live generation on this box is ~1200 tags (≈380 nodes plus their
|
|
// per-group egress copies and chain hops); the board must carry that — and a
|
|
// second generation's worth of overlap during a subscription rename — with no
|
|
// eviction at all, or the ceiling would be silently degrading real health data.
|
|
func TestBoardHoldsAGenerationWithoutEvicting(t *testing.T) {
|
|
read := captureEvictions(t)
|
|
s := NewHistoryStorage()
|
|
|
|
const generation = 1200
|
|
for gen := 0; gen < 2; gen++ {
|
|
for i := 0; i < generation; i++ {
|
|
s.StoreURLTestHistory("gen"+strconv.Itoa(gen)+"-node-"+strconv.Itoa(i),
|
|
&adapter.URLTestHistory{LastOK: time.Now(), Delay: 20})
|
|
}
|
|
}
|
|
if got := s.Evicted(); got != 0 {
|
|
t.Fatalf("two full generations (%d tags) evicted %d entries; the board must hold them",
|
|
2*generation, got)
|
|
}
|
|
if msgs := read(); len(msgs) != 0 {
|
|
t.Fatalf("unexpected eviction notices: %v", msgs)
|
|
}
|
|
// Everything is still readable.
|
|
if s.LoadURLTestHistory("gen0-node-0") == nil {
|
|
t.Fatalf("the first tag of the first generation was lost without an eviction")
|
|
}
|
|
}
|
|
|
|
// TestBoardEvictsOldestAndSaysSo is the "what happens when it overflows" half.
|
|
// Overflow must (a) actually bound the map, (b) drop the LEAST RECENTLY MEASURED
|
|
// tags — on this box, exactly the ones no config names any more — and (c) be
|
|
// audible: a silent eviction is a health board quietly forgetting nodes it is
|
|
// still being asked about.
|
|
func TestBoardEvictsOldestAndSaysSo(t *testing.T) {
|
|
read := captureEvictions(t)
|
|
s := NewHistoryStorage()
|
|
|
|
base := time.Now().Add(-24 * time.Hour)
|
|
// Stale generation first: measured a day ago, nothing since.
|
|
const stale = 1500
|
|
for i := 0; i < stale; i++ {
|
|
s.StoreURLTestHistory("stale-"+strconv.Itoa(i),
|
|
&adapter.URLTestHistory{LastOK: base.Add(time.Duration(i) * time.Millisecond), Delay: 30})
|
|
}
|
|
if s.Evicted() != 0 {
|
|
t.Fatalf("evicted before the ceiling was reached")
|
|
}
|
|
// Now push past the ceiling with fresh measurements.
|
|
for i := 0; i <= maxBoardEntries; i++ {
|
|
s.StoreURLTestHistory("fresh-"+strconv.Itoa(i),
|
|
&adapter.URLTestHistory{LastOK: time.Now(), Delay: 15})
|
|
}
|
|
|
|
if got := s.Evicted(); got == 0 {
|
|
t.Fatalf("board grew past %d entries without evicting anything — it is still unbounded", maxBoardEntries)
|
|
}
|
|
s.access.RLock()
|
|
size := len(s.delayHistory)
|
|
s.access.RUnlock()
|
|
if size > maxBoardEntries {
|
|
t.Fatalf("board holds %d entries, above the %d ceiling", size, maxBoardEntries)
|
|
}
|
|
|
|
// The day-old generation is what went, not the fresh one.
|
|
if s.LoadURLTestHistory("stale-0") != nil {
|
|
t.Fatalf("the oldest observation survived while newer ones were dropped")
|
|
}
|
|
if s.LoadURLTestHistory("fresh-"+strconv.Itoa(maxBoardEntries)) == nil {
|
|
t.Fatalf("the newest measurement was evicted")
|
|
}
|
|
|
|
msgs := read()
|
|
if len(msgs) == 0 {
|
|
t.Fatalf("entries were evicted with no notice — eviction must never be silent")
|
|
}
|
|
m := msgs[0]
|
|
for _, want := range []string{"health board full", "evicted", "re-probed"} {
|
|
if !strings.Contains(m, want) {
|
|
t.Fatalf("eviction notice %q does not say %q", m, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestBoardEvictionThroughMarkFailed pins the OTHER write path. MarkFailed is how
|
|
// a dead node is recorded, and a flood of dead renamed nodes is exactly the shape
|
|
// of the leak — so it has to prune too, not just the success path.
|
|
func TestBoardEvictionThroughMarkFailed(t *testing.T) {
|
|
captureEvictions(t)
|
|
s := NewHistoryStorage()
|
|
for i := 0; i <= maxBoardEntries; i++ {
|
|
s.MarkFailed("dead-" + strconv.Itoa(i))
|
|
}
|
|
s.access.RLock()
|
|
size := len(s.delayHistory)
|
|
s.access.RUnlock()
|
|
if size > maxBoardEntries {
|
|
t.Fatalf("MarkFailed grew the board to %d, above the %d ceiling", size, maxBoardEntries)
|
|
}
|
|
if s.Evicted() == 0 {
|
|
t.Fatalf("MarkFailed never prunes — the failure path is still unbounded")
|
|
}
|
|
}
|
|
|
|
// lx:end health-board
|