shater/common/urltest/board_lx.go

// lx:begin health-board

// Health board (plan §5.A): failure tracking and verdict computation on top of
// HistoryStorage. Successes keep flowing through StoreURLTestHistory; failures are
// recorded with MarkFailed instead of deleting the entry (deletion stays reserved
// for nodes removed from the configuration), and consumers classify a tag at read
// time with Verdict. One store, one truth: whoever learns about a death — the
// group's own checker, the observatory, or a failed user dial — marks it here.

package urltest

import (
	"sort"
	"strconv"
	"sync"
	"time"

	"github.com/sagernet/sing-box/adapter"
	"github.com/sagernet/sing-box/log"
)

// --- board capacity ---------------------------------------------------------
//
// The board is the one structure in the daemon whose key space is chosen by
// somebody else. Its keys are outbound TAGS, and on this box a tag is a node
// NAME straight out of the subscription — plus the derived per-group egress
// copies ("group-<g>-m<i>-<node>") and per-chain hop copies the probe planner
// creates for the same nodes. Providers rename their nodes freely, so a daily
// subscription refresh introduces a whole new generation of keys, while the
// store itself is pinned to the ENGINE's context (shater/engine.New) and so
// outlives every generation and every Apply — by design, so health survives a
// config change.
//
// Nothing ever removed a key. DeleteURLTestHistory exists but no shater path
// calls it (only daemon/ and clashapi/, which this fork does not run), so the
// map was strictly append-only for the life of the process — and the process is
// expected to live for months.
//
// The arithmetic: ~380 nodes, and a config with a couple of egress-bound groups
// plus a handful of chains puts a LIVE generation at roughly 380 base tags +
// 2x380 group copies + ~100 chain copies ≈ 1200 keys. One new generation per day
// is ~440k keys a year, at ~200 B per entry (map bucket + a tag string that is
// routinely 30-50 B with flag emoji, + a 56 B URLTestHistory) ≈ 88 MB of a
// 512 MB box — spent entirely on nodes that no longer exist.
const (
	// maxBoardEntries is the hard ceiling. 4096 is ~3.4 live generations, so the
	// board comfortably holds the current config plus the overlap while a
	// subscription refresh swaps names, and still costs under a megabyte. A tighter
	// bound would start evicting tags the running config actually uses; a looser one
	// would stop being a bound in any useful sense.
	maxBoardEntries = 4096
	// keepBoardEntries is the prune target: drop a quarter at a time so the
	// O(n log n) selection is amortised over ~1024 inserts instead of running on
	// every probe once the board is full.
	keepBoardEntries = 3072
)

// boardEvictionLog reports an eviction. A package var so tests can capture it;
// production leaves it writing to the process log, which under procd is the same
// syslog/logsink stream every other daemon line lands in.
//
// Eviction is NEVER silent. It is not free either: an evicted tag reverts to
// "untested" and its next probe re-measures it, so a board that evicts entries
// belonging to the LIVE config is a board whose ceiling is too low — and the only
// way anyone finds that out is this line.
var boardEvictionLog = func(msg string) { boardLogger().Warn(msg) }

// pruneLocked drops the least-recently-OBSERVED entries when the board exceeds
// maxBoardEntries. "Least recently observed" is max(LastOK, LastFail): the entry
// nothing has measured for the longest is, on this box, precisely a tag that no
// longer exists in any config — a renamed node, a removed group copy, a retired
// chain hop. Caller holds access.
func (s *HistoryStorage) pruneLocked() {
	if len(s.delayHistory) <= maxBoardEntries {
		return
	}
	type kv struct {
		tag  string
		seen time.Time
	}
	all := make([]kv, 0, len(s.delayHistory))
	for tag, h := range s.delayHistory {
		seen := h.LastOK
		if h.LastFail.After(seen) {
			seen = h.LastFail
		}
		all = append(all, kv{tag, seen})
	}
	sort.Slice(all, func(i, j int) bool { return all[i].seen.Before(all[j].seen) })
	drop := len(all) - keepBoardEntries
	var oldest time.Time
	for i := 0; i < drop; i++ {
		if i == 0 {
			oldest = all[i].seen
		}
		delete(s.delayHistory, all[i].tag)
	}
	s.evicted += uint64(drop)

	msg := "urltest: health board full (" + strconv.Itoa(maxBoardEntries) + " tags) — evicted " +
		strconv.Itoa(drop) + " least-recently-measured entries (" + strconv.FormatUint(s.evicted, 10) +
		" total since start); they revert to untested and will be re-probed"
	if !oldest.IsZero() {
		msg += "; oldest observation was " + time.Since(oldest).Truncate(time.Second).String() + " ago"
	}
	boardEvictionLog(msg)
}

// Evicted reports how many entries the capacity bound has dropped since the store
// was created. Nonzero means the board reached maxBoardEntries at least once.
func (s *HistoryStorage) Evicted() uint64 {
	if s == nil {
		return 0
	}
	s.access.RLock()
	defer s.access.RUnlock()
	return s.evicted
}

// boardLogger is the process-wide fallback logger for eviction notices. The store
// is built from a plain constructor with no logger in sight (box.New, the daemon,
// shater/engine all call NewHistoryStorage()), so rather than change that
// signature everywhere the notice goes to the standard logger — which on the
// router is the daemon's own stderr, i.e. the same sink logsink owns.
var (
	boardLogOnce sync.Once
	boardLog     log.ContextLogger
)

func boardLogger() log.ContextLogger {
	boardLogOnce.Do(func() { boardLog = log.StdLogger() })
	return boardLog
}

// HealthVerdict classifies a stored history entry at read time.
type HealthVerdict int

const (
	// VerdictUntested means nothing fresh enough is known either way: no entry,
	// or every observation is older than the caller's TTL.
	VerdictUntested HealthVerdict = iota
	// VerdictAlive means the newest fresh observation is a success.
	VerdictAlive
	// VerdictDead means the newest fresh observation is a failure.
	VerdictDead
)

func (v HealthVerdict) String() string {
	switch v {
	case VerdictAlive:
		return "alive"
	case VerdictDead:
		return "dead"
	default:
		return "untested"
	}
}

// MarkFailed records a failed probe or dial for tag: LastFail is set to now while
// LastOK/Delay of an existing entry are preserved, so a node that once worked keeps
// its last known latency for display. The entry is never deleted here — a marked
// tag stays distinguishable from "never measured" (plan §2 Д5).
func (s *HistoryStorage) MarkFailed(tag string) {
	if s == nil {
		return
	}
	s.access.Lock()
	updated := &adapter.URLTestHistory{LastFail: time.Now()}
	if previous := s.delayHistory[tag]; previous != nil {
		updated.LastOK = previous.LastOK
		updated.Delay = previous.Delay
	}
	s.delayHistory[tag] = updated
	s.pruneLocked()
	s.notifyUpdated()
	s.access.Unlock()
}

// Verdict classifies tag against the wall clock: alive when the last success is
// newer than the last failure and younger than ttl, dead when the last failure is
// newer than the last success and younger than ttl, untested otherwise.
func (s *HistoryStorage) Verdict(tag string, ttl time.Duration) HealthVerdict {
	return s.VerdictAt(tag, ttl, time.Now())
}

// VerdictAt is Verdict against an explicit clock, for deterministic tests.
func (s *HistoryStorage) VerdictAt(tag string, ttl time.Duration, now time.Time) HealthVerdict {
	history := s.LoadURLTestHistory(tag)
	if history == nil {
		return VerdictUntested
	}
	switch {
	case history.LastOK.After(history.LastFail) && now.Sub(history.LastOK) < ttl:
		return VerdictAlive
	case history.LastFail.After(history.LastOK) && now.Sub(history.LastFail) < ttl:
		return VerdictDead
	default:
		return VerdictUntested
	}
}

// lx:end health-board