Subscriptions: uniqueness by subnet, not just by exit IP

unique_ips collapsed proxies per exit address, so one operator holding many
adjacent addresses counted as many distinct nodes. On the live pool the NL
exit layer looked like 87 unique IPs but was 40 distinct /24s, with 33 of
them in a single block — roughly three of four circuits leaving through one
of two operators.

Add a scope alongside the existing metric, so the two axes are independent:

  unique_ips_scope  = ip | subnet     (ip = previous behaviour, default)
  unique_ips_metric = speed | latency (unchanged)
  unique_subnet_v4  = prefix bits, default 24
  unique_subnet_v6  = prefix bits, default 48

The collapse key generalises from the exit address to a masked netip.Prefix.
Prefix lengths are clamped (v4 8-32, v6 16-128) rather than rejected so a
stored subscription can never render an empty payload, and the API persists
the normalized value so the panel shows what is actually served. Exits that
are empty or unparseable still pass through uncollapsed — dropping them would
discard distinct nodes. IPv4-mapped IPv6 is unmapped before masking.

Schema upgrade is idempotent and defaults reproduce the old behaviour, so
existing subscriptions keep serving the same node set until switched over.

The same scope is exposed on the Proxies tab (and its export) so the effect
can be previewed before it is applied to a subscription.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
omar
2026-08-08 17:48:55 +03:00
co-authored by Claude Opus 5
parent 979c2a691a
commit 5785be4964
12 changed files with 506 additions and 91 deletions
+20 -5
View File
@@ -57,10 +57,25 @@ per-protocol query whitelist → sort query alphabetically → normalize form
(scheme case, trailing `/`, percent-encoding; vmess re-packed as canonical JSON)
→ dedup by canonical URL. Broken/unparseable lines rejected.
**unique_ips** (display/serve-time, opt-in): collapse proxies sharing one
`exit_ip` to a single winner, chosen by metric `speed` (max Mbps, default) or
`latency` (min ms); tie-break: lower latency. Does not mutate the DB. Order in
subscriptions: filters → unique_ips collapse → sort → limit top-N.
**unique_ips** (display/serve-time, opt-in): collapse proxies sharing one exit to
a single winner, chosen by metric `speed` (max Mbps, default) or `latency` (min
ms); tie-break: lower latency. Does not mutate the DB. Order in subscriptions:
filters → collapse → sort → limit top-N.
Two scopes decide what "the same exit" means:
- `unique_ips_scope = ip` (default, legacy behaviour) — one winner per exit
address.
- `unique_ips_scope = subnet` — one winner per network prefix
(`unique_subnet_v4`, default 24; `unique_subnet_v6`, default 48). This is what
de-duplicates a single operator holding many adjacent addresses: an exit pool
that looks like 87 distinct IPs can be 40 distinct /24s. Prefix lengths are
clamped (v4 8–32, v6 16–128) rather than rejected, so a stored subscription can
never render an empty payload.
Exits that are empty or unparseable as an IP pass through uncollapsed — dropping
them would silently discard distinct nodes. IPv4-mapped IPv6 literals are
unmapped first, so they mask with the IPv4 prefix length.
## Query whitelist (per protocol)
@@ -83,7 +98,7 @@ Auth: single admin (login+password from env), issues a signed session token.
- `GET /dashboard` — full stats bundle (counts, trends, protocol/country/source
breakdown, latency/speed distribution, dynamics, uptime, last check, live progress)
- `GET /proxies` — filter (protocol, country, source, latency, speed, search),
optional `unique_ips` + `metric`; pagination
optional `unique_ips` + `metric` + `scope` (+ `subnet_v4`/`subnet_v6`); pagination
- `GET /proxies/export.txt` — filtered list, `\n`-joined
- `POST /proxies/recheck` — recheck selected (ids) — enqueues manual op
- `DELETE /proxies` — delete selected (removes from current session view)
+4
View File
@@ -63,6 +63,10 @@ export interface Subscription {
limit_n: number | null;
unique_ips: boolean;
unique_ips_metric: string;
/** "ip" = one winner per exit address, "subnet" = one per network prefix. */
unique_ips_scope: string;
unique_subnet_v4: number;
unique_subnet_v6: number;
sort_by: string[];
expires_at: string | null;
request_count: number;
+28 -7
View File
@@ -21,6 +21,7 @@ export default function Proxies() {
const [search, setSearch] = React.useState("");
const [uniqueIPs, setUniqueIPs] = React.useState(false);
const [metric, setMetric] = React.useState("speed");
const [scope, setScope] = React.useState("ip");
const [page, setPage] = React.useState(0);
const [selected, setSelected] = React.useState<Set<number>>(new Set());
@@ -37,15 +38,19 @@ export default function Proxies() {
if (uniqueIPs) {
p.set("unique_ips", "true");
p.set("metric", metric);
p.set("scope", scope);
}
p.set("limit", String(PAGE_SIZE));
p.set("offset", String(page * PAGE_SIZE));
return p.toString();
}, [protocol, country, source, maxLatency, minSpeed, search, uniqueIPs, metric, page]);
}, [protocol, country, source, maxLatency, minSpeed, search, uniqueIPs, metric, scope, page]);
const list = useQuery({ queryKey: ["proxies", params], queryFn: () => api.proxies(params) });
React.useEffect(() => setPage(0), [protocol, country, source, maxLatency, minSpeed, search, uniqueIPs, metric]);
React.useEffect(
() => setPage(0),
[protocol, country, source, maxLatency, minSpeed, search, uniqueIPs, metric, scope],
);
const items = list.data?.items ?? [];
const total = list.data?.total ?? 0;
@@ -159,13 +164,29 @@ export default function Proxies() {
<div className="flex flex-wrap items-center justify-between gap-3">
<div className="flex items-center gap-3">
<div className="flex items-center gap-2 rounded-lg border border-ink-700 bg-ink-850 px-3 py-1.5">
<Toggle checked={uniqueIPs} onChange={setUniqueIPs} label="Unique exit IPs" />
<Toggle checked={uniqueIPs} onChange={setUniqueIPs} label="Collapse duplicate exits" />
<span className="text-sm text-fog-muted">unique_ips</span>
{uniqueIPs && (
<Select value={metric} onChange={(e) => setMetric(e.target.value)} className="h-7 w-28 text-xs">
<option value="speed">by speed</option>
<option value="latency">by latency</option>
</Select>
<>
<Select
value={scope}
onChange={(e) => setScope(e.target.value)}
className="h-7 w-28 text-xs"
aria-label="Uniqueness scope"
>
<option value="ip">by IP</option>
<option value="subnet">by subnet</option>
</Select>
<Select
value={metric}
onChange={(e) => setMetric(e.target.value)}
className="h-7 w-28 text-xs"
aria-label="Winner metric"
>
<option value="speed">by speed</option>
<option value="latency">by latency</option>
</Select>
</>
)}
</div>
</div>
+76 -8
View File
@@ -100,7 +100,10 @@ function SubCard({
if (sub.max_latency_ms) filters.push(`≤${sub.max_latency_ms}ms`);
if (sub.min_speed_mbps) filters.push(`≥${sub.min_speed_mbps}Mbps`);
if (sub.limit_n) filters.push(`top ${sub.limit_n}`);
if (sub.unique_ips) filters.push(`unique/${sub.unique_ips_metric}`);
if (sub.unique_ips) {
const by = sub.unique_ips_scope === "subnet" ? `/${sub.unique_subnet_v4}` : "ip";
filters.push(`unique/${by}/${sub.unique_ips_metric}`);
}
return (
<div className="panel p-5">
@@ -181,6 +184,9 @@ function SubscriptionForm({
const [limitN, setLimitN] = React.useState("");
const [uniqueIPs, setUniqueIPs] = React.useState(false);
const [metric, setMetric] = React.useState("speed");
const [scope, setScope] = React.useState("ip");
const [subnetV4, setSubnetV4] = React.useState("24");
const [subnetV6, setSubnetV6] = React.useState("48");
const [sortBy, setSortBy] = React.useState<string[]>([]);
const [expires, setExpires] = React.useState("");
@@ -200,6 +206,9 @@ function SubscriptionForm({
setLimitN(editing.limit_n ? String(editing.limit_n) : "");
setUniqueIPs(editing.unique_ips);
setMetric(editing.unique_ips_metric);
setScope(editing.unique_ips_scope ?? "ip");
setSubnetV4(String(editing.unique_subnet_v4 || 24));
setSubnetV6(String(editing.unique_subnet_v6 || 48));
setSortBy(editing.sort_by);
setExpires(editing.expires_at ? editing.expires_at.slice(0, 16) : "");
} else {
@@ -216,6 +225,9 @@ function SubscriptionForm({
setLimitN("");
setUniqueIPs(false);
setMetric("speed");
setScope("ip");
setSubnetV4("24");
setSubnetV6("48");
setSortBy([]);
setExpires("");
}
@@ -250,6 +262,10 @@ function SubscriptionForm({
limit_n: limitN ? Number(limitN) : null,
unique_ips: uniqueIPs,
unique_ips_metric: metric,
unique_ips_scope: scope,
// Server clamps these; empty input falls back to the defaults.
unique_subnet_v4: Number(subnetV4) || 24,
unique_subnet_v6: Number(subnetV6) || 48,
sort_by: sortBy,
expires_at: expires ? new Date(expires).toISOString() : null,
});
@@ -394,14 +410,66 @@ function SubscriptionForm({
</div>
<div className="grid grid-cols-2 gap-4">
<div className="flex items-center gap-3 rounded-lg border border-ink-700 bg-ink-900 px-3 py-2">
<Toggle checked={uniqueIPs} onChange={setUniqueIPs} label="Unique IPs" />
<span className="text-sm text-fog-muted">unique_ips</span>
<div className="rounded-lg border border-ink-700 bg-ink-900 px-3 py-2">
<div className="flex items-center gap-3">
<Toggle checked={uniqueIPs} onChange={setUniqueIPs} label="Collapse duplicate exits" />
<span className="text-sm text-fog-muted">unique_ips</span>
{uniqueIPs && (
<>
<Select
value={scope}
onChange={(e) => setScope(e.target.value)}
className="h-7 w-24 text-xs"
aria-label="Uniqueness scope"
>
<option value="ip">by IP</option>
<option value="subnet">by subnet</option>
</Select>
<Select
value={metric}
onChange={(e) => setMetric(e.target.value)}
className="h-7 w-24 text-xs"
aria-label="Winner metric"
>
<option value="speed">speed</option>
<option value="latency">latency</option>
</Select>
</>
)}
</div>
{uniqueIPs && scope === "subnet" && (
<div className="mt-2 flex items-center gap-2 border-t border-ink-700 pt-2">
<span className="font-mono text-[11px] text-fog-faint">prefix</span>
<label className="flex items-center gap-1 font-mono text-[11px] text-fog-faint">
v4 /
<Input
type="number"
min={8}
max={32}
value={subnetV4}
onChange={(e) => setSubnetV4(e.target.value)}
className="h-7 w-16 text-xs"
/>
</label>
<label className="flex items-center gap-1 font-mono text-[11px] text-fog-faint">
v6 /
<Input
type="number"
min={16}
max={128}
value={subnetV6}
onChange={(e) => setSubnetV6(e.target.value)}
className="h-7 w-16 text-xs"
/>
</label>
</div>
)}
{uniqueIPs && (
<Select value={metric} onChange={(e) => setMetric(e.target.value)} className="h-7 w-24 text-xs">
<option value="speed">speed</option>
<option value="latency">latency</option>
</Select>
<p className="mt-2 text-[11px] leading-snug text-fog-faint">
{scope === "subnet"
? `One node per network block — collapses a single operator holding many adjacent addresses.`
: `One node per exit address — an operator with many addresses still counts many times.`}
</p>
)}
</div>
<div>
+29 -7
View File
@@ -22,11 +22,9 @@ func (s *Server) handleListProxies(w http.ResponseWriter, r *http.Request) {
}
f := parseProxyFilter(r)
q := r.URL.Query()
uniqueIPs := q.Get("unique_ips") == "true"
metric := q.Get("metric")
u := parseUnique(r)
if uniqueIPs {
if u.Enabled {
// Collapse in memory, then paginate the collapsed set.
all, err := s.db.ListSessionProxies(ctx, cur.ID, db.ProxyFilter{
Protocols: f.Protocols, Countries: f.Countries, Sources: f.Sources,
@@ -36,7 +34,7 @@ func (s *Server) handleListProxies(w http.ResponseWriter, r *http.Request) {
writeErr(w, http.StatusInternalServerError, err.Error())
return
}
collapsed := subs.CollapseUniqueIPs(all, metric)
collapsed := subs.Collapse(all, u)
total := len(collapsed)
items := paginate(collapsed, f.Offset, f.Limit)
writeJSON(w, http.StatusOK, map[string]any{"items": items, "total": total, "session": cur})
@@ -59,6 +57,30 @@ func (s *Server) handleListProxies(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, map[string]any{"items": items, "total": total, "session": cur})
}
// parseUnique reads the exit de-duplication mode from the query string. It is
// shared by the proxies list and the export endpoint so a preview in the panel
// matches what an equivalently configured subscription would serve.
func parseUnique(r *http.Request) subs.Unique {
q := r.URL.Query()
return subs.Unique{
Enabled: q.Get("unique_ips") == "true",
Scope: q.Get("scope"),
Metric: q.Get("metric"),
SubnetV4: atoiOr(q.Get("subnet_v4"), 0),
SubnetV6: atoiOr(q.Get("subnet_v6"), 0),
}
}
// atoiOr parses s, returning def when s is absent or malformed. Out-of-range
// values are left to subs.Unique, which clamps them.
func atoiOr(s string, def int) int {
n, err := strconv.Atoi(s)
if err != nil {
return def
}
return n
}
func paginate(items []db.SessionProxy, offset, limit int) []db.SessionProxy {
if offset >= len(items) {
return []db.SessionProxy{}
@@ -84,8 +106,8 @@ func (s *Server) handleExportProxies(w http.ResponseWriter, r *http.Request) {
writeErr(w, http.StatusInternalServerError, err.Error())
return
}
if r.URL.Query().Get("unique_ips") == "true" {
items = subs.CollapseUniqueIPs(items, r.URL.Query().Get("metric"))
if u := parseUnique(r); u.Enabled {
items = subs.Collapse(items, u)
}
var b strings.Builder
for _, p := range items {
+10 -5
View File
@@ -57,11 +57,16 @@ func (s *Server) handleSub(w http.ResponseWriter, r *http.Request) {
limit = *sub.LimitN
}
body, contentType := subs.Build(items, subs.Options{
Format: sub.Format,
UniqueIPs: sub.UniqueIPs,
UniqueIPsMetric: sub.UniqueIPsMetric,
SortBy: sub.SortBy,
Limit: limit,
Format: sub.Format,
Unique: subs.Unique{
Enabled: sub.UniqueIPs,
Scope: sub.UniqueIPsScope,
Metric: sub.UniqueIPsMetric,
SubnetV4: sub.UniqueSubnetV4,
SubnetV6: sub.UniqueSubnetV6,
},
SortBy: sub.SortBy,
Limit: limit,
})
w.Header().Set("Content-Type", contentType)
w.Header().Set("Cache-Control", "no-store")
+23 -4
View File
@@ -7,6 +7,7 @@ import (
"time"
"git.qomar.pw/omar/zhguchiy_perchik/internal/db"
"git.qomar.pw/omar/zhguchiy_perchik/internal/subs"
)
func (s *Server) handleListSubscriptions(w http.ResponseWriter, r *http.Request) {
@@ -34,6 +35,9 @@ type subReq struct {
LimitN *int `json:"limit_n"`
UniqueIPs *bool `json:"unique_ips"`
UniqueIPsMetric string `json:"unique_ips_metric"`
UniqueIPsScope string `json:"unique_ips_scope"`
UniqueSubnetV4 int `json:"unique_subnet_v4"`
UniqueSubnetV6 int `json:"unique_subnet_v6"`
SortBy []string `json:"sort_by"`
ExpiresAt *time.Time `json:"expires_at"`
}
@@ -60,9 +64,18 @@ func (r subReq) apply(s *db.Subscription) {
s.UniqueIPs = *r.UniqueIPs
}
s.UniqueIPsMetric = r.UniqueIPsMetric
if s.UniqueIPsMetric != "latency" {
s.UniqueIPsMetric = "speed"
if s.UniqueIPsMetric != subs.MetricLatency {
s.UniqueIPsMetric = subs.MetricSpeed
}
s.UniqueIPsScope = r.UniqueIPsScope
if s.UniqueIPsScope != subs.ScopeSubnet {
s.UniqueIPsScope = subs.ScopeIP
}
// Persist already-normalized prefix lengths so what the panel shows on the
// next edit is exactly what the subscription serves.
norm := subs.Unique{SubnetV4: r.UniqueSubnetV4, SubnetV6: r.UniqueSubnetV6}.Normalized()
s.UniqueSubnetV4 = norm.SubnetV4
s.UniqueSubnetV6 = norm.SubnetV6
s.SortBy = nonNil(r.SortBy)
s.ExpiresAt = r.ExpiresAt
}
@@ -80,7 +93,10 @@ func (s *Server) handleCreateSubscription(w http.ResponseWriter, r *http.Request
writeErr(w, http.StatusBadRequest, "bad request")
return
}
sub := &db.Subscription{Token: newToken(), Enabled: true, Format: "plain", UniqueIPsMetric: "speed"}
sub := &db.Subscription{
Token: newToken(), Enabled: true, Format: "plain",
UniqueIPsMetric: subs.MetricSpeed, UniqueIPsScope: subs.ScopeIP,
}
req.apply(sub)
created, err := s.db.CreateSubscription(r.Context(), sub)
if err != nil {
@@ -101,7 +117,10 @@ func (s *Server) handleUpdateSubscription(w http.ResponseWriter, r *http.Request
writeErr(w, http.StatusBadRequest, "bad request")
return
}
sub := &db.Subscription{ID: id, Enabled: true, UniqueIPsMetric: "speed"}
sub := &db.Subscription{
ID: id, Enabled: true,
UniqueIPsMetric: subs.MetricSpeed, UniqueIPsScope: subs.ScopeIP,
}
req.apply(sub)
updated, err := s.db.UpdateSubscription(r.Context(), sub)
if err != nil {
+21 -15
View File
@@ -56,20 +56,26 @@ type SessionProxy struct {
// Subscription is a client sub-link definition.
type Subscription struct {
ID int64 `json:"id"`
Token string `json:"token"`
Name string `json:"name"`
Enabled bool `json:"enabled"`
Format string `json:"format"`
FilterProtocols []string `json:"filter_protocols"`
FilterCountries []string `json:"filter_countries"`
FilterSources []string `json:"filter_sources"`
URLSearch []string `json:"url_search"`
MaxLatencyMs *int `json:"max_latency_ms"`
MinSpeedMbps *float64 `json:"min_speed_mbps"`
LimitN *int `json:"limit_n"`
UniqueIPs bool `json:"unique_ips"`
UniqueIPsMetric string `json:"unique_ips_metric"`
ID int64 `json:"id"`
Token string `json:"token"`
Name string `json:"name"`
Enabled bool `json:"enabled"`
Format string `json:"format"`
FilterProtocols []string `json:"filter_protocols"`
FilterCountries []string `json:"filter_countries"`
FilterSources []string `json:"filter_sources"`
URLSearch []string `json:"url_search"`
MaxLatencyMs *int `json:"max_latency_ms"`
MinSpeedMbps *float64 `json:"min_speed_mbps"`
LimitN *int `json:"limit_n"`
UniqueIPs bool `json:"unique_ips"`
UniqueIPsMetric string `json:"unique_ips_metric"`
// UniqueIPsScope selects what counts as "the same exit": "ip" (one winner
// per address) or "subnet" (one winner per network prefix, which collapses
// a single operator running many adjacent addresses).
UniqueIPsScope string `json:"unique_ips_scope"`
UniqueSubnetV4 int `json:"unique_subnet_v4"`
UniqueSubnetV6 int `json:"unique_subnet_v6"`
SortBy []string `json:"sort_by"`
ExpiresAt *time.Time `json:"expires_at"`
RequestCount int64 `json:"request_count"`
@@ -87,7 +93,7 @@ type ProxyFilter struct {
MinSpeedMbps *float64
Search string
// URLSearches matches canonical_url against any of these substrings (OR).
URLSearches []string
URLSearches []string
// UniqueIPs and Metric are applied in-memory after the DB scan.
Limit int
Offset int
+10
View File
@@ -148,6 +148,9 @@ CREATE TABLE IF NOT EXISTS subscriptions (
limit_n integer,
unique_ips boolean NOT NULL DEFAULT false,
unique_ips_metric text NOT NULL DEFAULT 'speed', -- speed|latency
unique_ips_scope text NOT NULL DEFAULT 'ip', -- ip|subnet
unique_subnet_v4 integer NOT NULL DEFAULT 24, -- prefix bits, scope=subnet
unique_subnet_v6 integer NOT NULL DEFAULT 48, -- prefix bits, scope=subnet
sort_by text[] NOT NULL DEFAULT '{}', -- e.g. {speed:desc,latency:asc}
expires_at timestamptz,
request_count bigint NOT NULL DEFAULT 0,
@@ -158,6 +161,13 @@ CREATE TABLE IF NOT EXISTS subscriptions (
-- Idempotent upgrade for databases created before url_search existed.
ALTER TABLE subscriptions ADD COLUMN IF NOT EXISTS url_search text[] NOT NULL DEFAULT '{}';
-- Idempotent upgrade for databases created before subnet-scoped uniqueness.
-- Defaults reproduce the previous behaviour exactly: scope 'ip' collapses per
-- address, so existing subscriptions keep serving the same node set.
ALTER TABLE subscriptions ADD COLUMN IF NOT EXISTS unique_ips_scope text NOT NULL DEFAULT 'ip';
ALTER TABLE subscriptions ADD COLUMN IF NOT EXISTS unique_subnet_v4 integer NOT NULL DEFAULT 24;
ALTER TABLE subscriptions ADD COLUMN IF NOT EXISTS unique_subnet_v6 integer NOT NULL DEFAULT 48;
-- ---------------------------------------------------------------------------
-- Default settings (only inserted when missing).
-- ---------------------------------------------------------------------------
+11 -5
View File
@@ -9,6 +9,7 @@ import (
const subCols = `id, token, name, enabled, format, filter_protocols, filter_countries,
filter_sources, url_search, max_latency_ms, min_speed_mbps, limit_n, unique_ips, unique_ips_metric,
unique_ips_scope, unique_subnet_v4, unique_subnet_v6,
sort_by, expires_at, request_count, last_requested_at, created_at`
func scanSub(row pgx.Row) (*Subscription, error) {
@@ -16,6 +17,7 @@ func scanSub(row pgx.Row) (*Subscription, error) {
err := row.Scan(&s.ID, &s.Token, &s.Name, &s.Enabled, &s.Format,
&s.FilterProtocols, &s.FilterCountries, &s.FilterSources, &s.URLSearch,
&s.MaxLatencyMs, &s.MinSpeedMbps, &s.LimitN, &s.UniqueIPs, &s.UniqueIPsMetric,
&s.UniqueIPsScope, &s.UniqueSubnetV4, &s.UniqueSubnetV6,
&s.SortBy, &s.ExpiresAt, &s.RequestCount, &s.LastRequestedAt, &s.CreatedAt)
if errors.Is(err, pgx.ErrNoRows) {
return nil, nil
@@ -54,11 +56,13 @@ func (d *DB) CreateSubscription(ctx context.Context, s *Subscription) (*Subscrip
row := d.pool.QueryRow(ctx,
`INSERT INTO subscriptions
(token, name, enabled, format, filter_protocols, filter_countries, filter_sources,
url_search, max_latency_ms, min_speed_mbps, limit_n, unique_ips, unique_ips_metric, sort_by, expires_at)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15)
url_search, max_latency_ms, min_speed_mbps, limit_n, unique_ips, unique_ips_metric,
unique_ips_scope, unique_subnet_v4, unique_subnet_v6, sort_by, expires_at)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18)
RETURNING `+subCols,
s.Token, s.Name, s.Enabled, s.Format, s.FilterProtocols, s.FilterCountries, s.FilterSources,
s.URLSearch, s.MaxLatencyMs, s.MinSpeedMbps, s.LimitN, s.UniqueIPs, s.UniqueIPsMetric, s.SortBy, s.ExpiresAt)
s.URLSearch, s.MaxLatencyMs, s.MinSpeedMbps, s.LimitN, s.UniqueIPs, s.UniqueIPsMetric,
s.UniqueIPsScope, s.UniqueSubnetV4, s.UniqueSubnetV6, s.SortBy, s.ExpiresAt)
return scanSub(row)
}
@@ -68,10 +72,12 @@ func (d *DB) UpdateSubscription(ctx context.Context, s *Subscription) (*Subscrip
`UPDATE subscriptions SET
name=$2, enabled=$3, format=$4, filter_protocols=$5, filter_countries=$6, filter_sources=$7,
url_search=$8, max_latency_ms=$9, min_speed_mbps=$10, limit_n=$11, unique_ips=$12, unique_ips_metric=$13,
sort_by=$14, expires_at=$15
unique_ips_scope=$14, unique_subnet_v4=$15, unique_subnet_v6=$16,
sort_by=$17, expires_at=$18
WHERE id=$1 RETURNING `+subCols,
s.ID, s.Name, s.Enabled, s.Format, s.FilterProtocols, s.FilterCountries, s.FilterSources,
s.URLSearch, s.MaxLatencyMs, s.MinSpeedMbps, s.LimitN, s.UniqueIPs, s.UniqueIPsMetric, s.SortBy, s.ExpiresAt)
s.URLSearch, s.MaxLatencyMs, s.MinSpeedMbps, s.LimitN, s.UniqueIPs, s.UniqueIPsMetric,
s.UniqueIPsScope, s.UniqueSubnetV4, s.UniqueSubnetV6, s.SortBy, s.ExpiresAt)
return scanSub(row)
}
+126 -31
View File
@@ -1,27 +1,123 @@
package subs
import (
"net/netip"
"sort"
"strings"
"git.qomar.pw/omar/zhguchiy_perchik/internal/db"
)
// Options controls how a proxy set is turned into a subscription payload.
type Options struct {
Format string // plain | base64 | clash | singbox
UniqueIPs bool // collapse proxies sharing one exit IP
UniqueIPsMetric string // speed | latency (winner selection)
SortBy []string // e.g. ["speed:desc","latency:asc","country:asc"]
Limit int // top-N (0 = no limit)
// Uniqueness scopes. ScopeIP collapses proxies sharing one exit IP; ScopeSubnet
// collapses proxies whose exit IPs fall in the same network prefix, which is
// what actually de-duplicates a single operator running many adjacent addresses.
const (
ScopeIP = "ip"
ScopeSubnet = "subnet"
)
// Winner-selection metrics.
const (
MetricSpeed = "speed"
MetricLatency = "latency"
)
// Default prefix lengths for ScopeSubnet. /24 is the smallest block routinely
// allocated to one customer in IPv4; /48 is the standard site assignment in IPv6.
const (
DefaultSubnetV4 = 24
DefaultSubnetV6 = 48
)
// Prefix-length bounds. Anything outside is clamped rather than rejected so a
// stored subscription can never render an empty payload.
const (
minSubnetV4, maxSubnetV4 = 8, 32
minSubnetV6, maxSubnetV6 = 16, 128
)
// Unique controls how proxies sharing an exit are collapsed to one winner.
// The zero value disables collapsing entirely.
type Unique struct {
Enabled bool // apply collapsing at all
Scope string // ip | subnet (empty = ip)
Metric string // speed | latency (empty = speed)
SubnetV4 int // IPv4 prefix bits, ScopeSubnet only (0 = DefaultSubnetV4)
SubnetV6 int // IPv6 prefix bits, ScopeSubnet only (0 = DefaultSubnetV6)
}
// Build applies unique_ips → sort → limit and renders the requested format.
// Normalized fills defaults and clamps prefix lengths into range. Callers that
// persist a Unique should store the normalized form so the value shown in the
// panel matches the value actually served.
func (u Unique) Normalized() Unique {
if u.Scope != ScopeSubnet {
u.Scope = ScopeIP
}
if u.Metric != MetricLatency {
u.Metric = MetricSpeed
}
u.SubnetV4 = clampBits(u.SubnetV4, minSubnetV4, maxSubnetV4, DefaultSubnetV4)
u.SubnetV6 = clampBits(u.SubnetV6, minSubnetV6, maxSubnetV6, DefaultSubnetV6)
return u
}
func clampBits(bits, lo, hi, def int) int {
if bits <= 0 {
return def
}
if bits < lo {
return lo
}
if bits > hi {
return hi
}
return bits
}
// key returns the collapse key for p and whether one could be derived. Proxies
// without a usable exit address are reported as keyless and pass through
// uncollapsed — de-duplicating them would silently drop distinct nodes.
func (u Unique) key(p db.SessionProxy) (string, bool) {
if p.ExitIP == "" {
return "", false
}
if u.Scope != ScopeSubnet {
return p.ExitIP, true
}
addr, err := netip.ParseAddr(p.ExitIP)
if err != nil {
return "", false
}
// Unmap so an IPv4-mapped IPv6 literal is masked with the IPv4 prefix length.
addr = addr.Unmap()
bits := u.SubnetV4
if addr.Is6() {
bits = u.SubnetV6
}
if bits > addr.BitLen() {
bits = addr.BitLen()
}
pfx, err := addr.Prefix(bits)
if err != nil {
return "", false
}
return pfx.String(), true
}
// Options controls how a proxy set is turned into a subscription payload.
type Options struct {
Format string // plain | base64 | clash | singbox
Unique Unique // exit de-duplication (disabled when Unique.Enabled is false)
SortBy []string // e.g. ["speed:desc","latency:asc","country:asc"]
Limit int // top-N (0 = no limit)
}
// Build applies unique → sort → limit and renders the requested format.
// Order matters: the limit counts already-collapsed proxies (per the spec).
func Build(proxies []db.SessionProxy, opt Options) (body, contentType string) {
items := proxies
if opt.UniqueIPs {
items = collapseUniqueIPs(items, opt.UniqueIPsMetric)
if opt.Unique.Enabled {
items = Collapse(items, opt.Unique)
}
sortProxies(items, opt.SortBy)
if opt.Limit > 0 && len(items) > opt.Limit {
@@ -30,46 +126,45 @@ func Build(proxies []db.SessionProxy, opt Options) (body, contentType string) {
return render(items, opt.Format)
}
// CollapseUniqueIPs is the exported unique_ips collapse used by the proxies list
// endpoint (the subscription path uses it internally via Build).
func CollapseUniqueIPs(proxies []db.SessionProxy, metric string) []db.SessionProxy {
return collapseUniqueIPs(proxies, metric)
}
// Collapse keeps one winner per exit key. The key is the exit IP under ScopeIP
// and the masked network prefix under ScopeSubnet. The winner is chosen by
// metric (speed → max Mbps, latency → min ms); tie-break is always lower
// latency. Proxies with an unknown or unparseable exit address are kept as-is.
// Input order of first appearance is preserved; sorting happens downstream.
func Collapse(proxies []db.SessionProxy, u Unique) []db.SessionProxy {
u = u.Normalized()
// collapseUniqueIPs keeps one winner per exit IP. Winner is chosen by metric
// (speed → max Mbps, latency → min ms); tie-break is always lower latency.
// Proxies with an unknown exit IP are kept as-is (cannot be de-duplicated).
func collapseUniqueIPs(proxies []db.SessionProxy, metric string) []db.SessionProxy {
best := make(map[string]db.SessionProxy)
best := make(map[string]db.SessionProxy, len(proxies))
order := make([]string, 0, len(proxies))
var passthrough []db.SessionProxy
order := make([]string, 0)
for _, p := range proxies {
if p.ExitIP == "" {
k, ok := u.key(p)
if !ok {
passthrough = append(passthrough, p)
continue
}
cur, ok := best[p.ExitIP]
if !ok {
best[p.ExitIP] = p
order = append(order, p.ExitIP)
cur, seen := best[k]
if !seen {
best[k] = p
order = append(order, k)
continue
}
if betterWinner(p, cur, metric) {
best[p.ExitIP] = p
if betterWinner(p, cur, u.Metric) {
best[k] = p
}
}
out := make([]db.SessionProxy, 0, len(order)+len(passthrough))
for _, ip := range order {
out = append(out, best[ip])
for _, k := range order {
out = append(out, best[k])
}
return append(out, passthrough...)
}
// betterWinner reports whether candidate beats current under the metric.
func betterWinner(cand, cur db.SessionProxy, metric string) bool {
if metric == "latency" {
if metric == MetricLatency {
if cand.LatencyMs != cur.LatencyMs {
return cand.LatencyMs < cur.LatencyMs
}
+148 -4
View File
@@ -8,14 +8,14 @@ import (
"git.qomar.pw/omar/zhguchiy_perchik/internal/db"
)
func TestCollapseUniqueIPs(t *testing.T) {
func TestCollapseScopeIP(t *testing.T) {
in := []db.SessionProxy{
{ProxyID: 1, ExitIP: "1.1.1.1", SpeedMbps: 10, LatencyMs: 200},
{ProxyID: 2, ExitIP: "1.1.1.1", SpeedMbps: 25, LatencyMs: 300}, // faster winner
{ProxyID: 3, ExitIP: "2.2.2.2", SpeedMbps: 5, LatencyMs: 100},
{ProxyID: 4, ExitIP: "", SpeedMbps: 1, LatencyMs: 50}, // unknown ip: passthrough
}
out := CollapseUniqueIPs(in, "speed")
out := Collapse(in, Unique{Enabled: true, Metric: MetricSpeed})
if len(out) != 3 {
t.Fatalf("want 3, got %d", len(out))
}
@@ -26,7 +26,7 @@ func TestCollapseUniqueIPs(t *testing.T) {
}
}
outLat := CollapseUniqueIPs(in, "latency")
outLat := Collapse(in, Unique{Enabled: true, Metric: MetricLatency})
for _, p := range outLat {
if p.ExitIP == "1.1.1.1" && p.ProxyID != 1 {
t.Errorf("latency winner: want proxy 1 (lower ms), got %d", p.ProxyID)
@@ -34,6 +34,150 @@ func TestCollapseUniqueIPs(t *testing.T) {
}
}
// TestCollapseScopeSubnet covers the case the feature exists for: one operator
// holding many adjacent addresses must collapse to a single node.
func TestCollapseScopeSubnet(t *testing.T) {
tests := []struct {
name string
in []db.SessionProxy
u Unique
wantN int
wantIDs []int64 // expected survivors, in output order
}{
{
name: "one operator across a /24 collapses to its fastest",
in: []db.SessionProxy{
{ProxyID: 1, ExitIP: "193.29.139.10", SpeedMbps: 5, LatencyMs: 400},
{ProxyID: 2, ExitIP: "193.29.139.77", SpeedMbps: 30, LatencyMs: 900},
{ProxyID: 3, ExitIP: "193.29.139.201", SpeedMbps: 12, LatencyMs: 260},
{ProxyID: 4, ExitIP: "51.158.206.4", SpeedMbps: 8, LatencyMs: 410},
},
u: Unique{Enabled: true, Scope: ScopeSubnet, Metric: MetricSpeed},
wantN: 2,
wantIDs: []int64{2, 4},
},
{
name: "latency metric picks the lowest ms in the block",
in: []db.SessionProxy{
{ProxyID: 1, ExitIP: "193.29.139.10", SpeedMbps: 5, LatencyMs: 400},
{ProxyID: 2, ExitIP: "193.29.139.77", SpeedMbps: 30, LatencyMs: 900},
{ProxyID: 3, ExitIP: "193.29.139.201", SpeedMbps: 12, LatencyMs: 260},
},
u: Unique{Enabled: true, Scope: ScopeSubnet, Metric: MetricLatency},
wantN: 1,
wantIDs: []int64{3},
},
{
name: "/16 merges addresses a /24 keeps apart",
in: []db.SessionProxy{
{ProxyID: 1, ExitIP: "31.76.80.205", SpeedMbps: 5},
{ProxyID: 2, ExitIP: "31.76.113.187", SpeedMbps: 9},
},
u: Unique{Enabled: true, Scope: ScopeSubnet, SubnetV4: 16},
wantN: 1,
wantIDs: []int64{2},
},
{
name: "/24 keeps those same two apart",
in: []db.SessionProxy{
{ProxyID: 1, ExitIP: "31.76.80.205", SpeedMbps: 5},
{ProxyID: 2, ExitIP: "31.76.113.187", SpeedMbps: 9},
},
u: Unique{Enabled: true, Scope: ScopeSubnet, SubnetV4: 24},
wantN: 2,
wantIDs: []int64{1, 2},
},
{
name: "IPv6 uses the v6 prefix length, not the v4 one",
in: []db.SessionProxy{
{ProxyID: 1, ExitIP: "2001:df1:7880:100::117b", SpeedMbps: 5},
{ProxyID: 2, ExitIP: "2001:df1:7880:1ff::9", SpeedMbps: 9},
{ProxyID: 3, ExitIP: "2001:df1:9999::1", SpeedMbps: 7},
},
u: Unique{Enabled: true, Scope: ScopeSubnet, SubnetV6: 48},
wantN: 2,
wantIDs: []int64{2, 3},
},
{
name: "IPv4-mapped IPv6 is masked as IPv4",
in: []db.SessionProxy{
{ProxyID: 1, ExitIP: "::ffff:203.0.113.5", SpeedMbps: 5},
{ProxyID: 2, ExitIP: "203.0.113.200", SpeedMbps: 9},
},
u: Unique{Enabled: true, Scope: ScopeSubnet, SubnetV4: 24},
wantN: 1,
wantIDs: []int64{2},
},
{
name: "unparseable and empty exits pass through uncollapsed",
in: []db.SessionProxy{
{ProxyID: 1, ExitIP: "193.29.139.10", SpeedMbps: 5},
{ProxyID: 2, ExitIP: "193.29.139.11", SpeedMbps: 9},
{ProxyID: 3, ExitIP: "not-an-ip", SpeedMbps: 1},
{ProxyID: 4, ExitIP: "", SpeedMbps: 1},
},
u: Unique{Enabled: true, Scope: ScopeSubnet},
wantN: 3,
wantIDs: []int64{2, 3, 4},
},
{
name: "out-of-range prefix bits are clamped, not rejected",
in: []db.SessionProxy{
{ProxyID: 1, ExitIP: "10.0.0.1", SpeedMbps: 5},
{ProxyID: 2, ExitIP: "10.255.255.254", SpeedMbps: 9},
},
u: Unique{Enabled: true, Scope: ScopeSubnet, SubnetV4: 999}, // → 32
wantN: 2,
wantIDs: []int64{1, 2},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
out := Collapse(tt.in, tt.u)
if len(out) != tt.wantN {
t.Fatalf("want %d survivors, got %d", tt.wantN, len(out))
}
got := make([]int64, len(out))
for i, p := range out {
got[i] = p.ProxyID
}
for i, want := range tt.wantIDs {
if got[i] != want {
t.Errorf("survivor %d: want proxy %d, got %v", i, want, got)
}
}
})
}
}
func TestUniqueNormalized(t *testing.T) {
tests := []struct {
name string
in Unique
wantScope string
wantMetric string
wantV4, wantV6 int
}{
{"zero value defaults", Unique{}, ScopeIP, MetricSpeed, DefaultSubnetV4, DefaultSubnetV6},
{"garbage scope falls back to ip", Unique{Scope: "asn"}, ScopeIP, MetricSpeed, DefaultSubnetV4, DefaultSubnetV6},
{"subnet survives", Unique{Scope: ScopeSubnet, Metric: MetricLatency}, ScopeSubnet, MetricLatency, DefaultSubnetV4, DefaultSubnetV6},
{"below range clamps up", Unique{SubnetV4: 1, SubnetV6: 4}, ScopeIP, MetricSpeed, minSubnetV4, minSubnetV6},
{"above range clamps down", Unique{SubnetV4: 64, SubnetV6: 512}, ScopeIP, MetricSpeed, maxSubnetV4, maxSubnetV6},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tt.in.Normalized()
if got.Scope != tt.wantScope || got.Metric != tt.wantMetric {
t.Errorf("scope/metric: got %s/%s, want %s/%s", got.Scope, got.Metric, tt.wantScope, tt.wantMetric)
}
if got.SubnetV4 != tt.wantV4 || got.SubnetV6 != tt.wantV6 {
t.Errorf("bits: got v4=%d v6=%d, want v4=%d v6=%d", got.SubnetV4, got.SubnetV6, tt.wantV4, tt.wantV6)
}
})
}
}
func TestBuildFormats(t *testing.T) {
items := []db.SessionProxy{
{CanonicalURL: "vless://uuid@h.com:443?security=tls&sni=a.com&type=ws",
@@ -73,7 +217,7 @@ func TestBuildLimitAfterCollapse(t *testing.T) {
{CanonicalURL: "vless://c@h3:443", Protocol: "vless", ExitIP: "2.2.2.2", SpeedMbps: 7},
}
// unique collapses to 2; limit 5 keeps both.
out, _ := Build(items, Options{Format: "plain", UniqueIPs: true, Limit: 5})
out, _ := Build(items, Options{Format: "plain", Unique: Unique{Enabled: true}, Limit: 5})
if n := strings.Count(strings.TrimSpace(out), "\n"); n != 1 {
t.Errorf("want 2 collapsed lines, got %d newlines", n)
}