Add selectable source type, fix Gitea CI image push, compose pulls from registry
- sources: new `kind` (auto|plain|base64) — fetcher honors it (plain = no base64 decode; base64 = force decode; auto = try base64 then plain). Schema ALTER for existing DBs, API validation, Settings UI selector (add-form + inline). - CI: rewrite images job to the proven pattern (buildx + metadata-action + build-push-action) using a write:package PAT secret (REGISTRY_TOKEN); the automatic GITEA_TOKEN cannot push packages. Matrix over checker/api/web. - compose: docker-compose.yml now pulls git.qomar.pw/omar/zhguchiy_perchik/* images from the Gitea registry; docker-compose.build.yml is the local-build override. Verified on a real run: 3 aggregated lists → 7838 raw → 4842 unique (canonical dedup) → 194 working proxies across all four protocols, served through the panel and subscriptions. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014TtR4PP2JM9KadaBkhPGAE
This commit is contained in:
omar
+51
-32
@@ -3,27 +3,29 @@ name: CI
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
tags: ["v*"]
|
||||
pull_request:
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
REGISTRY: git.qomar.pw
|
||||
# git.qomar.pw/omar/zhguchiy_perchik — images are namespaced under the repo.
|
||||
IMAGE_BASE: git.qomar.pw/${{ github.repository }}
|
||||
|
||||
jobs:
|
||||
# Fast validation on every push/PR: Go vet + tests, and the frontend build
|
||||
# (which runs tsc typecheck first).
|
||||
# Fast validation on every push/PR: Go vet + tests + build, and the frontend
|
||||
# build (which runs tsc typecheck first).
|
||||
validate:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: "1.26"
|
||||
cache: true
|
||||
- name: go vet
|
||||
run: go vet ./...
|
||||
- name: go test
|
||||
run: go test ./...
|
||||
- name: go build
|
||||
run: go build ./...
|
||||
|
||||
- run: go vet ./...
|
||||
- run: go test ./...
|
||||
- run: go build ./...
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "22"
|
||||
@@ -34,34 +36,51 @@ jobs:
|
||||
npm run build
|
||||
|
||||
# Build and push the three images to the Gitea container registry. Runs only
|
||||
# on pushes to main. Requires a runner with Docker available.
|
||||
# on pushes to main / tags. Uses a PAT with write:package scope (the automatic
|
||||
# GITEA_TOKEN cannot push packages), stored as the REGISTRY_TOKEN secret.
|
||||
images:
|
||||
needs: validate
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
if: github.event_name != 'pull_request'
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
REGISTRY: git.qomar.pw
|
||||
OWNER: omar
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- { name: checker, dockerfile: deploy/Dockerfile.checker }
|
||||
- { name: api, dockerfile: deploy/Dockerfile.api }
|
||||
- { name: web, dockerfile: deploy/Dockerfile.web }
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to Gitea registry
|
||||
run: echo "${{ secrets.GITEA_TOKEN }}" | docker login "$REGISTRY" -u "${{ github.actor }}" --password-stdin
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ secrets.REGISTRY_USERNAME || 'omar' }}
|
||||
password: ${{ secrets.REGISTRY_TOKEN }}
|
||||
|
||||
- name: Build & push checker
|
||||
run: |
|
||||
docker build -f deploy/Dockerfile.checker -t "$REGISTRY/$OWNER/perchik-checker:latest" -t "$REGISTRY/$OWNER/perchik-checker:${GITHUB_SHA::8}" .
|
||||
docker push "$REGISTRY/$OWNER/perchik-checker:latest"
|
||||
docker push "$REGISTRY/$OWNER/perchik-checker:${GITHUB_SHA::8}"
|
||||
- name: Extract metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.IMAGE_BASE }}/${{ matrix.name }}
|
||||
tags: |
|
||||
type=ref,event=branch
|
||||
type=ref,event=tag
|
||||
type=sha,prefix=sha-
|
||||
type=raw,value=latest,enable={{is_default_branch}}
|
||||
|
||||
- name: Build & push api
|
||||
run: |
|
||||
docker build -f deploy/Dockerfile.api -t "$REGISTRY/$OWNER/perchik-api:latest" -t "$REGISTRY/$OWNER/perchik-api:${GITHUB_SHA::8}" .
|
||||
docker push "$REGISTRY/$OWNER/perchik-api:latest"
|
||||
docker push "$REGISTRY/$OWNER/perchik-api:${GITHUB_SHA::8}"
|
||||
|
||||
- name: Build & push web
|
||||
run: |
|
||||
docker build -f deploy/Dockerfile.web -t "$REGISTRY/$OWNER/perchik-web:latest" -t "$REGISTRY/$OWNER/perchik-web:${GITHUB_SHA::8}" .
|
||||
docker push "$REGISTRY/$OWNER/perchik-web:latest"
|
||||
docker push "$REGISTRY/$OWNER/perchik-web:${GITHUB_SHA::8}"
|
||||
- name: Build and push ${{ matrix.name }}
|
||||
uses: docker/build-push-action@v6
|
||||
env:
|
||||
DOCKER_BUILD_RECORD_UPLOAD: "false"
|
||||
DOCKER_BUILD_SUMMARY: "false"
|
||||
with:
|
||||
context: .
|
||||
file: ${{ matrix.dockerfile }}
|
||||
platforms: linux/amd64
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
# Local-build override. Compose with the base file to build images from source
|
||||
# instead of pulling them from the Gitea registry:
|
||||
# docker compose -f docker-compose.yml -f docker-compose.build.yml up -d --build
|
||||
|
||||
services:
|
||||
checker:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: deploy/Dockerfile.checker
|
||||
api:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: deploy/Dockerfile.api
|
||||
web:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: deploy/Dockerfile.web
|
||||
+13
-14
@@ -1,5 +1,13 @@
|
||||
# zhguchiy_perchik — full stack in one compose file.
|
||||
# Copy .env.example to .env and adjust before `docker compose up -d --build`.
|
||||
# zhguchiy_perchik — production stack. Pulls prebuilt images from the Gitea
|
||||
# container registry (built & pushed by .gitea/workflows/ci.yml).
|
||||
#
|
||||
# cp .env.example .env # set ADMIN_PASSWORD, SESSION_SECRET
|
||||
# docker compose pull
|
||||
# docker compose up -d
|
||||
#
|
||||
# For local development that builds the images from source instead of pulling,
|
||||
# add the build override:
|
||||
# docker compose -f docker-compose.yml -f docker-compose.build.yml up -d --build
|
||||
|
||||
services:
|
||||
postgres:
|
||||
@@ -18,10 +26,7 @@ services:
|
||||
retries: 10
|
||||
|
||||
checker:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: deploy/Dockerfile.checker
|
||||
image: ${REGISTRY:-zhguchiy_perchik}/checker:${TAG:-latest}
|
||||
image: ${REGISTRY:-git.qomar.pw/omar/zhguchiy_perchik}/checker:${TAG:-latest}
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
DB_DSN: postgres://${POSTGRES_USER:-perchik}:${POSTGRES_PASSWORD:-perchik}@postgres:5432/${POSTGRES_DB:-perchik}?sslmode=disable
|
||||
@@ -31,10 +36,7 @@ services:
|
||||
condition: service_healthy
|
||||
|
||||
api:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: deploy/Dockerfile.api
|
||||
image: ${REGISTRY:-zhguchiy_perchik}/api:${TAG:-latest}
|
||||
image: ${REGISTRY:-git.qomar.pw/omar/zhguchiy_perchik}/api:${TAG:-latest}
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
DB_DSN: postgres://${POSTGRES_USER:-perchik}:${POSTGRES_PASSWORD:-perchik}@postgres:5432/${POSTGRES_DB:-perchik}?sslmode=disable
|
||||
@@ -47,10 +49,7 @@ services:
|
||||
condition: service_healthy
|
||||
|
||||
web:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: deploy/Dockerfile.web
|
||||
image: ${REGISTRY:-zhguchiy_perchik}/web:${TAG:-latest}
|
||||
image: ${REGISTRY:-git.qomar.pw/omar/zhguchiy_perchik}/web:${TAG:-latest}
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "${WEB_PORT:-8088}:80"
|
||||
|
||||
@@ -73,9 +73,9 @@ export const api = {
|
||||
|
||||
// sources
|
||||
sources: () => request<{ items: Source[] }>("GET", "/api/v1/sources"),
|
||||
createSource: (body: { name: string; url: string; enabled: boolean }) =>
|
||||
createSource: (body: { name: string; url: string; kind: string; enabled: boolean }) =>
|
||||
request<{ id: number }>("POST", "/api/v1/sources", body),
|
||||
updateSource: (id: number, body: { name: string; url: string; enabled: boolean }) =>
|
||||
updateSource: (id: number, body: { name: string; url: string; kind: string; enabled: boolean }) =>
|
||||
request<{ status: string }>("PATCH", `/api/v1/sources/${id}`, body),
|
||||
deleteSource: (id: number) => request<{ status: string }>("DELETE", `/api/v1/sources/${id}`),
|
||||
|
||||
|
||||
@@ -40,6 +40,7 @@ export interface Source {
|
||||
id: number;
|
||||
name: string;
|
||||
url: string;
|
||||
kind: string;
|
||||
enabled: boolean;
|
||||
last_fetched_at: string | null;
|
||||
last_line_count: number;
|
||||
|
||||
@@ -2,7 +2,7 @@ import * as React from "react";
|
||||
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
import { Plus, Trash2, Check, X } from "lucide-react";
|
||||
import { api } from "@/lib/api";
|
||||
import { Badge, Button, Card, CardTitle, Input, Label, Spinner, Toggle } from "@/components/ui";
|
||||
import { Badge, Button, Card, CardTitle, Input, Label, Select, Spinner, Toggle } from "@/components/ui";
|
||||
import { useToast } from "@/components/Toast";
|
||||
import { fmtAgo, fmtInt } from "@/lib/format";
|
||||
import type { Settings as SettingsMap, Source } from "@/lib/types";
|
||||
@@ -28,19 +28,27 @@ function SourcesPanel() {
|
||||
const list = useQuery({ queryKey: ["sources"], queryFn: api.sources });
|
||||
const [name, setName] = React.useState("");
|
||||
const [url, setUrl] = React.useState("");
|
||||
const [kind, setKind] = React.useState("auto");
|
||||
|
||||
const create = useMutation({
|
||||
mutationFn: () => api.createSource({ name: name.trim() || url.trim(), url: url.trim(), enabled: true }),
|
||||
mutationFn: () => api.createSource({ name: name.trim() || url.trim(), url: url.trim(), kind, enabled: true }),
|
||||
onSuccess: () => {
|
||||
toast("success", "Source added");
|
||||
setName("");
|
||||
setUrl("");
|
||||
setKind("auto");
|
||||
qc.invalidateQueries({ queryKey: ["sources"] });
|
||||
},
|
||||
onError: (e: Error) => toast("error", e.message),
|
||||
});
|
||||
const toggle = useMutation({
|
||||
mutationFn: (s: Source) => api.updateSource(s.id, { name: s.name, url: s.url, enabled: !s.enabled }),
|
||||
mutationFn: (s: Source) => api.updateSource(s.id, { name: s.name, url: s.url, kind: s.kind, enabled: !s.enabled }),
|
||||
onSuccess: () => qc.invalidateQueries({ queryKey: ["sources"] }),
|
||||
onError: (e: Error) => toast("error", e.message),
|
||||
});
|
||||
const changeKind = useMutation({
|
||||
mutationFn: ({ s, kind }: { s: Source; kind: string }) =>
|
||||
api.updateSource(s.id, { name: s.name, url: s.url, kind, enabled: s.enabled }),
|
||||
onSuccess: () => qc.invalidateQueries({ queryKey: ["sources"] }),
|
||||
onError: (e: Error) => toast("error", e.message),
|
||||
});
|
||||
@@ -70,10 +78,18 @@ function SourcesPanel() {
|
||||
<Label>Name</Label>
|
||||
<Input value={name} onChange={(e) => setName(e.target.value)} placeholder="optional" />
|
||||
</div>
|
||||
<div className="min-w-[280px] flex-1">
|
||||
<div className="min-w-[240px] flex-1">
|
||||
<Label>List URL</Label>
|
||||
<Input value={url} onChange={(e) => setUrl(e.target.value)} placeholder="https://…/all_configs.txt" />
|
||||
</div>
|
||||
<div className="w-32">
|
||||
<Label>Type</Label>
|
||||
<Select value={kind} onChange={(e) => setKind(e.target.value)}>
|
||||
<option value="auto">auto</option>
|
||||
<option value="plain">plain (\n)</option>
|
||||
<option value="base64">base64</option>
|
||||
</Select>
|
||||
</div>
|
||||
<Button type="submit" variant="primary" size="sm" disabled={create.isPending || !url.trim()}>
|
||||
<Plus size={14} /> Add
|
||||
</Button>
|
||||
@@ -90,6 +106,7 @@ function SourcesPanel() {
|
||||
<tr className="eyebrow border-b border-ink-700 text-fog-faint">
|
||||
<th className="w-16 py-2 font-normal">On</th>
|
||||
<th className="py-2 font-normal">Name / URL</th>
|
||||
<th className="py-2 font-normal">Type</th>
|
||||
<th className="py-2 text-right font-normal">Last lines</th>
|
||||
<th className="py-2 text-right font-normal">Fetched</th>
|
||||
<th className="py-2 pr-2 text-right font-normal">Status</th>
|
||||
@@ -108,6 +125,18 @@ function SourcesPanel() {
|
||||
{s.url}
|
||||
</div>
|
||||
</td>
|
||||
<td className="py-2.5">
|
||||
<Select
|
||||
value={s.kind}
|
||||
onChange={(e) => changeKind.mutate({ s, kind: e.target.value })}
|
||||
className="h-7 w-24 text-xs"
|
||||
aria-label="Source type"
|
||||
>
|
||||
<option value="auto">auto</option>
|
||||
<option value="plain">plain</option>
|
||||
<option value="base64">base64</option>
|
||||
</Select>
|
||||
</td>
|
||||
<td className="num py-2.5 text-right text-fog-muted">{fmtInt(s.last_line_count)}</td>
|
||||
<td className="py-2.5 text-right font-mono text-fog-faint">{fmtAgo(s.last_fetched_at)}</td>
|
||||
<td className="py-2.5 pr-2 text-right">
|
||||
|
||||
@@ -22,9 +22,19 @@ func (s *Server) handleListSources(w http.ResponseWriter, r *http.Request) {
|
||||
type sourceReq struct {
|
||||
Name string `json:"name"`
|
||||
URL string `json:"url"`
|
||||
Kind string `json:"kind"`
|
||||
Enabled *bool `json:"enabled"`
|
||||
}
|
||||
|
||||
var validSourceKinds = map[string]struct{}{"auto": {}, "plain": {}, "base64": {}}
|
||||
|
||||
func normalizeKind(k string) string {
|
||||
if _, ok := validSourceKinds[k]; ok {
|
||||
return k
|
||||
}
|
||||
return "auto"
|
||||
}
|
||||
|
||||
func (s *Server) handleCreateSource(w http.ResponseWriter, r *http.Request) {
|
||||
var req sourceReq
|
||||
if err := decodeJSON(r, &req); err != nil {
|
||||
@@ -44,7 +54,7 @@ func (s *Server) handleCreateSource(w http.ResponseWriter, r *http.Request) {
|
||||
if req.Enabled != nil {
|
||||
enabled = *req.Enabled
|
||||
}
|
||||
id, err := s.db.CreateSource(r.Context(), req.Name, req.URL, enabled)
|
||||
id, err := s.db.CreateSource(r.Context(), req.Name, req.URL, normalizeKind(req.Kind), enabled)
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
@@ -67,7 +77,7 @@ func (s *Server) handleUpdateSource(w http.ResponseWriter, r *http.Request) {
|
||||
if req.Enabled != nil {
|
||||
enabled = *req.Enabled
|
||||
}
|
||||
if err := s.db.UpdateSource(r.Context(), id, strings.TrimSpace(req.Name), strings.TrimSpace(req.URL), enabled); err != nil {
|
||||
if err := s.db.UpdateSource(r.Context(), id, strings.TrimSpace(req.Name), strings.TrimSpace(req.URL), normalizeKind(req.Kind), enabled); err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ type Source struct {
|
||||
ID int64 `json:"id"`
|
||||
Name string `json:"name"`
|
||||
URL string `json:"url"`
|
||||
Kind string `json:"kind"`
|
||||
Enabled bool `json:"enabled"`
|
||||
LastFetchedAt *time.Time `json:"last_fetched_at"`
|
||||
LastLineCount int `json:"last_line_count"`
|
||||
|
||||
@@ -18,12 +18,15 @@ CREATE TABLE IF NOT EXISTS sources (
|
||||
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
||||
name text NOT NULL,
|
||||
url text NOT NULL UNIQUE,
|
||||
kind text NOT NULL DEFAULT 'auto', -- auto | plain | base64
|
||||
enabled boolean NOT NULL DEFAULT true,
|
||||
last_fetched_at timestamptz,
|
||||
last_line_count integer NOT NULL DEFAULT 0,
|
||||
last_error text NOT NULL DEFAULT '',
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
-- Idempotent upgrade for databases created before `kind` existed.
|
||||
ALTER TABLE sources ADD COLUMN IF NOT EXISTS kind text NOT NULL DEFAULT 'auto';
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- proxies: persistent per-canonical-URL history. Kept even while a proxy is
|
||||
|
||||
+13
-15
@@ -2,11 +2,11 @@ package db
|
||||
|
||||
import "context"
|
||||
|
||||
const sourceCols = `id, name, url, kind, enabled, last_fetched_at, last_line_count, last_error, created_at`
|
||||
|
||||
// ListSources returns all sources ordered by id.
|
||||
func (d *DB) ListSources(ctx context.Context) ([]Source, error) {
|
||||
rows, err := d.pool.Query(ctx,
|
||||
`SELECT id, name, url, enabled, last_fetched_at, last_line_count, last_error, created_at
|
||||
FROM sources ORDER BY id`)
|
||||
rows, err := d.pool.Query(ctx, `SELECT `+sourceCols+` FROM sources ORDER BY id`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -16,9 +16,7 @@ func (d *DB) ListSources(ctx context.Context) ([]Source, error) {
|
||||
|
||||
// ListActiveSources returns only enabled sources.
|
||||
func (d *DB) ListActiveSources(ctx context.Context) ([]Source, error) {
|
||||
rows, err := d.pool.Query(ctx,
|
||||
`SELECT id, name, url, enabled, last_fetched_at, last_line_count, last_error, created_at
|
||||
FROM sources WHERE enabled ORDER BY id`)
|
||||
rows, err := d.pool.Query(ctx, `SELECT `+sourceCols+` FROM sources WHERE enabled ORDER BY id`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -34,7 +32,7 @@ func scanSources(rows interface {
|
||||
var out []Source
|
||||
for rows.Next() {
|
||||
var s Source
|
||||
if err := rows.Scan(&s.ID, &s.Name, &s.URL, &s.Enabled,
|
||||
if err := rows.Scan(&s.ID, &s.Name, &s.URL, &s.Kind, &s.Enabled,
|
||||
&s.LastFetchedAt, &s.LastLineCount, &s.LastError, &s.CreatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -44,21 +42,21 @@ func scanSources(rows interface {
|
||||
}
|
||||
|
||||
// CreateSource inserts a source and returns its id.
|
||||
func (d *DB) CreateSource(ctx context.Context, name, url string, enabled bool) (int64, error) {
|
||||
func (d *DB) CreateSource(ctx context.Context, name, url, kind string, enabled bool) (int64, error) {
|
||||
var id int64
|
||||
err := d.pool.QueryRow(ctx,
|
||||
`INSERT INTO sources (name, url, enabled) VALUES ($1, $2, $3)
|
||||
ON CONFLICT (url) DO UPDATE SET name = EXCLUDED.name, enabled = EXCLUDED.enabled
|
||||
`INSERT INTO sources (name, url, kind, enabled) VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (url) DO UPDATE SET name = EXCLUDED.name, kind = EXCLUDED.kind, enabled = EXCLUDED.enabled
|
||||
RETURNING id`,
|
||||
name, url, enabled).Scan(&id)
|
||||
name, url, kind, enabled).Scan(&id)
|
||||
return id, err
|
||||
}
|
||||
|
||||
// UpdateSource updates name/url/enabled.
|
||||
func (d *DB) UpdateSource(ctx context.Context, id int64, name, url string, enabled bool) error {
|
||||
// UpdateSource updates name/url/kind/enabled.
|
||||
func (d *DB) UpdateSource(ctx context.Context, id int64, name, url, kind string, enabled bool) error {
|
||||
_, err := d.pool.Exec(ctx,
|
||||
`UPDATE sources SET name = $2, url = $3, enabled = $4 WHERE id = $1`,
|
||||
id, name, url, enabled)
|
||||
`UPDATE sources SET name = $2, url = $3, kind = $4, enabled = $5 WHERE id = $1`,
|
||||
id, name, url, kind, enabled)
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
@@ -16,6 +16,41 @@ type Source struct {
|
||||
Name string
|
||||
}
|
||||
|
||||
// Source kinds control how a fetched body is decoded into proxy lines.
|
||||
const (
|
||||
KindAuto = "auto" // try base64 first, fall back to plain lines
|
||||
KindPlain = "plain" // treat body as newline-separated text (no base64 decode)
|
||||
KindBase64 = "base64" // decode base64 (subscription), then split lines
|
||||
)
|
||||
|
||||
// FetchSource downloads one source and decodes it according to kind. An empty
|
||||
// kind is treated as "auto". Retries transient failures with backoff.
|
||||
func FetchSource(ctx context.Context, rawURL, kind string, log *slog.Logger) ([]string, error) {
|
||||
client := &http.Client{Timeout: 30 * time.Second}
|
||||
content, err := fetchRawWithRetry(ctx, client, rawURL, log)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return decodeByKind(content, kind), nil
|
||||
}
|
||||
|
||||
func decodeByKind(content, kind string) []string {
|
||||
switch kind {
|
||||
case KindPlain:
|
||||
return parseLines(content)
|
||||
case KindBase64:
|
||||
if dec, err := tryBase64(content); err == nil {
|
||||
content = dec
|
||||
}
|
||||
return parseLines(content)
|
||||
default: // auto
|
||||
if dec, err := tryBase64(content); err == nil {
|
||||
content = dec
|
||||
}
|
||||
return parseLines(content)
|
||||
}
|
||||
}
|
||||
|
||||
// FetchAll downloads all sources, retrying failed ones with exponential backoff.
|
||||
// A failure of one source does not abort the others — it is logged and skipped.
|
||||
func FetchAll(ctx context.Context, sources []Source, log *slog.Logger) ([]string, error) {
|
||||
@@ -89,6 +124,47 @@ func fetchOne(ctx context.Context, client *http.Client, rawURL string) ([]string
|
||||
return parseLines(content), nil
|
||||
}
|
||||
|
||||
// fetchRawWithRetry downloads a source body (undecoded) with backoff.
|
||||
func fetchRawWithRetry(ctx context.Context, client *http.Client, rawURL string, log *slog.Logger) (string, error) {
|
||||
delays := []time.Duration{0, time.Second, 2 * time.Second}
|
||||
var lastErr error
|
||||
for attempt, delay := range delays {
|
||||
if delay > 0 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return "", ctx.Err()
|
||||
case <-time.After(delay):
|
||||
}
|
||||
}
|
||||
content, err := fetchRaw(ctx, client, rawURL)
|
||||
if err == nil {
|
||||
return content, nil
|
||||
}
|
||||
if log != nil {
|
||||
log.Debug("fetch attempt failed", "url", rawURL, "attempt", attempt+1, "err", err.Error())
|
||||
}
|
||||
lastErr = err
|
||||
}
|
||||
return "", lastErr
|
||||
}
|
||||
|
||||
func fetchRaw(ctx context.Context, client *http.Client, rawURL string) (string, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 20<<20))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(body), nil
|
||||
}
|
||||
|
||||
func tryBase64(s string) (string, error) {
|
||||
s = strings.TrimSpace(s)
|
||||
if strings.Contains(s, "://") && strings.Contains(s, "\n") {
|
||||
|
||||
@@ -234,7 +234,7 @@ func (w *Worker) collect(ctx context.Context, sessionID int64) ([]candidate, int
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
lines, ferr := fetcher.FetchAll(ctx, []fetcher.Source{{URL: src.URL, Name: src.Name}}, w.log)
|
||||
lines, ferr := fetcher.FetchSource(ctx, src.URL, src.Kind, w.log)
|
||||
errStr := ""
|
||||
if ferr != nil {
|
||||
errStr = ferr.Error()
|
||||
|
||||
@@ -18,12 +18,14 @@ CREATE TABLE IF NOT EXISTS sources (
|
||||
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
||||
name text NOT NULL,
|
||||
url text NOT NULL UNIQUE,
|
||||
kind text NOT NULL DEFAULT 'auto', -- auto | plain | base64
|
||||
enabled boolean NOT NULL DEFAULT true,
|
||||
last_fetched_at timestamptz,
|
||||
last_line_count integer NOT NULL DEFAULT 0,
|
||||
last_error text NOT NULL DEFAULT '',
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
ALTER TABLE sources ADD COLUMN IF NOT EXISTS kind text NOT NULL DEFAULT 'auto';
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- proxies: persistent per-canonical-URL history. Kept even while a proxy is
|
||||
|
||||
Reference in New Issue
Block a user