proxy-manager.js

import fs from "fs";
import path from "path";
import net from "net";
import tls from "tls";
import crypto from "crypto";
import { chromium } from "playwright";

const SOCKS_PROTOCOLS = new Set(["socks4", "socks4a", "socks5", "socks5h"]);
const SUPPORTED_PROTOCOLS = new Set(["http", "https", ...SOCKS_PROTOCOLS]);

function isSocksProxy(proxy) {
  return SOCKS_PROTOCOLS.has(proxy?.protocol);
}
const DEFAULT_SETTINGS = {
  mode: "auto",
  poolSize: 10,
  minSuccessRate: 0.6,
  maxLatencyMs: 3500,
  collectIntervalMs: 15 * 60 * 1000,
  validateIntervalMs: 10 * 60 * 1000,
  connectTimeoutMs: 8000,
  requestTimeoutMs: 10000,
  browserTimeoutMs: 20000,
  validateConcurrency: 2,
  browserConcurrency: 1,
  maxValidationsPerRun: 20,
  allowInsecureTls: true
};

const DEFAULT_SOURCES = [];

const BLOCKED_KEYWORDS = [
  "unusual traffic",
  "detected unusual traffic",
  "verify you are a human",
  "access denied",
  "captcha",
  "robot check",
  "automated queries",
  "cloudflare",
  "attention required"
];

const TLS_ERROR_CODES = new Set([
  "UNABLE_TO_GET_ISSUER_CERT_LOCALLY",
  "SELF_SIGNED_CERT_IN_CHAIN",
  "DEPTH_ZERO_SELF_SIGNED_CERT",
  "UNABLE_TO_VERIFY_LEAF_SIGNATURE",
  "CERT_HAS_EXPIRED",
  "ERR_TLS_CERT_ALTNAME_INVALID"
]);

function isTlsCertError(err) {
  if (!err) return false;
  const code = String(err.code || "").toUpperCase();
  if (TLS_ERROR_CODES.has(code)) return true;
  const msg = String(err.message || "").toLowerCase();
  return msg.includes("tls") || msg.includes("certificate");
}

const DEFAULT_DB = {
  version: 1,
  settings: { ...DEFAULT_SETTINGS },
  proxies: {},
  rotation: { index: 0 }
};

class Semaphore {
  constructor(limit) {
    this.limit = Math.max(1, limit || 1);
    this.active = 0;
    this.queue = [];
  }
  async acquire() {
    if (this.active < this.limit) {
      this.active += 1;
      return;
    }
    await new Promise((resolve) => this.queue.push(resolve));
    this.active += 1;
  }
  release() {
    this.active = Math.max(0, this.active - 1);
    const next = this.queue.shift();
    if (next) next();
  }
}

function nowIso() {
  return new Date().toISOString();
}

function readJsonSafe(filePath) {
  if (!fs.existsSync(filePath)) return null;
  try {
    const raw = fs.readFileSync(filePath, "utf8");
    return JSON.parse(raw);
  } catch (err) {
    return null;
  }
}

function writeJsonAtomic(filePath, payload) {
  const dir = path.dirname(filePath);
  if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
  const tmpPath = `${filePath}.tmp`;
  fs.writeFileSync(tmpPath, JSON.stringify(payload, null, 2), "utf8");
  fs.renameSync(tmpPath, filePath);
}

function mergeSettings(base, override) {
  return { ...base, ...(override || {}) };
}

function scoreProxy(proxy) {
  const successes = proxy.successes || 0;
  const failures = proxy.failures || 0;
  const total = successes + failures;
  const successRate = total ? successes / total : 0;
  const latency = Number.isFinite(proxy.latencyMs) ? proxy.latencyMs : 10000;
  const latencyPenalty = Math.min(latency, 10000) / 1000;
  const stabilityBonus = proxy.healthy ? 0.2 : 0;
  return successRate * 1000 - latencyPenalty * 12 + stabilityBonus * 50;
}

function buildAuthHeader(username, password) {
  if (!username && !password) return null;
  const token = Buffer.from(`${username || ""}:${password || ""}`).toString("base64");
  return `Basic ${token}`;
}

function normalizeProxyFromUrl(url) {
  const protocol = url.protocol.replace(":", "").toLowerCase();
  if (!SUPPORTED_PROTOCOLS.has(protocol)) return null;
  const host = url.hostname;
  const defaultPort = protocol === "https" ? 443 : protocol.startsWith("socks") ? 1080 : 80;
  const port = Number(url.port || defaultPort);
  if (!host || !Number.isFinite(port) || port <= 0 || port > 65535) return null;
  const username = url.username ? decodeURIComponent(url.username) : null;
  const password = url.password ? decodeURIComponent(url.password) : null;
  const authHash =
    username || password
      ? crypto.createHash("sha1").update(`${username || ""}:${password || ""}`).digest("hex").slice(0, 8)
      : "";
  const id = `${protocol}://${host}:${port}${authHash ? `#${authHash}` : ""}`;
  return {
    id,
    protocol,
    host,
    port,
    username,
    password,
    display: `${protocol}://${host}:${port}`,
    hasAuth: !!(username || password)
  };
}

function parseProxyString(raw) {
  if (!raw) return null;
  let line = String(raw).trim();
  if (!line) return null;
  line = line.replace(/^"+|"+$/g, "");
  line = line.replace(/,$/, "");
  if (!line) return null;
  if (!line.includes("://") && !line.includes("@")) {
    const parts = line.split(":");
    if (parts.length === 4) {
      const [a, b, c, d] = parts;
      const portA = Number(b);
      const portD = Number(d);
      if (Number.isFinite(portA) && portA > 0 && portA < 65536) {
        try {
          const user = encodeURIComponent(c || "");
          const pass = encodeURIComponent(d || "");
          const url = new URL(`http://${user}:${pass}@${a}:${portA}`);
          return normalizeProxyFromUrl(url);
        } catch (err) {
          return null;
        }
      }
      if (Number.isFinite(portD) && portD > 0 && portD < 65536) {
        try {
          const user = encodeURIComponent(a || "");
          const pass = encodeURIComponent(b || "");
          const url = new URL(`http://${user}:${pass}@${c}:${portD}`);
          return normalizeProxyFromUrl(url);
        } catch (err) {
          return null;
        }
      }
    }
  }
  const hasProtocol = /^[a-zA-Z][a-zA-Z0-9+.-]*:\/\//.test(line);
  const candidate = hasProtocol ? line : `http://${line}`;
  let url;
  try {
    url = new URL(candidate);
  } catch (err) {
    return null;
  }
  return normalizeProxyFromUrl(url);
}

function extractCandidatesFromText(text) {
  const results = new Set();
  const lines = String(text || "").split(/\r?\n/);
  for (const line of lines) {
    const trimmed = line.trim();
    if (!trimmed || trimmed.startsWith("#")) continue;
    if (trimmed.includes("://")) {
      results.add(trimmed);
      continue;
    }
    if (trimmed.split(":").length === 4) {
      results.add(trimmed);
      continue;
    }
    const matches = trimmed.match(/[a-zA-Z0-9.-]+:\d{2,5}/g);
    if (matches) {
      for (const match of matches) results.add(match);
    }
  }
  return Array.from(results);
}

function formatProxyCandidate({ protocol, host, port, username, password }) {
  if (!host || !port) return null;
  const safeProtocol = protocol || "http";
  if (username || password) {
    const user = encodeURIComponent(username || "");
    const pass = encodeURIComponent(password || "");
    return `${safeProtocol}://${user}:${pass}@${host}:${port}`;
  }
  return `${safeProtocol}://${host}:${port}`;
}

function extractCandidatesFromJson(json) {
  const results = [];
  if (!json) return results;
  if (Array.isArray(json)) {
    for (const item of json) {
      if (typeof item === "string") {
        results.push(item);
      } else if (item && typeof item === "object") {
        const host = item.proxy_address || item.host || item.ip || item.address;
        const port = item.port || item.proxy_port;
        const protocol = item.protocol || (Array.isArray(item.protocols) && item.protocols.includes("https") ? "https" : "http");
        const candidate = formatProxyCandidate({
          protocol,
          host,
          port,
          username: item.username,
          password: item.password
        });
        if (candidate) results.push(candidate);
      }
    }
    return results;
  }
  if (Array.isArray(json.results)) {
    for (const row of json.results) {
      const host = row?.proxy_address || row?.host || row?.ip;
      const port = row?.port || row?.proxy_port;
      const protocol =
        row?.protocol || (Array.isArray(row?.protocols) && row.protocols.includes("https") ? "https" : "http");
      const candidate = formatProxyCandidate({
        protocol,
        host,
        port,
        username: row?.username,
        password: row?.password
      });
      if (candidate) results.push(candidate);
    }
  }
  if (Array.isArray(json.data)) {
    for (const row of json.data) {
      const host = row?.ip || row?.host || row?.proxy_address;
      const port = row?.port;
      const protocol = Array.isArray(row?.protocols) && row.protocols.includes("https") ? "https" : "http";
      const candidate = formatProxyCandidate({ protocol, host, port });
      if (candidate) results.push(candidate);
    }
  }
  return results;
}

async function fetchWithTimeout(url, timeoutMs, headers) {
  const controller = new AbortController();
  const timer = setTimeout(() => controller.abort(), timeoutMs);
  try {
    const res = await fetch(url, { signal: controller.signal, headers });
    return res;
  } finally {
    clearTimeout(timer);
  }
}

async function fetchText(url, timeoutMs, headers) {
  const res = await fetchWithTimeout(url, timeoutMs, headers);
  if (!res.ok) throw new Error(`fetch_failed:${res.status}`);
  return await res.text();
}

async function fetchJson(url, timeoutMs, headers) {
  const res = await fetchWithTimeout(url, timeoutMs, headers);
  if (!res.ok) throw new Error(`fetch_failed:${res.status}`);
  return await res.json();
}

function parseHtmlProxyTable(html) {
  const proxies = [];
  const regex = /<td>(\d{1,3}(?:\.\d{1,3}){3})<\/td>\s*<td>(\d{2,5})<\/td>/g;
  let match;
  while ((match = regex.exec(html))) {
    proxies.push(`${match[1]}:${match[2]}`);
  }
  return proxies;
}

function parseGeoNode(json) {
  return extractCandidatesFromJson(json);
}

function sanitizeProxyForAdmin(proxy) {
  const successes = proxy.successes || 0;
  const failures = proxy.failures || 0;
  const total = successes + failures;
  const successRate = total ? successes / total : 0;
  return {
    id: proxy.id,
    display: proxy.display,
    protocol: proxy.protocol,
    host: proxy.host,
    port: proxy.port,
    enabled: proxy.enabled !== false,
    isManual: !!proxy.isManual,
    healthy: !!proxy.healthy,
    latencyMs: proxy.latencyMs || null,
    lastCheckedAt: proxy.lastCheckedAt || null,
    lastSuccessAt: proxy.lastSuccessAt || null,
    lastFailureAt: proxy.lastFailureAt || null,
    lastError: proxy.lastError || null,
    blocked: !!proxy.blocked,
    successRate,
    hasAuth: !!proxy.hasAuth,
    insecureTls: !!proxy.insecureTls
  };
}

async function connectSocket(proxy, timeoutMs) {
  const { host, port, protocol } = proxy;
  return new Promise((resolve, reject) => {
    const socket =
      protocol === "https"
        ? tls.connect({ host, port, servername: host, timeout: timeoutMs })
        : net.connect({ host, port });
    let settled = false;
    const onError = (err) => {
      if (settled) return;
      settled = true;
      socket.destroy();
      reject(err);
    };
    const onTimeout = () => {
      if (settled) return;
      settled = true;
      socket.destroy(new Error("timeout"));
      const err = new Error("timeout");
      err.code = "ETIMEDOUT";
      reject(err);
    };
    socket.setTimeout(timeoutMs, onTimeout);
    socket.once("error", onError);
    const onConnect = () => {
      if (settled) return;
      settled = true;
      socket.setTimeout(0);
      resolve(socket);
    };
    if (protocol === "https") {
      socket.once("secureConnect", onConnect);
    } else {
      socket.once("connect", onConnect);
    }
  });
}

function readUntil(socket, matcher, timeoutMs, maxBytes = 16384) {
  return new Promise((resolve, reject) => {
    let buffer = Buffer.alloc(0);
    let timer = null;
    const cleanup = () => {
      if (timer) clearTimeout(timer);
      socket.removeListener("data", onData);
      socket.removeListener("error", onError);
      socket.removeListener("end", onEnd);
    };
    const onError = (err) => {
      cleanup();
      reject(err);
    };
    const onEnd = () => {
      cleanup();
      resolve(buffer);
    };
    const onData = (chunk) => {
      buffer = Buffer.concat([buffer, chunk]);
      if (buffer.length >= maxBytes || buffer.includes(matcher)) {
        cleanup();
        resolve(buffer);
      }
    };
    timer = setTimeout(() => {
      cleanup();
      const err = new Error("timeout");
      err.code = "ETIMEDOUT";
      reject(err);
    }, timeoutMs);
    socket.on("data", onData);
    socket.once("error", onError);
    socket.once("end", onEnd);
  });
}

async function testHttpsViaProxy(proxy, opts) {
  const authHeader = buildAuthHeader(proxy.username, proxy.password);
  const attempt = async (rejectUnauthorized) => {
    const start = Date.now();
    let socket;
    try {
      socket = await connectSocket(proxy, opts.connectTimeoutMs);
      const connectHeaders = [
        "CONNECT example.com:443 HTTP/1.1",
        "Host: example.com:443",
        "Proxy-Connection: keep-alive"
      ];
      if (authHeader) connectHeaders.push(`Proxy-Authorization: ${authHeader}`);
      socket.write(`${connectHeaders.join("\r\n")}\r\n\r\n`);
      const connectData = await readUntil(socket, "\r\n\r\n", opts.requestTimeoutMs);
      const connectText = connectData.toString("utf8");
      const statusLine = connectText.split("\r\n")[0] || "";
      const match = statusLine.match(/HTTP\/\d\.\d\s+(\d+)/i);
      const statusCode = match ? Number(match[1]) : 0;
      if (statusCode < 200 || statusCode >= 300) {
        return { ok: false, errorType: "proxy_connect_failed", statusCode };
      }
      const tlsSocket = tls.connect({ socket, servername: "example.com", rejectUnauthorized });
      await new Promise((resolve, reject) => {
        tlsSocket.once("secureConnect", resolve);
        tlsSocket.once("error", reject);
        tlsSocket.setTimeout(opts.requestTimeoutMs, () => {
          tlsSocket.destroy(new Error("timeout"));
          const err = new Error("timeout");
          err.code = "ETIMEDOUT";
          reject(err);
        });
      });
      tlsSocket.write(
        "GET / HTTP/1.1\r\nHost: example.com\r\nConnection: close\r\nUser-Agent: ProxyValidator/1.0\r\n\r\n"
      );
      const data = await readUntil(tlsSocket, "\r\n\r\n", opts.requestTimeoutMs);
      const text = data.toString("utf8");
      const line = text.split("\r\n")[0] || "";
      const statusMatch = line.match(/HTTP\/\d\.\d\s+(\d+)/i);
      const status = statusMatch ? Number(statusMatch[1]) : 0;
      const latencyMs = Date.now() - start;
      if (status < 200 || status >= 500) {
        return { ok: false, errorType: "https_status", statusCode: status, latencyMs };
      }
      return { ok: true, latencyMs, statusCode: status };
    } catch (err) {
      const code = err?.code === "ETIMEDOUT" ? "timeout" : "connect_failed";
      if (isTlsCertError(err)) {
        return { ok: false, errorType: "tls_failed" };
      }
      return { ok: false, errorType: code };
    } finally {
      if (socket && !socket.destroyed) socket.destroy();
    }
  };

  const primary = await attempt(true);
  if (primary.ok) return primary;
  if (primary.errorType === "tls_failed" && opts.allowInsecureTls) {
    const insecure = await attempt(false);
    if (insecure.ok) return { ...insecure, insecureTls: true };
    return insecure;
  }
  return primary;
}

async function testProxyInBrowser(proxy, opts, launchArgs) {
  const start = Date.now();
  let browser;
  try {
    browser = await chromium.launch({
      headless: true,
      args: Array.isArray(launchArgs) ? launchArgs : []
    });
    const context = await browser.newContext({
      ignoreHTTPSErrors: !!(proxy?.insecureTls || opts.allowInsecureTls),
      proxy: {
        server: `${proxy.protocol}://${proxy.host}:${proxy.port}`,
        username: proxy.username || undefined,
        password: proxy.password || undefined
      }
    });
    const page = await context.newPage();
    page.setDefaultTimeout(opts.browserTimeoutMs);
    const response = await page.goto("https://www.google.com/", { waitUntil: "domcontentloaded" });
    const status = response?.status() || 0;
    const content = await page.content();
    const title = await page.title();
    const url = page.url();
    const latencyMs = Date.now() - start;
    const combined = `${title}\n${url}\n${content}`.toLowerCase();
    const blocked = BLOCKED_KEYWORDS.some((word) => combined.includes(word));
    await context.close();
    await browser.close();
    if (blocked || status >= 400) {
      return { ok: false, blocked: true, latencyMs };
    }
    return { ok: true, blocked: false, latencyMs };
  } catch (err) {
    if (browser) {
      try {
        await browser.close();
      } catch (closeErr) {}
    }
    return { ok: false, blocked: false, errorType: "browser_failed" };
  }
}

async function runWithConcurrency(items, limit, fn) {
  if (!items.length) return [];
  const results = [];
  let index = 0;
  const workers = Array.from({ length: Math.max(1, limit) }, async () => {
    while (index < items.length) {
      const current = items[index];
      index += 1;
      try {
        const result = await fn(current);
        results.push(result);
      } catch (err) {}
    }
  });
  await Promise.all(workers);
  return results;
}

export function createProxyManager(options = {}) {
  return new ProxyManager(options);
}

class ProxyManager {
  constructor(options = {}) {
    this.dbPath = options.dbPath || path.join(process.cwd(), "proxy-db.json");
    this.launchArgs = options.launchArgs || [];
    this.logger = options.logger || console;
    this.sources = Array.isArray(options.sources) && options.sources.length ? options.sources : DEFAULT_SOURCES;
    this.store = null;
    this.saveTimer = null;
    this.collectTimer = null;
    this.validateTimer = null;
    this.collecting = false;
    this.validating = false;
    this.settingsOverride = options.settings || {};
    this.onPersist = options.onPersist || null;
    this.externalPersistEnabled = !!options.externalPersistEnabled;
  }

  loadStore() {
    const loaded = readJsonSafe(this.dbPath);
    if (!loaded || !loaded.settings || !loaded.proxies) {
      this.store = {
        ...DEFAULT_DB,
        settings: mergeSettings(DEFAULT_SETTINGS, this.settingsOverride)
      };
      this.persist();
      return;
    }
    this.store = {
      ...DEFAULT_DB,
      ...loaded,
      settings: mergeSettings(DEFAULT_SETTINGS, loaded.settings)
    };
    if (this.settingsOverride && Object.keys(this.settingsOverride).length) {
      this.store.settings = mergeSettings(this.store.settings, this.settingsOverride);
    }
    if (!this.store.rotation) this.store.rotation = { index: 0 };
  }

  persist() {
    writeJsonAtomic(this.dbPath, this.store);
    if (!this.onPersist || !this.externalPersistEnabled) return;
    try {
      const maybePromise = this.onPersist(this.store);
      if (maybePromise?.catch) {
        maybePromise.catch((err) => this.logger.warn("Proxy external persist failed", err?.message || err));
      }
    } catch (err) {
      this.logger.warn("Proxy external persist failed", err?.message || err);
    }
  }

  setExternalPersistEnabled(enabled) {
    this.externalPersistEnabled = !!enabled;
  }

  getStoreSnapshot() {
    if (!this.store) this.loadStore();
    return this.store;
  }

  setStoreFromExternal(store) {
    if (!store || !store.settings || !store.proxies) return false;
    this.store = {
      ...DEFAULT_DB,
      ...store,
      settings: mergeSettings(DEFAULT_SETTINGS, store.settings)
    };
    if (this.settingsOverride && Object.keys(this.settingsOverride).length) {
      this.store.settings = mergeSettings(this.store.settings, this.settingsOverride);
    }
    if (!this.store.rotation) this.store.rotation = { index: 0 };
    this.persist();
    return true;
  }

  queuePersist() {
    if (this.saveTimer) return;
    this.saveTimer = setTimeout(() => {
      this.saveTimer = null;
      this.persist();
    }, 200);
  }

  start() {
    this.loadStore();
    if (this.collectTimer || this.validateTimer) return;
    const { collectIntervalMs, validateIntervalMs } = this.store.settings;
    this.collectTimer = setInterval(() => {
      this.collectOnce().catch((err) => {
        this.logger.warn("Proxy collect failed", err?.message || err);
      });
    }, collectIntervalMs);
    this.validateTimer = setInterval(() => {
      this.validateOnce().catch((err) => {
        this.logger.warn("Proxy validate failed", err?.message || err);
      });
    }, validateIntervalMs);
    this.collectOnce().catch(() => {});
    this.validateOnce().catch(() => {});
  }

  stop() {
    if (this.collectTimer) clearInterval(this.collectTimer);
    if (this.validateTimer) clearInterval(this.validateTimer);
    this.collectTimer = null;
    this.validateTimer = null;
  }

  setMode(mode) {
    if (!this.store) this.loadStore();
    const nextMode = mode === "manual" ? "manual" : "auto";
    this.store.settings.mode = nextMode;
    this.queuePersist();
    return this.store.settings;
  }

  listProxies() {
    if (!this.store) this.loadStore();
    return Object.values(this.store.proxies || {});
  }

  getAdminSnapshot() {
    if (!this.store) this.loadStore();
    const proxies = this.listProxies()
      .map((proxy) => sanitizeProxyForAdmin(proxy))
      .sort((a, b) => (b.lastCheckedAt || "").localeCompare(a.lastCheckedAt || ""));
    const pool = this.getPool();
    const total = proxies.length;
    const manualCount = proxies.filter((p) => p.isManual).length;
    const enabledCount = proxies.filter((p) => p.enabled).length;
    const healthyCount = proxies.filter((p) => p.healthy).length;
    return {
      settings: this.store.settings,
      stats: {
        total,
        enabled: enabledCount,
        healthy: healthyCount,
        manual: manualCount,
        auto: total - manualCount,
        poolSize: pool.length
      },
      proxies
    };
  }

  addManualProxies(rawList) {
    if (!this.store) this.loadStore();
    const entries = Array.isArray(rawList) ? rawList : extractCandidatesFromText(String(rawList || ""));
    let added = 0;
    if (this.store.settings.mode !== "manual") {
      this.store.settings.mode = "manual";
    }
    for (const entry of entries) {
      const parsed = parseProxyString(entry);
      if (!parsed) continue;
      const existing = this.store.proxies[parsed.id];
      const now = nowIso();
      if (!existing) {
        this.store.proxies[parsed.id] = {
          ...parsed,
          sources: ["manual"],
          isManual: true,
          enabled: true,
          healthy: false,
          successes: 0,
          failures: 0,
          firstSeenAt: now,
          lastSeenAt: now
        };
        added += 1;
      } else {
        existing.isManual = true;
        existing.enabled = true;
        existing.lastSeenAt = now;
        if (!existing.sources?.includes("manual")) {
          existing.sources = [...(existing.sources || []), "manual"];
        }
        added += 1;
      }
    }
    this.queuePersist();
    return added;
  }

  toggleProxy(id, enabled) {
    if (!this.store) this.loadStore();
    const proxy = this.store.proxies[id];
    if (!proxy) return false;
    proxy.enabled = !!enabled;
    this.queuePersist();
    return true;
  }

  removeProxy(id) {
    if (!this.store) this.loadStore();
    if (!this.store.proxies[id]) return false;
    delete this.store.proxies[id];
    this.queuePersist();
    return true;
  }

  clearProxies() {
    if (!this.store) this.loadStore();
    const count = Object.keys(this.store.proxies || {}).length;
    this.store.proxies = {};
    this.store.rotation = { index: 0 };
    this.queuePersist();
    return count;
  }

  async collectOnce() {
    if (!this.store) this.loadStore();
    if (this.collecting) return { skipped: true };
    if (this.store.settings.mode === "manual") {
      return { skipped: true, reason: "manual_mode" };
    }
    if (!this.sources || this.sources.length === 0) {
      return { skipped: true, reason: "no_sources_configured" };
    }
    this.collecting = true;
    const collected = new Set();
    const timeoutMs = Math.max(2000, this.store.settings.requestTimeoutMs);
    try {
      for (const source of this.sources) {
        try {
          if (source.type === "text") {
            const text = await fetchText(source.url, timeoutMs, source.headers);
            for (const candidate of extractCandidatesFromText(text)) collected.add(candidate);
          } else if (source.type === "json") {
            const json = await fetchJson(source.url, timeoutMs, source.headers);
            const proxies = source.parser ? source.parser(json) : parseGeoNode(json);
            for (const proxy of proxies) collected.add(proxy);
          } else if (source.type === "html") {
            const html = await fetchText(source.url, timeoutMs, source.headers);
            const proxies = parseHtmlProxyTable(html);
            for (const proxy of proxies) collected.add(proxy);
          } else if (source.type === "webshare") {
            const headers = { ...(source.headers || {}) };
            if (source.apiKey) headers.Authorization = `Token ${source.apiKey}`;
            const params = new URLSearchParams();
            if (source.mode) params.set("mode", source.mode);
            if (source.pageSize) params.set("page_size", String(source.pageSize));
            const url = params.size ? `${source.url}?${params.toString()}` : source.url;
            const json = await fetchJson(url, timeoutMs, headers);
            const proxies = extractCandidatesFromJson(json);
            for (const proxy of proxies) collected.add(proxy);
          } else if (source.type === "url") {
            const res = await fetchWithTimeout(source.url, timeoutMs, source.headers);
            if (!res.ok) throw new Error(`fetch_failed:${res.status}`);
            const contentType = res.headers.get("content-type") || "";
            if (contentType.includes("application/json")) {
              const json = await res.json();
              const proxies = extractCandidatesFromJson(json);
              for (const proxy of proxies) collected.add(proxy);
            } else {
              const text = await res.text();
              for (const candidate of extractCandidatesFromText(text)) collected.add(candidate);
            }
          }
        } catch (err) {
          this.logger.warn(`Proxy source failed: ${source.id}`, err?.message || err);
        }
      }
      let added = 0;
      const now = nowIso();
      for (const item of collected) {
        const parsed = parseProxyString(item);
        if (!parsed) continue;
        const existing = this.store.proxies[parsed.id];
        if (!existing) {
          this.store.proxies[parsed.id] = {
            ...parsed,
            sources: ["auto"],
            isManual: false,
            enabled: true,
            healthy: false,
            successes: 0,
            failures: 0,
            firstSeenAt: now,
            lastSeenAt: now
          };
          added += 1;
        } else {
          existing.lastSeenAt = now;
          if (!existing.sources?.includes("auto")) {
            existing.sources = [...(existing.sources || []), "auto"];
          }
        }
      }
      this.store.settings.lastCollectAt = now;
      this.queuePersist();
      return { added };
    } finally {
      this.collecting = false;
    }
  }

  async validateOnce() {
    if (!this.store) this.loadStore();
    if (this.validating) return { skipped: true };
    this.validating = true;
    try {
      const settings = this.store.settings;
      const now = nowIso();
      let candidates = Object.values(this.store.proxies || {}).filter((proxy) => proxy.enabled !== false);
      if (settings.mode === "manual") {
        candidates = candidates.filter((proxy) => proxy.isManual);
      }
      candidates.sort((a, b) => {
        const aTime = a.lastCheckedAt ? new Date(a.lastCheckedAt).getTime() : 0;
        const bTime = b.lastCheckedAt ? new Date(b.lastCheckedAt).getTime() : 0;
        return aTime - bTime;
      });
      const batch = candidates.slice(0, settings.maxValidationsPerRun);
      const browserSemaphore = new Semaphore(settings.browserConcurrency || settings.validateConcurrency);
      const results = await runWithConcurrency(batch, settings.validateConcurrency, async (proxy) => {
        const socksOnly = isSocksProxy(proxy);
        const basic = socksOnly ? { ok: true, latencyMs: null } : await testHttpsViaProxy(proxy, settings);
        if (!basic.ok) return { proxy, result: { ...basic } };
        await browserSemaphore.acquire();
        try {
          const browser = await testProxyInBrowser(proxy, settings, this.launchArgs);
          return { proxy, result: { ...basic, browser } };
        } finally {
          browserSemaphore.release();
        }
      });

      for (const item of results) {
        const proxy = item.proxy;
        const { result } = item;
        proxy.lastCheckedAt = now;
        if (!result.ok) {
          proxy.failures = (proxy.failures || 0) + 1;
          proxy.healthy = false;
          proxy.lastFailureAt = now;
          proxy.lastError = result.errorType || "failed";
          proxy.blocked = result.blocked || false;
          continue;
        }
        const browser = result.browser;
        if (browser && (!browser.ok || browser.blocked)) {
          proxy.failures = (proxy.failures || 0) + 1;
          proxy.healthy = false;
          proxy.lastFailureAt = now;
          proxy.lastError = browser.blocked ? "blocked" : "browser_failed";
          proxy.blocked = browser.blocked || false;
          continue;
        }
        proxy.successes = (proxy.successes || 0) + 1;
        proxy.healthy = true;
        proxy.blocked = false;
        proxy.lastSuccessAt = now;
        proxy.lastError = null;
        const latency = result.latencyMs ?? result.browser?.latencyMs ?? proxy.latencyMs;
        if (latency) proxy.latencyMs = latency;
        proxy.insecureTls = !!result.insecureTls;
      }
      this.store.settings.lastValidateAt = now;
      this.queuePersist();
      return { validated: results.length };
    } finally {
      this.validating = false;
    }
  }

  async testProxy(id) {
    if (!this.store) this.loadStore();
    const proxy = this.store.proxies[id];
    if (!proxy) return null;
    const settings = this.store.settings;
    const now = nowIso();
    let result;
    if (isSocksProxy(proxy)) {
      const browser = await testProxyInBrowser(proxy, settings, this.launchArgs);
      result = {
        ok: !!browser.ok,
        blocked: !!browser.blocked,
        latencyMs: browser.latencyMs || null,
        errorType: browser.errorType || (browser.ok ? null : "browser_failed")
      };
    } else {
      result = await testHttpsViaProxy(proxy, settings);
    }

    proxy.lastCheckedAt = now;
    if (!result.ok) {
      proxy.failures = (proxy.failures || 0) + 1;
      proxy.healthy = false;
      proxy.lastFailureAt = now;
      proxy.lastError = result.errorType || "failed";
      proxy.blocked = !!result.blocked;
    } else {
      proxy.successes = (proxy.successes || 0) + 1;
      proxy.healthy = true;
      proxy.blocked = false;
      proxy.lastSuccessAt = now;
      proxy.lastError = null;
      if (result.latencyMs) proxy.latencyMs = result.latencyMs;
      proxy.insecureTls = !!result.insecureTls;
    }

    this.queuePersist();
    return result;
  }

  getPool() {
    if (!this.store) this.loadStore();
    const { mode, poolSize, minSuccessRate, maxLatencyMs } = this.store.settings;
    let pool = Object.values(this.store.proxies || {}).filter((proxy) => proxy.enabled !== false);
    if (mode === "manual") {
      pool = pool.filter((proxy) => proxy.isManual);
      return pool;
    }
    pool = pool.filter((proxy) => proxy.healthy);
    pool = pool.filter((proxy) => {
      const successes = proxy.successes || 0;
      const failures = proxy.failures || 0;
      const total = successes + failures;
      const successRate = total ? successes / total : 0;
      if (total === 0) return false;
      if (successRate < minSuccessRate) return false;
      if (Number.isFinite(proxy.latencyMs) && proxy.latencyMs > maxLatencyMs) return false;
      return true;
    });
    pool.sort((a, b) => scoreProxy(b) - scoreProxy(a));
    return pool.slice(0, poolSize);
  }

  getNextProxy() {
    if (!this.store) this.loadStore();
    const pool = this.getPool();
    if (!pool.length) return null;
    const rotation = this.store.rotation || { index: 0 };
    const index = rotation.index % pool.length;
    const proxy = pool[index];
    rotation.index = (index + 1) % pool.length;
    this.store.rotation = rotation;
    this.queuePersist();
    return proxy;
  }
}