feat(api): capture POST + upload + SSRF-safe URL fetch

safe_fetch.js validates URLs before fetch: rejects non-http(s), literal
or DNS-resolved loopback / RFC1918 / link-local / CGNAT / metadata
addresses; follows redirects manually with the same checks on each hop.
Test fixtures gate the check with VOID_INGEST_ALLOW_PRIVATE for offline
fixtures that hit 127.0.0.1.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
root
2026-06-01 03:42:54 +10:00
parent eceebd2947
commit afc20712cb
6 changed files with 278 additions and 1 deletions

View File

@@ -1,5 +1,6 @@
import crypto from 'node:crypto';
import { extract } from '../../ingest/readability.js';
import { safeFetch } from '../../ingest/safe_fetch.js';
import * as refs from '../../db/repos/refs.js';
import { pool } from '../../db/pool.js';
@@ -19,7 +20,7 @@ export async function handler(job) {
);
if (existing) return { ref_id: existing.id, idempotent: true };
const res = await fetch(url, {
const res = await safeFetch(url, {
headers: { 'User-Agent': 'void-ingest/2.0' },
signal: AbortSignal.timeout(15_000)
});