safe_fetch.js validates URLs before fetch: rejects non-http(s), literal or DNS-resolved loopback / RFC1918 / link-local / CGNAT / metadata addresses; follows redirects manually with the same checks on each hop. Test fixtures gate the check with VOID_INGEST_ALLOW_PRIVATE for offline fixtures that hit 127.0.0.1. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
80 lines
2.4 KiB
JavaScript
80 lines
2.4 KiB
JavaScript
import { Router } from 'express';
|
|
import { z } from 'zod';
|
|
import crypto from 'node:crypto';
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import os from 'node:os';
|
|
import multer from 'multer';
|
|
import * as queue from '../../jobs/queue.js';
|
|
import { pool } from '../../db/pool.js';
|
|
import { validate } from '../validate.js';
|
|
import { requireWrite } from '../cap.js';
|
|
import { asyncWrap } from '../errors.js';
|
|
|
|
const captureBody = z.object({
|
|
space_id: z.string().uuid(),
|
|
url: z.string().url(),
|
|
hint: z.object({
|
|
project_id: z.string().uuid().optional(),
|
|
title: z.string().optional(),
|
|
tags: z.array(z.string()).optional()
|
|
}).optional()
|
|
});
|
|
|
|
const UPLOAD_TMP = process.env.UPLOAD_TMP || path.join(os.tmpdir(), 'void-uploads');
|
|
fs.mkdirSync(UPLOAD_TMP, { recursive: true });
|
|
const upload = multer({ dest: UPLOAD_TMP, limits: { fileSize: 100 * 1024 * 1024 } });
|
|
|
|
function key(space_id, url) {
|
|
return crypto.createHash('sha256').update(space_id + '\x00' + url).digest('hex');
|
|
}
|
|
|
|
export const router = Router();
|
|
|
|
router.post('/',
|
|
requireWrite('ref'),
|
|
validate({ body: captureBody }),
|
|
asyncWrap(async (req, res) => {
|
|
const { space_id, url } = req.body;
|
|
const idem = key(space_id, url);
|
|
const { rows: [existing] } = await pool.query(
|
|
`SELECT id FROM refs WHERE source_kind='url' AND external_id=$1 LIMIT 1`,
|
|
[idem]
|
|
);
|
|
if (existing) {
|
|
return res.status(202).json({
|
|
job_id: null, idempotency_key: idem, ref_id: existing.id
|
|
});
|
|
}
|
|
const job_id = await queue.enqueue('ingest.url', { space_id, url });
|
|
res.status(202).json({ job_id, idempotency_key: idem });
|
|
})
|
|
);
|
|
|
|
router.post('/upload',
|
|
requireWrite('ref'),
|
|
upload.single('file'),
|
|
asyncWrap(async (req, res) => {
|
|
if (!req.file) {
|
|
return res.status(400).json({ error: { code: 'validation_failed', message: 'file required' } });
|
|
}
|
|
const space_id = req.body.space_id;
|
|
if (!space_id) {
|
|
return res.status(400).json({ error: { code: 'validation_failed', message: 'space_id required' } });
|
|
}
|
|
let meta = {};
|
|
if (req.body.meta) {
|
|
try { meta = JSON.parse(req.body.meta); }
|
|
catch { /* leave empty */ }
|
|
}
|
|
const job_id = await queue.enqueue('ingest.blob', {
|
|
space_id,
|
|
tmp_path: req.file.path,
|
|
filename: req.file.originalname,
|
|
content_type: req.file.mimetype,
|
|
meta
|
|
});
|
|
res.status(202).json({ job_id });
|
|
})
|
|
);
|