feat(api): capture POST + upload + SSRF-safe URL fetch
safe_fetch.js validates URLs before fetch: rejects non-http(s), literal or DNS-resolved loopback / RFC1918 / link-local / CGNAT / metadata addresses; follows redirects manually with the same checks on each hop. Test fixtures gate the check with VOID_INGEST_ALLOW_PRIVATE for offline fixtures that hit 127.0.0.1. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -21,6 +21,7 @@ import { router as pendingChangesRouter } from './routes/pending_changes.js';
|
||||
import { router as auditRouter } from './routes/audit.js';
|
||||
import { router as searchRouter } from './routes/search.js';
|
||||
import { router as jobsRouter } from './routes/jobs.js';
|
||||
import { router as captureRouter } from './routes/capture.js';
|
||||
|
||||
export function mountApi(app) {
|
||||
const api = Router();
|
||||
@@ -49,6 +50,7 @@ export function mountApi(app) {
|
||||
api.use('/audit', auditRouter);
|
||||
api.use('/search', searchRouter);
|
||||
api.use('/jobs', jobsRouter);
|
||||
api.use('/capture', captureRouter);
|
||||
api.use('/:entity_type/:entity_id/tags', tagsByEntityRouter);
|
||||
|
||||
api.use((_req, _res, next) => next(new NotFoundError('route not found')));
|
||||
|
||||
79
lib/api/routes/capture.js
Normal file
79
lib/api/routes/capture.js
Normal file
@@ -0,0 +1,79 @@
|
||||
import { Router } from 'express';
|
||||
import { z } from 'zod';
|
||||
import crypto from 'node:crypto';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import os from 'node:os';
|
||||
import multer from 'multer';
|
||||
import * as queue from '../../jobs/queue.js';
|
||||
import { pool } from '../../db/pool.js';
|
||||
import { validate } from '../validate.js';
|
||||
import { requireWrite } from '../cap.js';
|
||||
import { asyncWrap } from '../errors.js';
|
||||
|
||||
const captureBody = z.object({
|
||||
space_id: z.string().uuid(),
|
||||
url: z.string().url(),
|
||||
hint: z.object({
|
||||
project_id: z.string().uuid().optional(),
|
||||
title: z.string().optional(),
|
||||
tags: z.array(z.string()).optional()
|
||||
}).optional()
|
||||
});
|
||||
|
||||
const UPLOAD_TMP = process.env.UPLOAD_TMP || path.join(os.tmpdir(), 'void-uploads');
|
||||
fs.mkdirSync(UPLOAD_TMP, { recursive: true });
|
||||
const upload = multer({ dest: UPLOAD_TMP, limits: { fileSize: 100 * 1024 * 1024 } });
|
||||
|
||||
function key(space_id, url) {
|
||||
return crypto.createHash('sha256').update(space_id + '\x00' + url).digest('hex');
|
||||
}
|
||||
|
||||
export const router = Router();
|
||||
|
||||
router.post('/',
|
||||
requireWrite('ref'),
|
||||
validate({ body: captureBody }),
|
||||
asyncWrap(async (req, res) => {
|
||||
const { space_id, url } = req.body;
|
||||
const idem = key(space_id, url);
|
||||
const { rows: [existing] } = await pool.query(
|
||||
`SELECT id FROM refs WHERE source_kind='url' AND external_id=$1 LIMIT 1`,
|
||||
[idem]
|
||||
);
|
||||
if (existing) {
|
||||
return res.status(202).json({
|
||||
job_id: null, idempotency_key: idem, ref_id: existing.id
|
||||
});
|
||||
}
|
||||
const job_id = await queue.enqueue('ingest.url', { space_id, url });
|
||||
res.status(202).json({ job_id, idempotency_key: idem });
|
||||
})
|
||||
);
|
||||
|
||||
router.post('/upload',
|
||||
requireWrite('ref'),
|
||||
upload.single('file'),
|
||||
asyncWrap(async (req, res) => {
|
||||
if (!req.file) {
|
||||
return res.status(400).json({ error: { code: 'validation_failed', message: 'file required' } });
|
||||
}
|
||||
const space_id = req.body.space_id;
|
||||
if (!space_id) {
|
||||
return res.status(400).json({ error: { code: 'validation_failed', message: 'space_id required' } });
|
||||
}
|
||||
let meta = {};
|
||||
if (req.body.meta) {
|
||||
try { meta = JSON.parse(req.body.meta); }
|
||||
catch { /* leave empty */ }
|
||||
}
|
||||
const job_id = await queue.enqueue('ingest.blob', {
|
||||
space_id,
|
||||
tmp_path: req.file.path,
|
||||
filename: req.file.originalname,
|
||||
content_type: req.file.mimetype,
|
||||
meta
|
||||
});
|
||||
res.status(202).json({ job_id });
|
||||
})
|
||||
);
|
||||
Reference in New Issue
Block a user