feat(search): hybrid FTS + vector with RRF + graceful Ollama fallback
Replaces FTS-only /api/search in place. RRF (k=60) fuses ts_rank and pgvector cosine distance rankings. Vector branch silently skipped when Ollama times out / errors, keeping search snappy and resilient. Messages have no embeddings in Plan 3, so they participate in the FTS branch only. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
import { describe, it, expect, beforeAll, beforeEach } from 'vitest';
|
||||
import { describe, it, expect, beforeAll, beforeEach, afterEach, vi } from 'vitest';
|
||||
import { resetDb } from '../helpers/db.js';
|
||||
import { migrateUp } from '../../lib/db/migrate.js';
|
||||
import { pool } from '../../lib/db/pool.js';
|
||||
import * as search from '../../lib/db/repos/search.js';
|
||||
import * as spacesRepo from '../../lib/db/repos/spaces.js';
|
||||
import * as pagesRepo from '../../lib/db/repos/pages.js';
|
||||
@@ -19,7 +20,11 @@ beforeEach(async () => {
|
||||
await migrateUp();
|
||||
space = await spacesRepo.create({ slug: 's-main', name: 'Main' }, owner);
|
||||
otherSpace = await spacesRepo.create({ slug: 's-other', name: 'Other' }, owner);
|
||||
// Default: pretend Ollama is down so the vector branch is skipped and
|
||||
// existing FTS-only assertions still hold deterministically.
|
||||
global.fetch = vi.fn(async () => { throw new Error('Ollama unreachable (test default)'); });
|
||||
});
|
||||
afterEach(() => { vi.restoreAllMocks(); });
|
||||
|
||||
async function seedAll(word) {
|
||||
await pagesRepo.create(
|
||||
@@ -97,4 +102,54 @@ describe('search repo', () => {
|
||||
const hits = await search.fts({ q: 'whitelight' });
|
||||
expect(hits).toEqual([]);
|
||||
});
|
||||
|
||||
it('vector branch surfaces an FTS-miss when embedding is close to the query', async () => {
|
||||
// Page text does not include "blackflame", but its hand-crafted vector
|
||||
// is close to the query vector, so the vector branch should surface it.
|
||||
const page = await pagesRepo.create(
|
||||
{ space_id: space.id, slug: 'vec-only', title: 'Unrelated', body_md: 'nothing about it' },
|
||||
owner
|
||||
);
|
||||
const v = '[' + new Array(1024).fill(0.5).join(',') + ']';
|
||||
await pool.query('UPDATE pages SET embedding=$1::vector WHERE id=$2', [v, page.id]);
|
||||
global.fetch = vi.fn(async () => new Response(
|
||||
JSON.stringify({ embedding: new Array(768).fill(0.5) }),
|
||||
{ status: 200, headers: { 'content-type': 'application/json' } }
|
||||
));
|
||||
const hits = await search.fts({ q: 'whatever' });
|
||||
expect(hits.find(h => h.id === page.id)).toBeTruthy();
|
||||
});
|
||||
|
||||
it('Ollama down → FTS-only fallback still returns FTS hits', async () => {
|
||||
await pagesRepo.create(
|
||||
{ space_id: space.id, slug: 'fb', title: 'blackflame palette', body_md: '' },
|
||||
owner
|
||||
);
|
||||
// Default mock already throws — that simulates Ollama being unreachable.
|
||||
const hits = await search.fts({ q: 'blackflame' });
|
||||
expect(hits.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('RRF fuses FTS and vector for the same row (higher rank than FTS alone)', async () => {
|
||||
const page = await pagesRepo.create(
|
||||
{ space_id: space.id, slug: 'rrf', title: 'cradle blackflame motif', body_md: 'blackflame essay' },
|
||||
owner
|
||||
);
|
||||
const v = '[' + new Array(1024).fill(0.5).join(',') + ']';
|
||||
await pool.query('UPDATE pages SET embedding=$1::vector WHERE id=$2', [v, page.id]);
|
||||
|
||||
// FTS-only run (vector branch errors)
|
||||
const ftsOnly = await search.fts({ q: 'blackflame' });
|
||||
const ftsRank = ftsOnly.find(h => h.id === page.id)?.rank;
|
||||
expect(ftsRank).toBeGreaterThan(0);
|
||||
|
||||
// FTS + vector (query embedding matches the row's vector)
|
||||
global.fetch = vi.fn(async () => new Response(
|
||||
JSON.stringify({ embedding: new Array(768).fill(0.5) }),
|
||||
{ status: 200, headers: { 'content-type': 'application/json' } }
|
||||
));
|
||||
const hybrid = await search.fts({ q: 'blackflame' });
|
||||
const hybridRank = hybrid.find(h => h.id === page.id)?.rank;
|
||||
expect(hybridRank).toBeGreaterThan(ftsRank);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user