Void-Homelab/lib/db/repos/search.js

import { pool } from '../pool.js';
import { embedText, padTo } from '../../ai/ollama.js';
import { log } from '../../log.js';

// Hybrid FTS + vector search with reciprocal-rank fusion.
//
// - FTS branch: tsvector @@ plainto_tsquery on pages / refs / source_docs /
//   messages, unioned with a `kind` discriminator. Each expression matches
//   the corresponding GIN index in migrations 002 / 003 / 004.
// - Vector branch: pgvector cosine distance against the entity's embedding
//   column. The query is embedded via Ollama (nomic-embed-text, padded
//   1024-d). Messages are not embedded in Plan 3 — skipped in the vector
//   branch.
// - Graceful fallback: if Ollama times out or errors, the vector branch is
//   dropped silently and search continues as FTS-only.
// - RRF fuses the two rank lists with k=60 (Cormack et al. canonical).

const PAGES_TSV = `to_tsvector('english', p.title || ' ' || coalesce(p.body_md,''))`;
const REFS_TSV  = `to_tsvector('english', coalesce(r.title,'') || ' ' || coalesce(r.summary,'') || ' ' || coalesce(r.body_text,''))`;
const SD_TSV    = `to_tsvector('english', sd.name || ' ' || coalesce(sd.body_text,''))`;
const MSG_TSV   = `to_tsvector('english', m.body)`;

// ----- FTS branch (returns flat row list ordered by ts_rank) ----------------

function ftsBranches({ kinds, spaceFilterPresent }) {
  const want = k => !kinds || kinds.includes(k);
  const out = [];
  if (want('page')) out.push(`
    SELECT 'page'::text AS kind, p.id::text, p.space_id, p.title AS title_or_snippet,
           ts_rank(${PAGES_TSV}, q.tsq) AS rank
      FROM pages p, q
      WHERE ${PAGES_TSV} @@ q.tsq
        AND ($2::uuid IS NULL OR p.space_id = $2)`);
  if (want('ref')) out.push(`
    SELECT 'ref'::text, r.id::text, r.space_id,
           coalesce(r.title, r.source_url, '(untitled)'),
           ts_rank(${REFS_TSV}, q.tsq)
      FROM refs r, q
      WHERE ${REFS_TSV} @@ q.tsq
        AND ($2::uuid IS NULL OR r.space_id = $2)`);
  if (want('source_doc')) out.push(`
    SELECT 'source_doc'::text, sd.id::text, res.space_id, sd.name,
           ts_rank(${SD_TSV}, q.tsq)
      FROM source_docs sd JOIN resources res ON res.id = sd.resource_id, q
      WHERE ${SD_TSV} @@ q.tsq
        AND ($2::uuid IS NULL OR res.space_id = $2)`);
  if (want('message') && !spaceFilterPresent) out.push(`
    SELECT 'message'::text, m.id::text, NULL::uuid, substring(m.body, 1, 200),
           ts_rank(${MSG_TSV}, q.tsq)
      FROM messages m, q
      WHERE ${MSG_TSV} @@ q.tsq`);
  return out;
}

async function ftsRows({ q, space_id, kinds, perBranchLimit }) {
  const branches = ftsBranches({ kinds, spaceFilterPresent: space_id != null });
  if (!branches.length) return [];
  const sql = `WITH q AS (SELECT plainto_tsquery('english', $1) AS tsq)
    SELECT * FROM (${branches.join('\n UNION ALL ')}) u
    ORDER BY rank DESC LIMIT $3`;
  const { rows } = await pool.query(sql, [q, space_id, perBranchLimit]);
  return rows;
}

// ----- Vector branch (runs one ANN query per kind, concats results) --------

async function vectorRows({ qvec, space_id, kinds, perBranchLimit }) {
  const want = k => !kinds || kinds.includes(k);
  const literal = '[' + qvec.join(',') + ']';
  const all = [];

  if (want('page')) {
    const { rows } = await pool.query(`
      SELECT 'page'::text AS kind, p.id::text, p.space_id, p.title AS title_or_snippet,
             (p.embedding <=> $1::vector) AS dist
        FROM pages p
        WHERE p.embedding IS NOT NULL
          AND ($2::uuid IS NULL OR p.space_id = $2)
        ORDER BY p.embedding <=> $1::vector LIMIT $3
    `, [literal, space_id, perBranchLimit]);
    all.push(...rows);
  }
  if (want('ref')) {
    const { rows } = await pool.query(`
      SELECT 'ref'::text AS kind, r.id::text, r.space_id,
             coalesce(r.title, r.source_url, '(untitled)') AS title_or_snippet,
             (r.embedding <=> $1::vector) AS dist
        FROM refs r
        WHERE r.embedding IS NOT NULL
          AND ($2::uuid IS NULL OR r.space_id = $2)
        ORDER BY r.embedding <=> $1::vector LIMIT $3
    `, [literal, space_id, perBranchLimit]);
    all.push(...rows);
  }
  if (want('source_doc')) {
    const { rows } = await pool.query(`
      SELECT 'source_doc'::text AS kind, sd.id::text, res.space_id, sd.name AS title_or_snippet,
             (sd.embedding <=> $1::vector) AS dist
        FROM source_docs sd JOIN resources res ON res.id = sd.resource_id
        WHERE sd.embedding IS NOT NULL
          AND ($2::uuid IS NULL OR res.space_id = $2)
        ORDER BY sd.embedding <=> $1::vector LIMIT $3
    `, [literal, space_id, perBranchLimit]);
    all.push(...rows);
  }
  // Messages aren't embedded in Plan 3.

  return all.sort((a, b) => a.dist - b.dist);
}

// ----- RRF fusion ----------------------------------------------------------

function fuse(fts, vec, limit, offset) {
  const K = 60;
  const score = new Map();
  function add(rows) {
    rows.forEach((r, idx) => {
      const key = `${r.kind}:${r.id}`;
      const cur = score.get(key) || {
        kind: r.kind,
        id: r.id,
        space_id: r.space_id,
        title_or_snippet: r.title_or_snippet,
        rrf: 0
      };
      cur.rrf += 1 / (K + idx + 1);
      score.set(key, cur);
    });
  }
  add(fts);
  add(vec);
  const sorted = [...score.values()].sort((a, b) => b.rrf - a.rrf);
  return sorted.slice(offset, offset + limit).map(r => ({
    kind: r.kind,
    id: r.id,
    space_id: r.space_id,
    title_or_snippet: r.title_or_snippet,
    rank: r.rrf
  }));
}

// ----- Public API ----------------------------------------------------------

export async function fts({ q, space_id = null, kinds = null, limit = 50, offset = 0 } = {}) {
  if (!q || typeof q !== 'string') return [];
  const normalizedKinds = Array.isArray(kinds) && kinds.length ? kinds : null;
  const perBranchLimit = limit * 3;

  const ftsP = ftsRows({ q, space_id, kinds: normalizedKinds, perBranchLimit });

  let vec = [];
  try {
    const raw = await embedText(q, { timeoutMs: 5_000 });
    vec = await vectorRows({
      qvec: padTo(raw, 1024),
      space_id,
      kinds: normalizedKinds,
      perBranchLimit
    });
  } catch (e) {
    log.debug({ err: e }, 'search vector branch skipped — falling back to FTS-only');
  }

  const ftsResult = await ftsP;
  return fuse(ftsResult, vec, limit, offset);
}

// Alias for clarity at callsites that want the hybrid name.
export async function hybrid(args) { return fts(args); }