Hynes/Void-Homelab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(embed): chunk + mean-pool long text so large pages embed

Browse Source 
Split long page text into 1500-char chunks before calling Ollama, then
mean-pool the per-chunk vectors into one page vector. Removes the hard
6000-char slice that still caused 500s on dense markdown/table pages.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
root
2026-06-05 23:24:40 +10:00
parent 946a03883f
commit 71adc51c00
3 changed files with 76 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
lib/jobs/workers/embed.js
View File

@@ -1,4 +1,4 @@
import { embedText, padTo } from '../../ai/ollama.js';
import { embedTextPooled, padTo } from '../../ai/ollama.js';
import { pool } from '../../db/pool.js';
import { recordAudit } from '../../db/repos/audit.js';
@@ -19,8 +19,8 @@ export async function handler(job) {
if (!table) throw new Error(`unknown entity_type: ${entity_type}`);
const { rows: [row] } = await pool.query(`SELECT * FROM ${table} WHERE id=$1`, [entity_id]);
if (!row) return { skipped: 'gone' };
const text = STRING_BUILDERS[entity_type](row).slice(0, 6_000);
const v = await embedText(text);
const text = STRING_BUILDERS[entity_type](row);
const v = await embedTextPooled(text);
const padded = padTo(v, 1024);
const literal = '[' + padded.join(',') + ']';
await pool.query(`UPDATE ${table} SET embedding=$1::vector WHERE id=$2`, [literal, entity_id]);