Compare commits
8 Commits
6d5c3027ac
...
bc86d3e282
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bc86d3e282 | ||
|
|
5d1eb2396b | ||
|
|
70bdba1a24 | ||
|
|
bc55da6b1e | ||
|
|
e29bacbda1 | ||
|
|
fc1e93a58f | ||
|
|
2dc9d612de | ||
|
|
e2be462ecb |
16
deploy/whisper/README.md
Normal file
16
deploy/whisper/README.md
Normal file
@@ -0,0 +1,16 @@
|
||||
# faster-whisper service (Dross voice STT)
|
||||
|
||||
Runs on **CT 102** (the Ollama box, `192.168.1.185`), bare-metal (no Docker), on the
|
||||
RTX A2000 with CPU fallback. OpenAI-style `/transcribe` consumed by void-app
|
||||
`lib/voice/whisper.js` (`WHISPER_URL=http://192.168.1.185:8001`).
|
||||
|
||||
## Install (on CT 102)
|
||||
```
|
||||
scp deploy/whisper/{server.py,setup.sh} root@192.168.1.185:/opt/whisper_server.py /root/setup.sh
|
||||
ssh root@192.168.1.185 'bash /root/setup.sh && install -m644 /opt/whisper_server.py /opt/whisper/server.py && systemctl enable --now whisper'
|
||||
curl http://192.168.1.185:8001/health # {"ok":true,"model":"small.en","device":"cuda"}
|
||||
```
|
||||
- venv at `/opt/whisper/venv`; model `small.en` (env `WHISPER_MODEL`); CUDA libs via
|
||||
`nvidia-cublas-cu12`/`nvidia-cudnn-cu12` pip wheels (LD_LIBRARY_PATH in the unit).
|
||||
- GPU → CPU fallback is in `server.py` `load()`.
|
||||
- **CT 102 disk was expanded +20G** (was 89% full) before install.
|
||||
35
deploy/whisper/server.py
Normal file
35
deploy/whisper/server.py
Normal file
@@ -0,0 +1,35 @@
|
||||
import os, tempfile
|
||||
from fastapi import FastAPI, UploadFile, File, HTTPException
|
||||
from faster_whisper import WhisperModel
|
||||
|
||||
MODEL = os.environ.get("WHISPER_MODEL", "small.en")
|
||||
app = FastAPI()
|
||||
model = None
|
||||
device_used = None
|
||||
|
||||
def load():
|
||||
global model, device_used
|
||||
try:
|
||||
model = WhisperModel(MODEL, device="cuda", compute_type="int8_float16")
|
||||
device_used = "cuda"
|
||||
except Exception:
|
||||
model = WhisperModel(MODEL, device="cpu", compute_type="int8")
|
||||
device_used = "cpu"
|
||||
|
||||
load()
|
||||
|
||||
@app.get("/health")
|
||||
def health():
|
||||
return {"ok": True, "model": MODEL, "device": device_used}
|
||||
|
||||
@app.post("/transcribe")
|
||||
async def transcribe(file: UploadFile = File(...)):
|
||||
data = await file.read()
|
||||
if not data:
|
||||
raise HTTPException(400, "empty audio")
|
||||
with tempfile.NamedTemporaryFile(suffix=".bin") as f:
|
||||
f.write(data); f.flush()
|
||||
segments, info = model.transcribe(f.name, beam_size=1, vad_filter=True)
|
||||
text = "".join(s.text for s in segments).strip()
|
||||
return {"text": text, "language": info.language,
|
||||
"duration": round(info.duration, 2), "device": device_used}
|
||||
26
deploy/whisper/setup.sh
Normal file
26
deploy/whisper/setup.sh
Normal file
@@ -0,0 +1,26 @@
|
||||
#!/usr/bin/env bash
|
||||
set -e
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get update -qq
|
||||
apt-get install -y -qq python3-pip python3-venv ffmpeg >/dev/null
|
||||
mkdir -p /opt/whisper
|
||||
python3 -m venv /opt/whisper/venv
|
||||
/opt/whisper/venv/bin/pip install -q --upgrade pip
|
||||
/opt/whisper/venv/bin/pip install -q faster-whisper fastapi "uvicorn[standard]" python-multipart nvidia-cublas-cu12 nvidia-cudnn-cu12
|
||||
SITE=/opt/whisper/venv/lib/python3.12/site-packages
|
||||
cat > /etc/systemd/system/whisper.service <<UNIT
|
||||
[Unit]
|
||||
Description=faster-whisper transcription server (Dross voice)
|
||||
After=network.target
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/opt/whisper
|
||||
Environment=WHISPER_MODEL=small.en
|
||||
Environment=LD_LIBRARY_PATH=${SITE}/nvidia/cublas/lib:${SITE}/nvidia/cudnn/lib
|
||||
ExecStart=/opt/whisper/venv/bin/uvicorn server:app --host 0.0.0.0 --port 8001
|
||||
Restart=on-failure
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
UNIT
|
||||
systemctl daemon-reload
|
||||
echo "deps+unit installed"
|
||||
@@ -1,7 +1,7 @@
|
||||
# Floating Dross Chat — Design
|
||||
|
||||
**Date:** 2026-06-09
|
||||
**Status:** Approved (pending final spec sign-off)
|
||||
**Status:** Phase 1 SHIPPED (v2.11.0, 2026-06-10) — global floating bubble + avatars + settings + persona. Phase 2 (voice) and the "Keep voice clips" retention setting are next.
|
||||
**Goal:** Replace the docked, per-Space "Cradle Chat" with a global, movable floating-bubble Dross companion — mobile-first, with voice-clip input transcribed locally into instructions.
|
||||
|
||||
---
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
# Floating Dross Chat — Phase 2 (Voice) Design
|
||||
|
||||
**Date:** 2026-06-10
|
||||
**Status:** SHIPPED — P2a v2.12.0 (transcribe+mic), P2b v2.13.0 (retention), 2026-06-10. Known gap: clips HA-replicated Z↔Z3 but not yet in the offsite Farm backup. Future: whisper-model selector, configurable storage, encryption-at-rest, LAN-IP mic (https-on-LAN).
|
||||
**Builds on:** `2026-06-09-floating-dross-chat-design.md` (Phase 1 shipped in v2.11.0)
|
||||
**Goal:** Let the user record a voice clip in the Dross bubble, transcribe it locally, and drop the transcript into the input to review-and-send. Optionally retain each clip paired with its transcript, stored durably and owner-only.
|
||||
|
||||
---
|
||||
|
||||
## Locked decisions (from the Phase-1 brainstorm + 2026-06-10 follow-up)
|
||||
|
||||
1. **STT = local faster-whisper on CT 102** (the Ollama box, RTX A2000). GPU with CPU fallback (per the GPU/CPU-fallback HA rule). English model (`small.en`) for speed/accuracy. OpenAI-compatible HTTP API.
|
||||
2. **Flow = review-and-send first** (`voiceMode: 'review'`). Record → transcribe → transcript lands in the bubble input → user edits/sends. `handsfree` (auto-send) and `action` (interpret) are later (the setting already exists; only `review` is wired now).
|
||||
3. **Retention = "Keep voice clips" Dross setting**, default OFF. When ON, each clip is saved paired with its transcript. **Storage:** transcript + metadata in **void-db** (`voice_clips` table — in the Core-4 offsite backup + HA-replicated); audio files on a **dedicated owner-only ZFS dataset** (`localzfs/void-voiceclips`, bind-mounted into void-app at `/var/lib/void/voice-clips`, 0700), **added to the offsite backup + syncoid replication**. NOT on void-app's ephemeral rootfs (it's the rebuildable tier, excluded from backups). Encryption-at-rest is a documented **future** toggle (ZFS native encryption, key in Vaultwarden).
|
||||
|
||||
## Non-goals (this phase)
|
||||
|
||||
- `handsfree` / `action` voice modes (designed-for; only `review` wired).
|
||||
- Encryption-at-rest of clips (future).
|
||||
- Wake-word / always-listening.
|
||||
|
||||
---
|
||||
|
||||
## Architecture
|
||||
|
||||
### Components
|
||||
|
||||
| Unit | Responsibility |
|
||||
|---|---|
|
||||
| **faster-whisper service** (CT 102, infra) | OpenAI-compatible `/v1/audio/transcriptions` (e.g. `faster-whisper-server`/`speaches`), `small.en`, GPU+CPU fallback, systemd unit, bound to `192.168.1.185:<port>` (LAN-only). |
|
||||
| `lib/voice/whisper.js` (void-app) | Thin client: POST the audio buffer to the CT-102 service, return `{ text }`. Timeout + error surface. |
|
||||
| `lib/api/routes/voice.js` (void-app) | `POST /api/voice/transcribe` (owner-only, multipart, ≤25 MB / ≤60 s): transcribe; if `dross.keepClips` is on, persist (Task: retention). Returns `{ text, clip_id? }`. |
|
||||
| `lib/db/repos/voice_clips.js` + migration | `voice_clips` table (id, transcript, duration_ms, bytes, mime, path, created_at). |
|
||||
| `public/components/dross_bubble.js` (edit) | Enable the mic: `MediaRecorder` capture (tap start/stop), recording UI (timer/waveform), upload, transcript → input (review-and-send). |
|
||||
| Settings → Dross (edit) | Add the **"Keep voice clips"** toggle; a small **clips list** (play / delete) when retention is on. |
|
||||
| Infra | New ZFS dataset + bind-mount; add the dataset to the offsite-backup script + syncoid job. |
|
||||
|
||||
### Data flow
|
||||
|
||||
**Transcribe:** mic tap → `MediaRecorder` (`audio/webm;codecs=opus`) → on stop, blob → `POST /api/voice/transcribe` (multipart) → `whisper.js` → CT-102 faster-whisper → `{text}` → bubble drops text into the input (review-and-send). Errors never block typing.
|
||||
|
||||
**Retention (when `dross.keepClips`):** the transcribe route, after a successful transcript, writes the audio to `/var/lib/void/voice-clips/<uuid>.webm` (0600) and inserts a `voice_clips` row (transcript + metadata + path). `GET /api/voice/clips` lists; `GET /api/voice/clips/:id/audio` streams; `DELETE` removes row + file. Owner-only throughout.
|
||||
|
||||
## Error handling
|
||||
|
||||
- **Whisper down / GPU absent:** `/transcribe` returns a clear 503; bubble shows "couldn't transcribe — type instead", keeps the typed text. faster-whisper falls back to CPU on a GPU-less node (slower).
|
||||
- **Mic permission denied / unsupported:** hide recording UI, one-line hint, typing still works.
|
||||
- **Clip too large/long:** reject at the route (413) with a friendly message.
|
||||
- **CT 102 disk pressure** (currently 89% full / 6.4 GB free): install lean (CTranslate2, no torch); **may expand the CT disk first**. Flagged as a build risk.
|
||||
|
||||
## Testing
|
||||
|
||||
- **Unit:** `voice.js` route with a mocked whisper client (returns `{text}`); retention path writes a row + file (temp dir) and lists/deletes; size/duration guard returns 413.
|
||||
- **Live smoke:** record a short WAV via the CT-300 test harness → `/api/voice/transcribe` → non-empty text from the real CT-102 service.
|
||||
- **Headless:** mic button enabled; recording UI toggles; (MediaRecorder needs a fake audio device in Chromium — use `--use-fake-device-for-media-stream`).
|
||||
|
||||
## Build phases
|
||||
|
||||
- **P2a — Transcription path.** faster-whisper on CT 102 + `whisper.js` + `/api/voice/transcribe` (no retention) + enable the mic + record→review-send. Ship-able.
|
||||
- **P2b — Retention.** ZFS dataset + bind-mount + backup/replication wiring; `voice_clips` table + repo; save on transcribe when `keepClips`; clips list/play/delete UI; the "Keep voice clips" toggle.
|
||||
|
||||
## Documentation
|
||||
|
||||
Wiki + Gitea per the standing rule; update `project_cradle_chat_floating` memory. Encryption-at-rest recorded as a future toggle.
|
||||
@@ -39,6 +39,7 @@ import { router as backupsRouter } from './routes/backups.js';
|
||||
import { router as kuttRouter } from './routes/kutt.js';
|
||||
import { router as themeRouter } from './routes/theme.js';
|
||||
import { router as drossRouter } from './routes/dross.js';
|
||||
import { router as voiceRouter } from './routes/voice.js';
|
||||
|
||||
export function mountApi(app) {
|
||||
const api = Router();
|
||||
@@ -75,6 +76,7 @@ export function mountApi(app) {
|
||||
api.use('/kutt', kuttRouter);
|
||||
api.use('/theme', themeRouter);
|
||||
api.use('/dross', drossRouter);
|
||||
api.use('/voice', voiceRouter);
|
||||
api.use('/pending-changes', pendingChangesRouter);
|
||||
api.use('/audit', auditRouter);
|
||||
api.use('/search', searchRouter);
|
||||
|
||||
@@ -10,7 +10,7 @@ import * as agents from '../../db/repos/agents.js';
|
||||
import { runAgentTurn } from '../../ai/agent/run_turn.js';
|
||||
import { personaFor } from '../../ai/personas/index.js';
|
||||
|
||||
const DEFAULT_SETTINGS = { avatar: 'soft-eye', accent: '#a86adf', persona: '', voiceMode: 'review' };
|
||||
const DEFAULT_SETTINGS = { avatar: 'soft-eye', accent: '#a86adf', persona: '', voiceMode: 'review', keepClips: false };
|
||||
const COMPANION_SLUG = 'companion';
|
||||
|
||||
export const router = Router();
|
||||
@@ -23,7 +23,8 @@ const settingsBody = z.object({
|
||||
avatar: z.enum(['soft-eye', 'wisp', 'motes']),
|
||||
accent: z.string().regex(/^#[0-9a-fA-F]{6}$/),
|
||||
persona: z.string().max(8000),
|
||||
voiceMode: z.enum(['review', 'handsfree', 'action'])
|
||||
voiceMode: z.enum(['review', 'handsfree', 'action']),
|
||||
keepClips: z.boolean().default(false)
|
||||
});
|
||||
router.put('/settings', requireOwner, validate({ body: settingsBody }),
|
||||
asyncWrap(async (req, res) => res.json(await settings.set('dross', req.body))));
|
||||
|
||||
73
lib/api/routes/voice.js
Normal file
73
lib/api/routes/voice.js
Normal file
@@ -0,0 +1,73 @@
|
||||
import { Router } from 'express';
|
||||
import multer from 'multer';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { createReadStream } from 'node:fs';
|
||||
import { writeFile, unlink } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { asyncWrap } from '../errors.js';
|
||||
import { requireOwner } from '../cap.js';
|
||||
import * as whisper from '../../voice/whisper.js';
|
||||
import * as settings from '../../db/repos/app_settings.js';
|
||||
import * as clips from '../../db/repos/voice_clips.js';
|
||||
export const router = Router();
|
||||
|
||||
const CLIPS_DIR = process.env.VOICE_CLIPS_DIR || '/var/lib/void/voice-clips';
|
||||
// In-memory upload; clips are small voice notes. 25 MB ceiling.
|
||||
const upload = multer({ storage: multer.memoryStorage(), limits: { fileSize: 25 * 1024 * 1024 } });
|
||||
|
||||
function extFor(mime = '') {
|
||||
if (mime.includes('ogg')) return '.ogg';
|
||||
if (mime.includes('mp4') || mime.includes('m4a')) return '.m4a';
|
||||
if (mime.includes('wav')) return '.wav';
|
||||
return '.webm';
|
||||
}
|
||||
|
||||
// POST /api/voice/transcribe — owner-only. multipart field `audio`. Returns { text }.
|
||||
// When the Dross "keepClips" setting is on, the clip + transcript are retained.
|
||||
router.post('/transcribe', requireOwner, upload.single('audio'), asyncWrap(async (req, res) => {
|
||||
if (!req.file || !req.file.buffer?.length) {
|
||||
return res.status(400).json({ error: { code: 'no_audio', message: 'no audio supplied' } });
|
||||
}
|
||||
let r;
|
||||
try {
|
||||
r = await whisper.transcribe(
|
||||
req.file.buffer, req.file.originalname || 'clip.webm', req.file.mimetype || 'audio/webm');
|
||||
} catch {
|
||||
return res.status(503).json({ error: { code: 'stt_unavailable', message: 'transcription service unavailable' } });
|
||||
}
|
||||
|
||||
const cfg = await settings.get('dross', {});
|
||||
let clip_id = null;
|
||||
if (cfg?.keepClips) {
|
||||
try {
|
||||
const id = randomUUID();
|
||||
const mime = req.file.mimetype || 'audio/webm';
|
||||
const filePath = path.join(CLIPS_DIR, id + extFor(mime));
|
||||
await writeFile(filePath, req.file.buffer, { mode: 0o600 });
|
||||
const row = await clips.create({
|
||||
transcript: r.text, duration_ms: r.duration != null ? Math.round(r.duration * 1000) : null,
|
||||
bytes: req.file.buffer.length, mime, path: filePath
|
||||
});
|
||||
clip_id = row.id;
|
||||
} catch { /* retention is best-effort; never fail the transcript */ }
|
||||
}
|
||||
res.json({ text: r.text, duration: r.duration ?? null, clip_id });
|
||||
}));
|
||||
|
||||
// GET /api/voice/clips — list retained clips (owner).
|
||||
router.get('/clips', requireOwner, asyncWrap(async (_req, res) => res.json(await clips.list())));
|
||||
|
||||
// GET /api/voice/clips/:id/audio — stream the audio file (owner).
|
||||
router.get('/clips/:id/audio', requireOwner, asyncWrap(async (req, res) => {
|
||||
const c = await clips.get(req.params.id);
|
||||
if (!c) return res.status(404).json({ error: { code: 'not_found', message: 'clip not found' } });
|
||||
res.setHeader('Content-Type', c.mime || 'audio/webm');
|
||||
createReadStream(c.path).on('error', () => res.status(404).end()).pipe(res);
|
||||
}));
|
||||
|
||||
// DELETE /api/voice/clips/:id — remove the row + the file (owner).
|
||||
router.delete('/clips/:id', requireOwner, asyncWrap(async (req, res) => {
|
||||
const removed = await clips.remove(req.params.id);
|
||||
if (removed?.path) { try { await unlink(removed.path); } catch { /* file may be gone */ } }
|
||||
res.status(204).end();
|
||||
}));
|
||||
14
lib/db/migrations/029_voice_clips.sql
Normal file
14
lib/db/migrations/029_voice_clips.sql
Normal file
@@ -0,0 +1,14 @@
|
||||
-- 029_voice_clips.sql
|
||||
-- Optional retained Dross voice clips (when the "Keep voice clips" setting is on).
|
||||
-- Transcript + metadata here (durable, HA-replicated); audio bytes live as files
|
||||
-- on the owner-only ZFS subvol mounted at /var/lib/void/voice-clips.
|
||||
CREATE TABLE voice_clips (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
transcript text NOT NULL DEFAULT '',
|
||||
duration_ms integer,
|
||||
bytes bigint,
|
||||
mime text,
|
||||
path text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX idx_voice_clips_created ON voice_clips (created_at DESC);
|
||||
26
lib/db/repos/voice_clips.js
Normal file
26
lib/db/repos/voice_clips.js
Normal file
@@ -0,0 +1,26 @@
|
||||
import { pool } from '../pool.js';
|
||||
|
||||
export async function create({ transcript = '', duration_ms = null, bytes = null, mime = null, path }) {
|
||||
const { rows } = await pool.query(
|
||||
`INSERT INTO voice_clips (transcript, duration_ms, bytes, mime, path)
|
||||
VALUES ($1,$2,$3,$4,$5) RETURNING *`,
|
||||
[transcript, duration_ms, bytes, mime, path]);
|
||||
return rows[0];
|
||||
}
|
||||
|
||||
export async function list(limit = 100) {
|
||||
const { rows } = await pool.query(
|
||||
`SELECT id, transcript, duration_ms, bytes, mime, created_at
|
||||
FROM voice_clips ORDER BY created_at DESC LIMIT $1`, [limit]);
|
||||
return rows;
|
||||
}
|
||||
|
||||
export async function get(id) {
|
||||
const { rows } = await pool.query(`SELECT * FROM voice_clips WHERE id = $1`, [id]);
|
||||
return rows[0] || null;
|
||||
}
|
||||
|
||||
export async function remove(id) {
|
||||
const { rows } = await pool.query(`DELETE FROM voice_clips WHERE id = $1 RETURNING path`, [id]);
|
||||
return rows[0] || null; // returns {path} so the caller can unlink the file
|
||||
}
|
||||
22
lib/voice/whisper.js
Normal file
22
lib/voice/whisper.js
Normal file
@@ -0,0 +1,22 @@
|
||||
// Thin client for the local faster-whisper service on CT 102 (the Ollama box).
|
||||
// GPU with CPU fallback lives in the service itself; here we just POST the audio
|
||||
// buffer and return the transcript. LAN-only endpoint.
|
||||
const WHISPER_URL = process.env.WHISPER_URL || 'http://192.168.1.185:8001';
|
||||
|
||||
export async function transcribe(buffer, filename = 'clip.webm', mime = 'audio/webm') {
|
||||
const fd = new FormData();
|
||||
fd.append('file', new Blob([buffer], { type: mime }), filename);
|
||||
const res = await fetch(`${WHISPER_URL}/transcribe`, {
|
||||
method: 'POST', body: fd, signal: AbortSignal.timeout(120000)
|
||||
});
|
||||
if (!res.ok) throw new Error(`whisper ${res.status}`);
|
||||
const j = await res.json();
|
||||
return { text: (j.text || '').trim(), duration: j.duration, device: j.device };
|
||||
}
|
||||
|
||||
export async function health() {
|
||||
try {
|
||||
const res = await fetch(`${WHISPER_URL}/health`, { signal: AbortSignal.timeout(5000) });
|
||||
return res.ok ? await res.json() : null;
|
||||
} catch { return null; }
|
||||
}
|
||||
4
package-lock.json
generated
4
package-lock.json
generated
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "void-server",
|
||||
"version": "2.10.0",
|
||||
"version": "2.13.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "void-server",
|
||||
"version": "2.10.0",
|
||||
"version": "2.13.0",
|
||||
"dependencies": {
|
||||
"@modelcontextprotocol/sdk": "^1.29.0",
|
||||
"@mozilla/readability": "^0.6.0",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "void-server",
|
||||
"version": "2.11.0",
|
||||
"version": "2.13.0",
|
||||
"type": "module",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
|
||||
@@ -22,8 +22,9 @@ export async function renderDrossBubble() {
|
||||
const input = el('textarea', { rows: 1, placeholder: 'Ask Dross…' });
|
||||
const sendBtn = el('button', { class: 'dross-send', title: 'Send' },
|
||||
el('span', { html: '<svg viewBox="0 0 24 24" width="20" height="20" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M22 2 11 13M22 2l-7 20-4-9-9-4 20-7z"/></svg>' }));
|
||||
const mic = el('button', { class: 'dross-mic', disabled: true, title: 'Voice arrives in Phase 2' },
|
||||
el('span', { html: '<svg viewBox="0 0 24 24" width="22" height="22" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="9" y="3" width="6" height="11" rx="3"/><path d="M5 11a7 7 0 0 0 14 0"/><line x1="12" y1="18" x2="12" y2="22"/><line x1="8" y1="22" x2="16" y2="22"/></svg>' }), 'Hold to talk');
|
||||
const micLabel = el('span', {}, 'Tap to record');
|
||||
const mic = el('button', { class: 'dross-mic', title: 'Record a voice note' },
|
||||
el('span', { html: '<svg viewBox="0 0 24 24" width="22" height="22" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="9" y="3" width="6" height="11" rx="3"/><path d="M5 11a7 7 0 0 0 14 0"/><line x1="12" y1="18" x2="12" y2="22"/><line x1="8" y1="22" x2="16" y2="22"/></svg>' }), micLabel);
|
||||
const closeBtn = el('button', { class: 'dross-x', title: 'Close' }, '⤬');
|
||||
const header = el('div', { class: 'dross-hd' }, drossAvatar(cfg.avatar, 30),
|
||||
el('div', { class: 'dross-who' }, 'Dross', el('small', {}, 'always here, regrettably')), closeBtn);
|
||||
@@ -51,15 +52,59 @@ export async function renderDrossBubble() {
|
||||
const top = Math.max(8, Math.min(r.bottom - pr.height, innerHeight - pr.height - 8));
|
||||
panel.style.right = 'auto'; panel.style.bottom = 'auto'; panel.style.left = left + 'px'; panel.style.top = top + 'px';
|
||||
if (!loaded) { loaded = true; chat.load(); }
|
||||
input.focus();
|
||||
// NB: do NOT auto-focus the input — on mobile that pops the keyboard every
|
||||
// time Dross opens. The keyboard should only appear when the user taps the box.
|
||||
}
|
||||
function closePanel() { panel.classList.remove('open'); fab.style.display = 'block'; }
|
||||
fab.addEventListener('click', () => { if (fab._moved) { fab._moved = false; return; } openPanel(); });
|
||||
closeBtn.addEventListener('click', closePanel);
|
||||
collapse.addEventListener('click', closePanel);
|
||||
// Topbar ◆ button (and any caller) can summon/dismiss Dross.
|
||||
window.addEventListener('dross-toggle', () => panel.classList.contains('open') ? closePanel() : openPanel());
|
||||
|
||||
drag(fab, fab, true); drag(header, panel, false);
|
||||
|
||||
// ---- voice: tap mic to record, tap again to stop → transcribe → review-and-send ----
|
||||
let media = null, chunks = [], recording = false;
|
||||
function setMic(label, rec) { micLabel.textContent = label; mic.classList.toggle('rec', !!rec); }
|
||||
async function startRec() {
|
||||
try {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
chunks = [];
|
||||
const opt = (window.MediaRecorder && MediaRecorder.isTypeSupported('audio/webm;codecs=opus'))
|
||||
? { mimeType: 'audio/webm;codecs=opus' } : {};
|
||||
media = new MediaRecorder(stream, opt);
|
||||
media.ondataavailable = (e) => { if (e.data && e.data.size) chunks.push(e.data); };
|
||||
media.onstop = async () => {
|
||||
stream.getTracks().forEach(t => t.stop());
|
||||
await sendClip(new Blob(chunks, { type: media.mimeType || 'audio/webm' }));
|
||||
};
|
||||
media.start();
|
||||
recording = true; setMic('● Recording… tap to stop', true);
|
||||
} catch {
|
||||
setMic('Mic blocked', false); setTimeout(() => setMic('Tap to record', false), 1800);
|
||||
}
|
||||
}
|
||||
function stopRec() {
|
||||
if (media && recording) { recording = false; setMic('Transcribing…', false); media.stop(); }
|
||||
}
|
||||
async function sendClip(blob) {
|
||||
try {
|
||||
const fd = new FormData(); fd.append('audio', blob, 'clip.webm');
|
||||
const res = await fetch('/api/voice/transcribe', {
|
||||
method: 'POST', headers: { Authorization: 'Bearer ' + (localStorage.getItem('void_token') || '') }, body: fd
|
||||
});
|
||||
if (!res.ok) throw new Error('stt');
|
||||
const { text } = await res.json();
|
||||
setMic('Tap to record', false);
|
||||
if (text) { input.value = input.value ? (input.value + ' ' + text) : text; input.focus(); }
|
||||
// voiceMode 'handsfree'/'action' (Phase 2b+) would branch here.
|
||||
} catch {
|
||||
setMic('Transcribe failed', false); setTimeout(() => setMic('Tap to record', false), 2000);
|
||||
}
|
||||
}
|
||||
mic.addEventListener('click', () => recording ? stopRec() : startRec());
|
||||
|
||||
window.addEventListener('dross-settings-changed', async () => {
|
||||
try { cfg = { ...cfg, ...(await api.get('/api/dross/settings')) }; } catch { return; }
|
||||
applyAccent(fab, cfg.accent); applyAccent(panel, cfg.accent);
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
import { el, mount, clear } from '../dom.js';
|
||||
import { navigate } from '../router.js';
|
||||
import { on } from '../state.js';
|
||||
import { toggleSidebar, toggleRail } from './chrome.js';
|
||||
import { toggleSidebar } from './chrome.js';
|
||||
import { api } from '../api.js';
|
||||
|
||||
// Cluster health → topbar pill. Returns [status, label, title].
|
||||
@@ -72,7 +72,7 @@ export function renderTopbar(root) {
|
||||
el('div', { class: 'topbar-spacer' }),
|
||||
clusterPill,
|
||||
bell,
|
||||
el('button', { class: 'chrome-toggle', title: 'Toggle companion chat', onclick: toggleRail }, '◆'),
|
||||
el('button', { class: 'chrome-toggle', title: 'Summon Dross', onclick: () => window.dispatchEvent(new CustomEvent('dross-toggle')) }, '◆'),
|
||||
el('button', { class: 'icon-btn', onclick: () => alert('Agent-switching ships post-Plan-2.') }, 'Owner')
|
||||
);
|
||||
|
||||
|
||||
@@ -29,20 +29,18 @@ html, body { margin: 0; padding: 0; height: 100%; background: var(--bg); color:
|
||||
|
||||
#shell {
|
||||
display: grid;
|
||||
grid-template-columns: var(--sidebar-w) 1fr var(--rail-w);
|
||||
grid-template-columns: var(--sidebar-w) 1fr;
|
||||
grid-template-rows: var(--topbar-h) 1fr;
|
||||
grid-template-areas:
|
||||
"topbar topbar topbar"
|
||||
"sidebar main rail";
|
||||
"topbar topbar"
|
||||
"sidebar main";
|
||||
height: 100vh;
|
||||
width: 100vw;
|
||||
}
|
||||
#shell.rail-collapsed { grid-template-columns: var(--sidebar-w) 1fr var(--rail-w-min); }
|
||||
|
||||
#topbar { grid-area: topbar; border-bottom: 1px solid var(--border); background: var(--panel); display: flex; align-items: center; padding: 0 16px; gap: 12px; }
|
||||
#sidebar { grid-area: sidebar; border-right: 1px solid var(--border); background: var(--panel); overflow-y: auto; padding: 12px 0; }
|
||||
#main { grid-area: main; overflow-y: auto; padding: 24px 32px; }
|
||||
#rightrail{ grid-area: rail; border-left: 1px solid var(--border); background: var(--panel); overflow: hidden; display: flex; flex-direction: column; }
|
||||
|
||||
/* topbar */
|
||||
.brand { font-family: var(--font-display); font-weight: 700; letter-spacing: 0.18em; font-size: 14px; color: var(--accent); text-transform: uppercase; padding: 0 6px; }
|
||||
@@ -483,15 +481,11 @@ ul.plain li:last-child { border-bottom: none; }
|
||||
/* ===== Collapsible chrome + responsive layout (Plan 6 polish) ===== */
|
||||
:root { --sidebar-w-min: 0px; }
|
||||
#shell { transition: grid-template-columns .22s ease; }
|
||||
#sidebar, #rightrail { transition: transform .22s ease; }
|
||||
#sidebar { transition: transform .22s ease; }
|
||||
|
||||
/* Desktop collapse — shrink the grid columns */
|
||||
#shell.sidebar-collapsed { grid-template-columns: var(--sidebar-w-min) 1fr var(--rail-w); }
|
||||
#shell.sidebar-collapsed.rail-collapsed { grid-template-columns: var(--sidebar-w-min) 1fr var(--rail-w-min); }
|
||||
#shell.rail-collapsed { grid-template-columns: var(--sidebar-w) 1fr var(--rail-w-min); }
|
||||
/* Desktop collapse — shrink the sidebar column */
|
||||
#shell.sidebar-collapsed { grid-template-columns: var(--sidebar-w-min) 1fr; }
|
||||
#shell.sidebar-collapsed #sidebar { overflow: hidden; border-right: none; }
|
||||
/* Hide chat body when the rail is collapsed so the thin strip stays clean */
|
||||
#shell.rail-collapsed .rail-chat { display: none; }
|
||||
|
||||
/* Topbar toggle buttons */
|
||||
.chrome-toggle {
|
||||
@@ -514,13 +508,11 @@ body.drawer-open #scrim { opacity: 1; pointer-events: auto; }
|
||||
/* ---- Narrow / mobile / vertical: off-canvas drawers, single-column main ---- */
|
||||
@media (max-width: 860px) {
|
||||
#shell,
|
||||
#shell.sidebar-collapsed,
|
||||
#shell.rail-collapsed,
|
||||
#shell.sidebar-collapsed.rail-collapsed {
|
||||
#shell.sidebar-collapsed {
|
||||
grid-template-columns: 1fr;
|
||||
grid-template-areas: "topbar" "main";
|
||||
}
|
||||
#sidebar, #rightrail {
|
||||
#sidebar {
|
||||
position: fixed; top: var(--topbar-h); bottom: 0; z-index: 50;
|
||||
}
|
||||
#sidebar { left: 0; width: min(82vw, 300px); transform: translateX(-100%); border-right: 1px solid var(--border); }
|
||||
@@ -773,6 +765,8 @@ body.drawer-open #scrim { opacity: 1; pointer-events: auto; }
|
||||
.dross-btnrow{display:flex;gap:10px}
|
||||
.dross-mic{flex:1;height:50px;border-radius:12px;border:1px solid var(--dross-dim);background:var(--dross-soft);color:var(--dross-glow);cursor:pointer;display:flex;align-items:center;justify-content:center;gap:9px;font-family:var(--font-ui);font-size:13px}
|
||||
.dross-mic[disabled]{opacity:.5;cursor:not-allowed}
|
||||
.dross-mic.rec{background:#3a1010;border-color:var(--accent);color:#fff;animation:dross-rec 1.2s infinite}
|
||||
@keyframes dross-rec{0%,100%{box-shadow:0 0 0 0 rgba(255,79,46,.5)}50%{box-shadow:0 0 0 8px rgba(255,79,46,0)}}
|
||||
.dross-send{width:64px;height:50px;border-radius:12px;border:1px solid var(--dross-dim);background:linear-gradient(180deg,var(--dross),var(--dross-dim));color:#fff;cursor:pointer;display:grid;place-items:center}
|
||||
.dross-collapse{display:flex;align-items:center;justify-content:center;gap:8px;height:34px;cursor:pointer;color:var(--muted);
|
||||
font-family:var(--font-ui);font-size:11px;letter-spacing:.12em;text-transform:uppercase;background:#0b0810;border-top:1px solid var(--border)}
|
||||
@@ -782,3 +776,7 @@ body.drawer-open #scrim { opacity: 1; pointer-events: auto; }
|
||||
.dross-avopt{display:flex;flex-direction:column;align-items:center;gap:6px;padding:10px 14px;border:1px solid var(--border);border-radius:12px;background:var(--panel);color:var(--muted);cursor:pointer;font-size:11px}
|
||||
.dross-avopt.on{border-color:var(--dross);color:var(--dross-glow);background:var(--dross-soft)}
|
||||
.dross-persona{width:100%;background:#0d0d13;border:1px solid var(--border);border-radius:8px;padding:10px;color:var(--text);font-family:var(--font-mono);font-size:12px;resize:vertical;margin:10px 0}
|
||||
.dross-clips{display:flex;flex-direction:column;gap:2px}
|
||||
.dross-clip{display:flex;align-items:center;gap:8px;font-size:12px;padding:4px 0;border-bottom:1px solid #ffffff08}
|
||||
.dross-clip-txt{flex:1;color:var(--text);overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
|
||||
.dross-clip audio{display:none}
|
||||
|
||||
@@ -151,7 +151,7 @@ async function renderAgents(c) {
|
||||
|
||||
function drossBody() {
|
||||
const out = el('span', { class: 'muted', style: { marginLeft: '8px' } });
|
||||
let cur = { avatar: 'soft-eye', accent: '#a86adf', persona: '', voiceMode: 'review' };
|
||||
let cur = { avatar: 'soft-eye', accent: '#a86adf', persona: '', voiceMode: 'review', keepClips: false };
|
||||
const avatarRow = el('div', { class: 'dross-pick' });
|
||||
const accent = el('input', { type: 'color', value: cur.accent });
|
||||
const persona = el('textarea', { class: 'dross-persona', rows: 6, placeholder: "Dross's system prompt…" });
|
||||
@@ -159,6 +159,8 @@ function drossBody() {
|
||||
el('option', { value: 'review' }, 'Voice: review then send'),
|
||||
el('option', { value: 'handsfree' }, 'Voice: hands-free (Phase 2)'),
|
||||
el('option', { value: 'action' }, 'Voice: interpret to action (later)'));
|
||||
const keep = el('input', { type: 'checkbox' });
|
||||
const clipsWrap = el('div', { class: 'dross-clips' });
|
||||
|
||||
function paintAvatars() {
|
||||
mount(avatarRow, ['soft-eye', 'wisp', 'motes'].map(v => {
|
||||
@@ -169,22 +171,55 @@ function drossBody() {
|
||||
return card;
|
||||
}));
|
||||
}
|
||||
|
||||
async function loadClips() {
|
||||
if (!keep.checked) { mount(clipsWrap); return; }
|
||||
let rows = [];
|
||||
try { rows = await api.get('/api/voice/clips'); } catch { mount(clipsWrap, el('span', { class: 'muted' }, 'Clips unavailable')); return; }
|
||||
mount(clipsWrap, el('div', { class: 'st-lbl', style: { margin: '10px 0 4px' } }, `Saved clips (${rows.length})`),
|
||||
...rows.map(c => {
|
||||
const audio = el('audio');
|
||||
const play = el('button', { class: 'ghost' }, '▶');
|
||||
play.onclick = async () => {
|
||||
try {
|
||||
const res = await fetch('/api/voice/clips/' + c.id + '/audio',
|
||||
{ headers: { Authorization: 'Bearer ' + (localStorage.getItem('void_token') || '') } });
|
||||
audio.src = URL.createObjectURL(await res.blob()); audio.play();
|
||||
} catch { /* */ }
|
||||
};
|
||||
const del = el('button', { class: 'ghost dv-ignore' }, '✕');
|
||||
del.onclick = async () => { try { await api.del('/api/voice/clips/' + c.id); loadClips(); } catch { /* */ } };
|
||||
return el('div', { class: 'dross-clip' }, play, audio,
|
||||
el('span', { class: 'dross-clip-txt' }, c.transcript || '(no transcript)'),
|
||||
el('span', { class: 'muted', style: { fontSize: '10px' } },
|
||||
new Date(c.created_at).toLocaleString('en-AU', { month: 'short', day: 'numeric', hour: '2-digit', minute: '2-digit' })),
|
||||
del);
|
||||
}));
|
||||
}
|
||||
|
||||
(async () => {
|
||||
try { cur = { ...cur, ...(await api.get('/api/dross/settings')) }; } catch {}
|
||||
accent.value = cur.accent; persona.value = cur.persona; mode.value = cur.voiceMode; paintAvatars();
|
||||
accent.value = cur.accent; persona.value = cur.persona; mode.value = cur.voiceMode;
|
||||
keep.checked = !!cur.keepClips; paintAvatars(); loadClips();
|
||||
})();
|
||||
accent.addEventListener('input', () => { cur.accent = accent.value; paintAvatars(); });
|
||||
keep.addEventListener('change', loadClips);
|
||||
|
||||
const save = el('button', { class: 'primary' }, 'Save');
|
||||
save.onclick = async () => {
|
||||
try {
|
||||
await api.put('/api/dross/settings', { avatar: cur.avatar, accent: accent.value, persona: persona.value, voiceMode: mode.value });
|
||||
await api.put('/api/dross/settings', {
|
||||
avatar: cur.avatar, accent: accent.value, persona: persona.value, voiceMode: mode.value, keepClips: keep.checked
|
||||
});
|
||||
window.dispatchEvent(new CustomEvent('dross-settings-changed'));
|
||||
out.textContent = 'Saved.';
|
||||
} catch { out.textContent = 'Save failed'; }
|
||||
};
|
||||
return el('div', { class: 'settings-body' }, avatarRow, el('label', { class: 'st-lbl' }, 'Accent', accent),
|
||||
persona, el('div', { class: 'theme-actions' }, mode, save, out));
|
||||
persona,
|
||||
el('label', { class: 'st-lbl' }, keep, ' Keep voice clips (saves audio + transcript, owner-only)'),
|
||||
el('div', { class: 'theme-actions' }, mode, save, out),
|
||||
clipsWrap);
|
||||
}
|
||||
|
||||
export async function render(main) {
|
||||
|
||||
19
tests/repos/voice_clips.test.js
Normal file
19
tests/repos/voice_clips.test.js
Normal file
@@ -0,0 +1,19 @@
|
||||
import { describe, it, expect, beforeEach } from 'vitest';
|
||||
import { resetDb } from '../helpers/db.js';
|
||||
import { migrateUp } from '../../lib/db/migrate.js';
|
||||
import * as clips from '../../lib/db/repos/voice_clips.js';
|
||||
|
||||
beforeEach(async () => { await resetDb(); await migrateUp(); });
|
||||
|
||||
describe('voice_clips repo', () => {
|
||||
it('creates, lists newest-first, and removes (returning path)', async () => {
|
||||
const a = await clips.create({ transcript: 'first', bytes: 10, mime: 'audio/webm', path: '/x/a.webm' });
|
||||
const b = await clips.create({ transcript: 'second', bytes: 20, mime: 'audio/webm', path: '/x/b.webm' });
|
||||
const list = await clips.list();
|
||||
expect(list.length).toBe(2);
|
||||
expect(list[0].transcript).toBe('second'); // newest first
|
||||
const removed = await clips.remove(a.id);
|
||||
expect(removed.path).toBe('/x/a.webm');
|
||||
expect((await clips.list()).length).toBe(1);
|
||||
});
|
||||
});
|
||||
24
tests/routes/voice.test.js
Normal file
24
tests/routes/voice.test.js
Normal file
@@ -0,0 +1,24 @@
|
||||
import { describe, it, expect, beforeAll } from 'vitest';
|
||||
import request from 'supertest';
|
||||
import { createApp } from '../../server.js';
|
||||
import { resetDb } from '../helpers/db.js';
|
||||
import { migrateUp } from '../../lib/db/migrate.js';
|
||||
|
||||
let app;
|
||||
const owner = { Authorization: 'Bearer test-token' };
|
||||
beforeAll(async () => {
|
||||
await resetDb(); await migrateUp();
|
||||
process.env.OWNER_TOKEN = 'test-token';
|
||||
app = createApp();
|
||||
});
|
||||
|
||||
describe('voice transcribe route', () => {
|
||||
it('401 without a token', async () => {
|
||||
const res = await request(app).post('/api/voice/transcribe');
|
||||
expect(res.status).toBe(401);
|
||||
});
|
||||
it('400 when no audio supplied', async () => {
|
||||
const res = await request(app).post('/api/voice/transcribe').set(owner);
|
||||
expect(res.status).toBe(400);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user