diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a68a86..66fe715 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to Void 2.0 are documented here. Format: [Keep a Changelog](https://keepachangelog.com). +## 2.4.0 — Storage · capacity card (Sacred Valley) +- **New "Storage · capacity" card** (`public/views/cards/storage.js`, `/api/storage`, `lib/proxmox/storage.js`) — read-only Proxmox health via the same `PROXMOX_RO_TOKEN` as the cluster card. Shows: **ZFS pools** (health + usage meter), **dropped pools** (a configured zfspool storage that's no longer `available` — the donatello/leonardo SATA-bus signal, rendered red), and **per-container disk fill** (top LXC by rootfs %), with a HEALTHY/WATCH/ATTENTION roll-up badge. Thresholds: 80% warn, 90% crit; a non-ONLINE or dropped pool is always crit. +- Closes the monitoring gap from the 2026-06-09 audit (the Void couldn't previously see C1 = pools offline or H2 = a container at 95%). Pure `normalizeStorage()` is unit-tested. + ## 2.3.0 — MagicMirror² as a Void app - **New "MagicMirror" Apps view** (`#/mirror`, `public/views/mirror.js`) — embeds the smart-mirror dashboard (CT 111) via the shared `embedView` factory, like Timelapse / AI Usage. - **Exposure:** MagicMirror (LAN-only `192.168.1.224:8080`) is now published at **mirror.hynesy.com** through Traefik + the `*.hynesy.com` tunnel, private behind **CF Access** (Farm policy / Google IdP). A Traefik `mirror-frame` middleware replaces MM's `X-Frame-Options: SAMEORIGIN` with a CSP `frame-ancestors` allowing the Void origins so the iframe renders. diff --git a/lib/api/index.js b/lib/api/index.js index 71d68a9..7f020c8 100644 --- a/lib/api/index.js +++ b/lib/api/index.js @@ -34,6 +34,7 @@ import { router as littleblueRouter } from './routes/littleblue.js'; import { router as aiUsageRouter } from './routes/ai_usage.js'; import { router as infraRouter } from './routes/infra.js'; import { router as clusterRouter } from './routes/cluster.js'; +import { router as storageRouter } from './routes/storage.js'; import { router as kuttRouter } from './routes/kutt.js'; export function mountApi(app) { @@ -50,6 +51,7 @@ export function mountApi(app) { api.use('/actions', actionsRouter); api.use('/infra', infraRouter); api.use('/cluster', clusterRouter); + api.use('/storage', storageRouter); api.use('/little-blue', littleblueRouter); api.use('/ai-usage', aiUsageRouter); api.use('/projects', projectsRouter); diff --git a/lib/api/routes/storage.js b/lib/api/routes/storage.js new file mode 100644 index 0000000..15c4752 --- /dev/null +++ b/lib/api/routes/storage.js @@ -0,0 +1,17 @@ +import { Router } from 'express'; +import { asyncWrap } from '../errors.js'; +import { storageHealth } from '../../proxmox/storage.js'; + +// Read-only storage/capacity health for the Sacred Valley card. Cached briefly so +// multiple polling clients coalesce into one set of PVE calls. Owner or any authed agent. +export const router = Router(); + +let cache = { at: 0, data: null }; +const TTL = 15_000; + +router.get('/', asyncWrap(async (_req, res) => { + if (cache.data && Date.now() - cache.at < TTL) return res.json(cache.data); + const data = await storageHealth(); + cache = { at: Date.now(), data }; + res.json(data); +})); diff --git a/lib/proxmox/storage.js b/lib/proxmox/storage.js new file mode 100644 index 0000000..a5a92ff --- /dev/null +++ b/lib/proxmox/storage.js @@ -0,0 +1,94 @@ +import { Agent } from 'undici'; + +// Read-only Proxmox storage + capacity health for the Sacred Valley card. Same +// PVEAuditor token as the cluster card (PROXMOX_RO_TOKEN). Surfaces the two things +// that have actually bitten this homelab and were previously invisible: +// 1. a ZFS pool dropping out (the donatello/leonardo SATA-bus incident) — seen as +// a zfspool storage whose status is no longer 'available'. +// 2. a container rootfs filling up (mediastack hitting 95%) — per-LXC disk/maxdisk. + +let insecure; +function tlsDispatcher() { + if (process.env.PROXMOX_INSECURE_TLS !== '1') return undefined; + insecure ??= new Agent({ connect: { rejectUnauthorized: false } }); + return insecure; +} + +async function pveGet(path, { apiUrl, token, fetchImpl = fetch }) { + const res = await fetchImpl(`${apiUrl}/api2/json${path}`, { + headers: { Authorization: `PVEAPIToken=${token}` }, + dispatcher: tlsDispatcher() + }); + if (!res.ok) throw new Error(`pve ${path} -> ${res.status}`); + return (await res.json())?.data ?? []; +} + +export const WARN = 80, CRIT = 90; +const pct = (used, total) => (total > 0 ? Math.round((used / total) * 100) : null); +const sev = p => (p == null ? 'ok' : p >= CRIT ? 'crit' : p >= WARN ? 'warn' : 'ok'); +const worstOf = items => items.reduce( + (w, x) => (x.status === 'crit' || w === 'crit') ? 'crit' : (x.status === 'warn' || w === 'warn') ? 'warn' : 'ok', 'ok'); + +// Pure: fold /nodes/*/disks/zfs + /cluster/resources(storage,vm) into the card shape. +export function normalizeStorage(storageRes = [], vmRes = [], zfsByNode = {}) { + // Imported ZFS pools (health + usage) + const pools = []; + for (const [node, list] of Object.entries(zfsByNode)) { + for (const z of (list || [])) { + const p = pct(z.alloc, z.size); + pools.push({ + name: z.name, node, health: z.health, used: z.alloc, total: z.size, pct: p, + status: z.health !== 'ONLINE' ? 'crit' : sev(p) + }); + } + } + pools.sort((a, b) => a.name.localeCompare(b.name) || a.node.localeCompare(b.node)); + + // zfspool storages that are configured but NOT available = a pool that has dropped + // out (or never imported). This is the donatello/leonardo signal. + const down = storageRes + .filter(s => s.plugintype === 'zfspool' && s.status !== 'available') + .map(s => ({ name: s.storage, node: s.node, state: s.status || 'unavailable', status: 'crit' })) + .sort((a, b) => a.name.localeCompare(b.name) || a.node.localeCompare(b.node)); + + // Per-guest rootfs fill. LXC report disk/maxdisk; QEMU usually report disk=0 + // (no agent) so they're skipped rather than shown as 0%. + const guests = vmRes + .filter(v => v.type === 'lxc' && v.maxdisk > 0 && v.disk > 0) + .map(v => { + const p = pct(v.disk, v.maxdisk); + return { vmid: v.vmid, name: v.name, node: v.node, used: v.disk, total: v.maxdisk, pct: p, status: sev(p) }; + }) + .sort((a, b) => b.pct - a.pct); + + const alerts = [ + ...down.map(d => `${d.name} (${d.node}) ${d.state}`), + ...pools.filter(p => p.health !== 'ONLINE').map(p => `pool ${p.name} ${p.health}`), + ...guests.filter(g => g.status !== 'ok').map(g => `CT ${g.vmid} ${g.name} ${g.pct}%`) + ]; + + return { worst: worstOf([...pools, ...down, ...guests]), pools, down, guests, alerts }; +} + +export async function storageHealth(opts = {}) { + const cfg = { + apiUrl: opts.apiUrl || process.env.PROXMOX_API_URL, + token: opts.token || process.env.PROXMOX_RO_TOKEN || process.env.PROXMOX_API_TOKEN, + fetchImpl: opts.fetchImpl || fetch + }; + if (!cfg.apiUrl || !cfg.token) return { error: 'proxmox_not_configured', at: Date.now() }; + try { + const [storageRes, vmRes, nodes] = await Promise.all([ + pveGet('/cluster/resources?type=storage', cfg), + pveGet('/cluster/resources?type=vm', cfg), + pveGet('/nodes', cfg) + ]); + const zfsByNode = {}; + await Promise.all((nodes || []) + .filter(n => n.status === 'online') + .map(async n => { zfsByNode[n.node] = await pveGet(`/nodes/${n.node}/disks/zfs`, cfg).catch(() => []); })); + return { ...normalizeStorage(storageRes, vmRes, zfsByNode), at: Date.now() }; + } catch (e) { + return { error: String(e.message || e), at: Date.now() }; + } +} diff --git a/package.json b/package.json index c737075..4d2770b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "void-server", - "version": "2.3.0", + "version": "2.4.0", "type": "module", "private": true, "scripts": { diff --git a/public/style.css b/public/style.css index f35bf96..06e9868 100644 --- a/public/style.css +++ b/public/style.css @@ -635,3 +635,14 @@ body.drawer-open #scrim { opacity: 1; pointer-events: auto; } .sv-tray-chip:hover { border-color: var(--accent); color: var(--accent); } .hidden { display: none !important; } + +/* Storage card (sv-cluster container) — warn dot + capacity meter + subheader */ +.sv-cluster .status-warn .dot { background: var(--warn); box-shadow: 0 0 7px var(--warn); } +.sv-cluster .ok { color: var(--ok); } +.sv-cluster .bad { color: var(--bad); } +.sv-cluster .st-meter { height: 3px; background: var(--accent-soft); border-radius: 2px; margin: 3px 0 9px; overflow: hidden; } +.sv-cluster .st-fill { height: 100%; border-radius: 2px; } +.sv-cluster .st-fill.ok { background: var(--ok); } +.sv-cluster .st-fill.warn { background: var(--warn); } +.sv-cluster .st-fill.bad { background: var(--bad); } +.sv-cluster .sv-subhdr { color: var(--muted); font-size: 10px; text-transform: uppercase; letter-spacing: .08em; margin: 11px 0 5px; font-family: var(--font-mono); } diff --git a/public/views/cards/storage.js b/public/views/cards/storage.js new file mode 100644 index 0000000..221397f --- /dev/null +++ b/public/views/cards/storage.js @@ -0,0 +1,62 @@ +// public/views/cards/storage.js — Proxmox storage health: ZFS pools, dropped pools, +// and per-container disk fill. Surfaces the two failure modes that have actually bitten +// this homelab (a pool dropping off the SATA bus; a container rootfs filling up). +import { el, mount } from '../../dom.js'; +import { api } from '../../api.js'; + +let body, timer; + +const gb = b => (b >= 1e12 ? (b / 1e12).toFixed(1) + 'T' : Math.round(b / 1e9) + 'G'); +const cls = s => (s === 'crit' ? 'bad' : s === 'warn' ? 'warn' : 'ok'); +const dotClass = s => 'status-' + (s === 'crit' ? 'down' : s === 'warn' ? 'warn' : 'ok'); + +function meterRow(label, value, p, status) { + const wrap = el('div', { class: dotClass(status) }); + wrap.appendChild(el('div', { class: 'sv-row' }, + el('span', { class: 'k' }, el('span', { class: 'dot' }), label), + el('span', { class: cls(status) }, value))); + if (p != null) { + wrap.appendChild(el('div', { class: 'st-meter' }, + el('div', { class: 'st-fill ' + cls(status), style: { width: Math.min(p, 100) + '%' } }))); + } + return wrap; +} + +async function load() { + if (!body) return; + try { + const s = await api.get('/api/storage'); + if (s.error) { mount(body, el('span', { class: 'muted' }, 'Storage: ' + s.error)); return; } + + const kids = []; + // overall badge + kids.push(el('div', { class: 'sv-row' }, + el('span', { class: 'k' }, 'Status'), + el('span', { class: 'cl-badge ' + (s.worst === 'ok' ? 'ok' : 'bad') }, + s.worst === 'ok' ? 'HEALTHY' : s.worst === 'warn' ? 'WATCH' : 'ATTENTION'))); + + // dropped pools first (most urgent — e.g. donatello/leonardo off the bus) + for (const d of (s.down || [])) + kids.push(meterRow(d.name + ' · ' + d.node, '⚠ ' + String(d.state).toUpperCase(), null, 'crit')); + + // imported ZFS pools + for (const p of (s.pools || [])) + kids.push(meterRow(p.name + ' · ' + p.node, + (p.health !== 'ONLINE' ? p.health + ' · ' : '') + (p.pct ?? '–') + '%', p.pct, p.status)); + + // container disk fill (top few by %) + const top = (s.guests || []).slice(0, 5); + if (top.length) kids.push(el('div', { class: 'sv-subhdr' }, 'Container disk')); + for (const g of top) + kids.push(meterRow('CT ' + g.vmid + ' ' + g.name, g.pct + '% · ' + gb(g.used) + '/' + gb(g.total), g.pct, g.status)); + + mount(body, el('div', { class: 'sv-cluster' }, ...kids)); + } catch { mount(body, el('span', { class: 'muted' }, 'Storage unavailable')); } +} + +export default { + id: 'storage', title: 'Storage · capacity', size: 'm', + mount(e) { body = e; load(); }, + start() { timer = setInterval(load, 30000); }, + stop() { clearInterval(timer); body = null; } +}; diff --git a/public/views/sacred_valley.js b/public/views/sacred_valley.js index 655c638..36e406b 100644 --- a/public/views/sacred_valley.js +++ b/public/views/sacred_valley.js @@ -14,8 +14,9 @@ import search from './cards/search.js'; import speedtest from './cards/speedtest.js'; import aiUsage from './cards/ai_usage.js'; import cluster from './cards/cluster.js'; +import storage from './cards/storage.js'; -const CARD_MODULES = [clock, weather, hostPerf, cluster, jobs, inbox, search, speedtest, aiUsage]; +const CARD_MODULES = [clock, weather, hostPerf, cluster, storage, jobs, inbox, search, speedtest, aiUsage]; const BY_ID = new Map(CARD_MODULES.map(d => [d.id, d])); let active = []; // mounted cards needing stop() diff --git a/server.js b/server.js index 981a77a..fe58301 100644 --- a/server.js +++ b/server.js @@ -14,7 +14,7 @@ import { mcpAuth } from './lib/api/middleware/mcp_auth.js'; import { handleMcp } from './lib/mcp/http.js'; import httpProxy from 'http-proxy'; -const VERSION = '2.3.0'; +const VERSION = '2.4.0'; // Proxy /terminal (+ its WebSocket) to ttyd on CT 300, so the embedded terminal // works whether the Void is reached via Traefik (void2-app.hynesy.com) OR the diff --git a/tests/proxmox/storage.test.js b/tests/proxmox/storage.test.js new file mode 100644 index 0000000..2ef43b6 --- /dev/null +++ b/tests/proxmox/storage.test.js @@ -0,0 +1,70 @@ +import { describe, it, expect } from 'vitest'; +import { normalizeStorage, storageHealth } from '../../lib/proxmox/storage.js'; + +// Fixtures mirror real PVE payload shapes from this cluster. +const STORAGE = [ + { storage: 'localzfs', node: 'z', status: 'available', plugintype: 'zfspool', disk: 37e9, maxdisk: 516e9 }, + { storage: 'donatello-vm', node: 'z', status: 'unknown', plugintype: 'zfspool', disk: 0, maxdisk: 0 }, + { storage: 'leonardo-vm', node: 'z', status: 'unknown', plugintype: 'zfspool', disk: 0, maxdisk: 0 }, + { storage: 'local', node: 'z', status: 'available', plugintype: 'dir', disk: 1e9, maxdisk: 100e9 } +]; +const VMS = [ + { vmid: 100, name: 'mediastack', type: 'lxc', node: 'z', disk: 60e9, maxdisk: 63e9, status: 'running' }, // 95% + { vmid: 311, name: 'void-app', type: 'lxc', node: 'z', disk: 4e9, maxdisk: 16e9, status: 'running' }, // 25% + { vmid: 200, name: 'OpenClaw', type: 'qemu', node: 'z', disk: 0, maxdisk: 32e9, status: 'running' } // skipped (qemu/0) +]; +const ZFS = { z: [{ name: 'localzfs', health: 'ONLINE', alloc: 37e9, size: 516e9, frag: 6 }] }; + +describe('normalizeStorage', () => { + it('flags a dropped zfspool, a hot container, and rolls up worst=crit', () => { + const r = normalizeStorage(STORAGE, VMS, ZFS); + // dropped pools (donatello/leonardo) surface in `down` + expect(r.down.map(d => d.name).sort()).toEqual(['donatello-vm', 'leonardo-vm']); + expect(r.down.every(d => d.status === 'crit')).toBe(true); + // imported pool present + healthy + expect(r.pools).toHaveLength(1); + expect(r.pools[0].name).toBe('localzfs'); + // guests: qemu/0 skipped, sorted desc, CT100 at 95% is crit + expect(r.guests.map(g => g.vmid)).toEqual([100, 311]); + expect(r.guests[0].pct).toBe(95); + expect(r.guests[0].status).toBe('crit'); + expect(r.worst).toBe('crit'); + expect(r.alerts.some(a => a.includes('donatello-vm'))).toBe(true); + expect(r.alerts.some(a => a.includes('CT 100'))).toBe(true); + }); + + it('all-healthy rolls up to ok', () => { + const r = normalizeStorage( + [{ storage: 'localzfs', node: 'z', status: 'available', plugintype: 'zfspool' }], + [{ vmid: 311, name: 'void-app', type: 'lxc', node: 'z', disk: 4e9, maxdisk: 16e9 }], + { z: [{ name: 'localzfs', health: 'ONLINE', alloc: 37e9, size: 516e9 }] } + ); + expect(r.worst).toBe('ok'); + expect(r.down).toHaveLength(0); + expect(r.alerts).toHaveLength(0); + }); +}); + +describe('storageHealth', () => { + it('returns proxmox_not_configured without a token', async () => { + const r = await storageHealth({ apiUrl: '', token: '' }); + expect(r.error).toBe('proxmox_not_configured'); + }); + + it('fetches + normalizes via injected fetch', async () => { + const fetchImpl = async (url) => ({ + ok: true, + json: async () => { + if (url.includes('type=storage')) return { data: STORAGE }; + if (url.includes('type=vm')) return { data: VMS }; + if (url.includes('/nodes/z/disks/zfs')) return { data: ZFS.z }; + if (url.endsWith('/nodes')) return { data: [{ node: 'z', status: 'online' }] }; + return { data: [] }; + } + }); + const r = await storageHealth({ apiUrl: 'https://pve:8006', token: 'tok', fetchImpl }); + expect(r.worst).toBe('crit'); + expect(r.down).toHaveLength(2); + expect(typeof r.at).toBe('number'); + }); +});