feat(sv): Storage · capacity card — ZFS pools, dropped pools, per-CT disk

Read-only Proxmox storage health (same PROXMOX_RO_TOKEN as the cluster card): ZFS pool health+usage, dropped zfspool storages (the donatello/leonardo SATA signal), and per-LXC rootfs fill, with a HEALTHY/WATCH/ATTENTION roll-up. Closes the monitoring gap from the 2026-06-09 audit (C1 + H2 were invisible). Pure normalizeStorage() unit-tested (4 tests). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-09 03:27:15 +10:00
parent 91a45b4b6c
commit 1b960ec52b
10 changed files with 264 additions and 3 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,6 +3,10 @@
 All notable changes to Void 2.0 are documented here.
 Format: [Keep a Changelog](https://keepachangelog.com).

+## 2.4.0 — Storage · capacity card (Sacred Valley)
+- **New "Storage · capacity" card** (`public/views/cards/storage.js`, `/api/storage`, `lib/proxmox/storage.js`) — read-only Proxmox health via the same `PROXMOX_RO_TOKEN` as the cluster card. Shows: **ZFS pools** (health + usage meter), **dropped pools** (a configured zfspool storage that's no longer `available` — the donatello/leonardo SATA-bus signal, rendered red), and **per-container disk fill** (top LXC by rootfs %), with a HEALTHY/WATCH/ATTENTION roll-up badge. Thresholds: 80% warn, 90% crit; a non-ONLINE or dropped pool is always crit.
+- Closes the monitoring gap from the 2026-06-09 audit (the Void couldn't previously see C1 = pools offline or H2 = a container at 95%). Pure `normalizeStorage()` is unit-tested.
+
 ## 2.3.0 — MagicMirror² as a Void app
 - **New "MagicMirror" Apps view** (`#/mirror`, `public/views/mirror.js`) — embeds the smart-mirror dashboard (CT 111) via the shared `embedView` factory, like Timelapse / AI Usage.
 - **Exposure:** MagicMirror (LAN-only `192.168.1.224:8080`) is now published at **mirror.hynesy.com** through Traefik + the `*.hynesy.com` tunnel, private behind **CF Access** (Farm policy / Google IdP). A Traefik `mirror-frame` middleware replaces MM's `X-Frame-Options: SAMEORIGIN` with a CSP `frame-ancestors` allowing the Void origins so the iframe renders.
--- a/lib/api/index.js
+++ b/lib/api/index.js
@@ -34,6 +34,7 @@ import { router as littleblueRouter } from './routes/littleblue.js';
 import { router as aiUsageRouter } from './routes/ai_usage.js';
 import { router as infraRouter } from './routes/infra.js';
 import { router as clusterRouter } from './routes/cluster.js';
+import { router as storageRouter } from './routes/storage.js';
 import { router as kuttRouter } from './routes/kutt.js';

 export function mountApi(app) {
@@ -50,6 +51,7 @@ export function mountApi(app) {
  api.use('/actions', actionsRouter);
  api.use('/infra', infraRouter);
  api.use('/cluster', clusterRouter);
+  api.use('/storage', storageRouter);
  api.use('/little-blue', littleblueRouter);
  api.use('/ai-usage', aiUsageRouter);
  api.use('/projects', projectsRouter);
--- a/lib/api/routes/storage.js
+++ b/lib/api/routes/storage.js
@@ -0,0 +1,17 @@
+import { Router } from 'express';
+import { asyncWrap } from '../errors.js';
+import { storageHealth } from '../../proxmox/storage.js';
+
+// Read-only storage/capacity health for the Sacred Valley card. Cached briefly so
+// multiple polling clients coalesce into one set of PVE calls. Owner or any authed agent.
+export const router = Router();
+
+let cache = { at: 0, data: null };
+const TTL = 15_000;
+
+router.get('/', asyncWrap(async (_req, res) => {
+  if (cache.data && Date.now() - cache.at < TTL) return res.json(cache.data);
+  const data = await storageHealth();
+  cache = { at: Date.now(), data };
+  res.json(data);
+}));
--- a/lib/proxmox/storage.js
+++ b/lib/proxmox/storage.js
@@ -0,0 +1,94 @@
+import { Agent } from 'undici';
+
+// Read-only Proxmox storage + capacity health for the Sacred Valley card. Same
+// PVEAuditor token as the cluster card (PROXMOX_RO_TOKEN). Surfaces the two things
+// that have actually bitten this homelab and were previously invisible:
+//   1. a ZFS pool dropping out (the donatello/leonardo SATA-bus incident) — seen as
+//      a zfspool storage whose status is no longer 'available'.
+//   2. a container rootfs filling up (mediastack hitting 95%) — per-LXC disk/maxdisk.
+
+let insecure;
+function tlsDispatcher() {
+  if (process.env.PROXMOX_INSECURE_TLS !== '1') return undefined;
+  insecure ??= new Agent({ connect: { rejectUnauthorized: false } });
+  return insecure;
+}
+
+async function pveGet(path, { apiUrl, token, fetchImpl = fetch }) {
+  const res = await fetchImpl(`${apiUrl}/api2/json${path}`, {
+    headers: { Authorization: `PVEAPIToken=${token}` },
+    dispatcher: tlsDispatcher()
+  });
+  if (!res.ok) throw new Error(`pve ${path} -> ${res.status}`);
+  return (await res.json())?.data ?? [];
+}
+
+export const WARN = 80, CRIT = 90;
+const pct = (used, total) => (total > 0 ? Math.round((used / total) * 100) : null);
+const sev = p => (p == null ? 'ok' : p >= CRIT ? 'crit' : p >= WARN ? 'warn' : 'ok');
+const worstOf = items => items.reduce(
+  (w, x) => (x.status === 'crit' || w === 'crit') ? 'crit' : (x.status === 'warn' || w === 'warn') ? 'warn' : 'ok', 'ok');
+
+// Pure: fold /nodes/*/disks/zfs + /cluster/resources(storage,vm) into the card shape.
+export function normalizeStorage(storageRes = [], vmRes = [], zfsByNode = {}) {
+  // Imported ZFS pools (health + usage)
+  const pools = [];
+  for (const [node, list] of Object.entries(zfsByNode)) {
+    for (const z of (list || [])) {
+      const p = pct(z.alloc, z.size);
+      pools.push({
+        name: z.name, node, health: z.health, used: z.alloc, total: z.size, pct: p,
+        status: z.health !== 'ONLINE' ? 'crit' : sev(p)
+      });
+    }
+  }
+  pools.sort((a, b) => a.name.localeCompare(b.name) || a.node.localeCompare(b.node));
+
+  // zfspool storages that are configured but NOT available = a pool that has dropped
+  // out (or never imported). This is the donatello/leonardo signal.
+  const down = storageRes
+    .filter(s => s.plugintype === 'zfspool' && s.status !== 'available')
+    .map(s => ({ name: s.storage, node: s.node, state: s.status || 'unavailable', status: 'crit' }))
+    .sort((a, b) => a.name.localeCompare(b.name) || a.node.localeCompare(b.node));
+
+  // Per-guest rootfs fill. LXC report disk/maxdisk; QEMU usually report disk=0
+  // (no agent) so they're skipped rather than shown as 0%.
+  const guests = vmRes
+    .filter(v => v.type === 'lxc' && v.maxdisk > 0 && v.disk > 0)
+    .map(v => {
+      const p = pct(v.disk, v.maxdisk);
+      return { vmid: v.vmid, name: v.name, node: v.node, used: v.disk, total: v.maxdisk, pct: p, status: sev(p) };
+    })
+    .sort((a, b) => b.pct - a.pct);
+
+  const alerts = [
+    ...down.map(d => `${d.name} (${d.node}) ${d.state}`),
+    ...pools.filter(p => p.health !== 'ONLINE').map(p => `pool ${p.name} ${p.health}`),
+    ...guests.filter(g => g.status !== 'ok').map(g => `CT ${g.vmid} ${g.name} ${g.pct}%`)
+  ];
+
+  return { worst: worstOf([...pools, ...down, ...guests]), pools, down, guests, alerts };
+}
+
+export async function storageHealth(opts = {}) {
+  const cfg = {
+    apiUrl: opts.apiUrl || process.env.PROXMOX_API_URL,
+    token: opts.token || process.env.PROXMOX_RO_TOKEN || process.env.PROXMOX_API_TOKEN,
+    fetchImpl: opts.fetchImpl || fetch
+  };
+  if (!cfg.apiUrl || !cfg.token) return { error: 'proxmox_not_configured', at: Date.now() };
+  try {
+    const [storageRes, vmRes, nodes] = await Promise.all([
+      pveGet('/cluster/resources?type=storage', cfg),
+      pveGet('/cluster/resources?type=vm', cfg),
+      pveGet('/nodes', cfg)
+    ]);
+    const zfsByNode = {};
+    await Promise.all((nodes || [])
+      .filter(n => n.status === 'online')
+      .map(async n => { zfsByNode[n.node] = await pveGet(`/nodes/${n.node}/disks/zfs`, cfg).catch(() => []); }));
+    return { ...normalizeStorage(storageRes, vmRes, zfsByNode), at: Date.now() };
+  } catch (e) {
+    return { error: String(e.message || e), at: Date.now() };
+  }
+}
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
  "name": "void-server",
-  "version": "2.3.0",
+  "version": "2.4.0",
  "type": "module",
  "private": true,
  "scripts": {
--- a/public/style.css
+++ b/public/style.css
@@ -635,3 +635,14 @@ body.drawer-open #scrim { opacity: 1; pointer-events: auto; }
 .sv-tray-chip:hover { border-color: var(--accent); color: var(--accent); }

 .hidden { display: none !important; }
+
+/* Storage card (sv-cluster container) — warn dot + capacity meter + subheader */
+.sv-cluster .status-warn .dot { background: var(--warn); box-shadow: 0 0 7px var(--warn); }
+.sv-cluster .ok  { color: var(--ok); }
+.sv-cluster .bad { color: var(--bad); }
+.sv-cluster .st-meter { height: 3px; background: var(--accent-soft); border-radius: 2px; margin: 3px 0 9px; overflow: hidden; }
+.sv-cluster .st-fill { height: 100%; border-radius: 2px; }
+.sv-cluster .st-fill.ok  { background: var(--ok); }
+.sv-cluster .st-fill.warn { background: var(--warn); }
+.sv-cluster .st-fill.bad { background: var(--bad); }
+.sv-cluster .sv-subhdr { color: var(--muted); font-size: 10px; text-transform: uppercase; letter-spacing: .08em; margin: 11px 0 5px; font-family: var(--font-mono); }
--- a/public/views/cards/storage.js
+++ b/public/views/cards/storage.js
@@ -0,0 +1,62 @@
+// public/views/cards/storage.js — Proxmox storage health: ZFS pools, dropped pools,
+// and per-container disk fill. Surfaces the two failure modes that have actually bitten
+// this homelab (a pool dropping off the SATA bus; a container rootfs filling up).
+import { el, mount } from '../../dom.js';
+import { api } from '../../api.js';
+
+let body, timer;
+
+const gb = b => (b >= 1e12 ? (b / 1e12).toFixed(1) + 'T' : Math.round(b / 1e9) + 'G');
+const cls = s => (s === 'crit' ? 'bad' : s === 'warn' ? 'warn' : 'ok');
+const dotClass = s => 'status-' + (s === 'crit' ? 'down' : s === 'warn' ? 'warn' : 'ok');
+
+function meterRow(label, value, p, status) {
+  const wrap = el('div', { class: dotClass(status) });
+  wrap.appendChild(el('div', { class: 'sv-row' },
+    el('span', { class: 'k' }, el('span', { class: 'dot' }), label),
+    el('span', { class: cls(status) }, value)));
+  if (p != null) {
+    wrap.appendChild(el('div', { class: 'st-meter' },
+      el('div', { class: 'st-fill ' + cls(status), style: { width: Math.min(p, 100) + '%' } })));
+  }
+  return wrap;
+}
+
+async function load() {
+  if (!body) return;
+  try {
+    const s = await api.get('/api/storage');
+    if (s.error) { mount(body, el('span', { class: 'muted' }, 'Storage: ' + s.error)); return; }
+
+    const kids = [];
+    // overall badge
+    kids.push(el('div', { class: 'sv-row' },
+      el('span', { class: 'k' }, 'Status'),
+      el('span', { class: 'cl-badge ' + (s.worst === 'ok' ? 'ok' : 'bad') },
+        s.worst === 'ok' ? 'HEALTHY' : s.worst === 'warn' ? 'WATCH' : 'ATTENTION')));
+
+    // dropped pools first (most urgent — e.g. donatello/leonardo off the bus)
+    for (const d of (s.down || []))
+      kids.push(meterRow(d.name + ' · ' + d.node, '⚠ ' + String(d.state).toUpperCase(), null, 'crit'));
+
+    // imported ZFS pools
+    for (const p of (s.pools || []))
+      kids.push(meterRow(p.name + ' · ' + p.node,
+        (p.health !== 'ONLINE' ? p.health + ' · ' : '') + (p.pct ?? '–') + '%', p.pct, p.status));
+
+    // container disk fill (top few by %)
+    const top = (s.guests || []).slice(0, 5);
+    if (top.length) kids.push(el('div', { class: 'sv-subhdr' }, 'Container disk'));
+    for (const g of top)
+      kids.push(meterRow('CT ' + g.vmid + ' ' + g.name, g.pct + '% · ' + gb(g.used) + '/' + gb(g.total), g.pct, g.status));
+
+    mount(body, el('div', { class: 'sv-cluster' }, ...kids));
+  } catch { mount(body, el('span', { class: 'muted' }, 'Storage unavailable')); }
+}
+
+export default {
+  id: 'storage', title: 'Storage · capacity', size: 'm',
+  mount(e) { body = e; load(); },
+  start() { timer = setInterval(load, 30000); },
+  stop() { clearInterval(timer); body = null; }
+};
--- a/public/views/sacred_valley.js
+++ b/public/views/sacred_valley.js
@@ -14,8 +14,9 @@ import search from './cards/search.js';
 import speedtest from './cards/speedtest.js';
 import aiUsage from './cards/ai_usage.js';
 import cluster from './cards/cluster.js';
+import storage from './cards/storage.js';

-const CARD_MODULES = [clock, weather, hostPerf, cluster, jobs, inbox, search, speedtest, aiUsage];
+const CARD_MODULES = [clock, weather, hostPerf, cluster, storage, jobs, inbox, search, speedtest, aiUsage];
 const BY_ID = new Map(CARD_MODULES.map(d => [d.id, d]));

 let active = [];                 // mounted cards needing stop()
--- a/server.js
+++ b/server.js
@@ -14,7 +14,7 @@ import { mcpAuth } from './lib/api/middleware/mcp_auth.js';
 import { handleMcp } from './lib/mcp/http.js';
 import httpProxy from 'http-proxy';

-const VERSION = '2.3.0';
+const VERSION = '2.4.0';

 // Proxy /terminal (+ its WebSocket) to ttyd on CT 300, so the embedded terminal
 // works whether the Void is reached via Traefik (void2-app.hynesy.com) OR the
--- a/tests/proxmox/storage.test.js
+++ b/tests/proxmox/storage.test.js
@@ -0,0 +1,70 @@
+import { describe, it, expect } from 'vitest';
+import { normalizeStorage, storageHealth } from '../../lib/proxmox/storage.js';
+
+// Fixtures mirror real PVE payload shapes from this cluster.
+const STORAGE = [
+  { storage: 'localzfs', node: 'z', status: 'available', plugintype: 'zfspool', disk: 37e9, maxdisk: 516e9 },
+  { storage: 'donatello-vm', node: 'z', status: 'unknown', plugintype: 'zfspool', disk: 0, maxdisk: 0 },
+  { storage: 'leonardo-vm', node: 'z', status: 'unknown', plugintype: 'zfspool', disk: 0, maxdisk: 0 },
+  { storage: 'local', node: 'z', status: 'available', plugintype: 'dir', disk: 1e9, maxdisk: 100e9 }
+];
+const VMS = [
+  { vmid: 100, name: 'mediastack', type: 'lxc', node: 'z', disk: 60e9, maxdisk: 63e9, status: 'running' }, // 95%
+  { vmid: 311, name: 'void-app',  type: 'lxc', node: 'z', disk: 4e9,  maxdisk: 16e9, status: 'running' }, // 25%
+  { vmid: 200, name: 'OpenClaw',  type: 'qemu', node: 'z', disk: 0,   maxdisk: 32e9, status: 'running' }  // skipped (qemu/0)
+];
+const ZFS = { z: [{ name: 'localzfs', health: 'ONLINE', alloc: 37e9, size: 516e9, frag: 6 }] };
+
+describe('normalizeStorage', () => {
+  it('flags a dropped zfspool, a hot container, and rolls up worst=crit', () => {
+    const r = normalizeStorage(STORAGE, VMS, ZFS);
+    // dropped pools (donatello/leonardo) surface in `down`
+    expect(r.down.map(d => d.name).sort()).toEqual(['donatello-vm', 'leonardo-vm']);
+    expect(r.down.every(d => d.status === 'crit')).toBe(true);
+    // imported pool present + healthy
+    expect(r.pools).toHaveLength(1);
+    expect(r.pools[0].name).toBe('localzfs');
+    // guests: qemu/0 skipped, sorted desc, CT100 at 95% is crit
+    expect(r.guests.map(g => g.vmid)).toEqual([100, 311]);
+    expect(r.guests[0].pct).toBe(95);
+    expect(r.guests[0].status).toBe('crit');
+    expect(r.worst).toBe('crit');
+    expect(r.alerts.some(a => a.includes('donatello-vm'))).toBe(true);
+    expect(r.alerts.some(a => a.includes('CT 100'))).toBe(true);
+  });
+
+  it('all-healthy rolls up to ok', () => {
+    const r = normalizeStorage(
+      [{ storage: 'localzfs', node: 'z', status: 'available', plugintype: 'zfspool' }],
+      [{ vmid: 311, name: 'void-app', type: 'lxc', node: 'z', disk: 4e9, maxdisk: 16e9 }],
+      { z: [{ name: 'localzfs', health: 'ONLINE', alloc: 37e9, size: 516e9 }] }
+    );
+    expect(r.worst).toBe('ok');
+    expect(r.down).toHaveLength(0);
+    expect(r.alerts).toHaveLength(0);
+  });
+});
+
+describe('storageHealth', () => {
+  it('returns proxmox_not_configured without a token', async () => {
+    const r = await storageHealth({ apiUrl: '', token: '' });
+    expect(r.error).toBe('proxmox_not_configured');
+  });
+
+  it('fetches + normalizes via injected fetch', async () => {
+    const fetchImpl = async (url) => ({
+      ok: true,
+      json: async () => {
+        if (url.includes('type=storage')) return { data: STORAGE };
+        if (url.includes('type=vm')) return { data: VMS };
+        if (url.includes('/nodes/z/disks/zfs')) return { data: ZFS.z };
+        if (url.endsWith('/nodes')) return { data: [{ node: 'z', status: 'online' }] };
+        return { data: [] };
+      }
+    });
+    const r = await storageHealth({ apiUrl: 'https://pve:8006', token: 'tok', fetchImpl });
+    expect(r.worst).toBe('crit');
+    expect(r.down).toHaveLength(2);
+    expect(typeof r.at).toBe('number');
+  });
+});