feat(memory): opt-in persistent memory scaffold for providers

Adds a provider capability (usesMemoryScaffold) and a container-side boot
scaffold that materializes a persistent memory/ tree for providers that opt
in. Dormant for the default provider — the scaffold is only built when a
provider declares the capability, so existing installs are byte-identical
(asserted by a boot-gate wiring test).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

container/agent-runner/src/index.ts

@@ -27,6 +27,7 @@ import { fileURLToPath } from 'url';
 
 import { loadConfig } from './config.js';
 import { buildSystemPromptAddendum } from './destinations.js';
+import { ensureMemoryScaffold } from './memory-scaffold.js';
 // Providers barrel — each enabled provider self-registers on import.
 // Provider skills append imports to providers/index.ts.
 import './providers/index.js';
@@ -95,6 +96,12 @@ async function main(): Promise<void> {
     effort: config.effort,
   });
 
+  // Providers that lack native memory opt in via `usesMemoryScaffold`; for them
+  // the runner creates a persistent memory/ tree in its host-backed workspace at
+  // boot (idempotent). Default off — the trunk default (Claude) omits the flag
+  // and keeps its native memory untouched.
+  if (provider.usesMemoryScaffold) ensureMemoryScaffold();
+
   await runPollLoop({
     provider,
     providerName,

container/agent-runner/src/memory-scaffold.test.ts

@@ -0,0 +1,37 @@
+import { describe, expect, it } from 'bun:test';
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+
+import { ensureMemoryScaffold } from './memory-scaffold.js';
+
+describe('ensureMemoryScaffold', () => {
+  it('deterministically creates the memory tree', () => {
+    const base = fs.mkdtempSync(path.join(os.tmpdir(), 'nanoclaw-mem-'));
+    try {
+      ensureMemoryScaffold(base);
+
+      expect(fs.existsSync(path.join(base, 'memory', 'index.md'))).toBe(true);
+      expect(fs.existsSync(path.join(base, 'memory', 'system', 'definition.md'))).toBe(true);
+      expect(fs.existsSync(path.join(base, 'memory', 'memories'))).toBe(true);
+      expect(fs.existsSync(path.join(base, 'memory', 'data'))).toBe(true);
+    } finally {
+      fs.rmSync(base, { recursive: true, force: true });
+    }
+  });
+
+  it('is idempotent and never clobbers the agent edits', () => {
+    const base = fs.mkdtempSync(path.join(os.tmpdir(), 'nanoclaw-mem-'));
+    try {
+      ensureMemoryScaffold(base);
+      const indexFile = path.join(base, 'memory', 'index.md');
+      fs.writeFileSync(indexFile, '# my own index\n');
+
+      ensureMemoryScaffold(base);
+
+      expect(fs.readFileSync(indexFile, 'utf-8')).toBe('# my own index\n');
+    } finally {
+      fs.rmSync(base, { recursive: true, force: true });
+    }
+  });
+});

container/agent-runner/src/memory-scaffold.ts

@@ -0,0 +1,39 @@
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+/**
+ * Create the agent's persistent memory scaffold, container-side, at boot.
+ *
+ * The runner owns its own workspace: it writes the memory tree straight into
+ * `/workspace/agent` (the host-backed, RW group dir, so it persists across the
+ * ephemeral container). No host-side step, nothing mounted in.
+ *
+ * The default `definition.md` / `index.md` live as real markdown templates next
+ * to this module (under `memory-templates/`) — not as strings in code — so the
+ * doctrine is editable as markdown and the agent receives an unescaped copy.
+ * They ship in the mounted `/app/src` tree, so no image change is needed.
+ *
+ * Idempotent — only writes what's missing, so the agent's own edits and
+ * accumulated memory are never clobbered on a later wake. Provider-agnostic:
+ * the runner makes no assumption about which harness is running — a provider
+ * opts in via `usesMemoryScaffold`.
+ */
+const TEMPLATES_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), 'memory-templates');
+
+export function ensureMemoryScaffold(baseDir = '/workspace/agent'): void {
+  const memoryDir = path.join(baseDir, 'memory');
+  const systemDir = path.join(memoryDir, 'system');
+
+  for (const dir of [systemDir, path.join(memoryDir, 'memories'), path.join(memoryDir, 'data')]) {
+    fs.mkdirSync(dir, { recursive: true });
+  }
+
+  copyTemplateIfMissing('definition.md', path.join(systemDir, 'definition.md'));
+  copyTemplateIfMissing('index.md', path.join(memoryDir, 'index.md'));
+}
+
+function copyTemplateIfMissing(template: string, dest: string): void {
+  if (fs.existsSync(dest)) return;
+  fs.copyFileSync(path.join(TEMPLATES_DIR, template), dest);
+}

container/agent-runner/src/memory-scaffold.wiring.test.ts

@@ -0,0 +1,22 @@
+import { describe, expect, it } from 'bun:test';
+import fs from 'fs';
+import path from 'path';
+
+// Wiring guard for the memory-scaffold seam: the boot gate in index.ts
+// (`if (provider.usesMemoryScaffold) ensureMemoryScaffold()`) is the seam's
+// single functional reach-in. The unit tests in memory-scaffold.test.ts drive
+// ensureMemoryScaffold directly and stay green if the gate is deleted — this
+// test goes red. main() can't be driven in-process (it reads
+// /workspace/agent/container.json and enters the poll loop), so the guard is
+// structural: gate + import must both be present in the real entry point.
+describe('memory scaffold boot wiring', () => {
+  const indexSrc = fs.readFileSync(path.join(import.meta.dir, 'index.ts'), 'utf-8');
+
+  it('gates the scaffold on the provider capability in main()', () => {
+    expect(indexSrc).toContain('if (provider.usesMemoryScaffold) ensureMemoryScaffold()');
+  });
+
+  it('imports ensureMemoryScaffold from the seam module', () => {
+    expect(indexSrc).toContain("import { ensureMemoryScaffold } from './memory-scaffold.js'");
+  });
+});

container/agent-runner/src/memory-templates/definition.md

@@ -0,0 +1,23 @@
+# Agent Memory System
+
+This editable file defines how your persistent memory works. It is a starting
+point, not a contract — reorganize it as the work demands. If the user or another
+memory system replaces this definition, follow the replacement.
+
+Start every memory task at `memory/index.md`, then follow the narrowest relevant index.
+Treat indexes as core data: keep them accurate and concise.
+Every folder of durable memory has its own `index.md` describing its contents.
+When an index grows past roughly 20 entries, group related items into subfolders,
+and give each new subfolder its own `index.md` linked from the parent.
+
+Use `memory/memories/` for durable facts, project context, people, decisions, and entity notes.
+Use `memory/data/` for structured reference data, datasets, tables, and reusable records.
+Use entity folders for things that matter: projects, people, places, organizations, decisions.
+
+When the user shares something that should survive future turns, store it in the
+smallest useful file; prefer updating an existing file over creating duplicates.
+Write concise, source-aware notes; include dates when timing matters.
+If a fact is corrected, update the memory and keep only useful history.
+When you add, move, or remove memory, update the nearest index.
+Before answering from memory, read the relevant index or file instead of guessing;
+if memory is missing or uncertain, say so and verify when it matters.

container/agent-runner/src/memory-templates/index.md

@@ -0,0 +1,5 @@
+# Memory Index
+
+- [Memory system definition](system/definition.md)
+- [Memories](memories/) - durable facts, people, projects, decisions
+- [Data](data/) - structured reference data

container/agent-runner/src/providers/types.ts

@@ -6,6 +6,14 @@ export interface AgentProvider {
    */
   readonly supportsNativeSlashCommands: boolean;
 
+  /**
+   * Optional. When true, the runner scaffolds a persistent `memory/` tree in the
+   * agent's workspace at boot. Providers with their own native memory (e.g.
+   * Claude's `CLAUDE.local.md`) omit this and get nothing — memory is opt-in per
+   * provider, never gated on a provider name.
+   */
+  readonly usesMemoryScaffold?: boolean;
+
   /** Start a new query. Returns a handle for streaming input and output. */
   query(input: QueryInput): AgentQuery;
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "nanoclaw",
-  "version": "2.1.10",
+  "version": "2.1.11",
   "description": "Personal Claude assistant. Lightweight, secure, customizable.",
   "type": "module",
   "packageManager": "pnpm@10.33.0",

src/webhook-server-raw.test.ts

@@ -0,0 +1,97 @@
+/**
+ * Guard for the raw-route half of src/webhook-server.ts —
+ * registerWebhookHandler + the rawRoutes dispatch branch.
+ *
+ * Drives the REAL shared HTTP server on an ephemeral WEBHOOK_PORT (no
+ * mocking of the routing layer): a registered raw route must dispatch,
+ * unknown paths must 404, a throwing handler must surface as 500,
+ * raw routes must coexist with Chat SDK adapter routes on the same
+ * server, and stopWebhookServer must clear them.
+ */
+import { afterAll, describe, expect, it, vi } from 'vitest';
+
+import type { Chat } from 'chat';
+
+import { registerWebhookAdapter, registerWebhookHandler, stopWebhookServer } from './webhook-server.js';
+
+const PORT = 21000 + Math.floor(Math.random() * 20000);
+
+async function post(path: string, body = '{}'): Promise<globalThis.Response> {
+  for (let attempt = 0; ; attempt++) {
+    try {
+      return await fetch(`http://127.0.0.1:${PORT}/webhook/${path}`, { method: 'POST', body });
+    } catch (err) {
+      if (attempt >= 40) throw err;
+      await new Promise((r) => setTimeout(r, 50));
+    }
+  }
+}
+
+afterAll(async () => {
+  await stopWebhookServer();
+  delete process.env.WEBHOOK_PORT;
+});
+
+describe('webhook server raw routes', () => {
+  it('dispatches a registered raw route to its handler', async () => {
+    process.env.WEBHOOK_PORT = String(PORT);
+    const methods: string[] = [];
+    registerWebhookHandler('ping', (req, res) => {
+      methods.push(req.method || '');
+      res.writeHead(200, { 'Content-Type': 'text/plain' });
+      res.end('pong');
+    });
+
+    const res = await post('ping');
+    expect(res.status).toBe(200);
+    expect(await res.text()).toBe('pong');
+    expect(methods).toEqual(['POST']);
+  });
+
+  it('returns 404 for paths with no registered route', async () => {
+    const res = await post('nope');
+    expect(res.status).toBe(404);
+  });
+
+  it('turns a throwing handler into a 500 response', async () => {
+    registerWebhookHandler('boom', () => {
+      throw new Error('handler exploded');
+    });
+
+    const res = await post('boom');
+    expect(res.status).toBe(500);
+    expect(await res.text()).toBe('Internal Server Error');
+  });
+
+  it('coexists with Chat SDK adapter routes on the same server', async () => {
+    const handler = vi.fn(async () => new Response('ok-chat', { status: 200 }));
+    const chat = { webhooks: { fake: handler } } as unknown as Chat;
+    registerWebhookAdapter(chat, 'fake');
+
+    const chatRes = await post('fake');
+    expect(chatRes.status).toBe(200);
+    expect(await chatRes.text()).toBe('ok-chat');
+    expect(handler).toHaveBeenCalledTimes(1);
+
+    // The raw route registered earlier is still live alongside it.
+    const rawRes = await post('ping');
+    expect(rawRes.status).toBe(200);
+  });
+
+  it('clears raw routes on stopWebhookServer', async () => {
+    await stopWebhookServer();
+
+    // Restart the server with a fresh route; the old raw routes must be gone.
+    registerWebhookHandler('fresh', (_req, res) => {
+      res.writeHead(200, { 'Content-Type': 'text/plain' });
+      res.end('fresh');
+    });
+
+    const stale = await post('ping');
+    expect(stale.status).toBe(404);
+
+    const fresh = await post('fresh');
+    expect(fresh.status).toBe(200);
+    expect(await fresh.text()).toBe('fresh');
+  });
+});

src/webhook-server.ts

@@ -3,9 +3,12 @@
  *
  * Starts lazily on first adapter registration. Routes requests by path:
  *   /webhook/{adapterName} → chat.webhooks[adapterName](request)
+ *   /webhook/{path}        → raw handler from registerWebhookHandler(path, ...)
  *
  * Multiple Chat instances can register adapters — each adapter name maps
- * to its owning Chat instance.
+ * to its owning Chat instance. Raw routes let modules receive non-Chat-SDK
+ * webhooks (GitHub, payment providers, health checks) on the same server
+ * without editing this file or opening a second port.
  */
 import http from 'http';
 
@@ -20,7 +23,11 @@ interface WebhookEntry {
   adapterName: string;
 }
 
+/** Node-style handler for raw (non-Chat-SDK) webhook routes. */
+export type RawWebhookHandler = (req: http.IncomingMessage, res: http.ServerResponse) => void | Promise<void>;
+
 const routes = new Map<string, WebhookEntry>();
+const rawRoutes = new Map<string, RawWebhookHandler>();
 let server: http.Server | null = null;
 
 /** Convert Node.js IncomingMessage to a Web API Request. */
@@ -84,6 +91,22 @@ export function registerWebhookAdapter(chat: Chat, adapterName: string, routingP
   log.info('Webhook adapter registered', { adapter: adapterName, path: `/webhook/${routingPath}` });
 }
 
+/**
+ * Register a raw Node-style handler at /webhook/{path} on the shared server.
+ *
+ * For webhooks that don't flow through a Chat SDK adapter (GitHub, payment
+ * providers, health checks): modules register their endpoint here instead of
+ * editing this file or standing up a second HTTP server on another port.
+ * The handler owns the request/response directly.
+ *
+ * Starts the server lazily on first call.
+ */
+export function registerWebhookHandler(path: string, handler: RawWebhookHandler): void {
+  rawRoutes.set(path, handler);
+  ensureServer();
+  log.info('Webhook handler registered', { path: `/webhook/${path}` });
+}
+
 function ensureServer(): void {
   if (server) return;
 
@@ -101,14 +124,22 @@ function ensureServer(): void {
     }
 
     const adapterName = match[1];
-    const entry = routes.get(adapterName);
-    if (!entry) {
-      res.writeHead(404, { 'Content-Type': 'text/plain' });
-      res.end(`Unknown adapter: ${adapterName}`);
-      return;
-    }
 
     try {
+      // Raw routes take priority — the handler writes the response itself.
+      const rawHandler = rawRoutes.get(adapterName);
+      if (rawHandler) {
+        await rawHandler(req, res);
+        return;
+      }
+
+      const entry = routes.get(adapterName);
+      if (!entry) {
+        res.writeHead(404, { 'Content-Type': 'text/plain' });
+        res.end(`Unknown adapter: ${adapterName}`);
+        return;
+      }
+
       const webReq = await toWebRequest(req);
       // eslint-disable-next-line @typescript-eslint/no-explicit-any
       const webhooks = entry.chat.webhooks as Record<string, (r: Request, opts?: any) => Promise<Response>>;
@@ -121,8 +152,10 @@ function ensureServer(): void {
       await fromWebResponse(webRes, res);
     } catch (err) {
       log.error('Webhook handler error', { adapter: adapterName, url: req.url, err });
-      res.writeHead(500, { 'Content-Type': 'text/plain' });
-      res.end('Internal Server Error');
+      if (!res.headersSent) {
+        res.writeHead(500, { 'Content-Type': 'text/plain' });
+        res.end('Internal Server Error');
+      }
     }
   });
 
@@ -137,6 +170,7 @@ export async function stopWebhookServer(): Promise<void> {
     await new Promise<void>((resolve) => server!.close(() => resolve()));
     server = null;
     routes.clear();
+    rawRoutes.clear();
     log.info('Webhook server stopped');
   }
 }