diff --git a/.claude/skills/add-discord/apply-fixtures.json b/.claude/skills/add-discord/apply-fixtures.json new file mode 100644 index 000000000..de84e1eb9 --- /dev/null +++ b/.claude/skills/add-discord/apply-fixtures.json @@ -0,0 +1,15 @@ +{ + "notes": "Conformance fixtures for scripts/skill-conformance.test.ts — shaped fake values only, never real credentials. The applications/@me stub answers the multi-field JSON capture (application_id/.id, public_key/.verify_key, owner_handle/.owner.id); the users/@me/channels stub answers the platform_id capture.", + "scenarios": [ + { + "name": "bot-token", + "inputs": { + "bot_token": "MTA0fake.token.fake_fake-fake.fake0123456789abcdefghijklmnopqrstuvwxyz" + }, + "exec": [ + { "match": "oauth2/applications/@me", "stdout": "{\"id\":\"1234567890\",\"verify_key\":\"abcdef0123456789\",\"owner\":{\"id\":\"111222333444\"}}" }, + { "match": "users/@me/channels", "stdout": "discord:@me:D0FAKE" } + ] + } + ] +} diff --git a/.claude/skills/add-gchat/apply-fixtures.json b/.claude/skills/add-gchat/apply-fixtures.json new file mode 100644 index 000000000..a552ca98c --- /dev/null +++ b/.claude/skills/add-gchat/apply-fixtures.json @@ -0,0 +1,11 @@ +{ + "notes": "Conformance fixtures for scripts/skill-conformance.test.ts — shaped fake values only, never real credentials.", + "scenarios": [ + { + "name": "service-account", + "inputs": { + "gchat_credentials": "fake-service-account-json-blob" + } + } + ] +} diff --git a/.claude/skills/add-github/apply-fixtures.json b/.claude/skills/add-github/apply-fixtures.json new file mode 100644 index 000000000..9d2dbc46d --- /dev/null +++ b/.claude/skills/add-github/apply-fixtures.json @@ -0,0 +1,13 @@ +{ + "notes": "Conformance fixtures for scripts/skill-conformance.test.ts — shaped fake values only, never real credentials.", + "scenarios": [ + { + "name": "pat", + "inputs": { + "github_token": "ghp_fake0123456789fake0123456789fake01", + "webhook_secret": "fake-webhook-secret", + "bot_username": "nanoclaw-bot" + } + } + ] +} diff --git a/.claude/skills/add-imessage/apply-fixtures.json b/.claude/skills/add-imessage/apply-fixtures.json new file mode 100644 index 000000000..4da3dc729 --- /dev/null +++ b/.claude/skills/add-imessage/apply-fixtures.json @@ -0,0 +1,27 @@ +{ + "notes": "Conformance fixtures for scripts/skill-conformance.test.ts — shaped fake values only, never real credentials. Two scenarios cover both when:mode= legs; the echo stub answers the platform_id capture (the skill echoes the owner handle back).", + "scenarios": [ + { + "name": "local", + "inputs": { + "mode": "local", + "owner_handle": "+15551234567" + }, + "exec": [ + { "match": "echo \"+15551234567\"", "stdout": "+15551234567" } + ] + }, + { + "name": "remote", + "inputs": { + "mode": "remote", + "server_url": "https://imsg.example.com", + "api_key": "fake-photon-api-key", + "owner_handle": "owner@example.com" + }, + "exec": [ + { "match": "echo \"owner@example.com\"", "stdout": "owner@example.com" } + ] + } + ] +} diff --git a/.claude/skills/add-linear/apply-fixtures.json b/.claude/skills/add-linear/apply-fixtures.json new file mode 100644 index 000000000..0bfbc2df7 --- /dev/null +++ b/.claude/skills/add-linear/apply-fixtures.json @@ -0,0 +1,17 @@ +{ + "notes": "Conformance fixtures for scripts/skill-conformance.test.ts — shaped fake values only, never real credentials. The effect:wire ncl runs carry no capture, so the exec stub's no-op answer is fine.", + "scenarios": [ + { + "name": "workspace", + "inputs": { + "linear_client_id": "fake-client-id", + "linear_client_secret": "fake-client-secret", + "linear_webhook_secret": "fake-webhook-secret", + "linear_team_key": "ENG", + "linear_bot_username": "nanoclaw", + "agent_folder": "main", + "linear_sender_policy": "public" + } + } + ] +} diff --git a/.claude/skills/add-matrix/apply-fixtures.json b/.claude/skills/add-matrix/apply-fixtures.json new file mode 100644 index 000000000..4575f0be5 --- /dev/null +++ b/.claude/skills/add-matrix/apply-fixtures.json @@ -0,0 +1,16 @@ +{ + "notes": "Conformance fixtures for scripts/skill-conformance.test.ts — shaped fake values only, never real credentials. All prompts in this skill are unguarded (the login-method legs are prose alternatives, not when:-branches), so a fully-programmatic apply must answer every one — username/password AND access_token.", + "scenarios": [ + { + "name": "homeserver", + "inputs": { + "base_url": "https://matrix.example.com", + "user_id": "@nanoclaw:example.com", + "bot_username": "nanoclaw", + "username": "nanoclaw", + "password": "fake-password", + "access_token": "syt_fake_access_token" + } + } + ] +} diff --git a/.claude/skills/add-resend/apply-fixtures.json b/.claude/skills/add-resend/apply-fixtures.json new file mode 100644 index 000000000..aefa28927 --- /dev/null +++ b/.claude/skills/add-resend/apply-fixtures.json @@ -0,0 +1,16 @@ +{ + "notes": "Conformance fixtures for scripts/skill-conformance.test.ts — shaped fake values only, never real credentials. The effect:wire ncl runs carry no capture, so the exec stub's no-op answer is fine.", + "scenarios": [ + { + "name": "email", + "inputs": { + "api_key": "re_fake_api_key", + "webhook_secret": "whsec_fake", + "from_address": "assistant@mail.example.com", + "from_name": "NanoClaw", + "owner_email": "owner@example.com", + "agent_folder": "main" + } + } + ] +} diff --git a/.claude/skills/add-signal/apply-fixtures.json b/.claude/skills/add-signal/apply-fixtures.json new file mode 100644 index 000000000..c7d7d9313 --- /dev/null +++ b/.claude/skills/add-signal/apply-fixtures.json @@ -0,0 +1,10 @@ +{ + "notes": "Conformance fixtures for scripts/skill-conformance.test.ts — no prompts in this skill; the fixture exists to feed the effect:step device-link's terminal-block fields (capture:platform_id=ACCOUNT,owner_handle=ACCOUNT).", + "scenarios": [ + { + "name": "device-link", + "inputs": {}, + "stepFields": { "ACCOUNT": "+15550001111" } + } + ] +} diff --git a/.claude/skills/add-slack/apply-fixtures.json b/.claude/skills/add-slack/apply-fixtures.json new file mode 100644 index 000000000..7592b2de3 --- /dev/null +++ b/.claude/skills/add-slack/apply-fixtures.json @@ -0,0 +1,31 @@ +{ + "notes": "Conformance fixtures for scripts/skill-conformance.test.ts — shaped fake values only, never real credentials. Two scenarios cover both when:connection= legs; the auth.test stub answers the connected_as capture and conversations.open answers platform_id.", + "scenarios": [ + { + "name": "socket", + "inputs": { + "connection": "socket", + "bot_token": "xoxb-fake-bot-token", + "app_token": "xapp-fake-app-token", + "owner_handle": "U12345678" + }, + "exec": [ + { "match": "auth.test", "stdout": "@nano in Acme" }, + { "match": "conversations.open", "stdout": "slack:D0FAKE" } + ] + }, + { + "name": "webhook", + "inputs": { + "connection": "webhook", + "bot_token": "xoxb-fake-bot-token", + "signing_secret": "0123456789abcdef", + "owner_handle": "U12345678" + }, + "exec": [ + { "match": "auth.test", "stdout": "@nano in Acme" }, + { "match": "conversations.open", "stdout": "slack:D0FAKE" } + ] + } + ] +} diff --git a/.claude/skills/add-teams/apply-fixtures.json b/.claude/skills/add-teams/apply-fixtures.json new file mode 100644 index 000000000..e24b0cf3f --- /dev/null +++ b/.claude/skills/add-teams/apply-fixtures.json @@ -0,0 +1,75 @@ +{ + "notes": "Conformance fixtures for scripts/skill-conformance.test.ts — shaped fake values only, never real credentials. Four scenarios: existing-creds drop-through (the have_creds probe answers 'yes'); a fresh create wiring the logged-in owner directly; a fresh create wiring a DIFFERENT account (wire_target=other-account); and a fresh create re-confirming via the logged-in account (wire_target=logged-in-account). The `app create` stub answers the five-field JSON capture; `status --json` answers owner_upn/owner_aad_id; the smba stubs answer the DM-open chain. Guards driven by CAPTURES (have_creds) are selected by exec stdout, not inputs.", + "coverageExclude": [ + "wire_owner=no" + ], + "coverageExcludeReason": "wire_owner=no is exercised at runtime by the other-account and logged-in-account scenarios (their wire_target prompts only run under it), but BOTH of those legs deliberately re-bind wire_owner to 'yes' via capture (SKILL.md ~L257/L261) before the run ends, so no scenario can finish with vars.wire_owner='no'. Final-vars coverage is impossible by design, not missing.", + "scenarios": [ + { + "name": "existing-creds", + "inputs": {}, + "exec": [ + { "match": "TEAMS_APP_ID=.", "stdout": "yes" } + ] + }, + { + "name": "create-wire-owner", + "inputs": { + "public_url": "https://nc.example.com", + "app_name": "NanoClaw Bot", + "wire_owner": "yes", + "signout": "yes" + }, + "exec": [ + { "match": "TEAMS_APP_ID=.", "stdout": "no" }, + { "match": "app create", "stdout": "{\"credentials\":{\"CLIENT_ID\":\"11111111-2222-3333-4444-555555555555\",\"CLIENT_SECRET\":\"fake-client-secret\",\"TENANT_ID\":\"99999999-8888-7777-6666-555555555555\"},\"teamsAppId\":\"aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee\",\"installLink\":\"https://teams.microsoft.com/l/app/fake\"}" }, + { "match": "status --json", "stdout": "{\"username\":\"owner@contoso.com\",\"userObjectId\":\"12345678-1234-1234-1234-123456789abc\"}" }, + { "match": "oauth2/v2.0/token", "stdout": "eyJhbGciOiJIUzI1NiJ9.fake.sig" }, + { "match": "/members", "stdout": "{\"id\":\"29:1fakeownerid\",\"name\":\"Owner\"}" }, + { "match": "v3/conversations\"", "stdout": "a:1fakeconversationid" }, + { "match": "base64url", "stdout": "teams:YTpmYWtl:aHR0cHM6Ly9zbWJh" } + ] + }, + { + "name": "create-wire-other-account", + "inputs": { + "public_url": "https://nc.example.com", + "app_name": "NanoClaw Bot", + "wire_owner": "no", + "wire_target": "other-account", + "target_aad_id": "12345678-1234-1234-1234-123456789abc", + "signout": "no" + }, + "exec": [ + { "match": "TEAMS_APP_ID=.", "stdout": "no" }, + { "match": "app create", "stdout": "{\"credentials\":{\"CLIENT_ID\":\"11111111-2222-3333-4444-555555555555\",\"CLIENT_SECRET\":\"fake-client-secret\",\"TENANT_ID\":\"99999999-8888-7777-6666-555555555555\"},\"teamsAppId\":\"aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee\",\"installLink\":\"https://teams.microsoft.com/l/app/fake\"}" }, + { "match": "status --json", "stdout": "{\"username\":\"owner@contoso.com\",\"userObjectId\":\"12345678-1234-1234-1234-123456789abc\"}" }, + { "match": "printf '{\"aad\"", "stdout": "{\"aad\":\"12345678-1234-1234-1234-123456789abc\",\"wire\":\"yes\"}" }, + { "match": "oauth2/v2.0/token", "stdout": "eyJhbGciOiJIUzI1NiJ9.fake.sig" }, + { "match": "/members", "stdout": "{\"id\":\"29:1faketargetid\",\"name\":\"Target user\"}" }, + { "match": "v3/conversations\"", "stdout": "a:1fakeconversationid" }, + { "match": "base64url", "stdout": "teams:YTpmYWtl:aHR0cHM6Ly9zbWJh" } + ] + }, + { + "name": "create-wire-logged-in-account", + "inputs": { + "public_url": "https://nc.example.com", + "app_name": "NanoClaw Bot", + "wire_owner": "no", + "wire_target": "logged-in-account", + "signout": "no" + }, + "exec": [ + { "match": "TEAMS_APP_ID=.", "stdout": "no" }, + { "match": "app create", "stdout": "{\"credentials\":{\"CLIENT_ID\":\"11111111-2222-3333-4444-555555555555\",\"CLIENT_SECRET\":\"fake-client-secret\",\"TENANT_ID\":\"99999999-8888-7777-6666-555555555555\"},\"teamsAppId\":\"aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee\",\"installLink\":\"https://teams.microsoft.com/l/app/fake\"}" }, + { "match": "status --json", "stdout": "{\"username\":\"owner@contoso.com\",\"userObjectId\":\"12345678-1234-1234-1234-123456789abc\"}" }, + { "match": "echo yes", "stdout": "yes" }, + { "match": "oauth2/v2.0/token", "stdout": "eyJhbGciOiJIUzI1NiJ9.fake.sig" }, + { "match": "/members", "stdout": "{\"id\":\"29:1fakeownerid\",\"name\":\"Owner\"}" }, + { "match": "v3/conversations\"", "stdout": "a:1fakeconversationid" }, + { "match": "base64url", "stdout": "teams:YTpmYWtl:aHR0cHM6Ly9zbWJh" } + ] + } + ] +} diff --git a/.claude/skills/add-telegram/apply-fixtures.json b/.claude/skills/add-telegram/apply-fixtures.json new file mode 100644 index 000000000..1c8093549 --- /dev/null +++ b/.claude/skills/add-telegram/apply-fixtures.json @@ -0,0 +1,15 @@ +{ + "notes": "Conformance fixtures for scripts/skill-conformance.test.ts — shaped fake values only, never real credentials. The getMe stub answers the bot_username capture (the operator block after the restart renders it); stepFields feed the pairing step's terminal block (capture:platform_id=PLATFORM_ID,owner_handle=ADMIN_USER_ID).", + "scenarios": [ + { + "name": "pairing", + "inputs": { + "bot_token": "123456789:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + }, + "exec": [ + { "match": "getMe", "stdout": "nanoclaw_bot" } + ], + "stepFields": { "PLATFORM_ID": "telegram:123456789", "ADMIN_USER_ID": "987654321" } + } + ] +} diff --git a/.claude/skills/add-webex/apply-fixtures.json b/.claude/skills/add-webex/apply-fixtures.json new file mode 100644 index 000000000..13a93a3d2 --- /dev/null +++ b/.claude/skills/add-webex/apply-fixtures.json @@ -0,0 +1,12 @@ +{ + "notes": "Conformance fixtures for scripts/skill-conformance.test.ts — shaped fake values only, never real credentials.", + "scenarios": [ + { + "name": "bot", + "inputs": { + "bot_token": "fake-webex-bot-token", + "webhook_secret": "fake-webex-webhook-secret" + } + } + ] +} diff --git a/.claude/skills/add-whatsapp-cloud/apply-fixtures.json b/.claude/skills/add-whatsapp-cloud/apply-fixtures.json new file mode 100644 index 000000000..d4f78fc2f --- /dev/null +++ b/.claude/skills/add-whatsapp-cloud/apply-fixtures.json @@ -0,0 +1,14 @@ +{ + "notes": "Conformance fixtures for scripts/skill-conformance.test.ts — shaped fake values only, never real credentials.", + "scenarios": [ + { + "name": "cloud-api", + "inputs": { + "access_token": "EAAfake-access-token", + "phone_number_id": "123456789012345", + "app_secret": "fake-app-secret", + "verify_token": "fake-verify-token" + } + } + ] +} diff --git a/.claude/skills/add-whatsapp/apply-fixtures.json b/.claude/skills/add-whatsapp/apply-fixtures.json new file mode 100644 index 000000000..002a88260 --- /dev/null +++ b/.claude/skills/add-whatsapp/apply-fixtures.json @@ -0,0 +1,51 @@ +{ + "notes": "Conformance fixtures for scripts/skill-conformance.test.ts — shaped fake values only, never real credentials. Three scenarios cover all four number_mode/shared_confirm guard values, both auth methods, and both when:mode= legs. The echo stubs answer the mode/platform_id/owner_handle captures; stepFields feed the QR / pairing-code step's terminal block (capture:bot_phone=PHONE); the 'test(n)' stub answers the engage_pattern capture on the shared leg.", + "scenarios": [ + { + "name": "shared-qr", + "inputs": { + "number_mode": "shared", + "shared_confirm": "continue", + "auth_method": "qr", + "agent_name": "Nano", + "selfchat_engage": "mention" + }, + "exec": [ + { "match": "echo shared", "stdout": "shared" }, + { "match": "test(n)", "stdout": "^@Nano\\b" }, + { "match": "@s.whatsapp.net", "stdout": "15550001111@s.whatsapp.net" } + ], + "stepFields": { "PHONE": "15550001111" } + }, + { + "name": "dedicated-pairing-code", + "inputs": { + "number_mode": "dedicated", + "auth_method": "pairing-code", + "phone": "15550001111", + "chat_phone": "15559998888", + "agent_name": "Nano" + }, + "exec": [ + { "match": "echo dedicated", "stdout": "dedicated" }, + { "match": "@s.whatsapp.net", "stdout": "15559998888@s.whatsapp.net" } + ], + "stepFields": { "PHONE": "15550001111" } + }, + { + "name": "shared-bails-to-dedicated", + "inputs": { + "number_mode": "shared", + "shared_confirm": "dedicated", + "auth_method": "qr", + "chat_phone": "15559998888", + "agent_name": "Nano" + }, + "exec": [ + { "match": "echo dedicated", "stdout": "dedicated" }, + { "match": "@s.whatsapp.net", "stdout": "15559998888@s.whatsapp.net" } + ], + "stepFields": { "PHONE": "15550001111" } + } + ] +} diff --git a/scripts/skill-conformance.test.ts b/scripts/skill-conformance.test.ts new file mode 100644 index 000000000..3fa530c96 --- /dev/null +++ b/scripts/skill-conformance.test.ts @@ -0,0 +1,342 @@ +// CI conformance for programmatic skill apply. +// +// Auto-discovers every .claude/skills/*/SKILL.md that carries nc: directive +// fences (discovery-based, not a hardcoded list — a new fence-carrying skill is +// covered the day it lands) and, per skill: +// +// 1. parse + validate + warn-lints all clean (errors AND advisory warnings — +// in-tree skills must stay warning-free); +// 2. per branch-scenario from the colocated apply-fixtures.json, drives +// applySkill end-to-end with stubbed exec/execStream/resolveRemote in a +// scratch root and asserts a fully-programmatic green run: nothing +// deferred, nothing bounced to an agent, balanced step events; +// 3. every when:-guard value is exercised by at least one scenario (checked +// via ApplyResult.vars — guard vars are non-secret prompts/captures); +// 4. static effect-ordering invariants: code mutations → build → test, and +// restart only after build+test. NOTE deliberately NOT "restart last" — +// real skills restart BEFORE their pairing effect:step (the adapter must +// be live to pair) and may write env after; +// 5. the dynamic run-health gate: a failure injected at the first +// fetch/check/external run must block every later restart/step/wire +// (bounced to an agent, never executed); +// 6. fixture hygiene: scenario input keys ⊆ the skill's prompt vars, every +// unguarded prompt answered by every scenario, and a skill with prompts +// MUST ship a fixture file (actionable failure otherwise). +// +// Everything is stubbed — no network, no git, no pnpm add — so this runs in +// milliseconds inside the normal vitest CI step. + +import { describe, it, expect, afterAll } from 'vitest'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { applySkill, fullyApplied, type ApplyEvent, type ApplyResult } from './skill-apply.js'; +import { + parseDirectives, + validate, + resolveChatCoreVersion, + promptVar, + lintGateAmbiguity, + lintReferenceFloor, + type Directive, +} from './skill-directives.js'; + +const ROOT = process.cwd(); +const SKILLS_DIR = join(ROOT, '.claude/skills'); +const CHAT_VERSION = resolveChatCoreVersion(ROOT); + +// --------------------------------------------------------------------------- +// Discovery: every skill whose SKILL.md opens an nc: fence. +// --------------------------------------------------------------------------- + +const SKILLS = readdirSync(SKILLS_DIR).filter((n) => { + const p = join(SKILLS_DIR, n, 'SKILL.md'); + return existsSync(p) && /^```nc:/m.test(readFileSync(p, 'utf8')); +}); + +// --------------------------------------------------------------------------- +// Fixtures: .claude/skills//apply-fixtures.json, colocated so a skill +// edit and its fixture update land in one diff. Prompt-less skills fall back +// to a single empty default scenario. +// --------------------------------------------------------------------------- + +interface ExecStub { + match: string; // substring of the (var-substituted) command + stdout: string; +} +interface Scenario { + name: string; + inputs?: Record; + exec?: ExecStub[]; + stepFields?: Record; // effect:step terminal-block fields +} +interface Fixture { + notes?: string; + coverageExclude?: string[]; // "var=value" guards final-vars coverage can't see + coverageExcludeReason?: string; + scenarios: Scenario[]; +} + +function loadFixture(name: string): Fixture | undefined { + const p = join(SKILLS_DIR, name, 'apply-fixtures.json'); + if (!existsSync(p)) return undefined; + return JSON.parse(readFileSync(p, 'utf8')) as Fixture; +} + +// --------------------------------------------------------------------------- +// Scratch project root: every append/env-set/json-merge target a trunk skill +// writes to must pre-exist (appendFileSync creates files but not directories, +// and marker appends need the dormant marker region). +// --------------------------------------------------------------------------- + +const scratchRoots: string[] = []; +afterAll(() => { + for (const r of scratchRoots) rmSync(r, { recursive: true, force: true }); +}); + +function scratchRoot(): string { + const root = mkdtempSync(join(tmpdir(), 'skill-conformance-')); + scratchRoots.push(root); + mkdirSync(join(root, 'src/channels'), { recursive: true }); + mkdirSync(join(root, 'src/providers'), { recursive: true }); + mkdirSync(join(root, 'container/agent-runner/src/providers'), { recursive: true }); + mkdirSync(join(root, 'setup/providers'), { recursive: true }); + writeFileSync(join(root, 'package.json'), '{"name":"scratch"}\n'); + writeFileSync(join(root, '.env'), ''); + writeFileSync(join(root, 'src/channels/index.ts'), '// channel adapter barrel\n'); + writeFileSync(join(root, 'src/providers/index.ts'), '// provider barrel\n'); + writeFileSync(join(root, 'container/agent-runner/src/providers/index.ts'), '// container provider barrel\n'); + writeFileSync(join(root, 'setup/providers/index.ts'), '// setup provider barrel\n'); + writeFileSync( + join(root, 'setup/index.ts'), + ['const STEPS = {', ' // >>> nanoclaw:setup-steps', ' // <<< nanoclaw:setup-steps', '};', ''].join('\n'), + ); + writeFileSync(join(root, 'container/cli-tools.json'), '[]\n'); + return root; +} + +interface RunOutcome { + res: ApplyResult; + cmds: string[]; + streamed: string[]; + events: ApplyEvent[]; +} + +async function runScenario(skillDir: string, sc: Scenario): Promise { + const root = scratchRoot(); + const cmds: string[] = []; + const streamed: string[] = []; + const events: ApplyEvent[] = []; + const res = await applySkill(skillDir, root, { + inputs: sc.inputs ?? {}, + // resolveRemote MUST be injected: the default shells out to real + // `git remote` + `git ls-remote` — network in CI, nondeterministic on forks. + resolveRemote: () => 'origin', + exec: (c) => { + cmds.push(c); + return sc.exec?.find((e) => c.includes(e.match))?.stdout; + }, + execStream: async (c) => { + streamed.push(c); + return { ok: true, fields: sc.stepFields ?? {} }; + }, + onEvent: (e) => void events.push(e), + }); + return { res, cmds, streamed, events }; +} + +const isString = (x: unknown): x is string => typeof x === 'string'; +const SIDE_EFFECTS = new Set(['restart', 'step', 'wire']); +const SABOTEUR_EFFECTS = new Set(['fetch', 'check', 'external']); +const isSideEffectRun = (d: Directive | undefined): boolean => + d?.kind === 'run' && isString(d.attrs.effect) && SIDE_EFFECTS.has(d.attrs.effect); + +// --------------------------------------------------------------------------- + +describe('skill discovery', () => { + it('finds the fence-carrying skills', () => { + // Sanity floor: discovery walking the wrong directory (or the fence regex + // regressing) must fail loudly, not silently skip the whole suite. + expect(SKILLS).toContain('add-slack'); + expect(SKILLS).toContain('add-whatsapp'); + expect(SKILLS.length).toBeGreaterThanOrEqual(10); + }); +}); + +describe.each(SKILLS)('%s', (name) => { + const dir = join(SKILLS_DIR, name); + const md = readFileSync(join(dir, 'SKILL.md'), 'utf8'); + const directives = parseDirectives(md); + const byLine = new Map(directives.map((d) => [d.line, d])); + const promptVars = new Set( + directives.filter((d) => d.kind === 'prompt').map((d) => promptVar(d)).filter(isString), + ); + const guards = [...new Set(directives.map((d) => d.attrs.when).filter(isString))]; + const fixture = loadFixture(name); + const scenarios: Scenario[] = fixture?.scenarios ?? [{ name: 'default', inputs: {} }]; + + it('parses + validates + warn-lints clean', () => { + expect(validate(directives, { chatVersion: CHAT_VERSION })).toEqual([]); + // Advisory warnings too: in-tree skills must stay warning-free. + expect(lintGateAmbiguity(directives)).toEqual([]); + expect(lintReferenceFloor(md)).toEqual([]); + }); + + it('fixture hygiene: prompts have a fixture, inputs match prompt vars, unguarded prompts always answered', () => { + if (promptVars.size > 0) { + expect( + fixture, + `${name} declares nc:prompt directives but has no apply-fixtures.json — add .claude/skills/${name}/apply-fixtures.json with a "scenarios" array: shaped fake inputs per prompt var (satisfying each validate: regex), exec stubs (substring match → stdout) for every capture run, and stepFields for any effect:step`, + ).toBeDefined(); + } + const unguarded = directives + .filter((d) => d.kind === 'prompt' && !isString(d.attrs.when)) + .map((d) => promptVar(d)) + .filter(isString); + for (const sc of fixture?.scenarios ?? []) { + for (const k of Object.keys(sc.inputs ?? {})) { + expect(promptVars.has(k), `scenario "${sc.name}" supplies input "${k}" which is not a prompt var of ${name} — stale fixture?`).toBe(true); + } + for (const v of unguarded) { + expect( + sc.inputs?.[v], + `scenario "${sc.name}" is missing unguarded prompt var "${v}" — it would defer and the apply could never be fully programmatic`, + ).toBeDefined(); + } + } + // coverageExclude entries must reference guards that actually exist. + const guardSet = new Set(guards); + for (const g of fixture?.coverageExclude ?? []) { + expect(guardSet.has(g), `coverageExclude "${g}" is not a when:-guard of ${name} — stale exclusion?`).toBe(true); + } + }); + + it.each(scenarios)('applies fully programmatically: $name', async (sc) => { + const { res, events } = await runScenario(dir, sc); + expect(res.agentTasks).toEqual([]); // nothing degraded to an agent + expect(res.deferred).toEqual([]); // every prompt satisfied AND validate-at-bind passed + expect(fullyApplied(res)).toBe(true); + // Balanced step brackets, all green. + const starts = events.filter((e) => e.type === 'step-start'); + const ends = events.filter((e): e is Extract => e.type === 'step-end'); + expect(starts.length).toBe(ends.length); + expect(ends.every((e) => e.ok)).toBe(true); + // Every resolved non-secret var is non-empty — an empty capture means an + // exec/stepFields fixture entry is missing (bindCapture binds '' when the + // stub returned nothing and no validate: catches it). + for (const [k, v] of Object.entries(res.vars)) { + expect(v, `resolved {{${k}}} is empty in scenario "${sc.name}" — add/fix the exec or stepFields fixture entry answering that capture`).not.toBe(''); + } + }); + + it('covers every when:-guard value across scenarios', async () => { + if (guards.length === 0) return; + const excluded = new Set(fixture?.coverageExclude ?? []); + const results: ApplyResult[] = []; + for (const sc of scenarios) results.push((await runScenario(dir, sc)).res); + for (const g of guards) { + if (excluded.has(g)) continue; + const eq = g.indexOf('='); + const [v, val] = [g.slice(0, eq), g.slice(eq + 1)]; + expect( + results.some((r) => r.vars[v] === val), + `no scenario exercises when:${g} in ${name} — add a scenario to apply-fixtures.json (or coverageExclude it with a reason)`, + ).toBe(true); + } + }); + + // Static, document-order invariants. Deliberately NOT "restart last": + // telegram/whatsapp/signal restart BEFORE their pairing effect:step (the + // adapter must be live to pair) and whatsapp writes env after the restart. + // What DOES hold: code mutations land before the build, the build runs + // before the tests, and a restart never precedes the build or the tests + // that validate what it would load. + it('effect ordering: mutations → build → test; restart only after build+test', () => { + const firstBuild = directives.findIndex((d) => d.kind === 'run' && d.attrs.effect === 'build'); + const firstTest = directives.findIndex((d) => d.kind === 'run' && d.attrs.effect === 'test'); + if (firstBuild >= 0 && firstTest >= 0) expect(firstBuild).toBeLessThan(firstTest); + if (firstBuild >= 0) { + directives.forEach((d, i) => { + if (['copy', 'append', 'dep', 'json-merge'].includes(d.kind)) { + expect(i, `${d.kind} at line ${d.line} lands after the build — the build would not see it`).toBeLessThan(firstBuild); + } + }); + } + directives.forEach((d, i) => { + if (d.kind === 'run' && d.attrs.effect === 'restart') { + if (firstBuild >= 0) expect(i, `restart at line ${d.line} precedes the build`).toBeGreaterThan(firstBuild); + if (firstTest >= 0) expect(i, `restart at line ${d.line} precedes the tests`).toBeGreaterThan(firstTest); + } + }); + }); + + // A restart-shaped command on a bare `nc:run` (no effect:) would silently + // escape both skipEffects ownership and the run-health gate. + it('no restart-shaped command hides on a bare nc:run', () => { + for (const d of directives) { + if (d.kind !== 'run' || d.attrs.effect !== undefined) continue; + for (const cmd of d.body) { + expect( + /restart\.sh|kickstart|systemctl/.test(cmd), + `bare nc:run at line ${d.line} runs a restart-shaped command ("${cmd}") without effect:restart — it would evade the run-health gate`, + ).toBe(false); + } + } + }); + + // Dynamic twin of the engine's run-health-gate unit tests, per real skill: + // inject a failure at the first fetch/check/external run a scenario reaches + // and assert no restart/step/wire executes afterwards — authoring that + // evades the gate (or a gate regression against real documents) fails here. + it('run-health gate: a failed fetch/check/external blocks every later restart/step/wire', async () => { + for (const sc of scenarios) { + let current: Directive | undefined; + let sabotagedLine = -1; + let streamedAfterSabotage = 0; + const sideEffectStartsAfterSabotage: number[] = []; + const root = scratchRoot(); + const res = await applySkill(dir, root, { + inputs: sc.inputs ?? {}, + resolveRemote: () => 'origin', + onEvent: (e) => { + if (e.type !== 'step-start') return; + current = byLine.get(e.line); + if (sabotagedLine >= 0 && isSideEffectRun(current)) sideEffectStartsAfterSabotage.push(e.line); + }, + exec: (c) => { + if ( + sabotagedLine < 0 && + current?.kind === 'run' && + isString(current.attrs.effect) && + SABOTEUR_EFFECTS.has(current.attrs.effect) + ) { + sabotagedLine = current.line; + throw new Error('conformance sabotage'); + } + return sc.exec?.find((e) => c.includes(e.match))?.stdout; + }, + execStream: async () => { + if (sabotagedLine >= 0) streamedAfterSabotage++; + return { ok: true, fields: sc.stepFields ?? {} }; + }, + }); + if (sabotagedLine < 0) continue; // this scenario never reaches a saboteur-eligible run + // The sabotaged run itself bounced to an agent… + expect(fullyApplied(res)).toBe(false); + expect(res.agentTasks.some((t) => t.line === sabotagedLine)).toBe(true); + // …and no dangerous side effect fired on its own afterwards. + expect(sideEffectStartsAfterSabotage).toEqual([]); + expect(streamedAfterSabotage).toBe(0); + // Any unguarded later side effect must surface as a gated agentTask, so + // an agent finishes it from the prose once the failure is fixed. + const gatedExpected = directives.some( + (d) => isSideEffectRun(d) && !isString(d.attrs.when) && d.line > sabotagedLine, + ); + if (gatedExpected) { + expect(res.agentTasks.some((t) => /earlier step did not complete/.test(t.reason))).toBe(true); + } + return; // one sabotaged scenario per skill is enough + } + }); +}); diff --git a/scripts/skill-inputs.test.ts b/scripts/skill-inputs.test.ts new file mode 100644 index 000000000..f1997b6a7 --- /dev/null +++ b/scripts/skill-inputs.test.ts @@ -0,0 +1,82 @@ +import { describe, it, expect } from 'vitest'; +import { mkdtempSync, mkdirSync, readFileSync, writeFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { inputsFromEnv } from './skill-inputs.js'; +import { applySkill, fullyApplied } from './skill-apply.js'; + +describe('inputsFromEnv (docs/skill-engine-seam.md §6)', () => { + it('maps NC_INPUT_ env keys onto prompt vars, ignoring unset and unrelated keys', () => { + const md = [ + '```nc:prompt bot_token secret', + 'Paste the token.', + '```', + '```nc:prompt owner_handle', + 'Your handle?', + '```', + ].join('\n'); + const inputs = inputsFromEnv(md, { + NC_INPUT_BOT_TOKEN: 'xoxb-fake', + NC_INPUT_UNRELATED: 'ignored', // no matching prompt + PATH: '/usr/bin', // not NC_INPUT_-prefixed + // NC_INPUT_OWNER_HANDLE deliberately unset → omitted, not '' + }); + expect(inputs).toEqual({ bot_token: 'xoxb-fake' }); + }); + + it('errors on an uppercase collision instead of silently merging', () => { + const md = [ + '```nc:prompt bot_token', + 'Token?', + '```', + '```nc:prompt Bot_Token', + 'Token again?', + '```', + ].join('\n'); + expect(() => inputsFromEnv(md, {})).toThrow(/NC_INPUT_BOT_TOKEN/); + }); + + // The round-trip proof for one real skill: env → inputsFromEnv → applySkill + // goes fully green for add-slack (webhook leg) with stubbed exec — the exact + // pipeline-consumer path the seam doc's §6 contract describes. + it('round-trips the env convention through a full programmatic apply of add-slack', async () => { + const skillDir = join(process.cwd(), '.claude/skills/add-slack'); + const md = readFileSync(join(skillDir, 'SKILL.md'), 'utf8'); + const inputs = inputsFromEnv(md, { + NC_INPUT_CONNECTION: 'webhook', + NC_INPUT_BOT_TOKEN: 'xoxb-fake-token', + NC_INPUT_SIGNING_SECRET: '0123456789abcdef', + NC_INPUT_OWNER_HANDLE: 'U12345678', + }); + expect(inputs).toEqual({ + connection: 'webhook', + bot_token: 'xoxb-fake-token', + signing_secret: '0123456789abcdef', + owner_handle: 'U12345678', + }); + + const root = mkdtempSync(join(tmpdir(), 'skill-inputs-')); + try { + mkdirSync(join(root, 'src/channels'), { recursive: true }); + writeFileSync(join(root, 'src/channels/index.ts'), '// barrel\n'); + writeFileSync(join(root, '.env'), ''); + writeFileSync(join(root, 'package.json'), '{"name":"scratch"}\n'); + + const res = await applySkill(skillDir, root, { + inputs, + exec: (c) => { + if (c.includes('auth.test')) return '@nano in Acme'; + if (c.includes('conversations.open')) return 'slack:D0FAKE'; + }, + resolveRemote: () => 'origin', + }); + expect(res.deferred).toEqual([]); + expect(res.agentTasks).toEqual([]); + expect(fullyApplied(res)).toBe(true); + expect(res.vars.platform_id).toBe('slack:D0FAKE'); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/scripts/skill-inputs.ts b/scripts/skill-inputs.ts new file mode 100644 index 000000000..d79535be1 --- /dev/null +++ b/scripts/skill-inputs.ts @@ -0,0 +1,32 @@ +// inputsFromEnv — the pipeline consumer's input path (docs/skill-engine-seam.md §6). +// +// A CI/pipeline caller supplies prompt answers as environment variables using +// the `NC_INPUT_` convention (prompt var uppercased). This helper parses +// the skill's prompt vars via parseDirectives and returns the `inputs` record +// applySkill consumes. It is a helper, not an engine feature — the engine never +// reads process.env for inputs; `inputs` stays the only env-agnostic seam. +// +// Var names are case-sensitive in the grammar, so uppercasing can collide +// (`bot_token` vs `Bot_Token` both map to NC_INPUT_BOT_TOKEN); a collision is +// an error, never a silent last-writer-wins. + +import { parseDirectives, promptVar } from './skill-directives.js'; + +export function inputsFromEnv(md: string, env: Record = process.env): Record { + const inputs: Record = {}; + const byKey = new Map(); // NC_INPUT_ → the prompt var that claimed it + for (const d of parseDirectives(md)) { + if (d.kind !== 'prompt') continue; + const v = promptVar(d); + if (!v) continue; + const key = `NC_INPUT_${v.toUpperCase()}`; + const prior = byKey.get(key); + if (prior !== undefined && prior !== v) { + throw new Error(`inputsFromEnv: prompt vars "${prior}" and "${v}" both map to ${key} — rename one (var names must be case-insensitively unique)`); + } + byKey.set(key, v); + const val = env[key]; + if (val !== undefined) inputs[v] = val; + } + return inputs; +}