mirror of
https://github.com/qwibitai/nanoclaw.git
synced 2026-07-15 19:06:18 +08:00
test(skills): conformance suite — every fence-carrying skill applies programmatically
scripts/skill-conformance.test.ts auto-discovers each .claude/skills/*/SKILL.md with nc: fences (15 today) and asserts, per skill: lint clean (errors and warn-lints); per branch-scenario from a colocated apply-fixtures.json, a full applySkill run with stubbed exec/execStream/resolveRemote goes green — nothing deferred, nothing degraded to agent, every var bound; every when: guard value is exercised by some scenario (coverage read from ApplyResult.vars, with documented exclusions); static effect ordering (mutations before build, build before test, restart only after both); and a run-health sabotage probe — a failing fetch/check/external run must gate every later restart/step/wire. Fixture values satisfy the skills' real validate: regexes, so a fixture can't paper over a broken prompt. Skills with prompts but no fixture fail with an actionable message; prompt-less skills get a default empty scenario. scripts/skill-inputs.ts adds the inputsFromEnv helper from the seam spec §6 (NC_INPUT_<VAR>, collision error); its test round-trips env-supplied inputs through a real skill apply. Runs in the existing CI vitest step — no workflow change needed. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
gavrielc
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"notes": "Conformance fixtures for scripts/skill-conformance.test.ts — shaped fake values only, never real credentials. The applications/@me stub answers the multi-field JSON capture (application_id/.id, public_key/.verify_key, owner_handle/.owner.id); the users/@me/channels stub answers the platform_id capture.",
|
||||
"scenarios": [
|
||||
{
|
||||
"name": "bot-token",
|
||||
"inputs": {
|
||||
"bot_token": "MTA0fake.token.fake_fake-fake.fake0123456789abcdefghijklmnopqrstuvwxyz"
|
||||
},
|
||||
"exec": [
|
||||
{ "match": "oauth2/applications/@me", "stdout": "{\"id\":\"1234567890\",\"verify_key\":\"abcdef0123456789\",\"owner\":{\"id\":\"111222333444\"}}" },
|
||||
{ "match": "users/@me/channels", "stdout": "discord:@me:D0FAKE" }
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"notes": "Conformance fixtures for scripts/skill-conformance.test.ts — shaped fake values only, never real credentials.",
|
||||
"scenarios": [
|
||||
{
|
||||
"name": "service-account",
|
||||
"inputs": {
|
||||
"gchat_credentials": "fake-service-account-json-blob"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"notes": "Conformance fixtures for scripts/skill-conformance.test.ts — shaped fake values only, never real credentials.",
|
||||
"scenarios": [
|
||||
{
|
||||
"name": "pat",
|
||||
"inputs": {
|
||||
"github_token": "ghp_fake0123456789fake0123456789fake01",
|
||||
"webhook_secret": "fake-webhook-secret",
|
||||
"bot_username": "nanoclaw-bot"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"notes": "Conformance fixtures for scripts/skill-conformance.test.ts — shaped fake values only, never real credentials. Two scenarios cover both when:mode= legs; the echo stub answers the platform_id capture (the skill echoes the owner handle back).",
|
||||
"scenarios": [
|
||||
{
|
||||
"name": "local",
|
||||
"inputs": {
|
||||
"mode": "local",
|
||||
"owner_handle": "+15551234567"
|
||||
},
|
||||
"exec": [
|
||||
{ "match": "echo \"+15551234567\"", "stdout": "+15551234567" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "remote",
|
||||
"inputs": {
|
||||
"mode": "remote",
|
||||
"server_url": "https://imsg.example.com",
|
||||
"api_key": "fake-photon-api-key",
|
||||
"owner_handle": "owner@example.com"
|
||||
},
|
||||
"exec": [
|
||||
{ "match": "echo \"owner@example.com\"", "stdout": "owner@example.com" }
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"notes": "Conformance fixtures for scripts/skill-conformance.test.ts — shaped fake values only, never real credentials. The effect:wire ncl runs carry no capture, so the exec stub's no-op answer is fine.",
|
||||
"scenarios": [
|
||||
{
|
||||
"name": "workspace",
|
||||
"inputs": {
|
||||
"linear_client_id": "fake-client-id",
|
||||
"linear_client_secret": "fake-client-secret",
|
||||
"linear_webhook_secret": "fake-webhook-secret",
|
||||
"linear_team_key": "ENG",
|
||||
"linear_bot_username": "nanoclaw",
|
||||
"agent_folder": "main",
|
||||
"linear_sender_policy": "public"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"notes": "Conformance fixtures for scripts/skill-conformance.test.ts — shaped fake values only, never real credentials. All prompts in this skill are unguarded (the login-method legs are prose alternatives, not when:-branches), so a fully-programmatic apply must answer every one — username/password AND access_token.",
|
||||
"scenarios": [
|
||||
{
|
||||
"name": "homeserver",
|
||||
"inputs": {
|
||||
"base_url": "https://matrix.example.com",
|
||||
"user_id": "@nanoclaw:example.com",
|
||||
"bot_username": "nanoclaw",
|
||||
"username": "nanoclaw",
|
||||
"password": "fake-password",
|
||||
"access_token": "syt_fake_access_token"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"notes": "Conformance fixtures for scripts/skill-conformance.test.ts — shaped fake values only, never real credentials. The effect:wire ncl runs carry no capture, so the exec stub's no-op answer is fine.",
|
||||
"scenarios": [
|
||||
{
|
||||
"name": "email",
|
||||
"inputs": {
|
||||
"api_key": "re_fake_api_key",
|
||||
"webhook_secret": "whsec_fake",
|
||||
"from_address": "assistant@mail.example.com",
|
||||
"from_name": "NanoClaw",
|
||||
"owner_email": "owner@example.com",
|
||||
"agent_folder": "main"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"notes": "Conformance fixtures for scripts/skill-conformance.test.ts — no prompts in this skill; the fixture exists to feed the effect:step device-link's terminal-block fields (capture:platform_id=ACCOUNT,owner_handle=ACCOUNT).",
|
||||
"scenarios": [
|
||||
{
|
||||
"name": "device-link",
|
||||
"inputs": {},
|
||||
"stepFields": { "ACCOUNT": "+15550001111" }
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"notes": "Conformance fixtures for scripts/skill-conformance.test.ts — shaped fake values only, never real credentials. Two scenarios cover both when:connection= legs; the auth.test stub answers the connected_as capture and conversations.open answers platform_id.",
|
||||
"scenarios": [
|
||||
{
|
||||
"name": "socket",
|
||||
"inputs": {
|
||||
"connection": "socket",
|
||||
"bot_token": "xoxb-fake-bot-token",
|
||||
"app_token": "xapp-fake-app-token",
|
||||
"owner_handle": "U12345678"
|
||||
},
|
||||
"exec": [
|
||||
{ "match": "auth.test", "stdout": "@nano in Acme" },
|
||||
{ "match": "conversations.open", "stdout": "slack:D0FAKE" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "webhook",
|
||||
"inputs": {
|
||||
"connection": "webhook",
|
||||
"bot_token": "xoxb-fake-bot-token",
|
||||
"signing_secret": "0123456789abcdef",
|
||||
"owner_handle": "U12345678"
|
||||
},
|
||||
"exec": [
|
||||
{ "match": "auth.test", "stdout": "@nano in Acme" },
|
||||
{ "match": "conversations.open", "stdout": "slack:D0FAKE" }
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
{
|
||||
"notes": "Conformance fixtures for scripts/skill-conformance.test.ts — shaped fake values only, never real credentials. Four scenarios: existing-creds drop-through (the have_creds probe answers 'yes'); a fresh create wiring the logged-in owner directly; a fresh create wiring a DIFFERENT account (wire_target=other-account); and a fresh create re-confirming via the logged-in account (wire_target=logged-in-account). The `app create` stub answers the five-field JSON capture; `status --json` answers owner_upn/owner_aad_id; the smba stubs answer the DM-open chain. Guards driven by CAPTURES (have_creds) are selected by exec stdout, not inputs.",
|
||||
"coverageExclude": [
|
||||
"wire_owner=no"
|
||||
],
|
||||
"coverageExcludeReason": "wire_owner=no is exercised at runtime by the other-account and logged-in-account scenarios (their wire_target prompts only run under it), but BOTH of those legs deliberately re-bind wire_owner to 'yes' via capture (SKILL.md ~L257/L261) before the run ends, so no scenario can finish with vars.wire_owner='no'. Final-vars coverage is impossible by design, not missing.",
|
||||
"scenarios": [
|
||||
{
|
||||
"name": "existing-creds",
|
||||
"inputs": {},
|
||||
"exec": [
|
||||
{ "match": "TEAMS_APP_ID=.", "stdout": "yes" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "create-wire-owner",
|
||||
"inputs": {
|
||||
"public_url": "https://nc.example.com",
|
||||
"app_name": "NanoClaw Bot",
|
||||
"wire_owner": "yes",
|
||||
"signout": "yes"
|
||||
},
|
||||
"exec": [
|
||||
{ "match": "TEAMS_APP_ID=.", "stdout": "no" },
|
||||
{ "match": "app create", "stdout": "{\"credentials\":{\"CLIENT_ID\":\"11111111-2222-3333-4444-555555555555\",\"CLIENT_SECRET\":\"fake-client-secret\",\"TENANT_ID\":\"99999999-8888-7777-6666-555555555555\"},\"teamsAppId\":\"aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee\",\"installLink\":\"https://teams.microsoft.com/l/app/fake\"}" },
|
||||
{ "match": "status --json", "stdout": "{\"username\":\"owner@contoso.com\",\"userObjectId\":\"12345678-1234-1234-1234-123456789abc\"}" },
|
||||
{ "match": "oauth2/v2.0/token", "stdout": "eyJhbGciOiJIUzI1NiJ9.fake.sig" },
|
||||
{ "match": "/members", "stdout": "{\"id\":\"29:1fakeownerid\",\"name\":\"Owner\"}" },
|
||||
{ "match": "v3/conversations\"", "stdout": "a:1fakeconversationid" },
|
||||
{ "match": "base64url", "stdout": "teams:YTpmYWtl:aHR0cHM6Ly9zbWJh" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "create-wire-other-account",
|
||||
"inputs": {
|
||||
"public_url": "https://nc.example.com",
|
||||
"app_name": "NanoClaw Bot",
|
||||
"wire_owner": "no",
|
||||
"wire_target": "other-account",
|
||||
"target_aad_id": "12345678-1234-1234-1234-123456789abc",
|
||||
"signout": "no"
|
||||
},
|
||||
"exec": [
|
||||
{ "match": "TEAMS_APP_ID=.", "stdout": "no" },
|
||||
{ "match": "app create", "stdout": "{\"credentials\":{\"CLIENT_ID\":\"11111111-2222-3333-4444-555555555555\",\"CLIENT_SECRET\":\"fake-client-secret\",\"TENANT_ID\":\"99999999-8888-7777-6666-555555555555\"},\"teamsAppId\":\"aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee\",\"installLink\":\"https://teams.microsoft.com/l/app/fake\"}" },
|
||||
{ "match": "status --json", "stdout": "{\"username\":\"owner@contoso.com\",\"userObjectId\":\"12345678-1234-1234-1234-123456789abc\"}" },
|
||||
{ "match": "printf '{\"aad\"", "stdout": "{\"aad\":\"12345678-1234-1234-1234-123456789abc\",\"wire\":\"yes\"}" },
|
||||
{ "match": "oauth2/v2.0/token", "stdout": "eyJhbGciOiJIUzI1NiJ9.fake.sig" },
|
||||
{ "match": "/members", "stdout": "{\"id\":\"29:1faketargetid\",\"name\":\"Target user\"}" },
|
||||
{ "match": "v3/conversations\"", "stdout": "a:1fakeconversationid" },
|
||||
{ "match": "base64url", "stdout": "teams:YTpmYWtl:aHR0cHM6Ly9zbWJh" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "create-wire-logged-in-account",
|
||||
"inputs": {
|
||||
"public_url": "https://nc.example.com",
|
||||
"app_name": "NanoClaw Bot",
|
||||
"wire_owner": "no",
|
||||
"wire_target": "logged-in-account",
|
||||
"signout": "no"
|
||||
},
|
||||
"exec": [
|
||||
{ "match": "TEAMS_APP_ID=.", "stdout": "no" },
|
||||
{ "match": "app create", "stdout": "{\"credentials\":{\"CLIENT_ID\":\"11111111-2222-3333-4444-555555555555\",\"CLIENT_SECRET\":\"fake-client-secret\",\"TENANT_ID\":\"99999999-8888-7777-6666-555555555555\"},\"teamsAppId\":\"aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee\",\"installLink\":\"https://teams.microsoft.com/l/app/fake\"}" },
|
||||
{ "match": "status --json", "stdout": "{\"username\":\"owner@contoso.com\",\"userObjectId\":\"12345678-1234-1234-1234-123456789abc\"}" },
|
||||
{ "match": "echo yes", "stdout": "yes" },
|
||||
{ "match": "oauth2/v2.0/token", "stdout": "eyJhbGciOiJIUzI1NiJ9.fake.sig" },
|
||||
{ "match": "/members", "stdout": "{\"id\":\"29:1fakeownerid\",\"name\":\"Owner\"}" },
|
||||
{ "match": "v3/conversations\"", "stdout": "a:1fakeconversationid" },
|
||||
{ "match": "base64url", "stdout": "teams:YTpmYWtl:aHR0cHM6Ly9zbWJh" }
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"notes": "Conformance fixtures for scripts/skill-conformance.test.ts — shaped fake values only, never real credentials. The getMe stub answers the bot_username capture (the operator block after the restart renders it); stepFields feed the pairing step's terminal block (capture:platform_id=PLATFORM_ID,owner_handle=ADMIN_USER_ID).",
|
||||
"scenarios": [
|
||||
{
|
||||
"name": "pairing",
|
||||
"inputs": {
|
||||
"bot_token": "123456789:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
|
||||
},
|
||||
"exec": [
|
||||
{ "match": "getMe", "stdout": "nanoclaw_bot" }
|
||||
],
|
||||
"stepFields": { "PLATFORM_ID": "telegram:123456789", "ADMIN_USER_ID": "987654321" }
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"notes": "Conformance fixtures for scripts/skill-conformance.test.ts — shaped fake values only, never real credentials.",
|
||||
"scenarios": [
|
||||
{
|
||||
"name": "bot",
|
||||
"inputs": {
|
||||
"bot_token": "fake-webex-bot-token",
|
||||
"webhook_secret": "fake-webex-webhook-secret"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"notes": "Conformance fixtures for scripts/skill-conformance.test.ts — shaped fake values only, never real credentials.",
|
||||
"scenarios": [
|
||||
{
|
||||
"name": "cloud-api",
|
||||
"inputs": {
|
||||
"access_token": "EAAfake-access-token",
|
||||
"phone_number_id": "123456789012345",
|
||||
"app_secret": "fake-app-secret",
|
||||
"verify_token": "fake-verify-token"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
{
|
||||
"notes": "Conformance fixtures for scripts/skill-conformance.test.ts — shaped fake values only, never real credentials. Three scenarios cover all four number_mode/shared_confirm guard values, both auth methods, and both when:mode= legs. The echo stubs answer the mode/platform_id/owner_handle captures; stepFields feed the QR / pairing-code step's terminal block (capture:bot_phone=PHONE); the 'test(n)' stub answers the engage_pattern capture on the shared leg.",
|
||||
"scenarios": [
|
||||
{
|
||||
"name": "shared-qr",
|
||||
"inputs": {
|
||||
"number_mode": "shared",
|
||||
"shared_confirm": "continue",
|
||||
"auth_method": "qr",
|
||||
"agent_name": "Nano",
|
||||
"selfchat_engage": "mention"
|
||||
},
|
||||
"exec": [
|
||||
{ "match": "echo shared", "stdout": "shared" },
|
||||
{ "match": "test(n)", "stdout": "^@Nano\\b" },
|
||||
{ "match": "@s.whatsapp.net", "stdout": "15550001111@s.whatsapp.net" }
|
||||
],
|
||||
"stepFields": { "PHONE": "15550001111" }
|
||||
},
|
||||
{
|
||||
"name": "dedicated-pairing-code",
|
||||
"inputs": {
|
||||
"number_mode": "dedicated",
|
||||
"auth_method": "pairing-code",
|
||||
"phone": "15550001111",
|
||||
"chat_phone": "15559998888",
|
||||
"agent_name": "Nano"
|
||||
},
|
||||
"exec": [
|
||||
{ "match": "echo dedicated", "stdout": "dedicated" },
|
||||
{ "match": "@s.whatsapp.net", "stdout": "15559998888@s.whatsapp.net" }
|
||||
],
|
||||
"stepFields": { "PHONE": "15550001111" }
|
||||
},
|
||||
{
|
||||
"name": "shared-bails-to-dedicated",
|
||||
"inputs": {
|
||||
"number_mode": "shared",
|
||||
"shared_confirm": "dedicated",
|
||||
"auth_method": "qr",
|
||||
"chat_phone": "15559998888",
|
||||
"agent_name": "Nano"
|
||||
},
|
||||
"exec": [
|
||||
{ "match": "echo dedicated", "stdout": "dedicated" },
|
||||
{ "match": "@s.whatsapp.net", "stdout": "15559998888@s.whatsapp.net" }
|
||||
],
|
||||
"stepFields": { "PHONE": "15550001111" }
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,342 @@
|
||||
// CI conformance for programmatic skill apply.
|
||||
//
|
||||
// Auto-discovers every .claude/skills/*/SKILL.md that carries nc: directive
|
||||
// fences (discovery-based, not a hardcoded list — a new fence-carrying skill is
|
||||
// covered the day it lands) and, per skill:
|
||||
//
|
||||
// 1. parse + validate + warn-lints all clean (errors AND advisory warnings —
|
||||
// in-tree skills must stay warning-free);
|
||||
// 2. per branch-scenario from the colocated apply-fixtures.json, drives
|
||||
// applySkill end-to-end with stubbed exec/execStream/resolveRemote in a
|
||||
// scratch root and asserts a fully-programmatic green run: nothing
|
||||
// deferred, nothing bounced to an agent, balanced step events;
|
||||
// 3. every when:-guard value is exercised by at least one scenario (checked
|
||||
// via ApplyResult.vars — guard vars are non-secret prompts/captures);
|
||||
// 4. static effect-ordering invariants: code mutations → build → test, and
|
||||
// restart only after build+test. NOTE deliberately NOT "restart last" —
|
||||
// real skills restart BEFORE their pairing effect:step (the adapter must
|
||||
// be live to pair) and may write env after;
|
||||
// 5. the dynamic run-health gate: a failure injected at the first
|
||||
// fetch/check/external run must block every later restart/step/wire
|
||||
// (bounced to an agent, never executed);
|
||||
// 6. fixture hygiene: scenario input keys ⊆ the skill's prompt vars, every
|
||||
// unguarded prompt answered by every scenario, and a skill with prompts
|
||||
// MUST ship a fixture file (actionable failure otherwise).
|
||||
//
|
||||
// Everything is stubbed — no network, no git, no pnpm add — so this runs in
|
||||
// milliseconds inside the normal vitest CI step.
|
||||
|
||||
import { describe, it, expect, afterAll } from 'vitest';
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
|
||||
import { applySkill, fullyApplied, type ApplyEvent, type ApplyResult } from './skill-apply.js';
|
||||
import {
|
||||
parseDirectives,
|
||||
validate,
|
||||
resolveChatCoreVersion,
|
||||
promptVar,
|
||||
lintGateAmbiguity,
|
||||
lintReferenceFloor,
|
||||
type Directive,
|
||||
} from './skill-directives.js';
|
||||
|
||||
const ROOT = process.cwd();
|
||||
const SKILLS_DIR = join(ROOT, '.claude/skills');
|
||||
const CHAT_VERSION = resolveChatCoreVersion(ROOT);
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Discovery: every skill whose SKILL.md opens an nc: fence.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const SKILLS = readdirSync(SKILLS_DIR).filter((n) => {
|
||||
const p = join(SKILLS_DIR, n, 'SKILL.md');
|
||||
return existsSync(p) && /^```nc:/m.test(readFileSync(p, 'utf8'));
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Fixtures: .claude/skills/<name>/apply-fixtures.json, colocated so a skill
|
||||
// edit and its fixture update land in one diff. Prompt-less skills fall back
|
||||
// to a single empty default scenario.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface ExecStub {
|
||||
match: string; // substring of the (var-substituted) command
|
||||
stdout: string;
|
||||
}
|
||||
interface Scenario {
|
||||
name: string;
|
||||
inputs?: Record<string, string>;
|
||||
exec?: ExecStub[];
|
||||
stepFields?: Record<string, string>; // effect:step terminal-block fields
|
||||
}
|
||||
interface Fixture {
|
||||
notes?: string;
|
||||
coverageExclude?: string[]; // "var=value" guards final-vars coverage can't see
|
||||
coverageExcludeReason?: string;
|
||||
scenarios: Scenario[];
|
||||
}
|
||||
|
||||
function loadFixture(name: string): Fixture | undefined {
|
||||
const p = join(SKILLS_DIR, name, 'apply-fixtures.json');
|
||||
if (!existsSync(p)) return undefined;
|
||||
return JSON.parse(readFileSync(p, 'utf8')) as Fixture;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Scratch project root: every append/env-set/json-merge target a trunk skill
|
||||
// writes to must pre-exist (appendFileSync creates files but not directories,
|
||||
// and marker appends need the dormant marker region).
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const scratchRoots: string[] = [];
|
||||
afterAll(() => {
|
||||
for (const r of scratchRoots) rmSync(r, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function scratchRoot(): string {
|
||||
const root = mkdtempSync(join(tmpdir(), 'skill-conformance-'));
|
||||
scratchRoots.push(root);
|
||||
mkdirSync(join(root, 'src/channels'), { recursive: true });
|
||||
mkdirSync(join(root, 'src/providers'), { recursive: true });
|
||||
mkdirSync(join(root, 'container/agent-runner/src/providers'), { recursive: true });
|
||||
mkdirSync(join(root, 'setup/providers'), { recursive: true });
|
||||
writeFileSync(join(root, 'package.json'), '{"name":"scratch"}\n');
|
||||
writeFileSync(join(root, '.env'), '');
|
||||
writeFileSync(join(root, 'src/channels/index.ts'), '// channel adapter barrel\n');
|
||||
writeFileSync(join(root, 'src/providers/index.ts'), '// provider barrel\n');
|
||||
writeFileSync(join(root, 'container/agent-runner/src/providers/index.ts'), '// container provider barrel\n');
|
||||
writeFileSync(join(root, 'setup/providers/index.ts'), '// setup provider barrel\n');
|
||||
writeFileSync(
|
||||
join(root, 'setup/index.ts'),
|
||||
['const STEPS = {', ' // >>> nanoclaw:setup-steps', ' // <<< nanoclaw:setup-steps', '};', ''].join('\n'),
|
||||
);
|
||||
writeFileSync(join(root, 'container/cli-tools.json'), '[]\n');
|
||||
return root;
|
||||
}
|
||||
|
||||
interface RunOutcome {
|
||||
res: ApplyResult;
|
||||
cmds: string[];
|
||||
streamed: string[];
|
||||
events: ApplyEvent[];
|
||||
}
|
||||
|
||||
async function runScenario(skillDir: string, sc: Scenario): Promise<RunOutcome> {
|
||||
const root = scratchRoot();
|
||||
const cmds: string[] = [];
|
||||
const streamed: string[] = [];
|
||||
const events: ApplyEvent[] = [];
|
||||
const res = await applySkill(skillDir, root, {
|
||||
inputs: sc.inputs ?? {},
|
||||
// resolveRemote MUST be injected: the default shells out to real
|
||||
// `git remote` + `git ls-remote` — network in CI, nondeterministic on forks.
|
||||
resolveRemote: () => 'origin',
|
||||
exec: (c) => {
|
||||
cmds.push(c);
|
||||
return sc.exec?.find((e) => c.includes(e.match))?.stdout;
|
||||
},
|
||||
execStream: async (c) => {
|
||||
streamed.push(c);
|
||||
return { ok: true, fields: sc.stepFields ?? {} };
|
||||
},
|
||||
onEvent: (e) => void events.push(e),
|
||||
});
|
||||
return { res, cmds, streamed, events };
|
||||
}
|
||||
|
||||
const isString = (x: unknown): x is string => typeof x === 'string';
|
||||
const SIDE_EFFECTS = new Set(['restart', 'step', 'wire']);
|
||||
const SABOTEUR_EFFECTS = new Set(['fetch', 'check', 'external']);
|
||||
const isSideEffectRun = (d: Directive | undefined): boolean =>
|
||||
d?.kind === 'run' && isString(d.attrs.effect) && SIDE_EFFECTS.has(d.attrs.effect);
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('skill discovery', () => {
|
||||
it('finds the fence-carrying skills', () => {
|
||||
// Sanity floor: discovery walking the wrong directory (or the fence regex
|
||||
// regressing) must fail loudly, not silently skip the whole suite.
|
||||
expect(SKILLS).toContain('add-slack');
|
||||
expect(SKILLS).toContain('add-whatsapp');
|
||||
expect(SKILLS.length).toBeGreaterThanOrEqual(10);
|
||||
});
|
||||
});
|
||||
|
||||
describe.each(SKILLS)('%s', (name) => {
|
||||
const dir = join(SKILLS_DIR, name);
|
||||
const md = readFileSync(join(dir, 'SKILL.md'), 'utf8');
|
||||
const directives = parseDirectives(md);
|
||||
const byLine = new Map(directives.map((d) => [d.line, d]));
|
||||
const promptVars = new Set(
|
||||
directives.filter((d) => d.kind === 'prompt').map((d) => promptVar(d)).filter(isString),
|
||||
);
|
||||
const guards = [...new Set(directives.map((d) => d.attrs.when).filter(isString))];
|
||||
const fixture = loadFixture(name);
|
||||
const scenarios: Scenario[] = fixture?.scenarios ?? [{ name: 'default', inputs: {} }];
|
||||
|
||||
it('parses + validates + warn-lints clean', () => {
|
||||
expect(validate(directives, { chatVersion: CHAT_VERSION })).toEqual([]);
|
||||
// Advisory warnings too: in-tree skills must stay warning-free.
|
||||
expect(lintGateAmbiguity(directives)).toEqual([]);
|
||||
expect(lintReferenceFloor(md)).toEqual([]);
|
||||
});
|
||||
|
||||
it('fixture hygiene: prompts have a fixture, inputs match prompt vars, unguarded prompts always answered', () => {
|
||||
if (promptVars.size > 0) {
|
||||
expect(
|
||||
fixture,
|
||||
`${name} declares nc:prompt directives but has no apply-fixtures.json — add .claude/skills/${name}/apply-fixtures.json with a "scenarios" array: shaped fake inputs per prompt var (satisfying each validate: regex), exec stubs (substring match → stdout) for every capture run, and stepFields for any effect:step`,
|
||||
).toBeDefined();
|
||||
}
|
||||
const unguarded = directives
|
||||
.filter((d) => d.kind === 'prompt' && !isString(d.attrs.when))
|
||||
.map((d) => promptVar(d))
|
||||
.filter(isString);
|
||||
for (const sc of fixture?.scenarios ?? []) {
|
||||
for (const k of Object.keys(sc.inputs ?? {})) {
|
||||
expect(promptVars.has(k), `scenario "${sc.name}" supplies input "${k}" which is not a prompt var of ${name} — stale fixture?`).toBe(true);
|
||||
}
|
||||
for (const v of unguarded) {
|
||||
expect(
|
||||
sc.inputs?.[v],
|
||||
`scenario "${sc.name}" is missing unguarded prompt var "${v}" — it would defer and the apply could never be fully programmatic`,
|
||||
).toBeDefined();
|
||||
}
|
||||
}
|
||||
// coverageExclude entries must reference guards that actually exist.
|
||||
const guardSet = new Set(guards);
|
||||
for (const g of fixture?.coverageExclude ?? []) {
|
||||
expect(guardSet.has(g), `coverageExclude "${g}" is not a when:-guard of ${name} — stale exclusion?`).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it.each(scenarios)('applies fully programmatically: $name', async (sc) => {
|
||||
const { res, events } = await runScenario(dir, sc);
|
||||
expect(res.agentTasks).toEqual([]); // nothing degraded to an agent
|
||||
expect(res.deferred).toEqual([]); // every prompt satisfied AND validate-at-bind passed
|
||||
expect(fullyApplied(res)).toBe(true);
|
||||
// Balanced step brackets, all green.
|
||||
const starts = events.filter((e) => e.type === 'step-start');
|
||||
const ends = events.filter((e): e is Extract<ApplyEvent, { type: 'step-end' }> => e.type === 'step-end');
|
||||
expect(starts.length).toBe(ends.length);
|
||||
expect(ends.every((e) => e.ok)).toBe(true);
|
||||
// Every resolved non-secret var is non-empty — an empty capture means an
|
||||
// exec/stepFields fixture entry is missing (bindCapture binds '' when the
|
||||
// stub returned nothing and no validate: catches it).
|
||||
for (const [k, v] of Object.entries(res.vars)) {
|
||||
expect(v, `resolved {{${k}}} is empty in scenario "${sc.name}" — add/fix the exec or stepFields fixture entry answering that capture`).not.toBe('');
|
||||
}
|
||||
});
|
||||
|
||||
it('covers every when:-guard value across scenarios', async () => {
|
||||
if (guards.length === 0) return;
|
||||
const excluded = new Set(fixture?.coverageExclude ?? []);
|
||||
const results: ApplyResult[] = [];
|
||||
for (const sc of scenarios) results.push((await runScenario(dir, sc)).res);
|
||||
for (const g of guards) {
|
||||
if (excluded.has(g)) continue;
|
||||
const eq = g.indexOf('=');
|
||||
const [v, val] = [g.slice(0, eq), g.slice(eq + 1)];
|
||||
expect(
|
||||
results.some((r) => r.vars[v] === val),
|
||||
`no scenario exercises when:${g} in ${name} — add a scenario to apply-fixtures.json (or coverageExclude it with a reason)`,
|
||||
).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
// Static, document-order invariants. Deliberately NOT "restart last":
|
||||
// telegram/whatsapp/signal restart BEFORE their pairing effect:step (the
|
||||
// adapter must be live to pair) and whatsapp writes env after the restart.
|
||||
// What DOES hold: code mutations land before the build, the build runs
|
||||
// before the tests, and a restart never precedes the build or the tests
|
||||
// that validate what it would load.
|
||||
it('effect ordering: mutations → build → test; restart only after build+test', () => {
|
||||
const firstBuild = directives.findIndex((d) => d.kind === 'run' && d.attrs.effect === 'build');
|
||||
const firstTest = directives.findIndex((d) => d.kind === 'run' && d.attrs.effect === 'test');
|
||||
if (firstBuild >= 0 && firstTest >= 0) expect(firstBuild).toBeLessThan(firstTest);
|
||||
if (firstBuild >= 0) {
|
||||
directives.forEach((d, i) => {
|
||||
if (['copy', 'append', 'dep', 'json-merge'].includes(d.kind)) {
|
||||
expect(i, `${d.kind} at line ${d.line} lands after the build — the build would not see it`).toBeLessThan(firstBuild);
|
||||
}
|
||||
});
|
||||
}
|
||||
directives.forEach((d, i) => {
|
||||
if (d.kind === 'run' && d.attrs.effect === 'restart') {
|
||||
if (firstBuild >= 0) expect(i, `restart at line ${d.line} precedes the build`).toBeGreaterThan(firstBuild);
|
||||
if (firstTest >= 0) expect(i, `restart at line ${d.line} precedes the tests`).toBeGreaterThan(firstTest);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// A restart-shaped command on a bare `nc:run` (no effect:) would silently
|
||||
// escape both skipEffects ownership and the run-health gate.
|
||||
it('no restart-shaped command hides on a bare nc:run', () => {
|
||||
for (const d of directives) {
|
||||
if (d.kind !== 'run' || d.attrs.effect !== undefined) continue;
|
||||
for (const cmd of d.body) {
|
||||
expect(
|
||||
/restart\.sh|kickstart|systemctl/.test(cmd),
|
||||
`bare nc:run at line ${d.line} runs a restart-shaped command ("${cmd}") without effect:restart — it would evade the run-health gate`,
|
||||
).toBe(false);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Dynamic twin of the engine's run-health-gate unit tests, per real skill:
|
||||
// inject a failure at the first fetch/check/external run a scenario reaches
|
||||
// and assert no restart/step/wire executes afterwards — authoring that
|
||||
// evades the gate (or a gate regression against real documents) fails here.
|
||||
it('run-health gate: a failed fetch/check/external blocks every later restart/step/wire', async () => {
|
||||
for (const sc of scenarios) {
|
||||
let current: Directive | undefined;
|
||||
let sabotagedLine = -1;
|
||||
let streamedAfterSabotage = 0;
|
||||
const sideEffectStartsAfterSabotage: number[] = [];
|
||||
const root = scratchRoot();
|
||||
const res = await applySkill(dir, root, {
|
||||
inputs: sc.inputs ?? {},
|
||||
resolveRemote: () => 'origin',
|
||||
onEvent: (e) => {
|
||||
if (e.type !== 'step-start') return;
|
||||
current = byLine.get(e.line);
|
||||
if (sabotagedLine >= 0 && isSideEffectRun(current)) sideEffectStartsAfterSabotage.push(e.line);
|
||||
},
|
||||
exec: (c) => {
|
||||
if (
|
||||
sabotagedLine < 0 &&
|
||||
current?.kind === 'run' &&
|
||||
isString(current.attrs.effect) &&
|
||||
SABOTEUR_EFFECTS.has(current.attrs.effect)
|
||||
) {
|
||||
sabotagedLine = current.line;
|
||||
throw new Error('conformance sabotage');
|
||||
}
|
||||
return sc.exec?.find((e) => c.includes(e.match))?.stdout;
|
||||
},
|
||||
execStream: async () => {
|
||||
if (sabotagedLine >= 0) streamedAfterSabotage++;
|
||||
return { ok: true, fields: sc.stepFields ?? {} };
|
||||
},
|
||||
});
|
||||
if (sabotagedLine < 0) continue; // this scenario never reaches a saboteur-eligible run
|
||||
// The sabotaged run itself bounced to an agent…
|
||||
expect(fullyApplied(res)).toBe(false);
|
||||
expect(res.agentTasks.some((t) => t.line === sabotagedLine)).toBe(true);
|
||||
// …and no dangerous side effect fired on its own afterwards.
|
||||
expect(sideEffectStartsAfterSabotage).toEqual([]);
|
||||
expect(streamedAfterSabotage).toBe(0);
|
||||
// Any unguarded later side effect must surface as a gated agentTask, so
|
||||
// an agent finishes it from the prose once the failure is fixed.
|
||||
const gatedExpected = directives.some(
|
||||
(d) => isSideEffectRun(d) && !isString(d.attrs.when) && d.line > sabotagedLine,
|
||||
);
|
||||
if (gatedExpected) {
|
||||
expect(res.agentTasks.some((t) => /earlier step did not complete/.test(t.reason))).toBe(true);
|
||||
}
|
||||
return; // one sabotaged scenario per skill is enough
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,82 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { mkdtempSync, mkdirSync, readFileSync, writeFileSync, rmSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
|
||||
import { inputsFromEnv } from './skill-inputs.js';
|
||||
import { applySkill, fullyApplied } from './skill-apply.js';
|
||||
|
||||
describe('inputsFromEnv (docs/skill-engine-seam.md §6)', () => {
|
||||
it('maps NC_INPUT_<VAR> env keys onto prompt vars, ignoring unset and unrelated keys', () => {
|
||||
const md = [
|
||||
'```nc:prompt bot_token secret',
|
||||
'Paste the token.',
|
||||
'```',
|
||||
'```nc:prompt owner_handle',
|
||||
'Your handle?',
|
||||
'```',
|
||||
].join('\n');
|
||||
const inputs = inputsFromEnv(md, {
|
||||
NC_INPUT_BOT_TOKEN: 'xoxb-fake',
|
||||
NC_INPUT_UNRELATED: 'ignored', // no matching prompt
|
||||
PATH: '/usr/bin', // not NC_INPUT_-prefixed
|
||||
// NC_INPUT_OWNER_HANDLE deliberately unset → omitted, not ''
|
||||
});
|
||||
expect(inputs).toEqual({ bot_token: 'xoxb-fake' });
|
||||
});
|
||||
|
||||
it('errors on an uppercase collision instead of silently merging', () => {
|
||||
const md = [
|
||||
'```nc:prompt bot_token',
|
||||
'Token?',
|
||||
'```',
|
||||
'```nc:prompt Bot_Token',
|
||||
'Token again?',
|
||||
'```',
|
||||
].join('\n');
|
||||
expect(() => inputsFromEnv(md, {})).toThrow(/NC_INPUT_BOT_TOKEN/);
|
||||
});
|
||||
|
||||
// The round-trip proof for one real skill: env → inputsFromEnv → applySkill
|
||||
// goes fully green for add-slack (webhook leg) with stubbed exec — the exact
|
||||
// pipeline-consumer path the seam doc's §6 contract describes.
|
||||
it('round-trips the env convention through a full programmatic apply of add-slack', async () => {
|
||||
const skillDir = join(process.cwd(), '.claude/skills/add-slack');
|
||||
const md = readFileSync(join(skillDir, 'SKILL.md'), 'utf8');
|
||||
const inputs = inputsFromEnv(md, {
|
||||
NC_INPUT_CONNECTION: 'webhook',
|
||||
NC_INPUT_BOT_TOKEN: 'xoxb-fake-token',
|
||||
NC_INPUT_SIGNING_SECRET: '0123456789abcdef',
|
||||
NC_INPUT_OWNER_HANDLE: 'U12345678',
|
||||
});
|
||||
expect(inputs).toEqual({
|
||||
connection: 'webhook',
|
||||
bot_token: 'xoxb-fake-token',
|
||||
signing_secret: '0123456789abcdef',
|
||||
owner_handle: 'U12345678',
|
||||
});
|
||||
|
||||
const root = mkdtempSync(join(tmpdir(), 'skill-inputs-'));
|
||||
try {
|
||||
mkdirSync(join(root, 'src/channels'), { recursive: true });
|
||||
writeFileSync(join(root, 'src/channels/index.ts'), '// barrel\n');
|
||||
writeFileSync(join(root, '.env'), '');
|
||||
writeFileSync(join(root, 'package.json'), '{"name":"scratch"}\n');
|
||||
|
||||
const res = await applySkill(skillDir, root, {
|
||||
inputs,
|
||||
exec: (c) => {
|
||||
if (c.includes('auth.test')) return '@nano in Acme';
|
||||
if (c.includes('conversations.open')) return 'slack:D0FAKE';
|
||||
},
|
||||
resolveRemote: () => 'origin',
|
||||
});
|
||||
expect(res.deferred).toEqual([]);
|
||||
expect(res.agentTasks).toEqual([]);
|
||||
expect(fullyApplied(res)).toBe(true);
|
||||
expect(res.vars.platform_id).toBe('slack:D0FAKE');
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,32 @@
|
||||
// inputsFromEnv — the pipeline consumer's input path (docs/skill-engine-seam.md §6).
|
||||
//
|
||||
// A CI/pipeline caller supplies prompt answers as environment variables using
|
||||
// the `NC_INPUT_<VAR>` convention (prompt var uppercased). This helper parses
|
||||
// the skill's prompt vars via parseDirectives and returns the `inputs` record
|
||||
// applySkill consumes. It is a helper, not an engine feature — the engine never
|
||||
// reads process.env for inputs; `inputs` stays the only env-agnostic seam.
|
||||
//
|
||||
// Var names are case-sensitive in the grammar, so uppercasing can collide
|
||||
// (`bot_token` vs `Bot_Token` both map to NC_INPUT_BOT_TOKEN); a collision is
|
||||
// an error, never a silent last-writer-wins.
|
||||
|
||||
import { parseDirectives, promptVar } from './skill-directives.js';
|
||||
|
||||
export function inputsFromEnv(md: string, env: Record<string, string | undefined> = process.env): Record<string, string> {
|
||||
const inputs: Record<string, string> = {};
|
||||
const byKey = new Map<string, string>(); // NC_INPUT_<VAR> → the prompt var that claimed it
|
||||
for (const d of parseDirectives(md)) {
|
||||
if (d.kind !== 'prompt') continue;
|
||||
const v = promptVar(d);
|
||||
if (!v) continue;
|
||||
const key = `NC_INPUT_${v.toUpperCase()}`;
|
||||
const prior = byKey.get(key);
|
||||
if (prior !== undefined && prior !== v) {
|
||||
throw new Error(`inputsFromEnv: prompt vars "${prior}" and "${v}" both map to ${key} — rename one (var names must be case-insensitively unique)`);
|
||||
}
|
||||
byKey.set(key, v);
|
||||
const val = env[key];
|
||||
if (val !== undefined) inputs[v] = val;
|
||||
}
|
||||
return inputs;
|
||||
}
|
||||
Reference in New Issue
Block a user