#!/usr/bin/env node
// Generate evals/cases.full.json — a representative input for EVERY skill, so the eval
// harness can score the whole library (not just the ~20 hand-curated cases).
//
// Curated cases (evals/cases.json) are kept verbatim — they're tailored and authoritative.
// Every other skill gets an auto-generated "demonstrate on a realistic example" input,
// which lets the skill produce a full artifact the judge can score. Treat auto-generated
// scores as an honest baseline (the input isn't bespoke), not a tuned benchmark.
//
//   node scripts/gen-eval-cases.mjs
//   node evals/run-evals.mjs --cases evals/cases.full.json --models claude-haiku-4-5-20251001 --judge claude-haiku-4-5-20251001
//
// No dependencies.
import { readFileSync, writeFileSync, readdirSync, existsSync } from 'node:fs';
import { join, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';

const root = join(dirname(fileURLToPath(import.meta.url)), '..');
const skillsDir = join(root, 'skills');

function parse(md) {
  const m = md.match(/^---\n([\s\S]*?)\n---/);
  const meta = {};
  if (m) for (const line of m[1].split('\n')) {
    const mm = line.match(/^(\w+):\s*(.*)$/);
    if (mm) meta[mm[1]] = mm[2].replace(/^["']|["']$/g, '');
  }
  const h = md.match(/^#\s+(.+)$/m);
  return { name: meta.name, description: meta.description || '', title: (h ? h[1] : meta.name || '').replace(/\s+Skill$/i, '') };
}
const firstSentence = (s) => (s.split(/(?<=[.!?])\s/)[0] || s).slice(0, 200);

// Curated, tailored inputs win.
const curated = JSON.parse(readFileSync(join(root, 'evals', 'cases.json'), 'utf8')).cases;
const curatedBySkill = new Map(curated.map((c) => [c.skill, c]));

// Skills that an LLM-as-judge can't fairly score on a text "artifact" rubric — they need
// an image, a live URL/account, or they activate a behaviour rather than produce a document.
// Scoring them yields a misleading low (grounding tanks because there's nothing to ground on),
// so we exclude them from the leaderboard and mark them "not eval-applicable" instead.
const EVAL_EXCLUDE = new Set([
  'chart-data-extractor',        // needs an image to read
  'substack-notes-scraper',      // needs a live URL
  'instagram-post-downloader',   // needs a live URL / tool
  'thumbnail-creator',           // produces an image
  'email-triage',                // needs a connected Gmail
  'notebooklm-connector',        // external tool / live source
  'morning-intelligence',        // needs live web data
  'context-mode',                // activates a session behaviour, not an artifact
  'claude-superpowers',          // activates a coding discipline, not an artifact
]);

const cases = [];
let generated = 0, excluded = 0;
for (const name of readdirSync(skillsDir).sort()) {
  const file = join(skillsDir, name, 'SKILL.md');
  if (!existsSync(file)) continue;
  const s = parse(readFileSync(file, 'utf8'));
  const skill = s.name || name;
  if (EVAL_EXCLUDE.has(skill)) { excluded++; continue; }
  if (curatedBySkill.has(skill)) { cases.push(curatedBySkill.get(skill)); continue; }
  cases.push({
    skill,
    generated: true,
    input: `Demonstrate the "${s.title}" skill end-to-end on a realistic example. ${firstSentence(s.description)} Invent a concrete, plausible scenario appropriate to this skill and produce the complete artifact — don't ask for more input.`,
  });
  generated++;
}

const out = {
  _comment: 'Full eval set: curated inputs (from cases.json) + auto-generated representative inputs for the rest. Generated by scripts/gen-eval-cases.mjs. Auto-generated cases are flagged "generated":true — treat their scores as an honest baseline, not a tuned benchmark.',
  cases,
};
writeFileSync(join(root, 'evals', 'cases.full.json'), JSON.stringify(out, null, 2) + '\n');
console.log(`Wrote evals/cases.full.json — ${cases.length} cases (${cases.length - generated} curated, ${generated} auto-generated; ${excluded} excluded as not eval-applicable).`);
console.log('Cheapest full run:\n  ANTHROPIC_API_KEY=sk-ant-… node evals/run-evals.mjs --cases evals/cases.full.json --models claude-haiku-4-5-20251001 --judge claude-haiku-4-5-20251001');
