Eval Set Authoring

/**
 * EvalSetAuthoring — headless core (primitive, watching).
 * Pattern: eval-set-authoring · family: shared-context · lifecycle: watching
 * Solves: turning "I know it when I see it" into examples and a standard a
 * test can use.
 *
 * Generated from the five-part anatomy on /patterns/eval-set-authoring. Zero
 * CSS beyond data attributes; the styled skin lands in patterns-ui when the
 * anatomy stabilizes (see /methodology).
 *
 * Anatomy (who holds what):
 *   01 Source   — human     — where examples come from: traffic | generated | paste
 *   02 Cases    — shared    — each case has an input and, where known, a good answer
 *   03 Rubric   — human     — plain sentences: what a good answer does / does not do
 *   04 PassFail — human     — one passing and one failing answer beside each rule
 *   05 Version  — interface — set + rubric versioned beside the prompt version
 */
import * as React from 'react';
import { createEvent, type EventDraft, type InteractionEventHandler } from '@/lib/interaction-contract';

// ---------- types ----------

export type EvalSource = 'traffic' | 'generated' | 'paste';

export interface EvalCase {
  id: string;
  input: string;
  /** Known-good answer, if the author has one. Optional on purpose. */
  ideal?: string;
  source: EvalSource;
}

export interface RubricRule {
  id: string;
  /** Plain sentence. "The answer names the refund window." */
  text: string;
  /** What makes the rule a shared standard rather than one person's taste. */
  pass?: string;
  fail?: string;
}

export interface EvalSetVersion {
  set: string; // e.g. "v3"
  rubric: string; // e.g. "v2"
  prompt: string; // the prompt version this set was authored against
}

export interface EvalSetState {
  source: EvalSource;
  cases: EvalCase[];
  rubric: RubricRule[];
  version: EvalSetVersion;
}

export type EvalSetAction =
  | { type: 'setSource'; source: EvalSource }
  | { type: 'addCase'; case: EvalCase }
  | { type: 'removeCase'; id: string }
  | { type: 'addRule'; rule: RubricRule }
  | { type: 'updateRule'; id: string; patch: Partial<RubricRule> }
  | { type: 'removeRule'; id: string }
  | { type: 'bumpVersion'; part: keyof EvalSetVersion; value: string };

// ---------- pure helpers (tested without React) ----------

export function reducer(state: EvalSetState, a: EvalSetAction): EvalSetState {
  switch (a.type) {
    case 'setSource':
      return { ...state, source: a.source };
    case 'addCase':
      return { ...state, cases: [...state.cases, a.case] };
    case 'removeCase':
      return { ...state, cases: state.cases.filter((c) => c.id !== a.id) };
    case 'addRule':
      return { ...state, rubric: [...state.rubric, a.rule] };
    case 'updateRule':
      return {
        ...state,
        rubric: state.rubric.map((r) => (r.id === a.id ? { ...r, ...a.patch } : r)),
      };
    case 'removeRule':
      return { ...state, rubric: state.rubric.filter((r) => r.id !== a.id) };
    case 'bumpVersion':
      return { ...state, version: { ...state.version, [a.part]: a.value } };
  }
}

/** Rules missing a pass or a fail example — the "one person's taste" failure mode. */
export function unsharedRules(rubric: RubricRule[]): RubricRule[] {
  return rubric.filter((r) => !r.pass || !r.fail);
}

/** True when no case came from real traffic — the "synthetic drift" failure mode. */
export function isSyntheticOnly(cases: EvalCase[]): boolean {
  return cases.length > 0 && cases.every((c) => c.source !== 'traffic');
}

/** A case set as a JSONL fixture — one object per line, blank lines ignored. */
export function parseJsonl<T = unknown>(text: string): T[] {
  return text
    .split('\n')
    .map((l) => l.trim())
    .filter(Boolean)
    .map((l) => JSON.parse(l) as T);
}

/**
 * The contract event an authoring action produces (spec 2026-10-04 §1), or null.
 * Adding or removing a case or a rule and switching the source are
 * review.recorded; a version bump is baseline.versioned, because the set the
 * next comparison runs against has changed. Keystrokes inside a rule
 * (updateRule) emit nothing; `onChange` carries every state.
 */
export function authoringEvent(before: EvalSetState, a: EvalSetAction): EventDraft | null {
  switch (a.type) {
    case 'setSource':
      return before.source === a.source
        ? null
        : { type: 'review.recorded', actor: 'human', payload: { kind: 'source', source: a.source } };
    case 'addCase':
      return { type: 'review.recorded', actor: 'human', correlationId: a.case.id, payload: { kind: 'case', change: 'added', case: a.case } };
    case 'removeCase':
      return before.cases.some((c) => c.id === a.id)
        ? { type: 'review.recorded', actor: 'human', correlationId: a.id, payload: { kind: 'case', change: 'removed', caseId: a.id } }
        : null;
    case 'addRule':
      return { type: 'review.recorded', actor: 'human', correlationId: a.rule.id, payload: { kind: 'rubric', change: 'added', rule: a.rule } };
    case 'removeRule':
      return before.rubric.some((r) => r.id === a.id)
        ? { type: 'review.recorded', actor: 'human', correlationId: a.id, payload: { kind: 'rubric', change: 'removed', ruleId: a.id } }
        : null;
    case 'bumpVersion':
      return before.version[a.part] === a.value
        ? null
        : {
            type: 'baseline.versioned',
            actor: 'human',
            payload: { part: a.part, from: before.version[a.part], version: { ...before.version, [a.part]: a.value } },
          };
    case 'updateRule':
      return null;
  }
}

// ---------- context ----------

interface Ctx {
  state: EvalSetState;
  dispatch: React.Dispatch<EvalSetAction>;
  /** Rules missing a pass or fail example — the "one person's taste" failure mode. */
  unsharedRules: RubricRule[];
}

const EvalSetCtx = React.createContext<Ctx | null>(null);

export function useEvalSet(): Ctx {
  const ctx = React.useContext(EvalSetCtx);
  if (!ctx) throw new Error('useEvalSet must be used inside <EvalSetAuthoring.Root>');
  return ctx;
}

// ---------- Root ----------

export interface RootProps {
  initial: EvalSetState;
  onChange?: (state: EvalSetState) => void;
  /** One agent run; stamped on every contract event. Defaults to a generated id. */
  runId?: string;
  /** Contract events: review.recorded, baseline.versioned. */
  onEvent?: InteractionEventHandler;
  /** The parts, in any order; a story or a skin composes them. */
  children?: React.ReactNode;
}

function Root({ initial, onChange, runId, onEvent, children }: RootProps) {
  const [state, apply] = React.useReducer(reducer, initial);
  React.useEffect(() => onChange?.(state), [state, onChange]);
  const unshared = React.useMemo(() => unsharedRules(state.rubric), [state.rubric]);
  const latest = React.useRef(state);
  latest.current = state;
  const fallbackRun = React.useId();
  const run = runId ?? fallbackRun;
  const dispatch = React.useCallback<React.Dispatch<EvalSetAction>>(
    (a) => {
      const draft = authoringEvent(latest.current, a);
      apply(a);
      if (draft) onEvent?.(createEvent(run, draft));
    },
    [onEvent, run],
  );
  return (
    <EvalSetCtx.Provider value={{ state, dispatch, unsharedRules: unshared }}>
      <section data-pattern="eval-set-authoring" data-lifecycle="watching" aria-label="Eval set authoring">
        {children}
      </section>
    </EvalSetCtx.Provider>
  );
}

// ---------- 01 Source ----------

const SOURCES: { value: EvalSource; label: string; hint: string }[] = [
  { value: 'traffic', label: 'Real traffic', hint: 'Keeps the set honest to how people phrase things.' },
  { value: 'generated', label: 'Generated', hint: 'Cheap and plentiful; can drift from real phrasing.' },
  { value: 'paste', label: 'Paste', hint: 'Fast start; no provenance.' },
];

function Source() {
  const { state, dispatch } = useEvalSet();
  const synthetic = isSyntheticOnly(state.cases);
  return (
    <fieldset data-part="source" data-actor="human" data-synthetic-only={synthetic || undefined}>
      <legend>01 · Source</legend>
      {SOURCES.map((s) => (
        <label key={s.value} title={s.hint}>
          <input
            type="radio"
            name="eval-source"
            value={s.value}
            checked={state.source === s.value}
            onChange={() => dispatch({ type: 'setSource', source: s.value })}
          />
          {s.label}
        </label>
      ))}
      {synthetic && (
        <p role="status" data-warning="synthetic-only">
          No case came from real traffic — the set can drift from how people phrase things.
        </p>
      )}
    </fieldset>
  );
}

// ---------- 02 Cases ----------

function Cases() {
  const { state, dispatch } = useEvalSet();
  return (
    <div data-part="cases" data-actor="shared">
      <h3>
        02 · Cases <small>({state.cases.length})</small>
      </h3>
      <table>
        <thead>
          <tr>
            <th>Input</th>
            <th>Good answer</th>
            <th>From</th>
            <th>Remove</th>
          </tr>
        </thead>
        <tbody>
          {state.cases.map((c) => (
            <tr key={c.id} data-case-source={c.source}>
              <td>{c.input}</td>
              <td>{c.ideal ?? <em data-empty>not yet known</em>}</td>
              <td>{c.source}</td>
              <td>
                <button type="button" onClick={() => dispatch({ type: 'removeCase', id: c.id })}>
                  remove
                </button>
              </td>
            </tr>
          ))}
        </tbody>
      </table>
    </div>
  );
}

// ---------- 03 Rubric + 04 Pass and fail examples ----------

function newRuleId(): string {
  return typeof crypto !== 'undefined' && 'randomUUID' in crypto
    ? crypto.randomUUID()
    : `r-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
}

function Rubric() {
  const { state, dispatch, unsharedRules } = useEvalSet();
  return (
    <div data-part="rubric" data-actor="human">
      <h3>03 · Rubric</h3>
      {unsharedRules.length > 0 && (
        <p role="status" data-warning="unshared-rules">
          {unsharedRules.length} rule{unsharedRules.length === 1 ? '' : 's'} without a pass and a fail example — a
          preference, not yet a standard.
        </p>
      )}
      <ol>
        {state.rubric.map((r) => (
          <li key={r.id} data-rule-shared={Boolean(r.pass && r.fail)}>
            <textarea
              aria-label="Rule"
              value={r.text}
              onChange={(e) => dispatch({ type: 'updateRule', id: r.id, patch: { text: e.target.value } })}
            />
            <PassFail rule={r} />
          </li>
        ))}
      </ol>
      <button type="button" onClick={() => dispatch({ type: 'addRule', rule: { id: newRuleId(), text: '' } })}>
        Add rule
      </button>
    </div>
  );
}

function PassFail({ rule }: { rule: RubricRule }) {
  const { dispatch } = useEvalSet();
  return (
    <div data-part="pass-fail" data-actor="human">
      <label>
        <span data-verdict="pass">Passes</span>
        <textarea
          value={rule.pass ?? ''}
          placeholder="One answer that passes this rule"
          onChange={(e) => dispatch({ type: 'updateRule', id: rule.id, patch: { pass: e.target.value } })}
        />
      </label>
      <label>
        <span data-verdict="fail">Fails</span>
        <textarea
          value={rule.fail ?? ''}
          placeholder="One answer that fails this rule"
          onChange={(e) => dispatch({ type: 'updateRule', id: rule.id, patch: { fail: e.target.value } })}
        />
      </label>
    </div>
  );
}

// ---------- 05 Version ----------

function Version() {
  const { state } = useEvalSet();
  const { set, rubric, prompt } = state.version;
  return (
    <dl data-part="version" data-actor="interface">
      <dt>Set</dt>
      <dd>{set}</dd>
      <dt>Rubric</dt>
      <dd>{rubric}</dd>
      <dt>Prompt</dt>
      <dd>{prompt}</dd>
    </dl>
  );
}

// ---------- export ----------

export const EvalSetAuthoring = Object.assign(Root, {
  Root,
  Source,
  Cases,
  Rubric,
  PassFail,
  Version,
});