feat(results): ExperimentTrendAnalyzer — cross-run pass rate regression detection across timestamp directories

// packages/agent-eval/src/lib/trend.ts

export interface EvalRunPoint {
  timestamp: string;
  passRate: number;
  passedRuns: number;
  totalRuns: number;
}

export interface EvalTrendResult {
  evalName: string;
  runCount: number;
  slope: number;              // % per run via OLS
  direction: 'improving' | 'degrading' | 'stable';
  recentPassRate: number;
  firstPassRate: number;
}

export interface ExperimentTrendReport {
  experimentName: string;
  window: number;
  evalTrends: EvalTrendResult[];
  anyRegression: boolean;
}

export function analyzeExperimentTrend(
  resultsDir: string,
  experimentName: string,
  window = 10,
): ExperimentTrendReport;

export function analyzeExperimentTrend(
  resultsDir: string,
  experimentName: string,
  window = 10,
): ExperimentTrendReport {
  const experimentDir = join(resultsDir, experimentName);
  const timestamps = readdirSync(experimentDir)
    .filter((t) => !t.startsWith('.') && statSync(join(experimentDir, t)).isDirectory())
    .sort()  // ISO timestamps sort chronologically
    .slice(-window);

  const evalHistory = new Map<string, EvalRunPoint[]>();

  for (const timestamp of timestamps) {
    const tsDir = join(experimentDir, timestamp);
    for (const evalDir of readdirSync(tsDir).filter((d) => !d.startsWith('.'))) {
      const summaryPath = join(tsDir, evalDir, 'summary.json');
      if (!existsSync(summaryPath)) continue;
      const summary = JSON.parse(readFileSync(summaryPath, 'utf-8'));
      if (!evalHistory.has(evalDir)) evalHistory.set(evalDir, []);
      evalHistory.get(evalDir)!.push({
        timestamp,
        passRate: summary.passRate,
        passedRuns: summary.passedRuns,
        totalRuns: summary.totalRuns,
      });
    }
  }

  const evalTrends: EvalTrendResult[] = [];
  for (const [evalName, points] of evalHistory) {
    if (points.length < 2) continue;
    const slope = olsSlope(points.map((p, i) => [i, p.passRate]));
    evalTrends.push({
      evalName,
      runCount: points.length,
      slope,
      direction: slope < -0.5 ? 'degrading' : slope > 0.5 ? 'improving' : 'stable',
      recentPassRate: points.at(-1)!.passRate,
      firstPassRate: points[0].passRate,
    });
  }

  return {
    experimentName,
    window,
    evalTrends,
    anyRegression: evalTrends.some((t) => t.direction === 'degrading'),
  };
}

function olsSlope(pairs: [number, number][]): number {
  const n = pairs.length;
  const meanX = pairs.reduce((s, [x]) => s + x, 0) / n;
  const meanY = pairs.reduce((s, [, y]) => s + y, 0) / n;
  const num = pairs.reduce((s, [x, y]) => s + (x - meanX) * (y - meanY), 0);
  const den = pairs.reduce((s, [x]) => s + (x - meanX) ** 2, 0);
  return den === 0 ? 0 : num / den;
}

// In cli.ts — new `trend` subcommand
program
  .command('trend [experimentName]')
  .option('--window <n>', 'number of runs to analyze', '10')
  .option('--exit-on-regression', 'exit 1 if any eval is degrading')
  .action(async (experimentName, opts) => {
    const report = analyzeExperimentTrend(resultsDir, experimentName, +opts.window);
    console.table(report.evalTrends);
    if (opts.exitOnRegression && report.anyRegression) process.exit(1);
  });