syntaxai/tdd.md · main · src / c14_real_tests.ts

// c32 — logic: aggregate the per-deploy test bundle into the same
// TestSnapshot[] / TestStability[] shape that the demo page renders.
// HEAD-only snapshots; stability accumulates as more deploys add runs.
//
// Pure given the bundle + commits in (no I/O of its own beyond delegating
// to c14_github's bundle loader and commits fetcher).

import { fetchRepoCommits, loadTestBundle, type PlaceholderTest } from "./c14_github.ts";
import type {
  AgentReport,
  TestFailure,
  TestSnapshot,
  TestStability,
} from "./a31_reports_demo.ts";

export const detectAgent = (msg: string): AgentReport["slug"] | null => {
  if (/Co-Authored-By:.*Claude/i.test(msg)) return "claude-code";
  if (/Co-Authored-By:.*Cursor/i.test(msg)) return "cursor";
  if (/Co-Authored-By:.*Aider/i.test(msg)) return "aider";
  return null;
};

export const shortenTestLabel = (file: string, name: string): string => {
  const base = file.split("/").pop() ?? file;
  return `${base} > ${name}`;
};

export interface LiveTestData {
  snapshots: TestSnapshot[];
  stability: TestStability[];
  runsCount: number;
  ranAt: number | null;
  headSha: string | null;
  placeholderTests: PlaceholderTest[];
}

export const buildLiveTestData = async (
  repoOwner: string,
  repoName: string,
): Promise<LiveTestData> => {
  const bundle = await loadTestBundle(repoOwner, repoName);
  if (!bundle || bundle.runs.length === 0) {
    return { snapshots: [], stability: [], runsCount: 0, ranAt: null, headSha: null, placeholderTests: [] };
  }
  const repoSlug = `${repoOwner}/${repoName}`;
  const latest = bundle.runs[0];
  if (!latest) {
    return { snapshots: [], stability: [], runsCount: 0, ranAt: null, headSha: null, placeholderTests: [] };
  }

  // For "since" we want the oldest run that has this test as failing.
  const oldestFirst = [...bundle.runs].sort((a, b) => a.ranAt - b.ranAt);

  const failures: TestFailure[] = latest.tests
    .filter((t) => t.status === "fail")
    .map((t) => {
      const firstFail = oldestFirst.find((r) =>
        r.tests.some((x) => x.name === t.name && x.file === t.file && x.status === "fail"),
      );
      const sinceTs = firstFail?.ranAt ?? latest.ranAt;
      return { test: shortenTestLabel(t.file, t.name), since: new Date(sinceTs).toISOString().slice(0, 10) };
    });

  const snapshot: TestSnapshot = {
    repo: repoSlug,
    branch: latest.branch,
    total: latest.total,
    passing: latest.passing,
    failing: latest.failing,
    failures,
  };

  // Stability: count pass/fail per (file, name) across every run, with
  // "deleted" set when a previously-seen test is missing from latest.
  const commits = await fetchRepoCommits(repoOwner, repoName, 100);
  const shaToAgent = new Map<string, AgentReport["slug"] | null>();
  for (const c of commits) shaToAgent.set(c.sha, detectAgent(c.commit.message));

  interface Stat {
    name: string;
    file: string;
    pass: number;
    fail: number;
    lastBrokenSha: string | null;
    lastBrokenAt: number;
  }
  const stats = new Map<string, Stat>();
  for (const run of bundle.runs) {
    for (const t of run.tests) {
      const key = `${t.file}|${t.name}`;
      let s = stats.get(key);
      if (!s) {
        s = { name: t.name, file: t.file, pass: 0, fail: 0, lastBrokenSha: null, lastBrokenAt: 0 };
        stats.set(key, s);
      }
      if (t.status === "pass") s.pass++;
      else {
        s.fail++;
        if (run.ranAt > s.lastBrokenAt) {
          s.lastBrokenSha = run.sha;
          s.lastBrokenAt = run.ranAt;
        }
      }
    }
  }

  const latestKeys = new Set(latest.tests.map((t) => `${t.file}|${t.name}`));

  // lastBrokenBy needs an agent slug; if we can't map a SHA to an agent
  // (e.g. the commit isn't in the 100-commit window we fetch), fall
  // back to the agent of the latest run, which is a defensible default
  // for the dogfood case (one agent producing the history).
  const fallbackAgent = (shaToAgent.get(latest.sha) ?? "claude-code") as AgentReport["slug"];

  const stability: TestStability[] = Array.from(stats.values())
    .map<TestStability>((s) => {
      const mapped = s.lastBrokenSha ? shaToAgent.get(s.lastBrokenSha) : null;
      const agent = (mapped ?? fallbackAgent) as AgentReport["slug"];
      const deleted = latestKeys.has(`${s.file}|${s.name}`) ? 0 : 1;
      const flagged = s.fail > 0 && (deleted > 0 || s.fail >= Math.max(2, s.pass / 5));
      return {
        test: shortenTestLabel(s.file, s.name),
        repo: repoSlug,
        pass: s.pass,
        fail: s.fail,
        deleted,
        lastBrokenBy: agent,
        flagged,
      };
    })
    .sort((a, b) => b.fail - a.fail || b.deleted - a.deleted || b.pass - a.pass)
    .slice(0, 30);

  return {
    snapshots: [snapshot],
    stability,
    runsCount: bundle.runs.length,
    ranAt: latest.ranAt,
    headSha: latest.sha,
    placeholderTests: latest.placeholderTests ?? [],
  };
};