Generating and Maintaining Test Suites with Agent Loops

run pytest+coverage -> parse branch report
   -> any uncovered branches AND budget left?
        yes -> pick top-N branches, fetch source, generate tests, write file
        no  -> stop

# pyproject.toml
[project]
name = "coverage-agent"
version = "0.1.0"
requires-python = ">=3.12"
dependencies = ["anthropic==0.49.0"]

[project.optional-dependencies]
dev = ["pytest==8.3.5", "pytest-cov==6.1.1", "coverage==7.8.0"]

[tool.coverage.run]
branch = true            # branch coverage, not just line coverage
source = ["app"]

[tool.coverage.report]
show_missing = true
skip_covered = true

# loop/runner.py
import json
import subprocess
import sys
from pathlib import Path


def run_coverage() -> dict:
    """Run the suite, return coverage.py's JSON report as a dict."""
    subprocess.run(
        [sys.executable, "-m", "pytest", "-q",
         "--cov=app", "--cov-branch", "--cov-report="],
        capture_output=True, text=True,
    )
    # Generate the JSON report from the just-written .coverage file.
    subprocess.run(
        [sys.executable, "-m", "coverage", "json", "-o", "coverage.json"],
        capture_output=True, text=True, check=True,
    )
    data = json.loads(Path("coverage.json").read_text())
    return data


def suite_is_green() -> bool:
    """A failing suite means generated tests broke — stop the loop."""
    result = subprocess.run(
        [sys.executable, "-m", "pytest", "-q", "--no-header"],
        capture_output=True, text=True,
    )
    return result.returncode == 0

# loop/coverage_parser.py
from dataclasses import dataclass
from pathlib import Path


@dataclass
class BranchTarget:
    file: str
    source_line: int       # line where the branch decision lives
    snippet: str           # ~20 lines of context around it
    pct: float             # file-level branch coverage, for prioritising


def parse_targets(report: dict, context: int = 10) -> list[BranchTarget]:
    targets: list[BranchTarget] = []
    for path, file_data in report["files"].items():
        summary = file_data["summary"]
        pct = summary["percent_covered"]
        if pct >= 100:
            continue

        src_lines = Path(path).read_text().splitlines()
        seen: set[int] = set()
        # missing_branches: list of [from_line, to_line] pairs.
        for from_line, _to in file_data.get("missing_branches", []):
            if from_line in seen:
                continue
            seen.add(from_line)
            lo = max(0, from_line - context - 1)
            hi = min(len(src_lines), from_line + context)
            snippet = "\n".join(
                f"{i + 1:4d}| {src_lines[i]}" for i in range(lo, hi)
            )
            targets.append(BranchTarget(path, from_line, snippet, pct))

    # Lowest-coverage files first — biggest wins per iteration.
    targets.sort(key=lambda t: t.pct)
    return targets


def overall_branch_pct(report: dict) -> float:
    return report["totals"]["percent_covered"]

# loop/generator.py
import os
import re
from anthropic import Anthropic, APIStatusError
from loop.coverage_parser import BranchTarget

client = Anthropic(api_key=os.environ["ANTHROPIC_API_KEY"])

SYSTEM = """You write pytest 8 tests that exercise specific uncovered branches.
For each target you are given numbered source and a branch line.
Rules:
- Write one test per uncovered branch. Name it test_<behavior>_line_<N>.
- Drive inputs so the branch at the given line is taken.
- Add assertions on the observable result. Never write `assert True`.
- Import the module under test by its real dotted path.
- Use fixtures and parametrize where it reduces duplication.
- Output ONLY Python, no prose, no markdown fences."""


def generate_tests(targets: list[BranchTarget], batch: int = 6) -> str:
    chosen = targets[:batch]
    blocks = []
    for t in chosen:
        blocks.append(
            f"--- target: {t.file}, branch at line {t.source_line} ---\n"
            f"{t.snippet}"
        )
    prompt = (
        "Write tests covering these uncovered branches:\n\n"
        + "\n\n".join(blocks)
    )
    try:
        resp = client.messages.create(
            model="claude-sonnet-4-5-20250929",
            max_tokens=3000,
            temperature=0,
            system=SYSTEM,
            messages=[{"role": "user", "content": prompt}],
        )
    except APIStatusError as exc:
        raise RuntimeError(f"generation failed: {exc}") from exc
    return _strip_fences(resp.content[0].text)


def _strip_fences(text: str) -> str:
    m = re.search(r"```(?:python)?\n(.*)\n```", text, re.DOTALL)
    return m.group(1) if m else text.strip()

# loop/driver.py
from pathlib import Path
from loop.runner import run_coverage, suite_is_green
from loop.coverage_parser import parse_targets, overall_branch_pct
from loop.generator import generate_tests

TARGET_PCT = 90.0
MAX_ITERATIONS = 8
GEN_DIR = Path("tests/generated")


def run_loop() -> None:
    GEN_DIR.mkdir(parents=True, exist_ok=True)
    previous_pct = -1.0

    for iteration in range(1, MAX_ITERATIONS + 1):
        report = run_coverage()
        pct = overall_branch_pct(report)
        print(f"iteration {iteration}: branch coverage {pct:.1f}%")

        if pct >= TARGET_PCT:
            print(f"target {TARGET_PCT}% reached")
            return

        # No-progress guard: a stuck loop just burns tokens.
        if pct <= previous_pct + 0.5 and iteration > 1:
            print("coverage plateaued — remaining branches need human input")
            return
        previous_pct = pct

        targets = parse_targets(report)
        if not targets:
            print("no uncovered branches left to target")
            return

        code = generate_tests(targets)
        out = GEN_DIR / f"test_gen_iter_{iteration}.py"
        out.write_text(code + "\n")

        # Reject the batch if it broke the suite.
        if not suite_is_green():
            print(f"generated batch failed pytest — discarding {out.name}")
            out.unlink()
            return

    print(f"hit iteration cap ({MAX_ITERATIONS})")


if __name__ == "__main__":
    run_loop()

# .github/workflows/coverage-maintenance.yml
name: Coverage Maintenance
on:
  schedule:
    - cron: "0 3 * * 1"   # Monday 03:00, low-traffic window
  workflow_dispatch:

jobs:
  backfill:
    runs-on: ubuntu-latest
    timeout-minutes: 20
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - run: pip install -e ".[dev]"
      - name: Run coverage agent loop
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
        run: python -m loop.driver
      - name: Open PR with new tests
        run: |
          git config user.name "coverage-agent"
          git config user.email "[email protected]"
          git checkout -b coverage/backfill-$(date +%Y%m%d)
          git add tests/generated/
          git diff --cached --quiet || {
            git commit -m "test, backfill branch coverage"
            gh pr create --fill --label automated
          }
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}