import json import subprocess from pathlib import Path import pytest from scripts.build_benchmark_manifest import create_manifest def _fixture(tmp_path: Path) -> tuple[Path, Path, Path, Path, Path]: repo = tmp_path / "repo" repo.mkdir() subprocess.run(["git", "init", "-q", str(repo)], check=True) subprocess.run(["git", "-C", str(repo), "config", "user.email", "test@example.com"], check=True) subprocess.run(["git", "-C", str(repo), "config", "user.name", "Test"], check=True) (repo / "tracked.txt").write_text("fixed\n", encoding="utf-8") subprocess.run(["git", "-C", str(repo), "add", "tracked.txt"], check=True) subprocess.run(["git", "-C", str(repo), "commit", "-qm", "baseline"], check=True) pack = tmp_path / "pack" pack.mkdir() (pack / "manifest.json").write_text( json.dumps({"pack_id": "test-pack", "pack_version": "1.0.0"}), encoding="utf-8" ) source = tmp_path / "source.txt" source.write_text("source", encoding="utf-8") reference = tmp_path / "reference.json" reference.write_text("{}", encoding="utf-8") extractions = tmp_path / "extractions" extractions.mkdir() (extractions / "one.json").write_text( json.dumps( { "record": { "extraction_id": "extract-1", "parser_name": "test-parser", "parser_version": "1.0", "parser_config_sha256": "a" * 64, } } ), encoding="utf-8", ) return repo, pack, source, reference, extractions def test_manifest_records_all_inputs_and_pins(tmp_path: Path) -> None: repo, pack, source, reference, extractions = _fixture(tmp_path) result = create_manifest( repo=repo, pack=pack, sources=[source], reference=reference, extractions=extractions, model_id="vendor/model-version-2026-10", search_mode="hybrid-v1", budget_tokens=8000, ) assert result["backend"]["clean_worktree"] is True assert result["agent_pack"]["tree_sha256"] assert result["sources"][0]["sha256"] assert result["extractions"][0]["parser_version"] == "1.0" assert result["model_id"] == "vendor/model-version-2026-10" def test_manifest_refuses_dirty_checkout_and_unpinned_model(tmp_path: Path) -> None: repo, pack, source, reference, extractions = _fixture(tmp_path) (repo / "tracked.txt").write_text("changed\n", encoding="utf-8") args = { "repo": repo, "pack": pack, "sources": [source], "reference": reference, "extractions": extractions, "model_id": "vendor/model-1", "search_mode": "hybrid-v1", "budget_tokens": 8000, } with pytest.raises(ValueError, match="dirty"): create_manifest(**args) subprocess.run(["git", "-C", str(repo), "checkout", "--", "tracked.txt"], check=True) args["model_id"] = "default" with pytest.raises(ValueError, match="concrete model ID"): create_manifest(**args)