87 lines
3.0 KiB
Python
87 lines
3.0 KiB
Python
|
import json
|
||
|
|
import subprocess
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
import pytest
|
||
|
|
from scripts.build_benchmark_manifest import create_manifest
|
||
|
|
|
||
|
|
|
||
|
|
def _fixture(tmp_path: Path) -> tuple[Path, Path, Path, Path, Path]:
|
||
|
|
repo = tmp_path / "repo"
|
||
|
|
repo.mkdir()
|
||
|
|
subprocess.run(["git", "init", "-q", str(repo)], check=True)
|
||
|
|
subprocess.run(["git", "-C", str(repo), "config", "user.email", "test@example.com"], check=True)
|
||
|
|
subprocess.run(["git", "-C", str(repo), "config", "user.name", "Test"], check=True)
|
||
|
|
(repo / "tracked.txt").write_text("fixed\n", encoding="utf-8")
|
||
|
|
subprocess.run(["git", "-C", str(repo), "add", "tracked.txt"], check=True)
|
||
|
|
subprocess.run(["git", "-C", str(repo), "commit", "-qm", "baseline"], check=True)
|
||
|
|
|
||
|
|
pack = tmp_path / "pack"
|
||
|
|
pack.mkdir()
|
||
|
|
(pack / "manifest.json").write_text(
|
||
|
|
json.dumps({"pack_id": "test-pack", "pack_version": "1.0.0"}), encoding="utf-8"
|
||
|
|
)
|
||
|
|
source = tmp_path / "source.txt"
|
||
|
|
source.write_text("source", encoding="utf-8")
|
||
|
|
reference = tmp_path / "reference.json"
|
||
|
|
reference.write_text("{}", encoding="utf-8")
|
||
|
|
extractions = tmp_path / "extractions"
|
||
|
|
extractions.mkdir()
|
||
|
|
(extractions / "one.json").write_text(
|
||
|
|
json.dumps(
|
||
|
|
{
|
||
|
|
"record": {
|
||
|
|
"extraction_id": "extract-1",
|
||
|
|
"parser_name": "test-parser",
|
||
|
|
"parser_version": "1.0",
|
||
|
|
"parser_config_sha256": "a" * 64,
|
||
|
|
}
|
||
|
|
}
|
||
|
|
),
|
||
|
|
encoding="utf-8",
|
||
|
|
)
|
||
|
|
return repo, pack, source, reference, extractions
|
||
|
|
|
||
|
|
|
||
|
|
def test_manifest_records_all_inputs_and_pins(tmp_path: Path) -> None:
|
||
|
|
repo, pack, source, reference, extractions = _fixture(tmp_path)
|
||
|
|
|
||
|
|
result = create_manifest(
|
||
|
|
repo=repo,
|
||
|
|
pack=pack,
|
||
|
|
sources=[source],
|
||
|
|
reference=reference,
|
||
|
|
extractions=extractions,
|
||
|
|
model_id="vendor/model-version-2026-10",
|
||
|
|
search_mode="hybrid-v1",
|
||
|
|
budget_tokens=8000,
|
||
|
|
)
|
||
|
|
|
||
|
|
assert result["backend"]["clean_worktree"] is True
|
||
|
|
assert result["agent_pack"]["tree_sha256"]
|
||
|
|
assert result["sources"][0]["sha256"]
|
||
|
|
assert result["extractions"][0]["parser_version"] == "1.0"
|
||
|
|
assert result["model_id"] == "vendor/model-version-2026-10"
|
||
|
|
|
||
|
|
|
||
|
|
def test_manifest_refuses_dirty_checkout_and_unpinned_model(tmp_path: Path) -> None:
|
||
|
|
repo, pack, source, reference, extractions = _fixture(tmp_path)
|
||
|
|
(repo / "tracked.txt").write_text("changed\n", encoding="utf-8")
|
||
|
|
args = {
|
||
|
|
"repo": repo,
|
||
|
|
"pack": pack,
|
||
|
|
"sources": [source],
|
||
|
|
"reference": reference,
|
||
|
|
"extractions": extractions,
|
||
|
|
"model_id": "vendor/model-1",
|
||
|
|
"search_mode": "hybrid-v1",
|
||
|
|
"budget_tokens": 8000,
|
||
|
|
}
|
||
|
|
|
||
|
|
with pytest.raises(ValueError, match="dirty"):
|
||
|
|
create_manifest(**args)
|
||
|
|
|
||
|
|
subprocess.run(["git", "-C", str(repo), "checkout", "--", "tracked.txt"], check=True)
|
||
|
|
args["model_id"] = "default"
|
||
|
|
with pytest.raises(ValueError, match="concrete model ID"):
|
||
|
|
create_manifest(**args)
|