vasily
78033bb830
CI / quality (push) Canceled after 0s
Adds claim-bound online evidence records and citation verification scripts, node removal transport/adapter, retrieval benchmark and owner-review tooling, GraphCanvas/SourceDetail knowledge browser UI, and the ingest handoff helper. Bumps the agents/gewu pack to the online-verification branch.
136 lines
4.9 KiB
Python
136 lines
4.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Create a reproducibility manifest for a clean, pinned GEWU benchmark run."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import platform
|
|
import subprocess
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
def sha256(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as stream:
|
|
while chunk := stream.read(1024 * 1024):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def _git(repo: Path, *args: str) -> str:
|
|
result = subprocess.run(
|
|
["git", "-C", str(repo), *args],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
return result.stdout.strip()
|
|
|
|
|
|
def _pack_digest(pack: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
for path in sorted(item for item in pack.rglob("*") if item.is_file()):
|
|
relative = path.relative_to(pack).as_posix()
|
|
digest.update(relative.encode("utf-8"))
|
|
digest.update(b"\0")
|
|
digest.update(bytes.fromhex(sha256(path)))
|
|
return digest.hexdigest()
|
|
|
|
|
|
def _extractions(directory: Path) -> list[dict[str, str]]:
|
|
rows: list[dict[str, str]] = []
|
|
for path in sorted(directory.rglob("*.json")):
|
|
value = json.loads(path.read_text(encoding="utf-8"))
|
|
record = value.get("record", value)
|
|
required = ("extraction_id", "parser_name", "parser_version", "parser_config_sha256")
|
|
if not all(isinstance(record.get(key), str) for key in required):
|
|
raise ValueError(f"Extraction manifest is missing parser identity fields: {path}")
|
|
rows.append({key: record[key] for key in required})
|
|
if not rows:
|
|
raise ValueError("No extraction records found")
|
|
return rows
|
|
|
|
|
|
def create_manifest(
|
|
*,
|
|
repo: Path,
|
|
pack: Path,
|
|
sources: list[Path],
|
|
reference: Path,
|
|
extractions: Path,
|
|
model_id: str,
|
|
search_mode: str,
|
|
budget_tokens: int,
|
|
) -> dict[str, Any]:
|
|
dirty = _git(repo, "status", "--porcelain", "--untracked-files=all")
|
|
if dirty:
|
|
raise ValueError("Backend checkout is dirty; use a clean isolated checkout")
|
|
if not model_id.strip() or model_id.strip().lower() in {"default", "unknown", "latest"}:
|
|
raise ValueError("A concrete model ID is required; aliases are not reproducible pins")
|
|
if not 256 <= budget_tokens <= 50_000:
|
|
raise ValueError("budget_tokens must be between 256 and 50000")
|
|
for path in [repo, pack, reference, extractions, *sources]:
|
|
if not path.exists():
|
|
raise ValueError(f"Required benchmark input does not exist: {path}")
|
|
manifest_path = pack / "manifest.json"
|
|
pack_manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
|
return {
|
|
"schema_version": 1,
|
|
"created_utc": datetime.now(UTC).isoformat(),
|
|
"backend": {
|
|
"git_commit": _git(repo, "rev-parse", "HEAD"),
|
|
"clean_worktree": True,
|
|
},
|
|
"agent_pack": {
|
|
"pack_id": pack_manifest["pack_id"],
|
|
"pack_version": pack_manifest["pack_version"],
|
|
"manifest_sha256": sha256(manifest_path),
|
|
"tree_sha256": _pack_digest(pack),
|
|
},
|
|
"model_id": model_id.strip(),
|
|
"search_mode": search_mode,
|
|
"budget_tokens": budget_tokens,
|
|
"reference_sha256": sha256(reference),
|
|
"sources": [
|
|
{"filename": path.name, "sha256": sha256(path), "bytes": path.stat().st_size}
|
|
for path in sorted(sources)
|
|
],
|
|
"extractions": _extractions(extractions),
|
|
"runtime": {"python": platform.python_version(), "platform": platform.platform()},
|
|
}
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--repo", type=Path, required=True)
|
|
parser.add_argument("--pack", type=Path, required=True)
|
|
parser.add_argument("--source", type=Path, action="append", required=True)
|
|
parser.add_argument("--reference", type=Path, required=True)
|
|
parser.add_argument("--extractions", type=Path, required=True)
|
|
parser.add_argument("--model-id", required=True)
|
|
parser.add_argument("--search-mode", required=True)
|
|
parser.add_argument("--budget-tokens", type=int, required=True)
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
args = parser.parse_args()
|
|
manifest = create_manifest(
|
|
repo=args.repo.resolve(),
|
|
pack=args.pack.resolve(),
|
|
sources=[path.resolve() for path in args.source],
|
|
reference=args.reference.resolve(),
|
|
extractions=args.extractions.resolve(),
|
|
model_id=args.model_id,
|
|
search_mode=args.search_mode,
|
|
budget_tokens=args.budget_tokens,
|
|
)
|
|
with args.output.open("x", encoding="utf-8") as output:
|
|
output.write(json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n")
|
|
print(json.dumps({"status": "created", "output": str(args.output)}))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|