feat: online verification, node removals, and knowledge browser
CI / quality (push) Canceled after 0s
CI / quality (push) Canceled after 0s
Adds claim-bound online evidence records and citation verification scripts, node removal transport/adapter, retrieval benchmark and owner-review tooling, GraphCanvas/SourceDetail knowledge browser UI, and the ingest handoff helper. Bumps the agents/gewu pack to the online-verification branch.
This commit is contained in:
1 parent
a3156f4e08
commit
78033bb830
83 files changed
+13476
-1051
No files matched your search
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create a reproducibility manifest for a clean, pinned GEWU benchmark run."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import platform
|
||||
import subprocess
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
while chunk := stream.read(1024 * 1024):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _git(repo: Path, *args: str) -> str:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(repo), *args],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def _pack_digest(pack: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
for path in sorted(item for item in pack.rglob("*") if item.is_file()):
|
||||
relative = path.relative_to(pack).as_posix()
|
||||
digest.update(relative.encode("utf-8"))
|
||||
digest.update(b"\0")
|
||||
digest.update(bytes.fromhex(sha256(path)))
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _extractions(directory: Path) -> list[dict[str, str]]:
|
||||
rows: list[dict[str, str]] = []
|
||||
for path in sorted(directory.rglob("*.json")):
|
||||
value = json.loads(path.read_text(encoding="utf-8"))
|
||||
record = value.get("record", value)
|
||||
required = ("extraction_id", "parser_name", "parser_version", "parser_config_sha256")
|
||||
if not all(isinstance(record.get(key), str) for key in required):
|
||||
raise ValueError(f"Extraction manifest is missing parser identity fields: {path}")
|
||||
rows.append({key: record[key] for key in required})
|
||||
if not rows:
|
||||
raise ValueError("No extraction records found")
|
||||
return rows
|
||||
|
||||
|
||||
def create_manifest(
|
||||
*,
|
||||
repo: Path,
|
||||
pack: Path,
|
||||
sources: list[Path],
|
||||
reference: Path,
|
||||
extractions: Path,
|
||||
model_id: str,
|
||||
search_mode: str,
|
||||
budget_tokens: int,
|
||||
) -> dict[str, Any]:
|
||||
dirty = _git(repo, "status", "--porcelain", "--untracked-files=all")
|
||||
if dirty:
|
||||
raise ValueError("Backend checkout is dirty; use a clean isolated checkout")
|
||||
if not model_id.strip() or model_id.strip().lower() in {"default", "unknown", "latest"}:
|
||||
raise ValueError("A concrete model ID is required; aliases are not reproducible pins")
|
||||
if not 256 <= budget_tokens <= 50_000:
|
||||
raise ValueError("budget_tokens must be between 256 and 50000")
|
||||
for path in [repo, pack, reference, extractions, *sources]:
|
||||
if not path.exists():
|
||||
raise ValueError(f"Required benchmark input does not exist: {path}")
|
||||
manifest_path = pack / "manifest.json"
|
||||
pack_manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"created_utc": datetime.now(UTC).isoformat(),
|
||||
"backend": {
|
||||
"git_commit": _git(repo, "rev-parse", "HEAD"),
|
||||
"clean_worktree": True,
|
||||
},
|
||||
"agent_pack": {
|
||||
"pack_id": pack_manifest["pack_id"],
|
||||
"pack_version": pack_manifest["pack_version"],
|
||||
"manifest_sha256": sha256(manifest_path),
|
||||
"tree_sha256": _pack_digest(pack),
|
||||
},
|
||||
"model_id": model_id.strip(),
|
||||
"search_mode": search_mode,
|
||||
"budget_tokens": budget_tokens,
|
||||
"reference_sha256": sha256(reference),
|
||||
"sources": [
|
||||
{"filename": path.name, "sha256": sha256(path), "bytes": path.stat().st_size}
|
||||
for path in sorted(sources)
|
||||
],
|
||||
"extractions": _extractions(extractions),
|
||||
"runtime": {"python": platform.python_version(), "platform": platform.platform()},
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--repo", type=Path, required=True)
|
||||
parser.add_argument("--pack", type=Path, required=True)
|
||||
parser.add_argument("--source", type=Path, action="append", required=True)
|
||||
parser.add_argument("--reference", type=Path, required=True)
|
||||
parser.add_argument("--extractions", type=Path, required=True)
|
||||
parser.add_argument("--model-id", required=True)
|
||||
parser.add_argument("--search-mode", required=True)
|
||||
parser.add_argument("--budget-tokens", type=int, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
manifest = create_manifest(
|
||||
repo=args.repo.resolve(),
|
||||
pack=args.pack.resolve(),
|
||||
sources=[path.resolve() for path in args.source],
|
||||
reference=args.reference.resolve(),
|
||||
extractions=args.extractions.resolve(),
|
||||
model_id=args.model_id,
|
||||
search_mode=args.search_mode,
|
||||
budget_tokens=args.budget_tokens,
|
||||
)
|
||||
with args.output.open("x", encoding="utf-8") as output:
|
||||
output.write(json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n")
|
||||
print(json.dumps({"status": "created", "output": str(args.output)}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user