#!/usr/bin/env python3 """Create a deterministic, read-only evidence review ledger for benchmark proposals. This tool checks structure and source-span integrity. It does not make semantic support judgments and never writes owner decisions or canonical knowledge. """ from __future__ import annotations import argparse import hashlib import json import unicodedata from datetime import UTC, datetime from pathlib import Path from typing import Any def sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as stream: for chunk in iter(lambda: stream.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def load_extractions(directory: Path) -> dict[str, dict[str, Any]]: result: dict[str, dict[str, Any]] = {} for path in sorted(directory.rglob("*.json")): value = json.loads(path.read_text(encoding="utf-8")) record = value.get("record", value) extraction_id = record.get("extraction_id") if isinstance(extraction_id, str): if extraction_id in result: raise ValueError(f"Duplicate extraction ID: {extraction_id}") result[extraction_id] = record if not result: raise ValueError("No extraction manifests found") return result def _audit_claim(claim: dict[str, Any], extractions: dict[str, dict[str, Any]]) -> dict[str, Any]: evidence_rows = claim.get("evidence") issues: list[str] = [] checked = 0 claim_text = claim.get("text") expected_assertion_digest = ( hashlib.sha256( json.dumps( { "assertion": unicodedata.normalize( "NFC", claim_text.replace("\r\n", "\n").replace("\r", "\n") ) }, ensure_ascii=False, sort_keys=True, separators=(",", ":"), ).encode("utf-8") ).hexdigest() if isinstance(claim_text, str) else None ) if expected_assertion_digest is None: issues.append("CLAIM_TEXT_INVALID") if not isinstance(evidence_rows, list) or not evidence_rows: issues.append("CLAIM_EVIDENCE_REQUIRED") evidence_rows = [] for evidence in evidence_rows: checked += 1 if not isinstance(evidence, dict): issues.append("EVIDENCE_RECORD_INVALID") continue extraction_id = evidence.get("extraction_id") record = extractions.get(extraction_id) if isinstance(extraction_id, str) else None if record is None: issues.append("EXTRACTION_NOT_FOUND") continue if evidence.get("source_revision_id") != record.get("source_revision_id"): issues.append("SOURCE_REVISION_MISMATCH") if evidence.get("assertion_digest") != expected_assertion_digest: issues.append("ASSERTION_DIGEST_MISMATCH") locator = evidence.get("locator") if not isinstance(locator, dict): issues.append("LOCATOR_INVALID") continue block_id = locator.get("block_id") block = next( ( row for row in record.get("blocks", []) if isinstance(row, dict) and row.get("block_id") == block_id ), None, ) if block is None: issues.append("BLOCK_NOT_FOUND") continue text = block.get("normalized_text") start, end = locator.get("start"), locator.get("end") if ( not isinstance(text, str) or type(start) is not int or type(end) is not int or start < 0 or end <= start or end > len(text) ): issues.append("LOCATOR_RANGE_INVALID") continue if locator.get("page") != block.get("page"): issues.append("PAGE_MISMATCH") if text[start:end] != evidence.get("quote"): issues.append("QUOTE_SPAN_MISMATCH") return { "claim_id": claim.get("claim_id"), "assertion_digest": next( ( evidence.get("assertion_digest") for evidence in evidence_rows if isinstance(evidence, dict) ), None, ), "evidence_checked": checked, "structural_status": "valid" if not issues else "invalid", "issues": sorted(set(issues)), } def review( items: list[dict[str, Any]], extractions: dict[str, dict[str, Any]], *, input_sha256: str, ) -> dict[str, Any]: rows: list[dict[str, Any]] = [] for item in items: proposal = item.get("proposal", {}) evaluation = item.get("evaluation", {}) proposal_id = proposal.get("proposal_id") claims: list[dict[str, Any]] = [] supersession_edges: list[dict[str, Any]] = [] for operation in proposal.get("operations", []): node = operation.get("node") or {} claims.extend(_audit_claim(claim, extractions) for claim in node.get("claims", [])) edge = operation.get("edge") if isinstance(edge, dict): supersession_edges.append(edge) structural_issues = [issue for claim in claims for issue in claim["issues"]] policy_held = ( evaluation.get("status") == "needs-review" or evaluation.get("eligible_for_auto_accept") is False or "needs-review" in str(item.get("storage_state", "")) or item.get("source_stage") == "supersession" ) if structural_issues: recommendation = "revise-candidate" reason = "One or more evidence bindings failed deterministic checks." elif policy_held: recommendation = "defer" reason = "Evidence bindings are structurally valid, but policy requires owner review." elif not claims: recommendation = "defer" reason = "No atomic claims were available for review." else: recommendation = "defer" reason = "Semantic support has not been independently established by this tool." rows.append( { "proposal_id": proposal_id, "revision": proposal.get("revision"), "source_stage": item.get("source_stage"), "storage_state": item.get("storage_state"), "evaluation_status": evaluation.get("status"), "policy_held": policy_held, "recommendation": recommendation, "rationale": reason, "claims": claims, "supersession_edges_preserved": len(supersession_edges), "supersession_edges": supersession_edges, } ) return { "schema_version": 1, "kind": "machine_review_recommendations", "created_utc": datetime.now(UTC).isoformat(), "input_sha256": input_sha256, "review_method": "deterministic evidence binding checks; semantic support not assessed", "authority": "advisory only; no owner decision or canonical mutation", "summary": { "items": len(rows), "claims": sum(len(row["claims"]) for row in rows), "evidence_bindings_valid": sum( claim["structural_status"] == "valid" for row in rows for claim in row["claims"] ), "evidence_bindings_invalid": sum( claim["structural_status"] == "invalid" for row in rows for claim in row["claims"] ), "recommendations": { label: sum(row["recommendation"] == label for row in rows) for label in ("accept-candidate", "revise-candidate", "exclude-candidate", "defer") }, "owner_decisions_written": 0, "canonical_records_written": 0, }, "items": rows, } def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--input", type=Path, required=True) parser.add_argument("--extractions", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) args = parser.parse_args() raw = args.input.read_bytes() items = json.loads(raw) if not isinstance(items, list) or not items: parser.error("Input must be a nonempty JSON list") result = review( items, load_extractions(args.extractions), input_sha256=hashlib.sha256(raw).hexdigest(), ) with args.output.open("x", encoding="utf-8") as stream: stream.write(json.dumps(result, ensure_ascii=False, indent=2, sort_keys=True) + "\n") print( json.dumps({"status": "created", "summary": result["summary"], "output": str(args.output)}) ) if __name__ == "__main__": main()