233 lines
8.7 KiB
Python
233 lines
8.7 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
"""Create a deterministic, read-only evidence review ledger for benchmark proposals.
|
||
|
|
|
||
|
|
This tool checks structure and source-span integrity. It does not make semantic
|
||
|
|
support judgments and never writes owner decisions or canonical knowledge.
|
||
|
|
"""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import argparse
|
||
|
|
import hashlib
|
||
|
|
import json
|
||
|
|
import unicodedata
|
||
|
|
from datetime import UTC, datetime
|
||
|
|
from pathlib import Path
|
||
|
|
from typing import Any
|
||
|
|
|
||
|
|
|
||
|
|
def sha256(path: Path) -> str:
|
||
|
|
digest = hashlib.sha256()
|
||
|
|
with path.open("rb") as stream:
|
||
|
|
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||
|
|
digest.update(chunk)
|
||
|
|
return digest.hexdigest()
|
||
|
|
|
||
|
|
|
||
|
|
def load_extractions(directory: Path) -> dict[str, dict[str, Any]]:
|
||
|
|
result: dict[str, dict[str, Any]] = {}
|
||
|
|
for path in sorted(directory.rglob("*.json")):
|
||
|
|
value = json.loads(path.read_text(encoding="utf-8"))
|
||
|
|
record = value.get("record", value)
|
||
|
|
extraction_id = record.get("extraction_id")
|
||
|
|
if isinstance(extraction_id, str):
|
||
|
|
if extraction_id in result:
|
||
|
|
raise ValueError(f"Duplicate extraction ID: {extraction_id}")
|
||
|
|
result[extraction_id] = record
|
||
|
|
if not result:
|
||
|
|
raise ValueError("No extraction manifests found")
|
||
|
|
return result
|
||
|
|
|
||
|
|
|
||
|
|
def _audit_claim(claim: dict[str, Any], extractions: dict[str, dict[str, Any]]) -> dict[str, Any]:
|
||
|
|
evidence_rows = claim.get("evidence")
|
||
|
|
issues: list[str] = []
|
||
|
|
checked = 0
|
||
|
|
claim_text = claim.get("text")
|
||
|
|
expected_assertion_digest = (
|
||
|
|
hashlib.sha256(
|
||
|
|
json.dumps(
|
||
|
|
{
|
||
|
|
"assertion": unicodedata.normalize(
|
||
|
|
"NFC", claim_text.replace("\r\n", "\n").replace("\r", "\n")
|
||
|
|
)
|
||
|
|
},
|
||
|
|
ensure_ascii=False,
|
||
|
|
sort_keys=True,
|
||
|
|
separators=(",", ":"),
|
||
|
|
).encode("utf-8")
|
||
|
|
).hexdigest()
|
||
|
|
if isinstance(claim_text, str)
|
||
|
|
else None
|
||
|
|
)
|
||
|
|
if expected_assertion_digest is None:
|
||
|
|
issues.append("CLAIM_TEXT_INVALID")
|
||
|
|
if not isinstance(evidence_rows, list) or not evidence_rows:
|
||
|
|
issues.append("CLAIM_EVIDENCE_REQUIRED")
|
||
|
|
evidence_rows = []
|
||
|
|
for evidence in evidence_rows:
|
||
|
|
checked += 1
|
||
|
|
if not isinstance(evidence, dict):
|
||
|
|
issues.append("EVIDENCE_RECORD_INVALID")
|
||
|
|
continue
|
||
|
|
extraction_id = evidence.get("extraction_id")
|
||
|
|
record = extractions.get(extraction_id) if isinstance(extraction_id, str) else None
|
||
|
|
if record is None:
|
||
|
|
issues.append("EXTRACTION_NOT_FOUND")
|
||
|
|
continue
|
||
|
|
if evidence.get("source_revision_id") != record.get("source_revision_id"):
|
||
|
|
issues.append("SOURCE_REVISION_MISMATCH")
|
||
|
|
if evidence.get("assertion_digest") != expected_assertion_digest:
|
||
|
|
issues.append("ASSERTION_DIGEST_MISMATCH")
|
||
|
|
locator = evidence.get("locator")
|
||
|
|
if not isinstance(locator, dict):
|
||
|
|
issues.append("LOCATOR_INVALID")
|
||
|
|
continue
|
||
|
|
block_id = locator.get("block_id")
|
||
|
|
block = next(
|
||
|
|
(
|
||
|
|
row
|
||
|
|
for row in record.get("blocks", [])
|
||
|
|
if isinstance(row, dict) and row.get("block_id") == block_id
|
||
|
|
),
|
||
|
|
None,
|
||
|
|
)
|
||
|
|
if block is None:
|
||
|
|
issues.append("BLOCK_NOT_FOUND")
|
||
|
|
continue
|
||
|
|
text = block.get("normalized_text")
|
||
|
|
start, end = locator.get("start"), locator.get("end")
|
||
|
|
if (
|
||
|
|
not isinstance(text, str)
|
||
|
|
or type(start) is not int
|
||
|
|
or type(end) is not int
|
||
|
|
or start < 0
|
||
|
|
or end <= start
|
||
|
|
or end > len(text)
|
||
|
|
):
|
||
|
|
issues.append("LOCATOR_RANGE_INVALID")
|
||
|
|
continue
|
||
|
|
if locator.get("page") != block.get("page"):
|
||
|
|
issues.append("PAGE_MISMATCH")
|
||
|
|
if text[start:end] != evidence.get("quote"):
|
||
|
|
issues.append("QUOTE_SPAN_MISMATCH")
|
||
|
|
return {
|
||
|
|
"claim_id": claim.get("claim_id"),
|
||
|
|
"assertion_digest": next(
|
||
|
|
(
|
||
|
|
evidence.get("assertion_digest")
|
||
|
|
for evidence in evidence_rows
|
||
|
|
if isinstance(evidence, dict)
|
||
|
|
),
|
||
|
|
None,
|
||
|
|
),
|
||
|
|
"evidence_checked": checked,
|
||
|
|
"structural_status": "valid" if not issues else "invalid",
|
||
|
|
"issues": sorted(set(issues)),
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def review(
|
||
|
|
items: list[dict[str, Any]],
|
||
|
|
extractions: dict[str, dict[str, Any]],
|
||
|
|
*,
|
||
|
|
input_sha256: str,
|
||
|
|
) -> dict[str, Any]:
|
||
|
|
rows: list[dict[str, Any]] = []
|
||
|
|
for item in items:
|
||
|
|
proposal = item.get("proposal", {})
|
||
|
|
evaluation = item.get("evaluation", {})
|
||
|
|
proposal_id = proposal.get("proposal_id")
|
||
|
|
claims: list[dict[str, Any]] = []
|
||
|
|
supersession_edges: list[dict[str, Any]] = []
|
||
|
|
for operation in proposal.get("operations", []):
|
||
|
|
node = operation.get("node") or {}
|
||
|
|
claims.extend(_audit_claim(claim, extractions) for claim in node.get("claims", []))
|
||
|
|
edge = operation.get("edge")
|
||
|
|
if isinstance(edge, dict):
|
||
|
|
supersession_edges.append(edge)
|
||
|
|
structural_issues = [issue for claim in claims for issue in claim["issues"]]
|
||
|
|
policy_held = (
|
||
|
|
evaluation.get("status") == "needs-review"
|
||
|
|
or evaluation.get("eligible_for_auto_accept") is False
|
||
|
|
or "needs-review" in str(item.get("storage_state", ""))
|
||
|
|
or item.get("source_stage") == "supersession"
|
||
|
|
)
|
||
|
|
if structural_issues:
|
||
|
|
recommendation = "revise-candidate"
|
||
|
|
reason = "One or more evidence bindings failed deterministic checks."
|
||
|
|
elif policy_held:
|
||
|
|
recommendation = "defer"
|
||
|
|
reason = "Evidence bindings are structurally valid, but policy requires owner review."
|
||
|
|
elif not claims:
|
||
|
|
recommendation = "defer"
|
||
|
|
reason = "No atomic claims were available for review."
|
||
|
|
else:
|
||
|
|
recommendation = "defer"
|
||
|
|
reason = "Semantic support has not been independently established by this tool."
|
||
|
|
rows.append(
|
||
|
|
{
|
||
|
|
"proposal_id": proposal_id,
|
||
|
|
"revision": proposal.get("revision"),
|
||
|
|
"source_stage": item.get("source_stage"),
|
||
|
|
"storage_state": item.get("storage_state"),
|
||
|
|
"evaluation_status": evaluation.get("status"),
|
||
|
|
"policy_held": policy_held,
|
||
|
|
"recommendation": recommendation,
|
||
|
|
"rationale": reason,
|
||
|
|
"claims": claims,
|
||
|
|
"supersession_edges_preserved": len(supersession_edges),
|
||
|
|
"supersession_edges": supersession_edges,
|
||
|
|
}
|
||
|
|
)
|
||
|
|
return {
|
||
|
|
"schema_version": 1,
|
||
|
|
"kind": "machine_review_recommendations",
|
||
|
|
"created_utc": datetime.now(UTC).isoformat(),
|
||
|
|
"input_sha256": input_sha256,
|
||
|
|
"review_method": "deterministic evidence binding checks; semantic support not assessed",
|
||
|
|
"authority": "advisory only; no owner decision or canonical mutation",
|
||
|
|
"summary": {
|
||
|
|
"items": len(rows),
|
||
|
|
"claims": sum(len(row["claims"]) for row in rows),
|
||
|
|
"evidence_bindings_valid": sum(
|
||
|
|
claim["structural_status"] == "valid" for row in rows for claim in row["claims"]
|
||
|
|
),
|
||
|
|
"evidence_bindings_invalid": sum(
|
||
|
|
claim["structural_status"] == "invalid" for row in rows for claim in row["claims"]
|
||
|
|
),
|
||
|
|
"recommendations": {
|
||
|
|
label: sum(row["recommendation"] == label for row in rows)
|
||
|
|
for label in ("accept-candidate", "revise-candidate", "exclude-candidate", "defer")
|
||
|
|
},
|
||
|
|
"owner_decisions_written": 0,
|
||
|
|
"canonical_records_written": 0,
|
||
|
|
},
|
||
|
|
"items": rows,
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def main() -> None:
|
||
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
||
|
|
parser.add_argument("--input", type=Path, required=True)
|
||
|
|
parser.add_argument("--extractions", type=Path, required=True)
|
||
|
|
parser.add_argument("--output", type=Path, required=True)
|
||
|
|
args = parser.parse_args()
|
||
|
|
raw = args.input.read_bytes()
|
||
|
|
items = json.loads(raw)
|
||
|
|
if not isinstance(items, list) or not items:
|
||
|
|
parser.error("Input must be a nonempty JSON list")
|
||
|
|
result = review(
|
||
|
|
items,
|
||
|
|
load_extractions(args.extractions),
|
||
|
|
input_sha256=hashlib.sha256(raw).hexdigest(),
|
||
|
|
)
|
||
|
|
with args.output.open("x", encoding="utf-8") as stream:
|
||
|
|
stream.write(json.dumps(result, ensure_ascii=False, indent=2, sort_keys=True) + "\n")
|
||
|
|
print(
|
||
|
|
json.dumps({"status": "created", "summary": result["summary"], "output": str(args.output)})
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
main()
|