#!/usr/bin/env python3 """Run deterministic four-file benchmark gates and emit an owner-review bundle. The runner is read-only with respect to GEWU canonical stores. It writes a new exclusive output directory containing machine recommendations and validation reports. It never records owner decisions or edits source answer artifacts. """ from __future__ import annotations import argparse import hashlib import json import shutil import sys from datetime import UTC, datetime from pathlib import Path from typing import Any if __package__ in {None, ""}: sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from scripts.review_benchmark_candidates import load_extractions, review from scripts.score_retrieval_benchmark import score as score_retrieval from scripts.verify_answer_citations import load_governing_rules from scripts.verify_answer_citations import verify as verify_answers def file_sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as stream: for chunk in iter(lambda: stream.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def validate_source_pins(manifest: dict[str, Any], source_dir: Path) -> list[dict[str, Any]]: actual = {path.name: path for path in source_dir.iterdir() if path.is_file()} checks: list[dict[str, Any]] = [] for expected in manifest.get("sources", []): filename = expected.get("filename") path = actual.get(filename) digest = file_sha256(path) if path else None checks.append( { "filename": filename, "expected_sha256": expected.get("sha256"), "actual_sha256": digest, "status": "pass" if digest == expected.get("sha256") else "fail", } ) return checks def gate_summary( *, source_checks: list[dict[str, Any]], review_ledger: dict[str, Any], retrieval_score: dict[str, Any], answer_reports: dict[str, dict[str, Any]], support_judgments: dict[str, Any], answer_scores: dict[str, Any], behavioral_results: dict[str, Any] | None = None, semantic_results: dict[str, Any] | None = None, ) -> dict[str, Any]: behavioral_cases = (behavioral_results or {}).keys() behavioral_complete = { "exact_replay", "quote", "offset", "extraction", "revision", "meaning", "unauthorized_get_node", "unauthorized_search", }.issubset(behavioral_cases) gates: dict[str, dict[str, Any]] = { "source_pins": { "status": "pass" if source_checks and all(x["status"] == "pass" for x in source_checks) else "fail", "checked": len(source_checks), }, "evidence_binding_audit": { "status": "pass" if review_ledger["summary"]["evidence_bindings_invalid"] == 0 else "fail", "checked_claims": review_ledger["summary"]["claims"], "invalid_bindings": review_ledger["summary"]["evidence_bindings_invalid"], }, "retrieval_tuning": { "status": "pass" if retrieval_score.get("question_count") == 24 and retrieval_score.get("mean_conditional_recall_at_5", 0) >= 0.85 and retrieval_score.get("conditional_hit_rate", 0) >= 0.85 and retrieval_score.get("pagination_blocked_queries", 1) == 0 else "fail", "recall_at_5": retrieval_score.get("mean_conditional_recall_at_5"), "hit_rate": retrieval_score.get("conditional_hit_rate"), "pagination_blocked": retrieval_score.get("pagination_blocked_queries"), }, "answer_citations": { "status": "pass" if answer_reports and all( row.get("status") in {"valid", "repaired"} and not row.get("issues") for row in answer_reports.values() ) else "fail", "outputs": { key: { "status": row.get("status"), "checked": row.get("checked"), "issues": len(row.get("issues", [])), } for key, row in answer_reports.items() }, }, "accepted_claim_support": { "status": "pass" if support_judgments.get("strict_group_support_fraction", 0) >= 0.95 else "fail", "fully_supported_groups": support_judgments.get("fully_supported_groups"), "groups": support_judgments.get("groups"), "strict_fraction": support_judgments.get("strict_group_support_fraction"), "threshold": 0.95, }, "answer_quality": { "status": "pass" if answer_scores.get("50k", {}).get("content_points_before_citation_penalty", 0) / max(answer_scores.get("50k", {}).get("max_points", 1), 1) >= 0.85 else "fail", "scored_profile": "50k", "points_before_citation_penalty": answer_scores.get("50k", {}).get( "content_points_before_citation_penalty" ), "max_points": answer_scores.get("50k", {}).get("max_points"), "threshold": 0.85, "grading_method": answer_scores.get("50k", {}).get("graded_by"), }, "owner_protected_decisions": { "status": "pending_owner", "items": review_ledger["summary"]["items"], "machine_decisions_written": 0, }, "semantic_support_review": { "status": "advisory_complete_candidate_actions_pending" if semantic_results else "gap_demonstrated_not_judged" if "meaning" in behavioral_cases else "not_run", "claims_assessed": semantic_results.get("claim_count", 0) if semantic_results else 0, "outcomes": semantic_results.get("outcomes", {}) if semantic_results else {}, }, "answer_host_enforcement": {"status": "not_integrated"}, "behavioral_suite": { "status": "partial" if behavioral_complete else "not_run", "completed_cases": sorted(behavioral_cases), "missing_cases": [] if behavioral_complete else [ "full_reingest", "changed_file_revision", "source_instruction_injection", "delayed_index", "interruption_resume_under_budget", ], }, "reserved_questions": {"status": "not_run"}, "fresh_repetitions": {"status": "not_run", "required": 3, "completed": 0}, } hard_failures = [ name for name in ( "source_pins", "evidence_binding_audit", "accepted_claim_support", "retrieval_tuning", "answer_citations", "answer_quality", ) if gates[name]["status"] == "fail" ] remaining = [name for name, gate in gates.items() if gate["status"] not in {"pass"}] return { "qualification": "not_qualified" if hard_failures or remaining else "qualified", "hard_failures": hard_failures, "remaining_gates": remaining, "gates": gates, } def run(args: argparse.Namespace) -> dict[str, Any]: output: Path = args.output.resolve() output.mkdir(parents=True, exist_ok=False, mode=0o700) manifest = json.loads(args.manifest.read_text(encoding="utf-8")) shutil.copyfile(args.manifest, output / "base-manifest.json") raw_items = args.review_items.read_bytes() items = json.loads(raw_items) if not isinstance(items, list) or not items: raise ValueError("Review items must be a nonempty JSON array") review_copy = output / "held-and-correction-proposals-with-evidence.json" shutil.copyfile(args.review_items, review_copy) review_copy.chmod(0o600) source_checks = validate_source_pins(manifest, args.sources) extractions = load_extractions(args.extractions) review_ledger = review(items, extractions, input_sha256=hashlib.sha256(raw_items).hexdigest()) (output / "machine-review-recommendations.json").write_text( json.dumps(review_ledger, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8", ) retrieval_score = score_retrieval(args.retrieval, args.judgments) (output / "retrieval-score.json").write_text( json.dumps(retrieval_score, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8", ) governing = load_governing_rules(args.governing_rules, args.repo_root) answer_rule_map_doc = json.loads(args.answer_rule_map.read_text(encoding="utf-8")) if answer_rule_map_doc.get("schema_version") != 1 or not isinstance( answer_rule_map_doc.get("answers"), dict ): raise ValueError("Frozen answer-rule map has an unsupported shape") answer_rule_map = answer_rule_map_doc["answers"] answer_reports: dict[str, dict[str, Any]] = {} for answer_path in args.answers: verified, report = verify_answers( json.loads(answer_path.read_text(encoding="utf-8")), json.loads(args.accepted_records.read_text(encoding="utf-8")), repair_quotes=True, governing_rules=governing, governing_rule_answer_map=answer_rule_map, ) label = answer_path.stem answer_reports[label] = report if not report["issues"]: (output / f"{label}-verified.json").write_text( json.dumps(verified, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" ) (output / f"{label}-citation-report.json").write_text( json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8", ) behavioral_results = ( json.loads(args.behavioral_results.read_text(encoding="utf-8")) if args.behavioral_results else None ) if behavioral_results is not None: shutil.copyfile(args.behavioral_results, output / "behavioral-results.json") semantic_results = ( json.loads(args.semantic_results.read_text(encoding="utf-8")) if args.semantic_results else None ) if semantic_results is not None: shutil.copyfile(args.semantic_results, output / "semantic-review-consensus.json") by_claim_key = {row["claim_id"]: row for row in semantic_results.get("assessments", [])} if len(by_claim_key) != review_ledger["summary"]["claims"]: raise ValueError("Semantic review does not cover every audited proposal claim") attached_count = 0 for item in review_ledger["items"]: for claim in item["claims"]: claim_key = f"{item['proposal_id']}::{claim['claim_id']}" if claim_key in by_claim_key: claim["semantic_review"] = by_claim_key[claim_key] attached_count += 1 if attached_count != len(by_claim_key): raise ValueError("Semantic review contains claim IDs outside the owner review packet") review_path = output / "machine-review-recommendations.json" review_path.write_text( json.dumps(review_ledger, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8", ) status = gate_summary( source_checks=source_checks, review_ledger=review_ledger, retrieval_score=retrieval_score, answer_reports=answer_reports, support_judgments=json.loads(args.support_judgments.read_text(encoding="utf-8")), answer_scores=json.loads(args.answer_scores.read_text(encoding="utf-8")), behavioral_results=behavioral_results, semantic_results=semantic_results, ) report = { "schema_version": 1, "kind": "four_file_automation_run", "created_utc": datetime.now(UTC).isoformat(), "base_manifest_sha256": file_sha256(args.manifest), "inputs": { name: {"path": str(path.resolve()), "sha256": file_sha256(path)} for name, path in ( ("review_items", args.review_items), ("retrieval", args.retrieval), ("judgments", args.judgments), ("accepted_records", args.accepted_records), ("answer_rule_map", args.answer_rule_map), ("support_judgments", args.support_judgments), ("answer_scores", args.answer_scores), ) }, "answer_inputs": [ {"path": str(path.resolve()), "sha256": file_sha256(path)} for path in args.answers ], "behavioral_input": ( { "path": str(args.behavioral_results.resolve()), "sha256": file_sha256(args.behavioral_results), } if args.behavioral_results else None ), "semantic_input": ( { "path": str(args.semantic_results.resolve()), "sha256": file_sha256(args.semantic_results), } if args.semantic_results else None ), "source_pins": source_checks, "status": status, "machine_review_summary": review_ledger["summary"], "important_limitations": [ "Semantic entailment was not assessed by the deterministic review tool.", "No owner decision was written; policy-held proposals remain quarantined.", "Citation validation and deterministic quote/rule-basis repairs were run offline; " "no answering host output hook was available.", "Live behavioral checks are partial; missing cases are listed in the gate report." if behavioral_results else ( "Reserved questions and fresh repetitions were not run because answer and owner " "gates remain open." ), ], } (output / "run-report.json").write_text( json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8" ) markdown = [ "# Four-file automated gate report", "", f"Qualification: **{status['qualification']}**", "", "## Gate status", "", ] markdown.extend(f"- `{name}`: **{gate['status']}**" for name, gate in status["gates"].items()) markdown.extend( [ "", "## Machine review", "", f"Audited {review_ledger['summary']['claims']} claims and " f"{review_ledger['summary']['evidence_bindings_valid']} structurally valid " "evidence bindings. " f"All {review_ledger['summary']['items']} review items remain deferred pending " "protected owner decisions or semantic review.", "", "The full evidence ledger, per-output citation reports, retrieval scores, and pinned " "input hashes are beside this report. No canonical record or owner decision changed.", "", ] ) (output / "README.md").write_text("\n".join(markdown), encoding="utf-8") return report def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--manifest", type=Path, required=True) parser.add_argument("--sources", type=Path, required=True) parser.add_argument("--review-items", type=Path, required=True) parser.add_argument("--extractions", type=Path, required=True) parser.add_argument("--retrieval", type=Path, required=True) parser.add_argument("--judgments", type=Path, required=True) parser.add_argument("--answers", type=Path, action="append", required=True) parser.add_argument("--accepted-records", type=Path, required=True) parser.add_argument("--support-judgments", type=Path, required=True) parser.add_argument("--answer-scores", type=Path, required=True) parser.add_argument("--behavioral-results", type=Path) parser.add_argument("--semantic-results", type=Path) parser.add_argument( "--governing-rules", type=Path, default=Path(__file__).with_name("governing_answer_rules.json"), ) parser.add_argument( "--answer-rule-map", type=Path, default=Path(__file__).with_name("governing_answer_map.json"), ) parser.add_argument("--repo-root", type=Path, default=Path(__file__).resolve().parent.parent) parser.add_argument("--output", type=Path, required=True) args = parser.parse_args() report = run(args) print( json.dumps({"qualification": report["status"]["qualification"], "output": str(args.output)}) ) if __name__ == "__main__": main()