GEWU/scripts/run_four_file_automation.py

#!/usr/bin/env python3
"""Run deterministic four-file benchmark gates and emit an owner-review bundle.

The runner is read-only with respect to GEWU canonical stores. It writes a new
exclusive output directory containing machine recommendations and validation
reports. It never records owner decisions or edits source answer artifacts.
"""

from __future__ import annotations

import argparse
import hashlib
import json
import shutil
import sys
from datetime import UTC, datetime
from pathlib import Path
from typing import Any

if __package__ in {None, ""}:
    sys.path.insert(0, str(Path(__file__).resolve().parent.parent))

from scripts.review_benchmark_candidates import load_extractions, review
from scripts.score_retrieval_benchmark import score as score_retrieval
from scripts.verify_answer_citations import load_governing_rules
from scripts.verify_answer_citations import verify as verify_answers


def file_sha256(path: Path) -> str:
    digest = hashlib.sha256()
    with path.open("rb") as stream:
        for chunk in iter(lambda: stream.read(1024 * 1024), b""):
            digest.update(chunk)
    return digest.hexdigest()


def validate_source_pins(manifest: dict[str, Any], source_dir: Path) -> list[dict[str, Any]]:
    actual = {path.name: path for path in source_dir.iterdir() if path.is_file()}
    checks: list[dict[str, Any]] = []
    for expected in manifest.get("sources", []):
        filename = expected.get("filename")
        path = actual.get(filename)
        digest = file_sha256(path) if path else None
        checks.append(
            {
                "filename": filename,
                "expected_sha256": expected.get("sha256"),
                "actual_sha256": digest,
                "status": "pass" if digest == expected.get("sha256") else "fail",
            }
        )
    return checks


def gate_summary(
    *,
    source_checks: list[dict[str, Any]],
    review_ledger: dict[str, Any],
    retrieval_score: dict[str, Any],
    answer_reports: dict[str, dict[str, Any]],
    support_judgments: dict[str, Any],
    answer_scores: dict[str, Any],
    behavioral_results: dict[str, Any] | None = None,
    semantic_results: dict[str, Any] | None = None,
) -> dict[str, Any]:
    behavioral_cases = (behavioral_results or {}).keys()
    behavioral_complete = {
        "exact_replay",
        "quote",
        "offset",
        "extraction",
        "revision",
        "meaning",
        "unauthorized_get_node",
        "unauthorized_search",
    }.issubset(behavioral_cases)
    gates: dict[str, dict[str, Any]] = {
        "source_pins": {
            "status": "pass"
            if source_checks and all(x["status"] == "pass" for x in source_checks)
            else "fail",
            "checked": len(source_checks),
        },
        "evidence_binding_audit": {
            "status": "pass"
            if review_ledger["summary"]["evidence_bindings_invalid"] == 0
            else "fail",
            "checked_claims": review_ledger["summary"]["claims"],
            "invalid_bindings": review_ledger["summary"]["evidence_bindings_invalid"],
        },
        "retrieval_tuning": {
            "status": "pass"
            if retrieval_score.get("question_count") == 24
            and retrieval_score.get("mean_conditional_recall_at_5", 0) >= 0.85
            and retrieval_score.get("conditional_hit_rate", 0) >= 0.85
            and retrieval_score.get("pagination_blocked_queries", 1) == 0
            else "fail",
            "recall_at_5": retrieval_score.get("mean_conditional_recall_at_5"),
            "hit_rate": retrieval_score.get("conditional_hit_rate"),
            "pagination_blocked": retrieval_score.get("pagination_blocked_queries"),
        },
        "answer_citations": {
            "status": "pass"
            if answer_reports
            and all(
                row.get("status") in {"valid", "repaired"} and not row.get("issues")
                for row in answer_reports.values()
            )
            else "fail",
            "outputs": {
                key: {
                    "status": row.get("status"),
                    "checked": row.get("checked"),
                    "issues": len(row.get("issues", [])),
                }
                for key, row in answer_reports.items()
            },
        },
        "accepted_claim_support": {
            "status": "pass"
            if support_judgments.get("strict_group_support_fraction", 0) >= 0.95
            else "fail",
            "fully_supported_groups": support_judgments.get("fully_supported_groups"),
            "groups": support_judgments.get("groups"),
            "strict_fraction": support_judgments.get("strict_group_support_fraction"),
            "threshold": 0.95,
        },
        "answer_quality": {
            "status": "pass"
            if answer_scores.get("50k", {}).get("content_points_before_citation_penalty", 0)
            / max(answer_scores.get("50k", {}).get("max_points", 1), 1)
            >= 0.85
            else "fail",
            "scored_profile": "50k",
            "points_before_citation_penalty": answer_scores.get("50k", {}).get(
                "content_points_before_citation_penalty"
            ),
            "max_points": answer_scores.get("50k", {}).get("max_points"),
            "threshold": 0.85,
            "grading_method": answer_scores.get("50k", {}).get("graded_by"),
        },
        "owner_protected_decisions": {
            "status": "pending_owner",
            "items": review_ledger["summary"]["items"],
            "machine_decisions_written": 0,
        },
        "semantic_support_review": {
            "status": "advisory_complete_candidate_actions_pending"
            if semantic_results
            else "gap_demonstrated_not_judged"
            if "meaning" in behavioral_cases
            else "not_run",
            "claims_assessed": semantic_results.get("claim_count", 0) if semantic_results else 0,
            "outcomes": semantic_results.get("outcomes", {}) if semantic_results else {},
        },
        "answer_host_enforcement": {"status": "not_integrated"},
        "behavioral_suite": {
            "status": "partial" if behavioral_complete else "not_run",
            "completed_cases": sorted(behavioral_cases),
            "missing_cases": []
            if behavioral_complete
            else [
                "full_reingest",
                "changed_file_revision",
                "source_instruction_injection",
                "delayed_index",
                "interruption_resume_under_budget",
            ],
        },
        "reserved_questions": {"status": "not_run"},
        "fresh_repetitions": {"status": "not_run", "required": 3, "completed": 0},
    }
    hard_failures = [
        name
        for name in (
            "source_pins",
            "evidence_binding_audit",
            "accepted_claim_support",
            "retrieval_tuning",
            "answer_citations",
            "answer_quality",
        )
        if gates[name]["status"] == "fail"
    ]
    remaining = [name for name, gate in gates.items() if gate["status"] not in {"pass"}]
    return {
        "qualification": "not_qualified" if hard_failures or remaining else "qualified",
        "hard_failures": hard_failures,
        "remaining_gates": remaining,
        "gates": gates,
    }


def run(args: argparse.Namespace) -> dict[str, Any]:
    output: Path = args.output.resolve()
    output.mkdir(parents=True, exist_ok=False, mode=0o700)
    manifest = json.loads(args.manifest.read_text(encoding="utf-8"))
    shutil.copyfile(args.manifest, output / "base-manifest.json")
    raw_items = args.review_items.read_bytes()
    items = json.loads(raw_items)
    if not isinstance(items, list) or not items:
        raise ValueError("Review items must be a nonempty JSON array")
    review_copy = output / "held-and-correction-proposals-with-evidence.json"
    shutil.copyfile(args.review_items, review_copy)
    review_copy.chmod(0o600)
    source_checks = validate_source_pins(manifest, args.sources)
    extractions = load_extractions(args.extractions)
    review_ledger = review(items, extractions, input_sha256=hashlib.sha256(raw_items).hexdigest())
    (output / "machine-review-recommendations.json").write_text(
        json.dumps(review_ledger, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
        encoding="utf-8",
    )
    retrieval_score = score_retrieval(args.retrieval, args.judgments)
    (output / "retrieval-score.json").write_text(
        json.dumps(retrieval_score, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
        encoding="utf-8",
    )
    governing = load_governing_rules(args.governing_rules, args.repo_root)
    answer_rule_map_doc = json.loads(args.answer_rule_map.read_text(encoding="utf-8"))
    if answer_rule_map_doc.get("schema_version") != 1 or not isinstance(
        answer_rule_map_doc.get("answers"), dict
    ):
        raise ValueError("Frozen answer-rule map has an unsupported shape")
    answer_rule_map = answer_rule_map_doc["answers"]
    answer_reports: dict[str, dict[str, Any]] = {}
    for answer_path in args.answers:
        verified, report = verify_answers(
            json.loads(answer_path.read_text(encoding="utf-8")),
            json.loads(args.accepted_records.read_text(encoding="utf-8")),
            repair_quotes=True,
            governing_rules=governing,
            governing_rule_answer_map=answer_rule_map,
        )
        label = answer_path.stem
        answer_reports[label] = report
        if not report["issues"]:
            (output / f"{label}-verified.json").write_text(
                json.dumps(verified, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
            )
        (output / f"{label}-citation-report.json").write_text(
            json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
            encoding="utf-8",
        )
    behavioral_results = (
        json.loads(args.behavioral_results.read_text(encoding="utf-8"))
        if args.behavioral_results
        else None
    )
    if behavioral_results is not None:
        shutil.copyfile(args.behavioral_results, output / "behavioral-results.json")
    semantic_results = (
        json.loads(args.semantic_results.read_text(encoding="utf-8"))
        if args.semantic_results
        else None
    )
    if semantic_results is not None:
        shutil.copyfile(args.semantic_results, output / "semantic-review-consensus.json")
        by_claim_key = {row["claim_id"]: row for row in semantic_results.get("assessments", [])}
        if len(by_claim_key) != review_ledger["summary"]["claims"]:
            raise ValueError("Semantic review does not cover every audited proposal claim")
        attached_count = 0
        for item in review_ledger["items"]:
            for claim in item["claims"]:
                claim_key = f"{item['proposal_id']}::{claim['claim_id']}"
                if claim_key in by_claim_key:
                    claim["semantic_review"] = by_claim_key[claim_key]
                    attached_count += 1
        if attached_count != len(by_claim_key):
            raise ValueError("Semantic review contains claim IDs outside the owner review packet")
        review_path = output / "machine-review-recommendations.json"
        review_path.write_text(
            json.dumps(review_ledger, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
            encoding="utf-8",
        )
    status = gate_summary(
        source_checks=source_checks,
        review_ledger=review_ledger,
        retrieval_score=retrieval_score,
        answer_reports=answer_reports,
        support_judgments=json.loads(args.support_judgments.read_text(encoding="utf-8")),
        answer_scores=json.loads(args.answer_scores.read_text(encoding="utf-8")),
        behavioral_results=behavioral_results,
        semantic_results=semantic_results,
    )
    report = {
        "schema_version": 1,
        "kind": "four_file_automation_run",
        "created_utc": datetime.now(UTC).isoformat(),
        "base_manifest_sha256": file_sha256(args.manifest),
        "inputs": {
            name: {"path": str(path.resolve()), "sha256": file_sha256(path)}
            for name, path in (
                ("review_items", args.review_items),
                ("retrieval", args.retrieval),
                ("judgments", args.judgments),
                ("accepted_records", args.accepted_records),
                ("answer_rule_map", args.answer_rule_map),
                ("support_judgments", args.support_judgments),
                ("answer_scores", args.answer_scores),
            )
        },
        "answer_inputs": [
            {"path": str(path.resolve()), "sha256": file_sha256(path)} for path in args.answers
        ],
        "behavioral_input": (
            {
                "path": str(args.behavioral_results.resolve()),
                "sha256": file_sha256(args.behavioral_results),
            }
            if args.behavioral_results
            else None
        ),
        "semantic_input": (
            {
                "path": str(args.semantic_results.resolve()),
                "sha256": file_sha256(args.semantic_results),
            }
            if args.semantic_results
            else None
        ),
        "source_pins": source_checks,
        "status": status,
        "machine_review_summary": review_ledger["summary"],
        "important_limitations": [
            "Semantic entailment was not assessed by the deterministic review tool.",
            "No owner decision was written; policy-held proposals remain quarantined.",
            "Citation validation and deterministic quote/rule-basis repairs were run offline; "
            "no answering host output hook was available.",
            "Live behavioral checks are partial; missing cases are listed in the gate report."
            if behavioral_results
            else (
                "Reserved questions and fresh repetitions were not run because answer and owner "
                "gates remain open."
            ),
        ],
    }
    (output / "run-report.json").write_text(
        json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8"
    )
    markdown = [
        "# Four-file automated gate report",
        "",
        f"Qualification: **{status['qualification']}**",
        "",
        "## Gate status",
        "",
    ]
    markdown.extend(f"- `{name}`: **{gate['status']}**" for name, gate in status["gates"].items())
    markdown.extend(
        [
            "",
            "## Machine review",
            "",
            f"Audited {review_ledger['summary']['claims']} claims and "
            f"{review_ledger['summary']['evidence_bindings_valid']} structurally valid "
            "evidence bindings. "
            f"All {review_ledger['summary']['items']} review items remain deferred pending "
            "protected owner decisions or semantic review.",
            "",
            "The full evidence ledger, per-output citation reports, retrieval scores, and pinned "
            "input hashes are beside this report. No canonical record or owner decision changed.",
            "",
        ]
    )
    (output / "README.md").write_text("\n".join(markdown), encoding="utf-8")
    return report


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--manifest", type=Path, required=True)
    parser.add_argument("--sources", type=Path, required=True)
    parser.add_argument("--review-items", type=Path, required=True)
    parser.add_argument("--extractions", type=Path, required=True)
    parser.add_argument("--retrieval", type=Path, required=True)
    parser.add_argument("--judgments", type=Path, required=True)
    parser.add_argument("--answers", type=Path, action="append", required=True)
    parser.add_argument("--accepted-records", type=Path, required=True)
    parser.add_argument("--support-judgments", type=Path, required=True)
    parser.add_argument("--answer-scores", type=Path, required=True)
    parser.add_argument("--behavioral-results", type=Path)
    parser.add_argument("--semantic-results", type=Path)
    parser.add_argument(
        "--governing-rules",
        type=Path,
        default=Path(__file__).with_name("governing_answer_rules.json"),
    )
    parser.add_argument(
        "--answer-rule-map",
        type=Path,
        default=Path(__file__).with_name("governing_answer_map.json"),
    )
    parser.add_argument("--repo-root", type=Path, default=Path(__file__).resolve().parent.parent)
    parser.add_argument("--output", type=Path, required=True)
    args = parser.parse_args()
    report = run(args)
    print(
        json.dumps({"qualification": report["status"]["qualification"], "output": str(args.output)})
    )


if __name__ == "__main__":
    main()