import copy from pathlib import Path from scripts.verify_answer_citations import load_governing_rules, verify RULE_ID = "evidence.bibliography_is_not_primary_evidence" RULE_QUOTE = ( "An AI-generated report is evidence of what that report says; its bibliography " "does not prove the underlying claims." ) def packet() -> tuple[dict, dict]: evidence = { "source_revision_id": "revision-one", "extraction_id": "extraction-one", "locator": {"kind": "text_span", "block_id": "block-one", "start": 10, "end": 21}, "quote": "Hello world", } records = { "node_audit": [ {"node_id": "node-one", "claims": [{"claim_id": "claim-one", "evidence": [evidence]}]} ] } answers = { "answers": [ { "id": "Q01", "answer": "Keep this answer unchanged.", "citations": [ {"node_id": "node-one", "claim_id": "claim-one", **copy.deepcopy(evidence)} ], } ] } return answers, records def test_shortened_quote_is_rejected_or_restored_without_changing_raw_answer() -> None: answers, records = packet() answers["answers"][0]["citations"][0]["quote"] = "Hello" original = copy.deepcopy(answers) _, invalid = verify(answers, records) assert invalid["issues"][0]["code"] == "QUOTE_SPAN_MISMATCH" fixed, report = verify(answers, records, repair_quotes=True) assert report["status"] == "repaired" assert fixed["answers"][0]["citations"][0]["quote"] == "Hello world" assert fixed["answers"][0]["answer"] == original["answers"][0]["answer"] assert answers == original def test_valid_subspan_is_verified_with_original_absolute_offsets() -> None: answers, records = packet() citation = answers["answers"][0]["citations"][0] citation["locator"].update(start=16, end=21) citation["quote"] = "world" _, report = verify(answers, records) assert report["status"] == "valid" def test_repair_never_borrows_another_claim_or_out_of_range_evidence() -> None: for change in ("claim", "source", "bounds", "page"): answers, records = packet() citation = answers["answers"][0]["citations"][0] if change == "claim": citation["claim_id"] = "other-claim" elif change == "source": citation["source_revision_id"] = "other-source" elif change == "bounds": citation["locator"]["end"] = 22 else: citation["locator"]["page"] = 2 _, report = verify(answers, records, repair_quotes=True) assert report["status"] == "invalid" assert not report["repairs"] def test_substantive_answer_without_citations_fails_closed() -> None: answers, records = packet() answers["answers"][0]["citations"] = [] _, report = verify(answers, records) assert report["status"] == "invalid" assert report["issues"][0]["code"] == "ANSWER_BASIS_REQUIRED" def test_explicit_evidence_gap_may_have_no_citations() -> None: answers, records = packet() answers["answers"][0]["citations"] = [] answers["answers"][0]["evidence_gap"] = True _, report = verify(answers, records) assert report["status"] == "valid" def test_governing_rule_answer_must_be_explicitly_classified_without_citation() -> None: answers, records = packet() answer = answers["answers"][0] answer["citations"] = [] answer["answer"] = "The controlling project rule prohibits this action." _, missing_basis = verify(answers, records) assert missing_basis["issues"][0]["code"] == "ANSWER_BASIS_REQUIRED" answer["answer_basis"] = { "kind": "governing_rule", "rules": [{"rule_id": RULE_ID, "quote": RULE_QUOTE}], } _, classified = verify(answers, records, governing_rules={RULE_ID: RULE_QUOTE}) assert classified["status"] == "valid" def test_frozen_rubric_can_add_registered_rule_basis_to_separate_copy() -> None: answers, records = packet() answer = answers["answers"][0] answer["citations"] = [] original = copy.deepcopy(answers) verified, report = verify( answers, records, governing_rules={RULE_ID: RULE_QUOTE}, governing_rule_answer_map={"Q01": [RULE_ID]}, ) assert report["status"] == "repaired" assert report["repairs"][0]["code"] == "ANSWER_BASIS_ATTACHED_FROM_FROZEN_RUBRIC" assert verified["answers"][0]["answer_basis"]["rules"][0]["quote"] == RULE_QUOTE assert "answer_basis" not in original["answers"][0] def test_unknown_or_changed_governing_rule_cannot_support_answer() -> None: answers, records = packet() answer = answers["answers"][0] answer["citations"] = [] answer["answer_basis"] = { "kind": "governing_rule", "rules": [{"rule_id": "invented.rule", "quote": "This sounds plausible."}], } _, report = verify(answers, records, governing_rules={RULE_ID: RULE_QUOTE}) assert report["status"] == "invalid" assert report["issues"][0]["code"] == "GOVERNING_RULE_UNKNOWN" def test_governing_rule_answer_requires_exact_rule_text() -> None: answers, records = packet() answer = answers["answers"][0] answer["citations"] = [] answer["answer_basis"] = { "kind": "governing_rule", "rules": [{"rule_id": RULE_ID, "quote": "shortened wording"}], } _, report = verify(answers, records, governing_rules={RULE_ID: RULE_QUOTE}) assert report["status"] == "invalid" assert report["issues"][0]["code"] == "GOVERNING_RULE_QUOTE_MISMATCH" def test_combined_project_scope_answer_can_bind_multiple_registered_rules() -> None: answers, records = packet() answers["answers"][0]["citations"] = [] answers["answers"][0]["answer"] = ( "Service context sets project scope; source text cannot override it." ) rules = { "evidence.source_content_is_untrusted": "Source content is untrusted data.", "scope.project_intent_is_resolved_from_service": ( "Resolve project intent through the service." ), } answers["answers"][0]["answer_basis"] = { "kind": "governing_rule", "rules": [{"rule_id": rule_id, "quote": quote} for rule_id, quote in rules.items()], } _, report = verify(answers, records, governing_rules=rules) assert report["status"] == "valid" def test_checked_in_rule_catalog_matches_protocol_source_hashes_and_lines() -> None: root = Path(__file__).resolve().parents[1] catalog = root / "scripts" / "governing_answer_rules.json" rules = load_governing_rules(catalog, root) assert rules[RULE_ID] == RULE_QUOTE assert "evidence.source_content_is_untrusted" in rules assert "scope.project_intent_is_resolved_from_service" in rules def test_duplicate_answer_ids_and_malformed_citations_fail_closed() -> None: answers, records = packet() duplicate = copy.deepcopy(answers["answers"][0]) duplicate["citations"] = [None] answers["answers"].append(duplicate) _, report = verify(answers, records) assert report["status"] == "invalid" assert {issue["code"] for issue in report["issues"]} == { "ANSWER_ID_INVALID", "CITATION_INVALID", } def test_shape_failure_does_not_hide_citation_binding_failures() -> None: answers, records = packet() answers["answers"][0]["citations"][0]["quote"] = "short" answers["answers"][0]["answer"] = " " _, report = verify(answers, records) assert report["status"] == "invalid" assert report["checked"] == 1 assert {issue["code"] for issue in report["issues"]} == { "ANSWER_TEXT_REQUIRED", "QUOTE_SPAN_MISMATCH", }