77 lines
3.4 KiB
Python
77 lines
3.4 KiB
Python
|
from scripts.run_four_file_automation import gate_summary
|
||
|
|
|
||
|
|
|
||
|
|
def test_gate_summary_does_not_qualify_when_protected_and_answer_gates_remain():
|
||
|
|
report = gate_summary(
|
||
|
|
source_checks=[{"status": "pass"}],
|
||
|
|
review_ledger={"summary": {"evidence_bindings_invalid": 0, "claims": 2, "items": 1}},
|
||
|
|
retrieval_score={
|
||
|
|
"question_count": 24,
|
||
|
|
"mean_conditional_recall_at_5": 0.9,
|
||
|
|
"conditional_hit_rate": 1.0,
|
||
|
|
"pagination_blocked_queries": 0,
|
||
|
|
},
|
||
|
|
answer_reports={
|
||
|
|
"answers": {"status": "invalid", "checked": 1, "issues": [{"code": "BAD"}]}
|
||
|
|
},
|
||
|
|
support_judgments={"strict_group_support_fraction": 1.0},
|
||
|
|
answer_scores={"50k": {"content_points_before_citation_penalty": 48, "max_points": 48}},
|
||
|
|
behavioral_results={
|
||
|
|
"exact_replay": {"passed": True},
|
||
|
|
"quote": {"status": "invalid"},
|
||
|
|
"offset": {"status": "invalid"},
|
||
|
|
"extraction": {"status": "invalid"},
|
||
|
|
"revision": {"status": "invalid"},
|
||
|
|
"meaning": {"status": "eligible"},
|
||
|
|
"unauthorized_get_node": {"denied": True},
|
||
|
|
"unauthorized_search": {"denied": True},
|
||
|
|
},
|
||
|
|
)
|
||
|
|
|
||
|
|
assert report["qualification"] == "not_qualified"
|
||
|
|
assert report["gates"]["retrieval_tuning"]["status"] == "pass"
|
||
|
|
assert report["gates"]["answer_citations"]["status"] == "fail"
|
||
|
|
assert report["gates"]["owner_protected_decisions"]["status"] == "pending_owner"
|
||
|
|
assert report["gates"]["behavioral_suite"]["status"] == "partial"
|
||
|
|
assert report["gates"]["semantic_support_review"]["status"] == "gap_demonstrated_not_judged"
|
||
|
|
assert "reserved_questions" in report["remaining_gates"]
|
||
|
|
|
||
|
|
|
||
|
|
def test_gate_summary_marks_retrieval_failure_when_pagination_is_blocked():
|
||
|
|
report = gate_summary(
|
||
|
|
source_checks=[{"status": "pass"}],
|
||
|
|
review_ledger={"summary": {"evidence_bindings_invalid": 0, "claims": 0, "items": 0}},
|
||
|
|
retrieval_score={
|
||
|
|
"question_count": 24,
|
||
|
|
"mean_conditional_recall_at_5": 0.9,
|
||
|
|
"conditional_hit_rate": 1.0,
|
||
|
|
"pagination_blocked_queries": 1,
|
||
|
|
},
|
||
|
|
answer_reports={"answers": {"status": "valid", "checked": 0, "issues": []}},
|
||
|
|
support_judgments={"strict_group_support_fraction": 1.0},
|
||
|
|
answer_scores={"50k": {"content_points_before_citation_penalty": 48, "max_points": 48}},
|
||
|
|
)
|
||
|
|
|
||
|
|
assert report["gates"]["retrieval_tuning"]["status"] == "fail"
|
||
|
|
assert report["qualification"] == "not_qualified"
|
||
|
|
|
||
|
|
|
||
|
|
def test_semantic_judge_completion_remains_advisory_until_candidate_actions_finish():
|
||
|
|
report = gate_summary(
|
||
|
|
source_checks=[{"status": "pass"}],
|
||
|
|
review_ledger={"summary": {"evidence_bindings_invalid": 0, "claims": 1, "items": 1}},
|
||
|
|
retrieval_score={"question_count": 24, "mean_conditional_recall_at_5": 1.0},
|
||
|
|
answer_reports={"answers": {"status": "valid", "checked": 0, "issues": []}},
|
||
|
|
support_judgments={"strict_group_support_fraction": 1.0},
|
||
|
|
answer_scores={"50k": {"content_points_before_citation_penalty": 48, "max_points": 48}},
|
||
|
|
semantic_results={
|
||
|
|
"claim_count": 1,
|
||
|
|
"outcomes": {"advisory_support_consensus": 1},
|
||
|
|
},
|
||
|
|
)
|
||
|
|
|
||
|
|
assert (
|
||
|
|
report["gates"]["semantic_support_review"]["status"]
|
||
|
|
== "advisory_complete_candidate_actions_pending"
|
||
|
|
)
|
||
|
|
assert report["qualification"] == "not_qualified"
|