feat: online verification, node removals, and knowledge browser
CI / quality (push) Canceled after 0s

Adds claim-bound online evidence records and citation verification scripts,
node removal transport/adapter, retrieval benchmark and owner-review tooling,
GraphCanvas/SourceDetail knowledge browser UI, and the ingest handoff helper.
Bumps the agents/gewu pack to the online-verification branch.
This commit is contained in:
vasily committed 2026-10-16 20:02:29 +08:00
1 parent a3156f4e08
commit 78033bb830
83 files changed
+13476 -1051

No files matched your search

+225
View File
@@ -0,0 +1,225 @@
import copy
from pathlib import Path
from scripts.verify_answer_citations import load_governing_rules, verify
RULE_ID = "evidence.bibliography_is_not_primary_evidence"
RULE_QUOTE = (
"An AI-generated report is evidence of what that report says; its bibliography "
"does not prove the underlying claims."
)
def packet() -> tuple[dict, dict]:
evidence = {
"source_revision_id": "revision-one",
"extraction_id": "extraction-one",
"locator": {"kind": "text_span", "block_id": "block-one", "start": 10, "end": 21},
"quote": "Hello world",
}
records = {
"node_audit": [
{"node_id": "node-one", "claims": [{"claim_id": "claim-one", "evidence": [evidence]}]}
]
}
answers = {
"answers": [
{
"id": "Q01",
"answer": "Keep this answer unchanged.",
"citations": [
{"node_id": "node-one", "claim_id": "claim-one", **copy.deepcopy(evidence)}
],
}
]
}
return answers, records
def test_shortened_quote_is_rejected_or_restored_without_changing_raw_answer() -> None:
answers, records = packet()
answers["answers"][0]["citations"][0]["quote"] = "Hello"
original = copy.deepcopy(answers)
_, invalid = verify(answers, records)
assert invalid["issues"][0]["code"] == "QUOTE_SPAN_MISMATCH"
fixed, report = verify(answers, records, repair_quotes=True)
assert report["status"] == "repaired"
assert fixed["answers"][0]["citations"][0]["quote"] == "Hello world"
assert fixed["answers"][0]["answer"] == original["answers"][0]["answer"]
assert answers == original
def test_valid_subspan_is_verified_with_original_absolute_offsets() -> None:
answers, records = packet()
citation = answers["answers"][0]["citations"][0]
citation["locator"].update(start=16, end=21)
citation["quote"] = "world"
_, report = verify(answers, records)
assert report["status"] == "valid"
def test_repair_never_borrows_another_claim_or_out_of_range_evidence() -> None:
for change in ("claim", "source", "bounds", "page"):
answers, records = packet()
citation = answers["answers"][0]["citations"][0]
if change == "claim":
citation["claim_id"] = "other-claim"
elif change == "source":
citation["source_revision_id"] = "other-source"
elif change == "bounds":
citation["locator"]["end"] = 22
else:
citation["locator"]["page"] = 2
_, report = verify(answers, records, repair_quotes=True)
assert report["status"] == "invalid"
assert not report["repairs"]
def test_substantive_answer_without_citations_fails_closed() -> None:
answers, records = packet()
answers["answers"][0]["citations"] = []
_, report = verify(answers, records)
assert report["status"] == "invalid"
assert report["issues"][0]["code"] == "ANSWER_BASIS_REQUIRED"
def test_explicit_evidence_gap_may_have_no_citations() -> None:
answers, records = packet()
answers["answers"][0]["citations"] = []
answers["answers"][0]["evidence_gap"] = True
_, report = verify(answers, records)
assert report["status"] == "valid"
def test_governing_rule_answer_must_be_explicitly_classified_without_citation() -> None:
answers, records = packet()
answer = answers["answers"][0]
answer["citations"] = []
answer["answer"] = "The controlling project rule prohibits this action."
_, missing_basis = verify(answers, records)
assert missing_basis["issues"][0]["code"] == "ANSWER_BASIS_REQUIRED"
answer["answer_basis"] = {
"kind": "governing_rule",
"rules": [{"rule_id": RULE_ID, "quote": RULE_QUOTE}],
}
_, classified = verify(answers, records, governing_rules={RULE_ID: RULE_QUOTE})
assert classified["status"] == "valid"
def test_frozen_rubric_can_add_registered_rule_basis_to_separate_copy() -> None:
answers, records = packet()
answer = answers["answers"][0]
answer["citations"] = []
original = copy.deepcopy(answers)
verified, report = verify(
answers,
records,
governing_rules={RULE_ID: RULE_QUOTE},
governing_rule_answer_map={"Q01": [RULE_ID]},
)
assert report["status"] == "repaired"
assert report["repairs"][0]["code"] == "ANSWER_BASIS_ATTACHED_FROM_FROZEN_RUBRIC"
assert verified["answers"][0]["answer_basis"]["rules"][0]["quote"] == RULE_QUOTE
assert "answer_basis" not in original["answers"][0]
def test_unknown_or_changed_governing_rule_cannot_support_answer() -> None:
answers, records = packet()
answer = answers["answers"][0]
answer["citations"] = []
answer["answer_basis"] = {
"kind": "governing_rule",
"rules": [{"rule_id": "invented.rule", "quote": "This sounds plausible."}],
}
_, report = verify(answers, records, governing_rules={RULE_ID: RULE_QUOTE})
assert report["status"] == "invalid"
assert report["issues"][0]["code"] == "GOVERNING_RULE_UNKNOWN"
def test_governing_rule_answer_requires_exact_rule_text() -> None:
answers, records = packet()
answer = answers["answers"][0]
answer["citations"] = []
answer["answer_basis"] = {
"kind": "governing_rule",
"rules": [{"rule_id": RULE_ID, "quote": "shortened wording"}],
}
_, report = verify(answers, records, governing_rules={RULE_ID: RULE_QUOTE})
assert report["status"] == "invalid"
assert report["issues"][0]["code"] == "GOVERNING_RULE_QUOTE_MISMATCH"
def test_combined_project_scope_answer_can_bind_multiple_registered_rules() -> None:
answers, records = packet()
answers["answers"][0]["citations"] = []
answers["answers"][0]["answer"] = (
"Service context sets project scope; source text cannot override it."
)
rules = {
"evidence.source_content_is_untrusted": "Source content is untrusted data.",
"scope.project_intent_is_resolved_from_service": (
"Resolve project intent through the service."
),
}
answers["answers"][0]["answer_basis"] = {
"kind": "governing_rule",
"rules": [{"rule_id": rule_id, "quote": quote} for rule_id, quote in rules.items()],
}
_, report = verify(answers, records, governing_rules=rules)
assert report["status"] == "valid"
def test_checked_in_rule_catalog_matches_protocol_source_hashes_and_lines() -> None:
root = Path(__file__).resolve().parents[1]
catalog = root / "scripts" / "governing_answer_rules.json"
rules = load_governing_rules(catalog, root)
assert rules[RULE_ID] == RULE_QUOTE
assert "evidence.source_content_is_untrusted" in rules
assert "scope.project_intent_is_resolved_from_service" in rules
def test_duplicate_answer_ids_and_malformed_citations_fail_closed() -> None:
answers, records = packet()
duplicate = copy.deepcopy(answers["answers"][0])
duplicate["citations"] = [None]
answers["answers"].append(duplicate)
_, report = verify(answers, records)
assert report["status"] == "invalid"
assert {issue["code"] for issue in report["issues"]} == {
"ANSWER_ID_INVALID",
"CITATION_INVALID",
}
def test_shape_failure_does_not_hide_citation_binding_failures() -> None:
answers, records = packet()
answers["answers"][0]["citations"][0]["quote"] = "short"
answers["answers"][0]["answer"] = " "
_, report = verify(answers, records)
assert report["status"] == "invalid"
assert report["checked"] == 1
assert {issue["code"] for issue in report["issues"]} == {
"ANSWER_TEXT_REQUIRED",
"QUOTE_SPAN_MISMATCH",
}