feat: online verification, node removals, and knowledge browser
CI / quality (push) Canceled after 0s
CI / quality (push) Canceled after 0s
Adds claim-bound online evidence records and citation verification scripts, node removal transport/adapter, retrieval benchmark and owner-review tooling, GraphCanvas/SourceDetail knowledge browser UI, and the ingest handoff helper. Bumps the agents/gewu pack to the online-verification branch.
This commit is contained in:
1 parent
a3156f4e08
commit
78033bb830
83 files changed
+13476
-1051
No files matched your search
@@ -0,0 +1,74 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from scripts.score_retrieval_benchmark import score
|
||||
|
||||
|
||||
def _inputs(tmp_path: Path) -> tuple[Path, Path]:
|
||||
results = tmp_path / "results.jsonl"
|
||||
rows = [
|
||||
{
|
||||
"kind": "manifest",
|
||||
"question_count": 2,
|
||||
"questions_sha256": "frozen-hash",
|
||||
},
|
||||
{
|
||||
"kind": "retrieval",
|
||||
"question": {"id": "Q01"},
|
||||
"seconds": 0.2,
|
||||
"is_error": False,
|
||||
"returned_node_ids": ["node-a", "other"],
|
||||
"response": [{"canonical_revision": "a" * 40}],
|
||||
},
|
||||
{
|
||||
"kind": "retrieval",
|
||||
"question": {"id": "Q02"},
|
||||
"seconds": 0.4,
|
||||
"is_error": False,
|
||||
"returned_node_ids": ["node-c"],
|
||||
"response": [{"canonical_revision": "a" * 40}],
|
||||
},
|
||||
]
|
||||
results.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8")
|
||||
judgments = tmp_path / "judgments.json"
|
||||
judgments.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"method": "independent relevance review",
|
||||
"fully_answerable_positive_questions": {
|
||||
"Q01": ["node-a", "node-b"],
|
||||
"Q02": ["node-c"],
|
||||
},
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return results, judgments
|
||||
|
||||
|
||||
def test_scores_recall_on_judged_answerable_questions_only(tmp_path: Path) -> None:
|
||||
results, judgments = _inputs(tmp_path)
|
||||
|
||||
result = score(results, judgments)
|
||||
|
||||
assert result["question_count"] == 2
|
||||
assert result["judged_answerable_questions"] == 2
|
||||
assert result["mean_conditional_recall_at_5"] == 0.75
|
||||
assert result["conditional_hit_rate"] == 1.0
|
||||
assert result["canonical_revision"] == "a" * 40
|
||||
|
||||
|
||||
def test_rejects_missing_judged_questions_and_mixed_canonical_heads(tmp_path: Path) -> None:
|
||||
results, judgments = _inputs(tmp_path)
|
||||
rows = results.read_text(encoding="utf-8").splitlines()
|
||||
missing_results = tmp_path / "missing.jsonl"
|
||||
missing_results.write_text("\n".join(rows[:2]) + "\n", encoding="utf-8")
|
||||
with pytest.raises(ValueError, match="count does not match"):
|
||||
score(missing_results, judgments)
|
||||
|
||||
rows[2] = rows[2].replace("a" * 40, "b" * 40)
|
||||
mixed = tmp_path / "mixed.jsonl"
|
||||
mixed.write_text("\n".join(rows) + "\n", encoding="utf-8")
|
||||
with pytest.raises(ValueError, match="more than one canonical revision"):
|
||||
score(mixed, judgments)
|
||||
Reference in new issue
Block a user