vasily
78033bb830
CI / quality (push) Canceled after 0s
Adds claim-bound online evidence records and citation verification scripts, node removal transport/adapter, retrieval benchmark and owner-review tooling, GraphCanvas/SourceDetail knowledge browser UI, and the ingest handoff helper. Bumps the agents/gewu pack to the online-verification branch.
75 lines
2.4 KiB
Python
75 lines
2.4 KiB
Python
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from scripts.score_retrieval_benchmark import score
|
|
|
|
|
|
def _inputs(tmp_path: Path) -> tuple[Path, Path]:
|
|
results = tmp_path / "results.jsonl"
|
|
rows = [
|
|
{
|
|
"kind": "manifest",
|
|
"question_count": 2,
|
|
"questions_sha256": "frozen-hash",
|
|
},
|
|
{
|
|
"kind": "retrieval",
|
|
"question": {"id": "Q01"},
|
|
"seconds": 0.2,
|
|
"is_error": False,
|
|
"returned_node_ids": ["node-a", "other"],
|
|
"response": [{"canonical_revision": "a" * 40}],
|
|
},
|
|
{
|
|
"kind": "retrieval",
|
|
"question": {"id": "Q02"},
|
|
"seconds": 0.4,
|
|
"is_error": False,
|
|
"returned_node_ids": ["node-c"],
|
|
"response": [{"canonical_revision": "a" * 40}],
|
|
},
|
|
]
|
|
results.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8")
|
|
judgments = tmp_path / "judgments.json"
|
|
judgments.write_text(
|
|
json.dumps(
|
|
{
|
|
"method": "independent relevance review",
|
|
"fully_answerable_positive_questions": {
|
|
"Q01": ["node-a", "node-b"],
|
|
"Q02": ["node-c"],
|
|
},
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
return results, judgments
|
|
|
|
|
|
def test_scores_recall_on_judged_answerable_questions_only(tmp_path: Path) -> None:
|
|
results, judgments = _inputs(tmp_path)
|
|
|
|
result = score(results, judgments)
|
|
|
|
assert result["question_count"] == 2
|
|
assert result["judged_answerable_questions"] == 2
|
|
assert result["mean_conditional_recall_at_5"] == 0.75
|
|
assert result["conditional_hit_rate"] == 1.0
|
|
assert result["canonical_revision"] == "a" * 40
|
|
|
|
|
|
def test_rejects_missing_judged_questions_and_mixed_canonical_heads(tmp_path: Path) -> None:
|
|
results, judgments = _inputs(tmp_path)
|
|
rows = results.read_text(encoding="utf-8").splitlines()
|
|
missing_results = tmp_path / "missing.jsonl"
|
|
missing_results.write_text("\n".join(rows[:2]) + "\n", encoding="utf-8")
|
|
with pytest.raises(ValueError, match="count does not match"):
|
|
score(missing_results, judgments)
|
|
|
|
rows[2] = rows[2].replace("a" * 40, "b" * 40)
|
|
mixed = tmp_path / "mixed.jsonl"
|
|
mixed.write_text("\n".join(rows) + "\n", encoding="utf-8")
|
|
with pytest.raises(ValueError, match="more than one canonical revision"):
|
|
score(mixed, judgments)
|