feat: online verification, node removals, and knowledge browser
CI / quality (push) Canceled after 0s

Adds claim-bound online evidence records and citation verification scripts,
node removal transport/adapter, retrieval benchmark and owner-review tooling,
GraphCanvas/SourceDetail knowledge browser UI, and the ingest handoff helper.
Bumps the agents/gewu pack to the online-verification branch.
This commit is contained in:
vasily committed 2026-10-16 20:02:29 +08:00
1 parent a3156f4e08
commit 78033bb830
83 files changed
+13476 -1051

No files matched your search

@@ -0,0 +1,48 @@
from datetime import UTC, datetime, timedelta
import pytest
from gewu.adapters.sqlite_node_removals import RemovalConflict, SQLiteNodeRemovalStore
def test_removed_node_is_hidden_until_restored_and_restore_window_is_seven_days(tmp_path) -> None:
store = SQLiteNodeRemovalStore(tmp_path / "node-removals.sqlite3")
removed = store.request("dome", "node-a", "A node", "editor")
assert store.is_hidden("dome", "node-a")
assert store.list_restorable("dome") == (removed,)
restore_window = datetime.fromisoformat(removed.restore_until) - datetime.fromisoformat(
removed.requested_at
)
assert restore_window == timedelta(days=7)
restored = store.restore("dome", "node-a", "editor")
assert restored == removed
assert not store.is_hidden("dome", "node-a")
assert store.list_restorable("dome") == ()
def test_expired_removal_stays_hidden_and_cannot_be_restored(tmp_path) -> None:
store = SQLiteNodeRemovalStore(tmp_path / "node-removals.sqlite3")
removed = store.request("dome", "node-a", "A node", "editor")
with store._connect() as connection:
connection.execute(
"UPDATE node_removals SET restore_until=? WHERE removal_id=?",
((datetime.now(UTC) - timedelta(seconds=1)).isoformat(), removed.removal_id),
)
assert store.is_hidden("dome", "node-a")
assert store.list_restorable("dome") == ()
with pytest.raises(RemovalConflict, match="expired"):
store.restore("dome", "node-a", "editor")
def test_removal_survives_store_reopen_and_duplicate_removal_is_rejected(tmp_path) -> None:
path = tmp_path / "node-removals.sqlite3"
store = SQLiteNodeRemovalStore(path)
store.request("dome", "node-a", "A node", "editor")
reopened = SQLiteNodeRemovalStore(path)
assert reopened.is_hidden("dome", "node-a")
with pytest.raises(RemovalConflict):
reopened.request("dome", "node-a", "A node", "editor")
+30
View File
@@ -11,6 +11,7 @@ import pytest
import yaml
from gewu.adapters.git_repository import GitCanonicalRepository
from gewu.adapters.sqlite_node_removals import SQLiteNodeRemovalStore
from gewu.adapters.sqlite_search import SQLiteSearchIndex
from gewu.application.identity import Principal
from gewu.application.search import CanonicalSearchService, SearchError
@@ -210,6 +211,7 @@ def test_assumption_and_limitation_claims_are_searchable_and_cited(tmp_path: Pat
query="assumptionspecific",
scope=Scope(projects=("dome-performance",), include_shared=False),
budget_tokens=50_000,
include_evidence=True,
),
principal=reader,
)
@@ -700,3 +702,31 @@ def test_neighbor_source_metadata_requires_an_ingested_for_scope(tmp_path: Path)
neighbor = cast(dict[str, object], response.data[0]["neighbor"])
assert neighbor["kind"] == "source"
assert cast(dict[str, object], neighbor["record"])["revision_id"] == SOURCE_REVISION.revision_id
def test_durable_node_removal_hides_from_search_and_returns_after_restore(tmp_path: Path) -> None:
repository = GitCanonicalRepository(tmp_path / "knowledge.git")
_publish_nodes(repository, tmp_path / "publishing", (NODE,))
removals = SQLiteNodeRemovalStore(tmp_path / "node-removals.sqlite3")
index = SQLiteSearchIndex(
tmp_path / "search.sqlite3", repository, node_removals_path=removals.path
)
service = CanonicalSearchService(
index=index,
project_ids_provider=lambda: PROJECTS.keys(),
canonical_head=repository.head,
)
principal = Principal(
"editor", "editor-key", frozenset({"editor"}), frozenset({NODE.project_id})
)
request = SearchRequest(
query=NODE.title,
scope=Scope(projects=(NODE.project_id,), include_shared=False),
budget_tokens=5000,
)
assert service.search(request, principal=principal).data
removals.request(NODE.project_id, NODE.node_id, NODE.title, "editor")
assert service.search(request, principal=principal).data == ()
removals.restore(NODE.project_id, NODE.node_id, "editor")
assert service.search(request, principal=principal).data
+11
View File
@@ -40,6 +40,7 @@ def test_local_factory_composes_private_persistent_read_service(
"/health",
"/v1/diagnostics",
"/v1/admin/projects",
"/v1/admin/projects/{project_id}",
"/v1/admin/credentials",
"/v1/admin/credentials/{credential_id}",
"/v1/admin/events",
@@ -56,16 +57,26 @@ def test_local_factory_composes_private_persistent_read_service(
"/v1/projects/{project_id}/sources/{revision_id}",
"/v1/projects/{project_id}/sources/{revision_id}/artifact",
"/v1/projects/{project_id}/sources/{revision_id}/extractions/{extraction_id}/blocks/{block_id}",
"/v1/projects/{project_id}/sources/{revision_id}/nodes",
"/v1/neighbors",
"/v1/projects",
"/v1/search",
"/v1/knowledge/browse",
"/v1/knowledge/graph",
"/v1/sources",
"/v1/session",
"/auth/owner-session",
"/v1/ingest-reviews",
"/v1/ingest-reviews/{review_id}/decision",
"/v1/projects/{project_id}/node-removals",
"/v1/projects/{project_id}/nodes/{node_id}/removal",
"/v1/projects/{project_id}/nodes/{node_id}/restore-removal",
}
assert (data_root / "knowledge.git" / "HEAD").is_file()
assert (data_root / "identity.sqlite3").is_file()
assert (data_root / "search.sqlite3").is_file()
assert (data_root / "node-removals.sqlite3").is_file()
assert (data_root / "node-removals.sqlite3").stat().st_mode & 0o077 == 0
assert (data_root / "search.sqlite3").stat().st_mode & 0o077 == 0
assert (data_root.stat().st_mode & 0o777) == 0o700
+65 -8
View File
@@ -257,9 +257,7 @@ def test_mcp_source_to_citation_vertical_slice(tmp_path: Path) -> None:
{"proposal_json": prepared_payload["proposal_json"][:-1]},
)
assert not truncated_evaluation.is_error, truncated_evaluation.content
truncated_evaluation_payload = json.loads(
_text_payload(truncated_evaluation)
)
truncated_evaluation_payload = json.loads(_text_payload(truncated_evaluation))
assert truncated_evaluation_payload["proposal_id"] == (
prepared_proposal.proposal_id
)
@@ -277,9 +275,7 @@ def test_mcp_source_to_citation_vertical_slice(tmp_path: Path) -> None:
"proposal_evaluate", {"proposal_json": '{"schema_version":'}
)
assert not malformed_evaluation.is_error, malformed_evaluation.content
malformed_evaluation_payload = json.loads(
_text_payload(malformed_evaluation)
)
malformed_evaluation_payload = json.loads(_text_payload(malformed_evaluation))
assert malformed_evaluation_payload["status"] == "invalid"
assert malformed_evaluation_payload["findings"][0]["code"] == (
"PROPOSAL_JSON_INVALID"
@@ -298,8 +294,7 @@ def test_mcp_source_to_citation_vertical_slice(tmp_path: Path) -> None:
invalid_payload = json.loads(_text_payload(invalid_result))
assert invalid_payload["status"] == "invalid"
assert any(
issue["path"].endswith("locator")
and "PDF locators" in issue["message"]
issue["path"].endswith("locator") and "PDF locators" in issue["message"]
for issue in invalid_payload["issues"]
)
invalid_review = await mcp.call_tool(
@@ -313,6 +308,41 @@ def test_mcp_source_to_citation_vertical_slice(tmp_path: Path) -> None:
"extraction_id",
"relevance_rationale",
}
project_only_token = identities.provision(
credential_id="project-only-source-agent-key",
principal_id="agent-project-only",
roles=("agent",),
project_ids=("dome-performance",),
)
project_only_draft = json.loads(json.dumps(draft))
project_only_draft["proposal_id"] = "proposal-project-only-source"
project_only_draft["scope"]["include_shared"] = False
project_only_draft["proposer_id"] = "agent-project-only"
project_only_draft["operations"] = [project_only_draft["operations"][0]]
async with httpx2.AsyncClient(
transport=httpx2.ASGITransport(app=app),
base_url="http://localhost",
headers={"Authorization": f"Bearer {project_only_token}"},
) as project_only_http:
async with Client(
streamable_http_client(
"http://localhost/mcp", http_client=project_only_http
)
) as project_only_mcp:
project_source = await project_only_mcp.call_tool(
"proposal_prepare",
{"proposal_json": json.dumps(project_only_draft)},
)
assert not project_source.is_error, project_source.content
assert json.loads(_text_payload(project_source))["status"] == "prepared"
shared_draft = json.loads(json.dumps(project_only_draft))
shared_draft["proposal_id"] = "proposal-project-only-shared-source"
shared_draft["operations"][0]["source"]["ingested_for"] = ["shared"]
shared_source = await project_only_mcp.call_tool(
"proposal_prepare",
{"proposal_json": json.dumps(shared_draft)},
)
assert shared_source.is_error
denied_lens = await mcp.call_tool(
"get_project_context", {"project_id": "interactive-video"}
)
@@ -387,6 +417,11 @@ def test_mcp_source_to_citation_vertical_slice(tmp_path: Path) -> None:
}
],
}
invalid_run = await mcp.call_tool("run_create", {"request_json": "{}"})
assert not invalid_run.is_error
invalid_run_result = json.loads(_text_payload(invalid_run))
assert invalid_run_result["status"] == "invalid"
assert "run_id" in {issue["path"] for issue in invalid_run_result["issues"]}
created_run = await mcp.call_tool(
"run_create", {"request_json": json.dumps(run_request)}
)
@@ -403,6 +438,23 @@ def test_mcp_source_to_citation_vertical_slice(tmp_path: Path) -> None:
)
assert not saved_run.is_error
assert json.loads(_text_payload(saved_run))["revision"] == 1
invalid_checkpoint = await mcp.call_tool(
"run_checkpoint",
{
"request_json": json.dumps(
{
**checkpoint,
"expected_revision": 1,
"coverage": {"skipped": ["pending source"]},
}
)
},
)
assert not invalid_checkpoint.is_error
invalid_result = json.loads(_text_payload(invalid_checkpoint))
assert invalid_result["status"] == "invalid"
assert invalid_result["issues"][0]["path"] == "coverage.skipped[0]"
assert "pending source" not in _text_payload(invalid_checkpoint)
read_run = await mcp.call_tool("run_status", {"run_id": "mcp-run-pilot"})
assert not read_run.is_error
assert json.loads(_text_payload(read_run))["revision"] == 1
@@ -647,6 +699,7 @@ def test_mcp_source_to_citation_vertical_slice(tmp_path: Path) -> None:
)
assert not html_block_result.is_error
html_block = json.loads(_text_payload(html_block_result))
assert html_block["normalization_profile"] == "unicode-nfc-lf-v1"
assert "Exact HTML statement." in html_block["normalized_text"]
assert "original_text" not in html_block
assert "normalization_map" not in html_block
@@ -663,6 +716,9 @@ def test_mcp_source_to_citation_vertical_slice(tmp_path: Path) -> None:
)
assert not mapped_html_result.is_error
mapped_html = json.loads(_text_payload(mapped_html_result))
assert (
mapped_html["normalization_profile"] == html_block["normalization_profile"]
)
assert mapped_html["original_text"]
assert len(mapped_html["normalization_map"]) == len(
mapped_html["normalized_text"]
@@ -705,6 +761,7 @@ def test_mcp_source_to_citation_vertical_slice(tmp_path: Path) -> None:
)
assert not pdf_block_result.is_error
pdf_block = json.loads(_text_payload(pdf_block_result))
assert pdf_block["normalization_profile"] == "unicode-nfc-lf-v1"
assert pdf_block["page"] == 1
assert "Archived PDF evidence." in pdf_block["normalized_text"]
assert "normalization_map" not in pdf_block
+95 -2
View File
@@ -127,7 +127,7 @@ def test_authenticated_search_scope_through_http_and_mcp(tmp_path: Path) -> None
unbuilt_state = diagnostic_unbuilt.json()
assert unbuilt_state["index_state"] == "not_built"
assert unbuilt_state["index_revision"] is None
assert unbuilt_state["index_schema_version"] == 4
assert unbuilt_state["index_schema_version"] == 5
result = await http.post(
"/v1/search",
json={
@@ -138,6 +138,7 @@ def test_authenticated_search_scope_through_http_and_mcp(tmp_path: Path) -> None
},
"limit": 10,
"budget_tokens": 5000,
"include_evidence": False,
},
)
assert result.status_code == 200, result.text
@@ -145,7 +146,90 @@ def test_authenticated_search_scope_through_http_and_mcp(tmp_path: Path) -> None
assert envelope["retrieval_mode"] == "keyword"
assert envelope["index_revision"] == repository.head()
assert envelope["data"][0]["node"]["node_id"] == NODE.node_id
assert envelope["citations"][0]["quote"] == NODE.claims[0].evidence[0].quote
assert envelope["data"][0]["evidence_included"] is False
compact_claim = envelope["data"][0]["node"]["claims"][0]
assert "evidence" not in compact_claim
assert compact_claim["evidence_count"] == len(NODE.claims[0].evidence)
assert envelope["citations"] == []
full_result = await http.post(
"/v1/search",
json={
"query": "400 visitors",
"scope": {"projects": ["dome-performance"], "include_shared": False},
"limit": 10,
"budget_tokens": 5000,
"include_evidence": True,
},
)
assert full_result.status_code == 200, full_result.text
full_row = full_result.json()["data"][0]
assert full_row["evidence_included"] is True
assert full_row["node"]["claims"][0]["evidence"]
assert full_result.json()["citations"][0]["quote"] == NODE.claims[0].evidence[0].quote
browse = await http.post(
"/v1/knowledge/browse",
json={"scope": {"projects": ["dome-performance"], "include_projects": [], "include_shared": False}, "limit": 2},
)
assert browse.status_code == 200, browse.text
browse_body = browse.json()
assert browse_body["project_node_count"] == 4
assert browse_body["filtered_node_count"] == 4
assert len(browse_body["nodes"]) == 2
assert browse_body["next_cursor"]
assert [item["title"] for item in browse_body["nodes"]] == sorted(
item["title"] for item in browse_body["nodes"]
)
assert browse_body["facets"]["tags"]
browse_page_two = await http.post(
"/v1/knowledge/browse",
json={"scope": {"projects": ["dome-performance"], "include_projects": [], "include_shared": False}, "limit": 2, "cursor": browse_body["next_cursor"]},
)
assert browse_page_two.status_code == 200
assert len(browse_page_two.json()["nodes"]) == 2
assert browse_page_two.json()["next_cursor"] is None
filtered_browse = await http.post(
"/v1/knowledge/browse",
json={"scope": {"projects": ["dome-performance"], "include_projects": [], "include_shared": False}, "node_types": ["problem"]},
)
assert filtered_browse.status_code == 200
assert filtered_browse.json()["filtered_node_count"] == 1
assert filtered_browse.json()["nodes"][0]["node_type"] == "problem"
assert filtered_browse.json()["project_node_count"] == 4
graph = await http.post(
"/v1/knowledge/graph",
json={"scope": {"projects": ["dome-performance"], "include_projects": [], "include_shared": False}},
)
assert graph.status_code == 200, graph.text
assert len(graph.json()["nodes"]) == 4
assert len(graph.json()["edges"]) == 2
assert graph.json()["truncated"] is False
neighborhood = await http.post(
"/v1/knowledge/graph",
json={"scope": {"projects": ["dome-performance"], "include_projects": [], "include_shared": False}, "center_node_id": NODE.node_id},
)
assert neighborhood.status_code == 200, neighborhood.text
assert neighborhood.json()["center_node_id"] == NODE.node_id
assert NODE.node_id in {item["node_id"] for item in neighborhood.json()["nodes"]}
assert len(neighborhood.json()["nodes"]) == 3
center_only = await http.post(
"/v1/knowledge/graph",
json={"scope": {"projects": ["dome-performance"], "include_projects": [], "include_shared": False}, "center_node_id": NODE.node_id, "max_nodes": 1},
)
assert center_only.status_code == 200
assert [item["node_id"] for item in center_only.json()["nodes"]] == [NODE.node_id]
assert center_only.json()["edges"] == []
denied_neighborhood = await http.post(
"/v1/knowledge/graph",
json={"scope": {"projects": ["other-project"], "include_projects": [], "include_shared": False}, "center_node_id": NODE.node_id},
)
assert denied_neighborhood.status_code == 404
capped_graph = await http.post(
"/v1/knowledge/graph",
json={"scope": {"projects": ["dome-performance"], "include_projects": [], "include_shared": False}, "max_nodes": 1},
)
assert capped_graph.status_code == 200
assert len(capped_graph.json()["nodes"]) == 1
assert capped_graph.json()["truncated"] is True
listed_sources = await http.post(
"/v1/sources",
json={
@@ -155,6 +239,15 @@ def test_authenticated_search_scope_through_http_and_mcp(tmp_path: Path) -> None
},
)
assert listed_sources.status_code == 200, listed_sources.text
source_nodes_resp = await http.get(
f"/v1/projects/dome-performance/sources/{SOURCE_REVISION.revision_id}/nodes"
)
assert source_nodes_resp.status_code == 200, source_nodes_resp.text
source_nodes_data = source_nodes_resp.json()
assert source_nodes_data["total"] >= 1
assert source_nodes_data["nodes"][0]["relation_basis"] in ("citation", "edge", "both")
assert listed_sources.json()["filtered_count"] == 2
assert listed_sources.json()["data"][0]["source"]["revision_id"] == (
SOURCE_REVISION.revision_id
)
+142
View File
@@ -5,6 +5,7 @@ import sys
import threading
from collections.abc import Callable
from concurrent.futures import ThreadPoolExecutor
from datetime import timedelta
from pathlib import Path
import pyotp
@@ -154,6 +155,147 @@ def test_acceptance_commits_records_and_retry_returns_original_receipt(tmp_path:
assert error.value.code == "IDEMPOTENCY_CONFLICT"
def test_second_source_enriches_stable_node_and_preserves_history(tmp_path: Path) -> None:
service, repository, artifact_store, source, extraction = _service(tmp_path)
first = service.submit(_proposal(repository, source), actor=ACTOR)
assert first.receipt is not None
source_bytes = b"An independent attendance log records 400 visitors over the 2026 season."
upload = artifact_store.begin(
filename="independent-attendance.md",
media_type="text/markdown",
byte_length=len(source_bytes),
expected_sha256=hashlib.sha256(source_bytes).hexdigest(),
now=103,
)
artifact_store.put_chunk(
upload.upload_id,
offset=0,
content=source_bytes,
chunk_sha256=hashlib.sha256(source_bytes).hexdigest(),
now=104,
)
artifact_store.finalize(upload.upload_id, now=105)
second_source = source.model_copy(
update={
"source_id": "shared:independent-attendance",
"revision_id": "shared:independent-attendance:r1",
"artifact_sha256": hashlib.sha256(source_bytes).hexdigest(),
"byte_length": len(source_bytes),
"title": "Independent seasonal attendance log",
"authors": ("Independent observer",),
}
)
second_extraction = TextMarkdownExtractor().extract(
second_source.revision_id, source_bytes, "text/markdown"
)
block = second_extraction.blocks[0]
quote = block.normalized_text
old_evidence = NODE.claims[0].evidence[0]
new_evidence = old_evidence.model_copy(
update={
"source_revision_id": second_source.revision_id,
"extraction_id": second_extraction.extraction_id,
"quote": quote,
"locator": old_evidence.locator.model_copy(
update={"block_id": block.block_id, "start": 0, "end": len(quote)}
),
}
)
assessment = NODE.claims[0].assessment.model_copy(
update={
"assessment_id": "assessment-independent-attendance",
"evidence_digests": (evidence_digest(old_evidence), evidence_digest(new_evidence)),
"assessor_id": "agent-independent",
"run_id": "run-independent",
"method": "independent source cross-check",
"rationale": "The independently recorded log reports the same count and season.",
}
)
enriched_claim = NODE.claims[0].model_copy(
update={"evidence": (old_evidence, new_evidence), "assessment": assessment}
)
from tests.domain.proposal_fixtures import NOW
enriched = NODE.model_copy(
update={
"updated": NOW + timedelta(minutes=1),
"agent_id": "agent-independent",
"run_id": "run-independent",
"model_id": "fixture-second-agent",
"claims": (enriched_claim,),
"tags": (*NODE.tags, "independent-source-supported"),
}
)
context = ValidationContext(
projects=PROJECTS,
policies=POLICIES,
sources={source.revision_id: source, second_source.revision_id: second_source},
extractions={
extraction.extraction_id: extraction,
second_extraction.extraction_id: second_extraction,
},
existing_nodes={NODE.node_id: NODE},
)
update_service = AcceptanceService(
repository=repository,
artifact_store=artifact_store,
workflow_root=tmp_path / "private-data" / "staging",
context_provider=lambda: context,
actor_is_current=lambda _actor: True,
)
proposal_draft = ProposalRevision(
proposal_id="proposal-second-source-enrichment",
revision=1,
initiating_project_id=NODE.project_id,
scope=Scope(projects=(NODE.project_id,), include_shared=True),
base_commit=repository.head(),
payload_digest="0" * 64,
proposer_id="agent-independent",
run_id="run-independent",
operations=(
ProposalOperation(operation="register_source", source=second_source),
ProposalOperation(
operation="update_node", target_id=NODE.node_id, node=enriched
),
),
)
proposal_update = proposal_draft.model_copy(
update={"payload_digest": proposal_payload_digest(proposal_draft)}
)
result = update_service.submit(
proposal_update,
actor=ActorContext("agent-independent", "agent-key-2"),
)
assert result.receipt is not None, result.validation.findings
assert result.receipt.accepted_node_ids == (NODE.node_id,)
node_path = (
f"projects/{hashlib.sha256(NODE.project_id.encode()).hexdigest()}/nodes/"
f"{hashlib.sha256(NODE.node_id.encode()).hexdigest()}.yaml"
)
old_record = repository.read_file(first.receipt.commit_id, node_path)
new_record = repository.read_file(result.receipt.commit_id, node_path)
assert old_record is not None and new_record is not None
assert safe_yaml_record(old_record, KnowledgeNode) == NODE
accepted = safe_yaml_record(new_record, KnowledgeNode)
assert accepted.node_id == NODE.node_id
assert accepted.claims[0].claim_id == NODE.claims[0].claim_id
assert len(accepted.claims[0].evidence) == 2
assert {item.source_revision_id for item in accepted.claims[0].evidence} == {
source.revision_id,
second_source.revision_id,
}
before_retry = repository.head()
retry = update_service.submit(
proposal_update,
actor=ActorContext("agent-independent", "agent-key-2"),
)
assert retry.receipt == result.receipt
assert repository.head() == before_retry
def test_policy_exception_requires_exact_owner_decision_in_manifest(tmp_path: Path) -> None:
service, repository, _artifact_store, source, extraction = _service(tmp_path)
identities = SQLiteIdentityStore(
+38 -2
View File
@@ -9,12 +9,48 @@ import pytest
import yaml
from gewu.adapters.git_repository import GitCanonicalRepository
from gewu.application.registry import LocalContextProvider, LocalProjectRegistry
from gewu.domain.contracts import AutoAcceptPolicy, ProjectRecord, ProjectStatus
from gewu.application.registry import (
MAX_CONTEXT_RECORD_BYTES,
LocalContextProvider,
LocalProjectRegistry,
RegistryError,
)
from gewu.domain.contracts import AutoAcceptPolicy, ExtractionBlock, ProjectRecord, ProjectStatus
from gewu.domain.validation import extraction_digest
from tests.domain.proposal_fixtures import EXTRACTION, NODE, POLICIES, PROJECTS
def test_context_loads_large_normalization_maps_with_a_separate_bound(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
text = "a" * 100_000
block = ExtractionBlock(
block_id="long-block",
ordinal=0,
original_text=text,
normalized_text=text,
normalization_map=tuple((index, index + 1) for index in range(len(text))),
)
candidate = EXTRACTION.model_copy(update={"blocks": (block,)})
extraction = candidate.model_copy(update={"extraction_id": extraction_digest(candidate)})
raw = extraction.model_dump_json().encode()
assert len(raw) > MAX_CONTEXT_RECORD_BYTES
repository = GitCanonicalRepository(tmp_path / "knowledge.git")
repository.publish_files(
expected_head=repository.head(),
files={f"extractions/{extraction.extraction_id}.json": raw},
message="Publish long normalized extraction",
index_directory=tmp_path / "publishing",
)
provider = LocalContextProvider(repository, tmp_path / "registry.json")
assert provider().extractions[extraction.extraction_id] == extraction
monkeypatch.setattr("gewu.application.registry.MAX_CONTEXT_EXTRACTION_BYTES", len(raw) - 1)
with pytest.raises(RegistryError, match="Canonical extraction is unavailable or too large"):
provider()
def test_project_registry_create_and_archive_are_atomic_and_idempotent(tmp_path: Path) -> None:
registry_file = tmp_path / "control" / "registry.json"
registry = LocalProjectRegistry(
+179 -9
View File
@@ -45,11 +45,22 @@ def _validate(candidate: ProposalRevision) -> ValidationReport:
)
def test_supported_exact_citation_and_both_bundle_policies_are_eligible() -> None:
def test_supported_exact_citation_and_project_policy_are_eligible() -> None:
report = _validate(proposal())
assert report.status == "eligible", report.findings
assert report.eligible_for_auto_accept
assert set(report.affected_policy_ids) == {"dome-policy", "shared"}
assert set(report.affected_policy_ids) == {"dome-policy"}
def test_project_only_source_registration_does_not_write_shared_knowledge() -> None:
candidate = proposal(
node=None,
scope=Scope(projects=("dome-performance",), include_shared=False),
)
report = _validate(candidate)
assert report.status == "eligible", report.findings
assert report.eligible_for_auto_accept
assert set(report.affected_policy_ids) == {"dome-policy"}
def test_proposal_digest_preparation_fills_all_dependent_claim_digests() -> None:
@@ -186,11 +197,7 @@ def test_cross_project_source_bundle_requires_every_affected_policy() -> None:
current_base_commit=BASE_COMMIT,
)
assert eligible.status == "eligible", eligible.findings
assert set(eligible.affected_policy_ids) == {
"dome-policy",
"video-policy",
"shared",
}
assert set(eligible.affected_policy_ids) == {"dome-policy", "video-policy"}
restrictive_video_policy = POLICIES["video-policy"].model_copy(
update={"automatic_source_registration": False}
@@ -269,6 +276,77 @@ def test_node_and_assessor_attribution_cannot_impersonate_another_agent() -> Non
assert "AGENT_ATTRIBUTION_MISMATCH" in {finding.code for finding in node_report.findings}
def test_existing_node_can_gain_facets_without_replacing_its_assertion() -> None:
from datetime import timedelta
from tests.domain.proposal_fixtures import NOW
existing = NODE
enriched = NODE.model_copy(
update={"updated": NOW + timedelta(minutes=1), "tags": (*NODE.tags, "interactive-storytelling")}
)
draft = ProposalRevision(
proposal_id="proposal-enrich-facets",
revision=1,
initiating_project_id="dome-performance",
scope=Scope(projects=("dome-performance",), include_shared=False),
base_commit=BASE_COMMIT,
payload_digest="0" * 64,
proposer_id="agent-pilot",
run_id="run-pilot-2",
operations=(
ProposalOperation(operation="update_node", target_id=NODE.node_id, node=enriched),
),
)
candidate = draft.model_copy(update={"payload_digest": proposal_payload_digest(draft)})
report = validate_proposal(
candidate,
projects=PROJECTS,
policies=POLICIES,
sources={SOURCE_REVISION.revision_id: SOURCE_REVISION},
extractions={EXTRACTION.extraction_id: EXTRACTION},
existing_nodes={NODE.node_id: existing},
current_base_commit=BASE_COMMIT,
)
assert report.status == "eligible", report.findings
def test_existing_node_enrichment_rejects_removed_evidence() -> None:
from datetime import timedelta
from tests.domain.proposal_fixtures import NOW
changed_claim = CLAIM.model_copy(update={"text": "The accepted count was changed."})
enriched = NODE.model_copy(
update={"updated": NOW + timedelta(minutes=1), "claims": (changed_claim,)}
)
draft = ProposalRevision(
proposal_id="proposal-enrich-removal",
revision=1,
initiating_project_id="dome-performance",
scope=Scope(projects=("dome-performance",), include_shared=False),
base_commit=BASE_COMMIT,
payload_digest="0" * 64,
proposer_id="agent-pilot",
run_id="run-pilot-2",
operations=(
ProposalOperation(operation="update_node", target_id=NODE.node_id, node=enriched),
),
)
candidate = draft.model_copy(update={"payload_digest": proposal_payload_digest(draft)})
report = validate_proposal(
candidate,
projects=PROJECTS,
policies=POLICIES,
sources={SOURCE_REVISION.revision_id: SOURCE_REVISION},
extractions={EXTRACTION.extraction_id: EXTRACTION},
existing_nodes={NODE.node_id: NODE},
current_base_commit=BASE_COMMIT,
)
assert report.status == "invalid"
assert "NODE_ENRICHMENT_CHANGED_CLAIM" in {finding.code for finding in report.findings}
def test_entity_name_collision_requires_review_without_creating_a_link() -> None:
entity = NODE.model_copy(
update={"node_type": NodeType.ENTITY, "title": "The Dome", "aliases": ("Planetarium",)}
@@ -443,6 +521,69 @@ def test_changed_assertion_and_old_assessment_are_rejected() -> None:
assert "ASSESSMENT_STALE" in {finding.code for finding in report.findings}
def test_exact_evidence_quote_does_not_validate_mutated_table_number() -> None:
"""A structurally valid quote cannot establish that a changed claim is true."""
original_text = "The installation hosted 400 visitors during the 2026 season."
mutated_text = "The installation hosted 999 visitors during the 2026 season."
digest = assertion_digest(original_text)
evidence = EVIDENCE.model_copy(
update={
"assertion_path": "claims[0].text",
"assertion_digest": digest,
}
)
claim = CLAIM.model_copy(
update={
"text": original_text,
"evidence": (evidence,),
"assessment": CLAIM.assessment.model_copy(
update={
"assertion_digest": digest,
"evidence_digests": (evidence_digest(evidence),),
}
),
}
)
candidate = proposal(
node=NODE.model_copy(update={"claims": (claim,)}),
source=None,
)
candidate = candidate.model_copy(update={"payload_digest": proposal_payload_digest(candidate)})
structurally_valid = _validate(candidate)
mutated_claim = claim.model_copy(
update={
"text": mutated_text,
"evidence": (
evidence.model_copy(update={"assertion_digest": assertion_digest(mutated_text)}),
),
"assessment": claim.assessment.model_copy(
update={
"assertion_digest": assertion_digest(mutated_text),
"evidence_digests": (
evidence_digest(
evidence.model_copy(
update={"assertion_digest": assertion_digest(mutated_text)}
)
),
),
}
),
}
)
mutant = proposal(node=NODE.model_copy(update={"claims": (mutated_claim,)}), source=None)
mutant = mutant.model_copy(update={"payload_digest": proposal_payload_digest(mutant)})
structurally_valid_mutant = _validate(mutant)
assert structurally_valid.status == "eligible", structurally_valid.findings
assert structurally_valid_mutant.status == "needs-review"
assert "NUMERIC_CLAIM_UNGROUNDED" in {
finding.code for finding in structurally_valid_mutant.findings
}
assert evidence.quote == EVIDENCE.quote
assert "999" in mutated_claim.text and "999" not in evidence.quote
def test_assumption_and_limitation_claims_use_the_same_evidence_contract() -> None:
assumption_text = CLAIM.text
assumption_digest = assertion_digest(assumption_text)
@@ -510,12 +651,17 @@ def test_assumption_and_limitation_claims_use_the_same_evidence_contract() -> No
def test_missing_policy_and_explicit_scope_escape_do_not_accept() -> None:
no_shared_policy = {key: value for key, value in POLICIES.items() if key != "shared"}
candidate = proposal()
shared_source = SOURCE_REVISION.model_copy(update={"ingested_for": ("shared",)})
candidate = proposal(
node=None,
source=shared_source,
scope=Scope(projects=("dome-performance",), include_shared=True),
)
report = validate_proposal(
candidate,
projects=PROJECTS,
policies=no_shared_policy,
sources={SOURCE_REVISION.revision_id: SOURCE_REVISION},
sources={},
extractions={EXTRACTION.extraction_id: EXTRACTION},
existing_nodes={},
current_base_commit=BASE_COMMIT,
@@ -621,6 +767,30 @@ def test_relationship_evidence_is_bound_to_the_edge_and_exact_source_span() -> N
assert "EVIDENCE_ASSERTION_MISMATCH" in {finding.code for finding in invalid_report.findings}
def test_proposal_preparation_accepts_json_edge_evidence_arrays() -> None:
draft = proposal().model_dump(mode="json")
draft.pop("payload_digest")
draft["operations"].append(
{
"operation": "add_edge",
"edge": {
"edge_id": "edge-preparation-test",
"edge_type": "supports",
"source": {"kind": "node", "id": NODE.node_id},
"target": {"kind": "node", "id": "dome:other-node"},
"evidence": [EVIDENCE.model_dump(mode="json")],
},
}
)
prepared = prepare_proposal_digests(draft)
edge = next(item.edge for item in prepared.operations if item.edge is not None)
assert edge is not None
assert edge.evidence[0].assertion_path == "edge"
assert edge.evidence[0].assertion_digest == edge_assertion_digest(edge)
def test_contradiction_bundle_preserves_both_supported_claims() -> None:
import hashlib
+225
View File
@@ -0,0 +1,225 @@
import copy
from pathlib import Path
from scripts.verify_answer_citations import load_governing_rules, verify
RULE_ID = "evidence.bibliography_is_not_primary_evidence"
RULE_QUOTE = (
"An AI-generated report is evidence of what that report says; its bibliography "
"does not prove the underlying claims."
)
def packet() -> tuple[dict, dict]:
evidence = {
"source_revision_id": "revision-one",
"extraction_id": "extraction-one",
"locator": {"kind": "text_span", "block_id": "block-one", "start": 10, "end": 21},
"quote": "Hello world",
}
records = {
"node_audit": [
{"node_id": "node-one", "claims": [{"claim_id": "claim-one", "evidence": [evidence]}]}
]
}
answers = {
"answers": [
{
"id": "Q01",
"answer": "Keep this answer unchanged.",
"citations": [
{"node_id": "node-one", "claim_id": "claim-one", **copy.deepcopy(evidence)}
],
}
]
}
return answers, records
def test_shortened_quote_is_rejected_or_restored_without_changing_raw_answer() -> None:
answers, records = packet()
answers["answers"][0]["citations"][0]["quote"] = "Hello"
original = copy.deepcopy(answers)
_, invalid = verify(answers, records)
assert invalid["issues"][0]["code"] == "QUOTE_SPAN_MISMATCH"
fixed, report = verify(answers, records, repair_quotes=True)
assert report["status"] == "repaired"
assert fixed["answers"][0]["citations"][0]["quote"] == "Hello world"
assert fixed["answers"][0]["answer"] == original["answers"][0]["answer"]
assert answers == original
def test_valid_subspan_is_verified_with_original_absolute_offsets() -> None:
answers, records = packet()
citation = answers["answers"][0]["citations"][0]
citation["locator"].update(start=16, end=21)
citation["quote"] = "world"
_, report = verify(answers, records)
assert report["status"] == "valid"
def test_repair_never_borrows_another_claim_or_out_of_range_evidence() -> None:
for change in ("claim", "source", "bounds", "page"):
answers, records = packet()
citation = answers["answers"][0]["citations"][0]
if change == "claim":
citation["claim_id"] = "other-claim"
elif change == "source":
citation["source_revision_id"] = "other-source"
elif change == "bounds":
citation["locator"]["end"] = 22
else:
citation["locator"]["page"] = 2
_, report = verify(answers, records, repair_quotes=True)
assert report["status"] == "invalid"
assert not report["repairs"]
def test_substantive_answer_without_citations_fails_closed() -> None:
answers, records = packet()
answers["answers"][0]["citations"] = []
_, report = verify(answers, records)
assert report["status"] == "invalid"
assert report["issues"][0]["code"] == "ANSWER_BASIS_REQUIRED"
def test_explicit_evidence_gap_may_have_no_citations() -> None:
answers, records = packet()
answers["answers"][0]["citations"] = []
answers["answers"][0]["evidence_gap"] = True
_, report = verify(answers, records)
assert report["status"] == "valid"
def test_governing_rule_answer_must_be_explicitly_classified_without_citation() -> None:
answers, records = packet()
answer = answers["answers"][0]
answer["citations"] = []
answer["answer"] = "The controlling project rule prohibits this action."
_, missing_basis = verify(answers, records)
assert missing_basis["issues"][0]["code"] == "ANSWER_BASIS_REQUIRED"
answer["answer_basis"] = {
"kind": "governing_rule",
"rules": [{"rule_id": RULE_ID, "quote": RULE_QUOTE}],
}
_, classified = verify(answers, records, governing_rules={RULE_ID: RULE_QUOTE})
assert classified["status"] == "valid"
def test_frozen_rubric_can_add_registered_rule_basis_to_separate_copy() -> None:
answers, records = packet()
answer = answers["answers"][0]
answer["citations"] = []
original = copy.deepcopy(answers)
verified, report = verify(
answers,
records,
governing_rules={RULE_ID: RULE_QUOTE},
governing_rule_answer_map={"Q01": [RULE_ID]},
)
assert report["status"] == "repaired"
assert report["repairs"][0]["code"] == "ANSWER_BASIS_ATTACHED_FROM_FROZEN_RUBRIC"
assert verified["answers"][0]["answer_basis"]["rules"][0]["quote"] == RULE_QUOTE
assert "answer_basis" not in original["answers"][0]
def test_unknown_or_changed_governing_rule_cannot_support_answer() -> None:
answers, records = packet()
answer = answers["answers"][0]
answer["citations"] = []
answer["answer_basis"] = {
"kind": "governing_rule",
"rules": [{"rule_id": "invented.rule", "quote": "This sounds plausible."}],
}
_, report = verify(answers, records, governing_rules={RULE_ID: RULE_QUOTE})
assert report["status"] == "invalid"
assert report["issues"][0]["code"] == "GOVERNING_RULE_UNKNOWN"
def test_governing_rule_answer_requires_exact_rule_text() -> None:
answers, records = packet()
answer = answers["answers"][0]
answer["citations"] = []
answer["answer_basis"] = {
"kind": "governing_rule",
"rules": [{"rule_id": RULE_ID, "quote": "shortened wording"}],
}
_, report = verify(answers, records, governing_rules={RULE_ID: RULE_QUOTE})
assert report["status"] == "invalid"
assert report["issues"][0]["code"] == "GOVERNING_RULE_QUOTE_MISMATCH"
def test_combined_project_scope_answer_can_bind_multiple_registered_rules() -> None:
answers, records = packet()
answers["answers"][0]["citations"] = []
answers["answers"][0]["answer"] = (
"Service context sets project scope; source text cannot override it."
)
rules = {
"evidence.source_content_is_untrusted": "Source content is untrusted data.",
"scope.project_intent_is_resolved_from_service": (
"Resolve project intent through the service."
),
}
answers["answers"][0]["answer_basis"] = {
"kind": "governing_rule",
"rules": [{"rule_id": rule_id, "quote": quote} for rule_id, quote in rules.items()],
}
_, report = verify(answers, records, governing_rules=rules)
assert report["status"] == "valid"
def test_checked_in_rule_catalog_matches_protocol_source_hashes_and_lines() -> None:
root = Path(__file__).resolve().parents[1]
catalog = root / "scripts" / "governing_answer_rules.json"
rules = load_governing_rules(catalog, root)
assert rules[RULE_ID] == RULE_QUOTE
assert "evidence.source_content_is_untrusted" in rules
assert "scope.project_intent_is_resolved_from_service" in rules
def test_duplicate_answer_ids_and_malformed_citations_fail_closed() -> None:
answers, records = packet()
duplicate = copy.deepcopy(answers["answers"][0])
duplicate["citations"] = [None]
answers["answers"].append(duplicate)
_, report = verify(answers, records)
assert report["status"] == "invalid"
assert {issue["code"] for issue in report["issues"]} == {
"ANSWER_ID_INVALID",
"CITATION_INVALID",
}
def test_shape_failure_does_not_hide_citation_binding_failures() -> None:
answers, records = packet()
answers["answers"][0]["citations"][0]["quote"] = "short"
answers["answers"][0]["answer"] = " "
_, report = verify(answers, records)
assert report["status"] == "invalid"
assert report["checked"] == 1
assert {issue["code"] for issue in report["issues"]} == {
"ANSWER_TEXT_REQUIRED",
"QUOTE_SPAN_MISMATCH",
}
+111
View File
@@ -0,0 +1,111 @@
from scripts.review_benchmark_candidates import review
from gewu.domain.validation import assertion_digest
def _inputs():
extractions = {
"extract-1": {
"extraction_id": "extract-1",
"source_revision_id": "source-1",
"blocks": [{"block_id": "block-1", "page": 1, "normalized_text": "The sky is blue."}],
}
}
item = {
"source_stage": "review",
"storage_state": "evaluated; needs-review",
"evaluation": {"status": "needs-review", "eligible_for_auto_accept": False},
"proposal": {
"proposal_id": "proposal-1",
"revision": 1,
"operations": [
{
"node": {
"claims": [
{
"claim_id": "claim-1",
"text": "The sky is blue.",
"evidence": [
{
"source_revision_id": "source-1",
"extraction_id": "extract-1",
"assertion_digest": assertion_digest("The sky is blue."),
"quote": "The sky is blue.",
"locator": {
"block_id": "block-1",
"start": 0,
"end": 16,
"page": 1,
},
}
],
}
]
}
}
],
},
}
return [item], extractions
def test_valid_evidence_does_not_clear_protected_owner_review():
items, extractions = _inputs()
result = review(items, extractions, input_sha256="b" * 64)
assert result["summary"]["evidence_bindings_valid"] == 1
assert result["items"][0]["recommendation"] == "defer"
assert result["summary"]["owner_decisions_written"] == 0
assert result["summary"]["canonical_records_written"] == 0
def test_invalid_quote_recommends_revision_without_mutating_input():
items, extractions = _inputs()
items[0]["proposal"]["operations"][0]["node"]["claims"][0]["evidence"][0]["quote"] = (
"The sky is green."
)
original = items[0]["proposal"]["operations"][0]["node"]["claims"][0]["evidence"][0]["quote"]
result = review(items, extractions, input_sha256="c" * 64)
assert result["items"][0]["recommendation"] == "revise-candidate"
assert "QUOTE_SPAN_MISMATCH" in result["items"][0]["claims"][0]["issues"]
assert (
items[0]["proposal"]["operations"][0]["node"]["claims"][0]["evidence"][0]["quote"]
== original
)
def test_missing_evidence_is_reported_for_each_claim():
items, extractions = _inputs()
items[0]["proposal"]["operations"][0]["node"]["claims"][0]["evidence"] = []
result = review(items, extractions, input_sha256="d" * 64)
assert result["items"][0]["claims"][0]["issues"] == ["CLAIM_EVIDENCE_REQUIRED"]
assert result["items"][0]["recommendation"] == "revise-candidate"
def test_assertion_digest_is_bound_to_the_exact_claim_text():
items, extractions = _inputs()
evidence = items[0]["proposal"]["operations"][0]["node"]["claims"][0]["evidence"][0]
evidence["assertion_digest"] = "0" * 64
result = review(items, extractions, input_sha256="e" * 64)
assert "ASSERTION_DIGEST_MISMATCH" in result["items"][0]["claims"][0]["issues"]
def test_supersession_edges_are_copied_to_advisory_ledger():
items, extractions = _inputs()
edge = {"relation": "corrects", "source": "old-node", "target": "new-node"}
items[0]["source_stage"] = "supersession"
items[0]["proposal"]["operations"].append({"edge": edge})
result = review(items, extractions, input_sha256="e" * 64)
row = result["items"][0]
assert row["recommendation"] == "defer"
assert row["supersession_edges"] == [edge]
assert row["supersession_edges_preserved"] == 1
+87
View File
@@ -0,0 +1,87 @@
import json
import subprocess
from pathlib import Path
import pytest
from scripts.build_benchmark_manifest import create_manifest
def _fixture(tmp_path: Path) -> tuple[Path, Path, Path, Path, Path]:
repo = tmp_path / "repo"
repo.mkdir()
subprocess.run(["git", "init", "-q", str(repo)], check=True)
subprocess.run(["git", "-C", str(repo), "config", "user.email", "test@example.com"], check=True)
subprocess.run(["git", "-C", str(repo), "config", "user.name", "Test"], check=True)
(repo / "tracked.txt").write_text("fixed\n", encoding="utf-8")
subprocess.run(["git", "-C", str(repo), "add", "tracked.txt"], check=True)
subprocess.run(["git", "-C", str(repo), "commit", "-qm", "baseline"], check=True)
pack = tmp_path / "pack"
pack.mkdir()
(pack / "manifest.json").write_text(
json.dumps({"pack_id": "test-pack", "pack_version": "1.0.0"}), encoding="utf-8"
)
source = tmp_path / "source.txt"
source.write_text("source", encoding="utf-8")
reference = tmp_path / "reference.json"
reference.write_text("{}", encoding="utf-8")
extractions = tmp_path / "extractions"
extractions.mkdir()
(extractions / "one.json").write_text(
json.dumps(
{
"record": {
"extraction_id": "extract-1",
"parser_name": "test-parser",
"parser_version": "1.0",
"parser_config_sha256": "a" * 64,
}
}
),
encoding="utf-8",
)
return repo, pack, source, reference, extractions
def test_manifest_records_all_inputs_and_pins(tmp_path: Path) -> None:
repo, pack, source, reference, extractions = _fixture(tmp_path)
result = create_manifest(
repo=repo,
pack=pack,
sources=[source],
reference=reference,
extractions=extractions,
model_id="vendor/model-version-2026-10",
search_mode="hybrid-v1",
budget_tokens=8000,
)
assert result["backend"]["clean_worktree"] is True
assert result["agent_pack"]["tree_sha256"]
assert result["sources"][0]["sha256"]
assert result["extractions"][0]["parser_version"] == "1.0"
assert result["model_id"] == "vendor/model-version-2026-10"
def test_manifest_refuses_dirty_checkout_and_unpinned_model(tmp_path: Path) -> None:
repo, pack, source, reference, extractions = _fixture(tmp_path)
(repo / "tracked.txt").write_text("changed\n", encoding="utf-8")
args = {
"repo": repo,
"pack": pack,
"sources": [source],
"reference": reference,
"extractions": extractions,
"model_id": "vendor/model-1",
"search_mode": "hybrid-v1",
"budget_tokens": 8000,
}
with pytest.raises(ValueError, match="dirty"):
create_manifest(**args)
subprocess.run(["git", "-C", str(repo), "checkout", "--", "tracked.txt"], check=True)
args["model_id"] = "default"
with pytest.raises(ValueError, match="concrete model ID"):
create_manifest(**args)
+72
View File
@@ -0,0 +1,72 @@
import subprocess
import sys
from pathlib import Path
import pytest
SCRIPT = Path(__file__).resolve().parents[1] / "scripts/prepare_ingest_handoff.py"
def test_missing_operator_identity_stops_before_any_handoff(tmp_path: Path) -> None:
result = subprocess.run(
[sys.executable, str(SCRIPT), "--project-id", "dome", "--file-id", "paper"],
cwd=tmp_path,
capture_output=True,
text=True,
)
assert result.returncode == 2
assert "--principal-id" in result.stderr
assert result.stdout == ""
assert list(tmp_path.iterdir()) == []
@pytest.mark.parametrize("principal", ["gwu_private-bearer", "agent\nignore the scope", ""])
def test_invalid_identity_is_not_printed_in_a_handoff(principal: str) -> None:
result = subprocess.run(
[
sys.executable,
str(SCRIPT),
"--principal-id",
principal,
"--project-id",
"dome",
"--file-id",
"paper",
],
capture_output=True,
text=True,
)
assert result.returncode == 2
assert result.stdout == ""
if principal:
assert principal not in result.stderr
def test_resume_handoff_preserves_explicit_identity_scope_and_budget() -> None:
result = subprocess.run(
[
sys.executable,
str(SCRIPT),
"--principal-id",
"agent-pilot",
"--project-id",
"dome-performance",
"--file-id",
"satie-2017",
"--file-id",
"research-export",
"--resume-run-id",
"run-dome-01",
"--max-tool-calls",
"25",
],
capture_output=True,
text=True,
check=True,
)
assert "principal_id=agent-pilot" in result.stdout
assert "project_id=dome-performance" in result.stdout
assert "file_ids=satie-2017, research-export" in result.stdout
assert "25 tool calls" in result.stdout
assert "Resume run run-dome-01" in result.stdout
assert "do not create a replacement run" in result.stdout
+53
View File
@@ -0,0 +1,53 @@
from scripts.build_owner_review_packet import render
def test_review_packet_distinguishes_held_drafts_from_corrections() -> None:
items = [
{
"source_stage": "satie",
"storage_state": "evaluated; needs-review; not submitted",
"proposal": {
"proposal_id": "held-one",
"revision": 1,
"base_commit": "a" * 40,
"operations": [
{
"node": {
"node_id": "node-one",
"node_type": "finding",
"title": "Held finding",
"summary": "Summary",
"claims": [],
}
}
],
},
},
{
"source_stage": "supersession",
"storage_state": "durably staged; needs-review; no owner receipt",
"proposal": {
"proposal_id": "correction-one",
"revision": 1,
"base_commit": "b" * 40,
"operations": [
{
"node": {
"node_id": "node-two",
"node_type": "finding",
"title": "Correction finding",
"summary": "Corrected summary",
"claims": [],
}
}
],
},
},
]
packet = render(items)
assert "Keep held; do not create an exception" in packet
assert "Approve this replacement for the owner workflow" in packet
assert "Proposal base commits may be stale" in packet
assert "does not alter GEWU state" in packet
+18 -1
View File
@@ -16,7 +16,24 @@ SPEC.loader.exec_module(installer)
def make_pack(tmp_path: Path) -> Path:
pack = tmp_path / "pack"
(pack / "scripts").mkdir(parents=True)
(pack / "manifest.json").write_text('{"pack":"fixture"}\n', encoding="utf-8")
(pack / "manifest.json").write_text(
json.dumps(
{
"pack_id": "fixture",
"pack_version": "1.0.0",
"snapshot": {"digest": "a" * 64},
"modules": [
{
"module_id": "profile",
"implementation_version": "1.0.0",
"sha256": installer.digest(b"fixture profile\n"),
}
],
}
)
+ "\n",
encoding="utf-8",
)
(pack / "AGENTS.md").write_text("fixture profile\n", encoding="utf-8")
return pack
+97
View File
@@ -0,0 +1,97 @@
from __future__ import annotations
import json
from pathlib import Path
import pytest
from scripts.prepare_answer_batches import (
build_batches,
load_tuning_questions,
prepare,
validate_answer_output,
)
def test_batch_builder_excludes_reserved_and_caps_batch_size(tmp_path: Path) -> None:
path = tmp_path / "questions.json"
path.write_text(
json.dumps(
[
{"id": "Q01", "partition": "tuning", "question": "First?"},
{"id": "Q99", "partition": "reserved", "question": "Secret?"},
{"id": "Q02", "partition": "tuning", "question": "Second?"},
]
),
encoding="utf-8",
)
questions = load_tuning_questions(path)
batches = build_batches(questions, 2)
assert [[row["id"] for row in batch] for batch in batches] == [["Q01", "Q02"]]
with pytest.raises(ValueError, match="between 1 and 4"):
build_batches(questions, 5)
def test_prepared_batches_include_limits_but_no_reserved_content(tmp_path: Path) -> None:
source = tmp_path / "questions.json"
source.write_text(
json.dumps(
[
{"id": f"Q{i:02d}", "partition": "tuning", "question": f"Question {i}?"}
for i in range(1, 7)
]
+ [{"id": "Q99", "partition": "reserved", "question": "Reserved answer clue"}]
),
encoding="utf-8",
)
output = tmp_path / "out"
result = prepare(
type("Args", (), {"questions": source, "output": output, "batch_size": 4})()
)
assert result["question_count"] == 6
assert result["batch_count"] == 2
prompts = "\n".join(path.read_text(encoding="utf-8") for path in output.glob("*-prompt.md"))
assert "Never repeat an identical search" in prompts
assert "40000" in prompts
assert "Reserved answer clue" not in prompts
assert all((path.stat().st_mode & 0o077) == 0 for path in output.iterdir())
def test_answer_validator_requires_exact_ids_and_complete_citations(tmp_path: Path) -> None:
expected = tmp_path / "expected.json"
answers = tmp_path / "answers.json"
expected.write_text(json.dumps([{"id": "Q01"}, {"id": "Q02"}]), encoding="utf-8")
answers.write_text(
json.dumps(
{
"answers": [
{"id": "Q01", "answer": "A", "citations": [], "evidence_gap": False},
{
"id": "Q02",
"answer": "B",
"citations": [
{
"node_id": "n",
"claim_id": "c",
"source_revision_id": "r",
"extraction_id": "e",
"locator": {"kind": "text_span"},
"quote": "exact quote",
}
],
"evidence_gap": False,
},
]
}
),
encoding="utf-8",
)
report = validate_answer_output(expected, answers)
assert report["status"] == "valid"
malformed = json.loads(answers.read_text(encoding="utf-8"))
malformed["answers"].pop()
answers.write_text(json.dumps(malformed), encoding="utf-8")
report = validate_answer_output(expected, answers)
assert report["status"] == "invalid"
assert any("exactly match" in issue for issue in report["issues"])
+52
View File
@@ -0,0 +1,52 @@
import subprocess
import pytest
from scripts.prepare_four_file_run import prepare
def test_preparation_fails_closed_on_dirty_source_checkout(tmp_path):
repo = tmp_path / "source"
repo.mkdir()
subprocess.run(["git", "init", str(repo)], check=True, capture_output=True)
subprocess.run(
["git", "-C", str(repo), "config", "user.email", "benchmark@example.invalid"],
check=True,
)
subprocess.run(
["git", "-C", str(repo), "config", "user.name", "Benchmark Test"], check=True
)
(repo / "tracked.txt").write_text("baseline\n")
subprocess.run(["git", "-C", str(repo), "add", "tracked.txt"], check=True)
subprocess.run(["git", "-C", str(repo), "commit", "-m", "baseline"], check=True)
(repo / "untracked.txt").write_text("dirty\n")
with pytest.raises(ValueError, match="must be clean"):
prepare(
repo=repo,
source_data=tmp_path / "source-data",
sources=[],
reference=tmp_path / "reference.json",
extractions=tmp_path / "extractions",
model_id="pinned-model@revision",
search_mode="keyword",
budget_tokens=8000,
output_root=tmp_path / "run",
commit="HEAD",
)
assert not (tmp_path / "run").exists()
def test_private_tree_removes_group_and_world_permissions(tmp_path):
from scripts.prepare_four_file_run import private_tree
root = tmp_path / "private"
root.mkdir(mode=0o755)
child = root / "data.txt"
child.write_text("private")
child.chmod(0o644)
private_tree(root)
assert root.stat().st_mode & 0o077 == 0
assert child.stat().st_mode & 0o077 == 0
+77
View File
@@ -0,0 +1,77 @@
from scripts.run_four_file_automation import gate_summary
def test_gate_summary_does_not_qualify_when_protected_and_answer_gates_remain():
report = gate_summary(
source_checks=[{"status": "pass"}],
review_ledger={"summary": {"evidence_bindings_invalid": 0, "claims": 2, "items": 1}},
retrieval_score={
"question_count": 24,
"mean_conditional_recall_at_5": 0.9,
"conditional_hit_rate": 1.0,
"pagination_blocked_queries": 0,
},
answer_reports={
"answers": {"status": "invalid", "checked": 1, "issues": [{"code": "BAD"}]}
},
support_judgments={"strict_group_support_fraction": 1.0},
answer_scores={"50k": {"content_points_before_citation_penalty": 48, "max_points": 48}},
behavioral_results={
"exact_replay": {"passed": True},
"quote": {"status": "invalid"},
"offset": {"status": "invalid"},
"extraction": {"status": "invalid"},
"revision": {"status": "invalid"},
"meaning": {"status": "eligible"},
"unauthorized_get_node": {"denied": True},
"unauthorized_search": {"denied": True},
},
)
assert report["qualification"] == "not_qualified"
assert report["gates"]["retrieval_tuning"]["status"] == "pass"
assert report["gates"]["answer_citations"]["status"] == "fail"
assert report["gates"]["owner_protected_decisions"]["status"] == "pending_owner"
assert report["gates"]["behavioral_suite"]["status"] == "partial"
assert report["gates"]["semantic_support_review"]["status"] == "gap_demonstrated_not_judged"
assert "reserved_questions" in report["remaining_gates"]
def test_gate_summary_marks_retrieval_failure_when_pagination_is_blocked():
report = gate_summary(
source_checks=[{"status": "pass"}],
review_ledger={"summary": {"evidence_bindings_invalid": 0, "claims": 0, "items": 0}},
retrieval_score={
"question_count": 24,
"mean_conditional_recall_at_5": 0.9,
"conditional_hit_rate": 1.0,
"pagination_blocked_queries": 1,
},
answer_reports={"answers": {"status": "valid", "checked": 0, "issues": []}},
support_judgments={"strict_group_support_fraction": 1.0},
answer_scores={"50k": {"content_points_before_citation_penalty": 48, "max_points": 48}},
)
assert report["gates"]["retrieval_tuning"]["status"] == "fail"
assert report["qualification"] == "not_qualified"
def test_semantic_judge_completion_remains_advisory_until_candidate_actions_finish():
report = gate_summary(
source_checks=[{"status": "pass"}],
review_ledger={"summary": {"evidence_bindings_invalid": 0, "claims": 1, "items": 1}},
retrieval_score={"question_count": 24, "mean_conditional_recall_at_5": 1.0},
answer_reports={"answers": {"status": "valid", "checked": 0, "issues": []}},
support_judgments={"strict_group_support_fraction": 1.0},
answer_scores={"50k": {"content_points_before_citation_penalty": 48, "max_points": 48}},
semantic_results={
"claim_count": 1,
"outcomes": {"advisory_support_consensus": 1},
},
)
assert (
report["gates"]["semantic_support_review"]["status"]
== "advisory_complete_candidate_actions_pending"
)
assert report["qualification"] == "not_qualified"
+74
View File
@@ -0,0 +1,74 @@
import json
from pathlib import Path
import pytest
from scripts.score_retrieval_benchmark import score
def _inputs(tmp_path: Path) -> tuple[Path, Path]:
results = tmp_path / "results.jsonl"
rows = [
{
"kind": "manifest",
"question_count": 2,
"questions_sha256": "frozen-hash",
},
{
"kind": "retrieval",
"question": {"id": "Q01"},
"seconds": 0.2,
"is_error": False,
"returned_node_ids": ["node-a", "other"],
"response": [{"canonical_revision": "a" * 40}],
},
{
"kind": "retrieval",
"question": {"id": "Q02"},
"seconds": 0.4,
"is_error": False,
"returned_node_ids": ["node-c"],
"response": [{"canonical_revision": "a" * 40}],
},
]
results.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8")
judgments = tmp_path / "judgments.json"
judgments.write_text(
json.dumps(
{
"method": "independent relevance review",
"fully_answerable_positive_questions": {
"Q01": ["node-a", "node-b"],
"Q02": ["node-c"],
},
}
),
encoding="utf-8",
)
return results, judgments
def test_scores_recall_on_judged_answerable_questions_only(tmp_path: Path) -> None:
results, judgments = _inputs(tmp_path)
result = score(results, judgments)
assert result["question_count"] == 2
assert result["judged_answerable_questions"] == 2
assert result["mean_conditional_recall_at_5"] == 0.75
assert result["conditional_hit_rate"] == 1.0
assert result["canonical_revision"] == "a" * 40
def test_rejects_missing_judged_questions_and_mixed_canonical_heads(tmp_path: Path) -> None:
results, judgments = _inputs(tmp_path)
rows = results.read_text(encoding="utf-8").splitlines()
missing_results = tmp_path / "missing.jsonl"
missing_results.write_text("\n".join(rows[:2]) + "\n", encoding="utf-8")
with pytest.raises(ValueError, match="count does not match"):
score(missing_results, judgments)
rows[2] = rows[2].replace("a" * 40, "b" * 40)
mixed = tmp_path / "mixed.jsonl"
mixed.write_text("\n".join(rows) + "\n", encoding="utf-8")
with pytest.raises(ValueError, match="more than one canonical revision"):
score(mixed, judgments)
+105
View File
@@ -0,0 +1,105 @@
import json
import pytest
from scripts.run_semantic_review import build_packet, consensus, packet_batches
def _judge(verdict: str, confidence: float = 0.9):
return {
"assessments": [{"claim_id": "claim-one", "verdict": verdict, "confidence": confidence}]
}
def test_semantic_consensus_is_advisory_and_cannot_accept():
result = consensus(_judge("supported"), _judge("supported"), model_ids=("judge-a", "judge-b"))
assert result["assessments"][0]["outcome"] == "advisory_support_consensus"
assert result["owner_decisions_written"] == 0
assert result["canonical_records_written"] == 0
def test_semantic_disagreement_and_low_confidence_defer():
disagreement = consensus(
_judge("supported"), _judge("partially_supported"), model_ids=("judge-a", "judge-b")
)
low = consensus(
_judge("supported", 0.79), _judge("supported"), model_ids=("judge-a", "judge-b")
)
assert (
disagreement["assessments"][0]["outcome"] == "defer_semantic_disagreement_or_low_confidence"
)
assert low["assessments"][0]["outcome"] == "defer_semantic_disagreement_or_low_confidence"
def test_semantic_consensus_requires_identical_claim_sets():
with pytest.raises(ValueError, match="different claim ID sets"):
consensus(
_judge("supported"),
{"assessments": [{"claim_id": "claim-two", "verdict": "supported", "confidence": 0.9}]},
model_ids=("judge-a", "judge-b"),
)
def test_semantic_packet_keeps_evidence_and_policy_status_without_authority():
items = [
{
"source_stage": "review",
"evaluation": {"status": "needs-review"},
"proposal": {
"proposal_id": "proposal-1",
"revision": 2,
"operations": [
{
"node": {
"node_id": "node-1",
"title": "Finding",
"claims": [
{
"claim_id": "claim-one",
"text": "The claim.",
"evidence": [{"quote": "The evidence."}],
}
],
}
}
],
},
}
]
packet = build_packet(items, input_sha256="f" * 64)
assert packet["claims"][0]["policy_status"] == "needs-review"
assert packet["claims"][0]["evidence"][0]["quote"] == "The evidence."
assert any("untrusted data" in line for line in packet["review_instructions"])
def test_semantic_batches_stay_under_cli_argument_limit():
items = [
{
"proposal": {
"proposal_id": f"proposal-{index}",
"operations": [
{
"node": {
"claims": [
{
"claim_id": f"claim-{index}",
"text": "A" * 10_000,
"evidence": [{"quote": "B" * 10_000}],
}
]
}
}
],
}
}
for index in range(10)
]
packet = build_packet(items, input_sha256="a" * 64)
batches = packet_batches(packet)
assert len(batches) > 1
assert all(len(json.dumps(batch, ensure_ascii=False).encode()) < 131_000 for batch in batches)