Files
GEWU/tests/adapters/test_sqlite_search.py
T
vasily 78033bb830
CI / quality (push) Canceled after 0s
feat: online verification, node removals, and knowledge browser
Adds claim-bound online evidence records and citation verification scripts,
node removal transport/adapter, retrieval benchmark and owner-review tooling,
GraphCanvas/SourceDetail knowledge browser UI, and the ingest handoff helper.
Bumps the agents/gewu pack to the online-verification branch.
2026-10-16 20:02:29 +08:00

733 lines
26 KiB
Python

import hashlib
import sqlite3
import subprocess
import sys
from collections.abc import Callable
from datetime import UTC, date, datetime
from pathlib import Path
from typing import cast
import pytest
import yaml
from gewu.adapters.git_repository import GitCanonicalRepository
from gewu.adapters.sqlite_node_removals import SQLiteNodeRemovalStore
from gewu.adapters.sqlite_search import SQLiteSearchIndex
from gewu.application.identity import Principal
from gewu.application.search import CanonicalSearchService, SearchError
from gewu.domain.contracts import (
Confidence,
EdgeEndpoint,
KnowledgeNode,
NeighborRequest,
NodeType,
Scope,
SearchRequest,
TypedEdge,
)
from gewu.domain.validation import assertion_digest, evidence_digest
from tests.domain.proposal_fixtures import NODE, PROJECTS, SOURCE_REVISION
def _node_path(node: KnowledgeNode) -> str:
project_key = hashlib.sha256(node.project_id.encode()).hexdigest()
node_key = hashlib.sha256(node.node_id.encode()).hexdigest()
return f"projects/{project_key}/nodes/{node_key}.yaml"
def _publish_nodes(
repository: GitCanonicalRepository, directory: Path, nodes: tuple[KnowledgeNode, ...]
) -> str:
return repository.publish_files(
expected_head=repository.head(),
files={
_node_path(node): yaml.safe_dump(
node.model_dump(mode="json"), allow_unicode=True
).encode()
for node in nodes
},
message="Publish search fixtures",
index_directory=directory,
)
def _service(
repository: GitCanonicalRepository,
index_path: Path,
*,
head: Callable[[], str] | None = None,
) -> CanonicalSearchService:
return CanonicalSearchService(
index=SQLiteSearchIndex(index_path, repository),
project_ids_provider=lambda: PROJECTS.keys(),
canonical_head=head or repository.head,
)
def test_fts_search_enforces_explicit_project_and_shared_scope(tmp_path: Path) -> None:
repository = GitCanonicalRepository(tmp_path / "knowledge.git")
stale = NODE.model_copy(
update={
"as_of": datetime(2024, 1, 1, tzinfo=UTC),
"stale_after": datetime(2025, 1, 1, tzinfo=UTC),
}
)
video_node = NODE.model_copy(
update={
"node_id": "video:branching-workflow",
"project_id": "interactive-video",
"title": "Interactive branching workflow",
"summary": "Video authors control branching and state growth.",
}
)
shared_node = NODE.model_copy(
update={
"node_id": "shared:visitor-benchmark",
"project_id": "shared",
"title": "Shared visitor benchmark",
}
)
_publish_nodes(repository, tmp_path / "publishing", (stale, video_node, shared_node))
service = _service(repository, tmp_path / "search.sqlite3")
dome_reader = Principal(
"reader-dome", "reader-key", frozenset({"viewer"}), frozenset({"dome-performance"})
)
dome_result = service.search(
SearchRequest(
query="visitors",
scope=Scope(projects=("dome-performance",), include_shared=False),
budget_tokens=5000,
),
principal=dome_reader,
)
assert [cast(dict[str, object], row["node"])["node_id"] for row in dome_result.data] == [
NODE.node_id
]
assert any(warning.code == "STALE_NODE" for warning in dome_result.warnings)
assert dome_result.retrieval_mode == "keyword"
assert dome_result.canonical_revision == dome_result.index_revision
assert dome_result.tokens_used > 0
bounded = service.search(
SearchRequest(
query="visitors",
scope=Scope(projects=("dome-performance",), include_shared=False),
budget_tokens=1500,
),
principal=dome_reader,
)
assert bounded.data == ()
assert bounded.citations == ()
assert bounded.omissions
assert bounded.warnings
assert bounded.next_cursor is not None
recovered = service.search(
SearchRequest(
query="visitors",
scope=Scope(projects=("dome-performance",), include_shared=False),
cursor=bounded.next_cursor,
budget_tokens=5000,
),
principal=dome_reader,
)
recovered_node = cast(dict[str, object], recovered.data[0]["node"])
assert recovered_node["node_id"] == NODE.node_id
with pytest.raises(SearchError, match="too small"):
service.search(
SearchRequest(
query="visitors",
scope=Scope(projects=("dome-performance",), include_shared=False),
budget_tokens=256,
),
principal=dome_reader,
)
with pytest.raises(SearchError, match="shared knowledge"):
service.search(
SearchRequest(
query="visitors",
scope=Scope(projects=("dome-performance",)),
budget_tokens=5000,
),
principal=dome_reader,
)
with pytest.raises(SearchError, match="every requested project"):
service.search(
SearchRequest(
query='visitors" OR *',
scope=Scope(
projects=("dome-performance", "interactive-video"), include_shared=False
),
budget_tokens=5000,
),
principal=dome_reader,
)
def test_assumption_and_limitation_claims_are_searchable_and_cited(tmp_path: Path) -> None:
original_claim = NODE.claims[0]
statement = "The assumptionspecific condition affects seasonal visitor counts."
statement_digest = assertion_digest(statement)
evidence = original_claim.evidence[0].model_copy(
update={
"assertion_path": "assumptions[0].text",
"assertion_digest": statement_digest,
}
)
assessment = original_claim.assessment.model_copy(
update={
"assertion_digest": statement_digest,
"evidence_digests": (evidence_digest(evidence),),
}
)
context_claim = original_claim.model_copy(
update={
"claim_id": "dome:assumption-search-test",
"text": statement,
"evidence": (evidence,),
"assessment": assessment,
}
)
limitation_evidence = evidence.model_copy(update={"assertion_path": "limitations[0].text"})
limitation_claim = context_claim.model_copy(
update={
"claim_id": "dome:limitation-search-test",
"evidence": (limitation_evidence,),
"assessment": assessment.model_copy(
update={"evidence_digests": (evidence_digest(limitation_evidence),)}
),
}
)
contextual_node = NODE.model_copy(
update={"assumptions": (context_claim,), "limitations": (limitation_claim,)}
)
repository = GitCanonicalRepository(tmp_path / "knowledge.git")
_publish_nodes(repository, tmp_path / "publishing", (contextual_node,))
service = _service(repository, tmp_path / "search.sqlite3")
reader = Principal(
"reader-dome", "reader-key", frozenset({"viewer"}), frozenset({"dome-performance"})
)
result = service.search(
SearchRequest(
query="assumptionspecific",
scope=Scope(projects=("dome-performance",), include_shared=False),
budget_tokens=50_000,
include_evidence=True,
),
principal=reader,
)
found = cast(dict[str, object], result.data[0]["node"])
assert found["assumptions"][0]["text"] == statement
assert found["limitations"][0]["text"] == statement
assert any(item.quote == evidence.quote for item in result.citations)
def test_fts_search_indexes_entity_aliases(tmp_path: Path) -> None:
repository = GitCanonicalRepository(tmp_path / "knowledge.git")
entity = NODE.model_copy(
update={
"node_type": NodeType.ENTITY,
"node_id": "dome:planetarium-entity",
"title": "Dome installation",
"aliases": ("Planetarium",),
}
)
_publish_nodes(repository, tmp_path / "publishing", (entity,))
service = _service(repository, tmp_path / "search.sqlite3")
reader = Principal(
"reader-dome", "reader-key", frozenset({"viewer"}), frozenset({"dome-performance"})
)
result = service.search(
SearchRequest(
query="Planetarium",
scope=Scope(projects=("dome-performance",), include_shared=False),
budget_tokens=5000,
),
principal=reader,
)
assert [cast(dict[str, object], row["node"])["node_id"] for row in result.data] == [
entity.node_id
]
def test_fts_search_applies_node_type_and_freshness_facets_in_sql(tmp_path: Path) -> None:
repository = GitCanonicalRepository(tmp_path / "knowledge.git")
stale_problem = NODE.model_copy(
update={
"node_id": "dome:stale-visitor-problem",
"node_type": NodeType.PROBLEM,
"title": "Visitor count reporting problem",
"as_of": datetime(2023, 1, 1, tzinfo=UTC),
"stale_after": datetime(2024, 1, 1, tzinfo=UTC),
"value": None,
"unit": None,
}
)
fresh_finding = NODE.model_copy(
update={
"node_id": "dome:fresh-visitor-finding",
"node_type": NodeType.FINDING,
"title": "Visitor count finding",
"as_of": None,
"stale_after": None,
"value": None,
"unit": None,
"sample_size": None,
"reporting_year": None,
"tags": ("demo-only",),
"audience": ("sales",),
"use_for": ("demo",),
"domain": ("video-production",),
}
)
_publish_nodes(repository, tmp_path / "publishing", (stale_problem, fresh_finding))
index = SQLiteSearchIndex(tmp_path / "search.sqlite3", repository)
problem_hits = index.search(
query="visitor",
project_ids=("dome-performance",),
include_shared=False,
limit=10,
node_types=("problem",),
)
assert [item.node.node_id for item in problem_hits.nodes] == [stale_problem.node_id]
browse_hits = index.search(
query="",
project_ids=("dome-performance",),
include_shared=False,
limit=10,
node_types=("problem",),
)
assert [item.node.node_id for item in browse_hits.nodes] == [stale_problem.node_id]
stale_hits = index.search(
query="visitor",
project_ids=("dome-performance",),
include_shared=False,
limit=10,
stale_only=True,
)
assert [item.node.node_id for item in stale_hits.nodes] == [stale_problem.node_id]
scoped_facet_hits = index.search(
query="visitor",
project_ids=("dome-performance",),
include_shared=False,
limit=10,
tags=("attendance",),
audience=("product",),
use_for=("comparison",),
domain=("production",),
)
assert [item.node.node_id for item in scoped_facet_hits.nodes] == [stale_problem.node_id]
or_facet_hits = index.search(
query="visitor",
project_ids=("dome-performance",),
include_shared=False,
limit=10,
tags=("attendance", "demo-only"),
)
assert {item.node.node_id for item in or_facet_hits.nodes} == {
stale_problem.node_id,
fresh_finding.node_id,
}
confidence_date_hits = index.search(
query="visitor",
project_ids=("dome-performance",),
include_shared=False,
limit=10,
confidences=(Confidence.MEDIUM,),
as_of_from=date(2023, 1, 1),
as_of_to=date(2023, 12, 31),
)
assert [item.node.node_id for item in confidence_date_hits.nodes] == [stale_problem.node_id]
mismatched_confidence_hits = index.search(
query="visitor",
project_ids=("dome-performance",),
include_shared=False,
limit=10,
confidences=(Confidence.LOW,),
as_of_from=date(2023, 1, 1),
as_of_to=date(2023, 12, 31),
)
assert not mismatched_confidence_hits.nodes
def test_typed_browse_uses_scoped_non_overlapping_cursor_pages(tmp_path: Path) -> None:
repository = GitCanonicalRepository(tmp_path / "knowledge.git")
first = NODE.model_copy(
update={
"node_id": "dome:problem-a",
"node_type": NodeType.PROBLEM,
"title": "Problem A",
"value": None,
"unit": None,
}
)
second = NODE.model_copy(
update={
"node_id": "dome:problem-b",
"node_type": NodeType.PROBLEM,
"title": "Problem B",
"value": None,
"unit": None,
}
)
other_project = NODE.model_copy(
update={
"node_id": "video:problem-c",
"project_id": "interactive-video",
"node_type": NodeType.PROBLEM,
"title": "Problem C",
"value": None,
"unit": None,
}
)
_publish_nodes(repository, tmp_path / "publishing", (first, second, other_project))
service = _service(repository, tmp_path / "search.sqlite3")
reader = Principal(
"reader-dome", "reader-key", frozenset({"viewer"}), frozenset({"dome-performance"})
)
request = SearchRequest(
query="",
scope=Scope(projects=("dome-performance",), include_shared=False),
node_types=(NodeType.PROBLEM,),
limit=1,
budget_tokens=5000,
)
page_one = service.search(request, principal=reader)
page_two = service.search(
request.model_copy(update={"cursor": page_one.next_cursor}), principal=reader
)
ids = [
cast(dict[str, object], row["node"])["node_id"] for row in (*page_one.data, *page_two.data)
]
assert ids == [first.node_id, second.node_id]
assert page_one.next_cursor is not None
assert page_two.next_cursor is None
def test_search_index_rebuilds_from_git_after_index_deletion_and_new_revision(
tmp_path: Path,
) -> None:
repository = GitCanonicalRepository(tmp_path / "knowledge.git")
_publish_nodes(repository, tmp_path / "publishing", (NODE,))
index_path = tmp_path / "search.sqlite3"
index = SQLiteSearchIndex(index_path, repository)
first = index.search(
query="visitors",
project_ids=("dome-performance",),
include_shared=False,
limit=10,
)
assert [item.node.node_id for item in first.nodes] == [NODE.node_id]
video_node = NODE.model_copy(
update={
"node_id": "video:branching-workflow",
"project_id": "interactive-video",
"title": "Interactive video branches",
"summary": "Branching is costly for video authors.",
}
)
new_revision = _publish_nodes(repository, tmp_path / "publishing", (video_node,))
second = index.search(
query="branches",
project_ids=("interactive-video",),
include_shared=False,
limit=10,
)
assert second.index_revision == new_revision
assert [item.node.node_id for item in second.nodes] == [video_node.node_id]
rebuilt = SQLiteSearchIndex(tmp_path / "rebuilt-search.sqlite3", repository).search(
query="visitors",
project_ids=("dome-performance",),
include_shared=False,
limit=10,
)
assert rebuilt.index_revision == new_revision
assert [item.node.node_id for item in rebuilt.nodes] == [NODE.node_id]
def test_os_termination_inside_index_rebuild_rolls_back_before_retrying(
tmp_path: Path,
) -> None:
repository = GitCanonicalRepository(tmp_path / "knowledge.git")
nodes = tuple(
NODE.model_copy(
update={
"node_id": f"dome:rebuild-{index:02d}",
"title": f"Visitor evidence {index:02d}",
"summary": f"The installation hosted visitors in fixture record {index:02d}.",
}
)
for index in range(8)
)
first_revision = _publish_nodes(repository, tmp_path / "publishing", nodes)
index_path = tmp_path / "search.sqlite3"
index = SQLiteSearchIndex(index_path, repository)
initial = index.search(
query="visitors",
project_ids=("dome-performance",),
include_shared=False,
limit=20,
)
assert len(initial.nodes) == len(nodes)
added_nodes = tuple(
NODE.model_copy(
update={
"node_id": f"dome:rebuild-{index:02d}",
"title": f"Visitor evidence {index:02d}",
"summary": f"The installation hosted visitors in fixture record {index:02d}.",
}
)
for index in range(8, 12)
)
all_nodes = (*nodes, *added_nodes)
latest_revision = _publish_nodes(repository, tmp_path / "publishing", added_nodes)
crashed = subprocess.run(
[
sys.executable,
"-m",
"tests.adapters.crash_sqlite_rebuild_worker",
str(repository.path),
str(index_path),
],
cwd=Path(__file__).parents[2],
capture_output=True,
check=False,
timeout=60,
)
assert crashed.returncode == 74, crashed.stderr.decode("utf-8", errors="replace")
with sqlite3.connect(index_path) as connection:
assert connection.execute("SELECT count(*) FROM indexed_nodes").fetchone() == (len(nodes),)
assert connection.execute("SELECT count(*) FROM node_fts").fetchone() == (len(nodes),)
prior_revision = connection.execute("SELECT revision FROM index_state").fetchone()
assert prior_revision == (first_revision,)
recovered = SQLiteSearchIndex(index_path, repository)
result = recovered.search(
query="visitors",
project_ids=("dome-performance",),
include_shared=False,
limit=20,
)
assert {hit.node.node_id for hit in result.nodes} == {node.node_id for node in all_nodes}
assert result.index_revision == latest_revision
assert recovered.diagnostic_status()[0] == latest_revision
def test_search_discloses_canonical_revision_advancing_during_the_query(
tmp_path: Path,
) -> None:
repository = GitCanonicalRepository(tmp_path / "knowledge.git")
_publish_nodes(repository, tmp_path / "publishing", (NODE,))
video_node = NODE.model_copy(
update={
"node_id": "video:branching-workflow",
"project_id": "interactive-video",
"title": "Interactive video branching",
"summary": "Authors control video branches.",
}
)
advanced = False
def canonical_head_after_index() -> str:
nonlocal advanced
if not advanced:
advanced = True
_publish_nodes(repository, tmp_path / "publishing", (video_node,))
return repository.head()
service = _service(
repository,
tmp_path / "search.sqlite3",
head=canonical_head_after_index,
)
principal = Principal(
"reader-dome", "reader-key", frozenset({"viewer"}), frozenset({"dome-performance"})
)
response = service.search(
SearchRequest(
query="visitors",
scope=Scope(projects=("dome-performance",), include_shared=False),
budget_tokens=5000,
),
principal=principal,
)
assert response.index_revision != response.canonical_revision
assert any(item.code == "INDEX_LAG" for item in response.warnings)
def test_search_cursor_expires_when_the_index_revision_changes(tmp_path: Path) -> None:
repository = GitCanonicalRepository(tmp_path / "knowledge.git")
second = NODE.model_copy(update={"node_id": "z:season-visitors"})
_publish_nodes(repository, tmp_path / "publishing", (NODE, second))
service = _service(repository, tmp_path / "search.sqlite3")
reader = Principal("reader", "key", frozenset({"viewer"}), frozenset({NODE.project_id}))
request = SearchRequest(
query="visitors",
scope=Scope(projects=(NODE.project_id,), include_shared=False),
limit=1,
budget_tokens=5000,
)
first_page = service.search(request, principal=reader)
assert first_page.next_cursor is not None
third = NODE.model_copy(update={"node_id": "zz:season-visitors"})
_publish_nodes(repository, tmp_path / "publishing", (third,))
with pytest.raises(SearchError, match="cursor expired") as error:
service.search(
request.model_copy(update={"cursor": first_page.next_cursor}), principal=reader
)
assert error.value.status_code == 409
def test_neighbors_are_rebuilt_and_hide_cross_project_nodes(tmp_path: Path) -> None:
repository = GitCanonicalRepository(tmp_path / "knowledge.git")
other = NODE.model_copy(update={"node_id": "video:other", "project_id": "interactive-video"})
edge = TypedEdge(
edge_id="edge:cross-project",
edge_type="compared_with",
source=EdgeEndpoint(kind="node", id=NODE.node_id),
target=EdgeEndpoint(kind="node", id=other.node_id),
)
files = {
_node_path(node): yaml.safe_dump(node.model_dump(mode="json")).encode()
for node in (NODE, other)
}
files[f"edges/{hashlib.sha256(edge.edge_id.encode()).hexdigest()}.yaml"] = yaml.safe_dump(
edge.model_dump(mode="json")
).encode()
repository.publish_files(
expected_head=repository.head(),
files=files,
message="Publish typed relationship",
index_directory=tmp_path / "publishing",
)
index = SQLiteSearchIndex(tmp_path / "search.sqlite3", repository)
hits = index.neighbors(
node_id=NODE.node_id,
edge_types=(),
project_ids=(NODE.project_id, other.project_id),
include_shared=False,
limit=10,
)
assert hits.origin is not None and hits.origin.node_id == NODE.node_id
assert len(hits.edges) == 1
assert hits.edges[0].neighbor_node is not None
service = _service(repository, tmp_path / "search2.sqlite3")
reader = Principal("reader", "key", frozenset({"viewer"}), frozenset({NODE.project_id}))
response = service.neighbors(
NeighborRequest(
node_id=NODE.node_id,
scope=Scope(projects=(NODE.project_id,), include_shared=False),
budget_tokens=5000,
),
principal=reader,
)
assert response.data == ()
assert response.citations == ()
def test_neighbor_source_metadata_requires_an_ingested_for_scope(tmp_path: Path) -> None:
repository = GitCanonicalRepository(tmp_path / "knowledge.git")
edge = TypedEdge(
edge_id="edge:zzz-visible-source",
edge_type="derived_from",
source=EdgeEndpoint(kind="node", id=NODE.node_id),
target=EdgeEndpoint(kind="source", id=SOURCE_REVISION.revision_id),
)
private_source = SOURCE_REVISION.model_copy(
update={
"source_id": "video:private",
"revision_id": "video:private:r1",
"ingested_for": ("interactive-video",),
}
)
private_edge = TypedEdge(
edge_id="edge:aaa-private-source",
edge_type="derived_from",
source=EdgeEndpoint(kind="node", id=NODE.node_id),
target=EdgeEndpoint(kind="source", id=private_source.revision_id),
)
source_key = hashlib.sha256(SOURCE_REVISION.source_id.encode()).hexdigest()
revision_key = hashlib.sha256(SOURCE_REVISION.revision_id.encode()).hexdigest()
private_source_key = hashlib.sha256(private_source.source_id.encode()).hexdigest()
private_revision_key = hashlib.sha256(private_source.revision_id.encode()).hexdigest()
files = {
_node_path(NODE): yaml.safe_dump(NODE.model_dump(mode="json")).encode(),
f"sources/{source_key}/{revision_key}.yaml": yaml.safe_dump(
SOURCE_REVISION.model_dump(mode="json")
).encode(),
f"sources/{private_source_key}/{private_revision_key}.yaml": yaml.safe_dump(
private_source.model_dump(mode="json")
).encode(),
f"edges/{hashlib.sha256(edge.edge_id.encode()).hexdigest()}.yaml": yaml.safe_dump(
edge.model_dump(mode="json")
).encode(),
f"edges/{hashlib.sha256(private_edge.edge_id.encode()).hexdigest()}.yaml": yaml.safe_dump(
private_edge.model_dump(mode="json")
).encode(),
}
repository.publish_files(
expected_head=repository.head(),
files=files,
message="Publish source relationship",
index_directory=tmp_path / "publishing",
)
service = _service(repository, tmp_path / "search.sqlite3")
reader = Principal("reader", "key", frozenset({"viewer"}), frozenset({NODE.project_id}))
response = service.neighbors(
NeighborRequest(
node_id=NODE.node_id,
scope=Scope(projects=(NODE.project_id,), include_shared=False),
limit=1,
budget_tokens=5000,
),
principal=reader,
)
assert len(response.data) == 1
assert response.next_cursor is None
neighbor = cast(dict[str, object], response.data[0]["neighbor"])
assert neighbor["kind"] == "source"
assert cast(dict[str, object], neighbor["record"])["revision_id"] == SOURCE_REVISION.revision_id
def test_durable_node_removal_hides_from_search_and_returns_after_restore(tmp_path: Path) -> None:
repository = GitCanonicalRepository(tmp_path / "knowledge.git")
_publish_nodes(repository, tmp_path / "publishing", (NODE,))
removals = SQLiteNodeRemovalStore(tmp_path / "node-removals.sqlite3")
index = SQLiteSearchIndex(
tmp_path / "search.sqlite3", repository, node_removals_path=removals.path
)
service = CanonicalSearchService(
index=index,
project_ids_provider=lambda: PROJECTS.keys(),
canonical_head=repository.head,
)
principal = Principal(
"editor", "editor-key", frozenset({"editor"}), frozenset({NODE.project_id})
)
request = SearchRequest(
query=NODE.title,
scope=Scope(projects=(NODE.project_id,), include_shared=False),
budget_tokens=5000,
)
assert service.search(request, principal=principal).data
removals.request(NODE.project_id, NODE.node_id, NODE.title, "editor")
assert service.search(request, principal=principal).data == ()
removals.restore(NODE.project_id, NODE.node_id, "editor")
assert service.search(request, principal=principal).data