GEWU/tests/adapters/test_sqlite_search.py

import hashlib
import sqlite3
import subprocess
import sys
from collections.abc import Callable
from datetime import UTC, date, datetime
from pathlib import Path
from typing import cast

import pytest
import yaml

from gewu.adapters.git_repository import GitCanonicalRepository
from gewu.adapters.sqlite_search import SQLiteSearchIndex
from gewu.application.identity import Principal
from gewu.application.search import CanonicalSearchService, SearchError
from gewu.domain.contracts import (
    Confidence,
    EdgeEndpoint,
    KnowledgeNode,
    NeighborRequest,
    NodeType,
    Scope,
    SearchRequest,
    TypedEdge,
)
from gewu.domain.validation import assertion_digest, evidence_digest
from tests.domain.proposal_fixtures import NODE, PROJECTS, SOURCE_REVISION


def _node_path(node: KnowledgeNode) -> str:
    project_key = hashlib.sha256(node.project_id.encode()).hexdigest()
    node_key = hashlib.sha256(node.node_id.encode()).hexdigest()
    return f"projects/{project_key}/nodes/{node_key}.yaml"


def _publish_nodes(
    repository: GitCanonicalRepository, directory: Path, nodes: tuple[KnowledgeNode, ...]
) -> str:
    return repository.publish_files(
        expected_head=repository.head(),
        files={
            _node_path(node): yaml.safe_dump(
                node.model_dump(mode="json"), allow_unicode=True
            ).encode()
            for node in nodes
        },
        message="Publish search fixtures",
        index_directory=directory,
    )


def _service(
    repository: GitCanonicalRepository,
    index_path: Path,
    *,
    head: Callable[[], str] | None = None,
) -> CanonicalSearchService:
    return CanonicalSearchService(
        index=SQLiteSearchIndex(index_path, repository),
        project_ids_provider=lambda: PROJECTS.keys(),
        canonical_head=head or repository.head,
    )


def test_fts_search_enforces_explicit_project_and_shared_scope(tmp_path: Path) -> None:
    repository = GitCanonicalRepository(tmp_path / "knowledge.git")
    stale = NODE.model_copy(
        update={
            "as_of": datetime(2024, 1, 1, tzinfo=UTC),
            "stale_after": datetime(2025, 1, 1, tzinfo=UTC),
        }
    )
    video_node = NODE.model_copy(
        update={
            "node_id": "video:branching-workflow",
            "project_id": "interactive-video",
            "title": "Interactive branching workflow",
            "summary": "Video authors control branching and state growth.",
        }
    )
    shared_node = NODE.model_copy(
        update={
            "node_id": "shared:visitor-benchmark",
            "project_id": "shared",
            "title": "Shared visitor benchmark",
        }
    )
    _publish_nodes(repository, tmp_path / "publishing", (stale, video_node, shared_node))
    service = _service(repository, tmp_path / "search.sqlite3")
    dome_reader = Principal(
        "reader-dome", "reader-key", frozenset({"viewer"}), frozenset({"dome-performance"})
    )
    dome_result = service.search(
        SearchRequest(
            query="visitors",
            scope=Scope(projects=("dome-performance",), include_shared=False),
            budget_tokens=5000,
        ),
        principal=dome_reader,
    )
    assert [cast(dict[str, object], row["node"])["node_id"] for row in dome_result.data] == [
        NODE.node_id
    ]
    assert any(warning.code == "STALE_NODE" for warning in dome_result.warnings)
    assert dome_result.retrieval_mode == "keyword"
    assert dome_result.canonical_revision == dome_result.index_revision
    assert dome_result.tokens_used > 0
    bounded = service.search(
        SearchRequest(
            query="visitors",
            scope=Scope(projects=("dome-performance",), include_shared=False),
            budget_tokens=1500,
        ),
        principal=dome_reader,
    )
    assert bounded.data == ()
    assert bounded.citations == ()
    assert bounded.omissions
    assert bounded.warnings
    assert bounded.next_cursor is not None
    recovered = service.search(
        SearchRequest(
            query="visitors",
            scope=Scope(projects=("dome-performance",), include_shared=False),
            cursor=bounded.next_cursor,
            budget_tokens=5000,
        ),
        principal=dome_reader,
    )
    recovered_node = cast(dict[str, object], recovered.data[0]["node"])
    assert recovered_node["node_id"] == NODE.node_id
    with pytest.raises(SearchError, match="too small"):
        service.search(
            SearchRequest(
                query="visitors",
                scope=Scope(projects=("dome-performance",), include_shared=False),
                budget_tokens=256,
            ),
            principal=dome_reader,
        )
    with pytest.raises(SearchError, match="shared knowledge"):
        service.search(
            SearchRequest(
                query="visitors",
                scope=Scope(projects=("dome-performance",)),
                budget_tokens=5000,
            ),
            principal=dome_reader,
        )
    with pytest.raises(SearchError, match="every requested project"):
        service.search(
            SearchRequest(
                query='visitors" OR *',
                scope=Scope(
                    projects=("dome-performance", "interactive-video"), include_shared=False
                ),
                budget_tokens=5000,
            ),
            principal=dome_reader,
        )


def test_assumption_and_limitation_claims_are_searchable_and_cited(tmp_path: Path) -> None:
    original_claim = NODE.claims[0]
    statement = "The assumptionspecific condition affects seasonal visitor counts."
    statement_digest = assertion_digest(statement)
    evidence = original_claim.evidence[0].model_copy(
        update={
            "assertion_path": "assumptions[0].text",
            "assertion_digest": statement_digest,
        }
    )
    assessment = original_claim.assessment.model_copy(
        update={
            "assertion_digest": statement_digest,
            "evidence_digests": (evidence_digest(evidence),),
        }
    )
    context_claim = original_claim.model_copy(
        update={
            "claim_id": "dome:assumption-search-test",
            "text": statement,
            "evidence": (evidence,),
            "assessment": assessment,
        }
    )
    limitation_evidence = evidence.model_copy(update={"assertion_path": "limitations[0].text"})
    limitation_claim = context_claim.model_copy(
        update={
            "claim_id": "dome:limitation-search-test",
            "evidence": (limitation_evidence,),
            "assessment": assessment.model_copy(
                update={"evidence_digests": (evidence_digest(limitation_evidence),)}
            ),
        }
    )
    contextual_node = NODE.model_copy(
        update={"assumptions": (context_claim,), "limitations": (limitation_claim,)}
    )
    repository = GitCanonicalRepository(tmp_path / "knowledge.git")
    _publish_nodes(repository, tmp_path / "publishing", (contextual_node,))
    service = _service(repository, tmp_path / "search.sqlite3")
    reader = Principal(
        "reader-dome", "reader-key", frozenset({"viewer"}), frozenset({"dome-performance"})
    )

    result = service.search(
        SearchRequest(
            query="assumptionspecific",
            scope=Scope(projects=("dome-performance",), include_shared=False),
            budget_tokens=50_000,
        ),
        principal=reader,
    )
    found = cast(dict[str, object], result.data[0]["node"])
    assert found["assumptions"][0]["text"] == statement
    assert found["limitations"][0]["text"] == statement
    assert any(item.quote == evidence.quote for item in result.citations)


def test_fts_search_indexes_entity_aliases(tmp_path: Path) -> None:
    repository = GitCanonicalRepository(tmp_path / "knowledge.git")
    entity = NODE.model_copy(
        update={
            "node_type": NodeType.ENTITY,
            "node_id": "dome:planetarium-entity",
            "title": "Dome installation",
            "aliases": ("Planetarium",),
        }
    )
    _publish_nodes(repository, tmp_path / "publishing", (entity,))
    service = _service(repository, tmp_path / "search.sqlite3")
    reader = Principal(
        "reader-dome", "reader-key", frozenset({"viewer"}), frozenset({"dome-performance"})
    )
    result = service.search(
        SearchRequest(
            query="Planetarium",
            scope=Scope(projects=("dome-performance",), include_shared=False),
            budget_tokens=5000,
        ),
        principal=reader,
    )
    assert [cast(dict[str, object], row["node"])["node_id"] for row in result.data] == [
        entity.node_id
    ]


def test_fts_search_applies_node_type_and_freshness_facets_in_sql(tmp_path: Path) -> None:
    repository = GitCanonicalRepository(tmp_path / "knowledge.git")
    stale_problem = NODE.model_copy(
        update={
            "node_id": "dome:stale-visitor-problem",
            "node_type": NodeType.PROBLEM,
            "title": "Visitor count reporting problem",
            "as_of": datetime(2023, 1, 1, tzinfo=UTC),
            "stale_after": datetime(2024, 1, 1, tzinfo=UTC),
            "value": None,
            "unit": None,
        }
    )
    fresh_finding = NODE.model_copy(
        update={
            "node_id": "dome:fresh-visitor-finding",
            "node_type": NodeType.FINDING,
            "title": "Visitor count finding",
            "as_of": None,
            "stale_after": None,
            "value": None,
            "unit": None,
            "sample_size": None,
            "reporting_year": None,
            "tags": ("demo-only",),
            "audience": ("sales",),
            "use_for": ("demo",),
            "domain": ("video-production",),
        }
    )
    _publish_nodes(repository, tmp_path / "publishing", (stale_problem, fresh_finding))
    index = SQLiteSearchIndex(tmp_path / "search.sqlite3", repository)

    problem_hits = index.search(
        query="visitor",
        project_ids=("dome-performance",),
        include_shared=False,
        limit=10,
        node_types=("problem",),
    )
    assert [item.node.node_id for item in problem_hits.nodes] == [stale_problem.node_id]

    browse_hits = index.search(
        query="",
        project_ids=("dome-performance",),
        include_shared=False,
        limit=10,
        node_types=("problem",),
    )
    assert [item.node.node_id for item in browse_hits.nodes] == [stale_problem.node_id]

    stale_hits = index.search(
        query="visitor",
        project_ids=("dome-performance",),
        include_shared=False,
        limit=10,
        stale_only=True,
    )
    assert [item.node.node_id for item in stale_hits.nodes] == [stale_problem.node_id]

    scoped_facet_hits = index.search(
        query="visitor",
        project_ids=("dome-performance",),
        include_shared=False,
        limit=10,
        tags=("attendance",),
        audience=("product",),
        use_for=("comparison",),
        domain=("production",),
    )
    assert [item.node.node_id for item in scoped_facet_hits.nodes] == [stale_problem.node_id]

    or_facet_hits = index.search(
        query="visitor",
        project_ids=("dome-performance",),
        include_shared=False,
        limit=10,
        tags=("attendance", "demo-only"),
    )
    assert {item.node.node_id for item in or_facet_hits.nodes} == {
        stale_problem.node_id,
        fresh_finding.node_id,
    }

    confidence_date_hits = index.search(
        query="visitor",
        project_ids=("dome-performance",),
        include_shared=False,
        limit=10,
        confidences=(Confidence.MEDIUM,),
        as_of_from=date(2023, 1, 1),
        as_of_to=date(2023, 12, 31),
    )
    assert [item.node.node_id for item in confidence_date_hits.nodes] == [stale_problem.node_id]

    mismatched_confidence_hits = index.search(
        query="visitor",
        project_ids=("dome-performance",),
        include_shared=False,
        limit=10,
        confidences=(Confidence.LOW,),
        as_of_from=date(2023, 1, 1),
        as_of_to=date(2023, 12, 31),
    )
    assert not mismatched_confidence_hits.nodes


def test_typed_browse_uses_scoped_non_overlapping_cursor_pages(tmp_path: Path) -> None:
    repository = GitCanonicalRepository(tmp_path / "knowledge.git")
    first = NODE.model_copy(
        update={
            "node_id": "dome:problem-a",
            "node_type": NodeType.PROBLEM,
            "title": "Problem A",
            "value": None,
            "unit": None,
        }
    )
    second = NODE.model_copy(
        update={
            "node_id": "dome:problem-b",
            "node_type": NodeType.PROBLEM,
            "title": "Problem B",
            "value": None,
            "unit": None,
        }
    )
    other_project = NODE.model_copy(
        update={
            "node_id": "video:problem-c",
            "project_id": "interactive-video",
            "node_type": NodeType.PROBLEM,
            "title": "Problem C",
            "value": None,
            "unit": None,
        }
    )
    _publish_nodes(repository, tmp_path / "publishing", (first, second, other_project))
    service = _service(repository, tmp_path / "search.sqlite3")
    reader = Principal(
        "reader-dome", "reader-key", frozenset({"viewer"}), frozenset({"dome-performance"})
    )
    request = SearchRequest(
        query="",
        scope=Scope(projects=("dome-performance",), include_shared=False),
        node_types=(NodeType.PROBLEM,),
        limit=1,
        budget_tokens=5000,
    )

    page_one = service.search(request, principal=reader)
    page_two = service.search(
        request.model_copy(update={"cursor": page_one.next_cursor}), principal=reader
    )

    ids = [
        cast(dict[str, object], row["node"])["node_id"] for row in (*page_one.data, *page_two.data)
    ]
    assert ids == [first.node_id, second.node_id]
    assert page_one.next_cursor is not None
    assert page_two.next_cursor is None


def test_search_index_rebuilds_from_git_after_index_deletion_and_new_revision(
    tmp_path: Path,
) -> None:
    repository = GitCanonicalRepository(tmp_path / "knowledge.git")
    _publish_nodes(repository, tmp_path / "publishing", (NODE,))
    index_path = tmp_path / "search.sqlite3"
    index = SQLiteSearchIndex(index_path, repository)
    first = index.search(
        query="visitors",
        project_ids=("dome-performance",),
        include_shared=False,
        limit=10,
    )
    assert [item.node.node_id for item in first.nodes] == [NODE.node_id]

    video_node = NODE.model_copy(
        update={
            "node_id": "video:branching-workflow",
            "project_id": "interactive-video",
            "title": "Interactive video branches",
            "summary": "Branching is costly for video authors.",
        }
    )
    new_revision = _publish_nodes(repository, tmp_path / "publishing", (video_node,))
    second = index.search(
        query="branches",
        project_ids=("interactive-video",),
        include_shared=False,
        limit=10,
    )
    assert second.index_revision == new_revision
    assert [item.node.node_id for item in second.nodes] == [video_node.node_id]

    rebuilt = SQLiteSearchIndex(tmp_path / "rebuilt-search.sqlite3", repository).search(
        query="visitors",
        project_ids=("dome-performance",),
        include_shared=False,
        limit=10,
    )
    assert rebuilt.index_revision == new_revision
    assert [item.node.node_id for item in rebuilt.nodes] == [NODE.node_id]


def test_os_termination_inside_index_rebuild_rolls_back_before_retrying(
    tmp_path: Path,
) -> None:
    repository = GitCanonicalRepository(tmp_path / "knowledge.git")
    nodes = tuple(
        NODE.model_copy(
            update={
                "node_id": f"dome:rebuild-{index:02d}",
                "title": f"Visitor evidence {index:02d}",
                "summary": f"The installation hosted visitors in fixture record {index:02d}.",
            }
        )
        for index in range(8)
    )
    first_revision = _publish_nodes(repository, tmp_path / "publishing", nodes)
    index_path = tmp_path / "search.sqlite3"
    index = SQLiteSearchIndex(index_path, repository)
    initial = index.search(
        query="visitors",
        project_ids=("dome-performance",),
        include_shared=False,
        limit=20,
    )
    assert len(initial.nodes) == len(nodes)

    added_nodes = tuple(
        NODE.model_copy(
            update={
                "node_id": f"dome:rebuild-{index:02d}",
                "title": f"Visitor evidence {index:02d}",
                "summary": f"The installation hosted visitors in fixture record {index:02d}.",
            }
        )
        for index in range(8, 12)
    )
    all_nodes = (*nodes, *added_nodes)
    latest_revision = _publish_nodes(repository, tmp_path / "publishing", added_nodes)

    crashed = subprocess.run(
        [
            sys.executable,
            "-m",
            "tests.adapters.crash_sqlite_rebuild_worker",
            str(repository.path),
            str(index_path),
        ],
        cwd=Path(__file__).parents[2],
        capture_output=True,
        check=False,
        timeout=60,
    )
    assert crashed.returncode == 74, crashed.stderr.decode("utf-8", errors="replace")

    with sqlite3.connect(index_path) as connection:
        assert connection.execute("SELECT count(*) FROM indexed_nodes").fetchone() == (len(nodes),)
        assert connection.execute("SELECT count(*) FROM node_fts").fetchone() == (len(nodes),)
        prior_revision = connection.execute("SELECT revision FROM index_state").fetchone()
        assert prior_revision == (first_revision,)

    recovered = SQLiteSearchIndex(index_path, repository)
    result = recovered.search(
        query="visitors",
        project_ids=("dome-performance",),
        include_shared=False,
        limit=20,
    )
    assert {hit.node.node_id for hit in result.nodes} == {node.node_id for node in all_nodes}
    assert result.index_revision == latest_revision
    assert recovered.diagnostic_status()[0] == latest_revision


def test_search_discloses_canonical_revision_advancing_during_the_query(
    tmp_path: Path,
) -> None:
    repository = GitCanonicalRepository(tmp_path / "knowledge.git")
    _publish_nodes(repository, tmp_path / "publishing", (NODE,))
    video_node = NODE.model_copy(
        update={
            "node_id": "video:branching-workflow",
            "project_id": "interactive-video",
            "title": "Interactive video branching",
            "summary": "Authors control video branches.",
        }
    )
    advanced = False

    def canonical_head_after_index() -> str:
        nonlocal advanced
        if not advanced:
            advanced = True
            _publish_nodes(repository, tmp_path / "publishing", (video_node,))
        return repository.head()

    service = _service(
        repository,
        tmp_path / "search.sqlite3",
        head=canonical_head_after_index,
    )
    principal = Principal(
        "reader-dome", "reader-key", frozenset({"viewer"}), frozenset({"dome-performance"})
    )
    response = service.search(
        SearchRequest(
            query="visitors",
            scope=Scope(projects=("dome-performance",), include_shared=False),
            budget_tokens=5000,
        ),
        principal=principal,
    )
    assert response.index_revision != response.canonical_revision
    assert any(item.code == "INDEX_LAG" for item in response.warnings)


def test_search_cursor_expires_when_the_index_revision_changes(tmp_path: Path) -> None:
    repository = GitCanonicalRepository(tmp_path / "knowledge.git")
    second = NODE.model_copy(update={"node_id": "z:season-visitors"})
    _publish_nodes(repository, tmp_path / "publishing", (NODE, second))
    service = _service(repository, tmp_path / "search.sqlite3")
    reader = Principal("reader", "key", frozenset({"viewer"}), frozenset({NODE.project_id}))
    request = SearchRequest(
        query="visitors",
        scope=Scope(projects=(NODE.project_id,), include_shared=False),
        limit=1,
        budget_tokens=5000,
    )
    first_page = service.search(request, principal=reader)
    assert first_page.next_cursor is not None

    third = NODE.model_copy(update={"node_id": "zz:season-visitors"})
    _publish_nodes(repository, tmp_path / "publishing", (third,))
    with pytest.raises(SearchError, match="cursor expired") as error:
        service.search(
            request.model_copy(update={"cursor": first_page.next_cursor}), principal=reader
        )
    assert error.value.status_code == 409


def test_neighbors_are_rebuilt_and_hide_cross_project_nodes(tmp_path: Path) -> None:
    repository = GitCanonicalRepository(tmp_path / "knowledge.git")
    other = NODE.model_copy(update={"node_id": "video:other", "project_id": "interactive-video"})
    edge = TypedEdge(
        edge_id="edge:cross-project",
        edge_type="compared_with",
        source=EdgeEndpoint(kind="node", id=NODE.node_id),
        target=EdgeEndpoint(kind="node", id=other.node_id),
    )
    files = {
        _node_path(node): yaml.safe_dump(node.model_dump(mode="json")).encode()
        for node in (NODE, other)
    }
    files[f"edges/{hashlib.sha256(edge.edge_id.encode()).hexdigest()}.yaml"] = yaml.safe_dump(
        edge.model_dump(mode="json")
    ).encode()
    repository.publish_files(
        expected_head=repository.head(),
        files=files,
        message="Publish typed relationship",
        index_directory=tmp_path / "publishing",
    )
    index = SQLiteSearchIndex(tmp_path / "search.sqlite3", repository)
    hits = index.neighbors(
        node_id=NODE.node_id,
        edge_types=(),
        project_ids=(NODE.project_id, other.project_id),
        include_shared=False,
        limit=10,
    )
    assert hits.origin is not None and hits.origin.node_id == NODE.node_id
    assert len(hits.edges) == 1
    assert hits.edges[0].neighbor_node is not None

    service = _service(repository, tmp_path / "search2.sqlite3")
    reader = Principal("reader", "key", frozenset({"viewer"}), frozenset({NODE.project_id}))
    response = service.neighbors(
        NeighborRequest(
            node_id=NODE.node_id,
            scope=Scope(projects=(NODE.project_id,), include_shared=False),
            budget_tokens=5000,
        ),
        principal=reader,
    )
    assert response.data == ()
    assert response.citations == ()


def test_neighbor_source_metadata_requires_an_ingested_for_scope(tmp_path: Path) -> None:
    repository = GitCanonicalRepository(tmp_path / "knowledge.git")
    edge = TypedEdge(
        edge_id="edge:zzz-visible-source",
        edge_type="derived_from",
        source=EdgeEndpoint(kind="node", id=NODE.node_id),
        target=EdgeEndpoint(kind="source", id=SOURCE_REVISION.revision_id),
    )
    private_source = SOURCE_REVISION.model_copy(
        update={
            "source_id": "video:private",
            "revision_id": "video:private:r1",
            "ingested_for": ("interactive-video",),
        }
    )
    private_edge = TypedEdge(
        edge_id="edge:aaa-private-source",
        edge_type="derived_from",
        source=EdgeEndpoint(kind="node", id=NODE.node_id),
        target=EdgeEndpoint(kind="source", id=private_source.revision_id),
    )
    source_key = hashlib.sha256(SOURCE_REVISION.source_id.encode()).hexdigest()
    revision_key = hashlib.sha256(SOURCE_REVISION.revision_id.encode()).hexdigest()
    private_source_key = hashlib.sha256(private_source.source_id.encode()).hexdigest()
    private_revision_key = hashlib.sha256(private_source.revision_id.encode()).hexdigest()
    files = {
        _node_path(NODE): yaml.safe_dump(NODE.model_dump(mode="json")).encode(),
        f"sources/{source_key}/{revision_key}.yaml": yaml.safe_dump(
            SOURCE_REVISION.model_dump(mode="json")
        ).encode(),
        f"sources/{private_source_key}/{private_revision_key}.yaml": yaml.safe_dump(
            private_source.model_dump(mode="json")
        ).encode(),
        f"edges/{hashlib.sha256(edge.edge_id.encode()).hexdigest()}.yaml": yaml.safe_dump(
            edge.model_dump(mode="json")
        ).encode(),
        f"edges/{hashlib.sha256(private_edge.edge_id.encode()).hexdigest()}.yaml": yaml.safe_dump(
            private_edge.model_dump(mode="json")
        ).encode(),
    }
    repository.publish_files(
        expected_head=repository.head(),
        files=files,
        message="Publish source relationship",
        index_directory=tmp_path / "publishing",
    )
    service = _service(repository, tmp_path / "search.sqlite3")
    reader = Principal("reader", "key", frozenset({"viewer"}), frozenset({NODE.project_id}))
    response = service.neighbors(
        NeighborRequest(
            node_id=NODE.node_id,
            scope=Scope(projects=(NODE.project_id,), include_shared=False),
            limit=1,
            budget_tokens=5000,
        ),
        principal=reader,
    )
    assert len(response.data) == 1
    assert response.next_cursor is None
    neighbor = cast(dict[str, object], response.data[0]["neighbor"])
    assert neighbor["kind"] == "source"
    assert cast(dict[str, object], neighbor["record"])["revision_id"] == SOURCE_REVISION.revision_id