feat: online verification, node removals, and knowledge browser

Adds claim-bound online evidence records and citation verification scripts,
node removal transport/adapter, retrieval benchmark and owner-review tooling,
GraphCanvas/SourceDetail knowledge browser UI, and the ingest handoff helper.
Bumps the agents/gewu pack to the online-verification branch.

src/gewu/application/reading.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 import hashlib
 import json
 import secrets
+from collections.abc import Callable
 from dataclasses import dataclass
 
 import yaml
@@ -39,6 +40,7 @@ _GIT_COMMIT_ID = TypeAdapter(GitCommitId)
 @dataclass(frozen=True, slots=True)
 class CanonicalNodeReader:
     repository: CanonicalRepository
+    node_is_hidden: Callable[[str, str], bool] | None = None
 
     def get_node(
         self,
@@ -50,6 +52,8 @@ class CanonicalNodeReader:
     ) -> ReadEnvelope:
         if not principal.can_read_project(project_id):
             raise ReadError(403, "Credential cannot read this project")
+        if self.node_is_hidden is not None and self.node_is_hidden(project_id, node_id):
+            raise ReadError(404, "Knowledge node was not found")
         head = self.repository.head()
         path = f"projects/{_key(project_id)}/nodes/{_key(node_id)}.yaml"
         if revision is None:
@@ -98,6 +102,8 @@ class CanonicalNodeReader:
             raise ReadError(422, "History limit must be between one and twenty")
         if not principal.can_read_project(project_id):
             raise ReadError(403, "Credential cannot read this project")
+        if self.node_is_hidden is not None and self.node_is_hidden(project_id, node_id):
+            raise ReadError(404, "Knowledge node was not found")
         path = f"projects/{_key(project_id)}/nodes/{_key(node_id)}.yaml"
         head = self.repository.head()
         rows: list[dict[str, object]] = []

src/gewu/application/registry.py

@@ -30,6 +30,9 @@ from gewu.domain.validation import extraction_digest
 
 MAX_CONTEXT_RECORDS = 20_000
 MAX_CONTEXT_RECORD_BYTES = 1_048_576
+# Character-level normalization maps are substantially larger than source text.
+# Keep ordinary registry/node limits small while allowing bounded extraction records.
+MAX_CONTEXT_EXTRACTION_BYTES = 16 * 1024 * 1024
 MAX_PROJECT_REGISTRY_RESPONSE_BYTES = 1_048_576
 
 
@@ -162,7 +165,7 @@ class LocalContextProvider:
                     continue
                 record_count += 1
                 extraction_bytes = self.repository.read_file(revision, record_path)
-                if extraction_bytes is None or len(extraction_bytes) > MAX_CONTEXT_RECORD_BYTES:
+                if extraction_bytes is None or len(extraction_bytes) > MAX_CONTEXT_EXTRACTION_BYTES:
                     raise RegistryError("Canonical extraction is unavailable or too large")
                 extraction = ExtractionRecord.model_validate_json(extraction_bytes, strict=True)
                 if extraction_digest(extraction) != extraction.extraction_id:

src/gewu/application/search.py

@@ -11,7 +11,7 @@ from collections import defaultdict
 from collections.abc import Callable, Collection, Iterator
 from dataclasses import dataclass
 from datetime import UTC, date, datetime
-from typing import Literal, Protocol
+from typing import Any, Literal, Protocol
 
 from gewu.application.identity import Principal
 from gewu.application.ports import EmbeddingProvider, VectorGeneration, VectorIndex
@@ -22,6 +22,12 @@ from gewu.domain.contracts import (
     EvidenceReference,
     Finding,
     KnowledgeNode,
+    KnowledgeBrowseRequest,
+    KnowledgeBrowseEnvelope,
+    KnowledgeGraphEnvelope,
+    KnowledgeGraphRequest,
+    SourceRelatedNode,
+    SourceRelatedNodesEnvelope,
     NeighborRequest,
     ReadEnvelope,
     Scope,
@@ -58,6 +64,31 @@ class SearchIndexResult:
     warnings: tuple[Finding, ...] = ()
 
 
+@dataclass(frozen=True, slots=True)
+class BrowseIndexResult:
+    index_revision: str
+    nodes: tuple[KnowledgeNode, ...]
+    filtered_node_count: int
+    project_node_count: int
+    shared_node_count: int
+    facets: dict[str, dict[str, int]]
+    has_more: bool
+
+
+@dataclass(frozen=True, slots=True)
+class SourceRelatedNodeHit:
+    node: KnowledgeNode
+    relation_basis: Literal["citation", "edge", "both"]
+
+
+@dataclass(frozen=True, slots=True)
+class SourceNodesIndexResult:
+    index_revision: str
+    nodes: tuple[SourceRelatedNodeHit, ...]
+    total: int
+    has_more: bool
+
+
 @dataclass(frozen=True, slots=True)
 class SourceIndexResult:
     index_revision: str
@@ -65,6 +96,14 @@ class SourceIndexResult:
     has_more: bool = False
 
 
+@dataclass(frozen=True, slots=True)
+class GraphIndexResult:
+    index_revision: str
+    nodes: tuple[KnowledgeNode, ...]
+    edges: tuple[TypedEdge, ...]
+    truncated: bool
+
+
 @dataclass(frozen=True, slots=True)
 class SearchEdgeResult:
     index_revision: str
@@ -124,6 +163,42 @@ class SearchIndex(Protocol):
         offset: int = 0,
     ) -> SourceIndexResult: ...
 
+    def count_sources(self, *, project_ids: tuple[str, ...], include_shared: bool) -> int: ...
+
+    def graph(self, *, project_ids: tuple[str, ...], include_shared: bool, max_nodes: int, max_edges: int) -> GraphIndexResult: ...
+
+    def browse(
+        self,
+        *,
+        query: str,
+        project_ids: tuple[str, ...],
+        include_shared: bool,
+        node_types: tuple[str, ...] = (),
+        tags: tuple[str, ...] = (),
+        audience: tuple[str, ...] = (),
+        use_for: tuple[str, ...] = (),
+        domain: tuple[str, ...] = (),
+        confidences: tuple[Confidence, ...] = (),
+        as_of_from: date | None = None,
+        as_of_to: date | None = None,
+        stale_only: bool = False,
+        order: str = "title_asc",
+        limit: int = 20,
+        offset: int = 0,
+    ) -> BrowseIndexResult: ...
+
+    def source_nodes(
+        self,
+        *,
+        source_revision_id: str,
+        project_ids: tuple[str, ...],
+        include_shared: bool,
+        query: str = "",
+        node_type: str | None = None,
+        limit: int = 20,
+        offset: int = 0,
+    ) -> SourceNodesIndexResult | None: ...
+
 
 _VECTOR_CANDIDATE_LIMIT = 100
 _VECTOR_BUILD_BATCH_SIZE = 32
@@ -538,6 +613,18 @@ class HybridSearchIndex:
             offset=offset,
         )
 
+    def count_sources(self, *, project_ids: tuple[str, ...], include_shared: bool) -> int:
+        return self.lexical.count_sources(project_ids=project_ids, include_shared=include_shared)
+
+    def graph(self, *, project_ids: tuple[str, ...], include_shared: bool, max_nodes: int, max_edges: int) -> GraphIndexResult:
+        return self.lexical.graph(project_ids=project_ids, include_shared=include_shared, max_nodes=max_nodes, max_edges=max_edges)
+
+    def browse(self, **kwargs: Any) -> BrowseIndexResult:
+        return self.lexical.browse(**kwargs)
+
+    def source_nodes(self, **kwargs: Any) -> SourceNodesIndexResult | None:
+        return self.lexical.source_nodes(**kwargs)
+
 
 def _budgeted_envelope(
     *,
@@ -550,6 +637,7 @@ def _budgeted_envelope(
     index_revision: str,
     budget_tokens: int,
     retrieval_mode: Literal["keyword", "hybrid", "canonical"],
+    filtered_count: int | None = None,
     next_cursor_for: Callable[[int], str | None] | None = None,
 ) -> ReadEnvelope:
     for included_count in range(len(rows), -1, -1):
@@ -571,6 +659,7 @@ def _budgeted_envelope(
             request_id=request_id,
             resolved_scope=scope,
             data=tuple(rows[:included_count]),
+            filtered_count=filtered_count,
             citations=tuple(evidence[key] for key in sorted(evidence)),
             warnings=tuple(warnings),
             omissions=omissions,
@@ -620,6 +709,137 @@ class CanonicalSearchService:
         state = "current" if index_revision == canonical_revision else "lagging"
         return canonical_revision, index_revision, state, schema_version
 
+    def browse(self, request: KnowledgeBrowseRequest, *, principal: Principal) -> KnowledgeBrowseEnvelope:
+        project_ids = self._resolve_scope(request.scope, principal)
+        fingerprint = hashlib.sha256(canonical_json({
+            "projects": project_ids,
+            "shared": request.scope.include_shared,
+            "query": request.query,
+            "types": sorted(item.value for item in request.node_types),
+            "tags": sorted(request.tags), "audience": sorted(request.audience),
+            "use": sorted(request.use_for), "domain": sorted(request.domain),
+            "confidence": sorted(item.value for item in request.confidences),
+            "from": request.as_of_from.isoformat() if request.as_of_from else None,
+            "to": request.as_of_to.isoformat() if request.as_of_to else None,
+            "stale": request.stale_only, "order": request.order, "limit": request.limit,
+        })).hexdigest()
+        offset, expected_revision = self._cursor_offset(request.cursor, fingerprint, max_offset=100_000)
+        lexical = self.index.lexical if isinstance(self.index, HybridSearchIndex) else self.index
+        result = lexical.browse(
+            query=request.query,
+            project_ids=project_ids,
+            include_shared=request.scope.include_shared,
+            node_types=tuple(item.value for item in request.node_types),
+            tags=request.tags,
+            audience=request.audience,
+            use_for=request.use_for,
+            domain=request.domain,
+            confidences=request.confidences,
+            as_of_from=request.as_of_from,
+            as_of_to=request.as_of_to,
+            stale_only=request.stale_only,
+            order=request.order,
+            limit=request.limit,
+            offset=offset,
+        )
+        if expected_revision is not None and expected_revision != result.index_revision:
+            raise SearchError(409, "Browse cursor expired because the index revision changed")
+
+        source_count = self.index.count_sources(project_ids=project_ids, include_shared=False)
+        shared_source_count = (
+            self.index.count_sources(project_ids=(), include_shared=True)
+            if request.scope.include_shared
+            else 0
+        )
+        page = result.nodes
+        next_offset = offset + len(page)
+        next_cursor = (
+            self._encode_cursor(next_offset, fingerprint, result.index_revision)
+            if next_offset < result.filtered_node_count
+            else None
+        )
+        canonical_revision = self.canonical_head()
+        warnings: tuple[Finding, ...] = ()
+        if result.index_revision != canonical_revision:
+            warnings = (Finding(code="INDEX_LAG", message="Browse data reflects an older accepted revision."),)
+        return KnowledgeBrowseEnvelope(
+            request_id="browse-" + secrets.token_hex(12),
+            resolved_scope=Scope(projects=project_ids, include_shared=request.scope.include_shared),
+            nodes=page,
+            project_node_count=result.project_node_count,
+            project_source_count=source_count,
+            shared_node_count=result.shared_node_count,
+            shared_source_count=shared_source_count,
+            filtered_node_count=result.filtered_node_count,
+            facets=result.facets,
+            canonical_revision=canonical_revision,
+            index_revision=result.index_revision,
+            warnings=warnings,
+            next_cursor=next_cursor,
+        )
+
+    def source_nodes(
+        self,
+        *,
+        project_id: str,
+        revision_id: str,
+        principal: Principal,
+        include_shared: bool = False,
+        query: str = "",
+        node_type: str | None = None,
+        limit: int = 20,
+        cursor: str | None = None,
+    ) -> SourceRelatedNodesEnvelope:
+        if not principal.can_read_project(project_id):
+            raise SearchError(403, "Credential cannot read this project")
+        fingerprint = hashlib.sha256(canonical_json({
+            "project": project_id,
+            "revision": revision_id,
+            "shared": include_shared,
+            "query": query,
+            "node_type": node_type,
+            "limit": limit,
+        })).hexdigest()
+        offset, expected_revision = self._cursor_offset(cursor, fingerprint, max_offset=100_000)
+        lexical = self.index.lexical if isinstance(self.index, HybridSearchIndex) else self.index
+        result = lexical.source_nodes(
+            source_revision_id=revision_id,
+            project_ids=(project_id,),
+            include_shared=include_shared,
+            query=query,
+            node_type=node_type,
+            limit=limit,
+            offset=offset,
+        )
+        if result is None:
+            raise SearchError(404, "Source revision was not found")
+        if expected_revision is not None and expected_revision != result.index_revision:
+            raise SearchError(409, "Cursor expired because the index revision changed")
+        next_offset = offset + len(result.nodes)
+        next_cursor = (
+            self._encode_cursor(next_offset, fingerprint, result.index_revision)
+            if next_offset < result.total
+            else None
+        )
+        canonical_revision = self.canonical_head()
+        warnings: tuple[Finding, ...] = ()
+        if result.index_revision != canonical_revision:
+            warnings = (Finding(code="INDEX_LAG", message="Source nodes reflect an older accepted revision."),)
+        return SourceRelatedNodesEnvelope(
+            request_id="srcnodes-" + secrets.token_hex(12),
+            resolved_scope=Scope(projects=(project_id,), include_shared=include_shared),
+            source_revision_id=revision_id,
+            nodes=tuple(
+                SourceRelatedNode(node=hit.node, relation_basis=hit.relation_basis)
+                for hit in result.nodes
+            ),
+            total=result.total,
+            canonical_revision=canonical_revision,
+            index_revision=result.index_revision,
+            warnings=warnings,
+            next_cursor=next_cursor,
+        )
+
     def search(self, request: SearchRequest, *, principal: Principal) -> ReadEnvelope:
         project_ids = self._resolve_scope(request.scope, principal)
         fingerprint = hashlib.sha256(
@@ -638,6 +858,7 @@ class CanonicalSearchService:
                     "as_of_to": request.as_of_to.isoformat() if request.as_of_to else None,
                     "stale_only": request.stale_only,
                     "limit": request.limit,
+                    "include_evidence": request.include_evidence,
                     "retrieval_profile": (
                         self.index.cursor_profile()
                         if isinstance(self.index, HybridSearchIndex)
@@ -674,9 +895,25 @@ class CanonicalSearchService:
         rows: list[dict[str, object]] = []
         for hit in result.nodes:
             node = hit.node
+            node_payload = node.model_dump(mode="json")
+            if not request.include_evidence:
+                # Search returns claim text and result-level citations. Full
+                # claim-to-citation records remain available through get_node.
+                for claim_group in ("claims", "assumptions", "limitations"):
+                    compact_claims = []
+                    for claim in node_payload.get(claim_group, []):
+                        compact_claims.append(
+                            {
+                                **claim,
+                                "evidence_count": len(claim.get("evidence", [])),
+                            }
+                        )
+                        compact_claims[-1].pop("evidence", None)
+                    node_payload[claim_group] = compact_claims
             row: dict[str, object] = {
-                "node": node.model_dump(mode="json"),
+                "node": node_payload,
                 "retrieval_score": hit.score,
+                "evidence_included": request.include_evidence,
             }
             if hit.expanded_from is not None and hit.expansion_edge is not None:
                 row["retrieval_path"] = "graph_expansion"
@@ -692,14 +929,15 @@ class CanonicalSearchService:
                     )
                 )
             node_evidence: dict[str, EvidenceReference] = {}
-            for claim in node.evidence_claims:
-                for item in claim.evidence:
-                    key = evidence_digest(item)
-                    node_evidence[key] = item
-            if hit.expansion_edge is not None:
-                for item in hit.expansion_edge.evidence:
-                    key = evidence_digest(item)
-                    node_evidence[key] = item
+            if request.include_evidence:
+                for claim in node.evidence_claims:
+                    for item in claim.evidence:
+                        key = evidence_digest(item)
+                        node_evidence[key] = item
+                if hit.expansion_edge is not None:
+                    for item in hit.expansion_edge.evidence:
+                        key = evidence_digest(item)
+                        node_evidence[key] = item
             evidence_by_node.append(node_evidence)
         if result.index_revision != canonical_revision:
             warnings.append(
@@ -734,6 +972,56 @@ class CanonicalSearchService:
             ),
         )
 
+    def graph(self, request: KnowledgeGraphRequest, *, principal: Principal) -> KnowledgeGraphEnvelope:
+        project_ids = self._resolve_scope(request.scope, principal)
+        if request.center_node_id is None:
+            result = self.index.graph(
+                project_ids=project_ids,
+                include_shared=request.scope.include_shared,
+                max_nodes=request.max_nodes,
+                max_edges=request.max_edges,
+            )
+        else:
+            scoped = self.index.search(
+                query="", project_ids=project_ids, include_shared=request.scope.include_shared,
+                limit=1, candidate_node_ids=(request.center_node_id,),
+            )
+            origin = scoped.nodes[0].node if scoped.nodes else None
+            if origin is None:
+                raise SearchError(404, "Knowledge node was not found within the requested scope")
+            nodes_by_id = {origin.node_id: origin}
+            edges: list[TypedEdge] = []
+            neighbors = None
+            if request.max_nodes > 1:
+                neighbors = self.index.neighbors(
+                    node_id=origin.node_id, edge_types=(), project_ids=project_ids,
+                    include_shared=request.scope.include_shared, limit=min(50, request.max_nodes - 1),
+                )
+                for hit in neighbors.edges:
+                    endpoint = hit.neighbor_endpoint
+                    if endpoint.kind != "node" or hit.neighbor_node is None:
+                        continue
+                    nodes_by_id[hit.neighbor_node.node_id] = hit.neighbor_node
+                    edges.append(hit.edge)
+            result = GraphIndexResult(
+                index_revision=neighbors.index_revision if neighbors else scoped.index_revision,
+                nodes=tuple(nodes_by_id.values()),
+                edges=tuple(edges[:request.max_edges]),
+                truncated=(neighbors.has_more if neighbors else False) or len(edges) > request.max_edges,
+            )
+        canonical_revision = self.canonical_head()
+        warnings: tuple[Finding, ...] = ()
+        if result.index_revision != canonical_revision:
+            warnings = (Finding(code="INDEX_LAG", message="Graph data reflects an older accepted revision."),)
+        return KnowledgeGraphEnvelope(
+            request_id="graph-" + secrets.token_hex(12),
+            resolved_scope=Scope(projects=project_ids, include_shared=request.scope.include_shared),
+            nodes=result.nodes, edges=result.edges, truncated=result.truncated,
+            center_node_id=request.center_node_id,
+            canonical_revision=canonical_revision, index_revision=result.index_revision,
+            warnings=warnings,
+        )
+
     @staticmethod
     def _encode_cursor(offset: int, fingerprint: str, index_revision: str) -> str:
         payload = canonical_json(
@@ -956,6 +1244,10 @@ class CanonicalSearchService:
         rows: list[dict[str, object]] = [
             {"source": source.model_dump(mode="json")} for source in result.sources
         ]
+        source_count = self.index.count_sources(
+            project_ids=project_ids,
+            include_shared=request.scope.include_shared,
+        )
         warnings: list[Finding] = []
         if result.index_revision != canonical_revision:
             warnings.append(
@@ -977,6 +1269,7 @@ class CanonicalSearchService:
             index_revision=result.index_revision,
             budget_tokens=request.budget_tokens,
             retrieval_mode="keyword",
+            filtered_count=source_count,
             next_cursor_for=(
                 lambda included: (
                     self._encode_cursor(offset + included, fingerprint, result.index_revision)