feat: online verification, node removals, and knowledge browser
CI / quality (push) Canceled after 0s

Adds claim-bound online evidence records and citation verification scripts,
node removal transport/adapter, retrieval benchmark and owner-review tooling,
GraphCanvas/SourceDetail knowledge browser UI, and the ingest handoff helper.
Bumps the agents/gewu pack to the online-verification branch.
This commit is contained in:
vasily committed 2026-10-16 20:02:29 +08:00
1 parent a3156f4e08
commit 78033bb830
83 files changed
+13476 -1051

No files matched your search

+147
View File
@@ -0,0 +1,147 @@
"""Durable seven-day soft removals for user-facing knowledge visibility."""
from __future__ import annotations
import secrets
import sqlite3
from contextlib import closing
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from pathlib import Path
from gewu.application.paths import reject_symlink_components
REMOVAL_GRACE_PERIOD = timedelta(days=7)
@dataclass(frozen=True, slots=True)
class NodeRemoval:
removal_id: str
project_id: str
node_id: str
title: str
requested_by: str
requested_at: str
restore_until: str
class RemovalConflict(RuntimeError):
"""A node already has an active removal or its restore window expired."""
class SQLiteNodeRemovalStore:
def __init__(self, path: Path) -> None:
self.path = path.expanduser().absolute()
reject_symlink_components(self.path)
self.path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
self.path.parent.chmod(0o700)
if self.path.is_symlink():
raise RuntimeError("Node removal database must not be a symbolic link")
with closing(self._connect()) as connection:
connection.execute(
"""CREATE TABLE IF NOT EXISTS node_removals (
removal_id TEXT PRIMARY KEY,
project_id TEXT NOT NULL,
node_id TEXT NOT NULL,
title TEXT NOT NULL,
requested_by TEXT NOT NULL,
requested_at TEXT NOT NULL,
restore_until TEXT NOT NULL,
restored_at TEXT,
restored_by TEXT
)"""
)
connection.execute(
"CREATE UNIQUE INDEX IF NOT EXISTS node_removals_active_node "
"ON node_removals(project_id, node_id) WHERE restored_at IS NULL"
)
connection.execute(
"CREATE INDEX IF NOT EXISTS node_removals_project_restore "
"ON node_removals(project_id, restored_at, restore_until)"
)
self.path.chmod(0o600)
def _connect(self) -> sqlite3.Connection:
connection = sqlite3.connect(self.path, timeout=5, isolation_level=None)
connection.execute("PRAGMA foreign_keys=ON")
connection.execute("PRAGMA busy_timeout=5000")
connection.execute("PRAGMA synchronous=FULL")
return connection
@staticmethod
def _record(row: tuple[object, ...]) -> NodeRemoval:
return NodeRemoval(*(str(value) for value in row))
def is_hidden(self, project_id: str, node_id: str) -> bool:
with closing(self._connect()) as connection:
row = connection.execute(
"SELECT 1 FROM node_removals WHERE project_id=? AND node_id=? "
"AND restored_at IS NULL LIMIT 1",
(project_id, node_id),
).fetchone()
return row is not None
def request(self, project_id: str, node_id: str, title: str, requested_by: str) -> NodeRemoval:
now = datetime.now(UTC)
requested_at = now.isoformat()
restore_until = (now + REMOVAL_GRACE_PERIOD).isoformat()
record = NodeRemoval(
secrets.token_hex(16), project_id, node_id, title, requested_by,
requested_at, restore_until,
)
try:
with closing(self._connect()) as connection:
connection.execute(
"INSERT INTO node_removals(removal_id, project_id, node_id, title, "
"requested_by, requested_at, restore_until) VALUES (?, ?, ?, ?, ?, ?, ?)",
(
record.removal_id, record.project_id, record.node_id, record.title,
record.requested_by, record.requested_at, record.restore_until,
),
)
except sqlite3.IntegrityError as error:
raise RemovalConflict(
"This node is already removed or its restore period ended"
) from error
except sqlite3.Error as error:
raise RuntimeError("Node removal could not be saved") from error
return record
def list_restorable(self, project_id: str) -> tuple[NodeRemoval, ...]:
now = datetime.now(UTC).isoformat()
with closing(self._connect()) as connection:
rows = connection.execute(
"SELECT removal_id, project_id, node_id, title, requested_by, requested_at, "
"restore_until "
"FROM node_removals WHERE project_id=? AND restored_at IS NULL AND restore_until>? "
"ORDER BY requested_at DESC LIMIT 100",
(project_id, now),
).fetchall()
return tuple(self._record(row) for row in rows)
def restore(self, project_id: str, node_id: str, restored_by: str) -> NodeRemoval:
now = datetime.now(UTC)
now_text = now.isoformat()
with closing(self._connect()) as connection:
connection.execute("BEGIN IMMEDIATE")
row = connection.execute(
"SELECT removal_id, project_id, node_id, title, requested_by, requested_at, "
"restore_until "
"FROM node_removals WHERE project_id=? AND node_id=? AND restored_at IS NULL "
"ORDER BY requested_at DESC LIMIT 1",
(project_id, node_id),
).fetchone()
if row is None:
connection.rollback()
raise KeyError("Removed node was not found")
record = self._record(row)
if record.restore_until <= now_text:
connection.rollback()
raise RemovalConflict("The seven-day restore period has expired")
connection.execute(
"UPDATE node_removals SET restored_at=?, restored_by=? WHERE removal_id=? "
"AND restored_at IS NULL",
(now_text, restored_by, record.removal_id),
)
connection.commit()
return record
+449 -6
View File
@@ -16,7 +16,11 @@ from pydantic import ValidationError
from gewu.application.paths import reject_symlink_components
from gewu.application.ports import CanonicalRepository
from gewu.application.search import (
BrowseIndexResult,
GraphIndexResult,
IndexedNode,
SourceRelatedNodeHit,
SourceNodesIndexResult,
SearchEdgeHit,
SearchEdgeResult,
SearchIndexResult,
@@ -28,7 +32,7 @@ from gewu.domain.contracts import Confidence, KnowledgeNode, SourceRevisionRecor
MAX_INDEX_NODES = 100_000
MAX_INDEX_EDGES = 500_000
MAX_INDEX_RECORD_BYTES = 1_048_576
INDEX_SCHEMA_VERSION = 4
INDEX_SCHEMA_VERSION = 5
_WORD = re.compile(r"[^\W_]+", flags=re.UNICODE)
@@ -39,7 +43,9 @@ class SearchIndexError(RuntimeError):
class SQLiteSearchIndex:
"""SQLite FTS5 index that can be deleted and reconstructed from Git."""
def __init__(self, path: Path, repository: CanonicalRepository) -> None:
def __init__(
self, path: Path, repository: CanonicalRepository, *, node_removals_path: Path | None = None
) -> None:
self.path = path.expanduser().absolute()
reject_symlink_components(self.path)
self.path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
@@ -47,6 +53,11 @@ class SQLiteSearchIndex:
if self.path.is_symlink():
raise RuntimeError("Search index path must not be a symbolic link")
self.repository = repository
self.node_removals_path = (
node_removals_path.expanduser().absolute() if node_removals_path is not None else None
)
if self.node_removals_path is not None:
reject_symlink_components(self.node_removals_path)
self._vector_extension_loader: Callable[[sqlite3.Connection], None] | None = None
self._vector_generation_builder: (
Callable[[sqlite3.Connection, str, tuple[KnowledgeNode, ...]], None] | None
@@ -72,6 +83,10 @@ class SQLiteSearchIndex:
connection.execute("PRAGMA foreign_keys=ON")
connection.execute("PRAGMA busy_timeout=5000")
connection.execute("PRAGMA synchronous=NORMAL")
if self.node_removals_path is not None:
connection.execute(
"ATTACH DATABASE ? AS removal_state", (str(self.node_removals_path),)
)
if self._vector_extension_loader is not None:
try:
self._vector_extension_loader(connection)
@@ -96,6 +111,15 @@ class SQLiteSearchIndex:
"CREATE INDEX IF NOT EXISTS indexed_nodes_project "
"ON indexed_nodes(project_id, node_id)"
)
connection.execute(
"CREATE TABLE IF NOT EXISTS indexed_citations ("
"node_id TEXT NOT NULL, project_id TEXT NOT NULL, source_revision_id TEXT NOT NULL, "
"PRIMARY KEY(node_id, source_revision_id))"
)
connection.execute(
"CREATE INDEX IF NOT EXISTS indexed_citations_source "
"ON indexed_citations(source_revision_id, project_id)"
)
connection.execute(
"CREATE TABLE IF NOT EXISTS indexed_sources ("
"source_revision_id TEXT PRIMARY KEY, source_json TEXT NOT NULL)"
@@ -237,6 +261,7 @@ class SQLiteSearchIndex:
connection.execute("BEGIN IMMEDIATE")
connection.execute("DELETE FROM node_fts")
connection.execute("DELETE FROM indexed_nodes")
connection.execute("DELETE FROM indexed_citations")
connection.execute("DELETE FROM indexed_edges")
connection.execute("DELETE FROM indexed_sources")
connection.execute("DELETE FROM indexed_source_grants")
@@ -248,6 +273,13 @@ class SQLiteSearchIndex:
"VALUES (?, ?, ?)",
(node.node_id, node.project_id, node_json),
)
for claim in node.evidence_claims:
for evidence in claim.evidence:
connection.execute(
"INSERT OR IGNORE INTO indexed_citations("
"node_id, project_id, source_revision_id) VALUES (?, ?, ?)",
(node.node_id, node.project_id, evidence.source_revision_id),
)
connection.execute(
"INSERT INTO node_fts(node_id, project_id, title, summary, claims, tags) "
"VALUES (?, ?, ?, ?, ?, ?)",
@@ -338,6 +370,12 @@ class SQLiteSearchIndex:
placeholders = ", ".join("?" for _ in project_scope)
filters: list[str] = []
filter_parameters: list[str] = []
if self.node_removals_path is not None:
filters.append(
"NOT EXISTS (SELECT 1 FROM removal_state.node_removals removal "
"WHERE removal.project_id=indexed_nodes.project_id "
"AND removal.node_id=indexed_nodes.node_id AND removal.restored_at IS NULL)"
)
if node_types:
filters.append(
"json_extract(indexed_nodes.node_json, '$.node_type') IN ("
@@ -444,10 +482,16 @@ class SQLiteSearchIndex:
scope_placeholders = ", ".join("?" for _ in project_scope)
def eligible(kind_column: str, id_column: str) -> str:
visible = (
" AND NOT EXISTS (SELECT 1 FROM removal_state.node_removals removal "
"WHERE removal.project_id=eligible_node.project_id "
"AND removal.node_id=eligible_node.node_id AND removal.restored_at IS NULL)"
if self.node_removals_path is not None else ""
)
return (
f"(({kind_column}='node' AND EXISTS (SELECT 1 FROM indexed_nodes eligible_node "
f"WHERE eligible_node.node_id=e.{id_column} AND eligible_node.project_id IN "
f"({scope_placeholders}))) OR "
f"({scope_placeholders}){visible})) OR "
f"({kind_column}='project' AND e.{id_column} IN ({scope_placeholders})) OR "
f"({kind_column}='source' AND EXISTS (SELECT 1 FROM indexed_source_grants grant "
f"WHERE grant.source_revision_id=e.{id_column} AND grant.project_id IN "
@@ -488,9 +532,14 @@ class SQLiteSearchIndex:
if state is None:
raise SearchIndexError("Search index revision is unavailable")
revision = str(state[0])
origin_row = connection.execute(
"SELECT node_json FROM indexed_nodes WHERE node_id=?", (node_id,)
).fetchone()
origin_query = "SELECT node_json FROM indexed_nodes WHERE node_id=?"
if self.node_removals_path is not None:
origin_query += (
" AND NOT EXISTS (SELECT 1 FROM removal_state.node_removals removal "
"WHERE removal.project_id=indexed_nodes.project_id "
"AND removal.node_id=indexed_nodes.node_id AND removal.restored_at IS NULL)"
)
origin_row = connection.execute(origin_query, (node_id,)).fetchone()
origin = (
KnowledgeNode.model_validate_json(origin_row[0], strict=True)
if origin_row is not None
@@ -570,3 +619,397 @@ class SQLiteSearchIndex:
except (sqlite3.Error, ValidationError) as error:
raise SearchIndexError("Source query could not be evaluated") from error
return SourceIndexResult(revision, sources, len(rows) > limit)
def count_sources(self, *, project_ids: tuple[str, ...], include_shared: bool) -> int:
self._rebuild_if_needed()
project_scope = tuple(sorted(set(project_ids) | ({"shared"} if include_shared else set())))
if not project_scope:
return 0
placeholders = ", ".join("?" for _ in project_scope)
try:
with closing(self._connect()) as connection:
row = connection.execute(
"SELECT COUNT(DISTINCT source_revision_id) FROM indexed_source_grants "
f"WHERE project_id IN ({placeholders})", project_scope,
).fetchone()
except sqlite3.Error as error:
raise SearchIndexError("Source count could not be evaluated") from error
return int(row[0]) if row else 0
def graph(
self, *, project_ids: tuple[str, ...], include_shared: bool, max_nodes: int, max_edges: int
) -> GraphIndexResult:
nodes_result = self.search(
query="", project_ids=project_ids, include_shared=include_shared, limit=max_nodes
)
node_ids = tuple(hit.node.node_id for hit in nodes_result.nodes)
if not node_ids:
return GraphIndexResult(nodes_result.index_revision, (), (), nodes_result.has_more)
placeholders = ", ".join("?" for _ in node_ids)
statement = (
"SELECT edge_json FROM indexed_edges WHERE source_kind='node' AND target_kind='node' "
f"AND source_id IN ({placeholders}) AND target_id IN ({placeholders}) "
"ORDER BY edge_id LIMIT ?"
)
try:
with closing(self._connect()) as connection:
connection.execute("BEGIN")
state = connection.execute(
"SELECT revision FROM index_state WHERE singleton=1 AND schema_version=?",
(INDEX_SCHEMA_VERSION,),
).fetchone()
if state is None or str(state[0]) != nodes_result.index_revision:
raise SearchIndexError("Graph index changed while loading its nodes")
rows = connection.execute(
statement, (*node_ids, *node_ids, max_edges + 1)
).fetchall()
edges = tuple(
TypedEdge.model_validate_json(row[0], strict=True)
for row in rows[:max_edges]
)
connection.commit()
except (sqlite3.Error, ValidationError) as error:
raise SearchIndexError("Knowledge graph could not be evaluated") from error
return GraphIndexResult(
nodes=tuple(hit.node for hit in nodes_result.nodes),
edges=edges,
index_revision=nodes_result.index_revision,
truncated=nodes_result.has_more or len(rows) > max_edges,
)
def browse(
self,
*,
query: str,
project_ids: tuple[str, ...],
include_shared: bool,
node_types: tuple[str, ...] = (),
tags: tuple[str, ...] = (),
audience: tuple[str, ...] = (),
use_for: tuple[str, ...] = (),
domain: tuple[str, ...] = (),
confidences: tuple[Confidence, ...] = (),
as_of_from: date | None = None,
as_of_to: date | None = None,
stale_only: bool = False,
order: str = "title_asc",
limit: int = 20,
offset: int = 0,
) -> BrowseIndexResult:
revision = self._rebuild_if_needed()
project_scope = tuple(sorted(set(project_ids) | ({"shared"} if include_shared else set())))
if not project_scope:
return BrowseIndexResult(
revision, (), 0, 0, 0, {k: {} for k in ("tags", "audience", "use_for", "domain")}, False
)
removal_filter = (
" AND NOT EXISTS (SELECT 1 FROM removal_state.node_removals removal "
"WHERE removal.project_id=indexed_nodes.project_id "
"AND removal.node_id=indexed_nodes.node_id AND removal.restored_at IS NULL)"
if self.node_removals_path is not None
else ""
)
filters: list[str] = [
f"indexed_nodes.project_id IN ({", ".join("?" for _ in project_scope)})"
]
filter_parameters: list[object] = list(project_scope)
if self.node_removals_path is not None:
filters.append(
"NOT EXISTS (SELECT 1 FROM removal_state.node_removals removal "
"WHERE removal.project_id=indexed_nodes.project_id "
"AND removal.node_id=indexed_nodes.node_id AND removal.restored_at IS NULL)"
)
if node_types:
filters.append(
"json_extract(indexed_nodes.node_json, '$.node_type') IN ("
+ ", ".join("?" for _ in node_types)
+ ")"
)
filter_parameters.extend(node_types)
for field, values in (
("tags", tags),
("audience", audience),
("use_for", use_for),
("domain", domain),
):
if values:
value_slots = ", ".join("?" for _ in values)
filters.append(
"EXISTS (SELECT 1 FROM json_each(indexed_nodes.node_json, '$."
+ field
+ "') AS facet WHERE facet.value IN ("
+ value_slots
+ "))"
)
filter_parameters.extend(values)
if confidences:
filters.append(
"json_extract(indexed_nodes.node_json, '$.confidence') IN ("
+ ", ".join("?" for _ in confidences)
+ ")"
)
filter_parameters.extend(item.value for item in confidences)
if as_of_from is not None:
filters.append("substr(json_extract(indexed_nodes.node_json, '$.as_of'), 1, 10) >= ?")
filter_parameters.append(as_of_from.isoformat())
if as_of_to is not None:
filters.append("substr(json_extract(indexed_nodes.node_json, '$.as_of'), 1, 10) <= ?")
filter_parameters.append(as_of_to.isoformat())
if stale_only:
filters.append("json_extract(indexed_nodes.node_json, '$.stale_after') <= ?")
filter_parameters.append(datetime.now(UTC).date().isoformat() + "T00:00:00Z")
where_clause = " WHERE " + " AND ".join(filters)
match = self._match_expression(query)
if order == "title_asc":
order_by = "LOWER(json_extract(indexed_nodes.node_json, '$.title')) ASC, json_extract(indexed_nodes.node_json, '$.title') ASC, indexed_nodes.node_id ASC"
elif order == "title_desc":
order_by = "LOWER(json_extract(indexed_nodes.node_json, '$.title')) DESC, json_extract(indexed_nodes.node_json, '$.title') DESC, indexed_nodes.node_id DESC"
elif order == "date_newest":
order_by = "(json_extract(indexed_nodes.node_json, '$.as_of') IS NULL) ASC, json_extract(indexed_nodes.node_json, '$.as_of') DESC, indexed_nodes.node_id ASC"
elif order == "date_oldest":
order_by = "(json_extract(indexed_nodes.node_json, '$.as_of') IS NULL) ASC, json_extract(indexed_nodes.node_json, '$.as_of') ASC, indexed_nodes.node_id ASC"
elif order == "relevance" and match:
order_by = "score ASC, indexed_nodes.node_id ASC"
else:
order_by = "LOWER(json_extract(indexed_nodes.node_json, '$.title')) ASC, json_extract(indexed_nodes.node_json, '$.title') ASC, indexed_nodes.node_id ASC"
try:
with closing(self._connect()) as connection:
connection.execute("BEGIN")
state = connection.execute(
"SELECT revision FROM index_state WHERE singleton=1 AND schema_version=?",
(INDEX_SCHEMA_VERSION,),
).fetchone()
if state is None:
raise SearchIndexError("Search index revision is unavailable")
rev = str(state[0])
# Project node count
project_node_count = 0
if project_ids:
p_holders = ", ".join("?" for _ in project_ids)
p_row = connection.execute(
f"SELECT COUNT(*) FROM indexed_nodes WHERE project_id IN ({p_holders}){removal_filter}",
project_ids,
).fetchone()
project_node_count = int(p_row[0]) if p_row else 0
# Shared node count
shared_node_count = 0
if include_shared:
s_row = connection.execute(
f"SELECT COUNT(*) FROM indexed_nodes WHERE project_id='shared'{removal_filter}"
).fetchone()
shared_node_count = int(s_row[0]) if s_row else 0
# Filtered count
if match:
count_stmt = (
"SELECT COUNT(*) FROM node_fts JOIN indexed_nodes USING(node_id) "
f"{where_clause} AND node_fts MATCH ?"
)
count_row = connection.execute(
count_stmt, (*filter_parameters, match)
).fetchone()
else:
count_stmt = f"SELECT COUNT(*) FROM indexed_nodes {where_clause}"
count_row = connection.execute(count_stmt, filter_parameters).fetchone()
filtered_node_count = int(count_row[0]) if count_row else 0
# Paged node query
if match:
query_stmt = (
"SELECT indexed_nodes.node_json, bm25(node_fts) AS score "
f"FROM node_fts JOIN indexed_nodes USING(node_id) {where_clause} AND node_fts MATCH ? "
f"ORDER BY {order_by} LIMIT ? OFFSET ?"
)
node_rows = connection.execute(
query_stmt, (*filter_parameters, match, limit + 1, offset)
).fetchall()
else:
query_stmt = (
f"SELECT indexed_nodes.node_json FROM indexed_nodes {where_clause} "
f"ORDER BY {order_by} LIMIT ? OFFSET ?"
)
node_rows = connection.execute(
query_stmt, (*filter_parameters, limit + 1, offset)
).fetchall()
nodes = tuple(
KnowledgeNode.model_validate_json(row[0], strict=True)
for row in node_rows[:limit]
)
has_more = len(node_rows) > limit
# Facets
scope_holders = ", ".join("?" for _ in project_scope)
facets: dict[str, dict[str, int]] = {}
for field in ("tags", "audience", "use_for", "domain"):
facet_stmt = (
"SELECT facet.value, COUNT(DISTINCT indexed_nodes.node_id) "
f"FROM indexed_nodes, json_each(indexed_nodes.node_json, '$.' || ?) AS facet "
f"WHERE indexed_nodes.project_id IN ({scope_holders}){removal_filter} "
"GROUP BY facet.value"
)
facet_rows = connection.execute(facet_stmt, (field, *project_scope)).fetchall()
facets[field] = {str(r[0]): int(r[1]) for r in facet_rows}
connection.commit()
except (sqlite3.Error, ValidationError) as error:
raise SearchIndexError("Browse query could not be evaluated") from error
return BrowseIndexResult(
index_revision=rev,
nodes=nodes,
filtered_node_count=filtered_node_count,
project_node_count=project_node_count,
shared_node_count=shared_node_count,
facets=facets,
has_more=has_more,
)
def source_nodes(
self,
*,
source_revision_id: str,
project_ids: tuple[str, ...],
include_shared: bool,
query: str = "",
node_type: str | None = None,
limit: int = 20,
offset: int = 0,
) -> SourceNodesIndexResult | None:
revision = self._rebuild_if_needed()
project_scope = tuple(sorted(set(project_ids) | ({"shared"} if include_shared else set())))
if not project_scope:
return None
scope_holders = ", ".join("?" for _ in project_scope)
match = self._match_expression(query)
try:
with closing(self._connect()) as connection:
connection.execute("BEGIN")
state = connection.execute(
"SELECT revision FROM index_state WHERE singleton=1 AND schema_version=?",
(INDEX_SCHEMA_VERSION,),
).fetchone()
if state is None:
raise SearchIndexError("Search index revision is unavailable")
rev = str(state[0])
# Check source revision grant
grant_row = connection.execute(
f"SELECT 1 FROM indexed_source_grants WHERE source_revision_id=? AND project_id IN ({scope_holders}) LIMIT 1",
(source_revision_id, *project_scope),
).fetchone()
if grant_row is None:
return None
cte = f"""
WITH candidates AS (
SELECT node_id, 'citation' AS basis
FROM indexed_citations
WHERE source_revision_id = ? AND project_id IN ({scope_holders})
UNION ALL
SELECT
CASE WHEN source_kind = 'node' THEN source_id ELSE target_id END AS node_id,
'edge' AS basis
FROM indexed_edges
WHERE (
(source_kind = 'node' AND target_kind = 'source' AND target_id = ?) OR
(source_kind = 'source' AND target_kind = 'node' AND source_id = ?)
)
),
combined AS (
SELECT
node_id,
CASE
WHEN COUNT(DISTINCT basis) > 1 THEN 'both'
ELSE MAX(basis)
END AS relation_basis
FROM candidates
GROUP BY node_id
)
"""
filters = [
f"indexed_nodes.project_id IN ({scope_holders})",
]
filter_params: list[object] = [
source_revision_id,
*project_scope,
source_revision_id,
source_revision_id,
*project_scope,
]
if self.node_removals_path is not None:
filters.append(
"NOT EXISTS (SELECT 1 FROM removal_state.node_removals removal "
"WHERE removal.project_id=indexed_nodes.project_id "
"AND removal.node_id=indexed_nodes.node_id AND removal.restored_at IS NULL)"
)
if node_type:
filters.append("json_extract(indexed_nodes.node_json, '$.node_type') = ?")
filter_params.append(node_type)
where_clause = " WHERE " + " AND ".join(filters)
# Total count
if match:
count_stmt = (
f"{cte} SELECT COUNT(*) FROM combined JOIN indexed_nodes USING(node_id) "
f"JOIN node_fts USING(node_id) {where_clause} AND node_fts MATCH ?"
)
count_row = connection.execute(count_stmt, (*filter_params, match)).fetchone()
else:
count_stmt = (
f"{cte} SELECT COUNT(*) FROM combined JOIN indexed_nodes USING(node_id) {where_clause}"
)
count_row = connection.execute(count_stmt, filter_params).fetchone()
total = int(count_row[0]) if count_row else 0
# Rows query
order_by = "LOWER(json_extract(indexed_nodes.node_json, '$.title')) ASC, indexed_nodes.node_id ASC"
if match:
row_stmt = (
f"{cte} SELECT indexed_nodes.node_json, combined.relation_basis FROM combined "
f"JOIN indexed_nodes USING(node_id) JOIN node_fts USING(node_id) "
f"{where_clause} AND node_fts MATCH ? ORDER BY {order_by} LIMIT ? OFFSET ?"
)
rows = connection.execute(
row_stmt, (*filter_params, match, limit + 1, offset)
).fetchall()
else:
row_stmt = (
f"{cte} SELECT indexed_nodes.node_json, combined.relation_basis FROM combined "
f"JOIN indexed_nodes USING(node_id) {where_clause} ORDER BY {order_by} LIMIT ? OFFSET ?"
)
rows = connection.execute(
row_stmt, (*filter_params, limit + 1, offset)
).fetchall()
nodes = tuple(
SourceRelatedNodeHit(
node=KnowledgeNode.model_validate_json(r[0], strict=True),
relation_basis=r[1],
)
for r in rows[:limit]
)
has_more = len(rows) > limit
connection.commit()
except (sqlite3.Error, ValidationError) as error:
raise SearchIndexError("Source related nodes could not be evaluated") from error
return SourceNodesIndexResult(
index_revision=rev,
nodes=nodes,
total=total,
has_more=has_more,
)
+6
View File
@@ -5,6 +5,7 @@ from __future__ import annotations
import hashlib
import json
import secrets
from collections.abc import Callable
from dataclasses import dataclass
import yaml
@@ -39,6 +40,7 @@ _GIT_COMMIT_ID = TypeAdapter(GitCommitId)
@dataclass(frozen=True, slots=True)
class CanonicalNodeReader:
repository: CanonicalRepository
node_is_hidden: Callable[[str, str], bool] | None = None
def get_node(
self,
@@ -50,6 +52,8 @@ class CanonicalNodeReader:
) -> ReadEnvelope:
if not principal.can_read_project(project_id):
raise ReadError(403, "Credential cannot read this project")
if self.node_is_hidden is not None and self.node_is_hidden(project_id, node_id):
raise ReadError(404, "Knowledge node was not found")
head = self.repository.head()
path = f"projects/{_key(project_id)}/nodes/{_key(node_id)}.yaml"
if revision is None:
@@ -98,6 +102,8 @@ class CanonicalNodeReader:
raise ReadError(422, "History limit must be between one and twenty")
if not principal.can_read_project(project_id):
raise ReadError(403, "Credential cannot read this project")
if self.node_is_hidden is not None and self.node_is_hidden(project_id, node_id):
raise ReadError(404, "Knowledge node was not found")
path = f"projects/{_key(project_id)}/nodes/{_key(node_id)}.yaml"
head = self.repository.head()
rows: list[dict[str, object]] = []
+4 -1
View File
@@ -30,6 +30,9 @@ from gewu.domain.validation import extraction_digest
MAX_CONTEXT_RECORDS = 20_000
MAX_CONTEXT_RECORD_BYTES = 1_048_576
# Character-level normalization maps are substantially larger than source text.
# Keep ordinary registry/node limits small while allowing bounded extraction records.
MAX_CONTEXT_EXTRACTION_BYTES = 16 * 1024 * 1024
MAX_PROJECT_REGISTRY_RESPONSE_BYTES = 1_048_576
@@ -162,7 +165,7 @@ class LocalContextProvider:
continue
record_count += 1
extraction_bytes = self.repository.read_file(revision, record_path)
if extraction_bytes is None or len(extraction_bytes) > MAX_CONTEXT_RECORD_BYTES:
if extraction_bytes is None or len(extraction_bytes) > MAX_CONTEXT_EXTRACTION_BYTES:
raise RegistryError("Canonical extraction is unavailable or too large")
extraction = ExtractionRecord.model_validate_json(extraction_bytes, strict=True)
if extraction_digest(extraction) != extraction.extraction_id:
+303 -10
View File
@@ -11,7 +11,7 @@ from collections import defaultdict
from collections.abc import Callable, Collection, Iterator
from dataclasses import dataclass
from datetime import UTC, date, datetime
from typing import Literal, Protocol
from typing import Any, Literal, Protocol
from gewu.application.identity import Principal
from gewu.application.ports import EmbeddingProvider, VectorGeneration, VectorIndex
@@ -22,6 +22,12 @@ from gewu.domain.contracts import (
EvidenceReference,
Finding,
KnowledgeNode,
KnowledgeBrowseRequest,
KnowledgeBrowseEnvelope,
KnowledgeGraphEnvelope,
KnowledgeGraphRequest,
SourceRelatedNode,
SourceRelatedNodesEnvelope,
NeighborRequest,
ReadEnvelope,
Scope,
@@ -58,6 +64,31 @@ class SearchIndexResult:
warnings: tuple[Finding, ...] = ()
@dataclass(frozen=True, slots=True)
class BrowseIndexResult:
index_revision: str
nodes: tuple[KnowledgeNode, ...]
filtered_node_count: int
project_node_count: int
shared_node_count: int
facets: dict[str, dict[str, int]]
has_more: bool
@dataclass(frozen=True, slots=True)
class SourceRelatedNodeHit:
node: KnowledgeNode
relation_basis: Literal["citation", "edge", "both"]
@dataclass(frozen=True, slots=True)
class SourceNodesIndexResult:
index_revision: str
nodes: tuple[SourceRelatedNodeHit, ...]
total: int
has_more: bool
@dataclass(frozen=True, slots=True)
class SourceIndexResult:
index_revision: str
@@ -65,6 +96,14 @@ class SourceIndexResult:
has_more: bool = False
@dataclass(frozen=True, slots=True)
class GraphIndexResult:
index_revision: str
nodes: tuple[KnowledgeNode, ...]
edges: tuple[TypedEdge, ...]
truncated: bool
@dataclass(frozen=True, slots=True)
class SearchEdgeResult:
index_revision: str
@@ -124,6 +163,42 @@ class SearchIndex(Protocol):
offset: int = 0,
) -> SourceIndexResult: ...
def count_sources(self, *, project_ids: tuple[str, ...], include_shared: bool) -> int: ...
def graph(self, *, project_ids: tuple[str, ...], include_shared: bool, max_nodes: int, max_edges: int) -> GraphIndexResult: ...
def browse(
self,
*,
query: str,
project_ids: tuple[str, ...],
include_shared: bool,
node_types: tuple[str, ...] = (),
tags: tuple[str, ...] = (),
audience: tuple[str, ...] = (),
use_for: tuple[str, ...] = (),
domain: tuple[str, ...] = (),
confidences: tuple[Confidence, ...] = (),
as_of_from: date | None = None,
as_of_to: date | None = None,
stale_only: bool = False,
order: str = "title_asc",
limit: int = 20,
offset: int = 0,
) -> BrowseIndexResult: ...
def source_nodes(
self,
*,
source_revision_id: str,
project_ids: tuple[str, ...],
include_shared: bool,
query: str = "",
node_type: str | None = None,
limit: int = 20,
offset: int = 0,
) -> SourceNodesIndexResult | None: ...
_VECTOR_CANDIDATE_LIMIT = 100
_VECTOR_BUILD_BATCH_SIZE = 32
@@ -538,6 +613,18 @@ class HybridSearchIndex:
offset=offset,
)
def count_sources(self, *, project_ids: tuple[str, ...], include_shared: bool) -> int:
return self.lexical.count_sources(project_ids=project_ids, include_shared=include_shared)
def graph(self, *, project_ids: tuple[str, ...], include_shared: bool, max_nodes: int, max_edges: int) -> GraphIndexResult:
return self.lexical.graph(project_ids=project_ids, include_shared=include_shared, max_nodes=max_nodes, max_edges=max_edges)
def browse(self, **kwargs: Any) -> BrowseIndexResult:
return self.lexical.browse(**kwargs)
def source_nodes(self, **kwargs: Any) -> SourceNodesIndexResult | None:
return self.lexical.source_nodes(**kwargs)
def _budgeted_envelope(
*,
@@ -550,6 +637,7 @@ def _budgeted_envelope(
index_revision: str,
budget_tokens: int,
retrieval_mode: Literal["keyword", "hybrid", "canonical"],
filtered_count: int | None = None,
next_cursor_for: Callable[[int], str | None] | None = None,
) -> ReadEnvelope:
for included_count in range(len(rows), -1, -1):
@@ -571,6 +659,7 @@ def _budgeted_envelope(
request_id=request_id,
resolved_scope=scope,
data=tuple(rows[:included_count]),
filtered_count=filtered_count,
citations=tuple(evidence[key] for key in sorted(evidence)),
warnings=tuple(warnings),
omissions=omissions,
@@ -620,6 +709,137 @@ class CanonicalSearchService:
state = "current" if index_revision == canonical_revision else "lagging"
return canonical_revision, index_revision, state, schema_version
def browse(self, request: KnowledgeBrowseRequest, *, principal: Principal) -> KnowledgeBrowseEnvelope:
project_ids = self._resolve_scope(request.scope, principal)
fingerprint = hashlib.sha256(canonical_json({
"projects": project_ids,
"shared": request.scope.include_shared,
"query": request.query,
"types": sorted(item.value for item in request.node_types),
"tags": sorted(request.tags), "audience": sorted(request.audience),
"use": sorted(request.use_for), "domain": sorted(request.domain),
"confidence": sorted(item.value for item in request.confidences),
"from": request.as_of_from.isoformat() if request.as_of_from else None,
"to": request.as_of_to.isoformat() if request.as_of_to else None,
"stale": request.stale_only, "order": request.order, "limit": request.limit,
})).hexdigest()
offset, expected_revision = self._cursor_offset(request.cursor, fingerprint, max_offset=100_000)
lexical = self.index.lexical if isinstance(self.index, HybridSearchIndex) else self.index
result = lexical.browse(
query=request.query,
project_ids=project_ids,
include_shared=request.scope.include_shared,
node_types=tuple(item.value for item in request.node_types),
tags=request.tags,
audience=request.audience,
use_for=request.use_for,
domain=request.domain,
confidences=request.confidences,
as_of_from=request.as_of_from,
as_of_to=request.as_of_to,
stale_only=request.stale_only,
order=request.order,
limit=request.limit,
offset=offset,
)
if expected_revision is not None and expected_revision != result.index_revision:
raise SearchError(409, "Browse cursor expired because the index revision changed")
source_count = self.index.count_sources(project_ids=project_ids, include_shared=False)
shared_source_count = (
self.index.count_sources(project_ids=(), include_shared=True)
if request.scope.include_shared
else 0
)
page = result.nodes
next_offset = offset + len(page)
next_cursor = (
self._encode_cursor(next_offset, fingerprint, result.index_revision)
if next_offset < result.filtered_node_count
else None
)
canonical_revision = self.canonical_head()
warnings: tuple[Finding, ...] = ()
if result.index_revision != canonical_revision:
warnings = (Finding(code="INDEX_LAG", message="Browse data reflects an older accepted revision."),)
return KnowledgeBrowseEnvelope(
request_id="browse-" + secrets.token_hex(12),
resolved_scope=Scope(projects=project_ids, include_shared=request.scope.include_shared),
nodes=page,
project_node_count=result.project_node_count,
project_source_count=source_count,
shared_node_count=result.shared_node_count,
shared_source_count=shared_source_count,
filtered_node_count=result.filtered_node_count,
facets=result.facets,
canonical_revision=canonical_revision,
index_revision=result.index_revision,
warnings=warnings,
next_cursor=next_cursor,
)
def source_nodes(
self,
*,
project_id: str,
revision_id: str,
principal: Principal,
include_shared: bool = False,
query: str = "",
node_type: str | None = None,
limit: int = 20,
cursor: str | None = None,
) -> SourceRelatedNodesEnvelope:
if not principal.can_read_project(project_id):
raise SearchError(403, "Credential cannot read this project")
fingerprint = hashlib.sha256(canonical_json({
"project": project_id,
"revision": revision_id,
"shared": include_shared,
"query": query,
"node_type": node_type,
"limit": limit,
})).hexdigest()
offset, expected_revision = self._cursor_offset(cursor, fingerprint, max_offset=100_000)
lexical = self.index.lexical if isinstance(self.index, HybridSearchIndex) else self.index
result = lexical.source_nodes(
source_revision_id=revision_id,
project_ids=(project_id,),
include_shared=include_shared,
query=query,
node_type=node_type,
limit=limit,
offset=offset,
)
if result is None:
raise SearchError(404, "Source revision was not found")
if expected_revision is not None and expected_revision != result.index_revision:
raise SearchError(409, "Cursor expired because the index revision changed")
next_offset = offset + len(result.nodes)
next_cursor = (
self._encode_cursor(next_offset, fingerprint, result.index_revision)
if next_offset < result.total
else None
)
canonical_revision = self.canonical_head()
warnings: tuple[Finding, ...] = ()
if result.index_revision != canonical_revision:
warnings = (Finding(code="INDEX_LAG", message="Source nodes reflect an older accepted revision."),)
return SourceRelatedNodesEnvelope(
request_id="srcnodes-" + secrets.token_hex(12),
resolved_scope=Scope(projects=(project_id,), include_shared=include_shared),
source_revision_id=revision_id,
nodes=tuple(
SourceRelatedNode(node=hit.node, relation_basis=hit.relation_basis)
for hit in result.nodes
),
total=result.total,
canonical_revision=canonical_revision,
index_revision=result.index_revision,
warnings=warnings,
next_cursor=next_cursor,
)
def search(self, request: SearchRequest, *, principal: Principal) -> ReadEnvelope:
project_ids = self._resolve_scope(request.scope, principal)
fingerprint = hashlib.sha256(
@@ -638,6 +858,7 @@ class CanonicalSearchService:
"as_of_to": request.as_of_to.isoformat() if request.as_of_to else None,
"stale_only": request.stale_only,
"limit": request.limit,
"include_evidence": request.include_evidence,
"retrieval_profile": (
self.index.cursor_profile()
if isinstance(self.index, HybridSearchIndex)
@@ -674,9 +895,25 @@ class CanonicalSearchService:
rows: list[dict[str, object]] = []
for hit in result.nodes:
node = hit.node
node_payload = node.model_dump(mode="json")
if not request.include_evidence:
# Search returns claim text and result-level citations. Full
# claim-to-citation records remain available through get_node.
for claim_group in ("claims", "assumptions", "limitations"):
compact_claims = []
for claim in node_payload.get(claim_group, []):
compact_claims.append(
{
**claim,
"evidence_count": len(claim.get("evidence", [])),
}
)
compact_claims[-1].pop("evidence", None)
node_payload[claim_group] = compact_claims
row: dict[str, object] = {
"node": node.model_dump(mode="json"),
"node": node_payload,
"retrieval_score": hit.score,
"evidence_included": request.include_evidence,
}
if hit.expanded_from is not None and hit.expansion_edge is not None:
row["retrieval_path"] = "graph_expansion"
@@ -692,14 +929,15 @@ class CanonicalSearchService:
)
)
node_evidence: dict[str, EvidenceReference] = {}
for claim in node.evidence_claims:
for item in claim.evidence:
key = evidence_digest(item)
node_evidence[key] = item
if hit.expansion_edge is not None:
for item in hit.expansion_edge.evidence:
key = evidence_digest(item)
node_evidence[key] = item
if request.include_evidence:
for claim in node.evidence_claims:
for item in claim.evidence:
key = evidence_digest(item)
node_evidence[key] = item
if hit.expansion_edge is not None:
for item in hit.expansion_edge.evidence:
key = evidence_digest(item)
node_evidence[key] = item
evidence_by_node.append(node_evidence)
if result.index_revision != canonical_revision:
warnings.append(
@@ -734,6 +972,56 @@ class CanonicalSearchService:
),
)
def graph(self, request: KnowledgeGraphRequest, *, principal: Principal) -> KnowledgeGraphEnvelope:
project_ids = self._resolve_scope(request.scope, principal)
if request.center_node_id is None:
result = self.index.graph(
project_ids=project_ids,
include_shared=request.scope.include_shared,
max_nodes=request.max_nodes,
max_edges=request.max_edges,
)
else:
scoped = self.index.search(
query="", project_ids=project_ids, include_shared=request.scope.include_shared,
limit=1, candidate_node_ids=(request.center_node_id,),
)
origin = scoped.nodes[0].node if scoped.nodes else None
if origin is None:
raise SearchError(404, "Knowledge node was not found within the requested scope")
nodes_by_id = {origin.node_id: origin}
edges: list[TypedEdge] = []
neighbors = None
if request.max_nodes > 1:
neighbors = self.index.neighbors(
node_id=origin.node_id, edge_types=(), project_ids=project_ids,
include_shared=request.scope.include_shared, limit=min(50, request.max_nodes - 1),
)
for hit in neighbors.edges:
endpoint = hit.neighbor_endpoint
if endpoint.kind != "node" or hit.neighbor_node is None:
continue
nodes_by_id[hit.neighbor_node.node_id] = hit.neighbor_node
edges.append(hit.edge)
result = GraphIndexResult(
index_revision=neighbors.index_revision if neighbors else scoped.index_revision,
nodes=tuple(nodes_by_id.values()),
edges=tuple(edges[:request.max_edges]),
truncated=(neighbors.has_more if neighbors else False) or len(edges) > request.max_edges,
)
canonical_revision = self.canonical_head()
warnings: tuple[Finding, ...] = ()
if result.index_revision != canonical_revision:
warnings = (Finding(code="INDEX_LAG", message="Graph data reflects an older accepted revision."),)
return KnowledgeGraphEnvelope(
request_id="graph-" + secrets.token_hex(12),
resolved_scope=Scope(projects=project_ids, include_shared=request.scope.include_shared),
nodes=result.nodes, edges=result.edges, truncated=result.truncated,
center_node_id=request.center_node_id,
canonical_revision=canonical_revision, index_revision=result.index_revision,
warnings=warnings,
)
@staticmethod
def _encode_cursor(offset: int, fingerprint: str, index_revision: str) -> str:
payload = canonical_json(
@@ -956,6 +1244,10 @@ class CanonicalSearchService:
rows: list[dict[str, object]] = [
{"source": source.model_dump(mode="json")} for source in result.sources
]
source_count = self.index.count_sources(
project_ids=project_ids,
include_shared=request.scope.include_shared,
)
warnings: list[Finding] = []
if result.index_revision != canonical_revision:
warnings.append(
@@ -977,6 +1269,7 @@ class CanonicalSearchService:
index_revision=result.index_revision,
budget_tokens=request.budget_tokens,
retrieval_mode="keyword",
filtered_count=source_count,
next_cursor_for=(
lambda included: (
self._encode_cursor(offset + included, fingerprint, result.index_revision)
+113 -4
View File
@@ -19,6 +19,7 @@ from gewu.adapters.source_extractor import LocalSourceExtractor, SourceExtractor
from gewu.adapters.sqlite_facet_labels import SQLiteFacetLabelIndex
from gewu.adapters.sqlite_identity import SQLiteIdentityStore
from gewu.adapters.sqlite_ingest_reviews import SQLiteIngestReviewStore
from gewu.adapters.sqlite_node_removals import SQLiteNodeRemovalStore
from gewu.adapters.sqlite_search import SQLiteSearchIndex
from gewu.adapters.sqlite_vectors import SQLiteVectorIndex
from gewu.adapters.url_fetch import PublicUrlFetcher
@@ -47,6 +48,9 @@ from gewu.application.search import (
from gewu.application.uploads import LocalArtifactStore, UploadError
from gewu.domain.contracts import (
GitCommitId,
KnowledgeBrowseEnvelope,
KnowledgeGraphEnvelope,
SourceRelatedNodesEnvelope,
ProjectRecord,
ReadEnvelope,
RecordId,
@@ -56,6 +60,8 @@ from gewu.transports.accounts_http import register_account_routes
from gewu.transports.admin_http import register_admin_routes
from gewu.transports.http_middleware import ProjectRegistryBodyLimitMiddleware
from gewu.transports.http_models import (
KnowledgeBrowseRequestBody,
KnowledgeGraphRequestBody,
NeighborRequestBody,
OwnerSessionRequestBody,
ProjectContentUpdateBody,
@@ -65,6 +71,7 @@ from gewu.transports.http_models import (
)
from gewu.transports.ingest_reviews_http import register_ingest_review_routes
from gewu.transports.mcp_server import create_mcp_server
from gewu.transports.node_removals_http import register_node_removal_routes
from gewu.transports.owner_mcp_server import OwnerReviewProvider, create_owner_mcp_server
from gewu.version import SERVICE_VERSION
@@ -84,6 +91,7 @@ def create_app(
credential_administration: CredentialAdministration | None = None,
account_identity: AccountIdentity | None = None,
ingest_review_store: SQLiteIngestReviewStore | None = None,
node_removal_store: SQLiteNodeRemovalStore | None = None,
jev_assessor: OpenRouterJev | None = None,
facet_label_index: SQLiteFacetLabelIndex | None = None,
embedding_provider: EmbeddingProvider | None = None,
@@ -103,6 +111,7 @@ def create_app(
or credential_administration is not None
or account_identity is not None
or ingest_review_store is not None
or node_removal_store is not None
):
app.add_middleware(ProjectRegistryBodyLimitMiddleware)
@@ -163,6 +172,13 @@ def create_app(
app, ingest_review_store, authenticate, require_same_origin
)
if node_removal_store is not None:
if node_reader is None:
raise ValueError("Node removals require a canonical node reader")
register_node_removal_routes(
app, node_removal_store, node_reader, authenticate, require_same_origin
)
if owner_identity_provider is not None:
@app.post(
@@ -351,6 +367,11 @@ def create_app(
}
)
if node_removal_store is not None:
http_operations.update(
{"node_removal_list", "node_removal_request", "node_removal_restore"}
)
if project_registry is not None:
@app.get(
@@ -717,7 +738,17 @@ def create_app(
}
if search_service is not None:
http_operations.update({"list_projects", "search", "neighbors", "list_sources"})
http_operations.update(
{
"list_projects",
"search",
"neighbors",
"list_sources",
"knowledge_browse",
"knowledge_graph",
"get_source_nodes",
}
)
@app.get("/v1/projects", tags=["knowledge"], operation_id="list_projects")
def list_projects(
@@ -747,6 +778,78 @@ def create_app(
except SearchError as error:
raise HTTPException(status_code=error.status_code, detail=str(error)) from error
@app.post(
"/v1/knowledge/browse",
response_model=KnowledgeBrowseEnvelope,
tags=["knowledge"],
operation_id="knowledge_browse",
)
def knowledge_browse(
request: KnowledgeBrowseRequestBody,
authorization: str | None = Header(default=None),
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
csrf_token: str | None = Header(default=None, alias="X-GEWU-CSRF"),
) -> KnowledgeBrowseEnvelope:
principal = authenticate(
authorization, session_cookie, csrf_token, require_csrf=True
)
try:
return search_service.browse(request.to_contract(), principal=principal)
except SearchError as error:
raise HTTPException(status_code=error.status_code, detail=str(error)) from error
@app.post(
"/v1/knowledge/graph",
response_model=KnowledgeGraphEnvelope,
tags=["knowledge"],
operation_id="knowledge_graph",
)
def knowledge_graph(
request: KnowledgeGraphRequestBody,
authorization: str | None = Header(default=None),
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
csrf_token: str | None = Header(default=None, alias="X-GEWU-CSRF"),
) -> KnowledgeGraphEnvelope:
principal = authenticate(
authorization, session_cookie, csrf_token, require_csrf=True
)
try:
return search_service.graph(request.to_contract(), principal=principal)
except SearchError as error:
raise HTTPException(status_code=error.status_code, detail=str(error)) from error
@app.get(
"/v1/projects/{project_id}/sources/{revision_id}/nodes",
response_model=SourceRelatedNodesEnvelope,
tags=["knowledge"],
operation_id="get_source_nodes",
)
def get_source_nodes(
project_id: RecordId,
revision_id: RecordId,
query: str = "",
node_type: str | None = None,
include_shared: bool = False,
limit: int = 20,
cursor: str | None = None,
authorization: str | None = Header(default=None),
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
) -> SourceRelatedNodesEnvelope:
principal = authenticate(authorization, session_cookie)
try:
return search_service.source_nodes(
project_id=project_id,
revision_id=revision_id,
principal=principal,
include_shared=include_shared,
query=query,
node_type=node_type,
limit=limit,
cursor=cursor,
)
except SearchError as error:
raise HTTPException(status_code=error.status_code, detail=str(error)) from error
@app.post(
"/v1/neighbors",
response_model=ReadEnvelope,
@@ -891,6 +994,7 @@ def create_local_app() -> FastAPI:
)
artifacts = LocalArtifactStore(data_root)
ingest_reviews = SQLiteIngestReviewStore(data_root / "ingest-reviews.sqlite3")
node_removals = SQLiteNodeRemovalStore(data_root / "node-removals.sqlite3")
allowed_fetch_hosts = {
host.strip()
for host in os.environ.get("GEWU_FETCH_ALLOWED_HOSTS", "").split(",")
@@ -929,7 +1033,9 @@ def create_local_app() -> FastAPI:
embedding_provider = embeddings
facet_label_index = SQLiteFacetLabelIndex(search_path)
vectors = SQLiteVectorIndex(search_path)
lexical_index = SQLiteSearchIndex(search_path, repository)
lexical_index = SQLiteSearchIndex(
search_path, repository, node_removals_path=node_removals.path
)
hybrid_index = HybridSearchIndex(
lexical=lexical_index,
vectors=vectors,
@@ -942,7 +1048,9 @@ def create_local_app() -> FastAPI:
)
search_index = hybrid_index
else:
search_index = SQLiteSearchIndex(search_path, repository)
search_index = SQLiteSearchIndex(
search_path, repository, node_removals_path=node_removals.path
)
search_service = CanonicalSearchService(
index=search_index,
project_ids_provider=project_registry,
@@ -968,6 +1076,7 @@ def create_local_app() -> FastAPI:
credential_administration=identities,
account_identity=identities,
ingest_review_store=ingest_reviews,
node_removal_store=node_removals,
jev_assessor=(
OpenRouterJev(
os.environ["OPENROUTER_API_KEY"],
@@ -978,7 +1087,7 @@ def create_local_app() -> FastAPI:
),
owner_identity_provider=identities,
owner_service_audience=os.environ.get("GEWU_OWNER_SERVICE_AUDIENCE"),
node_reader=CanonicalNodeReader(repository),
node_reader=CanonicalNodeReader(repository, node_removals.is_hidden),
artifact_store=artifacts,
extractor=LocalSourceExtractor(),
acceptance_service=acceptance,
+143 -2
View File
@@ -2,11 +2,13 @@
from __future__ import annotations
import ipaddress
import re
import unicodedata
from datetime import date, datetime
from enum import StrEnum
from typing import Annotated, Literal
from urllib.parse import urlsplit
from pydantic import (
BaseModel,
@@ -475,12 +477,38 @@ class ExtractionRecord(StrictRecord):
warnings: tuple[str, ...] = ()
class OnlineEvidence(StrictRecord):
"""A short online excerpt with its relationship to a claim and verification URL."""
url: str = Field(min_length=1, max_length=4000)
text: str = Field(min_length=1, max_length=8000)
relation: Literal["supports", "contradicts", "context", "inconclusive"]
context: str = Field(min_length=1, max_length=4000)
@field_validator("url")
@classmethod
def public_http_url(cls, value: str) -> str:
parsed = urlsplit(value)
hostname = parsed.hostname
if parsed.scheme not in {"http", "https"} or not hostname or parsed.username or parsed.password:
raise ValueError("online evidence requires an HTTP(S) URL without embedded credentials")
if hostname.lower() == "localhost" or hostname.lower().endswith((".localhost", ".local")):
raise ValueError("web references must use a public hostname")
try:
address = ipaddress.ip_address(hostname)
except ValueError:
address = None
if address is not None and not address.is_global:
raise ValueError("web references must use a public hostname")
return value
class Claim(StrictRecord):
claim_id: RecordId
text: str = Field(min_length=1, max_length=4000)
evidence_type: Literal["measured", "reported", "opinion", "vendor_marketing"]
evidence: tuple[EvidenceReference, ...] = Field(min_length=1, max_length=20)
assessment: SupportAssessment
online_evidence: tuple[OnlineEvidence, ...] = Field(default=(), max_length=20)
@model_validator(mode="after")
def assessment_matches_evidence(self) -> Claim:
@@ -660,7 +688,9 @@ class TypedEdge(StrictRecord):
class ProposalOperation(StrictRecord):
operation: Literal["register_source", "upsert_node", "add_edge", "flag_stale"]
operation: Literal[
"register_source", "upsert_node", "update_node", "add_edge", "flag_stale"
]
source: SourceRevisionRecord | None = None
node: KnowledgeNode | None = None
edge: TypedEdge | None = None
@@ -671,6 +701,7 @@ class ProposalOperation(StrictRecord):
payloads = {
"register_source": self.source,
"upsert_node": self.node,
"update_node": self.node,
"add_edge": self.edge,
}
expected = payloads.get(self.operation)
@@ -682,8 +713,11 @@ class ProposalOperation(StrictRecord):
)
elif supplied != 1 or expected is None:
raise ValueError("operation must carry exactly its matching typed payload")
elif self.operation == "update_node":
if self.target_id is None or self.node is None or self.target_id != self.node.node_id:
raise ValueError("update_node target_id must match the stable node ID")
elif self.target_id is not None:
raise ValueError("target_id is only valid for flag_stale")
raise ValueError("target_id is only valid for update_node or flag_stale")
return self
@@ -742,6 +776,7 @@ class SearchRequest(StrictRecord):
limit: int = Field(default=20, ge=1, le=50)
cursor: str | None = Field(default=None, max_length=512)
budget_tokens: int = Field(ge=256, le=50_000)
include_evidence: bool = True
@field_validator("query")
@classmethod
@@ -781,6 +816,111 @@ class SearchRequest(StrictRecord):
return value
class KnowledgeBrowseRequest(StrictRecord):
"""Deterministic, paged project browsing independent of retrieval search."""
scope: Scope
query: str = Field(default="", max_length=1000)
node_types: tuple[NodeType, ...] = Field(default=(), max_length=9)
tags: tuple[SearchFacetValue, ...] = Field(default=(), max_length=20)
audience: tuple[SearchFacetValue, ...] = Field(default=(), max_length=20)
use_for: tuple[SearchFacetValue, ...] = Field(default=(), max_length=20)
domain: tuple[SearchFacetValue, ...] = Field(default=(), max_length=20)
confidences: tuple[Confidence, ...] = Field(default=(), max_length=3)
as_of_from: date | None = None
as_of_to: date | None = None
stale_only: bool = False
order: Literal["title_asc", "title_desc", "date_newest", "date_oldest", "relevance"] = "title_asc"
limit: int = Field(default=20, ge=1, le=50)
cursor: str | None = Field(default=None, max_length=512)
budget_tokens: int = Field(ge=256, le=50_000)
@model_validator(mode="after")
def validate_browse(self) -> KnowledgeBrowseRequest:
if self.as_of_from is not None and self.as_of_to is not None and self.as_of_from > self.as_of_to:
raise ValueError("as_of_from must not be later than as_of_to")
if self.order == "relevance" and not self.query:
raise ValueError("relevance ordering requires a search query")
return self
@field_validator("node_types")
@classmethod
def browse_node_types_are_unique(cls, value: tuple[NodeType, ...]) -> tuple[NodeType, ...]:
if len(value) != len(set(value)):
raise ValueError("browse node types must be unique")
return value
@field_validator("confidences")
@classmethod
def browse_confidences_are_unique(cls, value: tuple[Confidence, ...]) -> tuple[Confidence, ...]:
if len(value) != len(set(value)):
raise ValueError("browse confidences must be unique")
return value
@field_validator("tags", "audience", "use_for", "domain")
@classmethod
def browse_facets_are_unique_and_trimmed(cls, value: tuple[str, ...]) -> tuple[str, ...]:
if any(item != item.strip() for item in value) or len(value) != len(set(value)):
raise ValueError("browse facet values must be unique and trimmed")
return value
class KnowledgeBrowseEnvelope(StrictRecord):
"""A bounded browse page with scope-safe project metadata."""
api_version: Literal["v1"] = "v1"
request_id: RecordId
resolved_scope: Scope
nodes: tuple[KnowledgeNode, ...]
project_node_count: int = Field(ge=0)
project_source_count: int = Field(ge=0)
shared_node_count: int = Field(ge=0)
shared_source_count: int = Field(ge=0)
filtered_node_count: int = Field(ge=0)
facets: dict[str, dict[str, int]]
canonical_revision: GitCommitId | None
index_revision: GitCommitId | None
warnings: tuple[Finding, ...] = ()
next_cursor: str | None = None
class KnowledgeGraphRequest(StrictRecord):
scope: Scope
max_nodes: int = Field(default=200, ge=1, le=200)
max_edges: int = Field(default=500, ge=1, le=500)
center_node_id: RecordId | None = None
class KnowledgeGraphEnvelope(StrictRecord):
api_version: Literal["v1"] = "v1"
request_id: RecordId
resolved_scope: Scope
nodes: tuple[KnowledgeNode, ...]
edges: tuple[TypedEdge, ...]
truncated: bool
center_node_id: RecordId | None = None
canonical_revision: GitCommitId | None
index_revision: GitCommitId | None
warnings: tuple[Finding, ...] = ()
class SourceRelatedNode(StrictRecord):
node: KnowledgeNode
relation_basis: Literal["citation", "edge", "both"]
class SourceRelatedNodesEnvelope(StrictRecord):
api_version: Literal["v1"] = "v1"
request_id: RecordId
resolved_scope: Scope
source_revision_id: RecordId
nodes: tuple[SourceRelatedNode, ...]
total: int = Field(ge=0)
canonical_revision: GitCommitId | None
index_revision: GitCommitId | None
warnings: tuple[Finding, ...] = ()
next_cursor: str | None = None
class NeighborRequest(StrictRecord):
node_id: RecordId
scope: Scope
@@ -1114,6 +1254,7 @@ class ReadEnvelope(StrictRecord):
request_id: RecordId
resolved_scope: Scope
data: tuple[dict[str, object], ...]
filtered_count: int | None = Field(default=None, ge=0)
citations: tuple[EvidenceReference, ...] = ()
warnings: tuple[Finding, ...] = ()
omissions: tuple[Finding, ...] = ()
+6
View File
@@ -52,13 +52,19 @@ OPERATIONS: Final[tuple[Operation, ...]] = (
Operation("get_project_context", "knowledge:read", frozenset({Transport.MCP})),
Operation("ingest_review_list", "ingest:review", frozenset({Transport.HTTP})),
Operation("ingest_review_decide", "ingest:review", frozenset({Transport.HTTP})),
Operation("node_removal_list", "knowledge:read", frozenset({Transport.HTTP})),
Operation("node_removal_request", "knowledge:moderate", frozenset({Transport.HTTP})),
Operation("node_removal_restore", "knowledge:moderate", frozenset({Transport.HTTP})),
Operation("search", "knowledge:read", frozenset({Transport.HTTP, Transport.MCP})),
Operation("knowledge_browse", "knowledge:read", frozenset({Transport.HTTP})),
Operation("knowledge_graph", "knowledge:read", frozenset({Transport.HTTP})),
Operation("list_sources", "knowledge:read", frozenset({Transport.HTTP, Transport.MCP})),
Operation("get_node", "knowledge:read", frozenset({Transport.HTTP, Transport.MCP})),
Operation("node_history", "knowledge:read", frozenset({Transport.HTTP, Transport.MCP})),
Operation("get_source", "knowledge:read", frozenset({Transport.HTTP, Transport.MCP})),
Operation("get_source_artifact", "knowledge:read", frozenset({Transport.HTTP})),
Operation("get_source_location", "knowledge:read", frozenset({Transport.HTTP})),
Operation("get_source_nodes", "knowledge:read", frozenset({Transport.HTTP})),
Operation("neighbors", "knowledge:read", frozenset({Transport.HTTP, Transport.MCP})),
Operation("source_capture_url", "proposal:submit", frozenset({Transport.MCP})),
Operation("source_capture_text", "proposal:submit", frozenset({Transport.MCP})),
+145 -6
View File
@@ -4,6 +4,7 @@ from __future__ import annotations
import hashlib
import json
import re
import unicodedata
from collections.abc import Iterable
from copy import deepcopy
@@ -51,6 +52,12 @@ def evidence_digest(evidence: EvidenceReference) -> str:
return hashlib.sha256(canonical_json(evidence.model_dump(mode="json"))).hexdigest()
def _numeric_tokens(text: str) -> set[str]:
"""Return normalized numeric literals for a conservative evidence check."""
matches = re.findall(r"(?<!\w)\d+(?:,\d{3})*(?:\.\d+)?", text)
return {match.replace(",", "") for match in matches}
def extraction_digest(extraction: ExtractionRecord) -> str:
value = extraction.model_dump(mode="json", exclude={"extraction_id"})
return hashlib.sha256(canonical_json(value)).hexdigest()
@@ -58,6 +65,18 @@ def extraction_digest(extraction: ExtractionRecord) -> str:
def proposal_payload_digest(proposal: ProposalRevision) -> str:
operations = [item.model_dump(mode="json") for item in proposal.operations]
def omit_empty_online_evidence(value: Any) -> None:
if isinstance(value, dict):
if value.get("online_evidence") == []:
value.pop("online_evidence")
for child in value.values():
omit_empty_online_evidence(child)
elif isinstance(value, list):
for child in value:
omit_empty_online_evidence(child)
omit_empty_online_evidence(operations)
return hashlib.sha256(canonical_json(operations)).hexdigest()
@@ -112,7 +131,12 @@ def prepare_proposal_digests(draft: dict[str, Any]) -> ProposalRevision:
raise ValueError("Evidence references must be objects")
item["assertion_path"] = "edge"
item["assertion_digest"] = "0" * 64
parsed_edge = TypedEdge.model_validate(edge, strict=True)
# The draft originated as JSON, where arrays represent tuple fields.
# Revalidate through the JSON path so strict mode accepts the typed
# edge evidence array without weakening Python-object validation.
parsed_edge = TypedEdge.model_validate_json(
json.dumps(edge, ensure_ascii=False, allow_nan=False), strict=True
)
digest = edge_assertion_digest(parsed_edge)
for item in edge_evidence:
item["assertion_digest"] = digest
@@ -176,6 +200,71 @@ def _entity_names(node: KnowledgeNode) -> set[str]:
}
def _validate_node_enrichment(
previous: KnowledgeNode,
updated: KnowledgeNode,
proposer_id: str,
path: str,
) -> tuple[Finding, ...]:
"""Require an update to preserve accepted knowledge and add only assessed material."""
findings: list[Finding] = []
for field in ("node_id", "project_id", "node_type", "created", "lifecycle"):
if getattr(previous, field) != getattr(updated, field):
findings.append(
_finding("NODE_ENRICHMENT_IDENTITY_CHANGED", f"Enrichment cannot change {field}", path)
)
if updated.updated <= previous.updated:
findings.append(
_finding("NODE_ENRICHMENT_TIMESTAMP_INVALID", "Enrichment must advance updated", path)
)
if updated.summary != previous.summary:
findings.append(
_finding(
"NODE_ENRICHMENT_SUMMARY_CHANGED",
"Enrichment cannot replace the summary; add supported claims instead",
f"{path}.summary",
)
)
for field in ("claims", "assumptions", "limitations"):
old_items = {claim.claim_id: claim for claim in getattr(previous, field)}
new_items = {claim.claim_id: claim for claim in getattr(updated, field)}
if not old_items.keys() <= new_items.keys():
findings.append(
_finding("NODE_ENRICHMENT_REMOVED_CLAIM", "Enrichment cannot remove accepted claims", f"{path}.{field}")
)
continue
for claim_id, old_claim in old_items.items():
new_claim = new_items[claim_id]
claim_path = f"{path}.{field}.{claim_id}"
if (
old_claim.text != new_claim.text
or old_claim.evidence_type != new_claim.evidence_type
):
findings.append(
_finding("NODE_ENRICHMENT_CHANGED_CLAIM", "Enrichment cannot alter an accepted assertion", claim_path)
)
continue
if not set(old_claim.online_evidence) <= set(new_claim.online_evidence):
findings.append(
_finding("NODE_ENRICHMENT_REMOVED_ONLINE_EVIDENCE", "Enrichment cannot remove prior online evidence", claim_path)
)
old_evidence = set(old_claim.evidence)
if not old_evidence <= set(new_claim.evidence):
findings.append(
_finding("NODE_ENRICHMENT_REMOVED_EVIDENCE", "Enrichment cannot remove accepted evidence", claim_path)
)
if new_claim.evidence != old_claim.evidence and new_claim.assessment.assessor_id != proposer_id:
findings.append(
_finding("ASSESSOR_ATTRIBUTION_MISMATCH", "New evidence requires an assessment by its proposer", claim_path)
)
if not set(previous.tags) <= set(updated.tags):
findings.append(_finding("NODE_ENRICHMENT_REMOVED_FACET", "Enrichment cannot remove existing tags", f"{path}.tags"))
for field in ("audience", "domain", "use_for"):
if not set(getattr(previous, field)) <= set(getattr(updated, field)):
findings.append(_finding("NODE_ENRICHMENT_REMOVED_FACET", "Enrichment cannot remove existing facets", f"{path}.{field}"))
return tuple(findings)
def _entity_visible(
candidate: KnowledgeNode,
existing: KnowledgeNode,
@@ -220,7 +309,7 @@ def validate_proposal(
proposed_nodes = {
operation.node.node_id: operation.node
for operation in proposal.operations
if operation.node is not None
if operation.node is not None and operation.operation == "upsert_node"
}
known_nodes = {**existing_nodes, **proposed_nodes}
entity_nodes_seen: list[KnowledgeNode] = []
@@ -242,7 +331,10 @@ def validate_proposal(
)
)
known_sources[source.revision_id] = source
source_domains = {"shared", *source.ingested_for}
# Source relevance is explicit: a source ingested for one project
# does not become shared knowledge merely because its record is
# stored in the canonical source registry.
source_domains = set(source.ingested_for)
for project_id in source.ingested_for:
if project_id != "shared" and project_id not in projects:
findings.append(
@@ -269,8 +361,13 @@ def validate_proposal(
if operation.node is not None:
node = operation.node
previous_node = (
existing_nodes.get(operation.target_id or "")
if operation.operation == "update_node"
else None
)
eligibility_checks.append(((node.project_id,), "node", node.node_type.value))
if node.node_id in existing_nodes:
if operation.operation == "upsert_node" and node.node_id in existing_nodes:
findings.append(
_finding(
"NODE_ID_EXISTS",
@@ -278,6 +375,15 @@ def validate_proposal(
f"{path}.node.node_id",
)
)
elif operation.operation == "update_node":
if previous_node is None:
findings.append(
_finding("NODE_NOT_FOUND", "Update target is not an accepted node", f"{path}.target_id")
)
else:
findings.extend(
_validate_node_enrichment(previous_node, node, proposal.proposer_id, f"{path}.node")
)
if node.node_type.value == "entity":
possible_duplicates = [
other
@@ -325,7 +431,20 @@ def validate_proposal(
)
for group_name, claim_index, claim in claim_items:
claim_path = f"{path}.node.{group_name}[{claim_index}]"
if claim.assessment.assessor_id != proposal.proposer_id:
previous_claims = (
{item.claim_id: item for item in previous_node.evidence_claims}
if previous_node is not None
else {}
)
old_claim = previous_claims.get(claim.claim_id)
unchanged_prior_assessment = bool(
old_claim
and old_claim.text == claim.text
and old_claim.evidence_type == claim.evidence_type
and old_claim.evidence == claim.evidence
and old_claim.assessment == claim.assessment
)
if claim.assessment.assessor_id != proposal.proposer_id and not unchanged_prior_assessment:
findings.append(
_finding(
"ASSESSOR_ATTRIBUTION_MISMATCH",
@@ -362,6 +481,17 @@ def validate_proposal(
claim_path,
)
)
quoted_text = "\n".join(evidence.quote for evidence in claim.evidence)
unquoted_numbers = _numeric_tokens(claim.text) - _numeric_tokens(quoted_text)
if unquoted_numbers:
findings.append(
_finding(
"NUMERIC_CLAIM_UNGROUNDED",
"Claim contains numeric literals absent from attached evidence; "
"semantic review is required",
claim_path,
)
)
for evidence_index, evidence in enumerate(claim.evidence):
evidence_path = f"{claim_path}.evidence[{evidence_index}]"
expected_assertion_path = f"{group_name}[{claim_index}].text"
@@ -636,7 +766,7 @@ def validate_proposal(
)
)
if operation.target_id is not None:
if operation.operation == "flag_stale" and operation.target_id is not None:
target = existing_nodes.get(operation.target_id)
if target is None:
findings.append(
@@ -720,6 +850,15 @@ def validate_proposal(
"SOURCE_REVISION_CONFLICT",
"PROJECT_NOT_REGISTERED",
"NODE_ID_EXISTS",
"NODE_NOT_FOUND",
"NODE_ENRICHMENT_IDENTITY_CHANGED",
"NODE_ENRICHMENT_TIMESTAMP_INVALID",
"NODE_ENRICHMENT_SUMMARY_CHANGED",
"NODE_ENRICHMENT_REMOVED_CLAIM",
"NODE_ENRICHMENT_CHANGED_CLAIM",
"NODE_ENRICHMENT_REMOVED_ONLINE_EVIDENCE",
"NODE_ENRICHMENT_REMOVED_EVIDENCE",
"NODE_ENRICHMENT_REMOVED_FACET",
"ASSESSMENT_STALE",
"ASSESSMENT_EVIDENCE_MISMATCH",
"SUPPORT_UNAVAILABLE",
+54
View File
@@ -11,6 +11,8 @@ from gewu.domain.contracts import (
AutoAcceptPolicy,
Confidence,
NeighborRequest,
KnowledgeBrowseRequest,
KnowledgeGraphRequest,
NodeType,
ProjectRecord,
RecordId,
@@ -46,6 +48,7 @@ class SearchRequestBody(BaseModel):
limit: int = Field(default=20, ge=1, le=50)
cursor: str | None = Field(default=None, max_length=512)
budget_tokens: int = Field(ge=256, le=50_000)
include_evidence: bool = True
@field_validator("query")
@classmethod
@@ -95,6 +98,57 @@ class SearchRequestBody(BaseModel):
return SearchRequest.model_validate_json(self.model_dump_json(), strict=True)
class KnowledgeBrowseRequestBody(BaseModel):
model_config = ConfigDict(extra="forbid", strict=True, frozen=True)
query: str = Field(default="", max_length=1000)
scope: SearchScopeBody
node_types: list[str] = Field(default_factory=list, max_length=9)
tags: list[SearchFacetValue] = Field(default_factory=list, max_length=20)
audience: list[SearchFacetValue] = Field(default_factory=list, max_length=20)
use_for: list[SearchFacetValue] = Field(default_factory=list, max_length=20)
domain: list[SearchFacetValue] = Field(default_factory=list, max_length=20)
confidences: list[Confidence] = Field(default_factory=list, max_length=3)
as_of_from: date | None = None
as_of_to: date | None = None
stale_only: bool = False
order: str = "title_asc"
limit: int = Field(default=20, ge=1, le=50)
cursor: str | None = Field(default=None, max_length=512)
budget_tokens: int = Field(default=12000, ge=256, le=50_000)
@field_validator("query")
@classmethod
def normalize_query(cls, value: str) -> str:
return value.strip()
@field_validator("node_types")
@classmethod
def validate_node_types(cls, values: list[str]) -> list[str]:
try:
normalized = [NodeType(value).value for value in values]
except ValueError as error:
raise ValueError("node_types contains an unsupported knowledge type") from error
if len(normalized) != len(set(normalized)):
raise ValueError("node_types must be unique")
return normalized
def to_contract(self) -> KnowledgeBrowseRequest:
return KnowledgeBrowseRequest.model_validate_json(self.model_dump_json(), strict=True)
class KnowledgeGraphRequestBody(BaseModel):
model_config = ConfigDict(extra="forbid", strict=True, frozen=True)
scope: SearchScopeBody
max_nodes: int = Field(default=200, ge=1, le=200)
max_edges: int = Field(default=500, ge=1, le=500)
center_node_id: str | None = None
def to_contract(self) -> KnowledgeGraphRequest:
return KnowledgeGraphRequest.model_validate_json(self.model_dump_json(), strict=True)
class NeighborRequestBody(BaseModel):
model_config = ConfigDict(extra="forbid", strict=True, frozen=True)
+35 -14
View File
@@ -372,9 +372,7 @@ def _validation_issues(error: ValidationError) -> list[dict[str, str]]:
)[:64]:
path = ""
for part in detail.get("loc", ()):
path += f"[{part}]" if isinstance(part, int) else (
("." if path else "") + str(part)
)
path += f"[{part}]" if isinstance(part, int) else (("." if path else "") + str(part))
issues.append(
{
"path": path,
@@ -528,8 +526,9 @@ def create_mcp_server(
as_of_from: str | None = None,
as_of_to: str | None = None,
stale_only: bool = False,
include_evidence: bool = True,
) -> str:
"""Search accepted knowledge in an explicit, authorized project scope."""
"""Search accepted knowledge in scope. Set include_evidence=false for compact claim summaries; the default includes full citations."""
principal = _principal(ctx, identity_provider)
try:
@@ -555,6 +554,7 @@ def create_mcp_server(
stale_only=stale_only,
cursor=cursor,
budget_tokens=budget_tokens,
include_evidence=include_evidence,
)
envelope = search_service.search(request, principal=principal)
except SearchError as error:
@@ -1253,7 +1253,7 @@ def create_mcp_server(
ctx: Context,
include_original_mapping: bool = False,
) -> str:
"""Read a bounded normalized text slice and optionally its original-source offset map."""
"""Read a normalized slice with its profile and optional original-source offset map."""
principal = _principal(ctx, identity_provider)
_require_project_submission(principal, project_id)
@@ -1273,6 +1273,7 @@ def create_mcp_server(
"block_id": block_id,
"page": block.page,
"offset": offset,
"normalization_profile": extraction.normalization_profile,
"normalized_text": block.normalized_text[offset:end],
}
if include_original_mapping:
@@ -1495,7 +1496,10 @@ def create_mcp_server(
principal = _principal(ctx, identity_provider)
if "agent" not in principal.roles or principal.auth_method != "bearer":
raise PermissionError("Durable runs require an agent bearer credential")
request = _parse_run_request(request_json, RunCreateRequest)
try:
request = _parse_run_request(request_json, RunCreateRequest)
except ValueError as error:
return _invalid_run_request(error)
_require_run_scope(principal, request.scope)
try:
record = run_service.create(
@@ -1513,7 +1517,10 @@ def create_mcp_server(
principal = _principal(ctx, identity_provider)
if "agent" not in principal.roles or principal.auth_method != "bearer":
raise PermissionError("Durable runs require an agent bearer credential")
request = _parse_run_request(request_json, RunCheckpointRequest)
try:
request = _parse_run_request(request_json, RunCheckpointRequest)
except ValueError as error:
return _invalid_run_request(error)
actor = ActorContext(principal.principal_id, principal.credential_id)
try:
current = run_service.status(request.run_id, actor=actor)
@@ -1746,14 +1753,25 @@ def _parse_run_request[RequestT: (RunCreateRequest, RunCheckpointRequest)](
) -> RequestT:
if len(request_json.encode("utf-8")) > 1_000_000:
raise ValueError("Run request exceeds the supported size")
try:
payload = json.loads(request_json, object_pairs_hook=_reject_duplicate_keys)
request = request_type.model_validate_json(request_json, strict=True)
except (ValueError, ValidationError) as error:
raise ValueError("Run request does not match the GEWU v1 contract") from error
payload = json.loads(request_json, object_pairs_hook=_reject_duplicate_keys)
if not isinstance(payload, dict):
raise ValueError("Run request must be a JSON object")
return request
return request_type.model_validate_json(request_json, strict=True)
def _invalid_run_request(error: ValueError) -> str:
issues = (
_validation_issues(error)
if isinstance(error, ValidationError)
else [
{
"path": "",
"code": "invalid_run_request",
"message": "Run request must be a valid JSON object within the supported size",
}
]
)
return json.dumps({"status": "invalid", "issues": issues}, ensure_ascii=False)
def _require_run_scope(principal: Principal, scope: Scope) -> None:
@@ -1804,7 +1822,10 @@ def _authorize_proposal(
affected: set[str] = set()
for operation in proposal.operations:
if operation.source is not None:
affected.update(("shared", *operation.source.ingested_for))
# A revision is project-visible through its explicit ingestion grants.
# Requiring a shared write grant for every source registration would
# block project-only agents and turn project evidence into shared data.
affected.update(operation.source.ingested_for)
if operation.node is not None:
affected.add(operation.node.project_id)
if operation.edge is not None:
+141
View File
@@ -0,0 +1,141 @@
"""Authenticated, project-scoped node removal and restore routes."""
from collections.abc import Callable
from typing import Annotated, Literal
from fastapi import Cookie, Depends, FastAPI, Header, HTTPException, Request, Response
from pydantic import BaseModel, ConfigDict
from gewu.adapters.sqlite_node_removals import NodeRemoval, RemovalConflict, SQLiteNodeRemovalStore
from gewu.application.identity import Principal
from gewu.application.reading import CanonicalNodeReader, ReadError
from gewu.domain.contracts import RecordId
class NodeRemovalRequest(BaseModel):
model_config = ConfigDict(extra="forbid", strict=True)
confirmed: Literal[True]
class NodeRemovalItem(BaseModel):
removal_id: str
project_id: str
node_id: str
title: str
requested_at: str
restore_until: str
@classmethod
def from_record(cls, record: NodeRemoval) -> "NodeRemovalItem":
return cls(
removal_id=record.removal_id,
project_id=record.project_id,
node_id=record.node_id,
title=record.title,
requested_at=record.requested_at,
restore_until=record.restore_until,
)
class NodeRemovalList(BaseModel):
can_manage: bool
removals: list[NodeRemovalItem]
def register_node_removal_routes(
app: FastAPI,
store: SQLiteNodeRemovalStore,
reader: CanonicalNodeReader,
authenticate: Callable[..., Principal],
require_same_origin: Callable[[Request], None],
) -> None:
def human(
request: Request,
response: Response,
authorization: str | None = Header(default=None),
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
csrf_token: str | None = Header(default=None, alias="X-GEWU-CSRF"),
) -> Principal:
response.headers["Cache-Control"] = "private, no-store"
write = request.method not in {"GET", "HEAD"}
if write:
require_same_origin(request)
principal = authenticate(authorization, session_cookie, csrf_token, require_csrf=write)
if "agent" in principal.roles:
raise HTTPException(403, "A human account is required")
return principal
def require_manager(principal: Principal, project_id: str) -> None:
if not principal.can_read_project(project_id):
raise HTTPException(404, "Project removal queue is unavailable")
if not ({"editor", "admin", "owner"} & principal.roles):
raise HTTPException(403, "Editor role is required to remove or restore knowledge")
@app.get(
"/v1/projects/{project_id}/node-removals",
response_model=NodeRemovalList,
tags=["knowledge"],
operation_id="node_removal_list",
)
def list_removals(
project_id: RecordId,
principal: Annotated[Principal, Depends(human)],
) -> NodeRemovalList:
if not principal.can_read_project(project_id):
raise HTTPException(404, "Project removal queue is unavailable")
can_manage = bool({"editor", "admin", "owner"} & principal.roles)
removals = store.list_restorable(project_id) if can_manage else ()
return NodeRemovalList(
can_manage=can_manage,
removals=[NodeRemovalItem.from_record(record) for record in removals],
)
@app.post(
"/v1/projects/{project_id}/nodes/{node_id}/removal",
response_model=NodeRemovalItem,
status_code=201,
tags=["knowledge"],
operation_id="node_removal_request",
)
def request_removal(
project_id: RecordId,
node_id: RecordId,
body: NodeRemovalRequest,
principal: Annotated[Principal, Depends(human)],
) -> NodeRemovalItem:
require_manager(principal, project_id)
try:
envelope = reader.get_node(project_id, node_id, principal=principal)
title = str(envelope.data[0]["title"])
return NodeRemovalItem.from_record(
store.request(project_id, node_id, title, principal.principal_id)
)
except ReadError as error:
raise HTTPException(status_code=error.status_code, detail=str(error)) from error
except RemovalConflict as error:
raise HTTPException(status_code=409, detail=str(error)) from error
except RuntimeError as error:
raise HTTPException(status_code=503, detail=str(error)) from error
@app.post(
"/v1/projects/{project_id}/nodes/{node_id}/restore-removal",
response_model=NodeRemovalItem,
tags=["knowledge"],
operation_id="node_removal_restore",
)
def restore_removal(
project_id: RecordId,
node_id: RecordId,
principal: Annotated[Principal, Depends(human)],
) -> NodeRemovalItem:
require_manager(principal, project_id)
try:
return NodeRemovalItem.from_record(
store.restore(project_id, node_id, principal.principal_id)
)
except KeyError as error:
raise HTTPException(status_code=404, detail="Removed node was not found") from error
except RemovalConflict as error:
raise HTTPException(status_code=410, detail=str(error)) from error
except RuntimeError as error:
raise HTTPException(status_code=503, detail=str(error)) from error