feat: online verification, node removals, and knowledge browser
CI / quality (push) Canceled after 0s
CI / quality (push) Canceled after 0s
Adds claim-bound online evidence records and citation verification scripts, node removal transport/adapter, retrieval benchmark and owner-review tooling, GraphCanvas/SourceDetail knowledge browser UI, and the ingest handoff helper. Bumps the agents/gewu pack to the online-verification branch.
This commit is contained in:
1 parent
a3156f4e08
commit
78033bb830
83 files changed
+13476
-1051
No files matched your search
@@ -0,0 +1,147 @@
|
||||
"""Durable seven-day soft removals for user-facing knowledge visibility."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import secrets
|
||||
import sqlite3
|
||||
from contextlib import closing
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
from gewu.application.paths import reject_symlink_components
|
||||
|
||||
REMOVAL_GRACE_PERIOD = timedelta(days=7)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class NodeRemoval:
|
||||
removal_id: str
|
||||
project_id: str
|
||||
node_id: str
|
||||
title: str
|
||||
requested_by: str
|
||||
requested_at: str
|
||||
restore_until: str
|
||||
|
||||
|
||||
class RemovalConflict(RuntimeError):
|
||||
"""A node already has an active removal or its restore window expired."""
|
||||
|
||||
|
||||
class SQLiteNodeRemovalStore:
|
||||
def __init__(self, path: Path) -> None:
|
||||
self.path = path.expanduser().absolute()
|
||||
reject_symlink_components(self.path)
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
self.path.parent.chmod(0o700)
|
||||
if self.path.is_symlink():
|
||||
raise RuntimeError("Node removal database must not be a symbolic link")
|
||||
with closing(self._connect()) as connection:
|
||||
connection.execute(
|
||||
"""CREATE TABLE IF NOT EXISTS node_removals (
|
||||
removal_id TEXT PRIMARY KEY,
|
||||
project_id TEXT NOT NULL,
|
||||
node_id TEXT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
requested_by TEXT NOT NULL,
|
||||
requested_at TEXT NOT NULL,
|
||||
restore_until TEXT NOT NULL,
|
||||
restored_at TEXT,
|
||||
restored_by TEXT
|
||||
)"""
|
||||
)
|
||||
connection.execute(
|
||||
"CREATE UNIQUE INDEX IF NOT EXISTS node_removals_active_node "
|
||||
"ON node_removals(project_id, node_id) WHERE restored_at IS NULL"
|
||||
)
|
||||
connection.execute(
|
||||
"CREATE INDEX IF NOT EXISTS node_removals_project_restore "
|
||||
"ON node_removals(project_id, restored_at, restore_until)"
|
||||
)
|
||||
self.path.chmod(0o600)
|
||||
|
||||
def _connect(self) -> sqlite3.Connection:
|
||||
connection = sqlite3.connect(self.path, timeout=5, isolation_level=None)
|
||||
connection.execute("PRAGMA foreign_keys=ON")
|
||||
connection.execute("PRAGMA busy_timeout=5000")
|
||||
connection.execute("PRAGMA synchronous=FULL")
|
||||
return connection
|
||||
|
||||
@staticmethod
|
||||
def _record(row: tuple[object, ...]) -> NodeRemoval:
|
||||
return NodeRemoval(*(str(value) for value in row))
|
||||
|
||||
def is_hidden(self, project_id: str, node_id: str) -> bool:
|
||||
with closing(self._connect()) as connection:
|
||||
row = connection.execute(
|
||||
"SELECT 1 FROM node_removals WHERE project_id=? AND node_id=? "
|
||||
"AND restored_at IS NULL LIMIT 1",
|
||||
(project_id, node_id),
|
||||
).fetchone()
|
||||
return row is not None
|
||||
|
||||
def request(self, project_id: str, node_id: str, title: str, requested_by: str) -> NodeRemoval:
|
||||
now = datetime.now(UTC)
|
||||
requested_at = now.isoformat()
|
||||
restore_until = (now + REMOVAL_GRACE_PERIOD).isoformat()
|
||||
record = NodeRemoval(
|
||||
secrets.token_hex(16), project_id, node_id, title, requested_by,
|
||||
requested_at, restore_until,
|
||||
)
|
||||
try:
|
||||
with closing(self._connect()) as connection:
|
||||
connection.execute(
|
||||
"INSERT INTO node_removals(removal_id, project_id, node_id, title, "
|
||||
"requested_by, requested_at, restore_until) VALUES (?, ?, ?, ?, ?, ?, ?)",
|
||||
(
|
||||
record.removal_id, record.project_id, record.node_id, record.title,
|
||||
record.requested_by, record.requested_at, record.restore_until,
|
||||
),
|
||||
)
|
||||
except sqlite3.IntegrityError as error:
|
||||
raise RemovalConflict(
|
||||
"This node is already removed or its restore period ended"
|
||||
) from error
|
||||
except sqlite3.Error as error:
|
||||
raise RuntimeError("Node removal could not be saved") from error
|
||||
return record
|
||||
|
||||
def list_restorable(self, project_id: str) -> tuple[NodeRemoval, ...]:
|
||||
now = datetime.now(UTC).isoformat()
|
||||
with closing(self._connect()) as connection:
|
||||
rows = connection.execute(
|
||||
"SELECT removal_id, project_id, node_id, title, requested_by, requested_at, "
|
||||
"restore_until "
|
||||
"FROM node_removals WHERE project_id=? AND restored_at IS NULL AND restore_until>? "
|
||||
"ORDER BY requested_at DESC LIMIT 100",
|
||||
(project_id, now),
|
||||
).fetchall()
|
||||
return tuple(self._record(row) for row in rows)
|
||||
|
||||
def restore(self, project_id: str, node_id: str, restored_by: str) -> NodeRemoval:
|
||||
now = datetime.now(UTC)
|
||||
now_text = now.isoformat()
|
||||
with closing(self._connect()) as connection:
|
||||
connection.execute("BEGIN IMMEDIATE")
|
||||
row = connection.execute(
|
||||
"SELECT removal_id, project_id, node_id, title, requested_by, requested_at, "
|
||||
"restore_until "
|
||||
"FROM node_removals WHERE project_id=? AND node_id=? AND restored_at IS NULL "
|
||||
"ORDER BY requested_at DESC LIMIT 1",
|
||||
(project_id, node_id),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
connection.rollback()
|
||||
raise KeyError("Removed node was not found")
|
||||
record = self._record(row)
|
||||
if record.restore_until <= now_text:
|
||||
connection.rollback()
|
||||
raise RemovalConflict("The seven-day restore period has expired")
|
||||
connection.execute(
|
||||
"UPDATE node_removals SET restored_at=?, restored_by=? WHERE removal_id=? "
|
||||
"AND restored_at IS NULL",
|
||||
(now_text, restored_by, record.removal_id),
|
||||
)
|
||||
connection.commit()
|
||||
return record
|
||||
@@ -16,7 +16,11 @@ from pydantic import ValidationError
|
||||
from gewu.application.paths import reject_symlink_components
|
||||
from gewu.application.ports import CanonicalRepository
|
||||
from gewu.application.search import (
|
||||
BrowseIndexResult,
|
||||
GraphIndexResult,
|
||||
IndexedNode,
|
||||
SourceRelatedNodeHit,
|
||||
SourceNodesIndexResult,
|
||||
SearchEdgeHit,
|
||||
SearchEdgeResult,
|
||||
SearchIndexResult,
|
||||
@@ -28,7 +32,7 @@ from gewu.domain.contracts import Confidence, KnowledgeNode, SourceRevisionRecor
|
||||
MAX_INDEX_NODES = 100_000
|
||||
MAX_INDEX_EDGES = 500_000
|
||||
MAX_INDEX_RECORD_BYTES = 1_048_576
|
||||
INDEX_SCHEMA_VERSION = 4
|
||||
INDEX_SCHEMA_VERSION = 5
|
||||
_WORD = re.compile(r"[^\W_]+", flags=re.UNICODE)
|
||||
|
||||
|
||||
@@ -39,7 +43,9 @@ class SearchIndexError(RuntimeError):
|
||||
class SQLiteSearchIndex:
|
||||
"""SQLite FTS5 index that can be deleted and reconstructed from Git."""
|
||||
|
||||
def __init__(self, path: Path, repository: CanonicalRepository) -> None:
|
||||
def __init__(
|
||||
self, path: Path, repository: CanonicalRepository, *, node_removals_path: Path | None = None
|
||||
) -> None:
|
||||
self.path = path.expanduser().absolute()
|
||||
reject_symlink_components(self.path)
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
@@ -47,6 +53,11 @@ class SQLiteSearchIndex:
|
||||
if self.path.is_symlink():
|
||||
raise RuntimeError("Search index path must not be a symbolic link")
|
||||
self.repository = repository
|
||||
self.node_removals_path = (
|
||||
node_removals_path.expanduser().absolute() if node_removals_path is not None else None
|
||||
)
|
||||
if self.node_removals_path is not None:
|
||||
reject_symlink_components(self.node_removals_path)
|
||||
self._vector_extension_loader: Callable[[sqlite3.Connection], None] | None = None
|
||||
self._vector_generation_builder: (
|
||||
Callable[[sqlite3.Connection, str, tuple[KnowledgeNode, ...]], None] | None
|
||||
@@ -72,6 +83,10 @@ class SQLiteSearchIndex:
|
||||
connection.execute("PRAGMA foreign_keys=ON")
|
||||
connection.execute("PRAGMA busy_timeout=5000")
|
||||
connection.execute("PRAGMA synchronous=NORMAL")
|
||||
if self.node_removals_path is not None:
|
||||
connection.execute(
|
||||
"ATTACH DATABASE ? AS removal_state", (str(self.node_removals_path),)
|
||||
)
|
||||
if self._vector_extension_loader is not None:
|
||||
try:
|
||||
self._vector_extension_loader(connection)
|
||||
@@ -96,6 +111,15 @@ class SQLiteSearchIndex:
|
||||
"CREATE INDEX IF NOT EXISTS indexed_nodes_project "
|
||||
"ON indexed_nodes(project_id, node_id)"
|
||||
)
|
||||
connection.execute(
|
||||
"CREATE TABLE IF NOT EXISTS indexed_citations ("
|
||||
"node_id TEXT NOT NULL, project_id TEXT NOT NULL, source_revision_id TEXT NOT NULL, "
|
||||
"PRIMARY KEY(node_id, source_revision_id))"
|
||||
)
|
||||
connection.execute(
|
||||
"CREATE INDEX IF NOT EXISTS indexed_citations_source "
|
||||
"ON indexed_citations(source_revision_id, project_id)"
|
||||
)
|
||||
connection.execute(
|
||||
"CREATE TABLE IF NOT EXISTS indexed_sources ("
|
||||
"source_revision_id TEXT PRIMARY KEY, source_json TEXT NOT NULL)"
|
||||
@@ -237,6 +261,7 @@ class SQLiteSearchIndex:
|
||||
connection.execute("BEGIN IMMEDIATE")
|
||||
connection.execute("DELETE FROM node_fts")
|
||||
connection.execute("DELETE FROM indexed_nodes")
|
||||
connection.execute("DELETE FROM indexed_citations")
|
||||
connection.execute("DELETE FROM indexed_edges")
|
||||
connection.execute("DELETE FROM indexed_sources")
|
||||
connection.execute("DELETE FROM indexed_source_grants")
|
||||
@@ -248,6 +273,13 @@ class SQLiteSearchIndex:
|
||||
"VALUES (?, ?, ?)",
|
||||
(node.node_id, node.project_id, node_json),
|
||||
)
|
||||
for claim in node.evidence_claims:
|
||||
for evidence in claim.evidence:
|
||||
connection.execute(
|
||||
"INSERT OR IGNORE INTO indexed_citations("
|
||||
"node_id, project_id, source_revision_id) VALUES (?, ?, ?)",
|
||||
(node.node_id, node.project_id, evidence.source_revision_id),
|
||||
)
|
||||
connection.execute(
|
||||
"INSERT INTO node_fts(node_id, project_id, title, summary, claims, tags) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?)",
|
||||
@@ -338,6 +370,12 @@ class SQLiteSearchIndex:
|
||||
placeholders = ", ".join("?" for _ in project_scope)
|
||||
filters: list[str] = []
|
||||
filter_parameters: list[str] = []
|
||||
if self.node_removals_path is not None:
|
||||
filters.append(
|
||||
"NOT EXISTS (SELECT 1 FROM removal_state.node_removals removal "
|
||||
"WHERE removal.project_id=indexed_nodes.project_id "
|
||||
"AND removal.node_id=indexed_nodes.node_id AND removal.restored_at IS NULL)"
|
||||
)
|
||||
if node_types:
|
||||
filters.append(
|
||||
"json_extract(indexed_nodes.node_json, '$.node_type') IN ("
|
||||
@@ -444,10 +482,16 @@ class SQLiteSearchIndex:
|
||||
scope_placeholders = ", ".join("?" for _ in project_scope)
|
||||
|
||||
def eligible(kind_column: str, id_column: str) -> str:
|
||||
visible = (
|
||||
" AND NOT EXISTS (SELECT 1 FROM removal_state.node_removals removal "
|
||||
"WHERE removal.project_id=eligible_node.project_id "
|
||||
"AND removal.node_id=eligible_node.node_id AND removal.restored_at IS NULL)"
|
||||
if self.node_removals_path is not None else ""
|
||||
)
|
||||
return (
|
||||
f"(({kind_column}='node' AND EXISTS (SELECT 1 FROM indexed_nodes eligible_node "
|
||||
f"WHERE eligible_node.node_id=e.{id_column} AND eligible_node.project_id IN "
|
||||
f"({scope_placeholders}))) OR "
|
||||
f"({scope_placeholders}){visible})) OR "
|
||||
f"({kind_column}='project' AND e.{id_column} IN ({scope_placeholders})) OR "
|
||||
f"({kind_column}='source' AND EXISTS (SELECT 1 FROM indexed_source_grants grant "
|
||||
f"WHERE grant.source_revision_id=e.{id_column} AND grant.project_id IN "
|
||||
@@ -488,9 +532,14 @@ class SQLiteSearchIndex:
|
||||
if state is None:
|
||||
raise SearchIndexError("Search index revision is unavailable")
|
||||
revision = str(state[0])
|
||||
origin_row = connection.execute(
|
||||
"SELECT node_json FROM indexed_nodes WHERE node_id=?", (node_id,)
|
||||
).fetchone()
|
||||
origin_query = "SELECT node_json FROM indexed_nodes WHERE node_id=?"
|
||||
if self.node_removals_path is not None:
|
||||
origin_query += (
|
||||
" AND NOT EXISTS (SELECT 1 FROM removal_state.node_removals removal "
|
||||
"WHERE removal.project_id=indexed_nodes.project_id "
|
||||
"AND removal.node_id=indexed_nodes.node_id AND removal.restored_at IS NULL)"
|
||||
)
|
||||
origin_row = connection.execute(origin_query, (node_id,)).fetchone()
|
||||
origin = (
|
||||
KnowledgeNode.model_validate_json(origin_row[0], strict=True)
|
||||
if origin_row is not None
|
||||
@@ -570,3 +619,397 @@ class SQLiteSearchIndex:
|
||||
except (sqlite3.Error, ValidationError) as error:
|
||||
raise SearchIndexError("Source query could not be evaluated") from error
|
||||
return SourceIndexResult(revision, sources, len(rows) > limit)
|
||||
|
||||
def count_sources(self, *, project_ids: tuple[str, ...], include_shared: bool) -> int:
|
||||
self._rebuild_if_needed()
|
||||
project_scope = tuple(sorted(set(project_ids) | ({"shared"} if include_shared else set())))
|
||||
if not project_scope:
|
||||
return 0
|
||||
placeholders = ", ".join("?" for _ in project_scope)
|
||||
try:
|
||||
with closing(self._connect()) as connection:
|
||||
row = connection.execute(
|
||||
"SELECT COUNT(DISTINCT source_revision_id) FROM indexed_source_grants "
|
||||
f"WHERE project_id IN ({placeholders})", project_scope,
|
||||
).fetchone()
|
||||
except sqlite3.Error as error:
|
||||
raise SearchIndexError("Source count could not be evaluated") from error
|
||||
return int(row[0]) if row else 0
|
||||
|
||||
def graph(
|
||||
self, *, project_ids: tuple[str, ...], include_shared: bool, max_nodes: int, max_edges: int
|
||||
) -> GraphIndexResult:
|
||||
nodes_result = self.search(
|
||||
query="", project_ids=project_ids, include_shared=include_shared, limit=max_nodes
|
||||
)
|
||||
node_ids = tuple(hit.node.node_id for hit in nodes_result.nodes)
|
||||
if not node_ids:
|
||||
return GraphIndexResult(nodes_result.index_revision, (), (), nodes_result.has_more)
|
||||
placeholders = ", ".join("?" for _ in node_ids)
|
||||
statement = (
|
||||
"SELECT edge_json FROM indexed_edges WHERE source_kind='node' AND target_kind='node' "
|
||||
f"AND source_id IN ({placeholders}) AND target_id IN ({placeholders}) "
|
||||
"ORDER BY edge_id LIMIT ?"
|
||||
)
|
||||
try:
|
||||
with closing(self._connect()) as connection:
|
||||
connection.execute("BEGIN")
|
||||
state = connection.execute(
|
||||
"SELECT revision FROM index_state WHERE singleton=1 AND schema_version=?",
|
||||
(INDEX_SCHEMA_VERSION,),
|
||||
).fetchone()
|
||||
if state is None or str(state[0]) != nodes_result.index_revision:
|
||||
raise SearchIndexError("Graph index changed while loading its nodes")
|
||||
rows = connection.execute(
|
||||
statement, (*node_ids, *node_ids, max_edges + 1)
|
||||
).fetchall()
|
||||
edges = tuple(
|
||||
TypedEdge.model_validate_json(row[0], strict=True)
|
||||
for row in rows[:max_edges]
|
||||
)
|
||||
connection.commit()
|
||||
except (sqlite3.Error, ValidationError) as error:
|
||||
raise SearchIndexError("Knowledge graph could not be evaluated") from error
|
||||
return GraphIndexResult(
|
||||
nodes=tuple(hit.node for hit in nodes_result.nodes),
|
||||
edges=edges,
|
||||
index_revision=nodes_result.index_revision,
|
||||
truncated=nodes_result.has_more or len(rows) > max_edges,
|
||||
)
|
||||
|
||||
def browse(
|
||||
self,
|
||||
*,
|
||||
query: str,
|
||||
project_ids: tuple[str, ...],
|
||||
include_shared: bool,
|
||||
node_types: tuple[str, ...] = (),
|
||||
tags: tuple[str, ...] = (),
|
||||
audience: tuple[str, ...] = (),
|
||||
use_for: tuple[str, ...] = (),
|
||||
domain: tuple[str, ...] = (),
|
||||
confidences: tuple[Confidence, ...] = (),
|
||||
as_of_from: date | None = None,
|
||||
as_of_to: date | None = None,
|
||||
stale_only: bool = False,
|
||||
order: str = "title_asc",
|
||||
limit: int = 20,
|
||||
offset: int = 0,
|
||||
) -> BrowseIndexResult:
|
||||
revision = self._rebuild_if_needed()
|
||||
project_scope = tuple(sorted(set(project_ids) | ({"shared"} if include_shared else set())))
|
||||
if not project_scope:
|
||||
return BrowseIndexResult(
|
||||
revision, (), 0, 0, 0, {k: {} for k in ("tags", "audience", "use_for", "domain")}, False
|
||||
)
|
||||
|
||||
removal_filter = (
|
||||
" AND NOT EXISTS (SELECT 1 FROM removal_state.node_removals removal "
|
||||
"WHERE removal.project_id=indexed_nodes.project_id "
|
||||
"AND removal.node_id=indexed_nodes.node_id AND removal.restored_at IS NULL)"
|
||||
if self.node_removals_path is not None
|
||||
else ""
|
||||
)
|
||||
|
||||
filters: list[str] = [
|
||||
f"indexed_nodes.project_id IN ({", ".join("?" for _ in project_scope)})"
|
||||
]
|
||||
filter_parameters: list[object] = list(project_scope)
|
||||
|
||||
if self.node_removals_path is not None:
|
||||
filters.append(
|
||||
"NOT EXISTS (SELECT 1 FROM removal_state.node_removals removal "
|
||||
"WHERE removal.project_id=indexed_nodes.project_id "
|
||||
"AND removal.node_id=indexed_nodes.node_id AND removal.restored_at IS NULL)"
|
||||
)
|
||||
if node_types:
|
||||
filters.append(
|
||||
"json_extract(indexed_nodes.node_json, '$.node_type') IN ("
|
||||
+ ", ".join("?" for _ in node_types)
|
||||
+ ")"
|
||||
)
|
||||
filter_parameters.extend(node_types)
|
||||
for field, values in (
|
||||
("tags", tags),
|
||||
("audience", audience),
|
||||
("use_for", use_for),
|
||||
("domain", domain),
|
||||
):
|
||||
if values:
|
||||
value_slots = ", ".join("?" for _ in values)
|
||||
filters.append(
|
||||
"EXISTS (SELECT 1 FROM json_each(indexed_nodes.node_json, '$."
|
||||
+ field
|
||||
+ "') AS facet WHERE facet.value IN ("
|
||||
+ value_slots
|
||||
+ "))"
|
||||
)
|
||||
filter_parameters.extend(values)
|
||||
if confidences:
|
||||
filters.append(
|
||||
"json_extract(indexed_nodes.node_json, '$.confidence') IN ("
|
||||
+ ", ".join("?" for _ in confidences)
|
||||
+ ")"
|
||||
)
|
||||
filter_parameters.extend(item.value for item in confidences)
|
||||
if as_of_from is not None:
|
||||
filters.append("substr(json_extract(indexed_nodes.node_json, '$.as_of'), 1, 10) >= ?")
|
||||
filter_parameters.append(as_of_from.isoformat())
|
||||
if as_of_to is not None:
|
||||
filters.append("substr(json_extract(indexed_nodes.node_json, '$.as_of'), 1, 10) <= ?")
|
||||
filter_parameters.append(as_of_to.isoformat())
|
||||
if stale_only:
|
||||
filters.append("json_extract(indexed_nodes.node_json, '$.stale_after') <= ?")
|
||||
filter_parameters.append(datetime.now(UTC).date().isoformat() + "T00:00:00Z")
|
||||
|
||||
where_clause = " WHERE " + " AND ".join(filters)
|
||||
match = self._match_expression(query)
|
||||
|
||||
if order == "title_asc":
|
||||
order_by = "LOWER(json_extract(indexed_nodes.node_json, '$.title')) ASC, json_extract(indexed_nodes.node_json, '$.title') ASC, indexed_nodes.node_id ASC"
|
||||
elif order == "title_desc":
|
||||
order_by = "LOWER(json_extract(indexed_nodes.node_json, '$.title')) DESC, json_extract(indexed_nodes.node_json, '$.title') DESC, indexed_nodes.node_id DESC"
|
||||
elif order == "date_newest":
|
||||
order_by = "(json_extract(indexed_nodes.node_json, '$.as_of') IS NULL) ASC, json_extract(indexed_nodes.node_json, '$.as_of') DESC, indexed_nodes.node_id ASC"
|
||||
elif order == "date_oldest":
|
||||
order_by = "(json_extract(indexed_nodes.node_json, '$.as_of') IS NULL) ASC, json_extract(indexed_nodes.node_json, '$.as_of') ASC, indexed_nodes.node_id ASC"
|
||||
elif order == "relevance" and match:
|
||||
order_by = "score ASC, indexed_nodes.node_id ASC"
|
||||
else:
|
||||
order_by = "LOWER(json_extract(indexed_nodes.node_json, '$.title')) ASC, json_extract(indexed_nodes.node_json, '$.title') ASC, indexed_nodes.node_id ASC"
|
||||
|
||||
try:
|
||||
with closing(self._connect()) as connection:
|
||||
connection.execute("BEGIN")
|
||||
state = connection.execute(
|
||||
"SELECT revision FROM index_state WHERE singleton=1 AND schema_version=?",
|
||||
(INDEX_SCHEMA_VERSION,),
|
||||
).fetchone()
|
||||
if state is None:
|
||||
raise SearchIndexError("Search index revision is unavailable")
|
||||
rev = str(state[0])
|
||||
|
||||
# Project node count
|
||||
project_node_count = 0
|
||||
if project_ids:
|
||||
p_holders = ", ".join("?" for _ in project_ids)
|
||||
p_row = connection.execute(
|
||||
f"SELECT COUNT(*) FROM indexed_nodes WHERE project_id IN ({p_holders}){removal_filter}",
|
||||
project_ids,
|
||||
).fetchone()
|
||||
project_node_count = int(p_row[0]) if p_row else 0
|
||||
|
||||
# Shared node count
|
||||
shared_node_count = 0
|
||||
if include_shared:
|
||||
s_row = connection.execute(
|
||||
f"SELECT COUNT(*) FROM indexed_nodes WHERE project_id='shared'{removal_filter}"
|
||||
).fetchone()
|
||||
shared_node_count = int(s_row[0]) if s_row else 0
|
||||
|
||||
# Filtered count
|
||||
if match:
|
||||
count_stmt = (
|
||||
"SELECT COUNT(*) FROM node_fts JOIN indexed_nodes USING(node_id) "
|
||||
f"{where_clause} AND node_fts MATCH ?"
|
||||
)
|
||||
count_row = connection.execute(
|
||||
count_stmt, (*filter_parameters, match)
|
||||
).fetchone()
|
||||
else:
|
||||
count_stmt = f"SELECT COUNT(*) FROM indexed_nodes {where_clause}"
|
||||
count_row = connection.execute(count_stmt, filter_parameters).fetchone()
|
||||
filtered_node_count = int(count_row[0]) if count_row else 0
|
||||
|
||||
# Paged node query
|
||||
if match:
|
||||
query_stmt = (
|
||||
"SELECT indexed_nodes.node_json, bm25(node_fts) AS score "
|
||||
f"FROM node_fts JOIN indexed_nodes USING(node_id) {where_clause} AND node_fts MATCH ? "
|
||||
f"ORDER BY {order_by} LIMIT ? OFFSET ?"
|
||||
)
|
||||
node_rows = connection.execute(
|
||||
query_stmt, (*filter_parameters, match, limit + 1, offset)
|
||||
).fetchall()
|
||||
else:
|
||||
query_stmt = (
|
||||
f"SELECT indexed_nodes.node_json FROM indexed_nodes {where_clause} "
|
||||
f"ORDER BY {order_by} LIMIT ? OFFSET ?"
|
||||
)
|
||||
node_rows = connection.execute(
|
||||
query_stmt, (*filter_parameters, limit + 1, offset)
|
||||
).fetchall()
|
||||
|
||||
nodes = tuple(
|
||||
KnowledgeNode.model_validate_json(row[0], strict=True)
|
||||
for row in node_rows[:limit]
|
||||
)
|
||||
has_more = len(node_rows) > limit
|
||||
|
||||
# Facets
|
||||
scope_holders = ", ".join("?" for _ in project_scope)
|
||||
facets: dict[str, dict[str, int]] = {}
|
||||
for field in ("tags", "audience", "use_for", "domain"):
|
||||
facet_stmt = (
|
||||
"SELECT facet.value, COUNT(DISTINCT indexed_nodes.node_id) "
|
||||
f"FROM indexed_nodes, json_each(indexed_nodes.node_json, '$.' || ?) AS facet "
|
||||
f"WHERE indexed_nodes.project_id IN ({scope_holders}){removal_filter} "
|
||||
"GROUP BY facet.value"
|
||||
)
|
||||
facet_rows = connection.execute(facet_stmt, (field, *project_scope)).fetchall()
|
||||
facets[field] = {str(r[0]): int(r[1]) for r in facet_rows}
|
||||
|
||||
connection.commit()
|
||||
except (sqlite3.Error, ValidationError) as error:
|
||||
raise SearchIndexError("Browse query could not be evaluated") from error
|
||||
|
||||
return BrowseIndexResult(
|
||||
index_revision=rev,
|
||||
nodes=nodes,
|
||||
filtered_node_count=filtered_node_count,
|
||||
project_node_count=project_node_count,
|
||||
shared_node_count=shared_node_count,
|
||||
facets=facets,
|
||||
has_more=has_more,
|
||||
)
|
||||
|
||||
def source_nodes(
|
||||
self,
|
||||
*,
|
||||
source_revision_id: str,
|
||||
project_ids: tuple[str, ...],
|
||||
include_shared: bool,
|
||||
query: str = "",
|
||||
node_type: str | None = None,
|
||||
limit: int = 20,
|
||||
offset: int = 0,
|
||||
) -> SourceNodesIndexResult | None:
|
||||
revision = self._rebuild_if_needed()
|
||||
project_scope = tuple(sorted(set(project_ids) | ({"shared"} if include_shared else set())))
|
||||
if not project_scope:
|
||||
return None
|
||||
|
||||
scope_holders = ", ".join("?" for _ in project_scope)
|
||||
match = self._match_expression(query)
|
||||
try:
|
||||
with closing(self._connect()) as connection:
|
||||
connection.execute("BEGIN")
|
||||
state = connection.execute(
|
||||
"SELECT revision FROM index_state WHERE singleton=1 AND schema_version=?",
|
||||
(INDEX_SCHEMA_VERSION,),
|
||||
).fetchone()
|
||||
if state is None:
|
||||
raise SearchIndexError("Search index revision is unavailable")
|
||||
rev = str(state[0])
|
||||
|
||||
# Check source revision grant
|
||||
grant_row = connection.execute(
|
||||
f"SELECT 1 FROM indexed_source_grants WHERE source_revision_id=? AND project_id IN ({scope_holders}) LIMIT 1",
|
||||
(source_revision_id, *project_scope),
|
||||
).fetchone()
|
||||
if grant_row is None:
|
||||
return None
|
||||
|
||||
cte = f"""
|
||||
WITH candidates AS (
|
||||
SELECT node_id, 'citation' AS basis
|
||||
FROM indexed_citations
|
||||
WHERE source_revision_id = ? AND project_id IN ({scope_holders})
|
||||
UNION ALL
|
||||
SELECT
|
||||
CASE WHEN source_kind = 'node' THEN source_id ELSE target_id END AS node_id,
|
||||
'edge' AS basis
|
||||
FROM indexed_edges
|
||||
WHERE (
|
||||
(source_kind = 'node' AND target_kind = 'source' AND target_id = ?) OR
|
||||
(source_kind = 'source' AND target_kind = 'node' AND source_id = ?)
|
||||
)
|
||||
),
|
||||
combined AS (
|
||||
SELECT
|
||||
node_id,
|
||||
CASE
|
||||
WHEN COUNT(DISTINCT basis) > 1 THEN 'both'
|
||||
ELSE MAX(basis)
|
||||
END AS relation_basis
|
||||
FROM candidates
|
||||
GROUP BY node_id
|
||||
)
|
||||
"""
|
||||
|
||||
filters = [
|
||||
f"indexed_nodes.project_id IN ({scope_holders})",
|
||||
]
|
||||
filter_params: list[object] = [
|
||||
source_revision_id,
|
||||
*project_scope,
|
||||
source_revision_id,
|
||||
source_revision_id,
|
||||
*project_scope,
|
||||
]
|
||||
|
||||
if self.node_removals_path is not None:
|
||||
filters.append(
|
||||
"NOT EXISTS (SELECT 1 FROM removal_state.node_removals removal "
|
||||
"WHERE removal.project_id=indexed_nodes.project_id "
|
||||
"AND removal.node_id=indexed_nodes.node_id AND removal.restored_at IS NULL)"
|
||||
)
|
||||
if node_type:
|
||||
filters.append("json_extract(indexed_nodes.node_json, '$.node_type') = ?")
|
||||
filter_params.append(node_type)
|
||||
|
||||
where_clause = " WHERE " + " AND ".join(filters)
|
||||
|
||||
# Total count
|
||||
if match:
|
||||
count_stmt = (
|
||||
f"{cte} SELECT COUNT(*) FROM combined JOIN indexed_nodes USING(node_id) "
|
||||
f"JOIN node_fts USING(node_id) {where_clause} AND node_fts MATCH ?"
|
||||
)
|
||||
count_row = connection.execute(count_stmt, (*filter_params, match)).fetchone()
|
||||
else:
|
||||
count_stmt = (
|
||||
f"{cte} SELECT COUNT(*) FROM combined JOIN indexed_nodes USING(node_id) {where_clause}"
|
||||
)
|
||||
count_row = connection.execute(count_stmt, filter_params).fetchone()
|
||||
total = int(count_row[0]) if count_row else 0
|
||||
|
||||
# Rows query
|
||||
order_by = "LOWER(json_extract(indexed_nodes.node_json, '$.title')) ASC, indexed_nodes.node_id ASC"
|
||||
if match:
|
||||
row_stmt = (
|
||||
f"{cte} SELECT indexed_nodes.node_json, combined.relation_basis FROM combined "
|
||||
f"JOIN indexed_nodes USING(node_id) JOIN node_fts USING(node_id) "
|
||||
f"{where_clause} AND node_fts MATCH ? ORDER BY {order_by} LIMIT ? OFFSET ?"
|
||||
)
|
||||
rows = connection.execute(
|
||||
row_stmt, (*filter_params, match, limit + 1, offset)
|
||||
).fetchall()
|
||||
else:
|
||||
row_stmt = (
|
||||
f"{cte} SELECT indexed_nodes.node_json, combined.relation_basis FROM combined "
|
||||
f"JOIN indexed_nodes USING(node_id) {where_clause} ORDER BY {order_by} LIMIT ? OFFSET ?"
|
||||
)
|
||||
rows = connection.execute(
|
||||
row_stmt, (*filter_params, limit + 1, offset)
|
||||
).fetchall()
|
||||
|
||||
nodes = tuple(
|
||||
SourceRelatedNodeHit(
|
||||
node=KnowledgeNode.model_validate_json(r[0], strict=True),
|
||||
relation_basis=r[1],
|
||||
)
|
||||
for r in rows[:limit]
|
||||
)
|
||||
has_more = len(rows) > limit
|
||||
|
||||
connection.commit()
|
||||
except (sqlite3.Error, ValidationError) as error:
|
||||
raise SearchIndexError("Source related nodes could not be evaluated") from error
|
||||
|
||||
return SourceNodesIndexResult(
|
||||
index_revision=rev,
|
||||
nodes=nodes,
|
||||
total=total,
|
||||
has_more=has_more,
|
||||
)
|
||||
@@ -5,6 +5,7 @@ from __future__ import annotations
|
||||
import hashlib
|
||||
import json
|
||||
import secrets
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
|
||||
import yaml
|
||||
@@ -39,6 +40,7 @@ _GIT_COMMIT_ID = TypeAdapter(GitCommitId)
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CanonicalNodeReader:
|
||||
repository: CanonicalRepository
|
||||
node_is_hidden: Callable[[str, str], bool] | None = None
|
||||
|
||||
def get_node(
|
||||
self,
|
||||
@@ -50,6 +52,8 @@ class CanonicalNodeReader:
|
||||
) -> ReadEnvelope:
|
||||
if not principal.can_read_project(project_id):
|
||||
raise ReadError(403, "Credential cannot read this project")
|
||||
if self.node_is_hidden is not None and self.node_is_hidden(project_id, node_id):
|
||||
raise ReadError(404, "Knowledge node was not found")
|
||||
head = self.repository.head()
|
||||
path = f"projects/{_key(project_id)}/nodes/{_key(node_id)}.yaml"
|
||||
if revision is None:
|
||||
@@ -98,6 +102,8 @@ class CanonicalNodeReader:
|
||||
raise ReadError(422, "History limit must be between one and twenty")
|
||||
if not principal.can_read_project(project_id):
|
||||
raise ReadError(403, "Credential cannot read this project")
|
||||
if self.node_is_hidden is not None and self.node_is_hidden(project_id, node_id):
|
||||
raise ReadError(404, "Knowledge node was not found")
|
||||
path = f"projects/{_key(project_id)}/nodes/{_key(node_id)}.yaml"
|
||||
head = self.repository.head()
|
||||
rows: list[dict[str, object]] = []
|
||||
|
||||
@@ -30,6 +30,9 @@ from gewu.domain.validation import extraction_digest
|
||||
|
||||
MAX_CONTEXT_RECORDS = 20_000
|
||||
MAX_CONTEXT_RECORD_BYTES = 1_048_576
|
||||
# Character-level normalization maps are substantially larger than source text.
|
||||
# Keep ordinary registry/node limits small while allowing bounded extraction records.
|
||||
MAX_CONTEXT_EXTRACTION_BYTES = 16 * 1024 * 1024
|
||||
MAX_PROJECT_REGISTRY_RESPONSE_BYTES = 1_048_576
|
||||
|
||||
|
||||
@@ -162,7 +165,7 @@ class LocalContextProvider:
|
||||
continue
|
||||
record_count += 1
|
||||
extraction_bytes = self.repository.read_file(revision, record_path)
|
||||
if extraction_bytes is None or len(extraction_bytes) > MAX_CONTEXT_RECORD_BYTES:
|
||||
if extraction_bytes is None or len(extraction_bytes) > MAX_CONTEXT_EXTRACTION_BYTES:
|
||||
raise RegistryError("Canonical extraction is unavailable or too large")
|
||||
extraction = ExtractionRecord.model_validate_json(extraction_bytes, strict=True)
|
||||
if extraction_digest(extraction) != extraction.extraction_id:
|
||||
|
||||
+303
-10
@@ -11,7 +11,7 @@ from collections import defaultdict
|
||||
from collections.abc import Callable, Collection, Iterator
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, date, datetime
|
||||
from typing import Literal, Protocol
|
||||
from typing import Any, Literal, Protocol
|
||||
|
||||
from gewu.application.identity import Principal
|
||||
from gewu.application.ports import EmbeddingProvider, VectorGeneration, VectorIndex
|
||||
@@ -22,6 +22,12 @@ from gewu.domain.contracts import (
|
||||
EvidenceReference,
|
||||
Finding,
|
||||
KnowledgeNode,
|
||||
KnowledgeBrowseRequest,
|
||||
KnowledgeBrowseEnvelope,
|
||||
KnowledgeGraphEnvelope,
|
||||
KnowledgeGraphRequest,
|
||||
SourceRelatedNode,
|
||||
SourceRelatedNodesEnvelope,
|
||||
NeighborRequest,
|
||||
ReadEnvelope,
|
||||
Scope,
|
||||
@@ -58,6 +64,31 @@ class SearchIndexResult:
|
||||
warnings: tuple[Finding, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BrowseIndexResult:
|
||||
index_revision: str
|
||||
nodes: tuple[KnowledgeNode, ...]
|
||||
filtered_node_count: int
|
||||
project_node_count: int
|
||||
shared_node_count: int
|
||||
facets: dict[str, dict[str, int]]
|
||||
has_more: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SourceRelatedNodeHit:
|
||||
node: KnowledgeNode
|
||||
relation_basis: Literal["citation", "edge", "both"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SourceNodesIndexResult:
|
||||
index_revision: str
|
||||
nodes: tuple[SourceRelatedNodeHit, ...]
|
||||
total: int
|
||||
has_more: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SourceIndexResult:
|
||||
index_revision: str
|
||||
@@ -65,6 +96,14 @@ class SourceIndexResult:
|
||||
has_more: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class GraphIndexResult:
|
||||
index_revision: str
|
||||
nodes: tuple[KnowledgeNode, ...]
|
||||
edges: tuple[TypedEdge, ...]
|
||||
truncated: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SearchEdgeResult:
|
||||
index_revision: str
|
||||
@@ -124,6 +163,42 @@ class SearchIndex(Protocol):
|
||||
offset: int = 0,
|
||||
) -> SourceIndexResult: ...
|
||||
|
||||
def count_sources(self, *, project_ids: tuple[str, ...], include_shared: bool) -> int: ...
|
||||
|
||||
def graph(self, *, project_ids: tuple[str, ...], include_shared: bool, max_nodes: int, max_edges: int) -> GraphIndexResult: ...
|
||||
|
||||
def browse(
|
||||
self,
|
||||
*,
|
||||
query: str,
|
||||
project_ids: tuple[str, ...],
|
||||
include_shared: bool,
|
||||
node_types: tuple[str, ...] = (),
|
||||
tags: tuple[str, ...] = (),
|
||||
audience: tuple[str, ...] = (),
|
||||
use_for: tuple[str, ...] = (),
|
||||
domain: tuple[str, ...] = (),
|
||||
confidences: tuple[Confidence, ...] = (),
|
||||
as_of_from: date | None = None,
|
||||
as_of_to: date | None = None,
|
||||
stale_only: bool = False,
|
||||
order: str = "title_asc",
|
||||
limit: int = 20,
|
||||
offset: int = 0,
|
||||
) -> BrowseIndexResult: ...
|
||||
|
||||
def source_nodes(
|
||||
self,
|
||||
*,
|
||||
source_revision_id: str,
|
||||
project_ids: tuple[str, ...],
|
||||
include_shared: bool,
|
||||
query: str = "",
|
||||
node_type: str | None = None,
|
||||
limit: int = 20,
|
||||
offset: int = 0,
|
||||
) -> SourceNodesIndexResult | None: ...
|
||||
|
||||
|
||||
_VECTOR_CANDIDATE_LIMIT = 100
|
||||
_VECTOR_BUILD_BATCH_SIZE = 32
|
||||
@@ -538,6 +613,18 @@ class HybridSearchIndex:
|
||||
offset=offset,
|
||||
)
|
||||
|
||||
def count_sources(self, *, project_ids: tuple[str, ...], include_shared: bool) -> int:
|
||||
return self.lexical.count_sources(project_ids=project_ids, include_shared=include_shared)
|
||||
|
||||
def graph(self, *, project_ids: tuple[str, ...], include_shared: bool, max_nodes: int, max_edges: int) -> GraphIndexResult:
|
||||
return self.lexical.graph(project_ids=project_ids, include_shared=include_shared, max_nodes=max_nodes, max_edges=max_edges)
|
||||
|
||||
def browse(self, **kwargs: Any) -> BrowseIndexResult:
|
||||
return self.lexical.browse(**kwargs)
|
||||
|
||||
def source_nodes(self, **kwargs: Any) -> SourceNodesIndexResult | None:
|
||||
return self.lexical.source_nodes(**kwargs)
|
||||
|
||||
|
||||
def _budgeted_envelope(
|
||||
*,
|
||||
@@ -550,6 +637,7 @@ def _budgeted_envelope(
|
||||
index_revision: str,
|
||||
budget_tokens: int,
|
||||
retrieval_mode: Literal["keyword", "hybrid", "canonical"],
|
||||
filtered_count: int | None = None,
|
||||
next_cursor_for: Callable[[int], str | None] | None = None,
|
||||
) -> ReadEnvelope:
|
||||
for included_count in range(len(rows), -1, -1):
|
||||
@@ -571,6 +659,7 @@ def _budgeted_envelope(
|
||||
request_id=request_id,
|
||||
resolved_scope=scope,
|
||||
data=tuple(rows[:included_count]),
|
||||
filtered_count=filtered_count,
|
||||
citations=tuple(evidence[key] for key in sorted(evidence)),
|
||||
warnings=tuple(warnings),
|
||||
omissions=omissions,
|
||||
@@ -620,6 +709,137 @@ class CanonicalSearchService:
|
||||
state = "current" if index_revision == canonical_revision else "lagging"
|
||||
return canonical_revision, index_revision, state, schema_version
|
||||
|
||||
def browse(self, request: KnowledgeBrowseRequest, *, principal: Principal) -> KnowledgeBrowseEnvelope:
|
||||
project_ids = self._resolve_scope(request.scope, principal)
|
||||
fingerprint = hashlib.sha256(canonical_json({
|
||||
"projects": project_ids,
|
||||
"shared": request.scope.include_shared,
|
||||
"query": request.query,
|
||||
"types": sorted(item.value for item in request.node_types),
|
||||
"tags": sorted(request.tags), "audience": sorted(request.audience),
|
||||
"use": sorted(request.use_for), "domain": sorted(request.domain),
|
||||
"confidence": sorted(item.value for item in request.confidences),
|
||||
"from": request.as_of_from.isoformat() if request.as_of_from else None,
|
||||
"to": request.as_of_to.isoformat() if request.as_of_to else None,
|
||||
"stale": request.stale_only, "order": request.order, "limit": request.limit,
|
||||
})).hexdigest()
|
||||
offset, expected_revision = self._cursor_offset(request.cursor, fingerprint, max_offset=100_000)
|
||||
lexical = self.index.lexical if isinstance(self.index, HybridSearchIndex) else self.index
|
||||
result = lexical.browse(
|
||||
query=request.query,
|
||||
project_ids=project_ids,
|
||||
include_shared=request.scope.include_shared,
|
||||
node_types=tuple(item.value for item in request.node_types),
|
||||
tags=request.tags,
|
||||
audience=request.audience,
|
||||
use_for=request.use_for,
|
||||
domain=request.domain,
|
||||
confidences=request.confidences,
|
||||
as_of_from=request.as_of_from,
|
||||
as_of_to=request.as_of_to,
|
||||
stale_only=request.stale_only,
|
||||
order=request.order,
|
||||
limit=request.limit,
|
||||
offset=offset,
|
||||
)
|
||||
if expected_revision is not None and expected_revision != result.index_revision:
|
||||
raise SearchError(409, "Browse cursor expired because the index revision changed")
|
||||
|
||||
source_count = self.index.count_sources(project_ids=project_ids, include_shared=False)
|
||||
shared_source_count = (
|
||||
self.index.count_sources(project_ids=(), include_shared=True)
|
||||
if request.scope.include_shared
|
||||
else 0
|
||||
)
|
||||
page = result.nodes
|
||||
next_offset = offset + len(page)
|
||||
next_cursor = (
|
||||
self._encode_cursor(next_offset, fingerprint, result.index_revision)
|
||||
if next_offset < result.filtered_node_count
|
||||
else None
|
||||
)
|
||||
canonical_revision = self.canonical_head()
|
||||
warnings: tuple[Finding, ...] = ()
|
||||
if result.index_revision != canonical_revision:
|
||||
warnings = (Finding(code="INDEX_LAG", message="Browse data reflects an older accepted revision."),)
|
||||
return KnowledgeBrowseEnvelope(
|
||||
request_id="browse-" + secrets.token_hex(12),
|
||||
resolved_scope=Scope(projects=project_ids, include_shared=request.scope.include_shared),
|
||||
nodes=page,
|
||||
project_node_count=result.project_node_count,
|
||||
project_source_count=source_count,
|
||||
shared_node_count=result.shared_node_count,
|
||||
shared_source_count=shared_source_count,
|
||||
filtered_node_count=result.filtered_node_count,
|
||||
facets=result.facets,
|
||||
canonical_revision=canonical_revision,
|
||||
index_revision=result.index_revision,
|
||||
warnings=warnings,
|
||||
next_cursor=next_cursor,
|
||||
)
|
||||
|
||||
def source_nodes(
|
||||
self,
|
||||
*,
|
||||
project_id: str,
|
||||
revision_id: str,
|
||||
principal: Principal,
|
||||
include_shared: bool = False,
|
||||
query: str = "",
|
||||
node_type: str | None = None,
|
||||
limit: int = 20,
|
||||
cursor: str | None = None,
|
||||
) -> SourceRelatedNodesEnvelope:
|
||||
if not principal.can_read_project(project_id):
|
||||
raise SearchError(403, "Credential cannot read this project")
|
||||
fingerprint = hashlib.sha256(canonical_json({
|
||||
"project": project_id,
|
||||
"revision": revision_id,
|
||||
"shared": include_shared,
|
||||
"query": query,
|
||||
"node_type": node_type,
|
||||
"limit": limit,
|
||||
})).hexdigest()
|
||||
offset, expected_revision = self._cursor_offset(cursor, fingerprint, max_offset=100_000)
|
||||
lexical = self.index.lexical if isinstance(self.index, HybridSearchIndex) else self.index
|
||||
result = lexical.source_nodes(
|
||||
source_revision_id=revision_id,
|
||||
project_ids=(project_id,),
|
||||
include_shared=include_shared,
|
||||
query=query,
|
||||
node_type=node_type,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
if result is None:
|
||||
raise SearchError(404, "Source revision was not found")
|
||||
if expected_revision is not None and expected_revision != result.index_revision:
|
||||
raise SearchError(409, "Cursor expired because the index revision changed")
|
||||
next_offset = offset + len(result.nodes)
|
||||
next_cursor = (
|
||||
self._encode_cursor(next_offset, fingerprint, result.index_revision)
|
||||
if next_offset < result.total
|
||||
else None
|
||||
)
|
||||
canonical_revision = self.canonical_head()
|
||||
warnings: tuple[Finding, ...] = ()
|
||||
if result.index_revision != canonical_revision:
|
||||
warnings = (Finding(code="INDEX_LAG", message="Source nodes reflect an older accepted revision."),)
|
||||
return SourceRelatedNodesEnvelope(
|
||||
request_id="srcnodes-" + secrets.token_hex(12),
|
||||
resolved_scope=Scope(projects=(project_id,), include_shared=include_shared),
|
||||
source_revision_id=revision_id,
|
||||
nodes=tuple(
|
||||
SourceRelatedNode(node=hit.node, relation_basis=hit.relation_basis)
|
||||
for hit in result.nodes
|
||||
),
|
||||
total=result.total,
|
||||
canonical_revision=canonical_revision,
|
||||
index_revision=result.index_revision,
|
||||
warnings=warnings,
|
||||
next_cursor=next_cursor,
|
||||
)
|
||||
|
||||
def search(self, request: SearchRequest, *, principal: Principal) -> ReadEnvelope:
|
||||
project_ids = self._resolve_scope(request.scope, principal)
|
||||
fingerprint = hashlib.sha256(
|
||||
@@ -638,6 +858,7 @@ class CanonicalSearchService:
|
||||
"as_of_to": request.as_of_to.isoformat() if request.as_of_to else None,
|
||||
"stale_only": request.stale_only,
|
||||
"limit": request.limit,
|
||||
"include_evidence": request.include_evidence,
|
||||
"retrieval_profile": (
|
||||
self.index.cursor_profile()
|
||||
if isinstance(self.index, HybridSearchIndex)
|
||||
@@ -674,9 +895,25 @@ class CanonicalSearchService:
|
||||
rows: list[dict[str, object]] = []
|
||||
for hit in result.nodes:
|
||||
node = hit.node
|
||||
node_payload = node.model_dump(mode="json")
|
||||
if not request.include_evidence:
|
||||
# Search returns claim text and result-level citations. Full
|
||||
# claim-to-citation records remain available through get_node.
|
||||
for claim_group in ("claims", "assumptions", "limitations"):
|
||||
compact_claims = []
|
||||
for claim in node_payload.get(claim_group, []):
|
||||
compact_claims.append(
|
||||
{
|
||||
**claim,
|
||||
"evidence_count": len(claim.get("evidence", [])),
|
||||
}
|
||||
)
|
||||
compact_claims[-1].pop("evidence", None)
|
||||
node_payload[claim_group] = compact_claims
|
||||
row: dict[str, object] = {
|
||||
"node": node.model_dump(mode="json"),
|
||||
"node": node_payload,
|
||||
"retrieval_score": hit.score,
|
||||
"evidence_included": request.include_evidence,
|
||||
}
|
||||
if hit.expanded_from is not None and hit.expansion_edge is not None:
|
||||
row["retrieval_path"] = "graph_expansion"
|
||||
@@ -692,14 +929,15 @@ class CanonicalSearchService:
|
||||
)
|
||||
)
|
||||
node_evidence: dict[str, EvidenceReference] = {}
|
||||
for claim in node.evidence_claims:
|
||||
for item in claim.evidence:
|
||||
key = evidence_digest(item)
|
||||
node_evidence[key] = item
|
||||
if hit.expansion_edge is not None:
|
||||
for item in hit.expansion_edge.evidence:
|
||||
key = evidence_digest(item)
|
||||
node_evidence[key] = item
|
||||
if request.include_evidence:
|
||||
for claim in node.evidence_claims:
|
||||
for item in claim.evidence:
|
||||
key = evidence_digest(item)
|
||||
node_evidence[key] = item
|
||||
if hit.expansion_edge is not None:
|
||||
for item in hit.expansion_edge.evidence:
|
||||
key = evidence_digest(item)
|
||||
node_evidence[key] = item
|
||||
evidence_by_node.append(node_evidence)
|
||||
if result.index_revision != canonical_revision:
|
||||
warnings.append(
|
||||
@@ -734,6 +972,56 @@ class CanonicalSearchService:
|
||||
),
|
||||
)
|
||||
|
||||
def graph(self, request: KnowledgeGraphRequest, *, principal: Principal) -> KnowledgeGraphEnvelope:
|
||||
project_ids = self._resolve_scope(request.scope, principal)
|
||||
if request.center_node_id is None:
|
||||
result = self.index.graph(
|
||||
project_ids=project_ids,
|
||||
include_shared=request.scope.include_shared,
|
||||
max_nodes=request.max_nodes,
|
||||
max_edges=request.max_edges,
|
||||
)
|
||||
else:
|
||||
scoped = self.index.search(
|
||||
query="", project_ids=project_ids, include_shared=request.scope.include_shared,
|
||||
limit=1, candidate_node_ids=(request.center_node_id,),
|
||||
)
|
||||
origin = scoped.nodes[0].node if scoped.nodes else None
|
||||
if origin is None:
|
||||
raise SearchError(404, "Knowledge node was not found within the requested scope")
|
||||
nodes_by_id = {origin.node_id: origin}
|
||||
edges: list[TypedEdge] = []
|
||||
neighbors = None
|
||||
if request.max_nodes > 1:
|
||||
neighbors = self.index.neighbors(
|
||||
node_id=origin.node_id, edge_types=(), project_ids=project_ids,
|
||||
include_shared=request.scope.include_shared, limit=min(50, request.max_nodes - 1),
|
||||
)
|
||||
for hit in neighbors.edges:
|
||||
endpoint = hit.neighbor_endpoint
|
||||
if endpoint.kind != "node" or hit.neighbor_node is None:
|
||||
continue
|
||||
nodes_by_id[hit.neighbor_node.node_id] = hit.neighbor_node
|
||||
edges.append(hit.edge)
|
||||
result = GraphIndexResult(
|
||||
index_revision=neighbors.index_revision if neighbors else scoped.index_revision,
|
||||
nodes=tuple(nodes_by_id.values()),
|
||||
edges=tuple(edges[:request.max_edges]),
|
||||
truncated=(neighbors.has_more if neighbors else False) or len(edges) > request.max_edges,
|
||||
)
|
||||
canonical_revision = self.canonical_head()
|
||||
warnings: tuple[Finding, ...] = ()
|
||||
if result.index_revision != canonical_revision:
|
||||
warnings = (Finding(code="INDEX_LAG", message="Graph data reflects an older accepted revision."),)
|
||||
return KnowledgeGraphEnvelope(
|
||||
request_id="graph-" + secrets.token_hex(12),
|
||||
resolved_scope=Scope(projects=project_ids, include_shared=request.scope.include_shared),
|
||||
nodes=result.nodes, edges=result.edges, truncated=result.truncated,
|
||||
center_node_id=request.center_node_id,
|
||||
canonical_revision=canonical_revision, index_revision=result.index_revision,
|
||||
warnings=warnings,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _encode_cursor(offset: int, fingerprint: str, index_revision: str) -> str:
|
||||
payload = canonical_json(
|
||||
@@ -956,6 +1244,10 @@ class CanonicalSearchService:
|
||||
rows: list[dict[str, object]] = [
|
||||
{"source": source.model_dump(mode="json")} for source in result.sources
|
||||
]
|
||||
source_count = self.index.count_sources(
|
||||
project_ids=project_ids,
|
||||
include_shared=request.scope.include_shared,
|
||||
)
|
||||
warnings: list[Finding] = []
|
||||
if result.index_revision != canonical_revision:
|
||||
warnings.append(
|
||||
@@ -977,6 +1269,7 @@ class CanonicalSearchService:
|
||||
index_revision=result.index_revision,
|
||||
budget_tokens=request.budget_tokens,
|
||||
retrieval_mode="keyword",
|
||||
filtered_count=source_count,
|
||||
next_cursor_for=(
|
||||
lambda included: (
|
||||
self._encode_cursor(offset + included, fingerprint, result.index_revision)
|
||||
|
||||
+113
-4
@@ -19,6 +19,7 @@ from gewu.adapters.source_extractor import LocalSourceExtractor, SourceExtractor
|
||||
from gewu.adapters.sqlite_facet_labels import SQLiteFacetLabelIndex
|
||||
from gewu.adapters.sqlite_identity import SQLiteIdentityStore
|
||||
from gewu.adapters.sqlite_ingest_reviews import SQLiteIngestReviewStore
|
||||
from gewu.adapters.sqlite_node_removals import SQLiteNodeRemovalStore
|
||||
from gewu.adapters.sqlite_search import SQLiteSearchIndex
|
||||
from gewu.adapters.sqlite_vectors import SQLiteVectorIndex
|
||||
from gewu.adapters.url_fetch import PublicUrlFetcher
|
||||
@@ -47,6 +48,9 @@ from gewu.application.search import (
|
||||
from gewu.application.uploads import LocalArtifactStore, UploadError
|
||||
from gewu.domain.contracts import (
|
||||
GitCommitId,
|
||||
KnowledgeBrowseEnvelope,
|
||||
KnowledgeGraphEnvelope,
|
||||
SourceRelatedNodesEnvelope,
|
||||
ProjectRecord,
|
||||
ReadEnvelope,
|
||||
RecordId,
|
||||
@@ -56,6 +60,8 @@ from gewu.transports.accounts_http import register_account_routes
|
||||
from gewu.transports.admin_http import register_admin_routes
|
||||
from gewu.transports.http_middleware import ProjectRegistryBodyLimitMiddleware
|
||||
from gewu.transports.http_models import (
|
||||
KnowledgeBrowseRequestBody,
|
||||
KnowledgeGraphRequestBody,
|
||||
NeighborRequestBody,
|
||||
OwnerSessionRequestBody,
|
||||
ProjectContentUpdateBody,
|
||||
@@ -65,6 +71,7 @@ from gewu.transports.http_models import (
|
||||
)
|
||||
from gewu.transports.ingest_reviews_http import register_ingest_review_routes
|
||||
from gewu.transports.mcp_server import create_mcp_server
|
||||
from gewu.transports.node_removals_http import register_node_removal_routes
|
||||
from gewu.transports.owner_mcp_server import OwnerReviewProvider, create_owner_mcp_server
|
||||
from gewu.version import SERVICE_VERSION
|
||||
|
||||
@@ -84,6 +91,7 @@ def create_app(
|
||||
credential_administration: CredentialAdministration | None = None,
|
||||
account_identity: AccountIdentity | None = None,
|
||||
ingest_review_store: SQLiteIngestReviewStore | None = None,
|
||||
node_removal_store: SQLiteNodeRemovalStore | None = None,
|
||||
jev_assessor: OpenRouterJev | None = None,
|
||||
facet_label_index: SQLiteFacetLabelIndex | None = None,
|
||||
embedding_provider: EmbeddingProvider | None = None,
|
||||
@@ -103,6 +111,7 @@ def create_app(
|
||||
or credential_administration is not None
|
||||
or account_identity is not None
|
||||
or ingest_review_store is not None
|
||||
or node_removal_store is not None
|
||||
):
|
||||
app.add_middleware(ProjectRegistryBodyLimitMiddleware)
|
||||
|
||||
@@ -163,6 +172,13 @@ def create_app(
|
||||
app, ingest_review_store, authenticate, require_same_origin
|
||||
)
|
||||
|
||||
if node_removal_store is not None:
|
||||
if node_reader is None:
|
||||
raise ValueError("Node removals require a canonical node reader")
|
||||
register_node_removal_routes(
|
||||
app, node_removal_store, node_reader, authenticate, require_same_origin
|
||||
)
|
||||
|
||||
if owner_identity_provider is not None:
|
||||
|
||||
@app.post(
|
||||
@@ -351,6 +367,11 @@ def create_app(
|
||||
}
|
||||
)
|
||||
|
||||
if node_removal_store is not None:
|
||||
http_operations.update(
|
||||
{"node_removal_list", "node_removal_request", "node_removal_restore"}
|
||||
)
|
||||
|
||||
if project_registry is not None:
|
||||
|
||||
@app.get(
|
||||
@@ -717,7 +738,17 @@ def create_app(
|
||||
}
|
||||
|
||||
if search_service is not None:
|
||||
http_operations.update({"list_projects", "search", "neighbors", "list_sources"})
|
||||
http_operations.update(
|
||||
{
|
||||
"list_projects",
|
||||
"search",
|
||||
"neighbors",
|
||||
"list_sources",
|
||||
"knowledge_browse",
|
||||
"knowledge_graph",
|
||||
"get_source_nodes",
|
||||
}
|
||||
)
|
||||
|
||||
@app.get("/v1/projects", tags=["knowledge"], operation_id="list_projects")
|
||||
def list_projects(
|
||||
@@ -747,6 +778,78 @@ def create_app(
|
||||
except SearchError as error:
|
||||
raise HTTPException(status_code=error.status_code, detail=str(error)) from error
|
||||
|
||||
@app.post(
|
||||
"/v1/knowledge/browse",
|
||||
response_model=KnowledgeBrowseEnvelope,
|
||||
tags=["knowledge"],
|
||||
operation_id="knowledge_browse",
|
||||
)
|
||||
def knowledge_browse(
|
||||
request: KnowledgeBrowseRequestBody,
|
||||
authorization: str | None = Header(default=None),
|
||||
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
|
||||
csrf_token: str | None = Header(default=None, alias="X-GEWU-CSRF"),
|
||||
) -> KnowledgeBrowseEnvelope:
|
||||
principal = authenticate(
|
||||
authorization, session_cookie, csrf_token, require_csrf=True
|
||||
)
|
||||
try:
|
||||
return search_service.browse(request.to_contract(), principal=principal)
|
||||
except SearchError as error:
|
||||
raise HTTPException(status_code=error.status_code, detail=str(error)) from error
|
||||
|
||||
@app.post(
|
||||
"/v1/knowledge/graph",
|
||||
response_model=KnowledgeGraphEnvelope,
|
||||
tags=["knowledge"],
|
||||
operation_id="knowledge_graph",
|
||||
)
|
||||
def knowledge_graph(
|
||||
request: KnowledgeGraphRequestBody,
|
||||
authorization: str | None = Header(default=None),
|
||||
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
|
||||
csrf_token: str | None = Header(default=None, alias="X-GEWU-CSRF"),
|
||||
) -> KnowledgeGraphEnvelope:
|
||||
principal = authenticate(
|
||||
authorization, session_cookie, csrf_token, require_csrf=True
|
||||
)
|
||||
try:
|
||||
return search_service.graph(request.to_contract(), principal=principal)
|
||||
except SearchError as error:
|
||||
raise HTTPException(status_code=error.status_code, detail=str(error)) from error
|
||||
|
||||
@app.get(
|
||||
"/v1/projects/{project_id}/sources/{revision_id}/nodes",
|
||||
response_model=SourceRelatedNodesEnvelope,
|
||||
tags=["knowledge"],
|
||||
operation_id="get_source_nodes",
|
||||
)
|
||||
def get_source_nodes(
|
||||
project_id: RecordId,
|
||||
revision_id: RecordId,
|
||||
query: str = "",
|
||||
node_type: str | None = None,
|
||||
include_shared: bool = False,
|
||||
limit: int = 20,
|
||||
cursor: str | None = None,
|
||||
authorization: str | None = Header(default=None),
|
||||
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
|
||||
) -> SourceRelatedNodesEnvelope:
|
||||
principal = authenticate(authorization, session_cookie)
|
||||
try:
|
||||
return search_service.source_nodes(
|
||||
project_id=project_id,
|
||||
revision_id=revision_id,
|
||||
principal=principal,
|
||||
include_shared=include_shared,
|
||||
query=query,
|
||||
node_type=node_type,
|
||||
limit=limit,
|
||||
cursor=cursor,
|
||||
)
|
||||
except SearchError as error:
|
||||
raise HTTPException(status_code=error.status_code, detail=str(error)) from error
|
||||
|
||||
@app.post(
|
||||
"/v1/neighbors",
|
||||
response_model=ReadEnvelope,
|
||||
@@ -891,6 +994,7 @@ def create_local_app() -> FastAPI:
|
||||
)
|
||||
artifacts = LocalArtifactStore(data_root)
|
||||
ingest_reviews = SQLiteIngestReviewStore(data_root / "ingest-reviews.sqlite3")
|
||||
node_removals = SQLiteNodeRemovalStore(data_root / "node-removals.sqlite3")
|
||||
allowed_fetch_hosts = {
|
||||
host.strip()
|
||||
for host in os.environ.get("GEWU_FETCH_ALLOWED_HOSTS", "").split(",")
|
||||
@@ -929,7 +1033,9 @@ def create_local_app() -> FastAPI:
|
||||
embedding_provider = embeddings
|
||||
facet_label_index = SQLiteFacetLabelIndex(search_path)
|
||||
vectors = SQLiteVectorIndex(search_path)
|
||||
lexical_index = SQLiteSearchIndex(search_path, repository)
|
||||
lexical_index = SQLiteSearchIndex(
|
||||
search_path, repository, node_removals_path=node_removals.path
|
||||
)
|
||||
hybrid_index = HybridSearchIndex(
|
||||
lexical=lexical_index,
|
||||
vectors=vectors,
|
||||
@@ -942,7 +1048,9 @@ def create_local_app() -> FastAPI:
|
||||
)
|
||||
search_index = hybrid_index
|
||||
else:
|
||||
search_index = SQLiteSearchIndex(search_path, repository)
|
||||
search_index = SQLiteSearchIndex(
|
||||
search_path, repository, node_removals_path=node_removals.path
|
||||
)
|
||||
search_service = CanonicalSearchService(
|
||||
index=search_index,
|
||||
project_ids_provider=project_registry,
|
||||
@@ -968,6 +1076,7 @@ def create_local_app() -> FastAPI:
|
||||
credential_administration=identities,
|
||||
account_identity=identities,
|
||||
ingest_review_store=ingest_reviews,
|
||||
node_removal_store=node_removals,
|
||||
jev_assessor=(
|
||||
OpenRouterJev(
|
||||
os.environ["OPENROUTER_API_KEY"],
|
||||
@@ -978,7 +1087,7 @@ def create_local_app() -> FastAPI:
|
||||
),
|
||||
owner_identity_provider=identities,
|
||||
owner_service_audience=os.environ.get("GEWU_OWNER_SERVICE_AUDIENCE"),
|
||||
node_reader=CanonicalNodeReader(repository),
|
||||
node_reader=CanonicalNodeReader(repository, node_removals.is_hidden),
|
||||
artifact_store=artifacts,
|
||||
extractor=LocalSourceExtractor(),
|
||||
acceptance_service=acceptance,
|
||||
|
||||
@@ -2,11 +2,13 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import re
|
||||
import unicodedata
|
||||
from datetime import date, datetime
|
||||
from enum import StrEnum
|
||||
from typing import Annotated, Literal
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from pydantic import (
|
||||
BaseModel,
|
||||
@@ -475,12 +477,38 @@ class ExtractionRecord(StrictRecord):
|
||||
warnings: tuple[str, ...] = ()
|
||||
|
||||
|
||||
class OnlineEvidence(StrictRecord):
|
||||
"""A short online excerpt with its relationship to a claim and verification URL."""
|
||||
|
||||
url: str = Field(min_length=1, max_length=4000)
|
||||
text: str = Field(min_length=1, max_length=8000)
|
||||
relation: Literal["supports", "contradicts", "context", "inconclusive"]
|
||||
context: str = Field(min_length=1, max_length=4000)
|
||||
|
||||
@field_validator("url")
|
||||
@classmethod
|
||||
def public_http_url(cls, value: str) -> str:
|
||||
parsed = urlsplit(value)
|
||||
hostname = parsed.hostname
|
||||
if parsed.scheme not in {"http", "https"} or not hostname or parsed.username or parsed.password:
|
||||
raise ValueError("online evidence requires an HTTP(S) URL without embedded credentials")
|
||||
if hostname.lower() == "localhost" or hostname.lower().endswith((".localhost", ".local")):
|
||||
raise ValueError("web references must use a public hostname")
|
||||
try:
|
||||
address = ipaddress.ip_address(hostname)
|
||||
except ValueError:
|
||||
address = None
|
||||
if address is not None and not address.is_global:
|
||||
raise ValueError("web references must use a public hostname")
|
||||
return value
|
||||
|
||||
class Claim(StrictRecord):
|
||||
claim_id: RecordId
|
||||
text: str = Field(min_length=1, max_length=4000)
|
||||
evidence_type: Literal["measured", "reported", "opinion", "vendor_marketing"]
|
||||
evidence: tuple[EvidenceReference, ...] = Field(min_length=1, max_length=20)
|
||||
assessment: SupportAssessment
|
||||
online_evidence: tuple[OnlineEvidence, ...] = Field(default=(), max_length=20)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def assessment_matches_evidence(self) -> Claim:
|
||||
@@ -660,7 +688,9 @@ class TypedEdge(StrictRecord):
|
||||
|
||||
|
||||
class ProposalOperation(StrictRecord):
|
||||
operation: Literal["register_source", "upsert_node", "add_edge", "flag_stale"]
|
||||
operation: Literal[
|
||||
"register_source", "upsert_node", "update_node", "add_edge", "flag_stale"
|
||||
]
|
||||
source: SourceRevisionRecord | None = None
|
||||
node: KnowledgeNode | None = None
|
||||
edge: TypedEdge | None = None
|
||||
@@ -671,6 +701,7 @@ class ProposalOperation(StrictRecord):
|
||||
payloads = {
|
||||
"register_source": self.source,
|
||||
"upsert_node": self.node,
|
||||
"update_node": self.node,
|
||||
"add_edge": self.edge,
|
||||
}
|
||||
expected = payloads.get(self.operation)
|
||||
@@ -682,8 +713,11 @@ class ProposalOperation(StrictRecord):
|
||||
)
|
||||
elif supplied != 1 or expected is None:
|
||||
raise ValueError("operation must carry exactly its matching typed payload")
|
||||
elif self.operation == "update_node":
|
||||
if self.target_id is None or self.node is None or self.target_id != self.node.node_id:
|
||||
raise ValueError("update_node target_id must match the stable node ID")
|
||||
elif self.target_id is not None:
|
||||
raise ValueError("target_id is only valid for flag_stale")
|
||||
raise ValueError("target_id is only valid for update_node or flag_stale")
|
||||
return self
|
||||
|
||||
|
||||
@@ -742,6 +776,7 @@ class SearchRequest(StrictRecord):
|
||||
limit: int = Field(default=20, ge=1, le=50)
|
||||
cursor: str | None = Field(default=None, max_length=512)
|
||||
budget_tokens: int = Field(ge=256, le=50_000)
|
||||
include_evidence: bool = True
|
||||
|
||||
@field_validator("query")
|
||||
@classmethod
|
||||
@@ -781,6 +816,111 @@ class SearchRequest(StrictRecord):
|
||||
return value
|
||||
|
||||
|
||||
class KnowledgeBrowseRequest(StrictRecord):
|
||||
"""Deterministic, paged project browsing independent of retrieval search."""
|
||||
|
||||
scope: Scope
|
||||
query: str = Field(default="", max_length=1000)
|
||||
node_types: tuple[NodeType, ...] = Field(default=(), max_length=9)
|
||||
tags: tuple[SearchFacetValue, ...] = Field(default=(), max_length=20)
|
||||
audience: tuple[SearchFacetValue, ...] = Field(default=(), max_length=20)
|
||||
use_for: tuple[SearchFacetValue, ...] = Field(default=(), max_length=20)
|
||||
domain: tuple[SearchFacetValue, ...] = Field(default=(), max_length=20)
|
||||
confidences: tuple[Confidence, ...] = Field(default=(), max_length=3)
|
||||
as_of_from: date | None = None
|
||||
as_of_to: date | None = None
|
||||
stale_only: bool = False
|
||||
order: Literal["title_asc", "title_desc", "date_newest", "date_oldest", "relevance"] = "title_asc"
|
||||
limit: int = Field(default=20, ge=1, le=50)
|
||||
cursor: str | None = Field(default=None, max_length=512)
|
||||
budget_tokens: int = Field(ge=256, le=50_000)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_browse(self) -> KnowledgeBrowseRequest:
|
||||
if self.as_of_from is not None and self.as_of_to is not None and self.as_of_from > self.as_of_to:
|
||||
raise ValueError("as_of_from must not be later than as_of_to")
|
||||
if self.order == "relevance" and not self.query:
|
||||
raise ValueError("relevance ordering requires a search query")
|
||||
return self
|
||||
|
||||
@field_validator("node_types")
|
||||
@classmethod
|
||||
def browse_node_types_are_unique(cls, value: tuple[NodeType, ...]) -> tuple[NodeType, ...]:
|
||||
if len(value) != len(set(value)):
|
||||
raise ValueError("browse node types must be unique")
|
||||
return value
|
||||
|
||||
@field_validator("confidences")
|
||||
@classmethod
|
||||
def browse_confidences_are_unique(cls, value: tuple[Confidence, ...]) -> tuple[Confidence, ...]:
|
||||
if len(value) != len(set(value)):
|
||||
raise ValueError("browse confidences must be unique")
|
||||
return value
|
||||
|
||||
@field_validator("tags", "audience", "use_for", "domain")
|
||||
@classmethod
|
||||
def browse_facets_are_unique_and_trimmed(cls, value: tuple[str, ...]) -> tuple[str, ...]:
|
||||
if any(item != item.strip() for item in value) or len(value) != len(set(value)):
|
||||
raise ValueError("browse facet values must be unique and trimmed")
|
||||
return value
|
||||
|
||||
|
||||
class KnowledgeBrowseEnvelope(StrictRecord):
|
||||
"""A bounded browse page with scope-safe project metadata."""
|
||||
|
||||
api_version: Literal["v1"] = "v1"
|
||||
request_id: RecordId
|
||||
resolved_scope: Scope
|
||||
nodes: tuple[KnowledgeNode, ...]
|
||||
project_node_count: int = Field(ge=0)
|
||||
project_source_count: int = Field(ge=0)
|
||||
shared_node_count: int = Field(ge=0)
|
||||
shared_source_count: int = Field(ge=0)
|
||||
filtered_node_count: int = Field(ge=0)
|
||||
facets: dict[str, dict[str, int]]
|
||||
canonical_revision: GitCommitId | None
|
||||
index_revision: GitCommitId | None
|
||||
warnings: tuple[Finding, ...] = ()
|
||||
next_cursor: str | None = None
|
||||
|
||||
|
||||
class KnowledgeGraphRequest(StrictRecord):
|
||||
scope: Scope
|
||||
max_nodes: int = Field(default=200, ge=1, le=200)
|
||||
max_edges: int = Field(default=500, ge=1, le=500)
|
||||
center_node_id: RecordId | None = None
|
||||
|
||||
|
||||
class KnowledgeGraphEnvelope(StrictRecord):
|
||||
api_version: Literal["v1"] = "v1"
|
||||
request_id: RecordId
|
||||
resolved_scope: Scope
|
||||
nodes: tuple[KnowledgeNode, ...]
|
||||
edges: tuple[TypedEdge, ...]
|
||||
truncated: bool
|
||||
center_node_id: RecordId | None = None
|
||||
canonical_revision: GitCommitId | None
|
||||
index_revision: GitCommitId | None
|
||||
warnings: tuple[Finding, ...] = ()
|
||||
|
||||
class SourceRelatedNode(StrictRecord):
|
||||
node: KnowledgeNode
|
||||
relation_basis: Literal["citation", "edge", "both"]
|
||||
|
||||
|
||||
class SourceRelatedNodesEnvelope(StrictRecord):
|
||||
api_version: Literal["v1"] = "v1"
|
||||
request_id: RecordId
|
||||
resolved_scope: Scope
|
||||
source_revision_id: RecordId
|
||||
nodes: tuple[SourceRelatedNode, ...]
|
||||
total: int = Field(ge=0)
|
||||
canonical_revision: GitCommitId | None
|
||||
index_revision: GitCommitId | None
|
||||
warnings: tuple[Finding, ...] = ()
|
||||
next_cursor: str | None = None
|
||||
|
||||
|
||||
class NeighborRequest(StrictRecord):
|
||||
node_id: RecordId
|
||||
scope: Scope
|
||||
@@ -1114,6 +1254,7 @@ class ReadEnvelope(StrictRecord):
|
||||
request_id: RecordId
|
||||
resolved_scope: Scope
|
||||
data: tuple[dict[str, object], ...]
|
||||
filtered_count: int | None = Field(default=None, ge=0)
|
||||
citations: tuple[EvidenceReference, ...] = ()
|
||||
warnings: tuple[Finding, ...] = ()
|
||||
omissions: tuple[Finding, ...] = ()
|
||||
|
||||
@@ -52,13 +52,19 @@ OPERATIONS: Final[tuple[Operation, ...]] = (
|
||||
Operation("get_project_context", "knowledge:read", frozenset({Transport.MCP})),
|
||||
Operation("ingest_review_list", "ingest:review", frozenset({Transport.HTTP})),
|
||||
Operation("ingest_review_decide", "ingest:review", frozenset({Transport.HTTP})),
|
||||
Operation("node_removal_list", "knowledge:read", frozenset({Transport.HTTP})),
|
||||
Operation("node_removal_request", "knowledge:moderate", frozenset({Transport.HTTP})),
|
||||
Operation("node_removal_restore", "knowledge:moderate", frozenset({Transport.HTTP})),
|
||||
Operation("search", "knowledge:read", frozenset({Transport.HTTP, Transport.MCP})),
|
||||
Operation("knowledge_browse", "knowledge:read", frozenset({Transport.HTTP})),
|
||||
Operation("knowledge_graph", "knowledge:read", frozenset({Transport.HTTP})),
|
||||
Operation("list_sources", "knowledge:read", frozenset({Transport.HTTP, Transport.MCP})),
|
||||
Operation("get_node", "knowledge:read", frozenset({Transport.HTTP, Transport.MCP})),
|
||||
Operation("node_history", "knowledge:read", frozenset({Transport.HTTP, Transport.MCP})),
|
||||
Operation("get_source", "knowledge:read", frozenset({Transport.HTTP, Transport.MCP})),
|
||||
Operation("get_source_artifact", "knowledge:read", frozenset({Transport.HTTP})),
|
||||
Operation("get_source_location", "knowledge:read", frozenset({Transport.HTTP})),
|
||||
Operation("get_source_nodes", "knowledge:read", frozenset({Transport.HTTP})),
|
||||
Operation("neighbors", "knowledge:read", frozenset({Transport.HTTP, Transport.MCP})),
|
||||
Operation("source_capture_url", "proposal:submit", frozenset({Transport.MCP})),
|
||||
Operation("source_capture_text", "proposal:submit", frozenset({Transport.MCP})),
|
||||
|
||||
@@ -4,6 +4,7 @@ from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import unicodedata
|
||||
from collections.abc import Iterable
|
||||
from copy import deepcopy
|
||||
@@ -51,6 +52,12 @@ def evidence_digest(evidence: EvidenceReference) -> str:
|
||||
return hashlib.sha256(canonical_json(evidence.model_dump(mode="json"))).hexdigest()
|
||||
|
||||
|
||||
def _numeric_tokens(text: str) -> set[str]:
|
||||
"""Return normalized numeric literals for a conservative evidence check."""
|
||||
matches = re.findall(r"(?<!\w)\d+(?:,\d{3})*(?:\.\d+)?", text)
|
||||
return {match.replace(",", "") for match in matches}
|
||||
|
||||
|
||||
def extraction_digest(extraction: ExtractionRecord) -> str:
|
||||
value = extraction.model_dump(mode="json", exclude={"extraction_id"})
|
||||
return hashlib.sha256(canonical_json(value)).hexdigest()
|
||||
@@ -58,6 +65,18 @@ def extraction_digest(extraction: ExtractionRecord) -> str:
|
||||
|
||||
def proposal_payload_digest(proposal: ProposalRevision) -> str:
|
||||
operations = [item.model_dump(mode="json") for item in proposal.operations]
|
||||
|
||||
def omit_empty_online_evidence(value: Any) -> None:
|
||||
if isinstance(value, dict):
|
||||
if value.get("online_evidence") == []:
|
||||
value.pop("online_evidence")
|
||||
for child in value.values():
|
||||
omit_empty_online_evidence(child)
|
||||
elif isinstance(value, list):
|
||||
for child in value:
|
||||
omit_empty_online_evidence(child)
|
||||
|
||||
omit_empty_online_evidence(operations)
|
||||
return hashlib.sha256(canonical_json(operations)).hexdigest()
|
||||
|
||||
|
||||
@@ -112,7 +131,12 @@ def prepare_proposal_digests(draft: dict[str, Any]) -> ProposalRevision:
|
||||
raise ValueError("Evidence references must be objects")
|
||||
item["assertion_path"] = "edge"
|
||||
item["assertion_digest"] = "0" * 64
|
||||
parsed_edge = TypedEdge.model_validate(edge, strict=True)
|
||||
# The draft originated as JSON, where arrays represent tuple fields.
|
||||
# Revalidate through the JSON path so strict mode accepts the typed
|
||||
# edge evidence array without weakening Python-object validation.
|
||||
parsed_edge = TypedEdge.model_validate_json(
|
||||
json.dumps(edge, ensure_ascii=False, allow_nan=False), strict=True
|
||||
)
|
||||
digest = edge_assertion_digest(parsed_edge)
|
||||
for item in edge_evidence:
|
||||
item["assertion_digest"] = digest
|
||||
@@ -176,6 +200,71 @@ def _entity_names(node: KnowledgeNode) -> set[str]:
|
||||
}
|
||||
|
||||
|
||||
def _validate_node_enrichment(
|
||||
previous: KnowledgeNode,
|
||||
updated: KnowledgeNode,
|
||||
proposer_id: str,
|
||||
path: str,
|
||||
) -> tuple[Finding, ...]:
|
||||
"""Require an update to preserve accepted knowledge and add only assessed material."""
|
||||
findings: list[Finding] = []
|
||||
for field in ("node_id", "project_id", "node_type", "created", "lifecycle"):
|
||||
if getattr(previous, field) != getattr(updated, field):
|
||||
findings.append(
|
||||
_finding("NODE_ENRICHMENT_IDENTITY_CHANGED", f"Enrichment cannot change {field}", path)
|
||||
)
|
||||
if updated.updated <= previous.updated:
|
||||
findings.append(
|
||||
_finding("NODE_ENRICHMENT_TIMESTAMP_INVALID", "Enrichment must advance updated", path)
|
||||
)
|
||||
if updated.summary != previous.summary:
|
||||
findings.append(
|
||||
_finding(
|
||||
"NODE_ENRICHMENT_SUMMARY_CHANGED",
|
||||
"Enrichment cannot replace the summary; add supported claims instead",
|
||||
f"{path}.summary",
|
||||
)
|
||||
)
|
||||
for field in ("claims", "assumptions", "limitations"):
|
||||
old_items = {claim.claim_id: claim for claim in getattr(previous, field)}
|
||||
new_items = {claim.claim_id: claim for claim in getattr(updated, field)}
|
||||
if not old_items.keys() <= new_items.keys():
|
||||
findings.append(
|
||||
_finding("NODE_ENRICHMENT_REMOVED_CLAIM", "Enrichment cannot remove accepted claims", f"{path}.{field}")
|
||||
)
|
||||
continue
|
||||
for claim_id, old_claim in old_items.items():
|
||||
new_claim = new_items[claim_id]
|
||||
claim_path = f"{path}.{field}.{claim_id}"
|
||||
if (
|
||||
old_claim.text != new_claim.text
|
||||
or old_claim.evidence_type != new_claim.evidence_type
|
||||
):
|
||||
findings.append(
|
||||
_finding("NODE_ENRICHMENT_CHANGED_CLAIM", "Enrichment cannot alter an accepted assertion", claim_path)
|
||||
)
|
||||
continue
|
||||
if not set(old_claim.online_evidence) <= set(new_claim.online_evidence):
|
||||
findings.append(
|
||||
_finding("NODE_ENRICHMENT_REMOVED_ONLINE_EVIDENCE", "Enrichment cannot remove prior online evidence", claim_path)
|
||||
)
|
||||
old_evidence = set(old_claim.evidence)
|
||||
if not old_evidence <= set(new_claim.evidence):
|
||||
findings.append(
|
||||
_finding("NODE_ENRICHMENT_REMOVED_EVIDENCE", "Enrichment cannot remove accepted evidence", claim_path)
|
||||
)
|
||||
if new_claim.evidence != old_claim.evidence and new_claim.assessment.assessor_id != proposer_id:
|
||||
findings.append(
|
||||
_finding("ASSESSOR_ATTRIBUTION_MISMATCH", "New evidence requires an assessment by its proposer", claim_path)
|
||||
)
|
||||
if not set(previous.tags) <= set(updated.tags):
|
||||
findings.append(_finding("NODE_ENRICHMENT_REMOVED_FACET", "Enrichment cannot remove existing tags", f"{path}.tags"))
|
||||
for field in ("audience", "domain", "use_for"):
|
||||
if not set(getattr(previous, field)) <= set(getattr(updated, field)):
|
||||
findings.append(_finding("NODE_ENRICHMENT_REMOVED_FACET", "Enrichment cannot remove existing facets", f"{path}.{field}"))
|
||||
return tuple(findings)
|
||||
|
||||
|
||||
def _entity_visible(
|
||||
candidate: KnowledgeNode,
|
||||
existing: KnowledgeNode,
|
||||
@@ -220,7 +309,7 @@ def validate_proposal(
|
||||
proposed_nodes = {
|
||||
operation.node.node_id: operation.node
|
||||
for operation in proposal.operations
|
||||
if operation.node is not None
|
||||
if operation.node is not None and operation.operation == "upsert_node"
|
||||
}
|
||||
known_nodes = {**existing_nodes, **proposed_nodes}
|
||||
entity_nodes_seen: list[KnowledgeNode] = []
|
||||
@@ -242,7 +331,10 @@ def validate_proposal(
|
||||
)
|
||||
)
|
||||
known_sources[source.revision_id] = source
|
||||
source_domains = {"shared", *source.ingested_for}
|
||||
# Source relevance is explicit: a source ingested for one project
|
||||
# does not become shared knowledge merely because its record is
|
||||
# stored in the canonical source registry.
|
||||
source_domains = set(source.ingested_for)
|
||||
for project_id in source.ingested_for:
|
||||
if project_id != "shared" and project_id not in projects:
|
||||
findings.append(
|
||||
@@ -269,8 +361,13 @@ def validate_proposal(
|
||||
|
||||
if operation.node is not None:
|
||||
node = operation.node
|
||||
previous_node = (
|
||||
existing_nodes.get(operation.target_id or "")
|
||||
if operation.operation == "update_node"
|
||||
else None
|
||||
)
|
||||
eligibility_checks.append(((node.project_id,), "node", node.node_type.value))
|
||||
if node.node_id in existing_nodes:
|
||||
if operation.operation == "upsert_node" and node.node_id in existing_nodes:
|
||||
findings.append(
|
||||
_finding(
|
||||
"NODE_ID_EXISTS",
|
||||
@@ -278,6 +375,15 @@ def validate_proposal(
|
||||
f"{path}.node.node_id",
|
||||
)
|
||||
)
|
||||
elif operation.operation == "update_node":
|
||||
if previous_node is None:
|
||||
findings.append(
|
||||
_finding("NODE_NOT_FOUND", "Update target is not an accepted node", f"{path}.target_id")
|
||||
)
|
||||
else:
|
||||
findings.extend(
|
||||
_validate_node_enrichment(previous_node, node, proposal.proposer_id, f"{path}.node")
|
||||
)
|
||||
if node.node_type.value == "entity":
|
||||
possible_duplicates = [
|
||||
other
|
||||
@@ -325,7 +431,20 @@ def validate_proposal(
|
||||
)
|
||||
for group_name, claim_index, claim in claim_items:
|
||||
claim_path = f"{path}.node.{group_name}[{claim_index}]"
|
||||
if claim.assessment.assessor_id != proposal.proposer_id:
|
||||
previous_claims = (
|
||||
{item.claim_id: item for item in previous_node.evidence_claims}
|
||||
if previous_node is not None
|
||||
else {}
|
||||
)
|
||||
old_claim = previous_claims.get(claim.claim_id)
|
||||
unchanged_prior_assessment = bool(
|
||||
old_claim
|
||||
and old_claim.text == claim.text
|
||||
and old_claim.evidence_type == claim.evidence_type
|
||||
and old_claim.evidence == claim.evidence
|
||||
and old_claim.assessment == claim.assessment
|
||||
)
|
||||
if claim.assessment.assessor_id != proposal.proposer_id and not unchanged_prior_assessment:
|
||||
findings.append(
|
||||
_finding(
|
||||
"ASSESSOR_ATTRIBUTION_MISMATCH",
|
||||
@@ -362,6 +481,17 @@ def validate_proposal(
|
||||
claim_path,
|
||||
)
|
||||
)
|
||||
quoted_text = "\n".join(evidence.quote for evidence in claim.evidence)
|
||||
unquoted_numbers = _numeric_tokens(claim.text) - _numeric_tokens(quoted_text)
|
||||
if unquoted_numbers:
|
||||
findings.append(
|
||||
_finding(
|
||||
"NUMERIC_CLAIM_UNGROUNDED",
|
||||
"Claim contains numeric literals absent from attached evidence; "
|
||||
"semantic review is required",
|
||||
claim_path,
|
||||
)
|
||||
)
|
||||
for evidence_index, evidence in enumerate(claim.evidence):
|
||||
evidence_path = f"{claim_path}.evidence[{evidence_index}]"
|
||||
expected_assertion_path = f"{group_name}[{claim_index}].text"
|
||||
@@ -636,7 +766,7 @@ def validate_proposal(
|
||||
)
|
||||
)
|
||||
|
||||
if operation.target_id is not None:
|
||||
if operation.operation == "flag_stale" and operation.target_id is not None:
|
||||
target = existing_nodes.get(operation.target_id)
|
||||
if target is None:
|
||||
findings.append(
|
||||
@@ -720,6 +850,15 @@ def validate_proposal(
|
||||
"SOURCE_REVISION_CONFLICT",
|
||||
"PROJECT_NOT_REGISTERED",
|
||||
"NODE_ID_EXISTS",
|
||||
"NODE_NOT_FOUND",
|
||||
"NODE_ENRICHMENT_IDENTITY_CHANGED",
|
||||
"NODE_ENRICHMENT_TIMESTAMP_INVALID",
|
||||
"NODE_ENRICHMENT_SUMMARY_CHANGED",
|
||||
"NODE_ENRICHMENT_REMOVED_CLAIM",
|
||||
"NODE_ENRICHMENT_CHANGED_CLAIM",
|
||||
"NODE_ENRICHMENT_REMOVED_ONLINE_EVIDENCE",
|
||||
"NODE_ENRICHMENT_REMOVED_EVIDENCE",
|
||||
"NODE_ENRICHMENT_REMOVED_FACET",
|
||||
"ASSESSMENT_STALE",
|
||||
"ASSESSMENT_EVIDENCE_MISMATCH",
|
||||
"SUPPORT_UNAVAILABLE",
|
||||
|
||||
@@ -11,6 +11,8 @@ from gewu.domain.contracts import (
|
||||
AutoAcceptPolicy,
|
||||
Confidence,
|
||||
NeighborRequest,
|
||||
KnowledgeBrowseRequest,
|
||||
KnowledgeGraphRequest,
|
||||
NodeType,
|
||||
ProjectRecord,
|
||||
RecordId,
|
||||
@@ -46,6 +48,7 @@ class SearchRequestBody(BaseModel):
|
||||
limit: int = Field(default=20, ge=1, le=50)
|
||||
cursor: str | None = Field(default=None, max_length=512)
|
||||
budget_tokens: int = Field(ge=256, le=50_000)
|
||||
include_evidence: bool = True
|
||||
|
||||
@field_validator("query")
|
||||
@classmethod
|
||||
@@ -95,6 +98,57 @@ class SearchRequestBody(BaseModel):
|
||||
return SearchRequest.model_validate_json(self.model_dump_json(), strict=True)
|
||||
|
||||
|
||||
class KnowledgeBrowseRequestBody(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", strict=True, frozen=True)
|
||||
|
||||
query: str = Field(default="", max_length=1000)
|
||||
scope: SearchScopeBody
|
||||
node_types: list[str] = Field(default_factory=list, max_length=9)
|
||||
tags: list[SearchFacetValue] = Field(default_factory=list, max_length=20)
|
||||
audience: list[SearchFacetValue] = Field(default_factory=list, max_length=20)
|
||||
use_for: list[SearchFacetValue] = Field(default_factory=list, max_length=20)
|
||||
domain: list[SearchFacetValue] = Field(default_factory=list, max_length=20)
|
||||
confidences: list[Confidence] = Field(default_factory=list, max_length=3)
|
||||
as_of_from: date | None = None
|
||||
as_of_to: date | None = None
|
||||
stale_only: bool = False
|
||||
order: str = "title_asc"
|
||||
limit: int = Field(default=20, ge=1, le=50)
|
||||
cursor: str | None = Field(default=None, max_length=512)
|
||||
budget_tokens: int = Field(default=12000, ge=256, le=50_000)
|
||||
|
||||
@field_validator("query")
|
||||
@classmethod
|
||||
def normalize_query(cls, value: str) -> str:
|
||||
return value.strip()
|
||||
|
||||
@field_validator("node_types")
|
||||
@classmethod
|
||||
def validate_node_types(cls, values: list[str]) -> list[str]:
|
||||
try:
|
||||
normalized = [NodeType(value).value for value in values]
|
||||
except ValueError as error:
|
||||
raise ValueError("node_types contains an unsupported knowledge type") from error
|
||||
if len(normalized) != len(set(normalized)):
|
||||
raise ValueError("node_types must be unique")
|
||||
return normalized
|
||||
|
||||
def to_contract(self) -> KnowledgeBrowseRequest:
|
||||
return KnowledgeBrowseRequest.model_validate_json(self.model_dump_json(), strict=True)
|
||||
|
||||
|
||||
class KnowledgeGraphRequestBody(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", strict=True, frozen=True)
|
||||
|
||||
scope: SearchScopeBody
|
||||
max_nodes: int = Field(default=200, ge=1, le=200)
|
||||
max_edges: int = Field(default=500, ge=1, le=500)
|
||||
center_node_id: str | None = None
|
||||
|
||||
def to_contract(self) -> KnowledgeGraphRequest:
|
||||
return KnowledgeGraphRequest.model_validate_json(self.model_dump_json(), strict=True)
|
||||
|
||||
|
||||
class NeighborRequestBody(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", strict=True, frozen=True)
|
||||
|
||||
|
||||
@@ -372,9 +372,7 @@ def _validation_issues(error: ValidationError) -> list[dict[str, str]]:
|
||||
)[:64]:
|
||||
path = ""
|
||||
for part in detail.get("loc", ()):
|
||||
path += f"[{part}]" if isinstance(part, int) else (
|
||||
("." if path else "") + str(part)
|
||||
)
|
||||
path += f"[{part}]" if isinstance(part, int) else (("." if path else "") + str(part))
|
||||
issues.append(
|
||||
{
|
||||
"path": path,
|
||||
@@ -528,8 +526,9 @@ def create_mcp_server(
|
||||
as_of_from: str | None = None,
|
||||
as_of_to: str | None = None,
|
||||
stale_only: bool = False,
|
||||
include_evidence: bool = True,
|
||||
) -> str:
|
||||
"""Search accepted knowledge in an explicit, authorized project scope."""
|
||||
"""Search accepted knowledge in scope. Set include_evidence=false for compact claim summaries; the default includes full citations."""
|
||||
|
||||
principal = _principal(ctx, identity_provider)
|
||||
try:
|
||||
@@ -555,6 +554,7 @@ def create_mcp_server(
|
||||
stale_only=stale_only,
|
||||
cursor=cursor,
|
||||
budget_tokens=budget_tokens,
|
||||
include_evidence=include_evidence,
|
||||
)
|
||||
envelope = search_service.search(request, principal=principal)
|
||||
except SearchError as error:
|
||||
@@ -1253,7 +1253,7 @@ def create_mcp_server(
|
||||
ctx: Context,
|
||||
include_original_mapping: bool = False,
|
||||
) -> str:
|
||||
"""Read a bounded normalized text slice and optionally its original-source offset map."""
|
||||
"""Read a normalized slice with its profile and optional original-source offset map."""
|
||||
|
||||
principal = _principal(ctx, identity_provider)
|
||||
_require_project_submission(principal, project_id)
|
||||
@@ -1273,6 +1273,7 @@ def create_mcp_server(
|
||||
"block_id": block_id,
|
||||
"page": block.page,
|
||||
"offset": offset,
|
||||
"normalization_profile": extraction.normalization_profile,
|
||||
"normalized_text": block.normalized_text[offset:end],
|
||||
}
|
||||
if include_original_mapping:
|
||||
@@ -1495,7 +1496,10 @@ def create_mcp_server(
|
||||
principal = _principal(ctx, identity_provider)
|
||||
if "agent" not in principal.roles or principal.auth_method != "bearer":
|
||||
raise PermissionError("Durable runs require an agent bearer credential")
|
||||
request = _parse_run_request(request_json, RunCreateRequest)
|
||||
try:
|
||||
request = _parse_run_request(request_json, RunCreateRequest)
|
||||
except ValueError as error:
|
||||
return _invalid_run_request(error)
|
||||
_require_run_scope(principal, request.scope)
|
||||
try:
|
||||
record = run_service.create(
|
||||
@@ -1513,7 +1517,10 @@ def create_mcp_server(
|
||||
principal = _principal(ctx, identity_provider)
|
||||
if "agent" not in principal.roles or principal.auth_method != "bearer":
|
||||
raise PermissionError("Durable runs require an agent bearer credential")
|
||||
request = _parse_run_request(request_json, RunCheckpointRequest)
|
||||
try:
|
||||
request = _parse_run_request(request_json, RunCheckpointRequest)
|
||||
except ValueError as error:
|
||||
return _invalid_run_request(error)
|
||||
actor = ActorContext(principal.principal_id, principal.credential_id)
|
||||
try:
|
||||
current = run_service.status(request.run_id, actor=actor)
|
||||
@@ -1746,14 +1753,25 @@ def _parse_run_request[RequestT: (RunCreateRequest, RunCheckpointRequest)](
|
||||
) -> RequestT:
|
||||
if len(request_json.encode("utf-8")) > 1_000_000:
|
||||
raise ValueError("Run request exceeds the supported size")
|
||||
try:
|
||||
payload = json.loads(request_json, object_pairs_hook=_reject_duplicate_keys)
|
||||
request = request_type.model_validate_json(request_json, strict=True)
|
||||
except (ValueError, ValidationError) as error:
|
||||
raise ValueError("Run request does not match the GEWU v1 contract") from error
|
||||
payload = json.loads(request_json, object_pairs_hook=_reject_duplicate_keys)
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError("Run request must be a JSON object")
|
||||
return request
|
||||
return request_type.model_validate_json(request_json, strict=True)
|
||||
|
||||
|
||||
def _invalid_run_request(error: ValueError) -> str:
|
||||
issues = (
|
||||
_validation_issues(error)
|
||||
if isinstance(error, ValidationError)
|
||||
else [
|
||||
{
|
||||
"path": "",
|
||||
"code": "invalid_run_request",
|
||||
"message": "Run request must be a valid JSON object within the supported size",
|
||||
}
|
||||
]
|
||||
)
|
||||
return json.dumps({"status": "invalid", "issues": issues}, ensure_ascii=False)
|
||||
|
||||
|
||||
def _require_run_scope(principal: Principal, scope: Scope) -> None:
|
||||
@@ -1804,7 +1822,10 @@ def _authorize_proposal(
|
||||
affected: set[str] = set()
|
||||
for operation in proposal.operations:
|
||||
if operation.source is not None:
|
||||
affected.update(("shared", *operation.source.ingested_for))
|
||||
# A revision is project-visible through its explicit ingestion grants.
|
||||
# Requiring a shared write grant for every source registration would
|
||||
# block project-only agents and turn project evidence into shared data.
|
||||
affected.update(operation.source.ingested_for)
|
||||
if operation.node is not None:
|
||||
affected.add(operation.node.project_id)
|
||||
if operation.edge is not None:
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
"""Authenticated, project-scoped node removal and restore routes."""
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Annotated, Literal
|
||||
|
||||
from fastapi import Cookie, Depends, FastAPI, Header, HTTPException, Request, Response
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
from gewu.adapters.sqlite_node_removals import NodeRemoval, RemovalConflict, SQLiteNodeRemovalStore
|
||||
from gewu.application.identity import Principal
|
||||
from gewu.application.reading import CanonicalNodeReader, ReadError
|
||||
from gewu.domain.contracts import RecordId
|
||||
|
||||
|
||||
class NodeRemovalRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", strict=True)
|
||||
confirmed: Literal[True]
|
||||
|
||||
|
||||
class NodeRemovalItem(BaseModel):
|
||||
removal_id: str
|
||||
project_id: str
|
||||
node_id: str
|
||||
title: str
|
||||
requested_at: str
|
||||
restore_until: str
|
||||
|
||||
@classmethod
|
||||
def from_record(cls, record: NodeRemoval) -> "NodeRemovalItem":
|
||||
return cls(
|
||||
removal_id=record.removal_id,
|
||||
project_id=record.project_id,
|
||||
node_id=record.node_id,
|
||||
title=record.title,
|
||||
requested_at=record.requested_at,
|
||||
restore_until=record.restore_until,
|
||||
)
|
||||
|
||||
|
||||
class NodeRemovalList(BaseModel):
|
||||
can_manage: bool
|
||||
removals: list[NodeRemovalItem]
|
||||
|
||||
|
||||
def register_node_removal_routes(
|
||||
app: FastAPI,
|
||||
store: SQLiteNodeRemovalStore,
|
||||
reader: CanonicalNodeReader,
|
||||
authenticate: Callable[..., Principal],
|
||||
require_same_origin: Callable[[Request], None],
|
||||
) -> None:
|
||||
def human(
|
||||
request: Request,
|
||||
response: Response,
|
||||
authorization: str | None = Header(default=None),
|
||||
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
|
||||
csrf_token: str | None = Header(default=None, alias="X-GEWU-CSRF"),
|
||||
) -> Principal:
|
||||
response.headers["Cache-Control"] = "private, no-store"
|
||||
write = request.method not in {"GET", "HEAD"}
|
||||
if write:
|
||||
require_same_origin(request)
|
||||
principal = authenticate(authorization, session_cookie, csrf_token, require_csrf=write)
|
||||
if "agent" in principal.roles:
|
||||
raise HTTPException(403, "A human account is required")
|
||||
return principal
|
||||
|
||||
def require_manager(principal: Principal, project_id: str) -> None:
|
||||
if not principal.can_read_project(project_id):
|
||||
raise HTTPException(404, "Project removal queue is unavailable")
|
||||
if not ({"editor", "admin", "owner"} & principal.roles):
|
||||
raise HTTPException(403, "Editor role is required to remove or restore knowledge")
|
||||
|
||||
@app.get(
|
||||
"/v1/projects/{project_id}/node-removals",
|
||||
response_model=NodeRemovalList,
|
||||
tags=["knowledge"],
|
||||
operation_id="node_removal_list",
|
||||
)
|
||||
def list_removals(
|
||||
project_id: RecordId,
|
||||
principal: Annotated[Principal, Depends(human)],
|
||||
) -> NodeRemovalList:
|
||||
if not principal.can_read_project(project_id):
|
||||
raise HTTPException(404, "Project removal queue is unavailable")
|
||||
can_manage = bool({"editor", "admin", "owner"} & principal.roles)
|
||||
removals = store.list_restorable(project_id) if can_manage else ()
|
||||
return NodeRemovalList(
|
||||
can_manage=can_manage,
|
||||
removals=[NodeRemovalItem.from_record(record) for record in removals],
|
||||
)
|
||||
|
||||
@app.post(
|
||||
"/v1/projects/{project_id}/nodes/{node_id}/removal",
|
||||
response_model=NodeRemovalItem,
|
||||
status_code=201,
|
||||
tags=["knowledge"],
|
||||
operation_id="node_removal_request",
|
||||
)
|
||||
def request_removal(
|
||||
project_id: RecordId,
|
||||
node_id: RecordId,
|
||||
body: NodeRemovalRequest,
|
||||
principal: Annotated[Principal, Depends(human)],
|
||||
) -> NodeRemovalItem:
|
||||
require_manager(principal, project_id)
|
||||
try:
|
||||
envelope = reader.get_node(project_id, node_id, principal=principal)
|
||||
title = str(envelope.data[0]["title"])
|
||||
return NodeRemovalItem.from_record(
|
||||
store.request(project_id, node_id, title, principal.principal_id)
|
||||
)
|
||||
except ReadError as error:
|
||||
raise HTTPException(status_code=error.status_code, detail=str(error)) from error
|
||||
except RemovalConflict as error:
|
||||
raise HTTPException(status_code=409, detail=str(error)) from error
|
||||
except RuntimeError as error:
|
||||
raise HTTPException(status_code=503, detail=str(error)) from error
|
||||
|
||||
@app.post(
|
||||
"/v1/projects/{project_id}/nodes/{node_id}/restore-removal",
|
||||
response_model=NodeRemovalItem,
|
||||
tags=["knowledge"],
|
||||
operation_id="node_removal_restore",
|
||||
)
|
||||
def restore_removal(
|
||||
project_id: RecordId,
|
||||
node_id: RecordId,
|
||||
principal: Annotated[Principal, Depends(human)],
|
||||
) -> NodeRemovalItem:
|
||||
require_manager(principal, project_id)
|
||||
try:
|
||||
return NodeRemovalItem.from_record(
|
||||
store.restore(project_id, node_id, principal.principal_id)
|
||||
)
|
||||
except KeyError as error:
|
||||
raise HTTPException(status_code=404, detail="Removed node was not found") from error
|
||||
except RemovalConflict as error:
|
||||
raise HTTPException(status_code=410, detail=str(error)) from error
|
||||
except RuntimeError as error:
|
||||
raise HTTPException(status_code=503, detail=str(error)) from error
|
||||
Reference in new issue
Block a user