This commit is contained in:
1 parent
1c61c7ad04
commit
7b2c43ddaf
18 files changed
+1648
-2
No files matched your search
@@ -0,0 +1,125 @@
|
||||
"""Small fixed-endpoint client for optional OpenRouter Jev relevance judgments."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, cast
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.request import HTTPRedirectHandler, Request, build_opener
|
||||
|
||||
from gewu.domain.contracts import IngestReviewCreate, ProjectRecord, ResearchLens
|
||||
|
||||
ENDPOINT = "https://openrouter.ai/api/alpha/decisions"
|
||||
MODEL = "typesafe/jev-1.13"
|
||||
MAX_RESPONSE_BYTES = 32_000
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class JevAssessment:
|
||||
relation_probability: float
|
||||
lens: ResearchLens
|
||||
lens_confidence: float
|
||||
model: str
|
||||
|
||||
|
||||
class JevUnavailable(RuntimeError):
|
||||
"""Jev is not configured or returned an unusable response."""
|
||||
|
||||
|
||||
class _NoRedirect(HTTPRedirectHandler):
|
||||
def redirect_request(
|
||||
self, request: Any, response: Any, code: int, message: str, headers: Any, new_url: str
|
||||
) -> None:
|
||||
return None
|
||||
|
||||
|
||||
class OpenRouterJev:
|
||||
def __init__(self, api_key: str, *, model: str = MODEL) -> None:
|
||||
if not api_key or len(api_key) > 512 or any(char.isspace() for char in api_key):
|
||||
raise ValueError("OpenRouter API key is invalid")
|
||||
if model not in {MODEL, "~typesafe/jev-latest"}:
|
||||
raise ValueError("Unsupported Jev model identifier")
|
||||
self._api_key = api_key
|
||||
self._model = model
|
||||
|
||||
def assess(self, item: IngestReviewCreate, project: ProjectRecord) -> JevAssessment:
|
||||
state = {
|
||||
"project_title": project.title,
|
||||
"project_north_star": project.north_star,
|
||||
"project_questions": project.key_questions,
|
||||
"candidate_lens": item.lens,
|
||||
"candidate_summary": item.summary,
|
||||
"candidate_relevance_rationale": item.relevance_rationale,
|
||||
"source_excerpt": item.excerpt,
|
||||
}
|
||||
criteria = {
|
||||
"technical": "Engineering, implementation, technology, or operational practice.",
|
||||
"conceptual": "Ideas, theory, design concepts, or analytical frameworks.",
|
||||
"cultural-historical": "Cultural, artistic, historical, social, or audience context.",
|
||||
"financial-market": "Costs, revenues, business models, markets, or financing.",
|
||||
}
|
||||
payload = json.dumps(
|
||||
{
|
||||
"model": self._model,
|
||||
"state": state,
|
||||
"questions": {
|
||||
"project_relation": {
|
||||
"type": "noul",
|
||||
"instructions": (
|
||||
"Does this passage help answer a project question or explain "
|
||||
"the project's technical, conceptual, cultural-historical, or "
|
||||
"financial-market context? Treat indirect context as relevant when "
|
||||
"the connection is plausible."
|
||||
),
|
||||
"criteria": {
|
||||
"true": "A plausible, specific connection to the project exists.",
|
||||
"false": "The passage has no plausible project connection.",
|
||||
},
|
||||
},
|
||||
"best_lens": {
|
||||
"type": "choice",
|
||||
"instructions": "Which project research lens best describes this passage?",
|
||||
"criteria": criteria,
|
||||
},
|
||||
},
|
||||
},
|
||||
ensure_ascii=False,
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
request = Request(
|
||||
ENDPOINT,
|
||||
data=payload,
|
||||
headers={
|
||||
"Authorization": f"Bearer {self._api_key}",
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "application/json",
|
||||
},
|
||||
method="POST",
|
||||
)
|
||||
try:
|
||||
with build_opener(_NoRedirect()).open(request, timeout=12) as response:
|
||||
if response.status != 200:
|
||||
raise JevUnavailable("Jev request failed")
|
||||
raw = response.read(MAX_RESPONSE_BYTES + 1)
|
||||
except (HTTPError, URLError, TimeoutError, OSError) as error:
|
||||
raise JevUnavailable("Jev request failed") from error
|
||||
if len(raw) > MAX_RESPONSE_BYTES:
|
||||
raise JevUnavailable("Jev response exceeded its size limit")
|
||||
try:
|
||||
result = json.loads(raw)
|
||||
answers = result["answers"]
|
||||
relation = float(answers["project_relation"]["noul"])
|
||||
lens = answers["best_lens"]["choice"]
|
||||
confidence = float(answers["best_lens"]["confidence"])
|
||||
served_model = str(result["model"])
|
||||
except (KeyError, TypeError, ValueError, json.JSONDecodeError) as error:
|
||||
raise JevUnavailable("Jev response did not match its typed decision schema") from error
|
||||
if (
|
||||
not 0 <= relation <= 1
|
||||
or not 0 <= confidence <= 1
|
||||
or lens not in criteria
|
||||
or len(served_model) > 100
|
||||
):
|
||||
raise JevUnavailable("Jev response contained an invalid decision")
|
||||
return JevAssessment(relation, cast(ResearchLens, lens), confidence, served_model)
|
||||
@@ -0,0 +1,316 @@
|
||||
"""Durable, project-scoped user review queue for borderline ingest material."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import sqlite3
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
from gewu.application.paths import reject_symlink_components
|
||||
from gewu.domain.contracts import (
|
||||
IngestGuidance,
|
||||
IngestReviewCreate,
|
||||
IngestReviewDecision,
|
||||
IngestReviewItem,
|
||||
)
|
||||
|
||||
MAX_REVIEW_ITEMS = 100_000
|
||||
MAX_REVIEW_PAGE = 100
|
||||
|
||||
|
||||
class ReviewConflict(ValueError):
|
||||
"""An idempotency key was reused with different content or state."""
|
||||
|
||||
|
||||
class SQLiteIngestReviewStore:
|
||||
"""Store review candidates and immutable human decisions outside canonical knowledge."""
|
||||
|
||||
def __init__(self, database: Path) -> None:
|
||||
self.database = database.expanduser().absolute()
|
||||
reject_symlink_components(self.database)
|
||||
self.database.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
self.database.parent.chmod(0o700)
|
||||
if self.database.is_symlink():
|
||||
raise RuntimeError("Ingest review database must not be a symbolic link")
|
||||
with self._connect() as connection:
|
||||
connection.execute(
|
||||
"""CREATE TABLE IF NOT EXISTS ingest_review_items (
|
||||
review_id TEXT PRIMARY KEY,
|
||||
project_id TEXT NOT NULL,
|
||||
run_id TEXT NOT NULL,
|
||||
source_revision_id TEXT NOT NULL,
|
||||
extraction_id TEXT NOT NULL,
|
||||
locator_json TEXT NOT NULL,
|
||||
excerpt TEXT NOT NULL,
|
||||
lens TEXT NOT NULL,
|
||||
summary TEXT NOT NULL,
|
||||
relevance_rationale TEXT NOT NULL,
|
||||
relevance TEXT NOT NULL CHECK(relevance IN ('borderline', 'low')),
|
||||
jev_relation_probability REAL,
|
||||
jev_lens TEXT,
|
||||
jev_lens_confidence REAL,
|
||||
jev_model TEXT,
|
||||
status TEXT NOT NULL CHECK(status IN (
|
||||
'needs-user-review', 'included', 'excluded', 'deferred'
|
||||
)),
|
||||
submitted_by TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL,
|
||||
payload_digest TEXT NOT NULL
|
||||
)"""
|
||||
)
|
||||
existing_columns = {
|
||||
row[1] for row in connection.execute("PRAGMA table_info(ingest_review_items)")
|
||||
}
|
||||
for column, declaration in (
|
||||
("jev_relation_probability", "REAL"),
|
||||
("jev_lens", "TEXT"),
|
||||
("jev_lens_confidence", "REAL"),
|
||||
("jev_model", "TEXT"),
|
||||
):
|
||||
if column not in existing_columns:
|
||||
connection.execute(
|
||||
f"ALTER TABLE ingest_review_items ADD COLUMN {column} {declaration}"
|
||||
)
|
||||
connection.execute(
|
||||
"CREATE INDEX IF NOT EXISTS ingest_review_project_status_created "
|
||||
"ON ingest_review_items(project_id, status, created_at DESC, review_id)"
|
||||
)
|
||||
connection.execute(
|
||||
"""CREATE TABLE IF NOT EXISTS ingest_review_decisions (
|
||||
decision_id TEXT PRIMARY KEY,
|
||||
review_id TEXT NOT NULL REFERENCES ingest_review_items(review_id),
|
||||
project_id TEXT NOT NULL,
|
||||
decision TEXT NOT NULL CHECK(decision IN ('include', 'exclude', 'defer')),
|
||||
rationale TEXT NOT NULL,
|
||||
save_as_guidance INTEGER NOT NULL CHECK(save_as_guidance IN (0, 1)),
|
||||
guidance_rule TEXT,
|
||||
reviewer_id TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL,
|
||||
payload_digest TEXT NOT NULL
|
||||
)"""
|
||||
)
|
||||
connection.execute(
|
||||
"CREATE INDEX IF NOT EXISTS ingest_review_guidance "
|
||||
"ON ingest_review_decisions(project_id, save_as_guidance, created_at DESC)"
|
||||
)
|
||||
connection.execute(
|
||||
"""CREATE TRIGGER IF NOT EXISTS ingest_review_decisions_no_update
|
||||
BEFORE UPDATE ON ingest_review_decisions BEGIN
|
||||
SELECT RAISE(ABORT, 'review decisions are immutable');
|
||||
END"""
|
||||
)
|
||||
connection.execute(
|
||||
"""CREATE TRIGGER IF NOT EXISTS ingest_review_decisions_no_delete
|
||||
BEFORE DELETE ON ingest_review_decisions BEGIN
|
||||
SELECT RAISE(ABORT, 'review decisions are immutable');
|
||||
END"""
|
||||
)
|
||||
self.database.chmod(0o600)
|
||||
|
||||
def create(
|
||||
self,
|
||||
item: IngestReviewCreate,
|
||||
*,
|
||||
submitted_by: str,
|
||||
jev_relation_probability: float | None = None,
|
||||
jev_lens: str | None = None,
|
||||
jev_lens_confidence: float | None = None,
|
||||
jev_model: str | None = None,
|
||||
) -> IngestReviewItem:
|
||||
payload = item.model_dump(mode="json")
|
||||
digest = _digest({**payload, "submitted_by": submitted_by})
|
||||
created_at = datetime.now(UTC).isoformat()
|
||||
with self._connect() as connection:
|
||||
row = connection.execute(
|
||||
"SELECT payload_digest FROM ingest_review_items WHERE review_id=?",
|
||||
(item.review_id,),
|
||||
).fetchone()
|
||||
if row is not None:
|
||||
if row[0] != digest:
|
||||
raise ReviewConflict("Review ID was already used with different content")
|
||||
return self._read_item(connection, item.review_id)
|
||||
count = connection.execute("SELECT count(*) FROM ingest_review_items").fetchone()[0]
|
||||
if count >= MAX_REVIEW_ITEMS:
|
||||
raise ValueError("Ingest review queue is at its configured capacity")
|
||||
connection.execute(
|
||||
"""INSERT INTO ingest_review_items (
|
||||
review_id, project_id, run_id, source_revision_id, extraction_id,
|
||||
locator_json, excerpt, lens, summary, relevance_rationale, relevance,
|
||||
status, submitted_by, created_at, payload_digest,
|
||||
jev_relation_probability, jev_lens, jev_lens_confidence, jev_model
|
||||
) VALUES (
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'needs-user-review', ?, ?, ?, ?, ?, ?, ?
|
||||
)""",
|
||||
(
|
||||
item.review_id,
|
||||
item.project_id,
|
||||
item.run_id,
|
||||
item.source_revision_id,
|
||||
item.extraction_id,
|
||||
item.locator.model_dump_json(),
|
||||
item.excerpt,
|
||||
item.lens,
|
||||
item.summary,
|
||||
item.relevance_rationale,
|
||||
item.relevance,
|
||||
submitted_by,
|
||||
created_at,
|
||||
digest,
|
||||
jev_relation_probability,
|
||||
jev_lens,
|
||||
jev_lens_confidence,
|
||||
jev_model,
|
||||
),
|
||||
)
|
||||
return self._read_item(connection, item.review_id)
|
||||
|
||||
def list_items(
|
||||
self, project_id: str, *, status: str | None, limit: int, offset: int
|
||||
) -> list[IngestReviewItem]:
|
||||
if not 1 <= limit <= MAX_REVIEW_PAGE or offset < 0:
|
||||
raise ValueError("Review pagination is outside the supported range")
|
||||
with self._connect() as connection:
|
||||
if status is None:
|
||||
rows = connection.execute(
|
||||
"SELECT review_id FROM ingest_review_items WHERE project_id=? "
|
||||
"ORDER BY created_at DESC, review_id LIMIT ? OFFSET ?",
|
||||
(project_id, limit, offset),
|
||||
).fetchall()
|
||||
else:
|
||||
rows = connection.execute(
|
||||
"SELECT review_id FROM ingest_review_items WHERE project_id=? AND status=? "
|
||||
"ORDER BY created_at DESC, review_id LIMIT ? OFFSET ?",
|
||||
(project_id, status, limit, offset),
|
||||
).fetchall()
|
||||
return [self._read_item(connection, row[0]) for row in rows]
|
||||
|
||||
def get(self, review_id: str) -> IngestReviewItem | None:
|
||||
with self._connect() as connection:
|
||||
exists = connection.execute(
|
||||
"SELECT 1 FROM ingest_review_items WHERE review_id=?", (review_id,)
|
||||
).fetchone()
|
||||
return self._read_item(connection, review_id) if exists else None
|
||||
|
||||
def decide(
|
||||
self,
|
||||
review_id: str,
|
||||
decision: IngestReviewDecision,
|
||||
*,
|
||||
reviewer_id: str,
|
||||
) -> IngestReviewItem:
|
||||
now = datetime.now(UTC).isoformat()
|
||||
decision_payload = decision.model_dump(mode="json")
|
||||
digest = _digest({**decision_payload, "reviewer_id": reviewer_id})
|
||||
with self._connect() as connection:
|
||||
existing = connection.execute(
|
||||
"SELECT payload_digest, review_id FROM ingest_review_decisions WHERE decision_id=?",
|
||||
(decision.decision_id,),
|
||||
).fetchone()
|
||||
if existing is not None:
|
||||
if existing[0] != digest or existing[1] != review_id:
|
||||
raise ReviewConflict("Decision ID was already used with different content")
|
||||
return self._read_item(connection, review_id)
|
||||
row = connection.execute(
|
||||
"SELECT project_id, status FROM ingest_review_items WHERE review_id=?", (review_id,)
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise KeyError("Ingest review item is unavailable")
|
||||
if row[1] != "needs-user-review":
|
||||
raise ReviewConflict("Review item already has a human decision")
|
||||
project_id = row[0]
|
||||
connection.execute(
|
||||
"""INSERT INTO ingest_review_decisions (
|
||||
decision_id, review_id, project_id, decision, rationale,
|
||||
save_as_guidance, guidance_rule, reviewer_id, created_at, payload_digest
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
||||
(
|
||||
decision.decision_id,
|
||||
review_id,
|
||||
project_id,
|
||||
decision.decision,
|
||||
decision.rationale,
|
||||
int(decision.save_as_guidance),
|
||||
decision.guidance_rule,
|
||||
reviewer_id,
|
||||
now,
|
||||
digest,
|
||||
),
|
||||
)
|
||||
connection.execute(
|
||||
"UPDATE ingest_review_items SET status=? WHERE review_id=?",
|
||||
(
|
||||
{"include": "included", "exclude": "excluded", "defer": "deferred"}[
|
||||
decision.decision
|
||||
],
|
||||
review_id,
|
||||
),
|
||||
)
|
||||
return self._read_item(connection, review_id)
|
||||
|
||||
def guidance(self, project_id: str, *, limit: int = 100) -> tuple[IngestGuidance, ...]:
|
||||
with self._connect() as connection:
|
||||
rows = connection.execute(
|
||||
"""SELECT d.decision_id, i.lens, d.guidance_rule, d.created_at
|
||||
FROM ingest_review_decisions d
|
||||
JOIN ingest_review_items i USING(review_id)
|
||||
WHERE d.project_id=? AND d.save_as_guidance=1
|
||||
ORDER BY d.created_at DESC, d.decision_id LIMIT ?""",
|
||||
(project_id, min(max(limit, 1), 100)),
|
||||
).fetchall()
|
||||
return tuple(
|
||||
IngestGuidance(
|
||||
guidance_id=row[0],
|
||||
lens=row[1],
|
||||
rule=row[2],
|
||||
created_at=datetime.fromisoformat(row[3]),
|
||||
)
|
||||
for row in rows
|
||||
)
|
||||
|
||||
def _read_item(self, connection: sqlite3.Connection, review_id: str) -> IngestReviewItem:
|
||||
row = connection.execute(
|
||||
"""SELECT review_id, project_id, run_id, source_revision_id, extraction_id,
|
||||
locator_json, excerpt, lens, summary, relevance_rationale, relevance, status,
|
||||
submitted_by, created_at, jev_relation_probability, jev_lens,
|
||||
jev_lens_confidence, jev_model FROM ingest_review_items WHERE review_id=?""",
|
||||
(review_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise KeyError("Ingest review item is unavailable")
|
||||
return IngestReviewItem.model_validate(
|
||||
{
|
||||
"review_id": row[0],
|
||||
"project_id": row[1],
|
||||
"run_id": row[2],
|
||||
"source_revision_id": row[3],
|
||||
"extraction_id": row[4],
|
||||
"locator": json.loads(row[5]),
|
||||
"excerpt": row[6],
|
||||
"lens": row[7],
|
||||
"summary": row[8],
|
||||
"relevance_rationale": row[9],
|
||||
"relevance": row[10],
|
||||
"status": row[11],
|
||||
"submitted_by": row[12],
|
||||
"created_at": datetime.fromisoformat(row[13]),
|
||||
"jev_relation_probability": row[14],
|
||||
"jev_lens": row[15],
|
||||
"jev_lens_confidence": row[16],
|
||||
"jev_model": row[17],
|
||||
},
|
||||
strict=True,
|
||||
)
|
||||
|
||||
def _connect(self) -> sqlite3.Connection:
|
||||
connection = sqlite3.connect(self.database, timeout=5, isolation_level="IMMEDIATE")
|
||||
connection.row_factory = sqlite3.Row
|
||||
connection.execute("PRAGMA foreign_keys=ON")
|
||||
connection.execute("PRAGMA busy_timeout=5000")
|
||||
return connection
|
||||
|
||||
|
||||
def _digest(value: object) -> str:
|
||||
payload = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
||||
@@ -14,8 +14,10 @@ from starlette.routing import Mount
|
||||
|
||||
from gewu.adapters.embeddings import LocalSentenceTransformer
|
||||
from gewu.adapters.git_repository import GitCanonicalRepository
|
||||
from gewu.adapters.openrouter_jev import OpenRouterJev
|
||||
from gewu.adapters.source_extractor import LocalSourceExtractor, SourceExtractor
|
||||
from gewu.adapters.sqlite_identity import SQLiteIdentityStore
|
||||
from gewu.adapters.sqlite_ingest_reviews import SQLiteIngestReviewStore
|
||||
from gewu.adapters.sqlite_search import SQLiteSearchIndex
|
||||
from gewu.adapters.sqlite_vectors import SQLiteVectorIndex
|
||||
from gewu.adapters.url_fetch import PublicUrlFetcher
|
||||
@@ -60,6 +62,7 @@ from gewu.transports.http_models import (
|
||||
SearchRequestBody,
|
||||
SourceListRequestBody,
|
||||
)
|
||||
from gewu.transports.ingest_reviews_http import register_ingest_review_routes
|
||||
from gewu.transports.mcp_server import create_mcp_server
|
||||
from gewu.transports.owner_mcp_server import OwnerReviewProvider, create_owner_mcp_server
|
||||
from gewu.version import SERVICE_VERSION
|
||||
@@ -79,6 +82,8 @@ def create_app(
|
||||
project_registry: ProjectRegistry | None = None,
|
||||
credential_administration: CredentialAdministration | None = None,
|
||||
account_identity: AccountIdentity | None = None,
|
||||
ingest_review_store: SQLiteIngestReviewStore | None = None,
|
||||
jev_assessor: OpenRouterJev | None = None,
|
||||
url_fetcher: PublicUrlFetcher | None = None,
|
||||
) -> FastAPI:
|
||||
if (identity_provider is None) != (node_reader is None):
|
||||
@@ -94,6 +99,7 @@ def create_app(
|
||||
or owner_identity_provider is not None
|
||||
or credential_administration is not None
|
||||
or account_identity is not None
|
||||
or ingest_review_store is not None
|
||||
):
|
||||
app.add_middleware(ProjectRegistryBodyLimitMiddleware)
|
||||
|
||||
@@ -149,6 +155,11 @@ def create_app(
|
||||
if account_identity is not None:
|
||||
register_account_routes(app, account_identity, authenticate, require_same_origin)
|
||||
|
||||
if ingest_review_store is not None:
|
||||
register_ingest_review_routes(
|
||||
app, ingest_review_store, authenticate, require_same_origin
|
||||
)
|
||||
|
||||
if owner_identity_provider is not None:
|
||||
|
||||
@app.post(
|
||||
@@ -783,6 +794,8 @@ def create_app(
|
||||
acceptance_service=acceptance_service,
|
||||
context_provider=context_provider,
|
||||
url_fetcher=url_fetcher or PublicUrlFetcher(frozenset()),
|
||||
ingest_review_store=ingest_review_store,
|
||||
jev_assessor=jev_assessor,
|
||||
)
|
||||
mcp_asgi = mcp_server.streamable_http_app(
|
||||
streamable_http_path="/",
|
||||
@@ -854,6 +867,7 @@ def create_app(
|
||||
def create_local_app() -> FastAPI:
|
||||
"""Compose the local persistent service from one private data root."""
|
||||
|
||||
_load_local_jev_environment()
|
||||
data_root = Path(os.environ.get("GEWU_DATA_ROOT", "./.data/gewu")).expanduser().absolute()
|
||||
reject_symlink_components(data_root)
|
||||
data_root.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
@@ -870,6 +884,7 @@ def create_local_app() -> FastAPI:
|
||||
or None,
|
||||
)
|
||||
artifacts = LocalArtifactStore(data_root)
|
||||
ingest_reviews = SQLiteIngestReviewStore(data_root / "ingest-reviews.sqlite3")
|
||||
allowed_fetch_hosts = {
|
||||
host.strip()
|
||||
for host in os.environ.get("GEWU_FETCH_ALLOWED_HOSTS", "").split(",")
|
||||
@@ -942,6 +957,15 @@ def create_local_app() -> FastAPI:
|
||||
identity_provider=identities,
|
||||
credential_administration=identities,
|
||||
account_identity=identities,
|
||||
ingest_review_store=ingest_reviews,
|
||||
jev_assessor=(
|
||||
OpenRouterJev(
|
||||
os.environ["OPENROUTER_API_KEY"],
|
||||
model=os.environ.get("OPENROUTER_JEV_MODEL", "typesafe/jev-1.13"),
|
||||
)
|
||||
if os.environ.get("OPENROUTER_API_KEY")
|
||||
else None
|
||||
),
|
||||
owner_identity_provider=identities,
|
||||
owner_service_audience=os.environ.get("GEWU_OWNER_SERVICE_AUDIENCE"),
|
||||
node_reader=CanonicalNodeReader(repository),
|
||||
@@ -953,3 +977,25 @@ def create_local_app() -> FastAPI:
|
||||
project_registry=project_registry,
|
||||
url_fetcher=url_fetcher,
|
||||
)
|
||||
|
||||
|
||||
def _load_local_jev_environment() -> None:
|
||||
"""Load only Jev settings from the ignored, local project .env file."""
|
||||
|
||||
path = Path.cwd() / ".env"
|
||||
if path.is_symlink() or not path.is_file():
|
||||
return
|
||||
path.chmod(0o600)
|
||||
allowed = {"OPENROUTER_API_KEY", "OPENROUTER_JEV_MODEL"}
|
||||
for raw_line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
name, _, value = line.partition("=")
|
||||
name = name.strip()
|
||||
value = value.strip()
|
||||
if name not in allowed or not value:
|
||||
continue
|
||||
if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}:
|
||||
value = value[1:-1]
|
||||
os.environ.setdefault(name, value)
|
||||
@@ -19,6 +19,18 @@ from pydantic import (
|
||||
)
|
||||
|
||||
SCHEMA_VERSION = 1
|
||||
ResearchLens = Literal[
|
||||
"technical",
|
||||
"conceptual",
|
||||
"cultural-historical",
|
||||
"financial-market",
|
||||
]
|
||||
DEFAULT_RESEARCH_LENSES: tuple[ResearchLens, ...] = (
|
||||
"technical",
|
||||
"conceptual",
|
||||
"cultural-historical",
|
||||
"financial-market",
|
||||
)
|
||||
|
||||
RecordId = Annotated[
|
||||
str,
|
||||
@@ -58,6 +70,7 @@ class ProjectRecord(StrictRecord):
|
||||
owners: tuple[RecordId, ...] = ()
|
||||
north_star: str = Field(min_length=1, max_length=4000)
|
||||
key_questions: tuple[str, ...] = Field(min_length=1, max_length=30)
|
||||
research_lenses: tuple[ResearchLens, ...] = DEFAULT_RESEARCH_LENSES
|
||||
watch_list: tuple[str, ...] = Field(default=(), max_length=100)
|
||||
default_includes: tuple[RecordId, ...] = ("shared",)
|
||||
taxonomy_extensions: dict[str, tuple[str, ...]] = Field(default_factory=dict, max_length=100)
|
||||
@@ -71,20 +84,97 @@ class ProjectRecord(StrictRecord):
|
||||
return value
|
||||
|
||||
|
||||
class IngestGuidance(StrictRecord):
|
||||
guidance_id: RecordId
|
||||
lens: ResearchLens
|
||||
rule: str = Field(min_length=1, max_length=1000)
|
||||
created_at: datetime
|
||||
|
||||
@field_validator("created_at")
|
||||
@classmethod
|
||||
def guidance_timestamp_is_utc(cls, value: datetime) -> datetime:
|
||||
offset = value.utcoffset()
|
||||
if offset is None or offset.total_seconds() != 0:
|
||||
raise ValueError("guidance timestamp must be timezone-aware UTC")
|
||||
return value
|
||||
|
||||
|
||||
class ProjectLens(StrictRecord):
|
||||
project_id: RecordId
|
||||
title: str = Field(min_length=1, max_length=200)
|
||||
status: ProjectStatus
|
||||
north_star: str = Field(min_length=1, max_length=4000)
|
||||
key_questions: tuple[str, ...] = Field(min_length=1, max_length=30)
|
||||
research_lenses: tuple[ResearchLens, ...] = DEFAULT_RESEARCH_LENSES
|
||||
watch_list: tuple[str, ...] = Field(default=(), max_length=100)
|
||||
taxonomy_extensions: dict[str, tuple[str, ...]] = Field(default_factory=dict, max_length=100)
|
||||
ingest_guidance: tuple[IngestGuidance, ...] = Field(default=(), max_length=100)
|
||||
|
||||
|
||||
class ProjectContextResponse(StrictRecord):
|
||||
lens: ProjectLens
|
||||
|
||||
|
||||
class IngestReviewItem(StrictRecord):
|
||||
review_id: RecordId
|
||||
project_id: RecordId
|
||||
run_id: RecordId
|
||||
source_revision_id: RecordId
|
||||
extraction_id: Sha256
|
||||
locator: Locator
|
||||
excerpt: str = Field(min_length=1, max_length=4000)
|
||||
lens: ResearchLens
|
||||
summary: str = Field(min_length=1, max_length=2000)
|
||||
relevance_rationale: str = Field(min_length=1, max_length=2000)
|
||||
relevance: Literal["borderline", "low"] = "borderline"
|
||||
jev_relation_probability: FiniteFloat | None = Field(default=None, ge=0, le=1)
|
||||
jev_lens: ResearchLens | None = None
|
||||
jev_lens_confidence: FiniteFloat | None = Field(default=None, ge=0, le=1)
|
||||
jev_model: str | None = Field(default=None, max_length=100)
|
||||
status: Literal["needs-user-review", "included", "excluded", "deferred"] = "needs-user-review"
|
||||
submitted_by: RecordId
|
||||
created_at: datetime
|
||||
|
||||
@field_validator("created_at")
|
||||
@classmethod
|
||||
def review_timestamp_is_utc(cls, value: datetime) -> datetime:
|
||||
offset = value.utcoffset()
|
||||
if offset is None or offset.total_seconds() != 0:
|
||||
raise ValueError("review timestamp must be timezone-aware UTC")
|
||||
return value
|
||||
|
||||
|
||||
class IngestReviewCreate(StrictRecord):
|
||||
review_id: RecordId
|
||||
project_id: RecordId
|
||||
run_id: RecordId
|
||||
source_revision_id: RecordId
|
||||
extraction_id: Sha256
|
||||
locator: Locator
|
||||
excerpt: str = Field(min_length=1, max_length=4000)
|
||||
lens: ResearchLens
|
||||
summary: str = Field(min_length=1, max_length=2000)
|
||||
relevance_rationale: str = Field(min_length=1, max_length=2000)
|
||||
relevance: Literal["borderline", "low"] = "borderline"
|
||||
|
||||
|
||||
class IngestReviewDecision(StrictRecord):
|
||||
decision_id: RecordId
|
||||
review_id: RecordId
|
||||
decision: Literal["include", "exclude", "defer"]
|
||||
rationale: str = Field(min_length=1, max_length=2000)
|
||||
save_as_guidance: bool = False
|
||||
guidance_rule: str | None = Field(default=None, max_length=1000)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def guidance_requires_explicit_rule(self) -> IngestReviewDecision:
|
||||
if self.save_as_guidance and not (self.guidance_rule or "").strip():
|
||||
raise ValueError("saving project guidance requires a non-empty guidance rule")
|
||||
if not self.save_as_guidance and self.guidance_rule is not None:
|
||||
raise ValueError("guidance_rule requires explicit save_as_guidance")
|
||||
return self
|
||||
|
||||
|
||||
class ReliabilityGrade(StrEnum):
|
||||
LOW = "low"
|
||||
MEDIUM = "medium"
|
||||
|
||||
@@ -50,6 +50,8 @@ OPERATIONS: Final[tuple[Operation, ...]] = (
|
||||
Operation("project_archive", "project:admin", frozenset({Transport.HTTP})),
|
||||
Operation("list_projects", "knowledge:read", frozenset({Transport.HTTP, Transport.MCP})),
|
||||
Operation("get_project_context", "knowledge:read", frozenset({Transport.MCP})),
|
||||
Operation("ingest_review_list", "ingest:review", frozenset({Transport.HTTP})),
|
||||
Operation("ingest_review_decide", "ingest:review", frozenset({Transport.HTTP})),
|
||||
Operation("search", "knowledge:read", frozenset({Transport.HTTP, Transport.MCP})),
|
||||
Operation("list_sources", "knowledge:read", frozenset({Transport.HTTP, Transport.MCP})),
|
||||
Operation("get_node", "knowledge:read", frozenset({Transport.HTTP, Transport.MCP})),
|
||||
@@ -73,6 +75,7 @@ OPERATIONS: Final[tuple[Operation, ...]] = (
|
||||
Operation("run_create", "run:write", frozenset({Transport.MCP})),
|
||||
Operation("run_checkpoint", "run:write", frozenset({Transport.MCP})),
|
||||
Operation("run_status", "run:read", frozenset({Transport.MCP})),
|
||||
Operation("ingest_review_create", "ingest:review", frozenset({Transport.MCP})),
|
||||
Operation("owner_challenge_get", "owner:review", frozenset({Transport.OWNER_MCP})),
|
||||
Operation("owner_diff_get", "owner:review", frozenset({Transport.OWNER_MCP})),
|
||||
Operation("owner_decide", "owner:decide", frozenset({Transport.OWNER_MCP})),
|
||||
|
||||
@@ -25,6 +25,14 @@ class ProjectRegistryBodyLimitMiddleware:
|
||||
"/v1/me/agent-credentials": MAX_OWNER_AUTH_BODY_BYTES,
|
||||
}
|
||||
path = scope.get("path")
|
||||
if (
|
||||
scope["type"] == "http"
|
||||
and scope.get("method") == "POST"
|
||||
and isinstance(path, str)
|
||||
and path.startswith("/v1/ingest-reviews/")
|
||||
and path.endswith("/decision")
|
||||
):
|
||||
limits[path] = 12 * 1024
|
||||
if scope["type"] != "http" or scope.get("method") != "POST" or path not in limits:
|
||||
await self.app(scope, receive, send)
|
||||
return
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Human review routes for borderline ingest candidates."""
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import Cookie, Depends, FastAPI, Header, HTTPException, Query, Request, Response
|
||||
|
||||
from gewu.adapters.sqlite_ingest_reviews import ReviewConflict, SQLiteIngestReviewStore
|
||||
from gewu.application.identity import Principal
|
||||
from gewu.domain.contracts import IngestReviewDecision, IngestReviewItem, RecordId
|
||||
|
||||
|
||||
def register_ingest_review_routes(
|
||||
app: FastAPI,
|
||||
store: SQLiteIngestReviewStore,
|
||||
authenticate: Callable[..., Principal],
|
||||
require_same_origin: Callable[[Request], None],
|
||||
) -> None:
|
||||
def human(
|
||||
request: Request,
|
||||
response: Response,
|
||||
authorization: str | None = Header(default=None),
|
||||
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
|
||||
csrf_token: str | None = Header(default=None, alias="X-GEWU-CSRF"),
|
||||
) -> Principal:
|
||||
response.headers["Cache-Control"] = "private, no-store"
|
||||
write = request.method not in {"GET", "HEAD"}
|
||||
if write:
|
||||
require_same_origin(request)
|
||||
principal = authenticate(authorization, session_cookie, csrf_token, require_csrf=write)
|
||||
if "agent" in principal.roles:
|
||||
raise HTTPException(403, "A human account is required")
|
||||
return principal
|
||||
|
||||
@app.get(
|
||||
"/v1/ingest-reviews",
|
||||
tags=["ingest review"],
|
||||
operation_id="ingest_review_list",
|
||||
)
|
||||
def list_reviews(
|
||||
principal: Annotated[Principal, Depends(human)],
|
||||
project_id: RecordId,
|
||||
status: str | None = Query(default="needs-user-review", max_length=32),
|
||||
limit: int = Query(default=50, ge=1, le=100),
|
||||
offset: int = Query(default=0, ge=0),
|
||||
) -> list[IngestReviewItem]:
|
||||
if status not in {None, "needs-user-review", "included", "excluded", "deferred"}:
|
||||
raise HTTPException(422, "Review status is invalid")
|
||||
if not principal.can_read_project(project_id):
|
||||
raise HTTPException(404, "Project review queue is unavailable")
|
||||
return store.list_items(project_id, status=status, limit=limit, offset=offset)
|
||||
|
||||
@app.post(
|
||||
"/v1/ingest-reviews/{review_id}/decision",
|
||||
tags=["ingest review"],
|
||||
operation_id="ingest_review_decide",
|
||||
)
|
||||
def decide_review(
|
||||
review_id: RecordId,
|
||||
body: IngestReviewDecision,
|
||||
principal: Annotated[Principal, Depends(human)],
|
||||
) -> dict[str, object]:
|
||||
if body.review_id != review_id:
|
||||
raise HTTPException(422, "Decision review_id must match the route")
|
||||
item = store.get(review_id)
|
||||
if item is None or not principal.can_read_project(item.project_id):
|
||||
raise HTTPException(404, "Review item is unavailable")
|
||||
if not ({"editor", "admin"} & principal.roles):
|
||||
raise HTTPException(403, "Editor role is required to decide review items")
|
||||
try:
|
||||
updated = store.decide(review_id, body, reviewer_id=principal.principal_id)
|
||||
guidance = store.guidance(item.project_id) if body.save_as_guidance else ()
|
||||
except ReviewConflict as error:
|
||||
raise HTTPException(409, str(error)) from error
|
||||
except KeyError as error:
|
||||
raise HTTPException(404, "Review item is unavailable") from error
|
||||
return {"item": updated, "guidance": guidance}
|
||||
@@ -17,8 +17,10 @@ from mcp.server import MCPServer
|
||||
from mcp.server.mcpserver import Context
|
||||
from pydantic import ValidationError
|
||||
|
||||
from gewu.adapters.openrouter_jev import JevUnavailable, OpenRouterJev
|
||||
from gewu.adapters.pdf_extractor import PdfExtractionError
|
||||
from gewu.adapters.source_extractor import SourceExtractor
|
||||
from gewu.adapters.sqlite_ingest_reviews import SQLiteIngestReviewStore
|
||||
from gewu.adapters.url_fetch import PublicUrlFetcher, UrlCaptureError
|
||||
from gewu.application.acceptance import AcceptanceError, AcceptanceService, ActorContext
|
||||
from gewu.application.context import ValidationContext
|
||||
@@ -37,6 +39,7 @@ from gewu.domain.contracts import (
|
||||
EvidenceReference,
|
||||
ExtractionRecord,
|
||||
GitCommitId,
|
||||
IngestReviewCreate,
|
||||
NeighborRequest,
|
||||
NodeType,
|
||||
ProjectContextResponse,
|
||||
@@ -82,6 +85,8 @@ def create_mcp_server(
|
||||
acceptance_service: AcceptanceService | None = None,
|
||||
context_provider: Callable[[], ValidationContext] | None = None,
|
||||
url_fetcher: PublicUrlFetcher | None = None,
|
||||
ingest_review_store: SQLiteIngestReviewStore | None = None,
|
||||
jev_assessor: OpenRouterJev | None = None,
|
||||
) -> MCPServer:
|
||||
"""Create the allowlisted MCP server for the supplied application services."""
|
||||
|
||||
@@ -353,11 +358,103 @@ def create_mcp_server(
|
||||
"run_status",
|
||||
}
|
||||
)
|
||||
if ingest_review_store is not None:
|
||||
available_operations.add("ingest_review_create")
|
||||
run_service = LocalRunService(
|
||||
workflow_root=acceptance_service.root,
|
||||
actor_is_current=acceptance_service.actor_is_current,
|
||||
)
|
||||
|
||||
if ingest_review_store is not None:
|
||||
|
||||
@server.tool(name="ingest_review_create", structured_output=False)
|
||||
def ingest_review_create(review_json: str, ctx: Context) -> str:
|
||||
"""Persist a source-bound follow-up flag for human relevance feedback."""
|
||||
|
||||
principal = _principal(ctx, identity_provider)
|
||||
if "agent" not in principal.roles or principal.auth_method != "bearer":
|
||||
raise PermissionError(
|
||||
"Ingest review submission requires an agent bearer credential"
|
||||
)
|
||||
if len(review_json.encode("utf-8")) > 16_000:
|
||||
raise ValueError("Ingest review request exceeds the supported size")
|
||||
try:
|
||||
item = IngestReviewCreate.model_validate_json(review_json, strict=True)
|
||||
except ValidationError as error:
|
||||
raise ValueError(
|
||||
"Ingest review request does not match the GEWU v1 contract"
|
||||
) from error
|
||||
_require_project_submission(principal, item.project_id)
|
||||
actor = ActorContext(principal.principal_id, principal.credential_id)
|
||||
try:
|
||||
run = run_service.status(item.run_id, actor=actor)
|
||||
except RunError as error:
|
||||
raise ValueError(f"{error.code}: {error}") from error
|
||||
_require_run_scope(principal, run.scope)
|
||||
if item.project_id not in run.scope.projects:
|
||||
raise ValueError("Ingest review project must be in the durable run scope")
|
||||
extraction = artifact_store.get_extraction(
|
||||
item.extraction_id,
|
||||
owner_id=principal.principal_id,
|
||||
project_id=item.project_id,
|
||||
)
|
||||
if extraction is None or extraction.source_revision_id != item.source_revision_id:
|
||||
raise ValueError("Ingest review source extraction is unavailable")
|
||||
block = next(
|
||||
(
|
||||
candidate
|
||||
for candidate in extraction.blocks
|
||||
if candidate.block_id == item.locator.block_id
|
||||
),
|
||||
None,
|
||||
)
|
||||
if (
|
||||
block is None
|
||||
or item.locator.end > len(block.normalized_text)
|
||||
or block.normalized_text[item.locator.start : item.locator.end] != item.excerpt
|
||||
):
|
||||
raise ValueError(
|
||||
"Ingest review excerpt must exactly match its extraction locator"
|
||||
)
|
||||
expected_kind = (
|
||||
"pdf_page_span"
|
||||
if block.page is not None
|
||||
else "transcript_timestamp_span"
|
||||
if block.time_start_ms is not None
|
||||
else "text_span"
|
||||
)
|
||||
if (
|
||||
item.locator.kind != expected_kind
|
||||
or item.locator.page != block.page
|
||||
or item.locator.time_start_ms != block.time_start_ms
|
||||
or item.locator.time_end_ms != block.time_end_ms
|
||||
):
|
||||
raise ValueError(
|
||||
"Ingest review locator metadata does not match its extraction block"
|
||||
)
|
||||
assessment = None
|
||||
if jev_assessor is not None:
|
||||
project = context_provider().projects.get(item.project_id)
|
||||
if project is not None:
|
||||
try:
|
||||
assessment = jev_assessor.assess(item, project)
|
||||
except JevUnavailable:
|
||||
# The review flag and normal proposal flow do not depend on Jev uptime.
|
||||
assessment = None
|
||||
record = ingest_review_store.create(
|
||||
item,
|
||||
submitted_by=principal.principal_id,
|
||||
jev_relation_probability=(
|
||||
assessment.relation_probability if assessment is not None else None
|
||||
),
|
||||
jev_lens=assessment.lens if assessment is not None else None,
|
||||
jev_lens_confidence=(
|
||||
assessment.lens_confidence if assessment is not None else None
|
||||
),
|
||||
jev_model=assessment.model if assessment is not None else None,
|
||||
)
|
||||
return json.dumps(record.model_dump(mode="json"), ensure_ascii=False)
|
||||
|
||||
@server.tool(name="source_capture_text", structured_output=False)
|
||||
def source_capture_text(
|
||||
capture_id: str,
|
||||
@@ -1016,8 +1113,14 @@ def create_mcp_server(
|
||||
status=project.status,
|
||||
north_star=project.north_star,
|
||||
key_questions=project.key_questions,
|
||||
research_lenses=project.research_lenses,
|
||||
watch_list=project.watch_list,
|
||||
taxonomy_extensions=project.taxonomy_extensions,
|
||||
ingest_guidance=(
|
||||
ingest_review_store.guidance(project_id)
|
||||
if ingest_review_store is not None
|
||||
else ()
|
||||
),
|
||||
),
|
||||
)
|
||||
encoded = json.dumps(response.model_dump(mode="json"), ensure_ascii=False)
|
||||
|
||||
Reference in new issue
Block a user