feat(ingest): resolve project labels with Jev
CI / quality (push) Canceled after 0s

This commit is contained in:
vasily committed 2026-10-15 10:53:30 +08:00
1 parent 0e1a2ad008
commit 34f4923566
28 files changed
+3932 -81

No files matched your search

+151 -30
View File
@@ -9,9 +9,13 @@ from urllib.error import HTTPError, URLError
from urllib.request import HTTPRedirectHandler, Request, build_opener
from gewu.domain.contracts import (
FacetLabelProposal,
FacetName,
IngestReviewCreate,
JevFacetOptionScore,
JevFacetResolution,
JevFacetScore,
ProjectFacetLabel,
ProjectRecord,
ResearchLens,
)
@@ -28,6 +32,7 @@ class JevAssessment:
lens_confidence: float
model: str
facet_scores: tuple[JevFacetScore, ...] = ()
facet_resolutions: tuple[JevFacetResolution, ...] = ()
class JevUnavailable(RuntimeError):
@@ -50,31 +55,75 @@ class OpenRouterJev:
self._api_key = api_key
self._model = model
def assess(self, item: IngestReviewCreate, project: ProjectRecord) -> JevAssessment:
facet_candidates: tuple[tuple[FacetName, tuple[str, ...]], ...] = (
("tags", item.candidate_tags),
("audience", item.candidate_audience),
("use_for", item.candidate_use_for),
("domain", item.candidate_domain),
)
def assess(
self,
item: IngestReviewCreate,
project: ProjectRecord,
*,
closest_labels: dict[tuple[FacetName, str], tuple[ProjectFacetLabel, ...]] | None = None,
) -> JevAssessment:
proposals = item.facet_proposals()
closest_labels = closest_labels or {}
facet_questions: dict[str, object] = {}
facet_question_map: dict[str, tuple[FacetName, str]] = {}
for facet, candidates in facet_candidates:
for index, value in enumerate(candidates):
key = f"facet_{facet}_{index}"
facet_question_map[key] = (facet, value)
facet_questions[key] = {
"type": "noul",
"instructions": (
f"Does the candidate {facet} label accurately describe the passage "
"in the context of this project? Judge the label itself, not just "
"whether its words appear in the excerpt."
),
"criteria": {
"true": "This label is a useful, well-supported classification.",
"false": "This label is inaccurate, unsupported, or too broad.",
},
fit_question_map: dict[str, FacetLabelProposal] = {}
resolution_question_map: dict[
str, tuple[FacetLabelProposal, tuple[tuple[str, ProjectFacetLabel], ...]]
] = {}
facet_state: list[dict[str, object]] = []
for index, proposal in enumerate(proposals):
existing_labels = closest_labels.get((proposal.facet, proposal.value), ())[:5]
fit_key = f"facet_fit_{proposal.facet}_{index}"
match_key = f"facet_match_{proposal.facet}_{index}"
fit_question_map[fit_key] = proposal
facet_questions[fit_key] = {
"type": "noul",
"instructions": (
f"Is this candidate {proposal.facet} label useful and accurate for "
"classifying the passage in this project?"
),
"criteria": {
"true": "The label accurately captures a reusable facet of this passage.",
"false": "The label is inaccurate, unsupported, or too specific to reuse.",
},
}
choice_criteria: dict[str, str] = {}
option_map: list[tuple[str, ProjectFacetLabel]] = []
for option_index, label in enumerate(existing_labels):
option_key = f"existing_{option_index}"
choice_criteria[option_key] = (
f"Reuse canonical label {label.canonical_label!r}: {label.definition}"
)
option_map.append((option_key, label))
choice_criteria["new_label"] = (
"The candidate is useful and distinct from all listed existing labels; "
"retain the candidate wording as a new project label."
)
resolution_question_map[match_key] = (proposal, tuple(option_map))
facet_questions[match_key] = {
"type": "choice",
"instructions": (
f"For candidate {proposal.facet} label {proposal.value!r} (meaning: "
f"{proposal.definition}), which one option best captures its meaning? "
"Choose an existing label only when it means essentially the same thing; "
"related but distinct labels are not synonyms."
),
"criteria": choice_criteria,
}
facet_state.append(
{
"facet": proposal.facet,
"candidate": proposal.value,
"definition": proposal.definition,
"existing_options": [
{
"option": option_key,
"label": label.canonical_label,
"definition": label.definition,
}
for option_key, label in option_map
],
}
)
state = {
"project_title": project.title,
"project_north_star": project.north_star,
@@ -83,9 +132,7 @@ class OpenRouterJev:
"candidate_summary": item.summary,
"candidate_relevance_rationale": item.relevance_rationale,
"source_excerpt": item.excerpt,
"candidate_facets": {
facet: candidates for facet, candidates in facet_candidates if candidates
},
"candidate_facets": facet_state,
}
criteria = {
"technical": "Engineering, implementation, technology, or operational practice.",
@@ -151,21 +198,95 @@ class OpenRouterJev:
served_model = str(result["model"])
facet_scores = tuple(
JevFacetScore(
facet=facet,
value=value,
facet=proposal.facet,
value=proposal.value,
probability=float(answers[key]["noul"]),
)
for key, (facet, value) in facet_question_map.items()
for key, proposal in fit_question_map.items()
)
facet_resolutions_list: list[JevFacetResolution] = []
for key, (proposal, option_pairs) in resolution_question_map.items():
answer = answers[key]
selected = answer["choice"]
confidence_value = float(answer["confidence"])
probabilities = answer["probabilities"]
if not isinstance(probabilities, dict):
raise TypeError("choice probabilities are missing")
new_probability = float(probabilities["new_label"])
option_ids = {option_key for option_key, _ in option_pairs} | {"new_label"}
if selected not in option_ids:
raise ValueError("Jev selected an unknown facet label option")
if any(not 0 <= float(probabilities[option_id]) <= 1 for option_id in option_ids):
raise ValueError("Jev returned an invalid facet choice probability")
selected_label = next(
(label for option_key, label in option_pairs if option_key == selected),
None,
)
option_probabilities = tuple(
JevFacetOptionScore(
label_id=(
next(
(
label.label_id
for option_key, label in option_pairs
if option_key == option_id
),
None,
)
),
label=(
next(
(
label.canonical_label
for option_key, label in option_pairs
if option_key == option_id
),
proposal.value,
)
),
probability=float(probabilities[option_id]),
)
for option_id in sorted(option_ids)
)
facet_resolutions_list.append(
JevFacetResolution(
facet=proposal.facet,
value=proposal.value,
fit_probability=next(
score.probability
for score in facet_scores
if score.facet == proposal.facet and score.value == proposal.value
),
selected_label_id=(selected_label.label_id if selected_label else None),
selected_label=(selected_label.canonical_label if selected_label else None),
selected_new_label=selected == "new_label",
new_probability=new_probability,
choice_confidence=confidence_value,
option_probabilities=option_probabilities,
rank=1,
)
)
facet_resolutions = tuple(facet_resolutions_list)
except (KeyError, TypeError, ValueError, json.JSONDecodeError) as error:
raise JevUnavailable("Jev response did not match its typed decision schema") from error
if (
not 0 <= relation <= 1
or not 0 <= confidence <= 1
or any(
not 0 <= decision.fit_probability <= 1
or not 0 <= decision.new_probability <= 1
or not 0 <= decision.choice_confidence <= 1
for decision in facet_resolutions
)
or lens not in criteria
or len(served_model) > 100
):
raise JevUnavailable("Jev response contained an invalid decision")
return JevAssessment(
relation, cast(ResearchLens, lens), confidence, served_model, facet_scores
relation,
cast(ResearchLens, lens),
confidence,
served_model,
facet_scores,
facet_resolutions,
)
+176
View File
@@ -0,0 +1,176 @@
"""SQLite cache of project facet labels and their local semantic vectors."""
from __future__ import annotations
import hashlib
import json
import math
import sqlite3
from dataclasses import dataclass
from pathlib import Path
from gewu.application.paths import reject_symlink_components
from gewu.application.ports import EmbeddingProvider
from gewu.domain.contracts import FacetName, ProjectFacetLabel
MAX_LABELS_PER_PROJECT = 2000
@dataclass(frozen=True, slots=True)
class FacetLabelMatch:
label_id: str
canonical_label: str
definition: str
similarity: float
class SQLiteFacetLabelIndex:
"""Cache rebuildable project label vectors, keyed by label and model profile."""
def __init__(self, database: Path) -> None:
self.database = database.expanduser().absolute()
reject_symlink_components(self.database)
self.database.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
self.database.parent.chmod(0o700)
if self.database.is_symlink():
raise RuntimeError("Facet label index must not be a symbolic link")
with self._connect() as connection:
connection.execute(
"""CREATE TABLE IF NOT EXISTS facet_label_embeddings (
project_id TEXT NOT NULL,
facet TEXT NOT NULL CHECK(facet IN ('tags', 'audience', 'use_for', 'domain')),
label_id TEXT NOT NULL,
canonical_label TEXT NOT NULL,
definition TEXT NOT NULL,
aliases_json TEXT NOT NULL,
label_digest TEXT NOT NULL,
model_id TEXT NOT NULL,
artifact_sha256 TEXT NOT NULL,
dimensions INTEGER NOT NULL,
embedding_json TEXT NOT NULL,
PRIMARY KEY(project_id, facet, label_id, model_id, artifact_sha256)
)"""
)
connection.execute(
"CREATE INDEX IF NOT EXISTS facet_label_lookup "
"ON facet_label_embeddings(project_id, facet, model_id, artifact_sha256)"
)
self.database.chmod(0o600)
def closest(
self,
*,
project_id: str,
facet: FacetName,
query: str,
labels: tuple[ProjectFacetLabel, ...],
embeddings: EmbeddingProvider,
limit: int = 5,
) -> tuple[FacetLabelMatch, ...]:
if not 1 <= limit <= 10 or len(labels) > MAX_LABELS_PER_PROJECT:
raise ValueError("Facet label search exceeds its supported bounds")
active = tuple(
label for label in labels if label.facet == facet and label.status == "active"
)
if not active:
return ()
profile = (embeddings.model_id, embeddings.artifact_sha256, embeddings.dimensions)
cached: dict[str, tuple[float, ...]] = {}
pending: list[tuple[ProjectFacetLabel, str, str]] = []
with self._connect() as connection:
for label in active:
digest = _label_digest(label)
row = connection.execute(
"""SELECT embedding_json, dimensions FROM facet_label_embeddings
WHERE project_id=? AND facet=? AND label_id=? AND model_id=?
AND artifact_sha256=? AND label_digest=?""",
(project_id, facet, label.label_id, *profile[:2], digest),
).fetchone()
if row is not None and row[1] == profile[2]:
try:
vector = tuple(float(value) for value in json.loads(row[0]))
except (TypeError, ValueError, json.JSONDecodeError):
vector = ()
if _valid_vector(vector, profile[2]):
cached[label.label_id] = vector
continue
pending.append((label, digest, _label_text(label)))
if len(pending) > 0:
vectors = embeddings.embed_passages(tuple(item[2] for item in pending))
if len(vectors) != len(pending) or any(
not _valid_vector(vector, profile[2]) for vector in vectors
):
raise ValueError("Embedding provider returned invalid facet label vectors")
for (label, digest, _), vector in zip(pending, vectors, strict=True):
cached[label.label_id] = vector
connection.execute(
"""INSERT INTO facet_label_embeddings (
project_id, facet, label_id, canonical_label, definition,
aliases_json, label_digest, model_id, artifact_sha256,
dimensions, embedding_json
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(project_id, facet, label_id, model_id, artifact_sha256)
DO UPDATE SET canonical_label=excluded.canonical_label,
definition=excluded.definition, aliases_json=excluded.aliases_json,
label_digest=excluded.label_digest, dimensions=excluded.dimensions,
embedding_json=excluded.embedding_json""",
(
project_id,
facet,
label.label_id,
label.canonical_label,
label.definition,
json.dumps(label.aliases, ensure_ascii=False, separators=(",", ":")),
digest,
*profile,
json.dumps(vector, separators=(",", ":")),
),
)
query_vector = embeddings.embed_queries(("Label meaning: " + query,))[0]
if not _valid_vector(query_vector, profile[2]):
raise ValueError("Embedding provider returned an invalid facet query vector")
by_id = {label.label_id: label for label in active}
scores = (
FacetLabelMatch(
label_id=label_id,
canonical_label=by_id[label_id].canonical_label,
definition=by_id[label_id].definition,
similarity=_dot(query_vector, vector),
)
for label_id, vector in cached.items()
)
return tuple(sorted(scores, key=lambda match: (-match.similarity, match.label_id))[:limit])
def _connect(self) -> sqlite3.Connection:
connection = sqlite3.connect(self.database, timeout=5, isolation_level="IMMEDIATE")
connection.execute("PRAGMA foreign_keys=ON")
connection.execute("PRAGMA busy_timeout=5000")
return connection
def _label_text(label: ProjectFacetLabel) -> str:
aliases = ", ".join(label.aliases)
return (
f"Facet: {label.facet}. Label: {label.canonical_label}. "
f"Meaning: {label.definition}. Aliases: {aliases}"
)
def _label_digest(label: ProjectFacetLabel) -> str:
encoded = json.dumps(
[label.facet, label.canonical_label, label.definition, label.aliases],
ensure_ascii=False,
separators=(",", ":"),
).encode("utf-8")
return hashlib.sha256(encoded).hexdigest()
def _valid_vector(vector: tuple[float, ...], dimensions: int) -> bool:
if len(vector) != dimensions or not all(math.isfinite(value) for value in vector):
return False
norm = math.sqrt(sum(value * value for value in vector))
return math.isclose(norm, 1.0, rel_tol=1e-4, abs_tol=1e-4)
def _dot(left: tuple[float, ...], right: tuple[float, ...]) -> float:
return sum(a * b for a, b in zip(left, right, strict=True))
+25 -4
View File
@@ -14,6 +14,7 @@ from gewu.domain.contracts import (
IngestReviewCreate,
IngestReviewDecision,
IngestReviewItem,
JevFacetResolution,
JevFacetScore,
)
@@ -54,6 +55,7 @@ class SQLiteIngestReviewStore:
jev_lens_confidence REAL,
jev_model TEXT,
jev_facet_scores_json TEXT NOT NULL DEFAULT '[]',
jev_facet_resolutions_json TEXT NOT NULL DEFAULT '[]',
status TEXT NOT NULL CHECK(status IN (
'needs-user-review', 'included', 'excluded', 'deferred'
)),
@@ -71,6 +73,7 @@ class SQLiteIngestReviewStore:
("jev_lens_confidence", "REAL"),
("jev_model", "TEXT"),
("jev_facet_scores_json", "TEXT NOT NULL DEFAULT '[]'"),
("jev_facet_resolutions_json", "TEXT NOT NULL DEFAULT '[]'"),
):
if column not in existing_columns:
connection.execute(
@@ -122,6 +125,7 @@ class SQLiteIngestReviewStore:
jev_lens_confidence: float | None = None,
jev_model: str | None = None,
jev_facet_scores: tuple[JevFacetScore, ...] = (),
jev_facet_resolutions: tuple[JevFacetResolution, ...] = (),
) -> IngestReviewItem:
payload = item.model_dump(mode="json")
digest = _digest({**payload, "submitted_by": submitted_by})
@@ -144,9 +148,9 @@ class SQLiteIngestReviewStore:
locator_json, excerpt, lens, summary, relevance_rationale, relevance,
status, submitted_by, created_at, payload_digest,
jev_relation_probability, jev_lens, jev_lens_confidence, jev_model,
jev_facet_scores_json
jev_facet_scores_json, jev_facet_resolutions_json
) VALUES (
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'needs-user-review', ?, ?, ?, ?, ?, ?, ?, ?
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'needs-user-review', ?, ?, ?, ?, ?, ?, ?, ?, ?
)""",
(
item.review_id,
@@ -172,6 +176,11 @@ class SQLiteIngestReviewStore:
ensure_ascii=False,
separators=(",", ":"),
),
json.dumps(
[item.model_dump(mode="json") for item in jev_facet_resolutions],
ensure_ascii=False,
separators=(",", ":"),
),
),
)
return self._read_item(connection, item.review_id)
@@ -284,7 +293,8 @@ class SQLiteIngestReviewStore:
"""SELECT review_id, project_id, run_id, source_revision_id, extraction_id,
locator_json, excerpt, lens, summary, relevance_rationale, relevance, status,
submitted_by, created_at, jev_relation_probability, jev_lens,
jev_lens_confidence, jev_model, jev_facet_scores_json
jev_lens_confidence, jev_model, jev_facet_scores_json,
jev_facet_resolutions_json
FROM ingest_review_items WHERE review_id=?""",
(review_id,),
).fetchone()
@@ -310,7 +320,18 @@ class SQLiteIngestReviewStore:
"jev_lens": row[15],
"jev_lens_confidence": row[16],
"jev_model": row[17],
"jev_facet_scores": json.loads(row[18]),
"jev_facet_scores": tuple(
JevFacetScore.model_validate_json(
json.dumps(score, ensure_ascii=False), strict=True
)
for score in json.loads(row[18])
),
"jev_facet_resolutions": tuple(
JevFacetResolution.model_validate_json(
json.dumps(resolution, ensure_ascii=False), strict=True
)
for resolution in json.loads(row[19])
),
},
strict=True,
)
+8 -2
View File
@@ -9,7 +9,7 @@ from dataclasses import dataclass
from pathlib import Path, PurePosixPath
from typing import Protocol
from gewu.domain.contracts import AutoAcceptPolicy, ProjectRecord
from gewu.domain.contracts import AutoAcceptPolicy, ProjectFacetLabel, ProjectRecord
class CanonicalRepositoryError(RuntimeError):
@@ -74,10 +74,16 @@ class ProjectRegistry(Protocol):
def create(self, project: ProjectRecord, policy: AutoAcceptPolicy) -> ProjectRecord: ...
def update_content(self, project_id: str, *, description: str, north_star: str, key_questions: tuple[str, ...]) -> ProjectRecord: ...
def update_content(
self, project_id: str, *, description: str, north_star: str, key_questions: tuple[str, ...]
) -> ProjectRecord: ...
def archive(self, project_id: str) -> ProjectRecord: ...
def add_facet_labels(
self, project_id: str, labels: tuple[ProjectFacetLabel, ...]
) -> ProjectRecord: ...
class EmbeddingProvider(Protocol):
"""Replaceable, normalized text-vector provider for derived retrieval indexes."""
+87 -5
View File
@@ -3,6 +3,7 @@
from __future__ import annotations
import json
import unicodedata
from contextlib import AbstractContextManager
from dataclasses import dataclass
from pathlib import Path
@@ -20,6 +21,7 @@ from gewu.domain.contracts import (
AutoAcceptPolicy,
ExtractionRecord,
KnowledgeNode,
ProjectFacetLabel,
ProjectRecord,
ProjectStatus,
SourceRevisionRecord,
@@ -307,15 +309,23 @@ class LocalProjectRegistry:
self, project_id: str, *, description: str, north_star: str, key_questions: tuple[str, ...]
) -> ProjectRecord:
"""Update a project's human-authored description and research questions."""
if self.repository is None or self.index_directory is None or self.authority_lock_path is None:
if (
self.repository is None
or self.index_directory is None
or self.authority_lock_path is None
):
raise RegistryError("Canonical project publication is not configured")
with self._write_lock():
project = _index(self._projects(), "project_id").get(project_id)
if project is None:
raise RegistryNotFound("Project is not registered")
updated = project.model_copy(update={
"description": description, "north_star": north_star, "key_questions": key_questions,
})
updated = project.model_copy(
update={
"description": description,
"north_star": north_star,
"key_questions": key_questions,
}
)
try:
self.repository.publish_files(
expected_head=self.repository.head(),
@@ -324,7 +334,9 @@ class LocalProjectRegistry:
index_directory=self.index_directory,
)
except CanonicalConflictError as error:
raise RegistryConflict("Canonical project registry changed during update") from error
raise RegistryConflict(
"Canonical project registry changed during update"
) from error
return updated
def archive(self, project_id: str) -> ProjectRecord:
@@ -359,6 +371,72 @@ class LocalProjectRegistry:
) from error
return archived
def add_facet_labels(
self, project_id: str, labels: tuple[ProjectFacetLabel, ...]
) -> ProjectRecord:
"""Atomically add bounded Jev-qualified labels to the canonical project record."""
if not labels or len(labels) > 20:
raise ValueError("Facet label additions must contain between 1 and 20 labels")
if (
self.repository is None
or self.index_directory is None
or self.authority_lock_path is None
):
raise RegistryError("Canonical project publication is not configured")
with self._write_lock():
project = _index(self._projects(), "project_id").get(project_id)
if project is None:
raise RegistryNotFound("Project is not registered")
existing = {
(label.facet, _normalize_facet_label(label.canonical_label))
for label in project.facet_labels
if label.status == "active"
}
existing.update(
(label.facet, _normalize_facet_label(alias))
for label in project.facet_labels
if label.status == "active"
for alias in label.aliases
)
accepted: list[ProjectFacetLabel] = []
known_ids = {label.label_id for label in project.facet_labels}
for label in labels:
if label.status != "active" or label.merged_into is not None:
raise ValueError("Only active new facet labels may be added")
if label.label_id in known_ids:
raise RegistryConflict("Facet label ID is already registered")
normalized_names = (
label.canonical_label,
*label.aliases,
)
if any(
(label.facet, _normalize_facet_label(name)) in existing
for name in normalized_names
):
raise RegistryConflict("Facet label duplicates an existing project label")
for name in normalized_names:
existing.add((label.facet, _normalize_facet_label(name)))
known_ids.add(label.label_id)
accepted.append(label)
if len(project.facet_labels) + len(accepted) > 2_000:
raise RegistryError("Project facet label catalog exceeds its configured limit")
updated = project.model_copy(
update={"facet_labels": (*project.facet_labels, *accepted)}
)
try:
self.repository.publish_files(
expected_head=self.repository.head(),
files={_project_record_path(project_id): _project_markdown(updated)},
message=f"Add qualified facet labels to {project_id}",
index_directory=self.index_directory,
)
except CanonicalConflictError as error:
raise RegistryConflict(
"Canonical project registry changed while adding facet labels"
) from error
return updated
def _write_lock(self) -> AbstractContextManager[Any]:
lock_path = self.authority_lock_path
if lock_path is None:
@@ -381,6 +459,10 @@ def _project_record_path(project_id: str) -> str:
return f"projects/{project_id}/project.md"
def _normalize_facet_label(value: str) -> str:
return " ".join(unicodedata.normalize("NFC", value).casefold().split())
def _policy_record_path(policy_id: str) -> str:
import hashlib
+29 -3
View File
@@ -16,6 +16,7 @@ from gewu.adapters.embeddings import LocalSentenceTransformer
from gewu.adapters.git_repository import GitCanonicalRepository
from gewu.adapters.openrouter_jev import OpenRouterJev
from gewu.adapters.source_extractor import LocalSourceExtractor, SourceExtractor
from gewu.adapters.sqlite_facet_labels import SQLiteFacetLabelIndex
from gewu.adapters.sqlite_identity import SQLiteIdentityStore
from gewu.adapters.sqlite_ingest_reviews import SQLiteIngestReviewStore
from gewu.adapters.sqlite_search import SQLiteSearchIndex
@@ -28,7 +29,7 @@ from gewu.application.context import ValidationContext
from gewu.application.diagnostics import build_service_diagnostics
from gewu.application.identity import IdentityProvider, OwnerIdentityProvider, Principal
from gewu.application.paths import reject_symlink_components
from gewu.application.ports import ProjectRegistry
from gewu.application.ports import EmbeddingProvider, ProjectRegistry
from gewu.application.reading import CanonicalNodeReader, CanonicalSourceReader, ReadError
from gewu.application.registry import (
LocalContextProvider,
@@ -84,6 +85,8 @@ def create_app(
account_identity: AccountIdentity | None = None,
ingest_review_store: SQLiteIngestReviewStore | None = None,
jev_assessor: OpenRouterJev | None = None,
facet_label_index: SQLiteFacetLabelIndex | None = None,
embedding_provider: EmbeddingProvider | None = None,
url_fetcher: PublicUrlFetcher | None = None,
) -> FastAPI:
if (identity_provider is None) != (node_reader is None):
@@ -796,6 +799,9 @@ def create_app(
url_fetcher=url_fetcher or PublicUrlFetcher(frozenset()),
ingest_review_store=ingest_review_store,
jev_assessor=jev_assessor,
project_registry=project_registry,
facet_label_index=facet_label_index,
embedding_provider=embedding_provider,
)
mcp_asgi = mcp_server.streamable_http_app(
streamable_http_path="/",
@@ -901,6 +907,8 @@ def create_local_app() -> FastAPI:
)
search_path = data_root / "search.sqlite3"
search_index: SearchIndex
embedding_provider: LocalSentenceTransformer | None = None
facet_label_index: SQLiteFacetLabelIndex | None = None
model_path = os.environ.get("GEWU_EMBEDDING_MODEL_PATH")
model_digest = os.environ.get("GEWU_EMBEDDING_MODEL_SHA256")
if model_path or model_digest:
@@ -918,6 +926,8 @@ def create_local_app() -> FastAPI:
expected_sha256=model_digest,
dimensions=int(os.environ.get("GEWU_EMBEDDING_DIMENSIONS", "384")),
)
embedding_provider = embeddings
facet_label_index = SQLiteFacetLabelIndex(search_path)
vectors = SQLiteVectorIndex(search_path)
lexical_index = SQLiteSearchIndex(search_path, repository)
hybrid_index = HybridSearchIndex(
@@ -975,18 +985,34 @@ def create_local_app() -> FastAPI:
context_provider=context_provider,
search_service=search_service,
project_registry=project_registry,
facet_label_index=facet_label_index,
embedding_provider=embedding_provider,
url_fetcher=url_fetcher,
)
def _load_local_jev_environment() -> None:
"""Load only Jev settings from the ignored, local project .env file."""
"""Load only Jev, local embedding and label policy settings from .env."""
path = Path.cwd() / ".env"
if path.is_symlink() or not path.is_file():
return
path.chmod(0o600)
allowed = {"OPENROUTER_API_KEY", "OPENROUTER_JEV_MODEL"}
allowed = {
"OPENROUTER_API_KEY",
"OPENROUTER_JEV_MODEL",
"GEWU_EMBEDDING_MODEL_PATH",
"GEWU_EMBEDDING_MODEL_SHA256",
"GEWU_EMBEDDING_MODEL_ID",
"GEWU_EMBEDDING_DIMENSIONS",
"GEWU_FACET_LABEL_TOP_N_TAGS",
"GEWU_FACET_LABEL_TOP_N_AUDIENCE",
"GEWU_FACET_LABEL_TOP_N_USE_FOR",
"GEWU_FACET_LABEL_TOP_N_DOMAIN",
"GEWU_FACET_LABEL_FIT_THRESHOLD",
"GEWU_FACET_LABEL_NEW_THRESHOLD",
"GEWU_FACET_LABEL_CONFIDENCE_THRESHOLD",
}
for raw_line in path.read_text(encoding="utf-8").splitlines():
line = raw_line.strip()
if not line or line.startswith("#") or "=" not in line:
+139 -7
View File
@@ -26,6 +26,7 @@ ResearchLens = Literal[
"financial-market",
]
FacetName = Literal["tags", "audience", "use_for", "domain"]
FacetLabelStatus = Literal["active", "merged", "retired"]
DEFAULT_RESEARCH_LENSES: tuple[ResearchLens, ...] = (
"technical",
"conceptual",
@@ -49,6 +50,17 @@ Sha256 = Annotated[str, StringConstraints(pattern=r"^[a-f0-9]{64}$")]
GitCommitId = Annotated[str, StringConstraints(pattern=r"^(?:[a-f0-9]{40}|[a-f0-9]{64})$")]
def _normalized_facet_label(value: str) -> str:
return " ".join(unicodedata.normalize("NFC", value).casefold().split())
def _clean_facet_label(value: str) -> str:
cleaned = " ".join(unicodedata.normalize("NFC", value).split())
if not cleaned:
raise ValueError("Facet labels must contain non-whitespace text")
return cleaned
class StrictRecord(BaseModel):
"""Base for public records: reject implicit coercion and unknown fields."""
@@ -63,6 +75,45 @@ class ProjectStatus(StrEnum):
ARCHIVED = "archived"
class ProjectFacetLabel(StrictRecord):
label_id: RecordId
facet: FacetName
canonical_label: SearchFacetValue
definition: str = Field(min_length=1, max_length=500)
aliases: tuple[SearchFacetValue, ...] = Field(default=(), max_length=20)
status: FacetLabelStatus = "active"
merged_into: RecordId | None = None
origin_review_id: RecordId | None = None
jev_model: str | None = Field(default=None, max_length=100)
jev_fit_probability: FiniteFloat | None = Field(default=None, ge=0, le=1)
jev_new_probability: FiniteFloat | None = Field(default=None, ge=0, le=1)
created_at: datetime
@field_validator("canonical_label")
@classmethod
def normalize_canonical_label(cls, value: str) -> str:
return _clean_facet_label(value)
@field_validator("aliases")
@classmethod
def normalize_aliases(cls, values: tuple[str, ...]) -> tuple[str, ...]:
return tuple(_clean_facet_label(value) for value in values)
@model_validator(mode="after")
def validate_label_lifecycle(self) -> ProjectFacetLabel:
if self.status == "merged" and self.merged_into is None:
raise ValueError("Merged facet labels require a canonical target")
if self.status != "merged" and self.merged_into is not None:
raise ValueError("Only merged facet labels may have a merge target")
names = (self.canonical_label, *self.aliases)
if len({_normalized_facet_label(name) for name in names}) != len(names):
raise ValueError("Facet label and aliases must be unique")
offset = self.created_at.utcoffset()
if offset is None or offset.total_seconds() != 0:
raise ValueError("Facet label timestamp must be timezone-aware UTC")
return self
class ProjectRecord(StrictRecord):
project_id: RecordId
title: str = Field(min_length=1, max_length=200)
@@ -75,6 +126,7 @@ class ProjectRecord(StrictRecord):
watch_list: tuple[str, ...] = Field(default=(), max_length=100)
default_includes: tuple[RecordId, ...] = ("shared",)
taxonomy_extensions: dict[str, tuple[str, ...]] = Field(default_factory=dict, max_length=100)
facet_labels: tuple[ProjectFacetLabel, ...] = Field(default=(), max_length=2000)
autonomy_policy_id: RecordId
@field_validator("project_id")
@@ -109,6 +161,7 @@ class ProjectLens(StrictRecord):
research_lenses: tuple[ResearchLens, ...] = DEFAULT_RESEARCH_LENSES
watch_list: tuple[str, ...] = Field(default=(), max_length=100)
taxonomy_extensions: dict[str, tuple[str, ...]] = Field(default_factory=dict, max_length=100)
facet_labels: tuple[ProjectFacetLabel, ...] = Field(default=(), max_length=2000)
ingest_guidance: tuple[IngestGuidance, ...] = Field(default=(), max_length=100)
@@ -122,6 +175,45 @@ class JevFacetScore(StrictRecord):
probability: FiniteFloat = Field(ge=0, le=1)
class JevFacetOptionScore(StrictRecord):
label_id: RecordId | None = None
label: SearchFacetValue
probability: FiniteFloat = Field(ge=0, le=1)
class JevFacetResolution(StrictRecord):
facet: FacetName
value: SearchFacetValue
fit_probability: FiniteFloat = Field(ge=0, le=1)
selected_label_id: RecordId | None = None
selected_label: SearchFacetValue | None = None
selected_new_label: bool = False
new_probability: FiniteFloat = Field(ge=0, le=1)
choice_confidence: FiniteFloat = Field(ge=0, le=1)
option_probabilities: tuple[JevFacetOptionScore, ...] = Field(default=(), max_length=6)
rank: int = Field(ge=1, le=20)
added_to_project: bool = False
class FacetLabelProposal(StrictRecord):
facet: FacetName
value: SearchFacetValue
definition: str = Field(min_length=1, max_length=500)
@field_validator("value")
@classmethod
def normalize_value(cls, value: str) -> str:
return _clean_facet_label(value)
@field_validator("definition")
@classmethod
def normalize_definition(cls, value: str) -> str:
definition = value.strip()
if not definition:
raise ValueError("Facet label definitions must contain non-whitespace text")
return definition
class IngestReviewItem(StrictRecord):
review_id: RecordId
project_id: RecordId
@@ -135,6 +227,7 @@ class IngestReviewItem(StrictRecord):
relevance_rationale: str = Field(min_length=1, max_length=2000)
relevance: Literal["borderline", "low"] = "borderline"
jev_facet_scores: tuple[JevFacetScore, ...] = Field(default=(), max_length=20)
jev_facet_resolutions: tuple[JevFacetResolution, ...] = Field(default=(), max_length=20)
jev_relation_probability: FiniteFloat | None = Field(default=None, ge=0, le=1)
jev_lens: ResearchLens | None = None
jev_lens_confidence: FiniteFloat | None = Field(default=None, ge=0, le=1)
@@ -165,9 +258,10 @@ class IngestReviewCreate(StrictRecord):
relevance_rationale: str = Field(min_length=1, max_length=2000)
relevance: Literal["borderline", "low"] = "borderline"
candidate_tags: tuple[SearchFacetValue, ...] = Field(default=(), max_length=5)
candidate_audience: tuple[SearchFacetValue, ...] = Field(default=(), max_length=5)
candidate_use_for: tuple[SearchFacetValue, ...] = Field(default=(), max_length=5)
candidate_domain: tuple[SearchFacetValue, ...] = Field(default=(), max_length=5)
candidate_audience: tuple[SearchFacetValue, ...] = Field(default=(), max_length=3)
candidate_use_for: tuple[SearchFacetValue, ...] = Field(default=(), max_length=3)
candidate_domain: tuple[SearchFacetValue, ...] = Field(default=(), max_length=3)
candidate_facet_labels: tuple[FacetLabelProposal, ...] = Field(default=(), max_length=14)
@model_validator(mode="after")
def limit_total_facet_candidates(self) -> IngestReviewCreate:
@@ -177,17 +271,55 @@ class IngestReviewCreate(StrictRecord):
*self.candidate_use_for,
*self.candidate_domain,
)
if len(values) > 20:
raise ValueError("At most 20 facet candidates may be assessed")
if len({(facet, value.casefold()) for facet, candidates in (
proposed = (*values, *(proposal.value for proposal in self.candidate_facet_labels))
if len(proposed) > 14:
raise ValueError("At most 14 facet candidates may be assessed")
facet_limits = {"tags": 5, "audience": 3, "use_for": 3, "domain": 3}
facet_counts = {facet: 0 for facet in facet_limits}
for facet, candidates in (
("tags", self.candidate_tags),
("audience", self.candidate_audience),
("use_for", self.candidate_use_for),
("domain", self.candidate_domain),
) for value in candidates}) != len(values):
):
facet_counts[facet] += len(candidates)
for proposal in self.candidate_facet_labels:
facet_counts[proposal.facet] += 1
if any(facet_counts[facet] > limit for facet, limit in facet_limits.items()):
raise ValueError("Facet candidates exceed the per-facet limits (tags: 5; others: 3)")
keyed = {
(facet, _normalized_facet_label(value))
for facet, candidates in (
("tags", self.candidate_tags),
("audience", self.candidate_audience),
("use_for", self.candidate_use_for),
("domain", self.candidate_domain),
)
for value in candidates
}
keyed.update(
(proposal.facet, _normalized_facet_label(proposal.value))
for proposal in self.candidate_facet_labels
)
if len(keyed) != len(proposed):
raise ValueError("Facet candidates must be unique within each facet")
return self
def facet_proposals(self) -> tuple[FacetLabelProposal, ...]:
proposals: list[FacetLabelProposal] = list(self.candidate_facet_labels)
fields: tuple[tuple[FacetName, tuple[SearchFacetValue, ...]], ...] = (
("tags", self.candidate_tags),
("audience", self.candidate_audience),
("use_for", self.candidate_use_for),
("domain", self.candidate_domain),
)
for facet, values in fields:
proposals.extend(
FacetLabelProposal(facet=facet, value=value, definition=self.summary[:500])
for value in values
)
return tuple(proposals)
class IngestReviewDecision(StrictRecord):
decision_id: RecordId
+1
View File
@@ -69,6 +69,7 @@ OPERATIONS: Final[tuple[Operation, ...]] = (
Operation("source_extract", "proposal:submit", frozenset({Transport.MCP})),
Operation("source_extraction_block", "proposal:submit", frozenset({Transport.MCP})),
Operation("proposal_submit", "proposal:submit", frozenset({Transport.MCP}), True),
Operation("proposal_prepare", "proposal:evaluate", frozenset({Transport.MCP})),
Operation("proposal_evaluate", "proposal:evaluate", frozenset({Transport.MCP})),
Operation("proposal_status", "proposal:read", frozenset({Transport.MCP})),
Operation("proposal_withdraw", "proposal:withdraw", frozenset({Transport.MCP})),
+69
View File
@@ -6,7 +6,9 @@ import hashlib
import json
import unicodedata
from collections.abc import Iterable
from copy import deepcopy
from dataclasses import dataclass
from typing import Any
from gewu.domain.contracts import (
AutoAcceptPolicy,
@@ -59,6 +61,73 @@ def proposal_payload_digest(proposal: ProposalRevision) -> str:
return hashlib.sha256(canonical_json(operations)).hexdigest()
def prepare_proposal_digests(draft: dict[str, Any]) -> ProposalRevision:
"""Validate a proposal draft, fill its dependent evidence digests, and pin its payload."""
value = deepcopy(draft)
operations = value.get("operations")
if not isinstance(operations, list) or not operations:
raise ValueError("Proposal must contain a non-empty operations array")
for operation in operations:
if not isinstance(operation, dict):
raise ValueError("Proposal operations must be objects")
node = operation.get("node")
if isinstance(node, dict):
for group in ("claims", "assumptions", "limitations"):
claims = node.get(group, [])
if not isinstance(claims, list):
raise ValueError(f"Node {group} must be an array")
for index, claim in enumerate(claims):
if not isinstance(claim, dict) or not isinstance(claim.get("text"), str):
raise ValueError(f"Node {group}[{index}] must contain claim text")
digest = assertion_digest(claim["text"])
evidence = claim.get("evidence")
if not isinstance(evidence, list) or not evidence:
raise ValueError(f"Node {group}[{index}] must contain evidence")
path = f"{group}[{index}].text"
evidence_digests = []
for item in evidence:
if not isinstance(item, dict):
raise ValueError("Evidence references must be objects")
item["assertion_path"] = path
item["assertion_digest"] = digest
parsed_evidence = EvidenceReference.model_validate(item, strict=True)
item.clear()
item.update(parsed_evidence.model_dump(mode="json"))
evidence_digests.append(evidence_digest(parsed_evidence))
assessment = claim.get("assessment")
if not isinstance(assessment, dict):
raise ValueError(f"Node {group}[{index}] must contain an assessment")
assessment["assertion_digest"] = digest
assessment["evidence_digests"] = evidence_digests
edge = operation.get("edge")
if isinstance(edge, dict):
edge_evidence = edge.get("evidence", [])
if not isinstance(edge_evidence, list):
raise ValueError("Edge evidence must be an array")
for item in edge_evidence:
if not isinstance(item, dict):
raise ValueError("Evidence references must be objects")
item["assertion_path"] = "edge"
item["assertion_digest"] = "0" * 64
parsed_edge = TypedEdge.model_validate(edge, strict=True)
digest = edge_assertion_digest(parsed_edge)
for item in edge_evidence:
item["assertion_digest"] = digest
parsed_evidence = EvidenceReference.model_validate(item, strict=True)
item.clear()
item.update(parsed_evidence.model_dump(mode="json"))
edge["evidence"] = edge_evidence
value["payload_digest"] = "0" * 64
provisional = ProposalRevision.model_validate_json(
json.dumps(value, ensure_ascii=False, allow_nan=False), strict=True
)
return provisional.model_copy(update={"payload_digest": proposal_payload_digest(provisional)})
def edge_assertion_digest(edge: TypedEdge) -> str:
"""Bind edge evidence to the relationship, conditions and rationale."""
+431 -23
View File
@@ -4,9 +4,14 @@ from __future__ import annotations
import base64
import binascii
import difflib
import hashlib
import json
import logging
import os
import re
import sqlite3
import unicodedata
from collections.abc import Callable, Iterator
from datetime import UTC, date, datetime
from typing import Literal
@@ -17,15 +22,17 @@ from mcp.server import MCPServer
from mcp.server.mcpserver import Context
from pydantic import ValidationError
from gewu.adapters.openrouter_jev import JevUnavailable, OpenRouterJev
from gewu.adapters.openrouter_jev import JevAssessment, JevUnavailable, OpenRouterJev
from gewu.adapters.pdf_extractor import PdfExtractionError
from gewu.adapters.source_extractor import SourceExtractor
from gewu.adapters.sqlite_facet_labels import SQLiteFacetLabelIndex
from gewu.adapters.sqlite_ingest_reviews import SQLiteIngestReviewStore
from gewu.adapters.url_fetch import PublicUrlFetcher, UrlCaptureError
from gewu.application.acceptance import AcceptanceError, AcceptanceService, ActorContext
from gewu.application.context import ValidationContext
from gewu.application.diagnostics import build_service_diagnostics
from gewu.application.identity import IdentityProvider, Principal
from gewu.application.ports import EmbeddingProvider, ProjectRegistry
from gewu.application.reading import CanonicalNodeReader, CanonicalSourceReader, ReadError
from gewu.application.runs import LocalRunService, RunError
from gewu.application.search import CanonicalSearchService, SearchError
@@ -38,12 +45,17 @@ from gewu.domain.contracts import (
Confidence,
EvidenceReference,
ExtractionRecord,
FacetLabelProposal,
FacetName,
GitCommitId,
IngestReviewCreate,
JevFacetResolution,
NeighborRequest,
NodeType,
ProjectContextResponse,
ProjectFacetLabel,
ProjectLens,
ProjectRecord,
ProposalRevision,
ReliabilityGrade,
RunCheckpointRequest,
@@ -53,11 +65,288 @@ from gewu.domain.contracts import (
SourceListRequest,
SourceRevisionRecord,
)
from gewu.domain.validation import validate_proposal
from gewu.domain.validation import prepare_proposal_digests, validate_proposal
from gewu.version import SERVICE_VERSION
MAX_PASTED_TEXT_BYTES = 1_000_000
MAX_PROJECT_CONTEXT_BYTES = 64_000
MAX_PROJECT_FACET_LABELS = 2_000
logger = logging.getLogger(__name__)
def _normalize_facet_value(value: str) -> str:
return " ".join(unicodedata.normalize("NFC", value).casefold().split())
def _project_facet_catalog(
project: ProjectRecord, context: ValidationContext
) -> tuple[ProjectFacetLabel, ...]:
project_id = project.project_id
labels = [label for label in project.facet_labels if label.status == "active"]
seen = {
(label.facet, _normalize_facet_value(name))
for label in labels
for name in (label.canonical_label, *label.aliases)
}
collected: dict[tuple[FacetName, str], tuple[int, str]] = {}
facet_fields: tuple[tuple[FacetName, str], ...] = (
("tags", "tags"),
("audience", "audience"),
("use_for", "use_for"),
("domain", "domain"),
)
for node in context.existing_nodes.values():
if node.project_id != project_id:
continue
for facet, field in facet_fields:
for value in getattr(node, field):
normalized = _normalize_facet_value(value)
if (facet, normalized) not in seen:
count, display = collected.get((facet, normalized), (0, value.strip()))
collected[(facet, normalized)] = (count + 1, display)
epoch = datetime(1970, 1, 1, tzinfo=UTC)
for (facet, normalized), (_count, display) in sorted(collected.items()):
if len(labels) >= MAX_PROJECT_FACET_LABELS:
break
label_text = display
if not label_text:
continue
digest = hashlib.sha256(f"{project_id}\0{facet}\0{normalized}".encode()).hexdigest()[:24]
label_id = f"facet-observed-{digest}"
labels.append(
ProjectFacetLabel(
label_id=label_id,
facet=facet,
canonical_label=label_text,
definition="Observed on existing accepted knowledge records.",
created_at=epoch,
)
)
seen.add((facet, normalized))
return tuple(labels[:MAX_PROJECT_FACET_LABELS])
def _nearest_facet_labels(
*,
proposal: FacetLabelProposal,
project_id: str,
catalog: tuple[ProjectFacetLabel, ...],
index: SQLiteFacetLabelIndex | None,
embeddings: EmbeddingProvider | None,
) -> tuple[ProjectFacetLabel, ...]:
facet = proposal.facet
value = proposal.value
query = f"{value}. {proposal.definition}"
relevant = tuple(label for label in catalog if label.facet == facet)
normalized = _normalize_facet_value(value)
exact = tuple(
label
for label in relevant
if normalized
in {
_normalize_facet_value(label.canonical_label),
*(_normalize_facet_value(alias) for alias in label.aliases),
}
)
if exact:
return exact[:5]
ordered: dict[str, tuple[float, ProjectFacetLabel]] = {}
if index is not None and embeddings is not None and relevant:
matches = index.closest(
project_id=project_id,
facet=facet,
query=query,
labels=relevant,
embeddings=embeddings,
limit=5,
)
by_id = {label.label_id: label for label in relevant}
for match in matches:
label = by_id.get(match.label_id)
if label is not None:
ordered[label.label_id] = (match.similarity, label)
query_terms = set(re.findall(r"[\w-]+", normalized))
lexical: list[tuple[float, ProjectFacetLabel]] = []
for label in relevant:
label_terms = set(
re.findall(
r"[\w-]+",
_normalize_facet_value(
f"{label.canonical_label} {label.definition} {' '.join(label.aliases)}"
),
)
)
overlap = len(query_terms & label_terms) / max(len(query_terms | label_terms), 1)
sequence = difflib.SequenceMatcher(
None,
normalized,
_normalize_facet_value(label.canonical_label),
autojunk=False,
).ratio()
lexical.append((max(overlap, sequence), label))
for _rank, (score, label) in enumerate(
sorted(lexical, key=lambda item: (-item[0], item[1].label_id))[:5], start=1
):
if score > 0:
prior = ordered.get(label.label_id)
ordered[label.label_id] = (score + (prior[0] if prior else 0.0), label)
return tuple(
label
for _score, label in sorted(
ordered.values(), key=lambda item: (-item[0], item[1].label_id)
)[:5]
)
def _promotion_policy() -> tuple[dict[FacetName, int], float, float, float]:
try:
top_n: dict[FacetName, int] = {
"tags": int(os.environ.get("GEWU_FACET_LABEL_TOP_N_TAGS", "5")),
"audience": int(os.environ.get("GEWU_FACET_LABEL_TOP_N_AUDIENCE", "3")),
"use_for": int(os.environ.get("GEWU_FACET_LABEL_TOP_N_USE_FOR", "3")),
"domain": int(os.environ.get("GEWU_FACET_LABEL_TOP_N_DOMAIN", "3")),
}
fit_threshold = float(os.environ.get("GEWU_FACET_LABEL_FIT_THRESHOLD", "0.90"))
new_threshold = float(os.environ.get("GEWU_FACET_LABEL_NEW_THRESHOLD", "0.90"))
confidence_threshold = float(
os.environ.get("GEWU_FACET_LABEL_CONFIDENCE_THRESHOLD", "0.80")
)
except ValueError as error:
raise RuntimeError("Facet label promotion settings are invalid") from error
limits: dict[FacetName, int] = {"tags": 5, "audience": 3, "use_for": 3, "domain": 3}
if any(not 1 <= top_n[facet] <= maximum for facet, maximum in limits.items()) or any(
not 0 <= threshold <= 1
for threshold in (fit_threshold, new_threshold, confidence_threshold)
):
raise RuntimeError("Facet label promotion settings are outside supported bounds")
return top_n, fit_threshold, new_threshold, confidence_threshold
def _promote_jev_labels(
*,
item: IngestReviewCreate,
assessment: JevAssessment,
project_id: str,
project_registry: ProjectRegistry | None,
facet_label_index: SQLiteFacetLabelIndex | None,
embedding_provider: EmbeddingProvider | None,
existing_catalog: tuple[ProjectFacetLabel, ...],
semantic_search_ready: bool,
) -> tuple[JevFacetResolution, ...]:
resolutions = assessment.facet_resolutions
if not resolutions:
return ()
top_n, fit_threshold, new_threshold, confidence_threshold = _promotion_policy()
ranks: dict[tuple[FacetName, str], int] = {}
for facet in ("tags", "audience", "use_for", "domain"):
facet_resolutions = tuple(r for r in resolutions if r.facet == facet)
new_candidates = tuple(r for r in facet_resolutions if r.selected_new_label)
new_ranks = {
(r.facet, r.value): rank
for rank, r in enumerate(
sorted(
new_candidates,
key=lambda r: (
-(r.fit_probability * r.new_probability),
r.value.casefold(),
),
),
start=1,
)
}
existing_rank_start = len(new_candidates) + 1
existing_candidates = sorted(
(r for r in facet_resolutions if not r.selected_new_label),
key=lambda r: (-(r.fit_probability * r.new_probability), r.value.casefold()),
)
ranks.update(new_ranks)
ranks.update(
{
(r.facet, r.value): rank
for rank, r in enumerate(existing_candidates, start=existing_rank_start)
}
)
ranked_resolutions = tuple(
resolution.model_copy(update={"rank": ranks[(resolution.facet, resolution.value)]})
for resolution in resolutions
)
if (
project_registry is None
or facet_label_index is None
or embedding_provider is None
or not semantic_search_ready
):
return ranked_resolutions
proposals = {(proposal.facet, proposal.value): proposal for proposal in item.facet_proposals()}
existing = {
(label.facet, _normalize_facet_value(name))
for label in existing_catalog
if label.status == "active"
for name in (label.canonical_label, *label.aliases)
}
eligible = tuple(
resolution
for resolution in ranked_resolutions
if resolution.rank <= top_n[resolution.facet]
and resolution.selected_new_label
and resolution.fit_probability >= fit_threshold
and resolution.new_probability >= new_threshold
and resolution.choice_confidence >= confidence_threshold
and (resolution.facet, _normalize_facet_value(resolution.value)) not in existing
)
if not eligible:
return ranked_resolutions
by_facet_rank = {
(resolution.facet, resolution.value): resolution.rank for resolution in eligible
}
additions: list[ProjectFacetLabel] = []
for resolution in eligible:
proposal = proposals[(resolution.facet, resolution.value)]
label_digest = hashlib.sha256(
f"{project_id}\0{resolution.facet}\0{_normalize_facet_value(resolution.value)}".encode()
).hexdigest()[:24]
additions.append(
ProjectFacetLabel(
label_id=f"facet-{resolution.facet}-{label_digest}",
facet=resolution.facet,
canonical_label=resolution.value,
definition=proposal.definition,
origin_review_id=item.review_id,
jev_model=assessment.model,
jev_fit_probability=resolution.fit_probability,
jev_new_probability=resolution.new_probability,
created_at=datetime.now(UTC),
)
)
try:
project_registry.add_facet_labels(project_id, tuple(additions))
except (RuntimeError, ValueError) as error:
# The review flag and its Jev judgment remain durable if catalog publication races.
logger.warning("Could not publish Jev-qualified project facet labels: %s", error)
return ranked_resolutions
added_keys = {(label.facet, label.canonical_label) for label in additions}
for label in additions:
try:
facet_label_index.closest(
project_id=project_id,
facet=label.facet,
query=label.canonical_label + ". " + label.definition,
labels=(label,),
embeddings=embedding_provider,
limit=1,
)
except (OSError, RuntimeError, ValueError, sqlite3.Error) as error:
# Vectors are a cache and can be rebuilt from the canonical project catalog.
logger.warning("Could not cache embedding for new project facet label: %s", error)
return tuple(
resolution.model_copy(
update={
"rank": by_facet_rank.get((resolution.facet, resolution.value), resolution.rank),
"added_to_project": (resolution.facet, resolution.value) in added_keys,
}
)
for resolution in ranked_resolutions
)
def _principal(ctx: Context, identities: IdentityProvider) -> Principal:
@@ -74,6 +363,28 @@ def _principal(ctx: Context, identities: IdentityProvider) -> Principal:
return principal
def _validation_issues(error: ValidationError) -> list[dict[str, str]]:
issues: list[dict[str, str]] = []
for detail in error.errors(
include_context=False,
include_input=False,
include_url=False,
)[:64]:
path = ""
for part in detail.get("loc", ()):
path += f"[{part}]" if isinstance(part, int) else (
("." if path else "") + str(part)
)
issues.append(
{
"path": path,
"code": str(detail.get("type", "invalid_value")),
"message": str(detail.get("msg", "Invalid value")),
}
)
return issues
def create_mcp_server(
*,
identity_provider: IdentityProvider,
@@ -87,6 +398,9 @@ def create_mcp_server(
url_fetcher: PublicUrlFetcher | None = None,
ingest_review_store: SQLiteIngestReviewStore | None = None,
jev_assessor: OpenRouterJev | None = None,
project_registry: ProjectRegistry | None = None,
facet_label_index: SQLiteFacetLabelIndex | None = None,
embedding_provider: EmbeddingProvider | None = None,
) -> MCPServer:
"""Create the allowlisted MCP server for the supplied application services."""
@@ -381,9 +695,10 @@ def create_mcp_server(
try:
item = IngestReviewCreate.model_validate_json(review_json, strict=True)
except ValidationError as error:
raise ValueError(
"Ingest review request does not match the GEWU v1 contract"
) from error
return json.dumps(
{"status": "invalid", "issues": _validation_issues(error)},
ensure_ascii=False,
)
_require_project_submission(principal, item.project_id)
actor = ActorContext(principal.principal_id, principal.credential_id)
try:
@@ -433,11 +748,56 @@ def create_mcp_server(
"Ingest review locator metadata does not match its extraction block"
)
assessment = None
facet_resolutions: tuple[JevFacetResolution, ...] = ()
if jev_assessor is not None:
project = context_provider().projects.get(item.project_id)
context = context_provider() if context_provider is not None else None
project = context.projects.get(item.project_id) if context is not None else None
if project is not None:
try:
assessment = jev_assessor.assess(item, project)
catalog = _project_facet_catalog(project, context)
nearest: dict[tuple[FacetName, str], tuple[ProjectFacetLabel, ...]] = {}
semantic_search_ready = (
facet_label_index is not None and embedding_provider is not None
)
for proposal in item.facet_proposals():
try:
nearest[(proposal.facet, proposal.value)] = (
_nearest_facet_labels(
proposal=proposal,
project_id=item.project_id,
catalog=catalog,
index=(
facet_label_index if semantic_search_ready else None
),
embeddings=(
embedding_provider
if semantic_search_ready
else None
),
)
)
except (OSError, RuntimeError, ValueError, sqlite3.Error):
semantic_search_ready = False
nearest[(proposal.facet, proposal.value)] = (
_nearest_facet_labels(
proposal=proposal,
project_id=item.project_id,
catalog=catalog,
index=None,
embeddings=None,
)
)
assessment = jev_assessor.assess(item, project, closest_labels=nearest)
facet_resolutions = _promote_jev_labels(
item=item,
assessment=assessment,
project_id=item.project_id,
project_registry=project_registry,
facet_label_index=facet_label_index,
embedding_provider=embedding_provider,
existing_catalog=catalog,
semantic_search_ready=semantic_search_ready,
)
except JevUnavailable:
# The review flag and normal proposal flow do not depend on Jev uptime.
assessment = None
@@ -453,6 +813,7 @@ def create_mcp_server(
),
jev_model=assessment.model if assessment is not None else None,
jev_facet_scores=(assessment.facet_scores if assessment is not None else ()),
jev_facet_resolutions=facet_resolutions,
)
return json.dumps(record.model_dump(mode="json"), ensure_ascii=False)
@@ -890,8 +1251,9 @@ def create_mcp_server(
offset: int,
length: int,
ctx: Context,
include_original_mapping: bool = False,
) -> str:
"""Read a bounded normalized text slice and its source offset map."""
"""Read a bounded normalized text slice and optionally its original-source offset map."""
principal = _principal(ctx, identity_provider)
_require_project_submission(principal, project_id)
@@ -906,21 +1268,20 @@ def create_mcp_server(
if block is None or offset >= len(block.normalized_text):
raise ValueError("Extraction block or offset does not exist")
end = min(offset + length, len(block.normalized_text))
spans = block.normalization_map[offset:end]
original_start = min(start for start, _ in spans)
original_end = max(stop for _, stop in spans)
return json.dumps(
{
"extraction_id": extraction_id,
"block_id": block_id,
"page": block.page,
"offset": offset,
"normalized_text": block.normalized_text[offset:end],
"original_text": block.original_text[original_start:original_end],
"normalization_map": spans,
},
ensure_ascii=False,
)
result: dict[str, object] = {
"extraction_id": extraction_id,
"block_id": block_id,
"page": block.page,
"offset": offset,
"normalized_text": block.normalized_text[offset:end],
}
if include_original_mapping:
spans = block.normalization_map[offset:end]
original_start = min(start for start, _ in spans)
original_end = max(stop for _, stop in spans)
result["original_text"] = block.original_text[original_start:original_end]
result["normalization_map"] = spans
return json.dumps(result, ensure_ascii=False)
@server.tool(name="proposal_submit", structured_output=False)
def proposal_submit(proposal_json: str, ctx: Context) -> str:
@@ -969,6 +1330,52 @@ def create_mcp_server(
}
return json.dumps(result, ensure_ascii=False)
@server.tool(name="proposal_prepare", structured_output=False)
def proposal_prepare(proposal_json: str, ctx: Context) -> str:
"""Fill canonical evidence and payload digests without storing the proposal."""
principal = _principal(ctx, identity_provider)
if len(proposal_json.encode("utf-8")) > 1_000_000:
raise ValueError("Proposal exceeds the supported request size")
try:
draft = json.loads(proposal_json, object_pairs_hook=_reject_duplicate_keys)
except (ValueError, TypeError) as error:
raise ValueError("Proposal draft must be valid JSON") from error
if not isinstance(draft, dict):
raise ValueError("Proposal draft must be a JSON object")
try:
proposal = prepare_proposal_digests(draft)
except ValidationError as error:
return json.dumps(
{"status": "invalid", "issues": _validation_issues(error)},
ensure_ascii=False,
)
except ValueError as error:
return json.dumps(
{
"status": "invalid",
"issues": [
{
"path": "",
"code": "invalid_proposal_draft",
"message": str(error)[:500],
}
],
},
ensure_ascii=False,
)
_authorize_proposal(principal, proposal, context_provider())
return json.dumps(
{
"status": "prepared",
"proposal_id": proposal.proposal_id,
"revision": proposal.revision,
"payload_digest": proposal.payload_digest,
"proposal_json": proposal.model_dump_json(),
},
ensure_ascii=False,
)
@server.tool(name="proposal_evaluate", structured_output=False)
def proposal_evaluate(proposal_json: str, ctx: Context) -> str:
"""Evaluate one bounded proposal without storing or publishing it."""
@@ -1117,6 +1524,7 @@ def create_mcp_server(
research_lenses=project.research_lenses,
watch_list=project.watch_list,
taxonomy_extensions=project.taxonomy_extensions,
facet_labels=_project_facet_catalog(project, context),
ingest_guidance=(
ingest_review_store.guidance(project_id)
if ingest_review_store is not None