This commit is contained in:
1 parent
0e1a2ad008
commit
34f4923566
28 files changed
+3932
-81
No files matched your search
@@ -9,9 +9,13 @@ from urllib.error import HTTPError, URLError
|
||||
from urllib.request import HTTPRedirectHandler, Request, build_opener
|
||||
|
||||
from gewu.domain.contracts import (
|
||||
FacetLabelProposal,
|
||||
FacetName,
|
||||
IngestReviewCreate,
|
||||
JevFacetOptionScore,
|
||||
JevFacetResolution,
|
||||
JevFacetScore,
|
||||
ProjectFacetLabel,
|
||||
ProjectRecord,
|
||||
ResearchLens,
|
||||
)
|
||||
@@ -28,6 +32,7 @@ class JevAssessment:
|
||||
lens_confidence: float
|
||||
model: str
|
||||
facet_scores: tuple[JevFacetScore, ...] = ()
|
||||
facet_resolutions: tuple[JevFacetResolution, ...] = ()
|
||||
|
||||
|
||||
class JevUnavailable(RuntimeError):
|
||||
@@ -50,31 +55,75 @@ class OpenRouterJev:
|
||||
self._api_key = api_key
|
||||
self._model = model
|
||||
|
||||
def assess(self, item: IngestReviewCreate, project: ProjectRecord) -> JevAssessment:
|
||||
facet_candidates: tuple[tuple[FacetName, tuple[str, ...]], ...] = (
|
||||
("tags", item.candidate_tags),
|
||||
("audience", item.candidate_audience),
|
||||
("use_for", item.candidate_use_for),
|
||||
("domain", item.candidate_domain),
|
||||
)
|
||||
def assess(
|
||||
self,
|
||||
item: IngestReviewCreate,
|
||||
project: ProjectRecord,
|
||||
*,
|
||||
closest_labels: dict[tuple[FacetName, str], tuple[ProjectFacetLabel, ...]] | None = None,
|
||||
) -> JevAssessment:
|
||||
proposals = item.facet_proposals()
|
||||
closest_labels = closest_labels or {}
|
||||
facet_questions: dict[str, object] = {}
|
||||
facet_question_map: dict[str, tuple[FacetName, str]] = {}
|
||||
for facet, candidates in facet_candidates:
|
||||
for index, value in enumerate(candidates):
|
||||
key = f"facet_{facet}_{index}"
|
||||
facet_question_map[key] = (facet, value)
|
||||
facet_questions[key] = {
|
||||
"type": "noul",
|
||||
"instructions": (
|
||||
f"Does the candidate {facet} label accurately describe the passage "
|
||||
"in the context of this project? Judge the label itself, not just "
|
||||
"whether its words appear in the excerpt."
|
||||
),
|
||||
"criteria": {
|
||||
"true": "This label is a useful, well-supported classification.",
|
||||
"false": "This label is inaccurate, unsupported, or too broad.",
|
||||
},
|
||||
fit_question_map: dict[str, FacetLabelProposal] = {}
|
||||
resolution_question_map: dict[
|
||||
str, tuple[FacetLabelProposal, tuple[tuple[str, ProjectFacetLabel], ...]]
|
||||
] = {}
|
||||
facet_state: list[dict[str, object]] = []
|
||||
for index, proposal in enumerate(proposals):
|
||||
existing_labels = closest_labels.get((proposal.facet, proposal.value), ())[:5]
|
||||
fit_key = f"facet_fit_{proposal.facet}_{index}"
|
||||
match_key = f"facet_match_{proposal.facet}_{index}"
|
||||
fit_question_map[fit_key] = proposal
|
||||
facet_questions[fit_key] = {
|
||||
"type": "noul",
|
||||
"instructions": (
|
||||
f"Is this candidate {proposal.facet} label useful and accurate for "
|
||||
"classifying the passage in this project?"
|
||||
),
|
||||
"criteria": {
|
||||
"true": "The label accurately captures a reusable facet of this passage.",
|
||||
"false": "The label is inaccurate, unsupported, or too specific to reuse.",
|
||||
},
|
||||
}
|
||||
choice_criteria: dict[str, str] = {}
|
||||
option_map: list[tuple[str, ProjectFacetLabel]] = []
|
||||
for option_index, label in enumerate(existing_labels):
|
||||
option_key = f"existing_{option_index}"
|
||||
choice_criteria[option_key] = (
|
||||
f"Reuse canonical label {label.canonical_label!r}: {label.definition}"
|
||||
)
|
||||
option_map.append((option_key, label))
|
||||
choice_criteria["new_label"] = (
|
||||
"The candidate is useful and distinct from all listed existing labels; "
|
||||
"retain the candidate wording as a new project label."
|
||||
)
|
||||
resolution_question_map[match_key] = (proposal, tuple(option_map))
|
||||
facet_questions[match_key] = {
|
||||
"type": "choice",
|
||||
"instructions": (
|
||||
f"For candidate {proposal.facet} label {proposal.value!r} (meaning: "
|
||||
f"{proposal.definition}), which one option best captures its meaning? "
|
||||
"Choose an existing label only when it means essentially the same thing; "
|
||||
"related but distinct labels are not synonyms."
|
||||
),
|
||||
"criteria": choice_criteria,
|
||||
}
|
||||
facet_state.append(
|
||||
{
|
||||
"facet": proposal.facet,
|
||||
"candidate": proposal.value,
|
||||
"definition": proposal.definition,
|
||||
"existing_options": [
|
||||
{
|
||||
"option": option_key,
|
||||
"label": label.canonical_label,
|
||||
"definition": label.definition,
|
||||
}
|
||||
for option_key, label in option_map
|
||||
],
|
||||
}
|
||||
)
|
||||
state = {
|
||||
"project_title": project.title,
|
||||
"project_north_star": project.north_star,
|
||||
@@ -83,9 +132,7 @@ class OpenRouterJev:
|
||||
"candidate_summary": item.summary,
|
||||
"candidate_relevance_rationale": item.relevance_rationale,
|
||||
"source_excerpt": item.excerpt,
|
||||
"candidate_facets": {
|
||||
facet: candidates for facet, candidates in facet_candidates if candidates
|
||||
},
|
||||
"candidate_facets": facet_state,
|
||||
}
|
||||
criteria = {
|
||||
"technical": "Engineering, implementation, technology, or operational practice.",
|
||||
@@ -151,21 +198,95 @@ class OpenRouterJev:
|
||||
served_model = str(result["model"])
|
||||
facet_scores = tuple(
|
||||
JevFacetScore(
|
||||
facet=facet,
|
||||
value=value,
|
||||
facet=proposal.facet,
|
||||
value=proposal.value,
|
||||
probability=float(answers[key]["noul"]),
|
||||
)
|
||||
for key, (facet, value) in facet_question_map.items()
|
||||
for key, proposal in fit_question_map.items()
|
||||
)
|
||||
facet_resolutions_list: list[JevFacetResolution] = []
|
||||
for key, (proposal, option_pairs) in resolution_question_map.items():
|
||||
answer = answers[key]
|
||||
selected = answer["choice"]
|
||||
confidence_value = float(answer["confidence"])
|
||||
probabilities = answer["probabilities"]
|
||||
if not isinstance(probabilities, dict):
|
||||
raise TypeError("choice probabilities are missing")
|
||||
new_probability = float(probabilities["new_label"])
|
||||
option_ids = {option_key for option_key, _ in option_pairs} | {"new_label"}
|
||||
if selected not in option_ids:
|
||||
raise ValueError("Jev selected an unknown facet label option")
|
||||
if any(not 0 <= float(probabilities[option_id]) <= 1 for option_id in option_ids):
|
||||
raise ValueError("Jev returned an invalid facet choice probability")
|
||||
selected_label = next(
|
||||
(label for option_key, label in option_pairs if option_key == selected),
|
||||
None,
|
||||
)
|
||||
option_probabilities = tuple(
|
||||
JevFacetOptionScore(
|
||||
label_id=(
|
||||
next(
|
||||
(
|
||||
label.label_id
|
||||
for option_key, label in option_pairs
|
||||
if option_key == option_id
|
||||
),
|
||||
None,
|
||||
)
|
||||
),
|
||||
label=(
|
||||
next(
|
||||
(
|
||||
label.canonical_label
|
||||
for option_key, label in option_pairs
|
||||
if option_key == option_id
|
||||
),
|
||||
proposal.value,
|
||||
)
|
||||
),
|
||||
probability=float(probabilities[option_id]),
|
||||
)
|
||||
for option_id in sorted(option_ids)
|
||||
)
|
||||
facet_resolutions_list.append(
|
||||
JevFacetResolution(
|
||||
facet=proposal.facet,
|
||||
value=proposal.value,
|
||||
fit_probability=next(
|
||||
score.probability
|
||||
for score in facet_scores
|
||||
if score.facet == proposal.facet and score.value == proposal.value
|
||||
),
|
||||
selected_label_id=(selected_label.label_id if selected_label else None),
|
||||
selected_label=(selected_label.canonical_label if selected_label else None),
|
||||
selected_new_label=selected == "new_label",
|
||||
new_probability=new_probability,
|
||||
choice_confidence=confidence_value,
|
||||
option_probabilities=option_probabilities,
|
||||
rank=1,
|
||||
)
|
||||
)
|
||||
facet_resolutions = tuple(facet_resolutions_list)
|
||||
except (KeyError, TypeError, ValueError, json.JSONDecodeError) as error:
|
||||
raise JevUnavailable("Jev response did not match its typed decision schema") from error
|
||||
if (
|
||||
not 0 <= relation <= 1
|
||||
or not 0 <= confidence <= 1
|
||||
or any(
|
||||
not 0 <= decision.fit_probability <= 1
|
||||
or not 0 <= decision.new_probability <= 1
|
||||
or not 0 <= decision.choice_confidence <= 1
|
||||
for decision in facet_resolutions
|
||||
)
|
||||
or lens not in criteria
|
||||
or len(served_model) > 100
|
||||
):
|
||||
raise JevUnavailable("Jev response contained an invalid decision")
|
||||
return JevAssessment(
|
||||
relation, cast(ResearchLens, lens), confidence, served_model, facet_scores
|
||||
relation,
|
||||
cast(ResearchLens, lens),
|
||||
confidence,
|
||||
served_model,
|
||||
facet_scores,
|
||||
facet_resolutions,
|
||||
)
|
||||
@@ -0,0 +1,176 @@
|
||||
"""SQLite cache of project facet labels and their local semantic vectors."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import sqlite3
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from gewu.application.paths import reject_symlink_components
|
||||
from gewu.application.ports import EmbeddingProvider
|
||||
from gewu.domain.contracts import FacetName, ProjectFacetLabel
|
||||
|
||||
MAX_LABELS_PER_PROJECT = 2000
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class FacetLabelMatch:
|
||||
label_id: str
|
||||
canonical_label: str
|
||||
definition: str
|
||||
similarity: float
|
||||
|
||||
|
||||
class SQLiteFacetLabelIndex:
|
||||
"""Cache rebuildable project label vectors, keyed by label and model profile."""
|
||||
|
||||
def __init__(self, database: Path) -> None:
|
||||
self.database = database.expanduser().absolute()
|
||||
reject_symlink_components(self.database)
|
||||
self.database.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
self.database.parent.chmod(0o700)
|
||||
if self.database.is_symlink():
|
||||
raise RuntimeError("Facet label index must not be a symbolic link")
|
||||
with self._connect() as connection:
|
||||
connection.execute(
|
||||
"""CREATE TABLE IF NOT EXISTS facet_label_embeddings (
|
||||
project_id TEXT NOT NULL,
|
||||
facet TEXT NOT NULL CHECK(facet IN ('tags', 'audience', 'use_for', 'domain')),
|
||||
label_id TEXT NOT NULL,
|
||||
canonical_label TEXT NOT NULL,
|
||||
definition TEXT NOT NULL,
|
||||
aliases_json TEXT NOT NULL,
|
||||
label_digest TEXT NOT NULL,
|
||||
model_id TEXT NOT NULL,
|
||||
artifact_sha256 TEXT NOT NULL,
|
||||
dimensions INTEGER NOT NULL,
|
||||
embedding_json TEXT NOT NULL,
|
||||
PRIMARY KEY(project_id, facet, label_id, model_id, artifact_sha256)
|
||||
)"""
|
||||
)
|
||||
connection.execute(
|
||||
"CREATE INDEX IF NOT EXISTS facet_label_lookup "
|
||||
"ON facet_label_embeddings(project_id, facet, model_id, artifact_sha256)"
|
||||
)
|
||||
self.database.chmod(0o600)
|
||||
|
||||
def closest(
|
||||
self,
|
||||
*,
|
||||
project_id: str,
|
||||
facet: FacetName,
|
||||
query: str,
|
||||
labels: tuple[ProjectFacetLabel, ...],
|
||||
embeddings: EmbeddingProvider,
|
||||
limit: int = 5,
|
||||
) -> tuple[FacetLabelMatch, ...]:
|
||||
if not 1 <= limit <= 10 or len(labels) > MAX_LABELS_PER_PROJECT:
|
||||
raise ValueError("Facet label search exceeds its supported bounds")
|
||||
active = tuple(
|
||||
label for label in labels if label.facet == facet and label.status == "active"
|
||||
)
|
||||
if not active:
|
||||
return ()
|
||||
profile = (embeddings.model_id, embeddings.artifact_sha256, embeddings.dimensions)
|
||||
cached: dict[str, tuple[float, ...]] = {}
|
||||
pending: list[tuple[ProjectFacetLabel, str, str]] = []
|
||||
with self._connect() as connection:
|
||||
for label in active:
|
||||
digest = _label_digest(label)
|
||||
row = connection.execute(
|
||||
"""SELECT embedding_json, dimensions FROM facet_label_embeddings
|
||||
WHERE project_id=? AND facet=? AND label_id=? AND model_id=?
|
||||
AND artifact_sha256=? AND label_digest=?""",
|
||||
(project_id, facet, label.label_id, *profile[:2], digest),
|
||||
).fetchone()
|
||||
if row is not None and row[1] == profile[2]:
|
||||
try:
|
||||
vector = tuple(float(value) for value in json.loads(row[0]))
|
||||
except (TypeError, ValueError, json.JSONDecodeError):
|
||||
vector = ()
|
||||
if _valid_vector(vector, profile[2]):
|
||||
cached[label.label_id] = vector
|
||||
continue
|
||||
pending.append((label, digest, _label_text(label)))
|
||||
if len(pending) > 0:
|
||||
vectors = embeddings.embed_passages(tuple(item[2] for item in pending))
|
||||
if len(vectors) != len(pending) or any(
|
||||
not _valid_vector(vector, profile[2]) for vector in vectors
|
||||
):
|
||||
raise ValueError("Embedding provider returned invalid facet label vectors")
|
||||
for (label, digest, _), vector in zip(pending, vectors, strict=True):
|
||||
cached[label.label_id] = vector
|
||||
connection.execute(
|
||||
"""INSERT INTO facet_label_embeddings (
|
||||
project_id, facet, label_id, canonical_label, definition,
|
||||
aliases_json, label_digest, model_id, artifact_sha256,
|
||||
dimensions, embedding_json
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(project_id, facet, label_id, model_id, artifact_sha256)
|
||||
DO UPDATE SET canonical_label=excluded.canonical_label,
|
||||
definition=excluded.definition, aliases_json=excluded.aliases_json,
|
||||
label_digest=excluded.label_digest, dimensions=excluded.dimensions,
|
||||
embedding_json=excluded.embedding_json""",
|
||||
(
|
||||
project_id,
|
||||
facet,
|
||||
label.label_id,
|
||||
label.canonical_label,
|
||||
label.definition,
|
||||
json.dumps(label.aliases, ensure_ascii=False, separators=(",", ":")),
|
||||
digest,
|
||||
*profile,
|
||||
json.dumps(vector, separators=(",", ":")),
|
||||
),
|
||||
)
|
||||
query_vector = embeddings.embed_queries(("Label meaning: " + query,))[0]
|
||||
if not _valid_vector(query_vector, profile[2]):
|
||||
raise ValueError("Embedding provider returned an invalid facet query vector")
|
||||
by_id = {label.label_id: label for label in active}
|
||||
scores = (
|
||||
FacetLabelMatch(
|
||||
label_id=label_id,
|
||||
canonical_label=by_id[label_id].canonical_label,
|
||||
definition=by_id[label_id].definition,
|
||||
similarity=_dot(query_vector, vector),
|
||||
)
|
||||
for label_id, vector in cached.items()
|
||||
)
|
||||
return tuple(sorted(scores, key=lambda match: (-match.similarity, match.label_id))[:limit])
|
||||
|
||||
def _connect(self) -> sqlite3.Connection:
|
||||
connection = sqlite3.connect(self.database, timeout=5, isolation_level="IMMEDIATE")
|
||||
connection.execute("PRAGMA foreign_keys=ON")
|
||||
connection.execute("PRAGMA busy_timeout=5000")
|
||||
return connection
|
||||
|
||||
|
||||
def _label_text(label: ProjectFacetLabel) -> str:
|
||||
aliases = ", ".join(label.aliases)
|
||||
return (
|
||||
f"Facet: {label.facet}. Label: {label.canonical_label}. "
|
||||
f"Meaning: {label.definition}. Aliases: {aliases}"
|
||||
)
|
||||
|
||||
|
||||
def _label_digest(label: ProjectFacetLabel) -> str:
|
||||
encoded = json.dumps(
|
||||
[label.facet, label.canonical_label, label.definition, label.aliases],
|
||||
ensure_ascii=False,
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
return hashlib.sha256(encoded).hexdigest()
|
||||
|
||||
|
||||
def _valid_vector(vector: tuple[float, ...], dimensions: int) -> bool:
|
||||
if len(vector) != dimensions or not all(math.isfinite(value) for value in vector):
|
||||
return False
|
||||
norm = math.sqrt(sum(value * value for value in vector))
|
||||
return math.isclose(norm, 1.0, rel_tol=1e-4, abs_tol=1e-4)
|
||||
|
||||
|
||||
def _dot(left: tuple[float, ...], right: tuple[float, ...]) -> float:
|
||||
return sum(a * b for a, b in zip(left, right, strict=True))
|
||||
@@ -14,6 +14,7 @@ from gewu.domain.contracts import (
|
||||
IngestReviewCreate,
|
||||
IngestReviewDecision,
|
||||
IngestReviewItem,
|
||||
JevFacetResolution,
|
||||
JevFacetScore,
|
||||
)
|
||||
|
||||
@@ -54,6 +55,7 @@ class SQLiteIngestReviewStore:
|
||||
jev_lens_confidence REAL,
|
||||
jev_model TEXT,
|
||||
jev_facet_scores_json TEXT NOT NULL DEFAULT '[]',
|
||||
jev_facet_resolutions_json TEXT NOT NULL DEFAULT '[]',
|
||||
status TEXT NOT NULL CHECK(status IN (
|
||||
'needs-user-review', 'included', 'excluded', 'deferred'
|
||||
)),
|
||||
@@ -71,6 +73,7 @@ class SQLiteIngestReviewStore:
|
||||
("jev_lens_confidence", "REAL"),
|
||||
("jev_model", "TEXT"),
|
||||
("jev_facet_scores_json", "TEXT NOT NULL DEFAULT '[]'"),
|
||||
("jev_facet_resolutions_json", "TEXT NOT NULL DEFAULT '[]'"),
|
||||
):
|
||||
if column not in existing_columns:
|
||||
connection.execute(
|
||||
@@ -122,6 +125,7 @@ class SQLiteIngestReviewStore:
|
||||
jev_lens_confidence: float | None = None,
|
||||
jev_model: str | None = None,
|
||||
jev_facet_scores: tuple[JevFacetScore, ...] = (),
|
||||
jev_facet_resolutions: tuple[JevFacetResolution, ...] = (),
|
||||
) -> IngestReviewItem:
|
||||
payload = item.model_dump(mode="json")
|
||||
digest = _digest({**payload, "submitted_by": submitted_by})
|
||||
@@ -144,9 +148,9 @@ class SQLiteIngestReviewStore:
|
||||
locator_json, excerpt, lens, summary, relevance_rationale, relevance,
|
||||
status, submitted_by, created_at, payload_digest,
|
||||
jev_relation_probability, jev_lens, jev_lens_confidence, jev_model,
|
||||
jev_facet_scores_json
|
||||
jev_facet_scores_json, jev_facet_resolutions_json
|
||||
) VALUES (
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'needs-user-review', ?, ?, ?, ?, ?, ?, ?, ?
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'needs-user-review', ?, ?, ?, ?, ?, ?, ?, ?, ?
|
||||
)""",
|
||||
(
|
||||
item.review_id,
|
||||
@@ -172,6 +176,11 @@ class SQLiteIngestReviewStore:
|
||||
ensure_ascii=False,
|
||||
separators=(",", ":"),
|
||||
),
|
||||
json.dumps(
|
||||
[item.model_dump(mode="json") for item in jev_facet_resolutions],
|
||||
ensure_ascii=False,
|
||||
separators=(",", ":"),
|
||||
),
|
||||
),
|
||||
)
|
||||
return self._read_item(connection, item.review_id)
|
||||
@@ -284,7 +293,8 @@ class SQLiteIngestReviewStore:
|
||||
"""SELECT review_id, project_id, run_id, source_revision_id, extraction_id,
|
||||
locator_json, excerpt, lens, summary, relevance_rationale, relevance, status,
|
||||
submitted_by, created_at, jev_relation_probability, jev_lens,
|
||||
jev_lens_confidence, jev_model, jev_facet_scores_json
|
||||
jev_lens_confidence, jev_model, jev_facet_scores_json,
|
||||
jev_facet_resolutions_json
|
||||
FROM ingest_review_items WHERE review_id=?""",
|
||||
(review_id,),
|
||||
).fetchone()
|
||||
@@ -310,7 +320,18 @@ class SQLiteIngestReviewStore:
|
||||
"jev_lens": row[15],
|
||||
"jev_lens_confidence": row[16],
|
||||
"jev_model": row[17],
|
||||
"jev_facet_scores": json.loads(row[18]),
|
||||
"jev_facet_scores": tuple(
|
||||
JevFacetScore.model_validate_json(
|
||||
json.dumps(score, ensure_ascii=False), strict=True
|
||||
)
|
||||
for score in json.loads(row[18])
|
||||
),
|
||||
"jev_facet_resolutions": tuple(
|
||||
JevFacetResolution.model_validate_json(
|
||||
json.dumps(resolution, ensure_ascii=False), strict=True
|
||||
)
|
||||
for resolution in json.loads(row[19])
|
||||
),
|
||||
},
|
||||
strict=True,
|
||||
)
|
||||
|
||||
@@ -9,7 +9,7 @@ from dataclasses import dataclass
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Protocol
|
||||
|
||||
from gewu.domain.contracts import AutoAcceptPolicy, ProjectRecord
|
||||
from gewu.domain.contracts import AutoAcceptPolicy, ProjectFacetLabel, ProjectRecord
|
||||
|
||||
|
||||
class CanonicalRepositoryError(RuntimeError):
|
||||
@@ -74,10 +74,16 @@ class ProjectRegistry(Protocol):
|
||||
|
||||
def create(self, project: ProjectRecord, policy: AutoAcceptPolicy) -> ProjectRecord: ...
|
||||
|
||||
def update_content(self, project_id: str, *, description: str, north_star: str, key_questions: tuple[str, ...]) -> ProjectRecord: ...
|
||||
def update_content(
|
||||
self, project_id: str, *, description: str, north_star: str, key_questions: tuple[str, ...]
|
||||
) -> ProjectRecord: ...
|
||||
|
||||
def archive(self, project_id: str) -> ProjectRecord: ...
|
||||
|
||||
def add_facet_labels(
|
||||
self, project_id: str, labels: tuple[ProjectFacetLabel, ...]
|
||||
) -> ProjectRecord: ...
|
||||
|
||||
|
||||
class EmbeddingProvider(Protocol):
|
||||
"""Replaceable, normalized text-vector provider for derived retrieval indexes."""
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import unicodedata
|
||||
from contextlib import AbstractContextManager
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
@@ -20,6 +21,7 @@ from gewu.domain.contracts import (
|
||||
AutoAcceptPolicy,
|
||||
ExtractionRecord,
|
||||
KnowledgeNode,
|
||||
ProjectFacetLabel,
|
||||
ProjectRecord,
|
||||
ProjectStatus,
|
||||
SourceRevisionRecord,
|
||||
@@ -307,15 +309,23 @@ class LocalProjectRegistry:
|
||||
self, project_id: str, *, description: str, north_star: str, key_questions: tuple[str, ...]
|
||||
) -> ProjectRecord:
|
||||
"""Update a project's human-authored description and research questions."""
|
||||
if self.repository is None or self.index_directory is None or self.authority_lock_path is None:
|
||||
if (
|
||||
self.repository is None
|
||||
or self.index_directory is None
|
||||
or self.authority_lock_path is None
|
||||
):
|
||||
raise RegistryError("Canonical project publication is not configured")
|
||||
with self._write_lock():
|
||||
project = _index(self._projects(), "project_id").get(project_id)
|
||||
if project is None:
|
||||
raise RegistryNotFound("Project is not registered")
|
||||
updated = project.model_copy(update={
|
||||
"description": description, "north_star": north_star, "key_questions": key_questions,
|
||||
})
|
||||
updated = project.model_copy(
|
||||
update={
|
||||
"description": description,
|
||||
"north_star": north_star,
|
||||
"key_questions": key_questions,
|
||||
}
|
||||
)
|
||||
try:
|
||||
self.repository.publish_files(
|
||||
expected_head=self.repository.head(),
|
||||
@@ -324,7 +334,9 @@ class LocalProjectRegistry:
|
||||
index_directory=self.index_directory,
|
||||
)
|
||||
except CanonicalConflictError as error:
|
||||
raise RegistryConflict("Canonical project registry changed during update") from error
|
||||
raise RegistryConflict(
|
||||
"Canonical project registry changed during update"
|
||||
) from error
|
||||
return updated
|
||||
|
||||
def archive(self, project_id: str) -> ProjectRecord:
|
||||
@@ -359,6 +371,72 @@ class LocalProjectRegistry:
|
||||
) from error
|
||||
return archived
|
||||
|
||||
def add_facet_labels(
|
||||
self, project_id: str, labels: tuple[ProjectFacetLabel, ...]
|
||||
) -> ProjectRecord:
|
||||
"""Atomically add bounded Jev-qualified labels to the canonical project record."""
|
||||
|
||||
if not labels or len(labels) > 20:
|
||||
raise ValueError("Facet label additions must contain between 1 and 20 labels")
|
||||
if (
|
||||
self.repository is None
|
||||
or self.index_directory is None
|
||||
or self.authority_lock_path is None
|
||||
):
|
||||
raise RegistryError("Canonical project publication is not configured")
|
||||
with self._write_lock():
|
||||
project = _index(self._projects(), "project_id").get(project_id)
|
||||
if project is None:
|
||||
raise RegistryNotFound("Project is not registered")
|
||||
existing = {
|
||||
(label.facet, _normalize_facet_label(label.canonical_label))
|
||||
for label in project.facet_labels
|
||||
if label.status == "active"
|
||||
}
|
||||
existing.update(
|
||||
(label.facet, _normalize_facet_label(alias))
|
||||
for label in project.facet_labels
|
||||
if label.status == "active"
|
||||
for alias in label.aliases
|
||||
)
|
||||
accepted: list[ProjectFacetLabel] = []
|
||||
known_ids = {label.label_id for label in project.facet_labels}
|
||||
for label in labels:
|
||||
if label.status != "active" or label.merged_into is not None:
|
||||
raise ValueError("Only active new facet labels may be added")
|
||||
if label.label_id in known_ids:
|
||||
raise RegistryConflict("Facet label ID is already registered")
|
||||
normalized_names = (
|
||||
label.canonical_label,
|
||||
*label.aliases,
|
||||
)
|
||||
if any(
|
||||
(label.facet, _normalize_facet_label(name)) in existing
|
||||
for name in normalized_names
|
||||
):
|
||||
raise RegistryConflict("Facet label duplicates an existing project label")
|
||||
for name in normalized_names:
|
||||
existing.add((label.facet, _normalize_facet_label(name)))
|
||||
known_ids.add(label.label_id)
|
||||
accepted.append(label)
|
||||
if len(project.facet_labels) + len(accepted) > 2_000:
|
||||
raise RegistryError("Project facet label catalog exceeds its configured limit")
|
||||
updated = project.model_copy(
|
||||
update={"facet_labels": (*project.facet_labels, *accepted)}
|
||||
)
|
||||
try:
|
||||
self.repository.publish_files(
|
||||
expected_head=self.repository.head(),
|
||||
files={_project_record_path(project_id): _project_markdown(updated)},
|
||||
message=f"Add qualified facet labels to {project_id}",
|
||||
index_directory=self.index_directory,
|
||||
)
|
||||
except CanonicalConflictError as error:
|
||||
raise RegistryConflict(
|
||||
"Canonical project registry changed while adding facet labels"
|
||||
) from error
|
||||
return updated
|
||||
|
||||
def _write_lock(self) -> AbstractContextManager[Any]:
|
||||
lock_path = self.authority_lock_path
|
||||
if lock_path is None:
|
||||
@@ -381,6 +459,10 @@ def _project_record_path(project_id: str) -> str:
|
||||
return f"projects/{project_id}/project.md"
|
||||
|
||||
|
||||
def _normalize_facet_label(value: str) -> str:
|
||||
return " ".join(unicodedata.normalize("NFC", value).casefold().split())
|
||||
|
||||
|
||||
def _policy_record_path(policy_id: str) -> str:
|
||||
import hashlib
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@ from gewu.adapters.embeddings import LocalSentenceTransformer
|
||||
from gewu.adapters.git_repository import GitCanonicalRepository
|
||||
from gewu.adapters.openrouter_jev import OpenRouterJev
|
||||
from gewu.adapters.source_extractor import LocalSourceExtractor, SourceExtractor
|
||||
from gewu.adapters.sqlite_facet_labels import SQLiteFacetLabelIndex
|
||||
from gewu.adapters.sqlite_identity import SQLiteIdentityStore
|
||||
from gewu.adapters.sqlite_ingest_reviews import SQLiteIngestReviewStore
|
||||
from gewu.adapters.sqlite_search import SQLiteSearchIndex
|
||||
@@ -28,7 +29,7 @@ from gewu.application.context import ValidationContext
|
||||
from gewu.application.diagnostics import build_service_diagnostics
|
||||
from gewu.application.identity import IdentityProvider, OwnerIdentityProvider, Principal
|
||||
from gewu.application.paths import reject_symlink_components
|
||||
from gewu.application.ports import ProjectRegistry
|
||||
from gewu.application.ports import EmbeddingProvider, ProjectRegistry
|
||||
from gewu.application.reading import CanonicalNodeReader, CanonicalSourceReader, ReadError
|
||||
from gewu.application.registry import (
|
||||
LocalContextProvider,
|
||||
@@ -84,6 +85,8 @@ def create_app(
|
||||
account_identity: AccountIdentity | None = None,
|
||||
ingest_review_store: SQLiteIngestReviewStore | None = None,
|
||||
jev_assessor: OpenRouterJev | None = None,
|
||||
facet_label_index: SQLiteFacetLabelIndex | None = None,
|
||||
embedding_provider: EmbeddingProvider | None = None,
|
||||
url_fetcher: PublicUrlFetcher | None = None,
|
||||
) -> FastAPI:
|
||||
if (identity_provider is None) != (node_reader is None):
|
||||
@@ -796,6 +799,9 @@ def create_app(
|
||||
url_fetcher=url_fetcher or PublicUrlFetcher(frozenset()),
|
||||
ingest_review_store=ingest_review_store,
|
||||
jev_assessor=jev_assessor,
|
||||
project_registry=project_registry,
|
||||
facet_label_index=facet_label_index,
|
||||
embedding_provider=embedding_provider,
|
||||
)
|
||||
mcp_asgi = mcp_server.streamable_http_app(
|
||||
streamable_http_path="/",
|
||||
@@ -901,6 +907,8 @@ def create_local_app() -> FastAPI:
|
||||
)
|
||||
search_path = data_root / "search.sqlite3"
|
||||
search_index: SearchIndex
|
||||
embedding_provider: LocalSentenceTransformer | None = None
|
||||
facet_label_index: SQLiteFacetLabelIndex | None = None
|
||||
model_path = os.environ.get("GEWU_EMBEDDING_MODEL_PATH")
|
||||
model_digest = os.environ.get("GEWU_EMBEDDING_MODEL_SHA256")
|
||||
if model_path or model_digest:
|
||||
@@ -918,6 +926,8 @@ def create_local_app() -> FastAPI:
|
||||
expected_sha256=model_digest,
|
||||
dimensions=int(os.environ.get("GEWU_EMBEDDING_DIMENSIONS", "384")),
|
||||
)
|
||||
embedding_provider = embeddings
|
||||
facet_label_index = SQLiteFacetLabelIndex(search_path)
|
||||
vectors = SQLiteVectorIndex(search_path)
|
||||
lexical_index = SQLiteSearchIndex(search_path, repository)
|
||||
hybrid_index = HybridSearchIndex(
|
||||
@@ -975,18 +985,34 @@ def create_local_app() -> FastAPI:
|
||||
context_provider=context_provider,
|
||||
search_service=search_service,
|
||||
project_registry=project_registry,
|
||||
facet_label_index=facet_label_index,
|
||||
embedding_provider=embedding_provider,
|
||||
url_fetcher=url_fetcher,
|
||||
)
|
||||
|
||||
|
||||
def _load_local_jev_environment() -> None:
|
||||
"""Load only Jev settings from the ignored, local project .env file."""
|
||||
"""Load only Jev, local embedding and label policy settings from .env."""
|
||||
|
||||
path = Path.cwd() / ".env"
|
||||
if path.is_symlink() or not path.is_file():
|
||||
return
|
||||
path.chmod(0o600)
|
||||
allowed = {"OPENROUTER_API_KEY", "OPENROUTER_JEV_MODEL"}
|
||||
allowed = {
|
||||
"OPENROUTER_API_KEY",
|
||||
"OPENROUTER_JEV_MODEL",
|
||||
"GEWU_EMBEDDING_MODEL_PATH",
|
||||
"GEWU_EMBEDDING_MODEL_SHA256",
|
||||
"GEWU_EMBEDDING_MODEL_ID",
|
||||
"GEWU_EMBEDDING_DIMENSIONS",
|
||||
"GEWU_FACET_LABEL_TOP_N_TAGS",
|
||||
"GEWU_FACET_LABEL_TOP_N_AUDIENCE",
|
||||
"GEWU_FACET_LABEL_TOP_N_USE_FOR",
|
||||
"GEWU_FACET_LABEL_TOP_N_DOMAIN",
|
||||
"GEWU_FACET_LABEL_FIT_THRESHOLD",
|
||||
"GEWU_FACET_LABEL_NEW_THRESHOLD",
|
||||
"GEWU_FACET_LABEL_CONFIDENCE_THRESHOLD",
|
||||
}
|
||||
for raw_line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
|
||||
@@ -26,6 +26,7 @@ ResearchLens = Literal[
|
||||
"financial-market",
|
||||
]
|
||||
FacetName = Literal["tags", "audience", "use_for", "domain"]
|
||||
FacetLabelStatus = Literal["active", "merged", "retired"]
|
||||
DEFAULT_RESEARCH_LENSES: tuple[ResearchLens, ...] = (
|
||||
"technical",
|
||||
"conceptual",
|
||||
@@ -49,6 +50,17 @@ Sha256 = Annotated[str, StringConstraints(pattern=r"^[a-f0-9]{64}$")]
|
||||
GitCommitId = Annotated[str, StringConstraints(pattern=r"^(?:[a-f0-9]{40}|[a-f0-9]{64})$")]
|
||||
|
||||
|
||||
def _normalized_facet_label(value: str) -> str:
|
||||
return " ".join(unicodedata.normalize("NFC", value).casefold().split())
|
||||
|
||||
|
||||
def _clean_facet_label(value: str) -> str:
|
||||
cleaned = " ".join(unicodedata.normalize("NFC", value).split())
|
||||
if not cleaned:
|
||||
raise ValueError("Facet labels must contain non-whitespace text")
|
||||
return cleaned
|
||||
|
||||
|
||||
class StrictRecord(BaseModel):
|
||||
"""Base for public records: reject implicit coercion and unknown fields."""
|
||||
|
||||
@@ -63,6 +75,45 @@ class ProjectStatus(StrEnum):
|
||||
ARCHIVED = "archived"
|
||||
|
||||
|
||||
class ProjectFacetLabel(StrictRecord):
|
||||
label_id: RecordId
|
||||
facet: FacetName
|
||||
canonical_label: SearchFacetValue
|
||||
definition: str = Field(min_length=1, max_length=500)
|
||||
aliases: tuple[SearchFacetValue, ...] = Field(default=(), max_length=20)
|
||||
status: FacetLabelStatus = "active"
|
||||
merged_into: RecordId | None = None
|
||||
origin_review_id: RecordId | None = None
|
||||
jev_model: str | None = Field(default=None, max_length=100)
|
||||
jev_fit_probability: FiniteFloat | None = Field(default=None, ge=0, le=1)
|
||||
jev_new_probability: FiniteFloat | None = Field(default=None, ge=0, le=1)
|
||||
created_at: datetime
|
||||
|
||||
@field_validator("canonical_label")
|
||||
@classmethod
|
||||
def normalize_canonical_label(cls, value: str) -> str:
|
||||
return _clean_facet_label(value)
|
||||
|
||||
@field_validator("aliases")
|
||||
@classmethod
|
||||
def normalize_aliases(cls, values: tuple[str, ...]) -> tuple[str, ...]:
|
||||
return tuple(_clean_facet_label(value) for value in values)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_label_lifecycle(self) -> ProjectFacetLabel:
|
||||
if self.status == "merged" and self.merged_into is None:
|
||||
raise ValueError("Merged facet labels require a canonical target")
|
||||
if self.status != "merged" and self.merged_into is not None:
|
||||
raise ValueError("Only merged facet labels may have a merge target")
|
||||
names = (self.canonical_label, *self.aliases)
|
||||
if len({_normalized_facet_label(name) for name in names}) != len(names):
|
||||
raise ValueError("Facet label and aliases must be unique")
|
||||
offset = self.created_at.utcoffset()
|
||||
if offset is None or offset.total_seconds() != 0:
|
||||
raise ValueError("Facet label timestamp must be timezone-aware UTC")
|
||||
return self
|
||||
|
||||
|
||||
class ProjectRecord(StrictRecord):
|
||||
project_id: RecordId
|
||||
title: str = Field(min_length=1, max_length=200)
|
||||
@@ -75,6 +126,7 @@ class ProjectRecord(StrictRecord):
|
||||
watch_list: tuple[str, ...] = Field(default=(), max_length=100)
|
||||
default_includes: tuple[RecordId, ...] = ("shared",)
|
||||
taxonomy_extensions: dict[str, tuple[str, ...]] = Field(default_factory=dict, max_length=100)
|
||||
facet_labels: tuple[ProjectFacetLabel, ...] = Field(default=(), max_length=2000)
|
||||
autonomy_policy_id: RecordId
|
||||
|
||||
@field_validator("project_id")
|
||||
@@ -109,6 +161,7 @@ class ProjectLens(StrictRecord):
|
||||
research_lenses: tuple[ResearchLens, ...] = DEFAULT_RESEARCH_LENSES
|
||||
watch_list: tuple[str, ...] = Field(default=(), max_length=100)
|
||||
taxonomy_extensions: dict[str, tuple[str, ...]] = Field(default_factory=dict, max_length=100)
|
||||
facet_labels: tuple[ProjectFacetLabel, ...] = Field(default=(), max_length=2000)
|
||||
ingest_guidance: tuple[IngestGuidance, ...] = Field(default=(), max_length=100)
|
||||
|
||||
|
||||
@@ -122,6 +175,45 @@ class JevFacetScore(StrictRecord):
|
||||
probability: FiniteFloat = Field(ge=0, le=1)
|
||||
|
||||
|
||||
class JevFacetOptionScore(StrictRecord):
|
||||
label_id: RecordId | None = None
|
||||
label: SearchFacetValue
|
||||
probability: FiniteFloat = Field(ge=0, le=1)
|
||||
|
||||
|
||||
class JevFacetResolution(StrictRecord):
|
||||
facet: FacetName
|
||||
value: SearchFacetValue
|
||||
fit_probability: FiniteFloat = Field(ge=0, le=1)
|
||||
selected_label_id: RecordId | None = None
|
||||
selected_label: SearchFacetValue | None = None
|
||||
selected_new_label: bool = False
|
||||
new_probability: FiniteFloat = Field(ge=0, le=1)
|
||||
choice_confidence: FiniteFloat = Field(ge=0, le=1)
|
||||
option_probabilities: tuple[JevFacetOptionScore, ...] = Field(default=(), max_length=6)
|
||||
rank: int = Field(ge=1, le=20)
|
||||
added_to_project: bool = False
|
||||
|
||||
|
||||
class FacetLabelProposal(StrictRecord):
|
||||
facet: FacetName
|
||||
value: SearchFacetValue
|
||||
definition: str = Field(min_length=1, max_length=500)
|
||||
|
||||
@field_validator("value")
|
||||
@classmethod
|
||||
def normalize_value(cls, value: str) -> str:
|
||||
return _clean_facet_label(value)
|
||||
|
||||
@field_validator("definition")
|
||||
@classmethod
|
||||
def normalize_definition(cls, value: str) -> str:
|
||||
definition = value.strip()
|
||||
if not definition:
|
||||
raise ValueError("Facet label definitions must contain non-whitespace text")
|
||||
return definition
|
||||
|
||||
|
||||
class IngestReviewItem(StrictRecord):
|
||||
review_id: RecordId
|
||||
project_id: RecordId
|
||||
@@ -135,6 +227,7 @@ class IngestReviewItem(StrictRecord):
|
||||
relevance_rationale: str = Field(min_length=1, max_length=2000)
|
||||
relevance: Literal["borderline", "low"] = "borderline"
|
||||
jev_facet_scores: tuple[JevFacetScore, ...] = Field(default=(), max_length=20)
|
||||
jev_facet_resolutions: tuple[JevFacetResolution, ...] = Field(default=(), max_length=20)
|
||||
jev_relation_probability: FiniteFloat | None = Field(default=None, ge=0, le=1)
|
||||
jev_lens: ResearchLens | None = None
|
||||
jev_lens_confidence: FiniteFloat | None = Field(default=None, ge=0, le=1)
|
||||
@@ -165,9 +258,10 @@ class IngestReviewCreate(StrictRecord):
|
||||
relevance_rationale: str = Field(min_length=1, max_length=2000)
|
||||
relevance: Literal["borderline", "low"] = "borderline"
|
||||
candidate_tags: tuple[SearchFacetValue, ...] = Field(default=(), max_length=5)
|
||||
candidate_audience: tuple[SearchFacetValue, ...] = Field(default=(), max_length=5)
|
||||
candidate_use_for: tuple[SearchFacetValue, ...] = Field(default=(), max_length=5)
|
||||
candidate_domain: tuple[SearchFacetValue, ...] = Field(default=(), max_length=5)
|
||||
candidate_audience: tuple[SearchFacetValue, ...] = Field(default=(), max_length=3)
|
||||
candidate_use_for: tuple[SearchFacetValue, ...] = Field(default=(), max_length=3)
|
||||
candidate_domain: tuple[SearchFacetValue, ...] = Field(default=(), max_length=3)
|
||||
candidate_facet_labels: tuple[FacetLabelProposal, ...] = Field(default=(), max_length=14)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def limit_total_facet_candidates(self) -> IngestReviewCreate:
|
||||
@@ -177,17 +271,55 @@ class IngestReviewCreate(StrictRecord):
|
||||
*self.candidate_use_for,
|
||||
*self.candidate_domain,
|
||||
)
|
||||
if len(values) > 20:
|
||||
raise ValueError("At most 20 facet candidates may be assessed")
|
||||
if len({(facet, value.casefold()) for facet, candidates in (
|
||||
proposed = (*values, *(proposal.value for proposal in self.candidate_facet_labels))
|
||||
if len(proposed) > 14:
|
||||
raise ValueError("At most 14 facet candidates may be assessed")
|
||||
facet_limits = {"tags": 5, "audience": 3, "use_for": 3, "domain": 3}
|
||||
facet_counts = {facet: 0 for facet in facet_limits}
|
||||
for facet, candidates in (
|
||||
("tags", self.candidate_tags),
|
||||
("audience", self.candidate_audience),
|
||||
("use_for", self.candidate_use_for),
|
||||
("domain", self.candidate_domain),
|
||||
) for value in candidates}) != len(values):
|
||||
):
|
||||
facet_counts[facet] += len(candidates)
|
||||
for proposal in self.candidate_facet_labels:
|
||||
facet_counts[proposal.facet] += 1
|
||||
if any(facet_counts[facet] > limit for facet, limit in facet_limits.items()):
|
||||
raise ValueError("Facet candidates exceed the per-facet limits (tags: 5; others: 3)")
|
||||
keyed = {
|
||||
(facet, _normalized_facet_label(value))
|
||||
for facet, candidates in (
|
||||
("tags", self.candidate_tags),
|
||||
("audience", self.candidate_audience),
|
||||
("use_for", self.candidate_use_for),
|
||||
("domain", self.candidate_domain),
|
||||
)
|
||||
for value in candidates
|
||||
}
|
||||
keyed.update(
|
||||
(proposal.facet, _normalized_facet_label(proposal.value))
|
||||
for proposal in self.candidate_facet_labels
|
||||
)
|
||||
if len(keyed) != len(proposed):
|
||||
raise ValueError("Facet candidates must be unique within each facet")
|
||||
return self
|
||||
|
||||
def facet_proposals(self) -> tuple[FacetLabelProposal, ...]:
|
||||
proposals: list[FacetLabelProposal] = list(self.candidate_facet_labels)
|
||||
fields: tuple[tuple[FacetName, tuple[SearchFacetValue, ...]], ...] = (
|
||||
("tags", self.candidate_tags),
|
||||
("audience", self.candidate_audience),
|
||||
("use_for", self.candidate_use_for),
|
||||
("domain", self.candidate_domain),
|
||||
)
|
||||
for facet, values in fields:
|
||||
proposals.extend(
|
||||
FacetLabelProposal(facet=facet, value=value, definition=self.summary[:500])
|
||||
for value in values
|
||||
)
|
||||
return tuple(proposals)
|
||||
|
||||
|
||||
class IngestReviewDecision(StrictRecord):
|
||||
decision_id: RecordId
|
||||
|
||||
@@ -69,6 +69,7 @@ OPERATIONS: Final[tuple[Operation, ...]] = (
|
||||
Operation("source_extract", "proposal:submit", frozenset({Transport.MCP})),
|
||||
Operation("source_extraction_block", "proposal:submit", frozenset({Transport.MCP})),
|
||||
Operation("proposal_submit", "proposal:submit", frozenset({Transport.MCP}), True),
|
||||
Operation("proposal_prepare", "proposal:evaluate", frozenset({Transport.MCP})),
|
||||
Operation("proposal_evaluate", "proposal:evaluate", frozenset({Transport.MCP})),
|
||||
Operation("proposal_status", "proposal:read", frozenset({Transport.MCP})),
|
||||
Operation("proposal_withdraw", "proposal:withdraw", frozenset({Transport.MCP})),
|
||||
|
||||
@@ -6,7 +6,9 @@ import hashlib
|
||||
import json
|
||||
import unicodedata
|
||||
from collections.abc import Iterable
|
||||
from copy import deepcopy
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from gewu.domain.contracts import (
|
||||
AutoAcceptPolicy,
|
||||
@@ -59,6 +61,73 @@ def proposal_payload_digest(proposal: ProposalRevision) -> str:
|
||||
return hashlib.sha256(canonical_json(operations)).hexdigest()
|
||||
|
||||
|
||||
def prepare_proposal_digests(draft: dict[str, Any]) -> ProposalRevision:
|
||||
"""Validate a proposal draft, fill its dependent evidence digests, and pin its payload."""
|
||||
|
||||
value = deepcopy(draft)
|
||||
operations = value.get("operations")
|
||||
if not isinstance(operations, list) or not operations:
|
||||
raise ValueError("Proposal must contain a non-empty operations array")
|
||||
|
||||
for operation in operations:
|
||||
if not isinstance(operation, dict):
|
||||
raise ValueError("Proposal operations must be objects")
|
||||
node = operation.get("node")
|
||||
if isinstance(node, dict):
|
||||
for group in ("claims", "assumptions", "limitations"):
|
||||
claims = node.get(group, [])
|
||||
if not isinstance(claims, list):
|
||||
raise ValueError(f"Node {group} must be an array")
|
||||
for index, claim in enumerate(claims):
|
||||
if not isinstance(claim, dict) or not isinstance(claim.get("text"), str):
|
||||
raise ValueError(f"Node {group}[{index}] must contain claim text")
|
||||
digest = assertion_digest(claim["text"])
|
||||
evidence = claim.get("evidence")
|
||||
if not isinstance(evidence, list) or not evidence:
|
||||
raise ValueError(f"Node {group}[{index}] must contain evidence")
|
||||
path = f"{group}[{index}].text"
|
||||
evidence_digests = []
|
||||
for item in evidence:
|
||||
if not isinstance(item, dict):
|
||||
raise ValueError("Evidence references must be objects")
|
||||
item["assertion_path"] = path
|
||||
item["assertion_digest"] = digest
|
||||
parsed_evidence = EvidenceReference.model_validate(item, strict=True)
|
||||
item.clear()
|
||||
item.update(parsed_evidence.model_dump(mode="json"))
|
||||
evidence_digests.append(evidence_digest(parsed_evidence))
|
||||
assessment = claim.get("assessment")
|
||||
if not isinstance(assessment, dict):
|
||||
raise ValueError(f"Node {group}[{index}] must contain an assessment")
|
||||
assessment["assertion_digest"] = digest
|
||||
assessment["evidence_digests"] = evidence_digests
|
||||
|
||||
edge = operation.get("edge")
|
||||
if isinstance(edge, dict):
|
||||
edge_evidence = edge.get("evidence", [])
|
||||
if not isinstance(edge_evidence, list):
|
||||
raise ValueError("Edge evidence must be an array")
|
||||
for item in edge_evidence:
|
||||
if not isinstance(item, dict):
|
||||
raise ValueError("Evidence references must be objects")
|
||||
item["assertion_path"] = "edge"
|
||||
item["assertion_digest"] = "0" * 64
|
||||
parsed_edge = TypedEdge.model_validate(edge, strict=True)
|
||||
digest = edge_assertion_digest(parsed_edge)
|
||||
for item in edge_evidence:
|
||||
item["assertion_digest"] = digest
|
||||
parsed_evidence = EvidenceReference.model_validate(item, strict=True)
|
||||
item.clear()
|
||||
item.update(parsed_evidence.model_dump(mode="json"))
|
||||
edge["evidence"] = edge_evidence
|
||||
|
||||
value["payload_digest"] = "0" * 64
|
||||
provisional = ProposalRevision.model_validate_json(
|
||||
json.dumps(value, ensure_ascii=False, allow_nan=False), strict=True
|
||||
)
|
||||
return provisional.model_copy(update={"payload_digest": proposal_payload_digest(provisional)})
|
||||
|
||||
|
||||
def edge_assertion_digest(edge: TypedEdge) -> str:
|
||||
"""Bind edge evidence to the relationship, conditions and rationale."""
|
||||
|
||||
|
||||
@@ -4,9 +4,14 @@ from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import difflib
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
import unicodedata
|
||||
from collections.abc import Callable, Iterator
|
||||
from datetime import UTC, date, datetime
|
||||
from typing import Literal
|
||||
@@ -17,15 +22,17 @@ from mcp.server import MCPServer
|
||||
from mcp.server.mcpserver import Context
|
||||
from pydantic import ValidationError
|
||||
|
||||
from gewu.adapters.openrouter_jev import JevUnavailable, OpenRouterJev
|
||||
from gewu.adapters.openrouter_jev import JevAssessment, JevUnavailable, OpenRouterJev
|
||||
from gewu.adapters.pdf_extractor import PdfExtractionError
|
||||
from gewu.adapters.source_extractor import SourceExtractor
|
||||
from gewu.adapters.sqlite_facet_labels import SQLiteFacetLabelIndex
|
||||
from gewu.adapters.sqlite_ingest_reviews import SQLiteIngestReviewStore
|
||||
from gewu.adapters.url_fetch import PublicUrlFetcher, UrlCaptureError
|
||||
from gewu.application.acceptance import AcceptanceError, AcceptanceService, ActorContext
|
||||
from gewu.application.context import ValidationContext
|
||||
from gewu.application.diagnostics import build_service_diagnostics
|
||||
from gewu.application.identity import IdentityProvider, Principal
|
||||
from gewu.application.ports import EmbeddingProvider, ProjectRegistry
|
||||
from gewu.application.reading import CanonicalNodeReader, CanonicalSourceReader, ReadError
|
||||
from gewu.application.runs import LocalRunService, RunError
|
||||
from gewu.application.search import CanonicalSearchService, SearchError
|
||||
@@ -38,12 +45,17 @@ from gewu.domain.contracts import (
|
||||
Confidence,
|
||||
EvidenceReference,
|
||||
ExtractionRecord,
|
||||
FacetLabelProposal,
|
||||
FacetName,
|
||||
GitCommitId,
|
||||
IngestReviewCreate,
|
||||
JevFacetResolution,
|
||||
NeighborRequest,
|
||||
NodeType,
|
||||
ProjectContextResponse,
|
||||
ProjectFacetLabel,
|
||||
ProjectLens,
|
||||
ProjectRecord,
|
||||
ProposalRevision,
|
||||
ReliabilityGrade,
|
||||
RunCheckpointRequest,
|
||||
@@ -53,11 +65,288 @@ from gewu.domain.contracts import (
|
||||
SourceListRequest,
|
||||
SourceRevisionRecord,
|
||||
)
|
||||
from gewu.domain.validation import validate_proposal
|
||||
from gewu.domain.validation import prepare_proposal_digests, validate_proposal
|
||||
from gewu.version import SERVICE_VERSION
|
||||
|
||||
MAX_PASTED_TEXT_BYTES = 1_000_000
|
||||
MAX_PROJECT_CONTEXT_BYTES = 64_000
|
||||
MAX_PROJECT_FACET_LABELS = 2_000
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _normalize_facet_value(value: str) -> str:
|
||||
return " ".join(unicodedata.normalize("NFC", value).casefold().split())
|
||||
|
||||
|
||||
def _project_facet_catalog(
|
||||
project: ProjectRecord, context: ValidationContext
|
||||
) -> tuple[ProjectFacetLabel, ...]:
|
||||
project_id = project.project_id
|
||||
labels = [label for label in project.facet_labels if label.status == "active"]
|
||||
seen = {
|
||||
(label.facet, _normalize_facet_value(name))
|
||||
for label in labels
|
||||
for name in (label.canonical_label, *label.aliases)
|
||||
}
|
||||
collected: dict[tuple[FacetName, str], tuple[int, str]] = {}
|
||||
facet_fields: tuple[tuple[FacetName, str], ...] = (
|
||||
("tags", "tags"),
|
||||
("audience", "audience"),
|
||||
("use_for", "use_for"),
|
||||
("domain", "domain"),
|
||||
)
|
||||
for node in context.existing_nodes.values():
|
||||
if node.project_id != project_id:
|
||||
continue
|
||||
for facet, field in facet_fields:
|
||||
for value in getattr(node, field):
|
||||
normalized = _normalize_facet_value(value)
|
||||
if (facet, normalized) not in seen:
|
||||
count, display = collected.get((facet, normalized), (0, value.strip()))
|
||||
collected[(facet, normalized)] = (count + 1, display)
|
||||
epoch = datetime(1970, 1, 1, tzinfo=UTC)
|
||||
for (facet, normalized), (_count, display) in sorted(collected.items()):
|
||||
if len(labels) >= MAX_PROJECT_FACET_LABELS:
|
||||
break
|
||||
label_text = display
|
||||
if not label_text:
|
||||
continue
|
||||
digest = hashlib.sha256(f"{project_id}\0{facet}\0{normalized}".encode()).hexdigest()[:24]
|
||||
label_id = f"facet-observed-{digest}"
|
||||
labels.append(
|
||||
ProjectFacetLabel(
|
||||
label_id=label_id,
|
||||
facet=facet,
|
||||
canonical_label=label_text,
|
||||
definition="Observed on existing accepted knowledge records.",
|
||||
created_at=epoch,
|
||||
)
|
||||
)
|
||||
seen.add((facet, normalized))
|
||||
return tuple(labels[:MAX_PROJECT_FACET_LABELS])
|
||||
|
||||
|
||||
def _nearest_facet_labels(
|
||||
*,
|
||||
proposal: FacetLabelProposal,
|
||||
project_id: str,
|
||||
catalog: tuple[ProjectFacetLabel, ...],
|
||||
index: SQLiteFacetLabelIndex | None,
|
||||
embeddings: EmbeddingProvider | None,
|
||||
) -> tuple[ProjectFacetLabel, ...]:
|
||||
facet = proposal.facet
|
||||
value = proposal.value
|
||||
query = f"{value}. {proposal.definition}"
|
||||
relevant = tuple(label for label in catalog if label.facet == facet)
|
||||
normalized = _normalize_facet_value(value)
|
||||
exact = tuple(
|
||||
label
|
||||
for label in relevant
|
||||
if normalized
|
||||
in {
|
||||
_normalize_facet_value(label.canonical_label),
|
||||
*(_normalize_facet_value(alias) for alias in label.aliases),
|
||||
}
|
||||
)
|
||||
if exact:
|
||||
return exact[:5]
|
||||
ordered: dict[str, tuple[float, ProjectFacetLabel]] = {}
|
||||
if index is not None and embeddings is not None and relevant:
|
||||
matches = index.closest(
|
||||
project_id=project_id,
|
||||
facet=facet,
|
||||
query=query,
|
||||
labels=relevant,
|
||||
embeddings=embeddings,
|
||||
limit=5,
|
||||
)
|
||||
by_id = {label.label_id: label for label in relevant}
|
||||
for match in matches:
|
||||
label = by_id.get(match.label_id)
|
||||
if label is not None:
|
||||
ordered[label.label_id] = (match.similarity, label)
|
||||
query_terms = set(re.findall(r"[\w-]+", normalized))
|
||||
lexical: list[tuple[float, ProjectFacetLabel]] = []
|
||||
for label in relevant:
|
||||
label_terms = set(
|
||||
re.findall(
|
||||
r"[\w-]+",
|
||||
_normalize_facet_value(
|
||||
f"{label.canonical_label} {label.definition} {' '.join(label.aliases)}"
|
||||
),
|
||||
)
|
||||
)
|
||||
overlap = len(query_terms & label_terms) / max(len(query_terms | label_terms), 1)
|
||||
sequence = difflib.SequenceMatcher(
|
||||
None,
|
||||
normalized,
|
||||
_normalize_facet_value(label.canonical_label),
|
||||
autojunk=False,
|
||||
).ratio()
|
||||
lexical.append((max(overlap, sequence), label))
|
||||
for _rank, (score, label) in enumerate(
|
||||
sorted(lexical, key=lambda item: (-item[0], item[1].label_id))[:5], start=1
|
||||
):
|
||||
if score > 0:
|
||||
prior = ordered.get(label.label_id)
|
||||
ordered[label.label_id] = (score + (prior[0] if prior else 0.0), label)
|
||||
return tuple(
|
||||
label
|
||||
for _score, label in sorted(
|
||||
ordered.values(), key=lambda item: (-item[0], item[1].label_id)
|
||||
)[:5]
|
||||
)
|
||||
|
||||
|
||||
def _promotion_policy() -> tuple[dict[FacetName, int], float, float, float]:
|
||||
try:
|
||||
top_n: dict[FacetName, int] = {
|
||||
"tags": int(os.environ.get("GEWU_FACET_LABEL_TOP_N_TAGS", "5")),
|
||||
"audience": int(os.environ.get("GEWU_FACET_LABEL_TOP_N_AUDIENCE", "3")),
|
||||
"use_for": int(os.environ.get("GEWU_FACET_LABEL_TOP_N_USE_FOR", "3")),
|
||||
"domain": int(os.environ.get("GEWU_FACET_LABEL_TOP_N_DOMAIN", "3")),
|
||||
}
|
||||
fit_threshold = float(os.environ.get("GEWU_FACET_LABEL_FIT_THRESHOLD", "0.90"))
|
||||
new_threshold = float(os.environ.get("GEWU_FACET_LABEL_NEW_THRESHOLD", "0.90"))
|
||||
confidence_threshold = float(
|
||||
os.environ.get("GEWU_FACET_LABEL_CONFIDENCE_THRESHOLD", "0.80")
|
||||
)
|
||||
except ValueError as error:
|
||||
raise RuntimeError("Facet label promotion settings are invalid") from error
|
||||
limits: dict[FacetName, int] = {"tags": 5, "audience": 3, "use_for": 3, "domain": 3}
|
||||
if any(not 1 <= top_n[facet] <= maximum for facet, maximum in limits.items()) or any(
|
||||
not 0 <= threshold <= 1
|
||||
for threshold in (fit_threshold, new_threshold, confidence_threshold)
|
||||
):
|
||||
raise RuntimeError("Facet label promotion settings are outside supported bounds")
|
||||
return top_n, fit_threshold, new_threshold, confidence_threshold
|
||||
|
||||
|
||||
def _promote_jev_labels(
|
||||
*,
|
||||
item: IngestReviewCreate,
|
||||
assessment: JevAssessment,
|
||||
project_id: str,
|
||||
project_registry: ProjectRegistry | None,
|
||||
facet_label_index: SQLiteFacetLabelIndex | None,
|
||||
embedding_provider: EmbeddingProvider | None,
|
||||
existing_catalog: tuple[ProjectFacetLabel, ...],
|
||||
semantic_search_ready: bool,
|
||||
) -> tuple[JevFacetResolution, ...]:
|
||||
resolutions = assessment.facet_resolutions
|
||||
if not resolutions:
|
||||
return ()
|
||||
top_n, fit_threshold, new_threshold, confidence_threshold = _promotion_policy()
|
||||
ranks: dict[tuple[FacetName, str], int] = {}
|
||||
for facet in ("tags", "audience", "use_for", "domain"):
|
||||
facet_resolutions = tuple(r for r in resolutions if r.facet == facet)
|
||||
new_candidates = tuple(r for r in facet_resolutions if r.selected_new_label)
|
||||
new_ranks = {
|
||||
(r.facet, r.value): rank
|
||||
for rank, r in enumerate(
|
||||
sorted(
|
||||
new_candidates,
|
||||
key=lambda r: (
|
||||
-(r.fit_probability * r.new_probability),
|
||||
r.value.casefold(),
|
||||
),
|
||||
),
|
||||
start=1,
|
||||
)
|
||||
}
|
||||
existing_rank_start = len(new_candidates) + 1
|
||||
existing_candidates = sorted(
|
||||
(r for r in facet_resolutions if not r.selected_new_label),
|
||||
key=lambda r: (-(r.fit_probability * r.new_probability), r.value.casefold()),
|
||||
)
|
||||
ranks.update(new_ranks)
|
||||
ranks.update(
|
||||
{
|
||||
(r.facet, r.value): rank
|
||||
for rank, r in enumerate(existing_candidates, start=existing_rank_start)
|
||||
}
|
||||
)
|
||||
ranked_resolutions = tuple(
|
||||
resolution.model_copy(update={"rank": ranks[(resolution.facet, resolution.value)]})
|
||||
for resolution in resolutions
|
||||
)
|
||||
if (
|
||||
project_registry is None
|
||||
or facet_label_index is None
|
||||
or embedding_provider is None
|
||||
or not semantic_search_ready
|
||||
):
|
||||
return ranked_resolutions
|
||||
proposals = {(proposal.facet, proposal.value): proposal for proposal in item.facet_proposals()}
|
||||
existing = {
|
||||
(label.facet, _normalize_facet_value(name))
|
||||
for label in existing_catalog
|
||||
if label.status == "active"
|
||||
for name in (label.canonical_label, *label.aliases)
|
||||
}
|
||||
eligible = tuple(
|
||||
resolution
|
||||
for resolution in ranked_resolutions
|
||||
if resolution.rank <= top_n[resolution.facet]
|
||||
and resolution.selected_new_label
|
||||
and resolution.fit_probability >= fit_threshold
|
||||
and resolution.new_probability >= new_threshold
|
||||
and resolution.choice_confidence >= confidence_threshold
|
||||
and (resolution.facet, _normalize_facet_value(resolution.value)) not in existing
|
||||
)
|
||||
if not eligible:
|
||||
return ranked_resolutions
|
||||
by_facet_rank = {
|
||||
(resolution.facet, resolution.value): resolution.rank for resolution in eligible
|
||||
}
|
||||
additions: list[ProjectFacetLabel] = []
|
||||
for resolution in eligible:
|
||||
proposal = proposals[(resolution.facet, resolution.value)]
|
||||
label_digest = hashlib.sha256(
|
||||
f"{project_id}\0{resolution.facet}\0{_normalize_facet_value(resolution.value)}".encode()
|
||||
).hexdigest()[:24]
|
||||
additions.append(
|
||||
ProjectFacetLabel(
|
||||
label_id=f"facet-{resolution.facet}-{label_digest}",
|
||||
facet=resolution.facet,
|
||||
canonical_label=resolution.value,
|
||||
definition=proposal.definition,
|
||||
origin_review_id=item.review_id,
|
||||
jev_model=assessment.model,
|
||||
jev_fit_probability=resolution.fit_probability,
|
||||
jev_new_probability=resolution.new_probability,
|
||||
created_at=datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
try:
|
||||
project_registry.add_facet_labels(project_id, tuple(additions))
|
||||
except (RuntimeError, ValueError) as error:
|
||||
# The review flag and its Jev judgment remain durable if catalog publication races.
|
||||
logger.warning("Could not publish Jev-qualified project facet labels: %s", error)
|
||||
return ranked_resolutions
|
||||
added_keys = {(label.facet, label.canonical_label) for label in additions}
|
||||
for label in additions:
|
||||
try:
|
||||
facet_label_index.closest(
|
||||
project_id=project_id,
|
||||
facet=label.facet,
|
||||
query=label.canonical_label + ". " + label.definition,
|
||||
labels=(label,),
|
||||
embeddings=embedding_provider,
|
||||
limit=1,
|
||||
)
|
||||
except (OSError, RuntimeError, ValueError, sqlite3.Error) as error:
|
||||
# Vectors are a cache and can be rebuilt from the canonical project catalog.
|
||||
logger.warning("Could not cache embedding for new project facet label: %s", error)
|
||||
return tuple(
|
||||
resolution.model_copy(
|
||||
update={
|
||||
"rank": by_facet_rank.get((resolution.facet, resolution.value), resolution.rank),
|
||||
"added_to_project": (resolution.facet, resolution.value) in added_keys,
|
||||
}
|
||||
)
|
||||
for resolution in ranked_resolutions
|
||||
)
|
||||
|
||||
|
||||
def _principal(ctx: Context, identities: IdentityProvider) -> Principal:
|
||||
@@ -74,6 +363,28 @@ def _principal(ctx: Context, identities: IdentityProvider) -> Principal:
|
||||
return principal
|
||||
|
||||
|
||||
def _validation_issues(error: ValidationError) -> list[dict[str, str]]:
|
||||
issues: list[dict[str, str]] = []
|
||||
for detail in error.errors(
|
||||
include_context=False,
|
||||
include_input=False,
|
||||
include_url=False,
|
||||
)[:64]:
|
||||
path = ""
|
||||
for part in detail.get("loc", ()):
|
||||
path += f"[{part}]" if isinstance(part, int) else (
|
||||
("." if path else "") + str(part)
|
||||
)
|
||||
issues.append(
|
||||
{
|
||||
"path": path,
|
||||
"code": str(detail.get("type", "invalid_value")),
|
||||
"message": str(detail.get("msg", "Invalid value")),
|
||||
}
|
||||
)
|
||||
return issues
|
||||
|
||||
|
||||
def create_mcp_server(
|
||||
*,
|
||||
identity_provider: IdentityProvider,
|
||||
@@ -87,6 +398,9 @@ def create_mcp_server(
|
||||
url_fetcher: PublicUrlFetcher | None = None,
|
||||
ingest_review_store: SQLiteIngestReviewStore | None = None,
|
||||
jev_assessor: OpenRouterJev | None = None,
|
||||
project_registry: ProjectRegistry | None = None,
|
||||
facet_label_index: SQLiteFacetLabelIndex | None = None,
|
||||
embedding_provider: EmbeddingProvider | None = None,
|
||||
) -> MCPServer:
|
||||
"""Create the allowlisted MCP server for the supplied application services."""
|
||||
|
||||
@@ -381,9 +695,10 @@ def create_mcp_server(
|
||||
try:
|
||||
item = IngestReviewCreate.model_validate_json(review_json, strict=True)
|
||||
except ValidationError as error:
|
||||
raise ValueError(
|
||||
"Ingest review request does not match the GEWU v1 contract"
|
||||
) from error
|
||||
return json.dumps(
|
||||
{"status": "invalid", "issues": _validation_issues(error)},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
_require_project_submission(principal, item.project_id)
|
||||
actor = ActorContext(principal.principal_id, principal.credential_id)
|
||||
try:
|
||||
@@ -433,11 +748,56 @@ def create_mcp_server(
|
||||
"Ingest review locator metadata does not match its extraction block"
|
||||
)
|
||||
assessment = None
|
||||
facet_resolutions: tuple[JevFacetResolution, ...] = ()
|
||||
if jev_assessor is not None:
|
||||
project = context_provider().projects.get(item.project_id)
|
||||
context = context_provider() if context_provider is not None else None
|
||||
project = context.projects.get(item.project_id) if context is not None else None
|
||||
if project is not None:
|
||||
try:
|
||||
assessment = jev_assessor.assess(item, project)
|
||||
catalog = _project_facet_catalog(project, context)
|
||||
nearest: dict[tuple[FacetName, str], tuple[ProjectFacetLabel, ...]] = {}
|
||||
semantic_search_ready = (
|
||||
facet_label_index is not None and embedding_provider is not None
|
||||
)
|
||||
for proposal in item.facet_proposals():
|
||||
try:
|
||||
nearest[(proposal.facet, proposal.value)] = (
|
||||
_nearest_facet_labels(
|
||||
proposal=proposal,
|
||||
project_id=item.project_id,
|
||||
catalog=catalog,
|
||||
index=(
|
||||
facet_label_index if semantic_search_ready else None
|
||||
),
|
||||
embeddings=(
|
||||
embedding_provider
|
||||
if semantic_search_ready
|
||||
else None
|
||||
),
|
||||
)
|
||||
)
|
||||
except (OSError, RuntimeError, ValueError, sqlite3.Error):
|
||||
semantic_search_ready = False
|
||||
nearest[(proposal.facet, proposal.value)] = (
|
||||
_nearest_facet_labels(
|
||||
proposal=proposal,
|
||||
project_id=item.project_id,
|
||||
catalog=catalog,
|
||||
index=None,
|
||||
embeddings=None,
|
||||
)
|
||||
)
|
||||
assessment = jev_assessor.assess(item, project, closest_labels=nearest)
|
||||
facet_resolutions = _promote_jev_labels(
|
||||
item=item,
|
||||
assessment=assessment,
|
||||
project_id=item.project_id,
|
||||
project_registry=project_registry,
|
||||
facet_label_index=facet_label_index,
|
||||
embedding_provider=embedding_provider,
|
||||
existing_catalog=catalog,
|
||||
semantic_search_ready=semantic_search_ready,
|
||||
)
|
||||
except JevUnavailable:
|
||||
# The review flag and normal proposal flow do not depend on Jev uptime.
|
||||
assessment = None
|
||||
@@ -453,6 +813,7 @@ def create_mcp_server(
|
||||
),
|
||||
jev_model=assessment.model if assessment is not None else None,
|
||||
jev_facet_scores=(assessment.facet_scores if assessment is not None else ()),
|
||||
jev_facet_resolutions=facet_resolutions,
|
||||
)
|
||||
return json.dumps(record.model_dump(mode="json"), ensure_ascii=False)
|
||||
|
||||
@@ -890,8 +1251,9 @@ def create_mcp_server(
|
||||
offset: int,
|
||||
length: int,
|
||||
ctx: Context,
|
||||
include_original_mapping: bool = False,
|
||||
) -> str:
|
||||
"""Read a bounded normalized text slice and its source offset map."""
|
||||
"""Read a bounded normalized text slice and optionally its original-source offset map."""
|
||||
|
||||
principal = _principal(ctx, identity_provider)
|
||||
_require_project_submission(principal, project_id)
|
||||
@@ -906,21 +1268,20 @@ def create_mcp_server(
|
||||
if block is None or offset >= len(block.normalized_text):
|
||||
raise ValueError("Extraction block or offset does not exist")
|
||||
end = min(offset + length, len(block.normalized_text))
|
||||
spans = block.normalization_map[offset:end]
|
||||
original_start = min(start for start, _ in spans)
|
||||
original_end = max(stop for _, stop in spans)
|
||||
return json.dumps(
|
||||
{
|
||||
"extraction_id": extraction_id,
|
||||
"block_id": block_id,
|
||||
"page": block.page,
|
||||
"offset": offset,
|
||||
"normalized_text": block.normalized_text[offset:end],
|
||||
"original_text": block.original_text[original_start:original_end],
|
||||
"normalization_map": spans,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
result: dict[str, object] = {
|
||||
"extraction_id": extraction_id,
|
||||
"block_id": block_id,
|
||||
"page": block.page,
|
||||
"offset": offset,
|
||||
"normalized_text": block.normalized_text[offset:end],
|
||||
}
|
||||
if include_original_mapping:
|
||||
spans = block.normalization_map[offset:end]
|
||||
original_start = min(start for start, _ in spans)
|
||||
original_end = max(stop for _, stop in spans)
|
||||
result["original_text"] = block.original_text[original_start:original_end]
|
||||
result["normalization_map"] = spans
|
||||
return json.dumps(result, ensure_ascii=False)
|
||||
|
||||
@server.tool(name="proposal_submit", structured_output=False)
|
||||
def proposal_submit(proposal_json: str, ctx: Context) -> str:
|
||||
@@ -969,6 +1330,52 @@ def create_mcp_server(
|
||||
}
|
||||
return json.dumps(result, ensure_ascii=False)
|
||||
|
||||
@server.tool(name="proposal_prepare", structured_output=False)
|
||||
def proposal_prepare(proposal_json: str, ctx: Context) -> str:
|
||||
"""Fill canonical evidence and payload digests without storing the proposal."""
|
||||
|
||||
principal = _principal(ctx, identity_provider)
|
||||
if len(proposal_json.encode("utf-8")) > 1_000_000:
|
||||
raise ValueError("Proposal exceeds the supported request size")
|
||||
try:
|
||||
draft = json.loads(proposal_json, object_pairs_hook=_reject_duplicate_keys)
|
||||
except (ValueError, TypeError) as error:
|
||||
raise ValueError("Proposal draft must be valid JSON") from error
|
||||
if not isinstance(draft, dict):
|
||||
raise ValueError("Proposal draft must be a JSON object")
|
||||
try:
|
||||
proposal = prepare_proposal_digests(draft)
|
||||
except ValidationError as error:
|
||||
return json.dumps(
|
||||
{"status": "invalid", "issues": _validation_issues(error)},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
except ValueError as error:
|
||||
return json.dumps(
|
||||
{
|
||||
"status": "invalid",
|
||||
"issues": [
|
||||
{
|
||||
"path": "",
|
||||
"code": "invalid_proposal_draft",
|
||||
"message": str(error)[:500],
|
||||
}
|
||||
],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
_authorize_proposal(principal, proposal, context_provider())
|
||||
return json.dumps(
|
||||
{
|
||||
"status": "prepared",
|
||||
"proposal_id": proposal.proposal_id,
|
||||
"revision": proposal.revision,
|
||||
"payload_digest": proposal.payload_digest,
|
||||
"proposal_json": proposal.model_dump_json(),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
|
||||
@server.tool(name="proposal_evaluate", structured_output=False)
|
||||
def proposal_evaluate(proposal_json: str, ctx: Context) -> str:
|
||||
"""Evaluate one bounded proposal without storing or publishing it."""
|
||||
@@ -1117,6 +1524,7 @@ def create_mcp_server(
|
||||
research_lenses=project.research_lenses,
|
||||
watch_list=project.watch_list,
|
||||
taxonomy_extensions=project.taxonomy_extensions,
|
||||
facet_labels=_project_facet_catalog(project, context),
|
||||
ingest_guidance=(
|
||||
ingest_review_store.guidance(project_id)
|
||||
if ingest_review_store is not None
|
||||
|
||||
Reference in new issue
Block a user