diff --git a/agents/gewu b/agents/gewu index 3182698..6adc5a0 160000 --- a/agents/gewu +++ b/agents/gewu @@ -1 +1 @@ -Subproject commit 31826982e6882eb1e6bc26c77000cd3e7d657e97 +Subproject commit 6adc5a0d35ea0a29c16a262e800d0c7ad971546c diff --git a/schema/v1/openapi.json b/schema/v1/openapi.json index 80ca3db..441dcc6 100644 --- a/schema/v1/openapi.json +++ b/schema/v1/openapi.json @@ -1215,6 +1215,15 @@ "title": "Extraction Id", "type": "string" }, + "jev_facet_scores": { + "default": [], + "items": { + "$ref": "#/components/schemas/JevFacetScore" + }, + "maxItems": 20, + "title": "Jev Facet Scores", + "type": "array" + }, "jev_lens": { "anyOf": [ { @@ -1392,6 +1401,46 @@ "title": "IssuedCredential", "type": "object" }, + "JevFacetScore": { + "additionalProperties": false, + "properties": { + "facet": { + "enum": [ + "tags", + "audience", + "use_for", + "domain" + ], + "title": "Facet", + "type": "string" + }, + "probability": { + "maximum": 1.0, + "minimum": 0.0, + "title": "Probability", + "type": "number" + }, + "schema_version": { + "const": 1, + "default": 1, + "title": "Schema Version", + "type": "integer" + }, + "value": { + "maxLength": 100, + "minLength": 1, + "title": "Value", + "type": "string" + } + }, + "required": [ + "facet", + "value", + "probability" + ], + "title": "JevFacetScore", + "type": "object" + }, "KnowledgeNode": { "additionalProperties": false, "properties": { diff --git a/src/gewu/adapters/openrouter_jev.py b/src/gewu/adapters/openrouter_jev.py index 4089094..5fde5d2 100644 --- a/src/gewu/adapters/openrouter_jev.py +++ b/src/gewu/adapters/openrouter_jev.py @@ -8,7 +8,13 @@ from typing import Any, cast from urllib.error import HTTPError, URLError from urllib.request import HTTPRedirectHandler, Request, build_opener -from gewu.domain.contracts import IngestReviewCreate, ProjectRecord, ResearchLens +from gewu.domain.contracts import ( + FacetName, + IngestReviewCreate, + JevFacetScore, + ProjectRecord, + ResearchLens, +) ENDPOINT = "https://openrouter.ai/api/alpha/decisions" MODEL = "typesafe/jev-1.13" @@ -21,6 +27,7 @@ class JevAssessment: lens: ResearchLens lens_confidence: float model: str + facet_scores: tuple[JevFacetScore, ...] = () class JevUnavailable(RuntimeError): @@ -44,6 +51,30 @@ class OpenRouterJev: self._model = model def assess(self, item: IngestReviewCreate, project: ProjectRecord) -> JevAssessment: + facet_candidates: tuple[tuple[FacetName, tuple[str, ...]], ...] = ( + ("tags", item.candidate_tags), + ("audience", item.candidate_audience), + ("use_for", item.candidate_use_for), + ("domain", item.candidate_domain), + ) + facet_questions: dict[str, object] = {} + facet_question_map: dict[str, tuple[FacetName, str]] = {} + for facet, candidates in facet_candidates: + for index, value in enumerate(candidates): + key = f"facet_{facet}_{index}" + facet_question_map[key] = (facet, value) + facet_questions[key] = { + "type": "noul", + "instructions": ( + f"Does the candidate {facet} label accurately describe the passage " + "in the context of this project? Judge the label itself, not just " + "whether its words appear in the excerpt." + ), + "criteria": { + "true": "This label is a useful, well-supported classification.", + "false": "This label is inaccurate, unsupported, or too broad.", + }, + } state = { "project_title": project.title, "project_north_star": project.north_star, @@ -52,6 +83,9 @@ class OpenRouterJev: "candidate_summary": item.summary, "candidate_relevance_rationale": item.relevance_rationale, "source_excerpt": item.excerpt, + "candidate_facets": { + facet: candidates for facet, candidates in facet_candidates if candidates + }, } criteria = { "technical": "Engineering, implementation, technology, or operational practice.", @@ -59,30 +93,32 @@ class OpenRouterJev: "cultural-historical": "Cultural, artistic, historical, social, or audience context.", "financial-market": "Costs, revenues, business models, markets, or financing.", } + questions: dict[str, object] = { + "project_relation": { + "type": "noul", + "instructions": ( + "Does this passage help answer a project question or explain " + "the project's technical, conceptual, cultural-historical, or " + "financial-market context? Treat indirect context as relevant when " + "the connection is plausible." + ), + "criteria": { + "true": "A plausible, specific connection to the project exists.", + "false": "The passage has no plausible project connection.", + }, + }, + "best_lens": { + "type": "choice", + "instructions": "Which project research lens best describes this passage?", + "criteria": criteria, + }, + **facet_questions, + } payload = json.dumps( { "model": self._model, "state": state, - "questions": { - "project_relation": { - "type": "noul", - "instructions": ( - "Does this passage help answer a project question or explain " - "the project's technical, conceptual, cultural-historical, or " - "financial-market context? Treat indirect context as relevant when " - "the connection is plausible." - ), - "criteria": { - "true": "A plausible, specific connection to the project exists.", - "false": "The passage has no plausible project connection.", - }, - }, - "best_lens": { - "type": "choice", - "instructions": "Which project research lens best describes this passage?", - "criteria": criteria, - }, - }, + "questions": questions, }, ensure_ascii=False, separators=(",", ":"), @@ -113,6 +149,14 @@ class OpenRouterJev: lens = answers["best_lens"]["choice"] confidence = float(answers["best_lens"]["confidence"]) served_model = str(result["model"]) + facet_scores = tuple( + JevFacetScore( + facet=facet, + value=value, + probability=float(answers[key]["noul"]), + ) + for key, (facet, value) in facet_question_map.items() + ) except (KeyError, TypeError, ValueError, json.JSONDecodeError) as error: raise JevUnavailable("Jev response did not match its typed decision schema") from error if ( @@ -122,4 +166,6 @@ class OpenRouterJev: or len(served_model) > 100 ): raise JevUnavailable("Jev response contained an invalid decision") - return JevAssessment(relation, cast(ResearchLens, lens), confidence, served_model) + return JevAssessment( + relation, cast(ResearchLens, lens), confidence, served_model, facet_scores + ) diff --git a/src/gewu/adapters/sqlite_ingest_reviews.py b/src/gewu/adapters/sqlite_ingest_reviews.py index 7819e7f..2bc0d8a 100644 --- a/src/gewu/adapters/sqlite_ingest_reviews.py +++ b/src/gewu/adapters/sqlite_ingest_reviews.py @@ -14,6 +14,7 @@ from gewu.domain.contracts import ( IngestReviewCreate, IngestReviewDecision, IngestReviewItem, + JevFacetScore, ) MAX_REVIEW_ITEMS = 100_000 @@ -52,6 +53,7 @@ class SQLiteIngestReviewStore: jev_lens TEXT, jev_lens_confidence REAL, jev_model TEXT, + jev_facet_scores_json TEXT NOT NULL DEFAULT '[]', status TEXT NOT NULL CHECK(status IN ( 'needs-user-review', 'included', 'excluded', 'deferred' )), @@ -68,6 +70,7 @@ class SQLiteIngestReviewStore: ("jev_lens", "TEXT"), ("jev_lens_confidence", "REAL"), ("jev_model", "TEXT"), + ("jev_facet_scores_json", "TEXT NOT NULL DEFAULT '[]'"), ): if column not in existing_columns: connection.execute( @@ -118,6 +121,7 @@ class SQLiteIngestReviewStore: jev_lens: str | None = None, jev_lens_confidence: float | None = None, jev_model: str | None = None, + jev_facet_scores: tuple[JevFacetScore, ...] = (), ) -> IngestReviewItem: payload = item.model_dump(mode="json") digest = _digest({**payload, "submitted_by": submitted_by}) @@ -139,9 +143,10 @@ class SQLiteIngestReviewStore: review_id, project_id, run_id, source_revision_id, extraction_id, locator_json, excerpt, lens, summary, relevance_rationale, relevance, status, submitted_by, created_at, payload_digest, - jev_relation_probability, jev_lens, jev_lens_confidence, jev_model + jev_relation_probability, jev_lens, jev_lens_confidence, jev_model, + jev_facet_scores_json ) VALUES ( - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'needs-user-review', ?, ?, ?, ?, ?, ?, ? + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'needs-user-review', ?, ?, ?, ?, ?, ?, ?, ? )""", ( item.review_id, @@ -162,6 +167,11 @@ class SQLiteIngestReviewStore: jev_lens, jev_lens_confidence, jev_model, + json.dumps( + [score.model_dump(mode="json") for score in jev_facet_scores], + ensure_ascii=False, + separators=(",", ":"), + ), ), ) return self._read_item(connection, item.review_id) @@ -274,7 +284,8 @@ class SQLiteIngestReviewStore: """SELECT review_id, project_id, run_id, source_revision_id, extraction_id, locator_json, excerpt, lens, summary, relevance_rationale, relevance, status, submitted_by, created_at, jev_relation_probability, jev_lens, - jev_lens_confidence, jev_model FROM ingest_review_items WHERE review_id=?""", + jev_lens_confidence, jev_model, jev_facet_scores_json + FROM ingest_review_items WHERE review_id=?""", (review_id,), ).fetchone() if row is None: @@ -299,6 +310,7 @@ class SQLiteIngestReviewStore: "jev_lens": row[15], "jev_lens_confidence": row[16], "jev_model": row[17], + "jev_facet_scores": json.loads(row[18]), }, strict=True, ) diff --git a/src/gewu/domain/contracts.py b/src/gewu/domain/contracts.py index 82126f9..3979564 100644 --- a/src/gewu/domain/contracts.py +++ b/src/gewu/domain/contracts.py @@ -25,6 +25,7 @@ ResearchLens = Literal[ "cultural-historical", "financial-market", ] +FacetName = Literal["tags", "audience", "use_for", "domain"] DEFAULT_RESEARCH_LENSES: tuple[ResearchLens, ...] = ( "technical", "conceptual", @@ -115,6 +116,12 @@ class ProjectContextResponse(StrictRecord): lens: ProjectLens +class JevFacetScore(StrictRecord): + facet: FacetName + value: SearchFacetValue + probability: FiniteFloat = Field(ge=0, le=1) + + class IngestReviewItem(StrictRecord): review_id: RecordId project_id: RecordId @@ -127,6 +134,7 @@ class IngestReviewItem(StrictRecord): summary: str = Field(min_length=1, max_length=2000) relevance_rationale: str = Field(min_length=1, max_length=2000) relevance: Literal["borderline", "low"] = "borderline" + jev_facet_scores: tuple[JevFacetScore, ...] = Field(default=(), max_length=20) jev_relation_probability: FiniteFloat | None = Field(default=None, ge=0, le=1) jev_lens: ResearchLens | None = None jev_lens_confidence: FiniteFloat | None = Field(default=None, ge=0, le=1) @@ -156,6 +164,29 @@ class IngestReviewCreate(StrictRecord): summary: str = Field(min_length=1, max_length=2000) relevance_rationale: str = Field(min_length=1, max_length=2000) relevance: Literal["borderline", "low"] = "borderline" + candidate_tags: tuple[SearchFacetValue, ...] = Field(default=(), max_length=5) + candidate_audience: tuple[SearchFacetValue, ...] = Field(default=(), max_length=5) + candidate_use_for: tuple[SearchFacetValue, ...] = Field(default=(), max_length=5) + candidate_domain: tuple[SearchFacetValue, ...] = Field(default=(), max_length=5) + + @model_validator(mode="after") + def limit_total_facet_candidates(self) -> IngestReviewCreate: + values = ( + *self.candidate_tags, + *self.candidate_audience, + *self.candidate_use_for, + *self.candidate_domain, + ) + if len(values) > 20: + raise ValueError("At most 20 facet candidates may be assessed") + if len({(facet, value.casefold()) for facet, candidates in ( + ("tags", self.candidate_tags), + ("audience", self.candidate_audience), + ("use_for", self.candidate_use_for), + ("domain", self.candidate_domain), + ) for value in candidates}) != len(values): + raise ValueError("Facet candidates must be unique within each facet") + return self class IngestReviewDecision(StrictRecord): diff --git a/src/gewu/transports/mcp_server.py b/src/gewu/transports/mcp_server.py index cfc54ab..c2ea207 100644 --- a/src/gewu/transports/mcp_server.py +++ b/src/gewu/transports/mcp_server.py @@ -452,6 +452,7 @@ def create_mcp_server( assessment.lens_confidence if assessment is not None else None ), jev_model=assessment.model if assessment is not None else None, + jev_facet_scores=(assessment.facet_scores if assessment is not None else ()), ) return json.dumps(record.model_dump(mode="json"), ensure_ascii=False) diff --git a/web/src/generated/api.ts b/web/src/generated/api.ts index 878e276..00245de 100644 --- a/web/src/generated/api.ts +++ b/web/src/generated/api.ts @@ -966,6 +966,11 @@ export interface components { excerpt: string; /** Extraction Id */ extraction_id: string; + /** + * Jev Facet Scores + * @default [] + */ + jev_facet_scores: components["schemas"]["JevFacetScore"][]; /** Jev Lens */ jev_lens?: ("technical" | "conceptual" | "cultural-historical" | "financial-market") | null; /** Jev Lens Confidence */ @@ -1020,6 +1025,24 @@ export interface components { /** Token */ token: string; }; + /** JevFacetScore */ + JevFacetScore: { + /** + * Facet + * @enum {string} + */ + facet: "tags" | "audience" | "use_for" | "domain"; + /** Probability */ + probability: number; + /** + * Schema Version + * @default 1 + * @constant + */ + schema_version: 1; + /** Value */ + value: string; + }; /** KnowledgeNode */ KnowledgeNode: { /** Agent Id */ diff --git a/web/src/shell/Admin.tsx b/web/src/shell/Admin.tsx index d579896..83e4d79 100644 --- a/web/src/shell/Admin.tsx +++ b/web/src/shell/Admin.tsx @@ -25,6 +25,7 @@ type IngestReview = { excerpt: string; lens: string; summary: string; relevance_rationale: string; jev_relation_probability?: number | null; jev_lens?: string | null; jev_lens_confidence?: number | null; jev_model?: string | null; + jev_facet_scores?: { facet: 'tags' | 'audience' | 'use_for' | 'domain'; value: string; probability: number }[]; relevance: 'borderline' | 'low'; status: string; created_at: string; }; @@ -67,13 +68,14 @@ function IngestReviewSection({ csrf, projects, onError, onMessage }: { } return
-

Ingest review

These flags are recorded alongside normal ingestion and do not hold up proposals or publication. Your decision helps tune future project triage.

If configured, Jev sends the project context and excerpt to OpenRouter for a relevance probability and lens suggestion. That signal is advisory.

+

Ingest review

These flags are recorded alongside normal ingestion and do not hold up proposals or publication. Your decision helps tune future project triage.

If configured, Jev sends the project context and excerpt to OpenRouter for relevance, lens and candidate facet suggestions. These signals are advisory.

{!projectId ?

Select a project to view its review flags.

: items.length === 0 ?

No review flags need feedback.

:
    {items.map(item =>
  1. {item.lens.replaceAll('-', ' ')} · {item.relevance} relevance · {item.summary}

    {item.excerpt}

    {item.relevance_rationale}

    {item.jev_relation_probability != null ?

    Jev relevance: {Math.round(item.jev_relation_probability * 100)}% · lens: {item.jev_lens ?? 'unknown'} ({item.jev_lens_confidence != null ? `${Math.round(item.jev_lens_confidence * 100)}% confidence` : 'no confidence score'}) · {item.jev_model}

    :

    Jev signal unavailable or not configured.

    } + {!!item.jev_facet_scores?.length &&

    Facet suggestions: {item.jev_facet_scores.map(score => `${score.facet.replace('_', ' ')} “${score.value}” (${Math.round(score.probability * 100)}%)`).join(' · ')}

    }

    Source {item.source_revision_id} · {item.locator.kind.replaceAll('_', ' ')} {item.locator.page ? `· page ${item.locator.page}` : ''} · run {item.run_id}