This commit is contained in:
vasily committed
1 parent
4db3f45164
commit
d25c04cdd2
175 files changed
+56243
-15
No files matched your search
@@ -0,0 +1,955 @@
|
||||
"""FastAPI composition root, health check and authenticated canonical reads."""
|
||||
|
||||
import os
|
||||
from collections.abc import AsyncIterator, Callable, Iterator
|
||||
from contextlib import AsyncExitStack, asynccontextmanager
|
||||
from pathlib import Path
|
||||
from typing import Annotated, cast
|
||||
|
||||
import portalocker
|
||||
from fastapi import Cookie, FastAPI, Header, HTTPException, Query, Request, Response
|
||||
from mcp.server.transport_security import TransportSecuritySettings
|
||||
from starlette.responses import StreamingResponse
|
||||
from starlette.routing import Mount
|
||||
|
||||
from gewu.adapters.embeddings import LocalSentenceTransformer
|
||||
from gewu.adapters.git_repository import GitCanonicalRepository
|
||||
from gewu.adapters.source_extractor import LocalSourceExtractor, SourceExtractor
|
||||
from gewu.adapters.sqlite_identity import SQLiteIdentityStore
|
||||
from gewu.adapters.sqlite_search import SQLiteSearchIndex
|
||||
from gewu.adapters.sqlite_vectors import SQLiteVectorIndex
|
||||
from gewu.adapters.url_fetch import PublicUrlFetcher
|
||||
from gewu.application.acceptance import AcceptanceService
|
||||
from gewu.application.accounts import AccountIdentity
|
||||
from gewu.application.admin import CredentialAdministration
|
||||
from gewu.application.context import ValidationContext
|
||||
from gewu.application.diagnostics import build_service_diagnostics
|
||||
from gewu.application.identity import IdentityProvider, OwnerIdentityProvider, Principal
|
||||
from gewu.application.paths import reject_symlink_components
|
||||
from gewu.application.ports import ProjectRegistry
|
||||
from gewu.application.reading import CanonicalNodeReader, CanonicalSourceReader, ReadError
|
||||
from gewu.application.registry import (
|
||||
LocalContextProvider,
|
||||
LocalProjectRegistry,
|
||||
RegistryConflict,
|
||||
RegistryError,
|
||||
RegistryNotFound,
|
||||
)
|
||||
from gewu.application.search import (
|
||||
CanonicalSearchService,
|
||||
HybridSearchIndex,
|
||||
SearchError,
|
||||
SearchIndex,
|
||||
)
|
||||
from gewu.application.uploads import LocalArtifactStore, UploadError
|
||||
from gewu.domain.contracts import (
|
||||
GitCommitId,
|
||||
ProjectRecord,
|
||||
ReadEnvelope,
|
||||
RecordId,
|
||||
ServiceDiagnostics,
|
||||
)
|
||||
from gewu.transports.accounts_http import register_account_routes
|
||||
from gewu.transports.admin_http import register_admin_routes
|
||||
from gewu.transports.http_middleware import ProjectRegistryBodyLimitMiddleware
|
||||
from gewu.transports.http_models import (
|
||||
NeighborRequestBody,
|
||||
OwnerSessionRequestBody,
|
||||
ProjectContentUpdateBody,
|
||||
ProjectCreateBody,
|
||||
SearchRequestBody,
|
||||
SourceListRequestBody,
|
||||
)
|
||||
from gewu.transports.mcp_server import create_mcp_server
|
||||
from gewu.transports.owner_mcp_server import OwnerReviewProvider, create_owner_mcp_server
|
||||
from gewu.version import SERVICE_VERSION
|
||||
|
||||
|
||||
def create_app(
|
||||
*,
|
||||
identity_provider: IdentityProvider | None = None,
|
||||
owner_identity_provider: OwnerIdentityProvider | None = None,
|
||||
owner_service_audience: str | None = None,
|
||||
node_reader: CanonicalNodeReader | None = None,
|
||||
artifact_store: LocalArtifactStore | None = None,
|
||||
extractor: SourceExtractor | None = None,
|
||||
acceptance_service: AcceptanceService | None = None,
|
||||
context_provider: Callable[[], ValidationContext] | None = None,
|
||||
search_service: CanonicalSearchService | None = None,
|
||||
project_registry: ProjectRegistry | None = None,
|
||||
credential_administration: CredentialAdministration | None = None,
|
||||
account_identity: AccountIdentity | None = None,
|
||||
url_fetcher: PublicUrlFetcher | None = None,
|
||||
) -> FastAPI:
|
||||
if (identity_provider is None) != (node_reader is None):
|
||||
raise ValueError("Identity provider and canonical reader must be configured together")
|
||||
app = FastAPI(title="GEWU", version=SERVICE_VERSION)
|
||||
mcp_allowed_hosts = [
|
||||
host.strip()
|
||||
for host in os.environ.get("GEWU_MCP_ALLOWED_HOSTS", "").split(",")
|
||||
if host.strip()
|
||||
]
|
||||
if (
|
||||
project_registry is not None
|
||||
or owner_identity_provider is not None
|
||||
or credential_administration is not None
|
||||
or account_identity is not None
|
||||
):
|
||||
app.add_middleware(ProjectRegistryBodyLimitMiddleware)
|
||||
|
||||
@app.get("/health", tags=["health"], operation_id="health")
|
||||
def health() -> dict[str, str]:
|
||||
return {"status": "ok", "service": "gewu"}
|
||||
|
||||
if identity_provider is not None and node_reader is not None:
|
||||
|
||||
def authenticate(
|
||||
authorization: str | None,
|
||||
session_cookie: str | None = None,
|
||||
csrf_token: str | None = None,
|
||||
*,
|
||||
require_csrf: bool = False,
|
||||
) -> Principal:
|
||||
scheme, _, token = (authorization or "").partition(" ")
|
||||
if scheme.lower() == "bearer" and token:
|
||||
principal = identity_provider.authenticate(token)
|
||||
if principal is None:
|
||||
raise HTTPException(status_code=401, detail="Credential is invalid or revoked")
|
||||
return principal
|
||||
if session_cookie:
|
||||
principal = identity_provider.authenticate_browser_session(session_cookie)
|
||||
if principal is None:
|
||||
raise HTTPException(
|
||||
status_code=401, detail="Browser session is invalid or expired"
|
||||
)
|
||||
if require_csrf and not identity_provider.browser_csrf_valid(
|
||||
session_cookie, csrf_token or ""
|
||||
):
|
||||
raise HTTPException(status_code=403, detail="Browser session CSRF check failed")
|
||||
return principal
|
||||
raise HTTPException(
|
||||
status_code=401, detail="Bearer credential or browser session required"
|
||||
)
|
||||
|
||||
def require_same_origin(request: Request) -> None:
|
||||
origin = request.headers.get("origin")
|
||||
expected = f"{request.url.scheme}://{request.headers.get('host', '')}"
|
||||
if origin is not None and origin.rstrip("/") != expected.rstrip("/"):
|
||||
raise HTTPException(
|
||||
status_code=403, detail="Browser session requires a same-origin request"
|
||||
)
|
||||
|
||||
def require_admin(principal: Principal) -> None:
|
||||
if "admin" not in principal.roles or "agent" in principal.roles:
|
||||
raise HTTPException(status_code=403, detail="Administrator role is required")
|
||||
|
||||
if credential_administration is not None:
|
||||
register_admin_routes(app, credential_administration, authenticate, require_same_origin)
|
||||
|
||||
if account_identity is not None:
|
||||
register_account_routes(app, account_identity, authenticate, require_same_origin)
|
||||
|
||||
if owner_identity_provider is not None:
|
||||
|
||||
@app.post(
|
||||
"/auth/owner-session",
|
||||
tags=["identity"],
|
||||
operation_id="owner_session_create",
|
||||
)
|
||||
def create_owner_session(
|
||||
body: OwnerSessionRequestBody,
|
||||
request: Request,
|
||||
response: Response,
|
||||
) -> dict[str, str]:
|
||||
require_same_origin(request)
|
||||
response.headers["Cache-Control"] = "no-store"
|
||||
host = request.url.hostname or ""
|
||||
if request.url.scheme != "https" and host not in {
|
||||
"localhost",
|
||||
"127.0.0.1",
|
||||
"::1",
|
||||
}:
|
||||
raise HTTPException(
|
||||
status_code=403, detail="Owner authentication requires HTTPS"
|
||||
)
|
||||
audience = owner_service_audience
|
||||
if not audience or not owner_identity_provider.owner_authentication_enabled:
|
||||
raise HTTPException(status_code=503, detail="Owner authentication is disabled")
|
||||
owner_session = owner_identity_provider.authenticate_owner(
|
||||
principal_id=body.principal_id,
|
||||
password=body.password,
|
||||
totp_code=body.totp_code,
|
||||
challenge_id=body.challenge_id,
|
||||
audience=audience,
|
||||
)
|
||||
if owner_session is None:
|
||||
raise HTTPException(status_code=401, detail="Owner authentication failed")
|
||||
return {
|
||||
"session_token": owner_session.session_token,
|
||||
"challenge_id": owner_session.challenge_id,
|
||||
"expires_at": owner_session.expires_at.isoformat(),
|
||||
}
|
||||
|
||||
@app.post("/v1/session", tags=["identity"], operation_id="browser_session_create")
|
||||
def create_browser_session(
|
||||
request: Request,
|
||||
response: Response,
|
||||
authorization: str | None = Header(default=None),
|
||||
) -> dict[str, str]:
|
||||
require_same_origin(request)
|
||||
response.headers["Cache-Control"] = "no-store"
|
||||
scheme, _, bearer_token = (authorization or "").partition(" ")
|
||||
if scheme.lower() != "bearer" or not bearer_token:
|
||||
raise HTTPException(status_code=401, detail="Bearer credential required")
|
||||
principal = identity_provider.authenticate(bearer_token)
|
||||
if principal is None:
|
||||
raise HTTPException(status_code=401, detail="Credential is invalid or revoked")
|
||||
if "agent" in principal.roles or not principal.roles.intersection(
|
||||
{"admin", "editor", "viewer"}
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=403, detail="Browser sessions require a human identity"
|
||||
)
|
||||
try:
|
||||
session = identity_provider.create_browser_session(bearer_token)
|
||||
except PermissionError as error:
|
||||
raise HTTPException(status_code=403, detail=str(error)) from error
|
||||
response.set_cookie(
|
||||
key="gewu_session",
|
||||
value=session.session_token,
|
||||
httponly=True,
|
||||
secure=request.url.scheme == "https",
|
||||
samesite="strict",
|
||||
path="/v1",
|
||||
)
|
||||
return {"csrf_token": session.csrf_token, "expires_at": session.expires_at.isoformat()}
|
||||
|
||||
@app.get("/v1/session", tags=["identity"], operation_id="browser_session_restore")
|
||||
def restore_browser_session(
|
||||
request: Request,
|
||||
response: Response,
|
||||
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
|
||||
restore_header: str | None = Header(default=None, alias="X-GEWU-Session"),
|
||||
) -> dict[str, str | bool]:
|
||||
require_same_origin(request)
|
||||
if restore_header != "restore" or request.headers.get("sec-fetch-site") == "cross-site":
|
||||
raise HTTPException(403, "Session restoration requires a same-origin client")
|
||||
principal = authenticate(None, session_cookie)
|
||||
session = identity_provider.restore_browser_session(session_cookie or "")
|
||||
if session is None:
|
||||
raise HTTPException(
|
||||
401,
|
||||
"Browser session is invalid or expired",
|
||||
headers={"Cache-Control": "no-store"},
|
||||
)
|
||||
response.headers["Cache-Control"] = "private, no-store"
|
||||
return {
|
||||
"csrf_token": session.csrf_token,
|
||||
"expires_at": session.expires_at.isoformat(),
|
||||
"password_account": principal.credential_id.startswith("account:"),
|
||||
}
|
||||
|
||||
@app.delete("/v1/session", tags=["identity"], operation_id="browser_session_revoke")
|
||||
def revoke_browser_session(
|
||||
request: Request,
|
||||
response: Response,
|
||||
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
|
||||
csrf_token: str | None = Header(default=None, alias="X-GEWU-CSRF"),
|
||||
) -> dict[str, str]:
|
||||
require_same_origin(request)
|
||||
response.headers["Cache-Control"] = "no-store"
|
||||
if session_cookie:
|
||||
authenticate(None, session_cookie, csrf_token, require_csrf=True)
|
||||
identity_provider.revoke_browser_session(session_cookie)
|
||||
response.delete_cookie(
|
||||
key="gewu_session",
|
||||
path="/v1",
|
||||
httponly=True,
|
||||
secure=request.url.scheme == "https",
|
||||
samesite="strict",
|
||||
)
|
||||
return {"status": "signed_out"}
|
||||
|
||||
@app.get(
|
||||
"/v1/projects/{project_id}/nodes/{node_id}",
|
||||
response_model=ReadEnvelope,
|
||||
tags=["knowledge"],
|
||||
operation_id="get_node",
|
||||
)
|
||||
def get_node(
|
||||
project_id: RecordId,
|
||||
node_id: RecordId,
|
||||
revision: GitCommitId | None = None,
|
||||
authorization: str | None = Header(default=None),
|
||||
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
|
||||
) -> ReadEnvelope:
|
||||
principal = authenticate(authorization, session_cookie)
|
||||
try:
|
||||
envelope = node_reader.get_node(
|
||||
project_id, node_id, principal=principal, revision=revision
|
||||
)
|
||||
except ReadError as error:
|
||||
raise HTTPException(status_code=error.status_code, detail=str(error)) from error
|
||||
return envelope
|
||||
|
||||
source_reader = CanonicalSourceReader(node_reader.repository)
|
||||
http_operations = {
|
||||
"diagnostics",
|
||||
"browser_session_create",
|
||||
"browser_session_revoke",
|
||||
"browser_session_restore",
|
||||
"get_node",
|
||||
"node_history",
|
||||
"get_source",
|
||||
}
|
||||
if credential_administration is not None:
|
||||
http_operations.update(
|
||||
{
|
||||
"admin_list_credentials",
|
||||
"admin_create_credential",
|
||||
"admin_revoke_credential",
|
||||
"admin_update_credential",
|
||||
"admin_list_events",
|
||||
}
|
||||
)
|
||||
if account_identity is not None:
|
||||
http_operations.update(
|
||||
{
|
||||
"account_login",
|
||||
"account_current",
|
||||
"account_change_password",
|
||||
"account_list_agent_credentials",
|
||||
"account_create_agent_credential",
|
||||
"account_revoke_agent_credential",
|
||||
"admin_list_accounts",
|
||||
"admin_create_account",
|
||||
"admin_update_account",
|
||||
"admin_disable_account",
|
||||
}
|
||||
)
|
||||
if project_registry is not None:
|
||||
http_operations.update(
|
||||
{
|
||||
"admin_list_projects",
|
||||
"project_create",
|
||||
"project_update_content",
|
||||
"project_archive",
|
||||
}
|
||||
)
|
||||
|
||||
if project_registry is not None:
|
||||
|
||||
@app.get(
|
||||
"/v1/admin/projects",
|
||||
response_model=list[ProjectRecord],
|
||||
tags=["control"],
|
||||
operation_id="admin_list_projects",
|
||||
)
|
||||
def admin_list_projects(
|
||||
response: Response,
|
||||
authorization: str | None = Header(default=None),
|
||||
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
|
||||
) -> list[ProjectRecord]:
|
||||
response.headers["Cache-Control"] = "private, no-store"
|
||||
principal = authenticate(authorization, session_cookie)
|
||||
require_admin(principal)
|
||||
if project_registry is None:
|
||||
raise HTTPException(status_code=503, detail="Project registry is unavailable")
|
||||
try:
|
||||
records = list(project_registry.records())
|
||||
return records
|
||||
except RegistryError as error:
|
||||
raise HTTPException(status_code=503, detail=str(error)) from error
|
||||
|
||||
@app.post(
|
||||
"/v1/admin/projects",
|
||||
response_model=ProjectRecord,
|
||||
status_code=201,
|
||||
tags=["control"],
|
||||
operation_id="project_create",
|
||||
)
|
||||
def project_create(
|
||||
request: Request,
|
||||
body: ProjectCreateBody,
|
||||
response: Response,
|
||||
authorization: str | None = Header(default=None),
|
||||
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
|
||||
csrf_token: str | None = Header(default=None, alias="X-GEWU-CSRF"),
|
||||
) -> ProjectRecord:
|
||||
require_same_origin(request)
|
||||
if session_cookie is not None and request.headers.get("origin") is None:
|
||||
raise HTTPException(
|
||||
status_code=403, detail="Browser session requires an Origin"
|
||||
)
|
||||
principal = authenticate(
|
||||
authorization, session_cookie, csrf_token, require_csrf=True
|
||||
)
|
||||
require_admin(principal)
|
||||
if project_registry is None:
|
||||
raise HTTPException(status_code=503, detail="Project registry is unavailable")
|
||||
try:
|
||||
result = project_registry.create(body.project, body.policy)
|
||||
except RegistryConflict as error:
|
||||
raise HTTPException(status_code=409, detail=str(error)) from error
|
||||
except portalocker.exceptions.LockException as error:
|
||||
raise HTTPException(
|
||||
status_code=409, detail="Project registry is busy"
|
||||
) from error
|
||||
except ValueError as error:
|
||||
raise HTTPException(status_code=422, detail=str(error)) from error
|
||||
except RegistryError as error:
|
||||
raise HTTPException(status_code=503, detail=str(error)) from error
|
||||
response.headers["Cache-Control"] = "no-store"
|
||||
return result
|
||||
|
||||
@app.patch(
|
||||
"/v1/admin/projects/{project_id}",
|
||||
response_model=ProjectRecord,
|
||||
tags=["control"],
|
||||
operation_id="project_update_content",
|
||||
)
|
||||
def project_update_content(
|
||||
project_id: RecordId,
|
||||
body: ProjectContentUpdateBody,
|
||||
request: Request,
|
||||
response: Response,
|
||||
authorization: str | None = Header(default=None),
|
||||
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
|
||||
csrf_token: str | None = Header(default=None, alias="X-GEWU-CSRF"),
|
||||
) -> ProjectRecord:
|
||||
require_same_origin(request)
|
||||
if session_cookie is not None and request.headers.get("origin") is None:
|
||||
raise HTTPException(
|
||||
status_code=403, detail="Browser session requires an Origin"
|
||||
)
|
||||
principal = authenticate(
|
||||
authorization, session_cookie, csrf_token, require_csrf=True
|
||||
)
|
||||
require_admin(principal)
|
||||
if project_registry is None:
|
||||
raise HTTPException(status_code=503, detail="Project registry is unavailable")
|
||||
try:
|
||||
existing = next(
|
||||
(
|
||||
item
|
||||
for item in project_registry.records()
|
||||
if item.project_id == project_id
|
||||
),
|
||||
None,
|
||||
)
|
||||
if existing is None:
|
||||
raise RegistryNotFound("Project is not registered")
|
||||
result = project_registry.update_content(
|
||||
project_id,
|
||||
description=body.description,
|
||||
north_star=body.north_star,
|
||||
key_questions=tuple(body.key_questions),
|
||||
)
|
||||
except RegistryNotFound as error:
|
||||
raise HTTPException(status_code=404, detail=str(error)) from error
|
||||
except portalocker.exceptions.LockException as error:
|
||||
raise HTTPException(
|
||||
status_code=409, detail="Project registry is busy"
|
||||
) from error
|
||||
except ValueError as error:
|
||||
raise HTTPException(status_code=422, detail=str(error)) from error
|
||||
except RegistryError as error:
|
||||
raise HTTPException(status_code=503, detail=str(error)) from error
|
||||
response.headers["Cache-Control"] = "no-store"
|
||||
return result
|
||||
|
||||
@app.post(
|
||||
"/v1/admin/projects/{project_id}/archive",
|
||||
response_model=ProjectRecord,
|
||||
tags=["control"],
|
||||
operation_id="project_archive",
|
||||
)
|
||||
def project_archive(
|
||||
project_id: RecordId,
|
||||
request: Request,
|
||||
response: Response,
|
||||
authorization: str | None = Header(default=None),
|
||||
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
|
||||
csrf_token: str | None = Header(default=None, alias="X-GEWU-CSRF"),
|
||||
) -> ProjectRecord:
|
||||
require_same_origin(request)
|
||||
if session_cookie is not None and request.headers.get("origin") is None:
|
||||
raise HTTPException(
|
||||
status_code=403, detail="Browser session requires an Origin"
|
||||
)
|
||||
principal = authenticate(
|
||||
authorization, session_cookie, csrf_token, require_csrf=True
|
||||
)
|
||||
require_admin(principal)
|
||||
if project_registry is None:
|
||||
raise HTTPException(status_code=503, detail="Project registry is unavailable")
|
||||
try:
|
||||
result = project_registry.archive(project_id)
|
||||
except RegistryNotFound as error:
|
||||
raise HTTPException(status_code=404, detail=str(error)) from error
|
||||
except portalocker.exceptions.LockException as error:
|
||||
raise HTTPException(
|
||||
status_code=409, detail="Project registry is busy"
|
||||
) from error
|
||||
except ValueError as error:
|
||||
raise HTTPException(status_code=422, detail=str(error)) from error
|
||||
except RegistryError as error:
|
||||
raise HTTPException(status_code=503, detail=str(error)) from error
|
||||
response.headers["Cache-Control"] = "no-store"
|
||||
return result
|
||||
|
||||
@app.get(
|
||||
"/v1/diagnostics",
|
||||
response_model=ServiceDiagnostics,
|
||||
tags=["diagnostics"],
|
||||
operation_id="diagnostics",
|
||||
)
|
||||
def diagnostics(
|
||||
response: Response,
|
||||
projects: Annotated[list[str] | None, Query()] = None,
|
||||
include_shared: bool = False,
|
||||
authorization: str | None = Header(default=None),
|
||||
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
|
||||
) -> ServiceDiagnostics:
|
||||
response.headers["Cache-Control"] = "private, no-store"
|
||||
response.headers["X-Content-Type-Options"] = "nosniff"
|
||||
principal = authenticate(authorization, session_cookie)
|
||||
authorized_projects = (
|
||||
search_service.list_projects(principal=principal)
|
||||
if search_service is not None
|
||||
else tuple(
|
||||
sorted(
|
||||
project
|
||||
for project in principal.project_ids
|
||||
if principal.can_read_project(project)
|
||||
)
|
||||
)
|
||||
)
|
||||
index_status = (
|
||||
search_service.diagnostic_index_status()
|
||||
if search_service is not None
|
||||
else (node_reader.repository.head(), None, "not_configured", None)
|
||||
)
|
||||
try:
|
||||
return build_service_diagnostics(
|
||||
principal=principal,
|
||||
transport="http",
|
||||
authorized_projects=tuple(authorized_projects),
|
||||
available_operations=frozenset(http_operations),
|
||||
index_status=index_status,
|
||||
projects=tuple(projects) if projects is not None else None,
|
||||
include_shared=include_shared,
|
||||
)
|
||||
except PermissionError as error:
|
||||
raise HTTPException(status_code=403, detail=str(error)) from error
|
||||
except ValueError as error:
|
||||
raise HTTPException(status_code=422, detail=str(error)) from error
|
||||
|
||||
@app.get(
|
||||
"/v1/projects/{project_id}/nodes/{node_id}/history",
|
||||
response_model=ReadEnvelope,
|
||||
tags=["knowledge"],
|
||||
operation_id="node_history",
|
||||
)
|
||||
def get_node_history(
|
||||
project_id: RecordId,
|
||||
node_id: RecordId,
|
||||
limit: int = 20,
|
||||
authorization: str | None = Header(default=None),
|
||||
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
|
||||
) -> ReadEnvelope:
|
||||
principal = authenticate(authorization, session_cookie)
|
||||
try:
|
||||
return node_reader.get_history(
|
||||
project_id, node_id, principal=principal, limit=limit
|
||||
)
|
||||
except ReadError as error:
|
||||
raise HTTPException(status_code=error.status_code, detail=str(error)) from error
|
||||
|
||||
@app.get(
|
||||
"/v1/projects/{project_id}/sources/{revision_id}",
|
||||
response_model=ReadEnvelope,
|
||||
tags=["knowledge"],
|
||||
operation_id="get_source",
|
||||
)
|
||||
def get_source(
|
||||
project_id: RecordId,
|
||||
revision_id: RecordId,
|
||||
authorization: str | None = Header(default=None),
|
||||
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
|
||||
) -> ReadEnvelope:
|
||||
principal = authenticate(authorization, session_cookie)
|
||||
try:
|
||||
return source_reader.get_source(project_id, revision_id, principal=principal)
|
||||
except ReadError as error:
|
||||
raise HTTPException(status_code=error.status_code, detail=str(error)) from error
|
||||
|
||||
if artifact_store is not None:
|
||||
http_operations.update({"get_source_artifact", "get_source_location"})
|
||||
|
||||
@app.get(
|
||||
"/v1/projects/{project_id}/sources/{revision_id}/artifact",
|
||||
tags=["knowledge"],
|
||||
operation_id="get_source_artifact",
|
||||
)
|
||||
def get_source_artifact(
|
||||
project_id: RecordId,
|
||||
revision_id: RecordId,
|
||||
authorization: str | None = Header(default=None),
|
||||
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
|
||||
) -> StreamingResponse:
|
||||
principal = authenticate(authorization, session_cookie)
|
||||
try:
|
||||
source, _ = source_reader.get_source_record(
|
||||
project_id, revision_id, principal=principal
|
||||
)
|
||||
if source.archive_status != "archived" or source.artifact_sha256 is None:
|
||||
raise HTTPException(
|
||||
status_code=404, detail="This source revision has no archived artifact"
|
||||
)
|
||||
media_type = artifact_store.artifact_media_type_for_project(
|
||||
source.artifact_sha256, project_id=project_id
|
||||
)
|
||||
if media_type != source.media_type:
|
||||
raise HTTPException(
|
||||
status_code=503, detail="Archived source metadata is inconsistent"
|
||||
)
|
||||
stream = artifact_store.open_artifact(
|
||||
source.artifact_sha256, project_id=project_id
|
||||
)
|
||||
if os.fstat(stream.fileno()).st_size != source.byte_length:
|
||||
stream.close()
|
||||
raise HTTPException(
|
||||
status_code=503, detail="Archived source length does not match metadata"
|
||||
)
|
||||
except ReadError as error:
|
||||
raise HTTPException(status_code=error.status_code, detail=str(error)) from error
|
||||
except UploadError as error:
|
||||
raise HTTPException(
|
||||
status_code=404, detail="Archived source is unavailable"
|
||||
) from error
|
||||
|
||||
def chunks() -> Iterator[bytes]:
|
||||
try:
|
||||
while block := stream.read(1024 * 1024):
|
||||
yield block
|
||||
finally:
|
||||
stream.close()
|
||||
|
||||
return StreamingResponse(
|
||||
chunks(),
|
||||
media_type=media_type,
|
||||
headers={
|
||||
"Cache-Control": "private, no-store",
|
||||
"Content-Length": str(source.byte_length),
|
||||
"ETag": f'"{source.artifact_sha256}"',
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
"Content-Security-Policy": "default-src 'none'; sandbox",
|
||||
"Content-Disposition": (
|
||||
'attachment; filename="source.pdf"'
|
||||
if media_type == "application/pdf"
|
||||
else 'attachment; filename="source.txt"'
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
@app.get(
|
||||
"/v1/projects/{project_id}/sources/{revision_id}/extractions/{extraction_id}/blocks/{block_id}",
|
||||
tags=["knowledge"],
|
||||
operation_id="get_source_location",
|
||||
)
|
||||
def get_source_location(
|
||||
project_id: RecordId,
|
||||
revision_id: RecordId,
|
||||
extraction_id: str,
|
||||
block_id: RecordId,
|
||||
response: Response,
|
||||
offset: int = Query(ge=0),
|
||||
length: int = Query(ge=1, le=12_000),
|
||||
authorization: str | None = Header(default=None),
|
||||
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
|
||||
) -> dict[str, object]:
|
||||
response.headers["Cache-Control"] = "private, no-store"
|
||||
response.headers["X-Content-Type-Options"] = "nosniff"
|
||||
principal = authenticate(authorization, session_cookie)
|
||||
try:
|
||||
source, _ = source_reader.get_source_record(
|
||||
project_id, revision_id, principal=principal
|
||||
)
|
||||
except ReadError as error:
|
||||
raise HTTPException(status_code=error.status_code, detail=str(error)) from error
|
||||
extraction = artifact_store.get_extraction(extraction_id, project_id=project_id)
|
||||
if extraction is None or extraction.source_revision_id != source.revision_id:
|
||||
raise HTTPException(status_code=404, detail="Source location is unavailable")
|
||||
block = next(
|
||||
(item for item in extraction.blocks if item.block_id == block_id), None
|
||||
)
|
||||
if block is None or offset >= len(block.normalized_text):
|
||||
raise HTTPException(status_code=404, detail="Source location is unavailable")
|
||||
end = min(offset + length, len(block.normalized_text))
|
||||
spans = block.normalization_map[offset:end]
|
||||
original_start = min(start for start, _ in spans)
|
||||
original_end = max(stop for _, stop in spans)
|
||||
return {
|
||||
"source_revision_id": revision_id,
|
||||
"extraction_id": extraction.extraction_id,
|
||||
"parser_name": extraction.parser_name,
|
||||
"parser_version": extraction.parser_version,
|
||||
"block_id": block_id,
|
||||
"offset": offset,
|
||||
"normalized_text": block.normalized_text[offset:end],
|
||||
"original_text": block.original_text[original_start:original_end],
|
||||
"normalization_map": spans,
|
||||
}
|
||||
|
||||
if search_service is not None:
|
||||
http_operations.update({"list_projects", "search", "neighbors", "list_sources"})
|
||||
|
||||
@app.get("/v1/projects", tags=["knowledge"], operation_id="list_projects")
|
||||
def list_projects(
|
||||
authorization: str | None = Header(default=None),
|
||||
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
|
||||
) -> list[str]:
|
||||
principal = authenticate(authorization, session_cookie)
|
||||
return list(search_service.list_projects(principal=principal))
|
||||
|
||||
@app.post(
|
||||
"/v1/search",
|
||||
response_model=ReadEnvelope,
|
||||
tags=["knowledge"],
|
||||
operation_id="search",
|
||||
)
|
||||
def search(
|
||||
request: SearchRequestBody,
|
||||
authorization: str | None = Header(default=None),
|
||||
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
|
||||
csrf_token: str | None = Header(default=None, alias="X-GEWU-CSRF"),
|
||||
) -> ReadEnvelope:
|
||||
principal = authenticate(
|
||||
authorization, session_cookie, csrf_token, require_csrf=True
|
||||
)
|
||||
try:
|
||||
return search_service.search(request.to_contract(), principal=principal)
|
||||
except SearchError as error:
|
||||
raise HTTPException(status_code=error.status_code, detail=str(error)) from error
|
||||
|
||||
@app.post(
|
||||
"/v1/neighbors",
|
||||
response_model=ReadEnvelope,
|
||||
tags=["knowledge"],
|
||||
operation_id="neighbors",
|
||||
)
|
||||
def neighbors(
|
||||
request: NeighborRequestBody,
|
||||
authorization: str | None = Header(default=None),
|
||||
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
|
||||
csrf_token: str | None = Header(default=None, alias="X-GEWU-CSRF"),
|
||||
) -> ReadEnvelope:
|
||||
principal = authenticate(
|
||||
authorization, session_cookie, csrf_token, require_csrf=True
|
||||
)
|
||||
try:
|
||||
return search_service.neighbors(request.to_contract(), principal=principal)
|
||||
except SearchError as error:
|
||||
raise HTTPException(status_code=error.status_code, detail=str(error)) from error
|
||||
|
||||
@app.post(
|
||||
"/v1/sources",
|
||||
response_model=ReadEnvelope,
|
||||
tags=["knowledge"],
|
||||
operation_id="list_sources",
|
||||
)
|
||||
def list_sources(
|
||||
request: SourceListRequestBody,
|
||||
authorization: str | None = Header(default=None),
|
||||
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
|
||||
csrf_token: str | None = Header(default=None, alias="X-GEWU-CSRF"),
|
||||
) -> ReadEnvelope:
|
||||
principal = authenticate(
|
||||
authorization, session_cookie, csrf_token, require_csrf=True
|
||||
)
|
||||
try:
|
||||
return search_service.list_sources(request.to_contract(), principal=principal)
|
||||
except SearchError as error:
|
||||
raise HTTPException(status_code=error.status_code, detail=str(error)) from error
|
||||
|
||||
mcp_server = create_mcp_server(
|
||||
identity_provider=identity_provider,
|
||||
node_reader=node_reader,
|
||||
source_reader=source_reader,
|
||||
search_service=search_service,
|
||||
artifact_store=artifact_store,
|
||||
extractor=extractor,
|
||||
acceptance_service=acceptance_service,
|
||||
context_provider=context_provider,
|
||||
url_fetcher=url_fetcher or PublicUrlFetcher(frozenset()),
|
||||
)
|
||||
mcp_asgi = mcp_server.streamable_http_app(
|
||||
streamable_http_path="/",
|
||||
stateless_http=True,
|
||||
json_response=True,
|
||||
max_request_body_size=1_048_576,
|
||||
transport_security=TransportSecuritySettings(
|
||||
allowed_hosts=[
|
||||
"localhost",
|
||||
"localhost:*",
|
||||
"127.0.0.1",
|
||||
"127.0.0.1:*",
|
||||
"[::1]",
|
||||
"[::1]:*",
|
||||
*mcp_allowed_hosts,
|
||||
]
|
||||
),
|
||||
)
|
||||
owner_mcp_server = None
|
||||
owner_mcp_asgi = None
|
||||
if (
|
||||
owner_identity_provider is not None
|
||||
and owner_identity_provider.owner_authentication_enabled
|
||||
and owner_service_audience
|
||||
and callable(getattr(identity_provider, "get_owner_challenge", None))
|
||||
and callable(getattr(identity_provider, "decide_owner_challenge", None))
|
||||
):
|
||||
owner_mcp_server = create_owner_mcp_server(
|
||||
identities=owner_identity_provider,
|
||||
reviews=cast(OwnerReviewProvider, identity_provider),
|
||||
audience=owner_service_audience,
|
||||
)
|
||||
owner_mcp_asgi = owner_mcp_server.streamable_http_app(
|
||||
streamable_http_path="/",
|
||||
stateless_http=True,
|
||||
json_response=True,
|
||||
max_request_body_size=1_048_576,
|
||||
transport_security=TransportSecuritySettings(
|
||||
allowed_hosts=[
|
||||
"localhost",
|
||||
"localhost:*",
|
||||
"127.0.0.1",
|
||||
"127.0.0.1:*",
|
||||
"[::1]",
|
||||
"[::1]:*",
|
||||
*mcp_allowed_hosts,
|
||||
]
|
||||
),
|
||||
)
|
||||
original_lifespan = app.router.lifespan_context
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(application: FastAPI) -> AsyncIterator[None]:
|
||||
async with AsyncExitStack() as stack:
|
||||
await stack.enter_async_context(mcp_server.session_manager.run())
|
||||
if owner_mcp_server is not None:
|
||||
await stack.enter_async_context(owner_mcp_server.session_manager.run())
|
||||
async with original_lifespan(application):
|
||||
yield
|
||||
|
||||
app.router.lifespan_context = lifespan
|
||||
app.router.routes.append(Mount("/mcp", app=mcp_asgi))
|
||||
if owner_mcp_asgi is not None:
|
||||
app.router.routes.append(Mount("/owner-mcp", app=owner_mcp_asgi))
|
||||
|
||||
return app
|
||||
|
||||
|
||||
def create_local_app() -> FastAPI:
|
||||
"""Compose the local persistent service from one private data root."""
|
||||
|
||||
data_root = Path(os.environ.get("GEWU_DATA_ROOT", "./.data/gewu")).expanduser().absolute()
|
||||
reject_symlink_components(data_root)
|
||||
data_root.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
if data_root.is_symlink() or not data_root.is_dir():
|
||||
raise RuntimeError("GEWU data root must be a real directory")
|
||||
data_root.chmod(0o700)
|
||||
repository = GitCanonicalRepository(data_root / "knowledge.git")
|
||||
workflow_root = data_root / "workflow"
|
||||
authority_lock = workflow_root / "locks" / "canonical-writer.lock"
|
||||
identities = SQLiteIdentityStore(
|
||||
data_root / "identity.sqlite3",
|
||||
authority_lock_path=authority_lock,
|
||||
owner_encryption_key=os.environ.get("GEWU_OWNER_ENCRYPTION_KEY", "").encode("ascii")
|
||||
or None,
|
||||
)
|
||||
artifacts = LocalArtifactStore(data_root)
|
||||
allowed_fetch_hosts = {
|
||||
host.strip()
|
||||
for host in os.environ.get("GEWU_FETCH_ALLOWED_HOSTS", "").split(",")
|
||||
if host.strip()
|
||||
}
|
||||
url_fetcher = PublicUrlFetcher(allowed_fetch_hosts) if allowed_fetch_hosts else None
|
||||
registry_file = data_root / "control" / "registry.json"
|
||||
context_provider = LocalContextProvider(repository, registry_file)
|
||||
project_registry = LocalProjectRegistry(
|
||||
registry_file,
|
||||
repository=repository,
|
||||
index_directory=data_root / "publishing-index",
|
||||
authority_lock_path=authority_lock,
|
||||
)
|
||||
search_path = data_root / "search.sqlite3"
|
||||
search_index: SearchIndex
|
||||
model_path = os.environ.get("GEWU_EMBEDDING_MODEL_PATH")
|
||||
model_digest = os.environ.get("GEWU_EMBEDDING_MODEL_SHA256")
|
||||
if model_path or model_digest:
|
||||
if not model_path or not model_digest:
|
||||
raise RuntimeError(
|
||||
"GEWU_EMBEDDING_MODEL_PATH and GEWU_EMBEDDING_MODEL_SHA256 "
|
||||
"must be configured together"
|
||||
)
|
||||
embeddings = LocalSentenceTransformer(
|
||||
Path(model_path),
|
||||
model_id=os.environ.get(
|
||||
"GEWU_EMBEDDING_MODEL_ID",
|
||||
"intfloat/multilingual-e5-small@614241f622f53c4eeff9890bdc4f31cfecc418b3",
|
||||
),
|
||||
expected_sha256=model_digest,
|
||||
dimensions=int(os.environ.get("GEWU_EMBEDDING_DIMENSIONS", "384")),
|
||||
)
|
||||
vectors = SQLiteVectorIndex(search_path)
|
||||
lexical_index = SQLiteSearchIndex(search_path, repository)
|
||||
hybrid_index = HybridSearchIndex(
|
||||
lexical=lexical_index,
|
||||
vectors=vectors,
|
||||
embeddings=embeddings,
|
||||
)
|
||||
lexical_index.configure_vector_generation(
|
||||
extension_loader=vectors.load_extension,
|
||||
builder=hybrid_index.build_vector_generation,
|
||||
is_current=hybrid_index.vector_generation_is_current,
|
||||
)
|
||||
search_index = hybrid_index
|
||||
else:
|
||||
search_index = SQLiteSearchIndex(search_path, repository)
|
||||
search_service = CanonicalSearchService(
|
||||
index=search_index,
|
||||
project_ids_provider=project_registry,
|
||||
canonical_head=repository.head,
|
||||
default_includes_provider=project_registry.default_includes,
|
||||
)
|
||||
acceptance = AcceptanceService(
|
||||
repository=repository,
|
||||
artifact_store=artifacts,
|
||||
workflow_root=workflow_root,
|
||||
context_provider=context_provider,
|
||||
actor_is_current=lambda actor: identities.is_current(
|
||||
actor.principal_id,
|
||||
actor.credential_id,
|
||||
roles=actor.roles,
|
||||
project_ids=actor.project_ids,
|
||||
),
|
||||
owner_challenge_issuer=identities.issue_owner_challenge,
|
||||
owner_decision_lookup=identities.authorized_owner_challenges,
|
||||
)
|
||||
return create_app(
|
||||
identity_provider=identities,
|
||||
credential_administration=identities,
|
||||
account_identity=identities,
|
||||
owner_identity_provider=identities,
|
||||
owner_service_audience=os.environ.get("GEWU_OWNER_SERVICE_AUDIENCE"),
|
||||
node_reader=CanonicalNodeReader(repository),
|
||||
artifact_store=artifacts,
|
||||
extractor=LocalSourceExtractor(),
|
||||
acceptance_service=acceptance,
|
||||
context_provider=context_provider,
|
||||
search_service=search_service,
|
||||
project_registry=project_registry,
|
||||
url_fetcher=url_fetcher,
|
||||
)
|
||||
Reference in new issue
Block a user