956 lines
42 KiB
Python
956 lines
42 KiB
Python
"""FastAPI composition root, health check and authenticated canonical reads."""
|
|
|
|
import os
|
|
from collections.abc import AsyncIterator, Callable, Iterator
|
|
from contextlib import AsyncExitStack, asynccontextmanager
|
|
from pathlib import Path
|
|
from typing import Annotated, cast
|
|
|
|
import portalocker
|
|
from fastapi import Cookie, FastAPI, Header, HTTPException, Query, Request, Response
|
|
from mcp.server.transport_security import TransportSecuritySettings
|
|
from starlette.responses import StreamingResponse
|
|
from starlette.routing import Mount
|
|
|
|
from gewu.adapters.embeddings import LocalSentenceTransformer
|
|
from gewu.adapters.git_repository import GitCanonicalRepository
|
|
from gewu.adapters.source_extractor import LocalSourceExtractor, SourceExtractor
|
|
from gewu.adapters.sqlite_identity import SQLiteIdentityStore
|
|
from gewu.adapters.sqlite_search import SQLiteSearchIndex
|
|
from gewu.adapters.sqlite_vectors import SQLiteVectorIndex
|
|
from gewu.adapters.url_fetch import PublicUrlFetcher
|
|
from gewu.application.acceptance import AcceptanceService
|
|
from gewu.application.accounts import AccountIdentity
|
|
from gewu.application.admin import CredentialAdministration
|
|
from gewu.application.context import ValidationContext
|
|
from gewu.application.diagnostics import build_service_diagnostics
|
|
from gewu.application.identity import IdentityProvider, OwnerIdentityProvider, Principal
|
|
from gewu.application.paths import reject_symlink_components
|
|
from gewu.application.ports import ProjectRegistry
|
|
from gewu.application.reading import CanonicalNodeReader, CanonicalSourceReader, ReadError
|
|
from gewu.application.registry import (
|
|
LocalContextProvider,
|
|
LocalProjectRegistry,
|
|
RegistryConflict,
|
|
RegistryError,
|
|
RegistryNotFound,
|
|
)
|
|
from gewu.application.search import (
|
|
CanonicalSearchService,
|
|
HybridSearchIndex,
|
|
SearchError,
|
|
SearchIndex,
|
|
)
|
|
from gewu.application.uploads import LocalArtifactStore, UploadError
|
|
from gewu.domain.contracts import (
|
|
GitCommitId,
|
|
ProjectRecord,
|
|
ReadEnvelope,
|
|
RecordId,
|
|
ServiceDiagnostics,
|
|
)
|
|
from gewu.transports.accounts_http import register_account_routes
|
|
from gewu.transports.admin_http import register_admin_routes
|
|
from gewu.transports.http_middleware import ProjectRegistryBodyLimitMiddleware
|
|
from gewu.transports.http_models import (
|
|
NeighborRequestBody,
|
|
OwnerSessionRequestBody,
|
|
ProjectContentUpdateBody,
|
|
ProjectCreateBody,
|
|
SearchRequestBody,
|
|
SourceListRequestBody,
|
|
)
|
|
from gewu.transports.mcp_server import create_mcp_server
|
|
from gewu.transports.owner_mcp_server import OwnerReviewProvider, create_owner_mcp_server
|
|
from gewu.version import SERVICE_VERSION
|
|
|
|
|
|
def create_app(
|
|
*,
|
|
identity_provider: IdentityProvider | None = None,
|
|
owner_identity_provider: OwnerIdentityProvider | None = None,
|
|
owner_service_audience: str | None = None,
|
|
node_reader: CanonicalNodeReader | None = None,
|
|
artifact_store: LocalArtifactStore | None = None,
|
|
extractor: SourceExtractor | None = None,
|
|
acceptance_service: AcceptanceService | None = None,
|
|
context_provider: Callable[[], ValidationContext] | None = None,
|
|
search_service: CanonicalSearchService | None = None,
|
|
project_registry: ProjectRegistry | None = None,
|
|
credential_administration: CredentialAdministration | None = None,
|
|
account_identity: AccountIdentity | None = None,
|
|
url_fetcher: PublicUrlFetcher | None = None,
|
|
) -> FastAPI:
|
|
if (identity_provider is None) != (node_reader is None):
|
|
raise ValueError("Identity provider and canonical reader must be configured together")
|
|
app = FastAPI(title="GEWU", version=SERVICE_VERSION)
|
|
mcp_allowed_hosts = [
|
|
host.strip()
|
|
for host in os.environ.get("GEWU_MCP_ALLOWED_HOSTS", "").split(",")
|
|
if host.strip()
|
|
]
|
|
if (
|
|
project_registry is not None
|
|
or owner_identity_provider is not None
|
|
or credential_administration is not None
|
|
or account_identity is not None
|
|
):
|
|
app.add_middleware(ProjectRegistryBodyLimitMiddleware)
|
|
|
|
@app.get("/health", tags=["health"], operation_id="health")
|
|
def health() -> dict[str, str]:
|
|
return {"status": "ok", "service": "gewu"}
|
|
|
|
if identity_provider is not None and node_reader is not None:
|
|
|
|
def authenticate(
|
|
authorization: str | None,
|
|
session_cookie: str | None = None,
|
|
csrf_token: str | None = None,
|
|
*,
|
|
require_csrf: bool = False,
|
|
) -> Principal:
|
|
scheme, _, token = (authorization or "").partition(" ")
|
|
if scheme.lower() == "bearer" and token:
|
|
principal = identity_provider.authenticate(token)
|
|
if principal is None:
|
|
raise HTTPException(status_code=401, detail="Credential is invalid or revoked")
|
|
return principal
|
|
if session_cookie:
|
|
principal = identity_provider.authenticate_browser_session(session_cookie)
|
|
if principal is None:
|
|
raise HTTPException(
|
|
status_code=401, detail="Browser session is invalid or expired"
|
|
)
|
|
if require_csrf and not identity_provider.browser_csrf_valid(
|
|
session_cookie, csrf_token or ""
|
|
):
|
|
raise HTTPException(status_code=403, detail="Browser session CSRF check failed")
|
|
return principal
|
|
raise HTTPException(
|
|
status_code=401, detail="Bearer credential or browser session required"
|
|
)
|
|
|
|
def require_same_origin(request: Request) -> None:
|
|
origin = request.headers.get("origin")
|
|
expected = f"{request.url.scheme}://{request.headers.get('host', '')}"
|
|
if origin is not None and origin.rstrip("/") != expected.rstrip("/"):
|
|
raise HTTPException(
|
|
status_code=403, detail="Browser session requires a same-origin request"
|
|
)
|
|
|
|
def require_admin(principal: Principal) -> None:
|
|
if "admin" not in principal.roles or "agent" in principal.roles:
|
|
raise HTTPException(status_code=403, detail="Administrator role is required")
|
|
|
|
if credential_administration is not None:
|
|
register_admin_routes(app, credential_administration, authenticate, require_same_origin)
|
|
|
|
if account_identity is not None:
|
|
register_account_routes(app, account_identity, authenticate, require_same_origin)
|
|
|
|
if owner_identity_provider is not None:
|
|
|
|
@app.post(
|
|
"/auth/owner-session",
|
|
tags=["identity"],
|
|
operation_id="owner_session_create",
|
|
)
|
|
def create_owner_session(
|
|
body: OwnerSessionRequestBody,
|
|
request: Request,
|
|
response: Response,
|
|
) -> dict[str, str]:
|
|
require_same_origin(request)
|
|
response.headers["Cache-Control"] = "no-store"
|
|
host = request.url.hostname or ""
|
|
if request.url.scheme != "https" and host not in {
|
|
"localhost",
|
|
"127.0.0.1",
|
|
"::1",
|
|
}:
|
|
raise HTTPException(
|
|
status_code=403, detail="Owner authentication requires HTTPS"
|
|
)
|
|
audience = owner_service_audience
|
|
if not audience or not owner_identity_provider.owner_authentication_enabled:
|
|
raise HTTPException(status_code=503, detail="Owner authentication is disabled")
|
|
owner_session = owner_identity_provider.authenticate_owner(
|
|
principal_id=body.principal_id,
|
|
password=body.password,
|
|
totp_code=body.totp_code,
|
|
challenge_id=body.challenge_id,
|
|
audience=audience,
|
|
)
|
|
if owner_session is None:
|
|
raise HTTPException(status_code=401, detail="Owner authentication failed")
|
|
return {
|
|
"session_token": owner_session.session_token,
|
|
"challenge_id": owner_session.challenge_id,
|
|
"expires_at": owner_session.expires_at.isoformat(),
|
|
}
|
|
|
|
@app.post("/v1/session", tags=["identity"], operation_id="browser_session_create")
|
|
def create_browser_session(
|
|
request: Request,
|
|
response: Response,
|
|
authorization: str | None = Header(default=None),
|
|
) -> dict[str, str]:
|
|
require_same_origin(request)
|
|
response.headers["Cache-Control"] = "no-store"
|
|
scheme, _, bearer_token = (authorization or "").partition(" ")
|
|
if scheme.lower() != "bearer" or not bearer_token:
|
|
raise HTTPException(status_code=401, detail="Bearer credential required")
|
|
principal = identity_provider.authenticate(bearer_token)
|
|
if principal is None:
|
|
raise HTTPException(status_code=401, detail="Credential is invalid or revoked")
|
|
if "agent" in principal.roles or not principal.roles.intersection(
|
|
{"admin", "editor", "viewer"}
|
|
):
|
|
raise HTTPException(
|
|
status_code=403, detail="Browser sessions require a human identity"
|
|
)
|
|
try:
|
|
session = identity_provider.create_browser_session(bearer_token)
|
|
except PermissionError as error:
|
|
raise HTTPException(status_code=403, detail=str(error)) from error
|
|
response.set_cookie(
|
|
key="gewu_session",
|
|
value=session.session_token,
|
|
httponly=True,
|
|
secure=request.url.scheme == "https",
|
|
samesite="strict",
|
|
path="/v1",
|
|
)
|
|
return {"csrf_token": session.csrf_token, "expires_at": session.expires_at.isoformat()}
|
|
|
|
@app.get("/v1/session", tags=["identity"], operation_id="browser_session_restore")
|
|
def restore_browser_session(
|
|
request: Request,
|
|
response: Response,
|
|
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
|
|
restore_header: str | None = Header(default=None, alias="X-GEWU-Session"),
|
|
) -> dict[str, str | bool]:
|
|
require_same_origin(request)
|
|
if restore_header != "restore" or request.headers.get("sec-fetch-site") == "cross-site":
|
|
raise HTTPException(403, "Session restoration requires a same-origin client")
|
|
principal = authenticate(None, session_cookie)
|
|
session = identity_provider.restore_browser_session(session_cookie or "")
|
|
if session is None:
|
|
raise HTTPException(
|
|
401,
|
|
"Browser session is invalid or expired",
|
|
headers={"Cache-Control": "no-store"},
|
|
)
|
|
response.headers["Cache-Control"] = "private, no-store"
|
|
return {
|
|
"csrf_token": session.csrf_token,
|
|
"expires_at": session.expires_at.isoformat(),
|
|
"password_account": principal.credential_id.startswith("account:"),
|
|
}
|
|
|
|
@app.delete("/v1/session", tags=["identity"], operation_id="browser_session_revoke")
|
|
def revoke_browser_session(
|
|
request: Request,
|
|
response: Response,
|
|
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
|
|
csrf_token: str | None = Header(default=None, alias="X-GEWU-CSRF"),
|
|
) -> dict[str, str]:
|
|
require_same_origin(request)
|
|
response.headers["Cache-Control"] = "no-store"
|
|
if session_cookie:
|
|
authenticate(None, session_cookie, csrf_token, require_csrf=True)
|
|
identity_provider.revoke_browser_session(session_cookie)
|
|
response.delete_cookie(
|
|
key="gewu_session",
|
|
path="/v1",
|
|
httponly=True,
|
|
secure=request.url.scheme == "https",
|
|
samesite="strict",
|
|
)
|
|
return {"status": "signed_out"}
|
|
|
|
@app.get(
|
|
"/v1/projects/{project_id}/nodes/{node_id}",
|
|
response_model=ReadEnvelope,
|
|
tags=["knowledge"],
|
|
operation_id="get_node",
|
|
)
|
|
def get_node(
|
|
project_id: RecordId,
|
|
node_id: RecordId,
|
|
revision: GitCommitId | None = None,
|
|
authorization: str | None = Header(default=None),
|
|
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
|
|
) -> ReadEnvelope:
|
|
principal = authenticate(authorization, session_cookie)
|
|
try:
|
|
envelope = node_reader.get_node(
|
|
project_id, node_id, principal=principal, revision=revision
|
|
)
|
|
except ReadError as error:
|
|
raise HTTPException(status_code=error.status_code, detail=str(error)) from error
|
|
return envelope
|
|
|
|
source_reader = CanonicalSourceReader(node_reader.repository)
|
|
http_operations = {
|
|
"diagnostics",
|
|
"browser_session_create",
|
|
"browser_session_revoke",
|
|
"browser_session_restore",
|
|
"get_node",
|
|
"node_history",
|
|
"get_source",
|
|
}
|
|
if credential_administration is not None:
|
|
http_operations.update(
|
|
{
|
|
"admin_list_credentials",
|
|
"admin_create_credential",
|
|
"admin_revoke_credential",
|
|
"admin_update_credential",
|
|
"admin_list_events",
|
|
}
|
|
)
|
|
if account_identity is not None:
|
|
http_operations.update(
|
|
{
|
|
"account_login",
|
|
"account_current",
|
|
"account_change_password",
|
|
"account_list_agent_credentials",
|
|
"account_create_agent_credential",
|
|
"account_revoke_agent_credential",
|
|
"admin_list_accounts",
|
|
"admin_create_account",
|
|
"admin_update_account",
|
|
"admin_disable_account",
|
|
}
|
|
)
|
|
if project_registry is not None:
|
|
http_operations.update(
|
|
{
|
|
"admin_list_projects",
|
|
"project_create",
|
|
"project_update_content",
|
|
"project_archive",
|
|
}
|
|
)
|
|
|
|
if project_registry is not None:
|
|
|
|
@app.get(
|
|
"/v1/admin/projects",
|
|
response_model=list[ProjectRecord],
|
|
tags=["control"],
|
|
operation_id="admin_list_projects",
|
|
)
|
|
def admin_list_projects(
|
|
response: Response,
|
|
authorization: str | None = Header(default=None),
|
|
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
|
|
) -> list[ProjectRecord]:
|
|
response.headers["Cache-Control"] = "private, no-store"
|
|
principal = authenticate(authorization, session_cookie)
|
|
require_admin(principal)
|
|
if project_registry is None:
|
|
raise HTTPException(status_code=503, detail="Project registry is unavailable")
|
|
try:
|
|
records = list(project_registry.records())
|
|
return records
|
|
except RegistryError as error:
|
|
raise HTTPException(status_code=503, detail=str(error)) from error
|
|
|
|
@app.post(
|
|
"/v1/admin/projects",
|
|
response_model=ProjectRecord,
|
|
status_code=201,
|
|
tags=["control"],
|
|
operation_id="project_create",
|
|
)
|
|
def project_create(
|
|
request: Request,
|
|
body: ProjectCreateBody,
|
|
response: Response,
|
|
authorization: str | None = Header(default=None),
|
|
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
|
|
csrf_token: str | None = Header(default=None, alias="X-GEWU-CSRF"),
|
|
) -> ProjectRecord:
|
|
require_same_origin(request)
|
|
if session_cookie is not None and request.headers.get("origin") is None:
|
|
raise HTTPException(
|
|
status_code=403, detail="Browser session requires an Origin"
|
|
)
|
|
principal = authenticate(
|
|
authorization, session_cookie, csrf_token, require_csrf=True
|
|
)
|
|
require_admin(principal)
|
|
if project_registry is None:
|
|
raise HTTPException(status_code=503, detail="Project registry is unavailable")
|
|
try:
|
|
result = project_registry.create(body.project, body.policy)
|
|
except RegistryConflict as error:
|
|
raise HTTPException(status_code=409, detail=str(error)) from error
|
|
except portalocker.exceptions.LockException as error:
|
|
raise HTTPException(
|
|
status_code=409, detail="Project registry is busy"
|
|
) from error
|
|
except ValueError as error:
|
|
raise HTTPException(status_code=422, detail=str(error)) from error
|
|
except RegistryError as error:
|
|
raise HTTPException(status_code=503, detail=str(error)) from error
|
|
response.headers["Cache-Control"] = "no-store"
|
|
return result
|
|
|
|
@app.patch(
|
|
"/v1/admin/projects/{project_id}",
|
|
response_model=ProjectRecord,
|
|
tags=["control"],
|
|
operation_id="project_update_content",
|
|
)
|
|
def project_update_content(
|
|
project_id: RecordId,
|
|
body: ProjectContentUpdateBody,
|
|
request: Request,
|
|
response: Response,
|
|
authorization: str | None = Header(default=None),
|
|
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
|
|
csrf_token: str | None = Header(default=None, alias="X-GEWU-CSRF"),
|
|
) -> ProjectRecord:
|
|
require_same_origin(request)
|
|
if session_cookie is not None and request.headers.get("origin") is None:
|
|
raise HTTPException(
|
|
status_code=403, detail="Browser session requires an Origin"
|
|
)
|
|
principal = authenticate(
|
|
authorization, session_cookie, csrf_token, require_csrf=True
|
|
)
|
|
require_admin(principal)
|
|
if project_registry is None:
|
|
raise HTTPException(status_code=503, detail="Project registry is unavailable")
|
|
try:
|
|
existing = next(
|
|
(
|
|
item
|
|
for item in project_registry.records()
|
|
if item.project_id == project_id
|
|
),
|
|
None,
|
|
)
|
|
if existing is None:
|
|
raise RegistryNotFound("Project is not registered")
|
|
result = project_registry.update_content(
|
|
project_id,
|
|
description=body.description,
|
|
north_star=body.north_star,
|
|
key_questions=tuple(body.key_questions),
|
|
)
|
|
except RegistryNotFound as error:
|
|
raise HTTPException(status_code=404, detail=str(error)) from error
|
|
except portalocker.exceptions.LockException as error:
|
|
raise HTTPException(
|
|
status_code=409, detail="Project registry is busy"
|
|
) from error
|
|
except ValueError as error:
|
|
raise HTTPException(status_code=422, detail=str(error)) from error
|
|
except RegistryError as error:
|
|
raise HTTPException(status_code=503, detail=str(error)) from error
|
|
response.headers["Cache-Control"] = "no-store"
|
|
return result
|
|
|
|
@app.post(
|
|
"/v1/admin/projects/{project_id}/archive",
|
|
response_model=ProjectRecord,
|
|
tags=["control"],
|
|
operation_id="project_archive",
|
|
)
|
|
def project_archive(
|
|
project_id: RecordId,
|
|
request: Request,
|
|
response: Response,
|
|
authorization: str | None = Header(default=None),
|
|
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
|
|
csrf_token: str | None = Header(default=None, alias="X-GEWU-CSRF"),
|
|
) -> ProjectRecord:
|
|
require_same_origin(request)
|
|
if session_cookie is not None and request.headers.get("origin") is None:
|
|
raise HTTPException(
|
|
status_code=403, detail="Browser session requires an Origin"
|
|
)
|
|
principal = authenticate(
|
|
authorization, session_cookie, csrf_token, require_csrf=True
|
|
)
|
|
require_admin(principal)
|
|
if project_registry is None:
|
|
raise HTTPException(status_code=503, detail="Project registry is unavailable")
|
|
try:
|
|
result = project_registry.archive(project_id)
|
|
except RegistryNotFound as error:
|
|
raise HTTPException(status_code=404, detail=str(error)) from error
|
|
except portalocker.exceptions.LockException as error:
|
|
raise HTTPException(
|
|
status_code=409, detail="Project registry is busy"
|
|
) from error
|
|
except ValueError as error:
|
|
raise HTTPException(status_code=422, detail=str(error)) from error
|
|
except RegistryError as error:
|
|
raise HTTPException(status_code=503, detail=str(error)) from error
|
|
response.headers["Cache-Control"] = "no-store"
|
|
return result
|
|
|
|
@app.get(
|
|
"/v1/diagnostics",
|
|
response_model=ServiceDiagnostics,
|
|
tags=["diagnostics"],
|
|
operation_id="diagnostics",
|
|
)
|
|
def diagnostics(
|
|
response: Response,
|
|
projects: Annotated[list[str] | None, Query()] = None,
|
|
include_shared: bool = False,
|
|
authorization: str | None = Header(default=None),
|
|
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
|
|
) -> ServiceDiagnostics:
|
|
response.headers["Cache-Control"] = "private, no-store"
|
|
response.headers["X-Content-Type-Options"] = "nosniff"
|
|
principal = authenticate(authorization, session_cookie)
|
|
authorized_projects = (
|
|
search_service.list_projects(principal=principal)
|
|
if search_service is not None
|
|
else tuple(
|
|
sorted(
|
|
project
|
|
for project in principal.project_ids
|
|
if principal.can_read_project(project)
|
|
)
|
|
)
|
|
)
|
|
index_status = (
|
|
search_service.diagnostic_index_status()
|
|
if search_service is not None
|
|
else (node_reader.repository.head(), None, "not_configured", None)
|
|
)
|
|
try:
|
|
return build_service_diagnostics(
|
|
principal=principal,
|
|
transport="http",
|
|
authorized_projects=tuple(authorized_projects),
|
|
available_operations=frozenset(http_operations),
|
|
index_status=index_status,
|
|
projects=tuple(projects) if projects is not None else None,
|
|
include_shared=include_shared,
|
|
)
|
|
except PermissionError as error:
|
|
raise HTTPException(status_code=403, detail=str(error)) from error
|
|
except ValueError as error:
|
|
raise HTTPException(status_code=422, detail=str(error)) from error
|
|
|
|
@app.get(
|
|
"/v1/projects/{project_id}/nodes/{node_id}/history",
|
|
response_model=ReadEnvelope,
|
|
tags=["knowledge"],
|
|
operation_id="node_history",
|
|
)
|
|
def get_node_history(
|
|
project_id: RecordId,
|
|
node_id: RecordId,
|
|
limit: int = 20,
|
|
authorization: str | None = Header(default=None),
|
|
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
|
|
) -> ReadEnvelope:
|
|
principal = authenticate(authorization, session_cookie)
|
|
try:
|
|
return node_reader.get_history(
|
|
project_id, node_id, principal=principal, limit=limit
|
|
)
|
|
except ReadError as error:
|
|
raise HTTPException(status_code=error.status_code, detail=str(error)) from error
|
|
|
|
@app.get(
|
|
"/v1/projects/{project_id}/sources/{revision_id}",
|
|
response_model=ReadEnvelope,
|
|
tags=["knowledge"],
|
|
operation_id="get_source",
|
|
)
|
|
def get_source(
|
|
project_id: RecordId,
|
|
revision_id: RecordId,
|
|
authorization: str | None = Header(default=None),
|
|
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
|
|
) -> ReadEnvelope:
|
|
principal = authenticate(authorization, session_cookie)
|
|
try:
|
|
return source_reader.get_source(project_id, revision_id, principal=principal)
|
|
except ReadError as error:
|
|
raise HTTPException(status_code=error.status_code, detail=str(error)) from error
|
|
|
|
if artifact_store is not None:
|
|
http_operations.update({"get_source_artifact", "get_source_location"})
|
|
|
|
@app.get(
|
|
"/v1/projects/{project_id}/sources/{revision_id}/artifact",
|
|
tags=["knowledge"],
|
|
operation_id="get_source_artifact",
|
|
)
|
|
def get_source_artifact(
|
|
project_id: RecordId,
|
|
revision_id: RecordId,
|
|
authorization: str | None = Header(default=None),
|
|
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
|
|
) -> StreamingResponse:
|
|
principal = authenticate(authorization, session_cookie)
|
|
try:
|
|
source, _ = source_reader.get_source_record(
|
|
project_id, revision_id, principal=principal
|
|
)
|
|
if source.archive_status != "archived" or source.artifact_sha256 is None:
|
|
raise HTTPException(
|
|
status_code=404, detail="This source revision has no archived artifact"
|
|
)
|
|
media_type = artifact_store.artifact_media_type_for_project(
|
|
source.artifact_sha256, project_id=project_id
|
|
)
|
|
if media_type != source.media_type:
|
|
raise HTTPException(
|
|
status_code=503, detail="Archived source metadata is inconsistent"
|
|
)
|
|
stream = artifact_store.open_artifact(
|
|
source.artifact_sha256, project_id=project_id
|
|
)
|
|
if os.fstat(stream.fileno()).st_size != source.byte_length:
|
|
stream.close()
|
|
raise HTTPException(
|
|
status_code=503, detail="Archived source length does not match metadata"
|
|
)
|
|
except ReadError as error:
|
|
raise HTTPException(status_code=error.status_code, detail=str(error)) from error
|
|
except UploadError as error:
|
|
raise HTTPException(
|
|
status_code=404, detail="Archived source is unavailable"
|
|
) from error
|
|
|
|
def chunks() -> Iterator[bytes]:
|
|
try:
|
|
while block := stream.read(1024 * 1024):
|
|
yield block
|
|
finally:
|
|
stream.close()
|
|
|
|
return StreamingResponse(
|
|
chunks(),
|
|
media_type=media_type,
|
|
headers={
|
|
"Cache-Control": "private, no-store",
|
|
"Content-Length": str(source.byte_length),
|
|
"ETag": f'"{source.artifact_sha256}"',
|
|
"X-Content-Type-Options": "nosniff",
|
|
"Content-Security-Policy": "default-src 'none'; sandbox",
|
|
"Content-Disposition": (
|
|
'attachment; filename="source.pdf"'
|
|
if media_type == "application/pdf"
|
|
else 'attachment; filename="source.txt"'
|
|
),
|
|
},
|
|
)
|
|
|
|
@app.get(
|
|
"/v1/projects/{project_id}/sources/{revision_id}/extractions/{extraction_id}/blocks/{block_id}",
|
|
tags=["knowledge"],
|
|
operation_id="get_source_location",
|
|
)
|
|
def get_source_location(
|
|
project_id: RecordId,
|
|
revision_id: RecordId,
|
|
extraction_id: str,
|
|
block_id: RecordId,
|
|
response: Response,
|
|
offset: int = Query(ge=0),
|
|
length: int = Query(ge=1, le=12_000),
|
|
authorization: str | None = Header(default=None),
|
|
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
|
|
) -> dict[str, object]:
|
|
response.headers["Cache-Control"] = "private, no-store"
|
|
response.headers["X-Content-Type-Options"] = "nosniff"
|
|
principal = authenticate(authorization, session_cookie)
|
|
try:
|
|
source, _ = source_reader.get_source_record(
|
|
project_id, revision_id, principal=principal
|
|
)
|
|
except ReadError as error:
|
|
raise HTTPException(status_code=error.status_code, detail=str(error)) from error
|
|
extraction = artifact_store.get_extraction(extraction_id, project_id=project_id)
|
|
if extraction is None or extraction.source_revision_id != source.revision_id:
|
|
raise HTTPException(status_code=404, detail="Source location is unavailable")
|
|
block = next(
|
|
(item for item in extraction.blocks if item.block_id == block_id), None
|
|
)
|
|
if block is None or offset >= len(block.normalized_text):
|
|
raise HTTPException(status_code=404, detail="Source location is unavailable")
|
|
end = min(offset + length, len(block.normalized_text))
|
|
spans = block.normalization_map[offset:end]
|
|
original_start = min(start for start, _ in spans)
|
|
original_end = max(stop for _, stop in spans)
|
|
return {
|
|
"source_revision_id": revision_id,
|
|
"extraction_id": extraction.extraction_id,
|
|
"parser_name": extraction.parser_name,
|
|
"parser_version": extraction.parser_version,
|
|
"block_id": block_id,
|
|
"offset": offset,
|
|
"normalized_text": block.normalized_text[offset:end],
|
|
"original_text": block.original_text[original_start:original_end],
|
|
"normalization_map": spans,
|
|
}
|
|
|
|
if search_service is not None:
|
|
http_operations.update({"list_projects", "search", "neighbors", "list_sources"})
|
|
|
|
@app.get("/v1/projects", tags=["knowledge"], operation_id="list_projects")
|
|
def list_projects(
|
|
authorization: str | None = Header(default=None),
|
|
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
|
|
) -> list[str]:
|
|
principal = authenticate(authorization, session_cookie)
|
|
return list(search_service.list_projects(principal=principal))
|
|
|
|
@app.post(
|
|
"/v1/search",
|
|
response_model=ReadEnvelope,
|
|
tags=["knowledge"],
|
|
operation_id="search",
|
|
)
|
|
def search(
|
|
request: SearchRequestBody,
|
|
authorization: str | None = Header(default=None),
|
|
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
|
|
csrf_token: str | None = Header(default=None, alias="X-GEWU-CSRF"),
|
|
) -> ReadEnvelope:
|
|
principal = authenticate(
|
|
authorization, session_cookie, csrf_token, require_csrf=True
|
|
)
|
|
try:
|
|
return search_service.search(request.to_contract(), principal=principal)
|
|
except SearchError as error:
|
|
raise HTTPException(status_code=error.status_code, detail=str(error)) from error
|
|
|
|
@app.post(
|
|
"/v1/neighbors",
|
|
response_model=ReadEnvelope,
|
|
tags=["knowledge"],
|
|
operation_id="neighbors",
|
|
)
|
|
def neighbors(
|
|
request: NeighborRequestBody,
|
|
authorization: str | None = Header(default=None),
|
|
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
|
|
csrf_token: str | None = Header(default=None, alias="X-GEWU-CSRF"),
|
|
) -> ReadEnvelope:
|
|
principal = authenticate(
|
|
authorization, session_cookie, csrf_token, require_csrf=True
|
|
)
|
|
try:
|
|
return search_service.neighbors(request.to_contract(), principal=principal)
|
|
except SearchError as error:
|
|
raise HTTPException(status_code=error.status_code, detail=str(error)) from error
|
|
|
|
@app.post(
|
|
"/v1/sources",
|
|
response_model=ReadEnvelope,
|
|
tags=["knowledge"],
|
|
operation_id="list_sources",
|
|
)
|
|
def list_sources(
|
|
request: SourceListRequestBody,
|
|
authorization: str | None = Header(default=None),
|
|
session_cookie: str | None = Cookie(default=None, alias="gewu_session"),
|
|
csrf_token: str | None = Header(default=None, alias="X-GEWU-CSRF"),
|
|
) -> ReadEnvelope:
|
|
principal = authenticate(
|
|
authorization, session_cookie, csrf_token, require_csrf=True
|
|
)
|
|
try:
|
|
return search_service.list_sources(request.to_contract(), principal=principal)
|
|
except SearchError as error:
|
|
raise HTTPException(status_code=error.status_code, detail=str(error)) from error
|
|
|
|
mcp_server = create_mcp_server(
|
|
identity_provider=identity_provider,
|
|
node_reader=node_reader,
|
|
source_reader=source_reader,
|
|
search_service=search_service,
|
|
artifact_store=artifact_store,
|
|
extractor=extractor,
|
|
acceptance_service=acceptance_service,
|
|
context_provider=context_provider,
|
|
url_fetcher=url_fetcher or PublicUrlFetcher(frozenset()),
|
|
)
|
|
mcp_asgi = mcp_server.streamable_http_app(
|
|
streamable_http_path="/",
|
|
stateless_http=True,
|
|
json_response=True,
|
|
max_request_body_size=1_048_576,
|
|
transport_security=TransportSecuritySettings(
|
|
allowed_hosts=[
|
|
"localhost",
|
|
"localhost:*",
|
|
"127.0.0.1",
|
|
"127.0.0.1:*",
|
|
"[::1]",
|
|
"[::1]:*",
|
|
*mcp_allowed_hosts,
|
|
]
|
|
),
|
|
)
|
|
owner_mcp_server = None
|
|
owner_mcp_asgi = None
|
|
if (
|
|
owner_identity_provider is not None
|
|
and owner_identity_provider.owner_authentication_enabled
|
|
and owner_service_audience
|
|
and callable(getattr(identity_provider, "get_owner_challenge", None))
|
|
and callable(getattr(identity_provider, "decide_owner_challenge", None))
|
|
):
|
|
owner_mcp_server = create_owner_mcp_server(
|
|
identities=owner_identity_provider,
|
|
reviews=cast(OwnerReviewProvider, identity_provider),
|
|
audience=owner_service_audience,
|
|
)
|
|
owner_mcp_asgi = owner_mcp_server.streamable_http_app(
|
|
streamable_http_path="/",
|
|
stateless_http=True,
|
|
json_response=True,
|
|
max_request_body_size=1_048_576,
|
|
transport_security=TransportSecuritySettings(
|
|
allowed_hosts=[
|
|
"localhost",
|
|
"localhost:*",
|
|
"127.0.0.1",
|
|
"127.0.0.1:*",
|
|
"[::1]",
|
|
"[::1]:*",
|
|
*mcp_allowed_hosts,
|
|
]
|
|
),
|
|
)
|
|
original_lifespan = app.router.lifespan_context
|
|
|
|
@asynccontextmanager
|
|
async def lifespan(application: FastAPI) -> AsyncIterator[None]:
|
|
async with AsyncExitStack() as stack:
|
|
await stack.enter_async_context(mcp_server.session_manager.run())
|
|
if owner_mcp_server is not None:
|
|
await stack.enter_async_context(owner_mcp_server.session_manager.run())
|
|
async with original_lifespan(application):
|
|
yield
|
|
|
|
app.router.lifespan_context = lifespan
|
|
app.router.routes.append(Mount("/mcp", app=mcp_asgi))
|
|
if owner_mcp_asgi is not None:
|
|
app.router.routes.append(Mount("/owner-mcp", app=owner_mcp_asgi))
|
|
|
|
return app
|
|
|
|
|
|
def create_local_app() -> FastAPI:
|
|
"""Compose the local persistent service from one private data root."""
|
|
|
|
data_root = Path(os.environ.get("GEWU_DATA_ROOT", "./.data/gewu")).expanduser().absolute()
|
|
reject_symlink_components(data_root)
|
|
data_root.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
if data_root.is_symlink() or not data_root.is_dir():
|
|
raise RuntimeError("GEWU data root must be a real directory")
|
|
data_root.chmod(0o700)
|
|
repository = GitCanonicalRepository(data_root / "knowledge.git")
|
|
workflow_root = data_root / "workflow"
|
|
authority_lock = workflow_root / "locks" / "canonical-writer.lock"
|
|
identities = SQLiteIdentityStore(
|
|
data_root / "identity.sqlite3",
|
|
authority_lock_path=authority_lock,
|
|
owner_encryption_key=os.environ.get("GEWU_OWNER_ENCRYPTION_KEY", "").encode("ascii")
|
|
or None,
|
|
)
|
|
artifacts = LocalArtifactStore(data_root)
|
|
allowed_fetch_hosts = {
|
|
host.strip()
|
|
for host in os.environ.get("GEWU_FETCH_ALLOWED_HOSTS", "").split(",")
|
|
if host.strip()
|
|
}
|
|
url_fetcher = PublicUrlFetcher(allowed_fetch_hosts) if allowed_fetch_hosts else None
|
|
registry_file = data_root / "control" / "registry.json"
|
|
context_provider = LocalContextProvider(repository, registry_file)
|
|
project_registry = LocalProjectRegistry(
|
|
registry_file,
|
|
repository=repository,
|
|
index_directory=data_root / "publishing-index",
|
|
authority_lock_path=authority_lock,
|
|
)
|
|
search_path = data_root / "search.sqlite3"
|
|
search_index: SearchIndex
|
|
model_path = os.environ.get("GEWU_EMBEDDING_MODEL_PATH")
|
|
model_digest = os.environ.get("GEWU_EMBEDDING_MODEL_SHA256")
|
|
if model_path or model_digest:
|
|
if not model_path or not model_digest:
|
|
raise RuntimeError(
|
|
"GEWU_EMBEDDING_MODEL_PATH and GEWU_EMBEDDING_MODEL_SHA256 "
|
|
"must be configured together"
|
|
)
|
|
embeddings = LocalSentenceTransformer(
|
|
Path(model_path),
|
|
model_id=os.environ.get(
|
|
"GEWU_EMBEDDING_MODEL_ID",
|
|
"intfloat/multilingual-e5-small@614241f622f53c4eeff9890bdc4f31cfecc418b3",
|
|
),
|
|
expected_sha256=model_digest,
|
|
dimensions=int(os.environ.get("GEWU_EMBEDDING_DIMENSIONS", "384")),
|
|
)
|
|
vectors = SQLiteVectorIndex(search_path)
|
|
lexical_index = SQLiteSearchIndex(search_path, repository)
|
|
hybrid_index = HybridSearchIndex(
|
|
lexical=lexical_index,
|
|
vectors=vectors,
|
|
embeddings=embeddings,
|
|
)
|
|
lexical_index.configure_vector_generation(
|
|
extension_loader=vectors.load_extension,
|
|
builder=hybrid_index.build_vector_generation,
|
|
is_current=hybrid_index.vector_generation_is_current,
|
|
)
|
|
search_index = hybrid_index
|
|
else:
|
|
search_index = SQLiteSearchIndex(search_path, repository)
|
|
search_service = CanonicalSearchService(
|
|
index=search_index,
|
|
project_ids_provider=project_registry,
|
|
canonical_head=repository.head,
|
|
default_includes_provider=project_registry.default_includes,
|
|
)
|
|
acceptance = AcceptanceService(
|
|
repository=repository,
|
|
artifact_store=artifacts,
|
|
workflow_root=workflow_root,
|
|
context_provider=context_provider,
|
|
actor_is_current=lambda actor: identities.is_current(
|
|
actor.principal_id,
|
|
actor.credential_id,
|
|
roles=actor.roles,
|
|
project_ids=actor.project_ids,
|
|
),
|
|
owner_challenge_issuer=identities.issue_owner_challenge,
|
|
owner_decision_lookup=identities.authorized_owner_challenges,
|
|
)
|
|
return create_app(
|
|
identity_provider=identities,
|
|
credential_administration=identities,
|
|
account_identity=identities,
|
|
owner_identity_provider=identities,
|
|
owner_service_audience=os.environ.get("GEWU_OWNER_SERVICE_AUDIENCE"),
|
|
node_reader=CanonicalNodeReader(repository),
|
|
artifact_store=artifacts,
|
|
extractor=LocalSourceExtractor(),
|
|
acceptance_service=acceptance,
|
|
context_provider=context_provider,
|
|
search_service=search_service,
|
|
project_registry=project_registry,
|
|
url_fetcher=url_fetcher,
|
|
)
|