This commit is contained in:
vasily committed
1 parent
4db3f45164
commit
d25c04cdd2
175 files changed
+56243
-15
No files matched your search
@@ -0,0 +1,426 @@
|
||||
"""Local project/policy configuration and committed-record context provider."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from contextlib import AbstractContextManager
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import portalocker
|
||||
import yaml
|
||||
from pydantic import BaseModel, ValidationError
|
||||
|
||||
from gewu.application.context import ValidationContext
|
||||
from gewu.application.paths import reject_symlink_components
|
||||
from gewu.application.ports import CanonicalConflictError, CanonicalRepository
|
||||
from gewu.application.serialization import safe_yaml_record
|
||||
from gewu.domain.contracts import (
|
||||
AutoAcceptPolicy,
|
||||
ExtractionRecord,
|
||||
KnowledgeNode,
|
||||
ProjectRecord,
|
||||
ProjectStatus,
|
||||
SourceRevisionRecord,
|
||||
)
|
||||
from gewu.domain.validation import extraction_digest
|
||||
|
||||
MAX_CONTEXT_RECORDS = 20_000
|
||||
MAX_CONTEXT_RECORD_BYTES = 1_048_576
|
||||
MAX_PROJECT_REGISTRY_RESPONSE_BYTES = 1_048_576
|
||||
|
||||
|
||||
class RegistryError(RuntimeError):
|
||||
"""Configured or committed context is malformed; acceptance must stop."""
|
||||
|
||||
|
||||
class RegistryConflict(RegistryError):
|
||||
"""A registry write conflicts with current project or policy identity."""
|
||||
|
||||
|
||||
class RegistryNotFound(RegistryError):
|
||||
"""A requested project is not registered."""
|
||||
|
||||
|
||||
def _unique_object(pairs: list[tuple[str, object]]) -> dict[str, object]:
|
||||
result: dict[str, object] = {}
|
||||
for key, value in pairs:
|
||||
if key in result:
|
||||
raise ValueError("duplicate JSON key")
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
|
||||
def _records[RecordT: BaseModel](
|
||||
raw: bytes, field: str, record_type: type[RecordT]
|
||||
) -> tuple[RecordT, ...]:
|
||||
if len(raw) > MAX_CONTEXT_RECORD_BYTES:
|
||||
raise RegistryError("Registry configuration exceeds the supported size")
|
||||
try:
|
||||
value = json.loads(raw, object_pairs_hook=_unique_object)
|
||||
if (
|
||||
not isinstance(value, dict)
|
||||
or set(value) != {"schema_version", "projects", "policies"}
|
||||
or value.get("schema_version") != 1
|
||||
):
|
||||
raise ValueError("registry schema version or fields are invalid")
|
||||
records = value[field]
|
||||
if not isinstance(records, list):
|
||||
raise ValueError("registry field must be a list")
|
||||
return tuple(
|
||||
record_type.model_validate_json(json.dumps(item, ensure_ascii=False), strict=True)
|
||||
for item in records
|
||||
)
|
||||
except (KeyError, TypeError, ValueError, json.JSONDecodeError, ValidationError) as error:
|
||||
raise RegistryError("Registry configuration failed validation") from error
|
||||
|
||||
|
||||
def _index[RecordT: BaseModel](records: tuple[RecordT, ...], field: str) -> dict[str, RecordT]:
|
||||
result: dict[str, RecordT] = {}
|
||||
for record in records:
|
||||
key = getattr(record, field)
|
||||
if key in result:
|
||||
raise RegistryError("Registry contains duplicate stable IDs")
|
||||
result[key] = record
|
||||
return result
|
||||
|
||||
|
||||
def _canonical_records[RecordT: BaseModel](
|
||||
repository: CanonicalRepository,
|
||||
revision: str,
|
||||
prefix: str,
|
||||
record_type: type[RecordT],
|
||||
id_field: str,
|
||||
) -> dict[str, RecordT]:
|
||||
records: dict[str, RecordT] = {}
|
||||
for record_path in repository.list_files(revision, prefix):
|
||||
if not record_path.endswith(".yaml"):
|
||||
continue
|
||||
if prefix == "projects" and "/nodes/" not in record_path:
|
||||
continue
|
||||
raw = repository.read_file(revision, record_path)
|
||||
if raw is None or len(raw) > MAX_CONTEXT_RECORD_BYTES:
|
||||
raise RegistryError("Canonical context record is unavailable or too large")
|
||||
record = safe_yaml_record(raw, record_type)
|
||||
record_id = getattr(record, id_field)
|
||||
if record_id in records:
|
||||
raise RegistryError("Canonical repository contains duplicate stable IDs")
|
||||
records[record_id] = record
|
||||
if len(records) > MAX_CONTEXT_RECORDS:
|
||||
raise RegistryError("Canonical context exceeds the configured record limit")
|
||||
return records
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class LocalContextProvider:
|
||||
repository: CanonicalRepository
|
||||
registry_file: Path
|
||||
|
||||
def __call__(self) -> ValidationContext:
|
||||
path = self.registry_file.expanduser().absolute()
|
||||
try:
|
||||
reject_symlink_components(path)
|
||||
except ValueError as error:
|
||||
raise RegistryError("Local registry path traverses a symbolic link") from error
|
||||
if path.exists() and not path.is_file():
|
||||
raise RegistryError("Local registry file is unavailable")
|
||||
if not path.exists():
|
||||
projects: dict[str, ProjectRecord] = {}
|
||||
policies: dict[str, AutoAcceptPolicy] = {}
|
||||
else:
|
||||
try:
|
||||
registry_bytes = path.read_bytes()
|
||||
except OSError as error:
|
||||
raise RegistryError("Local registry file is unavailable") from error
|
||||
projects = _index(_records(registry_bytes, "projects", ProjectRecord), "project_id")
|
||||
policies = _index(_records(registry_bytes, "policies", AutoAcceptPolicy), "policy_id")
|
||||
|
||||
revision = self.repository.head()
|
||||
projects.update(_canonical_projects(self.repository, revision))
|
||||
policies.update(
|
||||
_canonical_records(
|
||||
self.repository,
|
||||
revision,
|
||||
"policies",
|
||||
AutoAcceptPolicy,
|
||||
"policy_id",
|
||||
)
|
||||
)
|
||||
sources = _canonical_records(
|
||||
self.repository, revision, "sources", SourceRevisionRecord, "revision_id"
|
||||
)
|
||||
nodes = _canonical_records(self.repository, revision, "projects", KnowledgeNode, "node_id")
|
||||
extractions: dict[str, ExtractionRecord] = {}
|
||||
record_count = 0
|
||||
try:
|
||||
record_count += len(sources) + len(nodes)
|
||||
for record_path in self.repository.list_files(revision, "extractions"):
|
||||
if not record_path.endswith(".json"):
|
||||
continue
|
||||
record_count += 1
|
||||
extraction_bytes = self.repository.read_file(revision, record_path)
|
||||
if extraction_bytes is None or len(extraction_bytes) > MAX_CONTEXT_RECORD_BYTES:
|
||||
raise RegistryError("Canonical extraction is unavailable or too large")
|
||||
extraction = ExtractionRecord.model_validate_json(extraction_bytes, strict=True)
|
||||
if extraction_digest(extraction) != extraction.extraction_id:
|
||||
raise RegistryError("Canonical extraction identity is invalid")
|
||||
if extraction.extraction_id in extractions:
|
||||
raise RegistryError("Canonical repository contains duplicate extraction IDs")
|
||||
extractions[extraction.extraction_id] = extraction
|
||||
except (OSError, ValueError, ValidationError) as error:
|
||||
raise RegistryError("Canonical context failed validation") from error
|
||||
if record_count > MAX_CONTEXT_RECORDS:
|
||||
raise RegistryError("Canonical context exceeds the configured record limit")
|
||||
return ValidationContext(
|
||||
projects=projects,
|
||||
policies=policies,
|
||||
sources=sources,
|
||||
extractions=extractions,
|
||||
existing_nodes=nodes,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class LocalProjectRegistry:
|
||||
"""Read legacy control entries and canonical project lifecycle records."""
|
||||
|
||||
registry_file: Path
|
||||
repository: CanonicalRepository | None = None
|
||||
index_directory: Path | None = None
|
||||
authority_lock_path: Path | None = None
|
||||
|
||||
def _legacy_projects(self) -> tuple[ProjectRecord, ...]:
|
||||
path = self.registry_file.expanduser().absolute()
|
||||
try:
|
||||
reject_symlink_components(path)
|
||||
except ValueError as error:
|
||||
raise RegistryError("Local registry path traverses a symbolic link") from error
|
||||
if not path.exists():
|
||||
return ()
|
||||
if path.is_symlink() or not path.is_file():
|
||||
raise RegistryError("Local registry file is unavailable")
|
||||
try:
|
||||
raw = path.read_bytes()
|
||||
except OSError as error:
|
||||
raise RegistryError("Local registry file is unavailable") from error
|
||||
return _records(raw, "projects", ProjectRecord)
|
||||
|
||||
def _projects(self) -> tuple[ProjectRecord, ...]:
|
||||
projects = _index(self._legacy_projects(), "project_id")
|
||||
if self.repository is not None:
|
||||
projects.update(_canonical_projects(self.repository, self.repository.head()))
|
||||
return tuple(projects.values())
|
||||
|
||||
def _policies(self) -> tuple[AutoAcceptPolicy, ...]:
|
||||
path = self.registry_file.expanduser().absolute()
|
||||
if not path.exists():
|
||||
policies: dict[str, AutoAcceptPolicy] = {}
|
||||
elif path.is_symlink() or not path.is_file():
|
||||
raise RegistryError("Local registry file is unavailable")
|
||||
else:
|
||||
try:
|
||||
raw = path.read_bytes()
|
||||
except OSError as error:
|
||||
raise RegistryError("Local registry file is unavailable") from error
|
||||
policies = _index(_records(raw, "policies", AutoAcceptPolicy), "policy_id")
|
||||
if self.repository is not None:
|
||||
policies.update(
|
||||
_canonical_records(
|
||||
self.repository,
|
||||
self.repository.head(),
|
||||
"policies",
|
||||
AutoAcceptPolicy,
|
||||
"policy_id",
|
||||
)
|
||||
)
|
||||
return tuple(policies.values())
|
||||
|
||||
def __call__(self) -> frozenset[str]:
|
||||
return frozenset(_index(self._projects(), "project_id"))
|
||||
|
||||
def records(self) -> tuple[ProjectRecord, ...]:
|
||||
records = tuple(sorted(self._projects(), key=lambda project: project.project_id))
|
||||
response_bytes = sum(
|
||||
len(
|
||||
json.dumps(
|
||||
project.model_dump(mode="json"),
|
||||
ensure_ascii=False,
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
)
|
||||
for project in records
|
||||
)
|
||||
if response_bytes > MAX_PROJECT_REGISTRY_RESPONSE_BYTES:
|
||||
raise RegistryError("Project registry listing exceeds the supported size")
|
||||
return records
|
||||
|
||||
def create(self, project: ProjectRecord, policy: AutoAcceptPolicy) -> ProjectRecord:
|
||||
"""Atomically add a project and its explicitly supplied restrictive policy."""
|
||||
|
||||
if project.project_id == "shared" or project.status != ProjectStatus.ACTIVE:
|
||||
raise ValueError("New projects must use a non-reserved ID and active status")
|
||||
if not project.owners:
|
||||
raise ValueError("At least one named project owner is required")
|
||||
if (
|
||||
project.autonomy_policy_id != policy.policy_id
|
||||
or policy.project_id != project.project_id
|
||||
):
|
||||
raise ValueError("Project and initial policy identities must match")
|
||||
if not policy.require_supported_assessment:
|
||||
raise ValueError("Project policies must require supported assessments")
|
||||
if (
|
||||
self.repository is None
|
||||
or self.index_directory is None
|
||||
or self.authority_lock_path is None
|
||||
):
|
||||
raise RegistryError("Canonical project publication is not configured")
|
||||
with self._write_lock():
|
||||
project_index = _index(self._projects(), "project_id")
|
||||
policy_index = _index(self._policies(), "policy_id")
|
||||
if project.project_id in project_index:
|
||||
raise RegistryConflict("Project ID is already registered")
|
||||
if policy.policy_id in policy_index:
|
||||
raise RegistryConflict("Policy ID is already registered")
|
||||
available_projects = set(project_index) | {project.project_id, "shared"}
|
||||
if any(item not in available_projects for item in project.default_includes):
|
||||
raise ValueError("Project default includes must reference registered projects")
|
||||
if project.project_id in project.default_includes:
|
||||
raise ValueError("A project cannot include itself by default")
|
||||
try:
|
||||
self.repository.publish_files(
|
||||
expected_head=self.repository.head(),
|
||||
files={
|
||||
_project_record_path(project.project_id): _project_markdown(project),
|
||||
_policy_record_path(policy.policy_id): _yaml_record(policy),
|
||||
},
|
||||
message=f"Register project {project.project_id}",
|
||||
index_directory=self.index_directory,
|
||||
)
|
||||
except CanonicalConflictError as error:
|
||||
raise RegistryConflict(
|
||||
"Canonical project registry changed during creation"
|
||||
) from error
|
||||
return project
|
||||
|
||||
def update_content(
|
||||
self, project_id: str, *, description: str, north_star: str, key_questions: tuple[str, ...]
|
||||
) -> ProjectRecord:
|
||||
"""Update a project's human-authored description and research questions."""
|
||||
if self.repository is None or self.index_directory is None or self.authority_lock_path is None:
|
||||
raise RegistryError("Canonical project publication is not configured")
|
||||
with self._write_lock():
|
||||
project = _index(self._projects(), "project_id").get(project_id)
|
||||
if project is None:
|
||||
raise RegistryNotFound("Project is not registered")
|
||||
updated = project.model_copy(update={
|
||||
"description": description, "north_star": north_star, "key_questions": key_questions,
|
||||
})
|
||||
try:
|
||||
self.repository.publish_files(
|
||||
expected_head=self.repository.head(),
|
||||
files={_project_record_path(project_id): _project_markdown(updated)},
|
||||
message=f"Update project description and questions for {project_id}",
|
||||
index_directory=self.index_directory,
|
||||
)
|
||||
except CanonicalConflictError as error:
|
||||
raise RegistryConflict("Canonical project registry changed during update") from error
|
||||
return updated
|
||||
|
||||
def archive(self, project_id: str) -> ProjectRecord:
|
||||
"""Mark a project archived while preserving its registry and read history."""
|
||||
|
||||
if project_id == "shared":
|
||||
raise ValueError("The shared scope cannot be archived")
|
||||
if (
|
||||
self.repository is None
|
||||
or self.index_directory is None
|
||||
or self.authority_lock_path is None
|
||||
):
|
||||
raise RegistryError("Canonical project publication is not configured")
|
||||
with self._write_lock():
|
||||
project_index = _index(self._projects(), "project_id")
|
||||
project = project_index.get(project_id)
|
||||
if project is None:
|
||||
raise RegistryNotFound("Project is not registered")
|
||||
if project.status == ProjectStatus.ARCHIVED:
|
||||
return project
|
||||
archived = project.model_copy(update={"status": ProjectStatus.ARCHIVED})
|
||||
try:
|
||||
self.repository.publish_files(
|
||||
expected_head=self.repository.head(),
|
||||
files={_project_record_path(project_id): _project_markdown(archived)},
|
||||
message=f"Archive project {project_id}",
|
||||
index_directory=self.index_directory,
|
||||
)
|
||||
except CanonicalConflictError as error:
|
||||
raise RegistryConflict(
|
||||
"Canonical project registry changed during archive"
|
||||
) from error
|
||||
return archived
|
||||
|
||||
def _write_lock(self) -> AbstractContextManager[Any]:
|
||||
lock_path = self.authority_lock_path
|
||||
if lock_path is None:
|
||||
lock_path = self.registry_file.with_suffix(self.registry_file.suffix + ".lock")
|
||||
lock_path = lock_path.expanduser().absolute()
|
||||
reject_symlink_components(lock_path)
|
||||
lock_path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
lock_path.parent.chmod(0o700)
|
||||
return portalocker.Lock(lock_path, mode="a", timeout=0)
|
||||
|
||||
def default_includes(self, project_id: str) -> tuple[str, ...]:
|
||||
projects = _index(self._projects(), "project_id")
|
||||
project = projects.get(project_id)
|
||||
if project is None:
|
||||
raise RegistryError("Project is unavailable for default scope resolution")
|
||||
return tuple(include for include in project.default_includes if include != "shared")
|
||||
|
||||
|
||||
def _project_record_path(project_id: str) -> str:
|
||||
return f"projects/{project_id}/project.md"
|
||||
|
||||
|
||||
def _policy_record_path(policy_id: str) -> str:
|
||||
import hashlib
|
||||
|
||||
digest = hashlib.sha256(policy_id.encode("utf-8")).hexdigest()
|
||||
return f"policies/{digest}.yaml"
|
||||
|
||||
|
||||
def _yaml_record(record: BaseModel) -> bytes:
|
||||
return yaml.safe_dump(
|
||||
record.model_dump(mode="json"),
|
||||
allow_unicode=True,
|
||||
sort_keys=True,
|
||||
).encode("utf-8")
|
||||
|
||||
|
||||
def _project_markdown(project: ProjectRecord) -> bytes:
|
||||
return b"---\n" + _yaml_record(project) + b"---\n"
|
||||
|
||||
|
||||
def _canonical_projects(repository: CanonicalRepository, revision: str) -> dict[str, ProjectRecord]:
|
||||
projects: dict[str, ProjectRecord] = {}
|
||||
total_bytes = 0
|
||||
for record_path in repository.list_files(revision, "projects"):
|
||||
if not record_path.endswith("/project.md"):
|
||||
continue
|
||||
raw = repository.read_file(revision, record_path)
|
||||
if raw is None or len(raw) > MAX_CONTEXT_RECORD_BYTES:
|
||||
raise RegistryError("Canonical project record is unavailable or too large")
|
||||
total_bytes += len(raw)
|
||||
if total_bytes > MAX_PROJECT_REGISTRY_RESPONSE_BYTES:
|
||||
raise RegistryError("Canonical project registry exceeds the supported size")
|
||||
if not raw.startswith(b"---\n") or b"\n---\n" not in raw[4:]:
|
||||
raise RegistryError("Canonical project frontmatter is invalid")
|
||||
frontmatter = raw[4:].split(b"\n---\n", maxsplit=1)[0]
|
||||
project = safe_yaml_record(frontmatter, ProjectRecord)
|
||||
if record_path != _project_record_path(project.project_id):
|
||||
raise RegistryError("Canonical project path does not match its stable ID")
|
||||
if project.project_id in projects:
|
||||
raise RegistryError("Canonical repository contains duplicate project IDs")
|
||||
projects[project.project_id] = project
|
||||
if len(projects) > MAX_CONTEXT_RECORDS:
|
||||
raise RegistryError("Canonical project registry exceeds the supported record count")
|
||||
return projects
|
||||
Reference in new issue
Block a user