feat: online verification, node removals, and knowledge browser
CI / quality (push) Canceled after 0s
CI / quality (push) Canceled after 0s
Adds claim-bound online evidence records and citation verification scripts, node removal transport/adapter, retrieval benchmark and owner-review tooling, GraphCanvas/SourceDetail knowledge browser UI, and the ingest handoff helper. Bumps the agents/gewu pack to the online-verification branch.
This commit is contained in:
1 parent
a3156f4e08
commit
78033bb830
83 files changed
+13476
-1051
No files matched your search
@@ -0,0 +1,233 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create a deterministic, read-only evidence review ledger for benchmark proposals.
|
||||
|
||||
This tool checks structure and source-span integrity. It does not make semantic
|
||||
support judgments and never writes owner decisions or canonical knowledge.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import unicodedata
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def load_extractions(directory: Path) -> dict[str, dict[str, Any]]:
|
||||
result: dict[str, dict[str, Any]] = {}
|
||||
for path in sorted(directory.rglob("*.json")):
|
||||
value = json.loads(path.read_text(encoding="utf-8"))
|
||||
record = value.get("record", value)
|
||||
extraction_id = record.get("extraction_id")
|
||||
if isinstance(extraction_id, str):
|
||||
if extraction_id in result:
|
||||
raise ValueError(f"Duplicate extraction ID: {extraction_id}")
|
||||
result[extraction_id] = record
|
||||
if not result:
|
||||
raise ValueError("No extraction manifests found")
|
||||
return result
|
||||
|
||||
|
||||
def _audit_claim(claim: dict[str, Any], extractions: dict[str, dict[str, Any]]) -> dict[str, Any]:
|
||||
evidence_rows = claim.get("evidence")
|
||||
issues: list[str] = []
|
||||
checked = 0
|
||||
claim_text = claim.get("text")
|
||||
expected_assertion_digest = (
|
||||
hashlib.sha256(
|
||||
json.dumps(
|
||||
{
|
||||
"assertion": unicodedata.normalize(
|
||||
"NFC", claim_text.replace("\r\n", "\n").replace("\r", "\n")
|
||||
)
|
||||
},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
).hexdigest()
|
||||
if isinstance(claim_text, str)
|
||||
else None
|
||||
)
|
||||
if expected_assertion_digest is None:
|
||||
issues.append("CLAIM_TEXT_INVALID")
|
||||
if not isinstance(evidence_rows, list) or not evidence_rows:
|
||||
issues.append("CLAIM_EVIDENCE_REQUIRED")
|
||||
evidence_rows = []
|
||||
for evidence in evidence_rows:
|
||||
checked += 1
|
||||
if not isinstance(evidence, dict):
|
||||
issues.append("EVIDENCE_RECORD_INVALID")
|
||||
continue
|
||||
extraction_id = evidence.get("extraction_id")
|
||||
record = extractions.get(extraction_id) if isinstance(extraction_id, str) else None
|
||||
if record is None:
|
||||
issues.append("EXTRACTION_NOT_FOUND")
|
||||
continue
|
||||
if evidence.get("source_revision_id") != record.get("source_revision_id"):
|
||||
issues.append("SOURCE_REVISION_MISMATCH")
|
||||
if evidence.get("assertion_digest") != expected_assertion_digest:
|
||||
issues.append("ASSERTION_DIGEST_MISMATCH")
|
||||
locator = evidence.get("locator")
|
||||
if not isinstance(locator, dict):
|
||||
issues.append("LOCATOR_INVALID")
|
||||
continue
|
||||
block_id = locator.get("block_id")
|
||||
block = next(
|
||||
(
|
||||
row
|
||||
for row in record.get("blocks", [])
|
||||
if isinstance(row, dict) and row.get("block_id") == block_id
|
||||
),
|
||||
None,
|
||||
)
|
||||
if block is None:
|
||||
issues.append("BLOCK_NOT_FOUND")
|
||||
continue
|
||||
text = block.get("normalized_text")
|
||||
start, end = locator.get("start"), locator.get("end")
|
||||
if (
|
||||
not isinstance(text, str)
|
||||
or type(start) is not int
|
||||
or type(end) is not int
|
||||
or start < 0
|
||||
or end <= start
|
||||
or end > len(text)
|
||||
):
|
||||
issues.append("LOCATOR_RANGE_INVALID")
|
||||
continue
|
||||
if locator.get("page") != block.get("page"):
|
||||
issues.append("PAGE_MISMATCH")
|
||||
if text[start:end] != evidence.get("quote"):
|
||||
issues.append("QUOTE_SPAN_MISMATCH")
|
||||
return {
|
||||
"claim_id": claim.get("claim_id"),
|
||||
"assertion_digest": next(
|
||||
(
|
||||
evidence.get("assertion_digest")
|
||||
for evidence in evidence_rows
|
||||
if isinstance(evidence, dict)
|
||||
),
|
||||
None,
|
||||
),
|
||||
"evidence_checked": checked,
|
||||
"structural_status": "valid" if not issues else "invalid",
|
||||
"issues": sorted(set(issues)),
|
||||
}
|
||||
|
||||
|
||||
def review(
|
||||
items: list[dict[str, Any]],
|
||||
extractions: dict[str, dict[str, Any]],
|
||||
*,
|
||||
input_sha256: str,
|
||||
) -> dict[str, Any]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
for item in items:
|
||||
proposal = item.get("proposal", {})
|
||||
evaluation = item.get("evaluation", {})
|
||||
proposal_id = proposal.get("proposal_id")
|
||||
claims: list[dict[str, Any]] = []
|
||||
supersession_edges: list[dict[str, Any]] = []
|
||||
for operation in proposal.get("operations", []):
|
||||
node = operation.get("node") or {}
|
||||
claims.extend(_audit_claim(claim, extractions) for claim in node.get("claims", []))
|
||||
edge = operation.get("edge")
|
||||
if isinstance(edge, dict):
|
||||
supersession_edges.append(edge)
|
||||
structural_issues = [issue for claim in claims for issue in claim["issues"]]
|
||||
policy_held = (
|
||||
evaluation.get("status") == "needs-review"
|
||||
or evaluation.get("eligible_for_auto_accept") is False
|
||||
or "needs-review" in str(item.get("storage_state", ""))
|
||||
or item.get("source_stage") == "supersession"
|
||||
)
|
||||
if structural_issues:
|
||||
recommendation = "revise-candidate"
|
||||
reason = "One or more evidence bindings failed deterministic checks."
|
||||
elif policy_held:
|
||||
recommendation = "defer"
|
||||
reason = "Evidence bindings are structurally valid, but policy requires owner review."
|
||||
elif not claims:
|
||||
recommendation = "defer"
|
||||
reason = "No atomic claims were available for review."
|
||||
else:
|
||||
recommendation = "defer"
|
||||
reason = "Semantic support has not been independently established by this tool."
|
||||
rows.append(
|
||||
{
|
||||
"proposal_id": proposal_id,
|
||||
"revision": proposal.get("revision"),
|
||||
"source_stage": item.get("source_stage"),
|
||||
"storage_state": item.get("storage_state"),
|
||||
"evaluation_status": evaluation.get("status"),
|
||||
"policy_held": policy_held,
|
||||
"recommendation": recommendation,
|
||||
"rationale": reason,
|
||||
"claims": claims,
|
||||
"supersession_edges_preserved": len(supersession_edges),
|
||||
"supersession_edges": supersession_edges,
|
||||
}
|
||||
)
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"kind": "machine_review_recommendations",
|
||||
"created_utc": datetime.now(UTC).isoformat(),
|
||||
"input_sha256": input_sha256,
|
||||
"review_method": "deterministic evidence binding checks; semantic support not assessed",
|
||||
"authority": "advisory only; no owner decision or canonical mutation",
|
||||
"summary": {
|
||||
"items": len(rows),
|
||||
"claims": sum(len(row["claims"]) for row in rows),
|
||||
"evidence_bindings_valid": sum(
|
||||
claim["structural_status"] == "valid" for row in rows for claim in row["claims"]
|
||||
),
|
||||
"evidence_bindings_invalid": sum(
|
||||
claim["structural_status"] == "invalid" for row in rows for claim in row["claims"]
|
||||
),
|
||||
"recommendations": {
|
||||
label: sum(row["recommendation"] == label for row in rows)
|
||||
for label in ("accept-candidate", "revise-candidate", "exclude-candidate", "defer")
|
||||
},
|
||||
"owner_decisions_written": 0,
|
||||
"canonical_records_written": 0,
|
||||
},
|
||||
"items": rows,
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--input", type=Path, required=True)
|
||||
parser.add_argument("--extractions", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
raw = args.input.read_bytes()
|
||||
items = json.loads(raw)
|
||||
if not isinstance(items, list) or not items:
|
||||
parser.error("Input must be a nonempty JSON list")
|
||||
result = review(
|
||||
items,
|
||||
load_extractions(args.extractions),
|
||||
input_sha256=hashlib.sha256(raw).hexdigest(),
|
||||
)
|
||||
with args.output.open("x", encoding="utf-8") as stream:
|
||||
stream.write(json.dumps(result, ensure_ascii=False, indent=2, sort_keys=True) + "\n")
|
||||
print(
|
||||
json.dumps({"status": "created", "summary": result["summary"], "output": str(args.output)})
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user