feat(agent): group ingestion proposals by learning

README.md

@@ -1,6 +1,6 @@
 # GEWU local research-agent pack
 
-Version 0.16.0. Adds four default research lenses, non-blocking ingest-review flags, project-scoped feedback guidance and optional Jev relevance scoring; native-host installation and pilot workflows remain unverified.
+Version 0.19.0. Adds proposal grouping by semantic learning, with source registration dependencies preserved across bundles. Includes explicit shared-scope requirements, agent-side duplicate checks, compact extraction reads and per-source budget checkpoints. Disposable service/evidence and Jev label probes passed; Antigravity authenticated and captured the packet, but a complete client proposal run stopped at the provider quota.
 
 This pack contains the [agent profile](AGENTS.md), Claude harness pointer ([CLAUDE.md](CLAUDE.md)), shared protocols, four skills and separate pipeline procedures. An existing local host supplies the model and tools. There is no standalone agent engine here.
 

contracts/service-v1.json

@@ -1162,7 +1162,7 @@
       "name": "source_extract"
     },
     {
-      "description": "Read a bounded normalized text slice and its source offset map.",
+      "description": "Read a bounded normalized text slice and optionally its original-source offset map.",
       "input_schema": {
         "properties": {
           "block_id": {
@@ -1173,6 +1173,11 @@
             "title": "Extraction Id",
             "type": "string"
           },
+          "include_original_mapping": {
+            "default": false,
+            "title": "Include Original Mapping",
+            "type": "boolean"
+          },
           "length": {
             "title": "Length",
             "type": "integer"

manifest.json

@@ -1,6 +1,6 @@
 {
   "pack_id": "gewu-research",
-  "pack_version": "0.16.0",
+  "pack_version": "0.19.0",
   "manifest_version": "1.0.0",
   "status": "local-contract-and-checkpoint-bound-hosts-not-qualified",
   "execution": "existing-local-agent-host",
@@ -10,7 +10,7 @@
       "contract_id": "gewu.service.v1",
       "contract_version": 1,
       "path": "contracts/service-v1.json",
-      "sha256": "a0120520aadb73fee08138b7b60a5ee05b82823e028ec1bc644ead6637920668",
+      "sha256": "1b70e5eaddbd0db869b679462fe14167c221aab44e3691aa38409c9c1f7c7799",
       "scope": "Generated local MCP tool input schemas and v1 record schemas; native host workflows remain NOT RUN."
     }
   ],
@@ -28,10 +28,10 @@
   "skills": [
     {
       "id": "kb-ingest",
-      "version": "0.2.0",
+      "version": "0.4.0",
       "entry": "skills/kb-ingest/SKILL.md",
       "pipeline": "pipelines/ingest.md",
-      "pipeline_version": "0.2.0"
+      "pipeline_version": "0.4.0"
     },
     {
       "id": "kb-query",
@@ -73,8 +73,8 @@
       "requires_contracts": [
         "gewu.protocol.evidence@0.1.0",
         "gewu.protocol.scope@0.1.0",
-        "gewu.protocol.contributions@0.2.0",
+        "gewu.protocol.contributions@0.3.0",
-        "gewu.skill.kb-ingest@0.2.0",
+        "gewu.skill.kb-ingest@0.4.0",
         "gewu.skill.kb-query@0.1.0",
         "gewu.skill.kb-curate@0.1.0",
         "gewu.skill.kb-synthesize@0.2.0"
@@ -112,7 +112,7 @@
       "compatible_checkpoint_contracts": [
         "gewu.local-checkpoint@0.1.0"
       ],
-      "sha256": "4de60c95fddabd94223df0bad3cd2090b04d6e7a484488c8ea300a9e181246e6",
+      "sha256": "94fb3d58a9534ab3c4ee35ba890825a21a65438e7d7e911544764d28de8c3503",
       "hash_encoding": "utf8-no-bom-lf"
     },
     {
@@ -137,10 +137,10 @@
       "module_id": "contributions",
       "kind": "protocol",
       "entry": "protocols/CONTRIBUTIONS.md",
-      "implementation_version": "0.4.0",
+      "implementation_version": "0.3.0",
       "provided_contract": {
         "id": "gewu.protocol.contributions",
-        "version": "0.2.0"
+        "version": "0.3.0"
       },
       "requires_contracts": [
         "gewu.local-checkpoint@0.1.0"
@@ -159,23 +159,23 @@
       "compatible_checkpoint_contracts": [
         "gewu.local-checkpoint@0.1.0"
       ],
-      "sha256": "4c5b6169daf675866355265d598a0338b67fd9b819862031cd24d66753ea7b05",
+      "sha256": "8dcd38e78dc79ec93097b17c4c54a68f3e0cc407535a7afa1c63a1369e21107f",
       "hash_encoding": "utf8-no-bom-lf"
     },
     {
       "module_id": "kb-ingest",
       "kind": "skill",
       "entry": "skills/kb-ingest/SKILL.md",
-      "implementation_version": "0.4.0",
+      "implementation_version": "0.7.0",
       "provided_contract": {
         "id": "gewu.skill.kb-ingest",
-        "version": "0.2.0"
+        "version": "0.5.0"
       },
       "requires_contracts": [
         "gewu.protocol.evidence@0.1.0",
         "gewu.protocol.scope@0.1.0",
-        "gewu.protocol.contributions@0.2.0",
+        "gewu.protocol.contributions@0.3.0",
-        "gewu.pipeline.kb-ingest-pipeline@0.2.0"
+        "gewu.pipeline.kb-ingest-pipeline@0.5.0"
       ],
       "required_service_capabilities": [
         "proposal:evaluate",
@@ -190,22 +190,22 @@
       "compatible_checkpoint_contracts": [
         "gewu.local-checkpoint@0.1.0"
       ],
-      "sha256": "39d559dbc43b14cff6816a92bc95e337128b4eacea899d12cd71becbd441cc12",
+      "sha256": "2ce5a1ff710389761bf48a1a0cd48965c477a2a8705cb1cd0dd710324380e941",
       "hash_encoding": "utf8-no-bom-lf"
     },
     {
       "module_id": "kb-ingest-pipeline",
       "kind": "pipeline",
       "entry": "pipelines/ingest.md",
-      "implementation_version": "0.3.0",
+      "implementation_version": "0.6.0",
       "provided_contract": {
         "id": "gewu.pipeline.kb-ingest-pipeline",
-        "version": "0.2.0"
+        "version": "0.5.0"
       },
       "requires_contracts": [
         "gewu.protocol.evidence@0.1.0",
         "gewu.protocol.scope@0.1.0",
-        "gewu.protocol.contributions@0.2.0"
+        "gewu.protocol.contributions@0.3.0"
       ],
       "required_service_capabilities": [
         "proposal:evaluate",
@@ -220,7 +220,7 @@
       "compatible_checkpoint_contracts": [
         "gewu.local-checkpoint@0.1.0"
       ],
-      "sha256": "2942147e529b30a4ed92c9d17c5a9262328de31ec1cc584737c8b6eb9c103f71",
+      "sha256": "045bad113be7e1a6e8f7ea0646c5bd083ced299745abb2a099e574a5f41c5916",
       "hash_encoding": "utf8-no-bom-lf"
     },
     {
@@ -235,7 +235,7 @@
       "requires_contracts": [
         "gewu.protocol.evidence@0.1.0",
         "gewu.protocol.scope@0.1.0",
-        "gewu.protocol.contributions@0.2.0",
+        "gewu.protocol.contributions@0.3.0",
         "gewu.pipeline.kb-query-pipeline@0.1.0"
       ],
       "required_service_capabilities": [
@@ -260,7 +260,7 @@
       "requires_contracts": [
         "gewu.protocol.evidence@0.1.0",
         "gewu.protocol.scope@0.1.0",
-        "gewu.protocol.contributions@0.2.0"
+        "gewu.protocol.contributions@0.3.0"
       ],
       "required_service_capabilities": [
         "knowledge:read"
@@ -284,7 +284,7 @@
       "requires_contracts": [
         "gewu.protocol.evidence@0.1.0",
         "gewu.protocol.scope@0.1.0",
-        "gewu.protocol.contributions@0.2.0",
+        "gewu.protocol.contributions@0.3.0",
         "gewu.pipeline.kb-curate-pipeline@0.1.0"
       ],
       "required_service_capabilities": [
@@ -315,7 +315,7 @@
       "requires_contracts": [
         "gewu.protocol.evidence@0.1.0",
         "gewu.protocol.scope@0.1.0",
-        "gewu.protocol.contributions@0.2.0"
+        "gewu.protocol.contributions@0.3.0"
       ],
       "required_service_capabilities": [
         "knowledge:read",
@@ -345,7 +345,7 @@
       "requires_contracts": [
         "gewu.protocol.evidence@0.1.0",
         "gewu.protocol.scope@0.1.0",
-        "gewu.protocol.contributions@0.2.0",
+        "gewu.protocol.contributions@0.3.0",
         "gewu.pipeline.kb-synthesize-pipeline@0.2.0"
       ],
       "required_service_capabilities": [
@@ -370,7 +370,7 @@
       "requires_contracts": [
         "gewu.protocol.evidence@0.1.0",
         "gewu.protocol.scope@0.1.0",
-        "gewu.protocol.contributions@0.2.0"
+        "gewu.protocol.contributions@0.3.0"
       ],
       "required_service_capabilities": [
         "knowledge:read"
@@ -397,7 +397,7 @@
       "compatible_checkpoint_contracts": [
         "gewu.local-checkpoint@0.1.0"
       ],
-      "sha256": "5032436c257a203a36be6f093b7ae5db61a844060963ba25f855b0ef2af0f422",
+      "sha256": "a8f06d35a0e839be9d6c4582b6a3d50cd6588f9a9c8bac291969fac5d4290178",
       "hash_encoding": "utf8-no-bom-lf"
     },
     {
@@ -435,7 +435,7 @@
       "compatible_checkpoint_contracts": [
         "gewu.local-checkpoint@0.1.0"
       ],
-      "sha256": "a0120520aadb73fee08138b7b60a5ee05b82823e028ec1bc644ead6637920668",
+      "sha256": "1b70e5eaddbd0db869b679462fe14167c221aab44e3691aa38409c9c1f7c7799",
       "hash_encoding": "utf8-no-bom-lf"
     },
     {
@@ -516,6 +516,6 @@
   "snapshot": {
     "algorithm": "sha256",
     "scope": "modules array serialized as UTF-8 JSON with sorted keys and compact separators",
-    "digest": "f00e1677c0abafd42b8d70d14847a1044ba43917424cda7c4b18b5494a959385"
+    "digest": "0ad378e98a149fbb49aca24edd06a5fecf07679fd488d0556ffd6c7f31951b02"
   }
 }

pipelines/ingest.md

@@ -1,14 +1,17 @@
 # Ingestion pipeline
 
-Version 0.3.0. Host-executed procedure for `kb-ingest`; not a background scheduler. Follow the contribution protocol for budgets, checkpoints and retry identity.
+Version 0.5.0. Host-executed procedure for `kb-ingest`; not a background scheduler. Follow the contribution protocol for budgets, checkpoints and retry identity.
 
 | Stage | Input | Output / checkpoint | Gate |
 | --- | --- | --- | --- |
 | Context | User-selected material, explicit project and `get_project_context` lens | Objective, authorized project lens including four default research lenses and saved guidance, scope, run identity and pinned pack versions | Required tools/schema are available; otherwise report blocked. |
 | Capture | Explicitly selected local files, user-pasted text or an allowlisted public URL | Verified artifact digest, stable source ID, content-bound revision ID, metadata candidate and extraction availability | Use the connector for local files, `source_capture_text` for explicitly supplied UTF-8 text/Markdown (maximum 1 MB), or `source_capture_url` for a configured public host. Pasted text requires a stable capture ID and source key, and returns an archived artifact, extraction and candidate marked `pasted-text`; retry with the same ID and identical inputs after uncertainty, since changed inputs conflict. The service retains at most 1,024 private capture receipts. The capture does not register the source. The connector returns a typed `source_candidate`; `.vtt` and `.srt` files are parsed into cue-timed blocks and tagged `transcript`. Cite the precise cue times together with the quote span. The connector can record a declared NotebookLM export provenance for supported text, Markdown or PDF files, but does not verify that provenance; omit `source_revision_id` to derive a revision from file identity, content digest and its durable retry checkpoint, or supply a stable revision ID when resuming an existing workflow. Submit candidates through `proposal_submit`; staging alone does not register a source. Failed fetches return a `capture-failed` candidate without artifact bytes; register it only when a bookmark is useful. PDF extraction currently requires the supported Linux sandbox. HTML citations refer to visible extracted text, not markup-byte offsets. |
+| Budget checkpoint | Current source stage and run budget | Per-source coverage checkpoint, remaining file/section plan and resume point | Process one source at a time. Use metadata to target relevant bounded slices; retain concise notes and exact evidence excerpts, not full source text. Checkpoint after each source. If quota or tool budget is near exhaustion, save the exact resume point and report remaining coverage without dropping a research lens. |
 | Triage | Source plus project lens and four default lenses (technical, conceptual, cultural-historical, financial-market) | Included sections, explicit clear-exclusion reasons and review flags | Treat lenses as coverage prompts, not quotas. Historical, philosophical, cultural or commercial evidence is not automatically out of scope. Exclude only clearly unrelated or unavailable material, or user-excluded topics. Continue normal evidence-backed proposal processing for plausible material; add `ingest_review_create` as a follow-up notification/filter flag when human relevance review would help. |
-| Extract | Included sections and immutable maps | Atomic proposal draft with field-level evidence | Exact quotes/locators; no invented revisions. |
+| Duplicate check | Candidate findings and explicit readable project/shared scope | Existing-node candidates, reuse/update decision, or explicit unresolved-review flag | Before creating a node, run multiple lexical searches over its title, distinctive terms and likely synonyms, then inspect plausible matches with `get_node`. Reuse the stable node ID and preserve supported content for a true update; do not create a second node when it represents the same concept. Keep materially distinct conditions as distinct records. The backend flags exact entity name/alias matches only and does not deduplicate other node types. |
+| Extract | Included sections and immutable maps | Atomic proposal draft with field-level evidence | Read captured source text with bounded `source_extraction_block` calls, using exact extraction and block IDs; inspect relevant pages/sections even when the host cannot directly open a PDF. Use the normalized text and offsets returned by the service for exact quotes/locators. Record partial/unavailable coverage; upload or extraction alone is not analysis. |
 | Assess | Assertion/evidence bindings | Support judgments, assessor/run and rationale | Unsupported/unavailable claims remain drafts. |
+| Proposal grouping | Assessed findings across captured sources | Small, coherent learning bundles with source dependencies | Treat documents as capture/provenance units and proposals as semantic learning units. Group by one project question or a tightly related set of findings, combining evidence across files when it supports the same learning. Do not create one proposal per file or one oversized proposal for the whole packet. Keep unrelated learnings separate so each can be evaluated, retried, reviewed and accepted independently. Include each new source candidate as `register_source` in the first proposal that cites it; after acceptance makes that revision readable, later proposals cite the accepted revision without registering it again. |
 | Evaluate | Complete bounded bundle and base revision | `proposal_evaluate` findings and eligible status | Evaluation is read-only; it does not reserve an ID or store the proposal. |
 | Submit | Evaluated bounded bundle and stable proposal identity | `proposal_submit` outcome and eventual receipt | Every affected domain's policy applies; protected decisions remain external. |
 | Human review | Stored review flag and authorized editor/admin decision | Include, exclude or defer feedback; optionally saved project/lens guidance | The flag never blocks proposal evaluation/submission or marks knowledge accepted. Decisions are feedback only. Save reusable guidance only on explicit reviewer opt-in. |

protocols/CONTRIBUTIONS.md

@@ -10,6 +10,10 @@ Use the stable principal ID assigned to the configured agent credential. The cre
 
 Capture tools stage and archive source material but do not register its source record. Before evaluating a proposal that cites a newly captured source, include the exact returned `source_candidate` as a `register_source` operation in the same proposal, or confirm that the source revision was already accepted and is readable. Use the candidate's exact `revision_id` as each citation's `source_revision_id`, and the capture's returned `extraction_id`; do not synthesize or copy IDs from another capture. A source operation and a node operation may be evaluated together.
 
+Every source registration affects both the shared-source policy and each project in `source.ingested_for`; therefore proposal scope must set `include_shared=true` and explicitly include the project(s), and the agent credential must be authorized for that scope. A project-only scope is invalid for a newly registered source even if it is relevant only to that project.
+
+Before creating a new node, search the explicit project and shared scope with multiple lexical queries: the proposed title, distinctive terms, aliases and likely synonyms. Inspect plausible matches with `get_node`. If the finding belongs to an existing node, preserve its stable ID and supported content when proposing an evidence-backed update; do not create a second node for the same concept. Keep materially different populations, time periods, settings, mechanisms or conclusions distinct. Similarity is only a review cue. The backend flags exact entity name/alias matches but does not detect duplicate notes, cases, concepts or datapoints; if likely duplicates remain unresolved after search, submit a non-blocking review flag and do not create another node yet.
+
 Finalize claim text and evidence before creating support assessments or digests. Every evidence `assertion_path` must name its exact typed claim field (`claims[n].text`, `assumptions[n].text` or `limitations[n].text`). Its `assertion_digest` must bind to that exact claim text. Set the assessment's `assertion_digest` to the same value and `evidence_digests` to the digests of the claim's final evidence objects. Any edit to claim text or evidence requires regenerating those dependent values and the proposal `payload_digest` before `proposal_evaluate`.
 
 Use the canonical GEWU digest rules, not a hash of the raw quote or raw request text. Canonical JSON is UTF-8 JSON with Unicode preserved, sorted keys, no insignificant whitespace and no NaN values. The assertion digest is SHA-256 of canonical JSON `{"assertion": normalized_text}`, where text has CRLF and CR converted to LF and is normalized to Unicode NFC. An evidence digest is SHA-256 of canonical JSON for the complete evidence object in schema JSON form. The assessment's `evidence_digests` must match those final evidence digests. The proposal payload digest is SHA-256 of canonical JSON for the final operations serialized as GEWU v1 contract records. Recompute it after adding/removing/changing any operation. If the client cannot reproduce these rules, stop before submission and report that digest generation is unavailable.

skills/kb-ingest/SKILL.md

@@ -17,3 +17,13 @@ For `tags`, `audience`, `use_for` and `domain`, inspect the `facet_labels` catal
 If available, Jev also assesses relevance and lens in the same advisory call. Use the saved scores to focus human review; they are not evidence assessment or an instruction to silently rewrite accepted knowledge. When a reviewer decides, record the feedback and apply reusable project guidance only when the reviewer explicitly opts in. Preserve the service's exact source/extraction/location bindings. Map each factual field to evidence; assess semantic support separately. Record clear exclusions and genuinely skipped/deferred sections. Optimize for useful coverage rather than maximum note count.
 
 Submit bounded bundles under the affected project/shared policies. Repair actionable findings without weakening evidence, changing scope to evade policy or assuming owner authority. Resume uncertain calls through status lookup. Report accepted node IDs and receipts, pending/rejected work, archive/extraction limits and actual coverage. A successful upload or schema check is not a completed ingestion.
+
+Every `register_source` operation affects both the source's `ingested_for` project policy and the shared-source policy. Set proposal scope `include_shared=true` as well as the project scope, even when the source is relevant only to one project. Confirm the credential authorizes both. If shared scope or authorization is unavailable, stop and report the missing policy/capability; do not submit a project-only bundle that will fail scope validation.
+
+For a captured source, use the returned extraction status and block metadata to plan review. Read source text through bounded `source_extraction_block` calls using the exact `extraction_id` and `block_id`; inspect relevant slices across all pages/sections, including PDFs that the host's file reader cannot open. The default response contains normalized text and its offset but omits the large original-source mapping; set `include_original_mapping=true` only when an original-text span is needed. Citation quote and locator offsets bind to the normalized extraction either way. Record partial or unavailable extraction coverage explicitly. Do not infer that a source was analyzed just because upload and extraction succeeded.
+
+Before creating a new node, search the explicit project and shared scope using its proposed title plus distinctive terms, aliases and alternate phrasings. Search likely synonyms separately when title search is inconclusive, then inspect candidate nodes and claims with `get_node`. Search is lexical, so use more than one query. If a finding belongs to an existing node, preserve its stable `node_id` and existing supported content when proposing an evidence-backed update; do not mint another ID for the same concept. Keep separate nodes when populations, time periods, settings, mechanisms or conclusions materially differ. Similarity is a review cue, not proof of identity; if the distinction remains uncertain, create a non-blocking review flag and do not create a second node until resolved. The current backend only flags exact entity name/alias matches; it does not detect duplicate notes, cases, concepts or datapoints for you.
+
+Manage context and provider budget across the whole packet. Process one source at a time, use extraction metadata to target relevant slices rather than fetching every character, and retain concise section notes plus exact evidence excerpts instead of full source text. Checkpoint coverage after each source. If the provider/tool budget is nearly exhausted, checkpoint the current file and report the remaining files/sections for continuation; do not reduce cost by dropping a research lens or deferring cultural-historical or financial-market material.
+
+Treat documents as capture/provenance units and proposals as semantic learning units. Group a proposal around one project question or one tightly related set of findings, and combine evidence from multiple documents when it supports the same learning. Do not make one proposal per file or one oversized bundle for the entire packet. Keep unrelated learnings separate so each can be evaluated, retried, reviewed and accepted independently. Include each new source candidate in the first proposal that cites it; after its source revision is accepted and readable, later learning proposals cite that revision without registering it again.