zettelkasten.tables_persist

zettelkasten.tables_persist

Table persistence + overlay/attribution helpers for the matrix engine.

Mechanically split out of tables.py: the _reviews/<name>.tables.json read/write substrate, org mirror upsert, table signature, overlay normalization and corrections attribution.

set_column_synthesis

set_column_synthesis(name: str, table_id: str, column_key: str, ai: str, *, graphs_dir: 'Path | None' = None) -> bool

Durably fold a column's on-demand AI narrative into synthesis[key]["ai"].

The on-demand synthesis endpoint (Synthesize / Synthesize all) generates a one-line narrative per column; without this it lived only in transient client state and vanished on reload. This persists it onto the stored grid the same way the streaming build does, so it survives a reload and a cross-session peek.

A narrow load→mutate→save under the review write lock (never a whole-grid clobber from a possibly-stale read) so a parallel Synthesize all — which fires one request per column concurrently — can't drop a sibling column's freshly-written narrative. Returns False when the table or column is absent.

Source code in zettelkasten/tables_persist.py

def set_column_synthesis(
    name: str,
    table_id: str,
    column_key: str,
    ai: str,
    *,
    graphs_dir: "Path | None" = None,
) -> bool:
    """Durably fold a column's on-demand AI narrative into ``synthesis[key]["ai"]``.

    The on-demand synthesis endpoint (``Synthesize`` / ``Synthesize all``) generates
    a one-line narrative per column; without this it lived only in transient client
    state and vanished on reload. This persists it onto the stored grid the same way
    the streaming build does, so it survives a reload and a cross-session peek.

    A narrow load→mutate→save under the review write lock (never a whole-grid
    clobber from a possibly-stale read) so a parallel ``Synthesize all`` — which
    fires one request per column concurrently — can't drop a sibling column's
    freshly-written narrative. Returns ``False`` when the table or column is absent.
    """
    base = Path(graphs_dir) if graphs_dir is not None else GRAPHS_DIR
    with review_write_lock(name, graphs_dir=base):
        store = _load_tables(name, base)
        table = store["tables"].get(table_id)
        if table is None:
            return False
        synthesis = table.get("synthesis")
        if not isinstance(synthesis, dict):
            synthesis = {}
            table["synthesis"] = synthesis
        entry = synthesis.get(column_key)
        if not isinstance(entry, dict):
            # No mechanical entry yet (e.g. a legacy grid) — seed a minimal one so
            # the narrative has a home; the next full build recomputes the stats.
            entry = {"filled": 0, "total": len(table.get("rows") or []), "kind": ""}
            synthesis[column_key] = entry
        entry["ai"] = ai
        path = _tables_path(name, base)
        path.parent.mkdir(parents=True, exist_ok=True)
        atomic_write_text(path, json.dumps(store, indent=2, ensure_ascii=False))
        _schedule_zettel_commit(str(path.resolve()))
    return True

delete_table

delete_table(name: str, table_id: str, *, graphs_dir: 'Path | None' = None) -> bool

Remove a table from the review's tables artifact. Returns True if removed.

Source code in zettelkasten/tables_persist.py

def delete_table(name: str, table_id: str, *, graphs_dir: "Path | None" = None) -> bool:
    """Remove a table from the review's tables artifact. Returns True if removed."""
    base = Path(graphs_dir) if graphs_dir is not None else GRAPHS_DIR
    with review_write_lock(name, graphs_dir=base):
        store = _load_tables(name, base)
        if table_id not in store["tables"]:
            return False
        del store["tables"][table_id]
        path = _tables_path(name, base)
        atomic_write_text(path, json.dumps(store, indent=2, ensure_ascii=False))
        _schedule_zettel_commit(str(path.resolve()))
    owner = _mirror_owner(name, table_id, "", "", base)
    if owner is not None:
        try:
            import zettelkasten.organizations as organizations

            organizations.delete_definition(owner[0], owner[1], review=name, table_id=table_id, graphs_dir=base)
        except Exception:  # pragma: no cover - defensive: registry mirror is best-effort
            logger.warning("org mirror delete failed for %s/%s", name, table_id, exc_info=True)
    return True

rename_table

rename_table(name: str, table_id: str, title: str, *, graphs_dir: 'Path | None' = None) -> bool

Patch a persisted table's display title. Returns True if the table existed.

Only the title (a display label) changes — table_id is the immutable artifact key that cell-note anchors and the grid signature depend on, so a rename never re-keys the table. Cheap: no re-gather, no LLM.

Source code in zettelkasten/tables_persist.py

def rename_table(name: str, table_id: str, title: str, *, graphs_dir: "Path | None" = None) -> bool:
    """Patch a persisted table's display title. Returns True if the table existed.

    Only the ``title`` (a display label) changes — ``table_id`` is the immutable
    artifact key that cell-note anchors and the grid signature depend on, so a
    rename never re-keys the table. Cheap: no re-gather, no LLM.
    """
    base = Path(graphs_dir) if graphs_dir is not None else GRAPHS_DIR
    with review_write_lock(name, graphs_dir=base):
        store = _load_tables(name, base)
        table = store["tables"].get(table_id)
        if table is None:
            return False
        table["title"] = (title or "").strip() or table_id
        path = _tables_path(name, base)
        atomic_write_text(path, json.dumps(store, indent=2, ensure_ascii=False))
        _schedule_zettel_commit(str(path.resolve()))
    owner = _mirror_owner(name, table_id, "", "", base)
    if owner is not None:
        try:
            import zettelkasten.organizations as organizations

            organizations.rename_definition(owner[0], owner[1], review=name, table_id=table_id, title=title, graphs_dir=base)
        except Exception:  # pragma: no cover - defensive: registry mirror is best-effort
            logger.warning("org mirror rename failed for %s/%s", name, table_id, exc_info=True)
    return True

list_tables

list_tables(name: str, *, graphs_dir: 'Path | None' = None) -> list[dict[str, Any]]

All persisted tables for a review (full grids), newest-first by id.

Source code in zettelkasten/tables_persist.py

def list_tables(name: str, *, graphs_dir: "Path | None" = None) -> list[dict[str, Any]]:
    """All persisted tables for a review (full grids), newest-first by id."""
    base = Path(graphs_dir) if graphs_dir is not None else GRAPHS_DIR
    store = _load_tables(name, base)
    return list(store["tables"].values())

list_table_meta

list_table_meta(name: str, *, graphs_dir: 'Path | None' = None) -> list[dict[str, Any]]

Metadata-only listing of a review's tables — NO rows/cells.

Same order as :func:list_tables but each entry is a shallow copy with the heavy per-cell rows payload dropped and lightweight rows_count / columns_count added. Every other field (title, columns, row axis, synthesis, signature, timestamps, lifecycle) is preserved so the dropdown, merge, and axis-typed subdivision keep working without loading a single full grid. The full grids stay available via :func:list_tables (used for the on-demand Excel export).

Source code in zettelkasten/tables_persist.py

def list_table_meta(name: str, *, graphs_dir: "Path | None" = None) -> list[dict[str, Any]]:
    """Metadata-only listing of a review's tables — NO ``rows``/cells.

    Same order as :func:`list_tables` but each entry is a shallow copy with the
    heavy per-cell ``rows`` payload dropped and lightweight ``rows_count`` /
    ``columns_count`` added. Every other field (title, columns, row axis,
    synthesis, signature, timestamps, lifecycle) is preserved so the dropdown,
    merge, and axis-typed subdivision keep working without loading a single
    full grid. The full grids stay available via :func:`list_tables` (used for
    the on-demand Excel export).
    """
    out: list[dict[str, Any]] = []
    for table in list_tables(name, graphs_dir=graphs_dir):
        meta = {k: v for k, v in table.items() if k not in _TABLE_META_HEAVY_KEYS}
        rows = table.get("rows")
        meta["rows_count"] = len(rows) if isinstance(rows, list) else 0
        cols = table.get("columns")
        meta["columns_count"] = len(cols) if isinstance(cols, list) else 0
        out.append(meta)
    return out

apply_overlay

apply_overlay(table: dict[str, Any], overlay: Any, *, valid_note_ids: 'set[str] | None' = None) -> dict[str, Any]

Merge a corrections overlay onto a freshly built table — the durability merge.

Returns a DERIVED, READ-ONLY view (a deep copy): corrections whose referenced row/col/note ids still exist are applied; the rest drop into removed_upstream rather than crashing. NEVER persist this view — only the stored overlay is durable. member_removes win over member_adds for the same (row, col, note) so a resync can't resurrect an ejected note.

valid_note_ids (when supplied — :func:reconcile passes the rebuilt corpus's note ids) gates member_add against vanished notes: a pin whose note_id no longer exists in the corpus is dropped into removed_upstream rather than fabricating a phantom member. When None (a bare merge over a pre-built grid) the existence check is skipped. A member_remove whose note_id no longer routes into its cell is ALSO surfaced in removed_upstream (rather than silently no-op'd) so a correction that no longer bites — e.g. against a lens column with no materialized edges — stays visible and reconcilable instead of vanishing without trace.

Source code in zettelkasten/tables_persist.py

def apply_overlay(
    table: dict[str, Any],
    overlay: Any,
    *,
    valid_note_ids: "set[str] | None" = None,
) -> dict[str, Any]:
    """Merge a corrections overlay onto a freshly built table — the durability merge.

    Returns a DERIVED, READ-ONLY view (a deep copy): corrections whose referenced
    row/col/note ids still exist are applied; the rest drop into
    ``removed_upstream`` rather than crashing. NEVER persist this view — only the
    stored overlay is durable. ``member_removes`` win over ``member_adds`` for the
    same ``(row, col, note)`` so a resync can't resurrect an ejected note.

    ``valid_note_ids`` (when supplied — :func:`reconcile` passes the rebuilt
    corpus's note ids) gates ``member_add`` against vanished notes: a pin whose
    ``note_id`` no longer exists in the corpus is dropped into ``removed_upstream``
    rather than fabricating a phantom member. When ``None`` (a bare merge over a
    pre-built grid) the existence check is skipped. A ``member_remove`` whose
    ``note_id`` no longer routes into its cell is ALSO surfaced in
    ``removed_upstream`` (rather than silently no-op'd) so a correction that no
    longer bites — e.g. against a lens column with no materialized edges — stays
    visible and reconcilable instead of vanishing without trace.
    """
    overlay = _normalize_table_overlay(overlay)
    out = copy.deepcopy(table)
    rows_by_id = {r["id"]: r for r in out.get("rows", [])}
    col_keys = {c["key"] for c in out.get("columns", [])}
    removed_upstream: list[dict[str, Any]] = []

    def _cell_for(rid: str, ck: str) -> "dict[str, Any] | None":
        row = rows_by_id.get(rid)
        if row is None or ck not in col_keys:
            return None
        cell = (row.get("cells") or {}).get(ck)
        return cell if isinstance(cell, dict) else None

    touched: list[dict[str, Any]] = []
    human_cells: list[dict[str, Any]] = []
    ejected: set[tuple[str, str, str]] = set()

    for e in overlay["member_removes"]:
        cell = _cell_for(e["row_id"], e["col_key"])
        ref = f"{e['row_id']}/{e['col_key']}/{e['note_id']}"
        if cell is None:
            removed_upstream.append({"kind": "member_remove", "ref": ref})
            continue
        ejected.add((e["row_id"], e["col_key"], e["note_id"]))
        before = cell.get("members") or []
        kept = [m for m in before if m.get("note_id") != e["note_id"]]
        if len(kept) != len(before):
            cell["members"] = kept
            touched.append(cell)
        elif e["note_id"] not in (cell.get("note_ids") or []):
            # The note isn't a member and isn't a legacy (v1) note-id reference, so
            # the remove no longer routes into this cell. Surface it rather than
            # silently dropping the correction (covers the lens no-op case).
            removed_upstream.append({"kind": "member_remove", "ref": ref})

    for e in overlay["member_adds"]:
        cell = _cell_for(e["row_id"], e["col_key"])
        ref = f"{e['row_id']}/{e['col_key']}/{e['note_id']}"
        if cell is None:
            removed_upstream.append({"kind": "member_add", "ref": ref})
            continue
        # A remove of the same note wins — never re-add an ejected member.
        if (e["row_id"], e["col_key"], e["note_id"]) in ejected:
            continue
        # A pin to a note that no longer exists in the corpus must NOT fabricate a
        # phantom member — drop it as a vanished upstream reference.
        if valid_note_ids is not None and e["note_id"] not in valid_note_ids:
            removed_upstream.append({"kind": "member_add", "ref": ref})
            continue
        members = cell.setdefault("members", [])
        if any(m.get("note_id") == e["note_id"] for m in members):
            continue
        is_ai = e["relation"] == _RELATION_AI
        members.append(
            _member(
                note_id=e["note_id"],
                type=e["type"],
                title=e["title"] or e["note_id"],
                graph=e["graph"],
                quote=e["quote"],
                provenance=_PROVENANCE_REVIEWER if is_ai else "",
                relation=e["relation"],
            )
        )
        # A human-pinned member makes the cell author-edited — it reads as
        # ``human`` (not deterministic), so edits are visible at a glance.
        if not is_ai:
            cell["source_kind"] = _SOURCE_HUMAN
            human_cells.append(cell)
        touched.append(cell)

    for cell in touched:
        _recompute_after_membership(cell)
    # ``_recompute_after_membership`` preserves an already-``human`` source_kind,
    # but re-stamp here in case the recompute order or a later edit reset it.
    for cell in human_cells:
        if cell.get("members"):
            cell["source_kind"] = _SOURCE_HUMAN

    for e in overlay["cell_overrides"]:
        cell = _cell_for(e["row_id"], e["col_key"])
        if cell is None:
            removed_upstream.append({"kind": "cell_override", "ref": f"{e['row_id']}/{e['col_key']}"})
            continue
        cell["summary"] = e["summary"]
        cell["value"] = e["summary"]
        cell["locked"] = bool(e["locked"])
        cell["source_kind"] = _SOURCE_HUMAN
        cell["provenance"] = ""
        if e["summary"]:
            cell["gap"] = False

    out["removed_upstream"] = removed_upstream
    out["overlay_applied"] = {
        "member_adds": len(overlay["member_adds"]),
        "member_removes": len(overlay["member_removes"]),
        "cell_overrides": len(overlay["cell_overrides"]),
        "attributions": len(overlay["attributions"]),
    }
    return out

corpus_note_ids

corpus_note_ids(get_graph: GetGraph, *, project: str = '', graph: str = '', graphs_dir: 'Path | None' = None, localize: 'Callable[[str], str] | None' = None) -> set[str]

Every note id in the build's scope — the existence oracle for the overlay.

:func:apply_overlay consults this to tell a still-valid pin from one that references a vanished note. Spans every scope graph (incl. _cross) and is type-agnostic so a legitimately-existing note of any kind is never mistaken for vanished. Best-effort: a load failure yields an empty set (no spurious drops — the existence check simply degrades to skipped per-id).

Source code in zettelkasten/tables_persist.py

def corpus_note_ids(
    get_graph: GetGraph,
    *,
    project: str = "",
    graph: str = "",
    graphs_dir: "Path | None" = None,
    localize: "Callable[[str], str] | None" = None,
) -> set[str]:
    """Every note id in the build's scope — the existence oracle for the overlay.

    :func:`apply_overlay` consults this to tell a still-valid pin from one that
    references a vanished note. Spans every scope graph (incl. ``_cross``) and is
    type-agnostic so a legitimately-existing note of any kind is never mistaken
    for vanished. Best-effort: a load failure yields an empty set (no spurious
    drops — the existence check simply degrades to skipped per-id).
    """
    base = Path(graphs_dir) if graphs_dir is not None else GRAPHS_DIR
    loc = localize or (lambda s: s)
    ids: set[str] = set()
    for _name, _zg, note in _universe_pairs(
        get_graph, project=project, graph=graph, base=base, loc=loc
    ):
        nid = getattr(note, "id", "")
        if nid:
            ids.add(str(nid))
    return ids

load_overlay

load_overlay(name: str, table_id: str, *, project: str = '', graph: str = '', graphs_dir: 'Path | None' = None) -> 'dict[str, Any] | None'

The corrections overlay stored on the table's mirror org, or None.

Resolves the owner-scoped org for (review, table_id) the same way the persist mirror does (:func:_mirror_owner) and returns its overlay block. Returns None when no org exists yet (a brand-new / unmirrored table) so a caller can fall back to the bare projection and behave exactly as before the overlay existed. Best-effort: any registry hiccup also yields None.

Source code in zettelkasten/tables_persist.py

def load_overlay(
    name: str,
    table_id: str,
    *,
    project: str = "",
    graph: str = "",
    graphs_dir: "Path | None" = None,
) -> "dict[str, Any] | None":
    """The corrections overlay stored on the table's mirror org, or ``None``.

    Resolves the owner-scoped org for ``(review, table_id)`` the same way the
    persist mirror does (:func:`_mirror_owner`) and returns its ``overlay`` block.
    Returns ``None`` when no org exists yet (a brand-new / unmirrored table) so a
    caller can fall back to the bare projection and behave exactly as before the
    overlay existed. Best-effort: any registry hiccup also yields ``None``.
    """
    base = Path(graphs_dir) if graphs_dir is not None else GRAPHS_DIR
    owner = _mirror_owner(name, table_id, project, graph, base)
    if owner is None:
        return None
    try:
        import zettelkasten.organizations as organizations

        org = organizations.load_organization(
            owner[0], owner[1], organizations.org_id_for(name, table_id), graphs_dir=base
        )
    except Exception:  # noqa: BLE001 — a registry miss must never fail a build
        return None
    if not isinstance(org, dict):
        return None
    return org.get("overlay")

record_attribution

record_attribution(name: str, table_id: str, *, member_uid: str, dim_id: str, decision: str, note_id: str = '', source_graph: str = '', title: str = '', project: str = '', graph: str = '', graphs_dir: 'Path | None' = None) -> bool

Durably record a blank-edge collision attribution on the table's overlay.

decision='own' CLAIMS the ambiguous member as THIS spine's — on the next build it is no longer excluded and rejoins the grid as a normal row. decision='foreign' marks it not-mine — it stays excluded but leaves the needs-attribution bucket. The entry is keyed by (member_uid, dim_id) (a re-record on the same key REPLACES the prior decision) so it survives rebuilds and re-partitions. Returns False when no backing org exists yet (build the table first so its definition is mirrored). Raises ValueError on a bad decision.

Source code in zettelkasten/tables_persist.py

def record_attribution(
    name: str,
    table_id: str,
    *,
    member_uid: str,
    dim_id: str,
    decision: str,
    note_id: str = "",
    source_graph: str = "",
    title: str = "",
    project: str = "",
    graph: str = "",
    graphs_dir: "Path | None" = None,
) -> bool:
    """Durably record a blank-edge collision attribution on the table's overlay.

    ``decision='own'`` CLAIMS the ambiguous member as THIS spine's — on the next
    build it is no longer excluded and rejoins the grid as a normal row.
    ``decision='foreign'`` marks it not-mine — it stays excluded but leaves the
    needs-attribution bucket. The entry is keyed by ``(member_uid, dim_id)`` (a
    re-record on the same key REPLACES the prior decision) so it survives rebuilds
    and re-partitions. Returns ``False`` when no backing org exists yet (build the
    table first so its definition is mirrored). Raises ``ValueError`` on a bad
    decision.
    """
    if decision not in ("own", "foreign"):
        raise ValueError("attribution decision must be 'own' or 'foreign'")
    base = Path(graphs_dir) if graphs_dir is not None else GRAPHS_DIR
    member_uid = (member_uid or "").strip()
    dim_id = (dim_id or "").strip()
    if not (member_uid and dim_id):
        raise ValueError("attribution requires a member_uid and a dim_id")
    owner = _mirror_owner(name, table_id, project, graph, base)
    if owner is None:
        return False
    try:
        import zettelkasten.organizations as organizations
        from zettelkasten.commit import review_write_lock

        oid = organizations.org_id_for(name, table_id)
        ot, on = owner[0], owner[1]
        # The whole load → mutate → save is ONE critical section under the owner
        # lock: two concurrent attributions to the same table would otherwise read
        # the same base overlay, each append its own entry, and the last writer
        # would drop the other's decision (lost update → a member silently reverts
        # to unresolved on the next build). Hold the NON-REENTRANT owner lock once
        # and persist via the lock-free ``_save_organization_locked`` so we do not
        # re-acquire it (which would deadlock). ``load_organization(migrate=False)``
        # is a lock-free read, safe to call inside the held lock.
        with review_write_lock(organizations._owner_lock_name(ot, on), graphs_dir=base):
            org = organizations.load_organization(ot, on, oid, graphs_dir=base, migrate=False)
            if not isinstance(org, dict):
                return False
            overlay = _normalize_table_overlay(org.get("overlay"))
            overlay["attributions"] = [
                a for a in overlay["attributions"]
                if not (a["member_uid"] == member_uid and a["dim_id"] == dim_id)
            ]
            overlay["attributions"].append(
                {
                    "member_uid": member_uid,
                    "dim_id": dim_id,
                    "decision": decision,
                    "note_id": note_id,
                    "graph": source_graph,
                    "title": title,
                }
            )
            org["overlay"] = overlay
            organizations._save_organization_locked(base, org)
    except Exception:  # noqa: BLE001 — a registry hiccup must surface as a no-op, not a crash
        logger.warning("record_attribution failed for %s/%s", name, table_id, exc_info=True)
        return False
    return True

count_unrouted

count_unrouted(get_graph: GetGraph, table: dict[str, Any], *, project: str = '', graph: str = '', graphs_dir: 'Path | None' = None, localize: 'Callable[[str], str] | None' = None) -> dict[str, Any]

A cheap drift signal: in-scope notes attached to NO cell/dimension.

Compares the corpus's current in-scope value notes against the payload's stamped attached_note_ids. Surfaces {"unrouted", "unrouted_ids", "attached", "stale"} — the per-view "N new notes unrouted · Resync" cue. The in-scope universe spans every scope graph (incl. _cross); stale flags a built signature that no longer matches the corpus.

Source code in zettelkasten/tables_persist.py

def count_unrouted(
    get_graph: GetGraph,
    table: dict[str, Any],
    *,
    project: str = "",
    graph: str = "",
    graphs_dir: "Path | None" = None,
    localize: "Callable[[str], str] | None" = None,
) -> dict[str, Any]:
    """A cheap drift signal: in-scope notes attached to NO cell/dimension.

    Compares the corpus's current in-scope value notes against the payload's
    stamped ``attached_note_ids``. Surfaces ``{"unrouted", "unrouted_ids",
    "attached", "stale"}`` — the per-view "N new notes unrouted · Resync" cue. The
    in-scope universe spans every scope graph (incl. ``_cross``); ``stale`` flags
    a built signature that no longer matches the corpus.
    """
    base = Path(graphs_dir) if graphs_dir is not None else GRAPHS_DIR
    loc = localize or (lambda s: s)
    attached = set(table.get("attached_note_ids") or [])
    unrouted: list[str] = []
    for _name, _zg, note in _universe_pairs(
        get_graph, project=project, graph=graph, base=base, loc=loc
    ):
        if note.type in _NON_VALUE_TYPES:
            continue
        if any(t in _NON_VALUE_TAGS for t in (note.tags or [])):
            continue
        if note.id not in attached:
            unrouted.append(note.id)
    name = graph or project or "all"
    try:
        current_sig = _table_signature(
            normalize_columns(table.get("columns")),
            row_axis=table.get("row_axis"),
            project=project,
            graph=graph,
            name=name,
            base=base,
        )
    except Exception:
        current_sig = ""
    built = str(table.get("built_signature") or table.get("signature") or "")
    return {
        "unrouted": len(unrouted),
        "unrouted_ids": unrouted,
        "attached": len(attached),
        "stale": bool(built and current_sig and built != current_sig),
    }