Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 29 additions & 30 deletions src/oikb/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,13 @@ def _resolve_connector(source: str, branch: str | None = None, path: str | None
)

# Default: local filesystem.
if ":" not in source and source.isascii() and source.isupper() and source.isalnum():
raise ValueError(
f"Unrecognized source {source!r}. Bare uppercase names map to "
f"/data/{source} on the filesystem. For Confluence spaces use "
f"confluence:{source}."
)

from oikb.connectors.filesystem import FilesystemConnector
return FilesystemConnector(source)

Expand Down Expand Up @@ -397,46 +404,38 @@ def sync(
sys.exit(1)

has_errors = False
for entry in entries:
entry_source = entry.get("source")
entry_kb = entry.get("kb-id")
entry_branch = entry.get("branch")
entry_path = entry.get("path")
entry_filter = entry.get("filter", {})

if not entry_source or not entry_kb:
click.echo(click.style(f"Skipping invalid entry (needs source + kb-id): {entry}", fg="yellow"), err=True)
from oikb.kb_sync import group_entries_by_kb, run_entries_sync, sources_label

for group in group_entries_by_kb(entries):
entry_kb = group[0].get("kb-id")
label = sources_label(group)

if not entry_kb or any(not e.get("source") for e in group):
click.echo(
click.style(
f"Skipping invalid group (needs source + kb-id): {group}",
fg="yellow",
),
err=True,
)
continue

try:
connector = _resolve_connector(entry_source, entry_branch, entry_path)
client = _make_client(url, token)

if not quiet:
click.echo(f"\n{'─' * 40}")
click.echo(f"Syncing: {entry_source} → {entry_kb}")

mf = None
inc = entry_filter.get("include")
exc = entry_filter.get("exclude")
ms = entry_filter.get("max-size") or max_file_size
if inc or exc or ms:
from oikb.sync import build_manifest_filter, parse_size
mf = build_manifest_filter(
include=inc,
exclude=exc,
max_size=parse_size(ms),
)

result = run_sync(
client=client,
connector=connector,
kb_id=entry_kb,
click.echo(f"Syncing: {label} → {entry_kb}")

result = run_entries_sync(
client,
group,
resolve_connector=_resolve_connector,
dry_run=dry_run,
verbose=verbose,
quiet=quiet,
manifest_filter=mf,
concurrency=entry.get("concurrency", concurrency),
concurrency=concurrency,
max_file_size=max_file_size,
)

if not quiet:
Expand Down
46 changes: 46 additions & 0 deletions src/oikb/connectors/composite.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
"""Composite connector — merge multiple sources into one KB manifest."""

from __future__ import annotations

from oikb.connectors import BaseConnector, ManifestEntry


class CompositeConnector(BaseConnector):
"""Route read_file calls across connectors with a pre-merged manifest.

Used when several .oikb.yaml sources target the same kb-id. Each source
sync alone would drop files from the others (OWUI diff is KB-wide).
"""

def __init__(self, parts: list[tuple[BaseConnector, list[ManifestEntry]]]):
self._parts = parts
self._route: dict[tuple[str, str], BaseConnector] = {}
self._manifest: list[ManifestEntry] = []

for connector, manifest in parts:
for entry in manifest:
key = (entry.path, entry.filename)
if key in self._route:
raise ValueError(
f"Duplicate manifest path across sources: {entry.display_path}"
)
self._route[key] = connector
self._manifest.append(entry)

def build_manifest(self) -> list[ManifestEntry]:
return list(self._manifest)

def read_file(self, path: str, filename: str) -> bytes:
connector = self._route.get((path, filename))
if not connector:
display = f"{path}/{filename}" if path else filename
raise FileNotFoundError(f"File not in merged manifest: {display}")
return connector.read_file(path, filename)

def close(self) -> None:
closed: set[int] = set()
for connector, _ in self._parts:
token = id(connector)
if token not in closed:
connector.close()
closed.add(token)
Loading