From 9c8f7e04d310e276c21c5a21f38a1ff68a5fcf7c Mon Sep 17 00:00:00 2001 From: edwardvaneechoud Date: Fri, 3 Jul 2026 19:27:49 +0200 Subject: [PATCH 1/2] Improve documentation structure --- .claude/launch.json | 6 + .../skills/flowfile-docs-and-writing/SKILL.md | 25 +- .claude/skills/flowfile-docs-review/SKILL.md | 136 +++ .gitignore | 1 + data/templates/flows/sales_pipeline.yaml | 111 ++ data/templates/flows/sales_to_catalog.yaml | 85 ++ data/templates/generate_template_data.py | 49 + data/templates/supermarket_sales.csv | 1031 +++++++++++++++++ docs/MakeFile | 23 - docs/ai/index.md | 2 +- docs/ai/providers.md | 15 +- docs/assets/flows/sales_pipeline.yaml | 111 ++ docs/assets/try-sales-pipeline.html | 11 + docs/community.md | 2 +- docs/conf.py | 91 -- docs/examples/catalog_analysis.py | 36 + docs/examples/code_to_flow.py | 59 + docs/examples/custom_node.py | 56 + docs/examples/first_pipeline.py | 21 + .../examples/integrations/cloud_storage_s3.py | 55 + docs/examples/integrations/database_read.py | 41 + docs/examples/integrations/kafka_read.py | 63 + docs/examples/ml_pipeline.py | 46 + docs/examples/python_api_aggregations.py | 49 + docs/examples/python_api_joins.py | 51 + docs/examples/python_api_operations.py | 52 + docs/examples/sales_pipeline.py | 31 + docs/for-developers/ai-architecture.md | 10 +- docs/for-developers/architecture.md | 118 +- docs/for-developers/creating-custom-nodes.md | 283 ++--- docs/for-developers/custom-node-tutorial.md | 313 +---- docs/for-developers/design-philosophy.md | 331 ++---- docs/for-developers/docker-deployment.md | 220 ---- docs/for-developers/flowfile-core.md | 66 +- docs/for-developers/index.md | 58 +- docs/for-developers/kernel-architecture.md | 43 +- docs/for-developers/python-api-reference.md | 6 +- docs/for-developers/visualizations.md | 10 +- docs/index.html | 567 ++++----- docs/quickstart.md | 666 ++--------- docs/stylesheets/extra.css | 125 +- docs/users/coming-from-excel.md | 62 + docs/users/connect/apis.md | 101 ++ docs/users/connect/index.md | 50 + docs/users/connect/kafka.md | 78 ++ docs/users/deployment/cli.md | 122 ++ docs/users/deployment/desktop.md | 38 +- docs/users/deployment/docker.md | 161 ++- docs/users/deployment/index.md | 14 - docs/users/deployment/lite.md | 45 +- docs/users/deployment/python.md | 50 +- docs/users/deployment/sharing.md | 93 ++ docs/users/formulas/index.md | 6 +- docs/users/index.md | 97 +- docs/users/projects.md | 78 ++ .../python-api/concepts/design-concepts.md | 257 +--- docs/users/python-api/concepts/expressions.md | 11 +- docs/users/python-api/concepts/formulas.md | 4 +- docs/users/python-api/concepts/index.md | 98 +- docs/users/python-api/index.md | 34 +- docs/users/python-api/quickstart.md | 190 +-- .../python-api/reference/aggregations.md | 74 +- .../reference/catalog-references.md | 11 +- .../python-api/reference/cloud-connections.md | 85 +- docs/users/python-api/reference/data-types.md | 16 +- .../reference/flowframe-operations.md | 83 +- docs/users/python-api/reference/index.md | 20 +- docs/users/python-api/reference/joins.md | 91 +- .../python-api/reference/reading-data.md | 128 +- docs/users/python-api/reference/visual-ui.md | 95 +- .../python-api/reference/writing-data.md | 41 +- .../tutorials/flowfile_frame_api.md | 103 +- docs/users/python-api/tutorials/index.md | 58 +- docs/users/visual-editor/building-flows.md | 247 +--- docs/users/visual-editor/catalog/index.md | 45 +- docs/users/visual-editor/catalog/schedules.md | 20 +- docs/users/visual-editor/catalog/secrets.md | 46 +- .../users/visual-editor/catalog/sql-editor.md | 15 +- .../visual-editor/catalog/virtual-tables.md | 51 +- .../visual-editor/catalog/visualizations.md | 10 +- docs/users/visual-editor/connections.md | 4 +- docs/users/visual-editor/index.md | 100 +- docs/users/visual-editor/kernels.md | 18 +- docs/users/visual-editor/node-designer.md | 85 +- docs/users/visual-editor/nodes/aggregate.md | 39 +- docs/users/visual-editor/nodes/combine.md | 74 +- docs/users/visual-editor/nodes/index.md | 44 +- docs/users/visual-editor/nodes/input.md | 27 +- docs/users/visual-editor/nodes/ml.md | 37 +- docs/users/visual-editor/nodes/output.md | 67 +- docs/users/visual-editor/nodes/transform.md | 68 +- docs/users/visual-editor/settings.md | 6 +- docs/users/visual-editor/subflows.md | 111 ++ .../tutorials/cloud-connections.md | 51 +- .../visual-editor/tutorials/code-generator.md | 28 +- .../tutorials/database-connectivity.md | 194 ++-- docs/users/visual-editor/tutorials/index.md | 31 +- .../visual-editor/tutorials/sales-pipeline.md | 74 ++ .../tests/docs_examples/__init__.py | 0 .../tests/docs_examples/test_docs_examples.py | 123 ++ mkdocs.yml | 146 ++- poetry.lock | 45 +- pyproject.toml | 1 + 103 files changed, 5169 insertions(+), 4107 deletions(-) create mode 100644 .claude/skills/flowfile-docs-review/SKILL.md create mode 100644 data/templates/flows/sales_pipeline.yaml create mode 100644 data/templates/flows/sales_to_catalog.yaml create mode 100644 data/templates/supermarket_sales.csv delete mode 100644 docs/MakeFile create mode 100644 docs/assets/flows/sales_pipeline.yaml create mode 100644 docs/assets/try-sales-pipeline.html delete mode 100644 docs/conf.py create mode 100644 docs/examples/catalog_analysis.py create mode 100644 docs/examples/code_to_flow.py create mode 100644 docs/examples/custom_node.py create mode 100644 docs/examples/first_pipeline.py create mode 100644 docs/examples/integrations/cloud_storage_s3.py create mode 100644 docs/examples/integrations/database_read.py create mode 100644 docs/examples/integrations/kafka_read.py create mode 100644 docs/examples/ml_pipeline.py create mode 100644 docs/examples/python_api_aggregations.py create mode 100644 docs/examples/python_api_joins.py create mode 100644 docs/examples/python_api_operations.py create mode 100644 docs/examples/sales_pipeline.py delete mode 100644 docs/for-developers/docker-deployment.md create mode 100644 docs/users/coming-from-excel.md create mode 100644 docs/users/connect/apis.md create mode 100644 docs/users/connect/index.md create mode 100644 docs/users/connect/kafka.md create mode 100644 docs/users/deployment/cli.md create mode 100644 docs/users/deployment/sharing.md create mode 100644 docs/users/projects.md create mode 100644 docs/users/visual-editor/subflows.md create mode 100644 docs/users/visual-editor/tutorials/sales-pipeline.md rename docs/requirements.txt => flowfile_core/tests/docs_examples/__init__.py (100%) create mode 100644 flowfile_core/tests/docs_examples/test_docs_examples.py diff --git a/.claude/launch.json b/.claude/launch.json index 156aab530..44f818965 100644 --- a/.claude/launch.json +++ b/.claude/launch.json @@ -6,6 +6,12 @@ "runtimeExecutable": "npm", "runtimeArgs": ["run", "dev", "--prefix", "flowfile_wasm"], "port": 5174 + }, + { + "name": "docs-site", + "runtimeExecutable": "python3", + "runtimeArgs": ["-m", "http.server", "8765", "--directory", "site"], + "port": 8765 } ] } diff --git a/.claude/skills/flowfile-docs-and-writing/SKILL.md b/.claude/skills/flowfile-docs-and-writing/SKILL.md index 317e7eb8b..e30f6f89b 100644 --- a/.claude/skills/flowfile-docs-and-writing/SKILL.md +++ b/.claude/skills/flowfile-docs-and-writing/SKILL.md @@ -7,6 +7,7 @@ description: How the Flowfile docs site (MkDocs + Material, the docs/ tree, mkdo ## When NOT to use this skill +- Editorial standard for docs pages (house style/voice rules, the persona nav map, fact-checking a docs claim against code, the tested-examples contract and add-an-example recipe) → `flowfile-docs-review` (this skill owns *how the site builds*; that one owns *what good pages say and how to verify it*). - CI gates that happen to touch docs (`check-formula-docs`, `check-stubs` jobs, version-sync) and release mechanics → `flowfile-change-control` (this skill explains *what* the formula-docs generator does and the doc-authoring traps; that skill owns *why the CI job exists and what blocks a merge*). - flowfile_frame public API design, `.pyi` stub authoring, expression codegen → `flowfile-frame-and-codegen`. - Writing/reviewing a custom node's `NodeSettings` docstrings for the Node Designer UI → `flowfile-node-development`. @@ -23,7 +24,7 @@ Two documentation surfaces exist; don't confuse them: | Surface | Source | Audience | |---|---|---| -| **MkDocs site** (`docs/`) | ~64 Markdown pages + `docs/index.html`, built by `mkdocs.yml`, deployed to `https://edwardvaneechoud.github.io/Flowfile/` | End users (visual editor, Python API) and external contributors | +| **MkDocs site** (`docs/`) | Markdown pages + `docs/index.html` (count them: `find docs -name '*.md' | wc -l`), built by `mkdocs.yml`, deployed to `https://edwardvaneechoud.github.io/Flowfile/` | End users (visual editor, Python API) and external contributors | | **CLAUDE.md files** (9: root + 8 packages) | Not part of the MkDocs build; plain files read by AI agents and human contributors | Anyone working *inside* the repo — see §6 | The `docs/` tree structure mirrors `mkdocs.yml`'s `nav:` block almost exactly (verify by diffing them — see §3.3 for the one place they currently disagree). Top-level layout: @@ -74,14 +75,7 @@ Every new page must be added to `mkdocs.yml`'s `nav:` block by hand — MkDocs d ### 3.3 Forgetting the `nav:` entry makes a page invisible, not broken -MkDocs builds every `.md` file under `docs/` whether or not it's listed in `mkdocs.yml`'s `nav:` — a missing nav entry does **not** fail the build (§2's "no `--strict`" gotcha) and does **not** stop the page from existing at its URL if something else links to it directly. It just means the page is absent from the sidebar and site search weighting. This is not hypothetical: as of 2026-07-03, `docs/for-developers/docker-deployment.md` (the doc covering Docker deployment and Group-Based Sharing for multi-user mode) exists on disk, is linked *to* from `users/visual-editor/catalog/secrets.md` (the only real inbound page link — `users/deployment/docker.md` merely references a similarly-named image asset directory), but is **not in `mkdocs.yml`'s `nav:` at all**. Running `poetry run mkdocs build` prints exactly: - -``` -INFO - The following pages exist in the docs directory, but are not included in the "nav" configuration: - - for-developers/docker-deployment.md -``` - -and still exits 0. If you add a new page and it doesn't show up in the sidebar after a build, this is almost certainly why — check `mkdocs.yml`'s `nav:` block before assuming something is actually broken. +MkDocs builds every `.md` file under `docs/` whether or not it's listed in `mkdocs.yml`'s `nav:` — a missing nav entry does **not** fail the build (§2's "no `--strict`" gotcha) and does **not** stop the page from existing at its URL if something else links to it directly. It just means the page is absent from the sidebar and site search weighting. The build prints an `INFO` line ("The following pages exist in the docs directory, but are not included in the 'nav' configuration") and still exits 0. If you add a new page and it doesn't show up in the sidebar after a build, this is almost certainly why — check `mkdocs.yml`'s `nav:` block before assuming something is actually broken. (The long-standing live example, an orphaned `for-developers/docker-deployment.md`, was merged into `users/deployment/docker.md` in the 2026-07 docs rebuild, with a `redirects`-plugin mapping preserving its URL — as of that rebuild the nav is orphan-free, and the redirect map in `mkdocs.yml` is where moved pages get recorded.) ### 3.4 The formula reference is generated — never hand-edit it @@ -102,12 +96,7 @@ If you bump the `polars-expr-transformer` version pin (or edit `tools/generate_f ## 4. House style for docs pages -Two registers coexist in `docs/`; know which one you're extending: - -1. **Legacy/marketing register** — `quickstart.md`, `users/index.md`, `nodes/index.md`, most `python-api/` pages, `design-philosophy.md`: first-person-plural, emoji section headers (🎨 🐍 ✅ 🚀), heavy inline-styled HTML (`clamp()` font sizing, gradient `
` cards), `
` collapsibles, hype prose. -2. **Modern/engineering register** — `ai/index.md`, `deployment/lite.md` + `deployment/index.md`, `kernel-architecture.md`, `ai-architecture.md`, `visualizations.md`, `docker-deployment.md`'s sharing section, `community.md`: tight declarative prose, Material admonitions (`!!! info`, `!!! tip`, `!!! warning`, with a quoted title), comparison tables, exact file paths and symbol names, explicit invariant statements ("There is no min/max gate and nothing that rejects…"), mermaid diagrams where a picture beats prose. - -**Write new pages in register 2.** Recent doc commits (LSP notebook docs, project-tracking docs, node-reference docs, formula-docs regeneration) all trend toward register 2, and it matches the voice of `CONTRIBUTING.md` and every `CLAUDE.md` file. Register 1 pages are legacy, not a style to imitate going forward. +Since the 2026-07 docs rebuild there is **one register**: tight declarative prose, Material admonitions (`!!! info`, `!!! tip`, `!!! warning`, with a quoted title), comparison tables, exact file paths and symbol names, explicit invariant statements, mermaid diagrams where a picture beats prose. The old marketing register (emoji headers, gradient `
` cards, `clamp()` inline CSS, hype prose) was eliminated page-by-page — do not reintroduce it. The full editorial standard — voice rules (including the ban on generic "Tips for Success" filler), the persona nav map, the claim→source verification index, and the tested-examples contract — lives in `flowfile-docs-review`; read that skill before writing or reviewing any page. Admonition syntax (register 2's signature move): @@ -185,9 +174,7 @@ Gaps worth knowing so you don't assume something is documented when it isn't, an - **No PR template, no feature-request issue template.** The only issue template (`.github/ISSUE_TEMPLATE/bug_report.md`) is GitHub's unmodified default — it still asks for "Smartphone… Device: [e.g. iPhone6]," which makes no sense for a desktop/server ETL tool. `CONTRIBUTING.md`'s prose ("what changed, why, how you tested it; screenshots for UI changes") is the only PR-description guidance that exists. - **No `SECURITY.md`** at the repo root or in `.github/`. The private-vulnerability-reporting policy (GitHub's private "Report a vulnerability" form) is documented only inside `CONTRIBUTING.md`'s "Reporting security issues" section. - **`claude.yml` and `claude-pr-review.yml`** (interactive `@claude`-mention agent, and an automatic Claude code review posted on every non-draft PR) are real, tracked workflow files in `.github/workflows/` but are not mentioned in any docs page or in `CONTRIBUTING.md`'s workflow list — a first-time contributor gets an automated review with no explanation of where it came from. -- **Flow-in-flow** (subflows — PR #568, merged) has no docs page anywhere under `docs/` as of 2026-07-03 (a `docs/users/python-api/tutorials/flowfile_frame_api.md` hit for "Flow in Flow" is a false positive — it's a `` caption unrelated to the feature). -- **Project git-tracking** (`flowfile project init|open|save`, `FLOWFILE_ENABLE_PROJECTS`) gets only a passing mention in `docs/users/deployment/docker.md`; there's no dedicated user guide despite it being a full feature (#524). -- **Group-based sharing** (multi-user resource sharing) is documented for developers in the orphaned `for-developers/docker-deployment.md` (§3.3) but has no end-user guide page. +- ~~Flow-in-flow, project git-tracking, group-based sharing undocumented~~ — closed by the 2026-07 rebuild: `users/visual-editor/subflows.md`, `users/projects.md`, `users/deployment/sharing.md`, plus `users/deployment/cli.md` (headless runs), `users/connect/{index,kafka,apis}.md`, and `users/coming-from-excel.md` now exist and are in nav. If you're asked to fill one of these gaps, write it in register 2 (§4), put user-facing content under `users/` and internals under `for-developers/`, and don't forget the `nav:` entry (§3.3). @@ -200,7 +187,7 @@ All facts above were spot-verified in this repo on 2026-07-03 against app versio ```bash # Docs site config and orphan-page check (reproduces the exact CI build) poetry run mkdocs build 2>&1 | grep -E "not included in the|^ERROR" -# expect: "for-developers/docker-deployment.md" flagged as not in nav, and no line starting "ERROR" +# expect: no orphan pages and no line starting "ERROR" (orphans were eliminated in the 2026-07 rebuild) # Confirm the formula reference is still generated (not hand-edited) and current head -1 docs/users/formulas/functions.md # expect the AUTO-GENERATED header diff --git a/.claude/skills/flowfile-docs-review/SKILL.md b/.claude/skills/flowfile-docs-review/SKILL.md new file mode 100644 index 000000000..a3488c760 --- /dev/null +++ b/.claude/skills/flowfile-docs-review/SKILL.md @@ -0,0 +1,136 @@ +--- +name: flowfile-docs-review +description: The editorial standard for Flowfile's docs site — the house style (register-2 voice rules), the persona-based nav map (which tab serves which arriving audience), the claim-type→source-of-truth verification index for fact-checking any docs statement against code, and the zero-drift examples contract (how tested .py and .yaml examples are structured, included via snippets, and auto-tested). Use when writing or reviewing any page under docs/, fact-checking a docs claim (node counts, Lite availability, API signatures, ports, providers, defaults), adding a worked example or tutorial, deciding where a new page belongs in the persona nav, or running an editorial/lint pass over docs changes. +--- + +# Flowfile docs review — style, personas, verification, examples + +## When NOT to use this skill + +- Site build mechanics, `mkdocs.yml` traps (`use_directory_urls`, nav invisibility, the raw-HTML home page), deploy pipeline, formula-docs regeneration, comment doctrine for source code, CLAUDE.md maintenance → `flowfile-docs-and-writing` (this skill owns *what good pages say and how to verify it*; that one owns *how the site is built*). +- CI gates and release mechanics → `flowfile-change-control`. +- Per-package test commands and Docker fixture mechanics beyond the examples contract → `flowfile-testing-and-validation`. + +## 1. House style (register 2 — the only register for new/edited pages) + +The canonical exemplar is `docs/ai/index.md`. Rules, all checkable: + +1. **Declarative present tense, second person where needed.** No first-person-plural marketing ("we've got you covered"), no hype adjectives (powerful, seamless, blazing, incredible, professional-grade), no exclamation-mark enthusiasm, no unverifiable stats ("thousands of users"). +2. **No emoji** in headings, bullets, or feature lists. No `## **Bold-in-heading**` markdown. +3. **No inline-styled HTML blocks** in `.md` pages (gradient divs, `clamp()` fonts, `style=` attributes). Visual variety is wanted — plain-markdown-only pages read as flat — but it comes exclusively from the shared brand layer: Material **grid cards** (`
`, brand accent bar styled globally in `docs/stylesheets/extra.css`), **content tabs** (`=== "Label"` — use for install methods and platform variants), **icon shortcodes** (`:material-*:` / `:octicons-*:` via `pymdownx.emoji`), the `ff-paths`/`ff-path-teal`/`ff-path-purple` chooser classes from `extra.css`, admonitions, and mermaid. New visual components get a class in `extra.css` (both color schemes), never a `style=` attribute. Section indexes and audience routers should use cards; high-traffic pages (quickstart, guides-by-audience) carry the most visual weight, deep reference pages the least. +4. **No "coming soon" / roadmap promises.** Document what ships. Aspirational notes only as a short "Future direction" admonition on developer pages, clearly labeled. +5. **Material admonitions** (`!!! note "Quoted Title"`, `tip`, `warning`, `info`) for asides; roughly ≤2 per screenful. +6. **Exact names everywhere**: UI labels as the component renders them (Group By's average is **Mean**, `GroupBy.vue`), API symbols as actually exported, file paths and env vars in backticks. When docs and UI disagree, the Vue component is the truth. +7. **Numbers rot — prefer pointers.** "See `configs/node_store/nodes.py`" beats "46 nodes". A literal count needs an entry in §3's index and, if fast-rotting (versions, model names, node counts), a date stamp ("as of 2026-07"). +8. **Runnable code is never hand-written in a page.** Any Python block presented as runnable is a `--8<--` include from `docs/examples/` (§4). Inline fragments are allowed only for formulas, UI config values, or shell one-liners — with verified syntax. +9. **Every page opens with one orientation paragraph**: who it's for, what they'll be able to do after reading. Then short sections; tables for enumerable facts with the explanation in surrounding prose. +10. **Links**: relative `.md` links between pages (MkDocs rewrites them); raw `.html`/directory forms only inside `docs/index.html`. Every new page gets a `nav:` entry; moved/merged URLs get a `redirects` plugin mapping in `mkdocs.yml`. +11. **Images**: never create or edit image files; never block on one. Reuse an existing `docs/assets/images/` asset if apt, else leave `` at the spot. In step-by-step walkthroughs, per-step screenshots go in fold-outs so they don't break the reading flow (maintainer-preferred pattern): `
See it: …` wrapping the image plus its placeholder/refresh comment. +12. **Cross-page consistency beats local polish**: one canonical statement per fact, siblings link to it (the join-type list, the Lite node inventory, and the custom-node `process()` signature have each previously diverged across three pages). +13. **No filler advice — maintainer-rejected pattern.** Generic "Tips for Success" bullet lists ("Start simple", "Save regularly", "Preview often", "Use descriptions", "Try both modes") are banned. If a tip isn't specific to the page's subject and non-obvious, cut it. Never state the obvious ("click Run to run the flow"). +14. **Length is a quality dimension.** Each page earns its length: getting-started/how-to pages are short concrete numbered steps; reference pages are complete tables; concept pages explain once and stop. If a section restates what an adjacent section or page already says, cut or link. +15. **Model division of labor**: subagent drafts are raw material. The final shipped text of user-facing pages is written/edited by the lead (Fable) after a dedicated scan for filler, obviousness, length, and cross-page consistency. + +## 2. Persona nav map (target IA) + +One tab per arriving audience; the Home page routes with persona cards (one hop). + +| Tab | Arriving reader | Owns | +|---|---|---| +| Home (`index.html`) | everyone | value prop, sales-pipeline showcase (tested flow download + demo.flowfile.org), persona router | +| Get Started | new user, any kind | `quickstart.md` (install + first visual flow + first Python pipeline), Coming from Excel, Flowfile Lite | +| Visual Editor | analysts building flows | overview, building flows, formulas (+ generated function reference), node reference (6 categories), kernels, node designer, worked examples | +| Connect Your Data | "my data lives elsewhere" | connector matrix, connections & secrets, databases, cloud storage (S3/ADLS/GCS), Kafka, REST APIs & Google Analytics | +| Catalog & Automation | analyzing/operating data | catalog, virtual tables, SQL editor, visualizations, schedules, subflows, projects & git | +| Python API | Python developers | quickstart, concepts, reference, tutorials (all examples tested) | +| AI Assistant | any | feature catalog, provider setup (BYOK) | +| Deploy & Operate | admins | desktop, pip, Docker (single merged page), users/groups/sharing, headless runs & CLI | +| For Developers | contributors | architecture, internals, kernel, AI architecture, custom nodes | + +The **analysis** journey (analyze data without building pipelines) is deliberately multi-hooked: Home persona card → Catalog tab; quickstart's visual track ends in Catalog Writer → SQL editor → visualization; `flowfile seed-demo` documented as the one-command populated catalog. + +## 3. Claim-type → source-of-truth index + +Verify against code, never against another prose doc (READMEs and CLAUDE.md drift too). The most drift-prone claim types and where each is decided: + +| Claim about | Source of truth | +|---|---| +| Core node types, categories, laziness, narrow/wide | `flowfile_core/flowfile_core/configs/node_store/nodes.py` (`get_all_standard_nodes`; plus dict-only `polars_lazy_frame`) | +| Lite/WASM node availability | `flowfile_wasm/src/config/nodeDescriptions.ts` + `flowfile_wasm/src/components/Canvas.vue` `nodeCategories` (`available: false` flags). 23 usable nodes as of 2026-07 — historic "18" was wrong | +| Join strategies, fuzzy algorithms, group-by agg options | `flowfile_core/.../schemas/transform_schema.py` (`JoinKeyStrategy` = inner/left/right/full/semi/anti/outer; `FuzzyTypeLiteral` = 6 algorithms); UI labels in `GroupBy.vue` (`mean` → "Mean") | +| Node settings fields, file formats, write modes | `flowfile_core/flowfile_core/schemas/input_schema.py`; cloud: `cloud_storage_schemas.py` (`CloudStorageType` s3/adls/gcs; `AuthMethod` — the CLI literal is `aws-cli`, hyphen; read formats include `iceberg`) | +| `ff.*` availability | `flowfile/flowfile/__init__.py` — NOT the same as `flowfile_frame/flowfile_frame/__init__.py` (e.g. `read_ipc`/`read_ndjson`/`read_avro` exist in frame but are not re-exported as `ff.*`) | +| Expression methods | `flowfile_frame/flowfile_frame/expr.py` + pinned Polars (renames: `cum_sum` not `cumsum`, `weekday` not `day_of_week`; no `FlowFrame.drop_duplicates`/`vstack`/`__len__`) | +| AI providers and models | `flowfile_core/flowfile_core/ai/providers/registry.py` (6 BYOK + local pseudo-provider); per-provider `default_model` vs per-surface models differ — Groq default is `qwen/qwen3-32b` | +| CLI verbs and flags | `flowfile/flowfile/__main__.py` (run ui/core/worker/flow, seed-demo, remove-demo, project init/open/save) | +| Ports | `flowfile/flowfile/api.py`, `shared/storage_config.py`; the web UI is hard-locked to 63578 (`flowfile/web/__init__.py` raises on any other port — `FLOWFILE_PORT` does not move it) | +| Health probes | `flowfile_core/routes/public.py` — `/health/status`; there is no `/health` on core or worker | +| Kernel images and defaults | `flowfile_core/flowfile_core/kernel/manager.py` (image tags), `kernel/models.py` (default memory 4 GB, CPU 2) | +| Password/auth policy | `flowfile_core/auth/password.py` (8 chars + number + special; no case rules) | +| Storage paths per mode | `shared/storage_config.py` + `docker-compose.yml` overrides (`FLOWFILE_USER_DATA_DIR=/app/user_data` in shipped compose) | +| Docker deployment facts | `docker-compose.yml` itself (volume `flowfile-internal-storage`, `shm_size`, scheduler enabled in shipped compose, `FLOWFILE_INTERNAL_TOKEN` required) | +| Catalog internals | `flowfile_core/flowfile_core/catalog/` (services/, `constants.py` — thumbnail cap 500 KB, SQL recursion limit 5) | +| Flow save format | `.yaml` default (`.yml`/`.json` accepted; `.flowfile` is legacy pickle, open-only) — `flowfile/manage/io_flowfile.py` | +| Formula functions | generated `docs/users/formulas/functions.md` (never hand-edit; `make formula_docs`) | +| App version | root `pyproject.toml` only — never hardcode in prose | + +## 4. Zero-drift examples contract + +Two example kinds, two automatic gates each (pytest at runtime, `pymdownx.snippets check_paths: true` at build time). + +**Visual flow examples** — committed at `data/templates/flows/.yaml`: +- Carry `_template_meta` (template_id, name, category Beginner/Intermediate/Advanced, tags, node_count, icon) and `_required_csv_files`; read nodes use `__TEMPLATE_DATA_DIR__/.csv`. +- Auto-tested with zero new code: `flowfile_core/tests/templates/test_template_flows.py` globs the directory, validates, substitutes the placeholder, opens via `open_flow`, runs with `execution_location="local"` (no worker), asserts success. +- Auto-surfaced in the in-app template browser (`GET /templates/`, `POST /templates/{id}/create`). +- Author flows by building them in-process and `FlowGraph.save_flow()` — never hand-write node bodies. + +**Python examples** — committed at `docs/examples/.py` (Docker-dependent: `docs/examples/integrations/`): +- One-line docstring; doc-visible code between `# --8<-- [start:example]` / `# --8<-- [end:example]`; real assertions BELOW the end marker (tests run them, pages never show them). +- Use `import flowfile as ff` and only `ff`-namespace exports; repo-root-relative data paths (`data/templates/...`); runner pins CWD to repo root. +- Runner: `flowfile_core/tests/docs_examples/test_docs_examples.py` (glob-parametrized; integrations gated on `test_utils` fixture availability — real Postgres/MinIO/etc., no mocks). +- Pages include with `--8<-- "docs/examples/.py:example"`. + +**Data**: only committed, seeded datasets (`data/templates/*.csv` via `generate_template_data.py`; new generators use their own `random.Random(n)` and are called last so existing CSVs stay byte-identical — verify with `git diff`). Assert exact values for deterministic transforms; schema/shape only for ML and fuzzy outputs. + +**Add-a-worked-example recipe** (the "blog post" flow — no new test code, ever): +1. Pick/extend a committed dataset. +2. Build the flow in-process, save, placeholder-ize paths, add `_template_meta` → drop into `data/templates/flows/`. +3. Optional Python twin in `docs/examples/`. +4. Run `pytest flowfile_core/tests/templates/ flowfile_core/tests/docs_examples/ -q`. +5. Write the page from the fixed skeleton (§5) under the owning persona tab; add the `nav:` entry and a gallery-index row. + +## 5. Worked-example page skeleton (fixed) + +```markdown +# + + + +**Flow:** [`.yaml`](https://github.com/edwardvaneechoud/Flowfile/blob/main/data/templates/flows/.yaml) · +In-app: Create → From template → "" · Data: `data/templates/.csv` + + + +## The data +## The flow +## Run it +## The result +## In Python +## Variations +``` + +## 6. Review checklist (run per touched page) + +0. Filler scan: zero generic tip lists, zero stating-the-obvious lines, no section that merely restates a sibling (§1.13–14). How-to pages read as short concrete steps. +1. Every factual claim is timeless or verified against §3 (spot-check at least the counts, labels, signatures, defaults). +2. Every runnable code block is a snippet include; inline fragments use verified syntax/labels. +3. Zero register-1 markers: emoji headings/bullets, hype adjectives, inline-styled divs, "coming soon", unverifiable stats. +4. Links resolve; anchors exist; page is in `nav:`; moves have redirect mappings. +5. Numbers are pointers or date-stamped. +6. Image spots are placeholders or existing assets — no image files created/edited. +7. `FLOWFILE_SKIP_STARTUP_MIGRATION=1 poetry run mkdocs build` exits 0 with no new WARNINGs (snippets `check_paths` makes missing includes fatal). +8. If examples were touched: `poetry run pytest flowfile_core/tests/templates/ flowfile_core/tests/docs_examples/ -q` green. + +## Provenance + +Distilled from a full-site audit + adversarial fact-check (361 claims verified against source) and a product-surface sweep, 2026-07-03, app version 0.12.7. The §3 index entries are the exact locations that resolved those claims. Re-verify fast-rotting values (counts, versions, model names) against their §3 source before quoting them in new prose. diff --git a/.gitignore b/.gitignore index a66c3e446..3e50c1950 100644 --- a/.gitignore +++ b/.gitignore @@ -104,3 +104,4 @@ error-linux # Signing checklist holds real cert/notarization secrets — never commit flowfile_frontend/src-tauri/SIGNING_CHECKLIST.md +/.claude/agents/ diff --git a/data/templates/flows/sales_pipeline.yaml b/data/templates/flows/sales_pipeline.yaml new file mode 100644 index 000000000..c9757eab6 --- /dev/null +++ b/data/templates/flows/sales_pipeline.yaml @@ -0,0 +1,111 @@ +flowfile_version: 0.8.2 +flowfile_id: 12 +flowfile_name: Sales Pipeline +flowfile_settings: + description: null + execution_mode: Development + execution_location: local + auto_save: false + show_detailed_progress: true + max_parallel_workers: 4 + source_registration_id: null + parameters: [] +nodes: +- id: 1 + type: read + is_start_node: true + description: '' + node_reference: null + x_position: 0 + y_position: 200 + left_input_id: null + right_input_id: null + input_ids: [] + outputs: [2] + setting_input: + cache_results: false + received_file: + path: __TEMPLATE_DATA_DIR__/supermarket_sales.csv + file_type: csv + table_settings: + file_type: csv +- id: 2 + type: unique + is_start_node: false + description: '' + node_reference: null + x_position: 250 + y_position: 200 + left_input_id: null + right_input_id: null + input_ids: [1] + outputs: [3] + setting_input: + cache_results: false + unique_input: + columns: null + strategy: any +- id: 3 + type: filter + is_start_node: false + description: '' + node_reference: null + x_position: 500 + y_position: 200 + left_input_id: null + right_input_id: null + input_ids: [2] + outputs: [4] + setting_input: + cache_results: false + filter_input: + mode: advanced + advanced_filter: '[quantity] > 7' +- id: 4 + type: group_by + is_start_node: false + description: '' + node_reference: null + x_position: 750 + y_position: 200 + left_input_id: null + right_input_id: null + input_ids: [3] + outputs: [5] + setting_input: + cache_results: false + groupby_input: + agg_cols: + - old_name: city + agg: groupby + new_name: city + - old_name: gross_income + agg: sum + new_name: total_income + - old_name: gross_income + agg: median + new_name: median_income +- id: 5 + type: explore_data + is_start_node: false + description: 'Explore the output!' + node_reference: null + x_position: 1000 + y_position: 200 + left_input_id: null + right_input_id: null + input_ids: [4] + outputs: [] + setting_input: + cache_results: false + graphic_walker_input: null +_template_meta: + template_id: sales_pipeline + name: 'Sales pipeline: clean, filter, aggregate' + description: Drop duplicate sales rows, keep high-quantity orders, then total and + median gross income per city. + category: Beginner + tags: [unique, filter, group_by] + node_count: 5 + icon: insights +_required_csv_files: [supermarket_sales.csv] diff --git a/data/templates/flows/sales_to_catalog.yaml b/data/templates/flows/sales_to_catalog.yaml new file mode 100644 index 000000000..7302c4588 --- /dev/null +++ b/data/templates/flows/sales_to_catalog.yaml @@ -0,0 +1,85 @@ +flowfile_version: 0.8.2 +flowfile_id: 13 +flowfile_name: Sales To Catalog +flowfile_settings: + description: null + execution_mode: Development + execution_location: local + auto_save: false + show_detailed_progress: true + max_parallel_workers: 4 + source_registration_id: null + parameters: [] +nodes: +- id: 1 + type: read + is_start_node: true + description: '' + node_reference: null + x_position: 0 + y_position: 200 + left_input_id: null + right_input_id: null + input_ids: [] + outputs: [2] + setting_input: + cache_results: false + received_file: + path: __TEMPLATE_DATA_DIR__/supermarket_sales.csv + file_type: csv + table_settings: + file_type: csv +- id: 2 + type: group_by + is_start_node: false + description: '' + node_reference: null + x_position: 250 + y_position: 200 + left_input_id: null + right_input_id: null + input_ids: [1] + outputs: [3] + setting_input: + cache_results: false + groupby_input: + agg_cols: + - old_name: city + agg: groupby + new_name: city + - old_name: product_line + agg: groupby + new_name: product_line + - old_name: gross_income + agg: sum + new_name: total_income + - old_name: quantity + agg: sum + new_name: units_sold +- id: 3 + type: catalog_writer + is_start_node: false + description: '' + node_reference: null + x_position: 500 + y_position: 200 + left_input_id: null + right_input_id: null + input_ids: [2] + outputs: [] + setting_input: + cache_results: false + catalog_write_settings: + table_name: sales_by_city + namespace_full_name: General.default + write_mode: overwrite +_template_meta: + template_id: sales_to_catalog + name: Sales to Catalog + description: Aggregate sales by city and product line, then publish the result as + a catalog table. + category: Intermediate + tags: [group_by, catalog_writer] + node_count: 3 + icon: inventory +_required_csv_files: [supermarket_sales.csv] diff --git a/data/templates/generate_template_data.py b/data/templates/generate_template_data.py index ac745d6b3..cf04e9f4f 100644 --- a/data/templates/generate_template_data.py +++ b/data/templates/generate_template_data.py @@ -433,6 +433,54 @@ def generate_customer_churn(): ) +def generate_supermarket_sales(): + """~1000 supermarket sales rows for the flagship docs example. + + Uses a dedicated RNG (seed 43) so it doesn't perturb the global-seed + outputs above; must run LAST in __main__. + """ + rng = random.Random(43) + cities = ["Yangon", "Naypyitaw", "Mandalay", "Bago", "Taunggyi"] + customer_types = ["Member", "Normal"] + product_lines = [ + "Health and beauty", + "Electronic accessories", + "Home and lifestyle", + "Sports and travel", + "Food and beverages", + "Fashion accessories", + ] + start = datetime(2024, 1, 1) + end = datetime(2024, 12, 31) + delta_days = (end - start).days + + rows = [] + for i in range(1, 1001): + unit_price = round(rng.uniform(10.0, 100.0), 2) + quantity = rng.randint(1, 10) + gross_income = round(unit_price * quantity * 0.05, 2) + date = (start + timedelta(days=rng.randint(0, delta_days))).strftime("%Y-%m-%d") + rows.append([ + f"INV-{i:05d}", + rng.choice(cities), + rng.choice(customer_types), + rng.choice(product_lines), + unit_price, + quantity, + gross_income, + date, + ]) + + for _ in range(30): + rows.append(list(rng.choice(rows))) + rng.shuffle(rows) + write_csv( + "supermarket_sales.csv", + ["invoice_id", "city", "customer_type", "product_line", "unit_price", "quantity", "gross_income", "date"], + rows, + ) + + if __name__ == "__main__": print("Generating template data files...") generate_sales_data() @@ -446,4 +494,5 @@ def generate_customer_churn(): generate_fuzzy_match_data() generate_house_prices() generate_customer_churn() + generate_supermarket_sales() print("Done!") diff --git a/data/templates/supermarket_sales.csv b/data/templates/supermarket_sales.csv new file mode 100644 index 000000000..b8ae7f48f --- /dev/null +++ b/data/templates/supermarket_sales.csv @@ -0,0 +1,1031 @@ +invoice_id,city,customer_type,product_line,unit_price,quantity,gross_income,date +INV-00516,Bago,Normal,Health and beauty,57.16,4,11.43,2024-01-11 +INV-00163,Bago,Normal,Home and lifestyle,34.84,7,12.19,2024-12-24 +INV-00047,Bago,Normal,Food and beverages,43.84,8,17.54,2024-11-19 +INV-00963,Mandalay,Normal,Food and beverages,37.55,10,18.78,2024-08-05 +INV-00882,Yangon,Normal,Fashion accessories,33.31,7,11.66,2024-05-25 +INV-00370,Mandalay,Member,Fashion accessories,51.9,6,15.57,2024-06-21 +INV-00337,Mandalay,Member,Home and lifestyle,73.47,3,11.02,2024-03-30 +INV-00257,Bago,Member,Food and beverages,69.91,7,24.47,2024-03-23 +INV-00684,Naypyitaw,Member,Sports and travel,45.13,2,4.51,2024-09-10 +INV-00663,Naypyitaw,Normal,Health and beauty,24.56,7,8.6,2024-09-26 +INV-00519,Naypyitaw,Normal,Home and lifestyle,65.29,5,16.32,2024-11-07 +INV-00087,Taunggyi,Member,Food and beverages,44.33,9,19.95,2024-12-24 +INV-00725,Taunggyi,Normal,Electronic accessories,23.49,3,3.52,2024-02-28 +INV-00108,Yangon,Normal,Sports and travel,24.78,8,9.91,2024-05-03 +INV-00431,Bago,Normal,Food and beverages,54.58,4,10.92,2024-04-16 +INV-00862,Mandalay,Normal,Fashion accessories,37.35,9,16.81,2024-01-17 +INV-00331,Yangon,Member,Food and beverages,14.33,1,0.72,2024-12-31 +INV-00405,Mandalay,Normal,Home and lifestyle,49.62,7,17.37,2024-12-05 +INV-00789,Naypyitaw,Normal,Sports and travel,95.36,3,14.3,2024-07-30 +INV-00485,Mandalay,Member,Food and beverages,73.94,9,33.27,2024-08-28 +INV-00306,Mandalay,Member,Food and beverages,61.47,9,27.66,2024-04-16 +INV-00623,Taunggyi,Member,Fashion accessories,12.22,4,2.44,2024-02-15 +INV-00755,Taunggyi,Member,Food and beverages,35.17,2,3.52,2024-02-02 +INV-00849,Mandalay,Member,Home and lifestyle,67.49,9,30.37,2024-10-23 +INV-00942,Bago,Normal,Food and beverages,88.31,3,13.25,2024-06-09 +INV-00900,Taunggyi,Normal,Sports and travel,15.83,8,6.33,2024-12-27 +INV-00269,Naypyitaw,Normal,Food and beverages,63.25,8,25.3,2024-03-26 +INV-00220,Taunggyi,Normal,Fashion accessories,41.63,3,6.24,2024-06-02 +INV-00402,Naypyitaw,Normal,Electronic accessories,78.94,7,27.63,2024-07-30 +INV-00715,Bago,Normal,Fashion accessories,12.29,6,3.69,2024-12-13 +INV-00827,Taunggyi,Normal,Sports and travel,15.59,3,2.34,2024-04-29 +INV-00599,Naypyitaw,Member,Electronic accessories,98.11,2,9.81,2024-07-12 +INV-00734,Naypyitaw,Member,Home and lifestyle,85.49,6,25.65,2024-03-28 +INV-00933,Mandalay,Member,Food and beverages,42.06,5,10.52,2024-08-09 +INV-00346,Yangon,Normal,Fashion accessories,64.95,7,22.73,2024-07-12 +INV-00064,Taunggyi,Member,Fashion accessories,45.23,3,6.78,2024-01-06 +INV-00449,Naypyitaw,Normal,Electronic accessories,73.04,3,10.96,2024-03-10 +INV-00451,Mandalay,Member,Fashion accessories,65.08,4,13.02,2024-12-07 +INV-00194,Bago,Member,Food and beverages,10.52,8,4.21,2024-12-08 +INV-00854,Taunggyi,Normal,Health and beauty,67.4,5,16.85,2024-12-18 +INV-00793,Naypyitaw,Normal,Electronic accessories,39.95,6,11.99,2024-12-07 +INV-00842,Bago,Normal,Electronic accessories,52.19,9,23.49,2024-04-12 +INV-00794,Mandalay,Member,Health and beauty,82.87,5,20.72,2024-03-28 +INV-00414,Yangon,Normal,Fashion accessories,18.77,5,4.69,2024-04-29 +INV-00168,Yangon,Member,Food and beverages,41.11,5,10.28,2024-10-25 +INV-00284,Naypyitaw,Normal,Health and beauty,53.27,4,10.65,2024-05-07 +INV-00879,Naypyitaw,Normal,Fashion accessories,10.46,2,1.05,2024-01-28 +INV-00835,Taunggyi,Normal,Health and beauty,37.07,3,5.56,2024-02-18 +INV-00044,Mandalay,Member,Fashion accessories,44.96,9,20.23,2024-04-05 +INV-00810,Yangon,Member,Health and beauty,17.54,5,4.38,2024-03-24 +INV-00430,Taunggyi,Member,Sports and travel,40.38,3,6.06,2024-04-26 +INV-00128,Taunggyi,Normal,Health and beauty,45.69,7,15.99,2024-12-26 +INV-00409,Bago,Normal,Sports and travel,42.45,7,14.86,2024-05-22 +INV-00587,Bago,Normal,Home and lifestyle,27.18,9,12.23,2024-05-10 +INV-00192,Mandalay,Normal,Electronic accessories,62.05,10,31.03,2024-07-27 +INV-00029,Naypyitaw,Member,Electronic accessories,22.05,10,11.03,2024-03-18 +INV-00670,Taunggyi,Member,Food and beverages,89.56,8,35.82,2024-12-11 +INV-00434,Naypyitaw,Member,Fashion accessories,65.45,6,19.64,2024-07-26 +INV-00496,Taunggyi,Normal,Fashion accessories,58.79,7,20.58,2024-01-25 +INV-00021,Naypyitaw,Normal,Electronic accessories,73.56,1,3.68,2024-12-08 +INV-00310,Taunggyi,Normal,Food and beverages,23.38,6,7.01,2024-04-09 +INV-00271,Taunggyi,Normal,Health and beauty,71.78,6,21.53,2024-08-18 +INV-00869,Bago,Normal,Fashion accessories,63.59,1,3.18,2024-02-21 +INV-00378,Yangon,Member,Food and beverages,64.73,6,19.42,2024-08-03 +INV-00726,Bago,Member,Health and beauty,99.97,4,19.99,2024-06-05 +INV-00322,Yangon,Member,Sports and travel,36.11,10,18.06,2024-07-12 +INV-00600,Bago,Member,Fashion accessories,58.11,1,2.91,2024-03-17 +INV-00690,Yangon,Member,Home and lifestyle,25.6,10,12.8,2024-11-07 +INV-00730,Yangon,Member,Food and beverages,85.96,4,17.19,2024-12-26 +INV-00312,Taunggyi,Normal,Food and beverages,24.41,8,9.76,2024-09-20 +INV-00446,Mandalay,Member,Electronic accessories,53.16,1,2.66,2024-06-03 +INV-00285,Bago,Member,Electronic accessories,69.24,4,13.85,2024-06-28 +INV-00742,Yangon,Normal,Health and beauty,38.61,9,17.37,2024-07-01 +INV-00239,Bago,Member,Sports and travel,92.32,3,13.85,2024-11-19 +INV-00095,Taunggyi,Normal,Home and lifestyle,27.89,9,12.55,2024-10-27 +INV-00435,Yangon,Normal,Sports and travel,36.05,9,16.22,2024-06-19 +INV-00394,Yangon,Member,Fashion accessories,32.47,10,16.23,2024-12-30 +INV-00992,Yangon,Normal,Electronic accessories,38.51,7,13.48,2024-12-09 +INV-00480,Bago,Member,Home and lifestyle,98.25,7,34.39,2024-01-26 +INV-00765,Yangon,Normal,Electronic accessories,67.43,2,6.74,2024-12-01 +INV-00401,Bago,Normal,Food and beverages,72.67,1,3.63,2024-02-17 +INV-00455,Naypyitaw,Member,Health and beauty,11.2,7,3.92,2024-06-01 +INV-00011,Taunggyi,Member,Home and lifestyle,75.55,10,37.77,2024-02-06 +INV-00836,Bago,Normal,Home and lifestyle,52.16,4,10.43,2024-01-04 +INV-00259,Taunggyi,Member,Food and beverages,40.39,6,12.12,2024-04-10 +INV-00117,Naypyitaw,Member,Sports and travel,61.14,2,6.11,2024-12-16 +INV-00110,Yangon,Normal,Health and beauty,14.88,8,5.95,2024-11-26 +INV-00300,Mandalay,Member,Electronic accessories,65.82,6,19.75,2024-01-06 +INV-00551,Naypyitaw,Normal,Electronic accessories,36.36,3,5.45,2024-08-17 +INV-00139,Taunggyi,Member,Food and beverages,38.37,10,19.18,2024-03-22 +INV-00993,Bago,Member,Electronic accessories,38.57,1,1.93,2024-01-19 +INV-00815,Yangon,Member,Fashion accessories,50.83,5,12.71,2024-07-26 +INV-00024,Taunggyi,Member,Food and beverages,87.48,9,39.37,2024-09-14 +INV-00219,Mandalay,Member,Fashion accessories,49.32,4,9.86,2024-06-10 +INV-00178,Yangon,Normal,Electronic accessories,76.06,5,19.02,2024-06-07 +INV-00205,Mandalay,Normal,Health and beauty,47.31,6,14.19,2024-06-28 +INV-00203,Yangon,Member,Sports and travel,40.66,10,20.33,2024-09-15 +INV-00018,Bago,Member,Fashion accessories,13.01,4,2.6,2024-12-10 +INV-00031,Naypyitaw,Member,Fashion accessories,73.3,3,10.99,2024-06-12 +INV-00769,Naypyitaw,Normal,Sports and travel,76.63,3,11.49,2024-12-31 +INV-00974,Naypyitaw,Member,Fashion accessories,45.09,9,20.29,2024-09-08 +INV-00688,Mandalay,Normal,Fashion accessories,40.22,7,14.08,2024-12-22 +INV-00874,Naypyitaw,Member,Sports and travel,37.18,10,18.59,2024-11-24 +INV-00344,Mandalay,Normal,Home and lifestyle,82.13,9,36.96,2024-02-27 +INV-00861,Naypyitaw,Member,Health and beauty,62.87,4,12.57,2024-12-03 +INV-00918,Bago,Normal,Sports and travel,60.83,9,27.37,2024-08-05 +INV-00148,Yangon,Member,Fashion accessories,17.12,6,5.14,2024-06-11 +INV-00707,Bago,Member,Electronic accessories,14.56,8,5.82,2024-05-05 +INV-00652,Yangon,Normal,Sports and travel,88.8,2,8.88,2024-09-29 +INV-00334,Taunggyi,Member,Home and lifestyle,28.57,5,7.14,2024-08-31 +INV-00019,Taunggyi,Normal,Home and lifestyle,94.17,7,32.96,2024-05-13 +INV-00982,Taunggyi,Normal,Fashion accessories,22.27,2,2.23,2024-01-28 +INV-00978,Taunggyi,Member,Health and beauty,98.36,2,9.84,2024-08-17 +INV-00362,Yangon,Normal,Health and beauty,35.12,7,12.29,2024-05-31 +INV-00175,Mandalay,Normal,Electronic accessories,48.78,5,12.2,2024-06-13 +INV-00422,Yangon,Normal,Home and lifestyle,98.58,3,14.79,2024-07-01 +INV-00561,Mandalay,Normal,Health and beauty,96.06,6,28.82,2024-03-26 +INV-00500,Taunggyi,Member,Sports and travel,44.81,5,11.2,2024-04-02 +INV-00695,Yangon,Normal,Home and lifestyle,42.88,7,15.01,2024-10-29 +INV-00581,Yangon,Member,Electronic accessories,17.41,6,5.22,2024-12-08 +INV-00198,Mandalay,Member,Health and beauty,86.31,6,25.89,2024-07-28 +INV-00582,Yangon,Normal,Home and lifestyle,67.0,9,30.15,2024-10-26 +INV-00464,Bago,Member,Electronic accessories,33.27,4,6.65,2024-12-16 +INV-00181,Bago,Member,Home and lifestyle,14.5,2,1.45,2024-06-03 +INV-00901,Bago,Normal,Sports and travel,41.2,4,8.24,2024-05-27 +INV-00109,Naypyitaw,Normal,Food and beverages,39.23,8,15.69,2024-08-01 +INV-00964,Naypyitaw,Member,Sports and travel,40.72,9,18.32,2024-05-16 +INV-00989,Naypyitaw,Member,Electronic accessories,57.87,10,28.93,2024-12-29 +INV-00934,Taunggyi,Member,Sports and travel,83.2,9,37.44,2024-12-02 +INV-00593,Naypyitaw,Normal,Sports and travel,92.41,2,9.24,2024-03-14 +INV-00682,Yangon,Normal,Sports and travel,68.65,2,6.87,2024-04-08 +INV-00763,Naypyitaw,Member,Food and beverages,62.16,1,3.11,2024-01-23 +INV-00949,Yangon,Member,Fashion accessories,45.77,5,11.44,2024-11-27 +INV-00702,Mandalay,Normal,Health and beauty,48.8,4,9.76,2024-07-14 +INV-00639,Taunggyi,Member,Fashion accessories,43.5,6,13.05,2024-06-21 +INV-00263,Bago,Member,Home and lifestyle,20.03,2,2.0,2024-02-15 +INV-00186,Yangon,Normal,Food and beverages,65.3,8,26.12,2024-06-15 +INV-00681,Naypyitaw,Normal,Health and beauty,26.57,2,2.66,2024-03-12 +INV-00723,Naypyitaw,Normal,Sports and travel,62.57,5,15.64,2024-02-07 +INV-00556,Mandalay,Member,Health and beauty,17.7,10,8.85,2024-10-13 +INV-00891,Mandalay,Normal,Food and beverages,14.41,2,1.44,2024-08-25 +INV-00926,Mandalay,Normal,Food and beverages,83.06,7,29.07,2024-06-11 +INV-00143,Naypyitaw,Normal,Sports and travel,12.75,8,5.1,2024-01-06 +INV-00522,Bago,Member,Electronic accessories,57.8,10,28.9,2024-10-29 +INV-00997,Taunggyi,Member,Sports and travel,89.35,6,26.8,2024-11-14 +INV-00825,Yangon,Member,Fashion accessories,58.36,5,14.59,2024-06-28 +INV-00417,Yangon,Normal,Electronic accessories,42.34,6,12.7,2024-03-14 +INV-00520,Yangon,Normal,Health and beauty,90.33,6,27.1,2024-08-20 +INV-00689,Yangon,Member,Home and lifestyle,49.58,7,17.35,2024-11-14 +INV-00984,Mandalay,Normal,Sports and travel,38.94,6,11.68,2024-07-30 +INV-00061,Yangon,Member,Health and beauty,16.8,2,1.68,2024-08-24 +INV-00408,Naypyitaw,Normal,Home and lifestyle,11.36,1,0.57,2024-07-10 +INV-00512,Mandalay,Member,Home and lifestyle,99.98,7,34.99,2024-06-29 +INV-00248,Mandalay,Member,Fashion accessories,85.99,6,25.8,2024-02-19 +INV-00548,Yangon,Normal,Electronic accessories,83.16,1,4.16,2024-11-05 +INV-00701,Yangon,Normal,Sports and travel,30.51,3,4.58,2024-03-26 +INV-00325,Naypyitaw,Member,Sports and travel,15.68,4,3.14,2024-02-12 +INV-00426,Yangon,Member,Fashion accessories,33.05,9,14.87,2024-06-08 +INV-00439,Yangon,Member,Sports and travel,68.22,3,10.23,2024-05-23 +INV-00945,Naypyitaw,Member,Electronic accessories,35.3,6,10.59,2024-03-11 +INV-00796,Taunggyi,Normal,Sports and travel,46.12,1,2.31,2024-08-22 +INV-00843,Naypyitaw,Member,Home and lifestyle,54.48,4,10.9,2024-05-16 +INV-00035,Yangon,Member,Fashion accessories,65.25,3,9.79,2024-11-06 +INV-00875,Yangon,Member,Fashion accessories,80.96,7,28.34,2024-01-07 +INV-00590,Yangon,Member,Sports and travel,38.56,7,13.5,2024-06-04 +INV-00124,Yangon,Member,Electronic accessories,88.53,7,30.99,2024-04-15 +INV-00147,Taunggyi,Member,Electronic accessories,37.12,6,11.14,2024-11-17 +INV-00494,Yangon,Member,Sports and travel,57.36,9,25.81,2024-06-05 +INV-00399,Mandalay,Member,Fashion accessories,49.83,6,14.95,2024-12-10 +INV-00022,Bago,Member,Health and beauty,41.09,4,8.22,2024-07-15 +INV-00046,Naypyitaw,Normal,Health and beauty,78.16,5,19.54,2024-02-18 +INV-00893,Naypyitaw,Normal,Home and lifestyle,22.38,4,4.48,2024-02-08 +INV-00749,Mandalay,Normal,Health and beauty,89.54,8,35.82,2024-11-11 +INV-00700,Yangon,Member,Sports and travel,22.26,9,10.02,2024-08-19 +INV-00122,Yangon,Member,Home and lifestyle,43.32,6,13.0,2024-02-06 +INV-00275,Bago,Member,Home and lifestyle,84.89,8,33.96,2024-08-17 +INV-00166,Mandalay,Normal,Electronic accessories,58.78,6,17.63,2024-01-18 +INV-00974,Naypyitaw,Member,Fashion accessories,45.09,9,20.29,2024-09-08 +INV-00884,Yangon,Member,Food and beverages,41.66,5,10.41,2024-09-05 +INV-00947,Taunggyi,Normal,Electronic accessories,24.26,9,10.92,2024-10-16 +INV-00228,Taunggyi,Member,Sports and travel,66.11,8,26.44,2024-04-12 +INV-00969,Bago,Normal,Health and beauty,85.32,1,4.27,2024-08-06 +INV-00484,Mandalay,Normal,Fashion accessories,43.43,7,15.2,2024-03-13 +INV-00302,Naypyitaw,Normal,Food and beverages,17.87,10,8.94,2024-03-28 +INV-00437,Mandalay,Member,Fashion accessories,46.06,4,9.21,2024-10-05 +INV-00559,Mandalay,Member,Home and lifestyle,16.69,2,1.67,2024-09-11 +INV-00091,Bago,Normal,Home and lifestyle,15.51,3,2.33,2024-11-16 +INV-00518,Yangon,Member,Sports and travel,97.34,5,24.34,2024-08-06 +INV-00158,Naypyitaw,Member,Home and lifestyle,12.43,9,5.59,2024-08-30 +INV-00679,Bago,Member,Food and beverages,64.34,5,16.09,2024-02-27 +INV-00564,Bago,Member,Food and beverages,58.66,5,14.66,2024-06-24 +INV-00391,Taunggyi,Normal,Health and beauty,95.75,7,33.51,2024-12-13 +INV-00970,Naypyitaw,Member,Sports and travel,82.69,4,16.54,2024-10-06 +INV-00111,Bago,Normal,Electronic accessories,89.72,5,22.43,2024-10-13 +INV-00131,Bago,Member,Food and beverages,93.84,4,18.77,2024-07-26 +INV-00367,Mandalay,Member,Sports and travel,95.78,5,23.95,2024-04-16 +INV-00005,Bago,Normal,Food and beverages,56.12,8,22.45,2024-03-06 +INV-00272,Taunggyi,Normal,Sports and travel,23.84,9,10.73,2024-01-28 +INV-00479,Naypyitaw,Member,Fashion accessories,90.22,5,22.56,2024-04-30 +INV-00629,Bago,Member,Home and lifestyle,92.65,3,13.9,2024-07-16 +INV-00027,Bago,Member,Health and beauty,66.02,4,13.2,2024-07-11 +INV-00486,Naypyitaw,Member,Fashion accessories,53.29,10,26.64,2024-02-17 +INV-00050,Yangon,Member,Home and lifestyle,23.26,3,3.49,2024-07-02 +INV-00721,Bago,Member,Electronic accessories,22.56,5,5.64,2024-04-21 +INV-00125,Bago,Member,Food and beverages,40.56,10,20.28,2024-11-08 +INV-00637,Taunggyi,Member,Food and beverages,16.44,2,1.64,2024-06-25 +INV-00258,Bago,Normal,Health and beauty,38.86,8,15.54,2024-07-02 +INV-00944,Taunggyi,Member,Sports and travel,90.96,7,31.84,2024-08-15 +INV-00786,Yangon,Member,Food and beverages,23.95,8,9.58,2024-02-21 +INV-00547,Taunggyi,Normal,Electronic accessories,87.46,9,39.36,2024-11-18 +INV-00589,Naypyitaw,Member,Sports and travel,31.49,5,7.87,2024-02-25 +INV-00133,Yangon,Normal,Fashion accessories,25.75,4,5.15,2024-02-09 +INV-00760,Taunggyi,Normal,Home and lifestyle,69.8,8,27.92,2024-04-22 +INV-00532,Naypyitaw,Member,Home and lifestyle,13.67,7,4.78,2024-12-16 +INV-00116,Bago,Member,Health and beauty,24.37,1,1.22,2024-10-09 +INV-00217,Taunggyi,Normal,Health and beauty,82.5,9,37.12,2024-07-07 +INV-00333,Mandalay,Member,Electronic accessories,78.24,10,39.12,2024-12-07 +INV-00811,Yangon,Normal,Electronic accessories,89.94,7,31.48,2024-12-06 +INV-00803,Bago,Member,Fashion accessories,77.07,2,7.71,2024-03-01 +INV-00666,Taunggyi,Normal,Health and beauty,33.86,4,6.77,2024-12-24 +INV-00665,Taunggyi,Normal,Home and lifestyle,28.52,7,9.98,2024-08-28 +INV-00185,Taunggyi,Normal,Sports and travel,44.24,10,22.12,2024-09-26 +INV-00004,Naypyitaw,Member,Electronic accessories,86.58,2,8.66,2024-07-11 +INV-00470,Mandalay,Normal,Food and beverages,24.82,8,9.93,2024-07-07 +INV-00852,Bago,Member,Sports and travel,37.58,7,13.15,2024-08-05 +INV-00325,Naypyitaw,Member,Sports and travel,15.68,4,3.14,2024-02-12 +INV-00014,Yangon,Normal,Sports and travel,75.8,10,37.9,2024-06-06 +INV-00541,Taunggyi,Member,Health and beauty,14.19,4,2.84,2024-05-31 +INV-00873,Naypyitaw,Normal,Home and lifestyle,26.39,5,6.6,2024-12-25 +INV-00871,Taunggyi,Normal,Electronic accessories,57.49,3,8.62,2024-01-03 +INV-00937,Taunggyi,Member,Sports and travel,14.8,7,5.18,2024-06-18 +INV-00127,Taunggyi,Member,Home and lifestyle,73.22,4,14.64,2024-06-19 +INV-00659,Mandalay,Member,Food and beverages,41.04,3,6.16,2024-12-17 +INV-00727,Yangon,Normal,Food and beverages,29.06,4,5.81,2024-05-13 +INV-00542,Naypyitaw,Normal,Health and beauty,70.05,8,28.02,2024-02-12 +INV-00039,Bago,Normal,Home and lifestyle,42.24,6,12.67,2024-02-21 +INV-00042,Naypyitaw,Member,Electronic accessories,67.37,3,10.11,2024-06-04 +INV-00319,Taunggyi,Normal,Health and beauty,28.15,3,4.22,2024-08-14 +INV-00820,Naypyitaw,Member,Electronic accessories,83.21,1,4.16,2024-05-07 +INV-00078,Bago,Normal,Health and beauty,91.21,5,22.8,2024-12-04 +INV-00240,Mandalay,Member,Fashion accessories,71.25,10,35.62,2024-07-28 +INV-00660,Mandalay,Member,Sports and travel,15.33,7,5.37,2024-01-28 +INV-00799,Mandalay,Member,Fashion accessories,25.75,2,2.58,2024-03-06 +INV-00155,Naypyitaw,Member,Health and beauty,16.25,7,5.69,2024-07-13 +INV-00067,Mandalay,Normal,Fashion accessories,73.04,10,36.52,2024-06-13 +INV-01000,Yangon,Normal,Home and lifestyle,45.12,10,22.56,2024-01-13 +INV-00193,Yangon,Member,Food and beverages,35.8,2,3.58,2024-02-06 +INV-00858,Yangon,Member,Food and beverages,19.22,7,6.73,2024-02-28 +INV-00114,Naypyitaw,Normal,Food and beverages,40.4,5,10.1,2024-07-02 +INV-00357,Naypyitaw,Member,Fashion accessories,62.83,3,9.42,2024-02-11 +INV-00657,Bago,Member,Sports and travel,92.3,1,4.62,2024-01-23 +INV-00936,Mandalay,Normal,Home and lifestyle,95.69,7,33.49,2024-05-27 +INV-00010,Yangon,Normal,Home and lifestyle,21.37,2,2.14,2024-06-12 +INV-00924,Naypyitaw,Normal,Health and beauty,23.8,4,4.76,2024-06-12 +INV-00932,Naypyitaw,Normal,Food and beverages,49.56,10,24.78,2024-05-14 +INV-00595,Naypyitaw,Normal,Electronic accessories,47.1,4,9.42,2024-06-26 +INV-00764,Bago,Member,Health and beauty,34.12,5,8.53,2024-05-08 +INV-00788,Naypyitaw,Normal,Sports and travel,76.56,5,19.14,2024-03-01 +INV-00578,Naypyitaw,Member,Home and lifestyle,39.3,10,19.65,2024-02-13 +INV-00778,Mandalay,Member,Health and beauty,37.29,9,16.78,2024-08-30 +INV-00206,Bago,Member,Fashion accessories,75.19,2,7.52,2024-08-15 +INV-00261,Naypyitaw,Normal,Health and beauty,35.01,7,12.25,2024-08-29 +INV-00647,Naypyitaw,Normal,Sports and travel,70.1,1,3.5,2024-09-12 +INV-00265,Mandalay,Normal,Food and beverages,40.61,7,14.21,2024-05-09 +INV-00961,Mandalay,Normal,Electronic accessories,15.13,6,4.54,2024-07-12 +INV-00550,Yangon,Normal,Health and beauty,51.38,5,12.85,2024-04-07 +INV-00737,Taunggyi,Normal,Food and beverages,97.14,7,34.0,2024-01-20 +INV-00280,Mandalay,Normal,Food and beverages,84.81,6,25.44,2024-07-24 +INV-00347,Mandalay,Normal,Home and lifestyle,71.68,3,10.75,2024-05-06 +INV-00655,Naypyitaw,Normal,Fashion accessories,44.4,8,17.76,2024-12-29 +INV-00631,Taunggyi,Member,Health and beauty,12.25,7,4.29,2024-12-02 +INV-00160,Mandalay,Member,Fashion accessories,17.72,4,3.54,2024-10-07 +INV-00513,Mandalay,Normal,Home and lifestyle,91.45,2,9.15,2024-07-06 +INV-00576,Yangon,Normal,Food and beverages,75.31,2,7.53,2024-06-08 +INV-00661,Taunggyi,Normal,Fashion accessories,75.23,6,22.57,2024-09-12 +INV-00759,Bago,Member,Sports and travel,96.07,8,38.43,2024-01-09 +INV-00237,Bago,Member,Fashion accessories,40.03,5,10.01,2024-06-20 +INV-00814,Taunggyi,Member,Electronic accessories,57.45,7,20.11,2024-10-31 +INV-00540,Bago,Member,Home and lifestyle,87.79,2,8.78,2024-07-01 +INV-00073,Yangon,Normal,Home and lifestyle,42.41,8,16.96,2024-05-24 +INV-00890,Mandalay,Member,Health and beauty,16.99,10,8.49,2024-09-06 +INV-00719,Mandalay,Member,Electronic accessories,15.16,10,7.58,2024-11-25 +INV-00088,Mandalay,Normal,Fashion accessories,84.01,6,25.2,2024-02-05 +INV-00173,Naypyitaw,Member,Sports and travel,38.46,2,3.85,2024-12-11 +INV-00033,Taunggyi,Normal,Food and beverages,75.04,8,30.02,2024-06-06 +INV-00201,Taunggyi,Normal,Electronic accessories,27.91,10,13.96,2024-06-07 +INV-00878,Bago,Member,Electronic accessories,50.21,7,17.57,2024-01-29 +INV-00713,Mandalay,Member,Food and beverages,33.38,9,15.02,2024-01-16 +INV-00161,Bago,Normal,Health and beauty,82.7,9,37.22,2024-05-01 +INV-00023,Yangon,Member,Food and beverages,51.25,1,2.56,2024-02-19 +INV-00467,Yangon,Normal,Health and beauty,71.4,5,17.85,2024-06-18 +INV-00733,Bago,Member,Sports and travel,38.36,10,19.18,2024-08-20 +INV-00382,Taunggyi,Normal,Electronic accessories,44.61,4,8.92,2024-12-14 +INV-00298,Mandalay,Member,Fashion accessories,41.38,3,6.21,2024-04-14 +INV-00466,Yangon,Normal,Electronic accessories,97.5,2,9.75,2024-03-09 +INV-00774,Mandalay,Member,Home and lifestyle,29.85,1,1.49,2024-06-11 +INV-00824,Yangon,Normal,Fashion accessories,84.57,10,42.28,2024-06-18 +INV-00899,Bago,Member,Health and beauty,55.23,8,22.09,2024-07-10 +INV-00184,Naypyitaw,Normal,Food and beverages,22.12,4,4.42,2024-12-05 +INV-00374,Naypyitaw,Member,Home and lifestyle,64.63,3,9.69,2024-05-10 +INV-00314,Yangon,Normal,Food and beverages,55.53,1,2.78,2024-10-17 +INV-00962,Bago,Member,Sports and travel,74.16,1,3.71,2024-08-23 +INV-00274,Mandalay,Normal,Electronic accessories,75.24,2,7.52,2024-08-22 +INV-00341,Bago,Normal,Food and beverages,94.3,7,33.01,2024-05-10 +INV-00207,Mandalay,Normal,Health and beauty,76.27,1,3.81,2024-07-01 +INV-00710,Taunggyi,Member,Sports and travel,12.56,6,3.77,2024-12-15 +INV-00864,Bago,Member,Electronic accessories,23.35,3,3.5,2024-04-29 +INV-00931,Bago,Normal,Health and beauty,33.37,2,3.34,2024-10-19 +INV-00856,Yangon,Member,Home and lifestyle,98.79,3,14.82,2024-09-07 +INV-00411,Naypyitaw,Normal,Electronic accessories,21.11,1,1.06,2024-03-14 +INV-00079,Mandalay,Member,Fashion accessories,11.85,1,0.59,2024-03-27 +INV-00499,Naypyitaw,Normal,Sports and travel,56.9,10,28.45,2024-08-13 +INV-00833,Naypyitaw,Normal,Electronic accessories,23.51,8,9.4,2024-08-30 +INV-00096,Naypyitaw,Normal,Health and beauty,30.6,4,6.12,2024-06-12 +INV-00350,Mandalay,Normal,Sports and travel,91.82,1,4.59,2024-11-04 +INV-00005,Bago,Normal,Food and beverages,56.12,8,22.45,2024-03-06 +INV-00698,Bago,Member,Home and lifestyle,24.75,3,3.71,2024-08-17 +INV-00145,Yangon,Normal,Food and beverages,13.04,8,5.22,2024-12-01 +INV-00048,Bago,Normal,Health and beauty,68.35,2,6.83,2024-01-05 +INV-00202,Taunggyi,Member,Home and lifestyle,80.66,9,36.3,2024-08-24 +INV-00544,Yangon,Member,Electronic accessories,67.93,2,6.79,2024-05-05 +INV-00099,Mandalay,Member,Health and beauty,45.12,1,2.26,2024-10-19 +INV-00183,Bago,Normal,Fashion accessories,43.63,1,2.18,2024-07-08 +INV-00798,Mandalay,Member,Electronic accessories,21.31,6,6.39,2024-10-07 +INV-00065,Yangon,Member,Fashion accessories,20.27,7,7.09,2024-01-20 +INV-00101,Mandalay,Member,Food and beverages,75.81,10,37.91,2024-08-06 +INV-00461,Mandalay,Normal,Food and beverages,54.69,8,21.88,2024-11-30 +INV-00303,Yangon,Normal,Home and lifestyle,44.15,3,6.62,2024-10-28 +INV-00771,Bago,Normal,Sports and travel,61.87,4,12.37,2024-08-29 +INV-00179,Taunggyi,Member,Electronic accessories,31.27,9,14.07,2024-12-17 +INV-00883,Naypyitaw,Member,Home and lifestyle,15.92,5,3.98,2024-04-11 +INV-00845,Bago,Member,Home and lifestyle,24.28,10,12.14,2024-04-30 +INV-00243,Taunggyi,Normal,Home and lifestyle,65.08,10,32.54,2024-10-10 +INV-00458,Taunggyi,Member,Electronic accessories,81.05,7,28.37,2024-02-03 +INV-00006,Taunggyi,Member,Electronic accessories,26.37,2,2.64,2024-01-26 +INV-00722,Bago,Member,Sports and travel,44.03,3,6.6,2024-06-19 +INV-00150,Naypyitaw,Normal,Health and beauty,80.42,9,36.19,2024-12-16 +INV-00075,Taunggyi,Normal,Home and lifestyle,53.1,4,10.62,2024-12-09 +INV-00986,Naypyitaw,Normal,Home and lifestyle,11.79,8,4.72,2024-11-25 +INV-00906,Mandalay,Member,Fashion accessories,22.29,6,6.69,2024-01-14 +INV-00001,Mandalay,Member,Sports and travel,13.47,3,2.02,2024-08-24 +INV-00311,Yangon,Normal,Health and beauty,87.25,8,34.9,2024-05-03 +INV-00762,Bago,Normal,Food and beverages,35.6,4,7.12,2024-07-20 +INV-00527,Mandalay,Normal,Fashion accessories,90.04,3,13.51,2024-06-12 +INV-00286,Bago,Member,Sports and travel,87.87,9,39.54,2024-04-07 +INV-00748,Mandalay,Member,Fashion accessories,44.94,2,4.49,2024-05-05 +INV-00474,Bago,Member,Fashion accessories,80.84,7,28.29,2024-12-22 +INV-00336,Mandalay,Member,Home and lifestyle,38.92,10,19.46,2024-07-02 +INV-00767,Yangon,Normal,Fashion accessories,65.71,8,26.28,2024-05-07 +INV-00643,Taunggyi,Normal,Electronic accessories,40.23,9,18.1,2024-10-16 +INV-00054,Naypyitaw,Member,Sports and travel,93.55,10,46.78,2024-04-18 +INV-00397,Mandalay,Normal,Food and beverages,39.15,5,9.79,2024-07-30 +INV-00094,Mandalay,Member,Electronic accessories,67.39,7,23.59,2024-06-20 +INV-00812,Yangon,Member,Home and lifestyle,83.12,2,8.31,2024-04-24 +INV-00808,Yangon,Member,Food and beverages,73.84,6,22.15,2024-09-19 +INV-00097,Mandalay,Member,Sports and travel,35.86,2,3.59,2024-09-03 +INV-00805,Bago,Normal,Sports and travel,97.45,9,43.85,2024-06-10 +INV-00070,Mandalay,Member,Health and beauty,36.86,6,11.06,2024-08-18 +INV-00938,Naypyitaw,Member,Sports and travel,39.89,8,15.96,2024-08-06 +INV-00267,Mandalay,Member,Home and lifestyle,90.58,9,40.76,2024-11-15 +INV-00154,Mandalay,Normal,Food and beverages,27.65,5,6.91,2024-01-27 +INV-00741,Bago,Member,Food and beverages,25.72,4,5.14,2024-12-30 +INV-00821,Naypyitaw,Member,Home and lifestyle,99.83,8,39.93,2024-03-14 +INV-00427,Bago,Normal,Home and lifestyle,17.89,5,4.47,2024-07-02 +INV-00894,Mandalay,Member,Home and lifestyle,67.57,5,16.89,2024-08-17 +INV-00137,Yangon,Normal,Sports and travel,97.45,8,38.98,2024-01-24 +INV-00552,Taunggyi,Member,Food and beverages,81.95,3,12.29,2024-07-07 +INV-00071,Mandalay,Normal,Health and beauty,94.54,7,33.09,2024-06-03 +INV-00080,Mandalay,Member,Food and beverages,41.48,10,20.74,2024-11-29 +INV-00644,Naypyitaw,Normal,Sports and travel,24.65,9,11.09,2024-10-23 +INV-00504,Bago,Normal,Health and beauty,76.34,9,34.35,2024-10-06 +INV-00282,Bago,Member,Health and beauty,24.41,4,4.88,2024-06-18 +INV-00403,Yangon,Member,Sports and travel,57.59,5,14.4,2024-01-05 +INV-00141,Mandalay,Member,Electronic accessories,53.0,10,26.5,2024-02-06 +INV-00851,Naypyitaw,Normal,Home and lifestyle,15.37,5,3.84,2024-09-26 +INV-00433,Taunggyi,Normal,Food and beverages,27.67,2,2.77,2024-10-31 +INV-00979,Taunggyi,Normal,Health and beauty,71.46,5,17.86,2024-12-31 +INV-00052,Mandalay,Member,Sports and travel,12.02,6,3.61,2024-04-14 +INV-00371,Yangon,Normal,Home and lifestyle,22.23,1,1.11,2024-01-02 +INV-00308,Yangon,Member,Fashion accessories,31.07,6,9.32,2024-11-17 +INV-00030,Mandalay,Member,Sports and travel,13.95,5,3.49,2024-09-02 +INV-00694,Taunggyi,Normal,Food and beverages,70.76,4,14.15,2024-10-28 +INV-00783,Bago,Normal,Home and lifestyle,90.23,1,4.51,2024-05-23 +INV-00459,Yangon,Normal,Food and beverages,39.62,9,17.83,2024-11-01 +INV-00115,Taunggyi,Member,Food and beverages,78.74,8,31.5,2024-09-16 +INV-00950,Yangon,Member,Health and beauty,36.99,9,16.65,2024-11-09 +INV-00156,Naypyitaw,Member,Food and beverages,62.01,2,6.2,2024-08-12 +INV-00445,Bago,Normal,Food and beverages,14.19,10,7.1,2024-07-08 +INV-00404,Naypyitaw,Normal,Sports and travel,78.22,3,11.73,2024-05-01 +INV-00938,Naypyitaw,Member,Sports and travel,39.89,8,15.96,2024-08-06 +INV-00672,Naypyitaw,Member,Sports and travel,49.83,9,22.42,2024-03-26 +INV-00016,Taunggyi,Normal,Home and lifestyle,76.11,6,22.83,2024-03-01 +INV-00107,Yangon,Normal,Fashion accessories,76.63,10,38.31,2024-09-19 +INV-00456,Bago,Member,Home and lifestyle,73.04,6,21.91,2024-04-10 +INV-00775,Taunggyi,Member,Food and beverages,81.56,8,32.62,2024-12-31 +INV-00804,Bago,Normal,Sports and travel,90.75,5,22.69,2024-04-12 +INV-00696,Yangon,Normal,Food and beverages,52.98,10,26.49,2024-07-02 +INV-00935,Bago,Member,Sports and travel,26.04,3,3.91,2024-01-12 +INV-00929,Bago,Member,Food and beverages,54.38,10,27.19,2024-01-09 +INV-00777,Taunggyi,Member,Fashion accessories,68.62,5,17.16,2024-10-30 +INV-00212,Yangon,Normal,Home and lifestyle,82.93,4,16.59,2024-02-12 +INV-00326,Taunggyi,Normal,Home and lifestyle,89.04,6,26.71,2024-06-12 +INV-00584,Mandalay,Normal,Electronic accessories,74.34,9,33.45,2024-12-22 +INV-00939,Naypyitaw,Member,Fashion accessories,17.16,9,7.72,2024-01-23 +INV-00526,Taunggyi,Normal,Home and lifestyle,27.18,4,5.44,2024-11-30 +INV-00501,Yangon,Normal,Electronic accessories,85.88,10,42.94,2024-11-11 +INV-00100,Naypyitaw,Member,Fashion accessories,52.7,2,5.27,2024-05-01 +INV-00381,Yangon,Normal,Health and beauty,23.84,3,3.58,2024-10-19 +INV-00872,Bago,Normal,Fashion accessories,79.4,2,7.94,2024-11-14 +INV-00718,Taunggyi,Normal,Sports and travel,67.71,3,10.16,2024-10-24 +INV-00625,Taunggyi,Normal,Fashion accessories,90.13,8,36.05,2024-10-17 +INV-00388,Naypyitaw,Normal,Food and beverages,71.7,5,17.93,2024-01-08 +INV-00667,Bago,Member,Home and lifestyle,89.0,6,26.7,2024-10-11 +INV-00448,Taunggyi,Normal,Electronic accessories,18.44,4,3.69,2024-12-05 +INV-00720,Naypyitaw,Member,Electronic accessories,12.27,6,3.68,2024-12-16 +INV-00008,Yangon,Member,Food and beverages,66.12,10,33.06,2024-11-30 +INV-00355,Taunggyi,Member,Home and lifestyle,35.85,4,7.17,2024-02-16 +INV-00991,Bago,Normal,Health and beauty,10.17,4,2.03,2024-10-22 +INV-00740,Yangon,Normal,Food and beverages,24.34,5,6.09,2024-09-19 +INV-00585,Mandalay,Member,Home and lifestyle,35.08,5,8.77,2024-03-09 +INV-00463,Taunggyi,Normal,Sports and travel,98.55,5,24.64,2024-02-20 +INV-00747,Yangon,Normal,Home and lifestyle,34.06,3,5.11,2024-04-29 +INV-00944,Taunggyi,Member,Sports and travel,90.96,7,31.84,2024-08-15 +INV-00678,Yangon,Member,Home and lifestyle,59.46,10,29.73,2024-09-05 +INV-00995,Bago,Member,Health and beauty,64.31,8,25.72,2024-02-02 +INV-00244,Mandalay,Member,Home and lifestyle,79.59,5,19.9,2024-08-27 +INV-00364,Taunggyi,Normal,Food and beverages,98.45,6,29.54,2024-10-01 +INV-00356,Bago,Member,Electronic accessories,44.24,5,11.06,2024-02-18 +INV-00372,Mandalay,Member,Fashion accessories,18.94,6,5.68,2024-08-16 +INV-00607,Yangon,Member,Fashion accessories,76.08,2,7.61,2024-09-01 +INV-00677,Taunggyi,Normal,Fashion accessories,81.86,6,24.56,2024-06-11 +INV-00215,Mandalay,Normal,Health and beauty,17.63,9,7.93,2024-12-04 +INV-00855,Yangon,Member,Health and beauty,69.3,5,17.32,2024-10-06 +INV-00752,Naypyitaw,Member,Electronic accessories,60.2,10,30.1,2024-12-29 +INV-00829,Taunggyi,Member,Food and beverages,27.2,8,10.88,2024-09-29 +INV-00462,Taunggyi,Member,Health and beauty,64.5,10,32.25,2024-06-01 +INV-00142,Bago,Member,Food and beverages,89.98,6,26.99,2024-09-02 +INV-00171,Bago,Member,Sports and travel,11.88,9,5.35,2024-03-24 +INV-00221,Bago,Normal,Home and lifestyle,79.46,1,3.97,2024-04-14 +INV-00262,Bago,Member,Food and beverages,20.67,8,8.27,2024-09-09 +INV-00232,Yangon,Member,Electronic accessories,14.15,9,6.37,2024-01-10 +INV-00442,Yangon,Normal,Health and beauty,41.17,2,4.12,2024-11-15 +INV-00608,Bago,Normal,Health and beauty,21.35,7,7.47,2024-02-13 +INV-00948,Bago,Normal,Fashion accessories,74.36,3,11.15,2024-07-02 +INV-00102,Bago,Member,Electronic accessories,32.37,5,8.09,2024-01-15 +INV-00782,Bago,Normal,Food and beverages,35.73,3,5.36,2024-01-15 +INV-00905,Taunggyi,Normal,Electronic accessories,20.37,2,2.04,2024-05-31 +INV-00342,Bago,Normal,Fashion accessories,52.21,5,13.05,2024-06-22 +INV-00098,Naypyitaw,Normal,Health and beauty,13.66,5,3.42,2024-06-16 +INV-00616,Naypyitaw,Member,Home and lifestyle,94.89,9,42.7,2024-11-23 +INV-00425,Naypyitaw,Member,Health and beauty,80.99,2,8.1,2024-10-14 +INV-00315,Bago,Member,Sports and travel,65.33,2,6.53,2024-12-30 +INV-00252,Naypyitaw,Member,Sports and travel,28.16,5,7.04,2024-10-09 +INV-00538,Taunggyi,Normal,Electronic accessories,18.67,9,8.4,2024-06-03 +INV-00999,Taunggyi,Member,Sports and travel,84.69,10,42.34,2024-05-31 +INV-00304,Taunggyi,Normal,Home and lifestyle,56.91,6,17.07,2024-03-31 +INV-00130,Bago,Normal,Electronic accessories,64.66,1,3.23,2024-08-25 +INV-00971,Bago,Member,Food and beverages,88.76,5,22.19,2024-02-03 +INV-00960,Mandalay,Member,Home and lifestyle,22.46,9,10.11,2024-12-08 +INV-00316,Mandalay,Normal,Food and beverages,25.32,2,2.53,2024-06-07 +INV-00943,Naypyitaw,Normal,Sports and travel,67.96,6,20.39,2024-08-16 +INV-00077,Taunggyi,Normal,Food and beverages,19.75,10,9.88,2024-09-17 +INV-00321,Yangon,Member,Home and lifestyle,37.03,3,5.55,2024-06-23 +INV-00998,Bago,Normal,Health and beauty,41.19,7,14.42,2024-11-28 +INV-00090,Mandalay,Normal,Home and lifestyle,23.86,8,9.54,2024-07-31 +INV-00692,Taunggyi,Normal,Food and beverages,58.69,8,23.48,2024-12-02 +INV-00416,Taunggyi,Normal,Sports and travel,48.7,5,12.18,2024-09-16 +INV-00904,Bago,Normal,Health and beauty,92.53,10,46.27,2024-05-12 +INV-00754,Bago,Normal,Health and beauty,64.46,1,3.22,2024-05-05 +INV-00807,Bago,Normal,Sports and travel,46.82,8,18.73,2024-02-11 +INV-00753,Naypyitaw,Member,Home and lifestyle,38.42,4,7.68,2024-12-15 +INV-00301,Yangon,Normal,Electronic accessories,78.56,10,39.28,2024-07-31 +INV-00920,Mandalay,Normal,Sports and travel,16.18,7,5.66,2024-08-30 +INV-00170,Yangon,Member,Food and beverages,11.38,2,1.14,2024-04-19 +INV-00618,Mandalay,Member,Sports and travel,95.33,6,28.6,2024-05-12 +INV-00468,Mandalay,Normal,Sports and travel,78.24,9,35.21,2024-01-06 +INV-00985,Mandalay,Member,Fashion accessories,28.72,9,12.92,2024-10-01 +INV-00658,Bago,Normal,Sports and travel,79.35,8,31.74,2024-07-11 +INV-00987,Naypyitaw,Normal,Food and beverages,36.03,3,5.4,2024-08-09 +INV-00068,Yangon,Member,Food and beverages,48.3,1,2.42,2024-08-12 +INV-00802,Bago,Normal,Electronic accessories,83.55,6,25.06,2024-02-23 +INV-00117,Naypyitaw,Member,Sports and travel,61.14,2,6.11,2024-12-16 +INV-00543,Naypyitaw,Normal,Health and beauty,18.12,4,3.62,2024-03-05 +INV-00787,Naypyitaw,Normal,Electronic accessories,30.42,10,15.21,2024-03-21 +INV-00360,Mandalay,Normal,Food and beverages,51.85,6,15.56,2024-08-05 +INV-00034,Taunggyi,Member,Electronic accessories,64.91,7,22.72,2024-09-22 +INV-00888,Taunggyi,Normal,Sports and travel,60.11,8,24.04,2024-06-28 +INV-00041,Taunggyi,Normal,Food and beverages,37.12,7,12.99,2024-05-02 +INV-00153,Taunggyi,Normal,Food and beverages,85.28,2,8.53,2024-01-07 +INV-00053,Yangon,Member,Health and beauty,51.84,1,2.59,2024-11-20 +INV-00913,Bago,Member,Fashion accessories,53.28,10,26.64,2024-02-13 +INV-00592,Yangon,Normal,Sports and travel,15.92,1,0.8,2024-02-26 +INV-00348,Bago,Normal,Fashion accessories,86.07,5,21.52,2024-05-08 +INV-00863,Naypyitaw,Normal,Sports and travel,57.77,9,26.0,2024-03-12 +INV-00675,Mandalay,Member,Health and beauty,97.82,10,48.91,2024-01-21 +INV-00296,Mandalay,Member,Home and lifestyle,62.07,9,27.93,2024-03-24 +INV-00640,Mandalay,Member,Food and beverages,41.42,9,18.64,2024-10-17 +INV-00975,Naypyitaw,Member,Home and lifestyle,17.62,3,2.64,2024-06-02 +INV-00299,Yangon,Normal,Health and beauty,55.9,7,19.57,2024-01-03 +INV-00959,Naypyitaw,Member,Food and beverages,94.62,3,14.19,2024-12-22 +INV-00886,Mandalay,Normal,Food and beverages,69.92,3,10.49,2024-02-16 +INV-00574,Taunggyi,Normal,Health and beauty,57.56,4,11.51,2024-04-03 +INV-00015,Yangon,Member,Electronic accessories,62.84,3,9.43,2024-01-02 +INV-00126,Mandalay,Member,Sports and travel,33.75,6,10.12,2024-01-22 +INV-00568,Mandalay,Member,Sports and travel,54.49,3,8.17,2024-08-30 +INV-00806,Yangon,Normal,Food and beverages,66.14,10,33.07,2024-08-24 +INV-00582,Yangon,Normal,Home and lifestyle,67.0,9,30.15,2024-10-26 +INV-00656,Bago,Normal,Health and beauty,63.92,8,25.57,2024-07-28 +INV-00384,Mandalay,Normal,Health and beauty,37.53,3,5.63,2024-10-05 +INV-00045,Bago,Normal,Food and beverages,56.39,3,8.46,2024-02-22 +INV-00624,Naypyitaw,Normal,Sports and travel,36.83,5,9.21,2024-07-30 +INV-00830,Mandalay,Member,Sports and travel,13.68,2,1.37,2024-10-28 +INV-00911,Naypyitaw,Member,Sports and travel,14.74,4,2.95,2024-07-21 +INV-00443,Mandalay,Member,Sports and travel,80.91,7,28.32,2024-08-30 +INV-00704,Yangon,Normal,Electronic accessories,49.96,6,14.99,2024-11-01 +INV-00885,Taunggyi,Member,Home and lifestyle,76.67,4,15.33,2024-04-10 +INV-00247,Mandalay,Normal,Electronic accessories,24.61,10,12.3,2024-03-15 +INV-00199,Bago,Normal,Electronic accessories,68.52,1,3.43,2024-02-22 +INV-00051,Taunggyi,Normal,Electronic accessories,39.79,1,1.99,2024-02-15 +INV-00421,Mandalay,Normal,Health and beauty,89.77,4,17.95,2024-03-13 +INV-00472,Taunggyi,Normal,Electronic accessories,65.55,7,22.94,2024-08-06 +INV-00687,Mandalay,Normal,Electronic accessories,83.59,6,25.08,2024-11-27 +INV-00038,Naypyitaw,Member,Home and lifestyle,51.26,5,12.82,2024-09-12 +INV-00469,Taunggyi,Member,Food and beverages,42.36,4,8.47,2024-01-28 +INV-00673,Taunggyi,Member,Food and beverages,71.1,9,32.0,2024-08-21 +INV-00335,Taunggyi,Normal,Health and beauty,82.51,5,20.63,2024-12-21 +INV-00340,Bago,Member,Health and beauty,27.0,8,10.8,2024-04-30 +INV-00129,Bago,Member,Fashion accessories,87.6,8,35.04,2024-05-14 +INV-00043,Naypyitaw,Normal,Sports and travel,22.61,10,11.3,2024-09-16 +INV-00988,Yangon,Normal,Fashion accessories,58.1,10,29.05,2024-07-28 +INV-00329,Mandalay,Member,Fashion accessories,36.88,1,1.84,2024-12-17 +INV-00680,Mandalay,Normal,Home and lifestyle,28.59,1,1.43,2024-04-01 +INV-00441,Mandalay,Normal,Food and beverages,78.47,9,35.31,2024-04-27 +INV-00761,Mandalay,Member,Home and lifestyle,67.72,6,20.32,2024-10-19 +INV-00558,Yangon,Normal,Sports and travel,16.91,2,1.69,2024-05-05 +INV-00708,Taunggyi,Normal,Sports and travel,16.52,2,1.65,2024-07-17 +INV-00328,Mandalay,Normal,Home and lifestyle,55.58,3,8.34,2024-12-12 +INV-00058,Mandalay,Normal,Home and lifestyle,46.93,7,16.43,2024-12-05 +INV-00507,Mandalay,Member,Home and lifestyle,63.13,3,9.47,2024-03-01 +INV-00566,Naypyitaw,Normal,Food and beverages,13.72,8,5.49,2024-11-01 +INV-00497,Mandalay,Normal,Health and beauty,20.75,10,10.38,2024-06-02 +INV-00646,Mandalay,Member,Food and beverages,55.49,5,13.87,2024-10-05 +INV-00583,Bago,Member,Health and beauty,21.51,9,9.68,2024-05-03 +INV-00429,Naypyitaw,Member,Home and lifestyle,43.39,10,21.7,2024-01-18 +INV-00478,Yangon,Normal,Health and beauty,60.12,5,15.03,2024-02-10 +INV-00167,Mandalay,Member,Home and lifestyle,82.76,3,12.41,2024-09-24 +INV-00253,Mandalay,Member,Fashion accessories,83.0,10,41.5,2024-08-19 +INV-00736,Mandalay,Normal,Electronic accessories,27.02,1,1.35,2024-01-19 +INV-00338,Taunggyi,Member,Food and beverages,53.14,4,10.63,2024-09-02 +INV-00636,Taunggyi,Member,Health and beauty,58.99,6,17.7,2024-12-27 +INV-00539,Yangon,Normal,Home and lifestyle,67.64,10,33.82,2024-09-09 +INV-00990,Taunggyi,Normal,Home and lifestyle,49.24,10,24.62,2024-07-05 +INV-00270,Mandalay,Normal,Health and beauty,80.54,10,40.27,2024-05-22 +INV-00641,Bago,Member,Fashion accessories,77.45,9,34.85,2024-12-04 +INV-00009,Taunggyi,Normal,Electronic accessories,51.17,6,15.35,2024-06-18 +INV-00731,Mandalay,Member,Sports and travel,90.43,10,45.22,2024-10-19 +INV-00273,Taunggyi,Normal,Home and lifestyle,69.93,7,24.48,2024-04-15 +INV-00480,Bago,Member,Home and lifestyle,98.25,7,34.39,2024-01-26 +INV-00251,Bago,Normal,Electronic accessories,99.41,7,34.79,2024-05-19 +INV-00025,Bago,Normal,Electronic accessories,41.1,4,8.22,2024-03-07 +INV-00174,Yangon,Normal,Health and beauty,34.84,1,1.74,2024-05-26 +INV-00653,Mandalay,Normal,Food and beverages,67.06,7,23.47,2024-07-06 +INV-00848,Mandalay,Normal,Health and beauty,54.74,2,5.47,2024-11-26 +INV-00072,Yangon,Normal,Home and lifestyle,41.77,4,8.35,2024-10-04 +INV-00083,Taunggyi,Member,Health and beauty,75.19,1,3.76,2024-10-16 +INV-00432,Bago,Normal,Electronic accessories,99.9,2,9.99,2024-07-15 +INV-00291,Yangon,Normal,Electronic accessories,90.16,9,40.57,2024-07-17 +INV-00383,Mandalay,Member,Sports and travel,54.68,8,21.87,2024-02-02 +INV-00665,Taunggyi,Normal,Home and lifestyle,28.52,7,9.98,2024-08-28 +INV-00866,Yangon,Normal,Food and beverages,28.49,6,8.55,2024-10-24 +INV-00816,Yangon,Member,Food and beverages,26.76,3,4.01,2024-11-20 +INV-00123,Mandalay,Member,Fashion accessories,72.95,10,36.48,2024-04-25 +INV-00368,Taunggyi,Normal,Home and lifestyle,47.26,7,16.54,2024-02-15 +INV-00972,Yangon,Member,Electronic accessories,53.66,10,26.83,2024-08-12 +INV-00159,Taunggyi,Normal,Electronic accessories,68.15,5,17.04,2024-09-22 +INV-00229,Mandalay,Member,Fashion accessories,22.49,6,6.75,2024-09-25 +INV-00537,Taunggyi,Member,Food and beverages,44.53,7,15.59,2024-01-02 +INV-00017,Bago,Member,Fashion accessories,94.36,8,37.74,2024-06-23 +INV-00055,Mandalay,Member,Health and beauty,13.54,2,1.35,2024-02-20 +INV-00136,Naypyitaw,Normal,Fashion accessories,37.46,6,11.24,2024-04-26 +INV-00530,Mandalay,Member,Electronic accessories,53.91,8,21.56,2024-10-28 +INV-00213,Naypyitaw,Member,Sports and travel,55.58,7,19.45,2024-03-27 +INV-00957,Naypyitaw,Normal,Fashion accessories,90.07,7,31.52,2024-07-02 +INV-00712,Bago,Member,Sports and travel,26.75,1,1.34,2024-07-25 +INV-00958,Yangon,Normal,Home and lifestyle,88.95,2,8.9,2024-08-05 +INV-00419,Bago,Normal,Sports and travel,54.12,7,18.94,2024-05-23 +INV-00571,Yangon,Member,Electronic accessories,91.72,7,32.1,2024-03-08 +INV-00317,Yangon,Normal,Food and beverages,97.34,10,48.67,2024-07-22 +INV-00831,Yangon,Member,Electronic accessories,68.93,3,10.34,2024-09-10 +INV-00860,Mandalay,Member,Fashion accessories,14.26,7,4.99,2024-05-07 +INV-00104,Mandalay,Member,Home and lifestyle,43.74,2,4.37,2024-08-10 +INV-00493,Naypyitaw,Member,Electronic accessories,98.91,8,39.56,2024-10-18 +INV-00359,Yangon,Normal,Food and beverages,21.15,3,3.17,2024-05-11 +INV-00361,Yangon,Member,Home and lifestyle,37.69,7,13.19,2024-11-26 +INV-00152,Naypyitaw,Member,Fashion accessories,93.71,5,23.43,2024-11-04 +INV-00358,Yangon,Member,Food and beverages,92.47,6,27.74,2024-06-28 +INV-00103,Bago,Normal,Fashion accessories,39.41,8,15.76,2024-11-15 +INV-00209,Bago,Normal,Fashion accessories,35.48,10,17.74,2024-05-10 +INV-00343,Yangon,Normal,Sports and travel,34.26,3,5.14,2024-08-29 +INV-00635,Bago,Normal,Home and lifestyle,64.38,3,9.66,2024-01-10 +INV-00084,Bago,Normal,Home and lifestyle,37.6,3,5.64,2024-03-02 +INV-00003,Bago,Member,Health and beauty,43.51,9,19.58,2024-08-10 +INV-00454,Taunggyi,Normal,Sports and travel,33.62,3,5.04,2024-09-09 +INV-00606,Taunggyi,Member,Food and beverages,81.85,3,12.28,2024-11-15 +INV-00686,Yangon,Normal,Home and lifestyle,63.93,3,9.59,2024-10-25 +INV-00511,Mandalay,Member,Fashion accessories,85.1,8,34.04,2024-08-30 +INV-00246,Naypyitaw,Member,Sports and travel,15.96,2,1.6,2024-03-22 +INV-00390,Taunggyi,Normal,Health and beauty,31.89,7,11.16,2024-05-23 +INV-00876,Bago,Member,Electronic accessories,27.26,7,9.54,2024-12-19 +INV-00620,Bago,Normal,Food and beverages,30.12,5,7.53,2024-11-17 +INV-00060,Yangon,Member,Health and beauty,57.02,3,8.55,2024-08-13 +INV-00776,Yangon,Normal,Food and beverages,56.89,7,19.91,2024-03-14 +INV-00699,Yangon,Member,Health and beauty,58.12,8,23.25,2024-01-12 +INV-00923,Naypyitaw,Normal,Fashion accessories,35.74,8,14.3,2024-05-29 +INV-00492,Taunggyi,Normal,Food and beverages,72.87,9,32.79,2024-02-20 +INV-00489,Naypyitaw,Member,Fashion accessories,14.48,5,3.62,2024-07-23 +INV-00332,Taunggyi,Member,Electronic accessories,20.61,10,10.3,2024-01-31 +INV-00028,Mandalay,Normal,Food and beverages,26.82,2,2.68,2024-06-17 +INV-00555,Mandalay,Normal,Food and beverages,28.48,7,9.97,2024-10-17 +INV-00739,Naypyitaw,Normal,Sports and travel,84.81,1,4.24,2024-04-14 +INV-00423,Mandalay,Normal,Health and beauty,29.96,9,13.48,2024-08-17 +INV-00946,Yangon,Normal,Food and beverages,39.8,8,15.92,2024-09-29 +INV-00233,Bago,Normal,Home and lifestyle,26.49,4,5.3,2024-09-02 +INV-00256,Naypyitaw,Member,Home and lifestyle,67.02,9,30.16,2024-08-10 +INV-00603,Taunggyi,Member,Fashion accessories,28.13,6,8.44,2024-08-29 +INV-00373,Yangon,Member,Fashion accessories,47.79,10,23.89,2024-03-06 +INV-00880,Taunggyi,Member,Fashion accessories,85.46,2,8.55,2024-03-17 +INV-00226,Mandalay,Normal,Home and lifestyle,11.75,4,2.35,2024-10-28 +INV-00996,Bago,Normal,Fashion accessories,57.19,9,25.74,2024-07-31 +INV-00577,Naypyitaw,Member,Fashion accessories,20.43,6,6.13,2024-10-29 +INV-00330,Bago,Member,Health and beauty,35.35,8,14.14,2024-10-18 +INV-00477,Naypyitaw,Normal,Home and lifestyle,35.05,7,12.27,2024-01-30 +INV-00817,Mandalay,Member,Fashion accessories,92.87,6,27.86,2024-05-18 +INV-00338,Taunggyi,Member,Food and beverages,53.14,4,10.63,2024-09-02 +INV-00642,Yangon,Normal,Food and beverages,35.38,5,8.85,2024-05-17 +INV-00049,Yangon,Normal,Sports and travel,35.85,4,7.17,2024-08-17 +INV-00134,Mandalay,Member,Fashion accessories,14.3,1,0.72,2024-08-22 +INV-00092,Naypyitaw,Member,Electronic accessories,85.38,10,42.69,2024-01-10 +INV-00683,Taunggyi,Member,Electronic accessories,49.8,7,17.43,2024-05-09 +INV-00570,Yangon,Normal,Electronic accessories,72.49,1,3.62,2024-12-02 +INV-00669,Naypyitaw,Member,Food and beverages,32.58,3,4.89,2024-04-27 +INV-00654,Taunggyi,Normal,Home and lifestyle,17.8,7,6.23,2024-07-11 +INV-00697,Naypyitaw,Normal,Fashion accessories,26.52,2,2.65,2024-08-27 +INV-00953,Naypyitaw,Member,Electronic accessories,20.35,4,4.07,2024-09-23 +INV-00967,Taunggyi,Member,Sports and travel,45.64,7,15.97,2024-11-19 +INV-00105,Yangon,Member,Electronic accessories,65.13,8,26.05,2024-10-18 +INV-00222,Naypyitaw,Member,Fashion accessories,13.31,7,4.66,2024-10-25 +INV-00651,Yangon,Member,Home and lifestyle,33.66,6,10.1,2024-09-29 +INV-00895,Yangon,Normal,Health and beauty,98.97,5,24.74,2024-06-01 +INV-00063,Mandalay,Normal,Health and beauty,54.67,1,2.73,2024-05-03 +INV-00242,Yangon,Normal,Food and beverages,92.29,8,36.92,2024-10-09 +INV-00746,Naypyitaw,Normal,Home and lifestyle,56.72,7,19.85,2024-10-17 +INV-00490,Taunggyi,Normal,Fashion accessories,93.96,8,37.58,2024-03-07 +INV-00482,Bago,Member,Home and lifestyle,73.7,2,7.37,2024-09-19 +INV-00028,Mandalay,Normal,Food and beverages,26.82,2,2.68,2024-06-17 +INV-00002,Taunggyi,Normal,Food and beverages,63.89,10,31.95,2024-01-10 +INV-00801,Mandalay,Member,Home and lifestyle,27.41,5,6.85,2024-10-06 +INV-00596,Bago,Normal,Home and lifestyle,17.0,5,4.25,2024-01-13 +INV-00278,Bago,Member,Food and beverages,97.11,4,19.42,2024-04-22 +INV-00188,Yangon,Normal,Health and beauty,39.7,2,3.97,2024-04-28 +INV-00461,Mandalay,Normal,Food and beverages,54.69,8,21.88,2024-11-30 +INV-00973,Yangon,Normal,Food and beverages,40.6,5,10.15,2024-05-04 +INV-00601,Taunggyi,Member,Health and beauty,45.48,3,6.82,2024-05-17 +INV-00562,Mandalay,Member,Sports and travel,72.09,3,10.81,2024-01-24 +INV-00717,Mandalay,Member,Fashion accessories,58.6,1,2.93,2024-10-01 +INV-00447,Bago,Normal,Fashion accessories,66.55,9,29.95,2024-10-08 +INV-00922,Taunggyi,Normal,Food and beverages,79.97,8,31.99,2024-01-02 +INV-00572,Mandalay,Member,Fashion accessories,50.13,1,2.51,2024-11-10 +INV-00385,Mandalay,Member,Food and beverages,61.42,6,18.43,2024-06-28 +INV-00089,Bago,Member,Health and beauty,42.35,2,4.24,2024-12-08 +INV-00745,Bago,Normal,Sports and travel,42.07,8,16.83,2024-03-09 +INV-00013,Naypyitaw,Member,Fashion accessories,64.17,2,6.42,2024-07-02 +INV-00245,Naypyitaw,Normal,Home and lifestyle,45.73,6,13.72,2024-11-07 +INV-00980,Bago,Normal,Electronic accessories,67.27,4,13.45,2024-04-25 +INV-00251,Bago,Normal,Electronic accessories,99.41,7,34.79,2024-05-19 +INV-00622,Mandalay,Normal,Health and beauty,15.12,9,6.8,2024-11-16 +INV-00211,Taunggyi,Normal,Electronic accessories,14.33,2,1.43,2024-03-30 +INV-00839,Naypyitaw,Member,Sports and travel,70.47,5,17.62,2024-12-17 +INV-00819,Taunggyi,Member,Food and beverages,11.28,3,1.69,2024-07-07 +INV-00242,Yangon,Normal,Food and beverages,92.29,8,36.92,2024-10-09 +INV-00549,Naypyitaw,Member,Electronic accessories,29.73,7,10.41,2024-12-12 +INV-00560,Mandalay,Normal,Food and beverages,26.95,7,9.43,2024-11-21 +INV-00826,Taunggyi,Normal,Health and beauty,68.48,10,34.24,2024-01-05 +INV-00915,Naypyitaw,Normal,Home and lifestyle,57.46,10,28.73,2024-05-19 +INV-00418,Naypyitaw,Member,Electronic accessories,56.18,5,14.04,2024-04-07 +INV-00445,Bago,Normal,Food and beverages,14.19,10,7.1,2024-07-08 +INV-00144,Bago,Normal,Home and lifestyle,33.29,6,9.99,2024-07-15 +INV-00377,Bago,Normal,Sports and travel,68.02,10,34.01,2024-04-15 +INV-00633,Bago,Member,Home and lifestyle,23.79,7,8.33,2024-02-27 +INV-00395,Naypyitaw,Member,Fashion accessories,16.86,6,5.06,2024-06-06 +INV-00157,Yangon,Member,Health and beauty,17.52,1,0.88,2024-08-22 +INV-00440,Taunggyi,Member,Home and lifestyle,57.79,4,11.56,2024-11-21 +INV-00398,Taunggyi,Member,Electronic accessories,66.43,9,29.89,2024-04-17 +INV-00941,Yangon,Member,Home and lifestyle,57.17,9,25.73,2024-07-03 +INV-00613,Bago,Normal,Fashion accessories,95.22,8,38.09,2024-12-09 +INV-00528,Bago,Normal,Food and beverages,44.78,7,15.67,2024-08-11 +INV-00505,Naypyitaw,Member,Sports and travel,30.41,2,3.04,2024-05-18 +INV-00436,Bago,Member,Health and beauty,14.37,3,2.16,2024-11-24 +INV-00061,Yangon,Member,Health and beauty,16.8,2,1.68,2024-08-24 +INV-00204,Taunggyi,Normal,Sports and travel,93.35,9,42.01,2024-12-24 +INV-00074,Bago,Normal,Sports and travel,55.29,3,8.29,2024-02-25 +INV-00758,Mandalay,Normal,Fashion accessories,66.1,10,33.05,2024-03-24 +INV-00822,Naypyitaw,Normal,Food and beverages,77.54,6,23.26,2024-10-29 +INV-00019,Taunggyi,Normal,Home and lifestyle,94.17,7,32.96,2024-05-13 +INV-00165,Bago,Member,Food and beverages,51.09,6,15.33,2024-01-18 +INV-00428,Taunggyi,Member,Fashion accessories,97.05,1,4.85,2024-03-07 +INV-00114,Naypyitaw,Normal,Food and beverages,40.4,5,10.1,2024-07-02 +INV-00119,Naypyitaw,Member,Electronic accessories,92.09,8,36.84,2024-09-25 +INV-00630,Mandalay,Member,Food and beverages,53.75,3,8.06,2024-07-27 +INV-00711,Taunggyi,Normal,Health and beauty,29.97,6,8.99,2024-07-31 +INV-00457,Yangon,Normal,Home and lifestyle,69.51,3,10.43,2024-03-08 +INV-00293,Bago,Member,Sports and travel,53.69,1,2.68,2024-05-07 +INV-00146,Yangon,Member,Electronic accessories,83.31,10,41.66,2024-01-29 +INV-00867,Naypyitaw,Member,Electronic accessories,15.65,3,2.35,2024-12-14 +INV-00790,Yangon,Normal,Fashion accessories,33.98,6,10.19,2024-09-28 +INV-00940,Mandalay,Normal,Electronic accessories,74.27,4,14.85,2024-10-31 +INV-00138,Taunggyi,Member,Sports and travel,20.38,6,6.11,2024-04-06 +INV-00909,Bago,Member,Sports and travel,12.14,6,3.64,2024-12-20 +INV-00579,Naypyitaw,Member,Sports and travel,60.63,6,18.19,2024-12-07 +INV-00012,Yangon,Normal,Sports and travel,54.54,6,16.36,2024-08-10 +INV-00056,Naypyitaw,Normal,Home and lifestyle,65.42,9,29.44,2024-12-08 +INV-00977,Naypyitaw,Normal,Fashion accessories,73.03,10,36.52,2024-01-23 +INV-00758,Mandalay,Normal,Fashion accessories,66.1,10,33.05,2024-03-24 +INV-00902,Mandalay,Member,Sports and travel,43.59,1,2.18,2024-06-30 +INV-00674,Taunggyi,Member,Sports and travel,13.03,10,6.51,2024-12-13 +INV-00523,Mandalay,Member,Fashion accessories,23.76,4,4.75,2024-03-12 +INV-00706,Bago,Member,Home and lifestyle,63.98,4,12.8,2024-05-03 +INV-00738,Mandalay,Member,Home and lifestyle,36.47,1,1.82,2024-02-27 +INV-00881,Bago,Member,Fashion accessories,65.46,1,3.27,2024-11-19 +INV-00916,Yangon,Normal,Sports and travel,65.09,2,6.51,2024-03-07 +INV-00573,Bago,Member,Fashion accessories,69.35,7,24.27,2024-11-03 +INV-00628,Yangon,Normal,Fashion accessories,20.53,8,8.21,2024-09-29 +INV-00320,Naypyitaw,Member,Electronic accessories,98.26,8,39.3,2024-05-01 +INV-00294,Bago,Member,Sports and travel,56.61,8,22.64,2024-12-24 +INV-00757,Mandalay,Normal,Health and beauty,12.53,2,1.25,2024-01-18 +INV-00546,Taunggyi,Member,Food and beverages,47.94,8,19.18,2024-12-10 +INV-00968,Naypyitaw,Member,Home and lifestyle,33.97,6,10.19,2024-10-28 +INV-00521,Naypyitaw,Normal,Sports and travel,62.41,5,15.6,2024-03-10 +INV-00531,Bago,Normal,Electronic accessories,69.43,6,20.83,2024-05-10 +INV-00919,Naypyitaw,Normal,Home and lifestyle,65.01,2,6.5,2024-08-15 +INV-00218,Mandalay,Member,Fashion accessories,43.98,10,21.99,2024-12-10 +INV-00868,Bago,Member,Home and lifestyle,69.3,10,34.65,2024-03-02 +INV-00366,Yangon,Member,Electronic accessories,45.07,4,9.01,2024-11-18 +INV-00224,Yangon,Member,Health and beauty,61.0,3,9.15,2024-12-19 +INV-00602,Bago,Member,Health and beauty,74.81,8,29.92,2024-07-09 +INV-00965,Naypyitaw,Member,Electronic accessories,57.61,6,17.28,2024-02-13 +INV-00345,Yangon,Normal,Sports and travel,33.5,4,6.7,2024-08-07 +INV-00910,Mandalay,Member,Health and beauty,36.38,6,10.91,2024-04-25 +INV-00387,Bago,Normal,Home and lifestyle,16.69,2,1.67,2024-10-20 +INV-00529,Taunggyi,Normal,Food and beverages,89.85,5,22.46,2024-05-10 +INV-00353,Bago,Normal,Electronic accessories,19.27,2,1.93,2024-11-13 +INV-00536,Bago,Member,Food and beverages,58.31,1,2.92,2024-10-23 +INV-00210,Bago,Normal,Sports and travel,67.15,3,10.07,2024-06-06 +INV-00956,Yangon,Member,Fashion accessories,47.49,3,7.12,2024-12-15 +INV-00928,Mandalay,Normal,Electronic accessories,90.57,1,4.53,2024-02-03 +INV-00255,Bago,Normal,Electronic accessories,41.17,8,16.47,2024-09-24 +INV-00132,Mandalay,Normal,Electronic accessories,23.42,10,11.71,2024-10-24 +INV-00268,Bago,Member,Home and lifestyle,54.32,6,16.3,2024-11-13 +INV-00828,Yangon,Member,Health and beauty,93.09,9,41.89,2024-11-14 +INV-00182,Yangon,Normal,Fashion accessories,21.25,8,8.5,2024-07-30 +INV-00897,Bago,Normal,Home and lifestyle,59.92,3,8.99,2024-08-24 +INV-00375,Mandalay,Member,Home and lifestyle,52.57,9,23.66,2024-05-04 +INV-00062,Taunggyi,Normal,Electronic accessories,66.62,5,16.66,2024-12-19 +INV-00307,Taunggyi,Member,Food and beverages,37.41,9,16.83,2024-06-01 +INV-00857,Yangon,Normal,Sports and travel,55.45,5,13.86,2024-10-08 +INV-00892,Bago,Normal,Health and beauty,84.81,7,29.68,2024-09-20 +INV-00264,Yangon,Member,Health and beauty,37.13,2,3.71,2024-04-02 +INV-00645,Yangon,Member,Fashion accessories,56.58,4,11.32,2024-05-03 +INV-00735,Yangon,Member,Home and lifestyle,89.95,10,44.98,2024-10-07 +INV-00208,Naypyitaw,Member,Electronic accessories,47.04,6,14.11,2024-12-03 +INV-00487,Mandalay,Member,Home and lifestyle,77.45,6,23.24,2024-10-26 +INV-00444,Taunggyi,Member,Home and lifestyle,97.97,5,24.49,2024-03-20 +INV-00135,Bago,Member,Health and beauty,74.07,5,18.52,2024-03-03 +INV-00505,Naypyitaw,Member,Sports and travel,30.41,2,3.04,2024-05-18 +INV-00488,Taunggyi,Normal,Health and beauty,20.0,2,2.0,2024-08-04 +INV-00800,Taunggyi,Normal,Sports and travel,88.93,8,35.57,2024-01-06 +INV-00085,Yangon,Normal,Home and lifestyle,23.3,4,4.66,2024-01-15 +INV-00668,Mandalay,Normal,Food and beverages,97.73,2,9.77,2024-11-01 +INV-00671,Taunggyi,Member,Electronic accessories,94.43,3,14.16,2024-08-07 +INV-00266,Yangon,Member,Health and beauty,28.58,9,12.86,2024-11-19 +INV-00621,Yangon,Normal,Health and beauty,21.26,6,6.38,2024-05-09 +INV-00907,Taunggyi,Member,Sports and travel,34.51,6,10.35,2024-07-07 +INV-00896,Bago,Normal,Food and beverages,65.36,3,9.8,2024-06-20 +INV-00870,Mandalay,Normal,Electronic accessories,18.5,6,5.55,2024-07-31 +INV-00352,Yangon,Member,Home and lifestyle,40.13,9,18.06,2024-05-27 +INV-00216,Yangon,Normal,Fashion accessories,82.72,6,24.82,2024-02-02 +INV-00649,Mandalay,Member,Fashion accessories,53.93,1,2.7,2024-03-01 +INV-00840,Naypyitaw,Normal,Sports and travel,30.68,10,15.34,2024-12-08 +INV-00664,Mandalay,Normal,Home and lifestyle,76.45,3,11.47,2024-04-15 +INV-00214,Mandalay,Normal,Fashion accessories,93.97,5,23.49,2024-07-18 +INV-00369,Bago,Member,Sports and travel,45.98,3,6.9,2024-03-26 +INV-00837,Yangon,Normal,Health and beauty,67.5,3,10.12,2024-02-16 +INV-00927,Taunggyi,Member,Home and lifestyle,24.71,2,2.47,2024-02-16 +INV-00196,Naypyitaw,Normal,Home and lifestyle,22.65,5,5.66,2024-07-04 +INV-00234,Taunggyi,Member,Sports and travel,54.67,6,16.4,2024-03-02 +INV-00563,Taunggyi,Member,Electronic accessories,27.2,5,6.8,2024-02-14 +INV-00295,Yangon,Normal,Health and beauty,77.89,6,23.37,2024-09-14 +INV-00509,Naypyitaw,Member,Electronic accessories,36.38,1,1.82,2024-06-20 +INV-00365,Bago,Normal,Health and beauty,53.57,1,2.68,2024-07-04 +INV-00180,Mandalay,Member,Sports and travel,89.6,8,35.84,2024-09-19 +INV-00925,Taunggyi,Normal,Home and lifestyle,88.54,3,13.28,2024-10-01 +INV-00614,Taunggyi,Member,Health and beauty,66.5,10,33.25,2024-01-16 +INV-00475,Taunggyi,Member,Fashion accessories,54.3,7,19.0,2024-10-27 +INV-00553,Taunggyi,Normal,Fashion accessories,57.32,6,17.2,2024-01-15 +INV-00914,Bago,Member,Home and lifestyle,91.53,1,4.58,2024-05-28 +INV-00318,Taunggyi,Normal,Health and beauty,99.52,8,39.81,2024-01-19 +INV-00611,Bago,Normal,Electronic accessories,55.76,4,11.15,2024-09-19 +INV-00768,Bago,Normal,Sports and travel,76.18,1,3.81,2024-08-12 +INV-00912,Naypyitaw,Member,Home and lifestyle,30.5,9,13.73,2024-06-13 +INV-00994,Naypyitaw,Normal,Health and beauty,21.9,7,7.66,2024-03-01 +INV-00625,Taunggyi,Normal,Fashion accessories,90.13,8,36.05,2024-10-17 +INV-00420,Taunggyi,Member,Electronic accessories,81.2,7,28.42,2024-06-11 +INV-00363,Naypyitaw,Normal,Health and beauty,69.97,2,7.0,2024-05-10 +INV-00951,Yangon,Normal,Fashion accessories,16.0,2,1.6,2024-08-31 +INV-00662,Naypyitaw,Normal,Fashion accessories,96.85,4,19.37,2024-06-04 +INV-00076,Mandalay,Normal,Home and lifestyle,54.5,7,19.07,2024-02-22 +INV-00983,Mandalay,Member,Home and lifestyle,53.06,6,15.92,2024-11-21 +INV-00580,Naypyitaw,Normal,Sports and travel,49.01,2,4.9,2024-11-25 +INV-00230,Yangon,Member,Health and beauty,57.89,3,8.68,2024-02-28 +INV-00238,Taunggyi,Member,Fashion accessories,98.01,9,44.1,2024-12-18 +INV-00452,Bago,Member,Fashion accessories,61.3,3,9.19,2024-09-13 +INV-00351,Yangon,Member,Electronic accessories,14.92,8,5.97,2024-05-30 +INV-00634,Yangon,Member,Food and beverages,73.72,9,33.17,2024-02-13 +INV-00082,Mandalay,Normal,Food and beverages,91.56,4,18.31,2024-10-11 +INV-00453,Mandalay,Normal,Sports and travel,74.07,10,37.03,2024-11-04 +INV-00508,Mandalay,Normal,Health and beauty,91.99,5,23.0,2024-03-03 +INV-00289,Taunggyi,Member,Health and beauty,47.42,7,16.6,2024-08-24 +INV-00465,Mandalay,Member,Home and lifestyle,53.34,2,5.33,2024-01-28 +INV-00691,Naypyitaw,Normal,Sports and travel,54.57,4,10.91,2024-04-30 +INV-00506,Bago,Member,Food and beverages,69.82,9,31.42,2024-04-14 +INV-00617,Mandalay,Normal,Health and beauty,68.06,8,27.22,2024-11-16 +INV-00392,Taunggyi,Member,Home and lifestyle,67.31,6,20.19,2024-06-11 +INV-00323,Bago,Normal,Health and beauty,12.61,6,3.78,2024-03-21 +INV-00438,Yangon,Member,Electronic accessories,11.47,1,0.57,2024-04-30 +INV-00061,Yangon,Member,Health and beauty,16.8,2,1.68,2024-08-24 +INV-00781,Mandalay,Normal,Electronic accessories,64.72,6,19.42,2024-05-11 +INV-00503,Naypyitaw,Member,Electronic accessories,15.62,5,3.9,2024-10-09 +INV-00619,Naypyitaw,Member,Electronic accessories,63.35,6,19.01,2024-12-30 +INV-00305,Yangon,Normal,Health and beauty,84.42,10,42.21,2024-03-21 +INV-00565,Taunggyi,Member,Fashion accessories,42.46,5,10.62,2024-06-01 +INV-00393,Mandalay,Member,Electronic accessories,18.56,4,3.71,2024-11-21 +INV-00510,Yangon,Normal,Food and beverages,27.75,6,8.33,2024-06-11 +INV-00567,Taunggyi,Normal,Home and lifestyle,48.07,10,24.04,2024-10-07 +INV-00242,Yangon,Normal,Food and beverages,92.29,8,36.92,2024-10-09 +INV-00169,Mandalay,Normal,Fashion accessories,21.69,10,10.85,2024-08-06 +INV-00424,Bago,Member,Fashion accessories,92.34,3,13.85,2024-08-27 +INV-00515,Yangon,Normal,Sports and travel,62.61,6,18.78,2024-04-01 +INV-00121,Taunggyi,Member,Sports and travel,33.8,8,13.52,2024-06-07 +INV-00292,Naypyitaw,Member,Food and beverages,52.45,4,10.49,2024-03-10 +INV-00249,Taunggyi,Normal,Sports and travel,33.78,5,8.45,2024-05-14 +INV-00032,Taunggyi,Normal,Home and lifestyle,28.67,3,4.3,2024-05-26 +INV-00809,Naypyitaw,Normal,Fashion accessories,44.27,2,4.43,2024-03-07 +INV-00797,Yangon,Member,Sports and travel,30.88,9,13.9,2024-09-02 +INV-00744,Taunggyi,Normal,Electronic accessories,22.01,1,1.1,2024-12-03 +INV-00626,Mandalay,Normal,Electronic accessories,38.68,9,17.41,2024-03-01 +INV-00190,Bago,Normal,Fashion accessories,29.16,8,11.66,2024-07-05 +INV-00732,Naypyitaw,Member,Home and lifestyle,29.72,8,11.89,2024-06-22 +INV-00036,Naypyitaw,Member,Home and lifestyle,72.97,4,14.59,2024-04-06 +INV-00413,Taunggyi,Normal,Food and beverages,94.15,10,47.08,2024-10-10 +INV-00040,Bago,Member,Home and lifestyle,37.87,4,7.57,2024-05-01 +INV-00309,Taunggyi,Member,Food and beverages,70.87,5,17.72,2024-02-11 +INV-00410,Bago,Normal,Sports and travel,35.5,3,5.33,2024-10-17 +INV-00930,Naypyitaw,Member,Home and lifestyle,38.74,3,5.81,2024-06-27 +INV-00676,Bago,Normal,Fashion accessories,26.5,4,5.3,2024-05-20 +INV-00597,Naypyitaw,Member,Home and lifestyle,83.64,10,41.82,2024-06-05 +INV-00818,Taunggyi,Member,Sports and travel,69.61,2,6.96,2024-02-06 +INV-00476,Taunggyi,Normal,Food and beverages,19.71,5,4.93,2024-06-03 +INV-00917,Mandalay,Normal,Fashion accessories,85.27,10,42.63,2024-09-18 +INV-00412,Taunggyi,Member,Fashion accessories,55.91,7,19.57,2024-01-06 +INV-00283,Naypyitaw,Member,Electronic accessories,81.61,2,8.16,2024-12-20 +INV-00610,Yangon,Member,Home and lifestyle,40.23,5,10.06,2024-11-23 +INV-00406,Taunggyi,Normal,Food and beverages,91.27,6,27.38,2024-10-04 +INV-00140,Taunggyi,Normal,Fashion accessories,35.37,9,15.92,2024-02-25 +INV-00498,Naypyitaw,Member,Sports and travel,30.64,10,15.32,2024-06-07 +INV-00813,Taunggyi,Normal,Food and beverages,88.22,3,13.23,2024-10-17 +INV-00254,Yangon,Member,Sports and travel,62.62,9,28.18,2024-09-03 +INV-00648,Yangon,Normal,Health and beauty,95.47,10,47.74,2024-01-15 +INV-00751,Mandalay,Member,Home and lifestyle,81.69,8,32.68,2024-01-16 +INV-00069,Mandalay,Normal,Food and beverages,10.93,1,0.55,2024-08-06 +INV-00981,Mandalay,Member,Health and beauty,47.06,7,16.47,2024-02-24 +INV-00473,Mandalay,Normal,Electronic accessories,69.44,9,31.25,2024-01-05 +INV-00236,Bago,Member,Sports and travel,50.48,4,10.1,2024-11-23 +INV-00288,Bago,Normal,Home and lifestyle,59.36,10,29.68,2024-05-13 +INV-00844,Bago,Member,Food and beverages,29.45,8,11.78,2024-03-17 +INV-00859,Bago,Member,Home and lifestyle,40.42,7,14.15,2024-04-25 +INV-00569,Taunggyi,Normal,Home and lifestyle,12.06,2,1.21,2024-12-03 +INV-00609,Taunggyi,Normal,Health and beauty,72.6,8,29.04,2024-05-31 +INV-00276,Yangon,Member,Food and beverages,72.27,1,3.61,2024-01-02 +INV-00743,Naypyitaw,Normal,Sports and travel,16.87,9,7.59,2024-09-12 +INV-00260,Yangon,Normal,Food and beverages,51.58,2,5.16,2024-01-14 +INV-00059,Bago,Normal,Health and beauty,63.16,4,12.63,2024-08-14 +INV-00525,Taunggyi,Normal,Food and beverages,62.04,4,12.41,2024-08-27 +INV-00591,Bago,Member,Food and beverages,10.72,1,0.54,2024-09-28 +INV-00838,Bago,Member,Fashion accessories,98.37,10,49.19,2024-01-06 +INV-00976,Taunggyi,Member,Fashion accessories,33.97,4,6.79,2024-08-14 +INV-00728,Yangon,Member,Food and beverages,71.57,8,28.63,2024-07-03 +INV-00703,Bago,Normal,Sports and travel,95.5,3,14.33,2024-01-01 +INV-00223,Taunggyi,Normal,Electronic accessories,78.18,9,35.18,2024-02-29 +INV-00037,Mandalay,Member,Home and lifestyle,25.29,8,10.12,2024-08-02 +INV-00865,Yangon,Normal,Fashion accessories,88.61,3,13.29,2024-05-24 +INV-00113,Mandalay,Normal,Health and beauty,59.53,10,29.77,2024-10-22 +INV-00290,Taunggyi,Member,Fashion accessories,46.09,7,16.13,2024-10-12 +INV-00162,Bago,Member,Sports and travel,99.68,6,29.9,2024-05-21 +INV-00287,Mandalay,Normal,Sports and travel,57.61,2,5.76,2024-03-25 +INV-00191,Yangon,Normal,Fashion accessories,51.24,8,20.5,2024-01-28 +INV-00586,Yangon,Member,Fashion accessories,56.46,8,22.58,2024-10-27 +INV-00535,Yangon,Normal,Health and beauty,79.47,3,11.92,2024-03-02 +INV-00187,Yangon,Member,Home and lifestyle,71.49,8,28.6,2024-01-09 +INV-00627,Bago,Normal,Electronic accessories,36.73,2,3.67,2024-01-09 +INV-00172,Mandalay,Member,Home and lifestyle,47.79,4,9.56,2024-01-12 +INV-00853,Yangon,Member,Sports and travel,64.04,9,28.82,2024-10-02 +INV-00785,Taunggyi,Member,Fashion accessories,51.08,5,12.77,2024-06-02 +INV-00773,Taunggyi,Normal,Food and beverages,73.94,5,18.48,2024-10-21 +INV-00026,Naypyitaw,Member,Food and beverages,32.43,2,3.24,2024-12-24 +INV-00007,Mandalay,Member,Food and beverages,94.77,3,14.22,2024-05-31 +INV-00887,Mandalay,Member,Food and beverages,98.78,7,34.57,2024-09-21 +INV-00483,Taunggyi,Normal,Health and beauty,51.84,4,10.37,2024-09-20 +INV-00400,Taunggyi,Normal,Electronic accessories,71.02,1,3.55,2024-11-13 +INV-00177,Mandalay,Member,Home and lifestyle,18.07,10,9.04,2024-10-18 +INV-00638,Bago,Member,Home and lifestyle,21.94,8,8.78,2024-01-08 +INV-00093,Taunggyi,Normal,Fashion accessories,90.66,6,27.2,2024-02-17 +INV-00604,Naypyitaw,Normal,Health and beauty,57.2,8,22.88,2024-04-01 +INV-00081,Taunggyi,Member,Home and lifestyle,63.91,8,25.56,2024-07-30 +INV-00106,Bago,Normal,Electronic accessories,18.46,5,4.62,2024-04-10 +INV-00241,Bago,Normal,Health and beauty,72.43,9,32.59,2024-12-30 +INV-00834,Naypyitaw,Member,Fashion accessories,91.88,7,32.16,2024-12-26 +INV-00189,Yangon,Member,Fashion accessories,88.98,8,35.59,2024-05-10 +INV-00083,Taunggyi,Member,Health and beauty,75.19,1,3.76,2024-10-16 +INV-00705,Mandalay,Member,Home and lifestyle,60.42,1,3.02,2024-08-31 +INV-00954,Mandalay,Normal,Home and lifestyle,70.79,3,10.62,2024-07-27 +INV-00716,Mandalay,Member,Fashion accessories,96.1,1,4.8,2024-02-20 +INV-00605,Bago,Normal,Health and beauty,15.12,7,5.29,2024-07-29 +INV-00724,Mandalay,Normal,Home and lifestyle,32.04,7,11.21,2024-03-13 +INV-00514,Mandalay,Normal,Health and beauty,17.7,9,7.96,2024-05-02 +INV-00349,Yangon,Member,Food and beverages,49.47,7,17.31,2024-07-21 +INV-00120,Yangon,Member,Home and lifestyle,31.91,5,7.98,2024-02-15 +INV-00313,Mandalay,Member,Fashion accessories,17.94,6,5.38,2024-05-18 +INV-00227,Naypyitaw,Member,Health and beauty,82.61,3,12.39,2024-10-04 +INV-00250,Bago,Member,Electronic accessories,22.04,5,5.51,2024-07-12 +INV-00632,Yangon,Normal,Sports and travel,65.37,9,29.42,2024-12-16 +INV-00339,Yangon,Normal,Food and beverages,86.51,4,17.3,2024-02-03 +INV-00020,Taunggyi,Member,Electronic accessories,45.81,1,2.29,2024-04-05 +INV-00554,Bago,Member,Sports and travel,79.24,5,19.81,2024-03-30 +INV-00450,Taunggyi,Member,Health and beauty,84.66,8,33.86,2024-10-09 +INV-00197,Naypyitaw,Member,Fashion accessories,71.17,9,32.03,2024-04-12 +INV-00685,Mandalay,Normal,Fashion accessories,17.21,9,7.74,2024-05-17 +INV-00921,Yangon,Normal,Food and beverages,31.21,1,1.56,2024-07-20 +INV-00545,Bago,Member,Sports and travel,27.58,10,13.79,2024-10-07 +INV-00297,Yangon,Normal,Sports and travel,22.16,6,6.65,2024-03-28 +INV-00396,Bago,Member,Sports and travel,29.1,9,13.1,2024-06-10 +INV-00281,Mandalay,Member,Health and beauty,23.86,3,3.58,2024-06-18 +INV-00225,Mandalay,Normal,Fashion accessories,47.45,2,4.75,2024-12-09 +INV-00195,Mandalay,Member,Home and lifestyle,69.22,8,27.69,2024-02-11 +INV-00354,Taunggyi,Member,Sports and travel,56.9,7,19.92,2024-06-09 +INV-00086,Yangon,Member,Health and beauty,25.5,8,10.2,2024-09-13 +INV-00176,Mandalay,Normal,Sports and travel,71.73,8,28.69,2024-10-23 +INV-00709,Taunggyi,Member,Sports and travel,87.0,10,43.5,2024-08-06 +INV-00598,Taunggyi,Member,Electronic accessories,76.09,7,26.63,2024-05-11 +INV-00843,Naypyitaw,Member,Home and lifestyle,54.48,4,10.9,2024-05-16 +INV-00750,Naypyitaw,Normal,Sports and travel,48.45,3,7.27,2024-10-19 +INV-00889,Mandalay,Normal,Home and lifestyle,59.65,9,26.84,2024-01-08 +INV-00877,Yangon,Member,Health and beauty,63.62,4,12.72,2024-07-05 +INV-00118,Taunggyi,Member,Electronic accessories,89.81,7,31.43,2024-08-21 +INV-00407,Mandalay,Normal,Fashion accessories,70.42,8,28.17,2024-06-17 +INV-00747,Yangon,Normal,Home and lifestyle,34.06,3,5.11,2024-04-29 +INV-00795,Naypyitaw,Normal,Fashion accessories,97.19,10,48.59,2024-09-24 +INV-00491,Naypyitaw,Normal,Health and beauty,93.42,5,23.36,2024-06-02 +INV-00898,Yangon,Normal,Home and lifestyle,87.17,7,30.51,2024-03-23 +INV-00846,Taunggyi,Normal,Food and beverages,44.2,10,22.1,2024-05-30 +INV-00784,Bago,Normal,Electronic accessories,70.19,8,28.08,2024-07-10 +INV-00615,Taunggyi,Member,Electronic accessories,60.47,7,21.16,2024-11-11 +INV-00151,Mandalay,Member,Health and beauty,14.22,2,1.42,2024-03-13 +INV-00279,Taunggyi,Member,Fashion accessories,69.46,8,27.78,2024-01-10 +INV-00376,Yangon,Member,Electronic accessories,70.22,6,21.07,2024-12-04 +INV-00823,Yangon,Member,Sports and travel,15.79,6,4.74,2024-09-08 +INV-00557,Mandalay,Member,Fashion accessories,84.49,10,42.25,2024-05-17 +INV-00766,Taunggyi,Normal,Fashion accessories,20.09,8,8.04,2024-10-07 +INV-00379,Yangon,Normal,Electronic accessories,40.85,7,14.3,2024-08-30 +INV-00277,Mandalay,Member,Food and beverages,87.03,10,43.52,2024-07-23 +INV-00547,Taunggyi,Normal,Electronic accessories,87.46,9,39.36,2024-11-18 +INV-00714,Taunggyi,Member,Fashion accessories,41.53,6,12.46,2024-06-08 +INV-00832,Taunggyi,Normal,Fashion accessories,63.35,3,9.5,2024-03-24 +INV-00460,Yangon,Member,Food and beverages,70.15,6,21.05,2024-03-25 +INV-00850,Mandalay,Member,Sports and travel,28.4,10,14.2,2024-02-15 +INV-00792,Bago,Normal,Electronic accessories,52.49,7,18.37,2024-09-16 +INV-00533,Yangon,Member,Electronic accessories,70.31,4,14.06,2024-08-26 +INV-00780,Taunggyi,Member,Food and beverages,21.39,1,1.07,2024-03-11 +INV-00650,Yangon,Normal,Electronic accessories,21.53,8,8.61,2024-05-12 +INV-00756,Bago,Normal,Health and beauty,97.08,9,43.69,2024-06-30 +INV-00235,Taunggyi,Member,Fashion accessories,86.52,3,12.98,2024-01-08 +INV-00723,Naypyitaw,Normal,Sports and travel,62.57,5,15.64,2024-02-07 +INV-00231,Naypyitaw,Member,Fashion accessories,60.55,7,21.19,2024-03-26 +INV-00380,Bago,Member,Sports and travel,88.04,7,30.81,2024-08-17 +INV-00164,Mandalay,Member,Health and beauty,18.72,3,2.81,2024-09-02 +INV-00386,Bago,Normal,Fashion accessories,15.13,2,1.51,2024-05-09 +INV-00779,Yangon,Member,Home and lifestyle,25.83,3,3.87,2024-02-14 +INV-00772,Bago,Normal,Electronic accessories,95.74,3,14.36,2024-12-29 +INV-00770,Mandalay,Member,Home and lifestyle,46.35,6,13.91,2024-01-18 +INV-00112,Mandalay,Normal,Fashion accessories,82.27,6,24.68,2024-04-26 +INV-00966,Mandalay,Member,Home and lifestyle,99.24,8,39.7,2024-04-29 +INV-00495,Naypyitaw,Normal,Health and beauty,86.31,1,4.32,2024-10-26 +INV-00952,Mandalay,Normal,Fashion accessories,61.07,5,15.27,2024-06-02 +INV-00955,Bago,Normal,Electronic accessories,11.45,4,2.29,2024-01-27 +INV-00502,Bago,Member,Food and beverages,45.71,6,13.71,2024-12-31 +INV-00575,Yangon,Normal,Food and beverages,15.31,9,6.89,2024-06-23 +INV-00903,Mandalay,Member,Health and beauty,91.34,9,41.1,2024-08-21 +INV-00004,Naypyitaw,Member,Electronic accessories,86.58,2,8.66,2024-07-11 +INV-00534,Naypyitaw,Normal,Home and lifestyle,38.19,4,7.64,2024-07-24 +INV-00327,Taunggyi,Member,Fashion accessories,80.02,2,8.0,2024-10-26 +INV-00524,Naypyitaw,Member,Food and beverages,75.36,8,30.14,2024-05-25 +INV-00693,Naypyitaw,Normal,Health and beauty,82.84,2,8.28,2024-10-27 +INV-00729,Mandalay,Member,Sports and travel,63.73,5,15.93,2024-10-17 +INV-00481,Yangon,Member,Food and beverages,60.58,10,30.29,2024-06-05 +INV-00066,Bago,Member,Home and lifestyle,69.12,9,31.1,2024-10-11 +INV-00149,Mandalay,Normal,Electronic accessories,12.8,3,1.92,2024-09-05 +INV-00415,Naypyitaw,Normal,Electronic accessories,50.99,4,10.2,2024-05-20 +INV-00791,Yangon,Member,Health and beauty,55.89,5,13.97,2024-01-02 +INV-00908,Yangon,Normal,Sports and travel,77.74,2,7.77,2024-02-14 +INV-00517,Taunggyi,Normal,Home and lifestyle,22.25,8,8.9,2024-07-08 +INV-00584,Mandalay,Normal,Electronic accessories,74.34,9,33.45,2024-12-22 +INV-00057,Mandalay,Normal,Food and beverages,45.81,3,6.87,2024-03-27 +INV-00471,Mandalay,Normal,Home and lifestyle,17.52,10,8.76,2024-04-04 +INV-00847,Naypyitaw,Member,Home and lifestyle,64.37,4,12.87,2024-05-19 +INV-00588,Naypyitaw,Normal,Home and lifestyle,76.95,3,11.54,2024-11-16 +INV-00841,Yangon,Member,Electronic accessories,93.15,7,32.6,2024-08-16 +INV-00594,Mandalay,Member,Sports and travel,80.15,9,36.07,2024-06-18 +INV-00324,Taunggyi,Normal,Sports and travel,22.95,8,9.18,2024-12-20 +INV-00389,Bago,Normal,Fashion accessories,19.08,1,0.95,2024-07-05 +INV-00200,Naypyitaw,Normal,Fashion accessories,95.78,2,9.58,2024-12-30 +INV-00612,Yangon,Member,Sports and travel,86.04,8,34.42,2024-08-08 diff --git a/docs/MakeFile b/docs/MakeFile deleted file mode 100644 index 3d0ce63c6..000000000 --- a/docs/MakeFile +++ /dev/null @@ -1,23 +0,0 @@ -# Minimal makefile for Sphinx documentation - -# You can set these variables from the command line, and also -# from the environment for the first two. -SPHINXOPTS ?= -SPHINXBUILD ?= sphinx-build -SOURCEDIR = . -BUILDDIR = _build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile clean - -# Custom clean command -clean: - rm -rf $(BUILDDIR)/* - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/docs/ai/index.md b/docs/ai/index.md index 4ab6f7579..9e92748df 100644 --- a/docs/ai/index.md +++ b/docs/ai/index.md @@ -34,7 +34,7 @@ The **Agent variant** picker in settings selects execution mode: - **Live (REPL)** *(default)* — each step applies immediately, the affected subgraph runs (Performance) or samples (Development), and the runtime observation feeds back to the model. Failed steps auto-undo and retry. No diff to accept — the canvas is the running record. Higher latency per step because each one does real work. - **Staged** — small/local-model-friendly. A tightly-scoped state machine makes one decision per LLM round and bundles proposals into a `GraphDiff` for **Accept** (atomic, one undo point) or **Reject** with an optional note that becomes context for the next attempt. -- **Single-shot full** — big-model mode (Sonnet, Opus, GPT-4.1, Gemini Pro). Exposes the full tool catalog in one call. Same staged-diff review flow. +- **Single-shot full** — big-model mode (the provider's strongest tool-capable model for the `agent_complex` surface). Exposes the full tool catalog in one call. Same staged-diff review flow. The **Verify plan completion** checkbox adds one extra LLM round after the agent decides it's done, to walk its plan as a checklist — catches multi-step plans that terminate after step 1. diff --git a/docs/ai/providers.md b/docs/ai/providers.md index d9bfddcb7..9b7238251 100644 --- a/docs/ai/providers.md +++ b/docs/ai/providers.md @@ -1,6 +1,6 @@ # Provider Setup (BYOK) -The [AI Assistant](index.md) runs against any major LLM provider — pick the one that fits your budget, latency needs, or compliance posture. Today that's Anthropic, OpenAI, Google, Groq, OpenRouter, and a local Ollama server. You bring your own API key; Flowfile encrypts it at rest with Fernet (using `FLOWFILE_MASTER_KEY` / `master_key.txt`), the same scheme that protects your other secrets. For air-gapped work, Ollama lets the entire AI layer run on your laptop with no traffic leaving your machine. +The [AI Assistant](index.md) runs against one of six LLM providers — pick the one that fits your budget, latency needs, or compliance posture: Anthropic, OpenAI, Google, Groq, OpenRouter, and a local Ollama server. You bring your own API key; Flowfile encrypts it at rest with Fernet (using `FLOWFILE_MASTER_KEY` / `master_key.txt`), the same scheme that protects your other secrets. For air-gapped work, Ollama lets the entire AI layer run on your laptop with no traffic leaving your machine. !!! info "Not in Flowfile Lite" The AI Assistant (and BYOK provider setup) requires the full desktop/server build, not the browser-only [Flowfile Lite](../users/deployment/lite.md) edition. @@ -9,17 +9,22 @@ The [AI Assistant](index.md) runs against any major LLM provider — pick the on ## Supported providers +Default models below reflect the provider classes as of 2026-07. + | Provider | Default model | Tools | Streaming | Key env var | Notes | |----------|---------------|:-----:|:---------:|-------------|-------| | **Anthropic** | `claude-sonnet-4-6` | ✓ | ✓ | `ANTHROPIC_API_KEY` | Best balance of quality and tool-use reliability. Haiku 4.5 is the default for fast surfaces (Cmd+K, ghost-node, autocomplete); Opus 4.7 for `agent_complex`. | | **OpenAI** | `gpt-4.1-mini` | ✓ | ✓ | `OPENAI_API_KEY` | Mini tier for the cheap surfaces; full `gpt-4.1` for `explain` / `agent_complex` / `docgen`. Strict structured outputs supported via litellm. | | **Google (Gemini)** | `gemini-2.5-flash` | ✓ | ✓ | `GEMINI_API_KEY` or `GOOGLE_API_KEY` | Generous free tier (~250–1000 req/day, no card). Pro for `agent_complex`. | -| **Groq** | `qwen/qwen3-coder-30b-a3b-instruct` | ✓ | ✓ | `GROQ_API_KEY` | Very fast inference (~30 RPM free tier); good fit for low-TTFB surfaces (Cmd+K, ghost-node). | -| **OpenRouter** | `qwen/qwen3-coder-30b-a3b-instruct` | ✓ | ✓ | `OPENROUTER_API_KEY` | Unified façade for 50+ models with a single key. The `agent_staged` default is `meta-llama/llama-3.3-70b-instruct` (free tier). | +| **Groq** | `qwen/qwen3-32b` | ✓ | ✓ | `GROQ_API_KEY` | Very fast inference (~30 RPM free tier); good fit for low-TTFB surfaces (Cmd+K, ghost-node). | +| **OpenRouter** | `qwen/qwen3-coder-30b-a3b-instruct` | ✓ | ✓ | `OPENROUTER_API_KEY` | One key routing to many hosted models. The `agent_staged` default is `meta-llama/llama-3.3-70b-instruct` (free tier). | | **Ollama** | `llama3.1:8b` | ✓ (model-dependent) | ✓ | *(none — local)* | Self-hosted; talks to your local Ollama server (default `http://localhost:11434`). Tool-use works on Llama 3.1+ and most newer instruct models. | The "Tools" column means the provider can return structured tool-call arguments — required for the Agent surface. The Agent refuses to start against a model that lacks tool support. +!!! note "Model IDs drift with releases" + The default and per-surface model IDs are hardcoded in the provider classes at `flowfile_core/flowfile_core/ai/providers/*.py` (one file per provider) and are bumped as new models ship. The table above is a snapshot; check those files for the current values before relying on a specific model name. + --- ## Configuring keys in the UI @@ -29,7 +34,7 @@ The recommended path is the in-app settings panel: 1. Open **Settings → AI Providers**. 2. Pick a provider from the list. The panel shows class-level metadata (default model, supports tools, supports streaming) plus your current credential status: **Configured** (key saved), **Env fallback** (no key saved but a recognised env var is set on the server), or **Unconfigured**. 3. Paste the API key into the *API key* field and click **Save**. For Ollama or self-hosted endpoints, set *API base* to the server URL. -4. Click **Test**. Flowfile issues a 1-token ping and records the result on the credential (`last_tested_at`, `last_test_status`). A green checkmark means you're good to go. +4. Click **Test**. Flowfile issues a 1-token ping and records the result on the credential (`last_tested_at`, `last_test_status`). ![AI Providers list with status chips](../assets/images/ai/byok_provider_list.png) @@ -57,7 +62,7 @@ If no credential row exists for a user, Flowfile falls back to the standard prov ## Choosing models per surface -Each AI feature is a *surface* (`chat`, `explain`, `agent_staged`, `agent_complex`, `cmd_k`, `ghost_node`, `settings_autocomplete`, `lineage`, `docgen`, `intent_classifier`). Each provider class ships sensible per-surface defaults — Haiku for fast paths, Sonnet for thinking paths, etc. — so you usually don't need to override anything. +Each AI feature is a *surface* (`chat`, `explain`, `agent_staged`, `agent_complex`, `cmd_k`, `ghost_node`, `settings_autocomplete`, `lineage`, `docgen`, `intent_classifier`, `cron`). Each provider class ships sensible per-surface defaults — Haiku for fast paths, Sonnet for thinking paths, etc. — so you usually don't need to override anything. When you do want to override: diff --git a/docs/assets/flows/sales_pipeline.yaml b/docs/assets/flows/sales_pipeline.yaml new file mode 100644 index 000000000..e769fa97b --- /dev/null +++ b/docs/assets/flows/sales_pipeline.yaml @@ -0,0 +1,111 @@ +flowfile_version: 0.8.2 +flowfile_id: 12 +flowfile_name: Sales Pipeline +flowfile_settings: + description: null + execution_mode: Development + execution_location: local + auto_save: false + show_detailed_progress: true + max_parallel_workers: 4 + source_registration_id: null + parameters: [] +nodes: +- id: 1 + type: read + is_start_node: true + description: '' + node_reference: null + x_position: 0 + y_position: 200 + left_input_id: null + right_input_id: null + input_ids: [] + outputs: [2] + setting_input: + cache_results: false + received_file: + path: https://raw.githubusercontent.com/edwardvaneechoud/flowfile/main/data/templates/supermarket_sales.csv + file_type: csv + table_settings: + file_type: csv +- id: 2 + type: unique + is_start_node: false + description: '' + node_reference: null + x_position: 250 + y_position: 200 + left_input_id: null + right_input_id: null + input_ids: [1] + outputs: [3] + setting_input: + cache_results: false + unique_input: + columns: null + strategy: any +- id: 3 + type: filter + is_start_node: false + description: '' + node_reference: null + x_position: 500 + y_position: 200 + left_input_id: null + right_input_id: null + input_ids: [2] + outputs: [4] + setting_input: + cache_results: false + filter_input: + mode: advanced + advanced_filter: '[quantity] > 7' +- id: 4 + type: group_by + is_start_node: false + description: '' + node_reference: null + x_position: 750 + y_position: 200 + left_input_id: null + right_input_id: null + input_ids: [3] + outputs: [5] + setting_input: + cache_results: false + groupby_input: + agg_cols: + - old_name: city + agg: groupby + new_name: city + - old_name: gross_income + agg: sum + new_name: total_income + - old_name: gross_income + agg: median + new_name: median_income +- id: 5 + type: explore_data + is_start_node: false + description: 'Explore the output!' + node_reference: null + x_position: 1000 + y_position: 200 + left_input_id: null + right_input_id: null + input_ids: [4] + outputs: [] + setting_input: + cache_results: false + graphic_walker_input: null +_template_meta: + template_id: sales_pipeline + name: 'Sales pipeline: clean, filter, aggregate' + description: Drop duplicate sales rows, keep high-quantity orders, then total and + median gross income per city. + category: Beginner + tags: [unique, filter, group_by] + node_count: 5 + icon: insights +_required_csv_files: [supermarket_sales.csv] diff --git a/docs/assets/try-sales-pipeline.html b/docs/assets/try-sales-pipeline.html new file mode 100644 index 000000000..edebef4ac --- /dev/null +++ b/docs/assets/try-sales-pipeline.html @@ -0,0 +1,11 @@ + + + + +Opening the sales pipeline in Flowfile Lite… + + + +

Opening the sales pipeline in Flowfile Lite… The flow and its sample data are encoded in the link itself — nothing is uploaded.

+ + diff --git a/docs/community.md b/docs/community.md index 072d66fa8..d7b7ace49 100644 --- a/docs/community.md +++ b/docs/community.md @@ -1,6 +1,6 @@ # Community -Flowfile is an MIT-licensed open source project, and it's built by the people who use it. Whether you want to ask a question, share a flow, report a bug, or propose a feature, there's a place for you here. +Flowfile is an MIT-licensed open source project. Use the channels below to ask a question, share a flow, report a bug, or propose a feature. ## Where to go diff --git a/docs/conf.py b/docs/conf.py deleted file mode 100644 index e890c1db2..000000000 --- a/docs/conf.py +++ /dev/null @@ -1,91 +0,0 @@ -# Configuration file for the Sphinx documentation builder. -import os -import sys - -sys.path.insert(0, os.path.abspath("..")) - -# Project information -project = "Flowfile" -copyright = "2024, Flowfile Contributors" -author = "Edward van Eechoud" -release = "0.1.2" # Update this with your version - -# Add any Sphinx extension module names here -extensions = [ - "sphinx.ext.autodoc", # Core documentation engine - "sphinx.ext.napoleon", # Support for Google-style docstrings - "sphinx.ext.viewcode", # Add links to source code - "sphinx.ext.githubpages", # Generate .nojekyll file - "myst_parser", # Markdown support - "sphinx_copybutton", # Add copy button to code blocks - "sphinx.ext.intersphinx", # Link to other projects' documentation - "sphinx_autodoc_typehints", # Better type hints support - "autoapi.extension", # API documentation -] - -autoapi_type = "python" -autoapi_dirs = [ - "../flowfile_core", - "../flowfile_worker", -] -autoapi_template_dir = "_templates/autoapi" -autoapi_python_class_content = "both" - -# Add any paths that contain templates here -templates_path = ["_templates"] -exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] - -# HTML theme settings -html_theme = "sphinx_rtd_theme" -html_static_path = ["_static"] -html_logo = "../.github/images/logo.png" -html_favicon = "../.github/images/logo.png" - -# Theme options -html_theme_options = { - "logo_only": True, - "display_version": True, - "prev_next_buttons_location": "bottom", - "style_external_links": False, - "style_nav_header_background": "#2980B9", - # Dark mode colors - "dark_mode_theme": "dark", - "dark_mode_css_variables": { - "color-brand-primary": "#2980B9", - "color-brand-content": "#2980B9", - }, -} - -# These folders are copied to the documentation's HTML output -html_static_path = ["_static"] - -# These paths are either relative to html_static_path or fully qualified paths -html_css_files = [ - "css/custom.css", -] - -# Intersphinx configuration -intersphinx_mapping = { - "python": ("https://docs.python.org/3", None), - "polars": ("https://pola-rs.github.io/polars/py-polars/html/", None), -} - -# MyST Markdown settings -myst_enable_extensions = [ - "colon_fence", - "deflist", -] - -# Napoleon settings -napoleon_google_docstring = True -napoleon_numpy_docstring = True -napoleon_include_init_with_doc = True -napoleon_include_private_with_doc = True -napoleon_include_special_with_doc = True -napoleon_use_admonition_for_examples = True -napoleon_use_admonition_for_notes = True -napoleon_use_admonition_for_references = True -napoleon_use_ivar = True -napoleon_use_param = True -napoleon_use_rtype = True -napoleon_type_aliases = None diff --git a/docs/examples/catalog_analysis.py b/docs/examples/catalog_analysis.py new file mode 100644 index 000000000..49cab16fc --- /dev/null +++ b/docs/examples/catalog_analysis.py @@ -0,0 +1,36 @@ +"""Write an aggregate to the Flowfile catalog, then query it back with SQL.""" + +# --8<-- [start:example] +import flowfile as ff +from flowfile_frame import read_catalog_sql + +income_by_city = ( + ff.read_csv("data/templates/supermarket_sales.csv") + .group_by("city") + .agg(ff.col("gross_income").sum().alias("total_income")) +) + +ff.write_catalog_table( + income_by_city, "docs_sales_by_city", schema=ff.default_schema(), write_mode="overwrite" +) + +top_cities = read_catalog_sql( + "SELECT city, total_income FROM docs_sales_by_city ORDER BY total_income DESC" +).collect() +# --8<-- [end:example] + +assert top_cities.columns == ["city", "total_income"] +assert top_cities.height == 5 + +rows = top_cities.to_dicts() +assert rows[0]["city"] == "Taunggyi" +assert round(rows[0]["total_income"], 2) == 3525.60 + +by_city = {row["city"]: round(row["total_income"], 2) for row in rows} +assert by_city == { + "Bago": 3051.94, + "Mandalay": 3520.57, + "Naypyitaw": 2521.64, + "Taunggyi": 3525.60, + "Yangon": 3312.56, +} diff --git a/docs/examples/code_to_flow.py b/docs/examples/code_to_flow.py new file mode 100644 index 000000000..dc6ec9bc5 --- /dev/null +++ b/docs/examples/code_to_flow.py @@ -0,0 +1,59 @@ +"""The two pipelines from the Building Flows with Code tutorial.""" + +# --8<-- [start:first] +import flowfile as ff +from flowfile import col + +df = ff.from_dict({ + "id": [1, 2, 2], + "value": [10, 20, 15] +}) + +result = ( + df.filter(col("value") > 12, description="filter value > 12") + .with_columns((col("value") * 10).alias("scaled_value"), description="get a scaled value") + .group_by(col("id")) + .agg( + col("value").sum().alias("sum_value"), + col("value").max().alias("max_value"), + col("value").min().alias("min_value"), + ) +) + +frame = result.collect() # a Polars DataFrame +# --8<-- [end:first] + +assert frame.height == 1 +row = frame.row(0, named=True) +assert row == {"id": 2, "sum_value": 35, "max_value": 20, "min_value": 15} + +# --8<-- [start:involved] +df = ff.from_dict({ + "id": [1, 2, 3, 4, 5], + "category": ["A", "B", "A", "C", "B"], + "value": [100, 200, 150, 300, 250] +}) + +aggregated = ( + df.filter(ff.col("value") > 120, description="Filter on value greater than 120") + .with_columns( + [ + (ff.col("value") * 1.1).alias("adjusted_value"), + ff.when(ff.col("category") == "A").then(ff.lit("Premium")) + .when(ff.col("category") == "B").then(ff.lit("Standard")) + .otherwise(ff.lit("Basic")).alias("tier"), + ], + description="Calculate their tier", + ) + .group_by("tier") + .agg( + ff.col("adjusted_value").sum().alias("total_value"), + ff.col("id").count().alias("count"), + ) +) +# --8<-- [end:involved] + +tiers = {r["tier"]: r for r in aggregated.collect().to_dicts()} +assert round(tiers["Standard"]["total_value"], 2) == 495.0 and tiers["Standard"]["count"] == 2 +assert round(tiers["Premium"]["total_value"], 2) == 165.0 and tiers["Premium"]["count"] == 1 +assert round(tiers["Basic"]["total_value"], 2) == 330.0 and tiers["Basic"]["count"] == 1 diff --git a/docs/examples/custom_node.py b/docs/examples/custom_node.py new file mode 100644 index 000000000..ecd909cd8 --- /dev/null +++ b/docs/examples/custom_node.py @@ -0,0 +1,56 @@ +"""Define a custom node with the Node Designer API and run its process() method.""" + +# --8<-- [start:example] +import polars as pl + +from flowfile import node_designer as nd + + +class GreetingSettings(nd.NodeSettings): + main_config: nd.Section = nd.Section( + title="Greeting Configuration", + description="Configure how to greet each row", + name_column=nd.ColumnSelector( + label="Name Column", + data_types=nd.Types.String, + required=True, + ), + greeting=nd.SingleSelect( + label="Greeting", + options=[ + ("formal", "Hello"), + ("casual", "Hey"), + ], + default="casual", + ), + ) + + +class GreetingNode(nd.CustomNodeBase): + node_name: str = "Greeting Generator" + node_category: str = "Text Processing" + title: str = "Add greetings" + intro: str = "Prefix a name column with a greeting." + + settings_schema: GreetingSettings = GreetingSettings() + + def process(self, *inputs: pl.DataFrame) -> pl.DataFrame: + df = inputs[0] + name_col = self.settings_schema.main_config.name_column.value + style = self.settings_schema.main_config.greeting.value + word = "Hello" if style == "formal" else "Hey" + return df.with_columns( + pl.concat_str([pl.lit(f"{word}, "), pl.col(name_col)]).alias("greeting") + ) +# --8<-- [end:example] + +node = GreetingNode() +node.settings_schema.populate_values( + {"main_config": {"name_column": "name", "greeting": "formal"}} +) + +frame = pl.DataFrame({"name": ["Alice", "Bob"]}) +result = node.process(frame) + +assert result.columns == ["name", "greeting"] +assert result["greeting"].to_list() == ["Hello, Alice", "Hello, Bob"] diff --git a/docs/examples/first_pipeline.py b/docs/examples/first_pipeline.py new file mode 100644 index 000000000..727106c14 --- /dev/null +++ b/docs/examples/first_pipeline.py @@ -0,0 +1,21 @@ +"""Your first Flowfile pipeline: a derived column, a filter, and a group-by.""" + +# --8<-- [start:example] +import flowfile as ff + +revenue_by_line = ( + ff.read_csv("data/templates/supermarket_sales.csv") + .with_columns((ff.col("unit_price") * ff.col("quantity")).alias("revenue")) + .filter(ff.col("quantity") > 5) + .group_by("product_line") + .agg(ff.col("revenue").sum().alias("total_revenue")) + .collect() +) +# --8<-- [end:example] + +assert revenue_by_line.columns == ["product_line", "total_revenue"] +assert revenue_by_line.height == 6 + +by_line = {row["product_line"]: row["total_revenue"] for row in revenue_by_line.to_dicts()} +assert round(by_line["Fashion accessories"], 2) == 43859.10 +assert round(by_line["Health and beauty"], 2) == 34165.95 diff --git a/docs/examples/integrations/cloud_storage_s3.py b/docs/examples/integrations/cloud_storage_s3.py new file mode 100644 index 000000000..d06076dbd --- /dev/null +++ b/docs/examples/integrations/cloud_storage_s3.py @@ -0,0 +1,55 @@ +"""Round-trip a Parquet aggregate through S3 with a stored cloud connection.""" + +import os +from uuid import uuid4 + +from pydantic import SecretStr + +import flowfile as ff +from flowfile import FullCloudStorageConnection + +# MinIO test fixture wiring — a real S3 bucket needs neither endpoint_url nor these keys. +endpoint_url = os.environ.get("DOCS_S3_ENDPOINT_URL", "http://localhost:9000") +ff.create_cloud_storage_connection_if_not_exists( + FullCloudStorageConnection( + connection_name="analytics-s3", + storage_type="s3", + auth_method="access_key", + aws_region="us-east-1", + endpoint_url=endpoint_url, + aws_allow_unsafe_html=True, + aws_access_key_id="minioadmin", + aws_secret_access_key=SecretStr("minioadmin"), + ) +) +s3_path = f"s3://flowfile-test/city_income_{uuid4().hex}.parquet" + +# --8<-- [start:example] +income_by_city = ( + ff.read_csv("data/templates/supermarket_sales.csv") + .group_by("city") + .agg(ff.col("gross_income").sum().alias("total_income")) +) + +income_by_city.write_parquet_to_cloud_storage(s3_path, connection_name="analytics-s3") + +reloaded = ff.scan_parquet_from_cloud_storage(s3_path, connection_name="analytics-s3").collect() +# --8<-- [end:example] + +from polars.testing import assert_frame_equal + +assert reloaded.height == 5 +assert_frame_equal( + income_by_city.collect().sort("city"), + reloaded.sort("city"), + check_row_order=False, +) + +by_city = {row["city"]: round(row["total_income"], 2) for row in reloaded.to_dicts()} +assert by_city == { + "Bago": 3051.94, + "Mandalay": 3520.57, + "Naypyitaw": 2521.64, + "Taunggyi": 3525.60, + "Yangon": 3312.56, +} diff --git a/docs/examples/integrations/database_read.py b/docs/examples/integrations/database_read.py new file mode 100644 index 000000000..2c4c93f44 --- /dev/null +++ b/docs/examples/integrations/database_read.py @@ -0,0 +1,41 @@ +"""Read from PostgreSQL through a stored database connection.""" + +import os + +import flowfile as ff + +# Point at the seeded Postgres test fixture (movies sample database). +ff.create_database_connection_if_not_exists( + "analytics-postgres", + database_type="postgresql", + host=os.environ.get("DOCS_PG_HOST", "localhost"), + port=int(os.environ.get("DOCS_PG_PORT", 5433)), + database="testdb", + username="testuser", + password="testpass", +) + +# --8<-- [start:example] +big_budget = ff.read_database( + "analytics-postgres", + query=""" + SELECT title, budget, vote_average + FROM public.movies + WHERE budget > 200000000 + ORDER BY budget DESC + """, +).collect() + +all_movies = ff.read_database( + "analytics-postgres", schema_name="public", table_name="movies" +).collect() +# --8<-- [end:example] + +assert all_movies.height == 4803 +assert "title" in all_movies.columns + +assert big_budget.height > 0 +assert big_budget["budget"].max() == 380000000 +titles = set(big_budget["title"].to_list()) +assert "Avatar" in titles +assert "Pirates of the Caribbean: On Stranger Tides" in titles diff --git a/docs/examples/integrations/kafka_read.py b/docs/examples/integrations/kafka_read.py new file mode 100644 index 000000000..ff58a702e --- /dev/null +++ b/docs/examples/integrations/kafka_read.py @@ -0,0 +1,63 @@ +"""Read JSON messages from a Kafka topic through a stored Kafka connection.""" + +import uuid + +from flowfile_core.database.connection import get_db_context +from flowfile_core.database.models import KafkaConnection +from flowfile_core.kafka.connection_manager import store_kafka_connection +from flowfile_core.schemas.kafka_schemas import KafkaConnectionCreate +from test_utils.kafka.fixtures import BOOTSTRAP_SERVERS, create_topic, produce_json_messages + +# A unique topic per run so the consumer reads exactly the messages produced here. +topic = f"docs_orders_{uuid.uuid4().hex[:8]}" +create_topic(topic, num_partitions=1) +produce_json_messages( + topic, + [ + {"order_id": 1, "product": "keyboard", "amount": 49.99}, + {"order_id": 2, "product": "mouse", "amount": 19.99}, + {"order_id": 3, "product": "monitor", "amount": 199.00}, + ], +) + +# Kafka connections are stored server-side (there is no ff.* helper). Register one +# pointing at the broker; ff.read_kafka resolves it by name at read time. +connection_name = f"docs-kafka-{uuid.uuid4().hex[:8]}" +with get_db_context() as db: + store_kafka_connection( + db, + KafkaConnectionCreate( + connection_name=connection_name, + bootstrap_servers=BOOTSTRAP_SERVERS, + security_protocol="PLAINTEXT", + ), + user_id=1, + ) + +# --8<-- [start:example] +import flowfile as ff + +orders = ff.read_kafka( + connection_name, + topic_name=topic, + start_offset="earliest", + poll_timeout_seconds=10.0, +).collect() +# --8<-- [end:example] + +assert orders.height == 3 +assert "product" in orders.columns +assert set(orders["product"].to_list()) == {"keyboard", "mouse", "monitor"} +# Kafka metadata columns are added alongside the message fields. +assert "_kafka_offset" in orders.columns +assert "_kafka_partition" in orders.columns + +with get_db_context() as db: + conn = ( + db.query(KafkaConnection) + .filter(KafkaConnection.connection_name == connection_name, KafkaConnection.user_id == 1) + .first() + ) + if conn is not None: + db.delete(conn) + db.commit() diff --git a/docs/examples/ml_pipeline.py b/docs/examples/ml_pipeline.py new file mode 100644 index 000000000..6fc40174e --- /dev/null +++ b/docs/examples/ml_pipeline.py @@ -0,0 +1,46 @@ +"""Split, train, apply, and evaluate a churn classifier with the FlowFrame ML API.""" + +# --8<-- [start:example] +import flowfile as ff + +raw = ff.read_csv("data/templates/customer_churn.csv") + +# Hold out 20% of rows for evaluation. +train, test = raw.random_split({"train": 80, "test": 20}, seed=42) + +# Fit a binary logistic-regression model on the training slice. +trained = train.train_model( + target="churned", + features=["tenure_months", "monthly_charges", "support_calls", "has_contract"], + model_type="logistic_regression", + params={"l2_reg": 0.1}, +) + +# Score the held-out rows. wait_for blocks the apply until the model exists. +scored = test.wait_for(trained).apply_model( + upstream=trained, + output_column="predicted_churn", +) + +# Compare predictions against the true labels — a long (metric, value) table. +metrics = scored.evaluate_model( + "churned", + predicted_column="predicted_churn", + task_type="classification", + upstream=trained, +) + +result = metrics.collect() +# --8<-- [end:example] + +assert result.columns == ["metric", "value"] +assert result.schema["metric"] == ff.String +assert result.schema["value"] == ff.Float64 + +reported = set(result["metric"].to_list()) +for metric in ("accuracy", "precision", "recall", "f1", "n_correct", "n_total"): + assert metric in reported, f"Expected metric '{metric}' in {reported}" + +scored_df = scored.collect() +assert "predicted_churn" in scored_df.columns +assert scored_df.height > 0 diff --git a/docs/examples/python_api_aggregations.py b/docs/examples/python_api_aggregations.py new file mode 100644 index 000000000..65d52a197 --- /dev/null +++ b/docs/examples/python_api_aggregations.py @@ -0,0 +1,49 @@ +"""Grouped aggregations, the all_ selector, and window functions (rank and cum_sum over groups).""" + +# --8<-- [start:example] +import flowfile as ff + +# One group_by node fuses the grouping and every aggregation. +summary = ( + ff.read_csv("data/templates/orders.csv") + .group_by("customer_id") + .agg( + ff.col("order_id").count().alias("n_orders"), + ff.col("quantity").sum().alias("total_qty"), + ff.col("quantity").mean().alias("avg_qty"), + ff.col("quantity").min().alias("min_qty"), + ff.col("quantity").max().alias("max_qty"), + ff.col("quantity").std().alias("std_qty"), + ff.col("quantity").first().alias("first_qty"), + ff.col("quantity").implode().alias("all_qty"), + ) + .collect() +) + +# Sum every numeric column at once with the all_ selector inside select. +totals = ( + ff.read_csv("data/templates/orders.csv") + .select(ff.col("quantity")) + .select(ff.all_().sum()) + .collect() +) + +# Window functions: rank and running total within each customer. +ranked = ( + ff.read_csv("data/templates/orders.csv") + .with_columns( + ff.col("quantity").rank(method="dense", descending=True).over("customer_id").alias("qty_rank"), + ff.col("quantity").cum_sum().over("customer_id").alias("cum_qty"), + ) + .collect() +) +# --8<-- [end:example] + +assert summary.height == 401 +assert summary["n_orders"].sum() == 1000 +assert summary["total_qty"].sum() == 5461 + +assert totals.to_dicts()[0]["quantity"] == 5461 + +assert ranked.height == 1000 +assert ranked["qty_rank"].min() == 1 diff --git a/docs/examples/python_api_joins.py b/docs/examples/python_api_joins.py new file mode 100644 index 000000000..05f5deb67 --- /dev/null +++ b/docs/examples/python_api_joins.py @@ -0,0 +1,51 @@ +"""Join types (inner/left/full/semi/anti/cross), concatenation with ff.concat, and join validation.""" + +# --8<-- [start:example] +import flowfile as ff + +orders = ff.read_csv("data/templates/orders.csv") +customers = ff.read_csv("data/templates/customers.csv") +products = ff.read_csv("data/templates/products.csv") + +# Enrich orders with customer and product attributes. +inner = orders.join(customers, on="customer_id", how="inner") +with_product = orders.join(products, on="product_id", how="left") + +# full keeps unmatched rows from both sides; semi/anti filter the left side. +full = orders.join(customers, on="customer_id", how="full") +repeat_customers = customers.join(orders, on="customer_id", how="semi") +dormant_customers = customers.join(orders, on="customer_id", how="anti") + +# Stack frames vertically with ff.concat (there is no vstack), then dedupe. +deduped = ff.concat([customers, customers]).unique() + +# Validate a join key before joining: compare distinct keys to total rows. +key_stats = customers.select( + ff.col("customer_id").n_unique().alias("distinct_customers"), + ff.col("customer_id").count().alias("total_rows"), +) + +inner_df = inner.collect() +full_df = full.collect() +repeat_df = repeat_customers.collect() +dormant_df = dormant_customers.collect() +key_stats_df = key_stats.collect() +# --8<-- [end:example] + +with_product_df = with_product.collect() +deduped_df = deduped.collect() + +assert inner_df.height == 1000 +assert "name" in inner_df.columns +assert with_product_df.height == 1000 + +assert full_df.height == 1099 +assert repeat_df.height == 401 +assert dormant_df.height == 99 +assert repeat_df.height + dormant_df.height == 500 + +assert deduped_df.height == 500 + +stats = key_stats_df.to_dicts()[0] +assert stats["distinct_customers"] == 500 +assert stats["total_rows"] == 500 diff --git a/docs/examples/python_api_operations.py b/docs/examples/python_api_operations.py new file mode 100644 index 000000000..2737dfd0b --- /dev/null +++ b/docs/examples/python_api_operations.py @@ -0,0 +1,52 @@ +"""Row and column operations: unique, cast, filter, sort, exclude, and string/date/window expressions.""" + +# --8<-- [start:example] +import flowfile as ff + +# Deduplicate, derive columns, filter, sort, then drop a column with a selector. +cleaned = ( + ff.read_csv("data/templates/orders.csv") + .unique(subset=["order_id"]) + .with_columns( + (ff.col("quantity") * ff.lit(2)).alias("double_qty"), + ff.col("quantity").cast(ff.Float64).alias("quantity_f"), + ) + .filter(ff.col("quantity") >= 3) + .sort("quantity", descending=True) + .select(ff.col("*").exclude("product_id")) + .collect() +) + +# String and date namespaces on expressions. +enriched = ( + ff.read_csv("data/templates/customers.csv") + .with_columns( + ff.col("name").str.to_uppercase().alias("name_upper"), + ff.col("email").str.slice(0, 5).alias("email_prefix"), + ff.col("name").str.contains("a").alias("name_has_a"), + ff.col("signup_date").dt.year().alias("signup_year"), + ff.col("signup_date").dt.weekday().alias("signup_weekday"), + ) + .collect() +) + +# A running total per customer with a window (cum_sum over a group). +running = ( + ff.read_csv("data/templates/orders.csv") + .sort("order_date") + .with_columns(ff.col("quantity").cum_sum().over("customer_id").alias("running_qty")) + .collect() +) +# --8<-- [end:example] + +assert "product_id" not in cleaned.columns +assert "double_qty" in cleaned.columns and "quantity_f" in cleaned.columns +assert cleaned.height == 800 + +assert enriched.height == 500 +assert enriched["signup_weekday"].min() >= 1 +assert enriched["signup_weekday"].max() <= 7 +assert enriched["email_prefix"].str.len_chars().max() <= 5 + +assert running.height == 1000 +assert running["running_qty"].max() > 0 diff --git a/docs/examples/sales_pipeline.py b/docs/examples/sales_pipeline.py new file mode 100644 index 000000000..f3067db4d --- /dev/null +++ b/docs/examples/sales_pipeline.py @@ -0,0 +1,31 @@ +"""Dedupe, filter, and aggregate supermarket sales by city with the FlowFrame API.""" + +# --8<-- [start:example] +import flowfile as ff + +result = ( + ff.read_csv("data/templates/supermarket_sales.csv") + .unique() + .filter(ff.col("quantity") > 7) + .group_by("city") + .agg( + ff.col("gross_income").sum().alias("total_income"), + ff.col("gross_income").median().alias("median_income"), + ) + .collect() +) +# --8<-- [end:example] + +by_city = {row["city"]: row for row in result.to_dicts()} + +assert result.height == 5 +assert set(by_city) == {"Bago", "Mandalay", "Naypyitaw", "Taunggyi", "Yangon"} + +assert round(by_city["Bago"]["total_income"], 2) == 1429.66 +assert round(by_city["Mandalay"]["total_income"], 2) == 1846.88 +assert round(by_city["Naypyitaw"]["total_income"], 2) == 1186.66 +assert round(by_city["Taunggyi"]["total_income"], 2) == 1635.53 +assert round(by_city["Yangon"]["total_income"], 2) == 1704.81 + +assert round(by_city["Mandalay"]["median_income"], 3) == 27.220 +assert round(by_city["Yangon"]["median_income"], 3) == 26.085 diff --git a/docs/for-developers/ai-architecture.md b/docs/for-developers/ai-architecture.md index 692b33df2..fc5d3f03f 100644 --- a/docs/for-developers/ai-architecture.md +++ b/docs/for-developers/ai-architecture.md @@ -159,6 +159,9 @@ So every AI endpoint inherits the gate without per-route boilerplate. The Mutabl ## Routes catalog +!!! info "Verify against the package" + The routes and Pydantic model names below are a snapshot of the `ai/*_routes.py` files. Request/response model names in particular drift as the surface evolves — treat this table as a map, and read the matching `*_routes.py` for the authoritative signature before depending on one. + | Route | Method | Request | Response | Source | |-------|--------|---------|----------|--------| | `/ai/health` | GET | — | `{"status": "skeleton"}` | `routes.py` | @@ -230,10 +233,13 @@ Per-vendor subclasses override class-level fields: | `AnthropicProvider` | `anthropic` | `anthropic/` | `claude-sonnet-4-6` | Haiku 4.5 for Cmd+K / ghost / autocomplete / agent_staged; Opus 4.7 for `agent_complex` | | `OpenAIProvider` | `openai` | `""` | `gpt-4.1-mini` | `gpt-4.1` for `explain` / `agent_complex` / `docgen` / `lineage` | | `GoogleProvider` | `google` | `gemini/` | `gemini-2.5-flash` | `gemini-2.5-pro` for `agent_complex` | -| `GroqProvider` | `groq` | `groq/` | `qwen/qwen3-coder-30b-a3b-instruct` | One model across surfaces | +| `GroqProvider` | `groq` | `groq/` | `qwen/qwen3-32b` | `qwen/qwen3-coder-30b-a3b-instruct` for most surfaces (ghost / explain / agent_* / docgen / lineage / …); `qwen/qwen3-32b` for `cmd_k` | | `OpenRouterProvider` | `openrouter` | `openrouter/` | `qwen/qwen3-coder-30b-a3b-instruct` | `meta-llama/llama-3.3-70b-instruct` for `agent_staged` (free tier) | | `OllamaProvider` | `ollama` | `ollama_chat/` | `llama3.1:8b` | `llama3.1:70b` for `agent_complex`. `default_api_base="http://localhost:11434"` | +!!! note "`default_model` vs surface models" + `default_model` is the terminal fallback the class uses when no other rule fires (see the resolution order below). It is **not** the model most surfaces run on — `surface_models[surface]` usually shadows it. Groq is a good example: the class default is `qwen/qwen3-32b`, but every surface except `cmd_k` maps to `qwen/qwen3-coder-30b-a3b-instruct`. The model names in this table rot as vendors rename releases; check each provider's `surface_models` dict in `providers/{name}.py` for the current values. + `provider_factory(name, *, model, surface, api_key, api_base)` in `providers/registry.py` is the construction entry point. The `PROVIDERS` dict registers all six classes; `list_supported_providers()` returns the names in registration order. --- @@ -537,7 +543,7 @@ Components in `features/ai/`: - `AiAssistant.vue` — the drawer (chat + agent). - `AiAgentRun.vue` / `AiAgentEvent.vue` — agent panel and per-event renderers. - `AiDiffPanel.vue` / `AiDiffPreview.vue` — diff review UI. -- `AiInlineActions.vue` — the ✨ menu on a node. +- `AiInlineActions.vue` — the inline-actions (sparkle) menu on a node. - `AiCommandPalette.vue` — Cmd+K. - `AiGhostNode.vue` — edge-stub suggestions. - `AiMentionAutocomplete.vue` (+ `mentionVocabulary.ts`) — `@flow` / `@node:id` completion. diff --git a/docs/for-developers/architecture.md b/docs/for-developers/architecture.md index 51556a41c..1450eee30 100644 --- a/docs/for-developers/architecture.md +++ b/docs/for-developers/architecture.md @@ -1,6 +1,6 @@ # Technical Architecture -Flowfile's architecture integrates visual design with high-performance data processing through three interconnected services and utilizes Polars' lazy evaluation. This design provides real-time feedback, processes large datasets efficiently, and maintains UI responsiveness during intensive computations. On this page, the three-service architecture, key technical features such as real-time schema prediction and efficient data exchange, and the role of Polars' lazy evaluation are explained. +Flowfile's architecture pairs visual design with data processing across three services, built on Polars' lazy evaluation. This page explains the three-service architecture, key technical features such as real-time schema prediction and efficient data exchange, and the role of Polars' lazy evaluation. ![Flowfile Architecture](../assets/images/architecture/flowfile_architecture.png) @@ -8,7 +8,7 @@ Flowfile's architecture integrates visual design with high-performance data proc ### Three-Service Architecture -**Designer (Electron + Vue)** The visual interface where data pipelines are build through drag-and-drop operations. It communicates with the Core service to provide real-time feedback and displays data previews. +**Designer (Tauri + Vue)** The visual interface where data pipelines are built through drag-and-drop operations. It is a Tauri 2 desktop shell (Rust) wrapping a Vue 3 renderer, and the same renderer serves the web build. It communicates with the Core service for real-time feedback and data previews. **Core Service (FastAPI)** The orchestration engine that manages workflows, predicts schemas, and coordinates execution. It maintains the Directed Acyclic Graph (DAG) structure and handles all UI interactions and overall flow logic. @@ -18,7 +18,7 @@ Flowfile's architecture integrates visual design with high-performance data proc ### Real-time Schema Prediction -When you add or configure a node, Flowfile immediately shows how your data structure will change — **without executing any transformations**. This continuous feedback significantly reduces user errors and increases predictability. This happens through: +When you add or configure a node, Flowfile immediately shows how your data structure will change — **without executing any transformations**. This happens through: - **Schema Callbacks**: Custom functions, defined per node type, that calculate output schemas based on node settings and input schemas. - **Lazy Evaluation**: Leveraging Polars' ability to determine the schema of a planned transformation (`LazyFrame`) without processing the full dataset. @@ -48,51 +48,41 @@ As you add and connect nodes, Flowfile builds a Directed Acyclic Graph (DAG) whe * **Nodes** represent data operations (read file, filter, join, write to database, etc.). * **Edges** represent the flow of data between operations. -The DAG is managed by the `EtlGraph` class in the Core service, which orchestrates the entire workflow: +The DAG is managed by the `FlowGraph` class (`flowfile_core/flowfile_core/flowfile/flow_graph.py`) in the Core service, which orchestrates the entire workflow. The class shape below is illustrative — see the [Python API Reference](python-api-reference.md#flowgraph) for the real signatures.
-View EtlGraph Python Implementation +View FlowGraph shape (illustrative) ```python -class EtlGraph: +class FlowGraph: """ Manages the ETL workflow as a DAG. Stores nodes, dependencies, and settings, and handles the execution order. """ uuid: str - _node_db: Dict[Union[str, int], NodeStep] # Internal storage for all node steps - _flow_starts: List[NodeStep] # Nodes that initiate data flow (e.g., readers) + _node_db: Dict[Union[str, int], FlowNode] # Internal storage for all nodes + _flow_starts: List[FlowNode] # Nodes that initiate data flow (e.g., readers) _node_ids: List[Union[str, int]] # Tracking node identifiers flow_settings: schemas.FlowSettings # Global configuration for the flow def add_node_step(self, node_id: Union[int, str], function: Callable, node_type: str, **kwargs) -> None: - """Adds a new processing node (NodeStep) to the graph.""" - node_step = NodeStep(node_id=node_id, function=function, node_type=node_type, **kwargs) - self._node_db[node_id] = node_step - # Additional logic to manage dependencies and flow starts... + """Adds a new FlowNode to the graph.""" + ... def run_graph(self) -> RunInformation: - """Executes the entire flow in the correct topological order.""" - execution_order = self.topological_sort() # Determine correct sequence - run_info = RunInformation() - for node in execution_order: - # Execute node based on mode (Development/Performance) - node_results = node.execute_node() # Simplified representation - run_info.add_result(node.node_id, node_results) - return run_info - - def topological_sort(self) -> List[NodeStep]: - """Determines the correct order to execute nodes based on dependencies.""" - # Standard DAG topological sort algorithm... - pass + """Executes the flow in the correct topological order.""" + ... ```
-Each `NodeStep` in the graph encapsulates information about its dependencies, transformation logic, and output schema. This structure allows Flowfile to determine execution order, track data lineage, optimize performance, and provide schema predictions throughout the pipeline. +Each `FlowNode` in the graph encapsulates its dependencies, transformation logic, and output schema. This lets Flowfile determine execution order, track data lineage, optimize performance, and predict schemas throughout the pipeline. ### Execution Modes +!!! info "Canonical explanation" + This is the reference description of Development vs Performance mode. The [Core Developer Guide](flowfile-core.md#execution-strategy-how-nodes-decide-where-to-run) and [Design Philosophy](design-philosophy.md) pages link here rather than repeat it. + By clicking on settings → execution modes you can set how the flow will be executed the next time you run the flow. ![execution settings](../assets/images/guides/technical_architecture/execution_settings.png) @@ -115,25 +105,28 @@ In Development mode, each node's transformation is triggered sequentially within In Performance mode, Flowfile fully embraces Polars' lazy evaluation. The Core service constructs the *entire* Polars execution plan based on the DAG. This plan (`LazyFrame`) is passed to the Worker service. The Worker only *materializes* (executes `.collect()` or `.sink_*()`) the plan when an output node (like writing to a file) requires the final result, or if a node is explicitly configured to cache its results (`node.cache_results`). This minimizes computation and memory usage by avoiding unnecessary intermediate materializations.
-View Performance Mode Python Example +View Performance Mode Python Example (simplified) ```python # Execution logic in Performance Mode (simplified) -def execute_performance_mode(self, node: NodeStep, is_output_node: bool): +def execute_performance_mode(self, node: FlowNode, is_output_node: bool): """Handles execution in performance mode, leveraging lazy evaluation.""" if is_output_node or node.cache_results: - # If result is needed (output or caching), trigger execution in Worker - external_df_fetcher = ExternalDfFetcher( - lf=node.get_resulting_data().data_frame, # Pass the LazyFrame plan - file_ref=node.hash, # Unique reference for caching - wait_on_completion=False # Usually async + # If the result is needed (output or caching), offload to the Worker. + # Offload happens inside ExternalDfFetcher.__init__ — constructing it + # serializes the LazyFrame and POSTs it to the worker. flow_id and + # node_id are required. + fetcher = ExternalDfFetcher( + flow_id=node.flow_id, + node_id=node.node_id, + lf=node.get_resulting_data().data_frame, # the LazyFrame plan + file_ref=node.hash, # unique reference for caching + wait_on_completion=False, # usually async ) - # Worker executes .collect() or .sink_*() and caches if needed - result = external_df_fetcher.get_result() # May return LazyFrame or trigger compute - return result # Or potentially just confirmation if sinking + result = fetcher.get_result() # Worker runs .collect()/.sink_*() and caches + return result else: - # If not output/cached, just pass the LazyFrame plan along - # No computation happens here for intermediate nodes + # Intermediate nodes just pass the LazyFrame plan along — no compute here. return node.get_resulting_data().data_frame ``` @@ -152,36 +145,36 @@ Flowfile uses Apache Arrow IPC format for efficient inter-process communication This ensures responsiveness, memory isolation, and efficiency. -Here’s how the Core might offload computation to the Worker, and how the Worker manages the separate process execution: +Here is how the Core offloads computation to the Worker, and how the Worker manages the separate process execution:
-View Core-Side Python Example +View Core-Side Python Example (simplified) ```python # Core side - Initiating remote execution in the Worker (simplified) def execute_remote(self, performance_mode: bool = False) -> None: """Offloads the execution of a node's LazyFrame to the Worker service.""" - # Create a fetcher instance to manage communication with the Worker - external_df_fetcher = ExternalDfFetcher( - lf=self.get_resulting_data().data_frame, # The Polars LazyFrame plan - file_ref=self.hash, # Unique identifier for the result/cache - wait_on_completion=False, # Operate asynchronously + # Constructing ExternalDfFetcher IS the offload: __init__ serializes the + # LazyFrame and sends it to the worker (no separate "start" call). The + # constructor requires flow_id and node_id. + fetcher = ExternalDfFetcher( + flow_id=self.flow_id, + node_id=self.node_id, + lf=self.get_resulting_data().data_frame, # the Polars LazyFrame plan + file_ref=self.hash, # unique identifier for result/cache + wait_on_completion=False, # operate asynchronously ) - # Store the fetcher to potentially retrieve results later - self._fetch_cached_df = external_df_fetcher - - # Request the Worker to start processing (this returns quickly) - # The actual computation happens asynchronously in the Worker - external_df_fetcher.start_processing_in_worker() # Hypothetical method name + # Store the fetcher to retrieve results later + self._fetch_cached_df = fetcher # For UI updates, request a sample separately - self.store_example_data_generator(external_df_fetcher) # Fetches sample async + self.store_example_data_generator(fetcher) # fetches sample async ```
-View Worker-Side Python Example +View Worker-Side Python Example (simplified) ```python # Worker side - Managing computation in a separate process (simplified) @@ -192,8 +185,10 @@ def start_process( # ... other args like operation type ) -> None: """Launches a separate OS process to handle the heavy computation.""" - # Use multiprocessing context for safety - mp_context = multiprocessing.get_context('spawn') # or 'fork' depending on OS/needs + # The worker forces spawn at module load (set_start_method('spawn', + # force=True) in flowfile_worker/__init__.py) so every child is a fresh, + # killable process regardless of platform — never fork. + mp_context = get_context('spawn') # Shared memory/queue for progress tracking and results/errors progress = mp_context.Value('i', 0) # Shared integer for progress % @@ -219,7 +214,7 @@ def start_process( ```
-## The Power of Lazy Evaluation +## Lazy Evaluation By building on Polars' lazy evaluation, Flowfile achieves: @@ -228,15 +223,6 @@ By building on Polars' lazy evaluation, Flowfile achieves: - **Parallel Execution**: Polars automatically parallelizes operations across all available CPU cores during execution. - **Predicate Pushdown**: Filters and selections are applied as early as possible in the plan, often directly at the data source level (like during file reading), minimizing the amount of data that needs to be processed downstream. -## Summary - -This design enables Flowfile to: - -- Provide instant feedback without processing data. -- Handle datasets of any size efficiently. -- Keep the UI responsive during heavy computations. -- Scale from simple transformations to complex ETL pipelines. - --- -*For a deep dive into the implementation details, see the [full technical article](https://dev.to/edwardvaneechoud/building-flowfile-architecting-a-visual-etl-tool-with-polars-576c).* \ No newline at end of file +*Background reading: the original [design article](https://dev.to/edwardvaneechoud/building-flowfile-architecting-a-visual-etl-tool-with-polars-576c) covers the motivation and early design. It predates some renames (the graph class is now `FlowGraph`, nodes are `FlowNode`), so treat class names there as historical.* \ No newline at end of file diff --git a/docs/for-developers/creating-custom-nodes.md b/docs/for-developers/creating-custom-nodes.md index a9e8a218d..de58afc17 100644 --- a/docs/for-developers/creating-custom-nodes.md +++ b/docs/for-developers/creating-custom-nodes.md @@ -1,20 +1,20 @@ # Creating Custom Nodes -Build your own data transformation nodes with custom UI components and processing logic. +This page is the reference for building a custom node in Python with the Node Designer API — its class structure, the UI components you can put in the settings panel, type filtering, and the `process()` contract. After reading it you can write a node that shows up in the editor with a generated settings form. For a guided end-to-end build, see the [Custom Node Tutorial](custom-node-tutorial.md). -!!! tip "Visual Alternative" - You can also create custom nodes visually using the [Node Designer](../users/visual-editor/node-designer.md) without writing Python files directly. +!!! tip "Visual alternative" + You can also create custom nodes in the browser with the [Node Designer](../users/visual-editor/node-designer.md), without writing Python files directly. -!!! warning "Beta Feature" - Custom nodes are currently in beta. Some features like changing the icon are still in development. +!!! warning "Beta feature" + Custom nodes are in beta. Some features (such as changing the node icon) are still in development. -## What Are Custom Nodes? +## What are custom nodes? -Custom nodes let you extend Flowfile with your own data transformations that appear as native nodes in the visual editor. Each custom node includes: +Custom nodes extend Flowfile with your own data transformations that appear alongside the built-in nodes in the visual editor. Each one has: -- **Custom UI** - Automatically generated settings panels with dropdowns, inputs, and toggles -- **Data Processing** - Polars-based transformation logic -- **Visual Integration** - Appears seamlessly in the node palette +- **A settings panel** — generated automatically from your schema (dropdowns, inputs, toggles). +- **Processing logic** — Polars code that transforms the data. +- **Palette placement** — under **User Defined Operations** in the node palette. ## Quick Start @@ -29,96 +29,25 @@ Create a new Python file in your custom nodes directory: !!! info "Custom Node Location" The `~/.flowfile/user_defined_nodes/` directory is automatically created when you first run Flowfile. Place all your custom nodes here. -Here's a simple example that adds a greeting column: +Here's a simple example that adds a greeting column. Note the `process` signature — it receives one or more eager `pl.DataFrame` inputs (variadic `*inputs`) and returns a `pl.DataFrame`: ```python -import polars as pl -from flowfile_core.flowfile.node_designer import ( - CustomNodeBase, - Section, - NodeSettings, - TextInput, - SingleSelect, - ColumnSelector, - Types -) - -class GreetingSettings(NodeSettings): - main_config: Section = Section( - title="Greeting Configuration", - description="Configure how to greet your data", - name_column=ColumnSelector( - label="Name Column", - data_types=Types.String, - required=True - ), - greeting_style=SingleSelect( - label="Greeting Style", - options=[ - ("formal", "Formal (Hello, Mr/Ms)"), - ("casual", "Casual (Hey there!)"), - ("enthusiastic", "Enthusiastic (OMG HI!!!)"), - ], - default="casual" - ), - custom_message=TextInput( - label="Custom Message", - default="Nice to meet you!", - placeholder="Enter your custom greeting..." - ) - ) - -class GreetingNode(CustomNodeBase): - node_name: str = "Greeting Generator" - node_category: str = "Text Processing" - title: str = "Add Personal Greetings" - intro: str = "Transform names into personalized greetings" - - settings_schema: GreetingSettings = GreetingSettings() - - def process(self, input_df: pl.LazyFrame) -> pl.LazyFrame: - # Get values from the UI - name_col = self.settings_schema.main_config.name_column.value - style = self.settings_schema.main_config.greeting_style.value - custom = self.settings_schema.main_config.custom_message.value - - # Define greeting logic - if style == "formal": - greeting_expr = pl.concat_str([ - pl.lit("Hello, "), - pl.col(name_col), - pl.lit(f". {custom}") - ]) - elif style == "casual": - greeting_expr = pl.concat_str([ - pl.lit("Hey "), - pl.col(name_col), - pl.lit(f"! {custom}") - ]) - else: # enthusiastic - greeting_expr = pl.concat_str([ - pl.lit("OMG HI "), - pl.col(name_col).str.to_uppercase(), - pl.lit(f"!!! {custom} 🎉") - ]) - - return input_df.with_columns([ - greeting_expr.alias("greeting") - ]) +--8<-- "docs/examples/custom_node.py:example" ``` +This example is tested against the current API. The node-designer symbols are imported through `flowfile.node_designer` (aliased `nd` above); they are the same classes as `from flowfile_core.flowfile.node_designer import ...`. ### 2. Use Your Node -1. **Restart Flowfile** to load your new node -2. **Open the visual editor** -3. **Find your node** in the "Text Processing" category -4. **Drag it** onto the canvas -5. **Configure the settings** in the right panel -6. **Run your flow** and see the results! +1. **Restart Flowfile** to load your new node. +2. **Open the visual editor.** +3. **Find your node** under **User Defined Operations** in the node palette — every custom node lands there (see [The palette section](#the-palette-section)). +4. **Drag it** onto the canvas. +5. **Configure the settings** in the right panel. +6. **Run your flow.**
- Visual overview of the result! +Visual overview of the result ![Flow visualized](../assets/images/developers/basic_overview.png) @@ -133,41 +62,35 @@ Every custom node has three main parts: ```python class MyCustomNode(CustomNodeBase): - # 1. Metadata - How the node appears in Flowfile - node_name: str = "My Amazing Node" + # 1. Metadata - how the node appears in Flowfile + node_name: str = "My Node" node_category: str = "Data Enhancement" - title: str = "Add Personal Greetings" - intro: str = "Transform names into personalized greetings" + title: str = "Add greetings" + intro: str = "Prefix a name column with a greeting." - # 2. Settings Schema - The UI configuration + # 2. Settings schema - the UI configuration settings_schema: MySettings = MySettings() - - # 3. Processing Logic - What the node actually does - def process(self, input_df: pl.LazyFrame) -> pl.LazyFrame: - # Your transformation logic here - return modified_df + + # 3. Processing logic - what the node does + def process(self, *inputs: pl.DataFrame) -> pl.DataFrame: + df = inputs[0] + # ... transformation logic ... + return df ``` #### 3. Process -!!! Warning "Only one input supported for now" - Currently, custom nodes only support a single input DataFrame. Support for multiple inputs is planned for future releases. - - -The process method is the engine of your node. This is where you write your Polars code to transform the data. - -- Input: The method receives the incoming data as a Polars LazyFrame. If your node has multiple inputs, they will be passed as separate arguments (e.g., process(self, df1)). - -- Accessing Settings: Inside this method, you can get the current values from your UI components using self.settings_schema...value. - -- Output: The method must return a single Polars LazyFrame, which becomes the output of your node. - - - +The `process` method is the engine of your node — where you write Polars code to transform the data. +- **Input:** the method receives its inputs as eager `pl.DataFrame` objects, passed as a variadic `*inputs`. A single-input node reads `inputs[0]`; a node with multiple inputs indexes further. +- **Accessing settings:** read current UI values with `self.settings_schema...value`. +- **Output:** return a single `pl.DataFrame` (or, for a multi-output node, a `dict[str, pl.DataFrame]` keyed by output name). +### The palette section +Every custom node appears in the node palette under one section: **User Defined Operations**. That placement is driven by `node_group`, which defaults to `"custom"` on `CustomNodeBase` — you don't need to set it. +`node_category` is separate, descriptive metadata (shown in the node header and browser modal). It does **not** create or select a palette section, so a value like `"Text Processing"` or `"Data Validation"` organizes nothing in the palette — all custom nodes still sit under User Defined Operations. ### Settings Architecture @@ -285,20 +208,25 @@ text_columns = ColumnSelector( ``` ### Secret Selector -Access stored secrets (API keys, credentials, tokens) securely: +Access stored secrets (API keys, credentials, tokens). `SecretSelector`, like every other component, is added inside a `Section` as a keyword argument (the keyword is the field name). It has no `name=` field — only `label`: ```python -from flowfile_core.flowfile.node_designer import SecretSelector +from flowfile_core.flowfile.node_designer import ( + CustomNodeBase, NodeSettings, Section, SecretSelector +) -class MyNode(CustomNodeBase): - settings_schema = NodeSettings( - components=[ - SecretSelector(name="api_key", label="API Key"), - ] +class ApiSettings(NodeSettings): + connection: Section = Section( + title="Connection", + api_key=SecretSelector(label="API Key"), ) + +class MyNode(CustomNodeBase): + node_name: str = "API Reader" + settings_schema: ApiSettings = ApiSettings() ``` -The SecretSelector displays a dropdown of available secrets configured by the user. Secrets are stored securely and retrieved at runtime. This is useful for nodes that connect to external APIs or services. +The dropdown lists the secrets configured by the current user. Read the decrypted value inside `process()` with `self.settings_schema.connection.api_key.secret_value` — it is only accessible during execution. This is useful for nodes that connect to external APIs or services. ### Dynamic Column Options Use `IncomingColumns` for dropdowns that populate with input columns: @@ -320,17 +248,18 @@ from flowfile_core.flowfile.node_designer import Types # Type groups Types.Numeric # All numeric types Types.String # String and categorical -Types.Date # Date, datetime, time +Types.AnyDate # Date, datetime, time, duration Types.Boolean # Boolean columns Types.All # All column types # Specific types Types.Int64 # 64-bit integers Types.Float # Float64 -Types.Decimal # Decimal types +Types.Decimal # Decimal type +Types.Date # The Date type specifically (not the date group) # Mix and match -data_types=[Types.Numeric, Types.Date] # Numbers and dates only +data_types=[Types.Numeric, Types.AnyDate] # Numbers and dates only ``` ## Real-World Examples @@ -341,8 +270,8 @@ data_types=[Types.Numeric, Types.Date] # Numbers and dates only class DataQualityNode(CustomNodeBase): node_name: str = "Data Quality Checker" node_category: str = "Data Validation" - - settings_schema: DataQualitySettings = DataQualitySettings( + + settings_schema: NodeSettings = NodeSettings( validation_rules=Section( title="Validation Rules", columns_to_check=ColumnSelector( @@ -363,29 +292,21 @@ class DataQualityNode(CustomNodeBase): ) ) - def process(self, input_df: pl.LazyFrame) -> pl.LazyFrame: + def process(self, *inputs: pl.DataFrame) -> pl.DataFrame: + df = inputs[0] columns = self.settings_schema.validation_rules.columns_to_check.value threshold = self.settings_schema.validation_rules.null_threshold.value - - # Calculate quality metrics - quality_checks = [] + + row_count = df.height + result = df for col in columns: - null_pct = (input_df[col].is_null().sum() / len(input_df)) * 100 - quality_checks.append({ - "column": col, - "null_percentage": null_pct, - "quality_flag": "PASS" if null_pct <= threshold else "FAIL" - }) - - # Add quality flags to original data - result_df = input_df - for check in quality_checks: - if check["quality_flag"] == "FAIL": - result_df = result_df.with_columns([ - pl.col(check["column"]).is_null().alias(f"{check['column']}_has_issues") - ]) - - return result_df + null_pct = (df[col].null_count() / row_count) * 100 if row_count else 0.0 + if null_pct > threshold: + # Flag the failing column with a per-row null indicator + result = result.with_columns( + pl.col(col).is_null().alias(f"{col}_has_issues") + ) + return result ``` ### Text Processing Node @@ -394,8 +315,8 @@ class DataQualityNode(CustomNodeBase): class TextCleanerNode(CustomNodeBase): node_name: str = "Text Cleaner" node_category: str = "Text Processing" - - settings_schema: TextCleanerSettings = TextCleanerSettings( + + settings_schema: NodeSettings = NodeSettings( cleaning_options=Section( title="Cleaning Options", text_column=ColumnSelector( @@ -421,14 +342,15 @@ class TextCleanerNode(CustomNodeBase): ) ) - def process(self, input_df: pl.LazyFrame) -> pl.LazyFrame: + def process(self, *inputs: pl.DataFrame) -> pl.DataFrame: + df = inputs[0] text_col = self.settings_schema.cleaning_options.text_column.value operations = self.settings_schema.cleaning_options.operations.value output_col = self.settings_schema.cleaning_options.output_column.value - - # Start with original text + + # Start with the original text expr = pl.col(text_col) - + # Apply selected operations if "lowercase" in operations: expr = expr.str.to_lowercase() @@ -440,48 +362,33 @@ class TextCleanerNode(CustomNodeBase): expr = expr.str.replace_all(r"\d+", "") if "trim" in operations: expr = expr.str.strip_chars() - - return input_df.with_columns([expr.alias(output_col)]) -``` -## Best Practices + return df.with_columns(expr.alias(output_col)) +``` -### 1. Performance -Try to use Polars expressions and lazy evaluation to keep your nodes efficient. -A collect will be executed in the core process and can cause issues when using remote compute. +## Performance note +Express transformations as Polars expressions on the incoming `pl.DataFrame` rather than iterating rows in Python. `process()` runs against an eager DataFrame, so keep per-row Python work (like `map_elements`) to cases that genuinely need it. ## Troubleshooting -### Node Doesn't Appear -1. Check the file is in `~/.flowfile/user_defined_nodes/` -2. Restart Flowfile completely -3. Check for Python syntax errors in terminal -4. Ensure your class inherits from `CustomNodeBase` - -### Settings Don't Work -1. Verify `settings_schema` is properly assigned -2. Check component imports -3. Ensure section structure is correct -4. Use `.value` to access component values in `process()` - -### Processing Errors -1. Check input DataFrame exists and has expected columns -2. Use `print()` or logging for debugging -3. Handle null values and edge cases -4. Ensure return type is `pl.LazyFrame` - - -## Coming Soon +### Node doesn't appear +1. Check the file is in `~/.flowfile/user_defined_nodes/`. +2. Restart Flowfile completely. +3. Check for Python syntax errors in the terminal. +4. Ensure your class inherits from `CustomNodeBase`. +5. Look under **User Defined Operations** in the palette — not a section named after your `node_category`. -The following features are planned for future releases: +### Settings don't work +1. Verify `settings_schema` is assigned. +2. Check component imports. +3. Ensure the section structure is correct (components are keyword arguments inside a `Section`). +4. Use `.value` to read component values in `process()`. -- **Node Templates** - Quick-start templates for common patterns -- **Custom Icons** - Upload custom icons for your nodes -- **Node Categories** - Create your own node categories -- **Testing Framework** - Built-in testing for custom nodes -- **Node Publishing** - Share nodes with the community +### Processing errors +1. Check the input DataFrame has the expected columns. +2. Ensure `process()` returns a `pl.DataFrame`. --- -Ready to build? Start with the [Custom Node Tutorial](custom-node-tutorial.md) for a step-by-step walkthrough! \ No newline at end of file +For a step-by-step walkthrough, see the [Custom Node Tutorial](custom-node-tutorial.md). \ No newline at end of file diff --git a/docs/for-developers/custom-node-tutorial.md b/docs/for-developers/custom-node-tutorial.md index a348fad9e..bf8ed738f 100644 --- a/docs/for-developers/custom-node-tutorial.md +++ b/docs/for-developers/custom-node-tutorial.md @@ -1,13 +1,14 @@ # Custom Node Tutorial: Build an Emoji Generator -Learn to create custom nodes by building a fun emoji generator that transforms boring data into expressive datasets with visual flair. +This tutorial builds one custom node end to end — an Emoji Generator that maps numeric values to emojis. It is a small but complete example of the [Node Designer](creating-custom-nodes.md) API: a multi-section settings schema, type-filtered column selection, and a `process()` method that transforms an incoming DataFrame. -!!! info "What You'll Build" - By the end of this tutorial, you'll have created a fully functional "Emoji Generator" node that: - - Takes numeric data and converts it into mood-based emojis - - Provides 7 different emoji themes (performance, temperature, money, etc.) - - Includes intensity controls and random sparkle effects - - Features a sophisticated multi-section UI +!!! info "What you'll build" + By the end you will have an "Emoji Generator" node that: + + - Maps a numeric column to mood-based emojis. + - Offers seven emoji themes (performance, temperature, money, and so on). + - Includes intensity controls and an optional random sparkle. + - Uses a two-section settings panel. ## Prerequisites @@ -156,10 +157,10 @@ class EmojiSettings(NodeSettings): class EmojiGenerator(CustomNodeBase): # Node metadata - how it appears in Flowfile node_name: str = "Emoji Generator 🎉" - node_category: str = "Fun Stuff" # This creates a new category - node_group: str = "custom" + node_category: str = "Fun Stuff" # descriptive label only; does NOT create a palette section + node_group: str = "custom" # node_group="custom" puts it under "User Defined Operations" title: str = "Emoji Generator" - intro: str = "Transform boring data into fun emoji-filled datasets! 🚀" + intro: str = "Add an emoji column derived from a numeric column." # I/O configuration number_of_inputs: int = 1 @@ -168,17 +169,18 @@ class EmojiGenerator(CustomNodeBase): # Link to our settings schema settings_schema: EmojiSettings = EmojiSettings() - def process(self, input_df: pl.DataFrame) -> pl.DataFrame: - # We'll implement this in the next step - pass + def process(self, *inputs: pl.DataFrame) -> pl.DataFrame: + # Implemented in the next step + ... ``` ## Step 5: Implement the Processing Logic -Now for the fun part - the actual emoji generation logic: +Now the processing logic — mapping each value to an emoji. The method receives its inputs as a variadic `*inputs`; this node has one input, so it reads `inputs[0]`: ```python -def process(self, input_df: pl.DataFrame) -> pl.DataFrame: +def process(self, *inputs: pl.DataFrame) -> pl.DataFrame: + input_df = inputs[0] # Get settings values from the UI column_name = self.settings_schema.mood_config.source_column.value mood_type = self.settings_schema.mood_config.mood_type.value @@ -298,285 +300,46 @@ Save your file and test it: flowfile run ui ``` -2. **Create a test flow**: - - Create a new flow - - Add a "Manual Input" node with some test data: +2. **Create a test flow.** + - Create a new flow. + - Add a "Manual Input" node. The `source_column` selector filters to numeric columns, and the threshold logic compares numbers, so use numeric `value` entries (not quoted strings): ``` [ - { - "name": "bob", - "value": "21" - }, - { - "name": "magret", - "value": "62.1" - }, - { - "name": "fish", - "value": "1.2" - }, - { - "name": "dog", - "value": "20" - } + {"name": "bob", "value": 21}, + {"name": "magret", "value": 62.1}, + {"name": "fish", "value": 1.2}, + {"name": "dog", "value": 20} ] ``` - - Look for your "Emoji Generator 🎉" in the "User defined operations" category - - Connect it to your manual input - - Configure the settings and run! + - Find your "Emoji Generator" under **User Defined Operations** in the palette. + - Connect it to your manual input. + - Configure the settings and run.
- Visual overview of the result! +Visual overview of the result ![Flow visualized](../assets/images/developers/emoji_settings.png)
-### Performance Tips +### Performance notes -1. **Use Polars expressions** instead of Python loops when possible -2. **Avoid collecting DataFrames** in the middle of processing -3. **Handle large datasets** by checking input size and warning users -4. **Cache expensive operations** like random number generation -5. **Use lazy evaluation** - let Polars optimize the query plan +1. Prefer Polars expressions over Python loops where the logic allows it. +2. `map_elements` (used here) runs a Python callback per row — fine for a small demo, slower on large data. +3. Read settings once at the top of `process()` rather than inside the row callback. ## Complete Working Example -Here's the complete, working emoji generator node: - -```python -import polars as pl -import random -from typing import List - -from flowfile_core.flowfile.node_designer import ( - CustomNodeBase, - Section, - NodeSettings, - TextInput, - NumericInput, - SingleSelect, - ToggleSwitch, - ColumnSelector, - MultiSelect, - Types -) - - -class EmojiMoodSection(Section): - source_column: ColumnSelector = ColumnSelector( - label="Analyze This Column", - multiple=False, - required=True, - data_types=Types.Numeric # Only show numeric columns - ) - - mood_type: SingleSelect = SingleSelect( - label="Emoji Mood Logic", - options=[ - ("performance", "📈 Performance Based (High = 😎, Low = 😰)"), - ("temperature", "🌡️ Temperature (Hot = 🔥, Cold = 🧊)"), - ("money", "💰 Money Mode (Rich = 🤑, Poor = 😢)"), - ("energy", "⚡ Energy Level (High = 🚀, Low = 🔋)"), - ("love", "❤️ Love Meter (High = 😍, Low = 💔)"), - ("chaos", "🎲 Pure Chaos (Random emojis!)"), - ("pizza", "🍕 Pizza Scale (Everything becomes pizza)") - ], - default="performance" - ) - - threshold_value: NumericInput = NumericInput( - label="Mood Threshold", - default=50.0, - min_value=0, - max_value=100 - ) - - emoji_column_name: TextInput = TextInput( - label="New Emoji Column Name", - default="mood_emoji", - placeholder="Name your emoji column..." - ) - - -class EmojiStyleSection(Section): - emoji_intensity: SingleSelect = SingleSelect( - label="Emoji Intensity", - options=[ - ("subtle", "😐 Subtle (One emoji)"), - ("normal", "😊 Normal (1-2 emojis)"), - ("extra", "🤩 Extra (2-3 emojis)"), - ("maximum", "🤯🎉🚀 MAXIMUM OVERDRIVE") - ], - default="normal" - ) - - add_random_sparkle: ToggleSwitch = ToggleSwitch( - label="Add Random Sparkles ✨", - default=True, - description="Randomly sprinkle ✨ for extra pizzazz" - ) - - emoji_categories: MultiSelect = MultiSelect( - label="Allowed Emoji Categories", - options=[ - ("faces", "😀 Faces & Emotions"), - ("animals", "🦄 Animals"), - ("food", "🍔 Food & Drink"), - ("nature", "🌈 Nature"), - ("objects", "🎮 Objects"), - ("symbols", "💯 Symbols"), - ("flags", "🏴‍☠️ Flags") - ], - default=["faces", "animals", "food"] - ) - - -class EmojiSettings(NodeSettings): - mood_config: EmojiMoodSection = EmojiMoodSection( - title="Mood Detection 😊", - description="Configure how to detect the vibe of your data" - ) - - style_options: EmojiStyleSection = EmojiStyleSection( - title="Emoji Style 🎨", - description="Fine-tune your emoji experience" - ) - - -class EmojiGenerator(CustomNodeBase): - # Node metadata - how it appears in Flowfile - node_name: str = "Emoji Generator 🎉" - node_category: str = "Fun Stuff" # This creates a new category - node_group: str = "custom" - title: str = "Emoji Generator" - intro: str = "Transform boring data into fun emoji-filled datasets! 🚀" - - # I/O configuration - number_of_inputs: int = 1 - number_of_outputs: int = 1 - - # Link to our settings schema - settings_schema: EmojiSettings = EmojiSettings() - - def process(self, input_df: pl.DataFrame) -> pl.DataFrame: - # Get settings values from the UI - # Note how we can just access these from the settings that we have defined - column_name = self.settings_schema.mood_config.source_column.value - mood_type = self.settings_schema.mood_config.mood_type.value - threshold = self.settings_schema.mood_config.threshold_value.value - emoji_col_name = self.settings_schema.mood_config.emoji_column_name.value - intensity = self.settings_schema.style_options.emoji_intensity.value - add_sparkle = self.settings_schema.style_options.add_random_sparkle.value - - # Define emoji sets for different moods - emoji_sets = { - "performance": { - "high": ["😎", "💪", "🏆", "👑", "🌟", "💯", "🔥"], - "low": ["😰", "😓", "📉", "😢", "💔", "🆘", "😵"] - }, - "temperature": { - "high": ["🔥", "🌋", "☀️", "🥵", "🌡️", "♨️", "🏖️"], - "low": ["🧊", "❄️", "⛄", "🥶", "🌨️", "🏔️", "🐧"] - }, - "money": { - "high": ["🤑", "💰", "💎", "🏦", "💳", "🪙", "📈"], - "low": ["😢", "💸", "📉", "🏚️", "😭", "🥺", "📊"] - }, - "energy": { - "high": ["🚀", "⚡", "💥", "🎯", "🏃", "🤸", "🎪"], - "low": ["🔋", "😴", "🛌", "🐌", "🥱", "😪", "💤"] - }, - "love": { - "high": ["😍", "❤️", "💕", "🥰", "💘", "💝", "👨‍❤️‍👨"], - "low": ["💔", "😢", "😭", "🥀", "😔", "💀", "🖤"] - }, - "chaos": { - "high": ["🦖", "🎸", "🚁", "🎪", "🦜", "🎭", "🏴‍☠️"], - "low": ["🥔", "🧦", "📎", "🦷", "🧲", "🔌", "🪣"] - }, - "pizza": { - "high": ["🍕", "🍕🍕", "🍕🔥", "🍕😍", "🍕🎉", "🍕💯", "🍕👑"], - "low": ["🍕", "🍕😢", "🍕💔", "🍕😭", "🍕🥺", "🍕😔", "🍕"] - } - } - - # Helper function to get emoji based on value - def get_emoji(value, mood_type, threshold, intensity): - if value is None: - return "❓" - - emoji_list = emoji_sets.get(mood_type, emoji_sets["performance"]) - - if mood_type == "chaos": - # Random emoji from both lists - all_emojis = emoji_list["high"] + emoji_list["low"] - base_emoji = random.choice(all_emojis) - elif mood_type == "pizza": - # Everything is pizza - base_emoji = "🍕" - else: - # Use threshold to determine high/low - if value >= threshold: - base_emoji = random.choice(emoji_list["high"]) - else: - base_emoji = random.choice(emoji_list["low"]) - - # Add intensity - if intensity == "subtle": - result = base_emoji - elif intensity == "normal": - result = base_emoji - if random.random() > 0.5: - result += random.choice(["", "✨", ""]) - elif intensity == "extra": - extras = ["✨", "💫", "⭐", ""] - result = base_emoji + random.choice(extras) + random.choice(extras) - else: # maximum - chaos_emojis = ["🎉", "🚀", "💥", "🌈", "✨", "🔥", "💯", "⚡"] - result = base_emoji + "".join(random.choices(chaos_emojis, k=3)) - - # Add random sparkle - if add_sparkle and random.random() > 0.7: - result += "✨" - - return result - - # Create emoji column using map_elements - emoji_expr = ( - pl.col(column_name) - .map_elements( - lambda x: get_emoji(x, mood_type, threshold, intensity), - return_dtype=pl.String - ) - .alias(emoji_col_name) - ) - - # Add bonus columns based on intensity - if intensity == "maximum": - # Add extra fun columns in maximum mode - return input_df.with_columns([ - emoji_expr, - pl.lit("🎉 PARTY MODE 🎉").alias("vibe_check"), - pl.col(column_name).map_elements( - lambda x: "🔥" * min(int((x or 0) / 20), 5) if x else "💤", - return_dtype=pl.String - ).alias("fire_meter") - ]) - else: - return input_df.with_columns([emoji_expr]) -``` +The complete node is the assembly of Steps 3–5: the two `Section` classes and `EmojiSettings` from Step 3, the `EmojiGenerator` class from Step 4, and its `process()` body from Step 5, in one file. -Save this as `~/.flowfile/user_defined_nodes/emoji_generator.py`, restart Flowfile, and enjoy your new emoji-powered data transformations! +Save that assembled file as `~/.flowfile/user_defined_nodes/emoji_generator.py`, then restart Flowfile. -## Congratulations! 🎉 +!!! note "One setting is defined but unused" + `emoji_categories` (the `MultiSelect` in the style section) is collected by the UI but never read in `process()`. It's left in as an exercise — wire it into `get_emoji` to restrict which emoji pools a theme draws from. -You've successfully created a fully functional custom node +## Recap -- ✅ Multi-section UI with 6 different component types -- ✅ Complex processing logic with multiple mood themes -- ✅ Advanced features like intensity control and random effects -- ✅ Professional documentation and structure +You built a custom node with a two-section settings panel, type-filtered column selection, and a `process()` method that maps values to emojis. See [Creating Custom Nodes](creating-custom-nodes.md) for the full component catalog and the `SecretSelector` for API-backed nodes. diff --git a/docs/for-developers/design-philosophy.md b/docs/for-developers/design-philosophy.md index e79bc84e7..2c2dde913 100644 --- a/docs/for-developers/design-philosophy.md +++ b/docs/for-developers/design-philosophy.md @@ -1,47 +1,28 @@ -# The Architecture Story: How Flowfile Bridges Code and Visual Worlds +# Design Philosophy: Code and Visual Are One Model -📋 TL;DR - Key Takeaways +This page explains the core design decision behind Flowfile: the Python API and the visual editor construct the same underlying objects. It is for contributors mapping the codebase, and for anyone curious how a drag-and-drop tool and a code API can stay in lockstep. -!!! abstract "Key Points" - - **Two ways to build pipelines**: Write Python code or use drag-and-drop UI - both create the same thing - - **Settings-based design**: Every transformation is just a configuration object (Pydantic model) - - **Clear separation**: Graph structure, settings, and execution are handled separately - - **Happy accident**: Started as a UI project, ended up with an architecture that works great for both UI and code +!!! info "Related pages" + - [Technical Architecture](architecture.md) — the three services and execution modes + - [Core Developer Guide](flowfile-core.md) — internal implementation, hands-on + - [Python API](../users/python-api/index.md) — using Flowfile in your own projects +## The problem -!!! info "Navigation" - - **This page**: Architecture overview and design decisions - - **[Core Developer Guide](flowfile-core.md)**: Technical implementation details - - **[Python API](../users/python-api/index.md)**: How to use Flowfile in your projects +Most data tools make you choose: a visual interface (approachable but limited) or code (expressive but harder). Flowfile aims for both in one tool, which raises one hard question — how do you make a drag-and-drop interface produce the exact same pipelines as writing code? -👥 Who Should Read This? -!!! question "Target Audience" - - Contributors who want to understand the codebase - - Users curious about how things work internally - - Developers building similar dual-interface tools - - Anyone interested in bridging UI and code approaches - -## The Problem We Solved - -Most data tools force you to choose: either use a visual interface (easy but limited) or write code (powerful but complex). We wanted both in the same tool. - -The challenge: How do you make a visual drag-and-drop interface that creates the exact same pipelines as writing code? - -The platform started with a clean, settings-based backend where every transformation is a declarative configuration object. This design is perfect for a UI. But developers don't think in configuration objects—they think in code: +The backend was built around a settings-based model: every transformation is a declarative configuration object (a Pydantic model). That model suits a UI well, but developers think in code, not configuration objects: ```python # How developers want to write data code df.filter(col("price") > 100).group_by("region").sum() ``` -The breakthrough came from realizing that the Polars API would be able to convert to our settings object and therefore creating the same settings object that the UI creates. Both interfaces become different ways to build the same underlying configuration, giving developers the expressiveness they want while maintaining the structured settings the UI needs. +The resolution is to have the Python API build those same settings objects. Both interfaces become different front-ends to one underlying configuration: developers get an expressive, chainable API, and the UI gets the structured settings it needs. ## The result -!!! abstract "How It Actually Happened" - This wasn't some grand plan. I started building a drag-and-drop UI and needed a clean way to configure nodes. Settings objects made sense for the UI. But the development of Flowfile has never been a planned approach, it was just about building what sounded _fun_. - Later, when looking at other projects, I realized I could just have the API methods create the same settings objects, well that is **_fun_**. Suddenly there were two equivalent interfaces almost by accident,. - Since Polars does the actual data processing, our settings just configure what Polars should do. This turned out to be an easy abstraction layer that showed it's potential from the start. +Flowfile grew out of a visual editor. Nodes were configured with settings objects because that is a clean fit for a UI. Later, the Python API was wired to construct those same settings objects, so a method call and a dropped node produce identical configuration. Because Polars does the actual data processing, the settings only describe *what* Polars should do — a thin, stable abstraction layer. The result is a Python API that constructs the exact same configuration objects as the visual editor: @@ -52,7 +33,7 @@ Both interfaces are different ways to build the same Directed Acyclic Graph (DAG ## One Pipeline, Two Ways -Let's build the same pipeline using both approaches to see how they produce identical results. +The same pipeline built both ways produces identical results. ### Sample Data @@ -101,29 +82,35 @@ df_4 = df_3.group_by(['region']).agg([ **Graph Introspection:** ```python -# Access all nodes that were created in the graph +# Access all nodes that were created in the graph. +# Node ids come from a global incrementing counter (generate_node_id), so +# the exact numbers depend on how many ids were minted before this run. print(graph._node_db) -# {1: Node id: 1 (manual_input), -# 3: Node id: 3 (formula), -# 4: Node id: 4 (filter), -# 5: Node id: 5 (group_by)} +# {: Node id: (manual_input), +# : Node id: (formula), +# : Node id: (filter), +# : Node id: (group_by)} # Find the starting node(s) of the graph print(graph._flow_starts) -# [Node id: 1 (manual_input)] +# [Node id: (manual_input)] + +# Each FlowFrame exposes the node id it produced +manual_id = df_1.node_id +formula_id = df_2.node_id # From every node, access the next node that depends on it -print(graph.get_node(1).leads_to_nodes) -# [Node id: 3 (formula)] +print(graph.get_node(manual_id).leads_to_nodes) +# [Node id: (formula)] # The other way around works too -print(graph.get_node(3).node_inputs) -# NodeStepInputs(Left Input: None, Right Input: None, -# Main Inputs: [Node id: 1 (manual_input)]) +print(graph.get_node(formula_id).node_inputs) +# NodeStepInputs(Left Input: None, Right Input: None, +# Main Inputs: [Node id: (manual_input)]) # Access the settings and type of any node -print(graph.get_node(4).setting_input) -print(graph.get_node(4).node_type) +print(graph.get_node(formula_id).setting_input) +print(graph.get_node(formula_id).node_type) ```
@@ -224,7 +211,7 @@ This is the polars query plan generated by both methods: #### 1. The DAG is Everything -Every Flowfile pipeline is a Directed Acyclic Graph where. This is captured in the [FlowGraph](python-api-reference.md#flowgraph) +Every Flowfile pipeline is a Directed Acyclic Graph where nodes are operations and edges are data dependencies, captured in the [FlowGraph](python-api-reference.md#flowgraph): - **Nodes** are transformations (filter, join, group_by, etc.) - **Edges** represent data flow between nodes @@ -232,7 +219,7 @@ Every Flowfile pipeline is a Directed Acyclic Graph where. This is captured in t #### 2. Settings Drive Everything -Every node is composed of two parts: the **Node class** (a Pydantic BaseModel) that holds metadata and the **Settings** (often dataclasses) that configure the transformation: +Every node is composed of two parts: the **Node class** (a Pydantic BaseModel) that holds metadata, and the **Settings** (also Pydantic BaseModels) that configure the transformation: Read more about [Nodes](python-api-reference.md#input_schema) and the [transformations](python-api-reference.md#transform_schema) @@ -253,67 +240,41 @@ class NodeBase(BaseModel): description: Optional[str] = None # ... graph metadata ... -# The Settings: transformation configuration (dataclass) -@dataclass -class GroupByInput: +# The Settings: transformation configuration (Pydantic BaseModel +# with a custom __init__ that also accepts positional args) +class GroupByInput(BaseModel): """Defines how to perform the group by operation""" agg_cols: List[AggColl] -@dataclass -class AggColl: +class AggColl(BaseModel): """Single aggregation operation""" - old_name: str # Column to aggregate - agg: str # Aggregation function ('sum', 'mean', etc.) + old_name: str # Column to aggregate + agg: str # Aggregation function ('sum', 'mean', etc.) new_name: Optional[str] # Output column name output_type: Optional[str] = None ``` -!!! tip "Settings Power The Backend" - This dual structure—Nodes for graph metadata, Settings for transformation logic—drives the backend: - - - 🔧 **Code generation** (method signatures match settings) - - 💾 **Serialization** (graphs can be saved/loaded) - - 🔮 **Schema prediction** (output types are inferred from AggColl) - - 🎨 **UI structure** (defines what the frontend needs to collect, though forms are manually built) +!!! tip "Settings power the backend" + This dual structure — Nodes for graph metadata, Settings for transformation logic — drives the backend: + + - **Code generation** — method signatures match settings. + - **Serialization** — graphs can be saved and loaded. + - **Schema prediction** — output types are inferred from settings like `AggColl`. + - **UI structure** — defines what the frontend collects (native-node forms are hand-built; custom nodes are auto-generated). #### 3. Execution is Everything -The [`FlowDataEngine`](python-api-reference.md#flowfile_core.flowfile.flow_data_engine.flow_data_engine.FlowDataEngine) orchestrates everything about execution. While the DAG defines structure and settings define configuration, FlowDataEngine is the runtime brain that makes it all happen. +The [`FlowDataEngine`](python-api-reference.md#flowfile_core.flowfile.flow_data_engine.flow_data_engine.FlowDataEngine) owns execution. While the DAG defines structure and settings define configuration, FlowDataEngine wraps a Polars LazyFrame/DataFrame and decides where, when, and how transformations run: -FlowDataEngine handles: - **Compute location** (worker service vs local execution) - **Caching strategy** (when to materialize, where to store) - **Schema caching** (avoiding redundant schema calculations) - **Lazy vs eager evaluation** (performance vs debugging modes) - **Data movement** (passing LazyFrames between transformations) -This separation is powerful: the DAG remains a pure specification, settings stay declarative, and FlowDataEngine owns all execution concerns. It wraps a Polars LazyFrame/DataFrame but is really the execution orchestrator—deciding where, when, and how transformations run. - ### Understanding FlowNode -The `FlowNode` class is the heart of each transformation in the graph. Each node encapsulates everything needed for a single transformation step: - -!!! info "Core FlowNode Components" - **Essential State:** - - - **`_function`**: The closure containing the transformation logic - - **`leads_to_nodes`**: List of downstream nodes that depend on this one - - **`node_information`**: Metadata (id, type, position, connections) - - **`_hash`**: Unique identifier based on settings and parent hashes - - **Runtime State:** - - - **`results`**: Holds the resulting data, errors, and example data paths - - **`node_stats`**: Tracks execution status (has_run, is_canceled, etc.) - - **`node_settings`**: Runtime settings (cache_results, streamable, etc.) - - **`state_needs_reset`**: Flag indicating if the node needs recalculation - - **Schema Information:** - - - **`node_schema`**: Input/output columns and predicted schemas - - **`schema_callback`**: Function to calculate schema without execution - -The beauty is that FlowNode doesn't know about specific transformations—it just orchestrates the execution of its `_function` closure with the right inputs and manages the resulting state. +Each `FlowNode` wraps a single transformation: it holds the `_function` closure that carries the node's logic, its downstream links, hash, runtime state, and schema. FlowNode doesn't know about specific transformations — it orchestrates its `_function` closure with the right inputs and manages the resulting state. For the full component-by-component walkthrough, see the [Core Developer Guide](flowfile-core.md). ## Flowfile: The Use of Closures @@ -358,7 +319,7 @@ graph LR func4 ==> |Final FlowDataEngine
with full LazyFrame plan| Result ``` -Each `_func` is a closure that wraps around the previous one, building up a chain. The beauty is that Polars can track the schema through this entire chain without executing any data transformations—it just builds the query plan! +Each `_func` is a closure that wraps around the previous one, building up a chain. Polars tracks the schema through this entire chain without executing any data transformations — it just builds the query plan. #### The Closure Pattern in Practice @@ -411,14 +372,6 @@ print(result.data_frame.collect_schema()) # Schema([('region', String), ('total_revenue', Float64), ('avg_transaction', Float64)]) ``` -!!! example "Why This Works" - 1. **Each `_func` is a closure** containing the node's settings - 2. **Functions only need FlowDataEngine as input** (or multiple for joins/unions) - 3. **LazyFrame tracks schema changes** through the entire chain - 4. **No data is processed**—Polars just builds the query plan - - The result: instant schema feedback without running expensive computations! - ### Fallback: Schema Callbacks For nodes that can't infer schemas automatically (external data sources), each FlowNode can have a `schema_callback`: @@ -430,187 +383,97 @@ def schema_callback(settings, input_schema): return new_schema ``` -## Execution Methods - -Flowfile offers flexible execution strategies depending on your needs: +## Running a flow -### 🚀 Available Execution Methods +The two execution modes — **Development** and **Performance** — are described in full on the [Technical Architecture page](architecture.md#execution-modes). The code below shows how each is invoked and how to read the query plan. -#### Performance Mode - -**When to use:** Production pipelines, large datasets +**Performance mode.** Pull the final result; Polars optimizes and runs the whole pipeline once, with no intermediate materialization. ```python -# Get the final result efficiently result = flow.get_node(final_node_id).get_resulting_data() ``` -**Characteristics:** - -- ⚡ Pull-based execution from the final node -- 🎯 Polars optimizes the entire pipeline -- 💨 Data flows once through optimized plan -- 🚫 No intermediate materialization - -#### Development Mode - -**When to use:** Debugging, inspection, incremental development +**Development mode.** Push-based execution in topological order; each node's output is written to disk so any intermediate result can be inspected. On re-run, only changed nodes (and their descendants) run again. ```python -# Execute with caching enabled import flowfile as ff flow = ff.create_flow_graph() flow.flow_settings.execution_mode = "Development" -# Add transformations here +# ... add transformations ... flow.run_graph() -# Inspect intermediate results -node_3_result = flow.get_node(3).results.get_example_data() - -flow.get_node(3).needs_run(performance_mode=False) # False - +# Inspect an intermediate result by node id +node = flow.get_node(some_node_id) +example = node.results.get_example_data() +node.needs_run(performance_mode=False) # False once it has run ``` -**Characteristics:** -- 📝 Push-based execution in topological order -- 💾 Each node's output written to disk -- 🔍 Inspect any intermediate result -- 🔄 When re-running the flow, only the steps that have changed(directly and indirectly) will run - -#### Explain Plan - -**When to use:** Optimization, understanding deeply the execution plan. - -!!! warning This feature uses directly the Polars implementation, when the full flow cannot be fully converted to Polars, it will show partial executions. +**Explain plan.** See the optimized Polars plan without executing. ```python -# See what Polars will actually do plan = flow.get_node(node_id).get_resulting_data().data_frame.explain() print(plan) ``` -**Characteristics:** -- 📊 Shows optimized query plan -- 🔍 Understand Polars optimizations -- 📈 Identify performance bottlenecks -- 🎯 No actual execution +!!! warning "Partial plans" + Explain uses the Polars plan directly. When part of the flow cannot be expressed as Polars, the plan shows only the convertible portion. ## System Architecture -### Service Architecture - -```mermaid -graph LR - subgraph "Frontend" - A[Designer
Vue/Electron] - end - - subgraph "Backend" - B[Core Service
FastAPI] - C[Worker Service
FastAPI] - end - - subgraph "Storage" - D[Arrow IPC
Cache] - end - - A <-->|Settings/Schema| B - B <-->|Execution| C - C <-->|Data| D -``` - -!!! info "Service Responsibilities" - **Designer:** - - Visual graph building interface - - Node configuration forms (manually implemented) - - Real-time schema feedback - - **Core:** - - DAG management - - Execution orchestration - - Schema prediction - - **Worker:** - - Polars transformations - - Data caching (Arrow IPC) - - Isolated from Core for scalability +The three-service split (Designer, Core, Worker) and Arrow IPC data exchange are covered on the [Technical Architecture page](architecture.md#three-service-architecture). ### Project Structure +Each Python package nests its importable code one level down (e.g. `flowfile_core/flowfile_core/`). The key directories: + ``` -flowfile/ -├── flowfile_core/ -│ ├── nodes/ # Node implementations -│ ├── schemas/ # Pydantic models -│ └── flowfile/ # Graph management -├── flowfile_worker/ -│ ├── execution/ # Polars execution -│ └── cache/ # Arrow IPC caching +Flowfile/ +├── flowfile_core/flowfile_core/ +│ ├── flowfile/ # FlowGraph, FlowNode, FlowDataEngine, node_designer +│ ├── schemas/ # Pydantic settings + request/response models +│ ├── configs/ # node_store/nodes.py registry, settings.py +│ ├── routes/ # FastAPI routers +│ ├── ai/ · kernel/ · catalog/ · auth/ · scheduler/ · alembic/ +├── flowfile_worker/flowfile_worker/ # flat modules: funcs.py, routes.py, +│ # process_manager.py, viz_session_worker.py, … +├── flowfile_frame/ # Polars-like Python API (FlowFrame, Expr) └── flowfile_frontend/ - ├── components/ # Vue components - └── electron/ # Desktop app + ├── src/renderer/ # Vue 3 renderer (shared by desktop + web) + └── src-tauri/ # Tauri 2 Rust shell + sidecar boot ``` ## Contributing -!!! warning "Current State of Node Development" - While the backend architecture elegantly uses settings-driven nodes, adding new nodes requires work across multiple layers. The frontend currently requires manual implementation for each node type—the visual editor doesn't automatically generate forms from Pydantic schemas yet. - - However, there are also opportunities for more focused contributions! Integration with databases and cloud services is needed—these are smaller, more targeted tasks since the core structure is already in place. There's a lot of active development happening, so it's an exciting time to contribute! - -### Adding a New Node: The Full Picture - -Adding a node isn't as simple as defining settings and a function. Here's what's actually required: +Adding a **built-in native node** touches multiple layers: the frontend needs a hand-written settings form (native nodes are not auto-generated from Pydantic schemas). Smaller, self-contained tasks — new database and cloud connectors — are a good entry point, because the surrounding structure already exists. -#### Backend Requirements +!!! tip "Prefer a custom node?" + If you want a new transformation without touching the frontend, build a [custom node](creating-custom-nodes.md) with the Node Designer API. Custom nodes get an auto-generated settings panel and land in the **User Defined Operations** palette section. -1. **Define the Pydantic settings model** in `schemas/` -2. **Implement the transformation method** on `FlowDataEngine` -3. **Add the node method** to `FlowGraph` (e.g., `add_custom_transform()`) -4. **Create the closure function** that captures settings -5. **Define schema callbacks** for predicting output schemas -6. **Register the node** in the node registry - -Example of what's really needed in FlowGraph: - -```python -def add_custom_transform(self, transform_settings: input_schema.NodeCustomTransform): - # Create the closure that captures settings - def _func(fl: FlowDataEngine) -> FlowDataEngine: - return fl.do_custom_transform(transform_settings.transform_input) - - # Register with the graph - self.add_node_step( - node_id=transform_settings.node_id, - function=_func, - node_type='custom_transform', - setting_input=transform_settings, - input_node_ids=[transform_settings.depending_on_id] - ) - - # Don't forget schema prediction! - node = self.get_node(transform_settings.node_id) - # ... schema callback setup ... -``` +### Adding a native node: the full picture -#### Frontend Requirements +A native node is more than a settings model and a function. -Currently, you'll need to: +#### Backend -1. Create a new Vue component for the node's configuration form -2. Handle the visual representation in the graph editor -3. Map the UI inputs to the backend settings structure -4. Add the node type to the visual editor's palette +1. Define the Pydantic settings model in `schemas/`. +2. Implement the transformation method on `FlowDataEngine`. +3. Add the node method to `FlowGraph` (e.g. `add_()`). +4. Create the closure function that captures settings. +5. Define a schema callback for predicting output schemas. +6. Register the node in `configs/node_store/nodes.py`. -This manual process ensures full control over the UI/UX but requires significant development effort. +The `FlowGraph.add_` method follows the same closure pattern shown earlier — construct a `_func` that captures the settings and hand it to `add_node_step`. See `add_group_by` / `add_union` above for real examples. -### Future Vision +#### Frontend -The goal is to eventually auto-generate UI from Pydantic schemas, which would complete the settings-driven architecture. This would make adding new nodes closer to just defining the backend settings and transformation logic, with the UI automatically following. +1. Create a Vue settings component for the node's form. +2. Handle its visual representation in the graph editor. +3. Map the UI inputs to the backend settings structure. +4. Add the node type to the palette. -The beauty of Flowfile's architecture—discovered through the organic evolution from a UI-first approach—is that even though adding nodes requires work across multiple layers today, the settings-based design provides a clear contract between visual and code interfaces. +!!! note "Future direction" + A long-term goal is to auto-generate the native-node settings UI from Pydantic schemas, the way custom nodes already work — which would reduce a native node to its backend settings and transformation logic. This is aspirational, not shipped. -I hope you enjoyed learning about Flowfile's architecture and found the dual-interface approach as exciting as I do! If you have questions, ideas, or want to contribute, ] -feel free to reach out via [GitHub](https://github.com/edwardvaneechoud/Flowfile) or check our [Core Developer Guide](flowfile-core.md). Happy building! \ No newline at end of file +Questions and ideas are welcome via [GitHub](https://github.com/edwardvaneechoud/Flowfile), and the [Core Developer Guide](flowfile-core.md) goes deeper on the internals. \ No newline at end of file diff --git a/docs/for-developers/docker-deployment.md b/docs/for-developers/docker-deployment.md deleted file mode 100644 index b94f63111..000000000 --- a/docs/for-developers/docker-deployment.md +++ /dev/null @@ -1,220 +0,0 @@ -# Docker Deployment Guide - -Deploy Flowfile using Docker Compose for development and production environments. - -## Prerequisites - -- Docker and Docker Compose installed -- Basic understanding of Docker concepts - -## Quick Start - -### Option A: Interactive Setup (Recommended) - -1. **Start the services:** - ```bash - docker compose up -d - ``` - -2. **Open Flowfile:** http://localhost:8080 - -3. **Follow the Setup Wizard:** - - Click "Generate Master Key" - - Copy the generated key - - Add to your `.env` file: `FLOWFILE_MASTER_KEY=` - - Restart: `docker compose restart` - -4. **Log in** with default credentials (`admin` / `changeme`) - -### Option B: Pre-configured Setup - -For automated deployments: - -#### Step 1: Generate the Master Key - -```bash -python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())" -``` - -Copy the output for the next step. - -#### Step 2: Configure Environment - -```bash -cp .env.example .env -``` - -Edit `.env`: - -```bash -FLOWFILE_MASTER_KEY= -FLOWFILE_ADMIN_USER=admin -FLOWFILE_ADMIN_PASSWORD=YourSecurePassword123! -JWT_SECRET_KEY=your-secure-jwt-secret-at-least-32-chars -``` - -> **Git project tracking** is admin-only in docker and is gated by `FLOWFILE_ENABLE_PROJECTS` -> (the `/project` router 404s when off). The bundled compose files default it to `true`; set -> `FLOWFILE_ENABLE_PROJECTS=false` to disable it. Each user's projects are confined to their own -> `/app/user_data/projects/` subtree, so tenants stay isolated. - -#### Step 3: Start Services - -```bash -docker compose up -d -``` - -Access at: http://localhost:8080 - -## Security Architecture - -| Component | Purpose | Configuration | -|-----------|---------|---------------| -| **Master Key** | Encrypts user secrets at rest | `FLOWFILE_MASTER_KEY` env var | -| **User Secrets** | API keys, passwords, tokens | Encrypted in database | -| **JWT Secret** | Signs authentication tokens | `JWT_SECRET_KEY` env var | -| **User Password** | Authenticates users | Hashed in database | - -### How They Work Together - -1. User logs in with username/password -2. Server issues a JWT token (signed with `JWT_SECRET_KEY`) -3. User creates secrets (e.g., "my_api_key" = "sk-xxx") -4. Secret value is encrypted using the master key before storage -5. At runtime, secrets are decrypted with the master key for use in flows - -## Production Checklist - -- [ ] Generate a unique `FLOWFILE_MASTER_KEY` -- [ ] Set a strong `FLOWFILE_ADMIN_PASSWORD` -- [ ] Generate secure `JWT_SECRET_KEY` with `openssl rand -hex 32` -- [ ] Never commit `.env` to version control -- [ ] Back up `.env` securely (losing master key = losing all encrypted secrets) -- [ ] Set up HTTPS (reverse proxy with nginx/traefik) -- [ ] Configure firewall rules - -## Docker Compose Services - -```yaml -services: - flowfile-frontend: # Web UI on port 8080 - flowfile-core: # API server on port 63578 - flowfile-worker: # Background job processor on port 63579 -``` - -All services share: -- The master key via `FLOWFILE_MASTER_KEY` environment variable -- User data volume (`./flowfile_data`) -- Internal network for communication - -## Troubleshooting - -### Setup wizard keeps appearing - -The master key is not configured. Add `FLOWFILE_MASTER_KEY` to your `.env` file and restart. - -### Invalid master key format - -The master key must be a valid Fernet key. Generate a new one: - -```bash -python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())" -``` - -**Warning:** Changing the key makes existing encrypted secrets unreadable. - -### Container fails to start - -Check the logs: - -```bash -docker compose logs flowfile-core -docker compose logs flowfile-worker -``` - -## Volumes - -| Volume | Purpose | -|--------|---------| -| `./flowfile_data` | User uploads, files | -| `./saved_flows` | Saved flow definitions | -| `flowfile-internal-storage` | Internal application data | - -## Scaling Considerations - -For high-availability deployments: -- Use external PostgreSQL instead of SQLite -- Deploy multiple worker instances -- Use Redis for distributed task queue -- Place services behind a load balancer - -## Group-Based Sharing (multi-user mode) - -In Docker (multi-user) mode, Flowfile supports sharing resources with **user groups**. -This feature is dormant in the desktop/Electron app. - -### ⚠️ Breaking change: the catalog is now private-by-default - -Previously the data catalog (namespaces, tables, flows, runs, visualizations, -dashboards) was **globally visible** to every authenticated user in Docker mode. -As of the release that introduces group sharing, the catalog is -**private-by-default**: each user sees only the resources they own, the ones -explicitly shared with a group they belong to, and the seeded *public* system -namespaces (`General`, `default`, `Unnamed Flows`, `Local Flows`) as tree -containers. Global admins still see everything. - -If your team relied on the old "everyone sees every flow/table" behavior, create a -group, add the relevant members, and share the namespaces/tables/flows you want -shared. No data is lost in the upgrade — only its *visibility* narrows. - -### How it works - -- **Groups**: only global admins create groups (`/user-groups`, or the **User - Groups** sidebar entry). Each group has member roles (`owner` / `manager` / - `member`) so a group owner can manage membership without being a global admin. -- **Sharing**: owners share a resource with a group at `use` (read/execute) or - `manage` (edit + re-share) level via the **Share** action in the UI. - - *Secrets* are **use-only** when shared: a group's flows can run with the - credential, but members can never view its value. Sharing a secret for use is - therefore security-equivalent to giving the group that credential. - - *Connections* (database, cloud, Google Analytics, Kafka): a shared connection - is usable in flows directly. A `manage`-grantee who changes the connection's - target (host/endpoint/protocol) must re-enter the credentials. -- **Schedules**: a `use`-level member may schedule a shared flow; the run executes - as that member, so its secret/connection references resolve against the member's - own grants. Make sure the member also has access to everything the flow uses. -- **Not shared**: AI BYOK keys and uploaded files stay outside the sharing model - (the Docker uploads directory is already common to all users). - -### How shared secrets decrypt for non-owners - -Sharing is **authorization-only**: granting a group access to a secret or -connection never copies, re-encrypts, or exposes the credential. - -Every secret is stored as `$ffsec$1$$`, encrypted with a -per-user key derived from the master key (HKDF with the owner's user id as -context). Because the owner's id is embedded in the stored value, decryption -never depends on *who is running the flow*: when a group member's flow resolves -a shared secret, core (and the worker, which derives keys independently) reads -the owner id out of the value and re-derives the **owner's** key. The member's -identity only matters for the authorization check — may this user resolve this -secret by name — which is grant-based: own secrets first, then group-shared ones. - -Consequences: - -- The plaintext is never returned to non-owners through the API; it only - resolves inside flow execution. -- When a `manage`-grantee rotates a shared connection's credential, the new - value is re-encrypted under the **owner's** key, so the stored format never - changes hands. -- Revoking a grant takes effect immediately: the grantee never held key - material, so there is nothing to rotate or re-encrypt. - -### Operator notes - -- `FLOWFILE_INTERNAL_SERVICE_USER_ID` (default `1`) selects which user the kernel - internal-service principal maps to when no kernel id is supplied. In hardened - deployments point it at a dedicated service account that owns no resources. -- Migration `020` adds the `user_groups`, `user_group_memberships`, and - `resource_grants` tables and an `is_public` flag on namespaces; it runs - automatically at core startup. diff --git a/docs/for-developers/flowfile-core.md b/docs/for-developers/flowfile-core.md index 04b852e15..2e11a9ac8 100644 --- a/docs/for-developers/flowfile-core.md +++ b/docs/for-developers/flowfile-core.md @@ -1,16 +1,11 @@ # Flowfile Core: A Developer's Guide -Welcome! This guide is for developers who want to understand, use, and contribute to `flowfile-core`. We'll dive into the architecture, see how data flows, and learn how to build powerful data pipelines. +This guide is for developers who want to understand and contribute to `flowfile-core`. It walks the architecture from the outside in — building a small pipeline in Python step by step, then explaining how the graph decides where each node runs. -!!! info "Looking for the API docs?" - - **[Python API Reference for users](../users/python-api/index.md)**: If you want to USE Flowfile - - **[Design Philosophy](design-philosophy.md)**: If you want to understand WHY Flowfile works this way - - **This page**: If you want to understand HOW Flowfile works internally - -!!! tip "New to Flowfile?" - If you're looking for the high-level Python API, start with the [Python API Overview](../users/python-api/index.md). This guide dives into the internal architecture. - -Ready? Let's build something! +!!! info "Looking for something else?" + - **[Python API Reference](../users/python-api/index.md)**: to USE Flowfile + - **[Design Philosophy](design-philosophy.md)**: to understand WHY Flowfile works this way + - **This page**: to understand HOW Flowfile works internally --- @@ -21,7 +16,7 @@ At its heart, `flowfile-core` is composed of three main objects: 1. **`FlowGraph`**: The central orchestrator. It holds your pipeline, manages the nodes, and controls the execution flow. 2. **`FlowNode`**: An individual step in your pipeline. It's a wrapper around your settings and logic, making it an executable part of the graph. -3. **`FlowDataEngine`**: The data itself, which flows between nodes. It's a smart wrapper around a [Polars LazyFrame](https://pola-rs.github.io/polars/py-polars/html/reference/lazyframe/index.html), carrying both the data and its schema. +3. **`FlowDataEngine`**: The data itself, which flows between nodes. It's a wrapper around a [Polars LazyFrame](https://pola-rs.github.io/polars/py-polars/html/reference/lazyframe/index.html), carrying both the data and its schema. Let's see these in action. @@ -90,9 +85,7 @@ It runs successfully but does nothing, as expected. The FlowGraph's job is to: Let's give it a node to manage. ## 2. Adding a Node: Where Settings Come to Life -You don't add raw functions or data directly to the graph. Instead, you provide **settings objects** (which are just Pydantic models). The graph then transforms these settings into executable `FlowNodes`. - -Watch this: +You don't add raw functions or data directly to the graph. Instead, you provide **settings objects** (Pydantic models). The graph turns these settings into executable `FlowNodes`. ```python from flowfile_core.schemas import input_schema @@ -178,7 +171,7 @@ print(f"Nodes executed: {result.nodes_completed}/{len(graph.nodes)}") # Nodes executed: 1/2 ``` -Only one node ran! Why? The graph is smart; it knows the filter node has no input, thus will never succeed running. +Only one node ran. The graph knows the filter node has no input, so it can never run successfully and is skipped.
Why the filter node was skipped @@ -244,10 +237,10 @@ for node_result in result.node_step_result: # - Node 2 ran successfully: True ``` -Success! Both nodes executed. The connection allowed data to flow from the input to the filter. +Both nodes executed. The connection let data flow from the input to the filter. ## 4. The FlowDataEngine: The Data Carrier -When data moves from one node to another, it's bundled up in a FlowDataEngine object. This isn't just raw data; it's an enhanced wrapper around a Polars LazyFrame. +When data moves from one node to another, it's bundled up in a FlowDataEngine object. This isn't just raw data; it's a wrapper around a Polars LazyFrame. ```python # Let's inspect the data after the run @@ -276,7 +269,7 @@ The `FlowDataEngine` is the boundary between `flowfile-core` and Polars. It: * Tracks metadata like record counts. * Manages lazy vs. eager execution. -## 5. The Hash System: Smart Change Detection +## 5. The Hash System: Change Detection How does the graph know when to re-run a node? Every `FlowNode` has a unique hash based on its configuration and its inputs. ```python @@ -310,10 +303,10 @@ This hash is calculated from: This creates a chain of **dependency**. If you change a node, `flowfile-core` knows that it and all downstream nodes need to be re-run, while upstream nodes can use their cached results. This is crucial for efficiency. -## 6. Schema Prediction: See the Future -One of the most powerful features for interactive UI is **schema prediction**. A node can predict its output schema _without_ processing any data. +## 6. Schema Prediction +**Schema prediction** is what makes the interactive UI responsive: a node computes its output schema _without_ processing any data. -Let's add a "formula" node to create a new column. +Add a "formula" node to create a new column. ```python from flowfile_core.schemas.transform_schema import FunctionInput, FieldInput @@ -351,7 +344,6 @@ print(f"\nHas the formula node run? {formula_node.node_stats.has_run_with_curren Predicted columns for Node 3: - name (Type: String) - age (Type: Int64) - - city (Type: String) - age_doubled (Type: Int64) ```
@@ -379,7 +371,7 @@ Let's recap the entire lifecycle: * **Results Flow Through:** The data, wrapped in the `FlowDataEngine`, moves down the pipeline, getting transformed at each step. -This architecture provides a powerful combination of flexibility, introspection, and performance, bridging the gap between a visual, no-code interface and a powerful, code-driven engine. +This architecture combines flexibility, introspection, and performance, bridging a visual no-code interface and a code-driven engine that share one graph. @@ -396,7 +388,7 @@ The behavior depends on two settings visible in the frontend: ### Execution Modes from the Frontend -In the UI, users choose between **Development** and **Performance** mode. Here's what each mode does when running remotely: +Users choose between **Development** and **Performance** mode in the UI. The [Technical Architecture page](architecture.md#execution-modes) is the canonical description of what each mode does; the section here focuses on how the mode maps to an execution *strategy* per node when running remotely. #### Development Mode (remote) @@ -404,9 +396,9 @@ Development mode is the interactive, debugging-friendly mode. It produces **prev | Node type | What happens | |---|---| -| **Narrow transforms** (select, sample, union) | Computation runs locally, only a 100-row sample is sent to the remote worker for preview (`LOCAL_WITH_SAMPLING`) | -| **Wide transforms** (sort, record_count) | Entire computation runs on the remote worker (`REMOTE`) | -| **Everything else** (filter, join, group_by, input nodes) | Entire computation runs on the remote worker (`REMOTE`) | +| **Narrow transforms** (select, sample, union, filter) | Computation runs locally, only a 100-row sample is sent to the remote worker for preview (`LOCAL_WITH_SAMPLING`) | +| **Wide transforms** (sort, record_count, join, group_by) | Entire computation runs on the remote worker (`REMOTE`) | +| **Other nodes** (input nodes, manual_input) | Entire computation runs on the remote worker (`REMOTE`) | | **Any node with `cache_results` enabled** | Always fully remote regardless of transform type (`REMOTE`) | #### Performance Mode (remote) @@ -502,26 +494,22 @@ The strategy routing depends on how nodes are classified in the node store: | select | narrow | LOCAL_WITH_SAMPLING | skip if already ran | | sample | narrow | LOCAL_WITH_SAMPLING | skip if already ran | | union | narrow | LOCAL_WITH_SAMPLING | skip if already ran | +| filter | narrow | LOCAL_WITH_SAMPLING | skip if already ran | | sort | wide | REMOTE | skip if already ran | | record_count | wide | REMOTE | skip if already ran | -| filter | other | REMOTE | skip if already ran | -| join | other | REMOTE | skip if already ran | -| group_by | other | REMOTE | skip if already ran | +| join | wide | REMOTE | skip if already ran | +| group_by | wide | REMOTE | skip if already ran | | manual_input | other | REMOTE | skip if already ran | -Narrow transforms are defined with `transform_type="narrow"` in `configs/node_store/nodes.py`. +Each node's `transform_type` (`narrow` / `wide` / `other`) is set in `configs/node_store/nodes.py`. Only `narrow` transforms take the `LOCAL_WITH_SAMPLING` path; `wide` and `other` go `REMOTE`. --- ## The FastAPI Service: Your API Layer -While `FlowGraph`, `FlowNode`, and `FlowDataEngine` power the core pipeline logic, the **FastAPI service** is what makes it accessible from the outside world. +While `FlowGraph`, `FlowNode`, and `FlowDataEngine` power the core pipeline logic, the **FastAPI service** exposes it over HTTP: -Think of it as the **control panel** for your pipelines: - -- **HTTP interface** – Wraps the core Python objects in a REST API so UIs (like Flowfile’s) or other systems can create, run, and inspect flows via standard web requests. -- **State management** – Keeps track of all active `FlowGraph` sessions. When the UI triggers a change, it’s really calling one of these endpoints, which updates the in-memory graph. +- **HTTP interface** – Wraps the core Python objects in a REST API so UIs (like Flowfile's) or other systems can create, run, and inspect flows via standard web requests. +- **State management** – Keeps track of all active `FlowGraph` sessions. When the UI triggers a change, it's really calling one of these endpoints, which updates the in-memory graph. - **Security** – Handles authentication and authorization so only the right users can access or modify flows. -- **Data previews** – When you view a node’s output in the UI, the API calls `.get_resulting_data()` on the corresponding `FlowNode` and returns a sample to the client. - -In short: **FastAPI turns the in-memory power of `flowfile-core` into a secure, interactive web service**, enabling rich, real-time applications to be built on top of your pipelines. +- **Data previews** – When you view a node's output in the UI, the API calls `.get_resulting_data()` on the corresponding `FlowNode` and returns a sample to the client. diff --git a/docs/for-developers/index.md b/docs/for-developers/index.md index b4d92535a..76cd4d375 100644 --- a/docs/for-developers/index.md +++ b/docs/for-developers/index.md @@ -1,6 +1,6 @@ -# **Flowfile: For Developers** +# Flowfile for developers -Welcome to the developer documentation for Flowfile. This is the home for anyone who wants to contribute to the platform or understand its internal architecture. +This section is for anyone contributing to Flowfile or working out how its internals fit together. After reading it you will know how the visual editor and the Python API construct the same objects, how the three services divide the work, and where to change code when you add a node, a kernel feature, or an AI surface. !!! note "Looking to use the Python API?" If you want to **use** Flowfile's Python API to build data pipelines, check out the [Python API User Guide](../users/python-api/index.md). This developer section focuses on Flowfile's internal architecture and design philosophy. @@ -10,13 +10,7 @@ Welcome to the developer documentation for Flowfile. This is the home for anyone Flowfile is built on an architecture where the Python API and the visual editor are two interfaces to the exact same underlying objects: the **`FlowGraph`** and its **`FlowNodes`**. -When you write `df.filter(...)`, you programmatically construct a `FlowNode` and attach it to the `FlowGraph`. When a user drags a "Filter" node in the UI, they create the identical object. This **dual interface philosophy** means your work is never locked into one paradigm. - -- **`FlowGraph`**: The central orchestrator that holds the complete definition of your pipeline—every node, setting, and connection. -- **`FlowNode`**: An individual, executable step in your pipeline that wraps settings and logic. -- **`FlowDataEngine`**: A smart wrapper around a Polars `LazyFrame` that carries the data and its schema between nodes. - -Learn more about this in our **[Dual Interface Philosophy](design-philosophy.md)** guide. +When you write `df.filter(...)`, you programmatically construct a `FlowNode` and attach it to the `FlowGraph`. When a user drags a "Filter" node in the UI, they create the identical object. The `FlowGraph` orchestrates the pipeline, each `FlowNode` wraps a step's settings and logic, and a `FlowDataEngine` carries the data and schema between nodes — see the [Dual Interface Philosophy](design-philosophy.md) guide. --- ## Getting Started with Development @@ -27,42 +21,24 @@ To contribute to Flowfile, you should be familiar with: - **Required Knowledge**: Python 3.10+, and a basic familiarity with Polars or Pandas. - **Helpful Knowledge**: Experience with Polars LazyFrames, Directed Acyclic Graphs (DAGs), and Pydantic. -### 2. Set Up Your Environment +### 2. Set up your environment -Before diving in, clone the repository and install the dependencies using Poetry: +Clone the repository and install the Python dependencies with Poetry: ```bash -# For development/contributing -git clone [https://github.com/edwardvaneechoud/Flowfile](https://github.com/edwardvaneechoud/Flowfile) +git clone https://github.com/edwardvaneechoud/Flowfile cd Flowfile poetry install ``` -### 3. See It in Action: A Quick Example -The following code builds a data pipeline using the Python API. This same pipeline can be generated visually in the UI. +`poetry install` is enough to run `flowfile_core`, use the Python API, and run the test suite. Building the desktop app additionally needs the PyInstaller group (`poetry install --with build`), and the visual editor needs the frontend toolchain (`cd flowfile_frontend && npm install`). See [CONTRIBUTING.md](https://github.com/edwardvaneechoud/Flowfile/blob/main/CONTRIBUTING.md) and the root `Makefile` for the full build. + +### 3. See it in action + +The following pipeline uses the Python API. Building the same steps visually produces the identical `FlowGraph`. ```python -import flowfile as ff -from flowfile import col - -# Create a FlowFrame from a local CSV -df = ff.read_csv("sales_data.csv", description="Load raw sales data") - -# Build a transformation pipeline with a familiar, chainable API -processed_sales = ( - df.filter(col("amount") > 100, description="Filter for significant sales") - .with_columns( - (col("quantity") * col("price")).alias("total_revenue") - ) - .group_by("region", description="Aggregate sales by region") - .agg( - col("total_revenue").sum() - ) -) - -# Get your results as a Polars DataFrame -results_df = processed_sales.collect() -print(results_df) +--8<-- "docs/examples/sales_pipeline.py:example" ``` ## Documentation Guides @@ -74,10 +50,12 @@ print(results_df) --- ## Contributing to Flowfile -We welcome contributions! Adding a new node requires changes across the stack: +Adding a built-in native node touches the whole stack: + +- **Backend**: define Pydantic settings models, implement the transformation logic on `FlowDataEngine`, and register the node on `FlowGraph`. +- **Frontend**: hand-write a Vue component for the node's settings form. Native nodes do not get an auto-generated form. -- **Backend**: You'll need to define Pydantic setting models, implement the transformation logic in the `FlowDataEngine`, and register the new node in the `FlowGraph`. -- **Frontend**: Currently, you must also manually create a Vue component for the node's configuration form in the visual editor. +Custom (user-defined) nodes are the exception: they get a schema-driven settings panel generated from the [Node Designer](../users/visual-editor/node-designer.md) API, so no Vue is needed. See [Creating Custom Nodes](creating-custom-nodes.md). -For a more detailed breakdown, please read the **[Contributing section in our Design Philosophy guide](design-philosophy.md#contributing)**. +For the full walkthrough, read the [Contributing section of the Design Philosophy guide](design-philosophy.md#contributing). diff --git a/docs/for-developers/kernel-architecture.md b/docs/for-developers/kernel-architecture.md index d0a5f610c..9ec439b1c 100644 --- a/docs/for-developers/kernel-architecture.md +++ b/docs/for-developers/kernel-architecture.md @@ -16,7 +16,7 @@ The kernel system consists of two main components: ```mermaid graph LR - Frontend["Frontend
(Vue/Electron)"] + Frontend["Frontend
(Vue/Tauri)"] Core["Core API
(port 63578)"] Manager["Kernel Manager
(Docker API)"] K1["Kernel Container
(port 9999)"] @@ -49,7 +49,7 @@ The `KernelManager` is a singleton that runs inside the Core service. It manages | Operation | What Happens | |-----------|-------------| | **Create** | Allocates a `KernelInfo` record, persists config to the database | -| **Start** | Verifies the `flowfile-kernel` Docker image exists, runs `docker.containers.run()`, polls `/health` until ready (120s timeout) | +| **Start** | Verifies the pinned kernel image (`flowfile-kernel-{base,ml,lite}:`) exists, runs `docker.containers.run()`, polls `/health` until ready (120s timeout) | | **Execute** | Serializes inputs to parquet, sends `ExecuteRequest` via HTTP, tracks kernel state | | **Stop** | Stops and removes the Docker container | | **Delete** | Stops if running, removes from in-memory registry and database | @@ -92,13 +92,13 @@ Flowfile does **not** define compatibility *ranges* between app and kernel versi ### Three independent version numbers -| Version | Where | Example | Role | -|---------|-------|---------|------| -| App / root | root `pyproject.toml` | `0.11.0` | The Flowfile release | -| Kernel **image** tag | `flowfile_core/flowfile_core/kernel/manager.py` (`_KERNEL_IMAGE_{BASE,ML,LITE}_DEFAULT`) | `0.4.0` | The image the app pulls / runs | -| Kernel **runtime API** | `kernel_runtime/__init__.py` (`__version__`) | `0.3.0` | The kernel's HTTP API version, reported by `/health` | +| Version | Where | Role | +|---------|-------|------| +| App / root | root `pyproject.toml` `version` | The Flowfile release | +| Kernel **image** tag | `flowfile_core/flowfile_core/kernel/manager.py` (`_KERNEL_IMAGE_{BASE,ML,LITE}_DEFAULT`, `0.4.0` as of 2026-07) | The image the app pulls / runs | +| Kernel **runtime API** | `kernel_runtime/__init__.py` (`__version__`) | The kernel's HTTP API version, reported by `/health` | -These evolve **independently** — bumping the app does not require bumping the kernel image, and vice versa. +Read each value from its source rather than assuming a number — the three are decoupled. They evolve **independently**: bumping the app does not require bumping the kernel image, and vice versa. ### How the pin works @@ -106,7 +106,7 @@ These evolve **independently** — bumping the app does not require bumping the - Core reads the running kernel's runtime version from `/health` into `KernelInfo.kernel_version` **for display only** (the "Kernel runtime" line in the Kernel Manager). There is no min/max gate and nothing that rejects or warns about an "out-of-range" kernel. - The only **hard** coupling is **polars**: `kernel_runtime` pins a polars (and the `polars-ds` plugin) compatible with the app's `polars >=1.8.2,<1.40`. These must be bumped together, but that compatibility is guaranteed at *image-build time* via the pinned tag — not by a runtime check. -So the practical contract is "use the pinned tag." An older kernel image is **not blocked** — it simply may lack fixes or features the app expects (for example, JSON-format global-artifact deserialization was only fixed in image `0.3.1`). That gap is surfaced as a non-blocking **"Update available"** hint in the Kernel Manager (per flavour) and the kernel details modal (per kernel), rather than a hard version gate. +So the practical contract is "use the pinned tag." An older kernel image is **not blocked** — it simply may lack fixes or features the app expects. That gap is surfaced as a non-blocking **"Update available"** hint in the Kernel Manager (per flavour) and the kernel details modal (per kernel), rather than a hard version gate. ### Shipping a kernel change @@ -265,13 +265,14 @@ All endpoints require JWT authentication and enforce user ownership: | `DELETE /kernels/{id}` | Delete kernel | | `POST /kernels/{id}/execute` | Execute code | | `POST /kernels/{id}/execute_cell` | Execute in interactive mode | -| `POST /kernels/{id}/interrupt` | Cancel execution | | `GET /kernels/{id}/artifacts` | List artifacts | | `POST /kernels/{id}/clear` | Clear all artifacts | | `GET /kernels/{id}/display_outputs` | Get display outputs | | `GET /kernels/{id}/memory` | Get memory usage | | `GET /kernels/docker-status` | Check Docker availability | +There is no core-side interrupt route. Cancellation is a kernel-runtime concern: the runtime container exposes `POST /interrupt` on its own port (see [Thread-based Execution](#thread-based-execution)), which Core reaches directly, not through a `/kernels/{id}/interrupt` proxy. + ### Database Persistence **Location:** `flowfile_core/flowfile_core/kernel/persistence.py` @@ -325,14 +326,18 @@ Artifact serialization uses pickle/cloudpickle. This is acceptable because: ### Building the Kernel Image +The resolver looks for `flowfile-kernel-{base,ml,lite}:local` when the pinned registry tag isn't present locally, so tag your local build to match the flavour you want to run: + ```bash -# Via docker compose -docker compose build flowfile-kernel +# Via docker compose (builds the base flavour as flowfile-kernel-base:local) +docker compose --profile kernel build flowfile-kernel -# Or directly -docker build -t flowfile-kernel -f kernel_runtime/Dockerfile kernel_runtime/ +# Or directly — tag the flavour explicitly +docker build -t flowfile-kernel-base:local -f kernel_runtime/Dockerfile kernel_runtime/ ``` +A bare `docker build -t flowfile-kernel …` produces a tag the app will not pick up. + The image is based on `python:3.12-slim` and includes: - **Data stack:** Polars, PyArrow, NumPy @@ -342,21 +347,21 @@ The image is based on `python:3.12-slim` and includes: ### Docker Compose Integration -The `docker-compose.yml` includes a build-only service for the kernel image: +The `docker-compose.yml` includes a build-only service per kernel flavour (`flowfile-kernel`, `flowfile-kernel-ml`, `flowfile-kernel-lite`). The base flavour: ```yaml flowfile-kernel: build: context: kernel_runtime dockerfile: Dockerfile - image: flowfile-kernel + image: flowfile-kernel-base:local entrypoint: ["true"] restart: "no" profiles: - kernel ``` -This service is not started by `docker compose up` — it only builds the image. The Core service creates kernel containers dynamically via the Docker API. +These services are not started by `docker compose up` — the `kernel` profile only builds the images (the `entrypoint: ["true"]` exits immediately). The Core service creates kernel containers dynamically via the Docker API, using the `flowfile-kernel-{base,ml,lite}:local` tags. !!! warning "Docker Socket" The Core service requires access to the Docker socket (`/var/run/docker.sock`) to manage kernel containers. In production, consider using a Docker socket proxy (e.g., `tecnativa/docker-socket-proxy`) to restrict API access. @@ -387,8 +392,8 @@ python -m pytest kernel_runtime/tests -v ### Integration Tests (Docker Required) ```bash -# Build the kernel image first -docker build -t flowfile-kernel -f kernel_runtime/Dockerfile kernel_runtime/ +# Build the base kernel image first (tag must match the resolver's flavour tag) +docker build -t flowfile-kernel-base:local -f kernel_runtime/Dockerfile kernel_runtime/ # Run kernel integration tests poetry run pytest flowfile_core/tests -m kernel -v diff --git a/docs/for-developers/python-api-reference.md b/docs/for-developers/python-api-reference.md index c171bf120..af4c06e35 100644 --- a/docs/for-developers/python-api-reference.md +++ b/docs/for-developers/python-api-reference.md @@ -53,10 +53,8 @@ The `FlowDataEngine` is the primary engine of the library, providing a rich API unwrap_annotated: true show_symbol_type_toc: true -#### FlowfileColumn - -### `FlowfileColumn` -The `FlowfileColumn` is a data class that holds the schema and rich metadata for a single column managed by the `FlowDataEngine`. +### FlowfileColumn +The `FlowfileColumn` holds the schema and metadata for a single column managed by the `FlowDataEngine`. ::: flowfile_core.flowfile.flow_data_engine.flow_file_column.main.FlowfileColumn options: diff --git a/docs/for-developers/visualizations.md b/docs/for-developers/visualizations.md index eb19d1854..09a560325 100644 --- a/docs/for-developers/visualizations.md +++ b/docs/for-developers/visualizations.md @@ -40,7 +40,7 @@ spawned child imports Polars, holds the LazyFrame, runs polars-gw Two boundaries do real work here: -- **Core never touches a `LazyFrame`.** It resolves *where* a visualization reads from — a Delta table, a SQL string, a Python flow — and ships a small descriptor to the worker. That keeps core small and responsive. +- **Core never touches a `LazyFrame`.** `VisualizationService` resolves *where* a visualization reads from — a Delta table, a SQL string, a Python flow — and ships a small descriptor to the worker. That keeps core small and responsive. - **The worker's FastAPI process doesn't import Polars either.** It dispatches. The actual heavy lifting — Polars, polars-gw, dataset memory — lives in spawned child processes that the parent only knows about through queues. The second boundary is the unusual one. Polars is memory-heavy and aggressively multithreaded; mixing it into the request-serving process means one slow query starves every other chart. Pushing it out to a child fixes that. @@ -58,7 +58,7 @@ Two properties fall out of how the pool is keyed: - **Same source, same child.** Successive queries from one visualization reuse the warm `LazyFrame`. A per-handle lock guarantees that two browser tabs editing the same visualization don't trip over each other's responses. - **Different sources, different children, in parallel.** Two users on two tables — or one user with two visualization tabs open — run in separate processes that don't block each other. There is no global lock between sessions. -`flowfile_worker/viz_sessions.py` is the registry. `flowfile_worker/viz_session_worker.py` is the entry point that runs inside each spawned process — and the **only** place anywhere in the worker that imports Polars and polars-gw. +`flowfile_worker/flowfile_worker/viz_sessions.py` is the registry. `flowfile_worker/flowfile_worker/viz_session_worker.py` is the entry point that runs inside each spawned process — and the **only** place in the worker that imports **polars-gw** and holds the visualization dataset in memory. (Plain `polars` is imported by other worker modules such as `funcs.py` and `catalog_reader.py`; it's the polars-gw session data that is confined to the spawned child.) --- @@ -74,13 +74,13 @@ Each child holds a real, live dataset, so left alone they accumulate and the wor Every one of these paths runs the child through the same shutdown sequence: send a graceful stop, wait briefly, terminate, kill if it's still alive, drop references. Nothing is kept around on the assumption it might be needed later. -The actual numeric thresholds live as constants at the top of `viz_sessions.py` and are tunable knobs, not load-bearing magic. +The actual numeric thresholds live as constants near the top of `flowfile_worker/flowfile_worker/viz_sessions.py` (`IDLE_TTL_SECONDS`, `MAX_SESSIONS`, `REAP_INTERVAL_SECONDS`, `MAX_REQUESTS_PER_CHILD`, `MAX_CHILD_LIFETIME_SECONDS`) and are tunable knobs, not load-bearing magic. --- ## What a "visualization" looks like at rest -A visualization is a row in `catalog_visualizations`: a name, a Graphic Walker chart spec (JSON, possibly multi-tab), a pointer to either a catalog table or an inline SQL query, and a thumbnail PNG captured client-side at save time. Schemas live in `flowfile_core/schemas/catalog_schema.py`. +A visualization is a row in `catalog_visualizations`: a name, a Graphic Walker chart spec (JSON, possibly multi-tab), a pointer to either a catalog table or an inline SQL query, and a thumbnail PNG captured client-side at save time. Schemas live in `flowfile_core/flowfile_core/schemas/catalog_schema.py`. A visualization never embeds the data — it embeds *how to find the data*. That's why deleting the underlying table doesn't cascade-delete the visualization (it becomes orphaned and the library labels it as such), why moving a visualization between namespaces is free, and why a SQL visualization can reference whatever tables exist in the catalog at query time. @@ -91,7 +91,7 @@ A visualization never embeds the data — it embeds *how to find the data*. That To support a new source — a remote Postgres, a parquet on S3, anything — the touchpoints are: 1. Extend the source descriptor in core, and the matching worker model, so the new kind is expressible. -2. Teach `CatalogService` to translate it into a worker descriptor and emit a deterministic `session_key`. The session key is what lets the pool reuse children. +2. Teach `VisualizationService` (`flowfile_core/flowfile_core/catalog/services/visualizations.py`) to translate it into a worker descriptor and emit a deterministic `session_key` (`_session_key_for_table` and the `session_key` fields). The session key is what lets the pool reuse children. 3. Teach `viz_session_worker._build_viz_loader_in_child` to open the new kind as a Polars `LazyFrame`. Path validation and source resolution live in core on purpose. The worker child should treat its inputs as already-validated names plus an opaque chart payload — defence-in-depth is core's job. diff --git a/docs/index.html b/docs/index.html index 04dce162f..353b16fc2 100644 --- a/docs/index.html +++ b/docs/index.html @@ -5,11 +5,9 @@ Flowfile - Visual ETL Tool - - - +

Flowfile

- An open-source data platform with a visual pipeline builder, AI assistant, data catalog, Delta Lake storage, - scheduling, Kafka ingestion, sandboxed Python execution, and a Polars-compatible API — all in a single pip install. + An open-source data platform with a visual pipeline builder, AI assistant, data catalog with Delta Lake storage, + scheduling, Kafka ingestion, sandboxed Python execution, and a Polars-compatible API — installed with a single pip install.

@@ -741,18 +535,15 @@

Flowfile

 + + Try the Live Demo + View on GitHub - - User Guides - - - -
@@ -762,12 +553,22 @@

Flowfile

Flowfile AI Assistant in action
- +
-

Why Flowfile?

-

A complete data platform — visual pipeline builder, catalog, scheduling, and more — powered by Polars

+

What's inside

+

A complete data platform built on Polars — every part works with every other part

+ +
🎨
+

Visual Pipeline Builder

+

+ 40+ nodes for joins, filters, aggregations, fuzzy matching, pivots, and more — + with a data preview at every step. +

+ → + +
✨

AI Assistant

@@ -779,28 +580,29 @@

AI Assistant

→ - -
🎨
-

Visual Pipeline Builder

+ +
📚
+

Data Catalog & Delta Lake

- 30+ nodes for joins, filters, aggregations, fuzzy matching, pivots, and more. - See your data flow in real-time with instant previews at each step. + Tables stored as Delta Lake with version history and time travel. Namespaces, lineage, + run history, and virtual tables in one place.

→ - -
📚
-

Data Catalog & Delta Lake

+ +
📊
+

Analyze & Explore

- Every table stored as Delta Lake with version history, time travel, and merge/upsert support. Track lineage, runs, and artifacts in one place. + Query any catalog table in the SQL editor, explore results in Graphic Walker + visualizations, and save the charts next to the data.

→ - +
⏰
-

Scheduling & Triggers

+

Scheduling & Triggers

Run flows on intervals or trigger them when catalog tables update. Built into the catalog — not a separate orchestration tool. @@ -810,9 +612,10 @@

Scheduling & Triggers
📡
-

Kafka & Cloud Storage

+

Kafka, Databases & Cloud Storage

- Ingest from Kafka/Redpanda as a canvas node. Read and write to S3, Azure Data Lake, and GCS. Connect to PostgreSQL, MySQL, and more. + Ingest from Kafka/Redpanda as a canvas node. Read and write S3, Azure Data Lake, and GCS. + Connect PostgreSQL, MySQL, and SQLite.

→ @@ -821,116 +624,139 @@

Kafka & Cloud Storage

🐍

Sandboxed Python

- Run arbitrary Python code in isolated Docker containers. Use any library — matplotlib, scikit-learn, or your own — output flows back into the pipeline. + Run arbitrary Python in isolated Docker containers. Use any library — + the output flows back into the pipeline.

→
🔄
-

Code Generation & Python API

+

Code Generation & Python API

- Export visual flows as Python/Polars scripts. Or build pipelines programmatically with a Polars-compatible API and visualize them on the canvas. + Export visual flows as standalone Python/Polars scripts, or build pipelines in code + with a Polars-compatible API and open them on the canvas.

→
- -
+ +
-

Build Pipelines Your Way

+

One pipeline, two ways

- Use the visual designer for exploration or write code for automation - seamlessly switch between both + The same sales pipeline — cleaned, filtered, and aggregated — on the canvas and in Python. + Both produce the same flow; switch whenever you like.

+ +
-
-

📊 Visual Approach

-
    -
  • Drag and drop nodes to build workflows
    • -
  • Configure transformations with intuitive forms
    • -
  • Preview data at each step instantly
    • -
  • Export your pipeline as Python code
    • -
  • Perfect for exploration and prototyping
    • -
+

On the canvas

+
    +
  1. Read — load supermarket_sales.csv
    2. +
  2. Drop Duplicates — remove repeated invoice rows
    3. +
  3. Filter — keep bulk orders: [quantity] > 7
    4. +
  4. Group By — per city, the sum and median of gross income
    5. +
-
-

🐍 Code Approach

+

In Python

+ 
import flowfile as ff
 
-df = ff.read_csv("data.csv")
-result = df.filter(
-    ff.col("amount") > 1000
-).group_by("region").agg(
-    ff.col("amount").sum()
+df = ff.read_csv("data/templates/supermarket_sales.csv")
+
+result = (
+    df.unique()
+    .filter(ff.col("quantity") > 7)
+    .group_by("city")
+    .agg(
+        ff.col("gross_income").sum().alias("total_income"),
+        ff.col("gross_income").median().alias("median_income"),
+    )
 )
 
-# Visualize your pipeline
-ff.open_graph_in_editor(result.flow_graph)
+print(result.collect())
- - Python API Documentation + + Build it step by step - - Visual Editor Guide + + Download the flow (.yaml) -
-
- - Read Developer Docs - + + Open this flow in your browser
+

+ Also available in the app: Create → From template → "Sales pipeline". The flow and the code above are + validated by the test suite on every commit. +

- -
-

See It For Yourself

-

- Start rediscovering how we bridge the gap between business users and technical users. -

+ +
+

Where to start

+

Pick the guide written for the way you work

-
- -
5 min
-
to first pipeline
+
+ +

Build flows visually

+

The canvas, nodes, and formulas — no code required.

+ + +

Coming from Excel

+

VLOOKUP, pivot tables, and IF-formulas, translated to flows.

+ + +

Connect your data

+

Databases, S3/ADLS/GCS, Kafka, REST APIs, Google Analytics.

 - -
100%
-
Familiar API
+ +

Analyze your data

+

Catalog tables, SQL editor, visualizations, schedules.

 - -
0
-
vendor lock-in
+ +

Write Python

+

A Polars-compatible API that builds visual flows as a side effect.

+ + +

Deploy & operate

+

Desktop, pip, or Docker — plus users, groups, and sharing.
+
-
+ +
+

Free and open source

+

+ One install gives you the visual editor, the Python API, the catalog, and all services. +

+ 
pip install flowfile
+
- Try Interactive Tutorial + Quickstart - - Browse Examples + + Guides by audience
-

- Free, open-source and customizable + MIT licensed, developed on GitHub

- - \ No newline at end of file + diff --git a/docs/quickstart.md b/docs/quickstart.md index b75ff9b7b..9cb4b2165 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -1,644 +1,212 @@ -# Quick Start Guide +# Quickstart -
- Flowfile Logo -

Get Started with Flowfile in 5 Minutes

-
- -## Installation +Install Flowfile, build a real pipeline on the canvas, and see the same pipeline as Python code — in about ten minutes. The example cleans a sales dataset: remove duplicate rows, keep bulk orders, and summarize income per city. Everything on this page is backed by files in the repository and validated by the test suite, so what you see is what will run. -
-

Recommended quickstart: Install from PyPI

- 
pip install flowfile
-

This installs everything you need - the Python API, visual editor, and all services.

-
+## Install -!!! tip "Want to try Flowfile without installing anything?" - [**Flowfile Lite**](users/deployment/lite.md) runs the visual editor entirely in your browser (Polars via WebAssembly) — no install, no signup. Try it at [demo.flowfile.org](https://demo.flowfile.org). It's a lightweight subset (18 nodes, no backend, databases, scheduler, or AI); install the full package below for everything else. +=== ":material-language-python: pip (recommended)" -### Alternative Installation Methods + Installs the visual editor, the Python API, and all services: -
-Desktop Application (Pre-built Installer) + ```bash + pip install flowfile + flowfile run ui + ``` -Download the latest installer for your platform: -- **Windows**: [Flowfile-Setup.exe](https://github.com/Edwardvaneechoud/Flowfile/releases) -- **macOS**: [Flowfile.dmg](https://github.com/Edwardvaneechoud/Flowfile/releases) + Your browser opens the Flowfile designer. If it doesn't, go to [http://127.0.0.1:63578/ui#/main/designer](http://127.0.0.1:63578/ui#/main/designer) manually. -> **Note**: You may see security warnings since the installer isn't signed. On Windows, click "More info" → "Run anyway". On macOS, right-click → "Open" → confirm. +=== ":material-monitor: Desktop app" -
+ Download the signed installer for macOS, Windows, or Linux from [GitHub Releases](https://github.com/edwardvaneechoud/Flowfile/releases) and run it. -
-Development Setup (From Source) +=== ":material-web: Browser, no install" -```bash -# Clone repository -git clone https://github.com/edwardvaneechoud/Flowfile.git -cd Flowfile + [Flowfile Lite](users/deployment/lite.md) runs the visual editor entirely in your browser at [demo.flowfile.org](https://demo.flowfile.org) — a subset of the full product (no backend services, databases, scheduler, or AI), good for a first look and small files. -# Install with Poetry -poetry install +=== ":material-source-branch: From source" -# Start services -poetry run flowfile_worker # Terminal 1 (port 63579) -poetry run flowfile_core # Terminal 2 (port 63578) + Clone the repo and see [For Developers](for-developers/index.md) for the Poetry + npm setup. -# Start frontend -cd flowfile_frontend -npm install -npm run dev:web # Terminal 3 (port 8080) -``` +## Choose your path -
- ---- - -## Choose Your Path - -
- -
-

Non-Technical Users

-

Perfect for: Analysts, business users, Excel power users

-

No coding required!

- -
    -
  • ✅ Drag and drop interface
    • -
  • ✅ Visual data preview
    • -
  • ✅ Export to Excel/CSV
    • -
  • ✅ Built-in transformations
    • -
- - Start Visual Tutorial → -
- -
-

Technical Users

-

Perfect for: Developers, data scientists, engineers

-

Full programmatic control!

- -
    -
  • ✅ Polars-compatible API
    • -
  • ✅ Cloud storage integration
    • -
  • ✅ Version control friendly
    • -
  • ✅ Complex dynamic logic
    • -
- - Start Python Tutorial → -
+Both tracks build the same pipeline — pick the one that matches how you work, or do both: +
+ +Build it visually +Drag nodes onto the canvas, preview the data at every step, publish the result to the catalog. No code. + + +Write it in Python +A Polars-style API that builds the identical flow — and opens it on the canvas whenever you want. +
--- -## Quick Start for Non-Technical Users {#non-technical-quickstart} +## Your first flow, visually -
-Goal: Clean and analyze sales data without writing any code -
+You'll build this pipeline: **read → drop duplicates → filter → group by**. -### Step 1: Start Flowfile, and create a Flow + -Open your terminal (Command Prompt on Windows, Terminal on Mac) and type: +!!! tip "Skip the typing" + The finished flow ships with Flowfile: **Create → From template → "Sales pipeline: clean, filter, aggregate"**. The sample data is provisioned automatically. You can also [download the flow](assets/flows/sales_pipeline.yaml) as a `.yaml` file. The steps below build the same thing by hand so you learn the moves. -```bash -flowfile run ui -``` -Your browser should automatically open to the Flowfile UI. -!!! warning "If the browser does not open automatically" - If the browser does not open automatically, you can manually navigate to [http://127.0.0.1:63578/ui#/main/designer](http://127.0.0.1:63578/ui#/main/designer) in your web browser. +### 1. Create a flow -
+1. Run `flowfile run ui` and click **Create** in the toolbar. +2. Name the flow (for example `sales_analysis`) and create it. +3. Open **Settings** (top right) and check the execution mode is **Development** — that gives you data previews at every node while you build. -**Creating your First Flow:** +
+See it: the empty flow -1. **Click "Create"** to create a new data pipeline -2. **Click "Create New File Here"** -3. **Name your flow** (e.g., "Sales Data Analysis") -4. **Click on "Settings"** in the top right to configure your flow -5. **Set the Execution mode** to "Development" + +![The empty flow after creation](assets/images/quickstart/start_page.png) -
- -Your should see now an empty flow: -![Flowfile Interface](assets/images/quickstart/start_page.png)New clean flow interface +
+### 2. Read the data -### Step 2: Load Your Data +The walkthrough uses a committed sample: [`supermarket_sales.csv`](https://raw.githubusercontent.com/edwardvaneechoud/Flowfile/main/data/templates/supermarket_sales.csv) — 1,030 sales rows with `city`, `quantity`, and `gross_income` columns (and 30 deliberately duplicated rows to clean up). Any CSV or Excel file of your own works too. -
+1. Drag **Read data** from the Input section onto the canvas. +2. Click the node, then **Browse** to your file. +3. Click **Run** (top toolbar), then click the node to preview the rows in the bottom panel. -**Loading a CSV or Excel file:** +
+See it: the data preview after reading -1. **Find the "Read Data" node** in the left panel under "Input" -2. **Drag it** onto the canvas (center area) -3. **Click the node** to open settings on the right -4. **Click "Browse"** and select your file -5. **Configure options** (if needed): - - For CSV: Check "Has Headers" if your file has column names - - For Excel: Select the sheet name -6. **Click "Run"** (top toolbar) to load the data -7. **Click the node** to preview your data in the bottom panel + +![Preview after reading the CSV](assets/images/quickstart/read_csv.png) -
+ -### Step 3: Clean Your Data +### 3. Drop duplicates -Let's remove duplicate records and filter for high-value transactions: +1. Drag **Drop duplicates** from the Transform section and connect **Read data** to it. +2. Leave the column selection empty to compare whole rows. +3. Run — the sample data goes from 1,030 rows to 1,000. -
+
+See it: after deduplication -
-

Remove Duplicates

-
    -
  1. Drag "Drop Duplicates" node from Transform section
    2. -
  2. Connect it to your Read Data node
    3. -
  3. Select columns to check for duplicates
    4. -
  4. Click Run
    5. -
-
+ +![After Drop duplicates](assets/images/quickstart/after_drop_duplicates.png) -
-

Filter Data

-
    -
  1. Drag "Filter Data" node from Transform section
    2. -
  2. Connect it to Drop Duplicates node
    3. -
  3. Enter formula: [Quantity] > 7
    4. -
  4. Click Run
    5. -
-
+
-
+### 4. Filter to bulk orders -### Step 4: Analyze Your Data +1. Drag **Filter data** onto the canvas and connect it. +2. Switch the filter to advanced mode and enter the formula: -**Create a summary by city:** + ```text + [quantity] > 7 + ``` -1. **Add a Group By node** from the Aggregate section -2. **Connect it** to your Filter node -3. **Configure the aggregation**: - - Group by: `city` - - Aggregations: - - `gross income` → Sum → Name it `total_sales` - - `gross income` → Average → Name it `avg_sale` - - `gross income` → Count → Name it `number_of_sales` -4. **Click Run** to see your summary +3. Run — 314 rows remain. The `[column]` syntax is Flowfile's [formula language](users/formulas/index.md); if you can write an Excel formula, you already know it.
-Data after group by +See it: after the filter -![Group By Configuration](assets/images/quickstart/after_group_by.png) + +![After the quantity filter](assets/images/quickstart/after_filter.png)
-### Step 5: Save Your Results +### 5. Group by city -
+1. Drag **Group by** from the Aggregate section and connect it. +2. Configure: group on `city`; aggregate `gross_income` twice — **Sum** named `total_income`, **Median** named `median_income`. +3. Run, and click the node — the preview shows one row per city with its total and median income. -**Export your cleaned data:** - -1. **Add a "Write Data" node** from Output section -2. **Connect it** to your final transformation -3. **Choose format**: - - **Excel**: Best for sharing with colleagues - - **CSV**: Best for Excel/Google Sheets - - **Parquet**: Best for large datasets -4. **Set file path** (e.g., `cleaned_sales.xlsx`) -5. **Click Run** to save +
+See it: the grouped result and the complete flow -
+ +![Data after Group by](assets/images/quickstart/after_group_by.png) -
+![The complete flow](assets/images/quickstart/result.png) -### Here's what your complete flow should look like: + -![Group By Configuration](assets/images/quickstart/result.png) +Save the flow (File → Save) — flows are plain `.yaml` files you can version, share, and run headlessly with `flowfile run flow `. -
+### 6. Put the result to work -### Congratulations! - -You've just built your first data pipeline! You can: -- **Save this flow** using File → Save (creates a `.flowfile`) -- **Share it** with colleagues who can run it without any setup -- **Schedule it** to run automatically (coming soon) -- **Export as Python code** if you want to see what's happening behind the scenes - -
-

Pro Tips for Non-Technical Users:

-
    -
  • Use descriptions: Right-click nodes and add descriptions to document your work
    • -
  • Preview often: Click nodes after running to see data at each step
    • -
  • Start small: Test with a sample of your data first
    • -
  • Save versions: Save different versions of your flow as you build
    • -
-
+Instead of exporting a file, publish the result into Flowfile's catalog and analyze it there: -### Next Steps +1. Drag **Write to Catalog** from the Output section, connect it to **Group by**, pick the default namespace, and name the table (for example `sales_by_city`). Run once. +2. Open the **Catalog** tab: your table is there as a Delta table with schema, preview, and history. +3. Query it in the [SQL editor](users/visual-editor/catalog/sql-editor.md), or open it in a [visualization](users/visual-editor/catalog/visualizations.md) and chart income per city. +4. When the numbers should stay fresh, [schedule the flow](users/visual-editor/catalog/schedules.md). -
- Complete Visual Guide - Learn All Nodes - Connect to Databases -
+That loop — build, publish, analyze, schedule — is the core of working in Flowfile. A `Write data` node exports to Excel/CSV/Parquet instead whenever a file is what you need. --- -## Quick Start for Technical Users {#technical-quickstart} - -
-Goal: Build a production-ready ETL pipeline with cloud integration -
- -### Step 1: Install and Import - -```bash -pip install flowfile -``` - -```python -import flowfile as ff -from flowfile import col, when, lit -import polars as pl # Flowfile returns Polars DataFrames -``` - -### Step 2: Build a Real-World ETL Pipeline - -Let's build a production pipeline that reads from S3, transforms data, and writes results: - -```python -# Configure S3 connection (one-time setup) -from pydantic import SecretStr - -import flowfile as ff - -ff.create_cloud_storage_connection_if_not_exists( - ff.FullCloudStorageConnection( - connection_name="production-data", - storage_type="s3", - auth_method="access_key", - aws_region="us-east-1", - aws_access_key_id="AKIAIOSFODNN7EXAMPLE", - aws_secret_access_key=SecretStr("wJalrXUtnFEMI/K7MDENG") - ) -) -``` - -### Step 3: Extract and Transform - -
- -```python -# Build the pipeline (lazy evaluation - no data loaded yet!) -import flowfile as ff -pipeline = ( - # Extract: Read partitioned parquet files from S3 - ff.scan_parquet_from_cloud_storage( - "s3://data-lake/sales/year=2024/month=*", - connection_name="production-data", - description="Load Q1-Q4 2024 sales data" - ) - - # Transform: Clean and enrich - .filter( - (ff.col("status") == "completed") & - (ff.col("amount") > 0), - description="Keep only valid completed transactions" - ) - - # Add calculated fields - .with_columns([ - # Business logic - (ff.col("amount") * ff.col("quantity")).alias("line_total"), - (ff.col("amount") * ff.col("quantity") * 0.1).alias("tax"), - - # Date features for analytics - ff.col("order_date").dt.quarter().alias("quarter"), - ff.col("order_date").dt.day_of_week().alias("day_of_week"), - - # Customer segmentation - ff.when(ff.col("customer_lifetime_value") > 10000) - .then(ff.lit("VIP")) - .when(ff.col("customer_lifetime_value") > 1000) - .then(ff.lit("Regular")) - .otherwise(ff.lit("New")) - .alias("customer_segment"), - - # Region mapping - ff.when(ff.col("state").is_in(["CA", "OR", "WA"])) - .then(ff.lit("West")) - .when(ff.col("state").is_in(["NY", "NJ", "PA"])) - .then(ff.lit("Northeast")) - .when(ff.col("state").is_in(["TX", "FL", "GA"])) - .then(ff.lit("South")) - .otherwise(ff.lit("Midwest")) - .alias("region") - ], description="Add business metrics and segments") - - # Complex aggregation - .group_by(["region", "quarter", "customer_segment"]) - .agg([ - # Revenue metrics - ff.col("line_total").sum().alias("total_revenue"), - ff.col("tax").sum().alias("total_tax"), - - # Order metrics - ff.col("order_id").n_unique().alias("unique_orders"), - ff.col("customer_id").n_unique().alias("unique_customers"), - - # Performance metrics - ff.col("line_total").mean().round(2).alias("avg_order_value"), - ff.col("quantity").sum().alias("units_sold"), - - # Statistical metrics - ff.col("line_total").std().round(2).alias("revenue_std"), - ff.col("line_total").quantile(0.5).alias("median_order_value") - ]) - - # Final cleanup - .sort(["region", "quarter", "total_revenue"], descending=[False, False, True]) - .filter(ff.col("total_revenue") > 1000) # Remove noise -) - -# Check the execution plan (no data processed yet!) -print(pipeline.explain()) # Shows optimized Polars query plan -``` - -
+## The same pipeline in Python -### Step 4: Load and Monitor +The `flowfile` package exposes a Polars-style API that builds the identical flow: ```python -# Option 1: Write to cloud storage -pipeline.write_parquet_to_cloud_storage( - "s3://data-warehouse/aggregated/sales_summary_2024.parquet", - connection_name="production-data", - compression="snappy", - description="Save aggregated results for BI tools" -) - -# Option 2: Write to Delta Lake for versioning -pipeline.write_delta( - "s3://data-warehouse/delta/sales_summary", - connection_name="production-data", - write_mode="append", # or "overwrite" - description="Append to Delta table" -) - -# Option 3: Collect for analysis -df_result = pipeline.collect() # NOW it executes everything! -print(f"Processed {len(df_result):,} aggregated records") -print(df_result.head()) +--8<-- "docs/examples/sales_pipeline.py:example" ``` -### Step 5: Advanced Features - -
-

Visualize Your Pipeline

+`result.collect()` returns the same five-city table as step 5. Nothing executes until `.collect()` — every method call just adds a node to a flow graph, which means you can also look at it on the canvas: ```python -# Open in visual editor -ff.open_graph_in_editor(pipeline.flow_graph) - -# This shows your entire pipeline -# as a visual flow diagram! +ff.open_graph_in_editor(result.flow_graph) ```
- Visual overview of pipeline +See it: the code-built pipeline on the canvas -![Flow visualized](assets/images/quickstart/python_example.png) + +![The pipeline opened in the visual editor](assets/images/quickstart/python_example.png)
-
- -
-

Export as Pure Python

- -```python -# Generate standalone code -code = pipeline.flow_graph.generate_code() - -# Deploy without Flowfile dependency! -# Uses only Polars -``` -
- -### Step 6: Production Patterns - -**Pattern 1: Data Quality Checks** - -
- -```python -from datetime import datetime -import flowfile as ff - -def data_quality_pipeline(df: ff.FlowFrame) -> ff.FlowFrame: - """Reusable data quality check pipeline""" - - # Record initial count - initial_count = df.select(ff.col("*").count().alias("count")) - - # Apply quality filters - clean_df = ( - df - # Remove nulls in critical fields - .drop_nulls(subset=["order_id", "customer_id", "amount"]) - - # Validate data ranges - .filter( - (ff.col("amount").is_between(0, 1000000)) & - (ff.col("quantity") > 0) & - (ff.col("order_date") <= datetime.now()) - ) - - # Remove duplicates - .unique(subset=["order_id"], keep="first") - ) - - # Log quality metrics - final_count = clean_df.select(ff.col("*").count().alias("count")) - print(f"Initial count: {initial_count.collect()[0]['count']}") - print(f"Final count after quality checks: {final_count.collect()[0]['count']}") - return clean_df -``` - -
- -**Pattern 2: Incremental Processing** - -
- -```python -# Read only new data since last run -from datetime import datetime -import flowfile as ff -last_processed = datetime(2024, 10, 1) - -incremental_pipeline = ( - ff.scan_parquet_from_cloud_storage( - "s3://data-lake/events/", - connection_name="production-data" - ) - .filter(ff.col("event_timestamp") > last_processed) - .group_by(ff.col("event_timestamp").dt.date().alias("date")) - .agg([ - ff.col("event_id").count().alias("event_count"), - ff.col("user_id").n_unique().alias("unique_users") - ]) -) - -# Process and append to existing data -incremental_pipeline.write_delta( - "s3://data-warehouse/delta/daily_metrics", - connection_name="production-data", - write_mode="append" -) -``` - -
- -**Pattern 3: Multi-Source Join** - -
- -```python -# Combine data from multiple sources -import flowfile as ff - -customers = ff.scan_parquet_from_cloud_storage( - "s3://data-lake/customers/", - connection_name="production-data" -) - -orders = ff.scan_csv_from_cloud_storage( - "s3://raw-data/orders/", - connection_name="production-data", - delimiter="|", - has_header=True -) - -products = ff.read_parquet("local_products.parquet") - -# Complex multi-join pipeline -enriched_orders = ( - orders - .join(customers, on="customer_id", how="left") - .join(products, on="product_id", how="left") - .with_columns([ - # Handle missing values from left joins - ff.col("customer_segment").fill_null("Unknown"), - ff.col("product_category").fill_null("Other"), - - # Calculate metrics - (ff.col("unit_price") * ff.col("quantity") * - (ff.lit(1) - ff.col("discount_rate").fill_null(0))).alias("net_revenue") - ]) -) - -# Materialize results -results = enriched_orders.collect() -``` - -
- -### Next Steps for Technical Users - -
- Complete API Reference - Architecture Deep Dive - Core Internals - Polars Documentation -
+This snippet is included from a repository file that runs in CI on every change — it cannot drift from the real API. To go deeper — expressions, joins, databases, cloud storage — continue with the [Python API quickstart](users/python-api/quickstart.md). --- -## 🌟 Why Flowfile? - -
- -
-

⚡ Performance

-

Built on Polars - Uses the speed of Polars

-
- -
-

🔄 Dual Interface

-

Same pipeline works in both visual and code. Switch anytime, no lock-in.

-
+## Troubleshooting -
-

📦 Export to Production

-

Generate pure Python/Polars code. Deploy anywhere without Flowfile.

-
+
+Port 63578 is already in use -
-

☁️ Cloud Support

-

Direct S3/cloud storage support, no need for expensive clusters to analyse your data

-
+The web UI is fixed to port 63578 — it cannot be moved to another port. Free it instead: -
+```bash +lsof -i :63578 # macOS/Linux +netstat -ano | findstr :63578 # Windows +``` ---- +Stop the process holding it (often a previous Flowfile session), then run `flowfile run ui` again. -## Troubleshooting + -
-Installation Issues +
+pip install fails ```bash -# If pip install fails, try: pip install --upgrade pip pip install flowfile - -# For M1/M2 Macs: -pip install flowfile --no-binary :all: - -# Behind corporate proxy: -pip install --proxy http://proxy.company.com:8080 flowfile ``` -
-
-Port Already in Use - -```bash -# Find what's using port 63578 -lsof -i :63578 # Mac/Linux -netstat -ano | findstr :63578 # Windows +Flowfile supports Python 3.10–3.13. Check `python --version` if the resolver complains. -# Kill the process or use different port: -FLOWFILE_PORT=8080 flowfile run ui -```
-### Get Help - -
-
- Documentation
- Full Documentation -
-
- Discussions
- GitHub Discussions -
-
- Issues
- GitHub Issues -
-
+For anything else: [GitHub Discussions](https://github.com/edwardvaneechoud/Flowfile/discussions) for questions, [Issues](https://github.com/edwardvaneechoud/Flowfile/issues) for bugs. ---- +## Where next -
-

Ready to Transform Your Data?

-

Join thousands of users building data pipelines with Flowfile

-
- Start Visual (No Code) → - Start Coding (Python) → -
-
\ No newline at end of file +- [Coming from Excel](users/coming-from-excel.md) — VLOOKUP, pivot tables, and IF-formulas translated to flows. +- [Node reference](users/visual-editor/nodes/index.md) — everything the canvas can do. +- [Connect your data](users/visual-editor/connections.md) — databases, S3/ADLS/GCS, Kafka, APIs. +- [The catalog](users/visual-editor/catalog/index.md) — tables, SQL, visualizations, schedules; `flowfile seed-demo` gives you a populated catalog to explore in one command. +- [Deploy for a team](users/deployment/index.md) — Docker, users and groups, sharing. diff --git a/docs/stylesheets/extra.css b/docs/stylesheets/extra.css index 379b69007..037ce8236 100644 --- a/docs/stylesheets/extra.css +++ b/docs/stylesheets/extra.css @@ -1,4 +1,125 @@ -/* docs/stylesheets/extra.css */ +/* docs/stylesheets/extra.css — Flowfile brand layer. + Pages must not use inline styles; visual variety comes from these shared + classes plus Material's grid cards / content tabs. */ + .md-grid { max-width: 1920px; -} \ No newline at end of file +} + +/* Brand palette (matches the landing page hero) */ +:root, +[data-md-color-scheme="default"] { + --ff-teal: #00CED1; + --ff-teal-dark: #008B8B; + --ff-purple: #6B46C1; + --ff-purple-light: #8B5CF6; + --md-primary-fg-color: #008B8B; + --md-primary-fg-color--light: #00CED1; + --md-primary-fg-color--dark: #006b6b; + --md-accent-fg-color: #6B46C1; +} + +[data-md-color-scheme="slate"] { + --ff-teal: #20E3E6; + --ff-teal-dark: #00CED1; + --ff-purple: #A78BFA; + --ff-purple-light: #8B5CF6; + --md-primary-fg-color: #008B8B; + --md-primary-fg-color--light: #20E3E6; + --md-primary-fg-color--dark: #006b6b; + --md-accent-fg-color: #8B5CF6; +} + +/* Header + nav tabs: translucent brand-teal bar, blurred so scrolled content + stays readable underneath */ +.md-header, +.md-tabs { + background: color-mix(in srgb, var(--md-primary-fg-color) 55%, transparent); + -webkit-backdrop-filter: blur(14px); + backdrop-filter: blur(14px); + color: #fff; +} + +.md-header { + box-shadow: none; +} + +.md-tabs__link--active, +.md-tabs__link:hover { + color: #fff; + opacity: 1; +} + +.md-search__input { + background-color: rgba(255, 255, 255, 0.16); + color: #fff; +} + +.md-search__input::placeholder, +.md-search__input + .md-search__icon { + color: rgba(255, 255, 255, 0.75); +} + +/* Material grid cards: brand accent bar + hover lift */ +.md-typeset .grid.cards > ol > li, +.md-typeset .grid.cards > ul > li { + border-radius: 0.4rem; + border-top: 3px solid transparent; + background: + linear-gradient(var(--md-default-bg-color), var(--md-default-bg-color)) padding-box, + linear-gradient(90deg, var(--ff-teal), var(--ff-purple)) border-box; + transition: transform 0.15s ease, box-shadow 0.15s ease; +} + +.md-typeset .grid.cards > ol > li:hover, +.md-typeset .grid.cards > ul > li:hover { + transform: translateY(-2px); + box-shadow: 0 6px 16px rgba(0, 0, 0, 0.12); + border-color: transparent; +} + +/* Two-path chooser (quickstart, guides-by-audience) */ +.md-typeset .ff-paths { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(16rem, 1fr)); + gap: 0.8rem; + margin: 1em 0; +} + +.md-typeset a.ff-path { + display: block; + padding: 0.9rem 1rem; + border-radius: 0.4rem; + border: 1px solid; + color: var(--md-default-fg-color); + transition: transform 0.15s ease, box-shadow 0.15s ease; +} + +.md-typeset a.ff-path:hover { + transform: translateY(-2px); + box-shadow: 0 6px 16px rgba(0, 0, 0, 0.12); +} + +.md-typeset a.ff-path > strong { + display: block; + font-size: 0.85rem; + margin-bottom: 0.2rem; +} + +.md-typeset a.ff-path-teal { + border-color: var(--ff-teal-dark); + background: color-mix(in srgb, var(--ff-teal) 8%, transparent); +} + +.md-typeset a.ff-path-teal > strong { + color: var(--ff-teal-dark); +} + +.md-typeset a.ff-path-purple { + border-color: var(--ff-purple); + background: color-mix(in srgb, var(--ff-purple) 8%, transparent); +} + +.md-typeset a.ff-path-purple > strong { + color: var(--ff-purple); +} diff --git a/docs/users/coming-from-excel.md b/docs/users/coming-from-excel.md new file mode 100644 index 000000000..5956cffd3 --- /dev/null +++ b/docs/users/coming-from-excel.md @@ -0,0 +1,62 @@ +# Coming from Excel + +If you clean and analyze data in Excel — filters, VLOOKUPs, pivot tables, formula columns — you already know Flowfile's concepts. What changes is that every step becomes a visible, repeatable node instead of something buried in cell references. This page translates the Excel vocabulary into Flowfile's, so you can start from what you know. + +## The mental model + +An Excel workflow usually lives in one sheet: raw data on the left, helper columns in the middle, a pivot table somewhere else. In Flowfile the same work is a **flow** — a left-to-right chain of steps on a canvas, where each step shows you its output data. Nothing is hidden in cells, and re-running everything on next month's file is one click. + +| In Excel | In Flowfile | +|---|---| +| A sheet of rows | A table flowing from node to node, previewed at every step | +| A formula column (`=IF(...)`) | A [Formula node](visual-editor/nodes/transform.md#formula) | +| VLOOKUP / XLOOKUP | A [Join node](visual-editor/nodes/combine.md#join) | +| A pivot table | [Group By or Pivot nodes](visual-editor/nodes/aggregate.md) | +| Data → Remove Duplicates | A [Drop Duplicates node](visual-editor/nodes/transform.md#drop-duplicates) | +| Filter and sort buttons | [Filter and Sort nodes](visual-editor/nodes/transform.md#filter-data) | +| Redoing the steps by hand next month | Re-running the flow — or [scheduling it](visual-editor/catalog/schedules.md) | +| Save As → .xlsx | A [Write Data node](visual-editor/nodes/output.md) — Excel, CSV, or Parquet | + +## Formulas will feel familiar + +Flowfile's [formula language](formulas/index.md) reads like spreadsheet formulas: reference columns by name in square brackets instead of by cell, and write conditions as `if ... then ... else ... endif`. + +| Excel | Flowfile formula | +|---|---| +| `=IF(B2>100,"High","Low")` | `if [amount] > 100 then "High" else "Low" endif` | +| `=ROUND(B2*C2,2)` | `round([price] * [quantity], 2)` | +| `=A2&" "&B2` | `concat([first_name], " ", [last_name])` | +| `=IFERROR(B2,0)` (blank handling) | `ifnull([discount], 0)` | +| `=TODAY()` | `today()` | +| `=DATEDIF(A2,TODAY(),"d")` | `date_diff_days(today(), [hire_date])` | + +Two differences worth knowing up front: a formula applies to a whole column at once (there is no dragging down), and boolean logic is written `and` / `or` rather than `AND()` / `OR()`. The built-in functions are all in the [function reference](formulas/functions.md), and there is an [interactive playground](https://edwardvaneechoud.github.io/polars_expr_transformer/) where you can try formulas against sample data in your browser. + +## VLOOKUP is a Join + +A VLOOKUP pulls columns from another sheet by matching a key. In Flowfile, that is a **Join** node with two inputs: your main table and the lookup table, matched on a key column. + +- A **left** join is the closest match to VLOOKUP: every row of your main table is kept, and matching columns are added where the key is found (non-matches become empty values instead of `#N/A`). +- An **inner** join keeps only the rows that matched — like filtering out the `#N/A`s afterwards. + +Where Excel needed the lookup value in the first column and returned one column at a time, a Join matches on any column and brings the whole lookup row along. And when the keys don't match exactly — "Acme Corp" vs "ACME Corporation" — the [Fuzzy Match node](visual-editor/nodes/combine.md#fuzzy-match) does approximate matching, something Excel cannot do natively. + +## Pivot tables are two nodes + +Excel's pivot table does two jobs at once, and Flowfile splits them: + +- **Group By** produces summary rows — one row per group with aggregations like sum, mean, median, or count. This covers most pivot-table uses ("total sales per city"). +- **Pivot** spreads a category column across the header — one column per category value, like putting a field in the pivot table's "Columns" area. [Unpivot](visual-editor/nodes/aggregate.md#unpivot-data) does the reverse, turning wide monthly columns back into tidy rows. + +## Where the spreadsheet runs out + +Excel stops at 1,048,576 rows and gets slow well before that. Flowfile runs on [Polars](https://pola.rs), a modern engine that comfortably processes millions of rows on a laptop. The practical differences: + +- **Size** — files bigger than Excel's limit open and process normally. +- **Repeatability** — point the flow at next month's file and press run; no re-doing steps, no stale pivot caches. +- **Transparency** — every transformation is a labeled node a colleague can read, not a formula hidden in column Q. +- **A way out of files entirely** — results can land in the [catalog](visual-editor/catalog/index.md), where you query them with SQL, chart them, and schedule refreshes. + +## Try it + +The [Quickstart](../quickstart.md) walks through a classic Excel job — deduplicate sales rows, filter them, and build a per-city summary — as a visual flow. To try the canvas without installing anything, open the [live demo](https://demo.flowfile.org) and load a template. diff --git a/docs/users/connect/apis.md b/docs/users/connect/apis.md new file mode 100644 index 000000000..7b4c8f081 --- /dev/null +++ b/docs/users/connect/apis.md @@ -0,0 +1,101 @@ +# REST APIs and Google Analytics + +Pull data from HTTP endpoints and Google Analytics 4 into a flow. This page covers the REST API Reader (node and `ff.read_api`) and the Google Analytics reader. Both are read-only sources: they fetch data in, they do not write back. + +!!! info "Not in Flowfile Lite" + Both readers require the full desktop or server build. The browser-only [Flowfile Lite](../deployment/lite.md) edition has no backend and cannot reach external APIs. + +## REST APIs + +The **REST API Reader** node fetches JSON from an HTTP endpoint, flattens the response into a table, and hands it downstream. It handles authentication and pagination so you don't script the request loop yourself. JSON is the only supported response format. + +### Request settings + +The node's request is built from these settings (the same fields `ff.read_api` accepts): + +| Setting | Meaning | +|---|---| +| URL | The request URL. | +| Method | `GET` or `POST`. | +| Headers | Request headers as key/value pairs. | +| Query params | URL query parameters. | +| JSON body | Request body for `POST`. | +| Record path | Dot-path to the record array inside the response (e.g. `data.items`). Empty uses the top-level response. | +| Timeout (seconds) | Per-request timeout (default 30). | +| Max retries | Retries for transient failures (default 3). | + +### Authentication + +Authentication is configured by `auth_type`, one of: + +- `none` — no authentication. +- `api_key` — an API key sent as a header (default name `X-API-Key`) or query parameter, per `api_key_location`. +- `bearer` — a bearer token in the `Authorization` header. +- `basic` — HTTP basic auth with a username and a secret password. + +In the node, the credential references a stored secret by name ([Secrets](../visual-editor/catalog/secrets.md)) so it never lands in the flow file. In `ff.read_api` you may instead pass an inline `secret` for programmatic use; it is encrypted with the master key and never persisted. + +### Pagination + +`pagination_type` selects the strategy: + +- `none` — a single request. +- `offset` — offset/limit paging (`offset_param`, `limit_param`, `page_size`). +- `page` — page-number paging (`page_param`, `start_page`, `page_size`). +- `cursor` — cursor / next-token paging (`cursor_param`, `cursor_response_path`, `initial_cursor`). + +Safety caps `max_pages` (default 1000), `max_records`, and `page_delay_seconds` bound the paging loop. + +### Read a REST API in Python + +`ff.read_api` builds a REST API Reader node and returns a [FlowFrame](../python-api/index.md). The fragment below is **illustrative** — it points at a placeholder host, so it will not run as written; substitute your own endpoint and credentials. + +```python +# Illustrative — not runnable as written. +import flowfile as ff + +users = ff.read_api( + "https://api.example.com/v1/users", + auth={"auth_type": "bearer", "secret": "YOUR_TOKEN"}, + pagination={"pagination_type": "offset", "page_size": 100}, + record_path="data", +).collect() +``` + +The full signature is: + +```python +ff.read_api( + url, + *, + method="GET", + headers=None, + params=None, + json_body=None, + auth=None, + pagination=None, + record_path="", + timeout_seconds=30.0, + max_retries=3, +) +``` + +`auth` and `pagination` accept a plain dict (as above) or the typed `RestApiAuthSettings` / `RestApiPaginationSettings` objects. + +## Google Analytics + +The **Google Analytics** reader pulls reports from a **GA4 property** — dimensions and metrics over a date range — into a flow. It reads through a saved Google Analytics connection; there is no `ff.*` helper for GA, so set the connection up first, then use the reader node on the canvas. + +### Connecting an account + +Create a Google Analytics connection on the **Connections** page under the **Google Analytics** tab. Two credential types are supported: + +- **OAuth 2.0** — sign in with Google and consent to the `analytics.readonly` scope. Flowfile stores the resulting refresh token encrypted; no raw credential is entered by hand. This requires a Google OAuth **web application client** (client id, client secret, redirect URI), which you paste into the Google OAuth card on the same page. Create the client at [Google Cloud Console → Credentials](https://console.cloud.google.com/apis/credentials). +- **Service account** — upload a service-account JSON key. Flowfile validates and encrypts it at rest. The service account's email must be granted **Viewer** on each GA4 property you want to read; a service-account connection sees whatever properties that email has access to. + +Each connection can carry a **default property id** (the GA4 property, e.g. `123456789`). Use **Test** on the connection to confirm the credential is valid — for a service account with a default property set, the test also verifies Viewer access to that property. + +!!! note "Credentials needed" + OAuth needs a Google OAuth web-application client (id, secret, redirect URI) plus the interactive sign-in. A service account needs the JSON key and Viewer access on the target GA4 property. In either case you supply a GA4 property id to run a report. + +No runnable example is provided for Google Analytics: it reads from a live Google account and property that can't be seeded in a test. diff --git a/docs/users/connect/index.md b/docs/users/connect/index.md new file mode 100644 index 000000000..947f548fe --- /dev/null +++ b/docs/users/connect/index.md @@ -0,0 +1,50 @@ +# Connect your data + +Your data lives somewhere else — a database, an object store, a Kafka topic, a REST API, or Google Analytics. This section is the map of every source Flowfile can read from and every sink it can write to, and where each one is configured. Start here to find the right connector, then follow the link to its setup page. + +Flowfile connects to external systems in two ways. In the visual editor you add a **reader** or **writer** node and point it at a saved connection. In the [Python API](../python-api/index.md) you call the matching `ff.*` function. Both paths go through the same stored connections, so a connection you save once in the UI is usable from code, and vice versa. + +## Connector matrix + +Each connector below is read-only, write-only, or both. "Where configured" names the tab on the **Connections** page (or the node that carries inline settings) where you set it up. + +| Connector | Reads | Writes | Where configured | Setup | +|---|---|---|---|---| +| PostgreSQL / MySQL / SQLite | yes | yes | Connections → Database | [Databases](../visual-editor/tutorials/database-connectivity.md) | +| Cloud storage (S3 / ADLS / GCS) | yes | yes | Connections → Cloud Storage | [Cloud storage](../visual-editor/tutorials/cloud-connections.md) | +| Kafka / Redpanda | yes | no | Connections → Kafka | [Kafka](kafka.md) | +| REST API | yes | no | REST API Reader node (inline) | [REST APIs](apis.md#rest-apis) | +| Google Analytics 4 | yes | no | Connections → Google Analytics | [Google Analytics](apis.md#google-analytics) | +| Data catalog (Delta Lake) | yes | yes | Catalog Reader / Writer nodes | [Catalog](../visual-editor/catalog/index.md) | +| Saved secrets & connections | — | — | Connections page | [Connections](../visual-editor/connections.md) · [Secrets](../visual-editor/catalog/secrets.md) | + +!!! info "Not in Flowfile Lite" + Saved connections and the secrets that back them require the full desktop or server build. The browser-only [Flowfile Lite](../deployment/lite.md) edition has no backend, so none of the connectors above are available there — Lite works with files you load directly in the browser. + +## Databases + +Typed connections to **PostgreSQL**, **MySQL**, and **SQLite**. Both directions are supported: the Database Reader node (`ff.read_database`) runs a query or reads a whole table, and the Database Writer node (`ff.write_database`) writes a frame back. Credentials are stored encrypted and referenced by name. + +See [Databases](../visual-editor/tutorials/database-connectivity.md) for the connection form and worked reader/writer examples. + +## Cloud storage + +Read and write **Amazon S3**, **Azure Data Lake Storage (ADLS)**, and **Google Cloud Storage (GCS)**. Eight authentication methods are available (`access_key`, `iam_role`, `service_principal`, `managed_identity`, `sas_token`, `aws-cli`, `env_vars`, and `service_account`), so a connection can use stored keys or delegate to the ambient cloud credentials. Read formats are CSV, Parquet, JSON, Delta, and Iceberg; write formats are CSV, Parquet, JSON, and Delta. + +See [Cloud storage](../visual-editor/tutorials/cloud-connections.md) for provider-specific setup. + +## Kafka + +Consume JSON messages from a **Kafka** or **Redpanda** topic through the Kafka Source node (`ff.read_kafka`). Offsets are tracked broker-side by consumer group, so a flow that runs on a schedule reads only what arrived since the last run. Kafka is **read-only** — there is no Kafka writer node. + +See [Kafka](kafka.md) for the connection form, security options, and a runnable example. + +## REST APIs and Google Analytics + +The **REST API Reader** node (`ff.read_api`) fetches JSON from an HTTP endpoint with configurable authentication and pagination. The **Google Analytics** reader pulls GA4 property reports through a stored OAuth or service-account connection. Both are read-only. + +See [REST APIs and Google Analytics](apis.md). + +## Catalog + +The [data catalog](../visual-editor/catalog/index.md) is Flowfile's own storage layer, backed by Delta Lake. The Catalog Reader node loads a registered table into a flow and the Catalog Writer node persists a result back as a versioned table. Unlike the external connectors above, the catalog is managed inside Flowfile — see the [Catalog](../visual-editor/catalog/index.md) section for details. diff --git a/docs/users/connect/kafka.md b/docs/users/connect/kafka.md new file mode 100644 index 000000000..ed8214f70 --- /dev/null +++ b/docs/users/connect/kafka.md @@ -0,0 +1,78 @@ +# Kafka + +Consume records from a Kafka or Redpanda topic, either as a node on the canvas or with `ff.read_kafka` in Python. This page covers the saved Kafka connection, the Kafka Source node's settings, the security options, and a runnable example. It serves analysts wiring a streaming source into a flow and developers reading a topic from code. + +Kafka support is **read-only**: Flowfile consumes from topics but does not produce to them. There is no Kafka writer node. + +!!! info "Not in Flowfile Lite" + Kafka connections require the full desktop or server build. The browser-only [Flowfile Lite](../deployment/lite.md) edition has no backend and cannot connect to a broker. + +## Saved Kafka connections + +A Kafka connection stores the broker address and (optionally) authentication, so nodes and scripts reference it by name instead of repeating credentials. Create one on the **Connections** page under the **Kafka** tab, or manage them alongside the other connection types described in [Connections](../visual-editor/connections.md). + +The connection form carries these fields: + +| Field | What it is | +|---|---| +| Connection name | The name you reference later (in the node or `ff.read_kafka`). | +| Bootstrap servers | Comma-separated `host:port` list for the broker(s), e.g. `localhost:19092`. | +| Security protocol | One of `PLAINTEXT`, `SSL`, `SASL_PLAINTEXT`, `SASL_SSL`. | +| SASL mechanism | The SASL mechanism (e.g. `PLAIN`, `SCRAM-SHA-256`) when a SASL protocol is selected. | +| SASL username / password | Credentials for SASL authentication. The password is stored encrypted. | +| SSL CA location | Path to the CA certificate for verifying the broker (SSL/SASL_SSL). | +| SSL certificate location | Path to the client certificate for mutual TLS. | +| SSL key (PEM) | The client private key. Stored encrypted. | +| Schema registry URL | Optional schema-registry endpoint. | +| Extra config | Optional passthrough of additional confluent-kafka settings. | + +The `sasl_password` and `ssl_key_pem` values are stored as encrypted secrets ([Secrets](../visual-editor/catalog/secrets.md)) and never written into flow files. Reserved keys (`bootstrap.servers`, `group.id`, `enable.auto.commit`, and any `sasl.*` / `ssl.*` / `security.protocol`) are filtered out of Extra config so they can't override the managed settings. + +!!! note "Message format" + The consumer parses each message value as **JSON**. `value_format` accepts `json`; other formats are not supported. + +## The Kafka Source node + +Add a **Kafka Source** node to the canvas, pick a saved connection, then choose a topic. The node offers these settings: + +| Setting | Meaning | +|---|---| +| Topic name | The topic to consume from. When the connection can reach the broker, the node lists available topics; otherwise type the name. | +| Start offset | `Latest` (only new messages) or `Earliest` (from the beginning) — used the first time a consumer group reads. | +| Max messages | Cap on messages consumed in one run (default 100000). | +| Poll timeout (seconds) | How long to wait for messages before returning (default 30). | +| Sync name | The consumer-group id. Two runs with the same sync name resume where the last one left off. | + +The node adds Kafka metadata columns to every row: `_kafka_key`, `_kafka_partition`, `_kafka_offset`, and `_kafka_timestamp`, alongside the parsed message fields. + +Offsets are committed **broker-side by consumer group** after the run's downstream nodes succeed — a failed run does not advance the offset, so the next run re-reads the same messages. Use **Reset Offsets** in the node to rewind a consumer group to the beginning of a topic and re-read everything. + + + +## Read Kafka in Python + +`ff.read_kafka` resolves a saved connection by name and returns a [FlowFrame](../python-api/index.md): + +```python +--8<-- "docs/examples/integrations/kafka_read.py:example" +``` + +The signature is: + +```python +ff.read_kafka( + connection_name, + *, + topic_name, + max_messages=100_000, + start_offset="latest", + poll_timeout_seconds=30.0, + value_format="json", +) +``` + +Kafka connections are created server-side (through the Connections page or the core API); there is no `ff.*` helper to create one. `connection_name` must match a connection that already exists for your user. Calling `.collect()` runs the consume in-process against the broker. + +## Kafka to catalog sync + +Beyond one-off reads, Flowfile can generate a **Kafka sync** flow that consumes a topic and appends it to a [catalog table](../visual-editor/catalog/index.md) as a `kafka_source` → `catalog_writer` pipeline. Run that flow on a [schedule](../visual-editor/catalog/schedules.md) and each run picks up only the messages that arrived since the last, keeping the table current. diff --git a/docs/users/deployment/cli.md b/docs/users/deployment/cli.md new file mode 100644 index 000000000..c747688a9 --- /dev/null +++ b/docs/users/deployment/cli.md @@ -0,0 +1,122 @@ +# Headless runs and the CLI + +This page is for anyone driving Flowfile from a terminal — launching the app, running a saved flow without opening the UI, seeding the demo catalog, or scripting project version control. It documents every `flowfile` command and its flags, and walks through a real headless run of the bundled sales pipeline. + +Installing Flowfile with `pip install flowfile` puts a `flowfile` command on your path. Run it with no arguments to print the version and a usage summary. + +## Command overview + +| Command | What it does | +|---|---| +| `flowfile run ui` | Start the web UI with the bundled services (the default way to use Flowfile). | +| `flowfile run flow ` | Run a saved flow headlessly and exit. | +| `flowfile run core` | Start only the core API service. | +| `flowfile run worker` | Start only the compute worker service. | +| `flowfile seed-demo` | Populate the optional demo catalog. | +| `flowfile remove-demo` | Remove the demo catalog. | +| `flowfile project init/open/save` | Git version control for your work (see [Projects](../projects.md)). | + +## Start the app + +```bash +flowfile run ui +``` + +This launches the core API with the worker co-hosted on the same port and opens the web UI in your browser at `http://127.0.0.1:63578/ui`. + +Flags: + +- `--no-browser` — start the services but don't open a browser tab. + +!!! warning "The web UI is fixed to 127.0.0.1:63578" + `run ui` always binds the bundled UI to `127.0.0.1` on port `63578` — the server refuses to start on any other host or port. The `--host` and `--port` flags exist for `run core` and `run worker` (below); they do not move the bundled UI. To expose Flowfile on another address, run it behind a reverse proxy or use the [Docker deployment](docker.md). + +## Run a flow headlessly + +```bash +flowfile run flow +``` + +Runs a saved flow to completion and exits with code `0` on success or `1` on failure. This is the command to use in cron jobs, CI, or any non-interactive context. + +Accepted file formats are `.yaml`, `.yml`, and `.json`. (The legacy `.flowfile` pickle format is open-only in the app and is not accepted here.) + +What a headless run does: + +- **Forces local execution.** The flow runs entirely in-process — no worker service needs to be running. +- **Skips UI-only nodes.** `explore_data` nodes are dropped before the run (they require the interactive UI); the command reports how many it skipped. +- **Prints progress.** It shows the flow name, node count, the node tree, and a per-node error summary if anything fails. + +Flags: + +- `--param KEY=VALUE` — override a flow parameter. Repeat the flag for several overrides. An override that names a parameter the flow doesn't have is added as a new one. +- `--run-id N` — report the run's result back to a pre-created run record (used by the scheduler). You normally don't set this by hand. + +```bash +# Override two parameters +flowfile run flow pipeline.yaml --param input_dir=/data --param threshold=100 +``` + +## Worked example: run the sales pipeline + +The sales pipeline that ships with Flowfile is a runnable artifact you can execute headlessly. + +Download [`sales_pipeline.yaml`](../../assets/flows/sales_pipeline.yaml) and run it: + +```bash +flowfile run flow sales_pipeline.yaml +``` + +The downloaded flow reads the sample CSV from its public GitHub URL, so it runs as-is with an internet connection — expect a five-city income summary and exit code `0`. To run fully offline, edit the read node's path to a local copy of [`supermarket_sales.csv`](https://raw.githubusercontent.com/edwardvaneechoud/flowfile/main/data/templates/supermarket_sales.csv), or create the flow from the in-app template browser (which provisions the data locally) and save it to a file. + +## Run individual services + +For advanced or split deployments you can start each service on its own. These accept `--host` and `--port`: + +```bash +flowfile run core # core API, default 127.0.0.1:63578 +flowfile run core --host 0.0.0.0 --port 63578 +flowfile run worker # compute worker, default 127.0.0.1:63579 +``` + +When you run the UI the two are combined into one process via single-file mode (`FLOWFILE_SINGLE_FILE_MODE`), which co-hosts the worker on the core port. When you run `core` and `worker` separately, core reaches the worker at `WORKER_HOST:FLOWFILE_WORKER_PORT`. + +## The PyInstaller / packaged path + +The desktop app and packaged builds run a saved flow through the core module directly rather than the `flowfile` wrapper: + +```bash +python -m flowfile_core.main --run-flow --run-id +``` + +This forces local execution and drops `explore_data` nodes, exactly like `flowfile run flow`. Here `--run-id` is required — the packaged path always reports back to a run record. Prefer `flowfile run flow` for your own headless runs; this form exists for the scheduler and installed builds. + +## Demo catalog + +```bash +flowfile seed-demo # create the demo catalog (idempotent) +flowfile remove-demo # remove everything under the demo catalog +``` + +`seed-demo` populates a `Demo` catalog with sample tables and two registered flows so you can explore the catalog with real content. It's safe to run more than once — it skips what already exists. See [Start with a populated catalog](../visual-editor/catalog/index.md#start-with-a-populated-catalog) for what it creates. + +## Projects + +```bash +flowfile project init # create a project in +flowfile project open # open a project and rebuild it into the database +flowfile project save "" # save a version with a commit message +``` + +These are the headless equivalents of the app's project controls. Opening a project reports what it imported and how many secret values still need to be set. See [Projects](../projects.md) for the full workflow, and note that in Docker mode the project commands require `FLOWFILE_ENABLE_PROJECTS` to be set. + +## Scheduling + +To run flows on a timer rather than on demand, use the scheduler. In the desktop app and pip package it runs inside the app; for a standalone service the `flowfile_scheduler` console script polls for due runs and executes each one through the same headless path documented above. See [Schedules](../visual-editor/catalog/schedules.md) for setup, cron syntax, and the standalone-service mode. + +## Related + +- [Schedules](../visual-editor/catalog/schedules.md) — run flows automatically on a timer. +- [Projects](../projects.md) — git version control from the CLI. +- [Docker deployment](docker.md) — multi-user and networked deployments. +- [Python API](../python-api/index.md) — build and run flows programmatically instead of from files. diff --git a/docs/users/deployment/desktop.md b/docs/users/deployment/desktop.md index df138d8da..a83cedc13 100644 --- a/docs/users/deployment/desktop.md +++ b/docs/users/deployment/desktop.md @@ -1,12 +1,14 @@ # Desktop App -The easiest way to get started with Flowfile. +Download and run Flowfile as a native desktop app on macOS, Windows, or Linux. ## Download Download the latest release for your platform: -[**Download for macOS / Windows →**](https://github.com/edwardvaneechoud/Flowfile/releases) +[Releases on GitHub →](https://github.com/edwardvaneechoud/Flowfile/releases) + +Installers are built for macOS (Apple Silicon and Intel), Windows, and Linux. ## Installation @@ -14,10 +16,7 @@ Download the latest release for your platform: 1. Download the `.dmg` file 2. Open it and drag Flowfile to **Applications** -3. Launch from Applications - -!!! note "First launch" - macOS may ask you to confirm opening an app from an unidentified developer. Go to **System Preferences → Security & Privacy** and click "Open Anyway". +3. Launch from Applications — the app is signed and notarized, so it opens without security prompts ### Windows @@ -25,27 +24,14 @@ Download the latest release for your platform: 2. Run the installer and follow the wizard 3. Launch from the Start menu -## Why Desktop? - -**Pros** - -- Zero configuration - just download and run -- No Docker, no terminal commands -- Works offline -- Data stays on your machine -- Master key managed automatically - -**Cons** +### Linux -- Single user only (no team sharing) -- No centralized secrets management -- Flows stored locally (manual backup needed) +1. Download the `.AppImage` (or `.deb`) build +2. For the AppImage, mark it executable (`chmod +x Flowfile*.AppImage`) and run it +3. For the `.deb`, install with your package manager (`sudo dpkg -i flowfile_*.deb`) -## When to Use Docker Instead +## What you get -Consider [Docker deployment](docker.md) if you need: +The desktop app runs offline with no Docker or terminal commands; data stays on your machine, and the setup screen generates and stores the encryption master key on first launch. It is single-user, with local flow storage (manual backup needed) and no centralized secrets management. -- Multiple users with separate accounts -- Centralized flow storage -- Team collaboration -- Server/production deployment +For multiple users, centralized flow storage, team collaboration, or server/production deployment, use [Docker deployment](docker.md) instead. diff --git a/docs/users/deployment/docker.md b/docs/users/deployment/docker.md index 22084a122..f572de7bf 100644 --- a/docs/users/deployment/docker.md +++ b/docs/users/deployment/docker.md @@ -1,6 +1,6 @@ # Docker Reference -Run Flowfile with Docker using pre-built images from Docker Hub. +Run Flowfile as a multi-user stack with Docker. This is the deployment for teams and servers: authentication, encrypted secrets, a shared catalog, and (optionally) Docker-spawned kernels for Python-script nodes. The bundled compose file **builds the images from source** on first `up`; pre-built images are also published to Docker Hub if you prefer `image:` entries over `build:`. This page covers quick start, configuration, security, kernels, and operations. ## Quick Start @@ -10,27 +10,46 @@ cd Flowfile docker compose up -d ``` -Access at `http://localhost:8080`. The setup wizard will guide you through master key configuration. +Access at `http://localhost:8080`. On first run the setup wizard guides you through master-key configuration. ![Setup Wizard](../../assets/images/guides/docker-deployment/setup_wizard.png) +### First-run master key + +The bundled compose leaves `FLOWFILE_MASTER_KEY` empty, so the setup screen prompts you to configure it: + +1. Open `http://localhost:8080`. +2. Click **Generate Master Key** and copy the key. +3. Add it to your `.env`: `FLOWFILE_MASTER_KEY=`. +4. Restart: `docker compose restart`. +5. Log in with the admin credentials (default `admin` / `changeme` — change these). + +For an automated deployment, generate the key ahead of time and set it before the first boot: + +```bash +python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())" +``` + +!!! warning "Losing the master key loses your secrets" + The master key encrypts every stored secret. If you change or lose it, existing encrypted secrets become unreadable. Back up `.env` securely and never commit it to version control. + ## Docker Images | Image | Description | |-------|-------------| | `edwardvaneechoud/flowfile-frontend` | Web UI | | `edwardvaneechoud/flowfile-core` | API server | -| `edwardvaneechoud/flowfile-worker` | Data processing | +| `edwardvaneechoud/flowfile-worker` | Compute worker (heavy work runs in spawned subprocesses) | | `edwardvaneechoud/flowfile-kernel-base` | Python-script kernel (base) | | `edwardvaneechoud/flowfile-kernel-ml` | Python-script kernel with sklearn / xgboost / lightgbm / statsmodels | +| `edwardvaneechoud/flowfile-kernel-lite` | Slimmed Python-script kernel for constrained hosts | -Application images (`flowfile-frontend`, `flowfile-core`, `flowfile-worker`) share -the project version (`latest`, `0.9.3`, `sha-...`). The kernel images carry their -own version (currently `0.3.0`) so the kernel runtime can evolve independently of -the rest of the application. +The application images (`flowfile-frontend`, `flowfile-core`, `flowfile-worker`) share the project version (tagged `latest`, the release version, and `sha-...`). The kernel images carry their own version so the kernel runtime can evolve independently of the rest of the application; the tag core pulls by default is set in `flowfile_core/flowfile_core/kernel/manager.py` (`_KERNEL_IMAGE_*_DEFAULT`). ## docker-compose.yml +The compose file bundled at the repo root is the source of truth. A trimmed version showing the load-bearing settings: + ```yaml services: flowfile-frontend: @@ -47,18 +66,26 @@ services: image: edwardvaneechoud/flowfile-core:latest ports: - "63578:63578" + shm_size: '2gb' environment: - FLOWFILE_MODE=docker - FLOWFILE_ADMIN_USER=${FLOWFILE_ADMIN_USER:-admin} - FLOWFILE_ADMIN_PASSWORD=${FLOWFILE_ADMIN_PASSWORD:-changeme} - - JWT_SECRET_KEY=${JWT_SECRET_KEY} + - JWT_SECRET_KEY=${JWT_SECRET_KEY:-flowfile-dev-secret-change-in-production} + # Shared secret for kernel → core authentication (must match the kernels) + - FLOWFILE_INTERNAL_TOKEN=${FLOWFILE_INTERNAL_TOKEN:-flowfile-dev-internal-token-change-in-production} - FLOWFILE_MASTER_KEY=${FLOWFILE_MASTER_KEY:-} - FLOWFILE_SCHEDULER_ENABLED=true + - FLOWFILE_ENABLE_PROJECTS=${FLOWFILE_ENABLE_PROJECTS:-true} - WORKER_HOST=flowfile-worker + - FLOWFILE_STORAGE_DIR=/app/internal_storage + - FLOWFILE_USER_DATA_DIR=/app/user_data volumes: + # Docker socket lets core create kernel containers on the host. + - /var/run/docker.sock:/var/run/docker.sock - ./flowfile_data:/app/user_data + - flowfile-internal-storage:/app/internal_storage - ./saved_flows:/app/flowfile_core/saved_flows - - flowfile-storage:/app/internal_storage networks: - flowfile-network @@ -66,21 +93,27 @@ services: image: edwardvaneechoud/flowfile-worker:latest ports: - "63579:63579" + shm_size: '2gb' environment: - FLOWFILE_MODE=docker - CORE_HOST=flowfile-core - FLOWFILE_MASTER_KEY=${FLOWFILE_MASTER_KEY:-} + - FLOWFILE_STORAGE_DIR=/app/internal_storage + - FLOWFILE_USER_DATA_DIR=/app/user_data volumes: - ./flowfile_data:/app/user_data - - flowfile-storage:/app/internal_storage + - flowfile-internal-storage:/app/internal_storage networks: - flowfile-network networks: flowfile-network: + driver: bridge + name: flowfile-network volumes: - flowfile-storage: + flowfile-internal-storage: + driver: local ``` ## Environment Variables @@ -90,20 +123,20 @@ volumes: | `FLOWFILE_MODE` | Set to `docker` for multi-user auth | `docker` | | `FLOWFILE_ADMIN_USER` | Admin username | `admin` | | `FLOWFILE_ADMIN_PASSWORD` | Admin password | `changeme` | -| `JWT_SECRET_KEY` | Token signing secret | Required | -| `FLOWFILE_MASTER_KEY` | Encryption key for secrets | Via setup wizard | -| `FLOWFILE_SCHEDULER_ENABLED` | Auto-start the flow scheduler | `false` | -| `FLOWFILE_ENABLE_PROJECTS` | Enable git project tracking (admin-only in docker; the `/project` router 404s when off). Accepts `true`/`1`/`yes`/`on`. | `true` in the bundled compose files | +| `JWT_SECRET_KEY` | Token signing secret. Required in Docker mode. | Insecure dev fallback in compose | +| `FLOWFILE_INTERNAL_TOKEN` | Shared secret for kernel → core authentication. Required in Docker mode. | Insecure dev fallback in compose | +| `FLOWFILE_MASTER_KEY` | Encryption key for secrets | Empty (setup wizard prompts) | +| `FLOWFILE_SCHEDULER_ENABLED` | Auto-start the flow scheduler | `true` in the bundled compose (the code default when the var is entirely unset is off) | +| `FLOWFILE_ENABLE_PROJECTS` | Enable git project tracking (admin-only in Docker; the `/project` router 404s when off). Accepts `true`/`1`/`yes`/`on`. | `true` in the bundled compose | +| `FLOWFILE_STORAGE_DIR` | Internal storage path | `/app/internal_storage` | +| `FLOWFILE_USER_DATA_DIR` | User data path | `/app/user_data` | | `WORKER_HOST` | Worker hostname | `flowfile-worker` | | `CORE_HOST` | Core hostname | `flowfile-core` | -| `FLOWFILE_KERNEL_IMAGE` | Kernel image to launch for Python-script nodes | `edwardvaneechoud/flowfile-kernel-base:0.3.0` | +| `FLOWFILE_KERNEL_IMAGE` | Override the base kernel image for Python-script nodes | Registry default (unset ⇒ the tag in `kernel/manager.py`) | ### Git project tracking -When `FLOWFILE_ENABLE_PROJECTS` is on, Flowfile can mirror your flows, connections and schedules -into a versioned, secret-free git folder. In docker it is **administrator-only**, and each user's -projects are confined to their own `/app/user_data/projects/` area, so tenants stay -isolated. Turn it off with `FLOWFILE_ENABLE_PROJECTS=false`. +When `FLOWFILE_ENABLE_PROJECTS` is on, Flowfile can mirror your flows, connections and schedules into a versioned, secret-free git folder. In Docker it is **administrator-only**, and each user's projects are confined to their own `/app/user_data/projects/` area, so tenants stay isolated. Turn it off with `FLOWFILE_ENABLE_PROJECTS=false`. ## .env Example @@ -111,6 +144,7 @@ isolated. Turn it off with `FLOWFILE_ENABLE_PROJECTS=false`. FLOWFILE_ADMIN_USER=admin FLOWFILE_ADMIN_PASSWORD=YourSecurePassword123! JWT_SECRET_KEY=generate-with-openssl-rand-hex-32 +FLOWFILE_INTERNAL_TOKEN=generate-with-openssl-rand-hex-32 FLOWFILE_MASTER_KEY=generated-from-setup-wizard ``` @@ -118,9 +152,9 @@ FLOWFILE_MASTER_KEY=generated-from-setup-wizard | Path | Purpose | |------|---------| -| `./flowfile_data` | User data | +| `./flowfile_data` | User data (uploads, files) | | `./saved_flows` | Flow definitions | -| `flowfile-storage` | Internal storage | +| `flowfile-internal-storage` | Internal application data (catalog DB, logs) | ## Commands @@ -129,55 +163,90 @@ docker compose up -d # Start docker compose down # Stop docker compose pull # Update images docker compose logs -f # View logs +docker compose restart # Restart after an .env change ``` ## Health Checks +Core exposes a setup/health probe; the worker and frontend have no `/health` route — check them via their port or logs. + | Service | Endpoint | |---------|----------| -| Core | `http://localhost:63578/health` | -| Worker | `http://localhost:63579/health` | +| Core | `http://localhost:63578/health/status` | | Frontend | `http://localhost:8080` | +## Security Architecture + +The master key, JWT secret, and internal token are set via the env vars above. Two more components live in the database rather than the environment: + +| Component | Purpose | Configuration | +|-----------|---------|---------------| +| **User Secrets** | API keys, passwords, tokens | Encrypted in the database | +| **User Password** | Authenticates users | Hashed in the database | + +How they work together: + +1. A user logs in with username/password. +2. The server issues a JWT (signed with `JWT_SECRET_KEY`). +3. The user creates a secret (e.g. `my_api_key = sk-xxx`). +4. The value is encrypted with a per-user key derived from the master key before storage. +5. At runtime, secrets are decrypted for use inside flow execution — never returned to the API. + +### Production checklist + +- [ ] Replace the compose's insecure dev fallbacks for `JWT_SECRET_KEY` and `FLOWFILE_INTERNAL_TOKEN` (both effectively required in Docker mode) — generate each with `openssl rand -hex 32` +- [ ] Generate a unique `FLOWFILE_MASTER_KEY` +- [ ] Set a strong `FLOWFILE_ADMIN_PASSWORD` +- [ ] Never commit `.env` to version control +- [ ] Back up `.env` securely (losing the master key = losing all encrypted secrets) +- [ ] Terminate TLS with a reverse proxy (nginx / traefik / Caddy) +- [ ] Restrict access to the Docker socket mount (consider a socket proxy) + +!!! info "Storage backend" + The catalog database is SQLite, stored in the `flowfile-internal-storage` volume. There is no supported external-database (PostgreSQL) or Redis configuration — Flowfile resolves its database URL to a local SQLite file. Keep the internal-storage volume on durable, backed-up storage. + +## Group-Based Sharing + +In Docker (multi-user) mode Flowfile supports sharing resources — secrets, connections, catalog namespaces, tables, and flows — with **user groups** at `use` or `manage` level. The catalog is **private-by-default** in Docker mode: each user sees only what they own, what's shared with a group they belong to, and the seeded public system namespaces. This feature is dormant in the desktop/Electron app. + +Sharing is **authorization-only**: granting a group access never copies or re-encrypts a credential. Secrets stay owner-keyed (`$ffsec$1$$…`), so a shared secret decrypts unchanged in both core and the worker. + +For the full model — creating groups, the `use`/`manage` levels, how shared secrets and connections resolve, the catalog private-by-default upgrade, and operator notes — see **[Users, Groups & Sharing](sharing.md)**. + ## Python Script (Kernel) Nodes -Python-script nodes run inside short-lived kernel containers spawned by `flowfile-core` via the host Docker socket. To enable them, mount the Docker socket into `flowfile-core` and pull the kernel image you want to use. +Python-script nodes run inside short-lived kernel containers spawned by `flowfile-core` via the host Docker socket. To enable them, mount the Docker socket into `flowfile-core` (the bundled compose already does) and pull the kernel image you want. ### 1. Pull the kernel image +Kernel images are versioned independently of the app; the default tag lives in `flowfile_core/flowfile_core/kernel/manager.py`. Pull the matching tag: + ```bash -docker pull edwardvaneechoud/flowfile-kernel-base:0.3.0 +docker pull edwardvaneechoud/flowfile-kernel-base: # Or, for ML workloads (sklearn, xgboost, lightgbm, statsmodels pre-baked): -docker pull edwardvaneechoud/flowfile-kernel-ml:0.3.0 +docker pull edwardvaneechoud/flowfile-kernel-ml: ``` ### 2. Mount the Docker socket and set the image -In your `docker-compose.yml`, on the `flowfile-core` service: +On the `flowfile-core` service: ```yaml flowfile-core: # ... existing config ... environment: # ... existing env vars ... - - FLOWFILE_KERNEL_IMAGE=${FLOWFILE_KERNEL_IMAGE:-edwardvaneechoud/flowfile-kernel-base:0.3.0} + - FLOWFILE_KERNEL_IMAGE=${FLOWFILE_KERNEL_IMAGE:-} volumes: - /var/run/docker.sock:/var/run/docker.sock # ... existing volumes ... ``` -Switch flavours by setting `FLOWFILE_KERNEL_IMAGE` (no rebuild required), e.g.: - -```bash -export FLOWFILE_KERNEL_IMAGE=edwardvaneechoud/flowfile-kernel-ml:0.3.0 -docker compose up -d -``` - -### Adding Extra Packages +Leave `FLOWFILE_KERNEL_IMAGE` empty to use the registry default, or set it to point at a local or alternative image. Kernels default to 4 GB of memory. -When you create a kernel from the UI, packages listed in the **Extra Python packages** field are baked into a per-kernel Docker image at creation time (a `FROM + RUN pip install` layer pinned against the flavour's constraints file). Creation takes ~30 s; subsequent kernel starts reuse the image and boot in seconds. The derived image is removed automatically when you delete the kernel. +### Adding extra packages -If you instead run the kernel container directly (no `flowfile-core` orchestration), the legacy `KERNEL_PACKAGES` env var still works as a runtime install — it is overridden to empty when launched by core. +When you create a kernel from the UI, packages listed in the **Extra Python packages** field are baked into a per-kernel Docker image at creation time (a `FROM + RUN pip install` layer pinned against the flavour's constraints file). Subsequent kernel starts reuse that image and boot quickly. The derived image is removed when you delete the kernel. --- @@ -191,11 +260,11 @@ The File Manager provides a web-based interface for uploading and downloading da *The File Manager showing uploaded files* -### Supported Formats +### Supported formats -CSV, Parquet, Excel (`.xlsx`), JSON, TSV, TXT +CSV, Parquet, Excel (`.xlsx` and `.xls`), JSON, TSV, TXT -### File Size Limit +### File size limit Maximum **500 MB** per file. @@ -209,3 +278,11 @@ Maximum **500 MB** per file. ### Access The File Manager is only available when `FLOWFILE_MODE=docker` is set. + +## Troubleshooting + +**Setup wizard keeps appearing.** The master key is not configured. Add `FLOWFILE_MASTER_KEY` to your `.env` and restart. + +**Invalid master key format.** The master key must be a valid Fernet key. Generate a new one with `python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())"`. Changing the key makes existing encrypted secrets unreadable. + +**Container fails to start.** Check the logs: `docker compose logs flowfile-core` / `docker compose logs flowfile-worker`. diff --git a/docs/users/deployment/index.md b/docs/users/deployment/index.md index 24905b2a3..1e81ce91d 100644 --- a/docs/users/deployment/index.md +++ b/docs/users/deployment/index.md @@ -8,17 +8,3 @@ Choose how to run Flowfile based on your needs. | [Desktop App](desktop.md) | Getting started, local development | No | | [Python Package](python.md) | Scripting, CI/CD, automation | No | | [Docker](docker.md) | Teams, servers, production | Yes | - -## Quick Comparison - -### Flowfile Lite (Browser) -No install at all — the visual editor runs entirely in your browser with Polars compiled to WebAssembly. A lightweight subset (18 nodes, no backend, no databases/cloud/scheduler/AI). Try it at [demo.flowfile.org](https://demo.flowfile.org). See the [Flowfile Lite guide](lite.md) for what's included. - -### Desktop App -Download and run. No setup required. Best for exploring Flowfile or building flows locally on macOS or Windows. - -### Python Package -`pip install flowfile`. Run flows programmatically or integrate into existing Python projects. - -### Docker -Full deployment with authentication, secrets management, and centralized storage. Best for teams and production environments. diff --git a/docs/users/deployment/lite.md b/docs/users/deployment/lite.md index 9472bceeb..fd53868b1 100644 --- a/docs/users/deployment/lite.md +++ b/docs/users/deployment/lite.md @@ -2,11 +2,7 @@ Flowfile Lite is the **zero-install, browser-only** edition of the visual editor. Polars runs entirely in WebAssembly via [Pyodide](https://pyodide.org/), so your flows execute client-side with **no backend, no account, and no data leaving your browser**. -
- ▶  Try Flowfile Lite in your browser  → -
- No install. No signup. Polars in the browser via Pyodide. -
+[Try Flowfile Lite in your browser →](https://demo.flowfile.org) — no install, no signup; Polars in the browser via Pyodide. It is also published as the embeddable npm package [`flowfile-editor`](https://www.npmjs.com/package/flowfile-editor), so you can drop the editor into your own web app. @@ -28,43 +24,30 @@ It is also published as the embeddable npm package [`flowfile-editor`](https://w ## What's included -Flowfile Lite ships **18 nodes** across 5 categories — the same canvas, settings panels, and Polars semantics as the full editor: +Flowfile Lite ships **23 nodes** (as of 2026-07) across five active categories — the same canvas, settings panels, and Polars semantics as the full editor. A sixth category, Machine Learning, is present in the palette but its nodes are locked (full-build only). | Category | Nodes | |----------|-------| -| **Input** | Read File (CSV · Excel · Parquet, local upload or remote URL), Manual Input, Read from Catalog, External Data¹ | -| **Transform** | Filter, Select, Sort, Unique, Take Sample, Polars Code | -| **Combine** | Join | -| **Aggregate** | Group By, Pivot, Unpivot | -| **Output** | Explore Data (Graphic Walker), Write Data (download CSV · Excel · Parquet), Write to Catalog, External Output¹ | +| **Input Sources** | Read File (CSV · Excel · Parquet, local upload or remote URL), Manual Input, External Data¹, Read from Catalog | +| **Transformations** | Filter, Select, Formula, Sort, Polars Code, Unique, Rename, Record ID, Take Sample | +| **Combine Operations** | Join, Cross Join, Union | +| **Aggregations** | Group By, Pivot, Unpivot | +| **Output Operations** | Explore Data (Graphic Walker), Write Data (download CSV · Excel · Parquet), Write to Catalog, External Output¹ | ¹ *External Data / External Output are host-integration nodes used when Flowfile Lite is embedded as a library — they let the host app feed in and read out datasets.* It also supports **exporting a flow to a standalone Python/Polars script** and a lightweight **in-browser catalog** (CSV-only) for saving and reusing tables between flows. -!!! tip "No Formula node? Use Polars Code" - Flowfile Lite swaps the visual [**Formula** node](../visual-editor/nodes/transform.md#formula) for the **Polars Code** node, where you can write any Polars expression directly (with autocompletion). Everything `with_columns` does in the full build is available here as code. +!!! tip "Formula and Polars Code both ship" + Lite includes the visual [**Formula** node](../visual-editor/nodes/transform.md#formula) for point-and-click column expressions *and* the **Polars Code** node for writing any Polars expression directly (with autocompletion). --- ## What's *not* included -Everything that depends on the Python backend, worker, or kernel containers is **unavailable** in Flowfile Lite: +Everything that depends on the Python backend, worker, or kernel containers is **unavailable** in Flowfile Lite; those nodes appear greyed-out in the palette so the full breadth stays discoverable. The [feature comparison](#feature-comparison) below lists what's excluded; the **Window Functions** node is also unavailable. -- **Databases** — no PostgreSQL / MySQL / SQL Server / Oracle readers or writers -- **Cloud storage** — no S3 / Azure ADLS / Google Cloud Storage -- **Kafka** ingestion and **REST API / Google Analytics** sources -- **Kernels / Python Script** — no sandboxed user-code execution -- **SQL editor** and SQL query node -- **Machine Learning nodes** (Train / Apply / Evaluate Model) -- **Scheduler** and flow automation -- **AI assistant** (Chat, Agent, Cmd+K, inline actions) -- **Secrets & connections manager** — no credentials are stored -- **Governed catalog** — no Delta Lake versioning, virtual tables, lineage, or saved visualizations -- **The Python API** (`flowfile_frame`) — Lite is visual-only -- Extra transforms: **Add Record ID, Text to Rows, Fuzzy Match, Union, Cross Join, Graph Solver** - -Memory is bounded by the browser heap, and data previews are capped (100k rows). +Memory is bounded by the browser heap, and the Explore Data view materializes at most 100k rows for charting. --- @@ -74,15 +57,17 @@ Memory is bounded by the browser heap, and data previews are capped (100k rows). |---------|:-------------:|:-------------:| | Install / runtime | pip · Desktop · Docker | None — runs in the browser | | Compute engine | Polars on Python backend + worker | Polars compiled to WebAssembly (Pyodide) | -| Nodes | 40+ | 18 | +| Nodes | 40+ | 23 (as of 2026-07) | | Local files (CSV / Excel / Parquet) | ✓ | ✓ | | Remote URL fetch | ✓ | ✓ | | Databases (Postgres / MySQL / …) | ✓ | ✗ | | Cloud storage (S3 / ADLS / GCS) | ✓ | ✗ | | Kafka / REST API / Google Analytics | ✓ | ✗ | +| Formula node | ✓ | ✓ | | Polars Code node | ✓ | ✓ | | Python Script / Kernels | ✓ | ✗ | -| SQL editor & SQL query | ✓ | ✗ | +| SQL Query node | ✓ | ✗ | +| Fuzzy Match / Graph Solver | ✓ | ✗ | | Machine Learning nodes | ✓ | ✗ | | Scheduler & automation | ✓ | ✗ | | AI assistant (BYOK) | ✓ | ✗ | diff --git a/docs/users/deployment/python.md b/docs/users/deployment/python.md index fb9c1f578..30258c43c 100644 --- a/docs/users/deployment/python.md +++ b/docs/users/deployment/python.md @@ -1,6 +1,6 @@ # Python Package -Install Flowfile as a Python package for programmatic use. +Install Flowfile as a Python package to build flows programmatically, run them in CI/CD, or open a flow you built in code in the visual editor. This page covers install and the two ways to launch the editor; the full API lives in the [Python API Guide](../python-api/index.md). ## Installation @@ -8,42 +8,42 @@ Install Flowfile as a Python package for programmatic use. pip install flowfile ``` -## Usage +## A first pipeline -```python -import flowfile as ff - -# Read data -df = ff.read_csv("data.csv") - -# Transform -df = df.with_columns( - flowfile_formulas=["[price] * [quantity]"], - output_column_names=["total"], -) +Every `flowfile` method builds a lazy flow graph; nothing runs until you call `.collect()` (or execute the flow). Writes are lazy too — `write_csv` appends an Output node to the graph and returns a `FlowFrame`; the file is written when the graph runs, not on the `write_csv` call itself. -# Write -df.write_csv("output.csv") +```python +--8<-- "docs/examples/first_pipeline.py:example" ``` +This reads a CSV, derives a column, filters, and aggregates — the same operations the visual editor exposes as nodes. + ## Running the Visual Editor -Launch the visual editor from Python: +There is no `ff.open_editor()`. Launch the editor one of two ways: + +Start the web UI (serves the editor at `http://localhost:63578`): ```python import flowfile as ff -# Open the visual editor -ff.open_editor() +ff.start_web_ui() ``` -## Features +Or open a graph you built in code directly in the editor: -- Full Python API for building flows programmatically -- Export visual flows to Python code -- Master key auto-generated and stored securely -- Integrates with existing Python projects +```python +import flowfile as ff + +graph = ff.create_flow_graph() +# ... add nodes to graph ... +ff.open_graph_in_editor(graph) +``` -## Documentation +The editor is normally started from the command line instead: + +```bash +flowfile run ui +``` -See the [Python API Guide](../python-api/index.md) for detailed documentation. +On first use, the setup screen generates and stores the encryption [master key](docker.md#first-run-master-key). diff --git a/docs/users/deployment/sharing.md b/docs/users/deployment/sharing.md new file mode 100644 index 000000000..257ca05dd --- /dev/null +++ b/docs/users/deployment/sharing.md @@ -0,0 +1,93 @@ +# Sharing resources with user groups + +This page is for admins and teams running Flowfile in Docker (multi-user) mode who want to share secrets, connections, and catalog resources across users. You'll learn how to create user groups, share a resource at the right access level, how Flowfile decides who can see what, and why shared secrets stay secure. + +!!! info "Docker mode only" + Group-based sharing exists only in the multi-user Docker deployment. In the desktop app and the pip-installed package there is one user, so the feature is dormant — the `/user-groups` and `/shares` endpoints return 404 and the sharing UI does not appear. + +## The model + +You don't share a resource with a person — you share it with a **group**, and people are members of groups. A share grants a group either **use** or **manage** access to one resource. A user can reach a resource if they own it, or if any group they belong to has been granted access to it. Sharing is authorization only: it changes who is allowed to use a resource, never the resource's stored data or its encryption. + +## The catalog is private by default + +In Docker mode every user sees only what they own, what has been shared with a group they belong to, and the seeded public system namespaces (`General`, `default`, `Unnamed Flows`, `Local Flows`) as tree containers. Global admins still see everything. + +If your team expects "everyone sees every flow and table," that visibility is not automatic — you have to create a group, add the members, and share the namespaces, tables, and flows you want everyone to see. No data is lost; only its visibility narrows. + +## User groups + +Groups are the principals you share with. A group has members, and each member has a role: + +| Role | Can do | +|---|---| +| **owner** | Manage membership and roles; administer the group. | +| **manager** | Manage membership and roles. | +| **member** | Belong to the group and reach whatever is shared with it. | + +Only a **global admin** creates a group; the creator is automatically added as its owner. From then on the group's owners and managers run membership themselves — a group owner does not need to be a global admin to add or remove members. Every group keeps at least one owner. + +Create and manage groups from the **User Groups** area of the app. + + + +## Sharing a resource + +A resource's owner (or a global admin) shares it with a group by choosing an access level: + +- **Use** — read and execute. A group member can run a flow, use a connection in a flow, or read a table, but cannot edit or re-share it. +- **Manage** — everything **use** allows, plus edit and re-share. + +Share resources with the **Share** action on the resource in the app. + +### What you can share + +Sharing covers secrets, the four connection types (database, cloud storage, Google Analytics, Kafka), and catalog content — namespaces, tables, flows, notebooks, visualizations, dashboards, and models. + +Sharing a **namespace** cascades: a grant on a namespace reaches the tables, flows, notebooks, visualizations, dashboards, and models inside it (and its direct child schemas). This is the efficient way to open up a whole area of the catalog to a group at once. + +### Secrets are use-only + +A secret can be shared for **use** but never for **manage**. A manage grant on a credential would imply the right to edit and re-share it, which collapses into handing over the plaintext. So a group's flows can run *with* a shared secret, but members can never view its value. Sharing a secret for use is therefore security-equivalent to giving the group that credential to run with. + +### Connections and the credential re-entry rule + +A shared connection is usable directly in a group member's flows. Because a connection bundles a target (host / endpoint / protocol) with the owner's credential, there is one guardrail: a **manage**-grantee who changes the connection's target must re-enter the credentials. Otherwise a grantee could repoint a shared connection at a server they control and harvest the owner's credential by capturing what it sends. When a manage-grantee rotates the credential, the new value is re-encrypted under the **owner's** key — the stored value never changes hands. + +## How access is resolved + +When you ask for a resource by name (a secret in a flow, a connection, a catalog table), Flowfile resolves it **own-first, then group-granted**: + +1. If you own a resource with that name, you get yours. +2. Otherwise, if a group you belong to has been granted access to a resource with that name, you get that one (lowest id wins on a name collision). + +Your own resources always shadow shared ones with the same name, so a shared resource can never silently override something you own. Global admins bypass these checks and can reach everything. + +## Shared secrets stay encrypted + +Granting a group access to a secret or connection never copies, re-encrypts, or exposes the credential. + +Every secret is stored as `$ffsec$1$$`, encrypted with a per-user key derived from the master key using the owner's user id. Because the owner's id is embedded in the stored value, decryption never depends on *who runs the flow*: when a group member's flow resolves a shared secret, Flowfile reads the owner id out of the value and re-derives the **owner's** key to decrypt it. The member's identity matters only for the authorization check — may this user resolve this secret by name. + +Consequences: + +- The plaintext is never returned to non-owners through the API; it only decrypts inside flow execution. +- Revoking a grant takes effect immediately. The grantee never held key material, so there is nothing to rotate or re-encrypt. +- The worker derives keys the same way and independently, so a shared secret decrypts unchanged wherever the flow runs. + +## Scheduling a shared flow + +A **use**-level member can schedule a shared flow. The scheduled run executes as that member, so the flow's secret and connection references resolve against *the member's own* grants. Before scheduling a shared flow, make sure the member also has access to everything the flow uses (its secrets, connections, and tables) — otherwise the run fails to resolve them. + +## Not covered by sharing + +Two things stay outside the sharing model: + +- **AI BYOK keys** — each user's provider keys are personal. +- **Uploaded files** — in Docker mode the uploads directory is already common to all users. + +## Related + +- [Docker deployment](docker.md) — running Flowfile in multi-user mode. +- [Connections & secrets](../visual-editor/connections.md) — creating the resources you share. +- [Schedules](../visual-editor/catalog/schedules.md) — automating shared flows. diff --git a/docs/users/formulas/index.md b/docs/users/formulas/index.md index f5021f64d..39a19ebbe 100644 --- a/docs/users/formulas/index.md +++ b/docs/users/formulas/index.md @@ -2,7 +2,7 @@ Flowfile formulas are a simple, Excel-like expression language for transforming data. Write `[column]` to reference a column, call functions like `round()` or `concat()`, and use `if ... then ... else ... endif` for conditional logic — the same way you would in a spreadsheet formula or a SQL `CASE` statement. -Every formula compiles to a native [Polars](https://pola.rs) expression before it runs, so there is no row-by-row Python overhead: formulas are as fast as hand-written Polars code. The same language is used everywhere in Flowfile — the visual editor and the Python API share it. +Every formula compiles to a native [Polars](https://pola.rs) expression before it runs, so there is no row-by-row Python overhead. The same language is used everywhere in Flowfile — the visual editor and the Python API share it. !!! tip "Try it in your browser" The [interactive formula playground](https://edwardvaneechoud.github.io/polars_expr_transformer/). Pick a sample dataset, type a formula, and watch the result, the generated Polars code, and the FlowFrame code update as you type. @@ -148,7 +148,3 @@ pl.when(pl.col("score") >= pl.lit(90)).then(pl.lit("A")) ``` This is also why flows that use formulas [export to clean Python code](../visual-editor/tutorials/code-generator.md): each formula has a direct Polars equivalent. - ---- - -*Next: browse the [function reference](functions.md), use formulas in the [Formula node](../visual-editor/nodes/transform.md#formula), or call them from the [Python API](../python-api/concepts/formulas.md).* diff --git a/docs/users/index.md b/docs/users/index.md index 7cdc43ae0..cc0c9eb88 100644 --- a/docs/users/index.md +++ b/docs/users/index.md @@ -1,80 +1,75 @@ -# Flowfile User Guides +# Guides by audience -Welcome to Flowfile! Whether you prefer visual drag-and-drop or writing code, we've got you covered. +Flowfile is one tool with several front doors. Pick the one that matches how you work — every path builds the same flows, and you can switch between them at any point. -## Choose Your Path +## Find your starting point -
+
-
+- :material-drag-variant: **Build flows visually** -### 🎨 [Visual Editor](visual-editor/index.md) + --- -Perfect for analysts and business users who want to build data pipelines visually. + Drag nodes onto a canvas and preview the data at every step — no code required. -**You'll learn:** + [:octicons-arrow-right-24: Visual Editor](visual-editor/index.md) -- Drag and drop nodes to build flows -- Configure transformations with forms -- Connect to databases and cloud storage -- Export your flows as Python code +- :material-microsoft-excel: **Coming from Excel** -[**Get Started with Visual Editor →**](visual-editor/index.md) + --- -
+ VLOOKUPs, pivot tables, and IF-formulas, translated into flows. -
+ [:octicons-arrow-right-24: Coming from Excel](coming-from-excel.md) -### 🐍 [Python API](python-api/index.md) +- :material-database: **Connect your data** -Perfect for developers and data scientists who prefer code. + --- -**You'll learn:** + Databases, S3/ADLS/GCS, Kafka, REST APIs — set up a connection once, use it everywhere. -- Build pipelines with Polars-compatible API -- Seamlessly integrate with existing code -- Visualize your code as flow graphs -- Use advanced features and optimizations + [:octicons-arrow-right-24: Connect Your Data](connect/index.md) -[**Get Started with Python →**](python-api/index.md) +- :material-chart-bar: **Analyze your data** -
+ --- -
+ Query catalog tables in the SQL editor, build visualizations, schedule refreshes. -## The Best of Both Worlds + [:octicons-arrow-right-24: Catalog](visual-editor/catalog/index.md) -The beauty of Flowfile is that **you don't have to choose**. You can: +- :material-language-python: **Write Python** -- Write code and visualize it instantly with `open_graph_in_editor()` -- Build visually and export as Python code -- Switch between visual and code at any time -- Collaborate across technical and non-technical teams + --- -## Quick Examples + Polars-style code that builds a visual flow as a side effect. -### Visual Approach -1. Drag a "Read Data" node onto canvas -2. Add a "Filter" node and connect them -3. Configure filter conditions in the form -4. Run and see results instantly + [:octicons-arrow-right-24: Python API](python-api/index.md) -### Code Approach -```python -import flowfile as ff +- :material-server: **Deploy for a team** -df = ff.read_csv("data.csv") -result = df.filter(ff.col("amount") > 100) -ff.open_graph_in_editor(result.flow_graph) # See it visually! -``` + --- + + Docker, users and groups, sharing, headless runs. + + [:octicons-arrow-right-24: Deploy & Operate](deployment/index.md) -## Where to Start? +
+ +## Visual and code are the same pipeline + +A flow built on the canvas and a pipeline written in Python construct the same graph underneath: + +- Write code, then inspect it on the canvas with `ff.open_graph_in_editor(df.flow_graph)`. +- Build visually, then [export the flow as standalone Python](visual-editor/tutorials/code-generator.md) — plain Polars, no Flowfile dependency required to run it. +- Hand a visual flow to a colleague who prefers code, or the other way around — both are views of the same graph. + +```python +--8<-- "docs/examples/sales_pipeline.py:example" +``` -- **New to Flowfile?** Start with our [Quick Start Guide](../quickstart.md) -- **Coming from Excel/Tableau?** Try the [Visual Editor](visual-editor/index.md) -- **Know Python/Pandas/Polars?** Jump into the [Python API](python-api/index.md) -- **Want to see real examples?** Check out our tutorials in either section +This is the same pipeline the [Quickstart](../quickstart.md) builds visually — read, deduplicate, filter, aggregate. ---- +## New here? -*Remember: Every visual flow can become code, and every code pipeline can be visualized. Choose what feels natural and switch whenever you want!* \ No newline at end of file +The [Quickstart](../quickstart.md) installs Flowfile and walks both paths in a few minutes: a visual flow that ends in the catalog, and the Python version of the same pipeline. diff --git a/docs/users/projects.md b/docs/users/projects.md new file mode 100644 index 000000000..ede92c546 --- /dev/null +++ b/docs/users/projects.md @@ -0,0 +1,78 @@ +# Projects: version your work with git + +This page is for anyone who wants a versioned, portable copy of their Flowfile work — flows, connections (without their secrets), schedules, and catalog metadata — tracked in a git repository. You'll learn how a project mirrors your install to a folder, how to save and open versions, and what does and doesn't end up in git. + +## What a project is + +A project is a folder that Flowfile keeps in sync with the resources you own. As you save flows, edit connections, and manage catalog metadata, Flowfile writes a deterministic, human-readable copy of everything into the project folder and tracks it with git. You get version history, diffs, and a folder you can commit to your own remote, share with a teammate, or copy to another machine. + +Your Flowfile database stays the source of truth at runtime. The project folder is a **mirror**: + +- **Projection (database → files)** happens automatically in the background whenever you save a flow, change a connection, edit a schedule, or update catalog metadata. You never trigger it by hand, and a sync error never blocks the save that caused it. +- **Import (files → database)** happens only at explicit moments: when you open a project, reload it from disk, or restore an older version. + +!!! info "Not in Flowfile Lite" + Projects require the full desktop or server build. The browser-only [Flowfile Lite](deployment/lite.md) edition does not track work in git. + +## Secrets never enter the folder + +A project folder is safe to commit. Secret **values** are never written into it — only the manifest of which secret names exist. Connections are mirrored without their bundled credentials. Flowfile also writes a `.gitignore` into every project (and hardens a hand-written one) so that credential and local-state files can never be committed, including `.env`, `*.secret`, `*.db`, `master_key.txt`, `*.key`, `*.pem`, and any materialized `data/` or `catalog_tables/`. + +When you open a project on a new machine, the secret manifest tells Flowfile which values are still missing. Set them by name (see the CLI output below) or fill them in the app afterward. + +## What's in the folder + +A project folder holds YAML mirrors of your resources: + +| Path | Contents | +|---|---| +| `project.yaml` | Project manifest — name, id, format version, data-tracking toggle. | +| `flows/` | One `.flow.yaml` per registered flow. | +| `connections/database/`, `connections/cloud/` | One `.yaml` per connection (no credentials). | +| `schedules/` | One `.yaml` per schedule. | +| `notebooks/` | One `.notebook.yaml` per notebook. | +| `secrets.yaml` | The names of secrets that must be supplied (no values). | +| `namespaces.yaml`, `tables.yaml`, `models.yaml`, `kernels.yaml`, `visualizations.yaml`, `dashboards.yaml` | Catalog metadata manifests. | +| `.gitignore` | Guards against committing secrets and local data. | + +### Tracking data artifacts + +The `project.yaml` manifest carries an all-or-nothing **track data artifacts** toggle (on by default). When on, catalog-table metadata (`tables.yaml`) and global-model metadata (`models.yaml`) are mirrored too. Turn it off to keep a project focused on flows and connections and leave data-artifact metadata out of git entirely. The materialized table data itself is never committed regardless — only its metadata. + +## Working with projects in the app + +Project actions live in the app's project controls. The core operations are: + +- **Initialize** a new project in a folder — writes the manifest and `.gitignore`, initializes git, and makes a first commit. +- **Open** an existing project folder — rebuilds your database resources from the files and reports what was imported. +- **Save a version** — projects the current state to files and commits it with your message. +- **Restore a version** — resets the folder to an earlier commit and rebuilds the database to match. +- **Reload from disk** — accepts changes made to the files outside Flowfile (for example a `git pull`) by rebuilding the database from them. + + + +!!! warning "Restore and reload rebuild from the files" + Restoring a version and reloading from disk both reset your resources to match the folder, pruning anything not present in the files. If you have unsaved changes, Flowfile stops and asks you to confirm; when you force the action it first autosaves your current state into git history so nothing is silently lost. + +## Working with projects from the command line + +Every core action has a headless equivalent, useful for scripting or CI. See the [CLI reference](deployment/cli.md#projects) for full details. + +```bash +flowfile project init # create a project in +flowfile project open # open a project and import it +flowfile project save "" # save a version with a commit message +``` + +When you open a project whose secrets aren't set locally, `flowfile project open` reports how many values are missing. Supply them with `FLOWFILE_SECRET_` environment variables or fill them in the app. + +## Availability by mode + +Projects are always available in the desktop app and the pip-installed package. In **Docker** (multi-tenant) mode the project router is disabled by default and returns 404; an operator enables it by setting `FLOWFILE_ENABLE_PROJECTS` to a truthy value (`true`/`1`/`yes`/`on`), and in that mode projects are admin-only and confined to each owner's own subtree. + +## Related + +- [CLI reference](deployment/cli.md) — headless `flowfile project` commands. +- [Catalog](visual-editor/catalog/index.md) — the flows, tables, and metadata a project mirrors. +- [Connections & secrets](visual-editor/connections.md) — connections are mirrored; their secrets are not. +- For the internals (projection hooks, import boundaries, the git folder contract), see [Architecture](../for-developers/architecture.md). diff --git a/docs/users/python-api/concepts/design-concepts.md b/docs/users/python-api/concepts/design-concepts.md index 3861ab5d8..09c4dd4f5 100644 --- a/docs/users/python-api/concepts/design-concepts.md +++ b/docs/users/python-api/concepts/design-concepts.md @@ -1,197 +1,84 @@ -# FlowFrame and FlowGraph Design Concepts +# FlowFrame and FlowGraph -Understanding how FlowFrame and FlowGraph work together is key to mastering Flowfile. This guide explains the core design principles that make Flowfile both powerful and intuitive. +FlowFrame and FlowGraph are the two objects behind every Python pipeline: the always-lazy frame you call methods on, and the graph that records each operation as a node. Understanding how they interact explains almost everything else about the API — why nothing executes until `.collect()`, why every pipeline can open in the visual editor, and why schemas resolve instantly. -!!! tip "Related Reading" - - **Practical Implementation**: See these concepts in action in our [Code to Flow guide](../tutorials/flowfile_frame_api.md) - - **Architecture Overview**: Learn about the system design in [Technical Architecture](../../../for-developers/architecture.md#technical-architecture) - - **Visual Building**: Compare with [Building Flows](../../visual-editor/building-flows.md) visually +## FlowFrame: always lazy, always connected -## FlowFrame: Always Lazy, Always Connected - -### What is FlowFrame? - -FlowFrame is Flowfile's version of a Polars DataFrame with a crucial difference: **it's always lazy and always connected to a graph**. +A FlowFrame looks like a Polars DataFrame but differs in two ways: it is always lazy, and it always belongs to a graph. ```python import flowfile as ff -# This creates a FlowFrame, not a regular DataFrame df = ff.FlowFrame({ "id": [1, 2, 3, 4, 5], "amount": [100, 250, 80, 300, 150], "category": ["A", "B", "A", "C", "B"] }) -print(type(df)) # -print(type(df.data)) # +print(type(df)) # +print(type(df.data)) # ``` -### Key Properties of FlowFrame +### Nothing executes until `.collect()` -#### 1. Always Lazy Evaluation -A `FlowFrame` never loads your actual data into memory until you explicitly call `.collect()`. This means you can build complex transformations on massive datasets without consuming memory: +Method calls only build the plan. Data is read and processed once, when you collect — with the whole plan visible to the Polars optimizer: ```python -# None of this processes any data yet df = ( ff.FlowFrame({ "id": [1, 2, 3, 4, 5], - "amount": [500, 1200, 800, 1500, 900], + "amount": [500, 1200, 800, 1500, 900], "category": ["A", "B", "A", "C", "B"] - }) # Creates manual input node - .filter(ff.col("amount") > 1000) # No filtering happens yet - .group_by("category") # No grouping happens yet - .agg(ff.col("amount").sum()) # No aggregation happens yet + }) + .filter(ff.col("amount") > 1000) + .group_by("category") + .agg(ff.col("amount").sum()) # still nothing has executed ) -# Only now does the data get processed -result = df.collect() # Everything executes at once, optimized +result = df.collect() # everything runs here, optimized as one plan ``` -!!! info "Performance Benefits" - This lazy evaluation is powered by Polars and explained in detail in our [Technical Architecture guide](../../../for-developers/architecture.md#the-power-of-lazy-evaluation). +### Every operation becomes a graph node -#### 2. Connected to a DAG (Directed Acyclic Graph) -Every FlowFrame has a reference to a FlowGraph that tracks every operation as a node: +Each FlowFrame knows its graph and its own position in it: ```python -df = ff.FlowFrame({ - "id": [1, 2, 3], - "name": ["Alice", "Bob", "Charlie"], - "active": [True, False, True] -}) -print(df.flow_graph) # Shows the graph this FlowFrame belongs to -print(df.node_id) # Shows which node in the graph this FlowFrame represents +df = ff.FlowFrame({"id": [1, 2, 3], "name": ["Alice", "Bob", "Charlie"]}) +print(df.flow_graph) # the graph this frame belongs to +print(df.node_id) # the node this frame represents ``` -For a deeper understanding of how this DAG works internally, see [FlowGraph in the Developers Guide](../../../for-developers/flowfile-core.md#2-adding-a-node-where-settings-come-to-life)). - -#### 3. Linear Operation Tracking -Each operation creates a new node in the graph, even if you repeat the same operation: +Operations always append — an identical operation applied twice creates two nodes, and because both frames share one graph, the node count is cumulative: ```python -df = ff.FlowFrame({ - "id": [1, 2, 3, 4], - "amount": [50, 150, 75, 200], - "status": ["active", "inactive", "active", "active"] -}) -print(f"Initial graph has {len(df.flow_graph.nodes)} nodes") +df = ff.FlowFrame({"id": [1, 2, 3, 4], "amount": [50, 150, 75, 200]}) +print(len(df.flow_graph.nodes)) # 1 -# First filter - creates node 1 df1 = df.filter(ff.col("amount") > 100) -print(f"After first filter: {len(df1.flow_graph.nodes)} nodes") - -# Second identical filter - creates node 2 (not reused!) -df2 = df.filter(ff.col("amount") > 100) -print(f"After second filter: {len(df2.flow_graph.nodes)} nodes") +print(len(df1.flow_graph.nodes)) # 2 -# Both operations are tracked separately in the graph +df2 = df.filter(ff.col("amount") > 100) # identical filter, new node +print(len(df2.flow_graph.nodes)) # 3 ``` -## FlowGraph: The Pipeline's Blueprint +!!! note "Not every method gets its own node type" + Operations with a visual-node equivalent (formula-based filters, group-by, joins) appear as that node type; anything else lands in a generic `polars_code` node. The pipeline still works identically — the difference is only how editable the step is in the visual editor. See [Expressions](expressions.md) and [Formulas in Python](formulas.md) for which form produces which. -### What is FlowGraph? +## FlowGraph: the pipeline's record -FlowGraph is the "brain" behind FlowFrame - it's a Directed Acyclic Graph (DAG) that tracks every step in your data transformation pipeline. +The FlowGraph is the DAG all of a pipeline's frames share: ```python -# Access the graph from any FlowFrame -df = ff.FlowFrame({ - "product": ["Widget", "Gadget", "Tool"], - "price": [10.99, 25.50, 8.75], - "quantity": [100, 50, 200] -}) graph = df.flow_graph - -print(f"Graph ID: {graph.flow_id}") -print(f"Number of operations: {len(graph.nodes)}") -print(f"Node connections: {graph.node_connections}") -``` - -## Visual Integration - -### Viewing Your Graph -Every FlowFrame can show its complete operation history in the visual editor: - -```python -# Build a pipeline -result = ( - ff.FlowFrame({ - "region": ["North", "South", "North", "East", "South"], - "amount": [1000, 0, 1500, 800, 1200], - "product": ["A", "B", "A", "C", "B"] - }, description="Load sales data") - .filter(ff.col("amount") > 0, description="Remove invalid amounts") - .group_by("region", description="Group by sales region") - .agg(ff.col("amount").sum().alias("total_sales")) -) - -# Open the entire pipeline in the visual editor -ff.open_graph_in_editor(result.flow_graph) -``` - -!!! tip "Learn More" - See [Visual UI Integration](../reference/visual-ui.md) for details on launching and controlling the visual editor from Python. - -This opens a visual representation showing: -- Each operation as a node -- Data flow between operations -- Descriptions you added for documentation -- Schema changes at each step - -### Real-time Schema Prediction -The DAG enables instant schema prediction without processing data: - -```python -df = ff.FlowFrame({ - "product": ["Widget", "Gadget"], - "price": [10.50, 25.00], - "quantity": [2, 3] -}) -print("Original schema:", df.schema) - -# Schema is predicted instantly, no data processed -transformed = df.with_columns([ - (ff.col("price") * ff.col("quantity")).alias("total") -]) -print("New schema:", transformed.schema) # Shows new 'total' column immediately -``` - -!!! info "How Schema Prediction Works" - Learn about the closure pattern that enables this in [The Magic of Closures](../../../for-developers/design-philosophy.md#flowfile-the-use-of-closures). - -## Practical Implications - -### Memory Efficiency -Since FlowFrame is always lazy: - -```python -# This can handle large datasets efficiently through lazy evaluation -large_pipeline = ( - ff.FlowFrame({ - "id": list(range(10000)), - "quality_score": [0.1, 0.9, 0.8, 0.95] * 2500, # Simulating large data - "value": list(range(10000)), - "category": ["A", "B", "C", "D"] * 2500 - }) - .filter(ff.col("quality_score") > 0.95) # Reduces data early - .select(["id", "value", "category"]) # Reduces columns early - .group_by("category") - .agg(ff.col("value").mean()) -) - -# Only processes what's needed when you collect -result = large_pipeline.collect() # Optimized execution plan +print(graph.flow_id) # graph id +print(len(graph.nodes)) # operations so far +print(graph.node_connections) # how they're wired ``` -!!! tip "Performance Guide" - For more on optimization strategies, see [Execution Methods](../../../for-developers/design-philosophy.md#3-execution-is-everything) in our philosophy guide. +### Branching shares the graph -### Graph Reuse and Copying -You can work with the same graph across multiple FlowFrames: +Branches created from a common base are endpoints in one graph, not copies: ```python -# Start with common base base = ff.FlowFrame({ "region": ["North", "South", "East"], "year": [2024, 2024, 2023], @@ -200,69 +87,47 @@ base = ff.FlowFrame({ "quantity": [10, 15, 8] }).filter(ff.col("year") == 2024) -# Create different branches (same graph, different endpoints) sales_summary = base.group_by("region").agg(ff.col("sales").sum()) product_summary = base.group_by("product").agg(ff.col("quantity").sum()) -# Both share the same underlying graph assert sales_summary.flow_graph is product_summary.flow_graph ``` -## Best Practices +### Schema prediction without execution -### 1. Use Descriptions for Complex Pipelines -```python -import flowfile as ff -pipeline = ( - ff.FlowFrame({ - "customer_id": [1, 2, 3, 4, 5], - "status": ["active", "inactive", "active", "active", "inactive"], - "signup_date": ["2024-01-15", "2023-12-10", "2024-02-20", "2023-11-05", "2024-03-01"], - "customer_segment": ["premium", "basic", "premium", "basic", "premium"], - "revenue": [1000, 500, 1500, 300, 2000] - }, description="Load raw customer data") - .filter(ff.col("status") == "active", description="Keep only active customers") - .with_columns([ - ff.col("signup_date").str.strptime(ff.Date, "%Y-%m-%d").alias("signup_date") - ], description="Parse signup dates") - .group_by("customer_segment", description="Aggregate by customer segment") - .agg([ - ff.col("revenue").sum().alias("total_revenue"), - ff.col("customer_id").count().alias("customer_count") - ], description="Calculate segment metrics") -) -``` +Because the graph knows every operation, output schemas resolve immediately — no data is read: -### 2. Visualize During Development ```python -# Check your pipeline structure frequently -ff.open_graph_in_editor(pipeline.flow_graph) +df = ff.FlowFrame({"product": ["Widget", "Gadget"], "price": [10.50, 25.00], "quantity": [2, 3]}) +transformed = df.with_columns((ff.col("price") * ff.col("quantity")).alias("total")) +print(transformed.schema) # Schema([('product', String), ('price', Float64), ('quantity', Int64), ('total', Float64)]) ``` -!!! example "Complete Examples" - - **Database Pipeline**: See [PostgreSQL Integration](../../visual-editor/tutorials/database-connectivity.md) for a real-world ui example - - **Cloud Pipeline**: Check [Cloud Connections](../../visual-editor/tutorials/cloud-connections.md) for S3 workflows - - **Export to Code**: Learn how your pipelines convert to pure Python in [Flow to Code](../../visual-editor/tutorials/code-generator.md) +`df.schema` returns a Polars `Schema` — a mapping keyed by column name (`transformed.schema["total"]`), not a list. + +## Opening a pipeline in the visual editor -## Summary +Any pipeline can be inspected on the canvas. The `description=` argument most methods accept becomes the node's description there, which is what makes a code-built graph readable to someone else: -FlowFrame and FlowGraph work together to provide: +```python +result = ( + ff.FlowFrame({ + "region": ["North", "South", "North", "East", "South"], + "amount": [1000, 0, 1500, 800, 1200] + }, description="Load sales data") + .filter(ff.col("amount") > 0, description="Remove invalid amounts") + .group_by("region") + .agg(ff.col("amount").sum().alias("total_sales")) +) -- **Lazy evaluation**: No memory waste, optimal performance -- **Complete lineage**: Every operation is tracked and visualizable -- **Real-time feedback**: Instant schema prediction and error detection -- **Seamless integration**: Switch between code and visual editing -- **Polars compatibility**: Very identical API with additional features -- **Automatic adaptation**: Complex operations automatically fall back to code nodes +ff.open_graph_in_editor(result.flow_graph) +``` -Understanding this design helps you build efficient, maintainable data pipelines that scale from quick analyses to production ETL workflows. +See [Visual UI Integration](../reference/visual-ui.md) for how the editor is launched and controlled from Python, and [Export to Python](../../visual-editor/tutorials/code-generator.md) for the reverse direction. -## Related Documentation +## Related -- **[FlowFrame Operations](../reference/flowframe-operations.md)** - Available transformations and methods -- **[Expressions](expressions.md)** - Polars-style column operations -- **[Formulas in Python](formulas.md)** - Methods that accept Flowfile formula strings -- **[Joins](../reference/joins.md)** - Combining datasets -- **[Aggregations](../reference/aggregations.md)** - Group by and summarization -- **[Visual UI Integration](../reference/visual-ui.md)** - Working with the visual editor -- **[Developers guide](../../../for-developers/index.md)** - Core architecture and design philosophy \ No newline at end of file +- [FlowFrame Operations](../reference/flowframe-operations.md) — the transformation methods +- [Expressions](expressions.md) — Polars-style column operations +- [Formulas in Python](formulas.md) — methods that accept Flowfile formula strings +- [Developers guide](../../../for-developers/index.md) — how these objects are implemented diff --git a/docs/users/python-api/concepts/expressions.md b/docs/users/python-api/concepts/expressions.md index 02858c0c4..4a422feb9 100644 --- a/docs/users/python-api/concepts/expressions.md +++ b/docs/users/python-api/concepts/expressions.md @@ -1,6 +1,9 @@ # Expressions -FlowFrame methods accept standard **Polars expressions** — `ff.col`, operators, `ff.when`, and the rest of the expression API you already know from Polars. Expressions are the default and most powerful way to express transformations, covering the entire Polars API. +FlowFrame methods accept standard **Polars expressions** — `ff.col`, operators, `ff.when`, and most of the expression API you already know from Polars. Expressions are the default way to express transformations. + +!!! note "Most of Polars, not all of it" + Nearly every Polars `Expr` method is available, but a few names track the pinned Polars version (`cum_sum`, not `cumsum`; `dt.weekday`, not `day_of_week`), some helpers are selectors rather than top-level functions (`ff.all_()`, not `ff.all()`; use `ff.col("*").exclude(...)`, there is no `ff.exclude()`), and expressions without a dedicated node render as `polars_code` nodes in the visual editor. See [FlowFrame and FlowGraph](design-concepts.md). ## Column references and arithmetic @@ -43,11 +46,7 @@ df = df.with_columns([ ]) ``` -Because expressions stay lazy, they compose into a single optimized query plan — see [FlowFrame and FlowGraph](design-concepts.md). +Because expressions stay lazy, they compose into one query plan that runs when you `.collect()` — see [FlowFrame and FlowGraph](design-concepts.md). !!! tip "Looking for the Excel-like `[column]` syntax?" That is the **Flowfile formula language** — a separate, simpler syntax shared with the visual editor. See [Formulas in Python](formulas.md) for the FlowFrame methods that accept it, and the [Formula Language guide](../../formulas/index.md) for the language itself. - ---- - -*Want to see more of the Flowfile Python API? Check out the [reference documentation](../reference/index.md).* diff --git a/docs/users/python-api/concepts/formulas.md b/docs/users/python-api/concepts/formulas.md index 3f48b5a80..c7524629b 100644 --- a/docs/users/python-api/concepts/formulas.md +++ b/docs/users/python-api/concepts/formulas.md @@ -6,7 +6,7 @@ A few FlowFrame methods accept **Flowfile formula** strings — the same Excel-l "[price] * [quantity]" # Equivalent to ff.col("price") * ff.col("quantity") ``` -This page documents the methods that accept formulas. The language itself — syntax, operators, and all 95 built-in functions — is documented in the [Formula Language guide](../../formulas/index.md). For everything else, use regular [Polars expressions](expressions.md). +This page documents the methods that accept formulas. The language itself — syntax, operators, and the full [built-in function reference](../../formulas/functions.md) — is documented in the [Formula Language guide](../../formulas/index.md). For everything else, use regular [Polars expressions](expressions.md). !!! tip "Try it in your browser" The [interactive playground](https://edwardvaneechoud.github.io/polars_expr_transformer/) shows the generated FlowFrame code for every formula you type. @@ -66,7 +66,7 @@ output = df.with_columns([ ) ``` -**When to use which?** [Polars expressions](expressions.md) cover the entire Polars API and are the better fit for complex transformations. Formulas shine for simple, readable column logic and render as editable Formula/Filter node settings when you [open the pipeline in the visual editor](../reference/visual-ui.md). +**When to use which?** [Polars expressions](expressions.md) cover most of the Polars API and are the better fit for complex transformations. Formulas are best for simple, readable column logic and render as editable Formula/Filter node settings when you [open the pipeline in the visual editor](../reference/visual-ui.md). --- diff --git a/docs/users/python-api/concepts/index.md b/docs/users/python-api/concepts/index.md index 12f9c6302..bcee8c33a 100644 --- a/docs/users/python-api/concepts/index.md +++ b/docs/users/python-api/concepts/index.md @@ -1,105 +1,21 @@ # Core Concepts -Understanding the key concepts behind Flowfile's Python API will help you build better pipelines. +These guides explain the model behind Flowfile's Python API: what a FlowFrame is, how expressions and formulas differ, and why every operation is lazy and graph-connected. -## Available Guides +## Guides ### [FlowFrame and FlowGraph](design-concepts.md) -The fundamental building blocks of Flowfile pipelines. -**You'll learn:** -- What FlowFrame is and how it differs from DataFrames -- How FlowGraph tracks your operations -- Why everything is lazy by default -- How visual and code representations connect - -**Key takeaways:** -- FlowFrame = Your data + its transformation history -- FlowGraph = The complete pipeline blueprint -- Every operation creates a node in the graph - ---- +The fundamental building blocks. Covers how a FlowFrame differs from a DataFrame, how the FlowGraph tracks each operation as a node, why everything is lazy, and how the code and visual representations connect. ### [Expressions](expressions.md) -Polars-style column operations — the default way to express transformations. - -**You'll learn:** -- Column references, arithmetic, and conditional logic with `ff.col` / `ff.when` -- Filtering with expression predicates -- How expressions compose into one lazy query plan ---- +Polars-style column operations — the default way to express transformations. Column references, arithmetic, conditional logic with `ff.when`, filtering, and the `.str` / `.dt` / `.list` namespaces. ### [Formulas in Python](formulas.md) -The FlowFrame methods that accept Flowfile formula strings. - -**You'll learn:** -- Creating columns with `with_columns(flowfile_formulas=...)` -- Filtering with `filter(flowfile_formula=...)` and `filter_split` -- When to reach for a formula instead of an expression - -**Key takeaway:** the formula language itself (syntax, operators, all functions) is documented separately in the [Formula Language guide](../../formulas/index.md). - -## Quick Overview - -### FlowFrame vs DataFrame - -| DataFrame (Pandas/Polars) | FlowFrame (Flowfile) | -|--------------------------|---------------------| -| Holds data in memory | Always lazy (data not loaded) | -| Operations execute immediately | Operations build a plan | -| No operation history | Full operation history in graph | -| Can't visualize workflow | Can open in visual editor | - -### The Lazy Advantage - -```python -# This doesn't load the 10GB file! -df = ff.read_csv("huge_file.csv") - -# Still no data loaded - just building the plan -df = df.filter(ff.col("country") == "USA") -df = df.select(["id", "amount"]) - -# NOW it loads only what's needed -result = df.collect() # Might only read 100MB! -``` - -### Visual Integration - -Every FlowFrame knows its history: - -```python -# Build a complex pipeline -pipeline = ( - ff.read_csv("input.csv") - .filter(ff.col("active") == True) - .group_by("category") - .agg(ff.col("revenue").sum()) -) - -# See the entire pipeline visually -ff.open_graph_in_editor(pipeline.flow_graph) - -# The graph shows all 4 operations as connected nodes -``` - -## Why These Concepts Matter - -Understanding these concepts helps you: - -1. **Write efficient code** - Leverage lazy evaluation -2. **Debug effectively** - Visualize your pipeline -3. **Collaborate better** - Share visual representations -4. **Optimize performance** - Understand what executes when - -## Learn More -- **Deep dive:** Read the full [FlowFrame and FlowGraph guide](design-concepts.md) -- **Expressions:** Learn [Polars-style expressions](expressions.md) -- **Formulas:** Use the [formula-accepting methods](formulas.md) -- **Practice:** Try the [tutorials](../tutorials/index.md) +The FlowFrame methods that accept Flowfile formula strings: `with_columns(flowfile_formulas=...)`, `filter(flowfile_formula=...)`, and `filter_split`. The formula language itself (syntax, operators, functions) is documented in the [Formula Language guide](../../formulas/index.md). ---- +## The lazy model -*These concepts are the foundation of Flowfile. Understanding them will make everything else click!* \ No newline at end of file +A FlowFrame is always lazy: operations build a plan and append a node to the FlowGraph, and nothing runs until `.collect()`. See [FlowFrame and FlowGraph](design-concepts.md) for a worked example. diff --git a/docs/users/python-api/index.md b/docs/users/python-api/index.md index 695203e29..2c3b0271a 100644 --- a/docs/users/python-api/index.md +++ b/docs/users/python-api/index.md @@ -1,22 +1,13 @@ # Python API -Build data pipelines programmatically with Flowfile's Polars-compatible API. +Build data pipelines programmatically with Flowfile's Polars-compatible API. For Python developers who want version-controllable, reproducible pipelines that can still be opened in the visual editor. !!! info "Backend required — not in Flowfile Lite" The Python API runs the full Flowfile engine in-process. It is **not** part of the browser-only [Flowfile Lite](../deployment/lite.md) edition, which is visual-only. Install the [Python package](../deployment/python.md) to use this API. -!!! info "If You Know Polars, You Know Flowfile" - Our API is designed to be a seamless extension of Polars. The majority of the methods are identical, so you can leverage your existing knowledge to be productive from day one. The main additions are features that connect your code to the broader Flowfile ecosystem, like cloud integrations and UI visualization. +If you know Polars, most of this API will look familiar: the `FlowFrame` mirrors a Polars `LazyFrame`, and expressions (`ff.col`, `ff.when`, the `.str`/`.dt` namespaces) work the same way. The additions are the parts that connect your code to the rest of Flowfile — cloud and database connectors, the catalog, and opening a pipeline in the visual editor. - -## Who This Is For - -- **Python developers** who prefer code over drag-and-drop -- **Data scientists** familiar with Polars or Pandas -- **Engineers** building automated data pipelines -- **Anyone** who needs version control and programmatic pipeline generation - -## Quick Example +## Quick example ```python import flowfile as ff @@ -26,20 +17,20 @@ result = df.filter(ff.col("amount") > 100).group_by("region").agg( ff.col("amount").sum() ) -# Visualize your pipeline +# Open the pipeline in the visual editor ff.open_graph_in_editor(result.flow_graph) ``` ## Documentation ### [Quick Start](quickstart.md) -Get up and running in 5 minutes with your first pipeline. +Install the package and build your first pipeline. ### [Core Concepts](concepts/index.md) -- [FlowFrame and FlowGraph](concepts/design-concepts.md) - Fundamental building blocks -- [Expressions](concepts/expressions.md) - Polars-style column operations -- [Formulas in Python](concepts/formulas.md) - Methods that accept the Excel-like [formula language](../formulas/index.md) +- [FlowFrame and FlowGraph](concepts/design-concepts.md) — the lazy, graph-connected data model +- [Expressions](concepts/expressions.md) — Polars-style column operations +- [Formulas in Python](concepts/formulas.md) — methods that accept the Excel-like [formula language](../formulas/index.md) ### [API Reference](reference/index.md) @@ -51,15 +42,12 @@ Get up and running in 5 minutes with your first pipeline. - [Joins](reference/joins.md) - [Cloud Storage](reference/cloud-connections.md) - [Visual UI Integration](reference/visual-ui.md) +- [Catalog References](reference/catalog-references.md) ### [Tutorials](tutorials/index.md) - [Building Flows with Code](tutorials/flowfile_frame_api.md) -## For Contributors - -Want to understand how Flowfile works internally or contribute to the project? See the [Developer Documentation](../../for-developers/index.md) for architecture details and internal API reference. - ---- +## For contributors -*Prefer visual workflows? Check out the [Visual Editor Guide](../visual-editor/index.md).* \ No newline at end of file +To understand how Flowfile works internally or contribute to the project, see the [Developer Documentation](../../for-developers/index.md). diff --git a/docs/users/python-api/quickstart.md b/docs/users/python-api/quickstart.md index 93ea1263e..b74c80d3c 100644 --- a/docs/users/python-api/quickstart.md +++ b/docs/users/python-api/quickstart.md @@ -1,6 +1,6 @@ # Python API Quick Start -Get up and running with Flowfile's Python API in 5 minutes. +Install the package, build a first pipeline, and tour the operations you'll reach for most. For Python developers writing Flowfile pipelines in code. ## Installation @@ -8,107 +8,97 @@ Get up and running with Flowfile's Python API in 5 minutes. pip install flowfile ``` -## Your First Pipeline +## Your first pipeline -```python -import flowfile as ff - -# Load data -df = ff.read_csv("sales.csv", description="Load sales data") - -# Transform -result = ( - df - .filter(ff.col("amount") > 100, description="Filter large sales") - .with_columns([ - (ff.col("amount") * 1.1).alias("amount_with_tax") - ], description="Add tax calculation") - .group_by("region") - .agg([ - ff.col("amount").sum().alias("total_sales"), - ff.col("amount").mean().alias("avg_sale") - ]) -) - -# Get results as Polars DataFrame -data = result.collect() -print(data) +This pipeline reads a CSV, derives a column, filters, and aggregates. It runs against committed sample data and is executed by the docs test suite: -# Visualize in the UI -ff.open_graph_in_editor(result.flow_graph) +```python +--8<-- "docs/examples/first_pipeline.py:example" ``` -## Key Concepts +`collect()` runs the plan and returns a Polars `DataFrame`. + +## Key ideas ### FlowFrame -Your data container - like a Polars LazyFrame but tracks all operations: + +Your data container — like a Polars `LazyFrame`, but every operation is also recorded as a graph node: ```python -# Create from various sources -df = ff.FlowFrame({"col1": [1, 2, 3]}) # From dict -df = ff.read_csv("file.csv") # From CSV -df = ff.read_parquet("file.parquet") # From Parquet +import flowfile as ff + +df = ff.FlowFrame({"col1": [1, 2, 3]}) # from a dict +df = ff.read_csv("file.csv") # from CSV +df = ff.read_parquet("file.parquet") # from Parquet ``` -### Always Lazy +### Always lazy + Operations don't execute until you call `.collect()`: ```python -# These operations just build the plan +# These calls just build the plan df = ff.read_csv("huge_file.csv") df = df.filter(ff.col("status") == "active") df = df.select(["id", "name", "amount"]) -# Now it executes everything efficiently +# Now it executes, reading only what's needed result = df.collect() ``` +Check `df.schema` to see column types without running anything. + ### Descriptions -Document your pipeline as you build: + +Any operation accepts a `description` that shows up on the node in the visual editor: ```python df = ( ff.read_csv("input.csv", description="Raw customer data") .filter(ff.col("active") == True, description="Keep active only") - .drop_duplicates(description="Remove duplicates") + .unique(description="Remove duplicates") ) ``` -## Common Operations +## Common operations ### Filtering + ```python -# Polars style +# Polars expression predicate df.filter(ff.col("age") > 21) -# Flowfile formula style -df.filter(flowfile_formula="[age] > 21 AND [status] = 'active'") +# Flowfile formula (renders as an editable Filter node) +df.filter(flowfile_formula="[age] > 21 and [status] = 'active'") ``` -### Adding Columns +### Adding columns + ```python -# Standard way +# Expression form df.with_columns([ (ff.col("price") * ff.col("quantity")).alias("total") ]) -# Formula syntax +# Formula form df.with_columns( flowfile_formulas=["[price] * [quantity]"], - output_column_names=["total"] + output_column_names=["total"], ) ``` -### Grouping & Aggregation +### Grouping and aggregation + ```python df.group_by("category").agg([ ff.col("sales").sum().alias("total_sales"), ff.col("sales").mean().alias("avg_sales"), - ff.col("id").count().alias("count") + ff.col("id").count().alias("count"), ]) ``` ### Joining + ```python customers = ff.read_csv("customers.csv") orders = ff.read_csv("orders.csv") @@ -116,106 +106,42 @@ orders = ff.read_csv("orders.csv") result = customers.join( orders, left_on="customer_id", - right_on="cust_id", - how="left" + right_on="customer_id", + how="left", ) ``` -## Cloud Storage +See the [Joins reference](reference/joins.md) for the full set of strategies. -```python -from pydantic import SecretStr - -# Set up S3 connection -ff.create_cloud_storage_connection_if_not_exists( - ff.FullCloudStorageConnection( - connection_name="my-s3", - storage_type="s3", - auth_method="access_key", - aws_region="us-east-1", - aws_access_key_id="your-key", - aws_secret_access_key=SecretStr("your-secret") - ) -) +## Cloud storage -# Read from S3 -df = ff.scan_parquet_from_cloud_storage( - "s3://bucket/data.parquet", - connection_name="my-s3" -) +Store an S3 connection once, then read and write with it by name. This tested example round-trips a Parquet aggregate through S3: -# Write to S3 -df.write_parquet_to_cloud_storage( - "s3://bucket/output.parquet", - connection_name="my-s3" -) +```python +--8<-- "docs/examples/integrations/cloud_storage_s3.py:example" ``` -## Visual Integration +!!! info "Local S3-compatible stacks vs plain S3" + That example sets `endpoint_url`, `aws_allow_unsafe_html=True`, and inline keys to reach a local MinIO stack. Against real AWS S3, the connection needs only its `connection_name`, `storage_type`, `auth_method`, `aws_region`, and credentials — no endpoint URL and no unsafe-HTML flag. Once the connection exists, reads and writes just reference it by name. See [Cloud Connection Management](reference/cloud-connections.md). -### Open in Editor -```python -# Build pipeline in code -pipeline = ff.read_csv("data.csv").filter(ff.col("value") > 100) +## Visual integration -# Open in visual editor -ff.open_graph_in_editor(pipeline.flow_graph) -``` +### Open in the editor -### Start Web UI ```python -# Launch the web interface -ff.start_web_ui() # Opens browser automatically +pipeline = ff.read_csv("data.csv").filter(ff.col("value") > 100) +ff.open_graph_in_editor(pipeline.flow_graph) ``` -## Complete Example +### Start the web UI ```python -import flowfile as ff - -# Build a complete ETL pipeline -pipeline = ( - ff.read_csv("raw_sales.csv", description="Load raw sales") - .filter(ff.col("amount") > 0, description="Remove invalid") - .with_columns([ - ff.col("date").str.strptime(ff.Date, "%Y-%m-%d"), - (ff.col("amount") * ff.col("quantity")).alias("total") - ], description="Parse dates and calculate totals") - .group_by([ff.col("date").dt.year().alias("year"), "product"]) - .agg([ - ff.col("total").sum().alias("revenue"), - ff.col("quantity").sum().alias("units_sold") - ]) - .sort("revenue", descending=True) -) - -# Execute and get results -results = pipeline.collect() -print(results) - -# Visualize the pipeline -ff.open_graph_in_editor(pipeline.flow_graph) - -# Save results -pipeline.write_parquet("yearly_sales.parquet") +ff.start_web_ui() # opens a browser tab ``` -## Next Steps - -- 📖 [Core Concepts](concepts/design-concepts.md) - Understand FlowFrame and FlowGraph -- 📚 [API Reference](reference/index.md) - Detailed documentation -- 🎯 [Tutorials](tutorials/index.md) - Real-world examples -- 🔄 [Visual Integration](reference/visual-ui.md) - Working with the UI - -## Tips - -1. **Use descriptions** - They appear in the visual editor -2. **Think lazy** - Build your entire pipeline before collecting -3. **Leverage formulas** - Use `[column]` syntax for simpler expressions -4. **Visualize often** - `open_graph_in_editor()` helps debug -5. **Check schemas** - Use `df.schema` to see structure without running - ---- +## Next steps -*Ready for more? Check out the [full API reference](reference/index.md) or learn about [core concepts](concepts/design-concepts.md).* -Or want to see another example? Checkout the [quickstart guide](../../quickstart.md#technical-quickstart)! +- [Core Concepts](concepts/design-concepts.md) — the FlowFrame and FlowGraph model +- [API Reference](reference/index.md) — method-by-method documentation +- [Tutorials](tutorials/index.md) — worked pipelines +- [Visual UI Integration](reference/visual-ui.md) — moving between code and the editor diff --git a/docs/users/python-api/reference/aggregations.md b/docs/users/python-api/reference/aggregations.md index bac5a6fc7..8206d523b 100644 --- a/docs/users/python-api/reference/aggregations.md +++ b/docs/users/python-api/reference/aggregations.md @@ -1,8 +1,18 @@ # Aggregations -Group by and aggregate operations for summarizing data. +Group rows and summarize them with `group_by().agg()`, and compute per-group running values with window functions. A `group_by` followed by `agg` fuses into a single group_by node in the graph, however many aggregations it carries. -## Basic Group By +## A worked example + +This runs against committed data and is executed by the docs test suite: + +```python +--8<-- "docs/examples/python_api_aggregations.py:example" +``` + +The sections below break down each part. + +## Basic group by ```python import flowfile as ff @@ -10,51 +20,54 @@ import flowfile as ff df = ff.FlowFrame({ "category": ["A", "B", "A", "B", "A"], "value": [10, 20, 30, 40, 50], - "quantity": [1, 2, 3, 4, 5] + "quantity": [1, 2, 3, 4, 5], }) -# Simple aggregation result = df.group_by("category").agg([ ff.col("value").sum().alias("total_value"), ff.col("value").mean().alias("avg_value"), - ff.col("quantity").count().alias("count") + ff.col("quantity").count().alias("count"), ]) -# With description +# A description shows up on the group_by node in the visual editor. result = df.group_by("category", description="Group by product category").agg([ - ff.col("value").sum().alias("total_value") + ff.col("value").sum().alias("total_value"), ]) ``` -## Multiple Grouping Columns +## Multiple grouping columns ```python result = df.group_by(["region", "category"]).agg([ ff.col("sales").sum().alias("total_sales"), - ff.col("sales").mean().alias("avg_sales") + ff.col("sales").mean().alias("avg_sales"), ]) ``` -## Complex Group By +## Grouping by an expression ```python -# Group by expression (creates polars_code node) +# Grouping by an expression emits a polars_code node (not an editable group_by node). result = df.group_by([ - ff.col("date").dt.year().alias("year") + ff.col("date").dt.year().alias("year"), ]).agg([ - ff.col("amount").sum() + ff.col("amount").sum(), ]) +``` -# Dynamic aggregation -result = df.group_by("category").agg([ - ff.all().sum() # Sum all numeric columns -]) +## Aggregating every numeric column + +The `ff.all_()` selector expands to all columns. Use it inside `select` (not directly inside a native `group_by().agg()`, which needs a concrete column name per aggregation): + +```python +# Sum a whole frame of numeric columns +totals = df.select(ff.numeric()).select(ff.all_().sum()) ``` -## Available Aggregations +## Available aggregations -| Function | Description | -|----------|-------------| +| Method | Description | +|--------|-------------| | `sum()` | Sum of values | | `mean()` | Average value | | `median()` | Median value | @@ -65,21 +78,24 @@ result = df.group_by("category").agg([ | `var()` | Variance | | `first()` | First value in group | | `last()` | Last value in group | -| `list()` | Collect values into list | +| `implode()` | Collect the group's values into a list | + +!!! note "`implode`, not `list`" + To gather a group's values into a single list column, use `implode()`. `.list` is a namespace accessor for existing list columns, not an aggregation. -## Window Functions +## Window functions + +Window functions compute a value per row *within* a group, without collapsing the frame. Use `.over(...)`: ```python -# Running calculations df = df.with_columns([ - ff.col("value").cumsum().over("category").alias("running_total"), - ff.col("value").rank().over("category").alias("rank") + ff.col("value").cum_sum().over("category").alias("running_total"), + ff.col("value").rank(descending=True).over("category").alias("rank"), ]) ``` -!!! note "Node Type Selection" - Simple group_by operations create UI nodes. Complex expressions in group_by create `polars_code` nodes. - +!!! note "Use `cum_sum`, not `cumsum`" + The cumulative-sum expression follows the pinned Polars name: `cum_sum()`. `cumsum()` raises `AttributeError`. --- -[← Previous: Expressions](flowframe-operations.md) | [Next: Joins →](joins.md) +[← Previous: DataFrame Operations](flowframe-operations.md) | [Next: Joins →](joins.md) diff --git a/docs/users/python-api/reference/catalog-references.md b/docs/users/python-api/reference/catalog-references.md index 851af6527..58fa570e2 100644 --- a/docs/users/python-api/reference/catalog-references.md +++ b/docs/users/python-api/reference/catalog-references.md @@ -62,9 +62,9 @@ for schema in catalog.list_schemas(): print(schema.name, schema.list_tables()) ``` -#### `list_tables() -> list[CatalogTableOut]` +#### `list_tables() -> list[CatalogTableOut]` — CatalogReference -Return tables across **every** schema in this catalog, as a flat list. Each row's `namespace_id` field tells you which schema it belongs to. For a per-schema view, use [`SchemaReference.list_tables()`](#list_tables-list-catalogtableout). +Return tables across **every** schema in this catalog, as a flat list. Each row's `namespace_id` field tells you which schema it belongs to. For a per-schema view, use [`SchemaReference.list_tables()`](#list_tables-listcatalogtableout-schemareference). ```python for table in catalog.list_tables(): @@ -104,7 +104,7 @@ Like `CatalogReference`, schema references are immutable, hashable, and picklabl ### Methods -#### `list_tables() -> list[CatalogTableOut]` +#### `list_tables() -> list[CatalogTableOut]` — SchemaReference Return tables registered in this schema. @@ -186,10 +186,13 @@ orders = raw.read_table("orders") clean = ( orders .filter(ff.col("status") != "cancelled") - .with_columns(ff.col("total").cast(float)) + .with_columns(ff.col("total").cast(ff.Float64)) ) staging.write_table(clean, "orders_clean", write_mode="overwrite") # Discover what's there print([t.name for t in catalog.list_tables()]) ``` + +--- +[← Previous: Visual UI Integration](visual-ui.md) diff --git a/docs/users/python-api/reference/cloud-connections.md b/docs/users/python-api/reference/cloud-connections.md index 95d95f413..42e3d29b7 100644 --- a/docs/users/python-api/reference/cloud-connections.md +++ b/docs/users/python-api/reference/cloud-connections.md @@ -1,18 +1,13 @@ -# Cloud Connection Management +# Cloud Connections in Python -Flowfile provides secure, centralized management for cloud storage connections. Connections can be created through code or the UI—both store credentials in an encrypted database. -On this page we will cover how to create and manage them in Python. If you want to learn how to create them in the UI, -check out the [UI guide](../../visual-editor/tutorials/cloud-connections.md). +A cloud storage connection stores a provider, its credentials (encrypted, scoped to your user), and a name to reference them by. Code and the UI share one connection store: a connection created here appears in the Cloud Storage Reader/Writer nodes' dropdowns, and one created in the [UI](../../visual-editor/tutorials/cloud-connections.md) is usable from `ff.*` functions by name. ## Creating Connections -### Code Approach - ```python import flowfile as ff from pydantic import SecretStr -# Create a new S3 connection ff.create_cloud_storage_connection( ff.FullCloudStorageConnection( connection_name="data-lake", @@ -25,28 +20,6 @@ ff.create_cloud_storage_connection( ) ``` -### Visual Editor Integration - -Connections created through code are immediately available in the Flowfile visual editor: - -```python -# Create connection in code -ff.create_cloud_storage_connection( - ff.FullCloudStorageConnection( - connection_name="data-lake", - # ... parameters - ) -) - -# This connection now appears in: -# - Cloud Storage Reader node's connection dropdown -# - Cloud Storage Writer node's connection dropdown -# - Any other nodes that use cloud connections -``` - -!!! info "Seamless Integration" - There's no difference between connections created via code or UI. Both are stored in the same encrypted database and are instantly available across all interfaces. - ## Connection Types ### S3 Connection (Access Key) @@ -67,11 +40,41 @@ ff.FullCloudStorageConnection( ff.FullCloudStorageConnection( connection_name="my-s3-cli", storage_type="s3", - auth_method="aws_cli", # Uses local AWS CLI credentials + auth_method="aws-cli", # Uses local AWS CLI credentials (note the hyphen) aws_region="us-east-1" ) ``` +!!! warning "The CLI auth literal is `aws-cli` (hyphen)" + `auth_method="aws_cli"` (underscore) raises a Pydantic `ValidationError`. + +### Connection fields + +`FullCloudStorageConnection` covers all three cloud backends. `storage_type` selects one; fill only the fields for that provider. + +| Field group | Fields | Used by | +|---|---|---| +| Identity | `connection_name`, `storage_type` (`"s3"` / `"adls"` / `"gcs"`), `auth_method` | all | +| AWS S3 | `aws_region`, `aws_access_key_id`, `aws_secret_access_key`, `aws_role_arn`, `aws_session_token`, `aws_allow_unsafe_html`, `endpoint_url` | `s3` | +| Azure ADLS | `azure_account_name`, `azure_account_key`, `azure_tenant_id`, `azure_client_id`, `azure_client_secret`, `azure_sas_token` | `adls` | +| Google GCS | `gcs_service_account_key`, `gcs_project_id` | `gcs` | + +`auth_method` accepts `access_key`, `iam_role`, `service_principal`, `managed_identity`, `sas_token`, `aws-cli`, `env_vars`, and `service_account` — pick the one your `storage_type` supports. Secret fields (`aws_secret_access_key`, `azure_account_key`, `gcs_service_account_key`, …) take a `SecretStr`. + +!!! note "`aws_allow_unsafe_html`" + This flag permits plain-HTTP (non-TLS) S3 endpoints. Set it to `True` only for local or dev stacks such as MinIO reached over `http://`; leave it unset for real AWS. + +### Round-trip example + +The tested integration example creates a connection, writes Parquet to S3, and reads it back: + +```python +--8<-- "docs/examples/integrations/cloud_storage_s3.py:example" +``` + +!!! info "S3-compatible local stacks vs plain S3" + The `endpoint_url`, `aws_allow_unsafe_html`, and inline keys in that example wire an S3-compatible local stack (MinIO). Against real AWS S3, a connection needs only `connection_name`, `storage_type`, `auth_method`, `aws_region`, and credentials — no `endpoint_url` and no unsafe-HTML flag. + ## Managing Connections ### Create If Not Exists @@ -126,21 +129,6 @@ df.write_parquet_to_cloud_storage( ) ``` -## Security Features - -### Credential Encryption - -- All credentials are encrypted before storage -- Secrets never appear in logs or error messages -- Use `SecretStr` wrapper for sensitive values - -### User Isolation - -- Connections are scoped to the current user -- Each user manages their own connections -- No cross-user credential access - - ## Troubleshooting ### Common Issues @@ -150,7 +138,7 @@ df.write_parquet_to_cloud_storage( | "Connection not found" | Ensure connection exists with `get_all_available_cloud_storage_connections()` | | "Access denied" | Verify credentials and permissions | | "Invalid endpoint" | Check `endpoint_url` for custom S3 services | -| "SSL verification failed" | Use `aws_allow_unsafe_html=True` for local/dev endpoints only | +| Cannot reach a plain-HTTP endpoint | Set `aws_allow_unsafe_html=True` — for local/dev endpoints only | ### Debug Connection @@ -167,8 +155,5 @@ if my_conn: print(f"Region: {my_conn.aws_region}") ``` -!!! tip "UI Integration" - All connections created via code are immediately available in the UI's connection dropdown when configuring nodes. - --- -[← Previous: Joins](joins.md) | [Next: visual Ui →](visual-ui.md) +[← Previous: Joins](joins.md) | [Next: Visual UI Integration →](visual-ui.md) diff --git a/docs/users/python-api/reference/data-types.md b/docs/users/python-api/reference/data-types.md index 83b77f7e9..7bf4f21d7 100644 --- a/docs/users/python-api/reference/data-types.md +++ b/docs/users/python-api/reference/data-types.md @@ -40,15 +40,21 @@ df = df.with_columns([ ## Schema Inspection +`df.schema` returns a Polars `Schema` — a dict-like mapping of column name to dtype, resolved without reading any data. Access dtypes by column name, not by position. + ```python -# Get schema without processing data +# Get the schema without processing data print(df.schema) -# [Column(name='int_col', dtype=Int64), ...] +# Schema([('int_col', Int64), ('str_col', String)]) -# Check specific column type -print(df.schema[0].dtype) +# Look up a specific column's type by name +print(df.schema["int_col"]) # Int64 + +# Enumerate columns and types +for name, dtype in df.schema.items(): + print(name, dtype) ``` --- -[← Previous: Writing data](writing-data.md) | [Next: FlowFile Operations →](flowframe-operations.md) +[← Previous: Writing Data](writing-data.md) | [Next: DataFrame Operations →](flowframe-operations.md) diff --git a/docs/users/python-api/reference/flowframe-operations.md b/docs/users/python-api/reference/flowframe-operations.md index 5af925c65..0c44d140d 100644 --- a/docs/users/python-api/reference/flowframe-operations.md +++ b/docs/users/python-api/reference/flowframe-operations.md @@ -1,6 +1,18 @@ # DataFrame Operations -Core operations for transforming data. All standard Polars operations are supported with additional Flowfile features. The `flowfile_formula` examples below use the [Flowfile formula language](../../formulas/index.md). +Core row- and column-level transforms: filter, select, add or modify columns, sort, deduplicate, and the string/date/list expression namespaces. Every FlowFrame method mirrors its Polars counterpart and accepts an optional `description` that shows up as node documentation in the visual editor. + +The `flowfile_formula` examples below use the [Flowfile formula language](../../formulas/index.md); everything else is a [Polars expression](../concepts/expressions.md). + +## A worked example + +This runs against committed data and is executed by the docs test suite: + +```python +--8<-- "docs/examples/python_api_operations.py:example" +``` + +The sections below break down each operation. ## Filtering @@ -9,52 +21,58 @@ import flowfile as ff df = ff.FlowFrame({"price": [10, 20, 30], "qty": [5, 0, 10]}) -# Standard Polars filter +# Polars expression predicate df = df.filter(ff.col("price") > 15) -# With description +# With a description (surfaces in the visual editor) df = df.filter(ff.col("price") > 15, description="Keep items over $15") # Flowfile formula syntax -df = df.filter(flowfile_formula="[price] > 15 AND [qty] > 0") +df = df.filter(flowfile_formula="[price] > 15 and [qty] > 0") ``` -## Selecting Columns +!!! note "Which node the filter becomes" + `filter(flowfile_formula=...)` emits an editable Filter node. A plain `filter(ff.col(...) > x)` predicate emits a `polars_code` node instead — the result is identical, but only the formula form is editable in the visual editor. + +## Selecting columns ```python -# Select specific columns +# Select specific columns by name df = df.select(["price", "qty"]) # Select with expressions df = df.select([ ff.col("price"), - ff.col("qty").alias("quantity") + ff.col("qty").alias("quantity"), ]) -# Exclude columns -df = df.select(ff.exclude("internal_id")) +# Keep everything except one column with a column selector. +# There is no ff.exclude() — use ff.col("*").exclude(...) or the selectors module. +df = df.select(ff.col("*").exclude("internal_id")) ``` -## Adding/Modifying Columns +!!! tip "Selectors" + `ff.numeric()`, `ff.string()`, `ff.all_()`, and the other [selector helpers](../concepts/expressions.md) pick columns by dtype or pattern — e.g. `df.select(ff.numeric())` keeps only numeric columns. + +## Adding and modifying columns ```python -# Standard with_columns +# Expression form df = df.with_columns([ - (ff.col("price") * ff.col("qty")).alias("total") + (ff.col("price") * ff.col("qty")).alias("total"), ]) -# Flowfile formula syntax +# Flowfile formula form df = df.with_columns( flowfile_formulas=["[price] * [qty]"], output_column_names=["total"], - description="Calculate line totals" + description="Calculate line totals", ) ``` ## Sorting ```python -# Sort by column df = df.sort("price") df = df.sort("price", descending=True) @@ -62,67 +80,68 @@ df = df.sort("price", descending=True) df = df.sort(["category", "price"], descending=[False, True]) ``` -## Unique Operations +## Removing duplicates ```python -# Get unique rows +# Drop fully duplicate rows df = df.unique() -# Unique by specific columns +# Deduplicate on a subset of columns df = df.unique(subset=["product_id"]) - -# Drop duplicates (alias) -df = df.drop_duplicates(subset=["product_id"]) ``` +!!! warning "No `drop_duplicates`" + FlowFrame does not expose `drop_duplicates`. Use `unique()` (optionally with `subset=[...]` and `keep="first"`). -## String Operations +## String operations ```python df = df.with_columns([ ff.col("name").str.to_uppercase().alias("name_upper"), ff.col("code").str.slice(0, 3).alias("prefix"), - ff.col("text").str.contains("pattern").alias("has_pattern") + ff.col("text").str.contains("pattern").alias("has_pattern"), ]) ``` -## Conditional Logic +## Conditional logic ```python -# When/then/otherwise df = df.with_columns([ ff.when(ff.col("price") > 100) .then(ff.lit("Premium")) .when(ff.col("price") > 50) .then(ff.lit("Standard")) .otherwise(ff.lit("Budget")) - .alias("tier") + .alias("tier"), ]) ``` -## Date Operations +## Date operations ```python df = df.with_columns([ ff.col("date").dt.year().alias("year"), ff.col("date").dt.month().alias("month"), ff.col("date").dt.day().alias("day"), - ff.col("date").dt.weekday().alias("weekday") + ff.col("date").dt.weekday().alias("weekday"), ]) ``` -## List Operations +!!! note "Polars renames apply" + The expression namespaces track the pinned Polars version: use `dt.weekday()` (not `day_of_week`) and `cum_sum()` (not `cumsum`). + +## List operations ```python df = df.with_columns([ ff.col("tags").list.len().alias("tag_count"), ff.col("values").list.sum().alias("total"), - ff.col("items").list.first().alias("first_item") + ff.col("items").list.first().alias("first_item"), ]) ``` -!!! note "Polars Compatibility" - All standard Polars DataFrame methods work identically. See [Polars docs](https://pola-rs.github.io/polars/py-polars/html/reference/dataframe/index.html) for complete reference. +!!! note "Polars compatibility" + Most Polars `Expr` methods are available. See the [Polars docs](https://pola-rs.github.io/polars/py-polars/html/reference/dataframe/index.html) for the full method reference; a few methods are renamed or fall back to `polars_code` nodes — see [Expressions](../concepts/expressions.md). --- [← Previous: Data Types](data-types.md) | [Next: Aggregations →](aggregations.md) diff --git a/docs/users/python-api/reference/index.md b/docs/users/python-api/reference/index.md index 8212684c5..b49b8a85f 100644 --- a/docs/users/python-api/reference/index.md +++ b/docs/users/python-api/reference/index.md @@ -20,7 +20,7 @@ This section documents Flowfile's Python API, focusing on extensions and differe - [**Catalog References**](catalog-references.md) - Typed catalog/schema handles for the Flowfile catalog - [**Cloud Storage**](cloud-connections.md) - S3 integration -- [**visualize pipelines**](visual-ui.md) - Working with the visual editor +- [**Visual UI Integration**](visual-ui.md) - Working with the visual editor ## Key Extensions to Polars @@ -33,7 +33,7 @@ df = df.filter(ff.col("active") == True, description="Keep active records") ### Flowfile Formula Syntax Alternative bracket-based syntax for expressions: ```python -df.filter(flowfile_formula="[price] > 100 AND [quantity] >= 10") +df.filter(flowfile_formula="[price] > 100 and [quantity] >= 10") ``` See [Formulas in Python](../concepts/formulas.md) for the methods that accept formulas, the [Formula Language guide](../../formulas/index.md) for the syntax, or try it in the [interactive playground](https://edwardvaneechoud.github.io/polars_expr_transformer/). @@ -55,18 +55,4 @@ ff.open_graph_in_editor(df.flow_graph) ## Architecture Deep Dives -For understanding how Flowfile works internally: - -- [**Core Architecture**](../../../for-developers/flowfile-core.md#1-the-flowgraph-your-pipeline-orchestrator) - FlowGraph, FlowNode, and FlowDataEngine internals -- [**Design Philosophy**](../../../for-developers/design-philosophy.md) - The dual interface approach - - -## Getting Help - -- **Not finding a method?** Check the [Polars documentation](https://pola-rs.github.io/polars/py-polars/html/reference/) - most methods work identically -- **Need examples?** See our [tutorials](../tutorials/index.md) -- **Understanding concepts?** Read about [FlowFrame and FlowGraph](../concepts/design-concepts.md) - ---- - -*This reference covers Flowfile-specific features. For standard Polars operations, see the [Polars API Reference](https://pola-rs.github.io/polars/py-polars/html/reference/).* \ No newline at end of file +For internals, see the [Developer Documentation](../../../for-developers/index.md). \ No newline at end of file diff --git a/docs/users/python-api/reference/joins.md b/docs/users/python-api/reference/joins.md index c9d50c943..5eaa3cc44 100644 --- a/docs/users/python-api/reference/joins.md +++ b/docs/users/python-api/reference/joins.md @@ -1,105 +1,130 @@ # Joins -Combining data from multiple FlowFrames. +Combine two FlowFrames on one or more keys, stack frames vertically, and validate keys before joining. Joins support the strategies `inner`, `left`, `right`, `full`, `semi`, `anti`, `outer`, plus `cross`. -## Basic Join +## A worked example + +This runs against committed data and is executed by the docs test suite: + +```python +--8<-- "docs/examples/python_api_joins.py:example" +``` + +The sections below break down each pattern. + +## Basic join ```python import flowfile as ff customers = ff.FlowFrame({ "id": [1, 2, 3], - "name": ["Alice", "Bob", "Charlie"] + "name": ["Alice", "Bob", "Charlie"], }) orders = ff.FlowFrame({ "order_id": [101, 102, 103], "customer_id": [1, 2, 1], - "amount": [100, 200, 150] + "amount": [100, 200, 150], }) -# Inner join result = customers.join( orders, left_on="id", right_on="customer_id", how="inner", - description="Join customers with orders" + description="Join customers with orders", ) ``` -## Join Types +## Join types ```python -# Inner join (default) +# Inner join (default): rows with a match on both sides df1.join(df2, on="key", how="inner") -# Left join +# Left join: all left rows, matched right columns or null df1.join(df2, on="key", how="left") -# Outer join -df1.join(df2, on="key", how="outer") +# Right join: all right rows +df1.join(df2, on="key", how="right") -# Semi join (filter df1 by df2) +# Full join: all rows from both sides (unmatched filled with null) +df1.join(df2, on="key", how="full") + +# Semi join: left rows that have a match (keeps only left columns) df1.join(df2, on="key", how="semi") -# Anti join (exclude matches) +# Anti join: left rows with no match df1.join(df2, on="key", how="anti") ``` -## Multiple Join Keys +!!! note "`full`, not `outer`" + Under the pinned Polars version the full outer strategy is `how="full"`. `how="outer"` is accepted as a legacy alias, but new code should use `full`. + +## Multiple join keys ```python result = df1.join( df2, - on=["region", "year"], # Join on multiple columns - how="inner" + on=["region", "year"], # join on several columns + how="inner", ) -# Different column names +# Different column names on each side result = df1.join( df2, left_on=["region_code", "period"], right_on=["region", "year"], - how="left" + how="left", ) ``` -## Cross Join +## Cross join ```python -# Cartesian product +# Cartesian product of both frames result = df1.join(df2, how="cross") ``` -## Union/Concatenation +## Stacking frames + +There is no `vstack` on FlowFrame — use `ff.concat` to stack frames vertically: ```python # Vertical concatenation combined = ff.concat([df1, df2, df3]) -# Union (removes duplicates) -union_df = df1.unique().vstack(df2.unique()).unique() +# Concatenate then drop duplicate rows +union_df = ff.concat([df1, df2]).unique() -# Diagonal concatenation (handles different schemas) +# Diagonal concatenation aligns differing schemas combined = ff.concat([df1, df2], how="diagonal") ``` -## Join Validation +## Join validation + +FlowFrame has no frame-level `len()` or `n_unique()`. Count distinct keys with an expression inside `select`, and count matched rows by collecting a small aggregate rather than taking `len()` of a frame: ```python -# Check for duplicates before joining -if df2.select("customer_id").n_unique() < len(df2): +# Are there duplicate keys in the right table? +key_stats = df2.select( + ff.col("customer_id").n_unique().alias("distinct_keys"), + ff.col("customer_id").count().alias("total_rows"), +).collect() +if key_stats["distinct_keys"][0] < key_stats["total_rows"][0]: print("Warning: duplicate keys in right table") -# Validate join results +# How many left rows found no match? result = df1.join(df2, on="id", how="left") -unmatched = result.filter(ff.col("amount").is_null()) -print(f"Unmatched records: {len(unmatched)}") +unmatched = result.filter(ff.col("amount").is_null()).select( + ff.col("id").count().alias("unmatched") +).collect() +print(f"Unmatched records: {unmatched['unmatched'][0]}") ``` -!!! warning "Unsupported Join Types" - Currently, `join_asof` and `join_where` are not supported in Flowfile. These operations will need to be implemented using alternative approaches or raw Polars code. +!!! warning "`join_asof` and `join_where` are not supported" + These methods are present on FlowFrame (they are injected from Polars) but raise at call time — Flowfile has no native node for them. Use raw Polars in a Python Script node for time- or predicate-based joins. --- -[← Previous: Aggregations](aggregations.md) | [Next: Cloud Connection →](cloud-connections.md) +[← Previous: Aggregations](aggregations.md) | [Next: Cloud Storage →](cloud-connections.md) diff --git a/docs/users/python-api/reference/reading-data.md b/docs/users/python-api/reference/reading-data.md index a50df0589..542687e2f 100644 --- a/docs/users/python-api/reference/reading-data.md +++ b/docs/users/python-api/reference/reading-data.md @@ -52,22 +52,24 @@ df = ff.read_parquet("sales_data.parquet", description="Q4 sales results") ### Arrow IPC / Feather, NDJSON, and Avro Files Flowfile also reads Arrow IPC/Feather, newline-delimited JSON (NDJSON), and Avro -files as first-class connectors: +files. These readers live in `flowfile_frame` and are **not** re-exported on the +`ff` namespace — import them directly: ```python +from flowfile_frame import read_ipc, read_ndjson, read_avro, scan_ipc, scan_ndjson + # Arrow IPC / Feather (lazy scan — like parquet) -df = ff.read_ipc("data.arrow", description="Arrow IPC source") +df = read_ipc("data.arrow", description="Arrow IPC source") # Newline-delimited JSON (lazy scan) -df = ff.read_ndjson("events.ndjson") +df = read_ndjson("events.ndjson") # Avro (eager read — offloaded to the worker so core never holds the dataset) -df = ff.read_avro("data.avro") +df = read_avro("data.avro") ``` -IPC and NDJSON are scanned lazily, so they expose `scan_ipc` / `scan_ndjson` -aliases too. Avro has no lazy scan in Polars, so its read is offloaded to the -worker. +IPC and NDJSON are scanned lazily, so they also provide `scan_ipc` / `scan_ndjson`. +Avro has no lazy scan in Polars, so its read is offloaded to the worker. ### Scanning vs Reading @@ -78,9 +80,11 @@ Flowfile provides both `read_*` and `scan_*` functions for Polars compatibility: df1 = ff.read_csv("data.csv") df2 = ff.scan_csv("data.csv") # Alias for read_csv -# Also available for the new lazy formats -df3 = ff.scan_ipc("data.arrow") -df4 = ff.scan_ndjson("events.ndjson") +# The lazy IPC/NDJSON scans are imported from flowfile_frame (not ff.*) +from flowfile_frame import scan_ipc, scan_ndjson + +df3 = scan_ipc("data.arrow") +df4 = scan_ndjson("events.ndjson") ``` ## Cloud Storage Reading @@ -134,7 +138,7 @@ df = ff.read_from_cloud_storage( ### Format-Specific Cloud Readers -### Cloud CSV Reading +#### Cloud CSV Reading ```python # Read from S3 with connection @@ -153,7 +157,10 @@ df = ff.scan_csv_from_cloud_storage( ) ``` -### Cloud Parquet Reading +!!! note "CSV delimiter default" + The cloud CSV readers default `delimiter=";"`. Pass `delimiter=","` explicitly for comma-separated files (as above). + +#### Cloud Parquet Reading ```python # Single file @@ -170,7 +177,7 @@ df = ff.scan_parquet_from_cloud_storage( ) ``` -### Cloud JSON Reading +#### Cloud JSON Reading ```python df = ff.scan_json_from_cloud_storage( @@ -179,7 +186,7 @@ df = ff.scan_json_from_cloud_storage( ) ``` -### Delta Lake Reading +#### Delta Lake Reading ```python # Latest version @@ -188,11 +195,11 @@ df = ff.scan_delta( connection_name="data-lake-connection" ) -# Specific version (if supported) +# Time-travel to a specific version df = ff.scan_delta( "s3://data-lake/delta-table", - connection_name="data-lake-connection" - # Note: version parameter support depends on implementation + connection_name="data-lake-connection", + version=5, ) ``` @@ -235,16 +242,16 @@ Returns a `FlowFrame`. Use `.collect()` to materialize, `.data` to access the un ### Query with SQL -Use `read_catalog_sql()` to execute SQL queries against all catalog tables — both physical and virtual. Tables are registered by name in a Polars SQL context. +Use `read_catalog_sql()` to execute SQL queries against all catalog tables — both physical and virtual. Tables are registered by name in a Polars SQL context. `read_catalog_sql` is imported from `flowfile_frame` (it is not on the `ff` namespace): ```python -import flowfile as ff +from flowfile_frame import read_catalog_sql # Query a single table -df = ff.read_catalog_sql("SELECT * FROM customers WHERE region = 'Europe'") +df = read_catalog_sql("SELECT * FROM customers WHERE region = 'Europe'") # Join across catalog tables -df = ff.read_catalog_sql(""" +df = read_catalog_sql(""" SELECT o.order_id, c.name, o.total FROM orders o JOIN customers c ON o.customer_id = c.id @@ -252,7 +259,7 @@ df = ff.read_catalog_sql(""" """) # Aggregate virtual and physical tables together -df = ff.read_catalog_sql(""" +df = read_catalog_sql(""" SELECT category, SUM(amount) as total FROM sales_summary GROUP BY category @@ -340,67 +347,18 @@ df = ff.read_database( !!! note "Return Type" `read_database()` returns a `FlowFrame` (not a raw Polars `LazyFrame`). The result supports `.collect()` to materialize data, `.data` to access the underlying `LazyFrame`, and `open_graph_in_editor()` to visualize the pipeline in the UI. -## Connection Management - -Before reading from cloud storage, set up connections: - -```python -import flowfile as ff -from pydantic import SecretStr - -# Create S3 connection -ff.create_cloud_storage_connection_if_not_exists( - ff.FullCloudStorageConnection( - connection_name="my-aws-connection", - storage_type="s3", - auth_method="access_key", - aws_region="us-east-1", - aws_access_key_id="your-access-key", - aws_secret_access_key=SecretStr("your-secret-key") - ) -) -``` - -## Flowfile-Specific Features - -### Description Parameter - -Every reader accepts an optional `description` for visual documentation: +The tested integration example reads a table and a query from PostgreSQL through a stored connection: ```python -df = ff.read_csv( - "quarterly_sales.csv", - description="Load Q4 2024 sales data for analysis" -) +--8<-- "docs/examples/integrations/database_read.py:example" ``` -### Automatic Scan Mode Detection - -Cloud storage readers automatically detect scan mode: - -```python -# Automatically detects single file -df = ff.scan_parquet_from_cloud_storage("s3://bucket/file.parquet") - -# Automatically detects directory scan -df = ff.scan_parquet_from_cloud_storage("s3://bucket/folder/") -``` - -### Integration with Visual UI - -All reading operations create nodes in the visual workflow: +## Connection Management -```python -df = ff.read_csv("data.csv", description="Source data") - -# Open in visual editor -ff.open_graph_in_editor(df.flow_graph) -``` +Set up cloud and database connections once, then reference them by name. See [Cloud Connection Management](cloud-connections.md). ## Examples -### Standard Data Pipeline - ```python import flowfile as ff @@ -418,25 +376,5 @@ orders = ff.scan_parquet_from_cloud_storage( result = customers.join(orders, on="customer_id") ``` -### Multi-Format Cloud Pipeline - -```python -# Different formats from same connection -config_data = ff.scan_json_from_cloud_storage( - "s3://configs/settings.json", - connection_name="app-data" -) - -sales_data = ff.scan_parquet_from_cloud_storage( - "s3://analytics/sales/", - connection_name="app-data" -) - -delta_data = ff.scan_delta( - "s3://warehouse/customer_dim", - connection_name="app-data" -) -``` - [← Previous: Introduction](index.md) | [Next: Writing Data →](writing-data.md) diff --git a/docs/users/python-api/reference/visual-ui.md b/docs/users/python-api/reference/visual-ui.md index b65667673..7dd356eff 100644 --- a/docs/users/python-api/reference/visual-ui.md +++ b/docs/users/python-api/reference/visual-ui.md @@ -1,6 +1,6 @@ # Visual UI Integration -Flowfile provides a web-based visual interface that can be launched directly from Python. This allows seamless transitions between code and visual pipeline development. +Flowfile provides a web-based visual interface that can be launched directly from Python, so you can move a code-built pipeline into the visual editor and back. ## Starting the Web UI @@ -27,7 +27,7 @@ flowfile run ui --no-browser ``` !!! info "Unified Mode" - The web UI runs in "unified mode" - a single service that combines the Core API, Worker, and Web UI. No separate services or Docker required! + The web UI runs in unified mode: one process hosting the Core API, the Worker, and the UI. No separate services or Docker involved. ## Opening Pipelines in the Editor @@ -55,19 +55,19 @@ ff.open_graph_in_editor(result.flow_graph) When you call `open_graph_in_editor()`: -1. **Saves the graph** to a temporary `.flowfile` -2. **Checks if server is running** at `http://localhost:63578` -3. **Starts server if needed** using `flowfile run ui --no-browser` -4. **Imports the flow** via API endpoint -5. **Opens browser tab** at `http://localhost:63578/ui/flow/{id}` +1. **Saves the graph** to a temporary `.yaml` flow file +2. **Checks if the server is running** by probing `http://localhost:63578` +3. **Starts the server if needed** using `flowfile run ui --no-browser` +4. **Imports the flow** via an API endpoint +5. **Opens a browser tab** at `http://localhost:63578/ui/flow/{id}` ### Advanced Options ```python -# Save to specific location instead of temp file +# Save to a specific location instead of a temp file ff.open_graph_in_editor( result.flow_graph, - storage_location="./my_pipeline.flowfile" + storage_location="./my_pipeline.yaml" ) # Don't automatically open browser @@ -119,13 +119,7 @@ stop_flowfile_server_process() ## Configuration -### Environment Variables - -| Variable | Default | Description | -|----------|---------|-------------| -| `FLOWFILE_HOST` | `127.0.0.1` | Host to bind server to | -| `FLOWFILE_PORT` | `63578` | Port for the server | -| `FLOWFILE_MODULE_NAME` | `flowfile` | Module name to run | +The web UI is hard-locked to `127.0.0.1:63578` — `start_server` raises `NotImplementedError` for any other host or port, so there is no environment variable that relocates it. The `FLOWFILE_MODULE_NAME` variable (default `flowfile`) selects which module the launcher runs. ### URLs and Endpoints @@ -133,29 +127,14 @@ Once running, the following are available: - **Web UI**: `http://localhost:63578/ui` - **API Docs**: `http://localhost:63578/docs` -- **Health Check**: `http://localhost:63578/docs` (used to verify server is running) + +`is_flowfile_running()` treats a reachable `/docs` as "server up" — it is the readiness probe the client library uses, not a dedicated health endpoint. ## Troubleshooting ### Server Won't Start -```python -# Check if port is already in use -import socket - -def is_port_in_use(port): - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - return s.connect_ex(('localhost', port)) == 0 - -if is_port_in_use(63578): - print("Port 63578 is already in use") -``` - -### Server Starts but UI Doesn't Open - -- Manually navigate to `http://localhost:63578/ui` -- Check server logs in terminal -- Verify no firewall blocking localhost connections +The UI is fixed to port 63578; if a previous session holds it, free it (`lsof -i :63578` / `netstat -ano | findstr :63578`) and retry. If the server starts but no tab opens, navigate to `http://localhost:63578/ui` manually. ### Import Fails @@ -185,51 +164,7 @@ os.environ["POETRY_PATH"] = "/path/to/poetry" ff.open_graph_in_editor(df.flow_graph) ``` -## Best Practices - -### 1. Let Auto-start Handle It - -```python -# ✅ Good: Let open_graph_in_editor start server -ff.open_graph_in_editor(df.flow_graph) - -# ❌ Avoid: Manual server management unless necessary -ff.start_web_ui() -time.sleep(5) -ff.open_graph_in_editor(df.flow_graph) -``` - -### 2. Use Temporary Files - -```python -# ✅ Good: Let Flowfile handle temp files -ff.open_graph_in_editor(df.flow_graph) - -# Only specify path if you need to keep the file -ff.open_graph_in_editor( - df.flow_graph, - storage_location="./important_pipeline.flowfile" -) -``` - -### 3. Single Server Instance - -The server is designed to be a singleton - multiple calls to `open_graph_in_editor()` will reuse the same server instance. - -```python -# First call starts server -ff.open_graph_in_editor(pipeline1.flow_graph) - -# Subsequent calls reuse server -ff.open_graph_in_editor(pipeline2.flow_graph) # No new server started -ff.open_graph_in_editor(pipeline3.flow_graph) # Still same server -``` +The server is a singleton: the first `open_graph_in_editor()` call starts it, and every later call reuses it — there is no need to start it yourself or to manage the temporary flow files it writes (pass `storage_location` only when you want to keep the `.yaml`). --- - -!!! tip "Where to Go Next" - - - **Explore Visual Nodes:** Learn the details of each node available in the [Visual Editor](../../visual-editor/nodes/index.md). - - **Convert Code to Visual:** See how your code translates into a visual workflow in the [Conversion Guide](../tutorials/flowfile_frame_api.md). - - **Build with Code:** Dive deeper into the [code-first approach](../../visual-editor/building-flows.md) for building pipelines. - - **Back to Index:** Return to the main [Python API Index](index.md). \ No newline at end of file +[← Previous: Cloud Storage](cloud-connections.md) | [Next: Catalog References →](catalog-references.md) \ No newline at end of file diff --git a/docs/users/python-api/reference/writing-data.md b/docs/users/python-api/reference/writing-data.md index 6c07597ca..e78a3d852 100644 --- a/docs/users/python-api/reference/writing-data.md +++ b/docs/users/python-api/reference/writing-data.md @@ -1,6 +1,6 @@ # Writing Data -Flowfile provides Polars-compatible writers with additional cloud storage integration and visual workflow features. +Flowfile's writers are Polars-compatible, with added cloud storage integration and visual workflow features. !!! info "Polars Compatibility" Local file writers work identically to Polars, plus optional `description` for visual documentation. @@ -125,7 +125,7 @@ ff.write_to_cloud_storage( ### Format-Specific Cloud Writers -### Cloud CSV Writing +#### Cloud CSV Writing ```python # Write to S3 @@ -145,7 +145,7 @@ df.write_csv_to_cloud_storage( - `delimiter`: CSV field separator (default: `;`) - `encoding`: File encoding (`utf8` or `utf8-lossy`) -### Cloud Parquet Writing +#### Cloud Parquet Writing ```python # Write to S3 with compression @@ -163,7 +163,7 @@ df.write_parquet_to_cloud_storage( - `connection_name`: Name of configured cloud storage connection - `compression`: Compression algorithm (`snappy`, `gzip`, `brotli`, `lz4`, `zstd`) -### Cloud JSON Writing +#### Cloud JSON Writing ```python # Write JSON to cloud storage @@ -174,7 +174,7 @@ df.write_json_to_cloud_storage( ) ``` -### Delta Lake Writing +#### Delta Lake Writing ```python # Write Delta table (supports append mode) @@ -204,6 +204,12 @@ new_data.write_delta( Write data to the Flowfile catalog as managed Delta tables. Available as both a standalone function and a FlowFrame method. +The tested example writes an aggregate to the catalog, then queries it back with SQL: + +```python +--8<-- "docs/examples/catalog_analysis.py:example" +``` + ### Standalone Function ```python @@ -328,30 +334,7 @@ ff.write_catalog_table( ## Connection Requirements -All cloud storage writing requires a configured connection: - -```python -import flowfile as ff -from pydantic import SecretStr - -# Set up connection before writing -ff.create_cloud_storage_connection_if_not_exists( - ff.FullCloudStorageConnection( - connection_name="data-lake", - storage_type="s3", - auth_method="access_key", - aws_region="us-east-1", - aws_access_key_id="your-key", - aws_secret_access_key=SecretStr("your-secret") - ) -) - -# Now you can write to cloud storage -df.write_parquet_to_cloud_storage( - "s3://data-lake/output.parquet", - connection_name="data-lake" -) -``` +All cloud storage writing requires a configured connection referenced by name. See [Cloud Connection Management](cloud-connections.md). --- diff --git a/docs/users/python-api/tutorials/flowfile_frame_api.md b/docs/users/python-api/tutorials/flowfile_frame_api.md index b9ea8f45b..60251b482 100644 --- a/docs/users/python-api/tutorials/flowfile_frame_api.md +++ b/docs/users/python-api/tutorials/flowfile_frame_api.md @@ -1,107 +1,64 @@ -# Building Flows with code +# Building Flows with Code -The `flowfile_frame` module provides a powerful, Polars-like API that allows you to define and execute data transformation pipelines in Python while automatically generating a visual ETL graph. -[Source: readme.md] - ---- +The `flowfile_frame` API lets you define and run data transformation pipelines in Python — with a Polars-like surface — while building a visual ETL graph as a side effect. This tutorial walks through a simple pipeline and a more involved one, and opens each in the Designer. ## Overview -`flowfile_frame` is designed to bridge the gap between writing code and visual workflow design. It offers: - -- A familiar API for those accustomed to Pandas or Polars. -- Automatic generation of an ETL graph from your Python code. -- The ability to visualize, save, and share your data pipelines in the Flowfile Designer UI. -- The performance benefits of the Polars engine. - ---- +`flowfile_frame` gives you a Polars-like API that generates an ETL graph from your Python code, which you can visualize, save, and share in the Flowfile Designer. Execution runs on the Polars engine. ## Installation -The `flowfile_frame` module is included with the standard `flowfile` package. +The `flowfile_frame` module ships with the standard `flowfile` package. ```bash pip install flowfile ``` -### Quick Start -You can create a data pipeline programmatically and see the results: +## A first pipeline +Build a pipeline programmatically and collect the result (this code runs in the test suite on every commit): ```python -import flowfile as ff -from flowfile import col, open_graph_in_editor - -df = ff.from_dict({ - "id": [1, 2, 2], - "value": [10, 20, 15] -}) - -result = df.filter(col("value") > 12, description="filter value > 12").with_columns( - (col("value") * 10).alias("scaled_value"), description="get a scaled value" -).group_by(col("id")).agg(col("value").sum().alias("sum_value"), - col("value").max().alias("max_value"), - col("value").min().alias("min_value")) -df = result.collect() # provides a polars dataframe -open_graph_in_editor(result.flow_graph) +--8<-- "docs/examples/code_to_flow.py:first" ``` + +Then open the graph in the Designer: + +```python +ff.open_graph_in_editor(result.flow_graph) +``` +
-Generated Flow in Flowfile UI +Generated flow in the Flowfile UI ![Created flow](../../../assets/images/guides/code_to_flow/code_to_flow.png)
-## Visualizing Your Pipeline +## A more involved pipeline -One of the most powerful features of `flowfile_frame` is its ability to convert your code into a visual graph that can be opened in the Flowfile UI. +You can add conditional logic, grouping, and aggregation, then visualize the result the same way: -You can build more advanced pipelines with conditional logic, grouping, and aggregation — and then instantly visualize them. +```python +--8<-- "docs/examples/code_to_flow.py:involved" +``` ```python -import flowfile as ff -from flowfile import open_graph_in_editor - -# Create a more complex data pipeline -df = ff.from_dict({ - "id": [1, 2, 3, 4, 5], - "category": ["A", "B", "A", "C", "B"], - "value": [100, 200, 150, 300, 250] -}) - -aggregated_df = ( - df - .filter(ff.col("value") > 120, description='Filter on value greater then 120') - .with_columns([ - (ff.col("value") * 1.1).alias("adjusted_value"), - ff.when(ff.col("category") == "A").then(ff.lit("Premium")) - .when(ff.col("category") == "B").then(ff.lit("Standard")) - .otherwise(ff.lit("Basic")).alias("tier") - ], description='Calculate the thier') - .group_by("tier") - .agg([ - ff.col("adjusted_value").sum().alias("total_value"), - ff.col("id").count().alias("count") - ]) -) - -# This will launch the Flowfile Designer UI and render your pipeline -open_graph_in_editor(aggregated_df.flow_graph) +ff.open_graph_in_editor(aggregated.flow_graph) ``` +
-Generated Flow in Flowfile UI +Generated flow in the Flowfile UI ![Created flow](../../../assets/images/guides/code_to_flow/code_to_flow_2.png)
-When you run `open_graph_in_editor(...)`, the Flowfile Designer UI will open and display a visual graph of your pipeline. You can: +When you call `open_graph_in_editor(...)`, the Designer opens and displays your pipeline, where you can inspect each node, continue editing visually, or save and export it. -* Inspect each transformation node -* Continue modifying your logic visually -* Share or export your pipeline +!!! note "Grouping by an expression" + `group_by(col("id"))` groups by an expression, which renders as a `polars_code` node rather than an editable group_by node. Grouping by a column name (`group_by("id")`) produces the editable node. -### Benefits Summary -By combining the declarative power of a Polars-like API with Flowfile’s interactive designer, `flowfile_frame` gives you: +## Related -* Code-first development with automatic visualization -* Zero-config ETL graph generation -* Easy debugging and collaboration \ No newline at end of file +- [FlowFrame and FlowGraph](../concepts/design-concepts.md) — the model behind these graphs +- [Visual UI Integration](../reference/visual-ui.md) — launching and controlling the Designer from Python +- [API Reference](../reference/index.md) — the full method set diff --git a/docs/users/python-api/tutorials/index.md b/docs/users/python-api/tutorials/index.md index 4269ca1e8..f8d78fd0b 100644 --- a/docs/users/python-api/tutorials/index.md +++ b/docs/users/python-api/tutorials/index.md @@ -1,45 +1,19 @@ # Python API Tutorials -Learn to build powerful data pipelines with code through practical, hands-on examples. +Worked, hands-on pipelines built with the Python API. Start with the code-to-flow walkthrough, then use the short patterns below as building blocks. -## Available Tutorials +## Tutorial ### [Building Flows with Code](flowfile_frame_api.md) -The complete guide to creating data pipelines programmatically while maintaining visual compatibility. -**You'll learn:** -- Creating pipelines with the FlowFrame API -- Using Polars-compatible operations -- Automatically generating visual graphs -- Switching between code and visual editing +Build a pipeline programmatically and open it as a visual graph in the Designer. Covers `from_dict`, expressions, conditional logic, grouping, and `open_graph_in_editor`. -**Perfect for:** -- Python developers new to Flowfile -- Data scientists wanting reproducible pipelines -- Anyone preferring code over drag-and-drop +## Patterns -## Coming Soon +These fragments are illustrative — adapt the paths and columns to your data. For fully tested, runnable pipelines see the reference pages' worked examples ([operations](../reference/flowframe-operations.md), [joins](../reference/joins.md), [aggregations](../reference/aggregations.md)). -### Data Pipeline Patterns -Common patterns for ETL, data cleaning, and analysis. +### ETL pipeline -### Performance Optimization -Advanced techniques for handling large datasets efficiently. - -### Integration Examples -Connecting Flowfile with pandas, scikit-learn, and other tools. - -## Tutorial Style - -Our Python tutorials focus on: -- **Real-world examples** - Practical use cases you'll actually encounter -- **Code-first approach** - Everything done programmatically -- **Visual integration** - How to leverage the UI when helpful -- **Best practices** - Production-ready patterns - -## Quick Examples - -### ETL Pipeline ```python import flowfile as ff @@ -61,29 +35,25 @@ transformed = ( transformed.write_parquet("output.parquet") ``` -### Data Validation +### Data validation + ```python -# Check for data quality issues df = ff.read_csv("input.csv") -# Find duplicates +# Find duplicate keys duplicates = df.group_by("id").agg( ff.count().alias("count") ).filter(ff.col("count") > 1) -# Find nulls +# Count nulls per column null_counts = df.select([ ff.col(c).is_null().sum().alias(f"{c}_nulls") for c in df.columns ]) ``` -## Resources - -- [API Reference](../reference/index.md) - Complete method documentation -- [Core Concepts](../concepts/index.md) - Understand the architecture -- [Quick Start](../quickstart.md) - Get running in 5 minutes - ---- +## Related -*Want more tutorials? Let us know what you'd like to see in our [GitHub Discussions](https://github.com/edwardvaneechoud/Flowfile/discussions)!* \ No newline at end of file +- [API Reference](../reference/index.md) — method-by-method documentation +- [Core Concepts](../concepts/index.md) — the FlowFrame and FlowGraph model +- [Quick Start](../quickstart.md) — install and build a first pipeline diff --git a/docs/users/visual-editor/building-flows.md b/docs/users/visual-editor/building-flows.md index f48f63270..f626ec79d 100644 --- a/docs/users/visual-editor/building-flows.md +++ b/docs/users/visual-editor/building-flows.md @@ -1,235 +1,72 @@ # Building Flows -Flowfile allows you to create data pipelines visually by connecting nodes that represent different data operations. This guide will walk you through the process of creating and running flows. +This page covers the canvas mechanics: creating a flow, adding and connecting nodes, running, and saving. For a complete worked pipeline, follow the [Quickstart](../../quickstart.md#your-first-flow-visually) — or open the finished [sales pipeline in your browser](https://demo.flowfile.org), no install needed. -!!! info "Looking for a quickstart overview?" - Check out our [Quick Start Guide](../../quickstart.md#non-technical-quickstart) to get up and running in minutes. - -## Interface Overview +## The interface ![Flowfile Interface Overview](../../assets/images/ui/full_ui.png){ width="800px" } -*The complete Flowfile interface showing:* - -- **Left sidebar**: Browse and select from available nodes -- **Center canvas**: Build your flow by arranging and connecting nodes -- **Right sidebar**: Configure node settings and parameters -- **Bottom panel**: Preview data at each step - -## Creating a Flow - - ![Flowfile Landing](../../assets/images/ui/landing.png){ width="1200px" } -
The Flowfile landing page when no flows are active, showing options to create a new flow or open an existing one
- -### Starting a New Flow -1. Click the **Create** button in the top toolbar -2. A new empty canvas will open -3. Save your flow at any time using the **Save** button -4. Flows are saved as `.yaml` or `.json` files (human-readable formats) - -### Adding Nodes +- **Left sidebar** — the node palette, grouped into six categories ([full reference](nodes/index.md)) +- **Center canvas** — the flow itself +- **Right sidebar** — settings for the selected node +- **Bottom panel** — data preview for the selected node -1. Browse nodes in the left sidebar, organized by category: - - Input Sources (for loading data) - - Transformations (for modifying data) - - Combine Operations (for joining data) - - Aggregations (for summarizing data) - - Output Destinations (for saving data) -2. Drag any node onto the canvas -3. Connect nodes to create a flow +| Canvas control | Action | +|---|---| +| Mouse wheel | Zoom | +| Drag empty canvas | Pan | +| Shift + drag | Select multiple nodes | +| Right-click | Context menu; add notes to the canvas | -## Configuring Nodes +## Create a flow -### Node Settings +1. Click **Create** in the top toolbar and name the flow. +2. An empty canvas opens. +3. **Save** writes it as a `.yaml` file (`.json` also supported) — plain text that diffs and versions cleanly in Git. Legacy `.flowfile` files still open; re-save to convert. -
+## Add and connect nodes -
-

Click any node on the canvas to open its settings in the right sidebar. Each node type has unique configuration options tailored to its function.

-

For example, the "Formula" node shown here includes sections for:

-
    -
  • 🎛️ General: Add a custom description via general settings
    • -
  • ⚙️ Performance tweaking: Define if the data needs to be cached for better performance via general settings
    • -
  • ↔️ Transformations: Define the formula to be applied on the incoming data
    • -
-
+1. Drag a node from the left sidebar onto the canvas. +2. Drag from a node's output handle to the next node's input to connect them. +3. Click a node to configure it in the right sidebar — alongside each node's own settings there is a shared **General Settings** tab with the node's description, its reference name (used for edge labels, see below), and a toggle to cache its result between runs. -
- Node settings panel showing configuration options for a Formula node -

The settings panel for a "Formula" node.

-
+![Node settings panel for a Formula node](../../assets/images/ui/node_settings_formula.png) -
+## Run -### Data Preview -1. After configuration, each node shows the output schema of the action -2. Click on the **run** button to execute the node -3. The preview panel will show the output data +1. Click **Run** in the top toolbar. +2. Node borders show execution state: green success, red failure, orange warning, grey not yet executed. +3. Click any executed node to inspect its output in the bottom panel. -## Running Your Flow +In **Development** mode (the default) every node's data is available for preview after a run. Switch to **Performance** mode when the flow is done: only the steps needed for outputs execute, and the query optimizer works across nodes. -### 1. Flow Settings +## Flow settings -Open the settings modal by clicking the **gear icon** in the top toolbar. This modal controls how your flow executes and how connections are displayed on the canvas. +The **gear icon** in the top toolbar opens the flow-level settings: ![Flow Settings modal](../../assets/images/guides/building-flows/flow-settings-modal.png) -*The Flow Settings modal showing execution mode, location, display options, and parallel workers* - -#### Execution Mode - -| Mode | Description | -|------|-------------| -| **Development** | Lets you view the data in every step of the process, at the cost of performance | -| **Performance** | Only executes steps needed for the output (e.g., writing data), allowing for query optimizations and better performance | - -#### Execution Location - -| Location | Description | -|----------|-------------| -| **Local** | Runs the flow in the core process. Nodes execute sequentially. | -| **Remote** | Offloads execution to the worker service. Enables parallel execution of independent nodes. | - -#### Show Edge Labels - -When enabled, each connection on the canvas displays its **connection name** as a label on the edge. The name is derived from the source node's **node reference** (or defaults to `df_{node_id}` if no reference is set). +| Setting | Options | +|---|---| +| Execution mode | **Development** (preview everything) / **Performance** (optimized, outputs only) | +| Execution location | **Local** (core process, sequential) / **Remote** (worker service, independent nodes run in parallel) | +| Parallel workers | 1–32, default 4 — remote execution only | +| Show detailed progress | More granular per-node status during runs | +| Show edge labels | Display each connection's name on the canvas | -![Edge labels on the canvas](../../assets/images/guides/building-flows/edge-labels-canvas.png) +### Edge labels and Python Script nodes -*Edge labels displayed on connections, showing the name of each data stream* - -This is especially useful when working with **Python Script** (kernel) nodes, where edge labels tell you the exact name to pass to `flowfile_ctx.read_input()`. For example, if two nodes with references `total_sales` and `sales_per_city` are connected to a Python Script node, you read them like this: +Each connection's name comes from the source node's **node reference** (default `df_{node_id}`). With edge labels on, the canvas shows the exact name a [Python Script node](kernels.md#writing-output-data) uses to read that input: ```python total_sales = flowfile_ctx.read_input("total_sales") sales_per_city = flowfile_ctx.read_input("sales_per_city") ``` -Similarly, Python Script nodes can publish **multiple named outputs** using `flowfile_ctx.publish_output(df, "name")`, with output names configured visually in the node settings. See [Kernel Execution — Writing Output Data](kernels.md#writing-output-data) for details. - -#### Show Detailed Progress - -When enabled, the execution progress indicator shows more granular status updates for each node during a flow run. - -#### Parallel Workers - -Controls how many independent nodes can run concurrently during flow execution. - -| Setting | Details | -|---------|---------| -| **Range** | 1–32 | -| **Default** | 4 | -| **Applies to** | Remote execution only (Local always runs sequentially) | - -Increasing the number of parallel workers can speed up flows with many independent branches. Reduce it if the worker machine has limited CPU or memory. - -### 2. Running the Flow -1. Click the **Run** button in the top toolbar -2. Watch the execution progress: - - 🟢 Green: Success - - 🔴 Red: Error - - 🟡 Yellow: Warning - - ⚪ White: Not executed - -### 3. Viewing Results -1. Click any node after execution to see its output data -2. Review the results in the preview panel -3. Check for any errors or warnings -4. Export results using output nodes - -## Saving and Loading Flows - -Flowfile saves your pipelines as human-readable YAML or JSON files, making them easy to version control, share, and collaborate on. - -### Supported Formats - -| Format | Extension | Best For | -|--------|-----------|----------| -| YAML | `.yaml`, `.yml` | Human readability, version control | -| JSON | `.json` | Programmatic access, API integration | - -### Saving a Flow - -1. Click the **Save** button in the toolbar -2. Choose a filename with `.yaml` or `.json` extension -3. Your flow is saved with all nodes, connections, and settings - -### What Gets Saved - -- **Flow settings**: Name, description, execution mode -- **All nodes**: Type, position, and configuration -- **Connections**: How nodes are linked together -- **Node settings**: All parameters and options - -### Loading a Flow - -1. Click **Open** in the toolbar -2. Select a `.yaml`, `.json`, or legacy `.flowfile` -3. The flow is fully restored and ready to run - -### Version Control - -YAML files work great with Git: - -- Track changes to your pipelines over time -- Review diffs to see what changed -- Collaborate with team members -- Roll back to previous versions - -!!! tip "Migrating from Legacy Format" - If you have flows saved in the old `.flowfile` format, simply open them and save as `.yaml` to convert. - -## Example Flow - -Here's a typical flow that demonstrates common operations: -![graph](../../assets/images/ui/graph.png){ width="1200px" } - -## Best Practices - -### Organization - - Give your flows clear, descriptive names - - Arrange nodes logically from left to right - - Group related operations together - - Use comments or node labels for documentation - -### Development - - Save your work frequently - - Test with a subset of data first - - Use the auto-run mode during development - - Break complex flows into smaller, manageable parts - -### Troubleshooting - - Check node configurations if errors occur - - Review data previews to understand transformations - - Ensure data types match between connected nodes - - Look for error messages in node status - -## Tips and Tricks - -- **Node Management**: - - Double-click canvas to pan - - Use mouse wheel to zoom - - Hold Shift to select multiple nodes - - Right-click for context menu - - Right-click on the text to add notes - -- **Data Handling**: - - Use sample nodes during development - - Preview data frequently - - Check column types early with *select* nodes - ---- - ---- -## Want to see another example? -Checkout the [quickstart guide](../../quickstart.md#non-technical-quickstart)! - -## Next Steps +Python Script nodes likewise publish named outputs with `flowfile_ctx.publish_output(df, "name")`. -After mastering basic flows, explore: +## From here - - [Input sources](nodes/input.md) - - [Complex transformations](nodes/transform.md) - - [Data aggregation techniques](nodes/aggregate.md) - - [Advanced joining methods](nodes/combine.md) - - [Output options](nodes/output.md) \ No newline at end of file +- Open the finished sales pipeline: in-app via **Create → From template**, [as a download](../../assets/flows/sales_pipeline.yaml), or [live in the browser](../../assets/try-sales-pipeline.html) — the link carries the flow and its data. +- [Worked examples](tutorials/index.md) — complete flows with data and expected results. +- [Node reference](nodes/index.md) — every node, per category. diff --git a/docs/users/visual-editor/catalog/index.md b/docs/users/visual-editor/catalog/index.md index ebb379dd9..0bc2012e3 100644 --- a/docs/users/visual-editor/catalog/index.md +++ b/docs/users/visual-editor/catalog/index.md @@ -2,7 +2,7 @@ Organize, track, and govern your data flows and tables in a central catalog. -The Catalog is your single pane of glass for managing flows, tracking execution history, registering data tables (physical and [virtual](virtual-tables.md)), querying data with [SQL](sql-editor.md), sharing artifacts across flows, and automating pipelines with [schedules](schedules.md). +The Catalog is a central place to manage flows and track execution history. It registers data tables (physical and [virtual](virtual-tables.md)), queries data with [SQL](sql-editor.md), shares artifacts across flows, and automates pipelines with [schedules](schedules.md). !!! info "Limited in Flowfile Lite" The browser-only [Flowfile Lite](../../deployment/lite.md) edition includes only a **lightweight in-browser catalog** (save and reuse CSV tables). The governed catalog described here — Delta-backed storage, version history, virtual tables, lineage, SQL, schedules, and secrets — requires the full desktop/server build. @@ -19,6 +19,32 @@ Click the **Catalog** icon in the left sidebar menu to open the Catalog page. --- +## Start with a populated catalog + +A fresh install starts with an empty catalog (only the `General > default` namespace). To explore the catalog with real content, seed the optional **Demo** catalog from the command line: + +```bash +flowfile seed-demo +``` + +This one command: + +- Creates a `Demo` catalog with two schemas — `sales_analytics` and `market` +- Writes four Delta tables under `Demo > sales_analytics` — `regions`, `products`, `customers`, and `sales` +- Registers a **Sales by Region** flow (under `sales_analytics`) and runs it once so its output table is populated +- Registers a **Daily FX Sync** flow (under `market`), schedules it with a `0 6 * * *` cron, and triggers an immediate first run + +`seed-demo` is idempotent — running it again skips tables and flows that already exist. To remove everything under the `Demo` catalog (tables, flows, schedules, and namespaces) in one call: + +```bash +flowfile remove-demo +``` + +!!! note "Demo flows are the source of truth" + The seeded flows are imported from bundled YAML. Edit those flows in the designer and re-run them — they persist like any other registered flow until you run `flowfile remove-demo`. + +--- + ## Dashboard When no item is selected, the Catalog shows an overview dashboard with key metrics and quick-access panels. @@ -144,10 +170,10 @@ Click a run to see its full detail: Register data tables in the catalog for reuse across flows. Catalog tables come in two types: -| Type | Icon | Description | -|------|------|-------------| -| **Physical** | | Data materialized as a Delta table on disk — fast reads, version history, full schema preservation | -| **Virtual** | | No data on disk — executes a producer flow on demand to produce results. See [Virtual Flow Tables](virtual-tables.md) | +| Type | Description | +|------|-------------| +| **Physical** | Data materialized as a Delta table on disk — fast reads, version history, full schema preservation | +| **Virtual** | No data on disk — executes a producer flow on demand to produce results. See [Virtual Flow Tables](virtual-tables.md) | !!! tip "Recommended: Register tables via a flow" Use a [Catalog Writer](../nodes/output.md#catalog-writer) node in your flow for the best experience. It supports more source types, ensures correct data interpretation, and enables lineage tracking. @@ -208,7 +234,7 @@ The catalog tracks full data lineage — which flows produce and consume each ta | **Producer flow** | For [virtual tables](virtual-tables.md): the flow that produces data on demand | | **Consumer flows** | Flows that read from this table via Catalog Reader nodes | -This lineage graph enables powerful automation: when a table is updated, any [table trigger schedule](schedules.md#table-trigger) watching it fires automatically, creating reactive data pipelines. +This lineage graph drives automation: when a table is updated, any [table trigger schedule](schedules.md#table-trigger) watching it fires automatically, creating reactive data pipelines. --- @@ -230,12 +256,13 @@ When you register a table or write via a [Catalog Writer](../nodes/output.md#cat 3. Metadata is extracted: row count, column count, file size, and column schema (names + Polars data types) 4. A database record links the table name, namespace, and file path -**Storage location:** +**Storage location:** the directory is resolved by `FLOWFILE_USER_DATA_DIR` (or `~/.flowfile` locally). | Environment | Path | |-------------|------| | Desktop / local | `~/.flowfile/catalog_tables/` | -| Docker | `/data/user/catalog_tables/` (mapped via `FLOWFILE_USER_DATA_DIR`) | +| Docker (code default) | `/data/user/catalog_tables/` | +| Docker (shipped `docker-compose.yml`) | `/app/user_data/catalog_tables/` — the compose file sets `FLOWFILE_USER_DATA_DIR=/app/user_data` | **File naming:** Each Delta table directory is named `{table_name}_{uuid}` (e.g., `sales_data_a3f1b2c4/`). The UUID suffix ensures uniqueness even when multiple tables share similar names. @@ -260,8 +287,6 @@ In the table detail panel, the **History** section shows: - **Version list** with timestamps, operation types, and metadata - **Preview at version** — select any historical version to see the data as it was at that point -This is especially useful for auditing changes, debugging data quality issues, and understanding how a table evolved over time. - !!! info "Delta versioning is only available for physical tables" Virtual tables have no physical storage and therefore no version history. If you need historical snapshots, use a physical table. diff --git a/docs/users/visual-editor/catalog/schedules.md b/docs/users/visual-editor/catalog/schedules.md index a0de9cea0..b811b32d5 100644 --- a/docs/users/visual-editor/catalog/schedules.md +++ b/docs/users/visual-editor/catalog/schedules.md @@ -15,13 +15,15 @@ Automate flow execution with schedules — run flows on a timer or trigger them The scheduling system allows you to automate registered flows without manual intervention. Schedules are managed from the **Schedules** tab in the Catalog, or from individual flow detail panels. -Three schedule types are supported: +The create-schedule dialog offers three kinds: -| Type | Description | +| Kind | Description | |------|-------------| -| **Interval** | Run a flow every N minutes (minimum 1 minute) | -| **Table Trigger** | Run a flow when a specific catalog table is refreshed | -| **Table Set Trigger** | Run a flow when **all** tables in a set have been refreshed | +| **On a schedule** | Run a flow on a cron expression — the builder covers everything from "every minute" (the minimum) to complex calendars | +| **When a table updates** | Run a flow when a specific catalog table is refreshed | +| **When several tables update** | Run a flow when **all** tables in a set have been refreshed | + +(Legacy interval-based schedules created by older versions still run and show read-only in the detail panel.) !!! info "Scheduler must be running" Schedules are only active while the scheduler process is running. See [Scheduler Status](#scheduler-status) for details on how the scheduler operates. @@ -114,7 +116,7 @@ The flow detail panel includes: ### Run Logs -Scheduled and manual runs write their output to log files. When viewing a run's detail panel, click **View log** to see the full execution log. Logs are stored at `~/.flowfile/logs/scheduled_run_{run_id}.log`. +Scheduled and manual runs write their output to log files. When viewing a run's detail panel, click **View log** to see the full execution log. Logs are stored under the internal storage directory at `logs/scheduled_run_{run_id}.log` — `~/.flowfile/logs/…` on desktop/local installs, and `/app/internal_storage/logs/…` in the shipped Docker image. --- @@ -138,13 +140,15 @@ When running Flowfile as a desktop app or via `pip install flowfile`, the schedu ### Standalone Mode -Run the scheduler as an independent background service: +Run the scheduler as an independent background service using its own console script (installed by `pip install flowfile`): ```bash pip install flowfile -flowfile run flowfile_scheduler +flowfile_scheduler ``` +Pass `--once` to run a single poll cycle and exit instead of running continuously. `flowfile_scheduler` is a separate console script — it is not a `flowfile run` subcommand. + In standalone mode: - The scheduler runs independently and survives UI restarts diff --git a/docs/users/visual-editor/catalog/secrets.md b/docs/users/visual-editor/catalog/secrets.md index e32df57de..8170a5600 100644 --- a/docs/users/visual-editor/catalog/secrets.md +++ b/docs/users/visual-editor/catalog/secrets.md @@ -7,42 +7,58 @@ Store sensitive credentials like database passwords and API keys securely. ## How It Works -Secrets are encrypted using a master key before storage. When a flow needs a credential, Flowfile decrypts it on-demand. The actual values never appear in flow definitions or logs. +Secrets are encrypted before storage using a key derived from the **master key**. When a flow needs a credential, Flowfile decrypts it on-demand. The actual values never appear in flow definitions or logs. + +## Two keys, two jobs + +Flowfile uses two distinct keys. They are configured separately, and the one you care about depends on your mode: + +| Key | Encrypts | Where it lives | +|-----|----------|----------------| +| **`FLOWFILE_MASTER_KEY`** (Fernet) | User secrets (this page) | Env var / setup wizard — see [Master key](#master-key) | +| Local secure-storage key (`.secret_key`) | OAuth tokens and locally-stored connection files | Auto-generated file managed by Flowfile — see below | + +The **local secure-storage key** is generated automatically the first time Flowfile writes locally-stored credentials, and you never configure it by hand. Its location depends on `FLOWFILE_MODE`: + +| Mode | Secure-storage directory | +|------|--------------------------| +| **Desktop (`electron` mode)** | `%APPDATA%\flowfile` on Windows, `~/.config/flowfile` elsewhere — the file is `.secret_key` | +| **Python API / package** | `SECURE_STORAGE_PATH` if set, otherwise `/tmp/.flowfile` | +| **Docker** | `SECURE_STORAGE_PATH` if set, otherwise `/tmp/.flowfile` | + +The rest of this page is about the **master key**, which is what encrypts the secrets you create in the UI. ## Master Key -The master key encrypts all secrets. Without it, secrets cannot be decrypted. +The master key is a Fernet key that encrypts all user secrets. Each user's secrets are encrypted with a per-user key derived from it. Without the master key, secrets cannot be decrypted. ### Configuration by Mode | Mode | Configuration | |------|---------------| -| **Desktop (Electron)** | Auto-generated on first open, stored at `~/.config/flowfile/` | -| **Python API** | Auto-generated on first use, stored at `~/.config/flowfile/` | -| **Docker** | Generate via setup wizard, set as `FLOWFILE_MASTER_KEY` env variable | +| **Desktop (`electron` mode)** | Managed automatically on first use — no manual configuration | +| **Python API / package** | Managed automatically on first use — no manual configuration | +| **Docker** | Generate via the setup wizard, then set as the `FLOWFILE_MASTER_KEY` env variable | ### Desktop & Python API -The master key is automatically generated on first use and stored securely. No manual configuration needed. - -!!! note "Backup recommended" - The key is stored in `~/.config/flowfile/`. Back up this directory to preserve access to your encrypted secrets. +The master key is resolved automatically on first use — no manual configuration is needed. In these modes there is no separate file for you to back up: to move an install, migrate the whole storage directory (`~/.flowfile` by default, or `FLOWFILE_STORAGE_DIR` if set) together with your database, and your secrets keep decrypting. ### Docker -On first start without a master key, Flowfile shows a setup screen: +In Docker mode the master key must be supplied explicitly — it is **not** auto-generated to a file. On first start without a key, Flowfile shows a setup screen: 1. Click **Generate Master Key** 2. Copy the generated key -3. Add to your `.env` file: `FLOWFILE_MASTER_KEY=` +3. Add it to your `.env` file: `FLOWFILE_MASTER_KEY=` (or provide it as a `master_key.txt` Docker secret — the env var wins) 4. Restart the containers ![Setup Wizard](../../../assets/images/guides/docker-deployment/setup_wizard.png) !!! danger "Protect your master key" - - Back up your `.env` file securely - - Never commit to version control - - Losing it = losing access to all encrypted secrets + - Store the value securely (`.env` file or Docker secret) and back it up + - Never commit it to version control + - Losing it means losing access to every encrypted secret — there is no recovery ## Creating Secrets @@ -75,5 +91,5 @@ id from the value and derives the owner's key to decrypt — the identity of the running the flow is only used to check that they have been granted access. Revoking the share takes effect immediately, with no rotation needed. -See [Group-Based Sharing](../../../for-developers/docker-deployment.md#group-based-sharing-multi-user-mode) +See [Group-Based Sharing](../../deployment/docker.md#group-based-sharing) for the full sharing model. diff --git a/docs/users/visual-editor/catalog/sql-editor.md b/docs/users/visual-editor/catalog/sql-editor.md index 6e99a0a95..2fc0c7678 100644 --- a/docs/users/visual-editor/catalog/sql-editor.md +++ b/docs/users/visual-editor/catalog/sql-editor.md @@ -1,6 +1,6 @@ # SQL Editor -Query your catalog tables directly using SQL — no need to build a flow for quick ad-hoc analysis. +Query your catalog tables directly using SQL. !!! info "Not in Flowfile Lite" The SQL editor requires the full desktop/server build and is not available in the browser-only [Flowfile Lite](../../deployment/lite.md) edition. @@ -10,7 +10,9 @@ Query your catalog tables directly using SQL — no need to build a flow for qui ## Opening the SQL Editor Click the **SQL** button in the catalog toolbar to open the SQL editor panel. + ![SQL editor](../../../assets/images/guides/catalog/sql-editor.png) + --- ## Writing Queries @@ -67,17 +69,10 @@ Each time the virtual table is read (via Catalog Reader, another SQL query, or t ## Python API -You can also run SQL queries against catalog tables programmatically: +SQL queries against catalog tables also run from Python. This tested example writes an aggregate to the catalog and queries it back (note `read_catalog_sql` is imported from `flowfile_frame` — it is not on the `ff` namespace): ```python -import flowfile as ff - -df = ff.read_catalog_sql(""" - SELECT o.order_id, c.name, o.total - FROM orders o - JOIN customers c ON o.customer_id = c.id - WHERE o.total > 1000 -""") +--8<-- "docs/examples/catalog_analysis.py:example" ``` See [Reading Data — Catalog SQL](../../python-api/reference/reading-data.md#query-with-sql) for full documentation. diff --git a/docs/users/visual-editor/catalog/virtual-tables.md b/docs/users/visual-editor/catalog/virtual-tables.md index dcae02ffb..f7635cf63 100644 --- a/docs/users/visual-editor/catalog/virtual-tables.md +++ b/docs/users/visual-editor/catalog/virtual-tables.md @@ -1,22 +1,11 @@ # Virtual Flow Tables -Create catalog tables that store **no data on disk**. When queried, a virtual table executes its producer flow on demand — delivering always-fresh results with zero storage overhead. +A virtual table is a catalog table that stores no data on disk. Reading one executes its producer flow on demand, so the result always reflects the current source data — a computed view rather than a snapshot. Virtual tables work everywhere physical tables do: Catalog Reader nodes, SQL queries, table triggers, and schedules. !!! info "Not in Flowfile Lite" Virtual tables require the full desktop/server build and are not available in the browser-only [Flowfile Lite](../../deployment/lite.md) edition. ---- - -## Why Virtual Tables? - -Virtual tables change the way you think about catalog data. Instead of materializing every intermediate result to disk, you can expose **computed views** of your data that stay up to date automatically. - -| Benefit | Description | -|---------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **Zero storage cost** | No files are written. The catalog entry holds only metadata and (when optimized) a serialized execution plan. | -| **Always-fresh data** | Every time a virtual table is read, it produces results from the latest version of the source data and flow logic. No stale snapshots. | -| **Automatic optimization** | Flowfile analyzes your pipeline and, when all upstream nodes support lazy execution, serializes the Polars execution plan for instant resolution — with predicate and projection pushdown. | -| **Full integration** | Virtual tables work everywhere physical tables work: Catalog Reader, SQL queries, table triggers, and schedules. | +The trade against a physical table: nothing is written (the catalog entry holds only metadata and, when optimized, a serialized execution plan), nothing goes stale, and reads pay the producer's compute cost instead of a disk read. Whether that cost is trivial or substantial depends on the laziness classification below. --- @@ -72,22 +61,23 @@ The SQL query is validated, executed once to derive the output schema, and then ## Optimization and Laziness -The key differentiator of virtual tables is the **laziness system**. Flowfile classifies every node in your pipeline as *lazy*, *eager*, or *conditional*, and uses this to determine whether a virtual table can be optimized. +What a read costs depends on the **laziness system**. Flowfile classifies every node in your pipeline as *lazy*, *eager*, or *conditional*, and uses this to determine whether a virtual table can be optimized. ### What Makes a Flow "Fully Lazy"? A virtual table is **optimized** when every node upstream of the Catalog Writer supports Polars' lazy evaluation. This means the entire pipeline can be represented as a deferred execution plan — no intermediate computation required. -When a flow is fully lazy, Flowfile serializes the Polars `LazyFrame` execution plan and stores it alongside the virtual table metadata. Reading an optimized virtual table is nearly instant — it deserializes the plan and executes it with full query optimization, including predicate pushdown and projection pushdown. +When a flow is fully lazy, Flowfile serializes the Polars `LazyFrame` execution plan and stores it alongside the virtual table metadata. Reading an optimized virtual table deserializes that plan instead of re-executing the flow, and runs it with full query optimization, including predicate and projection pushdown. ### Node Laziness Classification -Every node type has a fixed laziness classification: +Every node type has a fixed laziness classification, defined in `flowfile_core/configs/node_store/nodes.py`. There are three classes: -| Classification | Nodes | Behavior | -|---|---|-------------------------------------------------------------------------------| -| **Lazy** | Manual Input, Filter, Select, Formula, Join, Group By, Sort, Add Record ID, Take Sample, Unpivot, Union, Drop Duplicates, Graph Solver, Count Records, Cross Join, Text to Rows, SQL Query, Catalog Reader | Operations are deferred — computation happens only when results are collected | -| **Eager** | Write Data, External Source, Explore Data, Pivot, Fuzzy Match, Python Script, Database Reader, Database Writer, Cloud Storage Writer, Catalog Writer, Kafka Source | Forces execution of upstream flow — may breaks the execution plan | +| Classification | Nodes | Behavior | +|---|---|---| +| **Lazy** | Manual Input, Select data, Rename columns, Filter data, Formula, Join, Cross join, Group by, Window functions, Sort data, Add record Id, Take Sample, Random Split, Unpivot data, Union data, Drop duplicates, Graph solver, Count records, Text to rows, SQL Query, Read from Catalog, Flow Input, LazyFrame node | Operations are deferred — computation happens only when results are collected, so they keep the plan optimizable. | +| **Eager** | External source, Write data, API response, Fuzzy match, Explore data, Pivot data, Python Script, Read from Database, Write to Database, Write to Catalog, Write to cloud provider, Kafka Source, Google Analytics, REST API, Train Model, Apply Model, Evaluate Model, Wait For, Flow Output, Run Flow | Forces execution of upstream data — breaks the lazy plan, so the virtual table falls back to standard resolution. | +| **Conditional** | Read data, Polars code, Read from cloud provider | Lazy or eager depending on configuration (e.g. the file type read, or whether the custom Polars code stays lazy). Treated as a blocker unless the check can prove it stays lazy. | ### Optimized Resolution @@ -98,7 +88,8 @@ When a virtual table is optimized (`is_optimized = true`): 3. The query engine applies predicate pushdown, projection pushdown, and other optimizations 4. Only the needed data is computed -This path is **extremely fast**. Because the deserialized plan is a `LazyFrame`, Polars can push the consumer's filters and column selections *through* the producer's execution plan — query optimization crosses the flow boundary. Work that the consumer doesn't need never runs in the producer, and results always reflect the current source data. +Because the deserialized plan is a `LazyFrame`, Polars pushes the consumer's filters and column selections *through* the producer's execution plan — query optimization crosses the flow boundary. Work the consumer doesn't need never runs in the producer, and results always reflect the current source data. + ### Standard Resolution When a virtual table is **not** optimized (eager or conditional nodes upstream): @@ -125,7 +116,7 @@ Each blocker identifies: ## Using Virtual Tables -Virtual tables integrate seamlessly into the catalog ecosystem. +Virtual tables work everywhere physical tables do. ### Reading in Flows @@ -150,22 +141,18 @@ WHERE v.score > 0.8 ### Table Triggers -A schedule can fire when a virtual table's producer flow finishes running, instead of (or in addition to) a cron. When the producer of a virtual table completes, any schedule with a table trigger on that table runs. +A [table trigger schedule](schedules.md#table-trigger) fires when a watched catalog table's `updated_at` changes, instead of (or in addition to) a cron. That change happens whenever a producer flow finishes a run that updates the table — so a virtual table's producer completing a run is one way to drive the trigger. -This shines when you want to **centralize trigger logic** across a chain or fan-out of flows: +This centralizes trigger logic across a chain of flows: ```mermaid flowchart LR - A[Flow Aproduces table X] -->|fires| B[Flow Bproduces table Y] + A[Flow A produces table X] -->|fires| B[Flow B produces table Y] B -->|fires| C[Flow C] B -->|fires| D[Flow D] ``` -Instead of Flow C and Flow D each managing their own cron trying to line up after A and B, one `A → B → { C, D }` trigger chain drives the whole graph off a single upstream event. - -**Note:** Because virtual tables recompute their full lineage on read, you could also fan out directly from A — `A → B`, `A → C`, `A → D` — and C and D would still see the effect of B's transformations. Chain vs. fan-out is a choice about *where* to express the trigger edges, not about data flow. - -In many cases you won't need it. If a flow runs on its own cadence, a plain schedule is simpler. +Instead of Flow C and Flow D each managing their own cron trying to line up after A and B, one `A → B → { C, D }` trigger chain drives the whole graph off a single upstream event. If a flow runs on its own cadence, a plain schedule is simpler. !!! note Virtual tables store no data, so a consumer reading the table recomputes its full lineage — including the producer's logic. A table trigger is a scheduling signal, not a data hand-off. @@ -192,7 +179,9 @@ In many cases you won't need it. If a flow runs on its own cadence, a plain sche - **Non-optimized tables re-execute the full producer flow** on every read. For complex or slow flows, this can add significant latency. - **Requires a registered producer flow** — the flow must be saved and registered in the catalog before a virtual table can reference it. - **No Delta versioning** — since no physical data is stored, there's no version history or time-travel capability. -- **One virtual table per producer flow** — each registered flow can produce at most one virtual table entry (updates overwrite the existing entry). + +!!! note "Multiple virtual tables per flow" + A single producer flow can register **multiple** virtual tables — one per Catalog Writer node in virtual mode, each keyed by its table name. Re-running the flow updates each entry in place (matched by producer registration + table name); it does not overwrite the others. --- diff --git a/docs/users/visual-editor/catalog/visualizations.md b/docs/users/visual-editor/catalog/visualizations.md index 68df72c8e..999851aea 100644 --- a/docs/users/visual-editor/catalog/visualizations.md +++ b/docs/users/visual-editor/catalog/visualizations.md @@ -72,7 +72,7 @@ From a visualization's detail panel you can: - **Edit** the chart spec — opens Graphic Walker again with the saved layout pre-loaded. - **Rename** or change the description. - **Move** the visualization to a different namespace. The source link is unaffected — a visualization can live anywhere in the catalog tree, regardless of where its source table lives. -- **Update the thumbnail** — saving an edited chart re-captures the thumbnail automatically. +- **Update the thumbnail** — saving an edited chart re-captures the thumbnail automatically. Thumbnails are capped at 500 KB; an oversized one is dropped (with a console warning) and the card falls back to the chart-type icon. - **Delete** — removes only the visualization. The underlying table or SQL is never touched. --- @@ -94,14 +94,6 @@ Graphic Walker supports multiple chart tabs in a single workspace. Flowfile save --- -## Tips - -- Use the same namespace as the underlying table when a visualization is the canonical chart for that table; use a separate namespace (e.g. `dashboards`) when collecting cross-cutting views. -- For the fastest editor experience, prefer table sources. SQL sources re-execute the query on every open; table sources load once and run every drag-and-drop against the warm dataset. -- Thumbnails are capped at 200 KB. Very large or very colourful charts may exceed that — Flowfile drops oversized thumbnails silently and falls back to the chart-type icon. - ---- - ## Related documentation - [Catalog overview](index.md) — namespaces, tables, and registration diff --git a/docs/users/visual-editor/connections.md b/docs/users/visual-editor/connections.md index 618cd021c..a2f4b1cb0 100644 --- a/docs/users/visual-editor/connections.md +++ b/docs/users/visual-editor/connections.md @@ -37,7 +37,7 @@ and Cloud Storage Writer nodes without re-entering credentials each time. | Field | Description | Example | |-------|-------------|---------| | **Connection Name** | Unique identifier for this connection | `prod_postgres` | -| **Database Type** | PostgreSQL or MySQL | `postgresql` | +| **Database Type** | PostgreSQL, MySQL, or SQLite | `postgresql` | | **Host** | Database server hostname | `db.example.com` | | **Port** | Database port | `5432` | | **Database** | Database name | `analytics` | @@ -142,7 +142,7 @@ For a step-by-step tutorial, see [Manage Cloud Storage](tutorials/cloud-connecti ### Using Kafka Connections in Flows -Select your saved Kafka connection when configuring Kafka Reader or Kafka Writer nodes. +Select your saved Kafka connection when configuring the **Kafka Source** node. Kafka support is read-only — there is no Kafka writer node. See [Kafka](../connect/kafka.md) for the node's settings and a runnable example. --- diff --git a/docs/users/visual-editor/index.md b/docs/users/visual-editor/index.md index a884bc1f3..43c07cc75 100644 --- a/docs/users/visual-editor/index.md +++ b/docs/users/visual-editor/index.md @@ -1,98 +1,62 @@ -# Visual Editor Guide +# Visual Editor -Build powerful data pipelines without writing code using Flowfile's intuitive drag-and-drop interface. +Build data pipelines by dragging nodes onto a canvas and connecting them — no code required. Each node is one operation (read a file, filter rows, join two tables), and you can inspect the data after every step. !!! tip "Try it in your browser first" - Want the visual editor with zero install? [**Flowfile Lite**](../deployment/lite.md) runs this same canvas entirely in your browser at [demo.flowfile.org](https://demo.flowfile.org) — a lightweight subset with no backend, databases, scheduler, or AI. + [**Flowfile Lite**](../deployment/lite.md) runs this same canvas entirely in your browser at [demo.flowfile.org](https://demo.flowfile.org) — a lightweight subset with no backend, databases, scheduler, or AI. -## What You'll Learn +## Three concepts -- **Build flows visually** - Drag, drop, and connect nodes -- **Transform data** - Filter, aggregate, join, and more -- **Connect to data sources** - Databases, files, and cloud storage -- **Preview results** - See data at each step -- **Export to code** - Generate Python code from your visual flows +- **Nodes** — operations, grouped into six palette categories: [Input](nodes/input.md), [Transform](nodes/transform.md), [Combine](nodes/combine.md), [Aggregate](nodes/aggregate.md), [Output](nodes/output.md), and [Machine Learning](nodes/ml.md). +- **Connections** — drag between node handles to define how data flows, left to right. +- **Execution modes** — **Development** materializes every node so you can preview all intermediate data; **Performance** executes only what outputs need, with query optimization across nodes. -## Getting Started +If you haven't built a flow yet, the [Quickstart](../../quickstart.md#your-first-flow-visually) walks through a complete one in five steps. -### Your First Flow +## In this section -1. **Create a new flow** - Click "Create" in the toolbar -2. **Add an input node** - Drag a "Read Data" node from the left panel -3. **Configure the node** - Click it and set file path in the right panel -4. **Add transformations** - Connect filter, sort, or other nodes -5. **Run the flow** - Click "Run" and see your results +
-### Interface Overview +- :material-vector-polyline: **[Building Flows](building-flows.md)** -![Flowfile Interface](../../assets/images/ui/full_ui.png) + --- -- **Left Panel**: Node library organized by category -- **Center Canvas**: Build your flow here -- **Right Panel**: Configure selected nodes -- **Bottom Panel**: Preview data and logs + Canvas mechanics: create, connect, configure, run, save. -## Core Concepts +- :material-function-variant: **[Formulas](../formulas/index.md)** -### Nodes + --- -Each node represents a data operation: + The Excel-like expression language used in Formula and Filter nodes. -- **[Input nodes](nodes/input.md)** - Load data from files, databases, APIs -- **[Transform nodes](nodes/transform.md)** - Modify and clean your data -- **[Combine nodes](nodes/combine.md)** - Join and merge datasets -- **[Aggregate nodes](nodes/aggregate.md)** - Summarize and group data -- **[Output nodes](nodes/output.md)** - Save or export results +- :material-graph-outline: **[Node Reference](nodes/index.md)** -### Connections -Draw lines between nodes to define data flow. Data moves from top to bottom, left to right. + --- -### Execution + Every node, per category, with configuration tables. -- **Development mode** - See data at every step (great for debugging) -- **Performance mode** - Optimized execution for large datasets +- :material-language-python: **[Sandboxed Python](kernels.md)** -## Learn More + --- -### More Resources -- [Building Flows](building-flows.md) - Detailed workflow guide -- [Node Reference](nodes/index.md) - Complete documentation of all nodes + Run arbitrary Python in Docker-isolated kernels as a node. -### Tutorials -- [Connect to Databases](tutorials/database-connectivity.md) - PostgreSQL, MySQL, and more -- [Cloud Storage Setup](tutorials/cloud-connections.md) - Work with S3 data -- [Export to Python](tutorials/code-generator.md) - Convert visual flows to code +- :material-toy-brick-outline: **[Node Designer](node-designer.md)** -## Tips for Success + --- -1. **Start simple** - Build basic flows before adding complexity -2. **Use descriptions** - Document nodes for your future self -3. **Preview often** - Check data at each transformation -4. **Save regularly** - Flows are saved as `.flowfile` files -5. **Try both modes** - Development for testing, Performance for production + Build your own reusable nodes. -## Visual vs Code +- :material-school-outline: **[Worked Examples](tutorials/index.md)** -Wondering when to use visual vs Python? Here's a quick guide: + --- -**Use Visual Editor when:** + Complete flows with data, expected results, and downloads. -- Exploring new datasets -- Building one-off analyses -- Collaborating with non-technical users -- Creating documented workflows -- Learning data transformations +
-**Consider [Python API](../python-api/index.md) when:** +[Settings](settings.md) covers theme and user management. -- Integrating with existing code -- Building programmatic pipelines -- Need version control -- Require advanced custom logic -- Automating workflows +## Visual or Python? -Remember, you can always switch between them! - ---- - -*Ready to build? Start with [Building Flows](building-flows.md) or explore the [Node Reference](nodes/index.md).* \ No newline at end of file +Both build the same flow graph, so this is a preference, not a commitment: any visual flow [exports to Python](tutorials/code-generator.md), and any [Python pipeline](../python-api/index.md) opens on the canvas. The canvas shines for exploring unfamiliar data and for handing work to colleagues who don't code; the Python API fits automation, version control, and logic that outgrows node settings. diff --git a/docs/users/visual-editor/kernels.md b/docs/users/visual-editor/kernels.md index ead8011fc..4bfa476c8 100644 --- a/docs/users/visual-editor/kernels.md +++ b/docs/users/visual-editor/kernels.md @@ -58,7 +58,7 @@ When Docker is not running or the kernel image has not been built, a status bann | **Kernel ID** | Unique identifier (alphanumeric) | — | | **Name** | A human-readable display label | — | | **Packages** | Comma-separated pip packages to install at startup | *(none)* | -| **Memory (GB)** | Maximum memory the container can use (0.5–64 GB) | `2` | +| **Memory (GB)** | Maximum memory the container can use (0.5–64 GB) | `4` | | **CPU Cores** | Number of CPU cores allocated (0.5–32) | `2` | | **GPU** | Enable GPU passthrough (requires NVIDIA Docker) | `false` | @@ -164,7 +164,7 @@ Click the **?** button in the code editor header to open the built-in API refere ## Writing Code -Inside a Python Script node connected to a kernel, you write standard Python code. The `flowfile` module is available automatically — no imports needed. +Inside a Python Script node connected to a kernel, you write standard Python code. The `flowfile_ctx` object is available automatically — no imports needed. ### Reading Input Data @@ -197,7 +197,7 @@ all_inputs = flowfile_ctx.read_inputs() Input names come from the **node reference** of each source node. You can set or change a node's reference in its settings panel. If no reference is set, the default name is `df_{node_id}`. Names must be lowercase and can only contain letters, digits, and underscores. !!! tip "Showing connection names on the canvas" - To display connection names on the canvas, enable **Show edge labels** in the [Flow Settings](building-flows.md#1-flow-settings). + To display connection names on the canvas, enable **Show edge labels** in the [Flow Settings](building-flows.md#flow-settings). ### Writing Output Data @@ -343,8 +343,8 @@ for a in artifacts: flowfile_ctx.delete_global_artifact("sales_model_v2") ``` -!!! note "Registered Flows Required" - `publish_global` requires the flow to be registered in the catalog. It is not available in interactive (cell) mode. +!!! note "Registered flow required to persist" + `publish_global` needs a flow registration to persist the artifact. Core normally auto-provisions a scratch registration for you, so this works in most cases. When no registration is available it returns `-1` and skips persisting rather than raising. !!! tip "Artifact Persistence" Local artifacts are automatically saved to disk and recovered if the kernel restarts — no configuration needed. @@ -452,7 +452,7 @@ Your `process` method code stays the same — the `self.settings_schema` access 3. Runs your process method body 4. Publishes outputs via `flowfile_ctx.publish_output()` for each named output -The full `flowfile` API (artifacts, display, logging) is available inside kernel-enabled custom nodes. +The full `flowfile_ctx` API (artifacts, display, logging) is available inside kernel-enabled custom nodes. For details on building custom nodes, see [Node Designer](node-designer.md#kernel-execution). @@ -468,15 +468,15 @@ Kernel execution is in beta. The following limitations are known and being worke --- -## `flowfile` API Reference +## `flowfile_ctx` API Reference -The following functions are available inside kernel code via the `flowfile` module: +The following functions are available inside kernel code via the `flowfile_ctx` object: ### Data I/O | Function | Description | |----------|---------------------------------------------------------------------------------------------------------------| -| `read_input(name="main")` | Read input data as a `pl.LazyFrame` if more then one sources are provided. It attempts to concat all sources. | +| `read_input(name="main")` | Read input data as a `pl.LazyFrame`. If more than one source is provided, it attempts to concat all sources. | | `read_inputs()` | Read all named inputs as `dict[str, list[LazyFrame]]` | | `publish_output(df, name="main")` | Write a DataFrame/LazyFrame as output | diff --git a/docs/users/visual-editor/node-designer.md b/docs/users/visual-editor/node-designer.md index 65be9cb18..078a2f975 100644 --- a/docs/users/visual-editor/node-designer.md +++ b/docs/users/visual-editor/node-designer.md @@ -1,8 +1,6 @@ # Node Designer -Create custom transformation nodes visually—no Python files required. - -The Node Designer lets you build reusable nodes by dragging UI components onto a canvas, configuring their properties, and writing transformation logic. Your custom nodes appear in the node palette alongside built-in nodes. +The Node Designer lets you build reusable custom nodes visually — no Python files required. Drag UI components onto a canvas, configure their properties, and write transformation logic; your nodes then appear in the palette alongside the built-in ones. !!! info "Not in Flowfile Lite" The Node Designer requires the full desktop/server build and is not available in the browser-only [Flowfile Lite](../deployment/lite.md) edition. Use the **Polars Code** node for custom logic there. @@ -22,7 +20,7 @@ The Node Designer lets you build reusable nodes by dragging UI components onto a 5. Click **Save** to add your node to the palette !!! tip "Restart Required" - After saving a new node, refresh `cmd/cntr + r` Flowfile to load it into the node palette. + After saving a new node, refresh Flowfile (`Cmd/Ctrl + R`) to load it into the node palette. --- @@ -65,7 +63,7 @@ The center panel is where you define your node's identity and structure. | Field | Description | Example | |-------|--------------------------------------|---------| | **Node Name** | Internal identifier (no spaces) | `Prefixer` | -| **Category** | Where it will appears in the palette | `Custom`, `Text`, `Transform` | +| **Category** | Where it appears in the palette | `Custom`, `Text`, `Transform` | | **Title** | Display name shown on the node | `Add prefixes to columns` | | **Description** | Tooltip text explaining the node | `Make columns easy to recognize...` | | **Number of Inputs** | How many input connections | `1` (most common) | @@ -73,8 +71,8 @@ The center panel is where you define your node's identity and structure. | **Node Icon** | Visual identifier in the palette | Select from icon library | !!! warning "Features Under Development" - **Category** and **Number of Outputs** are currently under development. - For now, custom nodes will appear in the default category and support single outputs. + **Category** and **Number of Outputs** are currently under development. + For now, custom nodes appear in the default category, and locally-executed nodes support a single output. Nodes that run on a [kernel](#kernel-execution) can define multiple named outputs — see [Kernel Execution](#kernel-execution) below. ### UI Sections @@ -97,27 +95,27 @@ Drag these components from the left panel into your sections: ### Input Components -| Component | Use Case | Value Type | -|-----------|----------|------------| -| **Text Input** | Names, patterns, custom strings | `str` | -| **Numeric Input** | Thresholds, counts, percentages | `int` or `float` | -| **Toggle Switch** | Enable/disable features | `bool` | -| **Single Select** | Choose one option from a list | `str` | -| **Slider** | Select a value within a range | `int` or `float` | +| Component | Class | Use Case | Value Type | +|-----------|-------|----------|------------| +| **Text Input** | `TextInput` | Names, patterns, custom strings | `str` | +| **Numeric Input** | `NumericInput` | Thresholds, counts, percentages | `float` | +| **Toggle Switch** | `ToggleSwitch` | Enable/disable features | `bool` | +| **Single Select** | `SingleSelect` | Choose one option from a list | `str` | +| **Slider** | `SliderInput` | Select a value within a range | `float` | ### Column Components -| Component | Use Case | Value Type | -|-----------|----------|------------| -| **Column Selector** | Pick one column from input data | `str` | -| **Multi Select** | Select multiple columns | `list[str]` | -| **Column Action** | Column with operation choice | `dict` | +| Component | Class | Use Case | Value Type | +|-----------|-------|----------|------------| +| **Column Selector** | `ColumnSelector` | Pick one column from input data | `str` | +| **Multi Select** | `MultiSelect` | Select multiple columns | `list[str]` | +| **Column Action** | `ColumnActionInput` | Column with operation choice | `dict` | ### Special Components -| Component | Use Case | Value Type | -|-----------|----------|-------------------| -| **Secret Selector** | API keys, passwords, credentials | `str` (SecretStr) | +| Component | Class | Use Case | Value Type | +|-----------|-------|----------|------------| +| **Secret Selector** | `SecretSelector` | API keys, passwords, credentials | `SecretStr` | !!! note "No Secret Usage Validation" There is currently no scanning to verify that secrets are handled securely @@ -156,10 +154,10 @@ The bottom section contains the code editor where you write your transformation ### Function Signature ```python -def process(self, *inputs: pl.LazyFrame) -> pl.LazyFrame: +def process(self, *inputs: pl.DataFrame) -> pl.DataFrame: ``` -Your function receives Polars LazyFrames and must return a LazyFrame. +Your function receives Polars DataFrames and must return a DataFrame. ### Accessing Component Values @@ -189,19 +187,19 @@ is_enabled: bool = self.settings_schema.options.is_enabled.value Here's a full example that adds a prefix to selected column names: ```python -def process(self, *inputs: pl.LazyFrame) -> pl.LazyFrame: - # Get the first input LazyFrame - lf = inputs[0] +def process(self, *inputs: pl.DataFrame) -> pl.DataFrame: + # Get the first input DataFrame + df = inputs[0] prefix_text: str = self.settings_schema.main_section.prefix_text.value columns_to_change: list[str] = self.settings_schema.main_section.columns_to_change.value - + # Build expressions: rename selected columns, keep others unchanged exprs: list[pl.Expr] = [ - pl.col(col_name).alias(f"{prefix_text}_{col_name}") - if col_name in columns_to_change else pl.col(col_name) - for col_name in lf.columns + pl.col(col_name).alias(f"{prefix_text}_{col_name}") + if col_name in columns_to_change else pl.col(col_name) + for col_name in df.columns ] - return lf.select(exprs) + return df.select(exprs) ``` ### Generated Code @@ -244,18 +242,18 @@ class Prefixer(CustomNodeBase): number_of_outputs: int = 1 settings_schema: PrefixerSettings = PrefixerSettings() - def process(self, *inputs: pl.LazyFrame) -> pl.LazyFrame: - # Get the first input LazyFrame - lf = inputs[0] + def process(self, *inputs: pl.DataFrame) -> pl.DataFrame: + # Get the first input DataFrame + df = inputs[0] prefix_text: str = self.settings_schema.main_section.prefix_text.value columns_to_change: list[str] = self.settings_schema.main_section.columns_to_change.value exprs: list[pl.Expr] = [ - pl.col(col_name).alias(f"{prefix_text}_{col_name}") - if col_name in columns_to_change else pl.col(col_name) - for col_name in lf.columns + pl.col(col_name).alias(f"{prefix_text}_{col_name}") + if col_name in columns_to_change else pl.col(col_name) + for col_name in df.columns ] - return lf.select(exprs) + return df.select(exprs) ``` This generated class is what gets saved to your user-defined nodes directory. @@ -291,7 +289,7 @@ When kernel execution is enabled: - The Node Designer auto-generates a kernel script from your `process` method - Your `self.settings_schema` values are baked into the script as a lightweight proxy -- Inputs are read via `flowfile_ctx.read_input()` instead of being passed as LazyFrame arguments +- Inputs are read via `flowfile_ctx.read_input()` instead of being passed as DataFrame arguments - Return values are published via `flowfile_ctx.publish_output()` for each named output - The full `flowfile_ctx` API is available — artifacts, display, logging, and more @@ -310,13 +308,12 @@ When you use a kernel-enabled custom node in a flow, the node settings panel sho Here's a process method that trains a model and publishes it as an artifact: ```python -def process(self, *inputs: pl.LazyFrame) -> pl.LazyFrame: +def process(self, *inputs: pl.DataFrame) -> pl.DataFrame: from sklearn.ensemble import RandomForestClassifier - lf = inputs[0] + df = inputs[0] target_col: str = self.settings_schema.main_section.target_column.value - df = lf.collect() X = df.drop(target_col).to_numpy() y = df[target_col].to_numpy() @@ -325,7 +322,7 @@ def process(self, *inputs: pl.LazyFrame) -> pl.LazyFrame: flowfile_ctx.log_info(f"Model trained with accuracy: {model.score(X, y):.3f}") predictions = model.predict(X) - return df.with_columns(pl.Series("prediction", predictions)).lazy() + return df.with_columns(pl.Series("prediction", predictions)) ``` For more details on the `flowfile_ctx` API available inside kernels, see [Kernel Execution](kernels.md). diff --git a/docs/users/visual-editor/nodes/aggregate.md b/docs/users/visual-editor/nodes/aggregate.md index 5a717a550..1b5db14c3 100644 --- a/docs/users/visual-editor/nodes/aggregate.md +++ b/docs/users/visual-editor/nodes/aggregate.md @@ -1,9 +1,9 @@ # Aggregate Nodes -Aggregate nodes help you summarize and analyze your data by grouping and calculating statistics. These nodes are essential for creating summaries and transforming data structure. +Aggregate nodes summarize and analyze your data by grouping and calculating statistics. -!!! tip "Available in Flowfile Lite" - All aggregate nodes — **Group By**, **Pivot**, and **Unpivot** — are included in the browser-only [Flowfile Lite](../../deployment/lite.md) build. +!!! info "Aggregate nodes in Flowfile Lite" + **Group By**, **Pivot**, and **Unpivot** are included in the browser-only [Flowfile Lite](../../deployment/lite.md) build. **Count Records** is not — it requires the full desktop/server build. ## Node Details @@ -32,10 +32,10 @@ The **Group By** node aggregates data based on selected columns, allowing calcul | Parameter | Description | |------------------------|-----------------------------------------------------| | **Group By Columns** | Columns used to define groups. | -| **Aggregations** | Functions like `sum`, `count`, `avg`, `min`, `max`. | +| **Aggregations** | One per output column. See the list below. | | **Output Column Name** | Custom name for the aggregated result (optional). | -This node is essential for summarizing datasets and preparing structured outputs. +**Aggregation functions** (as shown in the drawer): **Sum**, **Count**, **Mean**, **Median**, **Min**, **Max**, **N_unique**, **First**, **Last**, and **Concat**. The average is **Mean** — there is no `avg` option. ### ![Pivot Data](../../../assets/images/nodes/pivot.svg){ width="50" height="50" } Pivot Data @@ -54,7 +54,7 @@ The **Pivot Data** node converts data from a long format to a wide format by cre 1. Select **index columns** to retain in the final output. 2. Choose a **pivot column** whose unique values will become new columns. 3. Select a **value column** containing the data to fill the new columns. -4. Apply aggregation functions (e.g., `sum`, `count`, `avg`) if needed. +4. Apply aggregation functions (e.g., **Sum**, **Count**, **Mean**) if needed. --- @@ -67,8 +67,6 @@ The **Pivot Data** node converts data from a long format to a wide format by cre | **Value Column** | The column containing values to be placed in the new columns. | | **Aggregations** | Functions applied when multiple values exist per pivot column entry. | -This node is useful for restructuring datasets into a **summary-friendly format**. - ### ![Unpivot Data](../../../assets/images/nodes/unpivot.svg){ width="50" height="50" } Unpivot Data The **Unpivot Data** node transforms data from wide format to long format, making it easier for analysis and reporting. @@ -98,32 +96,9 @@ The **Unpivot Data** node transforms data from wide format to long format, makin | **Data Type Selector** | Automatically select columns based on data type (e.g., `string`). | | **Selection Mode** | Choose between `column` or `data_type` for unpivot selection. | -This node helps in restructuring datasets, especially when working with **reporting or analytical tools**. - ### ![Count Records](../../../assets/images/nodes/record_count.svg){ width="50" height="50" } Count Records -The **Count Records** node calculates the total number of rows in the dataset. - ---- - -#### **Key Features** -- Simple row count operation -- No configuration required -- Adds a new column `number_of_records` - ---- - -#### **Usage** -1. Add the **Count Records** node to your workflow. -2. It will automatically count the total number of rows. - ---- - -#### **Configuration Options** - -This node has **no additional settings**—it simply returns the record count. - -This transformation is useful for **quick dataset validation** and **workflow monitoring**. +The **Count Records** node calculates the total number of rows in the dataset. It has no configuration and adds a new column `number_of_records`. --- [← Combine data](combine.md) | [Next: Write data →](output.md) \ No newline at end of file diff --git a/docs/users/visual-editor/nodes/combine.md b/docs/users/visual-editor/nodes/combine.md index 57b8c96fe..debb2aafa 100644 --- a/docs/users/visual-editor/nodes/combine.md +++ b/docs/users/visual-editor/nodes/combine.md @@ -1,13 +1,9 @@ # Combine Nodes -**Combine nodes** allow you to merge multiple datasets in different ways, enabling data integration and enrichment. These nodes help in aligning, linking, and structuring data from various sources to create a unified dataset. +**Combine nodes** merge multiple datasets by **matching values**, **stacking rows**, **finding similar records**, **generating all possible combinations**, or **grouping related elements in a network**. -Depending on the method used, datasets can be merged by **matching values**, **stacking rows**, **finding similar records**, **generating all possible combinations**, or **grouping related elements in a network**. - -These transformations are essential for tasks like **data preparation, consolidation, and relationship mapping** across datasets. - -!!! info "Only Join is in Flowfile Lite" - The browser-only [Flowfile Lite](../../deployment/lite.md) build supports the **Join** node. **Fuzzy Match**, **Union Data**, **Cross Join**, and **Graph Solver** require the full desktop/server build. +!!! info "Some combine nodes are not in Flowfile Lite" + The browser-only [Flowfile Lite](../../deployment/lite.md) build supports **Join**, **Cross Join**, and **Union Data**. **Fuzzy Match** and **Graph Solver** require the full desktop/server build. ## Node Details @@ -18,7 +14,7 @@ The **Join** node merges two datasets based on matching values in selected colum --- #### **Key Features** -- Supports multiple join types: **Inner, Left, Right, Outer** +- Supports seven join types: **inner**, **left**, **right**, **full**, **semi**, **anti**, and **cross** - Join on **one or more columns** - Handles duplicate column names with automatic renaming @@ -26,8 +22,8 @@ The **Join** node merges two datasets based on matching values in selected colum #### **Usage** 1. Connect two input datasets (**left** and **right**). -2. Select **join type** (`inner`, `left`, `right`, `anti` or `outer`). -3. Choose **columns to join on**. +2. Select a **join type** (see the table below). +3. Choose **columns to join on** (not needed for `cross`). 4. Select **which columns to keep** from each dataset. --- @@ -36,11 +32,25 @@ The **Join** node merges two datasets based on matching values in selected colum | Parameter | Description | |------------------|----------------------------------------------------------| -| **Join Type** | Choose `inner`, `left`, `right`, `anti` or `outer` join. | -| **Join Columns** | Columns used to match records between datasets. | +| **Join Type** | `inner`, `left`, `right`, `full`, `semi`, `anti`, or `cross`. | +| **Join Columns** | Columns used to match records between datasets. Not required for a `cross` join. | + +**Join types:** -This node is useful for merging related datasets, such as **combining customer data with orders** or **linking product details with inventory**. +| Type | Result | +|------|--------| +| `inner` | Only rows with a match in both inputs. | +| `left` | Every left row; matched right columns, nulls where there is no match. | +| `right` | Every right row; matched left columns, nulls where there is no match. | +| `full` | Every row from both inputs, matched where possible. | +| `semi` | Left rows that have a match on the right — right columns are not added. | +| `anti` | Left rows that have no match on the right. | +| `cross` | Every combination of left and right rows (Cartesian product). | +!!! note "`full` vs `outer`" + The Join node's dropdown labels the full outer join **`full`**. The programmatic API also accepts `outer` as an alias for the same strategy. For a Cartesian product with no keys, the dedicated [Cross Join](#cross-join) node is usually clearer than the `cross` type here. + +--- ### ![Fuzzy Match](../../../assets/images/nodes/fuzzy_match.svg){ width="50" height="50" } Fuzzy Match @@ -49,7 +59,7 @@ The **Fuzzy Match** node joins datasets based on **similar values** instead of e --- #### **Key Features** -- Supports **fuzzy matching algorithms** (e.g., **Levenshtein**) +- Six string-similarity algorithms: `levenshtein`, `jaro`, `jaro_winkler`, `hamming`, `damerau_levenshtein`, and `indel` - Configurable **similarity threshold** - Calculates **match scores** - Joins datasets based on approximate values @@ -60,20 +70,19 @@ The **Fuzzy Match** node joins datasets based on **similar values** instead of e 1. Connect two datasets (**left** and **right**). 2. Select **columns** to match on. 3. Choose a **fuzzy matching algorithm**. -4. Set a **similarity threshold** (e.g., 75%). +4. Set a **similarity threshold** (0-100; defaults to `80`). --- #### **Configuration Options** -| Parameter | Description | -|---------------------|-----------------------------------------------| -| **Join Columns** | Columns used for fuzzy matching. | -| **Fuzzy Algorithm** | Choose an algorithm (e.g., `Levenshtein`). | -| **Threshold Score** | Minimum similarity score for a match (0-100). | - -This node is useful for handling **typos, name variations, and inconsistent formatting** when merging datasets. +| Parameter | Description | +|---------------------|--------------------------------------------------------------------------------------------------------------| +| **Join Columns** | Columns used for fuzzy matching. | +| **Fuzzy Algorithm** | One of `levenshtein`, `jaro`, `jaro_winkler`, `hamming`, `damerau_levenshtein`, or `indel`. | +| **Threshold Score** | Minimum similarity score for a match, on a scale of 0-100. Defaults to `80`. | +--- ### ![Union Data](../../../assets/images/nodes/union.svg){ width="50" height="50" } Union Data @@ -96,9 +105,6 @@ The **Union Data** node merges multiple datasets by stacking rows together. --- -This node is useful for **combining similar datasets**, such as **monthly reports or regional data**. - - ### ![Cross Join](../../../assets/images/nodes/cross_join.svg){ width="50" height="50" } Cross Join The **Cross Join** node creates all possible combinations between two datasets. @@ -121,10 +127,6 @@ The **Cross Join** node creates all possible combinations between two datasets. --- -This node is useful for **creating test scenarios, generating all possible product combinations, or building comparison matrices**. - ---- - ### ![Graph Solver](../../../assets/images/nodes/graph_solver.svg){ width="50" height="50" } Graph Solver The **Graph Solver** node groups related records based on connections in a graph-structured dataset. @@ -152,7 +154,19 @@ The **Graph Solver** node groups related records based on connections in a graph | **To Column** | Defines the endpoint of each connection. | | **Output Column** | Stores the assigned group identifier. | -This node is useful for **detecting dependencies, clustering related entities, and analyzing network connections**. +--- + +### Run Flow + +The **Run Flow** node executes another, catalog-registered flow inside the current one — the calling side of a [subflow](../subflows.md). Its input and output handles are shaped by the child flow's Flow Input and Flow Output nodes, and its settings map values into the child's parameters, including running the child once per row of a driving table. + +| Parameter | Description | +|-----------|-------------| +| **Flow** | The catalog-registered child flow to run | +| **Parameter bindings** | Default, constant, or column-mapped value per child parameter | +| **Iteration mode** | Run once with the first row's values, or once per row (capped at 1000) | + +See [Subflows](../subflows.md) for registration, wiring, and error handling. --- [← Transform data](transform.md) | [Next: Aggregate data →](aggregate.md) \ No newline at end of file diff --git a/docs/users/visual-editor/nodes/index.md b/docs/users/visual-editor/nodes/index.md index d0dbc1fa9..96df9c342 100644 --- a/docs/users/visual-editor/nodes/index.md +++ b/docs/users/visual-editor/nodes/index.md @@ -1,33 +1,27 @@ -# Nodes Overview +# Node Reference -Flowfile's nodes are the **building blocks** of your data pipeline. Each node performs a specific operation, allowing you to **load, transform, combine, summarize, and output** data without writing code. +Nodes are the building blocks of a Flowfile pipeline. Each node performs one operation — read a file, filter rows, join two datasets, train a model, write a result — and you connect them on the canvas to move data from source to output. This page is the entry point to the full reference: pick the category that matches the operation you need. -By connecting nodes together, you can create **flexible and scalable workflows** to process data efficiently. +Nodes are grouped into six categories. The counts below are current as of 2026-07 (from `flowfile_core/flowfile_core/configs/node_store/nodes.py`) and may change as nodes are added. ---- +| Category | Nodes | What it does | +|----------|-------|--------------| +| [Input](input.md) | 10 | Load data from files, databases, cloud storage, APIs, Kafka, or the catalog. | +| [Transform](transform.md) | 12 | Reshape one dataset — filter, sort, add columns, run formulas, SQL, or Python. | +| [Combine](combine.md) | 7 | Merge two or more datasets by joining, unioning, fuzzy-matching, or graph-solving. | +| [Aggregate](aggregate.md) | 5 | Summarize and restructure — group, pivot, unpivot, count. | +| [Output](output.md) | 7 | Write results to files, databases, cloud storage, or the catalog — or explore them visually. | +| [Machine Learning](ml.md) | 4 | Split data, train and apply a model, and evaluate its predictions. | -## **Node Categories** +## How nodes work -Flowfile nodes are grouped into six main categories: +Every node follows the same pattern: -- **[Input Nodes](input.md)** – Load data from files, databases, or external sources. -- **[Transform Nodes](transform.md)** – Modify, clean, and reshape your data. -- **[Combine Nodes](combine.md)** – Merge, join, or match multiple datasets. -- **[Aggregate Nodes](aggregate.md)** – Summarize, group, and compute metrics. -- **[Output Nodes](output.md)** – Save data to files or explore it interactively. -- **[Machine Learning Nodes](ml.md)** – Split data, train and apply models, and evaluate results. +- **Inputs and outputs.** A node reads from the nodes connected to its input handles and passes its result to whatever connects to its output. The number of inputs and outputs is fixed per node type — a Join takes two, a Filter in split mode emits two. +- **Configuration.** Selecting a node opens its settings in the right panel. The available fields depend on the node type; see each category page for the exact options. +- **Schema preview.** After you configure a node, it shows the output schema (column names and types) without running the whole flow. Run the node to see the actual data in the preview panel. +- **Lazy by default.** Most nodes build a Polars query that only materializes when you run the flow, so intermediate steps cost nothing until you ask for a result. ---- +Not every node is available in every build. The browser-only [Flowfile Lite](../../deployment/lite.md) edition ships a subset (23 usable nodes as of 2026-07); each category page notes which of its nodes are included. -## **How Nodes Work** - -Each node is designed to be **intuitive and consistent**, ensuring a smooth workflow experience: - -✔ **Clear inputs and outputs** – Easily understand what data flows into and out of each node. -✔ **Visual feedback** – Monitor the data transformation process. -✔ **Preview capability** – Inspect results before applying changes. -✔ **Error handling** – Validate and debug data issues efficiently. - -Flowfile’s node-based approach makes data processing **fast, flexible, and code-free**, helping you build **powerful data pipelines with ease**. - ---- +[Start with input nodes →](input.md) diff --git a/docs/users/visual-editor/nodes/input.md b/docs/users/visual-editor/nodes/input.md index 1b12b5cd0..8e7733db3 100644 --- a/docs/users/visual-editor/nodes/input.md +++ b/docs/users/visual-editor/nodes/input.md @@ -3,7 +3,7 @@ Input nodes are the starting point for any data flow. Flowfile supports reading from **local files**, **databases**, **cloud storage**, **catalog tables**, and **manual input**. !!! info "Not all input nodes are in Flowfile Lite" - The browser-only [Flowfile Lite](../../deployment/lite.md) build reads **local files (CSV/Excel/Parquet)**, **remote URLs**, **Manual Input**, and **Read from Catalog** — but it has no backend, so **Cloud Storage Reader** and **Database Reader** are not available. + The browser-only [Flowfile Lite](../../deployment/lite.md) build reads **local files (CSV/Excel/Parquet)**, host-provided datasets (**External Data**), **Manual Input**, and **Read from Catalog** — but it has no backend, so **Cloud Storage Reader**, **Database Reader**, and the API/Kafka sources are not available. ## Node Details @@ -20,14 +20,6 @@ The **Read Data** node allows you to load local data into your flow. It supports - **NDJSON files** (`.ndjson`, `.jsonl`) - **Avro files** (`.avro`) -#### **Usage:** - -1. Select your input file. -2. Configure any format-specific options. -3. Preview and confirm your data. - ---- - #### CSV When a **CSV** file is selected, the following setup options are available: @@ -60,7 +52,7 @@ When an **Excel** file is selected, you can specify the sheet, select specific r --- #### Parquet -When a **Parquet** file is selected, no additional setup options are required. Parquet is a columnar storage format optimized for efficiency and performance. It retains schema information and data types, enabling faster reads and writes without manual configuration. +When a **Parquet** file is selected, no additional setup options are required. --- @@ -99,7 +91,7 @@ The **Cloud Storage Reader** node reads data directly from cloud object storage. | Parameter | Description | |--------------------|----------------------------------------------------------------------------------------------------------| | **File Path** | Path to the file or directory (e.g., `bucket-name/folder/file.csv`) | -| **File Format** | Supported formats: CSV, Parquet, JSON, Delta Lake | +| **File Format** | Supported formats: CSV, Parquet, JSON, Delta Lake, Iceberg | | **Scan Mode** | Single file or directory scan (reads all matching files in a directory) | #### **Format-Specific Options:** @@ -322,5 +314,18 @@ From your flow's perspective, a virtual table behaves identically to a physical The Catalog Reader also supports a **SQL mode** where you can write SQL queries against all catalog tables (both physical and virtual). Tables are registered by name in a Polars SQL context, so you can join and query across the entire catalog. See [SQL Editor](../catalog/sql-editor.md) for more details. +--- + +### Flow Input + +The **Flow Input** node is a named entry point for a [subflow](../subflows.md) — a flow called from another flow. It has no upstream connection; when the flow runs on its own it serves the sample data configured in its settings, and when a parent flow calls it through a Run Flow node, the parent's dataset replaces that sample. + +| Parameter | Description | +|-----------|-------------| +| **Input name** | The port name the parent binds data to (default `input`) | +| **Sample data** | The dataset served when the flow runs standalone | + +See [Subflows](../subflows.md) for the full pattern. + --- [← Node overview](index.md) | [Next: Transform data →](transform.md) diff --git a/docs/users/visual-editor/nodes/ml.md b/docs/users/visual-editor/nodes/ml.md index 622bc4c1e..caba9f53e 100644 --- a/docs/users/visual-editor/nodes/ml.md +++ b/docs/users/visual-editor/nodes/ml.md @@ -111,6 +111,10 @@ Use it whenever a downstream node depends on a side effect of a sibling branch — typically Apply Model needing Train Model's artifact to exist before it runs. +Wait For is a general-purpose synchronization node (grouped under **Combine**, +not Machine Learning), but it appears here because the train → apply chain is +its most common use. + #### Configuration There are no settings. Wire the data branch into the left input and the @@ -225,36 +229,13 @@ Model, and you pass that frame to `apply_model(upstream=...)` to chain them. ```python -import flowfile as ff - -raw = ff.read_csv("customer_churn.csv") -train, test = raw.random_split([("train", 80), ("test", 20)], seed=42) - -trained = train.train_model( - target="churned", - features=["tenure_months", "monthly_charges", "support_calls", "has_contract"], - model_type="logistic_regression", - params={"l2_reg": 0.1}, -) - -scored = ( - test.wait_for(trained) # block apply until train is done - .apply_model( - upstream=trained, - output_column="predicted_churn", - ) -) - -metrics = scored.evaluate_model( - actual="churned", - predicted="predicted_churn", - task_type="auto", - upstream_train=trained, -) - -ff.open_graph_in_editor(metrics.flow_graph) # see the canvas +--8<-- "docs/examples/ml_pipeline.py:example" ``` +`random_split` takes a mapping of split name to percentage (the percentages +must sum to 100). `evaluate_model`'s first argument is the actual (ground-truth) +column; the prediction column and task type are keyword-only. + The catalog path is also supported — pass `model_name=` (and optionally `version=`) to `apply_model` instead of `upstream=`. diff --git a/docs/users/visual-editor/nodes/output.md b/docs/users/visual-editor/nodes/output.md index c9f01ac81..36966e3c0 100644 --- a/docs/users/visual-editor/nodes/output.md +++ b/docs/users/visual-editor/nodes/output.md @@ -1,12 +1,11 @@ # Output Nodes -Output nodes represent the final steps in your data pipeline, allowing you to save your transformed data or explore it visually. These nodes help you deliver your results in the desired format or analyze them directly. +Output nodes represent the final steps in your data pipeline, allowing you to save your transformed data or explore it visually. !!! info "Some output nodes are not in Flowfile Lite" - In the browser-only [Flowfile Lite](../../deployment/lite.md) build, **Write Data** downloads files to your browser and **Write to Catalog** / **Explore Data** work as usual. **Cloud Storage Writer** and **Database Writer** are not available (no backend). + In the browser-only [Flowfile Lite](../../deployment/lite.md) build, **Write Data** downloads a file to your browser (CSV, Parquet, or Excel — not the full desktop format set) and **Write to Catalog** / **Explore Data** work as usual. **Cloud Storage Writer** and **Database Writer** are not available (no backend). ## Node Details -## Node Details ### ![Write Data](../../../assets/images/nodes/output.svg){ width="50" height="50" } Write Data @@ -25,14 +24,6 @@ The **Write Data** node allows you to save your processed data in different form --- -### **Usage** - -1. Configure the **output file path**. -2. Select the **file format**. -3. Set writing options (e.g., delimiter, compression). - ---- - ### CSV When a **CSV** file is selected, the following setup options are available: @@ -55,7 +46,7 @@ When an **Excel** file is selected, additional configurations allow customizing --- ### Parquet -When a **Parquet** file is selected, you can choose a compression codec. Parquet is a **columnar storage format**, optimized for efficient reading and writing. +When a **Parquet** file is selected, you can choose a compression codec. | Parameter | Description | |-----------------|------------------------------------------------------------------------------------------------| @@ -102,9 +93,6 @@ When an **Avro** file is selected, you can choose a compression codec. Avro is a | **File Format** | Selects the output format (`CSV`, `Excel`, `Parquet`, `IPC`, `NDJSON`, `Avro`). | | **Overwrite Mode** | Controls whether to replace or append data. When `new file` is selected it will throw an error when the file already exists | -This node ensures that your transformed data is **saved in the correct format**, ready for further use or analysis. - - --- ### ![Cloud Storage Writer](../../../assets/images/nodes/cloud_storage_writer.svg){ width="50" height="50" } Cloud Storage Writer @@ -143,11 +131,14 @@ The **Cloud Storage Writer** node saves your processed data directly to cloud ob - Supports both `overwrite` and `append` write modes - Automatically handles schema evolution when appending +!!! note "Parquet default differs from local Write Data" + The Cloud Storage Writer defaults Parquet to **Snappy**, while the local **Write Data** node defaults to **Zstd**. Set the codec explicitly if you need the two paths to match. + !!! warning "Overwrite Mode" - When using `overwrite` mode, any existing file or data at the target path will be replaced. Make sure to verify the path before executing. + When using `overwrite` mode, any existing file or data at the target path will be replaced. Verify the path before running. !!! info "Append Mode" - Available only for Delta Lake format. + Available only for the Delta Lake format. --- @@ -249,5 +240,45 @@ For the full guide on virtual tables, optimization, and when to use them, see [V ### ![Explore Data](../../../assets/images/nodes/explore_data.svg){ width="50" height="50" } Explore Data -The Explore Data node provides interactive data exploration and analysis capabilities. +The **Explore Data** node opens an interactive, drag-and-drop chart builder (powered by [Graphic Walker](https://github.com/Kanaries/graphic-walker)) directly on the node's input. Drag columns onto the x/y axes, color, and size shelves to build bar, line, scatter, and other chart types — no configuration up front, no code. + +It takes a single input and produces no output: it is a **terminal preview node** for eyeballing a dataset, not a step that transforms or writes data. + +#### **Usage:** + +1. Connect the dataset you want to explore to the **Explore Data** node. +2. Run the flow so the node has data to visualize (the builder shows an empty state until the upstream has run). +3. Drag fields onto the chart shelves to compose a visualization. +4. Chart configurations are saved with the node, so they persist when you reopen the flow. + +!!! note "UI-only node" + Explore Data renders only in the visual editor. Headless runs (the `flowfile run flow` CLI, the scheduler, and other non-UI execution paths) **skip** Explore Data nodes automatically, since there is nowhere to draw the chart. It has no effect on the data flowing through the rest of the pipeline. + +To persist a chart as a shareable, reusable artifact rather than an ad-hoc preview, use the catalog's [visualizations](../catalog/visualizations.md) instead. + +--- + +### Flow Output + +The **Flow Output** node is a named exit point for a [subflow](../subflows.md): each Flow Output exposes one dataset to the parent flow that calls it, appearing as an output handle on the parent's Run Flow node. A flow can carry several, each with its own name. + +| Parameter | Description | +|-----------|-------------| +| **Output name** | The port name the parent reads from (default `output`) | + +See [Subflows](../subflows.md) for the full pattern. + +--- + +### API Response + +The **API Response** node marks its input as the body of an HTTP endpoint. When a flow is published as an API, the data flowing into this node is serialized and returned to the caller; a published flow must contain exactly one. During interactive runs it passes data through unchanged, so previews keep working. + +| Parameter | Description | +|-----------|-------------| +| **Orientation** | `records` (list of row objects, default) or `columns` (column-oriented) | +| **Max rows** | Optional cap on the number of rows returned | + +--- +[← Aggregate data](aggregate.md) | [Next: Machine Learning →](ml.md) diff --git a/docs/users/visual-editor/nodes/transform.md b/docs/users/visual-editor/nodes/transform.md index e8acc6f4e..2cc11ed47 100644 --- a/docs/users/visual-editor/nodes/transform.md +++ b/docs/users/visual-editor/nodes/transform.md @@ -3,7 +3,7 @@ Transform nodes modify and shape your data. These nodes handle everything from basic operations like filtering and sorting to more complex transformations like custom formulas and text manipulation. !!! info "Some transform nodes are not in Flowfile Lite" - The browser-only [Flowfile Lite](../../deployment/lite.md) build includes **Select**, **Filter**, **Sort**, **Take Sample**, **Drop Duplicates**, and **Polars Code**. **Add Record ID**, **Formula**, **Text to Rows**, and **Python Script** are not available — use the **Polars Code** node for Formula-style column logic. + The browser-only [Flowfile Lite](../../deployment/lite.md) build includes **Add Record ID**, **Formula**, **Select**, **Filter**, **Sort**, **Take Sample**, **Drop Duplicates**, **Rename Columns**, and **Polars Code**. **Text to Rows**, **Window Functions**, **SQL Query**, and **Python Script** are not available in Lite. ## Node Details @@ -12,17 +12,6 @@ Transform nodes modify and shape your data. These nodes handle everything from b The **Add Record ID** transformation generates a unique identifier for each record in your dataset. You can create a simple sequential ID or generate grouped IDs based on one or more columns. -#### **Usage:** - -1. Add the **Add Record ID** node to your flow. -2. Configure the settings: - - Define the output column name. - - Set an optional offset for ID numbering. - - (Optional) Enable grouping and specify grouping columns. -3. Apply the transformation. - ---- - #### Configuration Options | Parameter | Description | @@ -41,13 +30,11 @@ The **Add Record ID** transformation generates a unique identifier for each reco - **Grouped Record ID** - When grouping is enabled, the record ID resets within each group based on the specified columns. -This transformation helps in creating unique keys, tracking row order, or structuring data for downstream processing. - --- ### ![Formula](../../../assets/images/nodes/formula.svg){ width="50" height="50" } Formula -The **Formula** node creates a new column — or replaces an existing one — by evaluating a formula for every row. Formulas are written in the [Flowfile formula language](../../formulas/index.md): reference columns as `[column]`, call any of the [95 built-in functions](../../formulas/functions.md), and use `if ... then ... else ... endif` for conditional logic. +The **Formula** node creates a new column — or replaces an existing one — by evaluating a formula for every row. Formulas are written in the [Flowfile formula language](../../formulas/index.md): reference columns as `[column]`, call any of the [built-in functions](../../formulas/functions.md), and use `if ... then ... elseif ... else ... endif` for conditional logic. !!! tip "Try formulas in your browser" The [interactive formula playground](https://edwardvaneechoud.github.io/polars_expr_transformer/) lets you experiment with the full language against sample data — nothing to install. @@ -66,11 +53,9 @@ The **Formula** node creates a new column — or replaces an existing one — by #### **Usage** -1. Drag the **Formula** node onto your canvas. -2. Connect input data. -3. Set the output column name. -4. Write your formula, e.g. `round([price] * (1 - [discount]), 2)`. -5. Preview the results and optionally set a data type. +1. Set the output column name. +2. Write your formula, e.g. `round([price] * (1 - [discount]), 2)`. +3. Preview the results and optionally set a data type. --- @@ -89,8 +74,6 @@ The **Formula** node creates a new column — or replaces an existing one — by - If the column name is **new**, the column is added to the dataset. - If the column name **already exists**, its values are replaced with the formula result. -This transformation is useful for feature engineering, data cleaning, and enriching datasets with computed values. - --- ### ![Select Data](../../../assets/images/nodes/select.svg){ width="50" height="50" } Select Data @@ -124,9 +107,6 @@ The **Select Data** node allows you to choose which columns to keep, rename, and - Columns can be **renamed** without affecting their original data. - Changing the order affects how the columns appear in downstream processing. -This transformation ensures that datasets are structured efficiently before further analysis or processing. - - --- ### ![Filter Data](../../../assets/images/nodes/filter.svg){ width="50" height="50" } Filter Data @@ -191,13 +171,36 @@ The **Sort Data** node orders your data based on one or more columns. | **Sort Columns** | Columns used to sort the dataset. | | **Sort Order** | Set ascending (`asc`) or descending (`desc`). | -This node ensures structured and ordered data for better analysis. - --- ### ![Take Sample](../../../assets/images/nodes/sample.svg){ width="50" height="50" } Take Sample -The Take Sample node lets you work with a subset of your data. +The **Take Sample** node keeps the first *N* rows of the input, so you can iterate on a small slice before running the full dataset. + +--- + +#### **Configuration Options** + +| Parameter | Description | +|-----------------|------------------------------------------------------| +| **Sample Size** | Number of rows to keep. Default is `1000`. | + +--- + +### Rename Columns + +The **Rename Columns** node renames many columns at once by applying a single rule, instead of editing names one by one in a Select node. + +#### **Rename modes** + +| Mode | Effect | +|------|--------| +| Prefix | Prepend a fixed string to each selected column name | +| Suffix | Append a fixed string to each selected column name | +| Formula | Compute the new name with a [formula](../../formulas/index.md) — `[column_name]` is bound to each column's current name, e.g. `uppercase([column_name])` or `"v2_" + [column_name]` | +| First row | Promote the first data row to column headers and drop it from the data | + +Only the columns you select are renamed; in first-row mode the first row is dropped regardless of the selection, and null or empty header values raise an error. --- @@ -207,7 +210,7 @@ The **Drop Duplicates** node removes duplicate rows based on selected columns. O --- -### **Key Features** +#### **Key Features** - Remove duplicate rows - Select columns to check for duplicates @@ -227,8 +230,6 @@ The **Drop Duplicates** node removes duplicate rows based on selected columns. O |-------------|---------------------------------------| | **Columns** | Columns used to check for duplicates. | -This node ensures a clean dataset by eliminating redundant rows. - --- ### ![Text to Rows](../../../assets/images/nodes/text_to_rows.svg){ width="50" height="50" } Text to Rows @@ -258,12 +259,9 @@ The **Text to Rows** node splits text from a selected column into multiple rows | **Column to Split** | The column containing text to be split. | | **Output Column Name** | Name of the new column after splitting (defaults to the original column). | | **Split by Fixed Value** | If `true`, use a fixed delimiter (default: `,`). | -| **Delimiter** | The character used to split text (e.g., `,`, ` |`, `;`). | +| **Delimiter** | The character used to split text (e.g., `,`, `;`, or `\|`). | | **Split by Column** | Instead of a fixed delimiter, use values from another column. | -This transformation helps normalize datasets by converting **text lists into structured rows**. - - --- ### Window Functions diff --git a/docs/users/visual-editor/settings.md b/docs/users/visual-editor/settings.md index 7caf5e119..9c28630ec 100644 --- a/docs/users/visual-editor/settings.md +++ b/docs/users/visual-editor/settings.md @@ -20,8 +20,7 @@ Your preference persists across sessions. Manage team access through the Admin panel. Click your username → **Admin**. - -![User Management](../../assets/images/guides/settings/user_management.png) +![Admin panel showing the user list](../../assets/images/guides/settings/user_management.png) ### Creating Users @@ -36,9 +35,8 @@ New users must change their password on first login. ### Password Requirements - Minimum 8 characters -- At least one uppercase letter -- At least one lowercase letter - At least one number +- At least one special character (`!@#$%^&*()_+-=[]{}|;:,.<>?`) ### Admin Privileges diff --git a/docs/users/visual-editor/subflows.md b/docs/users/visual-editor/subflows.md new file mode 100644 index 000000000..f90dfdc97 --- /dev/null +++ b/docs/users/visual-editor/subflows.md @@ -0,0 +1,111 @@ +# Subflows: run a flow inside another flow + +This page is for analysts who want to reuse a flow as a building block. You'll learn how to turn a flow into a callable component with named inputs and outputs, register it in the catalog, and drive it from another flow with the Run Flow node — passing data in, mapping parameters, and reading results back. + +!!! info "Not in Flowfile Lite" + Subflows depend on the catalog to resolve the child flow. The browser-only [Flowfile Lite](../deployment/lite.md) edition has no catalog, so the Flow Input, Flow Output, and Run Flow nodes are unavailable there. + +## When to use a subflow + +A subflow is a flow you call from another flow. Reach for one when you want to: + +- Reuse the same cleaning or scoring logic across several pipelines without copy-pasting nodes. +- Run one flow many times with different parameters — one execution per row of a driving table. +- Keep a large pipeline readable by extracting a self-contained stage into its own flow. + +Three nodes make this work. Two mark the child flow's interface; one calls it: + +| Node | Category | Role | +|---|---|---| +| Flow Input | Input | A named entry point where the parent injects data. | +| Flow Output | Output | A named exit point exposing a dataset to the parent. | +| Run Flow | Combine | Executes a catalog-registered flow, mapping data and parameters into it. | + +## Build the child flow + +The child is an ordinary flow with two additions: named ports and (optionally) parameters. + +### Add Flow Input nodes + +Drop a **Flow Input** node wherever the parent should hand data in. Each carries an `input_name` (default `input`) — the name the parent binds to. A Flow Input node has no upstream connection; it is a start node. When you run the child flow on its own, it serves the sample data configured in its settings (an empty frame when blank), so you can develop and test the child in isolation. When a Run Flow node calls it, that sample data is replaced with the real dataset the parent passes in. + +Give each Flow Input a distinct, meaningful name (for example `orders`, `customers`) — those names are what you'll wire connections to in the parent. + + + +### Add Flow Output nodes + +Drop a **Flow Output** node on each dataset you want to expose. Each carries an `output_name` (default `output`). A Flow Output must have exactly one input connected inside the child flow, and you can have several per flow — each becomes a separate output on the parent's Run Flow node. + +### Add parameters (optional) + +If the child's behavior should vary per call, add **flow parameters** in the child's flow settings. A parameter has a name, a type (`string`, `integer`, `float`, `boolean`, or `enum`), and a default value. Inside the child flow, reference the parameter the same way you would in any flow (for example in a filter formula). The parent maps values into these parameters at call time. + +## Register the child in the catalog + +Run Flow resolves the child by its **catalog registration**, not by file path — so before a parent can call it, the child flow must be registered in the catalog. Registration also gates access: a user can only run a subflow they own or one that has been [shared](../deployment/sharing.md) with them. + +Register the child from its flow detail panel in the catalog (the same registration step described in the [Catalog guide](catalog/index.md)). Once registered, it appears as a selectable flow in the Run Flow node's settings. + +!!! tip "Reference stays valid if the file moves" + The Run Flow node stores the child's registration id and stamps its flow UUID. If the registration id ever dangles, Flowfile repairs the reference from the UUID, so a re-registered or relocated child keeps working. + +## Call it with the Run Flow node + +Add a **Run Flow** node to the parent flow and pick the registered child in its settings. Flowfile reads the child's interface — its Flow Input names, Flow Output names, and parameters — and shapes the node's handles to match. + + + +### Wire data inputs + +Each Flow Input in the child becomes a data input handle on the Run Flow node, labeled with the port name. Connect the parent's upstream nodes to the matching handles. A subflow supports up to 9 data inputs. An input you leave unconnected falls back to the child's own sample data (the node logs a warning when that happens). + +The node also reserves one extra input handle labeled **Parameters** — the driving table for column-mapped parameters (see below). It appears only when the child has parameters. + +### Map parameters + +Each child parameter gets a binding that decides where its value comes from: + +| Source | Meaning | +|---|---| +| **Default** | Use the parameter's default value from the child flow. | +| **Constant** | Use a fixed value you type in the Run Flow settings. | +| **Column** | Read the value from a named column of the Parameters input table. | + +Column bindings are what make a subflow run repeatedly. Connect a table to the Parameters handle, map one or more parameters to its columns, and set the **iteration mode**: + +- **First value** (default) — run the child once, using the first row's values. +- **Iterate** — run the child once per row of the Parameters table, and concatenate the outputs. Iteration is capped at 1000 rows. + +When iterating, the **Append run metadata** option (on by default) adds a `param_` column for each mapped parameter and a `run_index` column to each output, so you can tell which run produced which rows. + +### Read outputs + +Each Flow Output in the child becomes an output handle on the Run Flow node, labeled with the port name (up to 10 outputs). Connect downstream nodes to the outputs you need. If the child flow has no Flow Output nodes, the Run Flow node instead emits a small run-summary table (one row per run, with the resolved parameter values). + +## What happens on run + +When the parent runs and reaches the Run Flow node: + +1. Flowfile resolves the registered child, checks the current user may use it, and loads it. +2. It computes the per-run parameter values from the bindings (one set of values per run in iterate mode). +3. For each run it opens a fresh copy of the child flow, injects the connected data into the matching Flow Input nodes, applies the parameter values, and runs the child locally. +4. It collects each Flow Output's dataset. Iterated runs are concatenated (with nulls filling any columns that differ between runs), and run metadata is appended when enabled. + +The child always runs in-process and locally — it never registers itself as a live flow and never touches the global worker-offload setting. Outputs stay lazy in core; only the small parameter table is read eagerly. + +!!! warning "Recursion and nesting limits" + A subflow may not call itself, directly or through a chain — Flowfile detects the cycle and stops with an error. Nesting is capped at 5 levels deep. + +### Errors you may hit + +- **Referenced flow no longer exists** — the child's catalog registration was removed. Re-register it, or point the Run Flow node at a different flow. +- **No access to the referenced flow** — you don't own the child and it hasn't been shared with you. See [Sharing](../deployment/sharing.md). +- **Subflow no longer has port(s)** / **parameter does not exist** — you edited the child's interface after wiring the parent. Reopen the Run Flow settings to re-sync its interface, then re-map. +- **Subflow output has no input connected** — a Flow Output node inside the child has nothing feeding it. Connect it or remove it. + +## Related + +- [Catalog](catalog/index.md) — register the child flow and manage catalog resources. +- [Sharing](../deployment/sharing.md) — let teammates run a subflow you own (Docker mode). +- [Schedules](catalog/schedules.md) — run a parent flow (subflows included) on a timer. diff --git a/docs/users/visual-editor/tutorials/cloud-connections.md b/docs/users/visual-editor/tutorials/cloud-connections.md index d320860f0..1401730e4 100644 --- a/docs/users/visual-editor/tutorials/cloud-connections.md +++ b/docs/users/visual-editor/tutorials/cloud-connections.md @@ -1,42 +1,21 @@ -# Manage S3 Connections +# Connect to AWS S3 and S3-compatible storage -This guide walks you through creating AWS S3 connections in Flowfile to access your cloud data. +This guide walks you through creating an AWS S3 (or S3-compatible, such as MinIO) connection in Flowfile so you can read and write cloud data from your flows. !!! info "Not in Flowfile Lite" Cloud storage connections require the full desktop/server build. This guide does not apply to the browser-only [Flowfile Lite](../../deployment/lite.md) edition, which has no backend. ## Overview -Cloud storage connections securely store your AWS credentials and configuration, allowing you to reuse them across multiple workflows without re-entering credentials. +A cloud storage connection stores your AWS credentials and configuration under a name, which you reference from reader and writer nodes across workflows. -## Steps to Create an S3 Connection +## Create an S3 connection -### 1. Access Cloud Storage Connections +**1. Open the dialog.** On the **Connections** page (left sidebar), select the **Cloud Storage** tab and click **"+ Add Connection"**. -Open the **Connections** page from the left sidebar and select the **Cloud Storage** tab. - -
-Screenshot: Cloud Storage Connections Page - - -![Cloud storage connection page](../../../assets/images/guides/create_cloud_connection/cloud_storage_connection_page.png) - - -
- -### 2. Add New Connection - -Click the **"+ Add Connection"** button to open the connection configuration dialog. - -
-Screenshot: Add Connection Dialog - - ![create_new_cloud_storage](../../../assets/images/guides/create_cloud_connection/add_cloud_connection.png) -
- -### 3. Configure Connection Settings +**2. Configure the connection.** #### Basic Settings @@ -64,19 +43,27 @@ Choose one of the following authentication methods: | Field | Description | |-------|-------------| | **Custom Endpoint URL** | For S3-compatible services (e.g., MinIO) | -| **Allow Unsafe HTML** | Enable if your S3 data contains HTML content | +| **Allow Unsafe HTTP** | Enable for non-HTTPS endpoints, such as a local MinIO server | | **Verify SSL** | Disable only for testing with self-signed certificates | -### 4. Save Connection - -Click **"Create Connection"** to save your configuration. +**3. Save.** Click **"Create Connection"**. ## Using S3 Connections in Workflows -Once created, your S3 connection will appear in the Cloud Storage Reader and Writer node's connection dropdown. Simply: +Once created, your S3 connection will appear in the Cloud Storage Reader and Writer node's connection dropdown. 1. Add a **Cloud Storage Reader** node to your workflow 2. Select your connection from the dropdown 3. Enter the S3 path (e.g., `s3://my-bucket/data/file.csv`) 4. Configure file format options 5. Run your workflow + +## In Python + +The same connection, created and used from code — this example is tested against a real S3-compatible service on every commit: + +```python +--8<-- "docs/examples/integrations/cloud_storage_s3.py:example" +``` + +See [Cloud Connections in Python](../../python-api/reference/cloud-connections.md) for all fields and auth methods. diff --git a/docs/users/visual-editor/tutorials/code-generator.md b/docs/users/visual-editor/tutorials/code-generator.md index c559eab6b..135e4390a 100644 --- a/docs/users/visual-editor/tutorials/code-generator.md +++ b/docs/users/visual-editor/tutorials/code-generator.md @@ -1,22 +1,19 @@ -# Building code with flows +# Export to Python -Flowfile's Code Generator allows you to export your visually designed data pipelines as clean, executable Python code. This feature is designed to empower users who wish to inspect the underlying data transformation logic, integrate Flowfile pipelines into existing Python projects, or extend their workflows with custom scripts. +The Code Generator exports a visually designed flow as executable Python. Use it to inspect the transformation logic behind a flow, integrate a Flowfile pipeline into an existing Python project, or extend a workflow with custom scripts. For pure transformation flows (filter, join, group by, etc.), the generated code is standalone Polars — no Flowfile dependency required. Flows that include I/O nodes (database, catalog, cloud storage, Kafka) generate code that uses `import flowfile as ff` for connection-aware operations. The transformation logic remains Polars in both cases. + ![code_generator](../../../assets/images/guides/code_generator/code_generator.gif) ## Key Characteristics of the Generated Code -When you generate code from your Flowfile graph, you can expect the output to be: - -* **Mostly Standalone**: Transformation logic uses only Polars. I/O operations (database, catalog, cloud storage, Kafka) use `flowfile` for managed connections. -* **Readable**: The structure mirrors your visual flow, making it easy to understand the sequence of operations. -* **Direct Translation**: Transformation nodes translate to Polars operations. I/O nodes translate to FlowFrame API calls (`ff.read_database()`, `ff.read_catalog_table()`, etc.). -* **Ready for Integration**: Pure transformation flows can be embedded anywhere. Flows with I/O nodes require `pip install Flowfile`. +* Transformation nodes translate to Polars operations; I/O nodes (database, catalog, cloud storage, Kafka) translate to FlowFrame API calls (`ff.read_database()`, `ff.read_catalog_table()`, etc.). +* The structure mirrors your visual flow. Pure transformation flows can be embedded anywhere; flows with I/O nodes require `pip install flowfile`. ## Examples of Generated Code -Here are some simplified examples illustrating what the generated Polars code looks like for common Flowfile operations. These examples highlight how your visual workflow seamlessly translates into Python. +These simplified examples show what the generated Polars code looks like for common Flowfile operations, and how a visual flow maps to Python. ### Example 1: Reading a CSV and Selecting Columns @@ -114,11 +111,12 @@ def run_etl_pipeline(): """ df_1 = pl.LazyFrame({"value": [1, 2, 3]}) - # Custom Polars code as defined in the Flowfile node - def _custom_code_node_name(input_df: pl.LazyFrame): + # Custom Polars code as defined in the Flowfile node. + # The wrapper name is derived from the node id — e.g. _polars_code_2 for node 2. + def _polars_code_2(input_df: pl.LazyFrame): return input_df.with_columns((pl.col('value') * 10).alias('scaled_value')) - df_2 = _custom_code_node_name(df_1) + df_2 = _polars_code_2(df_1) return df_2 if __name__ == "__main__": @@ -162,8 +160,6 @@ if __name__ == "__main__": !!! note "`.data` accessor" The generated code calls `.data` on FlowFrame results to extract the underlying Polars `LazyFrame`. This keeps the rest of the pipeline as standard Polars operations. -These examples provide a clear overview of the type of high-quality, executable Python code produced by Flowfile's Code Generator. - ## Project Export For more complex flows — especially flows that contain **notebook (Python script) nodes** or **custom user-defined nodes** — a single generated script becomes hard to read. The third export mode, **Project**, exports the flow as a structured multi-file Python project instead: @@ -190,5 +186,5 @@ Key points: From the Code panel you can either **download the project as a .zip** or **save it directly into a folder** using the built-in file browser. -!!! info "Future Direction" - In a future release, generated code will use the FlowFrame API directly for all operations, enabling round-trip editing between code and canvas. \ No newline at end of file +!!! info "Editing exported code" + Exported code runs standalone; it does not round-trip back into the visual canvas. To keep editing a flow visually, work in the Designer and re-export. \ No newline at end of file diff --git a/docs/users/visual-editor/tutorials/database-connectivity.md b/docs/users/visual-editor/tutorials/database-connectivity.md index c2dc12967..dc8db8861 100644 --- a/docs/users/visual-editor/tutorials/database-connectivity.md +++ b/docs/users/visual-editor/tutorials/database-connectivity.md @@ -1,129 +1,131 @@ -# How to Connect and Work with PostgreSQL Databases in Flowfile +# Connect to a PostgreSQL database -### ![Full overview](../../../assets/images/guides/database_connectivity/main_image.png) Full flow overview - -Flowfile's latest release introduces powerful database connectivity features that allow you to seamlessly integrate with PostgreSQL databases like Supabase. In this guide, I'll walk you through the entire process of connecting to a database, reading data, transforming it, and writing it back. +This tutorial connects Flowfile to a PostgreSQL database (the example uses Supabase, but any PostgreSQL server works), reads a table, transforms it, and writes the result back to a new table. !!! info "Not in Flowfile Lite" - Database connectivity requires the full desktop/server build. This guide does not apply to the browser-only [Flowfile Lite](../../deployment/lite.md) edition, which has no backend. + Database connectivity requires the full desktop/server build. This tutorial does not apply to the browser-only [Flowfile Lite](../../deployment/lite.md) edition, which has no backend. + +![Full flow overview](../../../assets/images/guides/database_connectivity/main_image.png) + +*The finished flow: read from a database, transform, and write back.* ## Prerequisites -Before diving in, make sure you have: - -- A Flowfile account (free tier works fine) -- A Supabase account ([sign up here](https://supabase.com) if needed) -- Sample data to work with (we're using the [Sales Forecasting Dataset](https://www.kaggle.com/datasets/rohitsahoo/sales-forecasting) from Kaggle **so you can easily follow along with the transformation examples**) - -## Step 1: Set Up Your Supabase Database - -1. Create a new project in Supabase. -2. Download the sample dataset from Kaggle. -3. Create a new table in your Supabase project (e.g., `superstore_sales_data`). -4. Import the dataset into your table (hint: use Supabase's built-in CSV import feature via the Table Editor). -5. Note your database connection details (host, port, username, password). +- A running Flowfile install (desktop, `pip install flowfile`, or Docker) +- A PostgreSQL database you can reach, with credentials (host, port, database, username, password). This example uses [Supabase](https://supabase.com); any PostgreSQL server works. +- A table of data to read. This example uses the [Sales Forecasting dataset](https://www.kaggle.com/datasets/rohitsahoo/sales-forecasting) from Kaggle so the transformation steps line up. + +## Step 1: Load sample data into your database + +If you already have a table to read, skip to Step 2. Otherwise, load the sample dataset: + +1. Create a database (or a Supabase project). +2. Download the sample dataset from Kaggle. +3. Create a table (e.g. `superstore_sales_data`). +4. Import the CSV into the table. Supabase offers a CSV import in the Table Editor; other tools have equivalents. +5. Note the connection details — host, port, username, and password. + +## Step 2: Create a database connection in Flowfile + +Save the credentials once so you can reference them by name in any flow. + +1. Open the **Connections** page from the left sidebar and select the **Database** tab. +2. Click **Create New Connection**. +3. Fill in the fields: + - **Connection Name**: `supabase_connection` (any name) + - **Database Type**: PostgreSQL + - **Host**: your database host (e.g. `aws-0-eu-central-1.pooler.supabase.com`) + - **Port**: `5432` + - **Database**: `postgres` + - **Username** / **Password**: your credentials + - **Enable SSL**: check if your database requires it (Supabase typically does) +4. Click **Update Connection** to save. + + +![Connection overview in Flowfile](../../../assets/images/guides/database_connectivity/db_connection.png) + +*A saved database connection on the Connections page.* -## Step 2: Configure Your Database Connection in Flowfile +See [Connections](../connections.md#database-connections) for a full field reference. -1. Open Flowfile and navigate to the **Connections** page from the left sidebar, then select the **Database** tab. -2. Click **"Create New Connection"**. -3. Fill in your connection details: - * Connection Name: `supa_base_connection` (or any name you prefer) - * Database Type: PostgreSQL - * Host: Your Supabase host (e.g., `aws-0-eu-central-1.pooler.supabase.com`) - * Port: `5432` - * Database: `postgres` - * Username: Your Supabase username - * Password: Your Supabase password - * Enable SSL: Check if required by your database (Supabase typically requires it). -4. Click **"Update Connection"** to save. +## Step 3: Read from the database - -### ![db_connection](../../../assets/images/guides/database_connectivity/db_connection.png) Connection overview in Flowfile +1. Create a new flow (**Create** or **New Flow**). +2. From the node palette, drag a **Read from Database** node onto the canvas. +3. Click the node to open its settings panel. +4. Set **Connection Mode** to **reference**, then select `supabase_connection` from the dropdown. +5. Set the table: + - **Schema**: `public` (or your schema) + - **Table**: `superstore_sales_data` +6. Click **Validate Settings**. A green confirmation appears when the query settings are valid. -## Step 3: Create a New Data Flow +![Database read configuration](../../../assets/images/guides/database_connectivity/configure_read_db.png) -1. Click **"Create"** or **"New Flow"** to start a fresh workflow. -2. Navigate to the "Data actions" panel on the left sidebar. -3. Find the "Read from Database" node (look for the database icon). -4. Drag and drop this node onto your canvas. +*The Read from Database node configured in reference mode.* -## Step 4: Configure Your Database Read Operation +## Step 4: Run and preview -1. Click on the "Read from Database" node to open its settings panel. -2. Select **"reference"** for Connection Mode (this tells Flowfile to use the connection you configured in Step 2). -3. Choose your `supa_base_connection` from the Connection dropdown. -4. Configure the table settings: - * Schema: `public` (or your specific schema) - * Table: `superstore_sales_data` -5. Click **"Validate Settings"** to ensure everything is working. -6. You should see a green confirmation message: "Query settings are valid". +1. Click **Run** in the toolbar. +2. Watch progress in the log panel at the bottom. +3. On success, click the node's output to preview the rows read from the database. -### ![configure_read_db](../../../assets/images/guides/database_connectivity/configure_read_db.png) Node Settings panel showing database read configuration + +![Successful database read run](../../../assets/images/guides/database_connectivity/initial_run.png) -## Step 5: Run Your Initial Flow +*A successful run of the read step.* -1. Click the **"Run"** button in the top toolbar. -2. Watch the flow execution in the log panel at the bottom. -3. When completed, you'll see a success message and the number of records processed. -4. You can now click on the node output dot to preview the data that was read from your database. +## Step 5: Add transformations -### ![initial_run_success](../../../assets/images/guides/database_connectivity/initial_run.png) Flow execution logs showing successful database read operation -*Ensure the image `initial_run.png` shows the successful run log/node status, not the configuration panel again.* +With data flowing in, add transformation nodes. For example, to compute shipping time per product category: -## Step 6: Add Data Transformations +1. Add a **Formula** node to derive `shipping_time_days` from the ship and delivery dates (e.g. `[delivery_date] - [shipping_date]`). Add a preceding Formula node to cast those columns to a date type first if they arrive as text. +2. Add a **Group by** node keyed on the product category, aggregating `min`, `max`, and `median` of `shipping_time_days`. +3. Connect the nodes in sequence by dragging from one node's output dot to the next node's input dot. -Now that you've successfully read data from your database, you can add transformation steps: +For per-node settings, see the [node reference](../nodes/transform.md). -1. Add transformation nodes from the "Data actions" panel to your workflow. - * For example, creating time-to-ship metrics by category: - * Add formula nodes to transform the `shipping_date` and `delivery_date` columns to a proper date type if needed. - * Add a "Formula" node to calculate shipping time (e.g., `delivery_date - shipping_date`). Name the new column `shipping_time_days`. - * Add a "Group by" node to aggregate by product category. - * In the "Group by" node, calculate `min`, `max`, and `median` of the `shipping_time_days` column. -2. Connect these nodes in sequence by dragging from the output dot of one node to the input dot of the next. -3. Configure each node with the specific transformations you need (refer to the Flowfile documentation for details on specific node configurations if needed). +![Connected transformation nodes](../../../assets/images/guides/database_connectivity/transformations.png) -### ![transformations](../../../assets/images/guides/database_connectivity/transformations.png) Overview of connected transformation nodes (Read -> Formula -> Group By) +*Read → Formula → Group by.* -## Step 7: Add a Write to Database Node +## Step 6: Write the result back -1. From the "Data actions" panel, find and drag the "Write to Database" node onto your canvas. -2. Connect it to the output of the last transformation node (e.g., the "Group by" node). -3. Configure the write operation in its settings panel: - * Connection Mode: Select **"reference"**. - * Connection: Choose your `supa_base_connection`. - * Schema: `public` (or your desired schema). - * Table: Enter a name for your new output table (e.g., `time_to_ship_per_category`). - * Write Mode: Select how to handle the table if it already exists: - * **"Append"**: Add new data to the table. - * **"Replace"**: Delete the existing table and create a new one with the output data. - * **"Fail"**: Abort the flow if the table already exists. +1. Drag a **Write to Database** node onto the canvas and connect it to the last transformation node. +2. In its settings panel: + - **Connection Mode**: **reference** + - **Connection**: `supabase_connection` + - **Schema**: `public` + - **Table**: a new table name (e.g. `time_to_ship_per_category`) + - **If Table Exists**: choose how to handle an existing table — + - **Append**: add rows to the existing table + - **Replace**: drop the existing table and recreate it with the new data + - **Fail**: abort the run if the table already exists -### ![configure_write_db](../../../assets/images/guides/database_connectivity/configure_write_db.png) Setup write to database node configuration panel -*Ensure the image `configure_write_db.png` actually shows the "Write to Database" node's configuration panel.* +![Write to Database configuration](../../../assets/images/guides/database_connectivity/configure_write_db.png) -## Step 8: Run Your Complete Workflow +*The Write to Database node in reference mode.* -1. Click **"Run"** to execute the full workflow from start to finish. -2. The system will: - * Read data from your source Supabase table (`superstore_sales_data`). - * Apply all the transformation steps (calculate shipping time, group by category). - * Write the aggregated results to your destination Supabase table (`time_to_ship_per_category`). -3. Check the logs to confirm successful execution, including messages about records read and written. -4. Navigate to your Supabase project, open the SQL Editor or Table Editor, and check the `public.time_to_ship_per_category` table to see your newly created data! +## Step 7: Run the full flow -### ![supabase_result_table](../../../assets/images/guides/database_connectivity/result.png) Overview of the final result table in Supabase -*Ensure the image `result.png` shows the data in the newly created Supabase table.* +1. Click **Run** to execute the whole flow. +2. Flowfile reads the source table, applies the transformations, and writes the aggregated result to your destination table. +3. Check the logs for records read and written. +4. Open your database and query the destination table to confirm the output. -## Conclusion + +![Result table in the database](../../../assets/images/guides/database_connectivity/result.png) -Flowfile's database integration capabilities make it incredibly simple to build professional-grade data pipelines without writing code. By connecting to Supabase or other PostgreSQL databases, you can easily extract, transform, and load data in a visual, intuitive environment. +*The destination table populated with the aggregated result.* -Whether you're creating business dashboards, data warehousing solutions, or just exploring your data, the combination of Flowfile's visual workflow and Supabase's powerful PostgreSQL hosting gives you a robust platform for all your data needs. +## Next steps -Feel free to experiment with different transformation nodes and workflow patterns to build increasingly sophisticated data pipelines! +- Reference the same connection in other flows — no need to re-enter credentials. +- Export the flow to Python with the [Code Generator](code-generator.md); reference-mode database nodes translate to `ff.read_database()` / `ff.write_database()` calls. +- Schedule the flow to refresh on a timer — see [Schedules](../catalog/schedules.md). ---- +## Related documentation -*This guide is based on Flowfile v0.2.0, which introduced database connectivity features including PostgreSQL support, secure credential storage, and flexible connection management options.* +- [Connections](../connections.md) — saved database, cloud, and Kafka connections +- [Input Nodes: Database Reader](../nodes/input.md#database-reader) +- [Output Nodes: Database Writer](../nodes/output.md#database-writer) +- [Secrets](../catalog/secrets.md) — how credentials are encrypted diff --git a/docs/users/visual-editor/tutorials/index.md b/docs/users/visual-editor/tutorials/index.md index 0d0017536..5f513f2f6 100644 --- a/docs/users/visual-editor/tutorials/index.md +++ b/docs/users/visual-editor/tutorials/index.md @@ -1,24 +1,21 @@ -# Tutorials +# Worked Examples -Hands-on guides for building data pipelines with Flowfile. +End-to-end walkthroughs that build a real flow from start to finish. Each one names the reader it serves, the nodes it uses, and the data it runs on, so you can pick the closest match to what you're building. -## Available Tutorials +## Examples -### [Database Connectivity](database-connectivity.md) -Connect to PostgreSQL databases, read data, apply transformations, and write results back. +| Example | For | Difficulty | Nodes | Data | +|---------|-----|-----------|-------|------| +| [Deduplicate and summarize sales data](sales-pipeline.md) | Analysts new to Flowfile | Beginner | 5 | `data/templates/supermarket_sales.csv` | +| [Export to Python](code-generator.md) | Python developers | Beginner | Any | Any flow | ---- +## Connecting external data -### [Cloud Storage](cloud-connections.md) -Connect to AWS S3 and other cloud storage services. +Two more walkthroughs live under the **Connect Your Data** tab, since they start by wiring up a connection: ---- +| Example | For | Difficulty | Nodes | Data | +|---------|-----|-----------|-------|------| +| [Connect to a PostgreSQL database](database-connectivity.md) | Analysts with a database | Intermediate | Read from Database → Formula → Group by → Write to Database | A PostgreSQL table | +| [Connect to AWS S3 and S3-compatible storage](cloud-connections.md) | Analysts with cloud data | Intermediate | Cloud Storage Reader / Writer | Files in S3 or MinIO | -### [Export to Python](code-generator.md) -Convert visual pipelines into executable Python code. - -## Reference Documentation - -- [Docker](../../deployment/docker.md) - Docker deployment -- [Secrets](../catalog/secrets.md) - Encrypted credential storage -- [Settings](../settings.md) - Theme and user management +Every example here is validated by the test suite on each commit: the flow files live in [`data/templates/flows/`](https://github.com/edwardvaneechoud/Flowfile/tree/main/data/templates/flows) and the Python twins in [`docs/examples/`](https://github.com/edwardvaneechoud/Flowfile/tree/main/docs/examples), and both are executed end-to-end against the committed sample data. diff --git a/docs/users/visual-editor/tutorials/sales-pipeline.md b/docs/users/visual-editor/tutorials/sales-pipeline.md new file mode 100644 index 000000000..1fa047237 --- /dev/null +++ b/docs/users/visual-editor/tutorials/sales-pipeline.md @@ -0,0 +1,74 @@ +# Deduplicate and summarize sales data + +This example builds a five-node flow that cleans a raw sales export, keeps the high-quantity orders, and totals gross income per city. It's aimed at analysts new to Flowfile who want to see a full read → clean → filter → aggregate → explore pipeline end to end. + +**Flow:** [download `sales_pipeline.yaml`](../../../assets/flows/sales_pipeline.yaml) · +In-app: Create → From template → "Sales pipeline: clean, filter, aggregate" · Data: `data/templates/supermarket_sales.csv` + + + +## The data + +`supermarket_sales.csv` holds 1030 transaction rows — including 30 exact duplicate rows to clean up. + +| Column | Meaning | +|--------|---------| +| `invoice_id` | Transaction identifier | +| `city` | Store city (Bago, Mandalay, Naypyitaw, Taunggyi, Yangon) | +| `customer_type` | Member or Normal | +| `product_line` | Product category | +| `unit_price` | Price per unit | +| `quantity` | Units sold | +| `gross_income` | Income for the line | +| `date` | Transaction date | + +## The flow + +Five nodes, connected left to right: + +1. **Read data** — reads `supermarket_sales.csv` (file type CSV). +2. **Drop duplicates** — strategy `any`, no key columns, so it removes fully-identical rows (drops the 30 duplicates). +3. **Filter data** — advanced filter `[quantity] > 7`, keeping only the higher-quantity orders. +4. **Group by** — groups by `city`, with two aggregations on `gross_income`: + - `sum` → `total_income` + - `median` → `median_income` +5. **Explore data** — opens the result in the Graphic Walker explorer so you can chart it interactively. + +## Run it + +Four ways to run this flow: + +- **In your browser, right now** — [open it in Flowfile Lite](../../../assets/try-sales-pipeline.html): the flow and its sample data are encoded in the link itself, so nothing to install and nothing uploaded. Click **Run** when it loads. +- **From the template browser** — Create → From template → "Sales pipeline: clean, filter, aggregate". This loads the flow with the sample data already wired in. Click **Run**. +- **Download and open** — grab [`sales_pipeline.yaml`](../../../assets/flows/sales_pipeline.yaml) and open it in the designer. Its Read node points at the sample CSV's public URL, so it runs as-is with an internet connection — or repoint it at a local copy of the file. +- **Headless** — once a flow is saved with a real data path, run it from the command line without opening the UI: + +```bash +flowfile run flow path/to/your_flow.yaml +``` + +## The result + +Grouped per city, over the deduplicated rows with `quantity > 7`: + +| City | `total_income` | `median_income` | +|------|---------------|-----------------| +| Bago | 1429.66 | 25.57 | +| Mandalay | 1846.88 | 27.22 | +| Naypyitaw | 1186.66 | 22.42 | +| Taunggyi | 1635.53 | 27.92 | +| Yangon | 1704.81 | 26.085 | + +## In Python + +The same pipeline with the [FlowFrame API](../../python-api/index.md) — deduplicate, filter, group, aggregate — returns the identical per-city totals: + +```python +--8<-- "docs/examples/sales_pipeline.py:example" +``` + +## Variations + +- **Swap your data** — point the Read node at your own CSV; the clean → filter → aggregate shape carries over to any sales-style export. +- **Add more aggregations** — the Group by node offers Sum, Mean, Median, Min, Max, Count, N-unique, First, Last, and Concat per column. +- **Persist the result** — replace the Explore data node with a [Catalog Writer](../nodes/output.md#catalog-writer) to save the summary as a catalog table, then chart it with a [visualization](../catalog/visualizations.md). diff --git a/docs/requirements.txt b/flowfile_core/tests/docs_examples/__init__.py similarity index 100% rename from docs/requirements.txt rename to flowfile_core/tests/docs_examples/__init__.py diff --git a/flowfile_core/tests/docs_examples/test_docs_examples.py b/flowfile_core/tests/docs_examples/test_docs_examples.py new file mode 100644 index 000000000..3fa97aeea --- /dev/null +++ b/flowfile_core/tests/docs_examples/test_docs_examples.py @@ -0,0 +1,123 @@ +"""Execute every docs example end-to-end so the docs can never drift from the API. + +Core examples (docs/examples/*.py) must always pass. Integration examples +(docs/examples/integrations/*.py) skip when their backing Docker service is +unavailable, reusing the same fixture helpers the rest of the suite gates on. +""" + +import runpy +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[3] +EXAMPLES_DIR = REPO_ROOT / "docs" / "examples" +INTEGRATIONS_DIR = EXAMPLES_DIR / "integrations" + + +def _core_examples() -> list[Path]: + return sorted(EXAMPLES_DIR.glob("*.py")) + + +def _integration_examples() -> list[Path]: + return sorted(INTEGRATIONS_DIR.glob("*.py")) + + +@pytest.mark.parametrize("example_path", _core_examples(), ids=lambda p: p.stem) +def test_core_example_runs(example_path: Path, monkeypatch): + """A core docs example runs cleanly with the repo root as CWD.""" + monkeypatch.chdir(REPO_ROOT) + runpy.run_path(str(example_path), run_name="__main__") + + +def _postgres_available() -> bool: + from test_utils.postgres import fixtures as pg_fixtures + + return pg_fixtures.can_connect_to_db() + + +def _minio_available() -> bool: + from test_utils.s3 import fixtures as s3_fixtures + + if not s3_fixtures.is_docker_available(): + return False + # start_minio_container is idempotent: True if already up, else brings it up. + return s3_fixtures.start_minio_container() + + +def _kafka_available() -> bool: + from test_utils.kafka import fixtures as kafka_fixtures + + if not kafka_fixtures.is_docker_available(): + return False + # start_redpanda_container is idempotent: True if already up, else brings it up. + return kafka_fixtures.start_redpanda_container() + + +INTEGRATION_GATES = { + "database_read": _postgres_available, + "cloud_storage_s3": _minio_available, + "kafka_read": _kafka_available, +} + + +def test_live_demo_deep_link_matches_sources(): + """The docs' try-in-browser redirect embeds the whole flow + CSV in its URL fragment + (the wasm share-link format: deflate-raw + base64url JSON). Pin it to the tested + sources so regenerating the dataset or changing the flow can't silently strand it.""" + import base64 + import json + import re + import zlib + + import yaml + + html = (REPO_ROOT / "docs" / "assets" / "try-sales-pipeline.html").read_text() + blob = re.search(r"#flow=([A-Za-z0-9_-]+)", html).group(1) + padded = blob.replace("-", "+").replace("_", "/") + padded += "=" * (-len(padded) % 4) + envelope = json.loads(zlib.decompress(base64.b64decode(padded), -15)) + + assert envelope["v"] == 1 + csv_text = (REPO_ROOT / "data" / "templates" / "supermarket_sales.csv").read_text() + files = envelope["files"] + assert files.get(1, files.get("1")) == csv_text + + template = yaml.safe_load( + (REPO_ROOT / "data" / "templates" / "flows" / "sales_pipeline.yaml").read_text() + ) + assert [n["type"] for n in envelope["flow"]["nodes"]] == [n["type"] for n in template["nodes"]] + + by_type = {n["type"]: n["setting_input"] for n in envelope["flow"]["nodes"]} + aggs = {(c["old_name"], c["agg"], c["new_name"]) for c in by_type["group_by"]["groupby_input"]["agg_cols"]} + assert aggs == { + ("city", "groupby", "city"), + ("gross_income", "sum", "total_income"), + ("gross_income", "median", "median_income"), + } + basic = by_type["filter"]["filter_input"]["basic_filter"] + assert (basic["field"], basic["operator"], basic["value"]) == ("quantity", "greater_than", "7") + + +def test_landing_flow_attachment_matches_source(): + """The site-served download is the tested template with its data path resolved to the + public GitHub raw URL, so a downloaded flow opens and runs without any local setup. + Regenerate: copy the template and substitute __TEMPLATE_DATA_DIR__ with the URL below.""" + from flowfile_core.templates.data_downloader import TEMPLATE_DATA_BASE_URL + + source = (REPO_ROOT / "data" / "templates" / "flows" / "sales_pipeline.yaml").read_text() + attachment = (REPO_ROOT / "docs" / "assets" / "flows" / "sales_pipeline.yaml").read_text() + assert attachment == source.replace("__TEMPLATE_DATA_DIR__", TEMPLATE_DATA_BASE_URL) + + +@pytest.mark.parametrize("example_path", _integration_examples(), ids=lambda p: p.stem) +def test_integration_example_runs(example_path: Path, monkeypatch): + """An integration docs example runs when its backing service is reachable, else skips.""" + gate = INTEGRATION_GATES.get(example_path.stem) + if gate is None: + pytest.fail(f"No availability gate registered for integration example '{example_path.stem}'") + if not gate(): + pytest.skip(f"Backing service for '{example_path.stem}' is not available") + + monkeypatch.chdir(REPO_ROOT) + runpy.run_path(str(example_path), run_name="__main__") diff --git a/mkdocs.yml b/mkdocs.yml index 4090f8613..8b88693dc 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -49,13 +49,16 @@ extra_javascript: plugins: - search + - redirects: + redirect_maps: + for-developers/docker-deployment.md: users/deployment/docker.md - mkdocstrings: handlers: python: options: # Global options for all Python objects show_root_heading: true - show_inherited_members: true + inherited_members: true show_bases: true # Options specific to the griffe-pydantic extension extensions: @@ -68,6 +71,15 @@ markdown_extensions: - md_in_html - admonition - pymdownx.details + - pymdownx.snippets: + base_path: ["."] + check_paths: true + dedent_subsections: true + - pymdownx.tabbed: + alternate_style: true + - pymdownx.emoji: + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg - pymdownx.superfences: custom_fences: - name: mermaid @@ -80,77 +92,87 @@ markdown_extensions: nav: - Home: index.html - - Quickstart: quickstart.md - - User Guides: - - Overview: users/index.md + - Get Started: + - Quickstart: quickstart.md + - Guides by Audience: users/index.md + - Coming from Excel: users/coming-from-excel.md + - Flowfile Lite (Browser): users/deployment/lite.md + - Visual Editor: + - Overview: users/visual-editor/index.md + - Building Flows: users/visual-editor/building-flows.md - Formulas: - Formula Language: users/formulas/index.md - Function Reference: users/formulas/functions.md + - Node Reference: + - Overview: users/visual-editor/nodes/index.md + - Input Nodes: users/visual-editor/nodes/input.md + - Transform Nodes: users/visual-editor/nodes/transform.md + - Combine Nodes: users/visual-editor/nodes/combine.md + - Aggregate Nodes: users/visual-editor/nodes/aggregate.md + - Output Nodes: users/visual-editor/nodes/output.md + - Machine Learning Nodes: users/visual-editor/nodes/ml.md + - Sandboxed Python (Kernels): users/visual-editor/kernels.md + - Node Designer: users/visual-editor/node-designer.md + - Worked Examples: + - Overview: users/visual-editor/tutorials/index.md + - Sales Pipeline: users/visual-editor/tutorials/sales-pipeline.md + - Export to Python: users/visual-editor/tutorials/code-generator.md + - Settings: users/visual-editor/settings.md + + - Connect Your Data: + - Overview: users/connect/index.md + - Connections: users/visual-editor/connections.md + - Secrets & Encryption: users/visual-editor/catalog/secrets.md + - Databases: users/visual-editor/tutorials/database-connectivity.md + - Cloud Storage: users/visual-editor/tutorials/cloud-connections.md + - Kafka: users/connect/kafka.md + - REST APIs & Google Analytics: users/connect/apis.md + + - Catalog & Automation: + - Overview: users/visual-editor/catalog/index.md + - Virtual Tables: users/visual-editor/catalog/virtual-tables.md + - SQL Editor: users/visual-editor/catalog/sql-editor.md + - Visualizations: users/visual-editor/catalog/visualizations.md + - Schedules: users/visual-editor/catalog/schedules.md + - Subflows: users/visual-editor/subflows.md + - Projects & Git: users/projects.md + + - Python API: + - Overview: users/python-api/index.md + - Quick Start: users/python-api/quickstart.md + - Core Concepts: + - Overview: users/python-api/concepts/index.md + - FlowFrame & FlowGraph: users/python-api/concepts/design-concepts.md + - Expressions: users/python-api/concepts/expressions.md + - Formulas in Python: users/python-api/concepts/formulas.md + - API Reference: + - Overview: users/python-api/reference/index.md + - Reading Data: users/python-api/reference/reading-data.md + - Writing Data: users/python-api/reference/writing-data.md + - Catalog References: users/python-api/reference/catalog-references.md + - Data Types: users/python-api/reference/data-types.md + - DataFrame Operations: users/python-api/reference/flowframe-operations.md + - Aggregations: users/python-api/reference/aggregations.md + - Joins: users/python-api/reference/joins.md + - Cloud Storage: users/python-api/reference/cloud-connections.md + - Visual UI Integration: users/python-api/reference/visual-ui.md + - Tutorials: + - Overview: users/python-api/tutorials/index.md + - Building Flows with Code: users/python-api/tutorials/flowfile_frame_api.md + + - AI Assistant: + - Overview: ai/index.md + - Provider Setup: ai/providers.md - - Visual Editor: - - Getting Started: users/visual-editor/index.md - - Building Flows: users/visual-editor/building-flows.md - - Node Reference: - - Overview: users/visual-editor/nodes/index.md - - Input Nodes: users/visual-editor/nodes/input.md - - Transform Nodes: users/visual-editor/nodes/transform.md - - Combine Nodes: users/visual-editor/nodes/combine.md - - Aggregate Nodes: users/visual-editor/nodes/aggregate.md - - Output Nodes: users/visual-editor/nodes/output.md - - Machine Learning Nodes: users/visual-editor/nodes/ml.md - - Catalog: - - Overview: users/visual-editor/catalog/index.md - - Virtual Tables: users/visual-editor/catalog/virtual-tables.md - - Visualizations: users/visual-editor/catalog/visualizations.md - - Schedules: users/visual-editor/catalog/schedules.md - - SQL Editor: users/visual-editor/catalog/sql-editor.md - - Secrets & Encryption: users/visual-editor/catalog/secrets.md - - Features: - - Kernel Execution: users/visual-editor/kernels.md - - Node Designer: users/visual-editor/node-designer.md - - Connections: users/visual-editor/connections.md - - Tutorials: - - Overview: users/visual-editor/tutorials/index.md - - Connect to PostgreSQL: users/visual-editor/tutorials/database-connectivity.md - - Manage Cloud Storage: users/visual-editor/tutorials/cloud-connections.md - - Export to Python: users/visual-editor/tutorials/code-generator.md - - Settings: users/visual-editor/settings.md - - - Python API: - - Overview: users/python-api/index.md - - Quick Start: users/python-api/quickstart.md - - Core Concepts: - - Overview: users/python-api/concepts/index.md - - FlowFrame & FlowGraph: users/python-api/concepts/design-concepts.md - - Expressions: users/python-api/concepts/expressions.md - - Formulas in Python: users/python-api/concepts/formulas.md - - API Reference: - - Overview: users/python-api/reference/index.md - - Reading Data: users/python-api/reference/reading-data.md - - Writing Data: users/python-api/reference/writing-data.md - - Catalog References: users/python-api/reference/catalog-references.md - - Data Types: users/python-api/reference/data-types.md - - DataFrame Operations: users/python-api/reference/flowframe-operations.md - - Aggregations: users/python-api/reference/aggregations.md - - Joins: users/python-api/reference/joins.md - - Cloud Storage: users/python-api/reference/cloud-connections.md - - Visual UI Integration: users/python-api/reference/visual-ui.md - - Tutorials: - - Overview: users/python-api/tutorials/index.md - - Building Flows with Code: users/python-api/tutorials/flowfile_frame_api.md - - - Deployment: + - Deploy & Operate: - Overview: users/deployment/index.md - - Flowfile Lite (Browser): users/deployment/lite.md - Desktop App: users/deployment/desktop.md - Python Package: users/deployment/python.md - Docker: users/deployment/docker.md - - - AI Assistant: - - Overview: ai/index.md - - Provider Setup: ai/providers.md + - Users, Groups & Sharing: users/deployment/sharing.md + - Headless Runs & CLI: users/deployment/cli.md - For Developers: - Introduction: for-developers/index.md diff --git a/poetry.lock b/poetry.lock index f4136ff13..6347ce405 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2935,6 +2935,21 @@ files = [ {file = "mkdocs_material_extensions-1.3.1.tar.gz", hash = "sha256:10c9511cea88f568257f960358a467d12b970e1f7b2c0e5fb2bb48cab1928443"}, ] +[[package]] +name = "mkdocs-redirects" +version = "1.2.3" +description = "A MkDocs plugin for dynamic page redirects to prevent broken links" +optional = false +python-versions = ">=3.10" +files = [ + {file = "mkdocs_redirects-1.2.3-py3-none-any.whl", hash = "sha256:ec7312fff462d03ec16395d0c001006a418f8d0c21cdf2b47ff11cf839dc3ce0"}, + {file = "mkdocs_redirects-1.2.3.tar.gz", hash = "sha256:5e980330999299729a2d6a125347d1af78023d68a23681a4de3053ce7dfe2e51"}, +] + +[package.dependencies] +mkdocs = ">=1.2,<=1.6.1" +properdocs = ">=1.6.5" + [[package]] name = "mkdocstrings" version = "0.30.1" @@ -3821,6 +3836,34 @@ files = [ {file = "propcache-0.5.2.tar.gz", hash = "sha256:01c4fc7480cd0598bb4b57022df55b9ca296da7fc5a8760bd8451a7e63a7d427"}, ] +[[package]] +name = "properdocs" +version = "1.6.7" +description = "Project documentation with Markdown." +optional = false +python-versions = ">=3.9" +files = [ + {file = "properdocs-1.6.7-py3-none-any.whl", hash = "sha256:6fa0cfa2e01bf338f684892c8a506cf70ea88ae7f3479c933b6fa20168101cbd"}, + {file = "properdocs-1.6.7.tar.gz", hash = "sha256:adc7b16e562890af0e098a7e5b02e3a81c20894a87d6a28d345c9300de73c26e"}, +] + +[package.dependencies] +click = ">=7.0" +colorama = {version = ">=0.4", markers = "platform_system == \"Windows\""} +ghp-import = ">=1.0" +jinja2 = ">=2.11.1" +markdown = ">=3.3.6" +markupsafe = ">=2.0.1" +packaging = ">=20.5" +pathspec = ">=0.11.1" +platformdirs = ">=2.2.0" +pyyaml = ">=5.1" +pyyaml-env-tag = ">=0.1" +watchdog = ">=2.0" + +[package.extras] +i18n = ["babel (>=2.9.0)"] + [[package]] name = "proto-plus" version = "1.28.0" @@ -5946,4 +5989,4 @@ type = ["pytest-mypy (>=1.0.1)"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.14" -content-hash = "39e6d4466acf3af81137ec81fb7ddfd4dc1852193dd8609dd143a19354a7baf5" +content-hash = "29988739a1ae37c0e133e6d1a229c8fa65bb7153d1cc77fa76a6c511870b7279" diff --git a/pyproject.toml b/pyproject.toml index bbca8e943..fb3f7aea6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -121,6 +121,7 @@ mkdocstrings-python = "^1.16.12" griffe = ">=1.0,<2" griffe-pydantic = ">=1.1.6,<1.3" ruff = "^0.8.0" +mkdocs-redirects = "^1.2.3" [build-system] requires = ["poetry-core"] From d992c582a7a0be2e5c6a955efec00db94bbec053 Mon Sep 17 00:00:00 2001 From: edwardvaneechoud Date: Sat, 4 Jul 2026 12:21:27 +0200 Subject: [PATCH 2/2] improve documentation by adding personas and proper introductions --- .claude/skills/flowfile-docs-review/SKILL.md | 20 +- docs/assets/try-sales-pipeline.html | 4 +- docs/examples/catalog_analysis.py | 4 +- docs/examples/catalog_references.py | 22 ++ docs/examples/first_pipeline.py | 4 +- docs/examples/sales_pipeline.py | 4 +- docs/for-developers/catalog-architecture.md | 113 ++++++ docs/for-developers/kernel-architecture.md | 4 +- docs/index.html | 83 +++-- docs/installation.md | 123 ++++++ docs/quickstart.md | 37 +- docs/stylesheets/extra.css | 156 ++++++-- docs/users/analyze-your-data.md | 70 ++++ docs/users/build-flows-visually.md | 70 ++++ docs/users/coming-from-excel.md | 55 +-- docs/users/data-elsewhere.md | 62 +++ docs/users/deploy-for-a-team.md | 70 ++++ docs/users/deployment/desktop.md | 2 +- docs/users/deployment/docker.md | 7 +- docs/users/index.md | 28 +- docs/users/python-api/index.md | 14 +- .../reference/catalog-references.md | 11 +- docs/users/python-api/tutorials/index.md | 40 +- docs/users/visual-editor/building-flows.md | 4 +- docs/users/visual-editor/catalog/index.md | 8 +- docs/users/visual-editor/catalog/notebooks.md | 40 ++ docs/users/visual-editor/kernel-api.md | 352 ++++++++++++++++++ docs/users/visual-editor/kernels.md | 316 +--------------- .../visual-editor/tutorials/sales-pipeline.md | 2 +- docs/users/write-python.md | 77 ++++ docs/what-is-flowfile-plain.md | 59 +++ docs/what-is-flowfile-technical.md | 41 ++ docs/what-is-flowfile.md | 22 ++ .../flowfile_core/catalog/services/sql.py | 4 +- .../tests/docs_examples/test_docs_examples.py | 30 +- mkdocs.yml | 18 +- 36 files changed, 1469 insertions(+), 507 deletions(-) create mode 100644 docs/examples/catalog_references.py create mode 100644 docs/for-developers/catalog-architecture.md create mode 100644 docs/installation.md create mode 100644 docs/users/analyze-your-data.md create mode 100644 docs/users/build-flows-visually.md create mode 100644 docs/users/data-elsewhere.md create mode 100644 docs/users/deploy-for-a-team.md create mode 100644 docs/users/visual-editor/catalog/notebooks.md create mode 100644 docs/users/visual-editor/kernel-api.md create mode 100644 docs/users/write-python.md create mode 100644 docs/what-is-flowfile-plain.md create mode 100644 docs/what-is-flowfile-technical.md create mode 100644 docs/what-is-flowfile.md diff --git a/.claude/skills/flowfile-docs-review/SKILL.md b/.claude/skills/flowfile-docs-review/SKILL.md index a3488c760..4482c0940 100644 --- a/.claude/skills/flowfile-docs-review/SKILL.md +++ b/.claude/skills/flowfile-docs-review/SKILL.md @@ -35,10 +35,22 @@ The canonical exemplar is `docs/ai/index.md`. Rules, all checkable: One tab per arriving audience; the Home page routes with persona cards (one hop). +**Persona cards route to persona landing pages, never to feature pages** (maintainer-corrected, 2026-07). Six persona routes live under Get Started → By Audience: `coming-from-excel`, `build-flows-visually`, `analyze-your-data`, `data-elsewhere`, `write-python`, `deploy-for-a-team` (all in `docs/users/`). The bar a persona page must meet (maintainer-refined twice — narrow personas and feature-list cadence were both rejected): + +- **Broad opening**: describe the *situation* through several recognizable faces (roles, jobs-to-be-done), not one caricature — "finance person reconciling exports, marketer sizing a campaign, researcher cleaning waves…" beats "your job is answers about cities". +- **A stated mental model near the top** (one or two sentences: how Flowfile thinks about this person's problem) plus an `IMAGE-PLACEHOLDER-TO-CHANGE` describing a mental-model diagram for the maintainer to draw. +- **Depth per stop, not feature lists**: each numbered stop = the person's problem at that point → how the product's model answers it (the *why*) → a concrete example → the deep link. A stop that only names features and links is below the bar. +- **A real example on every page**: tested `--8<--` includes for runnable Python, verified formula/shell fragments otherwise. +- **Feature pages are linked stops** — the catalog is a stop on the analyst's route, not the destination. A persona page summarizes, it never re-teaches. +- **No-code truth**: every route that works without code says so explicitly; never let a `ff.*` column imply Python is required. +- **Shared shape**: numbered stops, `---` rule, one-line `**Fastest first taste:**` footer. If a card's target is a feature index, either the card is mislabeled or the persona page is missing. + +Above the personas sits the **identity tree** (Get Started → What is Flowfile): a short **root** `docs/what-is-flowfile.md` that does positioning only — what space the product occupies (the gap after spreadsheets stop scaling, before a data-engineering team is the only option; four bundled parts; reproducible by construction) — and forks via lens cards into two tagged branches: `what-is-flowfile-plain.md` (non-technical: reproducibility as the felt promise, catalog as where it compounds) and `what-is-flowfile-technical.md` (engineers: **absence of glue** — secrets, Python envs with the right rights, database/cloud/Kafka connection plumbing, keeping data current without an orchestrator, data organization, visualization, explaining your work). Root stays short; depth lives in the branches; branches close on the same reproducible-by-construction line. **Register split between the lenses (maintainer-directed)**: the plain branch may use warm, figurative prose (recipes, kitchens — it's the one page where that's allowed); the technical branch is flat and declarative — every claim states its mechanism, no metaphors, headers like "What it automates", not "The annoyances it eats". Technical readers parse figurative language as padding. Persona pages answer "how do I, given who I am"; the identity tree answers "what is it" — don't blur them, and keep the root-and-branch cross-tags intact. + | Tab | Arriving reader | Owns | |---|---|---| -| Home (`index.html`) | everyone | value prop, sales-pipeline showcase (tested flow download + demo.flowfile.org), persona router | -| Get Started | new user, any kind | `quickstart.md` (install + first visual flow + first Python pipeline), Coming from Excel, Flowfile Lite | +| Home (`index.html`) | everyone | value prop, sales-pipeline showcase (tested flow download + live-demo deep link), persona router | +| Get Started | new user, any kind | the What-is identity tree, `installation.md` (**the canonical install home** — all five paths as tabs; quickstart carries only the pip fast-path and links here; deployment pages hold per-edition depth), `quickstart.md` (first visual flow + first Python pipeline), Flowfile Lite, and the six By-Audience persona routes | | Visual Editor | analysts building flows | overview, building flows, formulas (+ generated function reference), node reference (6 categories), kernels, node designer, worked examples | | Connect Your Data | "my data lives elsewhere" | connector matrix, connections & secrets, databases, cloud storage (S3/ADLS/GCS), Kafka, REST APIs & Google Analytics | | Catalog & Automation | analyzing/operating data | catalog, virtual tables, SQL editor, visualizations, schedules, subflows, projects & git | @@ -68,7 +80,7 @@ Verify against code, never against another prose doc (READMEs and CLAUDE.md drif | Kernel images and defaults | `flowfile_core/flowfile_core/kernel/manager.py` (image tags), `kernel/models.py` (default memory 4 GB, CPU 2) | | Password/auth policy | `flowfile_core/auth/password.py` (8 chars + number + special; no case rules) | | Storage paths per mode | `shared/storage_config.py` + `docker-compose.yml` overrides (`FLOWFILE_USER_DATA_DIR=/app/user_data` in shipped compose) | -| Docker deployment facts | `docker-compose.yml` itself (volume `flowfile-internal-storage`, `shm_size`, scheduler enabled in shipped compose, `FLOWFILE_INTERNAL_TOKEN` required) | +| Docker deployment facts | `docker-compose.yml` itself (volume `flowfile-internal-storage`, `shm_size`, scheduler enabled in shipped compose, `FLOWFILE_INTERNAL_TOKEN` required). The bundled compose **builds from source**; server/HTTPS deployments are the separate [flowfile-hosting](https://github.com/Edwardvaneechoud/flowfile-hosting) kit (published images pinned via `FLOWFILE_VERSION`, Caddy/Cloudflare-Tunnel/LAN ingress, `./install.sh`) — verify hosting claims against that repo, and make hosting *changes* there, never in this repo | | Catalog internals | `flowfile_core/flowfile_core/catalog/` (services/, `constants.py` — thumbnail cap 500 KB, SQL recursion limit 5) | | Flow save format | `.yaml` default (`.yml`/`.json` accepted; `.flowfile` is legacy pickle, open-only) — `flowfile/manage/io_flowfile.py` | | Formula functions | generated `docs/users/formulas/functions.md` (never hand-edit; `make formula_docs`) | @@ -92,6 +104,8 @@ Two example kinds, two automatic gates each (pytest at runtime, `pymdownx.snippe **Data**: only committed, seeded datasets (`data/templates/*.csv` via `generate_template_data.py`; new generators use their own `random.Random(n)` and are called last so existing CSVs stay byte-identical — verify with `git diff`). Assert exact values for deterministic transforms; schema/shape only for ML and fuzzy outputs. +**Reads are URL-first in user-facing examples** (maintainer-directed, 2026-07): sample reads use the public raw-GitHub URL (`TEMPLATE_DATA_BASE_URL` in `templates/data_downloader.py` + filename), not repo-relative paths — so a pasted snippet runs for pip-install users with no checkout, and flows opened in the WASM demo fetch data instead of embedding it (the share-link stays ~1 KB). The test runner skips (never mocks) URL-bearing examples when the URL is unreachable — which includes "not merged to main yet"; they execute for real once the data is public. Repo-relative paths remain fine for contributor-facing examples that already assume a checkout. + **Add-a-worked-example recipe** (the "blog post" flow — no new test code, ever): 1. Pick/extend a committed dataset. 2. Build the flow in-process, save, placeholder-ize paths, add `_template_meta` → drop into `data/templates/flows/`. diff --git a/docs/assets/try-sales-pipeline.html b/docs/assets/try-sales-pipeline.html index edebef4ac..7fd452648 100644 --- a/docs/assets/try-sales-pipeline.html +++ b/docs/assets/try-sales-pipeline.html @@ -3,9 +3,9 @@ Opening the sales pipeline in Flowfile Lite… - + -

Opening the sales pipeline in Flowfile Lite… The flow and its sample data are encoded in the link itself — nothing is uploaded.

+

Opening the sales pipeline in Flowfile Lite… The flow definition travels in the link; its sample data is read from the public sample CSV on GitHub.

diff --git a/docs/examples/catalog_analysis.py b/docs/examples/catalog_analysis.py index 49cab16fc..cef9a3be8 100644 --- a/docs/examples/catalog_analysis.py +++ b/docs/examples/catalog_analysis.py @@ -4,8 +4,10 @@ import flowfile as ff from flowfile_frame import read_catalog_sql +SALES = "https://raw.githubusercontent.com/edwardvaneechoud/flowfile/main/data/templates/supermarket_sales.csv" + income_by_city = ( - ff.read_csv("data/templates/supermarket_sales.csv") + ff.read_csv(SALES) .group_by("city") .agg(ff.col("gross_income").sum().alias("total_income")) ) diff --git a/docs/examples/catalog_references.py b/docs/examples/catalog_references.py new file mode 100644 index 000000000..1788e3a44 --- /dev/null +++ b/docs/examples/catalog_references.py @@ -0,0 +1,22 @@ +"""Name-based catalog handles: create a catalog and schema, write a table, read it back.""" + +# --8<-- [start:example] +import flowfile as ff + +SALES = "https://raw.githubusercontent.com/edwardvaneechoud/flowfile/main/data/templates/supermarket_sales.csv" + +catalog = ff.CatalogReference("docs_sales", auto_create=True) +schema = catalog.schema("raw", auto_create=True) + +orders = ff.read_csv(SALES).select("invoice_id", "city", "quantity", "gross_income") +schema.write_table(orders, "orders") + +# Use the schema handle anywhere a namespace_id used to be required +bulk = schema.read_table("orders").filter(ff.col("quantity") > 7) +schema.write_table(bulk, "bulk_orders") +# --8<-- [end:example] + +result = schema.read_table("bulk_orders").collect() +assert result.height == 326 # raw rows with quantity > 7 (incl. the 30 seeded duplicates) +assert set(result.columns) == {"invoice_id", "city", "quantity", "gross_income"} +assert result["quantity"].min() == 8 diff --git a/docs/examples/first_pipeline.py b/docs/examples/first_pipeline.py index 727106c14..9d1f79fc9 100644 --- a/docs/examples/first_pipeline.py +++ b/docs/examples/first_pipeline.py @@ -3,8 +3,10 @@ # --8<-- [start:example] import flowfile as ff +SALES = "https://raw.githubusercontent.com/edwardvaneechoud/flowfile/main/data/templates/supermarket_sales.csv" + revenue_by_line = ( - ff.read_csv("data/templates/supermarket_sales.csv") + ff.read_csv(SALES) .with_columns((ff.col("unit_price") * ff.col("quantity")).alias("revenue")) .filter(ff.col("quantity") > 5) .group_by("product_line") diff --git a/docs/examples/sales_pipeline.py b/docs/examples/sales_pipeline.py index f3067db4d..d8e8af031 100644 --- a/docs/examples/sales_pipeline.py +++ b/docs/examples/sales_pipeline.py @@ -3,8 +3,10 @@ # --8<-- [start:example] import flowfile as ff +SALES = "https://raw.githubusercontent.com/edwardvaneechoud/flowfile/main/data/templates/supermarket_sales.csv" + result = ( - ff.read_csv("data/templates/supermarket_sales.csv") + ff.read_csv(SALES) .unique() .filter(ff.col("quantity") > 7) .group_by("city") diff --git a/docs/for-developers/catalog-architecture.md b/docs/for-developers/catalog-architecture.md new file mode 100644 index 000000000..dedaffe65 --- /dev/null +++ b/docs/for-developers/catalog-architecture.md @@ -0,0 +1,113 @@ +# Catalog Architecture + +The catalog is Flowfile's largest subsystem: namespaces, physical and virtual tables, SQL, visualizations, dashboards, notebooks, schedules, runs, artifacts, and the sharing model all live behind one facade. This page is the contributor's map — how the pieces compose, where metadata and data actually live, the resolution paths that matter, and the invariants you must not break. It pairs with [Kernel Architecture](kernel-architecture.md) and [Visualizations & Graphic Walker](visualizations.md), which own their subsystems' deep dives. + +## The big picture + +Architecturally the catalog is **two substrates and a service layer**. Metadata — every namespace, table row, schedule, grant — lives in the shared SQLite catalog DB. Table *data* lives outside the DB as Delta directories (locally or on S3). Around those, per-domain services implement features, and everything heavy routes to neighbors: full-frame reads to the worker, user code to kernels, timed execution to the embedded scheduler. + + + +## Metadata vs data + +**Metadata** is SQLAlchemy models in `database/models.py`: `CatalogNamespace` (two-level catalog→schema tree; level-0 rows may carry `storage_uri`/`storage_connection_name`), `FlowRegistration` (+ stable `flow_uuid`), `FlowRun` (with a YAML `flow_snapshot`), `FlowSchedule` + `ScheduleTriggerTable`, `CatalogTable`, `CatalogTableReadLink` (read-lineage), `CatalogVisualization`, `CatalogDashboard`, `CatalogNotebook` (metadata only — cells live on disk), `GlobalArtifact` (metadata only — blobs live on the kernel shared volume), engagement tables, and the polymorphic `ResourceGrant` layer. + +**Data** is one Delta directory per table under `storage.catalog_tables_directory` (`_/` with `_delta_log/`; legacy single `.parquet` files still open — `delta_utils.py` detects the format). A `CatalogTable` row points at its directory via `file_path` and caches `schema_json`, `row_count`, `size_bytes` — for optimized virtual tables `file_path` is NULL because there is no data on disk at all. + +**The S3 backend is per-catalog, not global.** A level-0 namespace's `storage_uri` + `storage_connection_name` make *new* tables under that catalog write to object storage; existing local tables never move, and the metadata DB stays local. `storage_backend.resolve_for_namespace` returns a `CatalogStorageTarget` with two credential forms: decrypted `storage_options` for core's own bounded reads, and an **owner-encrypted** `worker_interface` for the worker hand-off — secrets never cross the wire in plaintext, and credentials always resolve as the *catalog owner*, never the calling user. `FLOWFILE_CATALOG_STORAGE_URI`/`_CONNECTION` are snapshotted onto a catalog once at creation, not read live. + +## The service layer + +`CatalogService` (`catalog/service.py`) is a **facade over composed sub-services**, one per domain under `catalog/services/`: namespaces, flows, runs, engagement, schedules, tables, virtual_tables, sql, previews, visualizations, notebooks, stats. Composition is constructor injection with two deliberate quirks: + +- **Cyclic late-binding** — `SqlService` and `VirtualTableService` need each other (SQL resolution resolves virtual tables; creating a query-virtual derives its schema by running SQL). The cycle is broken with `.bind()` calls after construction. +- **Facade back-binding** — visualization/SQL/schedule services call back through the facade (`bind_facade(self)`) so tests can monkeypatch `CatalogService` methods and have the patch take effect on internal paths. + +Data access goes exclusively through `CatalogRepository` (protocol) / `SQLAlchemyCatalogRepository`. Services raise the `CatalogError` hierarchy (`catalog/exceptions.py`), never `HTTPException`; the router (`routes/catalog.py`, mounted at `/catalog`) is a thin adapter that maps exceptions to status codes via `_CATALOG_EXCEPTION_MAP` and builds a fresh repository + service per request. The router-level auth dependency accepts a JWT **or** the kernel's `X-Internal-Token`. + +## Access control + +Authorization is the facade's job, implemented by `AccessResolver` (`catalog/access.py`) over `auth/sharing.py`: + +- The router always injects a resolver, but it only *restricts* when `sharing_enabled() and not user.is_admin and not is_synthetic_principal(user)`. Electron/single-user mode, admins, the kernel's `_internal_service` principal, and internal callers that construct `CatalogService` without an `access` argument (scheduler, flow execution, tests) are unrestricted. +- Resolution is **own-first, then group-granted**: accessible ids = rows the user owns ∪ ids granted to any of their groups, at `use` or `manage` level. A use-level grant is read-only — `writable_namespace_ids` requires ownership or a *manage* grant. +- **Namespace grants cascade**: a grant on a catalog or schema reaches the tables, flows, visualizations, dashboards, notebooks, and artifacts inside it (one-level child expansion, memoized per request). Tree listing prunes invisible items but keeps "context-only" ancestor namespaces as breadcrumbs with redacted descriptions, and stamps every DTO with its effective access level. + + + +!!! warning "The delete invariant" + Every resource-delete path must call `sharing.delete_grants_for_resource`. SQLite reuses rowids, so a surviving grant row would silently attach itself to a future, unrelated resource. The repository does this on every delete and an ORM `after_delete` backstop covers `session.delete` — but bulk `query.delete()` bypasses ORM events and must clean up explicitly. + +## Virtual tables + +A `catalog_writer` node in virtual mode registers a table that stores no data; what gets stored decides how reads resolve later: + +- At registration, `FlowGraph.check_flow_laziness` walks the writer's upstream using each node's declared `laziness` (`configs/node_store/nodes.py`). If everything is lazy *and* no source is on cloud storage, the write is **optimized**: the LazyFrame plan is serialized into `CatalogTable.serialized_lazy_frame`, together with the Delta versions of its source tables (`source_table_versions`) and a human-readable `polars_plan`. +- **Optimized resolution** deserializes that plan — no flow run — after `check_source_versions_current` confirms every recorded source Delta version still matches; staleness falls through to the standard path. +- **Standard resolution** opens the producer flow, finds the `catalog_writer` node whose configured `table_name` matches the table's name (that name-keying is what lets one producer flow back several virtual tables), and executes just that node in performance mode — on the worker when offload is enabled. +- **Serialized cloud plans are never replayed**: a serialized cloud scan embeds decrypted `storage_options`, so `serialized_frame_uses_cloud` forces such tables onto the standard path permanently. + +**Query virtual tables** (the SQL editor's "save as virtual table") store a `sql_query` instead of a plan. Resolution builds a `pl.SQLContext`, registering every referenced table and recursively resolving nested virtuals — bounded by `QUERY_VIRTUAL_TABLE_RECURSION_LIMIT = 5` plus a visited-set cycle check, with per-table grant checks in restricted mode. + + + +## SQL execution + +`SqlService.execute_sql_query` resolves every accessible table into a name map (qualified `catalog.schema.table`, `schema.table`, and bare name when unique), materializes referenced virtual tables to IPC files via the worker, then hands the whole query to the worker: a spawned child opens the Delta directories and IPC files in a `pl.SQLContext`, collects at most `max_rows`, and returns rows. **Core never collects** — it ships names and paths. Cloud-backed physical tables are excluded from this path with a clear error (the in-flow `sql_query` node has its own resolution path that handles them). "Save as flow" generates a `catalog_reader`-per-table → `sql_query` flow and registers it. + +## Schedules and triggers + +Schedule rows live in the shared DB, and the embedded scheduler (`flowfile_scheduler/engine.py`) deliberately **imports nothing from core** — it reads mirrored models from `shared/`, polls every 30 s, and coordinates instances through a single-row advisory lock with a 90 s stale-takeover. + +Table triggers fire through **two cooperating paths**: + +- **Push (synchronous)**: catalog writes call `safely_fire_table_trigger_schedules` — find enabled watchers, skip any with an active run, commit `last_trigger_table_updated_at` *before* spawning (that pre-commit is the double-fire guard), then spawn the run. +- **Poll (safety net)**: every tick the scheduler compares each watched table's `updated_at` with the schedule's recorded value and fires on newer. Set-triggers require *all* linked tables updated since the last fire. + +Fired schedules execute headlessly: `spawn_flow_run` creates the `FlowRun` row and spawns a subprocess (`flowfile run flow --run-id `, or the frozen-binary equivalent), which reports completion back through `shared.run_completion` — again with no core import. Maintenance operations (optimize/vacuum) deliberately keep `updated_at` unchanged so they never masquerade as data changes and fire triggers. + + + +## Notebooks + +Notebooks are the catalog's exploration console, and architecturally they're a deliberate split-identity design: **the DB row is registration, the file is the content.** A `CatalogNotebook` row carries name, namespace placement, owner, and `default_kernel_id`; the cells live on disk as one deterministic YAML file per notebook at `//.notebook.yaml` (`catalog/services/notebook_store.py`). That's the same row-plus-file pattern flows use (`FlowRegistration` + its `.yaml`), and for the same reason: it's what lets [Projects](../users/projects.md) version notebooks in git. + +The store is built for that job — atomic writes, fixed key order, multi-line cell `source` dumped as YAML literal blocks so diffs read like code review, and a server-derived filename (nothing user-controlled in the path). Reads degrade gracefully: a missing or torn file yields an empty notebook and a malformed cell is skipped rather than raising, so a bad `git merge` on a notebook file can't brick the catalog. + +Three design decisions worth knowing before touching this code: + +- **`NotebookService` owns no execution.** The cell model is mixed — `python` / `sql` / `markdown`, with per-cell `metadata` (e.g. SQL `max_rows`) — and each type routes elsewhere: SQL cells to the same `/catalog/sql/execute` worker path as the SQL editor, Python cells to the kernel's `execute_cell` with the full `flowfile_ctx` API ([Kernel Architecture](kernel-architecture.md)), Markdown rendered client-side. The shipped editor currently surfaces Python and Markdown cells (`notebook/types.ts` `CellType`); SQL cells are schema-supported ahead of the UI. The notebook service only persists. +- **`default_kernel_id` is a plain string, no FK — by design.** It just remembers the last kernel picked for Python cells; deleting that kernel means "nothing pre-selected," never a cascade into notebook rows. +- **The integer `id` is the public handle**: it's the grant key for [sharing](#access-control) (`catalog_notebook` is namespace-scoped, so catalog grants cascade to notebooks) and the frontend derives the kernel session key from it. The `notebook_uuid` exists for the on-disk filename's stability, not for addressing. + +Cell editing gets Jedi-backed code intelligence from core's LSP routes (`flowfile_core/lsp/`) — completions run server-side against the kernel's environment, not in the browser. + + + +## Neighbors + +- **Worker** — the only sanctioned reader of full table data: `catalog_reader.open_catalog_table` (Delta, local or cloud) and `open_virtual_result` (IPC), invoked through core's `trigger_*` calls; results are collected in spawned children and returned as paths/rows, never frames. Long-lived viz session children are the same pattern specialized for Graphic Walker — see [Visualizations](visualizations.md). +- **Kernels** — write Delta directories *directly* on the shared volume, read the resulting metadata from the Delta log without materializing, and POST it to core (`/catalog/tables/from-data` or `/{id}/refresh`); core stores the pre-computed metadata as-is and never scans the file. Kernel-exchange and artifact directories must stay under the kernel shared volume — see [Kernel Architecture](kernel-architecture.md). +- **Artifacts** — `GlobalArtifact` rows are metadata plus a `storage_key`; blobs live on the kernel shared volume with versioning and soft delete. +- **Projects** — catalog mutations fire never-raising `project_sync` hooks that mirror *metadata* (namespaces, table pointers, schedules, notebooks, model metadata — no data, no blobs, secrets as `${secret:NAME}` placeholders) into the active git project. The DB remains the runtime source of truth. + +## Invariants + +| Rule | Why | +|---|---| +| Core never collects full catalog frames — worker children own dataset memory | The core/worker contract; previews are the only bounded exception | +| Every delete calls `delete_grants_for_resource` (bulk deletes explicitly) | SQLite rowid reuse would re-attach stale grants | +| Serialized cloud plans are never deserialized | They embed decrypted storage credentials | +| Storage credentials resolve as the catalog owner; worker payloads stay owner-encrypted | Sharing must not leak or re-key secrets | +| Catalog storage is immutable once a physical table exists; changing it requires ownership | Prevents splitting one catalog's data across backends | +| `sharing_enabled()` reads `FLOWFILE_MODE` from `os.environ` per call | Import-time caching would make docker-mode untestable | +| optimize/vacuum never bump `updated_at` | Maintenance must not fire table triggers | +| Table names reject dots | Dots are the SQL qualification separator | + +## Extension points + +**A new catalog resource type** touches, in order: the model in `database/models.py` plus an Alembic migration (next `NNN_` prefix); a `ResourceSpec` in `sharing.RESOURCE_REGISTRY` (and `_NAMESPACE_SCOPED_TYPES` if namespace grants should cascade to it); a sub-service under `catalog/services/` wired into `CatalogService.__init__`; repository methods whose delete path cleans up grants; DTOs in `schemas/catalog_schema.py`; an exception in `catalog/exceptions.py` mapped in `_CATALOG_EXCEPTION_MAP`; router endpoints; access guards and DTO stamping; and a `project_sync` hook if it should be projectable. + +**A new schedule kind** adds the `schedule_type` semantics and validators, a `_process_*` method in the scheduler's `_tick`, and — if push-triggered — a fan-out method in `schedules.py`. Keep the scheduler free of core imports; reflect any new columns via `shared/models.py`. + +**A new storage backend** extends `storage_backend.py` (URI schemes, `CatalogStorageTarget`, worker payload) and mirrors the join/decrypt on the worker side in `catalog_reader.py`; blob artifacts implement the `shared.artifact_storage.ArtifactStorageBackend` ABC. diff --git a/docs/for-developers/kernel-architecture.md b/docs/for-developers/kernel-architecture.md index 9ec439b1c..e9f22da6b 100644 --- a/docs/for-developers/kernel-architecture.md +++ b/docs/for-developers/kernel-architecture.md @@ -3,7 +3,9 @@ The kernel system provides isolated Python code execution inside Docker containers. This page explains the internal architecture, component interactions, and key design decisions. !!! info "Looking for the user guide?" - See [Kernel Execution](../users/visual-editor/kernels.md) for the user-facing documentation on how to write code and use the `flowfile` API inside kernels. + See [Kernel Execution](../users/visual-editor/kernels.md) for the user-facing documentation on how to write code and use the `flowfile_ctx` API inside kernels. + +Kernels are one half of a pair: the catalog is where their inputs and outputs live — Delta write-back, global artifacts, and the notebooks whose Python cells execute here. [Catalog Architecture](catalog-architecture.md) (especially its [Notebooks](catalog-architecture.md#notebooks) section) covers that side of the contract. --- diff --git a/docs/index.html b/docs/index.html index 353b16fc2..3d48f45fe 100644 --- a/docs/index.html +++ b/docs/index.html @@ -9,13 +9,15 @@