diff --git a/Makefile b/Makefile index e7ff109..25cfd53 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build test test-ts test-py lint render clean setup +.PHONY: build test test-ts test-py lint preview render clean setup VERSION := $(shell grep '^version:' _extensions/marimo/_extension.yml | sed 's/.*: *//') @@ -47,7 +47,7 @@ preview: quarto preview render: - quarto render tutorials/intro.qmd --to html + quarto render clean: rm -rf _site .quarto _extensions/marimo/marimo-engine-v*.js diff --git a/_extensions/marimo/extract.py b/_extensions/marimo/extract.py index 7c85568..71205df 100755 --- a/_extensions/marimo/extract.py +++ b/_extensions/marimo/extract.py @@ -14,6 +14,7 @@ import asyncio import json +import keyword import os import re import sys @@ -27,6 +28,7 @@ import marimo from marimo import App, MarimoIslandGenerator +from marimo._convert.common.format import sql_to_marimo from marimo._session.notebook import AppFileManager try: @@ -60,6 +62,36 @@ "editor": False, } +SQL_DOT_FENCE_REGEX = re.compile( + r"^(\s*`{3,})\s*\{\s*sql\.marimo(?P[^}]*)\}\s*$", + re.MULTILINE, +) +DEFAULT_SQL_QUERY_TARGET = "_df" + + +def is_valid_python_identifier(name: str) -> bool: + return name.isidentifier() and not keyword.iskeyword(name) + + +def sql_query_target(query: Optional[str]) -> str: + if query and is_valid_python_identifier(query): + return query + return DEFAULT_SQL_QUERY_TARGET + + +def is_true_attr(value: Optional[str]) -> bool: + return str(value or "false").lower() == "true" + + +def sql_code_to_python( + code: str, + query: Optional[str], + hide_output: bool = False, + engine: Optional[str] = None, +) -> str: + """Convert a marimo markdown SQL cell into executable Python.""" + return sql_to_marimo(code, sql_query_target(query), hide_output, engine) + def extract_and_strip_quarto_config(block: str) -> tuple[dict[str, Any], str]: pattern = r"^\s*\#\|\s*(.*?)\s*:\s*(.*?)(?=\n|\Z)" @@ -293,6 +325,13 @@ def tree_to_pandoc_export(root: Element) -> SafeWrap: # type: ignore[valid-type code = str(child.text) config, code = extract_and_strip_quarto_config(code) + if child.attrib.get("language") == "sql": + code = sql_code_to_python( + code, + child.attrib.get("query"), + hide_output=is_true_attr(child.attrib.get("hide_output")), + engine=child.attrib.get("engine"), + ) try: stub = app.add_code( @@ -369,6 +408,7 @@ def convert_from_md_to_pandoc_export(text: str, mime_sensitive: bool) -> dict[st """ if not text: return {"header": "", "outputs": []} + text = SQL_DOT_FENCE_REGEX.sub(r"\1sql {.marimo\g}", text) if mime_sensitive: parser = MarimoPandocParser(output_format="marimo-pandoc-export-with-mime") # type: ignore[arg-type] else: diff --git a/_quarto.yml b/_quarto.yml index 18bd397..38d7086 100644 --- a/_quarto.yml +++ b/_quarto.yml @@ -1,6 +1,9 @@ project: type: website output-dir: _site + render: + - index.qmd + - tutorials/*.qmd post-render: python scripts/sync_local_frontend.py preview: port: 7777 @@ -31,6 +34,7 @@ website: - href: tutorials/plots.qmd - href: tutorials/layout.qmd - href: tutorials/fileformat.qmd + - href: tutorials/sql.qmd - href: tutorials/external_dependencies.qmd - href: tutorials/markdown_format.qmd - href: tutorials/for_jupyter_users.qmd diff --git a/lib/cell-execution-regex.ts b/lib/cell-execution-regex.ts index df78b24..ba60c22 100644 --- a/lib/cell-execution-regex.ts +++ b/lib/cell-execution-regex.ts @@ -7,22 +7,26 @@ * ```{python.marimo} ← pampa/dot-joined syntax * ```{python .marimo} ← preferred class syntax * ```python {.marimo} ← legacy (language outside braces) + * ```{sql.marimo} ← SQL dot-joined syntax + * ```sql {.marimo} ← SQL marimo cells * * Groups: * 1: backticks (```+) - * 2: language ("python.marimo" or "python") + * 2: language ("python.marimo", "python", "sql.marimo", or "sql") */ // Matches all marimo cell syntaxes with language always in group 2: // ```{python.marimo} → group 2: "python.marimo" // ```{python .marimo} → group 2: "python" // ```python {.marimo} → group 2: "python" (legacy) +// ```{sql.marimo} → group 2: "sql.marimo" +// ```sql {.marimo} → group 2: "sql" (SQL cells) // // Structure: // - Lookahead ensures .marimo appears somewhere // - \{? handles optional leading brace (present for braced syntax, absent for legacy) -// - Language capture: python or python.marimo +// - Language capture: python/sql or python.marimo/sql.marimo // - [^}]* consumes rest (classes, attributes) until closing brace // Note: accepts some invalid syntax (e.g. comma-separated) that will fail pampa parsing export const MARIMO_CELL_REGEX = - /^\s*(```+)\s*(?=.*\.marimo)\{?(python(?:\.marimo)?)[^}]*\}\s*$/; + /^\s*(```+)\s*(?=.*\.marimo)\{?((?:python|sql)(?:\.marimo)?)[^}]*\}\s*$/; diff --git a/lib/is-marimo-cell.ts b/lib/is-marimo-cell.ts index 5334407..4ff843c 100644 --- a/lib/is-marimo-cell.ts +++ b/lib/is-marimo-cell.ts @@ -11,12 +11,12 @@ export function isMarimoCell(cell: QuartoMdCell): boolean { return false; } const lang = cell.cell_type.language; - // Handle {python.marimo} syntax (quarto parses as language "python.marimo") - if (lang === "python.marimo") { + // Handle {python.marimo}/{sql.marimo} syntax. + if (lang === "python.marimo" || lang === "sql.marimo") { return true; } - // Handle {python .marimo} and legacy python {.marimo} syntax - if (lang === "python") { + // Handle class syntax and legacy language-outside-braces syntax. + if (lang === "python" || lang === "sql") { const firstLine = cell.sourceVerbatim.value.split('\n')[0] || ''; return /\.marimo/.test(firstLine); } diff --git a/tests/cell-execution-regex.test.ts b/tests/cell-execution-regex.test.ts index 42511a5..3dea186 100644 --- a/tests/cell-execution-regex.test.ts +++ b/tests/cell-execution-regex.test.ts @@ -128,6 +128,26 @@ Deno.test("python {.marimo} legacy with leading whitespace", () => { ); }); +Deno.test("sql {.marimo} legacy basic", () => { + assertMatch("```sql {.marimo}", "sql", "sql {.marimo} legacy basic"); +}); + +Deno.test("sql {.marimo} with query attribute", () => { + assertMatch( + '```sql {.marimo query="result"}', + "sql", + "sql {.marimo} with query attribute", + ); +}); + +Deno.test("{sql .marimo} class syntax", () => { + assertMatch("```{sql .marimo}", "sql", "{sql .marimo} class syntax"); +}); + +Deno.test("{sql.marimo} basic", () => { + assertMatch("```{sql.marimo}", "sql.marimo", "{sql.marimo} basic"); +}); + Deno.test("four backticks {python .marimo}", () => { assertMatch( "````{python .marimo}", diff --git a/tests/is-marimo-cell.test.ts b/tests/is-marimo-cell.test.ts index 60af89f..55a42cc 100644 --- a/tests/is-marimo-cell.test.ts +++ b/tests/is-marimo-cell.test.ts @@ -30,6 +30,16 @@ Deno.test("python {.marimo} legacy syntax returns true", () => { assertEquals(isMarimoCell(cell), true); }); +Deno.test("sql {.marimo} legacy syntax returns true", () => { + const cell = makeCell("sql", "```sql {.marimo query=\"result\"}\nSELECT 1"); + assertEquals(isMarimoCell(cell), true); +}); + +Deno.test("sql.marimo language returns true", () => { + const cell = makeCell("sql.marimo", "SELECT 1"); + assertEquals(isMarimoCell(cell), true); +}); + Deno.test("plain python without .marimo returns false", () => { const cell = makeCell("python", "print('hello')"); assertEquals(isMarimoCell(cell), false); diff --git a/tests/python/test_extract.py b/tests/python/test_extract.py index ed87334..f0d370c 100644 --- a/tests/python/test_extract.py +++ b/tests/python/test_extract.py @@ -12,6 +12,7 @@ extract_and_strip_quarto_config, get_mime_render, pyproject_to_script_metadata, + sql_code_to_python, ) @@ -59,7 +60,7 @@ def test_marimo_layout_maps_to_layout_file(self): assert config["layout_file"] == "grid.json" def test_marimo_version_stripped(self): - root = Element("root", attrib={"marimo-version": "0.14.0"}) + root = Element("root", attrib={"marimo-version": "0.23.1"}) config = app_config_from_root(root) assert "marimo-version" not in config @@ -94,7 +95,58 @@ def test_preserves_existing_script_metadata(self): assert metadata == "# /// script\n# dependencies = []\n# ///\n" +class TestSqlCodeToPython: + def test_without_query_renders_sql_expression(self): + result = sql_code_to_python("SELECT * FROM df;", None) + + assert "_df = mo.sql(" in result + assert "SELECT * FROM df;" in result + + def test_with_query_assigns_result(self): + result = sql_code_to_python("SELECT * FROM df;", "filtered") + + assert "filtered = mo.sql(" in result + assert "SELECT * FROM df;" in result + + def test_invalid_query_falls_back_to_default_target(self): + result = sql_code_to_python("SELECT * FROM df;", "not-valid") + + assert "_df = mo.sql(" in result + assert "not-valid = mo.sql(" not in result + + def test_keyword_query_falls_back_to_default_target(self): + result = sql_code_to_python("SELECT * FROM df;", "class") + + assert "_df = mo.sql(" in result + assert "class = mo.sql(" not in result + + def test_forwards_sql_options_to_marimo_formatter(self): + result = sql_code_to_python( + "SELECT * FROM df;", + "filtered", + hide_output=True, + engine="engine", + ) + + assert "filtered = mo.sql(" in result + assert "output=False" in result + assert "engine=engine" in result + + class TestConvertFromMdToPandocExport: + def _extract_notebook_code(self, header: str) -> str: + notebook_match = re.search(r"", header) + assert notebook_match is not None + return unquote(notebook_match.group(1)) + + def _convert_without_eval(self, markdown: str) -> dict: + original_eval = default_config["eval"] + default_config["eval"] = False + try: + return convert_from_md_to_pandoc_export(markdown, mime_sensitive=False) + finally: + default_config["eval"] = original_eval + def test_injects_pyproject_into_exported_notebook(self): markdown = """--- title: External dependencies @@ -118,9 +170,7 @@ def test_injects_pyproject_into_exported_notebook(self): assert "__MARIMO_EXPORT_CONTEXT__" in header assert "