CodeMirror stack 2/4: rich schema autocomplete, /-/editor-schema.json, defaultTable

datasette/app.py

@@ -85,6 +85,7 @@
     JumpView,
     InstanceSchemaView,
     DatabaseSchemaView,
+    DatabaseEditorSchemaView,
     TableSchemaView,
 )
 from .views.table import (
@@ -2716,6 +2717,10 @@ def add_route(view, regex):
             DatabaseSchemaView.as_view(self),
             r"/(?P<database>[^\/\.]+)/-/schema(\.(?P<format>json|md))?$",
         )
+        add_route(
+            DatabaseEditorSchemaView.as_view(self),
+            r"/(?P<database>[^\/\.]+)/-/editor-schema\.json$",
+        )
         add_route(
             QueryParametersView.as_view(self),
             r"/(?P<database>[^\/\.]+)/-/query/parameters$",

datasette/static/cm-editor.js

@@ -1,4 +1,5 @@
 import { EditorView, basicSetup } from "codemirror";
+import { Compartment } from "@codemirror/state";
 import { keymap } from "@codemirror/view";
 import { sql, SQLDialect } from "@codemirror/lang-sql";
 
@@ -17,10 +18,22 @@ const SQLite = SQLDialect.define({
   caseInsensitiveIdentifiers: true,
 });
 
+// Builds the sql() extension from a {schema, defaultTable, defaultSchema} conf object
+function sqlExtension(conf) {
+  return sql({
+    dialect: SQLite,
+    schema: conf.schema,
+    defaultTable: conf.defaultTable,
+    defaultSchema: conf.defaultSchema,
+  });
+}
+
 // Utility function from https://codemirror.net/docs/migration/
 export function editorFromTextArea(textarea, conf = {}) {
-  // This could also be configured with a set of tables and columns for better autocomplete:
-  // https://github.com/codemirror/lang-sql#user-content-sqlconfig.tables
+  // Wraps the sql() extension so it can be swapped out later via view.updateSchema()
+  // https://codemirror.net/examples/config/#dynamic-configuration
+  let sqlCompartment = new Compartment();
+
   let view = new EditorView({
     doc: textarea.value,
     extensions: [
@@ -46,15 +59,17 @@ export function editorFromTextArea(textarea, conf = {}) {
       // Meta-Enter from running
       basicSetup,
       EditorView.lineWrapping,
-      sql({
-        dialect: SQLite,
-        schema: conf.schema,
-        defaultTable: conf.defaultTable,
-        defaultSchema: conf.defaultSchema,
-      }),
+      sqlCompartment.of(sqlExtension(conf)),
     ],
   });
 
+  // Allows callers (and plugins) to update the schema/defaultTable/defaultSchema
+  // used for autocomplete after the editor has already been created.
+  view.updateSchema = (conf2) =>
+    view.dispatch({
+      effects: sqlCompartment.reconfigure(sqlExtension(conf2)),
+    });
+
   // Idea taken from https://discuss.codemirror.net/t/resizing-codemirror-6/3265.
   // Using CSS resize: both and scheduling a measurement when the element changes.
   let editorDOM = view.contentDOM.closest(".cm-editor");

datasette/templates/_codemirror_foot.html

@@ -15,6 +15,9 @@
     if (sqlInput) {
       var editor = (window.editor = cm.editorFromTextArea(sqlInput, {
         schema,
+        {% if default_table is defined and default_table %}
+        defaultTable: {{ default_table|tojson }},
+        {% endif %}
       }));
       if (sqlFormat) {
         sqlFormat.addEventListener("click", (ev) => {

datasette/templates/table.html

@@ -126,7 +126,7 @@ <h3>{{ extra_wheres_for_ui|length }} extra where clause{% if extra_wheres_for_ui
 {% endif %}
 
 {% if query.sql and allow_execute_sql %}
-    <p><a class="not-underlined" title="{{ query.sql }}" href="{{ urls.database(database) }}?{{ {'sql': query.sql}|urlencode|safe }}{% if query.params %}&amp;{{ query.params|urlencode|safe }}{% endif %}">&#x270e; <span class="underlined">View and edit SQL</span></a></p>
+    <p><a class="not-underlined" title="{{ query.sql }}" href="{{ urls.database(database) }}?{{ {'sql': query.sql, '_table': table}|urlencode|safe }}{% if query.params %}&amp;{{ query.params|urlencode|safe }}{% endif %}">&#x270e; <span class="underlined">View and edit SQL</span></a></p>
 {% endif %}
 
 <p class="export-links">This data as {% for name, url in renderers.items() %}<a href="{{ url }}">{{ name }}</a>{{ ", " if not loop.last }}{% endfor %}{% if display_rows %}, <a href="{{ url_csv }}">CSV</a> (<a href="#export">advanced</a>){% endif %}</p>

datasette/views/database.py

@@ -36,7 +36,10 @@
 from datasette.plugins import pm
 
 from .base import DatasetteError, View, stream_csv
-from .query_helpers import _ensure_stored_query_execution_permissions, _table_columns
+from .query_helpers import (
+    _ensure_stored_query_execution_permissions,
+    _editor_schema,
+)
 from .table_extras import (
     QueryExtraContext,
     resolve_query_extras,
@@ -201,7 +204,7 @@ async def database_actions():
             "queries_count": queries_count,
             "allow_execute_sql": allow_execute_sql,
             "table_columns": (
-                await _table_columns(datasette, database) if allow_execute_sql else {}
+                await _editor_schema(datasette, database) if allow_execute_sql else {}
             ),
             "metadata": await datasette.get_database_metadata(database),
         }
@@ -242,7 +245,7 @@ async def database_actions():
                     queries_count=queries_count,
                     allow_execute_sql=allow_execute_sql,
                     table_columns=(
-                        await _table_columns(datasette, database)
+                        await _editor_schema(datasette, database)
                         if allow_execute_sql
                         else {}
                     ),
@@ -454,6 +457,11 @@ class QueryContext(Context):
             "help": "Dictionary mapping table names to lists of column names, used to power SQL autocomplete."
         }
     )
+    default_table: str = field(
+        metadata={
+            "help": "Name of the focal table for this query, if any - set when the query page was reached from a table-scoped context (such as the table page's 'View and edit SQL' link) so the SQL editor can complete that table's columns unprefixed. ``None`` otherwise, including for stored/canned queries."
+        }
+    )
     alternate_url_json: str = field(
         metadata={"help": "URL for alternate JSON version of this page"}
     )
@@ -713,6 +721,15 @@ async def get(self, request, datasette):
         # Create lookup dict for quick access
         allowed_dict = {r.child: r for r in allowed_tables_page.resources}
 
+        # If the request carries a ?_table= pointing at a real (visible) table
+        # or view in this database, treat this as a table-scoped query - e.g.
+        # arriving here via the "View and edit SQL" link on a table page - so
+        # the SQL editor can offer that table's columns unprefixed. Anything
+        # else (including stored/canned queries, which may reference more
+        # than one table) leaves this as None.
+        requested_table = request.args.get("_table")
+        default_table = requested_table if requested_table in allowed_dict else None
+
         # Are we a stored query?
         stored_query = None
         stored_query_write = False
@@ -1094,10 +1111,11 @@ async def query_actions():
                             datasette, database, request, rows, columns
                         ),
                         table_columns=(
-                            await _table_columns(datasette, database)
+                            await _editor_schema(datasette, database)
                             if allow_execute_sql
                             else {}
                         ),
+                        default_table=default_table,
                         columns=columns,
                         renderers=renderers,
                         url_csv=datasette.urls.path(

datasette/views/execute_write.py

@@ -21,6 +21,7 @@
     _inserted_row_url,
     _json_or_form_payload,
     _prepare_execute_write,
+    _editor_schema,
     _table_columns,
     _wants_json,
 )
@@ -266,6 +267,7 @@ async def _render_form(
         write_template_tables = await _write_template_tables(
             self.ds, db, table_columns, hidden_table_names, request.actor
         )
+        editor_schema = await _editor_schema(self.ds, db.name)
         write_template_operations = _write_template_operations(write_template_tables)
         write_create_table_template_sql = await _create_table_template_sql(
             self.ds, db, request.actor
@@ -328,7 +330,7 @@ async def _render_form(
                 "sql_parameter_name_prefix": SQL_PARAMETER_FORM_PREFIX,
                 "execute_disabled": bool(execute_disabled_reason),
                 "execute_disabled_reason": execute_disabled_reason,
-                "table_columns": table_columns,
+                "table_columns": editor_schema,
                 "write_template_tables": write_template_tables,
                 "write_template_operations": write_template_operations,
                 "write_create_table_template_sql": write_create_table_template_sql,

datasette/views/query_helpers.py

@@ -634,3 +634,92 @@ async def _table_columns(datasette, database_name):
     for view_name in await db.view_names():
         table_columns[view_name] = []
     return table_columns
+
+
+def _column_completion(name, type_):
+    # A @codemirror/lang-sql Completion object for a single column. boost keeps
+    # columns ranked above bare SQL keywords in the autocomplete popup.
+    completion = {
+        "label": name,
+        "type": "property",
+        "boost": 10,
+    }
+    if type_:
+        completion["detail"] = type_
+    return completion
+
+
+async def _schema_tables(datasette, database_name, *, include_hidden=True):
+    """
+    Neutral introspection of a database's tables and views for SQL editors.
+
+    Returns an ordered list of dicts, one per table or view::
+
+        {"name": str, "view": bool,
+         "columns": [{"name": str, "type": str}, ...]}
+
+    ``type`` is the SQLite declared column type (empty string when the column
+    has no declared type). Regular-table columns come from the internal
+    ``catalog_columns`` catalog; views are absent from that catalog so their
+    columns are read directly via PRAGMA table_xinfo. Hidden tables (FTS shadow
+    tables and the like) are excluded unless ``include_hidden`` is True. This is
+    the shared, serialization-agnostic source for both ``_editor_schema`` (which
+    maps it to lang-sql Completion objects) and the ``/-/editor-schema.json``
+    endpoint (which emits it directly).
+    """
+    internal_db = datasette.get_internal_database()
+    result = await internal_db.execute(
+        "select table_name, name, type from catalog_columns where database_name = ?",
+        [database_name],
+    )
+    table_columns = {}
+    for row in result.rows:
+        table_columns.setdefault(row["table_name"], []).append(
+            {"name": row["name"], "type": row["type"]}
+        )
+    db = datasette.get_database(database_name)
+    hidden = set() if include_hidden else set(await db.hidden_table_names())
+    tables = []
+    for table_name, columns in table_columns.items():
+        if table_name in hidden:
+            continue
+        tables.append({"name": table_name, "view": False, "columns": columns})
+    # Views are not represented in catalog_columns, so pull their real columns
+    # directly (PRAGMA table_xinfo works against views too).
+    for view_name in await db.view_names():
+        columns = [
+            {"name": column.name, "type": column.type}
+            for column in await db.table_column_details(view_name)
+        ]
+        tables.append({"name": view_name, "view": True, "columns": columns})
+    return tables
+
+
+async def _editor_schema(datasette, database_name):
+    """
+    Build a lang-sql SQLNamespace for the CodeMirror SQL editor autocomplete.
+
+    Returns a dict keyed by table or view name. Table values are lists of
+    Completion objects (one per column, carrying the column's SQLite type as
+    ``detail``). Views are wrapped in a ``{"self": Completion, "children": [...]}``
+    container so the popup can label them as views while still completing their
+    real columns. See @codemirror/lang-sql >= 6.6 SQLNamespace / Completion.
+    """
+    schema = {}
+    for table in await _schema_tables(datasette, database_name, include_hidden=True):
+        completions = [
+            _column_completion(column["name"], column["type"])
+            for column in table["columns"]
+        ]
+        if table["view"]:
+            schema[table["name"]] = {
+                "self": {
+                    "label": table["name"],
+                    "type": "class",
+                    "detail": "view",
+                },
+                "children": completions,
+            }
+        else:
+            schema[table["name"]] = completions
+    return schema

datasette/views/special.py

@@ -1345,6 +1345,59 @@ async def get(self, request):
             return await self.format_html_response(request, schemas)
 
 
+class DatabaseEditorSchemaView(BaseView):
+    """
+    JSON introspection of a database's tables, views and columns shaped for SQL
+    editor autocomplete consumers (the CodeMirror ``<datasette-sql-editor>``
+    component and external clients such as datasette-paper).
+
+    Distinct from :class:`DatabaseSchemaView` (``/<db>/-/schema.json``), which
+    returns the raw DDL as a SQL string gated on ``view-database`` alone. This
+    endpoint returns a neutral structured payload and is gated on both
+    ``view-database`` and ``execute-sql`` — the same permissions as the inline
+    editor schema handed to the SQL query page.
+    """
+
+    name = "database_editor_schema"
+    has_json_alternate = False
+
+    async def get(self, request):
+        from .query_helpers import _schema_tables
+
+        database_name = request.url_vars["database"]
+
+        # view-database is checked first so actors without it cannot
+        # distinguish an existing database from a missing one, and a denied
+        # request only ever leaks the permission action name, never table names.
+        await self.ds.ensure_permission(
+            action="view-database",
+            resource=DatabaseResource(database=database_name),
+            actor=request.actor,
+        )
+        if database_name not in self.ds.databases:
+            headers = {}
+            if self.ds.cors:
+                add_cors_headers(headers)
+            return Response.json(
+                error_body("Database not found", 404), status=404, headers=headers
+            )
+        await self.ds.ensure_permission(
+            action="execute-sql",
+            resource=DatabaseResource(database=database_name),
+            actor=request.actor,
+        )
+
+        await self.ds.refresh_schemas()
+        tables = await _schema_tables(self.ds, database_name, include_hidden=False)
+
+        headers = {}
+        if self.ds.cors:
+            add_cors_headers(headers)
+        return Response.json(
+            {"database": database_name, "tables": tables}, headers=headers
+        )
+
+
 class TableSchemaView(SchemaBaseView):
     """
     Displays schema for a specific table.