unified: Hook up swift-syntax AST

At this point it's only a proof-of-concept translation -- it translates
`sourceFile` nodes, but everything else gets mapped to
'unsupported_node`.

Author: tausbn

Cargo.lock

@@ -167,6 +167,28 @@ impl Schema {
         id
     }
 
+    /// Register every kind (named and unnamed) and field *name* from `other`
+    /// into this schema (idempotent). Ids are assigned in this schema's own id
+    /// space; existing ids are unchanged.
+    ///
+    /// This is used when running desugaring rules over an AST that was built
+    /// against a different schema (e.g. from an external parser): the rules
+    /// build output nodes whose kind/field names come from `other`, and those
+    /// names must resolve in the AST's own schema. Only names are needed — the
+    /// rule engine resolves kinds/fields by name and does not consult
+    /// `other`'s field-type or supertype information.
+    pub fn register_names_from(&mut self, other: &Schema) {
+        for name in other.kind_ids.keys() {
+            self.register_kind(name);
+        }
+        for name in other.unnamed_kind_ids.keys() {
+            self.register_unnamed_kind(name);
+        }
+        for name in other.field_ids.keys() {
+            self.register_field(name);
+        }
+    }
+
     /// Track a name for a kind ID without registering it as named or
     /// unnamed. Useful when importing tree-sitter ID tables that may
     /// contain duplicate IDs across the named/unnamed split.

shared/yeast-schema/src/schema.rs

@@ -524,10 +524,15 @@ impl Ast {
         self.schema.register_field(name)
     }
 
-    fn union_source_range_of_children(
-        &self,
-        fields: &BTreeMap<FieldId, Vec<Id>>,
-    ) -> Option<Range> {
+    /// Register every kind and field name from `schema` into this AST's schema
+    /// (idempotent). Used before desugaring an externally-built AST so that
+    /// rules can build output nodes whose kind/field names come from the
+    /// desugarer's output schema.
+    pub fn register_names_from_schema(&mut self, schema: &schema::Schema) {
+        self.schema.register_names_from(schema);
+    }
+
+    fn union_source_range_of_children(&self, fields: &BTreeMap<FieldId, Vec<Id>>) -> Option<Range> {
         let mut start_byte: Option<usize> = None;
         let mut end_byte: Option<usize> = None;
         let mut start_point = Point { row: 0, column: 0 };
@@ -1448,6 +1453,18 @@ impl<'a, C: Clone + Default> Runner<'a, C> {
         let mut user_ctx = C::default();
         self.run_with_ctx(input, &mut user_ctx)
     }
+
+    /// Run all phases over an already-built `ast`, using the default context
+    /// (`C::default()`). Unlike [`run_from_tree`](Self::run_from_tree), the AST
+    /// is supplied by the caller (e.g. built from an external parser's output)
+    /// rather than constructed from a tree-sitter tree. The caller is
+    /// responsible for ensuring the AST's schema knows any output kind/field
+    /// names the rules will build (see [`Ast::register_names_from_schema`]).
+    pub fn run_from_ast(&self, mut ast: Ast) -> Result<Ast, String> {
+        let mut user_ctx = C::default();
+        self.run_phases(&mut ast, &mut user_ctx)?;
+        Ok(ast)
+    }
 }
 
 // ---------------------------------------------------------------------------
@@ -1470,6 +1487,12 @@ pub trait Desugarer: Send + Sync {
     /// Parse `tree` against `source` and run the desugaring pipeline.
     /// Each call constructs a fresh default user context internally.
     fn run_from_tree(&self, tree: &tree_sitter::Tree, source: &[u8]) -> Result<Ast, String>;
+
+    /// Run the desugaring pipeline over an already-built `ast` (e.g. produced
+    /// by an external parser rather than tree-sitter). The desugarer ensures
+    /// the AST's schema knows its output kind/field names before running the
+    /// rules. Each call constructs a fresh default user context internally.
+    fn run_from_ast(&self, ast: Ast) -> Result<Ast, String>;
 }
 
 /// A concrete [`Desugarer`] backed by a [`DesugaringConfig<C>`] for a
@@ -1507,4 +1530,12 @@ impl<C: Default + Clone + Send + Sync + 'static> Desugarer for ConcreteDesugarer
         let runner = Runner::with_schema(self.language.clone(), &self.schema, &self.config.phases);
         runner.run_from_tree(tree, source)
     }
+
+    fn run_from_ast(&self, mut ast: Ast) -> Result<Ast, String> {
+        // The AST was built against its own (external) schema; make sure the
+        // output kind/field names the rules build are resolvable in it.
+        ast.register_names_from_schema(&self.schema);
+        let runner = Runner::with_schema(self.language.clone(), &self.schema, &self.config.phases);
+        runner.run_from_ast(ast)
+    }
 }

shared/yeast/src/lib.rs

@@ -266,6 +266,59 @@ fn test_query_match() {
     assert!(captures.get_var("right").is_ok());
 }
 
+#[test]
+fn test_run_from_ast_desugars_hand_built_tree() {
+    use std::collections::BTreeMap;
+
+    // Output schema for the desugared tree. Its kind/field names must become
+    // resolvable in the hand-built AST's schema for the rule to build them.
+    let schema_yaml = r#"
+named:
+    assignment:
+        left: leaf
+    leaf:
+"#;
+
+    // A rule over an *input* kind (`wrapper`) that is not in the output schema,
+    // rewriting to an output `assignment` node.
+    let rules: Vec<Rule> = vec![yeast::rule!(
+        (wrapper)
+        =>
+        (assignment left: (leaf "lit"))
+    )];
+
+    let lang: tree_sitter::Language = tree_sitter_ruby::LANGUAGE.into();
+    let config = DesugaringConfig::<()>::new()
+        .add_phase("test", PhaseKind::OneShot, rules)
+        .with_output_node_types_yaml(schema_yaml);
+    let desugarer = ConcreteDesugarer::new(lang, config).unwrap();
+
+    // Build the input AST by hand, as an external parser adapter would. The
+    // schema starts empty and gains the `wrapper` input kind on the fly.
+    let mut ast = Ast::with_schema(yeast::schema::Schema::new());
+    let wrapper_kind = ast.register_kind("wrapper");
+    let root = ast.create_node_with_range(
+        wrapper_kind,
+        NodeContent::DynamicString(String::new()),
+        BTreeMap::new(),
+        true,
+        None,
+    );
+    ast.set_root(root);
+
+    // Desugaring the hand-built AST applies the rule, producing `assignment`
+    // even though the AST was built against a schema with no output kinds.
+    let out = desugarer
+        .run_from_ast(ast)
+        .expect("run_from_ast should succeed");
+    let out_root = out.get_node(out.get_root()).expect("root exists");
+    assert_eq!(out_root.kind_name(), "assignment");
+    let dump = dump_ast(&out, out.get_root(), "");
+    assert!(dump.contains("assignment"), "unexpected dump: {dump}");
+    assert!(dump.contains("left"), "unexpected dump: {dump}");
+    assert!(dump.contains("leaf"), "unexpected dump: {dump}");
+}
+
 #[test]
 fn test_query_no_match() {
     let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]);

shared/yeast/tests/test.rs

@@ -3,6 +3,15 @@ use codeql_extractor::extractor::simple;
 #[path = "swift/swift.rs"]
 mod swift;
 
+/// swift-syntax JSON -> `yeast::Ast` adapter for the Swift front-end.
+///
+/// Currently exercised by tests and the forthcoming runtime extraction path;
+/// `allow(dead_code)` because this is a binary crate, so its public API isn't
+/// counted as used until the binary itself calls it.
+#[path = "swift/adapter.rs"]
+#[allow(dead_code)]
+pub mod swift_adapter;
+
 /// Shared YEAST output AST schema for all languages.
 pub(crate) const OUTPUT_AST_SCHEMA: &str = include_str!("../../ast_types.yml");
 

unified/extractor/src/languages/mod.rs

@@ -1,6 +1,10 @@
-//! Adapter that converts the swift-syntax JSON tree (see [`crate::parse_to_json`])
-//! into a [`yeast::Ast`], the in-memory format the CodeQL desugaring rules
-//! operate on.
+//! Converts the swift-syntax JSON syntax tree into a [`yeast::Ast`], the
+//! in-memory format the CodeQL desugaring rules operate on.
+//!
+//! The JSON tree is produced by the `swift-syntax-rs` crate's Swift FFI shim
+//! (`parse_to_json`). This module is pure Rust (only `yeast` + `serde_json`),
+//! so the extractor consumes swift-syntax output without pulling in the Swift
+//! toolchain (the JSON is produced out-of-process).
 //!
 //! The mapping mirrors tree-sitter's node model, which is what yeast (and the
 //! extractor's rewrite rules) expect:
@@ -15,9 +19,8 @@
 //!   list-valued field maps directly to that field holding several children.
 //!
 //! Note: this preserves swift-syntax's own kind/field names. Aligning those
-//! names with the tree-sitter-swift schema (so the existing rewrite rules fire)
-//! is a separate, later step; this module is only concerned with getting the
-//! tree into yeast's format.
+//! names with the tree-sitter-swift schema (so the rewrite rules in
+//! [`super::swift`] fire) is done incrementally in the rules.
 
 use std::collections::BTreeMap;
 
@@ -409,40 +412,4 @@ mod tests {
             "comment should not appear as an AST node"
         );
     }
-
-    /// End-to-end: real Swift source parsed by the shim, then adapted into a
-    /// `yeast::Ast`. Requires the Swift toolchain (like the crate's FFI tests).
-    #[test]
-    fn end_to_end_from_swift_source() {
-        let json = crate::parse_to_json("func f(n: Int) -> Int { return n } // trailing")
-            .expect("parsing should succeed");
-        let adapted = json_to_ast(&json).expect("adapter should succeed");
-        let ast = &adapted.ast;
-
-        let root = ast.get_node(ast.get_root()).expect("root exists");
-        assert_eq!(root.kind_name(), "sourceFile");
-
-        // The tree contains a `functionDecl` layout node and an anonymous
-        // `func` keyword token keyed by its text.
-        let mut kinds: Vec<&str> = ast.nodes().iter().map(|n| n.kind_name()).collect();
-        kinds.sort_unstable();
-        assert!(
-            kinds.contains(&"functionDecl"),
-            "expected a functionDecl node, got kinds: {kinds:?}"
-        );
-        assert!(
-            kinds.contains(&"func"),
-            "expected an anonymous `func` token, got kinds: {kinds:?}"
-        );
-
-        // The trailing comment is recovered into the side channel.
-        assert!(
-            adapted
-                .trivia
-                .iter()
-                .any(|t| t.kind == "lineComment" && t.text == "// trailing"),
-            "expected the trailing comment in the trivia side channel, got: {:?}",
-            adapted.trivia
-        );
-    }
 }

…ied/swift-syntax-rs/src/yeast_adapter.rs …extractor/src/languages/swift/adapter.rsunified/swift-syntax-rs/src/yeast_adapter.rs renamed to unified/extractor/src/languages/swift/adapter.rs unified/swift-syntax-rs/src/yeast_adapter.rs renamed to unified/extractor/src/languages/swift/adapter.rs

@@ -135,6 +135,25 @@ fn translation_rules() -> Vec<Rule<SwiftContext>> {
         // Declarations may be wrapped in local/global wrapper nodes.
         rule!((global_declaration _ @inner) => stmt { inner }),
         rule!((local_declaration _ @inner) => stmt { inner }),
+        // ---- swift-syntax front-end (minimal hook-up) ----
+        // These rules target the swift-syntax AST (camelCase kind names),
+        // produced by the sibling `adapter` module. They coexist with the
+        // tree-sitter rules (snake_case names): rules are dispatched by exact
+        // kind name, and the two name spaces never collide, so these are inert
+        // on the tree-sitter path. Only the minimal top-level mapping lives here
+        // to demonstrate the pipeline end-to-end; the full translation is added
+        // separately. Unmatched swift-syntax nodes fall through to the
+        // `unsupported_node` fallback at the end.
+        //
+        // `sourceFile` holds its top-level statements in an (elided)
+        // `statements` collection; each element is a `codeBlockItem` wrapping
+        // the real node.
+        rule!(
+            (sourceFile statements: _* @items)
+            =>
+            (top_level body: (block stmt: {items}))
+        ),
+        rule!((codeBlockItem item: @item) => stmt { item }),
         // ---- Literals ----
         rule!((integer_literal) => (int_literal)),
         rule!((hex_literal) => (int_literal)),