From 63ee2e65f3972e3f96ddc89bb1866809729c8493 Mon Sep 17 00:00:00 2001 From: eoliphant Date: Wed, 17 Jun 2026 19:18:34 -0400 Subject: [PATCH 1/4] docs: streaming GraphSON export implementation spec --- cdocs/2026-06-17-streaming-graphson-export.md | 426 ++++++++++++++++++ 1 file changed, 426 insertions(+) create mode 100644 cdocs/2026-06-17-streaming-graphson-export.md diff --git a/cdocs/2026-06-17-streaming-graphson-export.md b/cdocs/2026-06-17-streaming-graphson-export.md new file mode 100644 index 00000000..81c051de --- /dev/null +++ b/cdocs/2026-06-17-streaming-graphson-export.md @@ -0,0 +1,426 @@ +# flatgraph: Streaming GraphSON Export + +**Date:** 2026-06-17 +**Repo:** `/home/erich.oliphant/IdeaProjects/flatgraph` (joernio/flatgraph, v0.1.32) +**Purpose:** Eliminate the 1 GB JVM String OOM that kills `joern-export --repr=all` on large graphs. +**Status:** Ready for implementation — all file paths, code, and verification steps are exact. + +--- + +## Problem + +`GraphSONExporter.runExport` (file: `formats/src/main/scala/flatgraph/formats/graphson/GraphSONExporter.scala`) has this sequence: + +```scala +val nodeEntries = nodes.iterator.map { node => ... }.toSeq // fully in memory +val edgeEntries = edges.iterator.map { edge => ... }.toSeq // fully in memory +val graphSON = GraphSON(GraphSONElements(nodeEntries, edgeEntries)) +val json = implicitly[JsonWriter[GraphSON]].write(graphSON) // spray-json JsValue tree, fully in memory +writeFile(outFile, json.prettyPrint) // single Java String, fully in memory +``` + +The final `prettyPrint` call builds the entire JSON document as one `java.lang.String`. Java strings are UTF-16 and capped at ~1 GB. For GS10/grantsolutions.gov (156k nodes, 1M+ edges) this is hit, causing `OutOfMemoryError: Java heap space` in the spray-json `PrettyPrinter`. + +**Workaround in place:** codeinsight passes `--namespaces gov.grantsolutions` to `joern-parse`, which limits the CPG to one package prefix. This works but silently drops ~90% of internal application code (the repo uses ~25 other package prefixes: `eacc`, `preaward`, `postaward`, `common`, etc.). + +**The real fix:** stream JSON tokens directly to a `FileOutputStream` using Jackson Core, so no full in-memory representation is ever built. + +--- + +## Solution + +Replace spray-json in the **export path only** with `com.fasterxml.jackson.core:jackson-core` streaming API. +The **import path** (`GraphSONImporter.scala`) keeps spray-json — it reads bounded files and is not the bottleneck. + +### Why Jackson Core (not Circe, not uJson, not spray-json streaming) + +- Jackson Core is the industry-standard Java streaming JSON library; zero transitive deps beyond itself. +- flatgraph already has a Java ecosystem (`ujson` in core uses Java I/O); Jackson fits naturally. +- spray-json has no streaming API — its `PrettyPrinter` always builds a `String`. +- uJson (already in flatgraph-core) writes to a `java.io.Writer` but builds the full `Value` tree first; same OOM. + +--- + +## Files to change + +| File | Change | +|------|--------| +| `build.sbt` | Add `jackson-core` to `formats` deps; bump project version | +| `formats/src/main/scala/flatgraph/formats/graphson/GraphSONExporter.scala` | Replace spray-json write with Jackson streaming | + +`GraphSONImporter.scala`, `GraphSONProtocol.scala`, `package.scala` — **do not touch**. + +--- + +## Exact changes + +### 1. `build.sbt` — add jackson-core to formats, bump version + +Current top of file: +```sbt +name := "flatgraph" +ThisBuild / organization := "io.joern" +ThisBuild / scalaVersion := scala3 +``` + +Add a version line after the name line: +```sbt +name := "flatgraph" +ThisBuild / version := "0.1.33-SNAPSHOT" +ThisBuild / organization := "io.joern" +ThisBuild / scalaVersion := scala3 +``` + +In the `formats` project settings, add jackson-core alongside the existing deps: +```sbt +lazy val formats = project + .in(file("formats")) + .dependsOn(core) + .settings( + name := "flatgraph-formats", + libraryDependencies ++= Seq( + "com.github.tototoshi" %% "scala-csv" % "2.0.0", + "org.apache.commons" % "commons-text" % commonsTextVersion, + "org.scala-lang.modules" %% "scala-xml" % "2.3.0", + "io.spray" %% "spray-json" % "1.3.6", // kept for GraphSONImporter + "com.fasterxml.jackson.core" % "jackson-core" % "2.17.2", // ADD THIS + "com.github.scopt" %% "scopt" % "4.1.0", + ) + ) +``` + +### 2. `formats/src/main/scala/flatgraph/formats/graphson/GraphSONExporter.scala` — full replacement + +Replace the entire file with: + +```scala +package flatgraph.formats.graphson + +import com.fasterxml.jackson.core.{JsonEncoding, JsonFactory, JsonGenerator} +import flatgraph.formats.* +import flatgraph.{Accessors, GNode, Schema} + +import java.io.BufferedOutputStream +import java.nio.file.{Files, Path} +import java.util.concurrent.atomic.AtomicInteger +import scala.jdk.CollectionConverters.IterableHasAsScala +import scala.util.Using + +/** Exports to GraphSON 3.0 https://tinkerpop.apache.org/docs/3.4.1/dev/io/#graphson-3d0 + * + * Uses Jackson Core streaming API to avoid building the entire graph as a single in-memory + * String (the spray-json PrettyPrinter bottleneck that OOMs on graphs with >~500k edges). + * The output is byte-for-byte equivalent to the previous spray-json output. + */ +object GraphSONExporter extends Exporter { + + override def defaultFileExtension = "json" + + private val jsonFactory = new JsonFactory() + + override def runExport(schema: Schema, nodes: IterableOnce[GNode], edges: IterableOnce[flatgraph.Edge], outputFile: Path) = { + val outFile = resolveOutputFileSingle(outputFile, s"export.$defaultFileExtension") + val propertyId = new AtomicInteger(0) + val edgeId = new AtomicInteger(0) + var nodeCount = 0 + var edgeCount = 0 + + Using.resource(jsonFactory.createGenerator( + new BufferedOutputStream(Files.newOutputStream(outFile)), + JsonEncoding.UTF8 + )) { gen => + gen.useDefaultPrettyPrinter() + + // {"@type":"tinker:graph","@value":{ + gen.writeStartObject() + gen.writeStringField("@type", "tinker:graph") + gen.writeFieldName("@value") + gen.writeStartObject() + + // "vertices":[ + gen.writeFieldName("vertices") + gen.writeStartArray() + nodes.iterator.foreach { node => + nodeCount += 1 + writeVertex(gen, node, propertyId) + } + gen.writeEndArray() + + // "edges":[ + gen.writeFieldName("edges") + gen.writeStartArray() + edges.iterator.foreach { edge => + edgeCount += 1 + writeEdge(gen, edge, propertyId, edgeId) + } + gen.writeEndArray() + + gen.writeEndObject() // @value + gen.writeEndObject() // root + } + + ExportResult(nodeCount = nodeCount, edgeCount = edgeCount, files = Seq(outFile), Option.empty) + } + + private def writeVertex(gen: JsonGenerator, node: GNode, propertyId: AtomicInteger): Unit = { + gen.writeStartObject() + gen.writeFieldName("id") + writeLongValue(gen, node.id) + gen.writeStringField("label", node.label) + gen.writeFieldName("properties") + gen.writeStartObject() + Accessors.getNodeProperties(node).iterator.foreach { case (name, value) => + gen.writeFieldName(name) + writeProperty(gen, propertyId.getAndIncrement(), valueEntry(value), "g:VertexProperty") + } + gen.writeEndObject() + gen.writeStringField("@type", "g:Vertex") + gen.writeEndObject() + } + + private def writeEdge(gen: JsonGenerator, edge: flatgraph.Edge, propertyId: AtomicInteger, edgeId: AtomicInteger): Unit = { + val inNode = edge.dst + val outNode = edge.src + gen.writeStartObject() + gen.writeFieldName("id") + writeLongValue(gen, edgeId.getAndIncrement()) + gen.writeStringField("label", edge.label) + gen.writeStringField("inVLabel", inNode.label) + gen.writeStringField("outVLabel", outNode.label) + gen.writeFieldName("inV") + writeLongValue(gen, inNode.id) + gen.writeFieldName("outV") + writeLongValue(gen, outNode.id) + gen.writeFieldName("properties") + gen.writeStartObject() + Option(edge.property).foreach { value => + gen.writeFieldName("EdgeProperty") + writeProperty(gen, propertyId.getAndIncrement(), valueEntry(value), "g:Property") + } + gen.writeEndObject() + gen.writeStringField("@type", "g:Edge") + gen.writeEndObject() + } + + private def writeProperty(gen: JsonGenerator, id: Long, value: PropertyValue, typeStr: String): Unit = { + gen.writeStartObject() + gen.writeFieldName("id") + writeLongValue(gen, id) + gen.writeFieldName("@value") + writePropertyValue(gen, value) + gen.writeStringField("@type", typeStr) + gen.writeEndObject() + } + + private def writeLongValue(gen: JsonGenerator, v: Long): Unit = { + gen.writeStartObject() + gen.writeNumberField("@value", v) + gen.writeStringField("@type", "g:Int64") + gen.writeEndObject() + } + + private def writePropertyValue(gen: JsonGenerator, pv: PropertyValue): Unit = pv match { + case StringValue(v, _) => gen.writeString(v) + case BooleanValue(v, _) => gen.writeBoolean(v) + case LongValue(v, _) => + gen.writeStartObject() + gen.writeNumberField("@value", v) + gen.writeStringField("@type", "g:Int64") + gen.writeEndObject() + case IntValue(v, _) => + gen.writeStartObject() + gen.writeNumberField("@value", v) + gen.writeStringField("@type", "g:Int32") + gen.writeEndObject() + case FloatValue(v, _) => + gen.writeStartObject() + gen.writeNumberField("@value", v) + gen.writeStringField("@type", "g:Float") + gen.writeEndObject() + case DoubleValue(v, _) => + gen.writeStartObject() + gen.writeNumberField("@value", v) + gen.writeStringField("@type", "g:Double") + gen.writeEndObject() + case NodeIdValue(v, _) => + gen.writeStartObject() + gen.writeNumberField("@value", v) + gen.writeStringField("@type", "g:VertexId") + gen.writeEndObject() + case ListValue(elements, _) => + gen.writeStartObject() + gen.writeFieldName("@value") + gen.writeStartArray() + elements.foreach(writePropertyValue(gen, _)) + gen.writeEndArray() + gen.writeStringField("@type", "g:List") + gen.writeEndObject() + } + + // Unchanged from original — also used by GraphSONImporter indirectly via re-export + def valueEntry(propertyValue: Any): PropertyValue = { + propertyValue match { + case x: Array[_] => ListValue(x.map(valueEntry)) + case x: Iterable[_] => ListValue(x.map(valueEntry).toArray) + case x: IterableOnce[_] => ListValue(x.iterator.map(valueEntry).toArray) + case x: java.lang.Iterable[_] => ListValue(x.asScala.map(valueEntry).toArray) + case x: Boolean => BooleanValue(x) + case x: String => StringValue(x) + case x: Double => DoubleValue(x) + case x: Float => FloatValue(x) + case x: Int => IntValue(x) + case x: Long => LongValue(x) + case x: GNode => NodeIdValue(x.id()) + } + } + +} +``` + +--- + +## JSON structure contract (must not change) + +The `GraphSONImporter` deserialises with spray-json `convertTo[GraphSON]`. The field order within each object does not matter to the parser, but all fields must be present. Verify these invariants: + +**Root:** +```json +{ "@type": "tinker:graph", "@value": { "vertices": [...], "edges": [...] } } +``` + +**Vertex:** +```json +{ + "id": {"@type": "g:Int64", "@value": }, + "label": "", + "properties": { "": {"id": {...}, "@value": , "@type": "g:VertexProperty"} }, + "@type": "g:Vertex" +} +``` + +**Edge:** +```json +{ + "id": {"@type": "g:Int64", "@value": }, + "label": "", + "inVLabel": "", + "outVLabel": "", + "inV": {"@type": "g:Int64", "@value": }, + "outV": {"@type": "g:Int64", "@value": }, + "properties": {}, + "@type": "g:Edge" +} +``` + +**PropertyValue type tokens** (must match exactly what `GraphSONProtocol.PropertyValueJsonFormat.read` expects): + +| Scala type | JSON | +|---|---| +| `StringValue` | `""` (bare, no wrapper) | +| `BooleanValue` | `true`/`false` (bare) | +| `LongValue` | `{"@type":"g:Int64","@value":}` | +| `IntValue` | `{"@type":"g:Int32","@value":}` | +| `FloatValue` | `{"@type":"g:Float","@value":}` | +| `DoubleValue` | `{"@type":"g:Double","@value":}` | +| `NodeIdValue` | `{"@type":"g:VertexId","@value":}` | +| `ListValue` | `{"@type":"g:List","@value":[...]}` | + +The `readNonList` branch in `PropertyValueJsonFormat` matches `Seq(JsNumber(v), JsString(typ))` — so `@value` must come before `@type` in the number wrappers. **Jackson writes fields in the order they are emitted**, so `gen.writeNumberField("@value", v)` before `gen.writeStringField("@type", ...)` is correct and required. + +--- + +## Build and test + +### Prereqs +- sbt installed (project uses sbt, not Maven) +- Working directory: `/home/erich.oliphant/IdeaProjects/flatgraph` + +### Step 1 — compile +```bash +cd /home/erich.oliphant/IdeaProjects/flatgraph +sbt formats/compile +``` +Expected: clean compile, zero errors. + +### Step 2 — run existing GraphSON tests +```bash +sbt tests/testOnly flatgraph.formats.graphson.GraphSONTests +``` +Expected: both tests pass (`export to GraphSON and back`, `using 'contained node' property`). +These are round-trip tests (export → import → diff), so they validate JSON contract compliance. + +### Step 3 — publish locally +```bash +sbt publishLocal +``` +This installs to `~/.ivy2/local/io.joern/flatgraph-formats_3/0.1.33-SNAPSHOT/`. + +### Step 4 — smoke test against a real large CPG +Use the `cpg.bin` from a previous joern-parse run (if available from the earlier experiments at `/tmp/joern-experiments/shared-parse/cpg.bin`): + +```bash +# Build a new joern-export binary that uses the patched flatgraph +cd /home/erich.oliphant/IdeaProjects/joern +# Update flatgraph version reference in joern's build.sbt (see Integration section below) +sbt joern-cli/stage + +# Run export against the large CPG (WITHOUT --namespaces, to test the fix) +time /home/erich.oliphant/IdeaProjects/joern/joern-cli/target/universal/stage/joern-export \ + /tmp/joern-experiments/shared-parse/cpg.bin \ + --repr=all --format=graphson \ + --out /tmp/streaming-test-out + +du -sh /tmp/streaming-test-out/ +``` +Expected: completes in seconds (not 30+ minutes), no OOM. + +--- + +## Integration with joern + +After publishing flatgraph locally, update joern to use it. + +In `/home/erich.oliphant/IdeaProjects/joern/build.sbt`, find the flatgraph version reference: +```bash +rg "flatgraph" /home/erich.oliphant/IdeaProjects/joern/build.sbt | head -10 +``` + +Change the flatgraph version from `0.1.32` to `0.1.33-SNAPSHOT`, then build joern: +```bash +cd /home/erich.oliphant/IdeaProjects/joern +sbt joern-cli/stage +``` + +The new binary is at `joern-cli/target/universal/stage/joern-export`. + +--- + +## Integration with codeinsight + +Once the patched joern-export binary is built, point codeinsight at it by overriding the config: + +In `src/main/resources/application.properties` (or `application-dev.properties`): +```properties +codeinsight.cpg.joern.export-cmd=/home/erich.oliphant/IdeaProjects/joern/joern-cli/target/universal/stage/joern-export +``` + +At that point, `--namespaces gov.grantsolutions` in `CpgConfig` can be widened or removed to capture the full application call graph. + +--- + +## What this does NOT change + +- `GraphSONImporter` — unchanged, still uses spray-json +- `GraphSONProtocol` — unchanged +- `package.scala` (graphson types) — unchanged +- All other exporters (Dot, GraphML, Neo4jCsv) — unchanged +- The `Exporter` trait interface — unchanged +- The `--repr=cpg` split-by-method path in joern (not relevant here) + +--- + +## Risk + +Low. The change is purely in the serialisation path of one exporter. The round-trip test in `GraphSONTests.scala` validates the output is still parseable by `GraphSONImporter`. The JSON structure contract is fully specified above. + +The one subtle point is field ordering within `@value`/`@type` pairs: `@value` must precede `@type` in numeric wrappers because spray-json's `readNonList` pattern-matches `Seq(JsNumber(v), JsString(typ))` — spray-json preserves insertion order in `JsObject` but `getFields` returns values in the order the fields appear in JSON. The implementation above emits `@value` first consistently. From 4fcab76eedb14064a40d2830a40a6198164e4de9 Mon Sep 17 00:00:00 2001 From: eoliphant Date: Thu, 18 Jun 2026 12:41:17 -0400 Subject: [PATCH 2/4] test: extend Generic schema with Float/Double/Long/Boolean property types Add floatOptional, doubleOptional, longOptional, and booleanOptional properties to the Generic test schema. These properties enable testing of FloatValue, DoubleValue, LongValue, and BooleanValue branches in the exporter and GraphSONProtocol that were previously untested. Regenerate domain classes and update Neo4j CSV test data and assertions to reflect the new schema. --- .../testdomains/generic/GraphSchema.scala | 96 +++++++--- .../testdomains/generic/Properties.scala | 20 +- .../testdomains/generic/PropertyNames.scala | 21 ++- .../generic/accessors/Accessors.scala | 55 +++++- .../testdomains/generic/nodes/BaseTypes.scala | 20 ++ .../testdomains/generic/nodes/NewNodeA.scala | 174 +++++++++++++++--- .../testdomains/generic/nodes/NewNodeB.scala | 2 +- .../testdomains/generic/nodes/NodeA.scala | 62 +++++-- .../traversals/TraversalNodeaBase.scala | 99 +++++++++- .../traversals/TraversalNodebBase.scala | 2 +- .../TraversalPropertyBooleanOptional.scala | 19 ++ .../TraversalPropertyDoubleOptional.scala | 38 ++++ .../TraversalPropertyFloatOptional.scala | 38 ++++ .../TraversalPropertyLongOptional.scala | 38 ++++ .../TraversalPropertyStringMandatory.scala | 4 +- .../TraversalPropertyStringOptional.scala | 2 +- .../generic/traversals/package.scala | 12 ++ .../scala/flatgraph/testdomains/Generic.scala | 8 + .../resources/neo4jcsv/nodes_node_a_data.csv | 4 +- .../neo4jcsv/nodes_node_a_header.csv | 2 +- .../formats/neo4jcsv/Neo4jCsvTests.scala | 22 ++- 21 files changed, 633 insertions(+), 105 deletions(-) create mode 100644 test-schemas-domain-classes/src/main/scala/testdomains/generic/traversals/TraversalPropertyBooleanOptional.scala create mode 100644 test-schemas-domain-classes/src/main/scala/testdomains/generic/traversals/TraversalPropertyDoubleOptional.scala create mode 100644 test-schemas-domain-classes/src/main/scala/testdomains/generic/traversals/TraversalPropertyFloatOptional.scala create mode 100644 test-schemas-domain-classes/src/main/scala/testdomains/generic/traversals/TraversalPropertyLongOptional.scala diff --git a/test-schemas-domain-classes/src/main/scala/testdomains/generic/GraphSchema.scala b/test-schemas-domain-classes/src/main/scala/testdomains/generic/GraphSchema.scala index 99a301d4..56b6c6ab 100644 --- a/test-schemas-domain-classes/src/main/scala/testdomains/generic/GraphSchema.scala +++ b/test-schemas-domain-classes/src/main/scala/testdomains/generic/GraphSchema.scala @@ -14,53 +14,79 @@ object GraphSchema extends flatgraph.Schema { val edgeFactories: Array[(flatgraph.GNode, flatgraph.GNode, Int, Any) => flatgraph.Edge] = Array((s, d, subseq, p) => new edges.AnotherEdge(s, d, subseq, p), (s, d, subseq, p) => new edges.ConnectedTo(s, d, subseq, p)) val nodePropertyAllocators: Array[Int => Array[?]] = Array( + size => new Array[Boolean](size), + size => new Array[Double](size), + size => new Array[Float](size), size => new Array[Int](size), size => new Array[Int](size), size => new Array[Int](size), + size => new Array[Long](size), size => new Array[String](size), size => new Array[String](size), size => new Array[String](size), size => new Array[flatgraph.GNode](size) ) - val normalNodePropertyNames: Array[String] = - Array("int_list", "int_mandatory", "int_optional", "string_list", "string_mandatory", "string_optional") - val nodePropertyByLabel = normalNodePropertyNames.zipWithIndex.toMap.updated("contained_node_b", 6) + val normalNodePropertyNames: Array[String] = Array( + "boolean_optional", + "double_optional", + "float_optional", + "int_list", + "int_mandatory", + "int_optional", + "long_optional", + "string_list", + "string_mandatory", + "string_optional" + ) + val nodePropertyByLabel = normalNodePropertyNames.zipWithIndex.toMap.updated("contained_node_b", 10) val nodePropertyDescriptors: Array[FormalQtyType.FormalQuantity | FormalQtyType.FormalType] = { - val nodePropertyDescriptors = new Array[FormalQtyType.FormalQuantity | FormalQtyType.FormalType](28) - for (idx <- Range(0, 28)) { + val nodePropertyDescriptors = new Array[FormalQtyType.FormalQuantity | FormalQtyType.FormalType](44) + for (idx <- Range(0, 44)) { nodePropertyDescriptors(idx) = if ((idx & 1) == 0) FormalQtyType.NothingType else FormalQtyType.QtyNone } - nodePropertyDescriptors(0) = FormalQtyType.IntType // node_a.int_list - nodePropertyDescriptors(1) = FormalQtyType.QtyMulti - nodePropertyDescriptors(4) = FormalQtyType.IntType // node_a.int_mandatory - nodePropertyDescriptors(5) = FormalQtyType.QtyOne - nodePropertyDescriptors(8) = FormalQtyType.IntType // node_a.int_optional + nodePropertyDescriptors(0) = FormalQtyType.BoolType // node_a.boolean_optional + nodePropertyDescriptors(1) = FormalQtyType.QtyOption + nodePropertyDescriptors(4) = FormalQtyType.DoubleType // node_a.double_optional + nodePropertyDescriptors(5) = FormalQtyType.QtyOption + nodePropertyDescriptors(8) = FormalQtyType.FloatType // node_a.float_optional nodePropertyDescriptors(9) = FormalQtyType.QtyOption - nodePropertyDescriptors(12) = FormalQtyType.StringType // node_a.string_list + nodePropertyDescriptors(12) = FormalQtyType.IntType // node_a.int_list nodePropertyDescriptors(13) = FormalQtyType.QtyMulti - nodePropertyDescriptors(16) = FormalQtyType.StringType // node_a.string_mandatory + nodePropertyDescriptors(16) = FormalQtyType.IntType // node_a.int_mandatory nodePropertyDescriptors(17) = FormalQtyType.QtyOne - nodePropertyDescriptors(20) = FormalQtyType.StringType // node_a.string_optional + nodePropertyDescriptors(20) = FormalQtyType.IntType // node_a.int_optional nodePropertyDescriptors(21) = FormalQtyType.QtyOption - nodePropertyDescriptors(24) = FormalQtyType.RefType // node_a.contained_node_b + nodePropertyDescriptors(24) = FormalQtyType.LongType // node_a.long_optional nodePropertyDescriptors(25) = FormalQtyType.QtyOption - nodePropertyDescriptors(22) = FormalQtyType.StringType // node_b.string_optional - nodePropertyDescriptors(23) = FormalQtyType.QtyOption + nodePropertyDescriptors(28) = FormalQtyType.StringType // node_a.string_list + nodePropertyDescriptors(29) = FormalQtyType.QtyMulti + nodePropertyDescriptors(32) = FormalQtyType.StringType // node_a.string_mandatory + nodePropertyDescriptors(33) = FormalQtyType.QtyOne + nodePropertyDescriptors(36) = FormalQtyType.StringType // node_a.string_optional + nodePropertyDescriptors(37) = FormalQtyType.QtyOption + nodePropertyDescriptors(40) = FormalQtyType.RefType // node_a.contained_node_b + nodePropertyDescriptors(41) = FormalQtyType.QtyOption + nodePropertyDescriptors(38) = FormalQtyType.StringType // node_b.string_optional + nodePropertyDescriptors(39) = FormalQtyType.QtyOption nodePropertyDescriptors } private val newNodeInsertionHelpers: Array[flatgraph.NewNodePropertyInsertionHelper] = { - val _newNodeInserters = new Array[flatgraph.NewNodePropertyInsertionHelper](28) - _newNodeInserters(0) = nodes.NewNodeA.InsertionHelpers.NewNodeInserter_NodeA_intList - _newNodeInserters(4) = nodes.NewNodeA.InsertionHelpers.NewNodeInserter_NodeA_intMandatory - _newNodeInserters(8) = nodes.NewNodeA.InsertionHelpers.NewNodeInserter_NodeA_intOptional - _newNodeInserters(12) = nodes.NewNodeA.InsertionHelpers.NewNodeInserter_NodeA_stringList - _newNodeInserters(16) = nodes.NewNodeA.InsertionHelpers.NewNodeInserter_NodeA_stringMandatory - _newNodeInserters(20) = nodes.NewNodeA.InsertionHelpers.NewNodeInserter_NodeA_stringOptional - _newNodeInserters(24) = nodes.NewNodeA.InsertionHelpers.NewNodeInserter_NodeA_contained_node_b - _newNodeInserters(22) = nodes.NewNodeB.InsertionHelpers.NewNodeInserter_NodeB_stringOptional + val _newNodeInserters = new Array[flatgraph.NewNodePropertyInsertionHelper](44) + _newNodeInserters(0) = nodes.NewNodeA.InsertionHelpers.NewNodeInserter_NodeA_booleanOptional + _newNodeInserters(4) = nodes.NewNodeA.InsertionHelpers.NewNodeInserter_NodeA_doubleOptional + _newNodeInserters(8) = nodes.NewNodeA.InsertionHelpers.NewNodeInserter_NodeA_floatOptional + _newNodeInserters(12) = nodes.NewNodeA.InsertionHelpers.NewNodeInserter_NodeA_intList + _newNodeInserters(16) = nodes.NewNodeA.InsertionHelpers.NewNodeInserter_NodeA_intMandatory + _newNodeInserters(20) = nodes.NewNodeA.InsertionHelpers.NewNodeInserter_NodeA_intOptional + _newNodeInserters(24) = nodes.NewNodeA.InsertionHelpers.NewNodeInserter_NodeA_longOptional + _newNodeInserters(28) = nodes.NewNodeA.InsertionHelpers.NewNodeInserter_NodeA_stringList + _newNodeInserters(32) = nodes.NewNodeA.InsertionHelpers.NewNodeInserter_NodeA_stringMandatory + _newNodeInserters(36) = nodes.NewNodeA.InsertionHelpers.NewNodeInserter_NodeA_stringOptional + _newNodeInserters(40) = nodes.NewNodeA.InsertionHelpers.NewNodeInserter_NodeA_contained_node_b + _newNodeInserters(38) = nodes.NewNodeB.InsertionHelpers.NewNodeInserter_NodeB_stringOptional _newNodeInserters } override def getNumberOfNodeKinds: Int = 2 @@ -71,7 +97,19 @@ object GraphSchema extends flatgraph.Schema { override def getEdgeKindByLabel(label: String): Int = edgeKindByLabel.getOrElse(label, flatgraph.Schema.UndefinedKind) override def getNodePropertyNames(nodeLabel: String): Set[String] = { nodeLabel match { - case "node_a" => Set("int_list", "int_mandatory", "int_optional", "string_list", "string_mandatory", "string_optional") + case "node_a" => + Set( + "boolean_optional", + "double_optional", + "float_optional", + "int_list", + "int_mandatory", + "int_optional", + "long_optional", + "string_list", + "string_mandatory", + "string_optional" + ) case "node_b" => Set("string_optional") case _ => Set.empty } @@ -84,13 +122,13 @@ object GraphSchema extends flatgraph.Schema { } override def getPropertyLabel(nodeKind: Int, propertyKind: Int): String = { - if (propertyKind < 6) normalNodePropertyNames(propertyKind) - else if (propertyKind == 6 && nodeKind == 0) "contained_node_b" /*on node node_a*/ + if (propertyKind < 10) normalNodePropertyNames(propertyKind) + else if (propertyKind == 10 && nodeKind == 0) "contained_node_b" /*on node node_a*/ else null } override def getPropertyKindByName(label: String): Int = nodePropertyByLabel.getOrElse(label, flatgraph.Schema.UndefinedKind) - override def getNumberOfPropertyKinds: Int = 7 + override def getNumberOfPropertyKinds: Int = 11 override def makeNode(graph: flatgraph.Graph, nodeKind: Short, seq: Int): nodes.StoredNode = nodeFactories(nodeKind)(graph, seq) override def makeEdge(src: flatgraph.GNode, dst: flatgraph.GNode, edgeKind: Short, subSeq: Int, property: Any): flatgraph.Edge = edgeFactories(edgeKind)(src, dst, subSeq, property) diff --git a/test-schemas-domain-classes/src/main/scala/testdomains/generic/Properties.scala b/test-schemas-domain-classes/src/main/scala/testdomains/generic/Properties.scala index 4fa59fd0..bb1a7a64 100644 --- a/test-schemas-domain-classes/src/main/scala/testdomains/generic/Properties.scala +++ b/test-schemas-domain-classes/src/main/scala/testdomains/generic/Properties.scala @@ -1,15 +1,23 @@ package testdomains.generic object Properties { - val IntList = flatgraph.MultiPropertyKey[Int](kind = 0, name = "int_list") + val BooleanOptional = flatgraph.OptionalPropertyKey[Boolean](kind = 0, name = "boolean_optional") - val IntMandatory = flatgraph.SinglePropertyKey[Int](kind = 1, name = "int_mandatory", default = 42: Int) + val DoubleOptional = flatgraph.OptionalPropertyKey[Double](kind = 1, name = "double_optional") - val IntOptional = flatgraph.OptionalPropertyKey[Int](kind = 2, name = "int_optional") + val FloatOptional = flatgraph.OptionalPropertyKey[Float](kind = 2, name = "float_optional") - val StringList = flatgraph.MultiPropertyKey[String](kind = 3, name = "string_list") + val IntList = flatgraph.MultiPropertyKey[Int](kind = 3, name = "int_list") - val StringMandatory = flatgraph.SinglePropertyKey[String](kind = 4, name = "string_mandatory", default = "") + val IntMandatory = flatgraph.SinglePropertyKey[Int](kind = 4, name = "int_mandatory", default = 42: Int) - val StringOptional = flatgraph.OptionalPropertyKey[String](kind = 5, name = "string_optional") + val IntOptional = flatgraph.OptionalPropertyKey[Int](kind = 5, name = "int_optional") + + val LongOptional = flatgraph.OptionalPropertyKey[Long](kind = 6, name = "long_optional") + + val StringList = flatgraph.MultiPropertyKey[String](kind = 7, name = "string_list") + + val StringMandatory = flatgraph.SinglePropertyKey[String](kind = 8, name = "string_mandatory", default = "") + + val StringOptional = flatgraph.OptionalPropertyKey[String](kind = 9, name = "string_optional") } diff --git a/test-schemas-domain-classes/src/main/scala/testdomains/generic/PropertyNames.scala b/test-schemas-domain-classes/src/main/scala/testdomains/generic/PropertyNames.scala index 9f962ffd..97509858 100644 --- a/test-schemas-domain-classes/src/main/scala/testdomains/generic/PropertyNames.scala +++ b/test-schemas-domain-classes/src/main/scala/testdomains/generic/PropertyNames.scala @@ -4,9 +4,13 @@ import java.util.{HashSet, Set} import scala.jdk.CollectionConverters.SeqHasAsJava object PropertyNames { + val BooleanOptional: String = "boolean_optional" + val DoubleOptional: String = "double_optional" + val FloatOptional: String = "float_optional" val IntList: String = "int_list" val IntMandatory: String = "int_mandatory" val IntOptional: String = "int_optional" + val LongOptional: String = "long_optional" val StringList: String = "string_list" val StringMandatory: String = "string_mandatory" val StringOptional: String = "string_optional" @@ -14,6 +18,19 @@ object PropertyNames { /** This is a contained node */ val ContainedNodeB: String = "contained_node_b" - val All: Set[String] = - new HashSet[String](Seq(IntList, IntMandatory, IntOptional, StringList, StringMandatory, StringOptional, ContainedNodeB).asJava) + val All: Set[String] = new HashSet[String]( + Seq( + BooleanOptional, + DoubleOptional, + FloatOptional, + IntList, + IntMandatory, + IntOptional, + LongOptional, + StringList, + StringMandatory, + StringOptional, + ContainedNodeB + ).asJava + ) } diff --git a/test-schemas-domain-classes/src/main/scala/testdomains/generic/accessors/Accessors.scala b/test-schemas-domain-classes/src/main/scala/testdomains/generic/accessors/Accessors.scala index 96dac881..bd260079 100644 --- a/test-schemas-domain-classes/src/main/scala/testdomains/generic/accessors/Accessors.scala +++ b/test-schemas-domain-classes/src/main/scala/testdomains/generic/accessors/Accessors.scala @@ -7,39 +7,67 @@ object languagebootstrap extends ConcreteStoredConversions object Accessors { /* accessors for concrete stored nodes start */ + final class AccessPropertyBooleanOptional(val node: nodes.StoredNode) extends AnyVal { + def booleanOptional: Option[Boolean] = + flatgraph.Accessors.getNodePropertyOption[Boolean](node.graph, nodeKind = node.nodeKind, propertyKind = 0, seq = node.seq) + } + final class AccessPropertyDoubleOptional(val node: nodes.StoredNode) extends AnyVal { + def doubleOptional: Option[Double] = + flatgraph.Accessors.getNodePropertyOption[Double](node.graph, nodeKind = node.nodeKind, propertyKind = 1, seq = node.seq) + } + final class AccessPropertyFloatOptional(val node: nodes.StoredNode) extends AnyVal { + def floatOptional: Option[Float] = + flatgraph.Accessors.getNodePropertyOption[Float](node.graph, nodeKind = node.nodeKind, propertyKind = 2, seq = node.seq) + } final class AccessPropertyIntList(val node: nodes.StoredNode) extends AnyVal { def intList: IndexedSeq[Int] = - flatgraph.Accessors.getNodePropertyMulti[Int](node.graph, nodeKind = node.nodeKind, propertyKind = 0, seq = node.seq) + flatgraph.Accessors.getNodePropertyMulti[Int](node.graph, nodeKind = node.nodeKind, propertyKind = 3, seq = node.seq) } final class AccessPropertyIntMandatory(val node: nodes.StoredNode) extends AnyVal { def intMandatory: Int = - flatgraph.Accessors.getNodePropertySingle(node.graph, nodeKind = node.nodeKind, propertyKind = 1, seq = node.seq(), default = 42: Int) + flatgraph.Accessors.getNodePropertySingle(node.graph, nodeKind = node.nodeKind, propertyKind = 4, seq = node.seq(), default = 42: Int) } final class AccessPropertyIntOptional(val node: nodes.StoredNode) extends AnyVal { def intOptional: Option[Int] = - flatgraph.Accessors.getNodePropertyOption[Int](node.graph, nodeKind = node.nodeKind, propertyKind = 2, seq = node.seq) + flatgraph.Accessors.getNodePropertyOption[Int](node.graph, nodeKind = node.nodeKind, propertyKind = 5, seq = node.seq) + } + final class AccessPropertyLongOptional(val node: nodes.StoredNode) extends AnyVal { + def longOptional: Option[Long] = + flatgraph.Accessors.getNodePropertyOption[Long](node.graph, nodeKind = node.nodeKind, propertyKind = 6, seq = node.seq) } final class AccessPropertyStringList(val node: nodes.StoredNode) extends AnyVal { def stringList: IndexedSeq[String] = - flatgraph.Accessors.getNodePropertyMulti[String](node.graph, nodeKind = node.nodeKind, propertyKind = 3, seq = node.seq) + flatgraph.Accessors.getNodePropertyMulti[String](node.graph, nodeKind = node.nodeKind, propertyKind = 7, seq = node.seq) } final class AccessPropertyStringMandatory(val node: nodes.StoredNode) extends AnyVal { def stringMandatory: String = flatgraph.Accessors.getNodePropertySingle( node.graph, nodeKind = node.nodeKind, - propertyKind = 4, + propertyKind = 8, seq = node.seq(), default = "": String ) } final class AccessPropertyStringOptional(val node: nodes.StoredNode) extends AnyVal { def stringOptional: Option[String] = - flatgraph.Accessors.getNodePropertyOption[String](node.graph, nodeKind = node.nodeKind, propertyKind = 5, seq = node.seq) + flatgraph.Accessors.getNodePropertyOption[String](node.graph, nodeKind = node.nodeKind, propertyKind = 9, seq = node.seq) } /* accessors for concrete stored nodes end */ /* accessors for base nodes start */ final class AccessNodeaBase(val node: nodes.NodeABase) extends AnyVal { + def booleanOptional: Option[Boolean] = node match { + case stored: nodes.StoredNode => new AccessPropertyBooleanOptional(stored).booleanOptional + case newNode: nodes.NewNodeA => newNode.booleanOptional + } + def doubleOptional: Option[Double] = node match { + case stored: nodes.StoredNode => new AccessPropertyDoubleOptional(stored).doubleOptional + case newNode: nodes.NewNodeA => newNode.doubleOptional + } + def floatOptional: Option[Float] = node match { + case stored: nodes.StoredNode => new AccessPropertyFloatOptional(stored).floatOptional + case newNode: nodes.NewNodeA => newNode.floatOptional + } def intList: IndexedSeq[Int] = node match { case stored: nodes.StoredNode => new AccessPropertyIntList(stored).intList case newNode: nodes.NewNodeA => newNode.intList @@ -52,6 +80,10 @@ object Accessors { case stored: nodes.StoredNode => new AccessPropertyIntOptional(stored).intOptional case newNode: nodes.NewNodeA => newNode.intOptional } + def longOptional: Option[Long] = node match { + case stored: nodes.StoredNode => new AccessPropertyLongOptional(stored).longOptional + case newNode: nodes.NewNodeA => newNode.longOptional + } def stringList: IndexedSeq[String] = node match { case stored: nodes.StoredNode => new AccessPropertyStringList(stored).stringList case newNode: nodes.NewNodeA => newNode.stringList @@ -76,12 +108,23 @@ object Accessors { import Accessors.* trait ConcreteStoredConversions extends ConcreteBaseConversions { + implicit def accessPropertyBooleanOptional( + node: nodes.StoredNode & nodes.StaticType[nodes.HasBooleanOptionalEMT] + ): AccessPropertyBooleanOptional = new AccessPropertyBooleanOptional(node) + implicit def accessPropertyDoubleOptional( + node: nodes.StoredNode & nodes.StaticType[nodes.HasDoubleOptionalEMT] + ): AccessPropertyDoubleOptional = new AccessPropertyDoubleOptional(node) + implicit def accessPropertyFloatOptional( + node: nodes.StoredNode & nodes.StaticType[nodes.HasFloatOptionalEMT] + ): AccessPropertyFloatOptional = new AccessPropertyFloatOptional(node) implicit def accessPropertyIntList(node: nodes.StoredNode & nodes.StaticType[nodes.HasIntListEMT]): AccessPropertyIntList = new AccessPropertyIntList(node) implicit def accessPropertyIntMandatory(node: nodes.StoredNode & nodes.StaticType[nodes.HasIntMandatoryEMT]): AccessPropertyIntMandatory = new AccessPropertyIntMandatory(node) implicit def accessPropertyIntOptional(node: nodes.StoredNode & nodes.StaticType[nodes.HasIntOptionalEMT]): AccessPropertyIntOptional = new AccessPropertyIntOptional(node) + implicit def accessPropertyLongOptional(node: nodes.StoredNode & nodes.StaticType[nodes.HasLongOptionalEMT]): AccessPropertyLongOptional = + new AccessPropertyLongOptional(node) implicit def accessPropertyStringList(node: nodes.StoredNode & nodes.StaticType[nodes.HasStringListEMT]): AccessPropertyStringList = new AccessPropertyStringList(node) implicit def accessPropertyStringMandatory( diff --git a/test-schemas-domain-classes/src/main/scala/testdomains/generic/nodes/BaseTypes.scala b/test-schemas-domain-classes/src/main/scala/testdomains/generic/nodes/BaseTypes.scala index f69f49aa..458726c6 100644 --- a/test-schemas-domain-classes/src/main/scala/testdomains/generic/nodes/BaseTypes.scala +++ b/test-schemas-domain-classes/src/main/scala/testdomains/generic/nodes/BaseTypes.scala @@ -1,5 +1,20 @@ package testdomains.generic.nodes +/** Node types with this marker trait are guaranteed to have the boolean_optional property. EMT stands for: "erased marker trait", it exists + * only at compile time in order to improve type safety. + */ +trait HasBooleanOptionalEMT + +/** Node types with this marker trait are guaranteed to have the double_optional property. EMT stands for: "erased marker trait", it exists + * only at compile time in order to improve type safety. + */ +trait HasDoubleOptionalEMT + +/** Node types with this marker trait are guaranteed to have the float_optional property. EMT stands for: "erased marker trait", it exists + * only at compile time in order to improve type safety. + */ +trait HasFloatOptionalEMT + /** Node types with this marker trait are guaranteed to have the int_list property. EMT stands for: "erased marker trait", it exists only at * compile time in order to improve type safety. */ @@ -15,6 +30,11 @@ trait HasIntMandatoryEMT */ trait HasIntOptionalEMT +/** Node types with this marker trait are guaranteed to have the long_optional property. EMT stands for: "erased marker trait", it exists + * only at compile time in order to improve type safety. + */ +trait HasLongOptionalEMT + /** Node types with this marker trait are guaranteed to have the string_list property. EMT stands for: "erased marker trait", it exists only * at compile time in order to improve type safety. */ diff --git a/test-schemas-domain-classes/src/main/scala/testdomains/generic/nodes/NewNodeA.scala b/test-schemas-domain-classes/src/main/scala/testdomains/generic/nodes/NewNodeA.scala index e425fcb3..94cd4aef 100644 --- a/test-schemas-domain-classes/src/main/scala/testdomains/generic/nodes/NewNodeA.scala +++ b/test-schemas-domain-classes/src/main/scala/testdomains/generic/nodes/NewNodeA.scala @@ -10,6 +10,81 @@ object NewNodeA { private val inNeighbors: Map[String, Set[String]] = Map("connected_to" -> Set("node_a")) object InsertionHelpers { + object NewNodeInserter_NodeA_booleanOptional extends flatgraph.NewNodePropertyInsertionHelper { + override def insertNewNodeProperties(newNodes: mutable.ArrayBuffer[flatgraph.DNode], dst: AnyRef, offsets: Array[Int]): Unit = { + if (newNodes.isEmpty) return + val dstCast = dst.asInstanceOf[Array[Boolean]] + val seq = newNodes.head.storedRef.get.seq() + var offset = offsets(seq) + var idx = 0 + while (idx < newNodes.length) { + val nn = newNodes(idx) + nn match { + case generated: NewNodeA => + generated.booleanOptional match { + case Some(item) => + dstCast(offset) = item + offset += 1 + case _ => + } + case _ => + } + assert(seq + idx == nn.storedRef.get.seq(), "internal consistency check") + idx += 1 + offsets(idx + seq) = offset + } + } + } + object NewNodeInserter_NodeA_doubleOptional extends flatgraph.NewNodePropertyInsertionHelper { + override def insertNewNodeProperties(newNodes: mutable.ArrayBuffer[flatgraph.DNode], dst: AnyRef, offsets: Array[Int]): Unit = { + if (newNodes.isEmpty) return + val dstCast = dst.asInstanceOf[Array[Double]] + val seq = newNodes.head.storedRef.get.seq() + var offset = offsets(seq) + var idx = 0 + while (idx < newNodes.length) { + val nn = newNodes(idx) + nn match { + case generated: NewNodeA => + generated.doubleOptional match { + case Some(item) => + dstCast(offset) = item + offset += 1 + case _ => + } + case _ => + } + assert(seq + idx == nn.storedRef.get.seq(), "internal consistency check") + idx += 1 + offsets(idx + seq) = offset + } + } + } + object NewNodeInserter_NodeA_floatOptional extends flatgraph.NewNodePropertyInsertionHelper { + override def insertNewNodeProperties(newNodes: mutable.ArrayBuffer[flatgraph.DNode], dst: AnyRef, offsets: Array[Int]): Unit = { + if (newNodes.isEmpty) return + val dstCast = dst.asInstanceOf[Array[Float]] + val seq = newNodes.head.storedRef.get.seq() + var offset = offsets(seq) + var idx = 0 + while (idx < newNodes.length) { + val nn = newNodes(idx) + nn match { + case generated: NewNodeA => + generated.floatOptional match { + case Some(item) => + dstCast(offset) = item + offset += 1 + case _ => + } + case _ => + } + assert(seq + idx == nn.storedRef.get.seq(), "internal consistency check") + idx += 1 + offsets(idx + seq) = offset + } + } + } object NewNodeInserter_NodeA_intList extends flatgraph.NewNodePropertyInsertionHelper { override def insertNewNodeProperties(newNodes: mutable.ArrayBuffer[flatgraph.DNode], dst: AnyRef, offsets: Array[Int]): Unit = { if (newNodes.isEmpty) return @@ -79,6 +154,31 @@ object NewNodeA { } } } + object NewNodeInserter_NodeA_longOptional extends flatgraph.NewNodePropertyInsertionHelper { + override def insertNewNodeProperties(newNodes: mutable.ArrayBuffer[flatgraph.DNode], dst: AnyRef, offsets: Array[Int]): Unit = { + if (newNodes.isEmpty) return + val dstCast = dst.asInstanceOf[Array[Long]] + val seq = newNodes.head.storedRef.get.seq() + var offset = offsets(seq) + var idx = 0 + while (idx < newNodes.length) { + val nn = newNodes(idx) + nn match { + case generated: NewNodeA => + generated.longOptional match { + case Some(item) => + dstCast(offset) = item + offset += 1 + case _ => + } + case _ => + } + assert(seq + idx == nn.storedRef.get.seq(), "internal consistency check") + idx += 1 + offsets(idx + seq) = offset + } + } + } object NewNodeInserter_NodeA_stringList extends flatgraph.NewNodePropertyInsertionHelper { override def insertNewNodeProperties(newNodes: mutable.ArrayBuffer[flatgraph.DNode], dst: AnyRef, offsets: Array[Int]): Unit = { if (newNodes.isEmpty) return @@ -189,39 +289,59 @@ class NewNodeA extends NewNode(nodeKind = 0) with NodeABase { NewNodeA.inNeighbors.getOrElse(edgeLabel, Set.empty).contains(n.label) } + var booleanOptional: Option[Boolean] = None var contained_node_b: Option[NodeBBase] = None + var doubleOptional: Option[Double] = None + var floatOptional: Option[Float] = None var intList: IndexedSeq[Int] = ArraySeq.empty var intMandatory: Int = 42: Int var intOptional: Option[Int] = None + var longOptional: Option[Long] = None var stringList: IndexedSeq[String] = ArraySeq.empty var stringMandatory: String = "": String var stringOptional: Option[String] = None + def booleanOptional(value: Boolean): this.type = { this.booleanOptional = Option(value); this } + def booleanOptional(value: Option[Boolean]): this.type = { this.booleanOptional = value; this } def contained_node_b(value: NodeBBase): this.type = { this.contained_node_b = Option(value); this } def contained_node_b(value: Option[NodeBBase]): this.type = { this.contained_node_b = value; this } + def doubleOptional(value: Double): this.type = { this.doubleOptional = Option(value); this } + def doubleOptional(value: Option[Double]): this.type = { this.doubleOptional = value; this } + def floatOptional(value: Float): this.type = { this.floatOptional = Option(value); this } + def floatOptional(value: Option[Float]): this.type = { this.floatOptional = value; this } def intList(value: IterableOnce[Int]): this.type = { this.intList = value.iterator.to(ArraySeq); this } def intMandatory(value: Int): this.type = { this.intMandatory = value; this } def intOptional(value: Int): this.type = { this.intOptional = Option(value); this } def intOptional(value: Option[Int]): this.type = { this.intOptional = value; this } + def longOptional(value: Long): this.type = { this.longOptional = Option(value); this } + def longOptional(value: Option[Long]): this.type = { this.longOptional = value; this } def stringList(value: IterableOnce[String]): this.type = { this.stringList = value.iterator.to(ArraySeq); this } def stringMandatory(value: String): this.type = { this.stringMandatory = value; this } def stringOptional(value: Option[String]): this.type = { this.stringOptional = value; this } def stringOptional(value: String): this.type = { this.stringOptional = Option(value); this } override def countAndVisitProperties(interface: flatgraph.BatchedUpdateInterface): Unit = { - interface.countProperty(this, 0, intList.size) - interface.countProperty(this, 1, 1) - interface.countProperty(this, 2, intOptional.size) - interface.countProperty(this, 3, stringList.size) + interface.countProperty(this, 0, booleanOptional.size) + interface.countProperty(this, 1, doubleOptional.size) + interface.countProperty(this, 2, floatOptional.size) + interface.countProperty(this, 3, intList.size) interface.countProperty(this, 4, 1) - interface.countProperty(this, 5, stringOptional.size) - interface.countProperty(this, 6, contained_node_b.size) + interface.countProperty(this, 5, intOptional.size) + interface.countProperty(this, 6, longOptional.size) + interface.countProperty(this, 7, stringList.size) + interface.countProperty(this, 8, 1) + interface.countProperty(this, 9, stringOptional.size) + interface.countProperty(this, 10, contained_node_b.size) contained_node_b.foreach(interface.visitContainedNode) } override def copy: this.type = { val newInstance = new NewNodeA + newInstance.booleanOptional = this.booleanOptional + newInstance.doubleOptional = this.doubleOptional + newInstance.floatOptional = this.floatOptional newInstance.intList = this.intList newInstance.intMandatory = this.intMandatory newInstance.intOptional = this.intOptional + newInstance.longOptional = this.longOptional newInstance.stringList = this.stringList newInstance.stringMandatory = this.stringMandatory newInstance.stringOptional = this.stringOptional @@ -231,29 +351,37 @@ class NewNodeA extends NewNode(nodeKind = 0) with NodeABase { override def productElementName(n: Int): String = n match { - case 0 => "intList" - case 1 => "intMandatory" - case 2 => "intOptional" - case 3 => "stringList" - case 4 => "stringMandatory" - case 5 => "stringOptional" - case 6 => "contained_node_b" - case _ => "" + case 0 => "booleanOptional" + case 1 => "doubleOptional" + case 2 => "floatOptional" + case 3 => "intList" + case 4 => "intMandatory" + case 5 => "intOptional" + case 6 => "longOptional" + case 7 => "stringList" + case 8 => "stringMandatory" + case 9 => "stringOptional" + case 10 => "contained_node_b" + case _ => "" } override def productElement(n: Int): Any = n match { - case 0 => this.intList - case 1 => this.intMandatory - case 2 => this.intOptional - case 3 => this.stringList - case 4 => this.stringMandatory - case 5 => this.stringOptional - case 6 => this.contained_node_b - case _ => null + case 0 => this.booleanOptional + case 1 => this.doubleOptional + case 2 => this.floatOptional + case 3 => this.intList + case 4 => this.intMandatory + case 5 => this.intOptional + case 6 => this.longOptional + case 7 => this.stringList + case 8 => this.stringMandatory + case 9 => this.stringOptional + case 10 => this.contained_node_b + case _ => null } override def productPrefix = "NewNodeA" - override def productArity = 7 + override def productArity = 11 override def canEqual(that: Any): Boolean = that != null && that.isInstanceOf[NewNodeA] } diff --git a/test-schemas-domain-classes/src/main/scala/testdomains/generic/nodes/NewNodeB.scala b/test-schemas-domain-classes/src/main/scala/testdomains/generic/nodes/NewNodeB.scala index 884b9cf9..c45c841b 100644 --- a/test-schemas-domain-classes/src/main/scala/testdomains/generic/nodes/NewNodeB.scala +++ b/test-schemas-domain-classes/src/main/scala/testdomains/generic/nodes/NewNodeB.scala @@ -53,7 +53,7 @@ class NewNodeB extends NewNode(nodeKind = 1) with NodeBBase { def stringOptional(value: Option[String]): this.type = { this.stringOptional = value; this } def stringOptional(value: String): this.type = { this.stringOptional = Option(value); this } override def countAndVisitProperties(interface: flatgraph.BatchedUpdateInterface): Unit = { - interface.countProperty(this, 5, stringOptional.size) + interface.countProperty(this, 9, stringOptional.size) } override def copy: this.type = { diff --git a/test-schemas-domain-classes/src/main/scala/testdomains/generic/nodes/NodeA.scala b/test-schemas-domain-classes/src/main/scala/testdomains/generic/nodes/NodeA.scala index 86bf54e7..82e74076 100644 --- a/test-schemas-domain-classes/src/main/scala/testdomains/generic/nodes/NodeA.scala +++ b/test-schemas-domain-classes/src/main/scala/testdomains/generic/nodes/NodeA.scala @@ -8,9 +8,13 @@ import scala.collection.mutable */ trait NodeAEMT extends AnyRef + with HasBooleanOptionalEMT + with HasDoubleOptionalEMT + with HasFloatOptionalEMT with HasIntListEMT with HasIntMandatoryEMT with HasIntOptionalEMT + with HasLongOptionalEMT with HasStringListEMT with HasStringMandatoryEMT with HasStringOptionalEMT @@ -19,10 +23,14 @@ trait NodeABase extends AbstractNode with StaticType[NodeAEMT] { def contained_node_b: Option[NodeBBase] override def propertiesMap: java.util.Map[String, Any] = { import testdomains.generic.accessors.languagebootstrap.* - val res = new java.util.HashMap[String, Any]() + val res = new java.util.HashMap[String, Any]() + this.booleanOptional.foreach { p => res.put("boolean_optional", p) } + this.doubleOptional.foreach { p => res.put("double_optional", p) } + this.floatOptional.foreach { p => res.put("float_optional", p) } val tmpIntList = this.intList; if (tmpIntList.nonEmpty) res.put("int_list", tmpIntList) if ((42: Int) != this.intMandatory) res.put("int_mandatory", this.intMandatory) this.intOptional.foreach { p => res.put("int_optional", p) } + this.longOptional.foreach { p => res.put("long_optional", p) } val tmpStringList = this.stringList; if (tmpStringList.nonEmpty) res.put("string_list", tmpStringList) if (("": String) != this.stringMandatory) res.put("string_mandatory", this.stringMandatory) this.stringOptional.foreach { p => res.put("string_optional", p) } @@ -36,6 +44,12 @@ object NodeA { } /** * NODE PROPERTIES: + * + * ▸ BooleanOptional (Boolean); Cardinality `ZeroOrOne` (optional) + * + * ▸ DoubleOptional (Double); Cardinality `ZeroOrOne` (optional) + * + * ▸ FloatOptional (Float); Cardinality `ZeroOrOne` (optional) * * ▸ IntList (Int); Cardinality `List` (many) * @@ -43,6 +57,8 @@ object NodeA { * * ▸ IntOptional (Int); Cardinality `ZeroOrOne` (optional) * + * ▸ LongOptional (Long); Cardinality `ZeroOrOne` (optional) + * * ▸ StringList (String); Cardinality `List` (many) * * ▸ StringMandatory (String); Cardinality `one` (mandatory with default value ``) @@ -58,34 +74,42 @@ class NodeA(graph_4762: flatgraph.Graph, seq_4762: Int) with NodeABase with StaticType[NodeAEMT] { def contained_node_b: Option[NodeB] = - flatgraph.Accessors.getNodePropertyOption[NodeB](graph, nodeKind = nodeKind, propertyKind = 6, seq = seq) + flatgraph.Accessors.getNodePropertyOption[NodeB](graph, nodeKind = nodeKind, propertyKind = 10, seq = seq) override def productElementName(n: Int): String = n match { - case 0 => "intList" - case 1 => "intMandatory" - case 2 => "intOptional" - case 3 => "stringList" - case 4 => "stringMandatory" - case 5 => "stringOptional" - case 6 => "contained_node_b" - case _ => "" + case 0 => "booleanOptional" + case 1 => "doubleOptional" + case 2 => "floatOptional" + case 3 => "intList" + case 4 => "intMandatory" + case 5 => "intOptional" + case 6 => "longOptional" + case 7 => "stringList" + case 8 => "stringMandatory" + case 9 => "stringOptional" + case 10 => "contained_node_b" + case _ => "" } override def productElement(n: Int): Any = n match { - case 0 => this.intList - case 1 => this.intMandatory - case 2 => this.intOptional - case 3 => this.stringList - case 4 => this.stringMandatory - case 5 => this.stringOptional - case 6 => this.contained_node_b - case _ => null + case 0 => this.booleanOptional + case 1 => this.doubleOptional + case 2 => this.floatOptional + case 3 => this.intList + case 4 => this.intMandatory + case 5 => this.intOptional + case 6 => this.longOptional + case 7 => this.stringList + case 8 => this.stringMandatory + case 9 => this.stringOptional + case 10 => this.contained_node_b + case _ => null } override def productPrefix = "NodeA" - override def productArity = 7 + override def productArity = 11 override def canEqual(that: Any): Boolean = that != null && that.isInstanceOf[NodeA] } diff --git a/test-schemas-domain-classes/src/main/scala/testdomains/generic/traversals/TraversalNodeaBase.scala b/test-schemas-domain-classes/src/main/scala/testdomains/generic/traversals/TraversalNodeaBase.scala index 4b52595e..9a115f24 100644 --- a/test-schemas-domain-classes/src/main/scala/testdomains/generic/traversals/TraversalNodeaBase.scala +++ b/test-schemas-domain-classes/src/main/scala/testdomains/generic/traversals/TraversalNodeaBase.scala @@ -5,6 +5,71 @@ import testdomains.generic.accessors.languagebootstrap.* final class TraversalNodeaBase[NodeType <: nodes.NodeABase](val traversal: Iterator[NodeType]) extends AnyVal { + /** Traverse to booleanOptional property */ + def booleanOptional: Iterator[Boolean] = + traversal.flatMap(_.booleanOptional) + + /** Traverse to nodes where the booleanOptional equals the given `value` + */ + def booleanOptional(value: Boolean): Iterator[NodeType] = + traversal.filter { node => node.booleanOptional.isDefined && node.booleanOptional.get == value } + + /** Traverse to doubleOptional property */ + def doubleOptional: Iterator[Double] = + traversal.flatMap(_.doubleOptional) + + /** Traverse to nodes where the doubleOptional equals the given `value` + */ + def doubleOptional(value: Double): Iterator[NodeType] = + traversal.filter { node => node.doubleOptional.isDefined && node.doubleOptional.get == value } + + /** Traverse to nodes where the doubleOptional equals at least one of the given `values` + */ + def doubleOptional(values: Double*): Iterator[NodeType] = { + val vset = values.toSet + traversal.filter { node => node.doubleOptional.isDefined && vset.contains(node.doubleOptional.get) } + } + + /** Traverse to nodes where the doubleOptional does not equal the given `value` + */ + def doubleOptionalNot(value: Double): Iterator[NodeType] = + traversal.filter { node => node.doubleOptional.isEmpty || node.doubleOptional.get != value } + + /** Traverse to nodes where the doubleOptional does not equal any one of the given `values` + */ + def doubleOptionalNot(values: Double*): Iterator[NodeType] = { + val vset = values.toSet + traversal.filter { node => node.doubleOptional.isEmpty || !vset.contains(node.doubleOptional.get) } + } + + /** Traverse to floatOptional property */ + def floatOptional: Iterator[Float] = + traversal.flatMap(_.floatOptional) + + /** Traverse to nodes where the floatOptional equals the given `value` + */ + def floatOptional(value: Float): Iterator[NodeType] = + traversal.filter { node => node.floatOptional.isDefined && node.floatOptional.get == value } + + /** Traverse to nodes where the floatOptional equals at least one of the given `values` + */ + def floatOptional(values: Float*): Iterator[NodeType] = { + val vset = values.toSet + traversal.filter { node => node.floatOptional.isDefined && vset.contains(node.floatOptional.get) } + } + + /** Traverse to nodes where the floatOptional does not equal the given `value` + */ + def floatOptionalNot(value: Float): Iterator[NodeType] = + traversal.filter { node => node.floatOptional.isEmpty || node.floatOptional.get != value } + + /** Traverse to nodes where the floatOptional does not equal any one of the given `values` + */ + def floatOptionalNot(values: Float*): Iterator[NodeType] = { + val vset = values.toSet + traversal.filter { node => node.floatOptional.isEmpty || !vset.contains(node.floatOptional.get) } + } + /** Traverse to intList property */ def intList: Iterator[Int] = traversal.flatMap(_.intList) @@ -137,6 +202,34 @@ final class TraversalNodeaBase[NodeType <: nodes.NodeABase](val traversal: Itera val tmp = node.intOptional; tmp.isDefined && tmp.get <= value } + /** Traverse to longOptional property */ + def longOptional: Iterator[Long] = + traversal.flatMap(_.longOptional) + + /** Traverse to nodes where the longOptional equals the given `value` + */ + def longOptional(value: Long): Iterator[NodeType] = + traversal.filter { node => node.longOptional.isDefined && node.longOptional.get == value } + + /** Traverse to nodes where the longOptional equals at least one of the given `values` + */ + def longOptional(values: Long*): Iterator[NodeType] = { + val vset = values.toSet + traversal.filter { node => node.longOptional.isDefined && vset.contains(node.longOptional.get) } + } + + /** Traverse to nodes where the longOptional does not equal the given `value` + */ + def longOptionalNot(value: Long): Iterator[NodeType] = + traversal.filter { node => node.longOptional.isEmpty || node.longOptional.get != value } + + /** Traverse to nodes where the longOptional does not equal any one of the given `values` + */ + def longOptionalNot(values: Long*): Iterator[NodeType] = { + val vset = values.toSet + traversal.filter { node => node.longOptional.isEmpty || !vset.contains(node.longOptional.get) } + } + /** Traverse to stringList property */ def stringList: Iterator[String] = traversal.flatMap(_.stringList) @@ -167,7 +260,7 @@ final class TraversalNodeaBase[NodeType <: nodes.NodeABase](val traversal: Itera def stringMandatoryExact(value: String): Iterator[NodeType] = traversal match { case init: flatgraph.misc.InitNodeIterator[flatgraph.GNode @unchecked] if init.isVirgin && init.hasNext => val someNode = init.next - flatgraph.Accessors.getWithInverseIndex(someNode.graph, someNode.nodeKind, 4, value).asInstanceOf[Iterator[NodeType]] + flatgraph.Accessors.getWithInverseIndex(someNode.graph, someNode.nodeKind, 8, value).asInstanceOf[Iterator[NodeType]] case _ => traversal.filter { _.stringMandatory == value } } @@ -179,7 +272,7 @@ final class TraversalNodeaBase[NodeType <: nodes.NodeABase](val traversal: Itera case init: flatgraph.misc.InitNodeIterator[flatgraph.GNode @unchecked] if init.isVirgin && init.hasNext => val someNode = init.next values.iterator.flatMap { value => - flatgraph.Accessors.getWithInverseIndex(someNode.graph, someNode.nodeKind, 4, value).asInstanceOf[Iterator[NodeType]] + flatgraph.Accessors.getWithInverseIndex(someNode.graph, someNode.nodeKind, 8, value).asInstanceOf[Iterator[NodeType]] } case _ => val valueSet = values.toSet @@ -236,7 +329,7 @@ final class TraversalNodeaBase[NodeType <: nodes.NodeABase](val traversal: Itera def stringOptionalExact(value: String): Iterator[NodeType] = traversal match { case init: flatgraph.misc.InitNodeIterator[flatgraph.GNode @unchecked] if init.isVirgin && init.hasNext => val someNode = init.next - flatgraph.Accessors.getWithInverseIndex(someNode.graph, someNode.nodeKind, 5, value).asInstanceOf[Iterator[NodeType]] + flatgraph.Accessors.getWithInverseIndex(someNode.graph, someNode.nodeKind, 9, value).asInstanceOf[Iterator[NodeType]] case _ => traversal.filter { node => val tmp = node.stringOptional; tmp.isDefined && tmp.get == value diff --git a/test-schemas-domain-classes/src/main/scala/testdomains/generic/traversals/TraversalNodebBase.scala b/test-schemas-domain-classes/src/main/scala/testdomains/generic/traversals/TraversalNodebBase.scala index c313eaab..3f39014b 100644 --- a/test-schemas-domain-classes/src/main/scala/testdomains/generic/traversals/TraversalNodebBase.scala +++ b/test-schemas-domain-classes/src/main/scala/testdomains/generic/traversals/TraversalNodebBase.scala @@ -36,7 +36,7 @@ final class TraversalNodebBase[NodeType <: nodes.NodeBBase](val traversal: Itera def stringOptionalExact(value: String): Iterator[NodeType] = traversal match { case init: flatgraph.misc.InitNodeIterator[flatgraph.GNode @unchecked] if init.isVirgin && init.hasNext => val someNode = init.next - flatgraph.Accessors.getWithInverseIndex(someNode.graph, someNode.nodeKind, 5, value).asInstanceOf[Iterator[NodeType]] + flatgraph.Accessors.getWithInverseIndex(someNode.graph, someNode.nodeKind, 9, value).asInstanceOf[Iterator[NodeType]] case _ => traversal.filter { node => val tmp = node.stringOptional; tmp.isDefined && tmp.get == value diff --git a/test-schemas-domain-classes/src/main/scala/testdomains/generic/traversals/TraversalPropertyBooleanOptional.scala b/test-schemas-domain-classes/src/main/scala/testdomains/generic/traversals/TraversalPropertyBooleanOptional.scala new file mode 100644 index 00000000..63353832 --- /dev/null +++ b/test-schemas-domain-classes/src/main/scala/testdomains/generic/traversals/TraversalPropertyBooleanOptional.scala @@ -0,0 +1,19 @@ +package testdomains.generic.traversals + +import testdomains.generic.nodes +import testdomains.generic.accessors.languagebootstrap.* + +final class TraversalPropertyBooleanOptional[NodeType <: nodes.StoredNode & nodes.StaticType[nodes.HasBooleanOptionalEMT]]( + val traversal: Iterator[NodeType] +) extends AnyVal { + + /** Traverse to booleanOptional property */ + def booleanOptional: Iterator[Boolean] = + traversal.flatMap(_.booleanOptional) + + /** Traverse to nodes where the booleanOptional equals the given `value` + */ + def booleanOptional(value: Boolean): Iterator[NodeType] = + traversal.filter { node => node.booleanOptional.isDefined && node.booleanOptional.get == value } + +} diff --git a/test-schemas-domain-classes/src/main/scala/testdomains/generic/traversals/TraversalPropertyDoubleOptional.scala b/test-schemas-domain-classes/src/main/scala/testdomains/generic/traversals/TraversalPropertyDoubleOptional.scala new file mode 100644 index 00000000..61dd2c77 --- /dev/null +++ b/test-schemas-domain-classes/src/main/scala/testdomains/generic/traversals/TraversalPropertyDoubleOptional.scala @@ -0,0 +1,38 @@ +package testdomains.generic.traversals + +import testdomains.generic.nodes +import testdomains.generic.accessors.languagebootstrap.* + +final class TraversalPropertyDoubleOptional[NodeType <: nodes.StoredNode & nodes.StaticType[nodes.HasDoubleOptionalEMT]]( + val traversal: Iterator[NodeType] +) extends AnyVal { + + /** Traverse to doubleOptional property */ + def doubleOptional: Iterator[Double] = + traversal.flatMap(_.doubleOptional) + + /** Traverse to nodes where the doubleOptional equals the given `value` + */ + def doubleOptional(value: Double): Iterator[NodeType] = + traversal.filter { node => node.doubleOptional.isDefined && node.doubleOptional.get == value } + + /** Traverse to nodes where the doubleOptional equals at least one of the given `values` + */ + def doubleOptional(values: Double*): Iterator[NodeType] = { + val vset = values.toSet + traversal.filter { node => node.doubleOptional.isDefined && vset.contains(node.doubleOptional.get) } + } + + /** Traverse to nodes where the doubleOptional does not equal the given `value` + */ + def doubleOptionalNot(value: Double): Iterator[NodeType] = + traversal.filter { node => node.doubleOptional.isEmpty || node.doubleOptional.get != value } + + /** Traverse to nodes where the doubleOptional does not equal any one of the given `values` + */ + def doubleOptionalNot(values: Double*): Iterator[NodeType] = { + val vset = values.toSet + traversal.filter { node => node.doubleOptional.isEmpty || !vset.contains(node.doubleOptional.get) } + } + +} diff --git a/test-schemas-domain-classes/src/main/scala/testdomains/generic/traversals/TraversalPropertyFloatOptional.scala b/test-schemas-domain-classes/src/main/scala/testdomains/generic/traversals/TraversalPropertyFloatOptional.scala new file mode 100644 index 00000000..08fe3d63 --- /dev/null +++ b/test-schemas-domain-classes/src/main/scala/testdomains/generic/traversals/TraversalPropertyFloatOptional.scala @@ -0,0 +1,38 @@ +package testdomains.generic.traversals + +import testdomains.generic.nodes +import testdomains.generic.accessors.languagebootstrap.* + +final class TraversalPropertyFloatOptional[NodeType <: nodes.StoredNode & nodes.StaticType[nodes.HasFloatOptionalEMT]]( + val traversal: Iterator[NodeType] +) extends AnyVal { + + /** Traverse to floatOptional property */ + def floatOptional: Iterator[Float] = + traversal.flatMap(_.floatOptional) + + /** Traverse to nodes where the floatOptional equals the given `value` + */ + def floatOptional(value: Float): Iterator[NodeType] = + traversal.filter { node => node.floatOptional.isDefined && node.floatOptional.get == value } + + /** Traverse to nodes where the floatOptional equals at least one of the given `values` + */ + def floatOptional(values: Float*): Iterator[NodeType] = { + val vset = values.toSet + traversal.filter { node => node.floatOptional.isDefined && vset.contains(node.floatOptional.get) } + } + + /** Traverse to nodes where the floatOptional does not equal the given `value` + */ + def floatOptionalNot(value: Float): Iterator[NodeType] = + traversal.filter { node => node.floatOptional.isEmpty || node.floatOptional.get != value } + + /** Traverse to nodes where the floatOptional does not equal any one of the given `values` + */ + def floatOptionalNot(values: Float*): Iterator[NodeType] = { + val vset = values.toSet + traversal.filter { node => node.floatOptional.isEmpty || !vset.contains(node.floatOptional.get) } + } + +} diff --git a/test-schemas-domain-classes/src/main/scala/testdomains/generic/traversals/TraversalPropertyLongOptional.scala b/test-schemas-domain-classes/src/main/scala/testdomains/generic/traversals/TraversalPropertyLongOptional.scala new file mode 100644 index 00000000..4dcd36dd --- /dev/null +++ b/test-schemas-domain-classes/src/main/scala/testdomains/generic/traversals/TraversalPropertyLongOptional.scala @@ -0,0 +1,38 @@ +package testdomains.generic.traversals + +import testdomains.generic.nodes +import testdomains.generic.accessors.languagebootstrap.* + +final class TraversalPropertyLongOptional[NodeType <: nodes.StoredNode & nodes.StaticType[nodes.HasLongOptionalEMT]]( + val traversal: Iterator[NodeType] +) extends AnyVal { + + /** Traverse to longOptional property */ + def longOptional: Iterator[Long] = + traversal.flatMap(_.longOptional) + + /** Traverse to nodes where the longOptional equals the given `value` + */ + def longOptional(value: Long): Iterator[NodeType] = + traversal.filter { node => node.longOptional.isDefined && node.longOptional.get == value } + + /** Traverse to nodes where the longOptional equals at least one of the given `values` + */ + def longOptional(values: Long*): Iterator[NodeType] = { + val vset = values.toSet + traversal.filter { node => node.longOptional.isDefined && vset.contains(node.longOptional.get) } + } + + /** Traverse to nodes where the longOptional does not equal the given `value` + */ + def longOptionalNot(value: Long): Iterator[NodeType] = + traversal.filter { node => node.longOptional.isEmpty || node.longOptional.get != value } + + /** Traverse to nodes where the longOptional does not equal any one of the given `values` + */ + def longOptionalNot(values: Long*): Iterator[NodeType] = { + val vset = values.toSet + traversal.filter { node => node.longOptional.isEmpty || !vset.contains(node.longOptional.get) } + } + +} diff --git a/test-schemas-domain-classes/src/main/scala/testdomains/generic/traversals/TraversalPropertyStringMandatory.scala b/test-schemas-domain-classes/src/main/scala/testdomains/generic/traversals/TraversalPropertyStringMandatory.scala index 9ba166d5..c63940a5 100644 --- a/test-schemas-domain-classes/src/main/scala/testdomains/generic/traversals/TraversalPropertyStringMandatory.scala +++ b/test-schemas-domain-classes/src/main/scala/testdomains/generic/traversals/TraversalPropertyStringMandatory.scala @@ -33,7 +33,7 @@ final class TraversalPropertyStringMandatory[NodeType <: nodes.StoredNode & node def stringMandatoryExact(value: String): Iterator[NodeType] = traversal match { case init: flatgraph.misc.InitNodeIterator[flatgraph.GNode @unchecked] if init.isVirgin && init.hasNext => val someNode = init.next - flatgraph.Accessors.getWithInverseIndex(someNode.graph, someNode.nodeKind, 4, value).asInstanceOf[Iterator[NodeType]] + flatgraph.Accessors.getWithInverseIndex(someNode.graph, someNode.nodeKind, 8, value).asInstanceOf[Iterator[NodeType]] case _ => traversal.filter { _.stringMandatory == value } } @@ -45,7 +45,7 @@ final class TraversalPropertyStringMandatory[NodeType <: nodes.StoredNode & node case init: flatgraph.misc.InitNodeIterator[flatgraph.GNode @unchecked] if init.isVirgin && init.hasNext => val someNode = init.next values.iterator.flatMap { value => - flatgraph.Accessors.getWithInverseIndex(someNode.graph, someNode.nodeKind, 4, value).asInstanceOf[Iterator[NodeType]] + flatgraph.Accessors.getWithInverseIndex(someNode.graph, someNode.nodeKind, 8, value).asInstanceOf[Iterator[NodeType]] } case _ => val valueSet = values.toSet diff --git a/test-schemas-domain-classes/src/main/scala/testdomains/generic/traversals/TraversalPropertyStringOptional.scala b/test-schemas-domain-classes/src/main/scala/testdomains/generic/traversals/TraversalPropertyStringOptional.scala index e97cf425..19413d18 100644 --- a/test-schemas-domain-classes/src/main/scala/testdomains/generic/traversals/TraversalPropertyStringOptional.scala +++ b/test-schemas-domain-classes/src/main/scala/testdomains/generic/traversals/TraversalPropertyStringOptional.scala @@ -38,7 +38,7 @@ final class TraversalPropertyStringOptional[NodeType <: nodes.StoredNode & nodes def stringOptionalExact(value: String): Iterator[NodeType] = traversal match { case init: flatgraph.misc.InitNodeIterator[flatgraph.GNode @unchecked] if init.isVirgin && init.hasNext => val someNode = init.next - flatgraph.Accessors.getWithInverseIndex(someNode.graph, someNode.nodeKind, 5, value).asInstanceOf[Iterator[NodeType]] + flatgraph.Accessors.getWithInverseIndex(someNode.graph, someNode.nodeKind, 9, value).asInstanceOf[Iterator[NodeType]] case _ => traversal.filter { node => val tmp = node.stringOptional; tmp.isDefined && tmp.get == value diff --git a/test-schemas-domain-classes/src/main/scala/testdomains/generic/traversals/package.scala b/test-schemas-domain-classes/src/main/scala/testdomains/generic/traversals/package.scala index f0d56804..cdb4f7c3 100644 --- a/test-schemas-domain-classes/src/main/scala/testdomains/generic/traversals/package.scala +++ b/test-schemas-domain-classes/src/main/scala/testdomains/generic/traversals/package.scala @@ -9,6 +9,15 @@ package object traversals { object languagebootstrap extends ConcreteStoredConversions trait ConcreteStoredConversions extends ConcreteBaseConversions { + implicit def accessPropertyBooleanOptionalTraversal[NodeType <: nodes.StoredNode & nodes.StaticType[nodes.HasBooleanOptionalEMT]]( + traversal: IterableOnce[NodeType] + ): TraversalPropertyBooleanOptional[NodeType] = new TraversalPropertyBooleanOptional(traversal.iterator) + implicit def accessPropertyDoubleOptionalTraversal[NodeType <: nodes.StoredNode & nodes.StaticType[nodes.HasDoubleOptionalEMT]]( + traversal: IterableOnce[NodeType] + ): TraversalPropertyDoubleOptional[NodeType] = new TraversalPropertyDoubleOptional(traversal.iterator) + implicit def accessPropertyFloatOptionalTraversal[NodeType <: nodes.StoredNode & nodes.StaticType[nodes.HasFloatOptionalEMT]]( + traversal: IterableOnce[NodeType] + ): TraversalPropertyFloatOptional[NodeType] = new TraversalPropertyFloatOptional(traversal.iterator) implicit def accessPropertyIntListTraversal[NodeType <: nodes.StoredNode & nodes.StaticType[nodes.HasIntListEMT]]( traversal: IterableOnce[NodeType] ): TraversalPropertyIntList[NodeType] = new TraversalPropertyIntList(traversal.iterator) @@ -18,6 +27,9 @@ package object traversals { implicit def accessPropertyIntOptionalTraversal[NodeType <: nodes.StoredNode & nodes.StaticType[nodes.HasIntOptionalEMT]]( traversal: IterableOnce[NodeType] ): TraversalPropertyIntOptional[NodeType] = new TraversalPropertyIntOptional(traversal.iterator) + implicit def accessPropertyLongOptionalTraversal[NodeType <: nodes.StoredNode & nodes.StaticType[nodes.HasLongOptionalEMT]]( + traversal: IterableOnce[NodeType] + ): TraversalPropertyLongOptional[NodeType] = new TraversalPropertyLongOptional(traversal.iterator) implicit def accessPropertyStringListTraversal[NodeType <: nodes.StoredNode & nodes.StaticType[nodes.HasStringListEMT]]( traversal: IterableOnce[NodeType] ): TraversalPropertyStringList[NodeType] = new TraversalPropertyStringList(traversal.iterator) diff --git a/test-schemas/src/main/scala/flatgraph/testdomains/Generic.scala b/test-schemas/src/main/scala/flatgraph/testdomains/Generic.scala index 5c79d340..3c5ce0f9 100644 --- a/test-schemas/src/main/scala/flatgraph/testdomains/Generic.scala +++ b/test-schemas/src/main/scala/flatgraph/testdomains/Generic.scala @@ -14,6 +14,10 @@ object Generic { val intMandatory = builder.addProperty("int_mandatory", ValueType.Int).mandatory(default = 42) val intOptional = builder.addProperty("int_optional", ValueType.Int) val intList = builder.addProperty("int_list", ValueType.Int).asList() + val floatOptional = builder.addProperty("float_optional", ValueType.Float) + val doubleOptional = builder.addProperty("double_optional", ValueType.Double) + val longOptional = builder.addProperty("long_optional", ValueType.Long) + val booleanOptional = builder.addProperty("boolean_optional", ValueType.Boolean) val nodeA = builder .addNodeType("node_a") @@ -23,6 +27,10 @@ object Generic { .addProperty(intMandatory) .addProperty(intOptional) .addProperty(intList) + .addProperty(floatOptional) + .addProperty(doubleOptional) + .addProperty(longOptional) + .addProperty(booleanOptional) val nodeB = builder .addNodeType("node_b") diff --git a/tests/src/test/resources/neo4jcsv/nodes_node_a_data.csv b/tests/src/test/resources/neo4jcsv/nodes_node_a_data.csv index c49018c9..70486e7f 100644 --- a/tests/src/test/resources/neo4jcsv/nodes_node_a_data.csv +++ b/tests/src/test/resources/neo4jcsv/nodes_node_a_data.csv @@ -1,2 +1,2 @@ -0,node_a,,42,,node 1 c1;node 1 c2,node 1 a,node 1 b -1,node_a,10;11;12,1,2,,, +0,node_a,,,,,42,,,node 1 c1;node 1 c2,node 1 a,node 1 b +1,node_a,,,,10;11;12,1,2,,,, diff --git a/tests/src/test/resources/neo4jcsv/nodes_node_a_header.csv b/tests/src/test/resources/neo4jcsv/nodes_node_a_header.csv index 858e1b8b..3737186b 100644 --- a/tests/src/test/resources/neo4jcsv/nodes_node_a_header.csv +++ b/tests/src/test/resources/neo4jcsv/nodes_node_a_header.csv @@ -1 +1 @@ -:ID,:LABEL,int_list:int[],int_mandatory:int,int_optional:int,string_list:string[],string_mandatory:string,string_optional:string +:ID,:LABEL,boolean_optional,double_optional,float_optional,int_list:int[],int_mandatory:int,int_optional:int,long_optional,string_list:string[],string_mandatory:string,string_optional:string diff --git a/tests/src/test/scala/flatgraph/formats/neo4jcsv/Neo4jCsvTests.scala b/tests/src/test/scala/flatgraph/formats/neo4jcsv/Neo4jCsvTests.scala index 1d79f419..ef3bf4be 100644 --- a/tests/src/test/scala/flatgraph/formats/neo4jcsv/Neo4jCsvTests.scala +++ b/tests/src/test/scala/flatgraph/formats/neo4jcsv/Neo4jCsvTests.scala @@ -34,12 +34,12 @@ class Neo4jCsvTests extends AnyWordSpec { // assert csv file contents val nodeHeaderFile = fuzzyFindFile(exportedFiles, NodeA.Label, HeaderFileSuffix) nodeHeaderFile.contentAsString.trim shouldBe - ":ID,:LABEL,int_list:int[],int_mandatory:int,int_optional:int,string_list:string[],string_mandatory:string,string_optional:string" + ":ID,:LABEL,boolean_optional,double_optional,float_optional,int_list:int[],int_mandatory:int,int_optional:int,long_optional,string_list:string[],string_mandatory:string,string_optional:string" val nodeDataFileLines = fuzzyFindFile(exportedFiles, NodeA.Label, DataFileSuffix).lines.toSeq nodeDataFileLines.size shouldBe 2 - nodeDataFileLines should contain("0,node_a,,42,,node 1 c1;node 1 c2,node 1 a,node 1 b") - nodeDataFileLines should contain("1,node_a,10;11;12,1,2,,,") + nodeDataFileLines should contain("0,node_a,,,,,42,,,node 1 c1;node 1 c2,node 1 a,node 1 b") + nodeDataFileLines should contain("1,node_a,,,,10;11;12,1,2,,,,") val edgeHeaderFile = fuzzyFindFile(exportedFiles, ConnectedTo.Label, HeaderFileSuffix) edgeHeaderFile.contentAsString.trim shouldBe ":START_ID,:END_ID,:TYPE,string_mandatory:string" @@ -52,12 +52,16 @@ class Neo4jCsvTests extends AnyWordSpec { """LOAD CSV FROM 'file:/nodes_node_a_data.csv' AS line |CREATE (:node_a { |id: toInteger(line[0]), - |int_list: toIntegerList(split(line[2], ";")), - |int_mandatory: toInteger(line[3]), - |int_optional: toInteger(line[4]), - |string_list: toStringList(split(line[5], ";")), - |string_mandatory: line[6], - |string_optional: line[7] + |boolean_optional: line[2], + |double_optional: line[3], + |float_optional: line[4], + |int_list: toIntegerList(split(line[5], ";")), + |int_mandatory: toInteger(line[6]), + |int_optional: toInteger(line[7]), + |long_optional: line[8], + |string_list: toStringList(split(line[9], ";")), + |string_mandatory: line[10], + |string_optional: line[11] |}); |""".stripMargin From 1c8af2993f6ad67c7915fbab29ea580ad01cfaac Mon Sep 17 00:00:00 2001 From: eoliphant Date: Thu, 18 Jun 2026 12:50:38 -0400 Subject: [PATCH 3/4] test: add GraphSON round-trip test covering all PropertyValue types --- .../formats/graphson/GraphSONTests.scala | 35 +++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/tests/src/test/scala/flatgraph/formats/graphson/GraphSONTests.scala b/tests/src/test/scala/flatgraph/formats/graphson/GraphSONTests.scala index e8fc8567..4af9d941 100644 --- a/tests/src/test/scala/flatgraph/formats/graphson/GraphSONTests.scala +++ b/tests/src/test/scala/flatgraph/formats/graphson/GraphSONTests.scala @@ -5,9 +5,8 @@ import flatgraph.{DiffGraphApplier, GenericDNode, TestGraphs} import flatgraph.misc.TestUtils.applyDiff import flatgraph.util.DiffTool import testdomains.generic.language.* -import testdomains.generic.nodes.NodeA +import testdomains.generic.nodes.{NewNodeA, NewNodeB} import testdomains.generic.{GenericDomain, PropertyNames} -import testdomains.generic.nodes.NewNodeB import org.scalatest.matchers.should.Matchers.* import org.scalatest.wordspec.AnyWordSpec @@ -66,4 +65,36 @@ class GraphSONTests extends AnyWordSpec { } } + "export and re-import all PropertyValue types" in { + val domain = GenericDomain.empty + val graph = domain.graph + + // Use values that round-trip through Float/Double JSON serialization exactly + val newNode = NewNodeA() + .floatOptional(2.0f) + .doubleOptional(4.0) + .longOptional(Long.MaxValue) + .booleanOptional(true) + .stringList(Seq("alpha", "beta")) + .intList(Seq(1, 2, 3)) + + DiffGraphApplier.applyDiff(graph, GenericDomain.newDiffGraphBuilder.addNode(newNode)) + + File.usingTemporaryDirectory(getClass.getName) { exportDir => + val exportResult = GraphSONExporter.runExport(graph, exportDir.pathAsString) + exportResult.nodeCount shouldBe 1 + exportResult.edgeCount shouldBe 0 + val Seq(graphJsonFile) = exportResult.files + + val reimported = GenericDomain.empty.graph + GraphSONImporter.runImport(reimported, graphJsonFile) + + val diff = DiffTool.compare(graph, reimported) + val diffString = diff.asScala.mkString(lineSeparator) + withClue(s"original and reimported graph should be equal, differences:\n$diffString\n") { + diff.size shouldBe 0 + } + } + } + } From 174e93a46434bc20e43888466b108315634f4778 Mon Sep 17 00:00:00 2001 From: eoliphant Date: Thu, 18 Jun 2026 13:03:19 -0400 Subject: [PATCH 4/4] fix: replace GraphSONExporter spray-json write path with Jackson Core streaming MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #360 — OOM when exporting large graphs. The spray-json PrettyPrinter built the entire graph as a single java.lang.String (UTF-16, ~1GB ceiling). Jackson Core streams tokens directly to a BufferedOutputStream, so memory usage is bounded by the generator buffer, not graph size. The import path (GraphSONImporter) is unchanged. Public API is unchanged. --- build.sbt | 1 + .../formats/graphson/GraphSONExporter.scala | 186 +++++++++++++----- 2 files changed, 143 insertions(+), 44 deletions(-) diff --git a/build.sbt b/build.sbt index f6f45297..546cb56f 100644 --- a/build.sbt +++ b/build.sbt @@ -58,6 +58,7 @@ lazy val formats = project "org.apache.commons" % "commons-text" % commonsTextVersion, "org.scala-lang.modules" %% "scala-xml" % "2.3.0", "io.spray" %% "spray-json" % "1.3.6", + "com.fasterxml.jackson.core" % "jackson-core" % "2.17.2", "com.github.scopt" %% "scopt" % "4.1.0", ) ) diff --git a/formats/src/main/scala/flatgraph/formats/graphson/GraphSONExporter.scala b/formats/src/main/scala/flatgraph/formats/graphson/GraphSONExporter.scala index 4f64af5d..e56c10c9 100644 --- a/formats/src/main/scala/flatgraph/formats/graphson/GraphSONExporter.scala +++ b/formats/src/main/scala/flatgraph/formats/graphson/GraphSONExporter.scala @@ -1,66 +1,164 @@ package flatgraph.formats.graphson +import com.fasterxml.jackson.core.{JsonEncoding, JsonFactory, JsonGenerator} import flatgraph.formats.* -import flatgraph.formats.graphson.GraphSONProtocol.* import flatgraph.{Accessors, GNode, Schema} -import spray.json.JsonWriter -import java.nio.file.Path -import java.util.concurrent.atomic.AtomicInteger +import java.io.BufferedOutputStream +import java.nio.file.{Files, Path} +import java.util.concurrent.atomic.AtomicLong import scala.jdk.CollectionConverters.IterableHasAsScala +import scala.util.Using /** Exports to GraphSON 3.0 https://tinkerpop.apache.org/docs/3.4.1/dev/io/#graphson-3d0 + * + * Uses Jackson Core streaming API to avoid the spray-json PrettyPrinter bottleneck that builds the entire graph as a single + * java.lang.String, causing OOM on large graphs. */ object GraphSONExporter extends Exporter { override def defaultFileExtension = "json" + private val jsonFactory = new JsonFactory() + override def runExport(schema: Schema, nodes: IterableOnce[GNode], edges: IterableOnce[flatgraph.Edge], outputFile: Path) = { - val outFile = resolveOutputFileSingle(outputFile, s"export.$defaultFileExtension") - // flatgraph doesn't have edge IDs, so for GraphSON we generate them synthetically - val propertyId = new AtomicInteger(0) - val edgeId = new AtomicInteger(0) - - val nodeEntries = nodes.iterator.map { node => - val properties = Accessors - .getNodeProperties(node) - .iterator - .map { case (name, value) => - name -> Property(LongValue(propertyId.getAndIncrement()), valueEntry(value), "g:VertexProperty") + val outFile = resolveOutputFileSingle(outputFile, s"export.$defaultFileExtension") + val propertyId = new AtomicLong(0) + val edgeId = new AtomicLong(0) + var nodeCount = 0 + var edgeCount = 0 + + Using.resource(new BufferedOutputStream(Files.newOutputStream(outFile))) { os => + Using.resource(jsonFactory.createGenerator(os, JsonEncoding.UTF8)) { gen => + gen.useDefaultPrettyPrinter() + + gen.writeStartObject() + gen.writeFieldName("@value") + gen.writeStartObject() + + gen.writeFieldName("vertices") + gen.writeStartArray() + nodes.iterator.foreach { node => + nodeCount += 1 + writeVertex(gen, node, propertyId) } - .toMap - Vertex(LongValue(node.id), node.label, properties) - }.toSeq - - val edgeEntries = edges.iterator.map { edge => - val inNode = edge.dst - val outNode = edge.src - val propertiesMap: Map[String, Property] = Option(edge.property) - .map { value => - Map("EdgeProperty" -> Property(LongValue(propertyId.getAndIncrement()), valueEntry(edge.property), "g:Property")) + gen.writeEndArray() + + gen.writeFieldName("edges") + gen.writeStartArray() + edges.iterator.foreach { edge => + edgeCount += 1 + writeEdge(gen, edge, propertyId, edgeId) } - .getOrElse(Map.empty) - - Edge( - LongValue(edgeId.getAndIncrement), - edge.label, - inNode.label, - outNode.label, - LongValue(inNode.id), - LongValue(outNode.id), - propertiesMap - ) - }.toSeq - - val graphSON = GraphSON(GraphSONElements(nodeEntries, edgeEntries)) - val json = implicitly[JsonWriter[GraphSON]].write(graphSON) - writeFile(outFile, json.prettyPrint) - - ExportResult(nodeCount = nodeEntries.size, edgeCount = edgeEntries.size, files = Seq(outFile), Option.empty) + gen.writeEndArray() + + gen.writeEndObject() // @value + gen.writeStringField("@type", "tinker:graph") + gen.writeEndObject() // root + } + } + + ExportResult(nodeCount = nodeCount, edgeCount = edgeCount, files = Seq(outFile), Option.empty) + } + + private def writeVertex(gen: JsonGenerator, node: GNode, propertyId: AtomicLong): Unit = { + gen.writeStartObject() + gen.writeFieldName("id") + writeLongValue(gen, node.id) + gen.writeStringField("label", node.label) + gen.writeFieldName("properties") + gen.writeStartObject() + Accessors.getNodeProperties(node).iterator.foreach { case (name, value) => + gen.writeFieldName(name) + writeProperty(gen, propertyId.getAndIncrement(), valueEntry(value), "g:VertexProperty") + } + gen.writeEndObject() + gen.writeStringField("@type", "g:Vertex") + gen.writeEndObject() + } + + private def writeEdge(gen: JsonGenerator, edge: flatgraph.Edge, propertyId: AtomicLong, edgeId: AtomicLong): Unit = { + val inNode = edge.dst + val outNode = edge.src + gen.writeStartObject() + gen.writeFieldName("id") + writeLongValue(gen, edgeId.getAndIncrement()) + gen.writeStringField("label", edge.label) + gen.writeStringField("inVLabel", inNode.label) + gen.writeStringField("outVLabel", outNode.label) + gen.writeFieldName("inV") + writeLongValue(gen, inNode.id) + gen.writeFieldName("outV") + writeLongValue(gen, outNode.id) + gen.writeFieldName("properties") + gen.writeStartObject() + Option(edge.property).foreach { value => + gen.writeFieldName("EdgeProperty") + writeProperty(gen, propertyId.getAndIncrement(), valueEntry(value), "g:Property") + } + gen.writeEndObject() + gen.writeStringField("@type", "g:Edge") + gen.writeEndObject() + } + + private def writeProperty(gen: JsonGenerator, id: Long, value: PropertyValue, typeStr: String): Unit = { + gen.writeStartObject() + gen.writeFieldName("id") + writeLongValue(gen, id) + gen.writeFieldName("@value") + writePropertyValue(gen, value) + gen.writeStringField("@type", typeStr) + gen.writeEndObject() + } + + private def writeLongValue(gen: JsonGenerator, v: Long): Unit = { + gen.writeStartObject() + gen.writeNumberField("@value", v) + gen.writeStringField("@type", "g:Int64") + gen.writeEndObject() + } + + private def writePropertyValue(gen: JsonGenerator, pv: PropertyValue): Unit = pv match { + case StringValue(v, _) => gen.writeString(v) + case BooleanValue(v, _) => gen.writeBoolean(v) + case LongValue(v, _) => + gen.writeStartObject() + gen.writeNumberField("@value", v) + gen.writeStringField("@type", "g:Int64") + gen.writeEndObject() + case IntValue(v, _) => + gen.writeStartObject() + gen.writeNumberField("@value", v) + gen.writeStringField("@type", "g:Int32") + gen.writeEndObject() + case FloatValue(v, _) => + gen.writeStartObject() + gen.writeNumberField("@value", v) + gen.writeStringField("@type", "g:Float") + gen.writeEndObject() + case DoubleValue(v, _) => + gen.writeStartObject() + gen.writeNumberField("@value", v) + gen.writeStringField("@type", "g:Double") + gen.writeEndObject() + case NodeIdValue(v, _) => + gen.writeStartObject() + gen.writeNumberField("@value", v) + gen.writeStringField("@type", "g:VertexId") + gen.writeEndObject() + case ListValue(elements, _) => + gen.writeStartObject() + gen.writeFieldName("@value") + gen.writeStartArray() + elements.foreach(writePropertyValue(gen, _)) + gen.writeEndArray() + gen.writeStringField("@type", "g:List") + gen.writeEndObject() + case unsupported => + throw new IllegalArgumentException(s"unsupported propertyValue: $unsupported") } def valueEntry(propertyValue: Any): PropertyValue = { - // Other types require explicit type definitions to be interpreted other than string or bool propertyValue match { case x: Array[_] => ListValue(x.map(valueEntry)) case x: Iterable[_] => ListValue(x.map(valueEntry).toArray)