diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..b58355c4 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,27 @@ +# Linux + Windows test matrix. The Linux leg exists deliberately: the LWS +# symlink-escape test (LWSStorageServletTest.resolveWithinRejectsSymlinkEscape) +# auto-skips on Windows without symlink privileges, so Windows-only development +# never exercises that protection. +name: CI + +on: + push: + branches: [ master, next ] + pull_request: + +jobs: + test: + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest, windows-latest ] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 25 + cache: maven + - name: Build and test + run: mvn -B -Plib clean test diff --git a/RDF_1.1-compliance.md b/RDF_1.1-compliance.md new file mode 100644 index 00000000..87b57e6e --- /dev/null +++ b/RDF_1.1-compliance.md @@ -0,0 +1,64 @@ +# BeakGraph RDF 1.1 Compliance Assessment + +*Assessed 2026-07-02 against [RDF 1.1 Concepts and Abstract Syntax](https://www.w3.org/TR/rdf11-concepts/) (W3C Recommendation, 25 February 2014). + +## Verdict + +Substantially compliant for the core data model as an RDF 1.1 dataset store queried through SPARQL. Three deliberate, documented deviations exist, of which one (numeric literal canonicalization) affects term identity. + +## Compliant areas + +### Term model and positions +- Subjects are IRI/blank node, predicates IRI, objects IRI/blank node/literal, graph names IRI/blank node — enforced with loud failures rather than silent acceptance of generalized RDF (`PositionalDictionaryWriterBuilder.ProcessQuad`, mirrored in `HugeBuildPipeline.countEntityKind`). +- RDF-star quoted triples are rejected with an exception. Correct behavior for an RDF **1.1** store; quoted triples are RDF 1.2 territory. + +### Literals +- Every literal carries its datatype IRI, per RDF 1.1 (simple literals are `xsd:string`; identification inherited from Jena 5). +- `rdf:langString` is fully supported: the `langs`/`langTags` datasets store language tags separately and literals reconstruct term-exact (`MultiTypeDictionaryWriter` / `MultiTypeDictionaryReader`). Tag normalization policy is inherited from Jena's parsers and applied consistently on both write and query paths. +- Ill-typed literals (e.g. `"abc"^^xsd:int`) are valid RDF 1.1 terms and are preserved term-exactly through the strings path instead of being rejected or "repaired". +- Unicode: UTF-8 storage throughout; the front-coded dictionary never splits UTF-16 surrogate pairs (`FCDWriter.commonPrefixLength`). + +### Blank nodes +- Blank node identifiers are not part of the RDF 1.1 abstract syntax; graphs are defined up to isomorphism. Rank-based storage (labels regenerated from dictionary ids on read) and the huge writer's keep-original-labels policy are both compliant under that rule. +- Blank node sharing across named graphs within one dataset is preserved, matching TriG/N-Quads dataset scoping. + +### Datasets +- Default graph plus named graphs, with IRI and blank-node graph names supported. The default graph is held under ARQ's sentinel IRI internally — a representation detail invisible at the SPARQL level (`NodeComparator` even orders the two ARQ sentinels as distinct terms so data that uses them survives). +- Union-graph queries apply RDF-correct set semantics (explicit row dedup in `HDF5Reader.readUnion`). + +### Syntax and query semantics +- Parsing/serialization and SPARQL 1.1 semantics are delegated to Jena 5.x (RIOT/ARQ); BeakGraph sits below as storage. +- The value-ordered dictionary preserves term identity: `NodeComparator` breaks value-equal ties on the exact RDF term (`"1"^^xsd:int` vs `"1"^^xsd:integer` keep distinct ids), and provides a self-consistent total order for timezone-sensitive temporal value spaces that agrees with XSD order wherever XSD order is determinate. + +## Deviations + +### 1. Numeric lexical canonicalization (affects term identity) +`canonicalizeNumericObject` rewrites `xsd:int` / `xsd:long` / `xsd:float` / `xsd:double` **objects** to canonical lexical form at ingest (`"01"^^xsd:int` → `"1"^^xsd:int`). Under RDF 1.1 these are distinct terms with equal values, so: + +- the stored graph is value-preserving and D-entailment-equivalent to the source, but **not isomorphic** when the source contains non-canonical spellings; +- `sameTerm`-style lookups for the original spelling miss. + +This is forced by the storage design: those four datatypes are stored by binary value and the reader regenerates the lexical form, so non-canonical spellings have nowhere to live. Strict term fidelity would require routing non-canonical numerics through the term-exact strings path (larger files, no numeric range pushdown for those literals). If archival round-tripping ever matters, this is the one behavior worth putting behind an off switch. + +Note: `xsd:integer`, `xsd:decimal`, `xsd:dateTime`, booleans, and all other datatypes are stored term-exactly via the strings path. (The `T`-substring on `xsd:dateTime` in `ProcessQuad` feeds string-length *statistics* only; the stored literal is the full lexical form.) + +### 2. Relative IRIs stored unresolved +RDF 1.1 abstract syntax requires absolute IRIs. BeakGraph deliberately stores document-relative references (`DataType.RELATIVE_IRI`) unresolved — parsed against a sentinel base, stripped back to relative form, and resolved against the serving URL at query time (`RelativeIRIResolver`). Until resolution, the in-file graph is not pure abstract syntax. This is an intentional extension serving the LWS/document use case. + +### 3. Injected metadata graphs +The stored dataset is a superset of the source: VoID/SD metadata (`urn:x-beakgraph:void`) always, spatial index graphs (`urn:x-beakgraph:Spatial`, grid-tile URN graphs) when spatial indexing is enabled. Valid RDF, but `GRAPH ?g` enumerates graphs the source never contained — a faithfulness caveat rather than a spec violation. Relatedly, the `numQuads` attribute counts source quads only, excluding injected metadata. + +## Summary table + +| Aspect | Status | +|---|---| +| Term positions (S/P/O/G kinds) | Compliant, enforced | +| Literal datatypes, `xsd:string` identity | Compliant | +| `rdf:langString` / language tags | Compliant | +| Ill-typed literals | Compliant (preserved term-exact) | +| Blank node semantics | Compliant (isomorphism) | +| Datasets / named graphs | Compliant | +| Generalized RDF / RDF-star | Rejected loudly (correct for 1.1) | +| Numeric literal term identity | **Deviation** — canonicalized at ingest | +| Absolute-IRI requirement | **Deviation** — relative IRIs stored, resolved at serving time | +| Dataset faithfulness | **Caveat** — metadata graphs injected | diff --git a/README.md b/README.md index 8e9a13ec..be458976 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ alt="BeakGraph" title="BeakGraph"> -Beakgraph is an [Apache Jena](https://jena.apache.org/) Graph Implmentation of [RDF HDT](https://www.rdfhdt.org/) technology pumped into a [HDF5](https://www.hdfgroup.org/solutions/hdf5/) file and extended to support a full RDF Dataset. +BeakGraph is an [Apache Jena](https://jena.apache.org/) Graph implementation of [RDF HDT](https://www.rdfhdt.org/) technology pumped into a [HDF5](https://www.hdfgroup.org/solutions/hdf5/) file and extended to support a full RDF Dataset.
@@ -30,26 +30,32 @@ Core Library Jar Library mvn -Plib clean package ``` -## Using BreakGraph in your code +## Using BeakGraph in your code ### Creating a BeakGraph from your data -``` -HDF5Writer.Builder() - .setSource(file) - .setSpatial(true) # only needed if GeoSPARQL spatial data is present - .setDestination(dest) + +The source syntax is detected from the file name (Turtle, TriG, N-Quads, +N-Triples; `.gz` compression is handled), so named graphs can be loaded from +quad-capable formats. + +```java +BG.getBGWriterBuilder() + .setSource(new File("mydata.ttl")) + .setDestination(new File("mydata.ttl.h5")) + .setSpatial(true) // only needed if GeoSPARQL spatial data is present + .setFeatures(false) // optional: derive 2D shape features for geometries .build() .write(); ``` ### Using a BeakGraph with Apache Jena -``` - File file = new File("mydata.h5"); - try (BeakGraph bg = BG.getBeakGraph(file)) { - Dataset ds = bg.getDataset(); - ds.getDefaultModel().write(System.out, "NTRIPLE"); - } +```java +File file = new File("mydata.ttl.h5"); +try (BeakGraph bg = BG.getBeakGraph(file)) { + Dataset ds = bg.getDataset(); + ds.getDefaultModel().write(System.out, "NTRIPLE"); +} ``` BeakGraph is a [Apache Jena](https://jena.apache.org/) Graph implementation backed by [HDF5](https://www.hdfgroup.org/solutions/hdf5/). @@ -57,15 +63,28 @@ Beakgraph's HDF5 design is heavily inspired by [RDF HDT](https://www.rdfhdt.org/ ### Limitations -* extremely limited GeoSPARQL support (only sfIntersects) +* BeakGraph files are read-only; the writer builds them in one pass and holds + the working set in RAM (very large datasets may need a correspondingly large + heap). +* GeoSPARQL support covers `geof:sfIntersects` only. It is fully functional: a + recall-safe Hilbert cell-cover index produces candidate geometries and every + candidate is verified with real JTS geometry, so results are exact. Other + GeoSPARQL functions are not implemented. +* `.h5` files written before the spatial-index redesign carry the old + corner-based index entries, which the query side no longer reads - rebuild + them from source for spatial queries (their spatial answers were unsound + anyway; non-spatial queries are unaffected). +* Numeric literals typed `xsd:int`, `xsd:long`, `xsd:float` or `xsd:double` are + stored by value and canonicalized at ingest: `"01"^^xsd:int` is stored - and + matched - as `"1"^^xsd:int`. ### Author's notes The first iteration of BeakGraph was backed by Apache Arrow instead of [HDF5](https://www.hdfgroup.org/solutions/hdf5/). An Apache Arrow version will return. Reasons for this are varied with some of these reasons being just experimentation. -The general idea of BeakGraph is a read-only, searchable, indexed set of binary [sussinct data structures](https://en.wikipedia.org/wiki/Succinct_data_structure) to represent an [RDF Dataset](https://www.w3.org/TR/rdf11-datasets/). -What these sussinct data structures are stored in, is somewhat immaterial, but the choice of container has it's pro and cons. HDF5 treats multi-dimensional arrays as first class citizens, and has a free viewer for -HDF5 files called [HDFView](https://www.hdfgroup.org/download-hdfview/). HDFView providing a nice way to debug the sussinct data structures during development. There are other perks to HDF5 which will become apparent in time. +The general idea of BeakGraph is a read-only, searchable, indexed set of binary [succinct data structures](https://en.wikipedia.org/wiki/Succinct_data_structure) to represent an [RDF Dataset](https://www.w3.org/TR/rdf11-datasets/). +What these succinct data structures are stored in, is somewhat immaterial, but the choice of container has its pros and cons. HDF5 treats multi-dimensional arrays as first class citizens, and has a free viewer for +HDF5 files called [HDFView](https://www.hdfgroup.org/download-hdfview/). HDFView provides a nice way to debug the succinct data structures during development. There are other perks to HDF5 which will become apparent in time. -Support for spatial indexing based on [GeoSPARQL](https://github.com/opengeospatial/ogc-geosparql) is being worked on. +Spatial indexing based on [GeoSPARQL](https://github.com/opengeospatial/ogc-geosparql) is supported for `geof:sfIntersects` (see Limitations above). The full list of containers under consideration are: * [HDF5](https://www.hdfgroup.org/solutions/hdf5/) diff --git a/REVIEW.md b/REVIEW.md new file mode 100644 index 00000000..9087fb42 --- /dev/null +++ b/REVIEW.md @@ -0,0 +1,151 @@ +# BeakGraph code review — full repo + +*Branch `next`, 2026-07-01. Five subsystem deep-reviews (storage/HDF5 I/O, core write pipeline, SPARQL query engine, spatial/Hilbert/geometry, services/pool/CLI) over ~16k LOC; key findings reproduced at runtime and mechanisms re-verified in code. Test suite at review time: 155/155 green (1 skipped). After the fixes below: 211/211 green (1 skipped).* + +**Overall:** the storage core is in genuinely good shape — the bit-packing, rank/select directories, front-coded dictionaries, VByte, and writer↔reader symmetry were traced end-to-end and came back clean. The bugs cluster at the *integration seams*: the comparator contract with Jena's value ordering, the spatial index's variable binding, Jena's `BindingBase` parent invariant, and the HTTP/IRI layer. Two findings are critical; both produce silently wrong query results or failed builds on realistic data. + +## Checklist + +**Critical** +- [x] 1. `NodeComparator` total-order violation (mixed-timezone temporals, durations) +- [x] 2. Constant-subject `sfIntersects` silently returns zero rows + +**High** +- [x] 3. `BindingBG` double-exposure breaks `DISTINCT` +- [x] 4. Negative coordinate space excluded from the spatial index +- [x] 5. Invalid stored geometries unqueryable (`Intersects` throws) +- [x] 6. `absoluteToStorage()` rewrites absolute-stored IRIs into guaranteed misses +- [x] 7. CLI `-src` without `-dest` silently "succeeds" + +**Medium** +- [x] M1. Failed rebuild deletes the previous good artifact +- [x] M2. CRS-prefixed WKT literals get no derived features +- [x] M3. LWS RDF listings emit doubled-slash URIs +- [x] M4. Percent-encoding mismatch 404s encodable filenames +- [x] M5. Link headers/linksets advertise the canonical base, not the live one +- [x] M6. Content negotiation ignores `*/*` (and container POST answered 406) +- [x] M7. `-version`/`-v` prints nothing +- [x] M8. Hilbert-layer external-API traps (`isRangeRelevant`, `getGridCells`) + +**Low** +- [x] L1. `DefaultGraphNode` sentinel collapses with the default-graph sentinel +- [x] L2. Pool never invalidates failed readers; validation checks only `isOpen()` +- [x] L3. Committed-response failures: truncated 200 + double `sendError` ISE +- [x] L4. `SPARQLEndPoint.getDataset()` NPE; `file:///` resolution base leak +- [x] L5. `LWSMetadataGenerator` indexes its own metadata cache +- [x] L6. `SparqlWebPageServlet` classpath lookup lacks dot-segment rejection +- [x] L7. `FileCounter` summary double-subtracts zero-length files +- [x] L8. Feature-math edge cases (thin polygons, INF literals, holed-polygon PCA bias, axis-convention mismatch) +- [x] L9. Mixed `GEOMETRYCOLLECTION` non-polygon members unindexed +- [x] L10. Coordinates ≥ 2³¹ silently alias in the Hilbert curve +- [x] L11. `VoidStats` per-predicate counts overwritten across graph partitions +- [x] L12. RDF-star guard asymmetry (silent dictionary desync landmine) +- [x] L13. `isInTransaction()` unconditionally true +- [x] L14. `BGIteratorSPO_All` ignores a pre-bound graph variable + +**Packaging & design notes** +- [x] P1. Self-contained fat jar by owner decision (verified standalone: convert + serve + query); trade-offs documented +- [x] P2. CI matrix with a Linux leg (symlink-escape test finally runs) +- [x] P3. Dead/misleading API pruned; `FCDWriter` block-size guard +- [x] P4. Large-store performance items (lazy per-graph iterators, graph-id set, negative-lookup caching, compact union dedup key, lazy tiered index) + +## Critical + +- [x] **1. `NodeComparator` is not a total order for mixed timezoned/naive `xsd:dateTime` — breaks every sort and binary search built on it** — `NodeComparator.java:52` + `NodeValue.compareAlways` uses value order when XSD comparison is determinate but falls back to lexical term order when it's *indeterminate* (tz vs no-tz within ±14h). Mixing the two orderings pairwise creates cycles — hand-verified 3-cycle: `"2020-01-02T00:00:00"` < `"2020-01-02T08:00:00+14:00"` < `"2020-01-01T20:00:00Z"` < the first. Reproduced against the built project: a 400-triple file of ordinary mixed local/UTC timestamps **fails to build** (`IllegalStateException: Cannot resolve Object (not in dictionary)`), and on files that do build, binary search missed 2550/3000 stored literals — i.e., **silent missing rows**. Same hazard applies to `xsd:date`/`xsd:time` (also optionally timezoned) and `xsd:duration` (year-month vs day-time indeterminacy). Everything sits on this comparator: `NodeSorter`, `PositionalDictionaryWriter`, the GSPO/GPOS quad sorts and their dedup, `NodeSearch.findPosition`, `searchFAST`, and the `ValueCluster` range pushdown. + *Status: **FIXED** (2026-07-01). Timezone-sensitive temporal spaces (dateTime/date/time/g\*) now order by a UTC-pinned timeline key and durations by (total months, total seconds), both with exact-term tie-breaks; every other value space stays on `compareAlways`, whose cross-space ordering was probe-verified to be a consistent value-space rank. The new order agrees with XSD value order on every determinate pair, so the `ValueCluster` pushdown stays over-inclusive and value-equal terms stay adjacent; healthy existing files keep an identical sort order (no format bump — files with mixed-timezone temporals were already unsearchable and need a rebuild). Regression: `TemporalTotalOrderTest` (comparator totality incl. the reproduced cycles, plus mixed-tz dateTime and mixed-kind duration build/query round-trips — all failing before the fix, the round-trips with the original `Cannot resolve Object` error).* + +- [x] **2. Spatial `sfIntersects` with a constant subject silently returns zero rows** — `PatternMatchBG.java:47-53` + `SpatialIndexIterator.java:76` + When the trigger triple's subject is concrete (`ex:small geo:asWKT ?w FILTER(geof:sfIntersects(?w, …))`), `varToBind` stays the geometry variable, and the spatial iterator binds it to candidate **subject** IDs (`NodeId(id, NodeType.SUBJECT)`). The subsequent pattern solve substitutes the WKT-literal position with a subject URI, which can never match — `ASK` returns false for a geometry that genuinely intersects. Found independently by two reviewers, both runtime-reproduced. + *Status: **FIXED** (2026-07-01). `findTriggerTriple` now only returns a triple with the geometry variable in the object position AND a variable subject, and that subject is the only thing ever seeded (recall-safe for any predicate, since the writer indexes the subject of every wktLiteral-object quad). With a concrete subject the index is skipped and the always-present `sfIntersects` filter verifies — correct, just unaccelerated. This also closes the adjacent latent case where the geometry variable found in a subject/predicate position would have been seeded with subject ids. Regression: `SpatialConcreteSubjectTest` (pinned-subject SELECT/ASK, disjoint negative, VALUES-pre-bound subject, variable-subject control — SELECT/ASK failing before the fix).* + +## High + +- [x] **3. Input bindings are double-exposed through `BindingBG`, breaking `DISTINCT`** — `SolverLibBeak.java:41`, `BindingBG.java:23` + `convert()` copies every var of the incoming binding into the `BindingNodeId` map, and `BindingBG` *also* mounts the original binding as its `BindingBase` parent. Jena assumes a child never re-binds a parent var: `size()` inflates (3 for 2 vars), `vars()` yields duplicates, and `BindingLib.equal` fails on the size mismatch — runtime-reproduced: `SELECT DISTINCT` returned two identical rows. Hits any BGP fed by `VALUES`, `BIND`, subqueries, or joins across graphs. + *Status: **FIXED** (2026-07-01). `BindingBG` now exposes only vars the parent does not bind (`ownVar` filter on `get1/vars1/size1/contains1`); parent-copied vars — including "does not exist" terms — resolve through the parent, which carries the identical term the ids were derived from. Regression: `InputBindingShapeTest` (binding size/vars shape, DISTINCT merge for VALUES and BIND, missing-term ride-through — all failing before the fix).* + +- [x] **4. Negative coordinate space is excluded from the spatial index on both sides** — `SpatialIndexIterator.java:99-104`, writer at `PositionalDictionaryWriterBuilder.java:296-302` + Fully-negative geometries are skipped at build (warning only), straddling geometries are clamped, and query regions with a negative max bail out to zero candidates — probe-verified silent false negatives. The iterator's own recall proof states `0 ≤ coord < 2^31`; fine for slide imagery, but the README advertises GeoSPARQL, where negative lon/lat is most of the planet. + *Status: **FIXED** (2026-07-01). Both sides now clamp bboxes into the non-negative Hilbert domain with the same monotone projection (never skip, never bail): overlap survives the clamp, so recall is preserved — negative-space geometry indexes coarsely at the axis cells and the exact JTS verification removes the clamp's false positives. No format change; old files simply lack the previously-skipped geometries until rebuilt. Regression: `SpatialNegativeCoordinateTest` (negative/straddling regions and geometries, origin-clustering false-positive check — failing before the fix).* + +- [x] **5. Topologically invalid stored geometries are indexed but permanently invisible** — `Intersects.java:74-76` + The verification stage throws on `!isValid()` (Jena drops the row) for exactly the self-intersecting polygons the write path deliberately indexes — `BadPolygonResilienceTest`'s own comment says real pathology data contains them. Probe-verified: a bowtie polygon is never returned by any `sfIntersects` query, with no warning. + *Status: **FIXED** (2026-07-01). Invalid geometries (stored or query region) are repaired with JTS `GeometryFixer.fix` before the intersection test — chosen over `buffer(0)` because it preserves the point set (bowtie → its two triangles) and doesn't discard lines/points. Regression: two new tests in `BadPolygonResilienceTest` (bowtie returned by a query over its real area; invalid query region answers instead of emptying — both failing before the fix).* + +- [x] **6. `absoluteToStorage()` rewrites same-host absolute IRIs into forms the dictionary never contains** — `RelativeIRIResolver.java:68` (applied at `BGSparqlService.java:92`) + Jena's `relativize` happily returns `/other/thing` and `../other.png` (both `isRelative()`), but the writer's storage forms — source-relative IRIs re-relativized against the one-segment sentinel base — can never start with `/` or `../`. So a query naming an absolute same-host IRI that's stored *absolute* gets rewritten into a guaranteed dictionary miss: zero rows while `?s ?p ?o` shows the triple. + *Status: **FIXED** (2026-07-01). `absoluteToStorage` now takes a dictionary-membership probe and rewrites only when the relative form IS stored and the absolute form is NOT (both stored → the exact term the query named wins). `BGSparqlService` supplies the probe from the BG node table (one binary search — deliberately not a per-graph `find()`), via `BGDatasetGraph.getBeakGraph()`; non-BG datasets never rewrite. Regression: `RelativeIRIResolverTest` gained absolute-stored, `../`-form, and both-forms cases; existing relativization tests updated to the probe API.* + +- [x] **7. CLI: `-src` without `-dest` "succeeds" while converting nothing** — `BeakGraphCLI.java:178` + `:126` + `params.dest` is never validated; `FileProcessor.call()` NPEs on `params.dest.toPath()` *before* its try/catch, inside a Future that's discarded at the `engine.submit()` call. Result: no log, no failure count, progress bar advances, summary says "Successful Conversions: N", exit code 0, zero files written. + *Status: **FIXED** (2026-07-01). `main()` refuses `-src` without `-dest` (stderr + usage + exit 1), the missing-src message now also goes to stderr with exit 1, `FileProcessor.call()` counts and logs every failure from the top of the method (nothing escapes into the discarded Future), and a run with failed conversions exits 2. Regression: `BeakGraphCLIConversionTest` (bad source counted while good source still converts; dest-less processing fails loudly into the counter).* + +## Medium + +- [x] **M1. Failed rebuild deletes the previous good artifact** — `HDF5Writer.java:58`. The cleanup deleted the destination even when the failure (parse error, finding #1…) happened before anything was written, destroying a pre-existing valid `.h5`. + *Fixed (2026-07-01): builds go to a sibling `.tmp` file and are atomically moved over the destination only on success; failure cleans up the temp file and never touches the destination, and readers never observe a half-written file at the published path. Regression: `WriteErrorHandlingTest.failedRebuildPreservesThePreviousGoodArtifact` (byte-identical artifact after a failed rebuild; no `.tmp` left behind) — failing before the fix.* +- [x] **M2. CRS-prefixed WKT literals get no derived features** — `PositionalDictionaryWriterBuilder.java:331-336`. `addSpatial` strips the `` prefix (the earlier fix) but `addFeatures` re-read the *unstripped* lexical form; JTS threw, both feature generators skipped. Probe-verified. + *Fixed (2026-07-01): `addFeatures` now receives the CRS-stripped WKT computed once in `addSpatial`. Regression: `FeatureGenerationResilienceTest.crsPrefixedGeometryGetsFeatures` — failing before the fix.* +- [x] **M3. LWS RDF listings emit doubled-slash URIs that 404** — `LWSStorageServlet.java:331` (also 368–388). `BASE` ends with `/` and the canonical remainder starts with `/`; the HTML branch was correct, the machine-readable branches weren't. The paged container's self URI also disagreed in shape with its own `first`/`last`/`prev`/`next` values, which were emitted as plain literals. + *Fixed (2026-07-01): all canonical→live mapping goes through `toLiveUri` (strips the leading slash) — item URIs, per-item property objects, and page URIs; navigation links are now resources with the same URI shape as the page's self URI. Regression: `LWSStorageServletTest.toLiveUriMapsCanonicalUrisWithoutDoubledSlashes`.* +- [x] **M4. Percent-encoding mismatch makes encodable filenames permanently 404** — `LWSStorageServlet.java:243`. Raw `getRequestURI()` was compared against model URIs built from raw filenames — the servlet's own encoded listing links couldn't be followed for `a b.h5`. + *Fixed (2026-07-01): `doGet`/`doPost` decode the raw request URI via `decodePath` (java.net.URI semantics — no `+`→space damage); a malformed URI answers 400. The traversal guard now sees the real decoded path, strengthening it against encoded dot-segments. Regression: `LWSStorageServletTest.decodePathDecodesPercentEncodingAndRejectsGarbage`.* +- [x] **M5. Link headers/linksets advertise the canonical base, not the live one** — `LWSStorageServlet.java:283`. On any non-default port every advertised linkset URI pointed at the wrong server; `/description`'s advertised `.meta` 404'd even on defaults. + *Fixed (2026-07-01): the Link header and the linkset's `anchor`/`up` are mapped to the live base via `toLiveUri`, and `GET /description.meta` is answered directly (the description document is not in the metadata model).* +- [x] **M6. Content negotiation ignores `*/*`** — `LWSStorageServlet.java:287`. `curl` (default `Accept: */*`) got 406 on containers; RFC 9110 requires `*/*` to match. `doPost` also answered 406 where 405 was meant. + *Fixed (2026-07-01): `acceptsHtmlRepresentation` treats `*/*`, `text/*`, and an absent Accept header as HTML-capable; POST to a container answers 405. Regression: `LWSStorageServletTest.wildcardAcceptHeadersGetARepresentationNotA406`.* +- [x] **M7. `-version`/`-v` prints nothing** — `BeakGraphCLI.java:86`. Only reachable inside the `ParameterException` catch; a bare `-v` parsed fine and fell through silently. + *Fixed (2026-07-01): handled on the success path immediately after parse — prints the version and exits 0. Verified by running the CLI (`beakgraph -v` → `beakgraph - Version : 0.15.0`).* +- [x] **M8. Latent external-API traps in the Hilbert layer** — `HilbertPolygon.isRangeRelevant` kept ranges only if the polygon covers the endpoint cell *corner points* (a sub-cell polygon produced an EMPTY cover; diagonal strips lost interior cells), and `PolygonScaler.getGridCells` double-applied the scale factor, disagreeing with the writer-persisted tiles for every s≥1. Neither is on the in-repo query path, but both are the public Polygon→Hilbert API surface. + *Fixed (2026-07-01): `isRangeRelevant` now tests whole-cell intersection across the range's cells (capped at 64 cells; longer ranges are kept unexamined — superset-safe), and `getGridCells` uses the constant `GRIDTILESIZE` on the already-scaled polygon, matching the writer's `generateGridURNs` grid exactly. Regressions: `HilbertSpaceFixesTest` (sub-cell polygon cover, interior-overlap range) and `PolygonScalerGridCellTest` — all failing before the fix.* + +## Low + +- [x] **L1.** `urn:x-arq:DefaultGraphNode` as a data term compares equal to the default-graph sentinel → two distinct terms collapse in the dictionary (`NodeComparator.java:30,83-86`). + *Fixed (2026-07-01): both sentinels still rank first, but the pair is now ordered by exact term (null ≡ the default-graph IRI, which keeps the lowest rank). Regression: `DefaultGraphSentinelTest` (comparator distinctness + sentinel-URI-as-data round-trip) — failing before the fix.* +- [x] **L2.** Pool never `invalidateObject`s after a failure and `validateObject` only checks `isOpen()` (`LWSStorageServlet.java:93`, `BeakGraphPoolFactory.java:47`). + *Fixed (2026-07-01): `handleSparqlQuery` invalidates on any execution/infrastructure failure (via `BGSparqlService.execute`'s new health signal), and `validateObject` probes real mapped data (one dictionary `extract`) under the existing `testOnBorrow`, so an open-but-broken reader is culled on next borrow. Regression: `BeakGraphPoolValidationTest` (pins probe wiring; the open-but-broken scenario isn't portably reproducible on Windows).* +- [x] **L3.** Committed-response error handling: mid-stream failure yields a complete-looking truncated 200, then two `sendError` calls throw `IllegalStateException` (`BGSparqlService.java:132`). + *Fixed (2026-07-01): `execute` throws `QueryExecutionFailedException` when the response is already committed; callers invalidate the reader, touch the response only when uncommitted, and let the exception propagate so the container aborts the connection — the honest signal for a truncated transfer. No servlet-mock harness exists, so this is code-reviewed rather than test-pinned.* +- [x] **L4.** `SPARQLEndPoint.getDataset()` NPEs in single-file mode; single-file `/rdf` uses the local file URI as resolution base, disclosing `file:///…` paths (`SPARQLEndPoint.java:155,134`). + *Fixed (2026-07-01): the endpoint holds its Dataset directly (no registry lookup), and the `/rdf` servlet resolves against the served URL (`BASE_URL + "rdf"`). Requires a running server to test; code-reviewed.* +- [x] **L5.** Re-running `LWSMetadataGenerator` indexes its own previous `beakgraph.ttl.gz`, exposing the raw metadata with `owl:sameAs ` paths (`LWSMetadataGenerator.java:71`). + *Fixed (2026-07-01): the walk skips `CACHE_FILE_NAME`. Regression: `LWSMetadataGeneratorTest` — failing before the fix.* +- [x] **L6.** `SparqlWebPageServlet` concatenates decoded `pathInfo` into classpath lookups with no dot-segment rejection (`SPARQLEndPoint.java:203`). + *Fixed (2026-07-01): paths containing `..` or `\` answer 404 before touching the classpath — containment no longer depends on Jetty compliance defaults. Private servlet, no harness; code-reviewed.* +- [x] **L7.** `FileCounter` summary math double-subtracts zero-length files (can print negative "Successful Conversions") (`FileCounter.java:74`). + *Fixed (2026-07-01): `getSuccessfulConversionCount()` = RDF − failed (empties never entered the RDF count). Regression: `FileCounterTest`.* +- [x] **L8.** Feature-math edge cases: sub-0.5-unit polygons lost ALL features (`ShapeAnalysis.java:234`); zero-area geometry emitted `"INF"^^xsd:double` literals (`ShapeAnalysis.java:98`); holed polygons biased the PCA centroid/axes (`MajorMinor.java:31`); drawn axis (2·σ vertex cloud) disagreed with `PYR.MajorAxisLength` (4·σ pixel cloud). + *Fixed (2026-07-01): raster dimensions come from the internal envelope clamped to ≥1px (degenerate envelopes no longer throw); non-finite feature values are skipped per-feature; `MajorMinor` now computes exact polygon AREA moments (Green's theorem, shell minus holes) with the pyradiomics 4·√λ axis convention — the same definition the raster feature uses. Regression: `FeatureEdgeCasesTest` (thin polygon, bowtie INF, donut centroid — the pre-fix bias reproduced exactly as POINT(1.7778 1.7778) — and the axis convention).* +- [x] **L9.** Mixed `GEOMETRYCOLLECTION`: non-polygon members unindexed when any polygon part exists (`PositionalDictionaryWriterBuilder.java:238`). + *Fixed (2026-07-01): every non-polygonal member gets its expanded-envelope stand-in, per member instead of only as a no-polygon fallback. Regression: `SpatialGeometryCollectionTest` — point-member query failing before the fix.* +- [x] **L10.** Coordinates ≥ 2³¹ silently alias (the Hilbert curve masks to 31 bits, no guard) (`HilbertSpace.java:31`). + *Fixed (2026-07-01): `HilbertSpace.clampToDomain` clamps into [0, 2³¹) on the writer, query, and `Polygon2Hilbert` sides — the same monotone-projection recall argument as the negative-space fix, and no more reliance on the library's undefined masking. Regression: `SpatialHugeCoordinateTest` (pins the defined behavior; the old masking happened to alias consistently in this scenario).* +- [x] **L11.** `VoidStats` per-predicate counts overwritten (not summed) across graph partitions (`VoidStats.java:84`). + *Fixed (2026-07-01): `merge(…, Long::sum)`. Regression: `VoidStatsTest` — 2-vs-5 failing before the fix.* +- [x] **L12.** Guard asymmetry landmine: an RDF-star triple term in the subject path desynchronizes the dictionary silently (`MultiTypeDictionaryWriter.java:190`, builder `:494`). + *Fixed (2026-07-01): the subject branch and `addNodeInternal` now throw like the graph/object branches. Unreachable via current parsers (that's the point of the guard); code-reviewed.* +- [x] **L13.** `isInTransaction()` unconditionally returns true (`BGDatasetGraph.java:253`). + *Fixed (2026-07-01): transaction lifecycle delegates to Jena's `TransactionalLock` (MRSW) — real per-thread state; write transactions still rejected. Regression: `TransactionStateTest` (lifecycle, `Txn.executeRead`, WRITE rejection) — all three failing before the fix.* +- [x] **L14.** `BGIteratorSPO_All` ignores a graph variable pre-bound in the `BindingNodeId` (`BGIteratorSPO_All.java:62`). + *Fixed (2026-07-01): resolves the graph from the binding exactly like the other three iterators behind the dispatcher. Regression: new case in `VariableGraphScanTest` — 0-vs-20 rows failing before the fix.* + +## Packaging & design notes + +- [x] **P1.** The default `lib` profile **shades all dependencies unrelocated into the main artifact** and embeds `log4j2.yml` at the classpath root — duplicate-class conflicts and logging hijack for library consumers. + *Resolved by owner decision (2026-07-01): the BeakGraph jar is DELIBERATELY self-contained — one jar that runs with nothing else on the classpath. The `lib` profile keeps its fat, executable jar (dependencies, `log4j2.yml`, SPARQL web UI, merged service files, Main-Class). Verified end-to-end with the bare artifact from a neutral directory: `-v`, a spatial TTL→HDF5 conversion, and a booted `-endpoint` answering an HTTP SPARQL query. Retained hygiene: the log4j backend + YAML-config Jackson modules stay `optional` in the POM (accurate — main code logs only through slf4j; shade still embeds them). Accepted residual trade-off, documented in the POM: a consumer that ALSO brings its own Jena/JTS sees duplicate unrelocated classes, and the embedded `log4j2.yml` takes over that application's logging. Operational note: package profiles with `clean` (as the README shows) — switching profiles over a stale `target/` can reuse the other profile's jar.* +- [x] **P2.** The LWS symlink-escape test auto-skips on Windows, so that protection is never exercised on this dev machine. + *Fixed (2026-07-01): `.github/workflows/ci.yml` runs `mvn -Plib clean test` on an ubuntu-latest + windows-latest matrix (Temurin 25, Maven cache). The Linux leg exercises the symlink test; the workflow itself needs a push to GitHub to be exercised.* +- [x] **P3.** Dead/misleading API pruned: `eRange` deleted (zero callers; its `join()` was a hull with no overlap precondition); `HDTBitmapDirectory.getId(rank)` (indexed by row, not rank), `rank1`, and `getBitCount` deleted (no callers); `UTIL.getBytes` (matched a nonexistent class name), `subBuffer`, `skipNullTerminatedStrings`, `readNullTerminatedString` deleted (no callers); `BindingNodeId.putAll` deleted (would have thrown if ever used); `FCDWriter` now rejects `blockSize < 2` at construction instead of writing an unreadable dictionary. +- [x] **P4.** Performance items for large stores, all addressed (2026-07-01): + - `containsGraph`/`isGraph` answers from a lazily-built id set instead of decoding the whole columnar graphs list per call; + - negative node-table lookups are cached (the store is immutable, so absence is permanent — a repeated foreign-term miss no longer re-runs two binary searches); + - `readUnion` chains per-graph iterators lazily and dedups on a record of three primitive longs instead of a boxed `List` (the set itself is inherent to union set-semantics); + - the unbound-graph scan (`BGIteratorMaster`), `find(ANY,…)`, and `findNG(ANY,…)` build each graph's sub-iterator lazily as iteration reaches it, and the master scan pre-binds the graph id from the columnar list (no `extract→locate` round trip; the rare `?g`-in-predicate shape keeps the substitution+compatibility-filter path because predicate ids live in an isolated id-space); + - the dictionary's tiered search index builds lazily on first search (volatile-published, race-benign) instead of doing `numEntries/1024` full extracts at every file-open. + *All behavior pinned by the existing suite (union/DISTINCT/variable-graph-scan/concurrent-read/reader-lifecycle tests) — 211/211 green after the changes.* + +## Verified sound + +Worth stating, since it's most of the risk surface: MSB-first bit-packing round-trips at all edge widths, the rank/select directory math (pinned exhaustively by `RankSelectDirectoryTest`), FCD front-coding including UTF-16 prefix handling and surrogate back-off, VByte, the L0 padding/select1 addressing scheme, the index-selection matrix across all four iterators, repeated-variable handling via `putCompatible`, `ValueCluster` pushdown (over-inclusive only, filter always re-applied), reader thread-safety (absolute-position reads, immutable dictionaries), pool borrow/return discipline at the single borrow site, and HDF5 resource cleanup on error paths. diff --git a/pom.xml b/pom.xml index 2fc8bc46..45a6c913 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.ebremer BeakGraph BeakGraph - 0.15.0 + 0.16.0 jar Library for creating indexed binary storages for Apache Jena Graphs and Datasets @@ -46,9 +46,28 @@ 4.5.0 - 3.0.0-beta2 + + 2.25.2 2.0.17 0.10.0 + + 1.14.3-1.5.10 + 2.1.1 + + linux-x86_64 3.0 0.2.3 1.20.0 @@ -139,11 +158,6 @@ jackson-databind 2.21.1 - - org.apache.logging.log4j - log4j-config-yaml - ${log4j.version} - io.airlift aircompressor-v3 @@ -191,7 +205,7 @@ maven-compiler-plugin ${maven-compiler-plugin.version} - 25 + ${java.version} true ${java.home}/bin/javac @@ -210,11 +224,28 @@ lib - true + + + !thin + BeakGraph-${project.version} + org.apache.maven.plugins maven-shade-plugin @@ -303,9 +334,15 @@ --no-fallback --enable-url-protocols=https -H:+AddAllCharsets + -march=native -J-Xmx32G -H:IncludeResources=META-INF/sparql/.* + -H:IncludeResources=beakgraph.png + -H:IncludeResources=beakgraph-version.properties -H:IncludeResources=log4j2.yml -H:IncludeResources=META-INF/services/.* -H:IncludeResourceBundles=org.apache.jena.ext.xerces.impl.msg.XMLSchemaMessages @@ -363,6 +400,121 @@ + + + + + hdf5-backend-javacpp + + + !hdf5.ffm + + + + + org.bytedeco + hdf5-platform + ${hdf5.javacpp.version} + true + + + + + + hdf5-backend-ffm + + + hdf5.ffm + + + + + github-hdfgroup + HDF Group GitHub Packages + https://maven.pkg.github.com/HDFGroup/hdf5 + true + false + + + + + org.hdfgroup + hdf5-java-ffm + ${hdf5.ffm.version} + ${hdf5.ffm.classifier} + true + + + + + + hdf5-ffm-windows + + windows + + + windows-x86_64 + + + + hdf5-ffm-linux + + linuxamd64 + + + linux-x86_64 + + + + hdf5-ffm-macos-intel + + macx86_64 + + + macos-x86_64 + + + + hdf5-ffm-macos-arm + + macaarch64 + + + macos-aarch64 + + @@ -375,17 +527,19 @@ org.ejml ejml-simple + com.fasterxml.jackson.dataformat jackson-dataformat-yaml + true com.fasterxml.jackson.core jackson-databind - - - org.apache.logging.log4j - log4j-config-yaml + true me.tongfei @@ -421,10 +575,12 @@ org.apache.logging.log4j log4j-api + true org.apache.logging.log4j log4j-core + true org.slf4j @@ -433,12 +589,16 @@ org.apache.logging.log4j log4j-slf4j2-impl + true io.jhdf jhdf ${jhdf.version} + org.jcommander jcommander @@ -447,10 +607,12 @@ org.apache.logging.log4j log4j-layout-template-json + true org.apache.logging.log4j log4j-1.2-api + true com.github.davidmoten diff --git a/src/main/java/com/ebremer/beakgraph/BG.java b/src/main/java/com/ebremer/beakgraph/BG.java index 761ab2cb..274f1487 100644 --- a/src/main/java/com/ebremer/beakgraph/BG.java +++ b/src/main/java/com/ebremer/beakgraph/BG.java @@ -19,11 +19,17 @@ public static HDF5Writer.Builder getBGWriterBuilder() { public static BeakGraph getBeakGraph(File file) throws IOException { HDF5Reader reader = new HDF5Reader(file); - return new BeakGraph(reader); + try { + return new BeakGraph(reader); + } catch (RuntimeException | Error e) { + // The reader pins the mapped file; if BeakGraph construction fails it + // must be released here or nobody ever can. + try { reader.close(); } catch (Exception ignore) {} + throw e; + } } - + public static BeakGraph getBeakGraph(Path path) throws IOException { - HDF5Reader reader = new HDF5Reader(path); - return new BeakGraph(reader); + return getBeakGraph(path.toFile()); } } diff --git a/src/main/java/com/ebremer/beakgraph/cmdline/BeakGraphCLI.java b/src/main/java/com/ebremer/beakgraph/cmdline/BeakGraphCLI.java index fbac92b8..118c4944 100644 --- a/src/main/java/com/ebremer/beakgraph/cmdline/BeakGraphCLI.java +++ b/src/main/java/com/ebremer/beakgraph/cmdline/BeakGraphCLI.java @@ -5,6 +5,7 @@ import com.ebremer.beakgraph.Params; import com.ebremer.beakgraph.core.fuseki.SPARQLEndPoint; import com.ebremer.beakgraph.hdf5.writers.HDF5Writer; +import com.ebremer.beakgraph.huge.HugeHDF5Writer; import java.io.FileNotFoundException; import java.io.IOException; import java.nio.file.Files; @@ -35,6 +36,11 @@ public class BeakGraphCLI { private final FileCounter fc; private Parameters params; + /** Conversion counters for this run (failed conversions drive the exit code). */ + public FileCounter getFileCounter() { + return fc; + } + public BeakGraphCLI(Parameters params) { JenaSystem.init(); this.params = params; @@ -58,6 +64,13 @@ public static void main(String[] args) throws FileNotFoundException, IOException if (args.length != 0) { try { jc.parse(args); + if (params.version) { + // Must be handled on the SUCCESS path: version was previously + // printed only inside the ParameterException catch, so a plain + // "-v" parsed fine, matched no branch, and printed nothing. + System.out.println("beakgraph - Version : " + Params.VERSION); + System.exit(0); + } if (params.help) { jc.usage(); System.exit(0); @@ -76,18 +89,34 @@ public static void main(String[] args) throws FileNotFoundException, IOException Thread.currentThread().interrupt(); } } else if (params.src != null && params.src.exists()) { + if (params.dest == null) { + // Without this guard every FileProcessor NPEs inside a + // discarded Future: nothing converts, nothing is logged, + // and the run exits 0 reporting success. + System.err.println("Error: -dest is required with -src"); + jc.usage(); + System.exit(1); + } JenaSystem.init(); BeakGraphCLI bg = new BeakGraphCLI(params); bg.traverse(); + if (bg.fc.getFailedConversionFileCount() > 0) { + System.exit(2); + } } else if (params.src != null) { - System.out.println("Source does not exist! " + params.src); + System.err.println("Error: -src does not exist: " + params.src); + System.exit(1); } } } catch (ParameterException ex) { if (params.version) { System.out.println("beakgraph - Version : " + Params.VERSION); } else { - System.out.println(ex.getMessage()); + // Bad arguments are an error: say so on stderr and exit non-zero + // (scripts used to see a successful exit 0 for a failed run). + System.err.println(ex.getMessage()); + jc.usage(); + System.exit(1); } } } @@ -130,6 +159,9 @@ public void traverse() { try { Thread.sleep(1000); } catch (InterruptedException ex) { + // Restore the flag and stop polling; the executor keeps draining. + Thread.currentThread().interrupt(); + break; } } } catch (IOException ex) { @@ -168,23 +200,44 @@ public FileProcessor(Path src, FileCounter fc) { @Override public Model call() { - Path dest = mapToDestinationWithNewExtension(src, params.src.toPath(), params.dest.toPath(), "h5"); - if (dest.toFile().exists() && dest.toFile().length() > 0) { - return null; - } - dest.getParent().toFile().mkdirs(); + // The Future from engine.submit() is never inspected, so anything + // escaping this method is swallowed silently by FutureTask and the + // file still counts as a success. EVERYTHING - including destination + // mapping - must be counted and logged inside this catch. try { - HDF5Writer.Builder() - .setSource(src.toFile()) - .setDestination(dest.toFile()) - .setSpatial(params.spatial) - .setFeatures(params.features) - .build() - .write(); + Path dest = mapToDestinationWithNewExtension(src, params.src.toPath(), params.dest.toPath(), "h5"); + if (dest.toFile().exists() && dest.toFile().length() > 0) { + return null; + } + dest.getParent().toFile().mkdirs(); + if (params.huge) { + // Disk-based build: same output format, but sorting/indexing + // spill to a workspace instead of the heap. Note the huge + // writer needs the native HDF5 backend on the classpath (the + // hdf5-backend-* profiles); with -threads N, N builds run + // concurrently, each with its own workspace. + HugeHDF5Writer.Builder builder = HugeHDF5Writer.Builder() + .setSource(src.toFile()) + .setDestination(dest.toFile()) + .setSpatial(params.spatial) + .setFeatures(params.features); + if (params.workdir != null) { + params.workdir.mkdirs(); + builder.setWorkDirectory(params.workdir.toPath()); + } + builder.build().write(); + } else { + HDF5Writer.Builder() + .setSource(src.toFile()) + .setDestination(dest.toFile()) + .setSpatial(params.spatial) + .setFeatures(params.features) + .build() + .write(); + } } catch (Exception ex) { fc.incrementFailedConversionFileCount(); - logger.error("Failed to convert {} -> {}", src, dest, ex); - throw new RuntimeException("Failed to convert " + src + " -> " + dest, ex); + logger.error("Failed to convert {}", src, ex); } return null; } diff --git a/src/main/java/com/ebremer/beakgraph/cmdline/FileCounter.java b/src/main/java/com/ebremer/beakgraph/cmdline/FileCounter.java index 8418806d..140a9252 100644 --- a/src/main/java/com/ebremer/beakgraph/cmdline/FileCounter.java +++ b/src/main/java/com/ebremer/beakgraph/cmdline/FileCounter.java @@ -49,6 +49,16 @@ public long getFailedConversionFileCount() { return failedConversionFileCount.get(); } + /** + * RDF files that converted (or already had a non-empty destination). + * Zero-length files are NOT subtracted here: the traverse filter rejects + * them before they are ever counted as RDF files, so subtracting them + * again understated the summary and could drive it negative. + */ + public long getSuccessfulConversionCount() { + return getRDFFileCount() - getFailedConversionFileCount(); + } + @Override public String toString() { StringBuilder sb = new StringBuilder(); @@ -71,7 +81,7 @@ public String toString() { """, getZeroFileCount(), getFailedConversionFileCount(), - getRDFFileCount()-getZeroFileCount()-getFailedConversionFileCount() + getSuccessfulConversionCount() )); return sb.toString(); } diff --git a/src/main/java/com/ebremer/beakgraph/cmdline/Parameters.java b/src/main/java/com/ebremer/beakgraph/cmdline/Parameters.java index 6c180e49..71ddf798 100644 --- a/src/main/java/com/ebremer/beakgraph/cmdline/Parameters.java +++ b/src/main/java/com/ebremer/beakgraph/cmdline/Parameters.java @@ -27,9 +27,20 @@ public class Parameters { @Parameter(names = {"-spatial"}, converter = BooleanConverter.class) public boolean spatial = false; - + @Parameter(names = {"-features"}, converter = BooleanConverter.class) public boolean features = false; + + @Parameter(names = {"-huge"}, converter = BooleanConverter.class, + description = "Use the disk-based writer (com.ebremer.beakgraph.huge): sorts and " + + "indexes on disk instead of RAM, for sources too large for the heap") + public boolean huge = false; + + @Parameter(names = "-workdir", + description = "Workspace directory for -huge spill files; needs free space on the " + + "order of a few times the uncompressed source (default: each " + + "destination file's directory)", required = false) + public File workdir = null; @Parameter(names = {"-version","-v"}, converter = BooleanConverter.class) public boolean version = false; diff --git a/src/main/java/com/ebremer/beakgraph/core/BGDatasetGraph.java b/src/main/java/com/ebremer/beakgraph/core/BGDatasetGraph.java index 46e9bc3a..11a7229e 100644 --- a/src/main/java/com/ebremer/beakgraph/core/BGDatasetGraph.java +++ b/src/main/java/com/ebremer/beakgraph/core/BGDatasetGraph.java @@ -1,35 +1,73 @@ package com.ebremer.beakgraph.core; import com.ebremer.beakgraph.hdf5.jena.BindingNodeId; -import java.io.IOException; +import com.ebremer.beakgraph.hdf5.jena.OpExecutorBG; import java.util.ArrayList; +import java.util.Collections; import java.util.Iterator; import java.util.List; -import org.apache.commons.collections4.iterators.IteratorChain; import org.apache.jena.graph.Graph; import org.apache.jena.graph.Node; import org.apache.jena.graph.Triple; -import org.apache.jena.query.DatasetFactory; import org.apache.jena.query.ReadWrite; import org.apache.jena.query.TxnType; import org.apache.jena.riot.system.PrefixMap; +import org.apache.jena.riot.system.PrefixMapFactory; import org.apache.jena.sparql.core.DatasetGraphBase; import org.apache.jena.sparql.core.Quad; +import org.apache.jena.sparql.core.Transactional; +import org.apache.jena.sparql.core.TransactionalLock; import org.apache.jena.sparql.core.Var; +import org.apache.jena.sparql.engine.main.QC; +import org.apache.jena.sparql.pfunction.PropertyFunctionRegistry; +import org.apache.jena.sparql.util.Context; import org.apache.jena.util.iterator.WrappedIterator; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.jena.vocabulary.RDFS; /** * * @author erich */ public class BGDatasetGraph extends DatasetGraphBase { - private static final Logger logger = LoggerFactory.getLogger(BGDatasetGraph.class); private final BeakGraph bg; - + // Per-dataset execution wiring (the TDB pattern): the engine merges this context + // over the global ARQ context when a query runs against this dataset, so BG's + // OpExecutor and property-function scoping apply here and ONLY here - never to + // other datasets in the JVM. + private final Context context = Context.create(); + // Tracks per-thread read-transaction lifecycle (begin/commit/abort/end pairing); + // the store is immutable, so the lock only provides honest bookkeeping. + private final Transactional txn = TransactionalLock.createMRSW(); + + /** + * The standard property-function registry minus rdfs:member. BG stores use + * rdfs:member as a plain stored predicate; Jena's container-membership property + * function would rewrite those patterns into rdf:_1/rdf:_2... lookups and answer + * nothing. Scoped here per dataset - the global registry is left untouched. + */ + private static final class BGPropertyFunctions { + static final PropertyFunctionRegistry INSTANCE = build(); + private static PropertyFunctionRegistry build() { + PropertyFunctionRegistry global = PropertyFunctionRegistry.get(); + PropertyFunctionRegistry reg = new PropertyFunctionRegistry(); + global.keys().forEachRemaining(uri -> { + if (!RDFS.member.getURI().equals(uri)) { + reg.put(uri, global.get(uri)); + } + }); + return reg; + } + } + public BGDatasetGraph(BeakGraph g) { this.bg = g; + QC.setFactory(context, OpExecutorBG.opExecFactoryBG); + PropertyFunctionRegistry.set(context, BGPropertyFunctions.INSTANCE); + } + + @Override + public Context getContext() { + return context; } @Override @@ -47,13 +85,9 @@ public Graph getDefaultGraph() { } @Override - public Graph getGraph(Node node) { - try { - return new BeakGraph(node, bg.getReader()); - } catch (IOException ex) { - logger.error("Failed to open named graph {}", node, ex); - } - return Graph.emptyGraph; + public Graph getGraph(Node node) { + // A non-owning view over the shared reader (closing it is a no-op). + return new BeakGraph(node, bg.getReader()); } @Override @@ -75,6 +109,12 @@ public Iterator listGraphNodes() { @Override public boolean containsGraph(Node graphNode) { + // The union and default graphs are synthetic names that never appear in + // the stored graphs list, but ARQ gates GRAPH execution on this + // method - they must answer true here (the TDB convention). + if (Quad.isUnionGraph(graphNode) || Quad.isDefaultGraph(graphNode)) { + return true; + } return bg.getReader().containsGraph(graphNode); } @@ -93,9 +133,20 @@ public Iterator find(Node g, Node s, Node p, Node o) { @Override public Iterator findNG(Node g, Node s, Node p, Node o) { - // Same logic as find, but specifically for Named Graphs. - // Since HDF5Reader includes all graphs in listGraphNodes, logic is identical. - return find(g, s, p, o); + // findNG matches NAMED graphs only - unlike find(ANY,...), the default + // graph's quads are excluded. + if (Quad.isUnionGraph(g)) { + return findInSpecificGraph(g, s, p, o); // read() handles the union semantics + } + if (g == null || Node.ANY.equals(g)) { + // Lazy per-graph chaining - see findInAnyGraph. + return org.apache.jena.atlas.iterator.Iter.flatMap(listGraphNodes(), + gn -> findInSpecificGraph(gn, s, p, o)); + } + if (Quad.isDefaultGraph(g)) { + return Collections.emptyIterator(); + } + return findInSpecificGraph(g, s, p, o); } private Iterator findInSpecificGraph(Node g, Node s, Node p, Node o) { @@ -124,28 +175,25 @@ private Iterator findInSpecificGraph(Node g, Node s, Node p, Node o) { } private Iterator findInAnyGraph(Node s, Node p, Node o) { - // 1. Default Graph - Iterator defaultGraphIter = findInSpecificGraph(Quad.defaultGraphIRI, s, p, o); - - // 2. All Named Graphs - Iterator graphs = listGraphNodes(); - List> iterators = new ArrayList<>(); - iterators.add(defaultGraphIter); - - while(graphs.hasNext()) { - Node graphNode = graphs.next(); - // Skip default if it appears in the list to avoid duplicates - if (!graphNode.equals(Quad.defaultGraphIRI)) { - iterators.add(findInSpecificGraph(graphNode, s, p, o)); - } - } - - return new IteratorChain<>(iterators); + // Lazily chain the default graph and every named graph: constructing + // each graph's iterator up front paid its index searches before the + // first quad came back, and spatial stores hold thousands of tile + // graphs. Skip default if it appears in the graph list (duplicates). + Iterator graphs = org.apache.jena.atlas.iterator.Iter.concat( + List.of(Quad.defaultGraphIRI).iterator(), + org.apache.jena.atlas.iterator.Iter.filter(listGraphNodes(), + gn -> !gn.equals(Quad.defaultGraphIRI))); + return org.apache.jena.atlas.iterator.Iter.flatMap(graphs, gn -> findInSpecificGraph(gn, s, p, o)); } + // One mutable prefix map per dataset: the old implementation built a whole + // fresh in-memory dataset on EVERY call and returned its (disconnected) + // prefixes, so registrations silently vanished between calls. + private final PrefixMap prefixMap = PrefixMapFactory.create(); + @Override public PrefixMap prefixes() { - return DatasetFactory.createGeneral().asDatasetGraph().prefixes(); + return prefixMap; } // --- Minimal Transaction Support (Read-Only) --- @@ -158,11 +206,13 @@ public boolean supportsTransactions() { @Override public void begin(TxnType type) { if (type == TxnType.WRITE) throw new UnsupportedOperationException("Write transactions not supported"); + txn.begin(type); } @Override public void begin(ReadWrite readWrite) { if (readWrite == ReadWrite.WRITE) throw new UnsupportedOperationException("Write transactions not supported"); + txn.begin(readWrite); } @Override @@ -172,32 +222,35 @@ public boolean promote(Promote mode) { @Override public void commit() { - // No-op for read-only + txn.commit(); } @Override public void abort() { - // No-op for read-only + txn.abort(); } @Override public void end() { - // No-op for read-only + txn.end(); } @Override public ReadWrite transactionMode() { - return ReadWrite.READ; + return txn.transactionMode(); } @Override public TxnType transactionType() { - return TxnType.READ; + return txn.transactionType(); } @Override public boolean isInTransaction() { - // Always behave as if in a transaction or allow access - return true; + // Real per-thread state, not an unconditional "true": lying that a + // transaction is always active masked mispaired begin/end in callers. + // The data itself is immutable, so reads need no lock protection - the + // delegate exists purely to track lifecycle honestly. + return txn.isInTransaction(); } } diff --git a/src/main/java/com/ebremer/beakgraph/core/BeakGraph.java b/src/main/java/com/ebremer/beakgraph/core/BeakGraph.java index ee079526..11b8aa12 100644 --- a/src/main/java/com/ebremer/beakgraph/core/BeakGraph.java +++ b/src/main/java/com/ebremer/beakgraph/core/BeakGraph.java @@ -1,11 +1,8 @@ package com.ebremer.beakgraph.core; import com.ebremer.beakgraph.hdf5.jena.BGReader; -import com.ebremer.beakgraph.hdf5.jena.OpExecutorBG; -import com.ebremer.beakgraph.hdf5.jena.QueryEngineBeak; import com.ebremer.beakgraph.hdf5.jena.StageGeneratorDirectorBG; import com.ebremer.beakgraph.turbo.Spatial; -import java.io.IOException; import java.net.URI; import java.util.stream.Stream; import org.apache.jena.graph.Node; @@ -17,7 +14,6 @@ import org.apache.jena.shared.AddDeniedException; import org.apache.jena.shared.DeleteDeniedException; import org.apache.jena.sparql.core.Quad; -import org.apache.jena.sparql.engine.main.QC; import org.apache.jena.sparql.engine.main.StageBuilder; import org.apache.jena.sparql.engine.main.StageGenerator; import org.apache.jena.sparql.engine.optimizer.reorder.ReorderLib; @@ -37,6 +33,10 @@ public class BeakGraph extends GraphBase implements AutoCloseable { private static volatile boolean initialized = false ; private final Node namedgraph; private final BGReader reader; + // Whether this instance owns (and on close() must close) the reader. Named-graph + // views handed out by BGDatasetGraph.getGraph share the dataset's reader; closing + // such a view must not shut the storage under every other graph of the dataset. + private final boolean ownsReader; private static final Logger logger = LoggerFactory.getLogger(BeakGraph.class); private final URI uri; // Lazily-computed triple count for this graph. -1 = not yet computed; the graph @@ -50,19 +50,20 @@ public class BeakGraph extends GraphBase implements AutoCloseable { Spatial.init(); } - public BeakGraph(BGReader reader, URI uri) throws IOException { + public BeakGraph(BGReader reader, URI uri) { this(reader, uri, uri); } - - public BeakGraph(BGReader reader, URI uri, URI base) throws IOException { + + public BeakGraph(BGReader reader, URI uri, URI base) { logger.trace("BeakGraph -> {}", uri.toString()); init(); this.uri = uri; this.reader = reader; this.namedgraph = Quad.defaultGraphIRI; + this.ownsReader = true; } - public BeakGraph(BGReader reader) throws IOException { + public BeakGraph(BGReader reader) { this( reader, reader.getURI(), null); } @@ -70,12 +71,13 @@ public URI getURI() { return uri; } - public BeakGraph(Node namedgraph, BGReader reader) throws IOException { + public BeakGraph(Node namedgraph, BGReader reader) { logger.trace("Create a SubBeakGraph -> {}", namedgraph); init(); this.uri = reader.getURI(); this.reader = reader; this.namedgraph = namedgraph; + this.ownsReader = false; // a view over a reader owned by the dataset } private static void init() { @@ -86,10 +88,16 @@ private static void init() { if ( initialized ) { return ; } - initialized = true ; - QC.setFactory(ARQ.getContext(), OpExecutorBG.opExecFactoryBG); - QueryEngineBeak.register(); + // The OpExecutor factory is wired per-dataset (BGDatasetGraph's own + // context), NOT into the global ARQ context: a global factory would + // change query execution for every other dataset in the JVM. Only the + // stage-generator director is global - the standard Jena pattern - + // because it dispatches on the active graph's type and delegates + // everything that is not a BeakGraph. wireIntoExecution() ; + // Publish only after wiring succeeded, so a failure here is retried by + // the next caller instead of leaving the JVM half-wired forever. + initialized = true ; } } @@ -99,6 +107,9 @@ public BGReader getReader() { @Override public void close() { + if (!ownsReader) { + return; // closing a named-graph view must not close the shared reader + } try { reader.close(); } catch (Exception ex) { diff --git a/src/main/java/com/ebremer/beakgraph/core/EmptyDictionaryWriter.java b/src/main/java/com/ebremer/beakgraph/core/EmptyDictionaryWriter.java index 99ac1f7a..5bd28e0e 100644 --- a/src/main/java/com/ebremer/beakgraph/core/EmptyDictionaryWriter.java +++ b/src/main/java/com/ebremer/beakgraph/core/EmptyDictionaryWriter.java @@ -1,6 +1,5 @@ package com.ebremer.beakgraph.core; -import com.ebremer.beakgraph.core.AbstractDictionary; import io.jhdf.api.WritableGroup; import java.util.ArrayList; import java.util.List; @@ -8,6 +7,9 @@ import org.apache.jena.graph.Node; /** + * Null-object dictionary for a section with no entries (e.g. a source with no + * literals). Every operation answers honestly for an empty dictionary instead + * of throwing template "Not supported yet." exceptions. * * @author Erich Bremer */ @@ -33,16 +35,18 @@ public long locate(Node element) { @Override public Node extract(long id) { - throw new UnsupportedOperationException("Not supported yet."); // Generated from nbfs://nbhost/SystemFileSystem/Templates/Classes/Code/GeneratedMethodBody + throw new IllegalArgumentException("id [" + id + "]: empty dictionary has no entries"); } @Override public Stream streamNodes() { - throw new UnsupportedOperationException("Not supported yet."); // Generated from nbfs://nbhost/SystemFileSystem/Templates/Classes/Code/GeneratedMethodBody + return Stream.empty(); } @Override public long search(Node element) { - throw new UnsupportedOperationException("Not supported yet."); // Generated from nbfs://nbhost/SystemFileSystem/Templates/Classes/Code/GeneratedMethodBody + // Nothing matches; the insertion point in an empty 1-based dictionary is 1, + // encoded -(insertion)-1 per the search contract. + return -2; } } diff --git a/src/main/java/com/ebremer/beakgraph/core/VoidStats.java b/src/main/java/com/ebremer/beakgraph/core/VoidStats.java index 1b688de8..395dc63e 100644 --- a/src/main/java/com/ebremer/beakgraph/core/VoidStats.java +++ b/src/main/java/com/ebremer/beakgraph/core/VoidStats.java @@ -81,7 +81,11 @@ static VoidStats load(BGReader reader) { for (Map.Entry e : partitionPredicate.entrySet()) { Long n = resourceTriples.get(e.getKey()); if (n != null) { - predicateCount.put(e.getValue(), n); + // One partition per (graph, predicate) pair: the same predicate can + // appear in several graph descriptions, so counts must SUM. A plain + // put kept whichever partition iteration visited last, feeding the + // reorder cost model a per-graph count against a dataset-wide total. + predicateCount.merge(e.getValue(), n, Long::sum); total += n; } } diff --git a/src/main/java/com/ebremer/beakgraph/core/fuseki/BGSparqlService.java b/src/main/java/com/ebremer/beakgraph/core/fuseki/BGSparqlService.java index 47e0cbf3..9490fc28 100644 --- a/src/main/java/com/ebremer/beakgraph/core/fuseki/BGSparqlService.java +++ b/src/main/java/com/ebremer/beakgraph/core/fuseki/BGSparqlService.java @@ -1,10 +1,16 @@ package com.ebremer.beakgraph.core.fuseki; +import com.ebremer.beakgraph.core.BGDatasetGraph; +import com.ebremer.beakgraph.core.NodeTable; +import com.ebremer.beakgraph.hdf5.jena.NodeId; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; +import org.apache.jena.graph.Node; import org.apache.jena.query.Dataset; import org.apache.jena.query.Query; import org.apache.jena.query.QueryExecution; @@ -29,6 +35,37 @@ */ public final class BGSparqlService { + private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BGSparqlService.class); + + /** Maximum accepted SPARQL request body. Real queries are tiny; an unbounded + * readAllBytes lets a single request allocate arbitrary heap. */ + static final int MAX_QUERY_BODY_BYTES = 1 << 20; // 1 MiB + + /** Hard wall-clock limit per query so one pathological query cannot pin the server. */ + private static final long QUERY_TIMEOUT_SECONDS = 30; + + /** Thrown when a POST body exceeds {@link #MAX_QUERY_BODY_BYTES}; callers map it to HTTP 413. */ + public static final class QueryBodyTooLargeException extends IOException { + QueryBodyTooLargeException(String message) { + super(message); + } + } + + /** + * Thrown when execution failed AFTER the response was committed: a 200 and + * part of the body are already on the wire, so the only honest signal left + * is an aborted transfer. Callers must NOT touch the response again (a + * sendError would throw IllegalStateException) and should let this + * propagate so the container closes the connection without completing the + * response - a client then sees a truncated transfer instead of a + * complete-looking 200 with silently missing rows. + */ + public static final class QueryExecutionFailedException extends IOException { + QueryExecutionFailedException(String message, Throwable cause) { + super(message, cause); + } + } + private BGSparqlService() { } @@ -43,18 +80,51 @@ public static String extractQuery(HttpServletRequest req) throws IOException { return req.getParameter("query"); } try (InputStream in = req.getInputStream()) { - return new String(in.readAllBytes(), StandardCharsets.UTF_8); + return readBody(in, MAX_QUERY_BODY_BYTES); } } return null; } + /** Read at most {@code max} bytes as UTF-8; reject anything larger. */ + static String readBody(InputStream in, int max) throws IOException { + byte[] body = in.readNBytes(max + 1); + if (body.length > max) { + throw new QueryBodyTooLargeException("SPARQL query body exceeds " + max + " bytes"); + } + return new String(body, StandardCharsets.UTF_8); + } + + /** + * Dictionary-membership probe backing the relativization rewrite decision. + * BG datasets answer from the node table (a binary search - deliberately + * NOT a find() over the dataset, which fans out across every named graph). + * Any other dataset has no relative-stored IRIs, so nothing is rewritten. + */ + private static Predicate storedTermProbe(Dataset ds) { + if (ds.asDatasetGraph() instanceof BGDatasetGraph bgd) { + NodeTable nodeTable = bgd.getBeakGraph().getReader().getNodeTable(); + return n -> { + NodeId id = nodeTable.getNodeIdForNode(n); + return id != null && !NodeId.isDoesNotExist(id); + }; + } + return n -> false; + } + /** * Execute {@code queryStr} against {@code ds} and serialize the result, * resolving relative IRIs against {@code baseURI} (the URL/URI the data is - * served from). On any failure an HTTP 400 is written. + * served from). Client-side problems (parse errors, unsupported query + * types) are answered with 400 and still return true - the reader is fine. + * + * @return true when the underlying reader behaved; false when execution + * failed (a 500 was written) - pooled callers should invalidate + * their instance rather than return it. + * @throws QueryExecutionFailedException when execution failed after the + * response was committed (see that exception's contract) */ - public static void execute(Dataset ds, String queryStr, String baseURI, + public static boolean execute(Dataset ds, String queryStr, String baseURI, String acceptHeader, HttpServletResponse resp) throws IOException { String accept = (acceptHeader == null) ? "" : acceptHeader.toLowerCase(); try { @@ -63,9 +133,10 @@ public static void execute(Dataset ds, String queryStr, String baseURI, // Relativize document IRIs the query names so they match the // dictionary. Skip the whole-query walk when there is no base. Query execQuery = resolver.isActive() - ? QueryTransformOps.transform(query, resolver.absoluteToStorage()) + ? QueryTransformOps.transform(query, resolver.absoluteToStorage(storedTermProbe(ds))) : query; - try (QueryExecution qexec = QueryExecution.dataset(ds).query(execQuery).build()) { + try (QueryExecution qexec = QueryExecution.dataset(ds).query(execQuery) + .timeout(QUERY_TIMEOUT_SECONDS, TimeUnit.SECONDS).build()) { if (execQuery.isSelectType()) { ResultSet rs = resolver.resolve(qexec.execSelect()); if (accept.contains("json")) { @@ -99,8 +170,23 @@ public static void execute(Dataset ds, String queryStr, String baseURI, resp.sendError(400, "Unsupported SPARQL query type"); } } + return true; + } catch (org.apache.jena.query.QueryParseException ex) { + // The client's own query text is at fault; the parse message is theirs. + resp.sendError(400, "Query parse error: " + ex.getMessage()); + return true; } catch (Exception ex) { - resp.sendError(400, "Query error: " + ex.getMessage()); + // Internal failure: log the details server-side, but do not echo + // exception internals (paths, class names, state) back to the client. + logger.error("SPARQL query execution failed", ex); + if (resp.isCommitted()) { + // Partial 200 body already flushed: sendError would throw + // IllegalStateException. Rethrow so the container aborts the + // connection - the honest signal for a truncated result. + throw new QueryExecutionFailedException("Query failed after the response was committed", ex); + } + resp.sendError(500, "Query execution failed"); + return false; } } } diff --git a/src/main/java/com/ebremer/beakgraph/core/fuseki/BGVoIDSD.java b/src/main/java/com/ebremer/beakgraph/core/fuseki/BGVoIDSD.java index 55f569e3..9324adda 100644 --- a/src/main/java/com/ebremer/beakgraph/core/fuseki/BGVoIDSD.java +++ b/src/main/java/com/ebremer/beakgraph/core/fuseki/BGVoIDSD.java @@ -204,9 +204,9 @@ private String computeLongestCommonPrefix() { private String getNamespaceBase(String uri) { int idx = uri.lastIndexOf('#'); if (idx == -1) idx = uri.lastIndexOf('/'); - if (idx == -1) return uri + "#"; - String ns = uri.substring(0, idx + 1); - return (uri.charAt(idx) == '#') ? ns : ns; + if (idx == -1) return uri + "#"; + // The substring already ends with the separator ('#' or '/'). + return uri.substring(0, idx + 1); } } } diff --git a/src/main/java/com/ebremer/beakgraph/core/fuseki/LWSStorageServlet.java b/src/main/java/com/ebremer/beakgraph/core/fuseki/LWSStorageServlet.java index 756f3adf..a92d7125 100644 --- a/src/main/java/com/ebremer/beakgraph/core/fuseki/LWSStorageServlet.java +++ b/src/main/java/com/ebremer/beakgraph/core/fuseki/LWSStorageServlet.java @@ -66,23 +66,54 @@ private boolean isSparqlRequest(HttpServletRequest req) { return "GET".equals(method) && req.getParameter("query") != null; } private void handleSparqlQuery(HttpServletRequest req, HttpServletResponse resp, Path h5File) throws IOException { - String queryStr = BGSparqlService.extractQuery(req); + String queryStr; + try { + queryStr = BGSparqlService.extractQuery(req); + } catch (BGSparqlService.QueryBodyTooLargeException e) { + resp.sendError(413, e.getMessage()); + return; + } if (queryStr == null || queryStr.isBlank()) { resp.sendError(400, "No SPARQL query provided"); return; } URI fileUri = h5File.toUri(); BeakGraph bg = null; + boolean healthy = true; try { bg = BeakGraphPool.getPool().borrowObject(fileUri); // resolve document-relative IRIs against the URL this .h5 is served from - BGSparqlService.execute(bg.getDataset(), queryStr, + healthy = BGSparqlService.execute(bg.getDataset(), queryStr, req.getRequestURL().toString(), req.getHeader("Accept"), resp); + } catch (BGSparqlService.QueryExecutionFailedException ex) { + // Failure after the response committed: nothing may touch the response + // now. Rethrow (after the finally invalidates the reader) so the + // container aborts the connection instead of finishing a truncated + // 200 body as if it were complete. + healthy = false; + throw ex; } catch (Exception ex) { - resp.sendError(400, "Query error: " + ex.getMessage()); + // Reaching here means infrastructure failure (pool, file). Log it, + // don't echo internals to the client. + healthy = false; + logger.error("SPARQL query handling failed for {}", h5File, ex); + if (!resp.isCommitted()) { + resp.sendError(500, "Query execution failed"); + } } finally { if (bg != null) { - BeakGraphPool.getPool().returnObject(fileUri, bg); + if (healthy) { + BeakGraphPool.getPool().returnObject(fileUri, bg); + } else { + // Never re-issue an instance that just failed: with only + // returnObject here, a broken-but-open reader circulated + // forever, 500ing every request for its file. + try { + BeakGraphPool.getPool().invalidateObject(fileUri, bg); + } catch (Exception invalidateEx) { + logger.warn("Failed to invalidate pooled BeakGraph for {}", fileUri, invalidateEx); + } + } } } } @@ -110,7 +141,9 @@ private void serveLinkset(HttpServletResponse resp, String resourceURI) throws I String updated = r.hasProperty(AS_UPDATED) ? r.getProperty(AS_UPDATED).getString() : null; resp.setContentType("application/linkset+json"); resp.setCharacterEncoding("UTF-8"); - resp.getWriter().write(linksetJson(resourceURI, typeHref, up, media, size, updated)); + // anchor/up are advertised on the LIVE base, never the canonical one. + resp.getWriter().write(linksetJson(toLiveUri(resourceURI), typeHref, + up == null ? null : toLiveUri(up), media, size, updated)); } /** @@ -163,6 +196,99 @@ private static String writeJson(JsonObject obj) { return sw.toString(); } + /** + * Minimal HTML escaping for text and attribute contexts. Request paths and + * on-disk filenames are attacker-influenced and end up in the container + * listing - unescaped they are a stored-XSS sink. + */ + static String escapeHtml(String s) { + StringBuilder sb = new StringBuilder(s.length()); + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + switch (c) { + case '&' -> sb.append("&"); + case '<' -> sb.append("<"); + case '>' -> sb.append(">"); + case '"' -> sb.append("""); + case '\'' -> sb.append("'"); + default -> sb.append(c); + } + } + return sb.toString(); + } + + /** Percent-encode a path for use in an href (keeps '/', encodes spaces, quotes, etc.). */ + static String encodeHref(String path) { + try { + return new java.net.URI(null, null, path, null).toASCIIString(); + } catch (java.net.URISyntaxException e) { + return java.net.URLEncoder.encode(path, StandardCharsets.UTF_8); + } + } + + /** + * Decode the raw request URI into the literal path the metadata model and the + * filesystem use. Servlet spec: {@code getRequestURI()} is UNDECODED, while the + * model holds raw filenames - comparing them directly made every resource whose + * name needs percent-encoding (spaces, '#', non-ASCII) permanently 404, including + * via this servlet's own encoded listing links. Returns null for a syntactically + * invalid URI (callers answer 400). + */ + static String decodePath(String rawRequestUri) { + try { + String p = new java.net.URI(rawRequestUri).getPath(); + return p == null ? "" : p; + } catch (java.net.URISyntaxException e) { + return null; + } + } + + /** + * Map a canonical-model URI (rooted at {@link LWSMetadataGenerator#CANONICAL_BASE}) + * onto the live serving base. {@code base} ends with '/' and the canonical + * remainder starts with '/', so naive concatenation minted double-slash URIs that + * 404 when dereferenced; the canonical host/port also leaked into Link headers and + * linksets whenever the server ran on a non-default port. + */ + static String toLiveUri(String canonicalUri, String base) { + String rest = canonicalUri.substring(HTTP_ROOT.length()); + while (rest.startsWith("/")) { + rest = rest.substring(1); + } + return base + rest; + } + + private String toLiveUri(String canonicalUri) { + return toLiveUri(canonicalUri, BASE); + } + + /** Container page URI on the live base; reqPath carries no leading slash and BASE ends with '/'. */ + private String pageUri(String reqPath, int page) { + return BASE + reqPath + "?page=" + page; + } + + /** + * Whether the (lower-cased) Accept header admits an HTML representation. + * RFC 9110: {@code *}{@code /*} and {@code text/*} match every/any text + * representation - curl's default {@code Accept: *}{@code /*} must get a + * page, not a 406. + */ + static boolean acceptsHtmlRepresentation(String lowerAccept) { + return lowerAccept.isEmpty() + || lowerAccept.contains("text/html") + || lowerAccept.contains("text/*") + || lowerAccept.contains("*/*"); + } + + /** + * Whether a statement object may be sent to clients. The metadata model links + * every resource to its on-disk location via {@code owl:sameAs }; + * those server-local URIs must never leave the server. + */ + static boolean exposableToClient(RDFNode obj) { + return !(obj.isURIResource() && obj.asResource().getURI().startsWith("file:")); + } + /** * Resolve a request path against the storage root, rejecting anything that escapes it via "..", * an absolute path, etc. Returns null when {@code root} is null or the path would leave the root. @@ -189,7 +315,10 @@ static Path resolveWithin(Path root, String reqPath) { } @Override protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { - String reqPath = req.getRequestURI(); + // Stop browsers from MIME-sniffing served content into something executable. + resp.setHeader("X-Content-Type-Options", "nosniff"); + String reqPath = decodePath(req.getRequestURI()); + if (reqPath == null) { resp.sendError(400, "Malformed request URI"); return; } if (reqPath.startsWith("/")) reqPath = reqPath.substring(1); if (reqPath.endsWith("/")) reqPath = reqPath.substring(0, reqPath.length()-1); if (reqPath.endsWith(".meta")) { @@ -198,6 +327,15 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IO basePath = basePath.substring("HalcyonStorage".length()); if (basePath.startsWith("/")) basePath = basePath.substring(1); } + if (basePath.equals("description")) { + // The /description document advertises this linkset itself; it is + // not in the metadata model, so answer it directly instead of 404. + resp.setContentType("application/linkset+json"); + resp.setCharacterEncoding("UTF-8"); + resp.getWriter().write(linksetJson(BASE + "description", + LWS.MetadataResource.getURI(), BASE, null, null, null)); + return; + } String baseResourceURI = basePath.isEmpty() ? HTTP_ROOT : HTTP_ROOT + "/" + basePath; serveLinkset(resp, baseResourceURI); return; @@ -205,7 +343,9 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IO if (reqPath.equals("description")) { resp.setContentType("application/ld+json"); resp.setHeader("Link", "<" + BASE + "description>; rel=\"storageDescription\""); - resp.setHeader("Link", "<" + getLinksetURI(BASE + "description") + ">; rel=\"linkset\"; type=\"application/linkset+json\""); + // addHeader, not setHeader: a second setHeader replaces the first Link + // header, which silently dropped the storageDescription link. + resp.addHeader("Link", "<" + getLinksetURI(BASE + "description") + ">; rel=\"linkset\"; type=\"application/linkset+json\""); resp.setHeader("Vary", "Accept"); resp.getWriter().write(descriptionJson(BASE)); return; @@ -227,7 +367,7 @@ && isHDF5(h5Candidate) && isSparqlRequest(req)) { handleSparqlQuery(req, resp, h5Candidate); return; } - String linksetURI = getLinksetURI(resourceURI); + String linksetURI = getLinksetURI(toLiveUri(resourceURI)); resp.addHeader("Link", "<" + linksetURI + ">; rel=\"linkset\"; type=\"application/linkset+json\""); resp.setHeader("Vary", "Accept"); boolean isContainer = r.hasProperty(RDF.type, LWS_CONTAINER); @@ -235,15 +375,18 @@ && isHDF5(h5Candidate) && isSparqlRequest(req)) { String formatParam = req.getParameter("format"); boolean forceTurtle = "turtle".equalsIgnoreCase(formatParam); boolean forceJsonLd = "jsonld".equalsIgnoreCase(formatParam); - boolean wantHtml = (accept.contains("text/html") || accept.isEmpty()) && !forceTurtle && !forceJsonLd; + boolean wantHtml = acceptsHtmlRepresentation(accept) && !forceTurtle && !forceJsonLd; boolean wantTurtle = accept.contains("turtle") || forceTurtle; boolean wantJson = (accept.contains("ld+json") || accept.contains("json")) || forceJsonLd; if (isContainer && wantHtml) { resp.setContentType("text/html; charset=utf-8"); + // The request path and item names come from the URL / the filesystem; + // escape them for HTML and percent-encode hrefs (stored-XSS sink). + String safePath = escapeHtml(reqPath); try (PrintWriter out = resp.getWriter()) { - out.println("LWS Storage – /" + (reqPath.isEmpty() ? "" : reqPath) + ""); + out.println("LWS Storage – /" + safePath + ""); out.println(""); - out.println("

Linked Web Storage: /" + (reqPath.isEmpty() ? "" : reqPath) + "

"); + out.println("

Linked Web Storage: /" + safePath + "

"); out.println("

Storage Description | "); out.println("Turtle | JSON-LD | "); out.println("SPARQL Endpoint

"); @@ -256,7 +399,8 @@ && isHDF5(h5Candidate) && isSparqlRequest(req)) { String name = item.getURI().substring(HTTP_ROOT.length()); if (name.isEmpty()) name = "(root)"; String link = name.startsWith("/") ? name : "/" + name; - out.printf("
  • %s
    • %n", link, name); + out.printf("
  • %s
    • %n", + escapeHtml(encodeHref(link)), escapeHtml(name)); } out.println(""); } @@ -269,9 +413,9 @@ && isHDF5(h5Candidate) && isSparqlRequest(req)) { Resource httpR = out.createResource(BASE + (reqPath.isEmpty() ? "" : reqPath)); r.listProperties().forEachRemaining(s -> { RDFNode obj = s.getObject(); + if (!exposableToClient(obj)) return; // never leak file:/// server paths if (obj.isResource() && obj.asResource().getURI() != null && obj.asResource().getURI().startsWith(HTTP_ROOT)) { - String newURI = BASE + obj.asResource().getURI().substring(HTTP_ROOT.length()); - httpR.addProperty(s.getPredicate(), out.createResource(newURI)); + httpR.addProperty(s.getPredicate(), out.createResource(toLiveUri(obj.asResource().getURI()))); } else { httpR.addProperty(s.getPredicate(), obj); } @@ -294,22 +438,29 @@ && isHDF5(h5Candidate) && isSparqlRequest(req)) { if (start >= total) { resp.sendError(404); return; } int startIdx = (int) start; List paged = items.subList(startIdx, Math.min(startIdx+size, total)); - Resource pageR = out.createResource(BASE + (reqPath.isEmpty() ? "" : reqPath) + (reqPath.isEmpty() ? "" : "/") + "?page=" + page); - r.listProperties().forEachRemaining(s -> { if (!s.getPredicate().equals(LWS_ITEMS)) pageR.addProperty(s.getPredicate(), s.getObject()); }); + // Self and first/last/prev/next all use the same URI shape, + // and the navigation links are RESOURCES (they are page URIs, + // and were emitted as plain literals with a divergent shape). + Resource pageR = out.createResource(pageUri(reqPath, page)); + r.listProperties().forEachRemaining(s -> { + if (!s.getPredicate().equals(LWS_ITEMS) && exposableToClient(s.getObject())) { + pageR.addProperty(s.getPredicate(), s.getObject()); + } + }); pageR.addProperty(RDF.type, LWS.ContainerPage); - pageR.addProperty(ResourceFactory.createProperty("https://www.w3.org/ns/activitystreams#first"), BASE + (reqPath.isEmpty() ? "" : reqPath) + "?page=1"); + pageR.addProperty(ResourceFactory.createProperty("https://www.w3.org/ns/activitystreams#first"), out.createResource(pageUri(reqPath, 1))); int pages = (total + size - 1) / size; - pageR.addProperty(ResourceFactory.createProperty("https://www.w3.org/ns/activitystreams#last"), BASE + (reqPath.isEmpty() ? "" : reqPath) + "?page=" + pages); - if (page > 1) pageR.addProperty(ResourceFactory.createProperty("https://www.w3.org/ns/activitystreams#prev"), BASE + (reqPath.isEmpty() ? "" : reqPath) + "?page=" + (page-1)); - if (page < pages) pageR.addProperty(ResourceFactory.createProperty("https://www.w3.org/ns/activitystreams#next"), BASE + (reqPath.isEmpty() ? "" : reqPath) + "?page=" + (page+1)); + pageR.addProperty(ResourceFactory.createProperty("https://www.w3.org/ns/activitystreams#last"), out.createResource(pageUri(reqPath, pages))); + if (page > 1) pageR.addProperty(ResourceFactory.createProperty("https://www.w3.org/ns/activitystreams#prev"), out.createResource(pageUri(reqPath, page-1))); + if (page < pages) pageR.addProperty(ResourceFactory.createProperty("https://www.w3.org/ns/activitystreams#next"), out.createResource(pageUri(reqPath, page+1))); for (Resource it : paged) { - String itHttp = BASE + it.getURI().substring(HTTP_ROOT.length()); + String itHttp = toLiveUri(it.getURI()); pageR.addProperty(LWS_ITEMS, out.createResource(itHttp)); it.listProperties().forEachRemaining(st -> { RDFNode obj = st.getObject(); + if (!exposableToClient(obj)) return; // never leak file:/// server paths if (obj.isResource() && obj.asResource().getURI() != null && obj.asResource().getURI().startsWith(HTTP_ROOT)) { - String newURI = BASE + obj.asResource().getURI().substring(HTTP_ROOT.length()); - out.getResource(itHttp).addProperty(st.getPredicate(), out.createResource(newURI)); + out.getResource(itHttp).addProperty(st.getPredicate(), out.createResource(toLiveUri(obj.asResource().getURI()))); } else { out.getResource(itHttp).addProperty(st.getPredicate(), obj); } @@ -317,12 +468,12 @@ && isHDF5(h5Candidate) && isSparqlRequest(req)) { } } else { for (Resource it : items) { - String itHttp = BASE + it.getURI().substring(HTTP_ROOT.length()); + String itHttp = toLiveUri(it.getURI()); it.listProperties().forEachRemaining(st -> { RDFNode obj = st.getObject(); + if (!exposableToClient(obj)) return; // never leak file:/// server paths if (obj.isResource() && obj.asResource().getURI() != null && obj.asResource().getURI().startsWith(HTTP_ROOT)) { - String newURI = BASE + obj.asResource().getURI().substring(HTTP_ROOT.length()); - out.getResource(itHttp).addProperty(st.getPredicate(), out.createResource(newURI)); + out.getResource(itHttp).addProperty(st.getPredicate(), out.createResource(toLiveUri(obj.asResource().getURI()))); } else { out.getResource(itHttp).addProperty(st.getPredicate(), obj); } @@ -350,16 +501,39 @@ && isHDF5(h5Candidate) && isSparqlRequest(req)) { media = Files.probeContentType(localFile); if (media == null) media = "application/octet-stream"; } + // Conditional GET support: the store is read-only between writes, so + // size+mtime make a stable validator. + long size = Files.size(localFile); + long lastModified = Files.getLastModifiedTime(localFile).toMillis(); + String etag = "\"" + size + "-" + lastModified + "\""; + resp.setHeader("ETag", etag); + resp.setDateHeader("Last-Modified", lastModified); + String ifNoneMatch = req.getHeader("If-None-Match"); + long ifModifiedSince = req.getDateHeader("If-Modified-Since"); + if (etag.equals(ifNoneMatch) + || (ifNoneMatch == null && ifModifiedSince >= 0 && lastModified / 1000 <= ifModifiedSince / 1000)) { + resp.setStatus(HttpServletResponse.SC_NOT_MODIFIED); + return; + } resp.setContentType(media); - resp.setContentLengthLong(Files.size(localFile)); - Files.copy(localFile, resp.getOutputStream()); + // Stored bytes are served as a download: with nosniff above, this keeps an + // HTML/SVG file someone placed under the root from rendering in this origin. + String filename = localFile.getFileName().toString().replace("\"", ""); + resp.setHeader("Content-Disposition", "attachment; filename=\"" + filename + "\""); + resp.setContentLengthLong(size); + // HEAD gets the same headers without the body (and without the file copy). + if (!"HEAD".equals(req.getMethod())) { + Files.copy(localFile, resp.getOutputStream()); + } } @Override protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws IOException { + resp.setHeader("X-Content-Type-Options", "nosniff"); logger.info("LWS {} {} ct={} query-param={} bodyLen={}", req.getMethod(), req.getRequestURI(), req.getContentType(), req.getParameter("query") != null, req.getContentLengthLong()); - String reqPath = req.getRequestURI(); + String reqPath = decodePath(req.getRequestURI()); + if (reqPath == null) { resp.sendError(400, "Malformed request URI"); return; } if (reqPath.startsWith("/")) reqPath = reqPath.substring(1); if (reqPath.endsWith("/")) reqPath = reqPath.substring(0, reqPath.length()-1); if (reqPath.startsWith("HalcyonStorage")) { @@ -373,7 +547,9 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws I return; } boolean isContainer = r.hasProperty(RDF.type, LWS_CONTAINER); - if (isContainer) { resp.sendError(406); return; } + // 405, not 406: POST to a container is an unsupported METHOD, not a + // content-negotiation failure. + if (isContainer) { resp.sendError(405, "Method not allowed"); return; } if (STORAGE_ROOT == null) { resp.sendError(500); return; } Path localFile = resolveWithin(STORAGE_ROOT, reqPath); if (localFile == null) { resp.sendError(403, "Forbidden"); return; } @@ -386,6 +562,8 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws I } @Override protected void doHead(HttpServletRequest req, HttpServletResponse resp) throws IOException { + // Same routing and headers as GET; the file-serving branch checks the + // method and skips the body copy for HEAD. doGet(req, resp); } } diff --git a/src/main/java/com/ebremer/beakgraph/core/fuseki/RelativeIRIResolver.java b/src/main/java/com/ebremer/beakgraph/core/fuseki/RelativeIRIResolver.java index b5fae13b..6722000f 100644 --- a/src/main/java/com/ebremer/beakgraph/core/fuseki/RelativeIRIResolver.java +++ b/src/main/java/com/ebremer/beakgraph/core/fuseki/RelativeIRIResolver.java @@ -3,6 +3,8 @@ import com.ebremer.beakgraph.utils.UTIL; import java.util.Iterator; import java.util.List; +import java.util.function.Predicate; +import org.apache.jena.graph.Node; import org.apache.jena.graph.NodeFactory; import org.apache.jena.graph.Triple; import org.apache.jena.irix.IRIx; @@ -28,8 +30,9 @@ * in at build time. This class converts between the stored "storage form" and * the absolute "result form" expected by SPARQL clients: *
      - *
    • {@link #absoluteToStorage()} - query input: an absolute IRI under the - * base is rewritten to the relative form held in the dictionary.
      • + *
    • {@link #absoluteToStorage(Predicate)} - query input: an absolute IRI + * under the base is rewritten to the relative form when the dictionary + * holds that relative form (and not the absolute one).
      • *
    • {@link #storageToAbsolute()} - query output: a relative IRI is resolved * against the base into an absolute IRI.
      • *
    @@ -60,19 +63,33 @@ public boolean isActive() { } /** - * Input transform: an absolute IRI that sits under the document base is - * rewritten to the relative form actually stored in the dictionary, so that - * queries naming a resource by its served URL still match. Every other node - * (variables, literals, blank nodes, unrelated IRIs) passes through. + * Input transform: an absolute IRI under the document base is rewritten to + * its relative form so that queries naming a resource by its served URL + * still match a relative-stored term. Every other node (variables, + * literals, blank nodes, unrelated IRIs) passes through. + *

    + * The rewrite fires only when {@code storedTerm} says the relative form IS + * in the store and the absolute form is NOT. The dictionary holds a term in + * exactly one of the two forms (relative only when the source document used + * a relative reference), so rewriting unconditionally turned every query + * naming an absolute-stored same-host IRI into a guaranteed miss - Jena's + * relativize also emits {@code /absolute/path} and {@code ../up} forms the + * writer never produces. When both forms are stored, the exact term the + * query named (the absolute one) wins. + * + * @param storedTerm whether a node exists in the store's dictionary */ - public NodeTransform absoluteToStorage() { + public NodeTransform absoluteToStorage(Predicate storedTerm) { return node -> { if (base != null && node != null && node.isURI() && !UTIL.isRelativeIRI(node.getURI())) { try { IRIx rel = base.relativize(IRIx.create(node.getURI())); if (rel != null && rel.isRelative()) { - return NodeFactory.createURI(rel.str()); + Node relNode = NodeFactory.createURI(rel.str()); + if (storedTerm.test(relNode) && !storedTerm.test(node)) { + return relNode; + } } } catch (RuntimeException ignore) { // leave the node unchanged on any IRI parsing failure diff --git a/src/main/java/com/ebremer/beakgraph/core/fuseki/SPARQLEndPoint.java b/src/main/java/com/ebremer/beakgraph/core/fuseki/SPARQLEndPoint.java index 6bb1aff8..e5a3842a 100644 --- a/src/main/java/com/ebremer/beakgraph/core/fuseki/SPARQLEndPoint.java +++ b/src/main/java/com/ebremer/beakgraph/core/fuseki/SPARQLEndPoint.java @@ -35,6 +35,7 @@ public class SPARQLEndPoint { private Model lwsModel; private Path storageRoot = null; private BeakGraph singleFileGraph; + private final Dataset dataset; static { JenaSystem.init(); @@ -98,6 +99,7 @@ private SPARQLEndPoint(Parameters params) throws Exception { } if (lwsModel == null) lwsModel = ModelFactory.createDefaultModel(); } + this.dataset = ds; boolean singleFile = !Files.isDirectory(endpointPath); var serverBuilder = FusekiServer.create() @@ -130,8 +132,11 @@ private SPARQLEndPoint(Parameters params) throws Exception { // Serve /rdf ourselves so document-relative IRIs in the HDF5 file are // resolved (against the file's own URI) instead of leaking out raw, // matching the behaviour of the LWS .h5 SPARQL path. + // Resolution base is the SERVED URL, never the local file URI: resolving + // stored-relative IRIs against endpointPath.toUri() sent every client + // file:////... IRIs - full filesystem disclosure. ServletHolder rdfHolder = new ServletHolder("hdf5-sparql", - new HDF5SparqlServlet(ds, endpointPath.toUri().toString())); + new HDF5SparqlServlet(ds, BASE_URL + "rdf")); context.addServlet(rdfHolder, "/rdf"); context.addServlet(rdfHolder, "/rdf/*"); } @@ -152,7 +157,10 @@ public static synchronized SPARQLEndPoint getSPARQLEndPoint(Parameters params) t } public DatasetGraph getDataset() { - return server.getDataAccessPointRegistry().get("/rdf").getDataService().getDataset(); + // The dataset this endpoint serves, held directly: the Fuseki registry + // only knows "/rdf" in directory mode (single-file mode serves /rdf via + // its own servlet), so the old registry lookup NPE'd in single-file mode. + return dataset.asDatasetGraph(); } public void shutdown() { @@ -192,7 +200,25 @@ private static class SparqlWebPageServlet extends HttpServlet { @Override protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { String pathInfo = req.getPathInfo(); - String resourcePath = (pathInfo == null || pathInfo.equals("/") || pathInfo.isEmpty()) ? "/META-INF/sparql/index.html" : "/META-INF/sparql" + pathInfo; + String resourcePath; + if (pathInfo == null || pathInfo.equals("/") || pathInfo.isEmpty()) { + resourcePath = "/META-INF/sparql/index.html"; + } else if (pathInfo.equals("/beakgraph.png")) { + // The logo ships exactly once, at the classpath root - an identical + // copy under META-INF/sparql used to double the jar by 1.6 MB. + resourcePath = "/beakgraph.png"; + } else { + // pathInfo is attacker-influenced and already URL-decoded: reject + // dot-segments (and backslashes) before splicing it into a classpath + // lookup, so an encoded "/../.." can never escape /META-INF/sparql - + // regardless of the container's URI-compliance mode or whether the + // classpath is a jar or exploded directories. + if (pathInfo.contains("..") || pathInfo.indexOf('\\') >= 0) { + resp.sendError(HttpServletResponse.SC_NOT_FOUND); + return; + } + resourcePath = "/META-INF/sparql" + pathInfo; + } InputStream is = getClass().getResourceAsStream(resourcePath); if (is == null) { resp.sendError(HttpServletResponse.SC_NOT_FOUND); return; } resp.setContentType(getContentType(resourcePath)); @@ -207,6 +233,7 @@ private String getContentType(String path) { if (path.endsWith(".css")) return "text/css"; if (path.endsWith(".js")) return "application/javascript"; if (path.endsWith(".json")) return "application/json"; + if (path.endsWith(".png")) return "image/png"; return "application/octet-stream"; } } @@ -235,7 +262,13 @@ private static class HDF5SparqlServlet extends HttpServlet { } private void handle(HttpServletRequest req, HttpServletResponse resp) throws IOException { - String queryStr = BGSparqlService.extractQuery(req); + String queryStr; + try { + queryStr = BGSparqlService.extractQuery(req); + } catch (BGSparqlService.QueryBodyTooLargeException e) { + resp.sendError(413, e.getMessage()); + return; + } if (queryStr == null || queryStr.isBlank()) { resp.sendError(400, "No SPARQL query provided"); return; diff --git a/src/main/java/com/ebremer/beakgraph/core/lib/NodeComparator.java b/src/main/java/com/ebremer/beakgraph/core/lib/NodeComparator.java index babaf009..518e8705 100644 --- a/src/main/java/com/ebremer/beakgraph/core/lib/NodeComparator.java +++ b/src/main/java/com/ebremer/beakgraph/core/lib/NodeComparator.java @@ -1,6 +1,11 @@ package com.ebremer.beakgraph.core.lib; +import java.math.BigDecimal; +import java.math.BigInteger; import java.util.Comparator; +import javax.xml.datatype.DatatypeConstants; +import javax.xml.datatype.Duration; +import javax.xml.datatype.XMLGregorianCalendar; import org.apache.jena.graph.Node; import org.apache.jena.sparql.core.Quad; import org.apache.jena.sparql.expr.NodeValue; @@ -8,7 +13,7 @@ /** * Enforces a strict Total Ordering of RDF Nodes. - * CRITICAL for HDF5 Binary Search: + * CRITICAL for HDF5 Binary Search: * Ensures the sorting order perfectly matches the Monolithic Dictionary ID assignments: * 1. Default Graph * 2. Blank Nodes @@ -26,8 +31,18 @@ public int compare(Node n1, Node n2) { // 1. Handle the Default Graph first (Always ID 0 or implicitly lowest) boolean n1isDefault = isDefaultGraph(n1); boolean n2isDefault = isDefaultGraph(n2); - - if (n1isDefault && n2isDefault) return 0; + + if (n1isDefault && n2isDefault) { + // Both ARQ sentinels rank before everything else, but they are two + // DISTINCT RDF terms: returning 0 for the pair collapsed them onto + // one dictionary id when a source used urn:x-arq:DefaultGraphNode as + // a data term. Order by exact term (null ≡ the default graph IRI; + // "urn:x-arq:DefaultGraph" still sorts first, keeping the graph + // dictionary's lowest-id assumption). + Node t1 = (n1 == null) ? Quad.defaultGraphIRI : n1; + Node t2 = (n2 == null) ? Quad.defaultGraphIRI : n2; + return NodeCmp.compareRDFTerms(t1, t2); + } if (n1isDefault) return -1; if (n2isDefault) return 1; @@ -35,7 +50,7 @@ public int compare(Node n1, Node n2) { // This ensures the sorted array perfectly aligns with how IDs are chunked int type1 = getMacroType(n1); int type2 = getMacroType(n2); - + if (type1 != type2) { return Integer.compare(type1, type2); } @@ -47,8 +62,30 @@ public int compare(Node n1, Node n2) { NodeValue nv1 = NodeValue.makeNode(n1); NodeValue nv2 = NodeValue.makeNode(n2); + // Timezone-sensitive value spaces cannot go through compareAlways: + // it answers value order for XSD-determinate pairs but silently falls + // back to TERM order for indeterminate ones (a timezone-less dateTime + // vs a timezoned one within +/-14h; a month-based duration vs a + // day-based one). Mixing the two orders pairwise is not transitive - + // e.g. "2020-01-02T00:00:00" < "2020-01-02T08:00:00+14:00" < + // "2020-01-01T20:00:00Z" < the first - and a cyclic comparator breaks + // every sort and binary search in the store (locate() misses stored + // literals; builds fail with "Cannot resolve Object"). These spaces + // get a self-consistent total order below that agrees with XSD value + // order wherever XSD order is determinate, so the ValueCluster range + // pushdown stays over-inclusive and value-equal terms stay adjacent. + int group = temporalGroup(nv1); + if (group != 0 && group == temporalGroup(nv2)) { + return group == GROUP_DURATION + ? compareDurationTotal(nv1, nv2, n1, n2) + : compareTemporalTotal(nv1, nv2, n1, n2); + } + // compareAlways provides a strict SPARQL "ORDER BY" ordering by VALUE, - // handling mixed datatypes safely without throwing exceptions. + // handling mixed datatypes safely without throwing exceptions. Across + // distinct value spaces (number vs string vs dateTime vs date ...) it + // orders by a fixed value-space rank independent of the actual values, + // so it stays transitive there. int byValue = NodeValue.compareAlways(nv1, nv2); if (byValue != 0) { return byValue; @@ -69,6 +106,95 @@ public int compare(Node n1, Node n2) { return NodeCmp.compareRDFTerms(n1, n2); } + private static final int GROUP_DURATION = 9; + + /** + * Classifies a literal into one of the timezone-sensitive temporal value + * spaces (or 0 for everything else). The grouping mirrors Jena's own value + * spaces - dateTime and dateTimeStamp share one space, date/time/g* each + * have their own - so the special-cased ordering below applies exactly where + * compareAlways would have compared by value-or-term, and never across two + * spaces that compareAlways ranks by value space. Ill-formed literals answer + * false to all predicates and stay on the compareAlways path. + */ + private static int temporalGroup(NodeValue nv) { + if (nv.isDateTime()) return 1; + if (nv.isDate()) return 2; + if (nv.isTime()) return 3; + if (nv.isGYear()) return 4; + if (nv.isGYearMonth()) return 5; + if (nv.isGMonth()) return 6; + if (nv.isGMonthDay()) return 7; + if (nv.isGDay()) return 8; + if (nv.isDuration()) return GROUP_DURATION; + return 0; + } + + /** + * Total order for two temporal values of the same value space: a missing + * timezone is pinned to UTC, which makes the XSD ordering total (both + * operands become determinate instants) while agreeing with it on every + * pair that was already determinate (UTC lies inside the +/-14h window XSD + * uses for timezone-less values). Instant-equal values (same point on the + * timeline, e.g. "...Z" vs "...+00:00") fall through to the exact-term + * tie-break, exactly as value-equal literals do on the compareAlways path. + */ + private static int compareTemporalTotal(NodeValue nv1, NodeValue nv2, Node n1, Node n2) { + XMLGregorianCalendar a = (XMLGregorianCalendar) nv1.getDateTime().clone(); + XMLGregorianCalendar b = (XMLGregorianCalendar) nv2.getDateTime().clone(); + if (a.getTimezone() == DatatypeConstants.FIELD_UNDEFINED) a.setTimezone(0); + if (b.getTimezone() == DatatypeConstants.FIELD_UNDEFINED) b.setTimezone(0); + int r = a.compare(b); + if (r == DatatypeConstants.LESSER) return -1; + if (r == DatatypeConstants.GREATER) return 1; + return NodeCmp.compareRDFTerms(n1, n2); + } + + /** + * Total order for durations: by total months, then by total seconds, then + * by exact term. XSD duration equality is exactly (months, seconds) + * equality, so value-equal durations ("P1D" vs "PT24H") stay adjacent for + * ValueCluster; and within each XSD-comparable kind (month-based with + * month-based, day/time-based with day/time-based) the order equals XSD + * value order. Cross-kind pairs - which SPARQL comparison rejects and + * compareAlways used to term-order pairwise-inconsistently - get the fixed + * months-first rank. + */ + private static int compareDurationTotal(NodeValue nv1, NodeValue nv2, Node n1, Node n2) { + Duration d1 = nv1.getDuration(); + Duration d2 = nv2.getDuration(); + int c = totalMonths(d1).compareTo(totalMonths(d2)); + if (c != 0) return c; + c = totalSeconds(d1).compareTo(totalSeconds(d2)); + if (c != 0) return c; + return NodeCmp.compareRDFTerms(n1, n2); + } + + private static BigInteger totalMonths(Duration d) { + BigInteger years = fieldInt(d, DatatypeConstants.YEARS); + BigInteger months = fieldInt(d, DatatypeConstants.MONTHS); + BigInteger total = years.multiply(BigInteger.valueOf(12)).add(months); + return d.getSign() < 0 ? total.negate() : total; + } + + private static BigDecimal totalSeconds(Duration d) { + BigDecimal seconds = new BigDecimal(fieldInt(d, DatatypeConstants.DAYS)).multiply(BigDecimal.valueOf(86400)) + .add(new BigDecimal(fieldInt(d, DatatypeConstants.HOURS)).multiply(BigDecimal.valueOf(3600))) + .add(new BigDecimal(fieldInt(d, DatatypeConstants.MINUTES)).multiply(BigDecimal.valueOf(60))) + .add(fieldDec(d, DatatypeConstants.SECONDS)); + return d.getSign() < 0 ? seconds.negate() : seconds; + } + + private static BigInteger fieldInt(Duration d, DatatypeConstants.Field f) { + Number n = d.getField(f); + return n == null ? BigInteger.ZERO : (BigInteger) n; + } + + private static BigDecimal fieldDec(Duration d, DatatypeConstants.Field f) { + Number n = d.getField(f); + return n == null ? BigDecimal.ZERO : (BigDecimal) n; + } + /** * Maps a Node to an integer rank to enforce BNode < URI < Literal. */ @@ -77,7 +203,7 @@ private int getMacroType(Node n) { if (n.isURI()) return 2; if (n.isLiteral()) return 3; // Should never happen in valid RDF, but safe fallback - return 4; + return 4; } private boolean isDefaultGraph(Node n) { diff --git a/src/main/java/com/ebremer/beakgraph/core/lib/Stats.java b/src/main/java/com/ebremer/beakgraph/core/lib/Stats.java index a00d1f54..21f68652 100644 --- a/src/main/java/com/ebremer/beakgraph/core/lib/Stats.java +++ b/src/main/java/com/ebremer/beakgraph/core/lib/Stats.java @@ -21,10 +21,13 @@ public class Stats { public int maxInteger = Integer.MIN_VALUE; public int minInteger = Integer.MAX_VALUE; public long numInteger = 0; - public float maxFloat = Float.MIN_VALUE; + // Seeds for running max must be the most NEGATIVE value: Float.MIN_VALUE / + // Double.MIN_VALUE are the smallest POSITIVE values, which made the reported + // max wrong for all-negative data. + public float maxFloat = -Float.MAX_VALUE; public float minFloat = Float.MAX_VALUE; - public long numFloat = 0; - public double maxDouble = Double.MIN_VALUE; + public long numFloat = 0; + public double maxDouble = -Double.MAX_VALUE; public double minDouble = Double.MAX_VALUE; public long numDouble = 0; diff --git a/src/main/java/com/ebremer/beakgraph/core/lib/VByte.java b/src/main/java/com/ebremer/beakgraph/core/lib/VByte.java index 97ab9486..54693764 100644 --- a/src/main/java/com/ebremer/beakgraph/core/lib/VByte.java +++ b/src/main/java/com/ebremer/beakgraph/core/lib/VByte.java @@ -1,12 +1,17 @@ package com.ebremer.beakgraph.core.lib; -import java.io.InputStream; -import java.io.OutputStream; +import com.ebremer.beakgraph.io.RandomAccessBytes; import java.io.IOException; -import java.nio.ByteBuffer; +import java.io.OutputStream; /** - * Variable-byte (VByte) encoding with signed and unsigned support. + * Variable-byte (VByte) encoding, trimmed to the two operations the + * front-coded dictionaries actually use: streaming encode at write time and + * position-independent decode at read time. The signed (zig-zag), byte-array + * and position-mutating ByteBuffer variants that accumulated here had no + * callers and were removed in the dead-code sweep. The decode side reads + * through {@link RandomAccessBytes} with long offsets (its only caller is + * FCDReader), so string buffers past 2 GiB decode without int truncation. */ public class VByte { @@ -14,7 +19,7 @@ public class VByte { * Encode an unsigned long to an OutputStream using VByte. * @param out * @param value non-negative - * @return + * @return number of bytes written * @throws IOException */ public static int encode(OutputStream out, long value) throws IOException { @@ -32,160 +37,20 @@ public static int encode(OutputStream out, long value) throws IOException { } /** - * Decode an unsigned long from an InputStream using VByte. - * @param in - * @return decoded value - * @throws IOException - */ - public static long decode(InputStream in) throws IOException { - long result = 0; - int shift = 0, b; - do { - if (shift >= 64) - throw new IOException("VByte sequence too long"); - b = in.read(); - if (b < 0) - throw new IOException("Unexpected end of stream"); - result |= (long)(b & 0x7F) << shift; - shift += 7; - } while ((b & 0x80) == 0); - return result; - } - - /** - * Encode a signed long with zig-zag transform. - * @param out - * @param value - * @throws java.io.IOException - */ - public static void encodeSigned(OutputStream out, long value) throws IOException { - // canonical zig-zag: interleave sign bit - long zig = (value << 1) ^ (value >> 63); - encode(out, zig); - } - - /** - * Decode a signed long with zig-zag transform. - * @param in - * @return - * @throws java.io.IOException - */ - public static long decodeSigned(InputStream in) throws IOException { - long zig = decode(in); - // reverse zig-zag - return (zig >>> 1) ^ -(zig & 1); - } - - /** - * Encode an unsigned long into a byte array at offset. - * @param array - * @param offset - * @param value - * @return new offset - */ - public static int encode(byte[] array, int offset, long value) { - if (value < 0) - throw new IllegalArgumentException("Value must be non-negative: " + value); - while (value > 0x7F) { - array[offset++] = (byte)(value & 0x7F); - value >>>= 7; - } - array[offset++] = (byte)(value | 0x80); - return offset; - } - - /** - * Decode an unsigned long from a byte array starting at offset. - * @param array - * @param offset - * @return DecodeResult.value and nextOffset in the array - */ - public static DecodeResult decode(byte[] array, int offset) { - long result = 0; - int shift = 0; - byte b; - do { - if (shift >= 64) - throw new IllegalArgumentException("VByte sequence too long"); - b = array[offset++]; - result |= (long)(b & 0x7F) << shift; - shift += 7; - } while ((b & 0x80) == 0); - return new DecodeResult(result, offset); - } - - /** - * Encode an unsigned long into a ByteBuffer. - * @param buffer - * @param value - * @return number of bytes written - */ - public static int encode(ByteBuffer buffer, long value) { - int start = buffer.position(); - if (value < 0) - throw new IllegalArgumentException("Value must be non-negative: " + value); - while (value > 0x7F) { - buffer.put((byte)(value & 0x7F)); - value >>>= 7; - } - buffer.put((byte)(value | 0x80)); - return buffer.position() - start; - } - - /** - * Decode an unsigned long from a ByteBuffer. - * @param buffer - * @return DecodeResult.value and bytesConsumed - */ - public static DecodeResult decode(ByteBuffer buffer) { - int start = buffer.position(); - long result = 0; - int shift = 0; - byte b; - do { - if (shift >= 64) - throw new IllegalArgumentException("VByte sequence too long"); - b = buffer.get(); - result |= (long)(b & 0x7F) << shift; - shift += 7; - } while ((b & 0x80) == 0); - int consumed = buffer.position() - start; - return new DecodeResult(result, consumed); - } - - /** - * Decode an unsigned long from a ByteBuffer. - * @param buffer - * @return DecodeResult.value and bytesConsumed - */ - public static long decodeSingle(ByteBuffer buffer) { - long result = 0; - int shift = 0; - byte b; - do { - if (shift >= 64) throw new IllegalArgumentException("VByte sequence too long "+shift); - b = buffer.get(); - result |= (long)(b & 0x7F) << shift; - shift += 7; - } while ((b & 0x80) == 0); - return result; - } - - /** - * Decode an unsigned long from a ByteBuffer at an absolute offset, WITHOUT moving the - * buffer's position. This lets a single buffer be read by concurrent threads safely. - * @param buffer the buffer to read from + * Decode an unsigned long at an absolute offset. Absolute reads only, so a + * single backing region can be read by concurrent threads safely. + * @param bytes the region to read from * @param offset the absolute byte offset to start decoding at * @return value and nextOffset (the absolute position just past the encoded value) */ - public static DecodeResult decodeAt(ByteBuffer buffer, int offset) { + public static DecodeResult decodeAt(RandomAccessBytes bytes, long offset) { long result = 0; int shift = 0; - int pos = offset; + long pos = offset; byte b; do { if (shift >= 64) throw new IllegalArgumentException("VByte sequence too long"); - b = buffer.get(pos++); + b = bytes.get(pos++); result |= (long)(b & 0x7F) << shift; shift += 7; } while ((b & 0x80) == 0); @@ -193,16 +58,15 @@ public static DecodeResult decodeAt(ByteBuffer buffer, int offset) { } /** - * Holder for decoded value and next offset/bytesConsumed. + * Holder for decoded value and next offset. */ public static class DecodeResult { public final long value; - public final int nextOffset; + public final long nextOffset; - public DecodeResult(long value, int nextOffset) { + public DecodeResult(long value, long nextOffset) { this.value = value; this.nextOffset = nextOffset; } } - } diff --git a/src/main/java/com/ebremer/beakgraph/features/MajorMinor.java b/src/main/java/com/ebremer/beakgraph/features/MajorMinor.java index cf613845..3176dc4f 100644 --- a/src/main/java/com/ebremer/beakgraph/features/MajorMinor.java +++ b/src/main/java/com/ebremer/beakgraph/features/MajorMinor.java @@ -2,119 +2,154 @@ import com.ebremer.ns.GEO; import com.ebremer.ns.HAL; import java.util.ArrayList; +import java.util.Locale; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.jena.datatypes.RDFDatatype; import org.apache.jena.graph.Node; import org.apache.jena.graph.NodeFactory; import org.apache.jena.rdf.model.Resource; import org.apache.jena.sparql.core.Quad; -import org.ejml.simple.SimpleEVD; -import org.ejml.simple.SimpleMatrix; import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.Polygon; import org.locationtech.jts.io.ParseException; import org.locationtech.jts.io.WKTReader; /** - * Adds centroid, major axis and minor axis as WKT literals to a geo:Feature + * Adds centroid, major axis and minor axis as WKT literals to a geo:Feature. + *

    + * All values come from the polygon's exact AREA moments (Green's theorem over + * the rings, shell positive and holes negative), not from a vertex point + * cloud: the vertex PCA double-weighted ring-closure vertices on holed + * polygons and measured boundary spread rather than the region, so the drawn + * axis disagreed with {@code PYR.MajorAxisLength} (raster-region based) for + * the same geometry. Axis length is the pyradiomics convention, 4*sqrt(λ) of + * the region's covariance - identical in definition to the raster feature. */ public class MajorMinor { + private static final Logger logger = LoggerFactory.getLogger(MajorMinor.class); + public static void add(Resource f, String wkt) { try { - WKTReader reader = new WKTReader(); - Geometry geom = reader.read(wkt); - if (!(geom instanceof Polygon)) return; - Polygon poly = (Polygon) geom; - Coordinate[] coords = poly.getCoordinates(); - // build point cloud - int n = coords.length - 1; // close ring - SimpleMatrix m = new SimpleMatrix(n, 2); - for (int i = 0; i < n; i++) { - m.set(i, 0, coords[i].x); - m.set(i, 1, coords[i].y); + Geometry geom = new WKTReader().read(wkt); + if (!(geom instanceof Polygon poly)) return; + double[] g = axes(poly); + if (g == null) { + logger.warn("Skipping centroid/axis features for {}: zero-area geometry", f); + return; } - // centroid - double mx = m.extractVector(false, 0).elementSum() / n; - double my = m.extractVector(false, 1).elementSum() / n; - // PCA - SimpleMatrix ones = SimpleMatrix.ones(n, 1); - SimpleMatrix meanVec = new SimpleMatrix(1, 2, true, mx, my); - SimpleMatrix centered = m.minus(ones.mult(meanVec)); - SimpleMatrix cov = centered.transpose().mult(centered).divide(n - 1.0); - SimpleEVD evd = cov.eig(); - double lambda0 = evd.getEigenvalue(0).getReal(); - double lambda1 = evd.getEigenvalue(1).getReal(); - SimpleMatrix v0 = (SimpleMatrix) evd.getEigenVector(0).copy(); - SimpleMatrix v1 = (SimpleMatrix) evd.getEigenVector(1).copy(); - if (lambda0 < lambda1) { - double t = lambda0; lambda0 = lambda1; lambda1 = t; - SimpleMatrix tv = v0; v0 = v1; v1 = tv; - } - if (v0.get(0) < 0) v0 = v0.scale(-1.0); - if (v1.get(0) < 0) v1 = v1.scale(-1.0); - double majorlen = 2 * Math.sqrt(lambda0); - double minorlen = 2 * Math.sqrt(lambda1); - // half-axis vectors - double ax = majorlen / 2 * v0.get(0); - double ay = majorlen / 2 * v0.get(1); - double bx = minorlen / 2 * v1.get(0); - double by = minorlen / 2 * v1.get(1); - // WKT strings - String centroidWKT = String.format("POINT(%.4f %.4f)", mx, my); - String majorWKT = String.format("LINESTRING(%.4f %.4f, %.4f %.4f)", - mx - ax, my - ay, mx + ax, my + ay); - String minorWKT = String.format("LINESTRING(%.4f %.4f, %.4f %.4f)", - mx - bx, my - by, mx + bx, my + by); - // add to the feature - f.addProperty(HAL.centroid, f.getModel().createTypedLiteral(centroidWKT, GEO.wktLiteral.getURI())); - f.addProperty(HAL.majorAxis, f.getModel().createTypedLiteral(majorWKT, GEO.wktLiteral.getURI())); - f.addProperty(HAL.minorAxis, f.getModel().createTypedLiteral(minorWKT, GEO.wktLiteral.getURI())); - } catch (ParseException ignored) {} + // WKT strings: Locale.ROOT so the decimal separator is always '.', + // not the default locale's (e.g. ',' on de_DE, which is invalid WKT). + f.addProperty(HAL.centroid, f.getModel().createTypedLiteral(centroidWkt(g), GEO.wktLiteral.getURI())); + f.addProperty(HAL.majorAxis, f.getModel().createTypedLiteral(majorWkt(g), GEO.wktLiteral.getURI())); + f.addProperty(HAL.minorAxis, f.getModel().createTypedLiteral(minorWkt(g), GEO.wktLiteral.getURI())); + } catch (ParseException | RuntimeException e) { + // JTS throws IllegalArgumentException - not just ParseException - for + // structurally invalid geometry (e.g. a two-point ring). One bad + // geometry skips ITS features with a warning; it must never escape + // into the spatial task and abort the whole build. + logger.warn("Skipping centroid/axis features for {}: {}", f, e.toString()); + } } - + public static void add(ArrayList quads, Node f, String wkt) { try { - WKTReader reader = new WKTReader(); - Geometry geom = reader.read(wkt); - if (!(geom instanceof Polygon)) return; - Polygon poly = (Polygon) geom; - Coordinate[] coords = poly.getCoordinates(); - int n = coords.length - 1; - SimpleMatrix m = new SimpleMatrix(n, 2); - for (int i = 0; i < n; i++) { - m.set(i, 0, coords[i].x); - m.set(i, 1, coords[i].y); - } - double mx = m.extractVector(false, 0).elementSum() / n; - double my = m.extractVector(false, 1).elementSum() / n; - SimpleMatrix ones = SimpleMatrix.ones(n, 1); - SimpleMatrix meanVec = new SimpleMatrix(1, 2, true, mx, my); - SimpleMatrix centered = m.minus(ones.mult(meanVec)); - SimpleMatrix cov = centered.transpose().mult(centered).divide(n - 1.0); - SimpleEVD evd = cov.eig(); - double lambda0 = evd.getEigenvalue(0).getReal(); - double lambda1 = evd.getEigenvalue(1).getReal(); - SimpleMatrix v0 = (SimpleMatrix) evd.getEigenVector(0).copy(); - SimpleMatrix v1 = (SimpleMatrix) evd.getEigenVector(1).copy(); - if (lambda0 < lambda1) { - double t = lambda0; lambda0 = lambda1; lambda1 = t; - SimpleMatrix tv = v0; v0 = v1; v1 = tv; + Geometry geom = new WKTReader().read(wkt); + if (!(geom instanceof Polygon poly)) return; + double[] g = axes(poly); + if (g == null) { + logger.warn("Skipping centroid/axis features for {}: zero-area geometry", f); + return; } - if (v0.get(0) < 0) v0 = v0.scale(-1.0); - if (v1.get(0) < 0) v1 = v1.scale(-1.0); - double majorlen = 2 * Math.sqrt(lambda0); - double minorlen = 2 * Math.sqrt(lambda1); - double ax = majorlen / 2 * v0.get(0); - double ay = majorlen / 2 * v0.get(1); - double bx = minorlen / 2 * v1.get(0); - double by = minorlen / 2 * v1.get(1); - String centroidWKT = String.format("POINT(%.4f %.4f)", mx, my); - String majorWKT = String.format("LINESTRING(%.4f %.4f, %.4f %.4f)", mx - ax, my - ay, mx + ax, my + ay); - String minorWKT = String.format("LINESTRING(%.4f %.4f, %.4f %.4f)", mx - bx, my - by, mx + bx, my + by); Node graph = Quad.defaultGraphIRI; RDFDatatype wktDT = NodeFactory.getType(GEO.wktLiteral.getURI()); - quads.add(Quad.create(graph, f, HAL.centroid.asNode(), NodeFactory.createLiteralDT(centroidWKT, wktDT))); - quads.add(Quad.create(graph, f, HAL.majorAxis.asNode(), NodeFactory.createLiteralDT(majorWKT, wktDT))); - quads.add(Quad.create(graph, f, HAL.minorAxis.asNode(), NodeFactory.createLiteralDT(minorWKT, wktDT))); - } catch (ParseException ignored) {} + quads.add(Quad.create(graph, f, HAL.centroid.asNode(), NodeFactory.createLiteralDT(centroidWkt(g), wktDT))); + quads.add(Quad.create(graph, f, HAL.majorAxis.asNode(), NodeFactory.createLiteralDT(majorWkt(g), wktDT))); + quads.add(Quad.create(graph, f, HAL.minorAxis.asNode(), NodeFactory.createLiteralDT(minorWkt(g), wktDT))); + } catch (ParseException | RuntimeException e) { + // See the Resource overload: skip-with-warning, never abort the build. + logger.warn("Skipping centroid/axis features for {}: {}", f, e.toString()); + } + } + + private static String centroidWkt(double[] g) { + return String.format(Locale.ROOT, "POINT(%.4f %.4f)", g[0], g[1]); + } + + private static String majorWkt(double[] g) { + return String.format(Locale.ROOT, "LINESTRING(%.4f %.4f, %.4f %.4f)", + g[0] - g[2], g[1] - g[3], g[0] + g[2], g[1] + g[3]); + } + + private static String minorWkt(double[] g) { + return String.format(Locale.ROOT, "LINESTRING(%.4f %.4f, %.4f %.4f)", + g[0] - g[4], g[1] - g[5], g[0] + g[4], g[1] + g[5]); + } + + /** + * {cx, cy, ax, ay, bx, by}: area centroid and the major/minor HALF-axis + * vectors, or null when the net area is zero/degenerate. Exact closed-form + * moments over each ring's edges; ring contributions are sign-normalized so + * the shell adds and every hole subtracts regardless of winding order in + * the source WKT. + */ + private static double[] axes(Polygon poly) { + double area = 0, sumX = 0, sumY = 0, sumX2 = 0, sumY2 = 0, sumXY = 0; + for (int r = -1; r < poly.getNumInteriorRing(); r++) { + Coordinate[] ring = (r < 0 ? poly.getExteriorRing() : poly.getInteriorRingN(r)).getCoordinates(); + double a = 0, sx = 0, sy = 0, x2 = 0, y2 = 0, xy = 0; + for (int i = 0; i < ring.length - 1; i++) { + double x0 = ring[i].x, y0 = ring[i].y; + double x1 = ring[i + 1].x, y1 = ring[i + 1].y; + double cross = x0 * y1 - x1 * y0; + a += cross; + sx += (x0 + x1) * cross; + sy += (y0 + y1) * cross; + x2 += (x0 * x0 + x0 * x1 + x1 * x1) * cross; + y2 += (y0 * y0 + y0 * y1 + y1 * y1) * cross; + xy += (x0 * y1 + 2 * x0 * y0 + 2 * x1 * y1 + x1 * y0) * cross; + } + double sign = ((r < 0) == (a >= 0)) ? 1 : -1; + area += sign * a / 2; + sumX += sign * sx / 6; + sumY += sign * sy / 6; + sumX2 += sign * x2 / 12; + sumY2 += sign * y2 / 12; + sumXY += sign * xy / 24; + } + if (!(area > 0) || !Double.isFinite(area)) { + return null; + } + double cx = sumX / area; + double cy = sumY / area; + // Central second moments per unit area: the region's covariance matrix. + double varX = sumX2 / area - cx * cx; + double varY = sumY2 / area - cy * cy; + double cov = sumXY / area - cx * cy; + double trace = varX + varY; + double det = varX * varY - cov * cov; + double disc = Math.sqrt(Math.max(0, trace * trace - 4 * det)); + double l0 = Math.max(0, (trace + disc) / 2); + double l1 = Math.max(0, (trace - disc) / 2); + double v0x, v0y, v1x, v1y; + if (Math.abs(cov) < 1e-12) { + if (varX >= varY) { v0x = 1; v0y = 0; v1x = 0; v1y = 1; } + else { v0x = 0; v0y = 1; v1x = 1; v1y = 0; } + } else { + v0x = cov; v0y = l0 - varX; + double n0 = Math.hypot(v0x, v0y); + v0x /= n0; v0y /= n0; + v1x = cov; v1y = l1 - varX; + double n1 = Math.hypot(v1x, v1y); + v1x /= n1; v1y /= n1; + } + if (v0x < 0) { v0x = -v0x; v0y = -v0y; } + if (v1x < 0) { v1x = -v1x; v1y = -v1y; } + // Half-axis = (4*sqrt(λ)) / 2, the pyradiomics full axis length halved. + double ax = 2 * Math.sqrt(l0) * v0x; + double ay = 2 * Math.sqrt(l0) * v0y; + double bx = 2 * Math.sqrt(l1) * v1x; + double by = 2 * Math.sqrt(l1) * v1y; + return new double[]{cx, cy, ax, ay, bx, by}; } } diff --git a/src/main/java/com/ebremer/beakgraph/features/ShapeAnalysis.java b/src/main/java/com/ebremer/beakgraph/features/ShapeAnalysis.java index baab355e..a054afef 100644 --- a/src/main/java/com/ebremer/beakgraph/features/ShapeAnalysis.java +++ b/src/main/java/com/ebremer/beakgraph/features/ShapeAnalysis.java @@ -49,14 +49,20 @@ public static int Area(BufferedImage bi) { public static boolean isEdge(BufferedImage bi, int a, int b) { int c = bi.getRGB(a, b) & 0xFF; if (c>0) { - c = ((bi.getRGB(a+1, b) & 0xFF)>0)?1:0; - c = c + (((bi.getRGB(a-1, b) & 0xFF)>0)?1:0); - c = c + (((bi.getRGB(a, b+1) & 0xFF)>0)?1:0); - c = c + (((bi.getRGB(a, b-1) & 0xFF)>0)?1:0); - return (c!=4); + // Out-of-bounds neighbours count as background, so a filled pixel on + // the image border is an edge instead of an ArrayIndexOutOfBounds. + int n = filled(bi, a+1, b) + filled(bi, a-1, b) + filled(bi, a, b+1) + filled(bi, a, b-1); + return (n!=4); } return false; } + + private static int filled(BufferedImage bi, int x, int y) { + if (x < 0 || y < 0 || x >= bi.getWidth() || y >= bi.getHeight()) { + return 0; + } + return ((bi.getRGB(x, y) & 0xFF) > 0) ? 1 : 0; + } public static int Circumference(BufferedImage bi) { int count = 0; @@ -223,12 +229,17 @@ public static double[] getPrincipalAxes(SimpleMatrix points, double refX, double } public static BufferedImage getBufferedImage(Polygon p) { - Geometry bb = p.getEnvelope(); - Coordinate[] c = bb.getCoordinates(); - int width = (int) Math.round(c[2].x-c[0].x); - int height = (int) Math.round(c[2].y - c[0].y); + // Dimensions come from the Envelope, not getEnvelope()'s coordinate array: + // a degenerate (point/line) envelope has fewer than 3 coordinates (the old + // c[2] access threw), and a valid polygon thinner than ~0.5 units rounded + // to a 0-sized image (BufferedImage rejects it) - either way ALL features + // were skipped, including the purely polygon-based ones. Clamping to 1px + // keeps the raster features defined and the polygon features exact. + org.locationtech.jts.geom.Envelope env = p.getEnvelopeInternal(); + int width = Math.max(1, (int) Math.round(env.getWidth())); + int height = Math.max(1, (int) Math.round(env.getHeight())); AffineTransformation af = new AffineTransformation(); - af.setToTranslation(-c[0].x, -c[0].y); + af.setToTranslation(-env.getMinX(), -env.getMinY()); // transform() returns a translated copy (it does not mutate p, which the caller // still uses for its other feature calcs). Draw that copy, shifted so the polygon's // bounding-box corner sits at (0,0) and lands inside the width x height image - diff --git a/src/main/java/com/ebremer/beakgraph/features/pyradiomics/Gen2DFeatures.java b/src/main/java/com/ebremer/beakgraph/features/pyradiomics/Gen2DFeatures.java index a8f6befd..57f787cc 100644 --- a/src/main/java/com/ebremer/beakgraph/features/pyradiomics/Gen2DFeatures.java +++ b/src/main/java/com/ebremer/beakgraph/features/pyradiomics/Gen2DFeatures.java @@ -9,27 +9,34 @@ import org.locationtech.jts.geom.GeometryFactory; import org.locationtech.jts.geom.Polygon; import org.locationtech.jts.io.WKTReader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class Gen2DFeatures { + private static final Logger logger = LoggerFactory.getLogger(Gen2DFeatures.class); public static void generate(Resource geo, String wktPolygon) { try { GeometryFactory gf = new GeometryFactory(); Polygon p = (Polygon) new WKTReader(gf).read(wktPolygon); java.awt.image.BufferedImage bi = ShapeAnalysis.getBufferedImage(p); org.ejml.simple.SimpleMatrix pts = ShapeAnalysis.BufferedImage2INDArray(bi); - geo.addProperty(PYR.MeshSurface, ResourceFactory.createTypedLiteral(ShapeAnalysis.getMeshSurfaceFeatureValue(p))); - geo.addProperty(PYR.PixelSurface, ResourceFactory.createTypedLiteral(ShapeAnalysis.getPixelSurfaceFeatureValue(bi))); - geo.addProperty(PYR.Perimeter, ResourceFactory.createTypedLiteral(ShapeAnalysis.getPerimeter(p))); - geo.addProperty(PYR.PerimeterSurfaceRatio, ResourceFactory.createTypedLiteral(ShapeAnalysis.getPerimeterSurfaceRatioFeatureValue(p))); - geo.addProperty(PYR.Sphericity, ResourceFactory.createTypedLiteral(ShapeAnalysis.getSphericityFeatureValue(p))); - geo.addProperty(PYR.SphericalDisproportion, ResourceFactory.createTypedLiteral(ShapeAnalysis.getSphericalDisproportionFeatureValue(p))); - geo.addProperty(PYR.Maximum2DDiameter, ResourceFactory.createTypedLiteral(ShapeAnalysis.getMaximum2DDiameterFeatureValue(p))); - geo.addProperty(PYR.MajorAxisLength, ResourceFactory.createTypedLiteral(ShapeAnalysis.getMajorAxisLengthFeatureValue(pts))); - geo.addProperty(PYR.MinorAxisLength, ResourceFactory.createTypedLiteral(ShapeAnalysis.getMinorAxisLengthFeatureValue(pts))); - geo.addProperty(PYR.Elongation, ResourceFactory.createTypedLiteral(ShapeAnalysis.getElongationFeatureValue(pts))); - } catch (Exception e) {} + addFinite(geo, PYR.MeshSurface, ShapeAnalysis.getMeshSurfaceFeatureValue(p)); + addFinite(geo, PYR.PixelSurface, ShapeAnalysis.getPixelSurfaceFeatureValue(bi)); + addFinite(geo, PYR.Perimeter, ShapeAnalysis.getPerimeter(p)); + addFinite(geo, PYR.PerimeterSurfaceRatio, ShapeAnalysis.getPerimeterSurfaceRatioFeatureValue(p)); + addFinite(geo, PYR.Sphericity, ShapeAnalysis.getSphericityFeatureValue(p)); + addFinite(geo, PYR.SphericalDisproportion, ShapeAnalysis.getSphericalDisproportionFeatureValue(p)); + addFinite(geo, PYR.Maximum2DDiameter, ShapeAnalysis.getMaximum2DDiameterFeatureValue(p)); + addFinite(geo, PYR.MajorAxisLength, ShapeAnalysis.getMajorAxisLengthFeatureValue(pts)); + addFinite(geo, PYR.MinorAxisLength, ShapeAnalysis.getMinorAxisLengthFeatureValue(pts)); + addFinite(geo, PYR.Elongation, ShapeAnalysis.getElongationFeatureValue(pts)); + } catch (Exception e) { + // A geometry whose features cannot be computed is skipped - but never + // silently: silently-missing features read as "no data" downstream. + logger.warn("Failed to generate 2D shape features for {}: {}", geo, e.toString()); + } } - + public static void generate(ArrayList quads, Node geo, String wkt) { try { GeometryFactory gf = new GeometryFactory(); @@ -37,16 +44,41 @@ public static void generate(ArrayList quads, Node geo, String wkt) { java.awt.image.BufferedImage bi = ShapeAnalysis.getBufferedImage(p); org.ejml.simple.SimpleMatrix pts = ShapeAnalysis.BufferedImage2INDArray(bi); Node graph = Quad.defaultGraphIRI; - quads.add(Quad.create(graph, geo, PYR.MeshSurface.asNode(), ResourceFactory.createTypedLiteral(ShapeAnalysis.getMeshSurfaceFeatureValue(p)).asNode())); - quads.add(Quad.create(graph, geo, PYR.PixelSurface.asNode(), ResourceFactory.createTypedLiteral(ShapeAnalysis.getPixelSurfaceFeatureValue(bi)).asNode())); - quads.add(Quad.create(graph, geo, PYR.Perimeter.asNode(), ResourceFactory.createTypedLiteral(ShapeAnalysis.getPerimeter(p)).asNode())); - quads.add(Quad.create(graph, geo, PYR.PerimeterSurfaceRatio.asNode(), ResourceFactory.createTypedLiteral(ShapeAnalysis.getPerimeterSurfaceRatioFeatureValue(p)).asNode())); - quads.add(Quad.create(graph, geo, PYR.Sphericity.asNode(), ResourceFactory.createTypedLiteral(ShapeAnalysis.getSphericityFeatureValue(p)).asNode())); - quads.add(Quad.create(graph, geo, PYR.SphericalDisproportion.asNode(), ResourceFactory.createTypedLiteral(ShapeAnalysis.getSphericalDisproportionFeatureValue(p)).asNode())); - quads.add(Quad.create(graph, geo, PYR.Maximum2DDiameter.asNode(), ResourceFactory.createTypedLiteral(ShapeAnalysis.getMaximum2DDiameterFeatureValue(p)).asNode())); - quads.add(Quad.create(graph, geo, PYR.MajorAxisLength.asNode(), ResourceFactory.createTypedLiteral(ShapeAnalysis.getMajorAxisLengthFeatureValue(pts)).asNode())); - quads.add(Quad.create(graph, geo, PYR.MinorAxisLength.asNode(), ResourceFactory.createTypedLiteral(ShapeAnalysis.getMinorAxisLengthFeatureValue(pts)).asNode())); - quads.add(Quad.create(graph, geo, PYR.Elongation.asNode(), ResourceFactory.createTypedLiteral(ShapeAnalysis.getElongationFeatureValue(pts)).asNode())); - } catch (Exception e) {} + addFinite(quads, graph, geo, PYR.MeshSurface.asNode(), ShapeAnalysis.getMeshSurfaceFeatureValue(p)); + addFinite(quads, graph, geo, PYR.PixelSurface.asNode(), ShapeAnalysis.getPixelSurfaceFeatureValue(bi)); + addFinite(quads, graph, geo, PYR.Perimeter.asNode(), ShapeAnalysis.getPerimeter(p)); + addFinite(quads, graph, geo, PYR.PerimeterSurfaceRatio.asNode(), ShapeAnalysis.getPerimeterSurfaceRatioFeatureValue(p)); + addFinite(quads, graph, geo, PYR.Sphericity.asNode(), ShapeAnalysis.getSphericityFeatureValue(p)); + addFinite(quads, graph, geo, PYR.SphericalDisproportion.asNode(), ShapeAnalysis.getSphericalDisproportionFeatureValue(p)); + addFinite(quads, graph, geo, PYR.Maximum2DDiameter.asNode(), ShapeAnalysis.getMaximum2DDiameterFeatureValue(p)); + addFinite(quads, graph, geo, PYR.MajorAxisLength.asNode(), ShapeAnalysis.getMajorAxisLengthFeatureValue(pts)); + addFinite(quads, graph, geo, PYR.MinorAxisLength.asNode(), ShapeAnalysis.getMinorAxisLengthFeatureValue(pts)); + addFinite(quads, graph, geo, PYR.Elongation.asNode(), ShapeAnalysis.getElongationFeatureValue(pts)); + } catch (Exception e) { + logger.warn("Failed to generate 2D shape features for {}: {}", geo, e.toString()); + } + } + + /** + * Emit a feature only when its value is a real number. Degenerate geometry + * (a zero-area bowtie, say) drives the ratio features to Infinity/NaN, and + * an "INF"^^xsd:double literal stored as a real feature value poisons every + * numeric consumer downstream; the finite features of the same geometry are + * still emitted. + */ + private static void addFinite(Resource geo, org.apache.jena.rdf.model.Property feature, double value) { + if (Double.isFinite(value)) { + geo.addProperty(feature, ResourceFactory.createTypedLiteral(value)); + } else { + logger.warn("Skipping non-finite feature {} for {}", feature.getLocalName(), geo); + } + } + + private static void addFinite(ArrayList quads, Node graph, Node geo, Node feature, double value) { + if (Double.isFinite(value)) { + quads.add(Quad.create(graph, geo, feature, ResourceFactory.createTypedLiteral(value).asNode())); + } else { + logger.warn("Skipping non-finite feature {} for {}", feature, geo); + } } } diff --git a/src/main/java/com/ebremer/beakgraph/hdf5/BitPackedUnSignedLongBuffer.java b/src/main/java/com/ebremer/beakgraph/hdf5/BitPackedUnSignedLongBuffer.java index 18e5510b..5b75add1 100644 --- a/src/main/java/com/ebremer/beakgraph/hdf5/BitPackedUnSignedLongBuffer.java +++ b/src/main/java/com/ebremer/beakgraph/hdf5/BitPackedUnSignedLongBuffer.java @@ -1,5 +1,8 @@ package com.ebremer.beakgraph.hdf5; +import com.ebremer.beakgraph.io.ByteBufferBytes; +import com.ebremer.beakgraph.io.RandomAccessBytes; +import com.ebremer.beakgraph.utils.UTIL; import io.jhdf.api.WritableDataset; import io.jhdf.api.WritableGroup; import java.io.ByteArrayOutputStream; @@ -15,58 +18,95 @@ /** * A buffer that supports writing and reading bit-packed unsigned integers/longs. + * + *

    All read paths go through {@link RandomAccessBytes} with long offsets, so + * a read view is no longer capped at 2 GiB by ByteBuffer's int indexing - the + * backing bytes may be a jHDF-mapped buffer, an FFM-mapped segment, or (for + * the write side after {@link #prepareForReading()}) the internally + * accumulated bytes. */ public class BitPackedUnSignedLongBuffer { private ByteBuffer buffer; + private RandomAccessBytes data; private final int bitWidth; - + // Writing State private long writeAccumulator; private int writeAccumulatorCount; private final boolean usesInternalStream; private ByteArrayOutputStream internalStream; - + // Reading State (Sequential) private long readAccumulator; private int readAccumulatorCount; - + private long readPos; + private final Path path; private long numEntries; public BitPackedUnSignedLongBuffer(Path path, ByteBuffer buffer, long numEntries, int bitWidth) { this.path = path; - // The pack/unpack accumulators (putValue/getValue/get/stream) hold a value together with its - // <=7-bit sub-byte offset in a single 64-bit long. That fits only for width <= 57 (7 + 57 = 64); - // width 64 is also safe because it is byte-aligned (offset always 0). Widths 58..63 would - // silently drop high bits on both read and write, so reject them up front rather than corrupt - // data. Unreachable in practice: width is MinBits(id count) and 57 bits already addresses - // > 1.4e17 ids. - boolean supported = (bitWidth >= 1 && bitWidth <= 57) || bitWidth == 64; - if (!supported) { - throw new IllegalArgumentException( - "Unsupported bit width: " + bitWidth + ". Supported: 1..57, or 64 (byte-aligned)."); - } + checkWidth(bitWidth); this.bitWidth = bitWidth; if (buffer == null) { this.internalStream = new ByteArrayOutputStream(); this.usesInternalStream = true; - this.buffer = ByteBuffer.allocate(0); + this.buffer = ByteBuffer.allocate(0); + this.data = new ByteBufferBytes(this.buffer); this.numEntries = 0; } else { this.buffer = buffer; // Enforce Big Endian so getLong() matches the stream byte order this.buffer.order(ByteOrder.BIG_ENDIAN); + this.data = new ByteBufferBytes(this.buffer); this.usesInternalStream = false; this.numEntries = numEntries; } resetState(); } + private BitPackedUnSignedLongBuffer(RandomAccessBytes data, long numEntries, int bitWidth) { + this.path = null; + checkWidth(bitWidth); + this.bitWidth = bitWidth; + this.buffer = null; // pure read view: the write-side API is unavailable + this.data = data; + this.usesInternalStream = false; + this.numEntries = numEntries; + resetState(); + } + + /** + * A read-only view over already-packed bytes. This is how the HDF5 readers + * construct buffers (via {@code DatasetBytes.of}); unlike the ByteBuffer + * constructor it carries no 2 GiB ceiling. A static factory rather than a + * constructor overload: the writers construct with a null ByteBuffer + * literal, which an overload would make ambiguous. + */ + public static BitPackedUnSignedLongBuffer readView(RandomAccessBytes data, long numEntries, int bitWidth) { + return new BitPackedUnSignedLongBuffer(data, numEntries, bitWidth); + } + + private static void checkWidth(int bitWidth) { + // The pack/unpack accumulators (putValue/getValue/get/stream) hold a value together with its + // <=7-bit sub-byte offset in a single 64-bit long. That fits only for width <= 57 (7 + 57 = 64); + // width 64 is also safe because it is byte-aligned (offset always 0). Widths 58..63 would + // silently drop high bits on both read and write, so reject them up front rather than corrupt + // data. Unreachable in practice: width is MinBits(id count) and 57 bits already addresses + // > 1.4e17 ids. + boolean supported = (bitWidth >= 1 && bitWidth <= 57) || bitWidth == 64; + if (!supported) { + throw new IllegalArgumentException( + "Unsupported bit width: " + bitWidth + ". Supported: 1..57, or 64 (byte-aligned)."); + } + } + private void resetState() { this.writeAccumulator = 0L; this.writeAccumulatorCount = 0; this.readAccumulator = 0L; this.readAccumulatorCount = 0; + this.readPos = 0L; } // --- QUERY METHODS --- @@ -82,14 +122,12 @@ public long select1(long rank) { // so trailing padding / arena tail bytes can't inflate popcount and skew the rank. long fullWords = maxIndex / 64; long fullWordBytes = fullWords * 8; - int bufferLimit = buffer.limit(); - int safeLimit = (int) Math.min(fullWordBytes, bufferLimit) - 8; - int bufferOffset = 0; + long safeLimit = Math.min(fullWordBytes, data.size()) - 8; + long bufferOffset = 0; long i = 0; - // FAST PATH: Iterate over full 64-bit words directly from buffer - // This eliminates the overhead of getWord64() + // FAST PATH: Iterate over full 64-bit words directly from the backing bytes while (bufferOffset <= safeLimit && i < maxIndex) { - long word = buffer.getLong(bufferOffset); + long word = data.getLong(bufferOffset); int pop = Long.bitCount(word); if (currentRank + pop >= rank) { // The target bit is in this word. @@ -105,7 +143,7 @@ public long select1(long rank) { for (; i < maxIndex; i += 64) { long word = getWord64SafeTail(i); // Use existing safe method for the edge int pop = Long.bitCount(word); - + if (currentRank + pop >= rank) { long needed = rank - currentRank; long resultIndex = i + selectInWordSafe(word, needed); @@ -115,92 +153,60 @@ public long select1(long rank) { } return -1; } - + /** - * Finds the index (0-63) of the k-th set bit in a 64-bit word using Broadword Selection. - * This is strictly O(1) -- exactly 6 checks, no loops. - * * @param word The 64-bit word (Big Endian context). - * @param k The rank to find (1-based). - * @return The 0-based index of the k-th set bit (from MSB). + * Finds the index (0-63, from the MSB) of the k-th set bit in a word. + * Delegates to the shared broadword implementation so this linear select1 + * and HDTBitmapDirectory's accelerated select1 can never disagree. */ private int selectInWordSafe(long word, long k) { - int result = 0; - int cnt; - // Check top 32 bits - // shift right to isolate the top 32 bits. - cnt = Long.bitCount(word >>> 32); - if (k > cnt) { - // The target is NOT in the top 32. It's in the lower 32. - // Move the lower 32 bits up, add 32 to our result index, and subtract the count we skipped. - word <<= 32; - result += 32; - k -= cnt; - } - // Check top 16 bits (of the remaining word) - cnt = Long.bitCount(word >>> 48); - if (k > cnt) { - word <<= 16; - result += 16; - k -= cnt; - } - // Check top 8 bits - cnt = Long.bitCount(word >>> 56); - if (k > cnt) { - word <<= 8; - result += 8; - k -= cnt; - } - // Check top 4 bits - cnt = Long.bitCount(word >>> 60); - if (k > cnt) { - word <<= 4; - result += 4; - k -= cnt; - } - // Check top 2 bits - cnt = Long.bitCount(word >>> 62); - if (k > cnt) { - word <<= 2; - result += 2; - k -= cnt; - } - // Check top 1 bit - // If k > cnt (where cnt is 0 or 1), it means the target is the 2nd bit of this pair. - cnt = Long.bitCount(word >>> 63); - if (k > cnt) { - result += 1; - } - return result; + return UTIL.selectInWord(word, k); } // --- WRITE METHODS --- public void writeInteger(int value) { + // Reject values whose bit pattern would not survive the width mask - silent + // truncation here corrupts the dictionary far from the cause. Widths 32 and + // 64 are exempt for negatives: the full two's-complement pattern round-trips + // (the reader casts back to int/long). + if (bitWidth != 32 && bitWidth != 64 && (value < 0 || value > ((1L << bitWidth) - 1))) { + throw new IllegalArgumentException( + "Value " + value + " does not fit in " + bitWidth + " bits"); + } putValue(value & ((bitWidth == 64) ? -1L : (1L << bitWidth) - 1)); numEntries++; } public void writeLong(long value) { + // See writeInteger: only width 64 carries a negative long's full pattern. + if (bitWidth != 64 && (value < 0 || value > ((1L << bitWidth) - 1))) { + throw new IllegalArgumentException( + "Value " + value + " does not fit in " + bitWidth + " bits"); + } putValue(value & ((bitWidth == 64) ? -1L : (1L << bitWidth) - 1)); numEntries++; } - + public long getBitWidth() { return bitWidth; } - + public long getNumEntries() { return numEntries; } private void putValue(long valToPack) { + if (buffer == null) { + throw new IllegalStateException("This buffer is a read view; the write API is unavailable"); + } writeAccumulator = (writeAccumulator << bitWidth) | valToPack; writeAccumulatorCount += bitWidth; while (writeAccumulatorCount >= 8) { int shift = writeAccumulatorCount - 8; byte b = (byte) (writeAccumulator >>> shift); - + if (usesInternalStream) { internalStream.write(b); } else { @@ -209,7 +215,7 @@ private void putValue(long valToPack) { } buffer.put(b); } - + writeAccumulator &= (1L << shift) - 1; writeAccumulatorCount -= 8; } @@ -218,7 +224,7 @@ private void putValue(long valToPack) { public void complete() { if (writeAccumulatorCount > 0) { byte b = (byte) (writeAccumulator << (8 - writeAccumulatorCount)); - + if (usesInternalStream) { internalStream.write(b); } else { @@ -238,8 +244,8 @@ public void complete() { public void prepareForReading() { complete(); if (usesInternalStream) { - byte[] data = internalStream.toByteArray(); - buffer = ByteBuffer.wrap(data); + byte[] bytes = internalStream.toByteArray(); + buffer = ByteBuffer.wrap(bytes); // Enforce Big Endian for internal buffers too buffer.order(ByteOrder.BIG_ENDIAN); } else { @@ -247,8 +253,11 @@ public void prepareForReading() { // Enforce Big Endian buffer.order(ByteOrder.BIG_ENDIAN); } + // Refresh the read view: the flip/wrap above changed the readable window. + data = new ByteBufferBytes(buffer); readAccumulator = 0L; readAccumulatorCount = 0; + readPos = 0L; } public long get(long index) { @@ -256,16 +265,16 @@ public long get(long index) { throw new IndexOutOfBoundsException("Index " + index + " out of bounds [0, " + numEntries + ")"); } long totalBitOffset = index * bitWidth; - int startByteIndex = Math.toIntExact(totalBitOffset / 8); + long startByteIndex = totalBitOffset / 8; int bitOffsetInFirstByte = (int) (totalBitOffset % 8); long acc = 0; int bitsCollected = 0; - int currentByteIndex = startByteIndex; + long currentByteIndex = startByteIndex; while (bitsCollected < bitOffsetInFirstByte + bitWidth) { - if (currentByteIndex >= buffer.limit()) { + if (currentByteIndex >= data.size()) { throw new BufferUnderflowException(); } - acc = (acc << 8) | (buffer.get(currentByteIndex) & 0xFFL); + acc = (acc << 8) | (data.get(currentByteIndex) & 0xFFL); currentByteIndex++; bitsCollected += 8; } @@ -276,24 +285,24 @@ public long get(long index) { } public long getWord64(long bitIndex) { - int byteIndex = Math.toIntExact(bitIndex / 8); + long byteIndex = bitIndex / 8; int bitOffset = (int) (bitIndex % 8); if (bitIndex + 64 > numEntries) { return getWord64SafeTail(bitIndex); } long raw; try { - raw = buffer.getLong(byteIndex); + raw = data.getLong(byteIndex); } catch (IndexOutOfBoundsException | BufferUnderflowException e) { return getWord64SafeTail(bitIndex); } if (bitOffset == 0) { return raw; } - if (byteIndex + 8 >= buffer.limit()) { + if (byteIndex + 8 >= data.size()) { return getWord64SafeTail(bitIndex); } - long nextByte = buffer.get(byteIndex + 8) & 0xFFL; + long nextByte = data.get(byteIndex + 8) & 0xFFL; return (raw << bitOffset) | (nextByte >>> (8 - bitOffset)); } @@ -302,8 +311,8 @@ private long getWord64SafeTail(long bitIndex) { for (int i = 0; i < 64; i++) { acc <<= 1; long entryIdx = bitIndex + i; - if (entryIdx < numEntries) { - acc |= get(entryIdx); + if (entryIdx < numEntries) { + acc |= get(entryIdx); } } return acc; @@ -312,22 +321,22 @@ private long getWord64SafeTail(long bitIndex) { public int get() { return (int) getValue(); } - + public long getLong() { return getValue(); } private long getValue() { while (readAccumulatorCount < bitWidth) { - if (!buffer.hasRemaining()) { + if (readPos >= data.size()) { throw new BufferUnderflowException(); } - readAccumulator = (readAccumulator << 8) | (buffer.get() & 0xFFL); + readAccumulator = (readAccumulator << 8) | (data.get(readPos++) & 0xFFL); readAccumulatorCount += 8; } int shift = readAccumulatorCount - bitWidth; long value = readAccumulator >>> shift; - + readAccumulator &= (1L << shift) - 1; readAccumulatorCount -= bitWidth; return value; @@ -338,6 +347,9 @@ public Path getName() { } public void add(WritableGroup group) { + if (buffer == null) { + throw new IllegalStateException("This buffer is a read view; the write API is unavailable"); + } ByteBuffer dup = buffer.duplicate(); dup.rewind(); byte[] data = new byte[dup.remaining()]; @@ -349,16 +361,20 @@ public void add(WritableGroup group) { ds.putAttribute("numEntries", numEntries); } } - + public long binarySearch(long start, long end, long value) { - long low = start; + long low = start; long high = end; while (low <= high) { long mid = (low + high) >>> 1; long midVal = get(mid); // Internal get is bit-unpacked - if (midVal < value) { + // Unsigned comparison, matching lowerBound/upperBound: the stored + // values are unsigned bit patterns, and mixing signed search with + // unsigned bounds on the same buffer invites subtle disagreement. + int cmp = Long.compareUnsigned(midVal, value); + if (cmp < 0) { low = mid + 1; - } else if (midVal > value) { + } else if (cmp > 0) { high = mid - 1; } else { return mid; // Value found @@ -366,14 +382,14 @@ public long binarySearch(long start, long end, long value) { } return -(low + 1); // Value not found, returns insertion point } - + /** - * Finds the first index in the range [start, end] where the value is + * Finds the first index in the range [start, end] where the value is * greater than or equal to the target. (Unsigned) * @param start * @param end * @param value - * @return + * @return */ public long lowerBound(long start, long end, long value) { long low = start; @@ -394,12 +410,12 @@ public long lowerBound(long start, long end, long value) { } /** - * Finds the last index in the range [start, end] where the value is + * Finds the last index in the range [start, end] where the value is * less than or equal to the target. (Unsigned) * @param start * @param end * @param value - * @return + * @return */ public long upperBound(long start, long end, long value) { long low = start; @@ -418,41 +434,32 @@ public long upperBound(long start, long end, long value) { } return result; } - + /** * Returns a sequential LongStream of all entries in the buffer. - * Note: This method duplicates the underlying buffer to ensure the stream - * doesn't interfere with the current read position of the buffer. - * @return + * Note: the stream reads through the shared read view with its own cursor, + * so it never disturbs this buffer's sequential read position. + * @return */ public LongStream stream() { - // Ensure the buffer is ready for reading (flipped, etc) - // If the user hasn't called prepareForReading, you might want to call it here, - // but typically it's safer to assume the object is in a read-state. - - ByteBuffer readOnlyCopy = buffer.duplicate(); - readOnlyCopy.order(ByteOrder.BIG_ENDIAN); - // Rewind so the stream reads from byte 0 regardless of the live buffer's current - // position (e.g. if sequential getValue()/getLong() calls have advanced it). - readOnlyCopy.rewind(); - // If we are using an internal stream and haven't 'prepared' yet, - // this stream will be empty. Usually, complete() should be called first. - - return StreamSupport.longStream(new BitPackedSpliterator(readOnlyCopy, numEntries, bitWidth), false); + return StreamSupport.longStream(new BitPackedSpliterator(data, numEntries, bitWidth), false); } private static class BitPackedSpliterator extends Spliterators.AbstractLongSpliterator { - private final ByteBuffer localBuf; + private final RandomAccessBytes bytes; + private final long byteSize; private final int bitWidth; private final long totalEntries; private long entriesRead = 0; - + private long pos = 0; + private long acc = 0L; private int accCount = 0; - BitPackedSpliterator(ByteBuffer buffer, long totalEntries, int bitWidth) { + BitPackedSpliterator(RandomAccessBytes bytes, long totalEntries, int bitWidth) { super(totalEntries, Spliterator.IMMUTABLE | Spliterator.ORDERED | Spliterator.SIZED | Spliterator.NONNULL); - this.localBuf = buffer; + this.bytes = bytes; + this.byteSize = bytes.size(); this.totalEntries = totalEntries; this.bitWidth = bitWidth; } @@ -464,20 +471,23 @@ public boolean tryAdvance(java.util.function.LongConsumer action) { } while (accCount < bitWidth) { - if (!localBuf.hasRemaining()) { - // This handles potential padding/truncation issues - break; + if (pos >= byteSize) { + // A buffer too short for its declared entry count is corrupt. + // Fail loudly like the sequential reader does - the old break + // left accCount < bitWidth, making the shift below negative + // (mod-64) and emitting silent garbage values. + throw new BufferUnderflowException(); } - acc = (acc << 8) | (localBuf.get() & 0xFFL); + acc = (acc << 8) | (bytes.get(pos++) & 0xFFL); accCount += 8; } int shift = accCount - bitWidth; long value = acc >>> shift; - + acc &= (1L << shift) - 1; accCount -= bitWidth; - + action.accept(value); entriesRead++; return true; diff --git a/src/main/java/com/ebremer/beakgraph/hdf5/Buffer.java b/src/main/java/com/ebremer/beakgraph/hdf5/Buffer.java deleted file mode 100644 index cc291a92..00000000 --- a/src/main/java/com/ebremer/beakgraph/hdf5/Buffer.java +++ /dev/null @@ -1,70 +0,0 @@ -package com.ebremer.beakgraph.hdf5; - -import java.io.ByteArrayOutputStream; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; - -public class Buffer { - private ByteArrayOutputStream mainbuffer; - private List entries; - - public Buffer() { - mainbuffer = new ByteArrayOutputStream(); - entries = new ArrayList<>(); - } - - public void add(String element) { - int startPosition = mainbuffer.size(); - byte[] bytes = element.getBytes(StandardCharsets.UTF_8); - mainbuffer.write(bytes, 0, bytes.length); - entries.add(new EntryMetadata(startPosition, DataType.STRING)); - } - - public void add(float f) { - int startPosition = mainbuffer.size(); - ByteBuffer buffer = ByteBuffer.allocate(Float.BYTES); - buffer.putFloat(f); - mainbuffer.write(buffer.array(), 0, Float.BYTES); - entries.add(new EntryMetadata(startPosition, DataType.FLOAT)); - } - - public void add(double d) { - int startPosition = mainbuffer.size(); - ByteBuffer buffer = ByteBuffer.allocate(Double.BYTES); - buffer.putDouble(d); - mainbuffer.write(buffer.array(), 0, Double.BYTES); - entries.add(new EntryMetadata(startPosition, DataType.DOUBLE)); - } - - public void add(int n) { - int startPosition = mainbuffer.size(); - ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES); - buffer.putInt(n); - mainbuffer.write(buffer.array(), 0, Integer.BYTES); - entries.add(new EntryMetadata(startPosition, DataType.INTEGER)); - } - - public void add(long n) { - int startPosition = mainbuffer.size(); - ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES); - buffer.putLong(n); - mainbuffer.write(buffer.array(), 0, Long.BYTES); - entries.add(new EntryMetadata(startPosition, DataType.LONG)); - } - - public int size() { - return mainbuffer.size(); - } - - private enum DataType { - STRING, - FLOAT, - DOUBLE, - INTEGER, - LONG - } - - private record EntryMetadata(int startPosition, DataType type) {} -} \ No newline at end of file diff --git a/src/main/java/com/ebremer/beakgraph/hdf5/Index.java b/src/main/java/com/ebremer/beakgraph/hdf5/Index.java index fba2a4d5..67f5631d 100644 --- a/src/main/java/com/ebremer/beakgraph/hdf5/Index.java +++ b/src/main/java/com/ebremer/beakgraph/hdf5/Index.java @@ -4,8 +4,14 @@ import org.apache.jena.sparql.core.Quad; import java.util.Comparator; +/** + * The quad orderings BeakGraph materializes. Exactly these two are written and + * read (HDF5Writer / BGIteratorMaster); the other 22 permutations that used to + * be enumerated here were never constructed and were removed in the dead-code + * sweep - add a constant back only together with a writer and an access path + * that use it. + */ public enum Index { - // === G Indexes (Graph first) === GSPO { // Graph, Subject, Predicate, Object @Override public Comparator getComparator() { @@ -16,16 +22,6 @@ public Comparator getComparator() { .thenComparing(Quad::getObject, NodeComparator.INSTANCE); } }, - GPSO { // Graph, Predicate, Subject, Object - @Override - public Comparator getComparator() { - return Comparator - .comparing(Quad::getGraph, NodeComparator.INSTANCE) - .thenComparing(Quad::getPredicate, NodeComparator.INSTANCE) - .thenComparing(Quad::getSubject, NodeComparator.INSTANCE) - .thenComparing(Quad::getObject, NodeComparator.INSTANCE); - } - }, GPOS { // Graph, Predicate, Object, Subject @Override public Comparator getComparator() { @@ -35,223 +31,7 @@ public Comparator getComparator() { .thenComparing(Quad::getObject, NodeComparator.INSTANCE) .thenComparing(Quad::getSubject, NodeComparator.INSTANCE); } - }, - GOSP { // Graph, Object, Subject, Predicate - @Override - public Comparator getComparator() { - return Comparator - .comparing(Quad::getGraph, NodeComparator.INSTANCE) - .thenComparing(Quad::getObject, NodeComparator.INSTANCE) - .thenComparing(Quad::getSubject, NodeComparator.INSTANCE) - .thenComparing(Quad::getPredicate, NodeComparator.INSTANCE); - } - }, - GSOE { // Graph, Subject, Object, Predicate - @Override - public Comparator getComparator() { - return Comparator - .comparing(Quad::getGraph, NodeComparator.INSTANCE) - .thenComparing(Quad::getSubject, NodeComparator.INSTANCE) - .thenComparing(Quad::getObject, NodeComparator.INSTANCE) - .thenComparing(Quad::getPredicate, NodeComparator.INSTANCE); - } - }, - GOPS { // Graph, Object, Predicate, Subject - @Override - public Comparator getComparator() { - return Comparator - .comparing(Quad::getGraph, NodeComparator.INSTANCE) - .thenComparing(Quad::getObject, NodeComparator.INSTANCE) - .thenComparing(Quad::getPredicate, NodeComparator.INSTANCE) - .thenComparing(Quad::getSubject, NodeComparator.INSTANCE); - } - }, - - // === S Indexes (Subject first) === - SGPO { // Subject, Graph, Predicate, Object - @Override - public Comparator getComparator() { - return Comparator - .comparing(Quad::getSubject, NodeComparator.INSTANCE) - .thenComparing(Quad::getGraph, NodeComparator.INSTANCE) - .thenComparing(Quad::getPredicate, NodeComparator.INSTANCE) - .thenComparing(Quad::getObject, NodeComparator.INSTANCE); - } - }, - SGOP { // Subject, Graph, Object, Predicate - @Override - public Comparator getComparator() { - return Comparator - .comparing(Quad::getSubject, NodeComparator.INSTANCE) - .thenComparing(Quad::getGraph, NodeComparator.INSTANCE) - .thenComparing(Quad::getObject, NodeComparator.INSTANCE) - .thenComparing(Quad::getPredicate, NodeComparator.INSTANCE); - } - }, - SPGO { // Subject, Predicate, Graph, Object - @Override - public Comparator getComparator() { - return Comparator - .comparing(Quad::getSubject, NodeComparator.INSTANCE) - .thenComparing(Quad::getPredicate, NodeComparator.INSTANCE) - .thenComparing(Quad::getGraph, NodeComparator.INSTANCE) - .thenComparing(Quad::getObject, NodeComparator.INSTANCE); - } - }, - SPOG { // Subject, Predicate, Object, Graph - @Override - public Comparator getComparator() { - return Comparator - .comparing(Quad::getSubject, NodeComparator.INSTANCE) - .thenComparing(Quad::getPredicate, NodeComparator.INSTANCE) - .thenComparing(Quad::getObject, NodeComparator.INSTANCE) - .thenComparing(Quad::getGraph, NodeComparator.INSTANCE); - } - }, - SOGP { // Subject, Object, Graph, Predicate - @Override - public Comparator getComparator() { - return Comparator - .comparing(Quad::getSubject, NodeComparator.INSTANCE) - .thenComparing(Quad::getObject, NodeComparator.INSTANCE) - .thenComparing(Quad::getGraph, NodeComparator.INSTANCE) - .thenComparing(Quad::getPredicate, NodeComparator.INSTANCE); - } - }, - SOPG { // Subject, Object, Predicate, Graph - @Override - public Comparator getComparator() { - return Comparator - .comparing(Quad::getSubject, NodeComparator.INSTANCE) - .thenComparing(Quad::getObject, NodeComparator.INSTANCE) - .thenComparing(Quad::getPredicate, NodeComparator.INSTANCE) - .thenComparing(Quad::getGraph, NodeComparator.INSTANCE); - } - }, - - // === P Indexes (Predicate first) === - PSGO { // Predicate, Subject, Graph, Object - @Override - public Comparator getComparator() { - return Comparator - .comparing(Quad::getPredicate, NodeComparator.INSTANCE) - .thenComparing(Quad::getSubject, NodeComparator.INSTANCE) - .thenComparing(Quad::getGraph, NodeComparator.INSTANCE) - .thenComparing(Quad::getObject, NodeComparator.INSTANCE); - } - }, - PSOG { // Predicate, Subject, Object, Graph - @Override - public Comparator getComparator() { - return Comparator - .comparing(Quad::getPredicate, NodeComparator.INSTANCE) - .thenComparing(Quad::getSubject, NodeComparator.INSTANCE) - .thenComparing(Quad::getObject, NodeComparator.INSTANCE) - .thenComparing(Quad::getGraph, NodeComparator.INSTANCE); - } - }, - PGSO { // Predicate, Graph, Subject, Object - @Override - public Comparator getComparator() { - return Comparator - .comparing(Quad::getPredicate, NodeComparator.INSTANCE) - .thenComparing(Quad::getGraph, NodeComparator.INSTANCE) - .thenComparing(Quad::getSubject, NodeComparator.INSTANCE) - .thenComparing(Quad::getObject, NodeComparator.INSTANCE); - } - }, - PGOS { // Predicate, Graph, Object, Subject - @Override - public Comparator getComparator() { - return Comparator - .comparing(Quad::getPredicate, NodeComparator.INSTANCE) - .thenComparing(Quad::getGraph, NodeComparator.INSTANCE) - .thenComparing(Quad::getObject, NodeComparator.INSTANCE) - .thenComparing(Quad::getSubject, NodeComparator.INSTANCE); - } - }, - POSG { // Predicate, Object, Subject, Graph - @Override - public Comparator getComparator() { - return Comparator - .comparing(Quad::getPredicate, NodeComparator.INSTANCE) - .thenComparing(Quad::getObject, NodeComparator.INSTANCE) - .thenComparing(Quad::getSubject, NodeComparator.INSTANCE) - .thenComparing(Quad::getGraph, NodeComparator.INSTANCE); - } - }, - POGS { // Predicate, Object, Graph, Subject - @Override - public Comparator getComparator() { - return Comparator - .comparing(Quad::getPredicate, NodeComparator.INSTANCE) - .thenComparing(Quad::getObject, NodeComparator.INSTANCE) - .thenComparing(Quad::getGraph, NodeComparator.INSTANCE) - .thenComparing(Quad::getSubject, NodeComparator.INSTANCE); - } - }, - - // === O Indexes (Object first) === - OPSG { // Object, Predicate, Subject, Graph - @Override - public Comparator getComparator() { - return Comparator - .comparing(Quad::getObject, NodeComparator.INSTANCE) - .thenComparing(Quad::getPredicate, NodeComparator.INSTANCE) - .thenComparing(Quad::getSubject, NodeComparator.INSTANCE) - .thenComparing(Quad::getGraph, NodeComparator.INSTANCE); - } - }, - OGSP { // Object, Graph, Subject, Predicate - @Override - public Comparator getComparator() { - return Comparator - .comparing(Quad::getObject, NodeComparator.INSTANCE) - .thenComparing(Quad::getGraph, NodeComparator.INSTANCE) - .thenComparing(Quad::getSubject, NodeComparator.INSTANCE) - .thenComparing(Quad::getPredicate, NodeComparator.INSTANCE); - } - }, - OGPS { // Object, Graph, Predicate, Subject - @Override - public Comparator getComparator() { - return Comparator - .comparing(Quad::getObject, NodeComparator.INSTANCE) - .thenComparing(Quad::getGraph, NodeComparator.INSTANCE) - .thenComparing(Quad::getPredicate, NodeComparator.INSTANCE) - .thenComparing(Quad::getSubject, NodeComparator.INSTANCE); - } - }, - OSGP { // Object, Subject, Graph, Predicate - @Override - public Comparator getComparator() { - return Comparator - .comparing(Quad::getObject, NodeComparator.INSTANCE) - .thenComparing(Quad::getSubject, NodeComparator.INSTANCE) - .thenComparing(Quad::getGraph, NodeComparator.INSTANCE) - .thenComparing(Quad::getPredicate, NodeComparator.INSTANCE); - } - }, - OSPG { // Object, Subject, Predicate, Graph - @Override - public Comparator getComparator() { - return Comparator - .comparing(Quad::getObject, NodeComparator.INSTANCE) - .thenComparing(Quad::getSubject, NodeComparator.INSTANCE) - .thenComparing(Quad::getPredicate, NodeComparator.INSTANCE) - .thenComparing(Quad::getGraph, NodeComparator.INSTANCE); - } - }, - OPGS { // Object, Predicate, Graph, Subject - @Override - public Comparator getComparator() { - return Comparator - .comparing(Quad::getObject, NodeComparator.INSTANCE) - .thenComparing(Quad::getPredicate, NodeComparator.INSTANCE) - .thenComparing(Quad::getGraph, NodeComparator.INSTANCE) - .thenComparing(Quad::getSubject, NodeComparator.INSTANCE); - } }; public abstract Comparator getComparator(); -} \ No newline at end of file +} diff --git a/src/main/java/com/ebremer/beakgraph/hdf5/jena/BGIteratorMaster.java b/src/main/java/com/ebremer/beakgraph/hdf5/jena/BGIteratorMaster.java index ab204038..52b4ef2f 100644 --- a/src/main/java/com/ebremer/beakgraph/hdf5/jena/BGIteratorMaster.java +++ b/src/main/java/com/ebremer/beakgraph/hdf5/jena/BGIteratorMaster.java @@ -63,17 +63,41 @@ public BGIteratorMaster(HDF5Reader reader, PositionalDictionaryReader dict, Bind } } else { // G is an unbound variable. Scan only the actual graphs (the columnar - // `graphs` list), not every entity: getGraphs().streamNodes() would also - // yield every URI/BNode in S/O positions, creating one (almost always empty) - // sub-iterator per entity. Bind the graph variable to each graph too, since - // each per-graph sub-iterator only ever sees a concrete graph. + // `graphs` list), not every entity - and LAZILY: constructing every + // graph's sub-iterator up front paid each one's index binary searches + // before the first row came back (spatial stores hold thousands of + // tile graphs). The graph id is taken straight from the columnar list + // (no extract() -> locate() round trip) and pre-bound in a child + // binding: every concrete iterator resolves a pre-bound graph var, + // the binding rides into every result row, and a pattern that also + // uses the var (GRAPH ?g { ?g ?p ?o }) is constrained through the + // ordinary bound-variable substitution instead of a post-filter. Var gVar = Var.alloc(quad.getGraph()); - dict.streamGraphs().forEach(n -> { - Iterator sub = new BGIteratorMaster(reader, dict, bnid, - new Quad(n, quad.getSubject(), quad.getPredicate(), quad.getObject()), filter, nodeTable); - NodeId gId = new NodeId(dict.getGraphs().locate(n), NodeType.GRAPH); - its.add(Iter.map(sub, b -> { b.put(gVar, gId); return b; })); - }); + // GRAPH/SUBJECT/OBJECT share the universal entity id-space, so a + // pre-bound graph id is valid in those positions - but PREDICATE ids + // live in an isolated dictionary. If the graph var also occupies the + // predicate position, fall back to concrete-graph substitution with + // the cross-space compatibility filter. + boolean gVarInPredicate = quad.getPredicate().isVariable() + && quad.getPredicate().getName().equals(gVar.getName()); + if (gVarInPredicate) { + Iterator graphNodes = dict.streamGraphs().iterator(); + its.add(Iter.flatMap(graphNodes, n -> { + NodeId gId = new NodeId(dict.getGraphs().locate(n), NodeType.GRAPH); + Iterator sub = new BGIteratorMaster(reader, dict, bnid, + new Quad(n, quad.getSubject(), quad.getPredicate(), quad.getObject()), filter, nodeTable); + return Iter.removeNulls(Iter.map(sub, + b -> b.putCompatible(gVar, gId, nodeTable) ? b : null)); + })); + } else { + Iterator graphIds = dict.streamGraphIds() + .mapToObj(gid -> new NodeId(gid, NodeType.GRAPH)).iterator(); + its.add(Iter.flatMap(graphIds, gId -> { + BindingNodeId child = new BindingNodeId(bnid); + child.put(gVar, gId); + return new BGIteratorMaster(reader, dict, child, quad, filter, nodeTable); + })); + } } chain = new IteratorChain<>(its); } diff --git a/src/main/java/com/ebremer/beakgraph/hdf5/jena/BGIteratorOS.java b/src/main/java/com/ebremer/beakgraph/hdf5/jena/BGIteratorOS.java index 0bc7ce65..b03e6ba4 100644 --- a/src/main/java/com/ebremer/beakgraph/hdf5/jena/BGIteratorOS.java +++ b/src/main/java/com/ebremer/beakgraph/hdf5/jena/BGIteratorOS.java @@ -119,15 +119,19 @@ public BGIteratorOS(PositionalDictionaryReader dict, IndexReader reader, Binding // F. Initialize if (specificSubId > 0) { - boolean found = false; - for (long k = i; k < j; k++) { - if (Ss.get(k) == specificSubId) { - i = k; - found = true; - break; - } + // The subject list under one (G,P,O) group is sorted ascending (GPOS + // ordering), so binary-search it instead of the previous linear scan. + long lo = i, hi = j - 1, found = -1; + while (lo <= hi) { + long mid = (lo + hi) >>> 1; + long v = Ss.get(mid); + if (v == specificSubId) { found = mid; break; } + if (v < specificSubId) lo = mid + 1; else hi = mid - 1; + } + if (found >= 0) { + i = found; + hasNext = true; } - hasNext = found; } else { advanceToNextValid(); } @@ -181,28 +185,24 @@ private String flipOp(String op) { private void applyBound(Var var, String op, Node value, PositionalDictionaryReader dict, Quad quad) { if (!var.equals(quad.getSubject())) return; - long rawResult = dict.getSubjects().search(value); - long id = (rawResult >= 0) ? rawResult : -rawResult - 1; - boolean found = (rawResult >= 0); + // Snap the bound to the edges of the whole value-equal cluster (degenerates + // to the plain insertion point for non-literal constants); see ValueCluster. + long[] c = ValueCluster.of(dict.getSubjects(), value); switch (op) { case ">" -> { - long target = found ? id + 1 : id; + long target = c[1] + 1; if (Long.compareUnsigned(target, minSubId) > 0) minSubId = target; } case ">=" -> { - if (Long.compareUnsigned(id, minSubId) > 0) minSubId = id; + if (Long.compareUnsigned(c[0], minSubId) > 0) minSubId = c[0]; } case "<" -> { - if (id <= 1) { maxSubId = 0; minSubId = 1; } else { - long target = id - 1; - if (Long.compareUnsigned(target, maxSubId) < 0) maxSubId = target; - } + long target = c[0] - 1; + if (Long.compareUnsigned(target, maxSubId) < 0) maxSubId = target; } case "<=" -> { - long target = found ? id : id - 1; - if (id <= 1 && !found) { maxSubId = 0; minSubId = 1; } else { - if (Long.compareUnsigned(target, maxSubId) < 0) maxSubId = target; - } + long target = c[1]; + if (Long.compareUnsigned(target, maxSubId) < 0) maxSubId = target; } } } diff --git a/src/main/java/com/ebremer/beakgraph/hdf5/jena/BGIteratorPOS.java b/src/main/java/com/ebremer/beakgraph/hdf5/jena/BGIteratorPOS.java index 20a4b2fb..2b824020 100644 --- a/src/main/java/com/ebremer/beakgraph/hdf5/jena/BGIteratorPOS.java +++ b/src/main/java/com/ebremer/beakgraph/hdf5/jena/BGIteratorPOS.java @@ -32,11 +32,13 @@ public class BGIteratorPOS implements Iterator { private long maxObjId = Long.MAX_VALUE; private boolean hasNext = false; private PositionalDictionaryReader dict; + private final NodeTable nodeTable; public BGIteratorPOS(PositionalDictionaryReader dict, IndexReader reader, BindingNodeId bnid, Quad quad, ExprList filter, NodeTable nodeTable) { this.parentBinding = bnid; this.queryQuad = quad; this.dict = dict; + this.nodeTable = nodeTable; this.Bp = reader.getBitmapBuffer('P'); this.Sp = reader.getIDBuffer('P'); @@ -91,8 +93,11 @@ public BGIteratorPOS(PositionalDictionaryReader dict, IndexReader reader, Bindin if (rawOStart == -1 || rawOStart > rawOEnd) return; // C. APPLY FILTER: Narrow the Object Range using Binary Search + // upperBound already returns the last index whose value is <= maxObjId + // (inclusive), so it is used as oEnd directly - subtracting 1 dropped the + // boundary object group from FILTER(?o <= X) results. this.oStart = (minObjId <= 0) ? rawOStart : So.lowerBound(rawOStart, rawOEnd, minObjId); - this.oEnd = (maxObjId == Long.MAX_VALUE) ? rawOEnd : So.upperBound(rawOStart, rawOEnd, maxObjId) - 1; + this.oEnd = (maxObjId == Long.MAX_VALUE) ? rawOEnd : So.upperBound(rawOStart, rawOEnd, maxObjId); if (oStart > oEnd || oStart < 0) return; @@ -154,31 +159,26 @@ private String flipOp(String op) { private void applyBound(Var var, String op, Node value, PositionalDictionaryReader dict, Quad quad) { if (!var.equals(quad.getObject())) return; - long rawResult = dict.getObjects().search(value); - long id = (rawResult >= 0) ? rawResult : -rawResult - 1; - boolean found = (rawResult >= 0); - + // Snap the bound to the edges of the whole value-equal cluster: value-equal + // but term-distinct literals ("5"^^xsd:int vs "5"^^xsd:integer) occupy + // adjacent distinct ids, and the raw exact-term insertion point can land + // inside that cluster, silently dropping qualifying boundary rows. + long[] c = ValueCluster.of(dict.getObjects(), value); switch (op) { case ">" -> { - long target = found ? id + 1 : id; + long target = c[1] + 1; if (Long.compareUnsigned(target, minObjId) > 0) minObjId = target; } case ">=" -> { - if (Long.compareUnsigned(id, minObjId) > 0) minObjId = id; + if (Long.compareUnsigned(c[0], minObjId) > 0) minObjId = c[0]; } case "<" -> { - if (id == 0) { maxObjId = 0; minObjId = 1; } - else { - long target = id - 1; - if (Long.compareUnsigned(target, maxObjId) < 0) maxObjId = target; - } + long target = c[0] - 1; + if (Long.compareUnsigned(target, maxObjId) < 0) maxObjId = target; } case "<=" -> { - long target = found ? id : id - 1; - if (id == 0 && !found) { maxObjId = 0; minObjId = 1; } - else { - if (Long.compareUnsigned(target, maxObjId) < 0) maxObjId = target; - } + long target = c[1]; + if (Long.compareUnsigned(target, maxObjId) < 0) maxObjId = target; } } } @@ -190,25 +190,56 @@ private long select1Safe(HDTBitmapDirectory dir, BitPackedUnSignedLongBuffer fal return (dir != null) ? dir.select1(rank) : fallback.select1(rank); } + // Look-ahead: the next deliverable row, or null. Rows whose repeated-variable + // bindings conflict are skipped here, so hasNext() only answers true when + // next() really has a row to return. + private BindingNodeId pending = null; + private boolean primed = false; + + /** + * Builds the next row whose bindings are all compatible. A variable repeated in + * the pattern (e.g. ?x

    ?x as subject and object) must bind to the same term + * in both positions; putCompatible reports the conflict and the row is skipped + * rather than emitted with the first value. + */ + private BindingNodeId computeNext() { + while (hasNext) { + BindingNodeId result = new BindingNodeId(this.parentBinding); + long currentObjectId = So.get(curOIndex); + long currentSubjectId = Ss.get(curSIndex); + boolean ok = true; + if (queryQuad.getObject().isVariable()) { + ok = result.putCompatible(Var.alloc(queryQuad.getObject()), new NodeId(currentObjectId, NodeType.OBJECT), nodeTable); + } + if (ok && queryQuad.getSubject().isVariable()) { + ok = result.putCompatible(Var.alloc(queryQuad.getSubject()), new NodeId(currentSubjectId, NodeType.SUBJECT), nodeTable); + } + curSIndex++; + advanceToNextValid(); + if (ok) return result; + } + return null; + } + + private void prime() { + if (!primed) { + primed = true; + pending = computeNext(); + } + } + @Override public boolean hasNext() { - return hasNext; + prime(); + return pending != null; } @Override public BindingNodeId next() { - if (!hasNext) throw new NoSuchElementException(); - BindingNodeId result = new BindingNodeId(this.parentBinding); - long currentObjectId = So.get(curOIndex); - long currentSubjectId = Ss.get(curSIndex); - if (queryQuad.getObject().isVariable()) { - result.put(Var.alloc(queryQuad.getObject()), new NodeId(currentObjectId, NodeType.OBJECT)); - } - if (queryQuad.getSubject().isVariable()) { - result.put(Var.alloc(queryQuad.getSubject()), new NodeId(currentSubjectId, NodeType.SUBJECT)); - } - curSIndex++; - advanceToNextValid(); - return result; + prime(); + if (pending == null) throw new NoSuchElementException(); + BindingNodeId r = pending; + pending = computeNext(); + return r; } } diff --git a/src/main/java/com/ebremer/beakgraph/hdf5/jena/BGIteratorSO.java b/src/main/java/com/ebremer/beakgraph/hdf5/jena/BGIteratorSO.java index a9bab5ea..1bc1c990 100644 --- a/src/main/java/com/ebremer/beakgraph/hdf5/jena/BGIteratorSO.java +++ b/src/main/java/com/ebremer/beakgraph/hdf5/jena/BGIteratorSO.java @@ -95,24 +95,32 @@ public BGIteratorSO(PositionalDictionaryReader dict, IndexReader reader, Binding if (rawOStart == -1 || rawOStart > rawOEnd) return; // D. Apply Specific Object Bound or Range Filters - long specificObjId = resolveNode(quad.getObject(), dict.getObjects(), bnid); + // Distinguish "object is an unbound variable" from "object is a concrete + // term (or bound variable)": resolveNode returns -1 for both an unbound + // variable and a term missing from the dictionary, and a missing term must + // yield no match - not a full range scan (which fabricated phantom rows, + // e.g. ASK with a non-existent object answered true). + Node oNode = quad.getObject(); + boolean oUnbound = oNode.isVariable() && (bnid == null || !bnid.containsKey(Var.alloc(oNode))); + + if (oUnbound) { + // Case: Object is a variable, apply min/max ID range filters + this.i = (minObjId <= 0) ? rawOStart : So.lowerBound(rawOStart, rawOEnd, minObjId); + this.j = (maxObjId == Long.MAX_VALUE) ? rawOEnd : So.upperBound(rawOStart, rawOEnd, maxObjId); - if (specificObjId > 0) { + if (this.i != -1 && this.i <= this.j) { + this.hasNext = true; + } + } else { // Case: Object is bound (e.g., G, S, P, O are all known, just checking existence) + long specificObjId = resolveNode(oNode, dict.getObjects(), bnid); + if (specificObjId < 1) return; // concrete term absent from this store -> no match long foundIdx = So.binarySearch(rawOStart, rawOEnd, specificObjId); if (foundIdx >= 0) { this.i = foundIdx; this.j = foundIdx; this.hasNext = true; } - } else { - // Case: Object is a variable, apply min/max ID range filters - this.i = (minObjId <= 0) ? rawOStart : So.lowerBound(rawOStart, rawOEnd, minObjId); - this.j = (maxObjId == Long.MAX_VALUE) ? rawOEnd : So.upperBound(rawOStart, rawOEnd, maxObjId); - - if (this.i != -1 && this.i <= this.j) { - this.hasNext = true; - } } } @@ -157,30 +165,26 @@ private String flipOp(String op) { private void applyBound(Var var, String op, Node value, PositionalDictionaryReader dict, Quad quad) { if (!var.equals(quad.getObject())) return; - long rawResult = dict.getObjects().search(value); - long id = (rawResult >= 0) ? rawResult : -rawResult - 1; - boolean found = (rawResult >= 0); + // Snap the bound to the edges of the whole value-equal cluster: value-equal + // but term-distinct literals ("5"^^xsd:int vs "5"^^xsd:integer) occupy + // adjacent distinct ids, and the raw exact-term insertion point can land + // inside that cluster, silently dropping qualifying boundary rows. + long[] c = ValueCluster.of(dict.getObjects(), value); switch (op) { case ">" -> { - long target = found ? id + 1 : id; + long target = c[1] + 1; if (Long.compareUnsigned(target, minObjId) > 0) minObjId = target; } case ">=" -> { - if (Long.compareUnsigned(id, minObjId) > 0) minObjId = id; + if (Long.compareUnsigned(c[0], minObjId) > 0) minObjId = c[0]; } case "<" -> { - if (id == 0) { maxObjId = 0; minObjId = 1; } - else { - long target = id - 1; - if (Long.compareUnsigned(target, maxObjId) < 0) maxObjId = target; - } + long target = c[0] - 1; + if (Long.compareUnsigned(target, maxObjId) < 0) maxObjId = target; } case "<=" -> { - long target = found ? id : id - 1; - if (id == 0 && !found) { maxObjId = 0; minObjId = 1; } - else { - if (Long.compareUnsigned(target, maxObjId) < 0) maxObjId = target; - } + long target = c[1]; + if (Long.compareUnsigned(target, maxObjId) < 0) maxObjId = target; } } } diff --git a/src/main/java/com/ebremer/beakgraph/hdf5/jena/BGIteratorSPO_All.java b/src/main/java/com/ebremer/beakgraph/hdf5/jena/BGIteratorSPO_All.java index 257bb5a6..1a956bea 100644 --- a/src/main/java/com/ebremer/beakgraph/hdf5/jena/BGIteratorSPO_All.java +++ b/src/main/java/com/ebremer/beakgraph/hdf5/jena/BGIteratorSPO_All.java @@ -31,15 +31,20 @@ public class BGIteratorSPO_All implements Iterator { private long resS, resP, resO; private final long gi; private boolean hasNext = false; - private long minSubId = 0, maxSubId = Long.MAX_VALUE; + // minSubId starts at 1, not 0: real subject ids are 1-based, and the writer pads + // empty graph blocks with (S=0,P=0,O=0) dummy rows. Starting at 0 leaked those + // padding rows as phantom bindings (and extract(0) throws on materialization). + private long minSubId = 1, maxSubId = Long.MAX_VALUE; private long minPid = 0, maxPid = Long.MAX_VALUE; private long minObjId = 0, maxObjId = Long.MAX_VALUE; private final PositionalDictionaryReader dict; + private final NodeTable nodeTable; public BGIteratorSPO_All(PositionalDictionaryReader dict, IndexReader reader, BindingNodeId bnid, Quad quad, ExprList filter, NodeTable nodeTable) { this.parentBinding = bnid; this.queryQuad = quad; this.dict = dict; + this.nodeTable = nodeTable; this.Bs = reader.getBitmapBuffer('S'); this.Ss = reader.getIDBuffer('S'); this.Bp = reader.getBitmapBuffer('P'); @@ -54,7 +59,18 @@ public BGIteratorSPO_All(PositionalDictionaryReader dict, IndexReader reader, Bi analyzeFilters(filter, dict, quad); } - gi = dict.getGraphs().locate(quad.getGraph()); + // Resolve the graph the same way the other three iterators behind + // BGIteratorMaster do: the dispatcher also routes here when the graph + // VARIABLE is pre-bound in the BindingNodeId, and locate() on the raw + // variable node returns -1 - silently yielding nothing for a graph + // that exists. + if (quad.getGraph().isVariable()) { + gi = (bnid != null && bnid.containsKey(Var.alloc(quad.getGraph()))) + ? bnid.get(Var.alloc(quad.getGraph())).getId() + : -1; + } else { + gi = dict.getGraphs().locate(quad.getGraph()); + } if (gi < 1) return; // Honour concrete subject / object terms named directly in the triple @@ -261,22 +277,22 @@ private String flipOp(String op) { } private void applyBound(Var var, String op, Node value, PositionalDictionaryReader dict, Quad quad) { - int type; + int type; if (var.equals(quad.getSubject())) type = 1; else if (var.equals(quad.getPredicate())) type = 2; else if (var.equals(quad.getObject())) type = 3; else return; - long rawResult; - rawResult = switch (type) { - case 1 -> dict.getSubjects().search(value); - case 2 -> dict.getPredicates().search(value); - default -> dict.getObjects().search(value); + // Snap the bound to the edges of the whole value-equal cluster: value-equal + // but term-distinct literals ("5"^^xsd:int vs "5"^^xsd:integer) occupy + // adjacent distinct ids, and the raw exact-term insertion point can land + // inside that cluster, silently dropping qualifying boundary rows. + long[] c = switch (type) { + case 1 -> ValueCluster.of(dict.getSubjects(), value); + case 2 -> ValueCluster.of(dict.getPredicates(), value); + default -> ValueCluster.of(dict.getObjects(), value); }; - long id = (rawResult >= 0) ? rawResult : -rawResult - 1; - boolean found = (rawResult >= 0); - long min, max; switch (type) { case 1 -> { min = minSubId; max = maxSubId; } @@ -285,10 +301,10 @@ private void applyBound(Var var, String op, Node value, PositionalDictionaryRead } switch (op) { - case ">" -> min = Math.max(min, found ? id + 1 : id); - case ">=" -> min = Math.max(min, id); - case "<" -> max = Math.min(max, id - 1); - case "<=" -> max = Math.min(max, id); + case ">" -> min = Math.max(min, c[1] + 1); + case ">=" -> min = Math.max(min, c[0]); + case "<" -> max = Math.min(max, c[0] - 1); + case "<=" -> max = Math.min(max, c[1]); } switch (type) { @@ -298,28 +314,60 @@ private void applyBound(Var var, String op, Node value, PositionalDictionaryRead } } + // Look-ahead: the next deliverable row, or null. Rows whose repeated-variable + // bindings conflict are skipped here, so hasNext() only answers true when + // next() really has a row to return. + private BindingNodeId pending = null; + private boolean primed = false; + + /** + * Builds the next row whose bindings are all compatible. A variable repeated in + * the pattern (?s ?p ?s, ?x ?x ?o, ...) must bind to the same term in every + * position it occupies; putCompatible reports a conflict and the row is skipped + * rather than emitted with the first value. (Subject/object vs predicate repeats + * span two id-spaces - putCompatible resolves those through the node table.) + */ + private BindingNodeId computeNext() { + while (hasNext) { + BindingNodeId result = new BindingNodeId(parentBinding); + boolean ok = true; + if (queryQuad.getGraph().isVariable()) { + ok = result.putCompatible(Var.alloc(queryQuad.getGraph()), new NodeId(gi, NodeType.GRAPH), nodeTable); + } + if (ok && queryQuad.getSubject().isVariable()) { + ok = result.putCompatible(Var.alloc(queryQuad.getSubject()), new NodeId(resS, NodeType.SUBJECT), nodeTable); + } + if (ok && queryQuad.getPredicate().isVariable()) { + ok = result.putCompatible(Var.alloc(queryQuad.getPredicate()), new NodeId(resP, NodeType.PREDICATE), nodeTable); + } + if (ok && queryQuad.getObject().isVariable()) { + ok = result.putCompatible(Var.alloc(queryQuad.getObject()), new NodeId(resO, NodeType.OBJECT), nodeTable); + } + advance(); + if (ok) return result; + } + return null; + } + + private void prime() { + if (!primed) { + primed = true; + pending = computeNext(); + } + } + @Override public boolean hasNext() { - return hasNext; + prime(); + return pending != null; } @Override public BindingNodeId next() { - if (!hasNext) throw new NoSuchElementException(); - BindingNodeId result = new BindingNodeId(parentBinding); - if (queryQuad.getGraph().isVariable()) { - result.put(Var.alloc(queryQuad.getGraph()), new NodeId(gi, NodeType.GRAPH)); - } - if (queryQuad.getSubject().isVariable()) { - result.put(Var.alloc(queryQuad.getSubject()), new NodeId(resS, NodeType.SUBJECT)); - } - if (queryQuad.getPredicate().isVariable()) { - result.put(Var.alloc(queryQuad.getPredicate()), new NodeId(resP, NodeType.PREDICATE)); - } - if (queryQuad.getObject().isVariable()) { - result.put(Var.alloc(queryQuad.getObject()), new NodeId(resO, NodeType.OBJECT)); - } - advance(); - return result; + prime(); + if (pending == null) throw new NoSuchElementException(); + BindingNodeId r = pending; + pending = computeNext(); + return r; } } diff --git a/src/main/java/com/ebremer/beakgraph/hdf5/jena/BGReader.java b/src/main/java/com/ebremer/beakgraph/hdf5/jena/BGReader.java index 79d80895..9c65b011 100644 --- a/src/main/java/com/ebremer/beakgraph/hdf5/jena/BGReader.java +++ b/src/main/java/com/ebremer/beakgraph/hdf5/jena/BGReader.java @@ -4,25 +4,30 @@ import com.ebremer.beakgraph.core.GSPODictionary; import java.net.URI; import java.util.Iterator; -import java.util.stream.Stream; import org.apache.jena.graph.Node; import org.apache.jena.graph.Triple; -import org.apache.jena.sparql.core.Quad; import org.apache.jena.sparql.expr.ExprList; import org.apache.jena.util.iterator.ExtendedIterator; /** + * Read-side storage contract. (Two former members - getNumberOfTriples and + * streamQuads - were stubs that answered 0 / empty and had no callers; they + * were removed rather than left to mislead.) * * @author Erich Bremer */ public interface BGReader extends AutoCloseable { public GSPODictionary getDictionary(); - public int getNumberOfTriples(String ng); public NodeTable getNodeTable(); public Iterator read(Node ng, BindingNodeId bnid, Triple triple, ExprList filter, NodeTable nodeTable); public ExtendedIterator graphBaseFind(Node graph, Triple tp); public Iterator listGraphNodes(); - public Stream streamQuads(); public boolean containsGraph(Node graphNode); public URI getURI(); + + /** + * Whether this reader is still usable. Pool validation uses this to evict + * instances whose underlying storage has been closed. + */ + public default boolean isOpen() { return true; } } diff --git a/src/main/java/com/ebremer/beakgraph/hdf5/jena/BindingBG.java b/src/main/java/com/ebremer/beakgraph/hdf5/jena/BindingBG.java index 018fd732..cd7d5e31 100644 --- a/src/main/java/com/ebremer/beakgraph/hdf5/jena/BindingBG.java +++ b/src/main/java/com/ebremer/beakgraph/hdf5/jena/BindingBG.java @@ -2,10 +2,12 @@ import com.ebremer.beakgraph.core.BeakGraph; import java.util.Iterator; +import org.apache.jena.atlas.iterator.Iter; import org.apache.jena.graph.Node; import org.apache.jena.sparql.core.Var; import org.apache.jena.sparql.engine.binding.Binding; import org.apache.jena.sparql.engine.binding.BindingBase; +import org.apache.jena.sparql.engine.binding.BindingBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -13,28 +15,45 @@ * * @author erich */ -public class BindingBG extends BindingBase { +public class BindingBG extends BindingBase { private final BeakGraph bGraph; private final BindingNodeId idBinding; + private final Binding parent; private static final Logger logger = LoggerFactory.getLogger(BindingBG.class); - + public BindingBG(BindingNodeId idBinding, BeakGraph bGraph) { super(idBinding.getParentBinding()); this.idBinding = idBinding; + this.parent = idBinding.getParentBinding(); this.bGraph = bGraph; } - + public BindingNodeId getBindingId() { return idBinding ; } - + + /** + * Whether this level of the binding exposes {@code var}. BindingBase's + * contract is that a child never re-binds a parent var - size() is + * size1() + parent.size() and vars() concatenates both levels - but the + * id-map deliberately COPIES every parent var (SolverLibBeak.convert) so + * the solvers can substitute at the id level. Exposing those copies here + * as well double-counted every input var: size() lied, vars() yielded + * duplicates, and DISTINCT could not merge a row of this shape with an + * equal-valued row of normal shape. Parent vars therefore resolve through + * the parent, which carries the identical term the ids were derived from - + * including terms not in this store at all (the "does not exist" ids). + */ + private boolean ownVar(Var var) { + return idBinding.containsKey(var) && (parent == null || !parent.contains(var)); + } + @Override protected Node get1(Var var) { - if (idBinding.containsKey(var)) { + if (ownVar(var)) { NodeId id = idBinding.get(var); - // A var bound to "does not exist" (e.g. a VALUES/BIND term not in this store) - // has no node here; return null so BindingBase falls back to the parent binding, - // which still carries the original term. + // A var bound to "does not exist" has no node here; return null so + // BindingBase falls back to the parent binding. if (NodeId.isDoesNotExist(id)) { return null; } @@ -45,26 +64,60 @@ protected Node get1(Var var) { @Override protected Iterator vars1() { - return idBinding.iterator(); + return Iter.filter(idBinding.iterator(), this::ownVar); } @Override protected int size1() { - return idBinding.size(); + int n = 0; + Iterator it = idBinding.iterator(); + while (it.hasNext()) { + if (ownVar(it.next())) { + n++; + } + } + return n; } @Override protected boolean isEmpty1() { - return idBinding.isEmpty(); + return size1() == 0; } @Override protected boolean contains1(Var var) { - return idBinding.containsKey(var); + return ownVar(var); } + /** + * Detaching must materialize the NodeId-backed bindings into plain nodes: + * Jena detaches bindings it copies or spills beyond the execution that + * created them, and a detached binding must not keep resolving lazily + * against this graph's reader (which may be closed by then). Both detach + * paths are overridden - the default "original parent" path would return + * {@code this}, still reader-backed. + */ @Override protected Binding detachWithNewParent(Binding newParent) { - throw new UnsupportedOperationException("Not supported yet."); + return materialize(newParent); + } + + @Override + protected Binding detachWithOriginalParent() { + return materialize(idBinding.getParentBinding()); + } + + private Binding materialize(Binding newParent) { + BindingBuilder builder = Binding.builder(newParent); + Iterator it = idBinding.iterator(); + while (it.hasNext()) { + Var v = it.next(); + if (builder.contains(v)) continue; + Node n = get1(v); + if (n != null) { + builder.add(v, n); + } + } + return builder.build(); } } diff --git a/src/main/java/com/ebremer/beakgraph/hdf5/jena/BindingNodeId.java b/src/main/java/com/ebremer/beakgraph/hdf5/jena/BindingNodeId.java index 0425a868..3c29e5d5 100644 --- a/src/main/java/com/ebremer/beakgraph/hdf5/jena/BindingNodeId.java +++ b/src/main/java/com/ebremer/beakgraph/hdf5/jena/BindingNodeId.java @@ -1,21 +1,14 @@ package com.ebremer.beakgraph.hdf5.jena; +import com.ebremer.beakgraph.core.NodeTable; import java.util.HashMap; -import java.util.Iterator; import java.util.Map; -import java.util.Set; import org.apache.jena.atlas.lib.Map2; +import org.apache.jena.graph.Node; import org.apache.jena.sparql.core.Var; import org.apache.jena.sparql.engine.binding.Binding; public class BindingNodeId extends Map2 { - public static BindingNodeId root = new BindingNodeId(null, null, null) { - @Override - public String toString() { - return ""; - } - }; - // This is the parent binding - which may be several steps up the chain. // This just carried around for later use when we go BindingNodeId back to Binding. private final Binding parentBinding; @@ -44,6 +37,14 @@ public BindingNodeId() { public Binding getParentBinding() { return parentBinding; } + /** + * Binds {@code v} to {@code n}; when {@code v} is already bound, the existing + * binding is kept. Only safe when the caller guarantees the re-put carries the + * same term (e.g. re-binding a value derived from this binding's own entry). + * Row-building code that can see a variable repeated within one triple pattern + * must use {@link #putCompatible} and reject the row on a conflict instead - + * silently keeping the first value made {@code ?s ?p ?s} match every triple. + */ @Override public void put(Var v, NodeId n) { if ( v == null || n == null ) @@ -54,30 +55,46 @@ public void put(Var v, NodeId n) { } } - public void putAll(BindingNodeId other) { - Iterator vIter = other.iterator(); - for (; vIter.hasNext() ; ) { - Var v = vIter.next(); - if ( v == null ) - throw new IllegalArgumentException("Null key"); - NodeId n = other.get(v); - if ( n == null ) - throw new IllegalArgumentException("("+v+","+n+")"); + /** + * Binds {@code v} to {@code n}, or reports a conflict: returns true when the + * binding was added or the existing binding denotes the same term, false when + * {@code v} is already bound to a different term (the caller must reject the + * row). {@code nodeTable} is needed only to compare ids across the predicate / + * entity id-spaces; pass null when that cross-space case cannot occur. + */ + public boolean putCompatible(Var v, NodeId n, NodeTable nodeTable) { + if ( v == null || n == null ) + throw new IllegalArgumentException("("+v+","+n+")"); + NodeId existing = get(v); + if (existing == null) { super.put(v, n); + return true; } + return sameTerm(existing, n, nodeTable); } - + /** - * Returns a Set of entries representing all bindings (local + parent). - * Added to support iteration in BGIterator classes. - * @return + * Whether two NodeIds denote the same RDF term. GRAPH, SUBJECT and OBJECT ids + * share the universal entity id-space (object literals are offset beyond the + * entity range), so within it equal ids mean equal terms. PREDICATE ids live in + * an isolated dictionary, so a predicate/entity pair is resolved to Nodes and + * compared as terms. */ - public Set> entrySet() { - Map allEntries = new HashMap<>(); - for (Var v : this) { - allEntries.put(v, get(v)); + private static boolean sameTerm(NodeId a, NodeId b, NodeTable nodeTable) { + if (a.equals(b)) return true; + NodeType ta = a.getType(); + NodeType tb = b.getType(); + if (ta == NodeType.SPECIAL || tb == NodeType.SPECIAL) return false; // a.equals(b) already said no + boolean aPred = (ta == NodeType.PREDICATE); + boolean bPred = (tb == NodeType.PREDICATE); + if (aPred == bPred) { + // Same id-space (both predicate, or both universal entity/object space). + return a.getId() == b.getId(); } - return allEntries.entrySet(); + if (nodeTable == null) return false; + Node na = nodeTable.getNodeForNodeId(a); + Node nb = nodeTable.getNodeForNodeId(b); + return na != null && na.equals(nb); } @Override diff --git a/src/main/java/com/ebremer/beakgraph/hdf5/jena/HilbertPolygon.java b/src/main/java/com/ebremer/beakgraph/hdf5/jena/HilbertPolygon.java index 812e04cc..76da9c6a 100644 --- a/src/main/java/com/ebremer/beakgraph/hdf5/jena/HilbertPolygon.java +++ b/src/main/java/com/ebremer/beakgraph/hdf5/jena/HilbertPolygon.java @@ -9,7 +9,6 @@ import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.GeometryFactory; import org.locationtech.jts.geom.Polygon; -import org.locationtech.jts.geom.Coordinate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,11 +37,12 @@ public static ArrayList Polygon2Hilbert(Polygon poly) { public static ArrayList Polygon2Hilbert(Polygon poly, int scale) { Envelope env = poly.getEnvelopeInternal(); - // 1. Calculate scaled integer bounds - long minX = (long) Math.floor(env.getMinX()) >> scale; - long maxX = (long) Math.floor(env.getMaxX()) >> scale; - long minY = (long) Math.floor(env.getMinY()) >> scale; - long maxY = (long) Math.floor(env.getMaxY()) >> scale; + // 1. Calculate scaled integer bounds, clamped into the curve's domain + // (negative or >= 2^31 coordinates would otherwise alias via bit masking). + long minX = HilbertSpace.clampToDomain((long) Math.floor(env.getMinX())) >> scale; + long maxX = HilbertSpace.clampToDomain((long) Math.floor(env.getMaxX())) >> scale; + long minY = HilbertSpace.clampToDomain((long) Math.floor(env.getMinY())) >> scale; + long maxY = HilbertSpace.clampToDomain((long) Math.floor(env.getMaxY())) >> scale; // 2. Execute query. We use .stream() to get the ranges. // This is the most common API for davidmoten's hilbert-curve query builder. @@ -61,16 +61,33 @@ public static ArrayList Polygon2Hilbert(Polygon poly, int scale) { return compact(filteredRanges); } + /** Cap on per-range cell tests; longer ranges are kept outright (superset-safe). */ + private static final int MAX_RELEVANCE_CELLS = 64; + + /** + * Whether any WHOLE CELL of the range intersects the polygon. The old test + * asked whether the polygon covers the range's two endpoint cell-corner + * POINTS, which dropped ranges the geometry genuinely intersects - any + * polygon smaller than a cell returned an EMPTY cover, and diagonal strips + * lost their interior cells: silent false negatives for every consumer of + * this Polygon-to-Hilbert API. Keeping a range can only add false + * positives, never lose a match, so ranges longer than the cap are kept + * without examination. + */ private static boolean isRangeRelevant(Range range, Polygon poly, int scale) { - // Check the start point of the range - long[] startCoord = hc.point(range.low()); - if (poly.covers(gf.createPoint(new Coordinate(startCoord[0] << scale, startCoord[1] << scale)))) { + if (range.high() - range.low() + 1 > MAX_RELEVANCE_CELLS) { return true; } - - // Check the end point of the range - long[] endCoord = hc.point(range.high()); - return poly.covers(gf.createPoint(new Coordinate(endCoord[0] << scale, endCoord[1] << scale))); + double cellSize = (double) (1L << scale); + for (long d = range.low(); d <= range.high(); d++) { + long[] p = hc.point(d); + double x = (double) (p[0] << scale); + double y = (double) (p[1] << scale); + if (poly.intersects(gf.toGeometry(new Envelope(x, x + cellSize, y, y + cellSize)))) { + return true; + } + } + return false; } /** diff --git a/src/main/java/com/ebremer/beakgraph/hdf5/jena/NumScale.java b/src/main/java/com/ebremer/beakgraph/hdf5/jena/NumScale.java deleted file mode 100644 index 12742e05..00000000 --- a/src/main/java/com/ebremer/beakgraph/hdf5/jena/NumScale.java +++ /dev/null @@ -1,58 +0,0 @@ -package com.ebremer.beakgraph.hdf5.jena; - -/** - * Utility class to calculate image scaling requirements. - */ -public class NumScale { - - /** - * Calculates reduction steps using a loop (Iterative approach). - * This is generally preferred for this use case because it avoids floating-point - * precision errors entirely and N is very small (rarely > 10). - * @param width - */ - public static int calculateScaleSteps(int width, int height, int boxSize) { - if (width <= 0 || height <= 0 || boxSize <= 0) { - System.err.println("Error: Dimensions must be greater than 0."); - return -1; - } - - int steps = 0; - double currentMaxDimension = Math.max(width, height); - - while (currentMaxDimension > boxSize) { - currentMaxDimension = currentMaxDimension / 2.0; - steps++; - } - - return steps; - } - - /** - * Calculates reduction steps using Logarithms (Mathematical approach). - * Formula: steps = ceil( log2(maxDimension / boxSize) ) - * * Note: This requires careful handling of floating point precision (epsilon) - * to avoid off-by-one errors when the result is extremely close to an integer. - */ - public static int calculateScaleStepsLog(int width, int height, int boxSize) { - if (width <= 0 || height <= 0 || boxSize <= 0) return -1; - - int maxDim = Math.max(width, height); - - // If it already fits, 0 steps - if (maxDim <= boxSize) return 0; - - // Calculate the ratio (e.g., 1024 / 256 = 4.0) - double ratio = (double) maxDim / boxSize; - - // log2(x) = ln(x) / ln(2) - double stepsExact = Math.log(ratio) / Math.log(2); - - // We subtract a tiny epsilon before ceiling to handle cases where - // floating point noise makes an exact integer result slightly higher. - // e.g., if result is 3.00000000000004, ceil would return 4 without this. - double epsilon = 1e-10; - - return (int) Math.ceil(stepsExact - epsilon); - } -} \ No newline at end of file diff --git a/src/main/java/com/ebremer/beakgraph/hdf5/jena/OpExecutorBG.java b/src/main/java/com/ebremer/beakgraph/hdf5/jena/OpExecutorBG.java index 0b438295..83860759 100644 --- a/src/main/java/com/ebremer/beakgraph/hdf5/jena/OpExecutorBG.java +++ b/src/main/java/com/ebremer/beakgraph/hdf5/jena/OpExecutorBG.java @@ -1,31 +1,21 @@ package com.ebremer.beakgraph.hdf5.jena; import com.ebremer.beakgraph.core.BeakGraph; -import com.ebremer.beakgraph.hdf5.readers.PositionalDictionaryReader; import org.apache.jena.graph.Graph; -import org.apache.jena.graph.Node; -import org.apache.jena.graph.Triple; import org.apache.jena.sparql.ARQInternalErrorException; import org.apache.jena.sparql.algebra.Op; import org.apache.jena.sparql.algebra.op.*; import org.apache.jena.sparql.core.BasicPattern; -import org.apache.jena.sparql.core.Quad; import org.apache.jena.sparql.core.Substitute; -import org.apache.jena.sparql.core.Var; import org.apache.jena.sparql.engine.ExecutionContext; import org.apache.jena.sparql.engine.QueryIterator; -import org.apache.jena.sparql.engine.binding.Binding; -import org.apache.jena.sparql.engine.binding.BindingFactory; import org.apache.jena.sparql.engine.iterator.QueryIterPeek; -import org.apache.jena.sparql.engine.iterator.QueryIterPlainWrapper; import org.apache.jena.sparql.engine.main.OpExecutor; import org.apache.jena.sparql.engine.main.OpExecutorFactory; import org.apache.jena.sparql.engine.main.QC; import org.apache.jena.sparql.engine.optimizer.reorder.ReorderProc; import org.apache.jena.sparql.engine.optimizer.reorder.ReorderTransformation; import org.apache.jena.sparql.expr.ExprList; -import java.util.Iterator; -import java.util.stream.Stream; import org.apache.jena.sparql.algebra.optimize.TransformFilterPlacement; /** @@ -44,104 +34,20 @@ public OpExecutor create(ExecutionContext execCxt) { private final boolean isForBeakGraph; - private enum Position { SUBJECT, PREDICATE, OBJECT, GRAPH, NONE } - public OpExecutorBG(ExecutionContext execCtx) { super(execCtx); isForBeakGraph = execCtx.getActiveGraph() instanceof BeakGraph; } - /** - * OPTIMIZATION: Overriding execute for OpDistinct to catch schema-listing queries. - * Logic: If we are asking for DISTINCT elements across a triple or quad pattern, - * we skip the index scan and stream directly from the specific Dictionary position. - * @param opDistinct - * @param input - * @return - */ - @Override - protected QueryIterator execute(OpDistinct opDistinct, QueryIterator input) { - if (!isForBeakGraph) { - return super.execute(opDistinct, input); - } - - Op subOp = opDistinct.getSubOp(); - - // Pattern: Distinct -> Project(?v) -> BGP(?s ?p ?o) - if (subOp instanceof OpProject project && project.getVars().size() == 1) { - Var targetVar = project.getVars().get(0); - Op innerOp = project.getSubOp(); - - Position pos = getFullScanPosition(innerOp, targetVar); - - if (pos != Position.NONE) { - // Determine if we can resolve the BeakGraph - Graph active = execCxt.getActiveGraph(); - if (active instanceof BeakGraph bg) { - PositionalDictionaryReader dict = (PositionalDictionaryReader) bg.getReader().getDictionary(); - - Stream stream = switch (pos) { - case GRAPH -> dict.streamGraphs(); - case PREDICATE -> dict.streamPredicates(); - case SUBJECT -> dict.streamSubjects(); - case OBJECT -> dict.streamObjects(); - default -> null; - }; - - if (stream != null) { - return new QueryIterDictionary(targetVar, stream, execCxt); - } - } - } - } - - return super.execute(opDistinct, input); - } - - /** - * Recursively checks if the pattern is a simple "?s ?p ?o" scan that encompasses - * the entire space, and identifies the target variable's position. - */ - private Position getFullScanPosition(Op op, Var targetVar) { - // Unwrap OpFilter (if any exists around the pattern) - if (op instanceof OpFilter filter) { - return getFullScanPosition(filter.getSubOp(), targetVar); - } - // Case 1: Triple { ?s ?p ?o } - if (op instanceof OpBGP bgp && bgp.getPattern().size() == 1) { - Triple t = bgp.getPattern().get(0); - if (t.getSubject().isVariable() && t.getPredicate().isVariable() && t.getObject().isVariable()) { - if (t.getPredicate().equals(targetVar)) return Position.PREDICATE; - if (t.getSubject().equals(targetVar)) return Position.SUBJECT; - if (t.getObject().equals(targetVar)) return Position.OBJECT; - } - } - // Case 2: Quad { GRAPH ?g { ?s ?p ?o } } - if (op instanceof OpQuadPattern qp && qp.getPattern().size() == 1) { - Quad q = qp.getPattern().get(0); - if (q.getGraph().isVariable() && q.getSubject().isVariable() && - q.getPredicate().isVariable() && q.getObject().isVariable()) { - if (q.getGraph().equals(targetVar)) return Position.GRAPH; - if (q.getPredicate().equals(targetVar)) return Position.PREDICATE; - if (q.getSubject().equals(targetVar)) return Position.SUBJECT; - if (q.getObject().equals(targetVar)) return Position.OBJECT; - } - } - // Case 3: OpGraph ?g { ?s ?p ?o } - if (op instanceof OpGraph opGraph && opGraph.getNode().isVariable()) { - if (opGraph.getNode().equals(targetVar)) { - if (opGraph.getSubOp() instanceof OpBGP bgp && bgp.getPattern().size() == 1) { - Triple t = bgp.getPattern().get(0); - if (t.getSubject().isVariable() && t.getPredicate().isVariable() && t.getObject().isVariable()) { - return Position.GRAPH; - } - } - } else { - return getFullScanPosition(opGraph.getSubOp(), targetVar); - } - } - return Position.NONE; - } + // NOTE: an earlier version intercepted OpDistinct ("SELECT DISTINCT ?p { ?s ?p ?o }") + // and streamed the dictionary's per-position id lists instead of executing the + // query. That was removed as unsound: the columnar lists are file-global (they + // span every named graph, including the always-present VoID metadata graph, so + // default-graph queries over-reported terms), any FILTER wrapped around the + // pattern was silently dropped, DISTINCT ?g included the default graph, and the + // incoming iterator (join semantics) was discarded. A correct fast path would + // need per-graph id lists in the format plus filter/join guards; until then + // DISTINCT executes normally. @Override protected QueryIterator execute(OpPropFunc opPropFunc, QueryIterator input) { @@ -222,13 +128,13 @@ public OpExecutor create(ExecutionContext execCxt) { private static class OpExecutorPlainBeak extends OpExecutor { final ExprList filter; - + public OpExecutorPlainBeak(ExecutionContext execCxt) { super(execCxt); ExecutionContextBG ecr = (ExecutionContextBG) execCxt; filter = ecr.getFilter(); } - + @Override public QueryIterator execute(OpBGP opBGP, QueryIterator input) { Graph g = execCxt.getActiveGraph(); @@ -239,22 +145,4 @@ public QueryIterator execute(OpBGP opBGP, QueryIterator input) { return super.execute(opBGP, input) ; } } - - /** - * Inner class to wrap the Dictionary Stream as a Jena QueryIterator. - */ - private static class QueryIterDictionary extends QueryIterPlainWrapper { - public QueryIterDictionary(Var var, Stream nodes, ExecutionContext execCxt) { - super(convert(var, nodes.iterator()), execCxt); - } - - private static Iterator convert(Var var, Iterator nodes) { - return new Iterator<>() { - @Override public boolean hasNext() { return nodes.hasNext(); } - @Override public Binding next() { - return BindingFactory.binding(var, nodes.next()); - } - }; - } - } } diff --git a/src/main/java/com/ebremer/beakgraph/hdf5/jena/PatternMatchBG.java b/src/main/java/com/ebremer/beakgraph/hdf5/jena/PatternMatchBG.java index fdc4c013..4ada20d2 100644 --- a/src/main/java/com/ebremer/beakgraph/hdf5/jena/PatternMatchBG.java +++ b/src/main/java/com/ebremer/beakgraph/hdf5/jena/PatternMatchBG.java @@ -34,32 +34,24 @@ public static QueryIterator execute(BeakGraph bGraph, BasicPattern bgp, QueryIte List killList = new ArrayList<>(); Iterator chain = Iter.map(input, SolverLibBeak.convFromBinding(bGraph)); - // Check for spatial filter optimization opportunity + // Spatial index acceleration: seed the chain with recall-safe candidates + // for the geometry's subject. The sfIntersects filter itself STAYS in the + // plan (the OpFilter wrapper produced by TransformFilterPlacement) - it is + // the verification stage that removes the candidates' false positives with + // real JTS geometry. The old code removed it, which made the lossy index + // pre-filter the final answer. SpatialContext spatialCtx = getSpatialContext(filter); - ExprList modifiedFilter = filter; - Triple triggerTriple = null; - if (spatialCtx != null) { - triggerTriple = findTriggerTriple(triples, spatialCtx.geometryVar); - + Triple triggerTriple = findTriggerTriple(triples, spatialCtx.geometryVar); if (triggerTriple != null) { - Var varToBind = spatialCtx.geometryVar; - if (triggerTriple.getObject().isVariable() && - triggerTriple.getObject().equals(spatialCtx.geometryVar)) { - if (triggerTriple.getSubject().isVariable()) { - varToBind = (Var) triggerTriple.getSubject(); - } - } - - chain = new SpatialIndexIterator(chain, bGraph, varToBind, spatialCtx); - modifiedFilter = removeSpatialFilter(filter); + chain = new SpatialIndexIterator(chain, bGraph, (Var) triggerTriple.getSubject(), spatialCtx); } } - - // Execute all triple patterns + + // Execute all triple patterns (the ExprList is range-pushdown hints only; + // full filter semantics are enforced by the surrounding OpFilter). for (Triple triple : triples) { - ExprList filterToUse = (triggerTriple != null && triple.equals(triggerTriple)) ? null : modifiedFilter; - chain = solve(bGraph, triple, filterToUse, chain, execCxt); + chain = solve(bGraph, triple, filter, chain, execCxt); chain = makeAbortable(chain, killList); } @@ -171,48 +163,31 @@ private static Integer extractScale(Expr expr) { return null; } - private static ExprList removeSpatialFilter(ExprList filters) { - if (filters == null || filters.isEmpty()) { - return null; - } - - ExprList newFilters = new ExprList(); - - for (Expr e : filters) { - if (e.isFunction()) { - ExprFunction func = e.getFunction(); - if (func instanceof E_Function) { - if (func.getFunctionIRI().equals(SF_INTERSECTS)) { - continue; - } - } - } - newFilters.add(e); - } - - return newFilters.isEmpty() ? null : newFilters; - } - + /** + * The triple whose VARIABLE subject the spatial index can seed. The + * candidates are geometry SUBJECT ids (the writer indexes the subject of + * every quad with a wktLiteral object, so any row that survives the + * sfIntersects verification has its subject in the candidate set - the + * seeding stays recall-safe for any predicate). The geometry variable must + * sit in the OBJECT position: it binds WKT literals, and seeding it - or + * any other position - injects subject ids into a literal slot, so every + * candidate row fails the triple pattern and the query silently returns + * nothing (the old behavior whenever the subject was concrete). With no + * variable subject there is nothing to seed: return null and leave the + * work to the sfIntersects OpFilter, which always stays in the plan - + * skipping the index costs speed, never rows. + */ private static Triple findTriggerTriple(List triples, Var targetVar) { String targetName = targetVar.getName(); - + for (Triple t : triples) { - if (t.getObject().isVariable() && - t.getObject().getName().equals(targetName)) { - return t; - } - - if (t.getSubject().isVariable() && - t.getSubject().getName().equals(targetName)) { - return t; - } - - if (t.getPredicate().isVariable() && - t.getPredicate().getName().equals(targetName)) { + if (t.getObject().isVariable() + && t.getObject().getName().equals(targetName) + && t.getSubject().isVariable()) { return t; } } - + return null; } } diff --git a/src/main/java/com/ebremer/beakgraph/hdf5/jena/QueryEngineBeak.java b/src/main/java/com/ebremer/beakgraph/hdf5/jena/QueryEngineBeak.java deleted file mode 100644 index fabf614f..00000000 --- a/src/main/java/com/ebremer/beakgraph/hdf5/jena/QueryEngineBeak.java +++ /dev/null @@ -1,59 +0,0 @@ -package com.ebremer.beakgraph.hdf5.jena; - -import org.apache.jena.query.Query; -import org.apache.jena.sparql.ARQInternalErrorException; -import org.apache.jena.sparql.algebra.Op; -import org.apache.jena.sparql.core.DatasetGraph; -import org.apache.jena.sparql.engine.Plan; -import org.apache.jena.sparql.engine.QueryEngineFactory; -import org.apache.jena.sparql.engine.QueryEngineRegistry; -import org.apache.jena.sparql.engine.binding.Binding; -import org.apache.jena.sparql.engine.main.QueryEngineMain; -import org.apache.jena.sparql.util.Context; - -/** - * - * @author erich - */ -public class QueryEngineBeak extends QueryEngineMain { - protected static final QueryEngineFactory factory = new TQueryEngineFactory() ; - static public QueryEngineFactory getFactory() { return factory; } - static public void register(){ QueryEngineRegistry.addFactory(factory); } - static public void unregister(){ QueryEngineRegistry.removeFactory(factory); } - - public QueryEngineBeak(Query query, DatasetGraph dataset, Binding input, Context context) { - super(query, dataset, input, context); - } - - public QueryEngineBeak(Op op, DatasetGraph dataset, Binding input, Context context) { - super(op, dataset, input, context); - } - - protected static class TQueryEngineFactory implements QueryEngineFactory { - - private static boolean isHandledByBG(DatasetGraph dataset) { - return false ; - } - - @Override - public boolean accept(Query query, DatasetGraph dataset, Context context) { - return isHandledByBG(dataset); - } - - @Override - public boolean accept(Op op, DatasetGraph dataset, Context context) { - return isHandledByBG(dataset); - } - - @Override - public Plan create(Query query, DatasetGraph dataset, Binding initial, Context context) { - QueryEngineBeak engine = new QueryEngineBeak(query, dataset, initial, context); - return engine.getPlan(); - } - - @Override - public Plan create(Op op, DatasetGraph dataset, Binding inputBinding, Context context) { - throw new ARQInternalErrorException("TQueryEngine: factory called directly with an algebra expression"); - } - } -} diff --git a/src/main/java/com/ebremer/beakgraph/hdf5/jena/QueryIterDictionary.java b/src/main/java/com/ebremer/beakgraph/hdf5/jena/QueryIterDictionary.java deleted file mode 100644 index 6d37c3ec..00000000 --- a/src/main/java/com/ebremer/beakgraph/hdf5/jena/QueryIterDictionary.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.ebremer.beakgraph.hdf5.jena; - -import org.apache.jena.graph.Node; -import org.apache.jena.sparql.core.Var; -import org.apache.jena.sparql.engine.binding.Binding; -import org.apache.jena.sparql.engine.binding.BindingFactory; -import org.apache.jena.sparql.engine.iterator.QueryIterPlainWrapper; -import java.util.Iterator; -import java.util.stream.Stream; -import org.apache.jena.sparql.engine.ExecutionContext; - -public class QueryIterDictionary extends QueryIterPlainWrapper { - public QueryIterDictionary(Var var, Stream nodes, ExecutionContext execCxt) { - super(convert(var, nodes.iterator()), execCxt); - } - - private static Iterator convert(Var var, Iterator nodes) { - return new Iterator<>() { - @Override public boolean hasNext() { return nodes.hasNext(); } - @Override public Binding next() { - return BindingFactory.binding(var, nodes.next()); - } - }; - } -} \ No newline at end of file diff --git a/src/main/java/com/ebremer/beakgraph/hdf5/jena/SimpleNodeTable.java b/src/main/java/com/ebremer/beakgraph/hdf5/jena/SimpleNodeTable.java index f854d7a0..e96e6376 100644 --- a/src/main/java/com/ebremer/beakgraph/hdf5/jena/SimpleNodeTable.java +++ b/src/main/java/com/ebremer/beakgraph/hdf5/jena/SimpleNodeTable.java @@ -5,8 +5,11 @@ import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; import org.apache.jena.graph.Node; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class SimpleNodeTable implements NodeTable { + private static final Logger logger = LoggerFactory.getLogger(SimpleNodeTable.class); private final PositionalDictionaryReader dict; @@ -85,11 +88,15 @@ public NodeId getNodeIdForNode(Node n) { } NodeId nid = findInDictionaries(n); - + if (nid != NodeId.NodeDoesNotExist) { nodeId2nodemap.put(nid, n); - node2nodeIdmap.put(n, nid); // authoritative, deterministic Node -> NodeId mapping } + // Cache misses too: the store is immutable, so absence is permanent, and + // an uncached miss re-ran up to two dictionary binary searches on every + // lookup of the same foreign term (VALUES/BIND-heavy queries). The shared + // does-not-exist sentinel is deliberately NOT seeded into nodeId2nodemap. + node2nodeIdmap.put(n, nid); // authoritative, deterministic Node -> NodeId mapping return nid; } @@ -128,8 +135,8 @@ public Node getNodeForNodeId(NodeId id) { public void status() { // Caffeine evaluates size concurrently, so we use estimatedSize() - IO.println(String.format("nodeId2nodemap size: %d, node2nodeIdmap size: %d", - nodeId2nodemap.estimatedSize(), node2nodeIdmap.estimatedSize())); + logger.debug("nodeId2nodemap size: {}, node2nodeIdmap size: {}", + nodeId2nodemap.estimatedSize(), node2nodeIdmap.estimatedSize()); } @Override diff --git a/src/main/java/com/ebremer/beakgraph/hdf5/jena/SpatialIndexIterator.java b/src/main/java/com/ebremer/beakgraph/hdf5/jena/SpatialIndexIterator.java index 8fc02b77..e01b4b89 100644 --- a/src/main/java/com/ebremer/beakgraph/hdf5/jena/SpatialIndexIterator.java +++ b/src/main/java/com/ebremer/beakgraph/hdf5/jena/SpatialIndexIterator.java @@ -3,138 +3,179 @@ import com.ebremer.beakgraph.Params; import com.ebremer.beakgraph.core.BeakGraph; import com.ebremer.beakgraph.core.NodeTable; +import com.ebremer.beakgraph.hdf5.Index; +import com.ebremer.beakgraph.hdf5.readers.HDF5Reader; +import com.ebremer.beakgraph.hdf5.readers.IndexReader; +import com.ebremer.beakgraph.hdf5.readers.PositionalDictionaryReader; +import static com.ebremer.beakgraph.hdf5.writers.PositionalDictionaryWriterBuilder.HILBERT_CELL_NS; +import static com.ebremer.beakgraph.hdf5.writers.PositionalDictionaryWriterBuilder.MAX_INDEX_SCALE; +import com.ebremer.beakgraph.utils.ImageTools; +import com.ebremer.halcyon.hilbert.HilbertSpace; +import java.util.ArrayList; +import java.util.Collections; import java.util.Iterator; +import java.util.LinkedHashSet; import java.util.List; -import java.util.NoSuchElementException; import org.apache.jena.atlas.iterator.Iter; import org.apache.jena.graph.Node; -import org.apache.jena.query.Dataset; -import org.apache.jena.query.ParameterizedSparqlString; -import org.apache.jena.query.QueryExecution; -import org.apache.jena.query.QueryExecutionFactory; -import org.apache.jena.query.ResultSet; +import org.apache.jena.graph.NodeFactory; +import org.apache.jena.sparql.core.Quad; import org.apache.jena.sparql.core.Var; +import org.apache.jena.sparql.expr.E_GreaterThanOrEqual; +import org.apache.jena.sparql.expr.E_LessThanOrEqual; +import org.apache.jena.sparql.expr.ExprList; +import org.apache.jena.sparql.expr.ExprVar; +import org.apache.jena.sparql.expr.NodeValue; import org.davidmoten.hilbert.Range; +import org.locationtech.jts.geom.Envelope; +import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.io.WKTReader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Optimized SpatialIndexIterator using lazy flatMapping and efficient pool management. - * @author erich + * Recall-safe spatial candidate generator. The writer stores, per geometry + * part, the Hilbert indices of the whole cells covering its bbox at the + * coarsest scale where that cover is small (hal:hilbertCell{scale}). This + * iterator covers the QUERY region's bbox with Hilbert ranges at every scale - + * with the same floor snapping the writer uses - and collects every subject + * whose stored cell falls inside a range, by scanning the GPOS index directly + * with value-range pushdown (no SPARQL sub-execution, no leaked executions). + *

    + * Two bboxes that overlap share at least one whole cell at any scale, so a + * stored geometry overlapping the query region is ALWAYS produced: candidates + * are a superset (bbox-cover overlap), never a miss. Both sides clamp their + * bbox into the non-negative Hilbert domain with the same monotone projection, + * which preserves that overlap for coordinates in negative space too (they + * cluster on the axis cells - coarse, but recall-safe). The geof:sfIntersects + * filter stays in the plan and removes the false positives with real JTS + * geometry. Candidates are deduplicated across scales and ranges, so one + * geometry yields one row. */ public class SpatialIndexIterator implements Iterator { private static final Logger logger = LoggerFactory.getLogger(SpatialIndexIterator.class); + /** Cap on query-side range fragmentation; merging beyond the cap only widens the superset. */ + private static final int MAX_QUERY_RANGES = 256; + private final Iterator outputIterator; - private int scale = 0; public SpatialIndexIterator(Iterator input, BeakGraph bGraph, Var targetVar, PatternMatchBG.SpatialContext context) { - logger.trace("SpatialIndexIterator: {} at scale {}", context.searchRegionWKT, context.scale); - List ranges = HilbertPolygon.Polygon2Hilbert(context.searchRegionWKT, 0); - logger.trace("# of ranges : {}", ranges.size()); - NodeTable currentTable = bGraph.getReader().getNodeTable(); - this.outputIterator = Iter.flatMap(input, parent -> { - Iterator lazyCandidateNodes = new LazyCandidateIterator(ranges, bGraph, context.scale); - - return Iter.map(lazyCandidateNodes, node -> { - NodeId nodeId = currentTable.getNodeIdForNode(node); + NodeTable nodeTable = bGraph.getReader().getNodeTable(); + // The candidate set depends only on the (constant) query region, so it is + // computed once and reused for every incoming row - the previous version + // re-ran every range sub-query per parent binding. + final List candidates; + Envelope env = queryEnvelope(context.searchRegionWKT); + if (env != null && bGraph.getReader() instanceof HDF5Reader hdf5) { + candidates = collectCandidates(hdf5, env); + } else { + candidates = List.of(); + } + logger.trace("sfIntersects region {} -> {} candidate geometries", context.searchRegionWKT, candidates.size()); + this.outputIterator = Iter.flatMap(input, parent -> + Iter.removeNulls(Iter.map(candidates.iterator(), id -> { BindingNodeId child = new BindingNodeId(parent); - child.put(targetVar, nodeId); - return child; - }); - }); - } - - @Override - public boolean hasNext() { - return outputIterator.hasNext(); + // A conflicting pre-existing binding for the target var drops the row. + return child.putCompatible(targetVar, new NodeId(id, NodeType.SUBJECT), nodeTable) ? child : null; + }))); } - @Override - public BindingNodeId next() { - return outputIterator.next(); + private static Envelope queryEnvelope(String wkt) { + try { + Geometry g = new WKTReader().read(ImageTools.stripCrs(wkt)); + return g.isEmpty() ? null : g.getEnvelopeInternal(); + } catch (Exception e) { + logger.warn("Unparseable sfIntersects search region; no index candidates: {}", e.getMessage()); + return null; + } } - private static ParameterizedSparqlString buildRangeQuery(Range range, int scale) { - ParameterizedSparqlString pss = new ParameterizedSparqlString( - """ - SELECT DISTINCT ?geo - WHERE { - GRAPH ?spatial { - ?geo hal:hilbertCorner?scale ?index . - FILTER (?index >= ?low && ?index <= ?high) + /** Distinct entity ids of subjects whose stored bbox cells overlap the region's cover. */ + private static List collectCandidates(HDF5Reader reader, Envelope env) { + LinkedHashSet out = new LinkedHashSet<>(); + PositionalDictionaryReader dict = (PositionalDictionaryReader) reader.getDictionary(); + IndexReader gpos = reader.getIndexReader(Index.GPOS); + if (gpos == null) { + return List.of(); + } + NodeTable nodeTable = reader.getNodeTable(); + // Clamp into the [0, 2^31) Hilbert domain with the SAME monotone + // projection the writer applies to geometry bboxes: two boxes that + // overlap still overlap after clamping, so a query region in negative + // (or beyond-2^31) space finds the equally clamped geometries there via + // the domain-edge cells instead of silently returning nothing. False + // positives are removed by the sfIntersects verification stage. + long minX = HilbertSpace.clampToDomain((long) Math.floor(env.getMinX())); + long maxX = HilbertSpace.clampToDomain((long) Math.floor(env.getMaxX())); + long minY = HilbertSpace.clampToDomain((long) Math.floor(env.getMinY())); + long maxY = HilbertSpace.clampToDomain((long) Math.floor(env.getMaxY())); + Var sVar = Var.alloc("hilbertCellSubject"); + Var oVar = Var.alloc("hilbertCellValue"); + for (int s = 0; s <= MAX_INDEX_SCALE; s++) { + Node pred = NodeFactory.createURI(HILBERT_CELL_NS + s); + if (dict.getPredicates().locate(pred) < 1) { + continue; // no geometry stored at this scale + } + long cell = 1L << s; + long[] lo = {Math.floorDiv(minX, cell), Math.floorDiv(minY, cell)}; + long[] hi = {Math.floorDiv(maxX, cell), Math.floorDiv(maxY, cell)}; + for (Range r : coverRanges(lo, hi)) { + ExprList bounds = new ExprList(); + bounds.add(new E_GreaterThanOrEqual(new ExprVar(oVar), + NodeValue.makeNode(NodeFactory.createLiteralByValue(r.low())))); + bounds.add(new E_LessThanOrEqual(new ExprVar(oVar), + NodeValue.makeNode(NodeFactory.createLiteralByValue(r.high())))); + Quad pattern = new Quad(Params.SPATIAL, sVar, pred, oVar); + BGIteratorPOS it = new BGIteratorPOS(dict, gpos, new BindingNodeId(), pattern, bounds, nodeTable); + while (it.hasNext()) { + NodeId sid = it.next().get(sVar); + if (sid != null) { + out.add(sid.getId()); + } } } - """ - ); - pss.setNsPrefix("hal", "https://halcyon.is/ns/"); - pss.setIri("spatial", Params.SPATIALSTRING); - pss.setLiteral("scale", scale); - pss.setLiteral("low", range.low()); - pss.setLiteral("high", range.high()); - return pss; + } + return new ArrayList<>(out); } - private static class LazyCandidateIterator implements Iterator, AutoCloseable { - private final Iterator rangeIterator; - private final BeakGraph bGraph; - private final int scale; - - private QueryExecution currentQexec = null; - private ResultSet currentResultSet = null; - private Node nextNode = null; - - public LazyCandidateIterator(Iterable ranges, BeakGraph bGraph, int scale) { - this.rangeIterator = ranges.iterator(); - this.bGraph = bGraph; - this.scale = scale; - advance(); + /** Hilbert range cover of the cell box [lo, hi], merged and capped (superset-safe). */ + private static List coverRanges(long[] lo, long[] hi) { + ArrayList ranges = new ArrayList<>(); + for (Range r : HilbertSpace.hc.query(lo, hi)) { + ranges.add(r); } - - private void advance() { - nextNode = null; - while (true) { - if (currentResultSet != null && currentResultSet.hasNext()) { - nextNode = currentResultSet.next().get("geo").asNode(); - return; + ranges.sort((a, b) -> Long.compare(a.low(), b.low())); + ArrayList merged = new ArrayList<>(); + Range last = null; + for (Range r : ranges) { + if (last != null && r.low() <= last.high() + 1) { + last = new Range(last.low(), Math.max(last.high(), r.high())); + } else { + if (last != null) { + merged.add(last); } - if (currentQexec != null) { - currentQexec.close(); - currentQexec = null; - currentResultSet = null; - } - if (!rangeIterator.hasNext()) { - return; - } - Range range = rangeIterator.next(); - Dataset ds = bGraph.getDataset(); - ParameterizedSparqlString pss = buildRangeQuery(range, scale); - currentQexec = QueryExecutionFactory.create(pss.asQuery(), ds); - currentResultSet = currentQexec.execSelect(); + last = r; } } - - @Override - public boolean hasNext() { - return nextNode != null; + if (last != null) { + merged.add(last); } - - @Override - public Node next() { - if (nextNode == null) { - throw new NoSuchElementException(); - } - Node current = nextNode; - advance(); - return current; + if (merged.size() > MAX_QUERY_RANGES) { + // Collapse to one spanning range: a wider superset costs extra candidate + // scanning, never recall. + Range one = new Range(merged.get(0).low(), merged.get(merged.size() - 1).high()); + merged = new ArrayList<>(Collections.singletonList(one)); } + return merged; + } - @Override - public void close() { - if (currentQexec != null) { - currentQexec.close(); - currentQexec = null; - currentResultSet = null; - } - } + @Override + public boolean hasNext() { + return outputIterator.hasNext(); + } + + @Override + public BindingNodeId next() { + return outputIterator.next(); } -} \ No newline at end of file +} diff --git a/src/main/java/com/ebremer/beakgraph/hdf5/jena/ValueCluster.java b/src/main/java/com/ebremer/beakgraph/hdf5/jena/ValueCluster.java new file mode 100644 index 00000000..a4392412 --- /dev/null +++ b/src/main/java/com/ebremer/beakgraph/hdf5/jena/ValueCluster.java @@ -0,0 +1,70 @@ +package com.ebremer.beakgraph.hdf5.jena; + +import com.ebremer.beakgraph.core.Dictionary; +import org.apache.jena.graph.Node; +import org.apache.jena.sparql.expr.Expr; +import org.apache.jena.sparql.expr.NodeValue; + +/** + * Locates the contiguous id range of dictionary entries that compare + * value-equal to a filter constant. + *

    + * The dictionaries order literals by value first (NodeValue.compareAlways) and + * break value ties on the exact RDF term, so value-equal but term-distinct + * literals ("5"^^xsd:int, "5"^^xsd:integer, "5.0"^^xsd:double) occupy a run of + * adjacent ids - and an exact-term binary search can land anywhere inside that + * run. Range-filter pushdown (FILTER(?o >= 5)) must therefore snap its bound to + * the edges of the whole cluster: with [lo, hi] from {@link #of}, + * 

    + *   >   -> min = hi + 1        >=  -> min = lo
+ *   <   -> max = lo - 1        <=  -> max = hi
+ *
    + * For a constant with no value-equal entries (including all non-literals) the + * cluster is empty ({@code hi == lo - 1}, both at the insertion point) and the + * formulas reduce exactly to the plain insertion-point bounds. + */ +final class ValueCluster { + + private ValueCluster() {} + + /** + * Returns {@code {lo, hi}}: the inclusive 1-based id range of entries in + * {@code dict} that are value-equal to {@code value}. Empty when {@code hi < lo}. + */ + static long[] of(Dictionary dict, Node value) { + long raw = dict.search(value); + boolean found = raw >= 0; + long pos = found ? raw : (-raw - 1); + long lo = pos; + long hi = found ? pos : pos - 1; + if (!value.isLiteral()) { + // URIs/bnodes are ordered by term, never by value: no cluster to grow. + return new long[]{lo, hi}; + } + long n = dict.getNumberOfNodes(); + while (hi + 1 <= n && valueEqual(extractOrNull(dict, hi + 1), value)) hi++; + while (lo - 1 >= 1 && valueEqual(extractOrNull(dict, lo - 1), value)) lo--; + return new long[]{lo, hi}; + } + + private static Node extractOrNull(Dictionary dict, long id) { + try { + return dict.extract(id); + } catch (RuntimeException e) { + return null; + } + } + + private static boolean valueEqual(Node a, Node b) { + if (a == null || !a.isLiteral()) return false; + try { + // Value-only comparison. compareAlways would be wrong here: it never + // answers "equal" for distinct terms (it breaks value ties on the term), + // which is exactly the distinction this cluster must ignore. Throws for + // non-comparable values (e.g. string vs number) - not value-equal. + return NodeValue.compare(NodeValue.makeNode(a), NodeValue.makeNode(b)) == Expr.CMP_EQUAL; + } catch (RuntimeException e) { + return false; + } + } +} diff --git a/src/main/java/com/ebremer/beakgraph/hdf5/readers/FCDReader.java b/src/main/java/com/ebremer/beakgraph/hdf5/readers/FCDReader.java index 04089d15..e25bb3fb 100644 --- a/src/main/java/com/ebremer/beakgraph/hdf5/readers/FCDReader.java +++ b/src/main/java/com/ebremer/beakgraph/hdf5/readers/FCDReader.java @@ -2,23 +2,24 @@ import com.ebremer.beakgraph.core.lib.VByte; import com.ebremer.beakgraph.hdf5.BitPackedUnSignedLongBuffer; +import com.ebremer.beakgraph.io.DatasetBytes; +import com.ebremer.beakgraph.io.RandomAccessBytes; import com.ebremer.beakgraph.utils.StringUtils; import io.jhdf.api.Group; import io.jhdf.api.dataset.ContiguousDataset; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; /** * Reader for a front-coded string dictionary. *

    - * Safe for concurrent reads: it never mutates the shared {@code stringbuffer}'s position - * (every read is absolute, threaded through a local cursor), and the Zstd decompressor - - * which airlift does not allow sharing across threads - is held per thread. + * Safe for concurrent reads: it never shares mutable position state (every read + * is absolute through {@link RandomAccessBytes}, with long offsets - so string + * buffers past 2 GiB are addressable), and the Zstd decompressor - which + * airlift does not allow sharing across threads - is held per thread. */ public class FCDReader { - private final ByteBuffer buffer; - private final ByteBuffer offsets; + private final RandomAccessBytes buffer; + private final RandomAccessBytes offsets; private final BitPackedUnSignedLongBuffer compressed; private final long blockSize; private final long numEntries; @@ -29,10 +30,10 @@ public FCDReader(Group strings) { ContiguousDataset stringbuffer = (ContiguousDataset) strings.getChild("stringbuffer"); ContiguousDataset off = (ContiguousDataset) strings.getChild("offsets"); ContiguousDataset xcompressed = (ContiguousDataset) strings.getChild("compressed"); - this.buffer = stringbuffer.getBuffer(); - this.offsets = off.getBuffer().order(ByteOrder.BIG_ENDIAN); - this.compressed = new BitPackedUnSignedLongBuffer( - null, xcompressed.getBuffer(), + this.buffer = DatasetBytes.of(stringbuffer); + this.offsets = DatasetBytes.of(off); + this.compressed = BitPackedUnSignedLongBuffer.readView( + DatasetBytes.of(xcompressed), (long) xcompressed.getAttribute("numEntries").getData(), (int) xcompressed.getAttribute("width").getData() ); @@ -42,18 +43,18 @@ public FCDReader(Group strings) { } /** A decoded fragment plus the absolute position just past it. */ - private record Fragment(String value, int nextPos) {} + private record Fragment(String value, long nextPos) {} /** * Reads the length-prefixed fragment that starts at absolute byte position {@code pos}. - * Uses absolute reads only, so it does not disturb the shared buffer's position. + * Absolute reads only, so concurrent readers never disturb each other. */ - private Fragment readFragment(int pos, int entryIndex) { + private Fragment readFragment(long pos, long entryIndex) { VByte.DecodeResult lenR = VByte.decodeAt(buffer, pos); int dataLen = (int) lenR.value; - int p = lenR.nextOffset; + long p = lenR.nextOffset; byte[] data = new byte[dataLen]; - buffer.get(p, data); // absolute bulk read; does not move the buffer position + buffer.get(p, data, 0, dataLen); // absolute bulk read p += dataLen; boolean isCompressed = compressed.get(entryIndex) == 1; String value = isCompressed ? su.get().decompress(data) : new String(data, StandardCharsets.UTF_8); @@ -64,10 +65,10 @@ public String get(long n) { if (n < 0 || n >= numEntries) throw new IndexOutOfBoundsException(); long block = n / blockSize; - int pos = (int) offsets.getLong((int) block * 8); + long pos = offsets.getLong(block * 8L); // The first string in the block is always at index (block * blockSize). - Fragment frag = readFragment(pos, (int) (block * blockSize)); + Fragment frag = readFragment(pos, block * blockSize); String current = frag.value(); pos = frag.nextPos(); @@ -77,45 +78,17 @@ public String get(long n) { int prefixLen = (int) pl.value; pos = pl.nextOffset; // Suffix fragment is at index (block * blockSize + i). - Fragment suffix = readFragment(pos, (int) (block * blockSize + i)); + Fragment suffix = readFragment(pos, block * blockSize + i); current = current.substring(0, prefixLen) + suffix.value(); pos = suffix.nextPos(); } return current; } - public long locate(String x) { - if (numEntries == 0) return -1; - // Binary search for the correct block. - long low = 0; - long high = numBlocks - 1; - while (low <= high) { - long mid = low + (high - low) / 2; - String midHeader = get(mid * blockSize); - int cmp = x.compareTo(midHeader); - if (cmp < 0) high = mid - 1; - else if (cmp > 0) low = mid + 1; - else return mid * blockSize; - } - long candidateBlock = high; - if (candidateBlock < 0) return -1; - // Linear search inside the block. - long blockStart = candidateBlock * blockSize; - long blockEnd = Math.min(blockStart + blockSize, numEntries); - int pos = (int) offsets.getLong((int) candidateBlock * 8); - Fragment frag = readFragment(pos, (int) blockStart); - String current = frag.value(); - pos = frag.nextPos(); - if (current.equals(x)) return blockStart; - for (long i = blockStart + 1; i < blockEnd; i++) { - VByte.DecodeResult pl = VByte.decodeAt(buffer, pos); - int prefixLen = (int) pl.value; - pos = pl.nextOffset; - Fragment suffix = readFragment(pos, (int) i); - current = current.substring(0, prefixLen) + suffix.value(); - pos = suffix.nextPos(); - if (current.equals(x)) return i; - } - return -1; - } + // NOTE: an unused locate(String) lived here that binary-searched blocks by + // String.compareTo. It was removed in the dead-code sweep - and must not be + // reintroduced as-was: the value-ordered dictionaries (strings holding typed + // literals) are NOT ordered by raw string comparison, so its block search was + // wrong for them. Term lookup goes through MultiTypeDictionaryReader.search, + // which compares with the same NodeComparator the writer sorted with. } diff --git a/src/main/java/com/ebremer/beakgraph/hdf5/readers/HDF5Reader.java b/src/main/java/com/ebremer/beakgraph/hdf5/readers/HDF5Reader.java index 181b1c1a..0fb8ab05 100644 --- a/src/main/java/com/ebremer/beakgraph/hdf5/readers/HDF5Reader.java +++ b/src/main/java/com/ebremer/beakgraph/hdf5/readers/HDF5Reader.java @@ -16,11 +16,15 @@ import java.io.File; import java.net.URI; import java.nio.file.Path; +import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.Iterator; +import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Stream; +import org.apache.jena.atlas.iterator.Iter; import org.apache.jena.graph.Node; import org.apache.jena.graph.Triple; import org.apache.jena.sparql.core.Quad; @@ -41,7 +45,10 @@ public class HDF5Reader implements BGReader { private final Map indexCache = new ConcurrentHashMap<>(); private final URI uri; private final long formatVersion; - + // Closed readers must be detectable (pool validation) and close() must be + // idempotent (a poisoned pooled instance is closed again on destroy). + private volatile boolean open = true; + static { JenaSystem.init(); Spatial.init(); @@ -53,20 +60,30 @@ public HDF5Reader(Path src) { public HDF5Reader(File src) { this.hdf = new HdfFile(src.toPath()); - this.hdt = (Group) hdf.getChild(Params.BG); - this.formatVersion = readFormatVersion(hdt); - if (formatVersion > Params.FORMAT_VERSION) { - hdf.close(); - throw new IllegalStateException( - "BeakGraph HDF5 format version " + formatVersion + " in " + src - + " is newer than this build supports (max " + Params.FORMAT_VERSION - + "). Upgrade BeakGraph."); + try { + this.hdt = (Group) hdf.getChild(Params.BG); + if (hdt == null) { + throw new IllegalStateException( + "Not a BeakGraph file (no '" + Params.BG + "' group): " + src); + } + this.formatVersion = readFormatVersion(hdt); + if (formatVersion > Params.FORMAT_VERSION) { + throw new IllegalStateException( + "BeakGraph HDF5 format version " + formatVersion + " in " + src + + " is newer than this build supports (max " + Params.FORMAT_VERSION + + "). Upgrade BeakGraph."); + } + Group dictionary = (Group) hdt.getChild(Params.DICTIONARY); + this.dict = new PositionalDictionaryReader(dictionary); + this.defaultGraph = Quad.defaultGraphIRI; + nodeTable = new SimpleNodeTable(dict); + this.uri = src.toURI(); + } catch (RuntimeException | Error e) { + // Close the mapped file before propagating: a leaked HdfFile pins the + // file handle (and on Windows, the file lock) with no way to release it. + try { hdf.close(); } catch (Exception ignore) {} + throw e; } - Group dictionary = (Group) hdt.getChild(Params.DICTIONARY); - this.dict = new PositionalDictionaryReader(dictionary); - this.defaultGraph = Quad.defaultGraphIRI; - nodeTable = new SimpleNodeTable(dict); - this.uri = src.toURI(); } /** @@ -116,6 +133,9 @@ public Iterator read(Node ng, BindingNodeId bnid, Triple triple, || boundToMissing(triple.getObject(), bnid)) { return Collections.emptyIterator(); } + if (Quad.isUnionGraph(ng)) { + return readUnion(bnid, triple, filter, nodeTable); + } boolean isDefault = ng.equals(Quad.defaultGraphNodeGenerated) || ng.equals(Quad.defaultGraphIRI); Node g = isDefault ? this.defaultGraph : ng; Node s = substitute(triple.getSubject(), bnid, nodeTable); @@ -125,6 +145,41 @@ public Iterator read(Node ng, BindingNodeId bnid, Triple triple, return new BGIteratorMaster(this, dict, bnid, quadPattern, filter, nodeTable); } + /** + * {@code urn:x-arq:unionGraph}: the union of all named graphs, with the + * SPARQL-mandated set semantics - a triple present in several named graphs + * appears once. Rows are deduplicated on the values of the pattern's + * variables (the concrete positions are identical across graphs by + * construction). + */ + private Iterator readUnion(BindingNodeId bnid, Triple triple, ExprList filter, NodeTable nodeTable) { + List vars = new ArrayList<>(3); + for (Node n : new Node[]{triple.getSubject(), triple.getPredicate(), triple.getObject()}) { + if (n.isVariable()) vars.add(Var.alloc(n)); + } + // Lazy per-graph chaining: constructing every graph's iterator up front + // paid each one's index binary searches before the first row came back + // (spatial stores hold thousands of tile graphs). + Iterator graphs = dict.streamGraphs() + .filter(n -> !(n.equals(Quad.defaultGraphIRI) || n.equals(Quad.defaultGraphNodeGenerated))) + .iterator(); + Iterator chain = Iter.flatMap(graphs, gn -> read(gn, bnid, triple, filter, nodeTable)); + // The dedup set is inherent to union set-semantics (rows arrive per + // graph, not globally sorted); the key is a record of three primitive + // longs rather than a boxed List, cutting the per-row footprint + // of a large union scan several-fold. + record RowKey(long a, long b, long c) {} + Set seen = new HashSet<>(); + return Iter.filter(chain, b -> { + long[] k = {Long.MIN_VALUE, Long.MIN_VALUE, Long.MIN_VALUE}; + for (int i = 0; i < vars.size(); i++) { + NodeId id = b.get(vars.get(i)); + if (id != null) k[i] = id.getId(); + } + return seen.add(new RowKey(k[0], k[1], k[2])); + }); + } + /** True when {@code n} is a variable already bound to a node that does not exist here. */ private static boolean boundToMissing(Node n, BindingNodeId bnid) { if (bnid != null && n.isVariable()) { @@ -191,10 +246,18 @@ public ExtendedIterator graphBaseFind(Node graph, Triple tp) { @Override public NodeTable getNodeTable() { return nodeTable; } - @Override public void close() { hdf.close(); } - @Override public GSPODictionary getDictionary() { return dict; } - @Override public int getNumberOfTriples(String ng) { return 0; } - @Override public Stream streamQuads() { return Stream.empty(); } + + @Override + public void close() { + if (!open) return; + open = false; + hdf.close(); + } + + @Override + public boolean isOpen() { return open; } + + @Override public GSPODictionary getDictionary() { return dict; } @Override public Iterator listGraphNodes() { @@ -203,7 +266,11 @@ public Iterator listGraphNodes() { @Override public boolean containsGraph(Node graphNode) { - return (dict.getGraphs().locate(graphNode) != -1); + // Graphs share the universal entity dictionary, so locate() alone matches + // every subject/object entity too; membership in the columnar graphs list + // is what makes an entity an actual graph. + long id = dict.getGraphs().locate(graphNode); + return id != -1 && dict.isGraph(id); } } diff --git a/src/main/java/com/ebremer/beakgraph/hdf5/readers/IndexReader.java b/src/main/java/com/ebremer/beakgraph/hdf5/readers/IndexReader.java index e34bd4c8..ffe00527 100644 --- a/src/main/java/com/ebremer/beakgraph/hdf5/readers/IndexReader.java +++ b/src/main/java/com/ebremer/beakgraph/hdf5/readers/IndexReader.java @@ -3,6 +3,7 @@ import com.ebremer.beakgraph.Params; import com.ebremer.beakgraph.hdf5.BitPackedUnSignedLongBuffer; import com.ebremer.beakgraph.hdf5.Index; +import com.ebremer.beakgraph.io.DatasetBytes; import com.ebremer.beakgraph.utils.HDTBitmapDirectory; import io.jhdf.api.Group; import io.jhdf.api.dataset.ContiguousDataset; @@ -57,7 +58,7 @@ private BitPackedUnSignedLongBuffer loadBuffer(Group index, String name) { if (ds == null) return null; long num = (Long) ds.getAttribute("numEntries").getData(); int width = (Integer) ds.getAttribute("width").getData(); - return new BitPackedUnSignedLongBuffer(null, ds.getBuffer(), num, width); + return BitPackedUnSignedLongBuffer.readView(DatasetBytes.of(ds), num, width); } public HDTBitmapDirectory getDirectory(char component) { diff --git a/src/main/java/com/ebremer/beakgraph/hdf5/readers/MultiTypeDictionaryReader.java b/src/main/java/com/ebremer/beakgraph/hdf5/readers/MultiTypeDictionaryReader.java index d8fd003c..e11fab58 100644 --- a/src/main/java/com/ebremer/beakgraph/hdf5/readers/MultiTypeDictionaryReader.java +++ b/src/main/java/com/ebremer/beakgraph/hdf5/readers/MultiTypeDictionaryReader.java @@ -4,10 +4,10 @@ import com.ebremer.beakgraph.core.lib.DataType; import com.ebremer.beakgraph.core.lib.NodeComparator; import com.ebremer.beakgraph.hdf5.BitPackedUnSignedLongBuffer; +import com.ebremer.beakgraph.io.DatasetBytes; +import com.ebremer.beakgraph.io.RandomAccessBytes; import io.jhdf.api.Group; import io.jhdf.api.dataset.ContiguousDataset; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.util.Arrays; import java.util.Optional; import java.util.stream.LongStream; @@ -25,8 +25,8 @@ public class MultiTypeDictionaryReader extends AbstractDictionary { private final BitPackedUnSignedLongBuffer longs; private final BitPackedUnSignedLongBuffer datatype; private final BitPackedUnSignedLongBuffer typedLiterals; - private final ByteBuffer floats; - private final ByteBuffer doubles; + private final RandomAccessBytes floats; + private final RandomAccessBytes doubles; private final FCDReader iri; private final FCDReader strings; private final FCDReader typedLiteralsDictionary; @@ -38,30 +38,36 @@ public class MultiTypeDictionaryReader extends AbstractDictionary { private final long numEntries; private final String name; - // Tiered Index Storage - private long[] tieredIds; - private Node[] tieredNodes; + // Tiered index, built LAZILY on first search: building it in the + // constructor performed numEntries/TIER_SPACING full extracts at every + // file-open, paying for a search accelerator the caller might never use. + // Volatile publication keeps concurrent first-searchers safe; the benign + // race builds identical content over immutable data. + private volatile TieredIndex tiered; + + private record TieredIndex(long[] ids, Node[] nodes) {} + private static final TieredIndex EMPTY_TIER = new TieredIndex(new long[0], new Node[0]); public MultiTypeDictionaryReader(Group d) { this.name = d.getName(); ContiguousDataset offsetsDS = (ContiguousDataset) d.getDatasetByPath("offsets"); this.numEntries = (Long) offsetsDS.getAttribute("numEntries").getData(); - this.offsets = new BitPackedUnSignedLongBuffer(null, offsetsDS.getBuffer(), numEntries, (Integer) offsetsDS.getAttribute("width").getData()); + this.offsets = BitPackedUnSignedLongBuffer.readView(DatasetBytes.of(offsetsDS), numEntries, (Integer) offsetsDS.getAttribute("width").getData()); ContiguousDataset datatypeDS = (ContiguousDataset) d.getDatasetByPath("datatypes"); - this.datatype = new BitPackedUnSignedLongBuffer(null, datatypeDS.getBuffer(), (Long) datatypeDS.getAttribute("numEntries").getData(), (Integer) datatypeDS.getAttribute("width").getData()); + this.datatype = BitPackedUnSignedLongBuffer.readView(DatasetBytes.of(datatypeDS), (Long) datatypeDS.getAttribute("numEntries").getData(), (Integer) datatypeDS.getAttribute("width").getData()); ContiguousDataset typedLiteralsDS = (ContiguousDataset) d.getChild("typedLiterals"); - this.typedLiterals = (typedLiteralsDS != null) ? new BitPackedUnSignedLongBuffer(null, typedLiteralsDS.getBuffer(), (Long) typedLiteralsDS.getAttribute("numEntries").getData(), (Integer) typedLiteralsDS.getAttribute("width").getData()) : null; + this.typedLiterals = (typedLiteralsDS != null) ? BitPackedUnSignedLongBuffer.readView(DatasetBytes.of(typedLiteralsDS), (Long) typedLiteralsDS.getAttribute("numEntries").getData(), (Integer) typedLiteralsDS.getAttribute("width").getData()) : null; - this.doubles = getDataSet(d, "doubles").map(ds -> ds.getBuffer().order(ByteOrder.BIG_ENDIAN)).orElse(null); - this.floats = getDataSet(d, "floats").map(ds -> ds.getBuffer().order(ByteOrder.BIG_ENDIAN)).orElse(null); + this.doubles = getDataSet(d, "doubles").map(DatasetBytes::of).orElse(null); + this.floats = getDataSet(d, "floats").map(DatasetBytes::of).orElse(null); this.integers = getDataSet(d, "integers").map(ds -> - new BitPackedUnSignedLongBuffer(null, ds.getBuffer(), (Long) ds.getAttribute("numEntries").getData(), (Integer) ds.getAttribute("width").getData())).orElse(null); + BitPackedUnSignedLongBuffer.readView(DatasetBytes.of(ds), (Long) ds.getAttribute("numEntries").getData(), (Integer) ds.getAttribute("width").getData())).orElse(null); this.longs = getDataSet(d, "longs").map(ds -> - new BitPackedUnSignedLongBuffer(null, ds.getBuffer(), (Long) ds.getAttribute("numEntries").getData(), (Integer) ds.getAttribute("width").getData())).orElse(null); + BitPackedUnSignedLongBuffer.readView(DatasetBytes.of(ds), (Long) ds.getAttribute("numEntries").getData(), (Integer) ds.getAttribute("width").getData())).orElse(null); Group stringsG = (Group) d.getChild("strings"); this.strings = (stringsG != null) ? new FCDReader(stringsG) : null; @@ -75,22 +81,35 @@ public MultiTypeDictionaryReader(Group d) { Group langsG = (Group) d.getChild("langs"); this.langs = (langsG != null) ? new FCDReader(langsG) : null; ContiguousDataset langTagsDS = (ContiguousDataset) d.getChild("langTags"); - this.langTags = (langTagsDS != null) ? new BitPackedUnSignedLongBuffer(null, langTagsDS.getBuffer(), (Long) langTagsDS.getAttribute("numEntries").getData(), (Integer) langTagsDS.getAttribute("width").getData()) : null; + this.langTags = (langTagsDS != null) ? BitPackedUnSignedLongBuffer.readView(DatasetBytes.of(langTagsDS), (Long) langTagsDS.getAttribute("numEntries").getData(), (Integer) langTagsDS.getAttribute("width").getData()) : null; + } - buildTieredIndex(); + private TieredIndex tieredIndex() { + TieredIndex t = tiered; + if (t == null) { + synchronized (this) { + t = tiered; + if (t == null) { + t = buildTieredIndex(); + tiered = t; + } + } + } + return t; } - private void buildTieredIndex() { - if (numEntries <= TIER_SPACING) return; + private TieredIndex buildTieredIndex() { + if (numEntries <= TIER_SPACING) return EMPTY_TIER; int tierSize = (int) (numEntries / TIER_SPACING); - tieredIds = new long[tierSize]; - tieredNodes = new Node[tierSize]; - + long[] ids = new long[tierSize]; + Node[] nodes = new Node[tierSize]; + for (int i = 0; i < tierSize; i++) { long id = (long) i * TIER_SPACING + 1; - tieredIds[i] = id; - tieredNodes[i] = extract(id); + ids[i] = id; + nodes[i] = extract(id); } + return new TieredIndex(ids, nodes); } private Optional getDataSet(Group g, String name) { @@ -111,8 +130,8 @@ public Node extract(long id) { Node na = switch (dt) { case INTEGER -> NodeFactory.createLiteralByValue((int) integers.get(off)); case LONG -> NodeFactory.createLiteralByValue(longs.get(off)); - case FLOAT -> NodeFactory.createLiteralByValue(floats.getFloat(Math.toIntExact(off * Float.BYTES))); - case DOUBLE -> NodeFactory.createLiteralByValue(doubles.getDouble(Math.toIntExact(off * Double.BYTES))); + case FLOAT -> NodeFactory.createLiteralByValue(floats.getFloat(off * Float.BYTES)); + case DOUBLE -> NodeFactory.createLiteralByValue(doubles.getDouble(off * Double.BYTES)); case STRING -> { // A language tag takes precedence: rdf:langString is reconstructed // as a lang-tagged literal (term-exact per RDF semantics). @@ -151,19 +170,20 @@ private long searchFAST(Node element) { long high = numEntries; // 1. Tiered Index Lookup to narrow the range - // This is safe because tieredNodes are actual Node objects compared using your specific Comparator - if (tieredNodes != null && tieredNodes.length > 0) { - int tierIdx = Arrays.binarySearch(tieredNodes, element, NodeComparator.INSTANCE); - if (tierIdx >= 0) return tieredIds[tierIdx]; + // This is safe because the tier nodes are actual Node objects compared using your specific Comparator + TieredIndex tier = tieredIndex(); + if (tier.nodes().length > 0) { + int tierIdx = Arrays.binarySearch(tier.nodes(), element, NodeComparator.INSTANCE); + if (tierIdx >= 0) return tier.ids()[tierIdx]; int insertionPoint = -(tierIdx + 1); if (insertionPoint > 0) { - // The element at tieredIds[insertionPoint-1] already compared < element, + // The element at ids[insertionPoint-1] already compared < element, // so the real match (if any) starts strictly after it. - low = tieredIds[insertionPoint - 1] + 1; + low = tier.ids()[insertionPoint - 1] + 1; } - if (insertionPoint < tieredIds.length) { - high = tieredIds[insertionPoint] - 1; + if (insertionPoint < tier.ids().length) { + high = tier.ids()[insertionPoint] - 1; } } diff --git a/src/main/java/com/ebremer/beakgraph/hdf5/readers/PositionalDictionaryReader.java b/src/main/java/com/ebremer/beakgraph/hdf5/readers/PositionalDictionaryReader.java index ebf9d94c..2599d286 100644 --- a/src/main/java/com/ebremer/beakgraph/hdf5/readers/PositionalDictionaryReader.java +++ b/src/main/java/com/ebremer/beakgraph/hdf5/readers/PositionalDictionaryReader.java @@ -3,6 +3,7 @@ import com.ebremer.beakgraph.core.GSPODictionary; import com.ebremer.beakgraph.core.Dictionary; import com.ebremer.beakgraph.hdf5.BitPackedUnSignedLongBuffer; +import com.ebremer.beakgraph.io.DatasetBytes; import io.jhdf.api.Group; import io.jhdf.api.dataset.ContiguousDataset; import java.util.Optional; @@ -38,11 +39,11 @@ public PositionalDictionaryReader(Group dictionary) { this.maxEntityId = (entities != null) ? entities.getNumberOfNodes() : 0; this.graphs = getDataSet(dictionary, "graphs").map(ds -> - new BitPackedUnSignedLongBuffer(null, ds.getBuffer(), (Long) ds.getAttribute("numEntries").getData(), (Integer) ds.getAttribute("width").getData())).orElse(null); + BitPackedUnSignedLongBuffer.readView(DatasetBytes.of(ds), (Long) ds.getAttribute("numEntries").getData(), (Integer) ds.getAttribute("width").getData())).orElse(null); this.subjects = getDataSet(dictionary, "subjects").map(ds -> - new BitPackedUnSignedLongBuffer(null, ds.getBuffer(), (Long) ds.getAttribute("numEntries").getData(), (Integer) ds.getAttribute("width").getData())).orElse(null); + BitPackedUnSignedLongBuffer.readView(DatasetBytes.of(ds), (Long) ds.getAttribute("numEntries").getData(), (Integer) ds.getAttribute("width").getData())).orElse(null); this.objects = getDataSet(dictionary, "objects").map(ds -> - new BitPackedUnSignedLongBuffer(null, ds.getBuffer(), (Long) ds.getAttribute("numEntries").getData(), (Integer) ds.getAttribute("width").getData())).orElse(null); + BitPackedUnSignedLongBuffer.readView(DatasetBytes.of(ds), (Long) ds.getAttribute("numEntries").getData(), (Integer) ds.getAttribute("width").getData())).orElse(null); this.objectsDict = makeObjectsDictionary(); } @@ -124,6 +125,34 @@ public long getNumberOfNodes() { }; } + // Lazily materialized set of the graph ids: isGraph() used to decode the + // whole columnar list per call - O(numGraphs) for every containsGraph, and + // spatial stores carry thousands of tile graphs. Benign publication race: + // both builders produce identical content over immutable data. + private volatile java.util.Set graphIdSet; + + /** + * True when {@code entityId} appears in the columnar list of actual graphs. + * Graphs share the universal entity ID space, so a bare dictionary lookup + * cannot distinguish a graph from any other entity - this can. + */ + public boolean isGraph(long entityId) { + if (graphs == null) { + return false; + } + java.util.Set s = graphIdSet; + if (s == null) { + s = graphs.stream().boxed().collect(java.util.stream.Collectors.toUnmodifiableSet()); + graphIdSet = s; + } + return s.contains(entityId); + } + + /** Raw ids of the actual graphs (the columnar list), in stored order. */ + public java.util.stream.LongStream streamGraphIds() { + return (graphs == null) ? java.util.stream.LongStream.empty() : graphs.stream(); + } + @Override public Stream streamGraphs() { if (graphs == null || entities == null) { diff --git a/src/main/java/com/ebremer/beakgraph/hdf5/writers/BGIndex.java b/src/main/java/com/ebremer/beakgraph/hdf5/writers/BGIndex.java index 4b911192..1a62e3f9 100644 --- a/src/main/java/com/ebremer/beakgraph/hdf5/writers/BGIndex.java +++ b/src/main/java/com/ebremer/beakgraph/hdf5/writers/BGIndex.java @@ -10,8 +10,11 @@ import java.util.Arrays; import org.apache.jena.graph.Node; import org.apache.jena.sparql.core.Quad; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class BGIndex { + private static final Logger logger = LoggerFactory.getLogger(BGIndex.class); private final BitPackedUnSignedLongBuffer B1, B2, B3; private final BitPackedUnSignedLongBuffer S1, S2, S3; @@ -81,7 +84,7 @@ private class LevelState { } public BGIndex(HDF5Writer.Builder builder, PositionalDictionaryWriter dictWriter, Index type, Quad[] allQuads) { - System.out.println("Creating Index " + type); + logger.info("Creating index {}", type); this.type = type; String indexName = type.name(); @@ -144,9 +147,11 @@ private long computeMaxL0Id(PositionalDictionaryWriter w) { } private void processQuads(PositionalDictionaryWriter w, Quad[] allQuads) { - System.out.print("Sorting quads for " + type.name() + "... "); + // INFO bracketing: sorting millions of quads takes minutes with no other output. + logger.info("Sorting {} quads for {}...", allQuads.length, type.name()); + long sortStart = System.nanoTime(); Arrays.parallelSort(allQuads, type.getComparator()); - System.out.println("done"); + logger.info("Sorted {} in {} s", type.name(), (System.nanoTime() - sortStart) / 1_000_000_000L); LevelState l1 = new LevelState(), l2 = new LevelState(), l3 = new LevelState(); Quad lastUnique = null; @@ -160,7 +165,7 @@ private void processQuads(PositionalDictionaryWriter w, Quad[] allQuads) { for (Quad curr : allQuads) { if (++count % 1_000_000 == 0) { - System.out.println(" " + type.name() + " processed " + count + " / " + totalQuads + " quads..."); + logger.info("{} processed {} / {} quads...", type.name(), count, totalQuads); } // 1. Duplicate Check if (lastUnique != null && diff --git a/src/main/java/com/ebremer/beakgraph/hdf5/writers/FCDWriter.java b/src/main/java/com/ebremer/beakgraph/hdf5/writers/FCDWriter.java index c59f0c90..2876448c 100644 --- a/src/main/java/com/ebremer/beakgraph/hdf5/writers/FCDWriter.java +++ b/src/main/java/com/ebremer/beakgraph/hdf5/writers/FCDWriter.java @@ -12,7 +12,6 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Path; -import java.util.UUID; public class FCDWriter implements HDF5Buffer, AutoCloseable { private final int blockSize; @@ -24,16 +23,21 @@ public class FCDWriter implements HDF5Buffer, AutoCloseable { private long numEntries = 0; private long position = 0; private final DataOutputBuffer offsets; - public final String ID = UUID.randomUUID().toString(); private final BitPackedUnSignedLongBuffer compressed = new BitPackedUnSignedLongBuffer(Path.of("compressed"), null, 0, 1); private final StringUtils su = new StringUtils(); public FCDWriter(Path path, int blockSize) throws FileNotFoundException { + if (blockSize < 2) { + // With blockSize 1 the add() block-head branch never closes a block: + // the offsets dataset would hold one entry total and FCDReader.get() + // for any later block reads past it. Fail construction loudly rather + // than write an unreadable dictionary. + throw new IllegalArgumentException("FCD blockSize must be >= 2, got " + blockSize); + } this.path = path; this.blockSize = blockSize; this.baos = new ByteArrayOutputStream(); this.offsets = new DataOutputBuffer(Path.of("offsets")); - IO.println(path+" "+ID); } private void writeFragment(byte[] data) throws IOException { @@ -87,10 +91,14 @@ public void add(String item) throws IOException { private int commonPrefixLength(String s1, String s2) { int minLength = Math.min(s1.length(), s2.length()); - for (int i = 0; i < minLength; i++) { - if (s1.charAt(i) != s2.charAt(i)) return i; - } - return minLength; + int i = 0; + while (i < minLength && s1.charAt(i) == s2.charAt(i)) i++; + // Never split a UTF-16 surrogate pair: the suffix is encoded to UTF-8 on its + // own, and a suffix starting with an unpaired low surrogate encodes as '?', + // silently corrupting the stored string. Back off so the whole pair stays + // in the suffix. + if (i > 0 && Character.isHighSurrogate(s1.charAt(i - 1))) i--; + return i; } @Override public long getNumEntries() { return numEntries; } diff --git a/src/main/java/com/ebremer/beakgraph/hdf5/writers/HDF5Writer.java b/src/main/java/com/ebremer/beakgraph/hdf5/writers/HDF5Writer.java index 87e8725f..2e4380ce 100644 --- a/src/main/java/com/ebremer/beakgraph/hdf5/writers/HDF5Writer.java +++ b/src/main/java/com/ebremer/beakgraph/hdf5/writers/HDF5Writer.java @@ -8,6 +8,10 @@ import io.jhdf.WritableHdfFile; import io.jhdf.api.WritableGroup; import java.io.IOException; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import org.apache.jena.sparql.core.Quad; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,30 +30,52 @@ private HDF5Writer(Builder builder) { @Override public void write() throws IOException { - IO.println("Writing..."); - PositionalDictionaryWriterBuilder db = new PositionalDictionaryWriterBuilder(); - try (PositionalDictionaryWriter w = db - .setSource(builder.getSource()) - .setDestination(builder.getDestination()) - .setName("dictionary") - .setSpatial(builder.getSpatial()) - .setFeatures(builder.getFeatures()) - .build()) { - Quad[] allQuads = w.getQuads(); - BGIndex gspo = new BGIndex(builder, w, Index.GSPO, allQuads); - BGIndex gpos = new BGIndex(builder, w, Index.GPOS, allQuads); + logger.info("Writing BeakGraph to {}", builder.getDestination()); + Path dest = builder.getDestination().toPath(); + // Build into a sibling temp file and swap it in only on success: a failed + // rebuild must never destroy a previous good artifact at dest (the old + // cleanup deleted dest even when the failure - a parse error, say - + // happened before a single byte was written), and readers never observe + // a half-written file at the published path. + Path tmp = dest.resolveSibling(dest.getFileName() + ".tmp"); + try { + PositionalDictionaryWriterBuilder db = new PositionalDictionaryWriterBuilder(); + try (PositionalDictionaryWriter w = db + .setSource(builder.getSource()) + .setDestination(builder.getDestination()) + .setName(Params.DICTIONARY) + .setSpatial(builder.getSpatial()) + .setFeatures(builder.getFeatures()) + .build()) { + Quad[] allQuads = w.getQuads(); + BGIndex gspo = new BGIndex(builder, w, Index.GSPO, allQuads); + BGIndex gpos = new BGIndex(builder, w, Index.GPOS, allQuads); - IO.print("Creating HDF5 File..." + builder.getDestination() + "..."); - try (WritableHdfFile hdfFile = HdfFile.write(builder.getDestination().toPath())) { - final WritableGroup hdt = hdfFile.putGroup(builder.getName()); - hdt.putAttribute("numQuads", w.getNumberOfQuads()); - hdt.putAttribute("formatVersion", Params.FORMAT_VERSION); - w.add(hdt); - gspo.add(hdt); - gpos.add(hdt); + logger.info("Creating HDF5 file {}", builder.getDestination()); + try (WritableHdfFile hdfFile = HdfFile.write(tmp)) { + final WritableGroup hdt = hdfFile.putGroup(builder.getName()); + hdt.putAttribute("numQuads", w.getNumberOfQuads()); + hdt.putAttribute("formatVersion", Params.FORMAT_VERSION); + w.add(hdt); + gspo.add(hdt); + gpos.add(hdt); + } } + try { + Files.move(tmp, dest, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); + } catch (AtomicMoveNotSupportedException e) { + Files.move(tmp, dest, StandardCopyOption.REPLACE_EXISTING); + } + } catch (IOException | RuntimeException ex) { + // Only the temp file is ever cleaned up; dest is untouched on failure. + try { + Files.deleteIfExists(tmp); + } catch (IOException cleanup) { + logger.warn("Failed to remove temp output {}", tmp, cleanup); + } + throw ex; } - IO.println("Done."); + logger.info("Write complete: {}", builder.getDestination()); } public static class Builder extends AbstractGraphBuilder { diff --git a/src/main/java/com/ebremer/beakgraph/hdf5/writers/MultiTypeDictionaryWriter.java b/src/main/java/com/ebremer/beakgraph/hdf5/writers/MultiTypeDictionaryWriter.java index 1604cbc7..8962e022 100644 --- a/src/main/java/com/ebremer/beakgraph/hdf5/writers/MultiTypeDictionaryWriter.java +++ b/src/main/java/com/ebremer/beakgraph/hdf5/writers/MultiTypeDictionaryWriter.java @@ -14,6 +14,7 @@ import io.jhdf.api.WritableGroup; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; @@ -22,9 +23,9 @@ import java.util.List; import java.util.Set; import java.util.concurrent.atomic.AtomicLong; -import java.util.logging.Level; -import java.util.logging.Logger; import java.util.stream.Stream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import static com.ebremer.beakgraph.utils.UTIL.isRelativeIRI; import org.apache.jena.graph.Node; import org.apache.jena.vocabulary.XSD; @@ -35,7 +36,7 @@ * * @author erbre */ public class MultiTypeDictionaryWriter implements DictionaryWriter, Dictionary, AutoCloseable { - private static final Logger logger = Logger.getLogger(MultiTypeDictionaryWriter.class.getName()); + private static final Logger logger = LoggerFactory.getLogger(MultiTypeDictionaryWriter.class); private final BitPackedUnSignedLongBuffer offsets; private final BitPackedUnSignedLongBuffer typedLiterals; @@ -64,16 +65,18 @@ public class MultiTypeDictionaryWriter implements DictionaryWriter, Dictionary, protected MultiTypeDictionaryWriter(Builder builder) throws FileNotFoundException, IOException { this.name = builder.getName(); - IO.println("Building Dictionary: " + name); - IO.println("Total Nodes : " + builder.getNodes().size()); + logger.info("Building dictionary '{}' ({} nodes)", name, builder.getNodes().size()); Stats stats = builder.getStats(); this.et = builder.getEnabledTypes(); // --- STEP 1: Strict Total Ordering --- - System.out.print("Sorting nodes..."); + // INFO bracketing: sorting tens of millions of nodes takes minutes with no + // other output - this is the writer's longest silent phase. + logger.info("Sorting {} nodes for dictionary '{}'...", builder.getNodes().size(), name); + long sortStart = System.nanoTime(); sorted = NodeSorter.parallelSort(builder.getNodes()); - System.out.println("Done."); + logger.info("Sorted dictionary '{}' in {} s", name, (System.nanoTime() - sortStart) / 1_000_000_000L); // --- STEP 2: Initialize Buffers --- // BitPackedUnSignedLongBuffer constructors do not throw; assign finals directly. @@ -83,6 +86,9 @@ protected MultiTypeDictionaryWriter(Builder builder) throws FileNotFoundExceptio // bit pattern survives the unsigned mask round-trip in BitPackedUnSignedLongBuffer. int intWidth = (stats.minInteger < 0) ? 32 : (1 + MinBits(stats.maxInteger)); int longWidth = (stats.minLong < 0) ? 64 : (1 + MinBits(stats.maxLong)); + // The bit-packed buffer supports widths 1..57 and 64 only, and this width is + // value-derived: a legal xsd:long in [2^56, 2^62) lands in 58..63. Round up. + if (longWidth > 57) longWidth = 64; this.integers = (!et.contains(Types.INTEGER) || (stats.numInteger == 0)) ? null : new BitPackedUnSignedLongBuffer(Path.of("integers"), null, 0, intWidth); this.longs = (!et.contains(Types.LONG) || (stats.numLong == 0)) ? null : new BitPackedUnSignedLongBuffer(Path.of("longs"), null, 0, longWidth); @@ -114,7 +120,8 @@ protected MultiTypeDictionaryWriter(Builder builder) throws FileNotFoundExceptio fcdTLD.add(s); dataTypesLookUp.put(s, fcdTLD.getNumEntries()); } catch (IOException ex) { - logger.log(Level.SEVERE, "Failed to add datatype to dictionary", ex); + // A missing datatype entry shifts every later datatype id. + throw new UncheckedIOException("Failed to add datatype to dictionary: " + s, ex); } }); } @@ -168,7 +175,9 @@ protected MultiTypeDictionaryWriter(Builder builder) throws FileNotFoundExceptio try { close(); } catch (Exception ex) { - logger.log(Level.SEVERE, "Error during dictionary buffer finalization", ex); + // A failed finalization means incomplete buffers; writing them out + // would produce a corrupt dictionary. Abort the build instead. + throw new IllegalStateException("Dictionary buffer finalization failed for '" + name + "'", ex); } } @@ -195,7 +204,10 @@ else if (node.isURI()) { if (literalsPresent) typedLiterals.writeLong(0); if (langTags != null) langTags.writeLong(0); } catch (IOException ex) { - logger.log(Level.SEVERE, null, ex); + // Continuing after a failed iri.add() would leave the offsets and + // datatypes buffers one entry ahead of the IRI dictionary, silently + // corrupting every node after this one. Abort the build instead. + throw new UncheckedIOException("Failed to add IRI to dictionary: " + node, ex); } } else if (node.isLiteral()) { @@ -206,36 +218,46 @@ else if (node.isLiteral()) { String lang = node.getLiteralLanguage(); langTags.writeLong((lang == null || lang.isEmpty()) ? 0L : langLookUp.getOrDefault(lang, 0L)); } - Object val = node.getLiteralValue(); - if (dt.equals(XSD.xlong.getURI()) && longs != null) { + // An ill-typed literal ("abc"^^xsd:int) has no parseable value but is a + // valid RDF term: route it to the strings branch below (term-exact, with + // its datatype IRI) instead of aborting the build here. ProcessQuad + // counts those same terms toward numStrings, so the buffer exists. + Object val; + try { + val = node.getLiteralValue(); + } catch (RuntimeException ex) { + val = null; + } + if (dt.equals(XSD.xlong.getURI()) && longs != null && val instanceof Number num) { offsets.writeLong(longs.getNumEntries()); nativedatatypes.writeInteger(DataType.LONG.ordinal()); - long l = (val instanceof Number n) ? n.longValue() : Long.parseLong(node.getLiteralLexicalForm()); - longs.writeLong(l); + longs.writeLong(num.longValue()); } - else if (dt.equals(XSD.xint.getURI()) && integers != null) { + else if (dt.equals(XSD.xint.getURI()) && integers != null && val instanceof Number num) { // Only xsd:int is bit-packed (32-bit). xsd:integer is unbounded and is // stored via the strings branch below so its value and datatype survive. offsets.writeLong(integers.getNumEntries()); nativedatatypes.writeInteger(DataType.INTEGER.ordinal()); - int i = (val instanceof Number n) ? n.intValue() : Integer.parseInt(node.getLiteralLexicalForm()); - integers.writeInteger(i); + integers.writeInteger(num.intValue()); } - else if (dt.equals(XSD.xdouble.getURI()) && doubles != null) { + else if (dt.equals(XSD.xdouble.getURI()) && doubles != null && val instanceof Number num) { offsets.writeLong(doubles.getNumEntries()); nativedatatypes.writeInteger(DataType.DOUBLE.ordinal()); try { - double d = (val instanceof Number n) ? n.doubleValue() : Double.parseDouble(node.getLiteralLexicalForm()); - doubles.writeDouble(d); - } catch (IOException ex) { logger.log(Level.SEVERE, null, ex); } + doubles.writeDouble(num.doubleValue()); + } catch (IOException ex) { + // See the IRI case: a skipped value desynchronises the dictionary. + throw new UncheckedIOException("Failed to add double literal to dictionary", ex); + } } - else if (dt.equals(XSD.xfloat.getURI()) && floats != null) { + else if (dt.equals(XSD.xfloat.getURI()) && floats != null && val instanceof Number num) { offsets.writeLong(floats.getNumEntries()); nativedatatypes.writeInteger(DataType.FLOAT.ordinal()); try { - float f = (val instanceof Number n) ? n.floatValue() : Float.parseFloat(node.getLiteralLexicalForm()); - floats.writeFloat(f); - } catch (IOException ex) { logger.log(Level.SEVERE, null, ex); } + floats.writeFloat(num.floatValue()); + } catch (IOException ex) { + throw new UncheckedIOException("Failed to add float literal to dictionary", ex); + } } else if (strings != null) { // Fallback for strings, booleans, dates, and custom types @@ -244,7 +266,9 @@ else if (strings != null) { nativedatatypes.writeInteger(DataType.STRING.ordinal()); try { strings.add(lex); - } catch (IOException ex) { logger.log(Level.SEVERE, null, ex); } + } catch (IOException ex) { + throw new UncheckedIOException("Failed to add string literal to dictionary", ex); + } } else { // Unreachable in normal operation: every string-stored datatype is @@ -258,7 +282,17 @@ else if (strings != null) { + "refusing to write a misaligned dictionary entry."); } } - cc.incrementAndGet(); + else { + // A node that is neither blank, URI nor literal (e.g. an RDF-star triple + // term) would write NO buffer entries at all, leaving offsets/datatypes + // one entry short of the sorted node list - every id after it silently + // shifts. Fail the build loudly instead. + throw new IllegalStateException("Unsupported node kind in dictionary '" + name + "': " + node); + } + long c = cc.incrementAndGet(); + if (c % 1_000_000 == 0) { + logger.info("Dictionary '{}': encoded {} / {} nodes", name, c, sorted.size()); + } } @Override diff --git a/src/main/java/com/ebremer/beakgraph/hdf5/writers/PositionalDictionaryWriter.java b/src/main/java/com/ebremer/beakgraph/hdf5/writers/PositionalDictionaryWriter.java index b34b033e..79f4e2d0 100644 --- a/src/main/java/com/ebremer/beakgraph/hdf5/writers/PositionalDictionaryWriter.java +++ b/src/main/java/com/ebremer/beakgraph/hdf5/writers/PositionalDictionaryWriter.java @@ -19,12 +19,15 @@ import java.util.stream.Stream; import org.apache.jena.graph.Node; import org.apache.jena.sparql.core.Quad; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Monolithic Entity Dictionary with Columnar ID lists for Graphs, Subjects, and Objects. * @author Erich Bremer */ public class PositionalDictionaryWriter implements GSPODictionary, AutoCloseable, DictionaryWriter { + private static final Logger logger = LoggerFactory.getLogger(PositionalDictionaryWriter.class); private final DictionaryWriter entitiesdict; private final DictionaryWriter predicatesdict; private final DictionaryWriter literalsdict; @@ -44,7 +47,7 @@ public PositionalDictionaryWriter(PositionalDictionaryWriterBuilder builder) thr this.quads = builder.getQuads(); Stats stats = builder.getStats(); - IO.println(stats); + logger.debug("{}", stats); // 1. Build the Monolithic Entity Dictionary (G, S, O URIs + BNodes) entitiesdict = new MultiTypeDictionaryWriter.Builder() @@ -85,7 +88,7 @@ public PositionalDictionaryWriter(PositionalDictionaryWriterBuilder builder) thr this.objects = new BitPackedUnSignedLongBuffer(Path.of("objects"), null, 0, oBits); // 5. Populate ID lists from the unique sets collected by the Builder - System.out.println("Populating columnar ID lists with unique entities..."); + logger.info("Populating columnar ID lists..."); ArrayList src = parallelSort(builder.getUniqueGraphs()); for (Node n : src) { graphs.writeLong(locateGraph(n)); @@ -103,6 +106,7 @@ public PositionalDictionaryWriter(PositionalDictionaryWriterBuilder builder) thr graphs.prepareForReading(); subjects.prepareForReading(); objects.prepareForReading(); + logger.info("Columnar ID lists populated"); } private static ArrayList parallelSort(Set nodes) { @@ -179,8 +183,12 @@ public void add(WritableGroup group) { if (predicatesdict.getNumberOfNodes() > 0) predicatesdict.add(dictionary); if (literalsdict.getNumberOfNodes() > 0) literalsdict.add(dictionary); - // Add columnar ID lists - if (numQuads > 0) { + // Add columnar ID lists whenever any quads are stored. Gating on the + // SOURCE quad count (numQuads) left an empty-source file internally + // inconsistent: the always-written VoID metadata graph was present in the + // indexes, but with no graphs list, containsGraph answered false and ARQ + // refused to execute GRAPH queries against rows that are demonstrably there. + if (graphs.getNumEntries() > 0) { graphs.add(dictionary); subjects.add(dictionary); objects.add(dictionary); diff --git a/src/main/java/com/ebremer/beakgraph/hdf5/writers/PositionalDictionaryWriterBuilder.java b/src/main/java/com/ebremer/beakgraph/hdf5/writers/PositionalDictionaryWriterBuilder.java index df4907fc..48b1986a 100644 --- a/src/main/java/com/ebremer/beakgraph/hdf5/writers/PositionalDictionaryWriterBuilder.java +++ b/src/main/java/com/ebremer/beakgraph/hdf5/writers/PositionalDictionaryWriterBuilder.java @@ -3,6 +3,7 @@ import com.ebremer.beakgraph.Params; import com.ebremer.beakgraph.core.fuseki.BGVoIDSD; import com.ebremer.beakgraph.core.lib.Stats; +import com.ebremer.beakgraph.utils.ImageTools; import com.ebremer.halcyon.hilbert.HilbertSpace; import com.ebremer.halcyon.hilbert.PolygonScaler; import com.ebremer.halcyon.hilbert.WKTDatatype; @@ -28,6 +29,7 @@ import org.apache.jena.irix.IRIx; import org.apache.jena.rdf.model.Model; import org.apache.jena.riot.Lang; +import org.apache.jena.riot.RDFLanguages; import org.apache.jena.riot.lang.LabelToNode; import org.apache.jena.riot.system.AsyncParser; import org.apache.jena.riot.system.AsyncParserBuilder; @@ -37,6 +39,7 @@ import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.GeometryFactory; import org.locationtech.jts.geom.Polygon; +import org.locationtech.jts.geom.Polygonal; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static com.ebremer.beakgraph.Params.BGVOID; @@ -121,17 +124,21 @@ public class PositionalDictionaryWriterBuilder { NodeFactory.createURI("https://halcyon.is/ns/asWKT12"), NodeFactory.createURI("https://halcyon.is/ns/asWKT13"), NodeFactory.createURI("https://halcyon.is/ns/asWKT14") }; - private static final Node[] hilbertCorner = { - NodeFactory.createURI("https://halcyon.is/ns/hilbertCorner0"), NodeFactory.createURI("https://halcyon.is/ns/hilbertCorner1"), - NodeFactory.createURI("https://halcyon.is/ns/hilbertCorner2"), NodeFactory.createURI("https://halcyon.is/ns/hilbertCorner3"), - NodeFactory.createURI("https://halcyon.is/ns/hilbertCorner4"), NodeFactory.createURI("https://halcyon.is/ns/hilbertCorner5"), - NodeFactory.createURI("https://halcyon.is/ns/hilbertCorner6"), NodeFactory.createURI("https://halcyon.is/ns/hilbertCorner7"), - NodeFactory.createURI("https://halcyon.is/ns/hilbertCorner8"), NodeFactory.createURI("https://halcyon.is/ns/hilbertCorner9"), - NodeFactory.createURI("https://halcyon.is/ns/hilbertCorner10"), NodeFactory.createURI("https://halcyon.is/ns/hilbertCorner11"), - NodeFactory.createURI("https://halcyon.is/ns/hilbertCorner12"), NodeFactory.createURI("https://halcyon.is/ns/hilbertCorner13"), - NodeFactory.createURI("https://halcyon.is/ns/hilbertCorner14") - }; - + /** + * Spatial index entry parameters. Each geometry part's bbox is covered by + * whole Hilbert cells at the coarsest scale where the cover holds at most + * {@link #MAX_INDEX_CELLS} cells, stored as hal:hilbertCell{scale} values. + * The query side covers ITS bbox with ranges at every scale using the same + * floor snapping (see SpatialIndexIterator), so any bbox overlap shares a + * cell at the stored scale: candidates have no false negatives, and the + * sfIntersects filter - which the rewrite no longer removes from the plan - + * eliminates the false positives with real JTS geometry. + */ + public static final String HILBERT_CELL_NS = "https://halcyon.is/ns/hilbertCell"; + public static final int MAX_INDEX_CELLS = 16; + public static final int MAX_INDEX_SCALE = 30; + + private BGVoIDSD xvoid = new BGVoIDSD("https://ebremer.com/void/"); public File getDestination() { return dest; } @@ -205,65 +212,137 @@ private List generateGridURNs(Polygon polygon, int resolutionLevel) { return intersectingURNs; } - private boolean isDegeneratePolygon(String wkt) { - try { - Geometry g = new WKTReader().read(wkt); - if (!(g instanceof Polygon p)) return false; - return p.getExteriorRing().getNumPoints() < 4; - } catch (Exception e) { - return true; - } - } - private ArrayList addSpatial(Quad quad) { final ArrayList qqq = new ArrayList<>(); - String wkt = quad.getObject().getLiteralLexicalForm(); - if (isDegeneratePolygon(wkt)) { - IO.println("Degenerate Polygon : "+wkt); - return qqq; - } - final Polygon[] scales; - scales = PolygonScaler.toPolygons(wkt); + // The GeoSPARQL-standard " WKT" form must be indexed too: strip the + // prefix once here so the parser and the scaler both see plain WKT + // (previously such geometries failed the parse and were silently dropped). + String wkt = ImageTools.stripCrs(quad.getObject().getLiteralLexicalForm()); if (features) { - addFeatures(qqq, quad); + // Same containment as the geometry block below: feature generation runs + // inside the spatial task, so anything escaping here feeds Future.get() + // and fails the whole write over one bad geometry. + try { + addFeatures(qqq, quad, wkt); + } catch (Exception ex) { + logger.warn("Failed to generate features for {}: {}", quad.getSubject(), ex.toString()); + } } + // Everything geometry-related sits inside the catch-all below: a single bad + // geometry must never abort the build (these tasks feed Future.get(), whose + // ExecutionException would otherwise fail the whole write). try { - if (scales == null) { + // Index EVERY polygonal part: MULTIPOLYGON members each get their own + // pyramid and corner entries (indexing only the first part made the + // others unfindable). Non-areal members (POINT, LINESTRING) are indexed + // via their expanded envelopes - PER MEMBER, not only as a fallback + // when no polygon exists: in a mixed GEOMETRYCOLLECTION the point/line + // members next to a polygon member were silently unindexed, invisible + // to every spatial query. + Geometry g = new WKTReader().read(wkt); + if (g.isEmpty()) { return qqq; } - final String[] wktScales = PolygonScaler.toWKT(scales); - for (int s=0; s tiles = generateGridURNs(scales[s],s); - try { - for (int ii=0; ii parts = new ArrayList<>(ImageTools.wktToPolygons(wkt)); + GeometryFactory gf = new GeometryFactory(); + for (int i = 0; i < g.getNumGeometries(); i++) { + Geometry member = g.getGeometryN(i); + if (!(member instanceof Polygonal) && !member.isEmpty()) { + Envelope env = member.getEnvelopeInternal(); + env.expandBy(0.5); // a point/degenerate envelope still needs area + parts.add((Polygon) gf.toGeometry(env)); } } + for (Polygon part : parts) { + addSpatialIndexCells(qqq, quad, part); + addSpatialScales(qqq, quad, wkt, PolygonScaler.toPolygons(part)); + } } catch (Exception ex) { - logger.error("Failed to add spatial data for {}", wkt, ex); + // Expected data condition (pathology exports contain degenerate + // geometries such as two-point rings): one line per skip, no stack - + // a slide can contain thousands of these. + logger.warn("Skipping spatial indexing for {}: {} ({})", + quad.getSubject(), ex.toString(), abbrevWkt(wkt)); } return qqq; } + + private void addSpatialScales(ArrayList qqq, Quad quad, String wkt, Polygon[] scales) { + if (scales == null) { + return; + } + final String[] wktScales = PolygonScaler.toWKT(scales); + for (int s=0; s tiles = generateGridURNs(scales[s],s); + try { + for (int ii=0; ii qqq, Quad quad, Polygon part) { + Envelope env = part.getEnvelopeInternal(); + // The Hilbert domain is [0, 2^31), so bboxes are CLAMPED into it - never + // skipped, never allowed to alias (the curve masks out-of-range bits). + // Clamping is a monotone projection applied identically on the query + // side, so overlapping boxes still overlap after it and recall is + // preserved; out-of-domain geometry just indexes coarsely at the domain + // edge cells (false positives there are killed by the sfIntersects + // verification on the exact original WKT). Skipping fully-negative + // geometry instead made it unfindable by ANY query. + long minX = HilbertSpace.clampToDomain((long) Math.floor(env.getMinX())); + long maxX = HilbertSpace.clampToDomain((long) Math.floor(env.getMaxX())); + long minY = HilbertSpace.clampToDomain((long) Math.floor(env.getMinY())); + long maxY = HilbertSpace.clampToDomain((long) Math.floor(env.getMaxY())); + int s = 0; + while (s < MAX_INDEX_SCALE && cellCount(minX, maxX, minY, maxY, s) > MAX_INDEX_CELLS) { + s++; + } + long cell = 1L << s; + Node pred = NodeFactory.createURI(HILBERT_CELL_NS + s); + HashSet cells = new HashSet<>(); + for (long x = Math.floorDiv(minX, cell); x <= Math.floorDiv(maxX, cell); x++) { + for (long y = Math.floorDiv(minY, cell); y <= Math.floorDiv(maxY, cell); y++) { + cells.add(HilbertSpace.hc.index(new long[]{x, y})); + } + } + for (Long c : cells) { + qqq.add(Quad.create(Params.SPATIAL, quad.getSubject(), pred, NodeFactory.createLiteralByValue((long) c))); + } + } + + private static long cellCount(long minX, long maxX, long minY, long maxY, int s) { + long cell = 1L << s; + long nx = Math.floorDiv(maxX, cell) - Math.floorDiv(minX, cell) + 1; + long ny = Math.floorDiv(maxY, cell) - Math.floorDiv(minY, cell) + 1; + return nx * ny; + } + + private static String abbrevWkt(String wkt) { + return (wkt != null && wkt.length() > 200) ? wkt.substring(0, 200) + "..." : wkt; + } - private void addFeatures(ArrayList qqq, Quad quad) { + // Takes the CRS-stripped WKT computed once in addSpatial: re-reading the raw + // lexical form here made JTS throw on every " WKT"-form literal, so + // CRS-prefixed geometries silently got no derived features while identical + // unprefixed ones did. + private void addFeatures(ArrayList qqq, Quad quad, String wkt) { Node geo = quad.getSubject(); - String wkt = quad.getObject().getLiteralLexicalForm(); Gen2DFeatures.generate(qqq, geo, wkt); MajorMinor.add(qqq, geo, wkt); } @@ -351,7 +430,62 @@ private Node relativizeNode(Node n) { u.equals(REL_BASE) ? "" : u.substring(REL_BASE_PREFIX.length())); } - private void ProcessQuad(Quad quad) { + /** + * Numeric literal canonicalization policy: xsd:int / xsd:long / xsd:float / + * xsd:double objects are stored by VALUE and the reader regenerates the + * canonical lexical form, so two lexical variants of one value ("01" vs + * "1"^^xsd:int) would become two term-distinct dictionary entries that both + * extract to the same canonical term - duplicate "equal" entries that break + * the strict ordering the dictionary binary search relies on, leaving some + * triples unreachable by term lookup. Rewriting the object to its canonical + * term at ingest collapses the variants onto one entry and keeps locate() + * and extract() symmetric. (The value-typed storage never preserved the + * non-canonical lexical form anyway.) + */ + private Quad canonicalizeNumericObject(Quad quad) { + Node o = quad.getObject(); + if (!o.isLiteral()) return quad; + String dt = o.getLiteralDatatypeURI(); + try { + Node canonical = null; + if (XSD.xint.getURI().equals(dt)) { + if (o.getLiteralValue() instanceof Number n) canonical = NodeFactory.createLiteralByValue(n.intValue()); + } else if (XSD.xlong.getURI().equals(dt)) { + if (o.getLiteralValue() instanceof Number n) canonical = NodeFactory.createLiteralByValue(n.longValue()); + } else if (XSD.xfloat.getURI().equals(dt)) { + if (o.getLiteralValue() instanceof Number n) canonical = NodeFactory.createLiteralByValue(n.floatValue()); + } else if (XSD.xdouble.getURI().equals(dt)) { + if (o.getLiteralValue() instanceof Number n) canonical = NodeFactory.createLiteralByValue(n.doubleValue()); + } + if (canonical == null || canonical.equals(o)) return quad; + return new Quad(quad.getGraph(), quad.getSubject(), quad.getPredicate(), canonical); + } catch (RuntimeException e) { + // Malformed numeric literal: leave it untouched; downstream handling decides. + return quad; + } + } + + /** + * getLiteralValue(), or null for an ill-typed literal ("abc"^^xsd:int). RDF + * permits such terms and parsers accept them with a warning; they are counted + * toward (and stored via) the lexical strings path instead of aborting the + * whole build on a DatatypeFormatException. + */ + private static Object literalValueOrNull(Node o) { + try { + return o.getLiteralValue(); + } catch (RuntimeException e) { + return null; + } + } + + private void countStringStored(String lex) { + this.stats.longestStringLength = Math.max(this.stats.longestStringLength, lex.length()); + this.stats.shortestStringLength = Math.min(this.stats.shortestStringLength, lex.length()); + this.stats.numStrings++; + } + + private void ProcessQuad(Quad quad) { Node g = quad.getGraph(); Node s = quad.getSubject(); Node p = quad.getPredicate(); @@ -374,6 +508,11 @@ private void ProcessQuad(Quad quad) { stats.numBlankNodes++; } else if (s.isURI()) { stats.numIRI++; + } else { + // Same guard as the graph and object positions: any other node kind + // (e.g. a triple term) writes no dictionary entry and silently + // desynchronizes every id after it. Fail the build loudly instead. + throw new IllegalStateException("Unexpected subject node type (not URI or blank): " + s); } entities.add(s); } @@ -386,43 +525,49 @@ private void ProcessQuad(Quad quad) { String dt = o.getLiteralDatatypeURI(); dataTypes.add(dt); if (dt.equals(XSD.xlong.getURI())) { - Number n = (Number) o.getLiteralValue(); - this.stats.maxLong = Math.max(this.stats.maxLong, n.longValue()); - this.stats.minLong = Math.min(this.stats.minLong, n.longValue()); - this.stats.numLong++; + if (literalValueOrNull(o) instanceof Number n) { + this.stats.maxLong = Math.max(this.stats.maxLong, n.longValue()); + this.stats.minLong = Math.min(this.stats.minLong, n.longValue()); + this.stats.numLong++; + } else { + countStringStored(o.getLiteralLexicalForm()); // ill-typed: strings path + } } else if (dt.equals(XSD.xint.getURI())) { // Only xsd:int (32-bit bounded) is bit-packed here. xsd:integer is // unbounded, so it is handled by the string fallback below instead; // bit-packing it would truncate large values and change the datatype // to xsd:int on read-back. - Number n = (Number) o.getLiteralValue(); - this.stats.maxInteger = Math.max(this.stats.maxInteger, n.intValue()); - this.stats.minInteger = Math.min(this.stats.minInteger, n.intValue()); - this.stats.numInteger++; + if (literalValueOrNull(o) instanceof Number n) { + this.stats.maxInteger = Math.max(this.stats.maxInteger, n.intValue()); + this.stats.minInteger = Math.min(this.stats.minInteger, n.intValue()); + this.stats.numInteger++; + } else { + countStringStored(o.getLiteralLexicalForm()); // ill-typed: strings path + } } else if (dt.equals(XSD.xfloat.getURI())) { - Number n = (Number) o.getLiteralValue(); - this.stats.maxFloat = Math.max(this.stats.maxFloat, n.floatValue()); - this.stats.minFloat = Math.min(this.stats.minFloat, n.floatValue()); - this.stats.numFloat++; + if (literalValueOrNull(o) instanceof Number n) { + this.stats.maxFloat = Math.max(this.stats.maxFloat, n.floatValue()); + this.stats.minFloat = Math.min(this.stats.minFloat, n.floatValue()); + this.stats.numFloat++; + } else { + countStringStored(o.getLiteralLexicalForm()); // ill-typed: strings path + } } else if (dt.equals(XSD.xdouble.getURI())) { - Number n = (Number) o.getLiteralValue(); - this.stats.maxDouble = Math.max(this.stats.maxDouble, n.doubleValue()); - this.stats.minDouble = Math.min(this.stats.minDouble, n.doubleValue()); - this.stats.numDouble++; + if (literalValueOrNull(o) instanceof Number n) { + this.stats.maxDouble = Math.max(this.stats.maxDouble, n.doubleValue()); + this.stats.minDouble = Math.min(this.stats.minDouble, n.doubleValue()); + this.stats.numDouble++; + } else { + countStringStored(o.getLiteralLexicalForm()); // ill-typed: strings path + } } else if (dt.equals(XSD.xstring.getURI()) || dt.equals(GEO.wktLiteral.getURI()) || dt.equals(XSD.xboolean.getURI()) || dt.equals(RDF.langString.getURI())) { // rdf:langString shares the strings buffer; its language tag is // stored separately by MultiTypeDictionaryWriter (langs/langTags). - String wow = (String) o.getLiteralLexicalForm(); - this.stats.longestStringLength = Math.max(this.stats.longestStringLength, wow.length()); - this.stats.shortestStringLength = Math.min(this.stats.shortestStringLength, wow.length()); - this.stats.numStrings++; + countStringStored(o.getLiteralLexicalForm()); } else if (dt.equals(XSD.dateTime.getURI())) { - String lex = o.getLiteralLexicalForm(); + String lex = o.getLiteralLexicalForm(); int t = lex.indexOf('T'); - String wow = (t > 0) ? lex.substring(0, t) : lex; - this.stats.longestStringLength = Math.max(this.stats.longestStringLength, wow.length()); - this.stats.shortestStringLength = Math.min(this.stats.shortestStringLength, wow.length()); - this.stats.numStrings++; + countStringStored((t > 0) ? lex.substring(0, t) : lex); } else { // Any other datatype (xsd:integer, xsd:decimal, xsd:date, custom // datatypes, ...) is stored verbatim in the strings buffer by @@ -430,13 +575,10 @@ private void ProcessQuad(Quad quad) { // toward numStrings so that buffer is always allocated; otherwise the // writer would have nowhere to put it and would drop the node, // desynchronising the offset/datatype buffers and corrupting the dictionary. - String wow = o.getLiteralLexicalForm(); - this.stats.longestStringLength = Math.max(this.stats.longestStringLength, wow.length()); - this.stats.shortestStringLength = Math.min(this.stats.shortestStringLength, wow.length()); - this.stats.numStrings++; + countStringStored(o.getLiteralLexicalForm()); } literals.add(o); - } + } } else { if (!entities.contains(o)) { if (o.isBlank()) { @@ -457,12 +599,21 @@ public PositionalDictionaryWriter build() throws IOException { try (InputStream xis = src.toString().endsWith(".gz") ? new GZIPInputStream(new FileInputStream(src)) : new FileInputStream(src)) { + // Detect the syntax from the file name (TriG, N-Quads, N-Triples, + // Turtle, ...; .gz handled) instead of hardcoding Turtle: this is a + // quad store, and named graphs can only arrive through a quad-capable + // syntax. + String fname = src.getName(); + if (fname.endsWith(".gz")) { + fname = fname.substring(0, fname.length() - 3); + } + Lang lang = RDFLanguages.filenameToLang(fname, Lang.TURTLE); // Parse relative references against a stable sentinel base so they // resolve deterministically (not against the process working // directory). The relativize() step below strips the sentinel back // off; the relative form is resolved at query time against the URL // the .h5 file is served from. - AsyncParserBuilder parserBuilder = AsyncParser.of(xis, Lang.TURTLE, REL_BASE); + AsyncParserBuilder parserBuilder = AsyncParser.of(xis, lang, REL_BASE); parserBuilder.mutateSources(rdfBuilder -> rdfBuilder.labelToNode(LabelToNode.createUseLabelAsGiven())); final List>> spatialTasks = new ArrayList<>(); @@ -477,10 +628,11 @@ public PositionalDictionaryWriter build() throws IOException { : quad) .map(this::relativize) .map(this::AlignBnodes) + .map(this::canonicalizeNumericObject) .forEach(quad -> { quadcount.incrementAndGet(); if (quadcount.get() % 100_000 == 0) { - System.out.println("Loaded " + quadcount.get() + " quads..."); + logger.info("Loaded {} quads...", quadcount.get()); } quadslist.add(quad); // Let an invalid quad abort the write rather than silently skipping @@ -508,8 +660,9 @@ public PositionalDictionaryWriter build() throws IOException { throw new IOException("Spatial processing failed for " + src, ex.getCause()); } extraQuads.forEach(q -> { - quadslist.add(q); - ProcessQuad(q); + Quad canon = canonicalizeNumericObject(q); + quadslist.add(canon); + ProcessQuad(canon); }); } Model xxx = xvoid.getModel(); @@ -524,7 +677,7 @@ public PositionalDictionaryWriter build() throws IOException { xxx.setNsPrefix("exif", "http://www.w3.org/2003/12/exif/ns#"); xvoid.getModel().listStatements().forEach(s->{ Triple ff = s.asTriple(); - Quad qqq = Quad.create(BGVOID, ff); + Quad qqq = canonicalizeNumericObject(Quad.create(BGVOID, ff)); ProcessQuad(qqq); quadslist.add(qqq); }); @@ -542,7 +695,7 @@ public PositionalDictionaryWriter build() throws IOException { this.numQuads = quadcount.get(); this.quads = quadslist.toArray(Quad[]::new); quadslist.clear(); - System.out.println("Dictionary created. Total Quads: " + this.numQuads); + logger.info("Dictionary created. Total quads: {}", this.numQuads); return new PositionalDictionaryWriter(this); } diff --git a/src/main/java/com/ebremer/beakgraph/huge/ExternalSorter.java b/src/main/java/com/ebremer/beakgraph/huge/ExternalSorter.java new file mode 100644 index 00000000..829bd96e --- /dev/null +++ b/src/main/java/com/ebremer/beakgraph/huge/ExternalSorter.java @@ -0,0 +1,254 @@ +package com.ebremer.beakgraph.huge; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.DataOutput; +import java.io.DataOutputStream; +import java.io.EOFException; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.PriorityQueue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * External merge sort over records of type {@code T}: records are buffered in + * RAM, sorted and spilled as runs to temp files when the buffer fills, and the + * runs are k-way merged (multi-level when there are more runs than the fan-in) + * into one sorted stream. This is what lets the huge writer sort node and quad + * populations far larger than the heap - the RAM writer's + * {@code Arrays.parallelSort} over all quads is the size ceiling this removes. + * + *

    Single-consumer: {@link #sorted()} may be called once. Duplicate records + * are preserved (callers dedup while streaming, exactly like the RAM writers + * dedup after sorting). + * + * @author Erich Bremer + */ +final class ExternalSorter implements AutoCloseable { + + private static final Logger logger = LoggerFactory.getLogger(ExternalSorter.class); + + /** Serializes records for spill runs. {@code read} throws {@link EOFException} at run end. */ + interface Codec { + void write(DataOutput out, T record) throws IOException; + + T read(DataInput in) throws IOException; + } + + /** A sorted record stream that owns temp files; must be closed. */ + interface SortedStream extends Iterator, AutoCloseable { + @Override + void close(); + } + + private final Path workDir; + private final String tag; + private final Codec codec; + private final Comparator comparator; + private final int maxRecordsInMemory; + private final int mergeFanIn; + private ArrayList buffer = new ArrayList<>(); + private final List runs = new ArrayList<>(); + private long size = 0; + private int runCounter = 0; + private boolean consumed = false; + + ExternalSorter(Path workDir, String tag, Codec codec, Comparator comparator, + int maxRecordsInMemory, int mergeFanIn) { + if (maxRecordsInMemory < 1 || mergeFanIn < 2) { + throw new IllegalArgumentException("maxRecordsInMemory >= 1 and mergeFanIn >= 2 required"); + } + this.workDir = workDir; + this.tag = tag; + this.codec = codec; + this.comparator = comparator; + this.maxRecordsInMemory = maxRecordsInMemory; + this.mergeFanIn = mergeFanIn; + } + + void add(T record) throws IOException { + if (consumed) { + throw new IllegalStateException("Sorter '" + tag + "' already consumed"); + } + buffer.add(record); + size++; + if (buffer.size() >= maxRecordsInMemory) { + spillRun(); + } + } + + /** Total records added. */ + long size() { + return size; + } + + @SuppressWarnings("unchecked") + private T[] sortedBufferArray() { + Object[] arr = buffer.toArray(); + // parallelSort: the comparator (NodeComparator for term records) is the + // dominant cost of a run; use the cores exactly like the RAM writer does. + Arrays.parallelSort(arr, (Comparator) comparator); + return (T[]) arr; + } + + private void spillRun() throws IOException { + if (buffer.isEmpty()) return; + T[] arr = sortedBufferArray(); + Path run = workDir.resolve(tag + ".run" + (runCounter++)); + try (DataOutputStream out = new DataOutputStream( + new BufferedOutputStream(Files.newOutputStream(run), 1 << 16))) { + for (T rec : arr) { + codec.write(out, rec); + } + } + runs.add(run); + logger.debug("Sorter '{}': spilled run {} ({} records)", tag, run.getFileName(), arr.length); + buffer = new ArrayList<>(); + } + + /** + * Finishes ingestion and returns the fully sorted stream. All-in-RAM inputs + * (no spilled run) sort and iterate without touching disk. + */ + SortedStream sorted() throws IOException { + if (consumed) { + throw new IllegalStateException("Sorter '" + tag + "' already consumed"); + } + consumed = true; + if (runs.isEmpty()) { + T[] arr = sortedBufferArray(); + buffer = new ArrayList<>(); + Iterator it = Arrays.asList(arr).iterator(); + return new SortedStream() { + @Override public boolean hasNext() { return it.hasNext(); } + @Override public T next() { return it.next(); } + @Override public void close() {} + }; + } + spillRun(); + // Multi-level merge: collapse groups of fan-in runs until one merge pass + // can stream them all. + while (runs.size() > mergeFanIn) { + List group = new ArrayList<>(runs.subList(0, mergeFanIn)); + runs.subList(0, mergeFanIn).clear(); + Path merged = workDir.resolve(tag + ".run" + (runCounter++)); + logger.debug("Sorter '{}': intermediate merge of {} runs", tag, group.size()); + try (MergeIterator mergeIt = new MergeIterator(group); + DataOutputStream out = new DataOutputStream( + new BufferedOutputStream(Files.newOutputStream(merged), 1 << 16))) { + while (mergeIt.hasNext()) { + codec.write(out, mergeIt.next()); + } + } + runs.add(merged); + } + List finalRuns = new ArrayList<>(runs); + runs.clear(); + return new MergeIterator(finalRuns); + } + + @Override + public void close() { + buffer = new ArrayList<>(); + for (Path run : runs) { + try { + Files.deleteIfExists(run); + } catch (IOException e) { + logger.warn("Failed to delete spill run {}", run, e); + } + } + runs.clear(); + } + + private final class RunReader { + final Path path; + final DataInputStream in; + T head; + + RunReader(Path path) throws IOException { + this.path = path; + this.in = new DataInputStream(new BufferedInputStream(Files.newInputStream(path), 1 << 16)); + } + + /** Loads the next record into {@code head}; false (and cleanup) at run end. */ + boolean advance() throws IOException { + try { + head = codec.read(in); + return true; + } catch (EOFException eof) { + head = null; + closeAndDelete(); + return false; + } + } + + void closeAndDelete() { + try { in.close(); } catch (IOException ignored) {} + try { Files.deleteIfExists(path); } catch (IOException ignored) {} + } + } + + private final class MergeIterator implements SortedStream { + private final PriorityQueue heap; + private final List readers = new ArrayList<>(); + + MergeIterator(List runPaths) throws IOException { + this.heap = new PriorityQueue<>(Math.max(2, runPaths.size()), + (a, b) -> comparator.compare(a.head, b.head)); + try { + for (Path p : runPaths) { + RunReader r = new RunReader(p); + readers.add(r); + if (r.advance()) { + heap.add(r); + } + } + } catch (IOException e) { + close(); + throw e; + } + } + + @Override + public boolean hasNext() { + return !heap.isEmpty(); + } + + @Override + public T next() { + RunReader r = heap.poll(); + if (r == null) { + throw new NoSuchElementException(); + } + T result = r.head; + try { + if (r.advance()) { + heap.add(r); + } + } catch (IOException e) { + close(); + throw new UncheckedIOException("Merge failed for sorter '" + tag + "'", e); + } + return result; + } + + @Override + public void close() { + for (RunReader r : readers) { + r.closeAndDelete(); + } + heap.clear(); + } + } +} diff --git a/src/main/java/com/ebremer/beakgraph/huge/HugeBuildPipeline.java b/src/main/java/com/ebremer/beakgraph/huge/HugeBuildPipeline.java new file mode 100644 index 00000000..5c8bfdc1 --- /dev/null +++ b/src/main/java/com/ebremer/beakgraph/huge/HugeBuildPipeline.java @@ -0,0 +1,763 @@ +package com.ebremer.beakgraph.huge; + +import com.ebremer.beakgraph.Params; +import com.ebremer.beakgraph.core.fuseki.BGVoIDSD; +import com.ebremer.beakgraph.core.lib.NodeComparator; +import com.ebremer.beakgraph.core.lib.Stats; +import com.ebremer.beakgraph.hdf5.Index; +import com.ebremer.beakgraph.hdf5.Types; +import com.ebremer.beakgraph.huge.HugeRecords.IdQuad; +import com.ebremer.beakgraph.huge.HugeRecords.RowId; +import com.ebremer.beakgraph.huge.HugeRecords.TermRow; +import static com.ebremer.beakgraph.utils.UTIL.MinBits; +import com.ebremer.ns.GEO; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.file.Path; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.PriorityQueue; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.zip.GZIPInputStream; +import org.apache.jena.graph.Node; +import org.apache.jena.graph.NodeFactory; +import org.apache.jena.graph.Triple; +import org.apache.jena.irix.IRIx; +import org.apache.jena.riot.Lang; +import org.apache.jena.riot.RDFLanguages; +import org.apache.jena.riot.lang.LabelToNode; +import org.apache.jena.riot.system.AsyncParser; +import org.apache.jena.riot.system.AsyncParserBuilder; +import org.apache.jena.sparql.core.Quad; +import org.apache.jena.vocabulary.RDF; +import org.apache.jena.vocabulary.XSD; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The disk-based BeakGraph build: parse once, spill per-column (term, row) + * records, external-sort them, derive the dictionaries by merge-dedup, assign + * ids by sort-merge join (no random lookups), zip the id columns into encoded + * quads, external-sort those twice for the GSPO/GPOS indexes, and stream every + * buffer into the HDF5 file. RAM stays bounded by the sorter batch sizes plus + * the small in-memory populations (predicates, datatypes, language tags, VoID + * statistics) regardless of quad count. + * + *

    Quad-level transforms (default-graph rewrite, relative-IRI handling, + * numeric canonicalization, spatial/feature augmentation, VoID metadata) mirror + * {@link com.ebremer.beakgraph.hdf5.writers.PositionalDictionaryWriterBuilder}. + * The ONE deliberate divergence: blank nodes are NOT relabelled to + * first-appearance b%020d form (that map is unbounded RAM). Labels only order + * bnodes among themselves - the format stores bnodes by dictionary rank and + * readers regenerate labels from ids - so the output is isomorphic, not + * byte-identical, to the RAM writer's. + * + * @author Erich Bremer + */ +final class HugeBuildPipeline implements AutoCloseable { + + private static final Logger logger = LoggerFactory.getLogger(HugeBuildPipeline.class); + + // Same sentinel base as the RAM builder: relative references resolve against + // it during parsing and are stripped back to relative form for storage. + private static final String REL_BASE = "http://beakgraph.invalid/document"; + private static final String REL_BASE_PREFIX = "http://beakgraph.invalid/"; + private static final IRIx REL_BASE_IRIX = IRIx.create(REL_BASE); + + private final File src; + private final boolean spatial; + private final boolean features; + private final Path workDir; + private final int mergeFanIn; + + // ---- Pass A state ---- + private final Stats stats = new Stats(); + private final TreeSet dataTypes = new TreeSet<>(); + private final TreeSet langSet = new TreeSet<>(); + // Predicates are the one population kept in RAM (real-world predicate counts + // are tiny next to entities/literals); they get temp ids during the parse + // and final rank ids once the set is complete. + private final HashMap predTempIds = new HashMap<>(); + private final ArrayList predByTempId = new ArrayList<>(); + private final BGVoIDSD xvoid = new BGVoIDSD("https://ebremer.com/void/"); + private ExternalSorter gSorter; + private ExternalSorter sSorter; + private ExternalSorter oSorter; + private RecordFile pTempFile; + private final int idSpillBatch; + private long rows = 0; + private long parsedQuads = 0; + + // ---- Build products (owned; released in close()) ---- + private final List resources = new ArrayList<>(); + + HugeBuildPipeline(File src, boolean spatial, boolean features, Path workDir, + int termSpillBatch, int idSpillBatch, int mergeFanIn) { + this.src = src; + this.spatial = spatial; + this.features = features; + this.workDir = workDir; + this.mergeFanIn = mergeFanIn; + this.gSorter = track(new ExternalSorter<>(workDir, "gcol", HugeRecords.TERM_ROW_CODEC, + HugeRecords.TERM_ORDER, termSpillBatch, mergeFanIn)); + this.sSorter = track(new ExternalSorter<>(workDir, "scol", HugeRecords.TERM_ROW_CODEC, + HugeRecords.TERM_ORDER, termSpillBatch, mergeFanIn)); + this.oSorter = track(new ExternalSorter<>(workDir, "ocol", HugeRecords.TERM_ROW_CODEC, + HugeRecords.TERM_ORDER, termSpillBatch, mergeFanIn)); + this.idSpillBatch = idSpillBatch; + } + + private T track(T resource) { + resources.add(resource); + return resource; + } + + /** Runs the whole build, producing the finished HDF5 file at {@code tmpH5}. */ + void run(Path tmpH5) throws IOException { + ingest(); + + // ---- Predicates: final rank ids from the in-RAM population ---- + Node[] sortedPreds = predByTempId.toArray(Node[]::new); + Arrays.parallelSort(sortedPreds, NodeComparator.INSTANCE); + long numPredicates = sortedPreds.length; + long[] tempToFinal = new long[(int) Math.min(Integer.MAX_VALUE, predByTempId.size())]; + { + HashMap finalIds = new HashMap<>(); + for (int i = 0; i < sortedPreds.length; i++) { + finalIds.put(sortedPreds[i], (long) i + 1); // 1-based + } + for (int t = 0; t < predByTempId.size(); t++) { + tempToFinal[t] = finalIds.get(predByTempId.get(t)); + } + } + + // ---- Materialize the sorted columns (each is re-read several times) ---- + logger.info("Sorting {} rows per column (external merge sort)...", rows); + RecordFile gSorted = materialize(gSorter, "gcol.sorted"); + gSorter = null; + RecordFile sSorted = materialize(sSorter, "scol.sorted"); + sSorter = null; + RecordFile oSorted = materialize(oSorter, "ocol.sorted"); + oSorter = null; + + // ---- Distinct sorted dictionaries on disk ---- + RecordFile entFile = new RecordFile<>(workDir.resolve("entities.sorted"), NodeCodec.INSTANCE); + mergeDistinctEntities(entFile, gSorted, sSorted, oSorted); + long numEntities = entFile.count(); + + RecordFile litFile = new RecordFile<>(workDir.resolve("literals.sorted"), NodeCodec.INSTANCE); + distinctLiterals(litFile, oSorted); + long numLiterals = litFile.count(); + long numObjects = numEntities + numLiterals; + logger.info("Dictionary populations: {} entities, {} predicates, {} literals", + numEntities, numPredicates, numLiterals); + + // ---- Encode the three dictionary sections ---- + StreamingDictionaryWriter entitiesDict = null; + StreamingDictionaryWriter predicatesDict = null; + StreamingDictionaryWriter literalsDict = null; + if (numEntities > 0) { + entitiesDict = track(new StreamingDictionaryWriter(workDir, "entities", numEntities, stats, + Set.of(Types.IRI, Types.BNODE), new TreeSet<>(), new TreeSet<>())); + try (var s = entFile.read()) { + entitiesDict.encode(s); + } + } + if (numPredicates > 0) { + predicatesDict = track(new StreamingDictionaryWriter(workDir, "predicates", numPredicates, stats, + Set.of(Types.IRI), new TreeSet<>(), new TreeSet<>())); + predicatesDict.encode(Arrays.asList(sortedPreds).iterator()); + } + if (numLiterals > 0) { + literalsDict = track(new StreamingDictionaryWriter(workDir, "literals", numLiterals, stats, + Set.of(Types.DOUBLE, Types.FLOAT, Types.LONG, Types.INTEGER, Types.STRING), + dataTypes, langSet)); + try (var s = litFile.read()) { + literalsDict.encode(s); + } + } + + // ---- Columnar unique-id lists (same widths as PositionalDictionaryWriter) ---- + int gBits = (int) (Math.ceil(MinBits(numEntities + 1) / 8.0) * 8); + int sBits = (int) (Math.ceil(MinBits(numEntities + 1) / 8.0) * 8); + int oBits = (int) (Math.ceil(MinBits(numObjects + 1) / 8.0) * 8); + SpillBitPackedBuffer graphsList = track(new SpillBitPackedBuffer(workDir.resolve("columnar.graphs"), gBits)); + SpillBitPackedBuffer subjectsList = track(new SpillBitPackedBuffer(workDir.resolve("columnar.subjects"), sBits)); + SpillBitPackedBuffer objectsList = track(new SpillBitPackedBuffer(workDir.resolve("columnar.objects"), oBits)); + + // ---- Sort-merge id joins ---- + logger.info("Assigning ids (sort-merge joins)..."); + ExternalSorter gIds = track(new ExternalSorter<>(workDir, "gid", HugeRecords.ROW_ID_CODEC, + HugeRecords.ROW_ORDER, idSpillBatch, mergeFanIn)); + joinColumn(gSorted, "Graph", entityCursor(entFile, null, numEntities), graphsList, gIds); + gSorted.delete(); + + ExternalSorter sIds = track(new ExternalSorter<>(workDir, "sid", HugeRecords.ROW_ID_CODEC, + HugeRecords.ROW_ORDER, idSpillBatch, mergeFanIn)); + joinColumn(sSorted, "Subject", entityCursor(entFile, null, numEntities), subjectsList, sIds); + sSorted.delete(); + + ExternalSorter oIds = track(new ExternalSorter<>(workDir, "oid", HugeRecords.ROW_ID_CODEC, + HugeRecords.ROW_ORDER, idSpillBatch, mergeFanIn)); + joinColumn(oSorted, "Object", entityCursor(entFile, litFile, numEntities), objectsList, oIds); + oSorted.delete(); + entFile.delete(); + litFile.delete(); + + // ---- Predicate column: temp ids -> final ids, already in row order ---- + RecordFile pFinal = new RecordFile<>(workDir.resolve("pcol.final"), HugeRecords.VAR_LONG_CODEC); + try (var s = pTempFile.read()) { + while (s.hasNext()) { + pFinal.append(tempToFinal[Math.toIntExact(s.next())]); + } + } + pFinal.finish(); + pTempFile.delete(); + if (pFinal.count() != rows) { + throw new IllegalStateException("Predicate column has " + pFinal.count() + " entries for " + rows + " rows"); + } + + // ---- Zip id columns into encoded quads, feeding both index sorters ---- + logger.info("Encoding {} quads...", rows); + ExternalSorter gspoSorter = track(new ExternalSorter<>(workDir, "gspo", HugeRecords.ID_QUAD_CODEC, + HugeRecords.GSPO_ORDER, idSpillBatch, mergeFanIn)); + ExternalSorter gposSorter = track(new ExternalSorter<>(workDir, "gpos", HugeRecords.ID_QUAD_CODEC, + HugeRecords.GPOS_ORDER, idSpillBatch, mergeFanIn)); + try (var gs = gIds.sorted(); var ss = sIds.sorted(); var os = oIds.sorted(); var ps = pFinal.read()) { + for (long row = 0; row < rows; row++) { + RowId g = gs.next(); + RowId s = ss.next(); + RowId o = os.next(); + long p = ps.next(); + if (g.row() != row || s.row() != row || o.row() != row) { + throw new IllegalStateException("Row misalignment at " + row + ": g=" + g.row() + + " s=" + s.row() + " o=" + o.row()); + } + IdQuad q = new IdQuad(g.id(), s.id(), p, o.id()); + gspoSorter.add(q); + gposSorter.add(q); + } + } + pFinal.delete(); + + // ---- Indexes ---- + HugeIndexWriter gspo = track(new HugeIndexWriter(workDir, Index.GSPO, + numEntities, numEntities, numPredicates, numObjects, rows)); + try (var it = gspoSorter.sorted()) { + gspo.build(it); + } + HugeIndexWriter gpos = track(new HugeIndexWriter(workDir, Index.GPOS, + numEntities, numEntities, numPredicates, numObjects, rows)); + try (var it = gposSorter.sorted()) { + gpos.build(it); + } + + // ---- Assemble the HDF5 file (same names/attributes as HDF5Writer) ---- + logger.info("Writing HDF5 file {}", tmpH5); + try (StreamingHdf5File hdf = StreamingHdf5.create(tmpH5)) { + StreamingHdf5Group hdt = hdf.rootGroup().putGroup(Params.BG); + hdt.putAttribute("numQuads", parsedQuads); + hdt.putAttribute("formatVersion", Params.FORMAT_VERSION); + StreamingHdf5Group dictionary = hdt.putGroup(Params.DICTIONARY); + if (entitiesDict != null) entitiesDict.transferTo(dictionary); + if (predicatesDict != null) predicatesDict.transferTo(dictionary); + if (literalsDict != null) literalsDict.transferTo(dictionary); + if (graphsList.getNumEntries() > 0) { + graphsList.transferTo(dictionary, "graphs"); + subjectsList.transferTo(dictionary, "subjects"); + objectsList.transferTo(dictionary, "objects"); + } + gspo.transferTo(hdt); + gpos.transferTo(hdt); + } + logger.info("HDF5 assembly complete"); + } + + // ------------------------------------------------------------------ + // Pass A: parse + transforms + spills + // ------------------------------------------------------------------ + + private void ingest() throws IOException { + logger.info("Parsing {} (disk-based build)", src); + this.pTempFile = new RecordFile<>(workDir.resolve("pcol.tmpids"), HugeRecords.VAR_LONG_CODEC); + try (InputStream xis = src.toString().endsWith(".gz") + ? new GZIPInputStream(new FileInputStream(src)) + : new FileInputStream(src)) { + String fname = src.getName(); + if (fname.endsWith(".gz")) { + fname = fname.substring(0, fname.length() - 3); + } + Lang lang = RDFLanguages.filenameToLang(fname, Lang.TURTLE); + AsyncParserBuilder parserBuilder = AsyncParser.of(xis, lang, REL_BASE); + parserBuilder.mutateSources(rdfBuilder -> + rdfBuilder.labelToNode(LabelToNode.createUseLabelAsGiven())); + SpatialAugmenter augmenter = new SpatialAugmenter(features); + // Spatial tasks run concurrently exactly like the RAM writer, but the + // completion window is BOUNDED: results are drained and spilled as + // soon as the window fills instead of accumulating until end of parse. + final int maxInFlight = Math.max(8, Runtime.getRuntime().availableProcessors() * 4); + final ArrayDeque>> inFlight = new ArrayDeque<>(); + try (ExecutorService scope = Executors.newVirtualThreadPerTaskExecutor()) { + parserBuilder.streamQuads() + .map(quad -> quad.isDefaultGraph() + ? new Quad(Quad.defaultGraphIRI, quad.getSubject(), quad.getPredicate(), quad.getObject()) + : quad) + .map(this::relativize) + .map(this::canonicalizeNumericObject) + .forEach(quad -> { + parsedQuads++; + if (parsedQuads % 1_000_000 == 0) { + logger.info("Loaded {} quads...", parsedQuads); + } + try { + acceptRow(quad); + xvoid.add(quad); + if (spatial && isGeoLiteral(quad)) { + inFlight.add(scope.submit(() -> augmenter.addSpatial(quad))); + while (inFlight.size() >= maxInFlight) { + drainOne(inFlight); + } + } + } catch (IOException ex) { + throw new UncheckedIOException(ex); + } + }); + while (!inFlight.isEmpty()) { + drainOne(inFlight); + } + } catch (UncheckedIOException ex) { + throw new IOException("Failed while parsing/processing RDF source: " + src, ex.getCause()); + } catch (Exception ex) { + throw new IOException("Failed while parsing/processing RDF source: " + src, ex); + } + // VoID/SD metadata quads (mirrors the RAM writer; the model's + // namespace prefixes are presentation-only and irrelevant to quads). + for (Iterator it = xvoid.getModel().listStatements(); it.hasNext(); ) { + Triple ff = it.next().asTriple(); + Quad qqq = canonicalizeNumericObject(Quad.create(Params.BGVOID, ff)); + acceptRow(qqq); + } + } + pTempFile.finish(); + logger.info("Parse complete: {} source quads, {} total rows", parsedQuads, rows); + } + + private void drainOne(ArrayDeque>> inFlight) throws IOException { + Future> task = inFlight.poll(); + if (task == null) return; + ArrayList extraQuads; + try { + extraQuads = task.get(); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted while collecting spatial results: " + src, ex); + } catch (ExecutionException ex) { + throw new IOException("Spatial processing failed for " + src, ex.getCause()); + } + for (Quad q : extraQuads) { + acceptRow(canonicalizeNumericObject(q)); + } + } + + /** One row of the store: spill each column and account stats (mirrors ProcessQuad). */ + private void acceptRow(Quad quad) throws IOException { + Node g = quad.getGraph(); + Node s = quad.getSubject(); + Node p = quad.getPredicate(); + Node o = quad.getObject(); + countEntityKind(g, "graph"); + countEntityKind(s, "subject"); + if (o.isLiteral()) { + collectLiteralStats(o); + } else { + countEntityKind(o, "object"); + } + stats.numIRI++; // the predicate (RAM counts distinct; only >0 gates buffer allocation) + + long row = rows++; + gSorter.add(new TermRow(g, row)); + sSorter.add(new TermRow(s, row)); + oSorter.add(new TermRow(o, row)); + pTempFile.append(predTempIds.computeIfAbsent(p, k -> { + predByTempId.add(k); + return (long) (predByTempId.size() - 1); + })); + } + + private void countEntityKind(Node n, String position) { + if (n.isBlank()) { + stats.numBlankNodes++; + } else if (n.isURI()) { + stats.numIRI++; + } else { + // Same guard as ProcessQuad: an unstorable node kind aborts the build. + throw new IllegalStateException("Unexpected " + position + " node type: " + n); + } + } + + /** + * Literal stats accounting, mirroring ProcessQuad's branches. RAM counts + * distinct literals; per-occurrence counting yields identical min/max and + * identical zero-ness of every count, which is all the dictionary widths + * and buffer-allocation gates consume. + */ + private void collectLiteralStats(Node o) { + String dt = o.getLiteralDatatypeURI(); + dataTypes.add(dt); + String lang = o.getLiteralLanguage(); + if (lang != null && !lang.isEmpty()) { + langSet.add(lang); + } + if (dt.equals(XSD.xlong.getURI())) { + if (literalValueOrNull(o) instanceof Number n) { + stats.maxLong = Math.max(stats.maxLong, n.longValue()); + stats.minLong = Math.min(stats.minLong, n.longValue()); + stats.numLong++; + } else { + countStringStored(o.getLiteralLexicalForm()); + } + } else if (dt.equals(XSD.xint.getURI())) { + if (literalValueOrNull(o) instanceof Number n) { + stats.maxInteger = Math.max(stats.maxInteger, n.intValue()); + stats.minInteger = Math.min(stats.minInteger, n.intValue()); + stats.numInteger++; + } else { + countStringStored(o.getLiteralLexicalForm()); + } + } else if (dt.equals(XSD.xfloat.getURI())) { + if (literalValueOrNull(o) instanceof Number n) { + stats.maxFloat = Math.max(stats.maxFloat, n.floatValue()); + stats.minFloat = Math.min(stats.minFloat, n.floatValue()); + stats.numFloat++; + } else { + countStringStored(o.getLiteralLexicalForm()); + } + } else if (dt.equals(XSD.xdouble.getURI())) { + if (literalValueOrNull(o) instanceof Number n) { + stats.maxDouble = Math.max(stats.maxDouble, n.doubleValue()); + stats.minDouble = Math.min(stats.minDouble, n.doubleValue()); + stats.numDouble++; + } else { + countStringStored(o.getLiteralLexicalForm()); + } + } else if (dt.equals(XSD.xstring.getURI()) || dt.equals(GEO.wktLiteral.getURI()) + || dt.equals(XSD.xboolean.getURI()) || dt.equals(RDF.langString.getURI())) { + countStringStored(o.getLiteralLexicalForm()); + } else if (dt.equals(XSD.dateTime.getURI())) { + String lex = o.getLiteralLexicalForm(); + int t = lex.indexOf('T'); + countStringStored((t > 0) ? lex.substring(0, t) : lex); + } else { + countStringStored(o.getLiteralLexicalForm()); + } + } + + private void countStringStored(String lex) { + stats.longestStringLength = Math.max(stats.longestStringLength, lex.length()); + stats.shortestStringLength = Math.min(stats.shortestStringLength, lex.length()); + stats.numStrings++; + } + + private static Object literalValueOrNull(Node o) { + try { + return o.getLiteralValue(); + } catch (RuntimeException e) { + return null; + } + } + + private static boolean isGeoLiteral(Quad quad) { + return SpatialAugmenter.isGeoLiteral(quad); + } + + // ------------------------------------------------------------------ + // Quad transforms (mirrors of PositionalDictionaryWriterBuilder) + // ------------------------------------------------------------------ + + private Quad relativize(Quad q) { + Node qg = q.getGraph(); + Node qs = q.getSubject(); + Node qp = q.getPredicate(); + Node qo = q.getObject(); + Node g = relativizeNode(qg); + Node s = relativizeNode(qs); + Node p = relativizeNode(qp); + Node o = relativizeNode(qo); + if (g == qg && s == qs && p == qp && o == qo) { + return q; + } + return new Quad(g, s, p, o); + } + + private Node relativizeNode(Node n) { + if (n == null || !n.isURI() || !n.getURI().startsWith(REL_BASE_PREFIX)) { + return n; + } + String u = n.getURI(); + try { + IRIx rel = REL_BASE_IRIX.relativize(IRIx.create(u)); + if (rel != null && rel.isRelative()) { + return NodeFactory.createURI(rel.str()); + } + } catch (RuntimeException ignore) { + // fall through to textual stripping + } + return NodeFactory.createURI( + u.equals(REL_BASE) ? "" : u.substring(REL_BASE_PREFIX.length())); + } + + private Quad canonicalizeNumericObject(Quad quad) { + Node o = quad.getObject(); + if (!o.isLiteral()) return quad; + String dt = o.getLiteralDatatypeURI(); + try { + Node canonical = null; + if (XSD.xint.getURI().equals(dt)) { + if (o.getLiteralValue() instanceof Number n) canonical = NodeFactory.createLiteralByValue(n.intValue()); + } else if (XSD.xlong.getURI().equals(dt)) { + if (o.getLiteralValue() instanceof Number n) canonical = NodeFactory.createLiteralByValue(n.longValue()); + } else if (XSD.xfloat.getURI().equals(dt)) { + if (o.getLiteralValue() instanceof Number n) canonical = NodeFactory.createLiteralByValue(n.floatValue()); + } else if (XSD.xdouble.getURI().equals(dt)) { + if (o.getLiteralValue() instanceof Number n) canonical = NodeFactory.createLiteralByValue(n.doubleValue()); + } + if (canonical == null || canonical.equals(o)) return quad; + return new Quad(quad.getGraph(), quad.getSubject(), quad.getPredicate(), canonical); + } catch (RuntimeException e) { + return quad; + } + } + + // ------------------------------------------------------------------ + // Phase helpers + // ------------------------------------------------------------------ + + private RecordFile materialize(ExternalSorter sorter, String name) throws IOException { + RecordFile rf = new RecordFile<>(workDir.resolve(name), HugeRecords.TERM_ROW_CODEC); + try (ExternalSorter.SortedStream s = sorter.sorted()) { + while (s.hasNext()) { + rf.append(s.next()); + } + } + rf.finish(); + sorter.close(); + return rf; + } + + /** + * Entities = distinct {G union S union non-literal O} in NodeComparator + * order: a 3-way merge of the sorted columns with consecutive-dedup. The + * object stream stops at its first literal - NodeComparator's macro order + * (bnode < URI < literal) makes literals a contiguous suffix. + */ + private void mergeDistinctEntities(RecordFile out, RecordFile g, + RecordFile s, RecordFile o) throws IOException { + try (var gs = g.read(); var ss = s.read(); var os = o.read()) { + List> streams = List.of( + termsOf(gs), termsOf(ss), nonLiteralPrefix(termsOf(os))); + PriorityQueue heap = new PriorityQueue<>( + Comparator.comparing(pi -> pi.head, NodeComparator.INSTANCE)); + for (Iterator it : streams) { + PeekedIterator pi = new PeekedIterator(it); + if (pi.head != null) heap.add(pi); + } + Node last = null; + while (!heap.isEmpty()) { + PeekedIterator pi = heap.poll(); + Node term = pi.head; + if (pi.advance()) heap.add(pi); + // equals() is the cheap equivalent of compare()==0 here: the + // comparator tie-breaks on the exact RDF term, so it returns 0 + // only for term-equal nodes. + if (last == null || !last.equals(term)) { + out.append(term); + last = term; + } + } + } + out.finish(); + } + + /** Distinct literals = dedup of the literal suffix of the sorted object column. */ + private void distinctLiterals(RecordFile out, RecordFile o) throws IOException { + try (var os = o.read()) { + Node last = null; + while (os.hasNext()) { + Node term = os.next().term(); + if (!term.isLiteral()) continue; // pre-literal prefix + if (last == null || !last.equals(term)) { + out.append(term); + last = term; + } + } + } + out.finish(); + } + + private static Iterator termsOf(Iterator rows) { + return new Iterator<>() { + @Override public boolean hasNext() { return rows.hasNext(); } + @Override public Node next() { return rows.next().term(); } + }; + } + + private static Iterator nonLiteralPrefix(Iterator in) { + return new Iterator<>() { + private Node next = advance(); + + private Node advance() { + if (in.hasNext()) { + Node n = in.next(); + // Sorted stream: the first literal ends the entity prefix. + return n.isLiteral() ? null : n; + } + return null; + } + + @Override public boolean hasNext() { return next != null; } + + @Override public Node next() { + if (next == null) throw new NoSuchElementException(); + Node n = next; + next = advance(); + return n; + } + }; + } + + private static final class PeekedIterator { + private final Iterator it; + Node head; + + PeekedIterator(Iterator it) { + this.it = it; + advance(); + } + + /** Loads the next head; false at stream end. */ + final boolean advance() { + head = it.hasNext() ? it.next() : null; + return head != null; + } + } + + /** + * A monotone cursor over the sorted dictionary (entities, optionally + * followed by literals): the id of a term is its 1-based rank in the + * concatenated stream - exactly locateGraph/locateSubject/locateObject + * semantics, computed without random access. + */ + private DictCursor entityCursor(RecordFile entities, RecordFile literals, long numEntities) + throws IOException { + var entStream = entities.read(); + var litStream = (literals == null) ? null : literals.read(); + Iterator concat = new Iterator<>() { + @Override + public boolean hasNext() { + return entStream.hasNext() || (litStream != null && litStream.hasNext()); + } + + @Override + public Node next() { + return entStream.hasNext() ? entStream.next() : litStream.next(); + } + }; + return new DictCursor(concat, () -> { + entStream.close(); + if (litStream != null) litStream.close(); + }); + } + + private static final class DictCursor implements AutoCloseable { + private final Iterator stream; + private final Runnable onClose; + private Node current = null; + private long id = 0; + + DictCursor(Iterator stream, Runnable onClose) { + this.stream = stream; + this.onClose = onClose; + } + + /** 1-based id of {@code term}; the cursor only moves forward. */ + long locate(Node term, String role) { + // Cheap term-equality fast path before any comparator work. + if (current != null && current.equals(term)) { + return id; + } + while (current == null || NodeComparator.INSTANCE.compare(current, term) < 0) { + if (!stream.hasNext()) { + throw new IllegalStateException("Cannot resolve " + role + " (not in dictionary): " + term); + } + current = stream.next(); + id++; + if (current.equals(term)) { + return id; + } + } + if (!current.equals(term)) { + throw new IllegalStateException("Cannot resolve " + role + " (not in dictionary): " + term); + } + return id; + } + + @Override + public void close() { + onClose.run(); + } + } + + /** + * Walks a sorted column against the dictionary cursor, emitting (row, id) + * for every row and the id of each DISTINCT term - in term order, i.e. + * ascending id order - into the columnar list. + */ + private void joinColumn(RecordFile column, String role, DictCursor cursor, + SpillBitPackedBuffer columnar, ExternalSorter out) throws IOException { + try (cursor; var col = column.read()) { + Node prevTerm = null; + long prevId = -1; + while (col.hasNext()) { + TermRow tr = col.next(); + long id; + // equals() suffices for the repeat check (compare()==0 iff term-equal). + if (prevTerm != null && prevTerm.equals(tr.term())) { + id = prevId; + } else { + id = cursor.locate(tr.term(), role); + columnar.writeLong(id); + prevTerm = tr.term(); + prevId = id; + } + out.add(new RowId(tr.row(), id)); + } + } + } + + @Override + public void close() { + if (pTempFile != null) pTempFile.delete(); + for (AutoCloseable r : resources) { + try { + r.close(); + } catch (Exception e) { + logger.warn("Cleanup failure", e); + } + } + resources.clear(); + } +} diff --git a/src/main/java/com/ebremer/beakgraph/huge/HugeHDF5Writer.java b/src/main/java/com/ebremer/beakgraph/huge/HugeHDF5Writer.java new file mode 100644 index 00000000..30201611 --- /dev/null +++ b/src/main/java/com/ebremer/beakgraph/huge/HugeHDF5Writer.java @@ -0,0 +1,165 @@ +package com.ebremer.beakgraph.huge; + +import com.ebremer.beakgraph.Params; +import com.ebremer.beakgraph.core.AbstractGraphBuilder; +import com.ebremer.beakgraph.core.BeakGraphWriter; +import java.io.IOException; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.StandardCopyOption; +import java.nio.file.attribute.BasicFileAttributes; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Disk-based BeakGraph writer for very large graphs. Produces the same HDF5 + * format as {@link com.ebremer.beakgraph.hdf5.writers.HDF5Writer} (readable by + * the existing {@code HDF5Reader}), but sorts and indexes on disk: quads are + * spilled to a temp workspace, dictionaries are derived by external merge sort, + * ids are assigned by sort-merge joins, and every dataset streams into the file + * through the {@link StreamingHdf5} backend. Peak heap stays bounded by the + * configured spill batch sizes; the size ceiling moves from RAM to free disk in + * the workspace (roughly a few times the source size, transiently). + * + *

    Usage mirrors HDF5Writer: + * 

    {@code
+ * HugeHDF5Writer.Builder()
+ *     .setSource(new File("data.nq.gz"))
+ *     .setDestination(new File("data.h5"))
+ *     .setSpatial(true)
+ *     .build()
+ *     .write();
+ * }
    + * + *

    Output is semantically identical to the RAM writer's (same dictionaries, + * indexes, VoID metadata, spatial augmentation); the only divergence is blank + * node ordering (labels are kept as parsed rather than relabelled), which + * yields an isomorphic - not byte-identical - store. + * + * @author Erich Bremer + */ +public class HugeHDF5Writer implements BeakGraphWriter { + + private static final Logger logger = LoggerFactory.getLogger(HugeHDF5Writer.class); + + private final Builder builder; + + private HugeHDF5Writer(Builder builder) { + this.builder = builder; + } + + @Override + public void write() throws IOException { + logger.info("Writing BeakGraph (huge/disk-based) to {}", builder.getDestination()); + Path dest = builder.getDestination().toPath(); + // Same publish discipline as HDF5Writer: build into a sibling temp file, + // swap in atomically on success, never disturb a previous good artifact. + Path tmp = dest.resolveSibling(dest.getFileName() + ".tmp"); + Path workBase = (builder.workDir != null) ? builder.workDir + : (dest.toAbsolutePath().getParent() != null ? dest.toAbsolutePath().getParent() : Path.of(".")); + Path workspace = Files.createTempDirectory(workBase, ".bghuge-"); + try { + try (HugeBuildPipeline pipeline = new HugeBuildPipeline( + builder.getSource(), builder.getSpatial(), builder.getFeatures(), + workspace, builder.termSpillBatch, builder.idSpillBatch, builder.mergeFanIn)) { + pipeline.run(tmp); + } + try { + Files.move(tmp, dest, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); + } catch (AtomicMoveNotSupportedException e) { + Files.move(tmp, dest, StandardCopyOption.REPLACE_EXISTING); + } + } catch (IOException | RuntimeException ex) { + try { + Files.deleteIfExists(tmp); + } catch (IOException cleanup) { + logger.warn("Failed to remove temp output {}", tmp, cleanup); + } + throw ex; + } finally { + deleteRecursively(workspace); + } + logger.info("Write complete: {}", builder.getDestination()); + } + + private static void deleteRecursively(Path dir) { + try { + Files.walkFileTree(dir, new SimpleFileVisitor<>() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Files.deleteIfExists(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path d, IOException exc) throws IOException { + Files.deleteIfExists(d); + return FileVisitResult.CONTINUE; + } + }); + } catch (IOException e) { + logger.warn("Failed to remove workspace {}", dir, e); + } + } + + public static class Builder extends AbstractGraphBuilder { + + private Path workDir; + private int termSpillBatch = 1 << 18; // 262144 (term, row) records per column before a run spills + private int idSpillBatch = 1 << 21; // 2M numeric records per run for the id/quad sorts + private int mergeFanIn = 64; + + /** + * Directory for the build workspace (spill runs, staged buffers). + * Defaults to the destination's directory - the workspace transiently + * needs disk on the order of a few times the (uncompressed) source. + */ + public Builder setWorkDirectory(Path dir) { + this.workDir = dir; + return this; + } + + /** Records buffered in RAM per term column before spilling a sorted run. */ + public Builder setTermSpillBatch(int records) { + if (records < 1) throw new IllegalArgumentException("termSpillBatch must be >= 1"); + this.termSpillBatch = records; + return this; + } + + /** Records buffered in RAM per numeric (id/quad) sorter before spilling. */ + public Builder setIdSpillBatch(int records) { + if (records < 1) throw new IllegalArgumentException("idSpillBatch must be >= 1"); + this.idSpillBatch = records; + return this; + } + + /** Maximum spill runs merged in one pass. */ + public Builder setMergeFanIn(int fanIn) { + if (fanIn < 2) throw new IllegalArgumentException("mergeFanIn must be >= 2"); + this.mergeFanIn = fanIn; + return this; + } + + @Override + protected Builder self() { + return this; + } + + @Override + public String getName() { + return Params.BG; + } + + @Override + public HugeHDF5Writer build() { + return new HugeHDF5Writer(this); + } + } + + public static Builder Builder() { + return new Builder(); + } +} diff --git a/src/main/java/com/ebremer/beakgraph/huge/HugeIO.java b/src/main/java/com/ebremer/beakgraph/huge/HugeIO.java new file mode 100644 index 00000000..a62e20f8 --- /dev/null +++ b/src/main/java/com/ebremer/beakgraph/huge/HugeIO.java @@ -0,0 +1,73 @@ +package com.ebremer.beakgraph.huge; + +import java.io.BufferedInputStream; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * Small shared I/O helpers for the huge writer: temp-file-to-dataset streaming + * and unsigned varint coding for the spill record formats. + * + * @author Erich Bremer + */ +final class HugeIO { + + /** Copy buffer for temp-file-to-HDF5 streaming; bounds writer RAM per transfer. */ + static final int TRANSFER_BUFFER_BYTES = 8 * 1024 * 1024; + + private HugeIO() {} + + /** Streams an entire temp file into an already-created dataset. */ + static void copyFileIntoDataset(Path file, StreamingHdf5Dataset ds) throws IOException { + try (InputStream in = new BufferedInputStream(Files.newInputStream(file), 1 << 20)) { + byte[] buf = new byte[TRANSFER_BUFFER_BYTES]; + int n; + while ((n = in.read(buf)) > 0) { + ds.write(buf, 0, n); + } + } + } + + /** Unsigned LEB128-style varint (7 bits per byte, high bit = continuation). */ + static void writeVarLong(DataOutput out, long value) throws IOException { + if (value < 0) { + throw new IllegalArgumentException("varint value must be non-negative: " + value); + } + while ((value & ~0x7FL) != 0) { + out.writeByte((int) ((value & 0x7F) | 0x80)); + value >>>= 7; + } + out.writeByte((int) value); + } + + static long readVarLong(DataInput in) throws IOException { + long result = 0; + int shift = 0; + while (true) { + byte b = in.readByte(); + result |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + return result; + } + shift += 7; + if (shift >= 64) { + throw new IOException("Malformed varint (too long)"); + } + } + } + + /** + * Reads exactly {@code len} bytes or throws {@link EOFException}; DataInput's + * readFully equivalent, factored for symmetry with the writers here. + */ + static byte[] readBytes(DataInput in, int len) throws IOException { + byte[] data = new byte[len]; + in.readFully(data); + return data; + } +} diff --git a/src/main/java/com/ebremer/beakgraph/huge/HugeIndexWriter.java b/src/main/java/com/ebremer/beakgraph/huge/HugeIndexWriter.java new file mode 100644 index 00000000..00b3242d --- /dev/null +++ b/src/main/java/com/ebremer/beakgraph/huge/HugeIndexWriter.java @@ -0,0 +1,264 @@ +package com.ebremer.beakgraph.huge; + +import static com.ebremer.beakgraph.Params.BLOCKSIZE; +import static com.ebremer.beakgraph.Params.SUPERBLOCKSIZE; +import com.ebremer.beakgraph.hdf5.Index; +import com.ebremer.beakgraph.huge.HugeRecords.IdQuad; +import static com.ebremer.beakgraph.utils.UTIL.MinBits; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Iterator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Streaming twin of {@link com.ebremer.beakgraph.hdf5.writers.BGIndex}: builds + * one quad index (GSPO or GPOS) - S/B id+bitmap levels plus the SB/BB + * rank/select directory - from an externally sorted stream of + * dictionary-encoded quads. Because dictionary ids are rank-assigned in + * {@code NodeComparator} order, numeric id order equals the term order BGIndex + * sorts with, and this writer's output is identical to BGIndex's for the same + * data. The level/padding/dedup logic mirrors BGIndex line for line; keep the + * two in lockstep. + * + * @author Erich Bremer + */ +final class HugeIndexWriter implements AutoCloseable { + + private static final Logger logger = LoggerFactory.getLogger(HugeIndexWriter.class); + + private final SpillBitPackedBuffer B1, B2, B3; + private final SpillBitPackedBuffer S1, S2, S3; + private final SpillBitPackedBuffer SB1, SB2, SB3; + private final SpillBitPackedBuffer BB1, BB2, BB3; + private final Index type; + private final char[] positions; + private final String[] names; + private final long numGraphs, numSubjects, numPredicates, numObjects; + + private static final class LevelState { + long bitsProcessed = 0; + long onesSoFar = 0; + long onesInCurrentSuperblock = 0; + // Same seeding as BGIndex: start at 0, paired with the seeded BB[0]=0, + // so BB[k] = ones-before-block-k within its superblock. + long lastSuperblockWritten = 0; + long lastBlockWritten = 0; + } + + /** + * @param totalRows total encoded quads INCLUDING duplicates - BGIndex sizes + * the superblock width from the full pre-dedup quad array + */ + HugeIndexWriter(Path workDir, Index type, long numGraphs, long numSubjects, + long numPredicates, long numObjects, long totalRows) throws IOException { + logger.info("Creating index {} (disk-backed)", type); + this.type = type; + this.numGraphs = numGraphs; + this.numSubjects = numSubjects; + this.numPredicates = numPredicates; + this.numObjects = numObjects; + String indexName = type.name(); + this.positions = new char[]{indexName.charAt(0), indexName.charAt(1), indexName.charAt(2), indexName.charAt(3)}; + this.names = new String[4]; + for (int i = 0; i < 4; i++) { + names[i] = String.valueOf(Character.toLowerCase(positions[i])); + } + + Path dir = Files.createDirectories(workDir.resolve("index." + indexName)); + + long maxCumulativeOnes = totalRows + maxL0Id() + 128L; + int sbBits = MinBits(maxCumulativeOnes); + sbBits = (int) (Math.ceil(sbBits / 8.0) * 8); + int bbBits = MinBits(SUPERBLOCKSIZE); + bbBits = (int) (Math.ceil(bbBits / 8.0) * 8); + + B1 = new SpillBitPackedBuffer(dir.resolve("B" + names[1]), 1); + B2 = new SpillBitPackedBuffer(dir.resolve("B" + names[2]), 1); + B3 = new SpillBitPackedBuffer(dir.resolve("B" + names[3]), 1); + + S1 = new SpillBitPackedBuffer(dir.resolve("S" + names[1]), bitSize(positions[1])); + S2 = new SpillBitPackedBuffer(dir.resolve("S" + names[2]), bitSize(positions[2])); + S3 = new SpillBitPackedBuffer(dir.resolve("S" + names[3]), bitSize(positions[3])); + + SB1 = new SpillBitPackedBuffer(dir.resolve("SB" + names[1]), sbBits); + SB2 = new SpillBitPackedBuffer(dir.resolve("SB" + names[2]), sbBits); + SB3 = new SpillBitPackedBuffer(dir.resolve("SB" + names[3]), sbBits); + + BB1 = new SpillBitPackedBuffer(dir.resolve("BB" + names[1]), bbBits); + BB2 = new SpillBitPackedBuffer(dir.resolve("BB" + names[2]), bbBits); + BB3 = new SpillBitPackedBuffer(dir.resolve("BB" + names[3]), bbBits); + + // Seed the "ones before the first superblock/block" directory entries to 0. + SB1.writeLong(0); SB2.writeLong(0); SB3.writeLong(0); + BB1.writeLong(0); BB2.writeLong(0); BB3.writeLong(0); + } + + private long count(char component) { + return switch (component) { + case 'G' -> numGraphs; + case 'S' -> numSubjects; + case 'P' -> numPredicates; + case 'O' -> numObjects; + default -> throw new IllegalStateException("Unknown component: " + component); + }; + } + + /** Mirror of BGIndex.IndexPosition.getBitSize: byte-rounded MinBits(count+1), 0 -> 8. */ + private int bitSize(char component) { + int needed = MinBits(count(component) + 1); + if (needed == 0) return 8; + return (int) (Math.ceil(needed / 8.0) * 8); + } + + private long maxL0Id() { + return count(positions[0]); + } + + private static long component(IdQuad q, char c) { + return switch (c) { + case 'G' -> q.g(); + case 'S' -> q.s(); + case 'P' -> q.p(); + case 'O' -> q.o(); + default -> throw new IllegalStateException(); + }; + } + + /** Consumes the id-quads, which MUST be sorted in this index's order. */ + void build(Iterator sorted) throws IOException { + LevelState l1 = new LevelState(), l2 = new LevelState(), l3 = new LevelState(); + IdQuad lastUnique = null; + long count = 0; + long maxL0Id = maxL0Id(); + long currentL0 = 1; + + while (sorted.hasNext()) { + IdQuad curr = sorted.next(); + if (++count % 1_000_000 == 0) { + logger.info("{} processed {} quads...", type.name(), count); + } + // 1. Duplicate check (numeric id equality == term equality) + if (lastUnique != null + && component(lastUnique, positions[0]) == component(curr, positions[0]) + && component(lastUnique, positions[1]) == component(curr, positions[1]) + && component(lastUnique, positions[2]) == component(curr, positions[2]) + && component(lastUnique, positions[3]) == component(curr, positions[3])) { + continue; + } + + boolean changeL0 = (lastUnique == null) || component(lastUnique, positions[0]) != component(curr, positions[0]); + boolean changeL1 = (lastUnique == null) || changeL0 || component(lastUnique, positions[1]) != component(curr, positions[1]); + boolean changeL2 = (lastUnique == null) || changeL1 || component(lastUnique, positions[2]) != component(curr, positions[2]); + + long thisL0 = component(curr, positions[0]); + + // 2. Pad missing L0 ids with empty lists (see BGIndex) + if (changeL0) { + long skipped = (lastUnique == null) ? (thisL0 - 1) : (thisL0 - currentL0 - 1); + padEmptyL0(skipped, l1, l2, l3); + currentL0 = thisL0; + } + + // 3. Level 3 + S3.writeLong(component(curr, positions[3])); + int bit3 = changeL2 ? 1 : 0; + B3.writeInteger(bit3); + advanceLevel(l3, bit3, SB3, BB3); + + // 4. Level 2 + if (changeL2) { + S2.writeLong(component(curr, positions[2])); + int bit2 = changeL1 ? 1 : 0; + B2.writeInteger(bit2); + advanceLevel(l2, bit2, SB2, BB2); + } + + // 5. Level 1 + if (changeL1) { + S1.writeLong(component(curr, positions[1])); + int bit1 = changeL0 ? 1 : 0; + B1.writeInteger(bit1); + advanceLevel(l1, bit1, SB1, BB1); + } + + lastUnique = curr; + } + + // Pad remaining ids up to the dictionary's maximum + long skipped = maxL0Id - currentL0; + padEmptyL0(skipped, l1, l2, l3); + + completeAll(); + } + + private void padEmptyL0(long count, LevelState l1, LevelState l2, LevelState l3) { + for (long k = 0; k < count; k++) { + S1.writeLong(0); + B1.writeInteger(1); + advanceLevel(l1, 1, SB1, BB1); + + S2.writeLong(0); + B2.writeInteger(1); + advanceLevel(l2, 1, SB2, BB2); + + S3.writeLong(0); + B3.writeInteger(1); + advanceLevel(l3, 1, SB3, BB3); + } + } + + private void advanceLevel(LevelState state, int bitValue, SpillBitPackedBuffer SB, SpillBitPackedBuffer BB) { + if (bitValue == 1) { + state.onesSoFar++; + state.onesInCurrentSuperblock++; + } + state.bitsProcessed++; + + long currentSuperblock = state.bitsProcessed / SUPERBLOCKSIZE; + if (currentSuperblock != state.lastSuperblockWritten) { + SB.writeLong(state.onesSoFar); + state.lastSuperblockWritten = currentSuperblock; + state.onesInCurrentSuperblock = 0; + } + + long currentBlock = state.bitsProcessed / BLOCKSIZE; + if (currentBlock != state.lastBlockWritten) { + BB.writeLong(state.onesInCurrentSuperblock); + state.lastBlockWritten = currentBlock; + } + } + + private void completeAll() throws IOException { + B1.complete(); B2.complete(); B3.complete(); + S1.complete(); S2.complete(); S3.complete(); + SB1.complete(); SB2.complete(); SB3.complete(); + BB1.complete(); BB2.complete(); BB3.complete(); + } + + /** Writes the index group, mirroring BGIndex.add() (names and order). */ + void transferTo(StreamingHdf5Group hdt) throws IOException { + StreamingHdf5Group index = hdt.putGroup(type.name()); + S1.transferTo(index, "S" + names[1]); + S2.transferTo(index, "S" + names[2]); + S3.transferTo(index, "S" + names[3]); + B1.transferTo(index, "B" + names[1]); + B2.transferTo(index, "B" + names[2]); + B3.transferTo(index, "B" + names[3]); + SB1.transferTo(index, "SB" + names[1]); + SB2.transferTo(index, "SB" + names[2]); + SB3.transferTo(index, "SB" + names[3]); + BB1.transferTo(index, "BB" + names[1]); + BB2.transferTo(index, "BB" + names[2]); + BB3.transferTo(index, "BB" + names[3]); + } + + @Override + public void close() throws IOException { + B1.close(); B2.close(); B3.close(); + S1.close(); S2.close(); S3.close(); + SB1.close(); SB2.close(); SB3.close(); + BB1.close(); BB2.close(); BB3.close(); + } +} diff --git a/src/main/java/com/ebremer/beakgraph/huge/HugeRecords.java b/src/main/java/com/ebremer/beakgraph/huge/HugeRecords.java new file mode 100644 index 00000000..8dc09473 --- /dev/null +++ b/src/main/java/com/ebremer/beakgraph/huge/HugeRecords.java @@ -0,0 +1,114 @@ +package com.ebremer.beakgraph.huge; + +import com.ebremer.beakgraph.core.lib.NodeComparator; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.Comparator; +import org.apache.jena.graph.Node; + +/** + * Record types flowing through the huge writer's spill files, with their + * codecs and orderings. + * + *

    {@code TermRow} carries one quad component (a term) plus the row number of + * its quad; sorted by term it drives both dictionary construction (dedup) and + * the sort-merge id join. {@code RowId} is the join output (row -> id), sorted + * back into row order. {@code IdQuad} is a fully dictionary-encoded quad; its + * two orderings are exactly the {@link com.ebremer.beakgraph.hdf5.Index} + * comparators, because dictionary ids are rank-assigned from the same + * {@link NodeComparator} order the RAM writer sorts quads with - numeric id + * order and term order agree component-wise. + * + * @author Erich Bremer + */ +final class HugeRecords { + + private HugeRecords() {} + + record TermRow(Node term, long row) {} + + record RowId(long row, long id) {} + + record IdQuad(long g, long s, long p, long o) {} + + /** Term order only; equal terms may interleave rows arbitrarily (the join maps them to one id). */ + static final Comparator TERM_ORDER = + (a, b) -> NodeComparator.INSTANCE.compare(a.term(), b.term()); + + static final Comparator ROW_ORDER = Comparator.comparingLong(RowId::row); + + static final Comparator GSPO_ORDER = Comparator + .comparingLong(IdQuad::g) + .thenComparingLong(IdQuad::s) + .thenComparingLong(IdQuad::p) + .thenComparingLong(IdQuad::o); + + static final Comparator GPOS_ORDER = Comparator + .comparingLong(IdQuad::g) + .thenComparingLong(IdQuad::p) + .thenComparingLong(IdQuad::o) + .thenComparingLong(IdQuad::s); + + static final ExternalSorter.Codec TERM_ROW_CODEC = new ExternalSorter.Codec<>() { + @Override + public void write(DataOutput out, TermRow record) throws IOException { + NodeCodec.writeNode(out, record.term()); + HugeIO.writeVarLong(out, record.row()); + } + + @Override + public TermRow read(DataInput in) throws IOException { + Node term = NodeCodec.readNode(in); + long row = HugeIO.readVarLong(in); + return new TermRow(term, row); + } + }; + + static final ExternalSorter.Codec ROW_ID_CODEC = new ExternalSorter.Codec<>() { + @Override + public void write(DataOutput out, RowId record) throws IOException { + HugeIO.writeVarLong(out, record.row()); + HugeIO.writeVarLong(out, record.id()); + } + + @Override + public RowId read(DataInput in) throws IOException { + long row = HugeIO.readVarLong(in); + long id = HugeIO.readVarLong(in); + return new RowId(row, id); + } + }; + + /** Bare non-negative longs (the per-row predicate temp/final id files). */ + static final ExternalSorter.Codec VAR_LONG_CODEC = new ExternalSorter.Codec<>() { + @Override + public void write(DataOutput out, Long record) throws IOException { + HugeIO.writeVarLong(out, record); + } + + @Override + public Long read(DataInput in) throws IOException { + return HugeIO.readVarLong(in); + } + }; + + static final ExternalSorter.Codec ID_QUAD_CODEC = new ExternalSorter.Codec<>() { + @Override + public void write(DataOutput out, IdQuad record) throws IOException { + HugeIO.writeVarLong(out, record.g()); + HugeIO.writeVarLong(out, record.s()); + HugeIO.writeVarLong(out, record.p()); + HugeIO.writeVarLong(out, record.o()); + } + + @Override + public IdQuad read(DataInput in) throws IOException { + long g = HugeIO.readVarLong(in); + long s = HugeIO.readVarLong(in); + long p = HugeIO.readVarLong(in); + long o = HugeIO.readVarLong(in); + return new IdQuad(g, s, p, o); + } + }; +} diff --git a/src/main/java/com/ebremer/beakgraph/huge/NativeHdf5File.java b/src/main/java/com/ebremer/beakgraph/huge/NativeHdf5File.java new file mode 100644 index 00000000..74360caa --- /dev/null +++ b/src/main/java/com/ebremer/beakgraph/huge/NativeHdf5File.java @@ -0,0 +1,282 @@ +package com.ebremer.beakgraph.huge; + +import hdf.hdf5lib.H5; +import hdf.hdf5lib.HDF5Constants; +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayDeque; +import java.util.Arrays; +import java.util.Deque; + +/** + * {@link StreamingHdf5File} backed by the HDF Group's native HDF5 library + * (https://github.com/HDFGroup/hdf5) through the official {@code hdf.hdf5lib} + * Java API. This is the only backend that can write a dataset piecewise today; + * jHDF's writer needs the whole dataset in memory. + * + *

    This class is written against the official {@code hdf.hdf5lib.H5} API and + * is binding-agnostic: which jar supplies that API is a build-time choice (the + * {@code hdf5-backend-*} profiles in the POM). The default is the JavaCPP + * preset ({@code org.bytedeco:hdf5-platform}), which bundles the classes AND + * the native library for the common platforms - nothing to install. Building + * with {@code -Dhdf5.ffm} swaps in the HDF Group's own + * {@code org.hdfgroup:hdf5-java-ffm} bindings (HDF5 2.1.x, Java 25 FFM, no + * JNI), which come from GitHub Packages (authenticated) and require a system + * HDF5 install; see the profile comments in pom.xml. The two provide identical + * class names, so they are mutually exclusive on the classpath - this is a + * dependency swap, not a runtime switch. + * + *

    Reader compatibility (see {@link StreamingHdf5File}): datasets are created + * with a fixed 1-D dataspace and the library-default CONTIGUOUS layout, then + * filled by sequential hyperslab {@code H5Dwrite} calls; attributes are scalar + * {@code H5T_STD_I32LE} / {@code H5T_STD_I64LE}. jHDF maps these to + * {@code ContiguousDataset} and {@link Integer}/{@link Long} attribute values, + * exactly what the existing BeakGraph readers expect. + * + *

    Not thread-safe; the huge writer drives it from a single thread. + * + * @author Erich Bremer + */ +public final class NativeHdf5File implements StreamingHdf5File { + + private static volatile Throwable unavailableCause; + + private final long fileId; + private final NativeGroup root; + // Every open native handle (groups, datasets) is tracked so close() can + // release them all before H5Fclose even if a caller leaked one. + private final Deque openHandles = new ArrayDeque<>(); + private boolean closed = false; + + private NativeHdf5File(long fileId) { + this.fileId = fileId; + this.root = new NativeGroup(fileId, "/", false); + } + + /** + * True when the native HDF5 library can be loaded on this platform. Tests + * use this to skip rather than fail where no natives exist. + */ + public static boolean isAvailable() { + try { + H5.loadH5Lib(); + H5.H5open(); + return true; + } catch (Throwable t) { + unavailableCause = t; + return false; + } + } + + /** The reason {@link #isAvailable()} answered false, if it did. */ + public static Throwable getUnavailableCause() { + return unavailableCause; + } + + /** Creates a new HDF5 file at {@code path}, truncating any existing file. */ + public static NativeHdf5File create(Path path) throws IOException { + try { + H5.loadH5Lib(); + } catch (Throwable t) { + throw new IOException( + "Native HDF5 library unavailable (needed by the huge writer backend). " + + "Ensure org.bytedeco:hdf5-platform natives are on the classpath for this platform.", t); + } + try { + long fid = H5.H5Fcreate(path.toString(), HDF5Constants.H5F_ACC_TRUNC, + HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT); + return new NativeHdf5File(fid); + } catch (Exception e) { + throw new IOException("H5Fcreate failed for " + path, e); + } + } + + @Override + public StreamingHdf5Group rootGroup() { + return root; + } + + @Override + public void close() throws IOException { + if (closed) return; + closed = true; + // Release children before the file: HDF5 keeps a file alive while any + // object handle in it is open, and a leaked handle would strand the file. + while (!openHandles.isEmpty()) { + openHandles.pop().run(); + } + try { + H5.H5Fclose(fileId); + } catch (Exception e) { + throw new IOException("H5Fclose failed", e); + } + } + + private static void quietly(Runnable r) { + try { r.run(); } catch (RuntimeException ignored) {} + } + + /** Scalar int attribute: file type I32LE, so jHDF reads an Integer. */ + private static void writeIntAttribute(long objId, String name, int value) throws IOException { + try { + long space = H5.H5Screate(HDF5Constants.H5S_SCALAR); + try { + long attr = H5.H5Acreate(objId, name, HDF5Constants.H5T_STD_I32LE, space, + HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT); + try { + H5.H5Awrite_int(attr, HDF5Constants.H5T_NATIVE_INT, new int[]{value}); + } finally { + H5.H5Aclose(attr); + } + } finally { + H5.H5Sclose(space); + } + } catch (Exception e) { + throw new IOException("Failed to write int attribute '" + name + "'", e); + } + } + + /** Scalar long attribute: file type I64LE, so jHDF reads a Long. */ + private static void writeLongAttribute(long objId, String name, long value) throws IOException { + try { + long space = H5.H5Screate(HDF5Constants.H5S_SCALAR); + try { + long attr = H5.H5Acreate(objId, name, HDF5Constants.H5T_STD_I64LE, space, + HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT); + try { + H5.H5Awrite_long(attr, HDF5Constants.H5T_NATIVE_INT64, new long[]{value}); + } finally { + H5.H5Aclose(attr); + } + } finally { + H5.H5Sclose(space); + } + } catch (Exception e) { + throw new IOException("Failed to write long attribute '" + name + "'", e); + } + } + + private final class NativeGroup implements StreamingHdf5Group { + private final long groupId; + private final boolean ownsHandle; + + NativeGroup(long groupId, String name, boolean ownsHandle) { + this.groupId = groupId; + this.ownsHandle = ownsHandle; + } + + @Override + public StreamingHdf5Group putGroup(String name) throws IOException { + try { + long gid = H5.H5Gcreate(groupId, name, HDF5Constants.H5P_DEFAULT, + HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT); + NativeGroup g = new NativeGroup(gid, name, true); + openHandles.push(() -> quietly(() -> { + try { H5.H5Gclose(gid); } catch (Exception e) { throw new RuntimeException(e); } + })); + return g; + } catch (Exception e) { + throw new IOException("H5Gcreate failed for group '" + name + "'", e); + } + } + + @Override + public void putAttribute(String name, int value) throws IOException { + writeIntAttribute(groupId, name, value); + } + + @Override + public void putAttribute(String name, long value) throws IOException { + writeLongAttribute(groupId, name, value); + } + + @Override + public StreamingHdf5Dataset createByteDataset(String name, long length) throws IOException { + if (length <= 0) { + // The RAM writer never emits empty datasets (BitPacked/DataOutput + // buffers skip them), and jHDF's getBuffer() cannot map one. + throw new IOException("Refusing to create empty dataset '" + name + "'"); + } + try { + long space = H5.H5Screate_simple(1, new long[]{length}, null); + long dset = H5.H5Dcreate(groupId, name, HDF5Constants.H5T_STD_I8LE, space, + HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT); + NativeDataset d = new NativeDataset(dset, space, name, length); + openHandles.push(() -> quietly(d::releaseHandles)); + return d; + } catch (Exception e) { + throw new IOException("H5Dcreate failed for dataset '" + name + "'", e); + } + } + } + + private static final class NativeDataset implements StreamingHdf5Dataset { + private final long datasetId; + private final long fileSpaceId; + private final String name; + private final long length; + private long position = 0; + private boolean released = false; + + NativeDataset(long datasetId, long fileSpaceId, String name, long length) { + this.datasetId = datasetId; + this.fileSpaceId = fileSpaceId; + this.name = name; + this.length = length; + } + + @Override + public void write(byte[] buf, int off, int len) throws IOException { + if (len == 0) return; + if (position + len > length) { + throw new IOException("Dataset '" + name + "' overflow: " + (position + len) + + " > declared length " + length); + } + // H5Dwrite reads elements from the START of the Java array, so a + // non-zero offset needs a compact copy of just the slice. + byte[] data = (off == 0) ? buf : Arrays.copyOfRange(buf, off, off + len); + try { + H5.H5Sselect_hyperslab(fileSpaceId, HDF5Constants.H5S_SELECT_SET, + new long[]{position}, null, new long[]{len}, null); + long memSpace = H5.H5Screate_simple(1, new long[]{len}, null); + try { + H5.H5Dwrite(datasetId, HDF5Constants.H5T_NATIVE_INT8, memSpace, fileSpaceId, + HDF5Constants.H5P_DEFAULT, data); + } finally { + H5.H5Sclose(memSpace); + } + } catch (Exception e) { + throw new IOException("H5Dwrite failed for dataset '" + name + "' at offset " + position, e); + } + position += len; + } + + @Override + public void putAttribute(String name, int value) throws IOException { + writeIntAttribute(datasetId, name, value); + } + + @Override + public void putAttribute(String name, long value) throws IOException { + writeLongAttribute(datasetId, name, value); + } + + @Override + public void close() throws IOException { + if (released) return; + boolean complete = (position == length); + releaseHandles(); + if (!complete) { + throw new IOException("Dataset '" + name + "' closed after " + position + + " of " + length + " bytes - refusing to leave undefined data"); + } + } + + private void releaseHandles() { + if (released) return; + released = true; + try { H5.H5Sclose(fileSpaceId); } catch (Exception ignored) {} + try { H5.H5Dclose(datasetId); } catch (Exception ignored) {} + } + } +} diff --git a/src/main/java/com/ebremer/beakgraph/huge/NodeCodec.java b/src/main/java/com/ebremer/beakgraph/huge/NodeCodec.java new file mode 100644 index 00000000..a7309d8a --- /dev/null +++ b/src/main/java/com/ebremer/beakgraph/huge/NodeCodec.java @@ -0,0 +1,105 @@ +package com.ebremer.beakgraph.huge; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import org.apache.jena.datatypes.TypeMapper; +import org.apache.jena.graph.Node; +import org.apache.jena.graph.NodeFactory; + +/** + * Compact, exact round-trip serialization of RDF terms for the spill files. + * Term identity is preserved: URIs and blank node labels verbatim; literals as + * lexical form + datatype URI (or + language tag), reconstructed through the + * same {@code TypeMapper} path the HDF5 readers use, so a deserialized node is + * {@code equals()} to - and {@code NodeComparator}-orders identically to - the + * parsed original. + * + *

    Node kinds the BeakGraph format cannot store (RDF-star triple terms, + * variables) fail loudly here, matching the RAM writer's ProcessQuad guards. + * + * @author Erich Bremer + */ +final class NodeCodec implements ExternalSorter.Codec { + + static final NodeCodec INSTANCE = new NodeCodec(); + + private static final TypeMapper TM = TypeMapper.getInstance(); + private static final byte T_URI = 0; + private static final byte T_BNODE = 1; + private static final byte T_LITERAL_DT = 2; + private static final byte T_LITERAL_LANG = 3; + + private NodeCodec() {} + + @Override + public void write(DataOutput out, Node n) throws IOException { + writeNode(out, n); + } + + @Override + public Node read(DataInput in) throws IOException { + return readNode(in); + } + + static void writeNode(DataOutput out, Node n) throws IOException { + if (n.isURI()) { + out.writeByte(T_URI); + writeString(out, n.getURI()); + } else if (n.isBlank()) { + out.writeByte(T_BNODE); + writeString(out, n.getBlankNodeLabel()); + } else if (n.isLiteral()) { + String lang = n.getLiteralLanguage(); + if (lang != null && !lang.isEmpty()) { + out.writeByte(T_LITERAL_LANG); + writeString(out, n.getLiteralLexicalForm()); + writeString(out, lang); + } else { + out.writeByte(T_LITERAL_DT); + writeString(out, n.getLiteralLexicalForm()); + writeString(out, n.getLiteralDatatypeURI()); + } + } else { + // Same stance as ProcessQuad: a node kind the store cannot represent + // must abort the build, not silently skew it. + throw new IllegalStateException("Unsupported node kind in huge writer spill: " + n); + } + } + + static Node readNode(DataInput in) throws IOException { + byte tag = in.readByte(); + return switch (tag) { + case T_URI -> NodeFactory.createURI(readString(in)); + case T_BNODE -> NodeFactory.createBlankNode(readString(in)); + case T_LITERAL_DT -> { + String lex = readString(in); + String dt = readString(in); + yield NodeFactory.createLiteralDT(lex, TM.getSafeTypeByName(dt)); + } + case T_LITERAL_LANG -> { + String lex = readString(in); + String lang = readString(in); + yield NodeFactory.createLiteralLang(lex, lang); + } + default -> throw new IOException("Corrupt spill record: unknown node tag " + tag); + }; + } + + static void writeString(DataOutput out, String s) throws IOException { + // Explicit varint + UTF-8 bytes: DataOutput.writeUTF caps at 65535 bytes, + // far too small for WKT literals. + byte[] bytes = s.getBytes(StandardCharsets.UTF_8); + HugeIO.writeVarLong(out, bytes.length); + out.write(bytes); + } + + static String readString(DataInput in) throws IOException { + long len = HugeIO.readVarLong(in); + if (len < 0 || len > Integer.MAX_VALUE - 8) { + throw new IOException("Corrupt spill record: string length " + len); + } + return new String(HugeIO.readBytes(in, (int) len), StandardCharsets.UTF_8); + } +} diff --git a/src/main/java/com/ebremer/beakgraph/huge/RecordFile.java b/src/main/java/com/ebremer/beakgraph/huge/RecordFile.java new file mode 100644 index 00000000..8d2f8b8a --- /dev/null +++ b/src/main/java/com/ebremer/beakgraph/huge/RecordFile.java @@ -0,0 +1,123 @@ +package com.ebremer.beakgraph.huge; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.EOFException; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.NoSuchElementException; + +/** + * A flat temp file of codec-encoded records that is written once and can be + * streamed any number of times. The huge writer materializes each sorted + * column / dictionary stream this way because several later phases re-read it + * (dedup count, dictionary encode, id join) - an {@link ExternalSorter} stream + * is single-shot. + * + * @author Erich Bremer + */ +final class RecordFile { + + private final Path path; + private final ExternalSorter.Codec codec; + private DataOutputStream out; + private long count = 0; + private boolean finished = false; + + RecordFile(Path path, ExternalSorter.Codec codec) { + this.path = path; + this.codec = codec; + } + + void append(T record) throws IOException { + if (finished) { + throw new IllegalStateException("RecordFile already finished: " + path); + } + if (out == null) { + out = new DataOutputStream(new BufferedOutputStream(Files.newOutputStream(path), 1 << 16)); + } + codec.write(out, record); + count++; + } + + void finish() throws IOException { + if (finished) return; + finished = true; + if (out != null) { + out.close(); + out = null; + } else { + // Zero records: create the empty file so readers see a valid stream. + Files.deleteIfExists(path); + Files.createFile(path); + } + } + + long count() { + return count; + } + + /** A fresh sequential stream over all records; close it when done. */ + ExternalSorter.SortedStream read() throws IOException { + if (!finished) { + throw new IllegalStateException("RecordFile not finished: " + path); + } + DataInputStream in = new DataInputStream( + new BufferedInputStream(Files.newInputStream(path), 1 << 16)); + return new ExternalSorter.SortedStream() { + private T head = advance(); + private boolean closed = false; + + private T advance() { + try { + return codec.read(in); + } catch (EOFException eof) { + closeStream(); + return null; + } catch (IOException e) { + closeStream(); + throw new UncheckedIOException("Failed reading " + path, e); + } + } + + private void closeStream() { + if (!closed) { + closed = true; + try { in.close(); } catch (IOException ignored) {} + } + } + + @Override + public boolean hasNext() { + return head != null; + } + + @Override + public T next() { + if (head == null) throw new NoSuchElementException(); + T result = head; + head = advance(); + return result; + } + + @Override + public void close() { + closeStream(); + } + }; + } + + void delete() { + try { + if (out != null) { + out.close(); + out = null; + } + Files.deleteIfExists(path); + } catch (IOException ignored) {} + } +} diff --git a/src/main/java/com/ebremer/beakgraph/huge/SpatialAugmenter.java b/src/main/java/com/ebremer/beakgraph/huge/SpatialAugmenter.java new file mode 100644 index 00000000..7b847137 --- /dev/null +++ b/src/main/java/com/ebremer/beakgraph/huge/SpatialAugmenter.java @@ -0,0 +1,198 @@ +package com.ebremer.beakgraph.huge; + +import com.ebremer.beakgraph.Params; +import com.ebremer.beakgraph.features.MajorMinor; +import com.ebremer.beakgraph.features.pyradiomics.Gen2DFeatures; +import static com.ebremer.beakgraph.hdf5.writers.PositionalDictionaryWriterBuilder.HILBERT_CELL_NS; +import static com.ebremer.beakgraph.hdf5.writers.PositionalDictionaryWriterBuilder.MAX_INDEX_CELLS; +import static com.ebremer.beakgraph.hdf5.writers.PositionalDictionaryWriterBuilder.MAX_INDEX_SCALE; +import com.ebremer.beakgraph.utils.ImageTools; +import com.ebremer.halcyon.hilbert.HilbertSpace; +import com.ebremer.halcyon.hilbert.PolygonScaler; +import com.ebremer.halcyon.hilbert.WKTDatatype; +import com.ebremer.ns.GEO; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import org.apache.jena.graph.Node; +import org.apache.jena.graph.NodeFactory; +import org.apache.jena.sparql.core.Quad; +import org.locationtech.jts.geom.Envelope; +import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.geom.GeometryFactory; +import org.locationtech.jts.geom.Polygon; +import org.locationtech.jts.geom.Polygonal; +import org.locationtech.jts.io.WKTReader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Spatial index + derived-feature quad generation for the huge writer: a + * faithful port of the spatial half of + * {@link com.ebremer.beakgraph.hdf5.writers.PositionalDictionaryWriterBuilder} + * (addSpatial / addSpatialScales / addSpatialIndexCells / generateGridURNs / + * addFeatures), sharing that class's public tuning constants so the two writers + * index geometry identically. Stateless and thread-safe: instances of the JTS + * readers/factories are created per call, matching the original. + * + * @author Erich Bremer + */ +final class SpatialAugmenter { + + private static final Logger logger = LoggerFactory.getLogger(SpatialAugmenter.class); + + /** hal:asWKT0..14, generated instead of hand-enumerated (same URIs as the RAM writer). */ + private static final Node[] asWKT = new Node[15]; + static { + for (int i = 0; i < asWKT.length; i++) { + asWKT[i] = NodeFactory.createURI("https://halcyon.is/ns/asWKT" + i); + } + } + + private final boolean features; + + SpatialAugmenter(boolean features) { + this.features = features; + } + + static boolean isGeoLiteral(Quad quad) { + Node o = quad.getObject(); + return o.isLiteral() && GEO.wktLiteral.getURI().equals(o.getLiteralDatatypeURI()); + } + + /** + * Generates all derived quads for one geometry-carrying quad. A bad + * geometry never aborts the build - it is logged and skipped, exactly like + * the RAM writer. + */ + ArrayList addSpatial(Quad quad) { + final ArrayList qqq = new ArrayList<>(); + String wkt = ImageTools.stripCrs(quad.getObject().getLiteralLexicalForm()); + if (features) { + try { + addFeatures(qqq, quad, wkt); + } catch (Exception ex) { + logger.warn("Failed to generate features for {}: {}", quad.getSubject(), ex.toString()); + } + } + try { + Geometry g = new WKTReader().read(wkt); + if (g.isEmpty()) { + return qqq; + } + // Index EVERY polygonal part, and every non-areal member via its + // expanded envelope (see the RAM writer for the full rationale). + List parts = new ArrayList<>(ImageTools.wktToPolygons(wkt)); + GeometryFactory gf = new GeometryFactory(); + for (int i = 0; i < g.getNumGeometries(); i++) { + Geometry member = g.getGeometryN(i); + if (!(member instanceof Polygonal) && !member.isEmpty()) { + Envelope env = member.getEnvelopeInternal(); + env.expandBy(0.5); + parts.add((Polygon) gf.toGeometry(env)); + } + } + for (Polygon part : parts) { + addSpatialIndexCells(qqq, quad, part); + addSpatialScales(qqq, quad, wkt, PolygonScaler.toPolygons(part)); + } + } catch (Exception ex) { + logger.warn("Skipping spatial indexing for {}: {} ({})", + quad.getSubject(), ex.toString(), abbrevWkt(wkt)); + } + return qqq; + } + + private void addSpatialScales(ArrayList qqq, Quad quad, String wkt, Polygon[] scales) { + if (scales == null) { + return; + } + final String[] wktScales = PolygonScaler.toWKT(scales); + for (int s = 0; s < Math.min(scales.length, asWKT.length); s++) { + List tiles = generateGridURNs(scales[s], s); + try { + for (int ii = 0; ii < tiles.size(); ii++) { + qqq.add(Quad.create(tiles.get(ii), quad.getSubject(), asWKT[s], + NodeFactory.createLiteralDT(wktScales[s], WKTDatatype.INSTANCE))); + } + } catch (Exception ex) { + logger.error("Failed to add spatial tile quads for {}", abbrevWkt(wkt), ex); + } + try { + qqq.add(Quad.create(Params.SPATIAL, quad.getSubject(), asWKT[s], + NodeFactory.createLiteralDT(wktScales[s], WKTDatatype.INSTANCE))); + } catch (Exception ex) { + logger.error("Failed to add scaled WKT quad for {}", abbrevWkt(wkt), ex); + } + } + } + + private void addSpatialIndexCells(ArrayList qqq, Quad quad, Polygon part) { + Envelope env = part.getEnvelopeInternal(); + long minX = HilbertSpace.clampToDomain((long) Math.floor(env.getMinX())); + long maxX = HilbertSpace.clampToDomain((long) Math.floor(env.getMaxX())); + long minY = HilbertSpace.clampToDomain((long) Math.floor(env.getMinY())); + long maxY = HilbertSpace.clampToDomain((long) Math.floor(env.getMaxY())); + int s = 0; + while (s < MAX_INDEX_SCALE && cellCount(minX, maxX, minY, maxY, s) > MAX_INDEX_CELLS) { + s++; + } + long cell = 1L << s; + Node pred = NodeFactory.createURI(HILBERT_CELL_NS + s); + HashSet cells = new HashSet<>(); + for (long x = Math.floorDiv(minX, cell); x <= Math.floorDiv(maxX, cell); x++) { + for (long y = Math.floorDiv(minY, cell); y <= Math.floorDiv(maxY, cell); y++) { + cells.add(HilbertSpace.hc.index(new long[]{x, y})); + } + } + for (Long c : cells) { + qqq.add(Quad.create(Params.SPATIAL, quad.getSubject(), pred, + NodeFactory.createLiteralByValue((long) c))); + } + } + + private static long cellCount(long minX, long maxX, long minY, long maxY, int s) { + long cell = 1L << s; + long nx = Math.floorDiv(maxX, cell) - Math.floorDiv(minX, cell) + 1; + long ny = Math.floorDiv(maxY, cell) - Math.floorDiv(minY, cell) + 1; + return nx * ny; + } + + private List generateGridURNs(Polygon polygon, int resolutionLevel) { + List intersectingURNs = new ArrayList<>(); + Envelope env = polygon.getEnvelopeInternal(); + double cellSize = Params.GRIDTILESIZE; + long minTileX = (long) Math.floor(env.getMinX() / cellSize); + long maxTileX = (long) Math.floor(env.getMaxX() / cellSize); + long minTileY = (long) Math.floor(env.getMinY() / cellSize); + long maxTileY = (long) Math.floor(env.getMaxY() / cellSize); + GeometryFactory gf = polygon.getFactory(); + for (long x = minTileX; x <= maxTileX; x++) { + double tileMinX = x * cellSize; + double tileMaxX = tileMinX + cellSize; + for (long y = minTileY; y <= maxTileY; y++) { + double tileMinY = y * cellSize; + double tileMaxY = tileMinY + cellSize; + Envelope tileEnv = new Envelope(tileMinX, tileMaxX, tileMinY, tileMaxY); + if (env.intersects(tileEnv)) { + Polygon tilePoly = (Polygon) gf.toGeometry(tileEnv); + if (polygon.intersects(tilePoly)) { + intersectingURNs.add(NodeFactory.createURI( + String.format("urn:x-beakgraph:grid:%d:%d:%d", resolutionLevel, x, y))); + } + } + } + } + return intersectingURNs; + } + + private void addFeatures(ArrayList qqq, Quad quad, String wkt) { + Node geo = quad.getSubject(); + Gen2DFeatures.generate(qqq, geo, wkt); + MajorMinor.add(qqq, geo, wkt); + } + + private static String abbrevWkt(String wkt) { + return (wkt != null && wkt.length() > 200) ? wkt.substring(0, 200) + "..." : wkt; + } +} diff --git a/src/main/java/com/ebremer/beakgraph/huge/SpillBitPackedBuffer.java b/src/main/java/com/ebremer/beakgraph/huge/SpillBitPackedBuffer.java new file mode 100644 index 00000000..cbffc024 --- /dev/null +++ b/src/main/java/com/ebremer/beakgraph/huge/SpillBitPackedBuffer.java @@ -0,0 +1,127 @@ +package com.ebremer.beakgraph.huge; + +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * Disk-backed, write-only twin of + * {@link com.ebremer.beakgraph.hdf5.BitPackedUnSignedLongBuffer}: identical + * MSB-first bit packing and identical dataset shape ({@code width} / + * {@code numEntries} attributes), but the packed bytes accumulate in a temp + * file instead of a {@code ByteArrayOutputStream}, so the buffer size is + * bounded by disk, not heap. Values must fit their declared width - the same + * silent-truncation guard as the RAM twin. + * + * @author Erich Bremer + */ +final class SpillBitPackedBuffer implements AutoCloseable { + + private final Path file; + private final OutputStream out; + private final int bitWidth; + private long writeAccumulator = 0L; + private int writeAccumulatorCount = 0; + private long numEntries = 0; + private long bytesWritten = 0; + private boolean completed = false; + + SpillBitPackedBuffer(Path file, int bitWidth) throws IOException { + // Same width envelope as the RAM buffer: the pack accumulator carries a + // value plus a <=7-bit sub-byte offset in one long, so 58..63 are unsafe; + // 64 is byte-aligned and fine. + boolean supported = (bitWidth >= 1 && bitWidth <= 57) || bitWidth == 64; + if (!supported) { + throw new IllegalArgumentException( + "Unsupported bit width: " + bitWidth + ". Supported: 1..57, or 64 (byte-aligned)."); + } + this.file = file; + this.bitWidth = bitWidth; + this.out = new BufferedOutputStream(Files.newOutputStream(file), 1 << 16); + } + + void writeInteger(int value) { + if (bitWidth != 32 && bitWidth != 64 && (value < 0 || value > ((1L << bitWidth) - 1))) { + throw new IllegalArgumentException("Value " + value + " does not fit in " + bitWidth + " bits"); + } + putValue(value & ((bitWidth == 64) ? -1L : (1L << bitWidth) - 1)); + numEntries++; + } + + void writeLong(long value) { + if (bitWidth != 64 && (value < 0 || value > ((1L << bitWidth) - 1))) { + throw new IllegalArgumentException("Value " + value + " does not fit in " + bitWidth + " bits"); + } + putValue(value & ((bitWidth == 64) ? -1L : (1L << bitWidth) - 1)); + numEntries++; + } + + private void putValue(long valToPack) { + if (completed) { + throw new IllegalStateException("Buffer already completed: " + file); + } + writeAccumulator = (writeAccumulator << bitWidth) | valToPack; + writeAccumulatorCount += bitWidth; + try { + while (writeAccumulatorCount >= 8) { + int shift = writeAccumulatorCount - 8; + out.write((byte) (writeAccumulator >>> shift)); + bytesWritten++; + writeAccumulator &= (1L << shift) - 1; + writeAccumulatorCount -= 8; + } + } catch (IOException ex) { + throw new UncheckedIOException("Failed writing spill buffer " + file, ex); + } + } + + /** Flushes the trailing partial byte and closes the temp file for writing. */ + void complete() throws IOException { + if (completed) return; + completed = true; + if (writeAccumulatorCount > 0) { + out.write((byte) (writeAccumulator << (8 - writeAccumulatorCount))); + bytesWritten++; + writeAccumulator = 0; + writeAccumulatorCount = 0; + } + out.close(); + } + + long getNumEntries() { + return numEntries; + } + + int getBitWidth() { + return bitWidth; + } + + /** + * Streams the packed bytes into {@code group} as a dataset named + * {@code name} with the {@code width}/{@code numEntries} attributes, then + * deletes the temp file. Mirrors the RAM buffer's {@code add()}: an empty + * buffer writes no dataset at all. + */ + void transferTo(StreamingHdf5Group group, String name) throws IOException { + complete(); + if (bytesWritten == 0) { + Files.deleteIfExists(file); + return; + } + try (StreamingHdf5Dataset ds = group.createByteDataset(name, bytesWritten)) { + HugeIO.copyFileIntoDataset(file, ds); + ds.putAttribute("width", bitWidth); + ds.putAttribute("numEntries", numEntries); + } + Files.deleteIfExists(file); + } + + @Override + public void close() throws IOException { + complete(); + Files.deleteIfExists(file); + } +} diff --git a/src/main/java/com/ebremer/beakgraph/huge/SpillDataBuffer.java b/src/main/java/com/ebremer/beakgraph/huge/SpillDataBuffer.java new file mode 100644 index 00000000..7290c9db --- /dev/null +++ b/src/main/java/com/ebremer/beakgraph/huge/SpillDataBuffer.java @@ -0,0 +1,92 @@ +package com.ebremer.beakgraph.huge; + +import java.io.BufferedOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * Disk-backed twin of {@link com.ebremer.beakgraph.hdf5.DataOutputBuffer}: + * big-endian primitive stream (via {@link DataOutputStream}, exactly like the + * RAM twin) accumulated in a temp file, transferred to a dataset with the + * {@code numEntries} attribute. + * + * @author Erich Bremer + */ +final class SpillDataBuffer implements AutoCloseable { + + private final Path file; + private final DataOutputStream dos; + private long numEntries = 0; + private long bytesWritten = 0; + private boolean completed = false; + + SpillDataBuffer(Path file) throws IOException { + this.file = file; + this.dos = new DataOutputStream(new BufferedOutputStream(Files.newOutputStream(file), 1 << 16)); + } + + void writeInt(int v) throws IOException { + numEntries++; + bytesWritten += Integer.BYTES; + dos.writeInt(v); + } + + void writeLong(long v) throws IOException { + numEntries++; + bytesWritten += Long.BYTES; + dos.writeLong(v); + } + + void writeFloat(float v) throws IOException { + numEntries++; + bytesWritten += Float.BYTES; + dos.writeFloat(v); + } + + void writeDouble(double v) throws IOException { + numEntries++; + bytesWritten += Double.BYTES; + dos.writeDouble(v); + } + + long getNumEntries() { + return numEntries; + } + + void complete() throws IOException { + if (completed) return; + completed = true; + dos.close(); + } + + /** + * Streams the bytes into a dataset named {@code name} with the + * {@code numEntries} attribute, then deletes the temp file. An empty buffer + * (never produced by the writer for a dataset that is actually added, same + * as the RAM path) is skipped. + */ + void transferTo(StreamingHdf5Group group, String name) throws IOException { + complete(); + if (bytesWritten == 0) { + if (numEntries > 0) { + throw new IllegalStateException( + "CRITICAL ERROR: dataset " + name + " has no bytes but numEntries is " + numEntries); + } + Files.deleteIfExists(file); + return; + } + try (StreamingHdf5Dataset ds = group.createByteDataset(name, bytesWritten)) { + HugeIO.copyFileIntoDataset(file, ds); + ds.putAttribute("numEntries", numEntries); + } + Files.deleteIfExists(file); + } + + @Override + public void close() throws IOException { + complete(); + Files.deleteIfExists(file); + } +} diff --git a/src/main/java/com/ebremer/beakgraph/huge/SpillFCDWriter.java b/src/main/java/com/ebremer/beakgraph/huge/SpillFCDWriter.java new file mode 100644 index 00000000..66f5e4a5 --- /dev/null +++ b/src/main/java/com/ebremer/beakgraph/huge/SpillFCDWriter.java @@ -0,0 +1,142 @@ +package com.ebremer.beakgraph.huge; + +import static com.ebremer.beakgraph.Params.COMPRESSION_THRESHOLD; +import com.ebremer.beakgraph.core.lib.VByte; +import com.ebremer.beakgraph.utils.StringUtils; +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * Disk-backed twin of {@link com.ebremer.beakgraph.hdf5.writers.FCDWriter}: the + * same front-coded block format (block heads, VByte prefix lengths, Zstd + * compression above {@code COMPRESSION_THRESHOLD}, surrogate-safe prefixes) and + * the same HDF5 group shape (stringbuffer/offsets/compressed datasets plus + * blockSize/numBlocks/numEntries/compression_threshold attributes), but the + * string buffer grows in a temp file, so dictionary size is bounded by disk. + * + * @author Erich Bremer + */ +final class SpillFCDWriter implements AutoCloseable { + + private final String name; + private final int blockSize; + private int stringsInCurrentBlock = 0; + private String prevString = null; + private final Path stringFile; + private final OutputStream stringOut; + private long numBlocks = 0; + private long numEntries = 0; + private long position = 0; + private final SpillDataBuffer offsets; + private final SpillBitPackedBuffer compressed; + private final StringUtils su = new StringUtils(); + private boolean completed = false; + + SpillFCDWriter(Path workDir, String name, int blockSize) throws IOException { + if (blockSize < 2) { + // Same guard as FCDWriter: blockSize 1 never closes a block and the + // offsets dataset would be unreadable. + throw new IllegalArgumentException("FCD blockSize must be >= 2, got " + blockSize); + } + this.name = name; + this.blockSize = blockSize; + this.stringFile = workDir.resolve(name + ".fcd.strings"); + this.stringOut = new BufferedOutputStream(Files.newOutputStream(stringFile), 1 << 16); + this.offsets = new SpillDataBuffer(workDir.resolve(name + ".fcd.offsets")); + this.compressed = new SpillBitPackedBuffer(workDir.resolve(name + ".fcd.compressed"), 1); + } + + private void writeFragment(byte[] data) throws IOException { + boolean shouldCompress = data.length >= COMPRESSION_THRESHOLD; + byte[] finalData; + if (shouldCompress) { + // Byte-identical to FCDWriter: decode to String, Zstd-compress that. + finalData = su.compress(new String(data, StandardCharsets.UTF_8)); + compressed.writeLong(1); + } else { + finalData = data; + compressed.writeLong(0); + } + position += VByte.encode(stringOut, finalData.length); + stringOut.write(finalData); + position += finalData.length; + } + + void add(String item) throws IOException { + numEntries++; + if (stringsInCurrentBlock == 0) { + offsets.writeLong(position); + writeFragment(item.getBytes(StandardCharsets.UTF_8)); + prevString = item; + stringsInCurrentBlock = 1; + } else { + int prefixLength = commonPrefixLength(prevString, item); + position += VByte.encode(stringOut, prefixLength); + String suffix = item.substring(prefixLength); + writeFragment(suffix.getBytes(StandardCharsets.UTF_8)); + prevString = item; + stringsInCurrentBlock++; + if (stringsInCurrentBlock == blockSize) { + stringsInCurrentBlock = 0; + numBlocks++; + } + } + } + + private int commonPrefixLength(String s1, String s2) { + int minLength = Math.min(s1.length(), s2.length()); + int i = 0; + while (i < minLength && s1.charAt(i) == s2.charAt(i)) i++; + // Never split a UTF-16 surrogate pair (see FCDWriter). + if (i > 0 && Character.isHighSurrogate(s1.charAt(i - 1))) i--; + return i; + } + + long getNumEntries() { + return numEntries; + } + + private void complete() throws IOException { + if (completed) return; + completed = true; + stringOut.close(); + offsets.complete(); + compressed.complete(); + } + + /** + * Writes the whole dictionary as a subgroup named after this writer, + * mirroring {@code FCDWriter.add()} exactly (attribute set, dataset names, + * numBlocks accounting). Temp files are deleted afterwards. + */ + void transferTo(StreamingHdf5Group parent) throws IOException { + complete(); + StreamingHdf5Group strings = parent.putGroup(name); + strings.putAttribute("blockSize", blockSize); + long validBlocks = (stringsInCurrentBlock == 0 && numEntries > 0) ? numBlocks : numBlocks + 1; + strings.putAttribute("numBlocks", (numEntries == 0) ? 0 : validBlocks); + strings.putAttribute("numEntries", numEntries); + strings.putAttribute("compression_threshold", COMPRESSION_THRESHOLD); + long size = Files.size(stringFile); + if (size > 0) { + try (StreamingHdf5Dataset ds = strings.createByteDataset("stringbuffer", size)) { + HugeIO.copyFileIntoDataset(stringFile, ds); + } + } + Files.deleteIfExists(stringFile); + offsets.transferTo(strings, "offsets"); + compressed.transferTo(strings, "compressed"); + } + + @Override + public void close() throws IOException { + complete(); + Files.deleteIfExists(stringFile); + offsets.close(); + compressed.close(); + } +} diff --git a/src/main/java/com/ebremer/beakgraph/huge/StreamingDictionaryWriter.java b/src/main/java/com/ebremer/beakgraph/huge/StreamingDictionaryWriter.java new file mode 100644 index 00000000..3cd26e1e --- /dev/null +++ b/src/main/java/com/ebremer/beakgraph/huge/StreamingDictionaryWriter.java @@ -0,0 +1,274 @@ +package com.ebremer.beakgraph.huge; + +import com.ebremer.beakgraph.core.lib.DataType; +import com.ebremer.beakgraph.core.lib.Stats; +import com.ebremer.beakgraph.hdf5.Types; +import static com.ebremer.beakgraph.utils.UTIL.MinBits; +import static com.ebremer.beakgraph.utils.UTIL.isRelativeIRI; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Set; +import java.util.SortedSet; +import org.apache.jena.graph.Node; +import org.apache.jena.vocabulary.XSD; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Streaming twin of + * {@link com.ebremer.beakgraph.hdf5.writers.MultiTypeDictionaryWriter}: encodes + * an already-sorted, already-distinct node stream into the identical dictionary + * group layout (offsets / datatypes / typedLiterals / value stores / FCDs / + * langs), with every buffer disk-backed. Where the RAM writer sorts a HashSet + * of all nodes in memory, this writer receives the stream from an external + * sort-merge, so dictionary size is bounded by disk. + * + *

    The per-node encoding logic (buffer choice, offset bookkeeping, error + * stance) is a line-for-line mirror of {@code MultiTypeDictionaryWriter.addNodeInternal}; + * any change there needs a matching change here to keep the two writers + * producing identical files. + * + * @author Erich Bremer + */ +final class StreamingDictionaryWriter implements AutoCloseable { + + private static final Logger logger = LoggerFactory.getLogger(StreamingDictionaryWriter.class); + private static final int FCD_BLOCK_SIZE = 16; + + private final String name; + private final long nodeCount; + private final SpillBitPackedBuffer offsets; + private final SpillBitPackedBuffer nativedatatypes; + private final SpillBitPackedBuffer typedLiterals; + private final SpillBitPackedBuffer integers; + private final SpillBitPackedBuffer longs; + private final SpillDataBuffer floats; + private final SpillDataBuffer doubles; + private final SpillFCDWriter iri; + private final SpillFCDWriter strings; + private final SpillFCDWriter typedLiteralsDictionary; + private final SpillFCDWriter langs; + private final SpillBitPackedBuffer langTags; + private final HashMap dataTypesLookUp = new HashMap<>(); + private final HashMap langLookUp = new HashMap<>(); + private final boolean literalsPresent; + private long encoded = 0; + + /** + * @param nodeCount the exact number of distinct nodes {@link #encode} will + * deliver; fixes the offsets bit width, exactly like the + * RAM writer's {@code builder.getNodes().size()} + * @param dataTypes distinct literal datatype IRIs (natural String order), + * empty for dictionaries without literals + * @param langSet distinct language tags among the literals, natural order + */ + StreamingDictionaryWriter(Path workDir, String name, long nodeCount, Stats stats, + Set et, SortedSet dataTypes, SortedSet langSet) + throws IOException { + this.name = name; + this.nodeCount = nodeCount; + logger.info("Building dictionary '{}' ({} nodes, disk-backed)", name, nodeCount); + + // Per-dictionary temp subdirectory: FCD writers use their name both for + // temp files and for the HDF5 group they emit, so the isolation must + // come from the directory, not a name prefix. + Path dictDir = Files.createDirectories(workDir.resolve("dict." + name)); + this.offsets = new SpillBitPackedBuffer(dictDir.resolve("offsets"), 1 + MinBits(nodeCount)); + this.nativedatatypes = new SpillBitPackedBuffer(dictDir.resolve("datatypes"), + 1 + MinBits(DataType.values().length)); + + this.integers = (!et.contains(Types.INTEGER) || (stats.numInteger == 0)) ? null + : new SpillBitPackedBuffer(dictDir.resolve("integers"), + (stats.minInteger < 0) ? 32 : (1 + MinBits(stats.maxInteger))); + int longWidth = 0; + if (et.contains(Types.LONG) && stats.numLong > 0) { + longWidth = (stats.minLong < 0) ? 64 : (1 + MinBits(stats.maxLong)); + // Same rounding as the RAM writer: the bit-packed format supports + // widths 1..57 and 64 only. + if (longWidth > 57) longWidth = 64; + } + this.longs = (longWidth == 0) ? null + : new SpillBitPackedBuffer(dictDir.resolve("longs"), longWidth); + + this.doubles = (!et.contains(Types.DOUBLE) || (stats.numDouble == 0)) ? null + : new SpillDataBuffer(dictDir.resolve("doubles")); + this.floats = (!et.contains(Types.FLOAT) || (stats.numFloat == 0)) ? null + : new SpillDataBuffer(dictDir.resolve("floats")); + + this.literalsPresent = et.contains(Types.DOUBLE) || et.contains(Types.FLOAT) + || et.contains(Types.INTEGER) || et.contains(Types.LONG) || et.contains(Types.STRING); + if (literalsPresent) { + this.typedLiteralsDictionary = new SpillFCDWriter(dictDir, "typedLiteralsDictionary", FCD_BLOCK_SIZE); + this.typedLiterals = new SpillBitPackedBuffer(dictDir.resolve("typedLiterals"), + 1 + MinBits(dataTypes.size())); + for (String dt : dataTypes) { + typedLiteralsDictionary.add(dt); + dataTypesLookUp.put(dt, typedLiteralsDictionary.getNumEntries()); + } + } else { + this.typedLiteralsDictionary = null; + this.typedLiterals = null; + } + + this.iri = (!et.contains(Types.IRI) || (stats.numIRI == 0)) ? null + : new SpillFCDWriter(dictDir, "iri", FCD_BLOCK_SIZE); + this.strings = (!et.contains(Types.STRING) || (stats.numStrings == 0)) ? null + : new SpillFCDWriter(dictDir, "strings", FCD_BLOCK_SIZE); + + if (literalsPresent && !langSet.isEmpty()) { + this.langs = new SpillFCDWriter(dictDir, "langs", FCD_BLOCK_SIZE); + for (String lang : langSet) { + langs.add(lang); + langLookUp.put(lang, langs.getNumEntries()); // 1-based id + } + this.langTags = new SpillBitPackedBuffer(dictDir.resolve("langTags"), + 1 + MinBits(langSet.size())); + } else { + this.langs = null; + this.langTags = null; + } + } + + /** Encodes the sorted distinct node stream; must deliver exactly {@code nodeCount} nodes. */ + void encode(Iterator sortedDistinct) throws IOException { + while (sortedDistinct.hasNext()) { + addNodeInternal(sortedDistinct.next()); + } + if (encoded != nodeCount) { + throw new IllegalStateException("Dictionary '" + name + "': expected " + nodeCount + + " nodes but encoded " + encoded); + } + completeAll(); + } + + private void addNodeInternal(Node node) { + if (node.isBlank()) { + // Rank-based BNodes: offset 0, label regenerated from ID on read. + nativedatatypes.writeInteger(DataType.BNODE.ordinal()); + offsets.writeLong(0); + if (literalsPresent) typedLiterals.writeLong(0); + if (langTags != null) langTags.writeLong(0); + } else if (node.isURI()) { + try { + boolean relative = isRelativeIRI(node.getURI()); + offsets.writeLong(iri.getNumEntries()); + nativedatatypes.writeInteger((relative ? DataType.RELATIVE_IRI : DataType.IRI).ordinal()); + iri.add(node.getURI()); + if (literalsPresent) typedLiterals.writeLong(0); + if (langTags != null) langTags.writeLong(0); + } catch (IOException ex) { + throw new UncheckedIOException("Failed to add IRI to dictionary: " + node, ex); + } + } else if (node.isLiteral()) { + String dt = node.getLiteralDatatypeURI(); + long dtId = dataTypesLookUp.getOrDefault(dt, 0L); + if (literalsPresent) typedLiterals.writeLong(dtId); + if (langTags != null) { + String lang = node.getLiteralLanguage(); + langTags.writeLong((lang == null || lang.isEmpty()) ? 0L : langLookUp.getOrDefault(lang, 0L)); + } + Object val; + try { + val = node.getLiteralValue(); + } catch (RuntimeException ex) { + val = null; // ill-typed literal: strings branch below + } + if (dt.equals(XSD.xlong.getURI()) && longs != null && val instanceof Number num) { + offsets.writeLong(longs.getNumEntries()); + nativedatatypes.writeInteger(DataType.LONG.ordinal()); + longs.writeLong(num.longValue()); + } else if (dt.equals(XSD.xint.getURI()) && integers != null && val instanceof Number num) { + offsets.writeLong(integers.getNumEntries()); + nativedatatypes.writeInteger(DataType.INTEGER.ordinal()); + integers.writeInteger(num.intValue()); + } else if (dt.equals(XSD.xdouble.getURI()) && doubles != null && val instanceof Number num) { + offsets.writeLong(doubles.getNumEntries()); + nativedatatypes.writeInteger(DataType.DOUBLE.ordinal()); + try { + doubles.writeDouble(num.doubleValue()); + } catch (IOException ex) { + throw new UncheckedIOException("Failed to add double literal to dictionary", ex); + } + } else if (dt.equals(XSD.xfloat.getURI()) && floats != null && val instanceof Number num) { + offsets.writeLong(floats.getNumEntries()); + nativedatatypes.writeInteger(DataType.FLOAT.ordinal()); + try { + floats.writeFloat(num.floatValue()); + } catch (IOException ex) { + throw new UncheckedIOException("Failed to add float literal to dictionary", ex); + } + } else if (strings != null) { + String lex = node.getLiteralLexicalForm(); + offsets.writeLong(strings.getNumEntries()); + nativedatatypes.writeInteger(DataType.STRING.ordinal()); + try { + strings.add(lex); + } catch (IOException ex) { + throw new UncheckedIOException("Failed to add string literal to dictionary", ex); + } + } else { + throw new IllegalStateException( + "No writer buffer for literal datatype " + dt + " (strings buffer not allocated); " + + "refusing to write a misaligned dictionary entry."); + } + } else { + throw new IllegalStateException("Unsupported node kind in dictionary '" + name + "': " + node); + } + encoded++; + if (encoded % 1_000_000 == 0) { + logger.info("Dictionary '{}': encoded {} / {} nodes", name, encoded, nodeCount); + } + } + + private void completeAll() throws IOException { + offsets.complete(); + nativedatatypes.complete(); + if (typedLiterals != null) typedLiterals.complete(); + if (integers != null) integers.complete(); + if (longs != null) longs.complete(); + if (floats != null) floats.complete(); + if (doubles != null) doubles.complete(); + if (langTags != null) langTags.complete(); + } + + long getNumberOfNodes() { + return nodeCount; + } + + /** Writes this dictionary as a subgroup, mirroring MultiTypeDictionaryWriter.add(). */ + void transferTo(StreamingHdf5Group group) throws IOException { + StreamingHdf5Group subGroup = group.putGroup(name); + if (typedLiterals != null) typedLiterals.transferTo(subGroup, "typedLiterals"); + offsets.transferTo(subGroup, "offsets"); + if (typedLiteralsDictionary != null) typedLiteralsDictionary.transferTo(subGroup); + if (integers != null) integers.transferTo(subGroup, "integers"); + if (longs != null) longs.transferTo(subGroup, "longs"); + if (floats != null) floats.transferTo(subGroup, "floats"); + if (doubles != null) doubles.transferTo(subGroup, "doubles"); + if (iri != null && iri.getNumEntries() > 0) iri.transferTo(subGroup); + if (strings != null) strings.transferTo(subGroup); + if (langs != null && langs.getNumEntries() > 0) langs.transferTo(subGroup); + if (langTags != null) langTags.transferTo(subGroup, "langTags"); + if (nativedatatypes.getNumEntries() > 0) nativedatatypes.transferTo(subGroup, "datatypes"); + } + + @Override + public void close() throws IOException { + offsets.close(); + nativedatatypes.close(); + if (typedLiterals != null) typedLiterals.close(); + if (integers != null) integers.close(); + if (longs != null) longs.close(); + if (floats != null) floats.close(); + if (doubles != null) doubles.close(); + if (iri != null) iri.close(); + if (strings != null) strings.close(); + if (typedLiteralsDictionary != null) typedLiteralsDictionary.close(); + if (langs != null) langs.close(); + if (langTags != null) langTags.close(); + } +} diff --git a/src/main/java/com/ebremer/beakgraph/huge/StreamingHdf5.java b/src/main/java/com/ebremer/beakgraph/huge/StreamingHdf5.java new file mode 100644 index 00000000..72bed54d --- /dev/null +++ b/src/main/java/com/ebremer/beakgraph/huge/StreamingHdf5.java @@ -0,0 +1,38 @@ +package com.ebremer.beakgraph.huge; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Objects; + +/** + * Factory for {@link StreamingHdf5File} backends. + * + *

    The default provider is {@link NativeHdf5File} (the HDF Group's native + * hdf5 library, the only backend that currently supports writing datasets in + * chunks). A different backend - e.g. jHDF once it supports chunked/streaming + * writes - is installed with {@link #setProvider}. + * + * @author Erich Bremer + */ +public final class StreamingHdf5 { + + /** Creates a new writable HDF5 file at the given path (truncating). */ + @FunctionalInterface + public interface Provider { + StreamingHdf5File create(Path path) throws IOException; + } + + private static volatile Provider provider = NativeHdf5File::create; + + private StreamingHdf5() {} + + /** Replaces the backend used for all subsequently created files. */ + public static void setProvider(Provider p) { + provider = Objects.requireNonNull(p, "provider"); + } + + /** Creates a new writable HDF5 file at {@code path} using the current backend. */ + public static StreamingHdf5File create(Path path) throws IOException { + return provider.create(path); + } +} diff --git a/src/main/java/com/ebremer/beakgraph/huge/StreamingHdf5Dataset.java b/src/main/java/com/ebremer/beakgraph/huge/StreamingHdf5Dataset.java new file mode 100644 index 00000000..54358e8c --- /dev/null +++ b/src/main/java/com/ebremer/beakgraph/huge/StreamingHdf5Dataset.java @@ -0,0 +1,30 @@ +package com.ebremer.beakgraph.huge; + +import java.io.IOException; + +/** + * A 1-D byte dataset being written incrementally. Bytes are appended strictly + * sequentially; the dataset must receive exactly the length declared at + * creation before {@link #close()} (closing an under-filled dataset is an + * error - it would leave undefined bytes for the readers to map). + * + * @author Erich Bremer + */ +public interface StreamingHdf5Dataset extends AutoCloseable { + + /** Appends {@code len} bytes of {@code buf} starting at {@code off}. */ + void write(byte[] buf, int off, int len) throws IOException; + + default void write(byte[] buf) throws IOException { + write(buf, 0, buf.length); + } + + /** Writes a scalar 32-bit integer attribute (jHDF reads it as Integer). */ + void putAttribute(String name, int value) throws IOException; + + /** Writes a scalar 64-bit integer attribute (jHDF reads it as Long). */ + void putAttribute(String name, long value) throws IOException; + + @Override + void close() throws IOException; +} diff --git a/src/main/java/com/ebremer/beakgraph/huge/StreamingHdf5File.java b/src/main/java/com/ebremer/beakgraph/huge/StreamingHdf5File.java new file mode 100644 index 00000000..47ad601e --- /dev/null +++ b/src/main/java/com/ebremer/beakgraph/huge/StreamingHdf5File.java @@ -0,0 +1,33 @@ +package com.ebremer.beakgraph.huge; + +import java.io.IOException; + +/** + * A writable HDF5 file whose datasets are populated incrementally (piece by + * piece) instead of from a single in-memory array. + * + *

    This is the backend seam of the huge (disk-based) BeakGraph writer. jHDF's + * write API ({@code WritableHdfFile.putDataset(name, data)}) requires the whole + * dataset in RAM, which is exactly the limit the huge writer removes, so the + * default implementation is {@link NativeHdf5File}, backed by the HDF Group's + * native library (https://github.com/HDFGroup/hdf5) which supports writing a + * dataset in chunks. When jHDF gains equivalent streaming/chunked writing, a + * jHDF-backed implementation of these interfaces can be swapped in via + * {@link StreamingHdf5#setProvider} without touching any writer code. + * + *

    Implementations MUST produce files the existing jHDF-based readers can + * open: 1-D 8-bit datasets with CONTIGUOUS layout (the readers cast to + * {@code io.jhdf.api.dataset.ContiguousDataset} and memory-map via + * {@code getBuffer()}), and scalar 32-bit / 64-bit integer attributes that jHDF + * surfaces as {@link Integer} / {@link Long}. + * + * @author Erich Bremer + */ +public interface StreamingHdf5File extends AutoCloseable { + + /** The root group of the file. */ + StreamingHdf5Group rootGroup(); + + @Override + void close() throws IOException; +} diff --git a/src/main/java/com/ebremer/beakgraph/huge/StreamingHdf5Group.java b/src/main/java/com/ebremer/beakgraph/huge/StreamingHdf5Group.java new file mode 100644 index 00000000..72972487 --- /dev/null +++ b/src/main/java/com/ebremer/beakgraph/huge/StreamingHdf5Group.java @@ -0,0 +1,37 @@ +package com.ebremer.beakgraph.huge; + +import java.io.IOException; + +/** + * A group inside a {@link StreamingHdf5File}. + * + *

    Attribute types are deliberately restricted to the two the BeakGraph + * format uses: a 32-bit integer (read back by jHDF as {@link Integer}) and a + * 64-bit integer (read back as {@link Long}). The existing readers cast + * attribute values to exactly those boxed types, so an implementation must + * preserve the width faithfully. + * + * @author Erich Bremer + */ +public interface StreamingHdf5Group { + + /** Creates (and returns) a child group. */ + StreamingHdf5Group putGroup(String name) throws IOException; + + /** Writes a scalar 32-bit integer attribute (jHDF reads it as Integer). */ + void putAttribute(String name, int value) throws IOException; + + /** Writes a scalar 64-bit integer attribute (jHDF reads it as Long). */ + void putAttribute(String name, long value) throws IOException; + + /** + * Creates a 1-D dataset of 8-bit elements with the given total length and + * CONTIGUOUS layout. The returned dataset is populated by sequential + * {@link StreamingHdf5Dataset#write} calls and must be written completely + * (exactly {@code length} bytes) before it is closed. + * + * @param name dataset name within this group + * @param length total number of bytes the dataset will hold; must be > 0 + */ + StreamingHdf5Dataset createByteDataset(String name, long length) throws IOException; +} diff --git a/src/main/java/com/ebremer/beakgraph/huge/package-info.java b/src/main/java/com/ebremer/beakgraph/huge/package-info.java new file mode 100644 index 00000000..79be18b0 --- /dev/null +++ b/src/main/java/com/ebremer/beakgraph/huge/package-info.java @@ -0,0 +1,38 @@ +/** + * Disk-based construction of very large BeakGraphs. + * + *

    The RAM writers ({@code com.ebremer.beakgraph.hdf5.writers}) hold every + * quad, every dictionary node and every buffer in memory, which caps the size + * of a buildable store at the heap. This package rebuilds the same pipeline on + * disk: + * + *

      + *
    • {@link com.ebremer.beakgraph.huge.HugeHDF5Writer} - the public entry + * point (a {@code BeakGraphWriter}, drop-in alternative to + * {@code HDF5Writer});
      • + *
    • {@link com.ebremer.beakgraph.huge.HugeBuildPipeline} - parse once, + * spill (term, row) columns, external-sort, merge-dedup into + * dictionaries, sort-merge id joins, external-sort encoded quads for the + * GSPO/GPOS indexes;
      • + *
    • {@code Spill*} buffers - file-backed twins of the bit-packed / + * front-coded / primitive buffers, byte-identical on disk;
      • + *
    • {@link com.ebremer.beakgraph.huge.StreamingHdf5File} and friends - the + * backend seam for writing HDF5 datasets incrementally. The default + * backend is {@link com.ebremer.beakgraph.huge.NativeHdf5File}, the HDF + * Group's native library (https://github.com/HDFGroup/hdf5) via the + * official {@code hdf.hdf5lib} API - today the only implementation that + * supports chunk-at-a-time writing. Which jar supplies {@code hdf.hdf5lib} + * is a build-time choice: the JavaCPP preset with bundled natives by + * default, or the HDF Group's Java 25 FFM bindings with + * {@code -Dhdf5.ffm} (see the {@code hdf5-backend-*} profiles in + * pom.xml). When jHDF (https://github.com/jamesmudd/jhdf) gains + * chunked/streaming writes, a jHDF backend can be installed with + * {@link com.ebremer.beakgraph.huge.StreamingHdf5#setProvider} without + * touching writer code.
      • + *
    + * + *

    Files produced here are read by the existing jHDF-based readers + * ({@code HDF5Reader}); datasets are CONTIGUOUS layout and attributes are + * scalar 32/64-bit integers, exactly as those readers require. + */ +package com.ebremer.beakgraph.huge; diff --git a/src/main/java/com/ebremer/beakgraph/io/ByteBufferBytes.java b/src/main/java/com/ebremer/beakgraph/io/ByteBufferBytes.java new file mode 100644 index 00000000..ccd8fb38 --- /dev/null +++ b/src/main/java/com/ebremer/beakgraph/io/ByteBufferBytes.java @@ -0,0 +1,66 @@ +package com.ebremer.beakgraph.io; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +/** + * {@link RandomAccessBytes} over a {@link ByteBuffer} - the classic path for + * jHDF-mapped datasets (which are capped at 2 GiB by ByteBuffer's int + * indexing; larger datasets use {@link MemorySegmentBytes}). + * + *

    The source buffer is duplicated at construction (position/limit/order of + * the original are never touched afterwards) and forced BIG-ENDIAN; only + * absolute reads are issued, so instances are safe for concurrent readers. + * + * @author Erich Bremer + */ +public final class ByteBufferBytes implements RandomAccessBytes { + + private final ByteBuffer buffer; + private final long size; + + public ByteBufferBytes(ByteBuffer source) { + // duplicate: shares content, isolates position/limit/order from the caller + this.buffer = source.duplicate().order(ByteOrder.BIG_ENDIAN); + this.size = this.buffer.limit(); + } + + private static int idx(long offset) { + // The wrapped buffer is < 2 GiB by construction; a larger offset is a + // caller bug and surfaces as the same IndexOutOfBounds the reads throw. + if (offset < 0 || offset > Integer.MAX_VALUE) { + throw new IndexOutOfBoundsException("offset " + offset); + } + return (int) offset; + } + + @Override + public long size() { + return size; + } + + @Override + public byte get(long offset) { + return buffer.get(idx(offset)); + } + + @Override + public long getLong(long offset) { + return buffer.getLong(idx(offset)); + } + + @Override + public float getFloat(long offset) { + return buffer.getFloat(idx(offset)); + } + + @Override + public double getDouble(long offset) { + return buffer.getDouble(idx(offset)); + } + + @Override + public void get(long offset, byte[] dst, int dstOffset, int length) { + buffer.get(idx(offset), dst, dstOffset, length); + } +} diff --git a/src/main/java/com/ebremer/beakgraph/io/DatasetBytes.java b/src/main/java/com/ebremer/beakgraph/io/DatasetBytes.java new file mode 100644 index 00000000..24b7e45d --- /dev/null +++ b/src/main/java/com/ebremer/beakgraph/io/DatasetBytes.java @@ -0,0 +1,59 @@ +package com.ebremer.beakgraph.io; + +import io.jhdf.api.dataset.ContiguousDataset; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.ByteBuffer; + +/** + * Builds the {@link RandomAccessBytes} view of a jHDF contiguous dataset. + * + *

    Datasets that fit a ByteBuffer keep today's exact path - jHDF's own + * mapped buffer, whose lifetime jHDF manages via {@code HdfFile.close()}. + * Datasets past the 2 GiB ByteBuffer ceiling (which {@code getBuffer()} + * cannot serve at all) are FFM-mapped directly from the underlying file at + * {@code dataAddress + userBlockSize} - the same file offset jHDF itself + * would map - with an automatic arena managing the unmap. + * + *

    The threshold is overridable for testing and operations via the + * {@code beakgraph.ffm.threshold} system property (bytes; datasets strictly + * larger go through FFM). Setting it to 0 forces every dataset onto the FFM + * path. + * + * @author Erich Bremer + */ +public final class DatasetBytes { + + /** Above this many bytes a dataset is FFM-mapped instead of using getBuffer(). */ + private static volatile long ffmThreshold = + Long.getLong("beakgraph.ffm.threshold", Integer.MAX_VALUE); + + private DatasetBytes() {} + + /** Overrides the FFM threshold (tests); returns the previous value. */ + static long setFfmThreshold(long thresholdBytes) { + long previous = ffmThreshold; + ffmThreshold = thresholdBytes; + return previous; + } + + public static RandomAccessBytes of(ContiguousDataset dataset) { + long size = dataset.getSizeInBytes(); + if (size == 0) { + return new ByteBufferBytes(ByteBuffer.allocate(0)); + } + long address = dataset.getDataAddress(); + if (size > ffmThreshold && address >= 0) { + try { + long fileOffset = address + dataset.getHdfFile().getUserBlockSize(); + return MemorySegmentBytes.map(dataset.getFileAsPath(), fileOffset, size); + } catch (IOException e) { + throw new UncheckedIOException( + "Failed to FFM-map dataset '" + dataset.getPath() + "' (" + + size + " bytes) from " + dataset.getFileAsPath(), e); + } + } + // <= 2 GiB: jHDF's own mapped buffer, exactly as before. + return new ByteBufferBytes(dataset.getBuffer()); + } +} diff --git a/src/main/java/com/ebremer/beakgraph/io/MemorySegmentBytes.java b/src/main/java/com/ebremer/beakgraph/io/MemorySegmentBytes.java new file mode 100644 index 00000000..bd3c48fb --- /dev/null +++ b/src/main/java/com/ebremer/beakgraph/io/MemorySegmentBytes.java @@ -0,0 +1,83 @@ +package com.ebremer.beakgraph.io; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.ByteOrder; +import java.nio.channels.FileChannel; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; + +/** + * {@link RandomAccessBytes} over an FFM {@link MemorySegment} - the path for + * datasets past ByteBuffer's 2 GiB ceiling (and, at parity performance, an + * alternative below it). Since JDK 22 a file region of any size can be mapped + * straight to a segment; absolute segment reads JIT to the same machine code + * as direct-ByteBuffer access. + * + *

    Mappings use an automatic {@link Arena}: the region unmaps when the + * segment becomes unreachable, so no close() plumbing is needed through the + * reader stack (the file channel is closed immediately after mapping - the + * mapping survives it). Windows caveat: as with any mapping, the file stays + * in-use until the unmap actually happens. + * + *

    All reads are absolute and the layouts are unaligned BIG-ENDIAN (the + * bit-packed data honors no alignment). Auto arenas are shared, so instances + * are safe for concurrent readers. + * + * @author Erich Bremer + */ +public final class MemorySegmentBytes implements RandomAccessBytes { + + private static final ValueLayout.OfLong LONG_BE = + ValueLayout.JAVA_LONG_UNALIGNED.withOrder(ByteOrder.BIG_ENDIAN); + private static final ValueLayout.OfFloat FLOAT_BE = + ValueLayout.JAVA_FLOAT_UNALIGNED.withOrder(ByteOrder.BIG_ENDIAN); + private static final ValueLayout.OfDouble DOUBLE_BE = + ValueLayout.JAVA_DOUBLE_UNALIGNED.withOrder(ByteOrder.BIG_ENDIAN); + + private final MemorySegment segment; + + public MemorySegmentBytes(MemorySegment segment) { + this.segment = segment; + } + + /** Maps {@code size} bytes of {@code file} starting at {@code offset}, read-only. */ + public static MemorySegmentBytes map(Path file, long offset, long size) throws IOException { + try (FileChannel fc = FileChannel.open(file, StandardOpenOption.READ)) { + MemorySegment seg = fc.map(FileChannel.MapMode.READ_ONLY, offset, size, Arena.ofAuto()); + return new MemorySegmentBytes(seg); + } + } + + @Override + public long size() { + return segment.byteSize(); + } + + @Override + public byte get(long offset) { + return segment.get(ValueLayout.JAVA_BYTE, offset); + } + + @Override + public long getLong(long offset) { + return segment.get(LONG_BE, offset); + } + + @Override + public float getFloat(long offset) { + return segment.get(FLOAT_BE, offset); + } + + @Override + public double getDouble(long offset) { + return segment.get(DOUBLE_BE, offset); + } + + @Override + public void get(long offset, byte[] dst, int dstOffset, int length) { + MemorySegment.copy(segment, ValueLayout.JAVA_BYTE, offset, dst, dstOffset, length); + } +} diff --git a/src/main/java/com/ebremer/beakgraph/io/RandomAccessBytes.java b/src/main/java/com/ebremer/beakgraph/io/RandomAccessBytes.java new file mode 100644 index 00000000..d5e62b0e --- /dev/null +++ b/src/main/java/com/ebremer/beakgraph/io/RandomAccessBytes.java @@ -0,0 +1,43 @@ +package com.ebremer.beakgraph.io; + +/** + * Absolute-offset random access over an immutable byte region - the seam + * between BeakGraph's readers and whatever holds the bytes. + * + *

    All offsets are {@code long} (no 2 GiB ceiling) and all multi-byte reads + * are BIG-ENDIAN, matching the on-disk format (DataOutputStream primitives and + * MSB-first bit packing). The API is deliberately position-free: every read is + * absolute, so one instance may be shared by concurrent query threads - the + * same contract the readers previously relied on with absolute ByteBuffer + * reads. + * + *

    Implementations: {@link ByteBufferBytes} (wraps the jHDF-mapped buffer, + * the default for datasets under 2 GiB), {@link MemorySegmentBytes} (an FFM + * mapped segment for datasets beyond ByteBuffer's reach). A future chunked + * implementation (e.g. Zarr with a decompressed-chunk cache) plugs in here + * without touching the readers. + * + *

    Out-of-range offsets throw {@link IndexOutOfBoundsException}. + * + * @author Erich Bremer + */ +public interface RandomAccessBytes { + + /** Total number of readable bytes. */ + long size(); + + /** The byte at {@code offset}. */ + byte get(long offset); + + /** The big-endian 64-bit value at {@code offset} (unaligned allowed). */ + long getLong(long offset); + + /** The big-endian 32-bit float at {@code offset} (unaligned allowed). */ + float getFloat(long offset); + + /** The big-endian 64-bit double at {@code offset} (unaligned allowed). */ + double getDouble(long offset); + + /** Copies {@code length} bytes starting at {@code offset} into {@code dst}. */ + void get(long offset, byte[] dst, int dstOffset, int length); +} diff --git a/src/main/java/com/ebremer/beakgraph/lws/LWSMetadataGenerator.java b/src/main/java/com/ebremer/beakgraph/lws/LWSMetadataGenerator.java index d7e25810..e135be2a 100644 --- a/src/main/java/com/ebremer/beakgraph/lws/LWSMetadataGenerator.java +++ b/src/main/java/com/ebremer/beakgraph/lws/LWSMetadataGenerator.java @@ -43,7 +43,8 @@ public static void main(String[] args) { writeModelToGZ(model, outputPath); System.out.println("Metadata successfully written to " + outputPath.toAbsolutePath()); } catch (IOException e) { - e.printStackTrace(); + System.err.println("Failed to generate LWS metadata: " + e.getMessage()); + System.exit(1); } } @@ -71,6 +72,13 @@ public static Model generateLWSModel(Path rootPath) throws IOException { @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + // Never index the metadata cache itself: on regeneration the previous + // beakgraph.ttl.gz would become a listed DataResource, making the raw + // model - including the owl:sameAs file:/// server paths the servlet + // exists to withhold - downloadable by any client. + if (CACHE_FILE_NAME.equals(file.getFileName().toString())) { + return FileVisitResult.CONTINUE; + } String httpUri = toHttpUri(rootPath, file); processResource(model, httpUri, file, attrs, dataType, mediaType, size, modified); linkToParent(model, rootPath, file, items, totalItems); diff --git a/src/main/java/com/ebremer/beakgraph/pool/BeakGraphKeyedPool.java b/src/main/java/com/ebremer/beakgraph/pool/BeakGraphKeyedPool.java index 1872d5b9..9f0df5bc 100644 --- a/src/main/java/com/ebremer/beakgraph/pool/BeakGraphKeyedPool.java +++ b/src/main/java/com/ebremer/beakgraph/pool/BeakGraphKeyedPool.java @@ -19,13 +19,18 @@ public BeakGraphKeyedPool(BeakGraphPoolFactory factory, BeakGraphKeyedPoolConfig @Override public BeakGraph borrowObject(final URI key) throws Exception { - logger.trace("borrowObject {}\n{}", key, getStatus()); + // getStatus() takes five pool-lock metrics; only pay for it when TRACE is on. + if (logger.isTraceEnabled()) { + logger.trace("borrowObject {}\n{}", key, getStatus()); + } return super.borrowObject(key); } - + @Override public void returnObject(final URI key, final BeakGraph reader) { - logger.trace("returnObject {}\n{}", key, getStatus()); + if (logger.isTraceEnabled()) { + logger.trace("returnObject {}\n{}", key, getStatus()); + } super.returnObject(key, reader); } diff --git a/src/main/java/com/ebremer/beakgraph/pool/BeakGraphPool.java b/src/main/java/com/ebremer/beakgraph/pool/BeakGraphPool.java index e37a6e03..97b6242e 100644 --- a/src/main/java/com/ebremer/beakgraph/pool/BeakGraphPool.java +++ b/src/main/java/com/ebremer/beakgraph/pool/BeakGraphPool.java @@ -15,8 +15,12 @@ private static class Holder { private static final BeakGraphKeyedPool INSTANCE; static { BeakGraphKeyedPoolConfig config = new BeakGraphKeyedPoolConfig<>(); - config.setMaxTotalPerKey(100); - config.setMaxTotal(200); + // A reader is safe for concurrent use (absolute buffer reads throughout), + // so extra instances per file mostly multiply memory: each holds mapped + // buffers, a tiered node index and two 1M-entry caches. Two per key + // covers validation-replacement; the old 100/key invited OOM. + config.setMaxTotalPerKey(2); + config.setMaxTotal(64); config.setMinIdlePerKey(0); config.setTestOnBorrow(true); config.setMaxWait(Duration.ofMillis(60000)); diff --git a/src/main/java/com/ebremer/beakgraph/pool/BeakGraphPoolFactory.java b/src/main/java/com/ebremer/beakgraph/pool/BeakGraphPoolFactory.java index 17ce3b35..59449432 100644 --- a/src/main/java/com/ebremer/beakgraph/pool/BeakGraphPoolFactory.java +++ b/src/main/java/com/ebremer/beakgraph/pool/BeakGraphPoolFactory.java @@ -24,7 +24,13 @@ public BeakGraphPoolFactory() {} public BeakGraph create(URI uri) throws Exception { logger.trace("Creating BeakGraph {}", uri); HDF5Reader reader = new HDF5Reader(new File(uri)); - return new BeakGraph(reader, uri, null); + try { + return new BeakGraph(reader, uri, null); + } catch (RuntimeException | Error e) { + // Release the mapped file if wrapping fails - a leaked reader pins it. + try { reader.close(); } catch (Exception ignore) {} + throw e; + } } @Override @@ -32,6 +38,33 @@ public PooledObject wrap(BeakGraph value) { return new DefaultPooledObject<>(value); } + /** + * Without this override, BaseKeyedPooledObjectFactory.validateObject always + * returns true and setTestOnBorrow(true) validates nothing - an instance whose + * reader was closed (poisoned) would be re-issued and fail on first use. + * The probe additionally touches REAL mapped data, not just the open flag: + * a reader whose backing file was replaced or truncated underneath stays + * "open" but throws on access, and an isOpen-only check re-issued it to + * every borrower forever. + */ + @Override + public boolean validateObject(URI uri, PooledObject p) { + BeakGraph bg = p.getObject(); + if (!bg.getReader().isOpen()) { + return false; + } + try { + var predicates = bg.getReader().getDictionary().getPredicates(); + if (predicates.getNumberOfNodes() > 0) { + predicates.extract(1); + } + return true; + } catch (RuntimeException | Error e) { + logger.warn("Pooled BeakGraph for {} failed data-access validation; discarding", uri, e); + return false; + } + } + @Override public void destroyObject(URI uri, PooledObject p, DestroyMode mode) throws Exception { logger.trace("destroyObject {}", uri); diff --git a/src/main/java/com/ebremer/beakgraph/turbo/Intersects.java b/src/main/java/com/ebremer/beakgraph/turbo/Intersects.java index d72dba8b..e86d30cc 100644 --- a/src/main/java/com/ebremer/beakgraph/turbo/Intersects.java +++ b/src/main/java/com/ebremer/beakgraph/turbo/Intersects.java @@ -4,17 +4,25 @@ import java.util.List; import org.apache.jena.atlas.lib.Lib; import org.apache.jena.query.QueryBuildException; +import org.apache.jena.sparql.expr.ExprEvalException; import org.apache.jena.sparql.expr.ExprList; import org.apache.jena.sparql.expr.NodeValue; import org.apache.jena.sparql.function.FunctionBase; import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.geom.util.GeometryFixer; import org.locationtech.jts.io.ParseException; import org.locationtech.jts.io.WKTReader; +/** + * JTS-backed implementation of geof:sfIntersects. This is the fallback for any + * sfIntersects the Hilbert index rewrite does not capture, so it must answer + * honestly - the previous version returned TRUE unconditionally, which made + * every uncaptured spatial filter match everything. + */ public class Intersects extends FunctionBase { private static final String WKT_DATATYPE_URI = GEO.wktLiteral.getURI(); - + @Override public void checkBuild(String uri, ExprList args) { if (( args.size() < 2 ) || ( args.size() > 3 )) { @@ -22,38 +30,35 @@ public void checkBuild(String uri, ExprList args) { } } - /* @Override - public NodeValue exec(NodeValue v1, NodeValue v2) { - return NodeValue.TRUE; - /* + public NodeValue exec(List args) { + NodeValue v1 = args.get(0); + NodeValue v2 = args.get(1); if (v1 == null || v2 == null) { - throw new ExprEvalException("sfWithin: Arguments cannot be null"); + throw new ExprEvalException("sfIntersects: arguments cannot be null"); } if (!isValidGeometryLiteral(v1) || !isValidGeometryLiteral(v2)) { - throw new ExprEvalException("sfWithin: Arguments must be geo:wktLiteral"); + throw new ExprEvalException("sfIntersects: arguments must be geo:wktLiteral"); } - String geo1 = v1.asNode().getLiteralLexicalForm(); - String geo2 = v2.asNode().getLiteralLexicalForm(); try { - boolean isWithin = performSpatialCheck(geo1, geo2); - return isWithin ? NodeValue.TRUE : NodeValue.FALSE; + boolean intersects = performSpatialCheck( + v1.asNode().getLiteralLexicalForm(), + v2.asNode().getLiteralLexicalForm()); + return intersects ? NodeValue.TRUE : NodeValue.FALSE; } catch (Exception e) { - throw new ExprEvalException("sfWithin: Calculation failed: " + e.getMessage()); + throw new ExprEvalException("sfIntersects: " + e.getMessage()); } - }*/ + } private boolean isValidGeometryLiteral(NodeValue nv) { - if (!nv.isLiteral()) return false; + if (!nv.isLiteral()) return false; String dtURI = nv.asNode().getLiteralDatatypeURI(); return WKT_DATATYPE_URI.equals(dtURI); } /** - * Uses JTS to check if wkt1 is within wkt2. - * @param wkt1 The subject geometry (e.g., the point) - * @param wkt2 The containing geometry (e.g., the polygon) - * @return true if wkt1 is within wkt2 + * Uses JTS to check whether the two WKT geometries intersect (share at + * least one point) - the geof:sfIntersects relation. * @throws ParseException if WKT is invalid */ private boolean performSpatialCheck(String wkt1, String wkt2) throws ParseException { @@ -65,13 +70,23 @@ private boolean performSpatialCheck(String wkt1, String wkt2) throws ParseExcept // JTS only accepts "POINT(1 1)", so we must strip the URI prefix. String cleanWkt1 = extractWkt(wkt1); String cleanWkt2 = extractWkt(wkt2); - Geometry g1 = reader.read(cleanWkt1); - Geometry g2 = reader.read(cleanWkt2); - if (!g1.isValid() || !g2.isValid()) { - throw new ParseException("Encountered invalid geometry topology."); - } + Geometry g1 = repaired(reader.read(cleanWkt1)); + Geometry g2 = repaired(reader.read(cleanWkt2)); + + return g1.intersects(g2); + } - return g1.within(g2); + /** + * Topologically invalid geometry (self-intersecting rings - a data reality + * in pathology exports) must not make this function error out: the build + * deliberately indexes such geometries, and throwing here made Jena drop + * the row for every candidate whose stored WKT is invalid - an indexed + * geometry that no sfIntersects query could ever return. GeometryFixer + * preserves the point set (a bowtie becomes its two triangles), unlike + * buffer(0), which discards zero-area parts and lines/points. + */ + private static Geometry repaired(Geometry g) { + return g.isValid() ? g : GeometryFixer.fix(g); } /** @@ -90,9 +105,4 @@ private String extractWkt(String geoSparqlLiteral) { } return trimmed; } - - @Override - public NodeValue exec(List args) { - return NodeValue.TRUE; - } } \ No newline at end of file diff --git a/src/main/java/com/ebremer/beakgraph/turbo/SineWaveGenerator.java b/src/main/java/com/ebremer/beakgraph/turbo/SineWaveGenerator.java deleted file mode 100644 index 86bac53c..00000000 --- a/src/main/java/com/ebremer/beakgraph/turbo/SineWaveGenerator.java +++ /dev/null @@ -1,41 +0,0 @@ -package com.ebremer.beakgraph.turbo; - -import org.locationtech.jts.geom.Coordinate; -import org.locationtech.jts.geom.GeometryFactory; -import org.locationtech.jts.geom.LineString; - -public class SineWaveGenerator { - - private static final GeometryFactory geomFactory = new GeometryFactory(); - - /** - * Generates a JTS LineString representing a Sine Wave with an offset. - * - * @param A Amplitude (Height from center to peak). Total vertical span is 2*A. - * @param W Total Width (length along X axis). - * @param n Number of full cycles to fit within W. - * @param numPoints Resolution: How many points to use to draw the wave. - * @param offX X offset (shifts the start of the wave horizontally). - * @param offY Y offset (shifts the center of the wave vertically). - * @return A JTS LineString. - */ - public static LineString createSinePolyline(double A, double W, double n, int numPoints, double offX, double offY) { - Coordinate[] coords = new Coordinate[numPoints + 1]; - - // Calculate the angular frequency - double k = (2 * Math.PI * n) / W; - - for (int i = 0; i <= numPoints; i++) { - // 1. Calculate the local x (0 to W) used for the shape math - double xLocal = (double) i / numPoints * W; - - // 2. Calculate the local y based on the sine function - double yLocal = A * Math.sin(k * xLocal); - - // 3. Apply the offsets to get the final world coordinates - coords[i] = new Coordinate(xLocal + offX, yLocal + offY); - } - - return geomFactory.createLineString(coords); - } -} \ No newline at end of file diff --git a/src/main/java/com/ebremer/beakgraph/turbo/Spatial.java b/src/main/java/com/ebremer/beakgraph/turbo/Spatial.java index 8a58540a..5cd160fc 100644 --- a/src/main/java/com/ebremer/beakgraph/turbo/Spatial.java +++ b/src/main/java/com/ebremer/beakgraph/turbo/Spatial.java @@ -2,8 +2,6 @@ import com.ebremer.ns.GEOF; import org.apache.jena.sparql.function.FunctionRegistry; -import org.apache.jena.sparql.pfunction.PropertyFunctionRegistry; -import org.apache.jena.vocabulary.RDFS; /** @@ -14,10 +12,16 @@ public final class Spatial { private static Spatial spatial = null; private Spatial() { - PropertyFunctionRegistry.get().remove(RDFS.member.getURI()); - FunctionRegistry.get().put(GEOF.sfIntersects.getURI(), Intersects.class); + // Register a real, JTS-backed geof:sfIntersects so spatial filters that the + // index rewrite does not capture still evaluate correctly. Registering a + // filter function globally is the normal Jena pattern; what this must NOT + // do is alter standard semantics for other datasets - the old code here + // remove()d the rdfs:member property function JVM-wide. BG datasets opt out + // of rdfs:member rewriting via their own dataset-scoped registry instead + // (see BGDatasetGraph). + FunctionRegistry.get().put(GEOF.sfIntersects.getURI(), Intersects.class); } - + public synchronized static void init() { if (spatial == null) { spatial = new Spatial(); diff --git a/src/main/java/com/ebremer/beakgraph/utils/HDTBitmapDirectory.java b/src/main/java/com/ebremer/beakgraph/utils/HDTBitmapDirectory.java index 050defe6..facbc76c 100644 --- a/src/main/java/com/ebremer/beakgraph/utils/HDTBitmapDirectory.java +++ b/src/main/java/com/ebremer/beakgraph/utils/HDTBitmapDirectory.java @@ -23,7 +23,6 @@ public class HDTBitmapDirectory { private final long numBitmapEntries; private final long numSuperblockEntries; private final long numBlockEntries; - private final long numIdEntries; public HDTBitmapDirectory(BitPackedUnSignedLongBuffer superblock, BitPackedUnSignedLongBuffer block, @@ -48,38 +47,10 @@ public HDTBitmapDirectory(BitPackedUnSignedLongBuffer superblock, this.numBitmapEntries = bitmap.getNumEntries(); this.numSuperblockEntries = superblock.getNumEntries(); this.numBlockEntries = block.getNumEntries(); - this.numIdEntries = ids.getNumEntries(); } public long getNumBitmapEntries() { return numBitmapEntries; } - public long rank1(long pos) { - if (pos <= 0) return 0L; - if (pos > numBitmapEntries) pos = numBitmapEntries; - - long targetBit = pos - 1; - long sbIdx = targetBit / superblockSize; - long bIdx = targetBit / blockSize; - - long rank = (sbIdx < numSuperblockEntries) ? superblock.get(sbIdx) : superblock.get(numSuperblockEntries - 1); - - if (bIdx < numBlockEntries) { - rank += block.get(bIdx); - } - - long currentPos = bIdx * blockSize; - while (currentPos + 64 <= pos) { - long word = bitmap.getWord64(currentPos); - rank += Long.bitCount(word); - currentPos += 64; - } - while (currentPos < pos) { - if (bitmap.get(currentPos) == 1) rank++; - currentPos++; - } - return rank; - } - public long select1(long rank) { if (rank <= 0) return -1L; @@ -162,28 +133,10 @@ public long select1(long rank) { } private long findNthSetBitInWord(long word, long n) { - // Broadword selection: 0-based index (from MSB) of the n-th set bit (n >= 1), - // O(1) via 6 popcount narrowing steps. Replaces an O(64) bit-by-bit loop; the - // result is identical to the linear select1's selectInWordSafe. - long result = 0; - int cnt; - cnt = Long.bitCount(word >>> 32); if (n > cnt) { word <<= 32; result += 32; n -= cnt; } - cnt = Long.bitCount(word >>> 48); if (n > cnt) { word <<= 16; result += 16; n -= cnt; } - cnt = Long.bitCount(word >>> 56); if (n > cnt) { word <<= 8; result += 8; n -= cnt; } - cnt = Long.bitCount(word >>> 60); if (n > cnt) { word <<= 4; result += 4; n -= cnt; } - cnt = Long.bitCount(word >>> 62); if (n > cnt) { word <<= 2; result += 2; n -= cnt; } - cnt = Long.bitCount(word >>> 63); if (n > cnt) { result += 1; } - return result; + // Delegates to the shared broadword implementation so the accelerated + // select1 and the buffer's linear select1 can never disagree. + return UTIL.selectInWord(word, n); } public BitPackedUnSignedLongBuffer getIds() { return ids; } - - public long getId(long rank) { - if (rank <= 0 || rank > numIdEntries) { - throw new IndexOutOfBoundsException("Rank " + rank + " out of bounds [1.." + numIdEntries + "]"); - } - return ids.get(rank - 1); - } - - public long getBitCount() { return rank1(numBitmapEntries); } } \ No newline at end of file diff --git a/src/main/java/com/ebremer/beakgraph/utils/ImageTools.java b/src/main/java/com/ebremer/beakgraph/utils/ImageTools.java index 7befb113..b99b4b5b 100644 --- a/src/main/java/com/ebremer/beakgraph/utils/ImageTools.java +++ b/src/main/java/com/ebremer/beakgraph/utils/ImageTools.java @@ -95,11 +95,28 @@ public static Polygon wktToPolygon(String wkt) throws Exception { return polygons.isEmpty() ? null : polygons.get(0); } + /** + * Strips the optional CRS/SRS URI prefix from a GeoSPARQL wktLiteral + * ("<http://...crs...> POLYGON(...)" -> "POLYGON(...)"). The GeoSPARQL + * spec allows the prefix and data in the wild commonly carries it; JTS's + * WKTReader does not accept it. + */ + public static String stripCrs(String wkt) { + String trimmed = wkt.trim(); + if (trimmed.startsWith("<")) { + int end = trimmed.indexOf('>'); + if (end != -1) { + return trimmed.substring(end + 1).trim(); + } + } + return trimmed; + } + public static List wktToPolygons(String wkt) throws Exception { if (wkt == null || wkt.trim().isEmpty()) { return Collections.emptyList(); } - Geometry geom = WKT_READER.read(wkt.trim()); + Geometry geom = WKT_READER.read(stripCrs(wkt)); if (geom.isEmpty()) { return Collections.emptyList(); } diff --git a/src/main/java/com/ebremer/beakgraph/utils/UTIL.java b/src/main/java/com/ebremer/beakgraph/utils/UTIL.java index 005495bd..5d5e95a1 100644 --- a/src/main/java/com/ebremer/beakgraph/utils/UTIL.java +++ b/src/main/java/com/ebremer/beakgraph/utils/UTIL.java @@ -39,73 +39,6 @@ public static String toBinaryString(ByteBuffer buffer, int offset) { return sb.toString(); } - public static byte[] getBytes(ByteBuffer buffer) { - String cn = buffer.getClass().getName(); - if (cn.equals("java.nio.HeapByteBuffer2")) { - buffer.position(0); - int r = buffer.capacity(); - byte[] b = new byte[r]; - buffer.get(b); - return b; - } - return buffer.array(); - } - - public static ByteBuffer subBuffer(ByteBuffer src, int offset, int length) { - ByteBuffer dup = src.duplicate(); - dup.position(offset); - dup.limit(offset + length); - return dup.slice(); - } - - public static void skipNullTerminatedStrings(ByteBuffer buffer, int skip) { - if (skip > 0) { - int currentPosition = buffer.position(); - int limit = buffer.limit(); - for (int i = 0; i < skip; i++) { - while (currentPosition < limit) { - if (buffer.get(currentPosition) == 0) { - currentPosition++; - break; - } - currentPosition++; - } - } - buffer.position(currentPosition); - } - } - - public static String readNullTerminatedString(ByteBuffer buffer) { - if (!buffer.hasRemaining()) { - return ""; - } - int startPosition = buffer.position(); - int limit = buffer.limit(); - int endPosition = startPosition; - // Find the null terminator - while (endPosition < limit && buffer.get(endPosition) != 0) { - endPosition++; - } - int length = endPosition - startPosition; - byte[] stringBytes = new byte[length]; - // Use bulk get for better performance - buffer.get(stringBytes); - // If we stopped at a null terminator, skip over it - if (buffer.hasRemaining() && buffer.get() != 0) { - // This handles the edge case where the loop stopped at limit - // but we still need to advance position correctly. - // Usually, buffer.get() above moves position to endPosition. - // If the byte at endPosition was 0, buffer.get() consumes it. - } - // Correctly advance position to after the null terminator if it exists - if (endPosition < limit) { - buffer.position(endPosition + 1); - } else { - buffer.position(limit); - } - return new String(stringBytes, StandardCharsets.UTF_8); - } - public static WritableDataset putAttributes( WritableDataset ds, Map attributes ) { attributes.forEach((k,v)->{ ds.putAttribute(k, v); @@ -118,6 +51,25 @@ public static int MinBits(long x) { return Long.SIZE - Long.numberOfLeadingZeros(x); } + /** + * Broadword selection: the 0-based index (from the MSB) of the k-th set bit + * (k >= 1) of {@code word}, in O(1) via six popcount narrowing steps. The + * SINGLE implementation shared by the bit-packed buffer's linear select1 and + * the rank/select directory's accelerated select1 - the two must agree + * bit-for-bit or the directory fast path and the fallback diverge. + */ + public static int selectInWord(long word, long k) { + int result = 0; + int cnt; + cnt = Long.bitCount(word >>> 32); if (k > cnt) { word <<= 32; result += 32; k -= cnt; } + cnt = Long.bitCount(word >>> 48); if (k > cnt) { word <<= 16; result += 16; k -= cnt; } + cnt = Long.bitCount(word >>> 56); if (k > cnt) { word <<= 8; result += 8; k -= cnt; } + cnt = Long.bitCount(word >>> 60); if (k > cnt) { word <<= 4; result += 4; k -= cnt; } + cnt = Long.bitCount(word >>> 62); if (k > cnt) { word <<= 2; result += 2; k -= cnt; } + cnt = Long.bitCount(word >>> 63); if (k > cnt) { result += 1; } + return result; + } + /** * True when {@code iri} is a relative reference (RFC 3986) - i.e. it has no * scheme. The empty string (a same-document reference such as {@code <>}) is diff --git a/src/main/java/com/ebremer/halcyon/geometry/Vector2D.java b/src/main/java/com/ebremer/halcyon/geometry/Vector2D.java index d49753df..c9e22188 100644 --- a/src/main/java/com/ebremer/halcyon/geometry/Vector2D.java +++ b/src/main/java/com/ebremer/halcyon/geometry/Vector2D.java @@ -14,27 +14,30 @@ public class Vector2D { } public double Magnitude() { - return Math.sqrt((a*a)+(b*b)); + // double math: int*int overflows for coordinates above ~46340, which + // slide-scale coordinates routinely exceed. + return Math.sqrt(((double) a * a) + ((double) b * b)); } - + public static double Magnitude(long[] a) { - long sum = 0; + double sum = 0; for (int c=0; c + * Trimmed to the members with live callers; the legacy viewer rendering and + * rasterization utilities that accumulated here (image generation, JSON + * export, alternative Polygon2Hilbert variants, range merging) were unused - + * several demonstrably broken - and were removed in the dead-code sweep. * * @author erich */ public final class HilbertSpace { private static final Logger logger = LoggerFactory.getLogger(HilbertSpace.class); public static final SmallHilbertCurve hc = HilbertCurve.small().bits(31).dimensions(2); + + /** Largest coordinate the 31-bit curve can address. */ + public static final long MAX_COORD = (1L << 31) - 1; + public static final byte N = 0; public static final byte NE = 1; public static final byte E = 2; @@ -46,9 +43,26 @@ public final class HilbertSpace { public static final byte NW = 7; private HilbertSpace() {} - + + /** + * Clamps a coordinate into the curve's domain [0, 2^31). The davidmoten + * curve silently MASKS coordinates to their low 31 bits, so out-of-domain + * values alias onto unrelated cells - written and queried covers then + * disagree and matches are silently lost. Clamping is a monotone + * projection: applied identically on the write and query sides, two + * overlapping bboxes still overlap after it, so recall is preserved + * (out-of-domain geometry indexes coarsely at the domain-edge cells and + * the exact JTS verification removes the false positives). + */ + public static long clampToDomain(long v) { + return Math.max(0, Math.min(MAX_COORD, v)); + } + public static boolean inRange(ArrayList rr, Point p, Byte neighbor) { - switch (neighbor) { + // Return the membership result - the old switch *statement* computed + // contains(...) and discarded it, so this always answered false and the + // getPolygon boundary walk never advanced. + return switch (neighbor) { case N -> contains(rr,p.x,p.y-1); case NE -> contains(rr,p.x+1,p.y-1); case E -> contains(rr,p.x+1,p.y); @@ -57,12 +71,12 @@ public static boolean inRange(ArrayList rr, Point p, Byte neighbor) { case SW -> contains(rr,p.x-1,p.y+1); case W -> contains(rr,p.x-1,p.y); case NW -> contains(rr,p.x-1,p.y-1); - } - return false; + default -> false; + }; } - + public static boolean inRange(Ranges rr, Point p, Byte neighbor) { - switch (neighbor) { + return switch (neighbor) { case N -> contains(rr,p.x,p.y-1); case NE -> contains(rr,p.x+1,p.y-1); case E -> contains(rr,p.x+1,p.y); @@ -71,32 +85,10 @@ public static boolean inRange(Ranges rr, Point p, Byte neighbor) { case SW -> contains(rr,p.x-1,p.y+1); case W -> contains(rr,p.x-1,p.y); case NW -> contains(rr,p.x-1,p.y-1); - } - return false; - } - - public static Point getPoint(long p) { - long[] c = hc.point(p); - return new Point((int) c[0], (int) c[1]); + default -> false; + }; } - public static Polygon getFatPoint(long p) { - long[] c = hc.point(p); - int a = (int) c[0]; - int b = (int) c[1]; - int[] x = new int[4]; - int[] y = new int[4]; - x[0] = a; - y[0] = b; - x[1] = a+1; - y[1] = b; - x[2] = a+1; - y[2] = b+1; - x[3] = a; - y[3] = b+1; - return new Polygon(x,y,4); - } - public static Polygon getSkinnyPoint(int a, int b) { int[] x = new int[4]; int[] y = new int[4]; @@ -107,37 +99,15 @@ public static Polygon getSkinnyPoint(int a, int b) { x[2] = a; y[2] = b; x[3] = a; - y[3] = b; + y[3] = b; return new Polygon(x,y,4); } - + public static Polygon getSkinnyPoint(long p) { long[] c = hc.point(p); - int a = (int) c[0]; - int b = (int) c[1]; - int[] x = new int[4]; - int[] y = new int[4]; - x[0] = a; - y[0] = b; - x[1] = a; - y[1] = b; - x[2] = a; - y[2] = b; - x[3] = a; - y[3] = b; - return new Polygon(x,y,4); + return getSkinnyPoint((int) c[0], (int) c[1]); } - - public static long getCentroidHilbertIndex(org.locationtech.jts.geom.Polygon polygon) { - if (polygon == null || polygon.isEmpty()) { - throw new IllegalArgumentException("Polygon cannot be null or empty"); - } - org.locationtech.jts.geom.Point centroid = polygon.getCentroid(); - int x = (int) Math.round(centroid.getX()); - int y = (int) Math.round(centroid.getY()); - return hc.index(new long[] {x, y}); - } - + public static long[] getBoundingBoxHilbertIndices(org.locationtech.jts.geom.Polygon polygon) { if (polygon == null || polygon.isEmpty()) { throw new IllegalArgumentException("Polygon cannot be null or empty"); @@ -152,44 +122,8 @@ public static long[] getBoundingBoxHilbertIndices(org.locationtech.jts.geom.Poly long tr = hc.index(new long[] {maxX, maxY}); long br = hc.index(new long[] {maxX, minY}); return new long[] {bl, tl, tr, br}; - } - - public static Ranges Fatten(Ranges rr) { - Iterator i = rr.iterator(); - ArrayList nr = new ArrayList(); - while (i.hasNext()) { - Range r = i.next(); - for (long c = r.low(); c<=r.high();c++) { - long[] p = hc.point(c); - if (!nr.contains(c)) nr.add(c); - p[0] = p[0]+1; - long nn = hc.index(p); - if (!nr.contains(nn)) nr.add(nn); - p[0] = p[0]-1; - p[1] = p[1]+1; - nn = hc.index(p); - if (!nr.contains(nn)) nr.add(nn); - } - } - Collections.sort(nr); - Ranges neo = new Ranges(nr.size()); - Iterator ii = nr.iterator(); - long first; - long last; - while (ii.hasNext()) { - first = ii.next(); - last = first; - while (ii.hasNext()) { - long next = ii.next(); - if ((next-last)==1) { - last = next; - } - } - neo.add(new Range(first,last)); - } - return neo; } - + public static Point NextPoint(Point p, Byte neighbor) { switch (neighbor) { case N -> { return new Point(p.x,p.y-1); } @@ -203,11 +137,6 @@ public static Point NextPoint(Point p, Byte neighbor) { } return null; } - - public static Point GetXY(long p) { - long[] c = hc.point(p); - return new Point((int) c[0],(int) c[1]); - } public static boolean contains(ArrayList rr, int x, int y) { long target = hc.index(new long[] {x,y}); @@ -218,7 +147,7 @@ public static boolean contains(ArrayList rr, int x, int y) { } return false; } - + public static boolean contains(Ranges rr, int x, int y) { long target = hc.index(new long[] {x,y}); for (Range r : rr) { @@ -228,200 +157,7 @@ public static boolean contains(Ranges rr, int x, int y) { } return false; } - - public static BufferedImage GetBI(int px, int py, int width, int height, Ranges ranges) { - BufferedImage bi = new BufferedImage(width,height,BufferedImage.TYPE_INT_ARGB); - Graphics2D g2 = bi.createGraphics(); - g2.setColor(Color.RED); - g2.fillRect(0, 0, width, height); - for (Range pp : ranges) { - for (long k=pp.low(); k<(pp.high()+1); k++) { - long[] point = hc.point(k); - int x = (int)(point[0]- px); - int y = (int)(point[1]- py); - if ((x0)&&(y0)) { - bi.setRGB(x, y, 0xFF0000FF); - } - } - } - return bi; - } - public static Ranges search(int x, int y, int width, int height) { - long[] a = new long[] {x,y}; - long[] b = new long[] {x+width-1,y+height-1}; - return hc.query(a, b); - } - - public static hsPolygon Box(int x, int y, int width, int height) { - return new hsPolygon(search(x,y,width,height)); - } - - public static BufferedImage GetBI(int px, int py, int width, int height, HashMap> ranges, HashMap cvalues, HashMap pvalues) { - BufferedImage bi = new BufferedImage(width,height,BufferedImage.TYPE_INT_ARGB); - Graphics2D g2 = bi.createGraphics(); - g2.setColor(new Color(128,0,128,128)); - g2.fillRect(0, 0, width, height); - for (Integer id : ranges.keySet()) { - float pc = pvalues.get(id); - int classid = cvalues.get(id); - int prob = (int) (pc*255f+0.5); - int color = (0xFF000000)+(classid<<16)+(prob<<8); - ArrayList rs = ranges.get(id); - for (Range pp : rs) { - long len = pp.high()-pp.low()+1; - if ((pp.low()>0)&&(len<((width*height)+1))) { - for (long k=pp.low(); k<(pp.high()+1); k++) { - long[] point = hc.point(k); - int x = (int)(point[0]- px); - int y = (int)(point[1]- py); - if ((x=0)&&(y=0)) { - bi.setRGB(x, y, color); - } - } - } - } - } - return bi; - } - - public static BufferedImage GetBIbyClass(int px, int py, int width, int height, HashMap> ranges, HashMap values) { - BufferedImage bi = new BufferedImage(width,height,BufferedImage.TYPE_INT_ARGB); - Graphics2D g2 = bi.createGraphics(); - g2.setColor(new Color(128,0,128,128)); - g2.fillRect(0, 0, width, height); - for (Integer id : ranges.keySet()) { - int color = values.get(id); - color = (0xFF000000)+(color<<16)+(color<<8)+(color); - ArrayList rs = ranges.get(id); - for (Range pp : rs) { - long len = pp.high()-pp.low()+1; - if ((pp.low()>0)&&(len<((width*height)+1))) { - for (long k=pp.low(); k<(pp.high()+1); k++) { - long[] point = hc.point(k); - int x = (int)(point[0]- px); - int y = (int)(point[1]- py); - if ((x=0)&&(y=0)) { - bi.setRGB(x, y, color); - } - } - } - } - } - return bi; - } - - public static BufferedImage GetBIbyProbability(int px, int py, int width, int height, HashMap> ranges, HashMap values) { - BufferedImage bi = new BufferedImage(width,height,BufferedImage.TYPE_INT_ARGB); - Graphics2D g2 = bi.createGraphics(); - g2.setColor(new Color(128,0,128,128)); - g2.fillRect(0, 0, width, height); - for (Integer id : ranges.keySet()) { - float pc = values.get(id); - int color = (int) (pc*255f+0.5); - color = (0xFF000000)+(color<<16)+(color<<8)+(color); - ArrayList rs = ranges.get(id); - for (Range pp : rs) { - long len = pp.high()-pp.low()+1; - if ((pp.low()>0)&&(len<((width*height)+1))) { - for (long k=pp.low(); k<(pp.high()+1); k++) { - long[] point = hc.point(k); - int x = (int)(point[0]- px); - int y = (int)(point[1]- py); - if ((x=0)&&(y=0)) { - bi.setRGB(x, y, color); - } - } - } - } - } - return bi; - } - - public static Box BoundingBox(Ranges rr) { - long start = System.nanoTime(); - Iterator i = rr.iterator(); - long c = 0; - Box bb = new Box(Integer.MAX_VALUE, Integer.MAX_VALUE, Integer.MIN_VALUE, Integer.MIN_VALUE); - while (i.hasNext()) { - Range r = i.next(); - c = c + (r.high()-r.low()+1); - } - int cc = (int) c; - int[] px = new int[cc]; - int[] py = new int[cc]; - int k = -1; - i = rr.iterator(); - while (i.hasNext()) { - k++; - Range r = i.next(); - for (long a=r.low();a> ranges, HashMap values, HashMap classids) { - logger.debug("Convert to JSON String..."); - long start = System.nanoTime(); - JsonArrayBuilder jab = Json.createArrayBuilder(); - for (Integer id : ranges.keySet()) { - ArrayList rs = ranges.get(id); - Ranges rr = hTools.Tran(rs); - Polygon p = getPolygon(rr); - JsonArrayBuilder cab = Json.createArrayBuilder(); - if (p!=null) { - JsonObjectBuilder job = Json.createObjectBuilder(); - for (int k=0;k i = rr.iterator(); Point top = new Point(Integer.MAX_VALUE,Integer.MAX_VALUE); @@ -442,12 +178,7 @@ public static Point GetUpperLeft(Ranges rr) { } return top; } - - public static hNode DetermineNeighbors(Long p) { - hNode node = new hNode(); - return node; - } - + public static Polygon getPolygon(Ranges rr) { if (rr.size()==1) { Iterator ri = rr.iterator(); @@ -469,7 +200,7 @@ public static Polygon getPolygon(Ranges rr) { do { boolean jumped; do { - jumped = true; + jumped = true; if (inRange(big, cp, N) &&!inRange(big,cp, NW) && (v.getNumVisits(cp)>v.getNumVisits(NextPoint(cp, N)))) { ld = cd; lp.x = cp.x; lp.y = cp.y; cp.y--; cd = N; @@ -511,213 +242,24 @@ public static Polygon getPolygon(Ranges rr) { p.addPoint(cp.x, cp.y); return p; } - - public static String SpaceIT(String c) { - String e = ""; - int units = c.length()/2; - int s; - if ((units*2) Polygon2Hilbert(Polygon p, int scale) { - if (scale>0) { - for (int i=0; i>scale; - p.ypoints[i]=p.ypoints[i]>>scale; - } - return Polygon2Hilbert(p); - } else { - return Polygon2Hilbert(p); - } - } - - public static boolean mergable(Range a, Range b) { - if (a.high()= a.low()) { - return true; - } - return a.high() >= b.low(); - } - public static Range merge(Range a, Range b) { - return new Range(Math.min(a.low(), b.low()),Math.max(a.high(), b.high())); - } - - public static long Area(ArrayList list) { - long area = 0; - for (Range item : list) { - area = area + item.high()-item.low()+1; - } - return area; - } - - public static ArrayList compact(ArrayList list) { - if (list.size()==1) { - return list; - } else { - ArrayList neo = new ArrayList<>(); - while (list.size()>1) { - Range item = list.get(0); - int i = 1; - while ((list.size()>1)&&(i list) { - Iterator i = list.iterator(); - while (i.hasNext()) { - logger.debug("{}", i.next()); - } - } - - public static int len(int x1, int x2, int y1, int y2) { - return (int) Math.sqrt(((x1-x2)*(x1-x2))+((y1-y2)*(y1-y2))); - } - - public static boolean isSquare(Polygon p) { - if (p.npoints!=4) return false; - int s1 = len(p.xpoints[0],p.xpoints[1],p.ypoints[0],p.ypoints[1]); - int s2 = len(p.xpoints[1],p.xpoints[2],p.ypoints[1],p.ypoints[2]); - int s3 = len(p.xpoints[2],p.xpoints[3],p.ypoints[2],p.ypoints[3]); - int s4 = len(p.xpoints[3],p.xpoints[0],p.ypoints[3],p.ypoints[0]); - return (s1==s2)&&(s2==s3)&&(s3==s4); - } - - public static ArrayList Polygon2Hilbert(Polygon p) { - ArrayList rah = new ArrayList(); - Area a = new Area(p); - Rectangle2D r = a.getBounds2D(); - int minx = (int) r.getMinX(); - int maxx = (int) r.getMaxX(); - int miny = (int) r.getMinY(); - int maxy = (int) r.getMaxY(); - ArrayList pp = new ArrayList(); - for (int x=minx;x WKT" prefix; JTS does not + // accept it. Failures raise a catchable exception (with the offending + // input), never a bare java.lang.Error. + String clean = ImageTools.stripCrs(wkt); try { - geom = reader.read(wkt); + geom = reader.read(clean); if (!(geom instanceof org.locationtech.jts.geom.Polygon)) { logger.error("WKT is not a Polygon: {}", geom.getGeometryType()); - throw new Error("WKT is not a Polygon: " + geom.getGeometryType()); + throw new IllegalArgumentException("WKT is not a Polygon: " + geom.getGeometryType()); } return (org.locationtech.jts.geom.Polygon) geom; } catch (ParseException ex) { - throw new Error("Parsing Error!"); - } - } - - public static ArrayList Polygon2Hilbert(String wkt) { - return Polygon2Hilbert(fromWkt(wkt)); - } - - public static ArrayList Polygon2Hilbert(org.locationtech.jts.geom.Polygon p) { - ArrayList rah = new ArrayList(); - GeometryFactory geometryFactory = new GeometryFactory(); - Envelope r = p.getEnvelopeInternal(); - int minx = (int) r.getMinX(); - int maxx = (int) r.getMaxX(); - int miny = (int) r.getMinY(); - int maxy = (int) r.getMaxY(); - ArrayList pp = new ArrayList(); - for (int x=minx;x 100 ? clean.substring(0, 100) + "..." : clean), ex); } } -} \ No newline at end of file +} diff --git a/src/main/java/com/ebremer/halcyon/hilbert/PolygonScaler.java b/src/main/java/com/ebremer/halcyon/hilbert/PolygonScaler.java index e1194aa7..865ecfd7 100644 --- a/src/main/java/com/ebremer/halcyon/hilbert/PolygonScaler.java +++ b/src/main/java/com/ebremer/halcyon/hilbert/PolygonScaler.java @@ -19,23 +19,42 @@ public class PolygonScaler { private static final AffineTransformation half = AffineTransformation.scaleInstance(0.5, 0.5); /** - * Parses WKT and generates a sequence of progressively quarter-area scaled polygons. + * Parses WKT and generates a sequence of progressively quarter-area scaled + * polygons. Returns null (skip this geometry) for any input that cannot be + * turned into a usable polygon - unparseable WKT, a geometry that collapses + * under integer snapping, or one the fixer cannot repair. The policy is + * uniform: one bad geometry is skipped with a warning, never silently and + * never by aborting the (possibly multi-hour) build that contains it. * @param wkt The Well-Known Text string. - * @return Array of Polygons. + * @return Array of Polygons, or null to skip. */ public static Polygon[] toPolygons(String wkt) { Polygon original; try { original = ImageTools.wktToPolygon(wkt); + if (original == null) { + logger.warn("No polygon in WKT; skipping spatial indexing: {}", abbrev(wkt)); + return null; + } original.apply(new IntSnapFilter()); original = snapAndSimplify(original); } catch (Exception ex) { - logger.error("Failed to parse input WKT as Polygon {} {}", wkt, ex.getMessage()); - original = gf.createPolygon(); + logger.warn("Failed to parse WKT as Polygon; skipping spatial indexing: {} ({})", + abbrev(wkt), ex.getMessage()); + return null; + } + if (original == null) { + logger.warn("Polygon collapsed during integer snapping; skipping spatial indexing: {}", + abbrev(wkt)); + return null; } return toPolygons(original); } - + + private static String abbrev(String wkt) { + return (wkt != null && wkt.length() > 200) ? wkt.substring(0, 200) + "..." : wkt; + } + public static Polygon fixPolygon(Polygon polygon) { // Check if already valid if (polygon.isValid()) { @@ -80,7 +99,10 @@ public static Polygon fixPolygon(Polygon polygon) { return largest; } } - throw new IllegalStateException("Unable to fix invalid polygon: " + polygon); + // Describe the polygon by size and extent - embedding the full WKT of a + // possibly-100k-vertex polygon in an exception message helps nobody. + throw new IllegalStateException("Unable to fix invalid polygon (" + + polygon.getNumPoints() + " points, envelope " + polygon.getEnvelopeInternal() + ")"); } /** @@ -90,14 +112,31 @@ public static Polygon fixPolygon(Polygon polygon) { * @return Array of Polygons (Levels). */ public static Polygon[] toPolygons(Polygon original) { + if (original == null || original.isEmpty()) { + return null; + } + // Snap/clean here (not only in the String entry) so callers passing raw + // JTS parts - e.g. individual MULTIPOLYGON members - get the same + // integer-grid treatment. + original.apply(new IntSnapFilter()); + original = snapAndSimplify(original); if (original == null) { + logger.warn("Polygon collapsed during integer snapping; skipping spatial indexing"); return null; } List scaledPolygons = new ArrayList<>(); - Polygon current = fixPolygon(original); + Polygon current; + try { + current = fixPolygon(original); + } catch (RuntimeException ex) { + // One unfixable geometry must not abort the whole build: skip its + // spatial indexing - the source quad itself is still stored. + logger.warn("Skipping spatial indexing of unfixable polygon: {}", ex.getMessage()); + return null; + } if (current == null || !current.isValid()) { return new Polygon[0]; - } + } int maxIterations = 20; // Safety limit to prevent infinite loops int iterations = 0; while (iterations < maxIterations) { @@ -166,14 +205,17 @@ public static List getGridCells(Polygon poly, int numscales) { /** * Determines which grid cells the polygon intersects at a specific resolution scale. - * Logic: - * Scale 0: Tile covers 256 coordinates. - * Scale 1: Tile covers 512 coordinates (1/2 resolution). - * Scale 2: Tile covers 1024 coordinates (1/4 resolution). - * Formula: EffectiveSize = 256 * 2^scale - * + *

    + * The polygon is expected in the coordinates of ITS OWN scale level (already + * divided by 2^scale, as produced by {@link #toPolygons}), and every level's + * tiles are {@code GRIDTILESIZE} units on a side in those level coordinates - + * exactly the grid the writer persists (see generateGridURNs). The old + * formula multiplied the tile size by 2^scale ON TOP of the already-scaled + * coordinates, so for scale >= 1 the computed cells disagreed with every + * stored tile graph URN. + * * @param cells List to accumulate intersecting grid cells - * @param poly The input JTS Polygon (in base scale 0 coordinates) + * @param poly The input JTS Polygon, in scale-level coordinates * @param scale The pyramid scale level (0, 1, 2, 3...) * @return List of GridCell indices at the requested scale */ @@ -181,8 +223,8 @@ public static List getGridCells(List cells, Polygon poly, sh if (poly == null || poly.isEmpty() || scale < 0) { return cells; } - - double effectiveTileSize = GRIDTILESIZE * (1 << scale); + + double effectiveTileSize = GRIDTILESIZE; Envelope env = poly.getEnvelopeInternal(); int minGridX = (int) Math.floor(env.getMinX() / effectiveTileSize); @@ -238,40 +280,62 @@ public boolean isGeometryChanged() { } private static Polygon removeDuplicateAndCollinearVertices(Polygon poly) { - Coordinate[] coords = poly.getExteriorRing().getCoordinates(); - List cleaned = new ArrayList<>(); + // Clean every ring, not just the shell: interior rings (holes - lumens in + // pathology annotations) must survive cleanup. A hole that collapses under + // snapping is dropped while the outer shape survives. + Coordinate[] shellCoords = cleanRing(poly.getExteriorRing().getCoordinates()); + if (shellCoords == null) { + return null; + } + List holes = new ArrayList<>(); + for (int h = 0; h < poly.getNumInteriorRing(); h++) { + Coordinate[] ring = cleanRing(poly.getInteriorRingN(h).getCoordinates()); + if (ring != null) { + try { + holes.add(gf.createLinearRing(ring)); + } catch (Exception e) { + // collapsed/invalid hole: drop it, keep the polygon + } + } + } + try { + LinearRing shell = gf.createLinearRing(shellCoords); + return gf.createPolygon(shell, holes.toArray(LinearRing[]::new)); + } catch (Exception e) { + return null; + } + } + + /** De-duplicates and de-collinearizes one ring; null when it collapses (<4 points). */ + private static Coordinate[] cleanRing(Coordinate[] coords) { + List cleaned = new ArrayList<>(); // Remove duplicate consecutive vertices for (Coordinate coord : coords) { if (cleaned.isEmpty() || !coord.equals2D(cleaned.get(cleaned.size() - 1))) { cleaned.add(coord); } - } + } // Remove collinear points int i = 0; while (i < cleaned.size() - 2) { Coordinate a = cleaned.get(i); Coordinate b = cleaned.get(i + 1); Coordinate c = cleaned.get(i + 2); - + // Check if b is collinear with a and c if (Orientation.index(a, b, c) == 0) { cleaned.remove(i + 1); } else { i++; } - } + } // Ensure ring closure if (!cleaned.isEmpty() && !cleaned.get(0).equals2D(cleaned.get(cleaned.size() - 1))) { cleaned.add(new Coordinate(cleaned.get(0))); - } + } if (cleaned.size() < 4) { return null; - } - try { - LinearRing shell = gf.createLinearRing(cleaned.toArray(new Coordinate[0])); - return gf.createPolygon(shell); - } catch (Exception e) { - return null; } + return cleaned.toArray(new Coordinate[0]); } } diff --git a/src/main/java/com/ebremer/halcyon/hilbert/WKTDatatype.java b/src/main/java/com/ebremer/halcyon/hilbert/WKTDatatype.java index 9543992c..44a8621b 100644 --- a/src/main/java/com/ebremer/halcyon/hilbert/WKTDatatype.java +++ b/src/main/java/com/ebremer/halcyon/hilbert/WKTDatatype.java @@ -2,7 +2,6 @@ import org.apache.jena.datatypes.BaseDatatype; import org.apache.jena.datatypes.DatatypeFormatException; -import org.apache.jena.graph.impl.LiteralLabel; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.io.ParseException; import org.locationtech.jts.io.WKTReader; @@ -10,9 +9,16 @@ public class WKTDatatype extends BaseDatatype { - public static final String URI = "http://www.opengis.net/ont/geosparql#wktLiteral"; + public static final String URI = "http://www.opengis.net/ont/geosparql#wktLiteral"; public static final WKTDatatype INSTANCE = new WKTDatatype(); + static { + // Self-register on first touch: without this, the parse/equality machinery + // below was inert - TypeMapper handed out a generic datatype for + // geo:wktLiteral and nothing ever consulted this class. + org.apache.jena.datatypes.TypeMapper.getInstance().registerDatatype(INSTANCE); + } + private WKTDatatype() { super(URI); } @@ -21,7 +27,7 @@ private WKTDatatype() { * Parse the Lexical Form (String) into a Java Object (JTS Geometry). * GeoSPARQL literals can look like: " POINT(10 20)" * @param lexicalForm - * @return + * @return */ @Override public Object parse(String lexicalForm) throws DatatypeFormatException { @@ -38,10 +44,16 @@ public Object parse(String lexicalForm) throws DatatypeFormatException { } WKTReader reader = new WKTReader(); return reader.read(cleanWkt); - } catch (ParseException e) { + } catch (ParseException | RuntimeException e) { + // JTS throws IllegalArgumentException - not ParseException - for + // structurally invalid geometry (e.g. a two-point ring), and RIOT + // validates every wktLiteral through this method now that the + // datatype is registered. Anything escaping here turns one bad + // literal into a fatal abort of the whole parse; wrapping it makes + // it the ill-typed-literal warning it should be. throw new DatatypeFormatException( - lexicalForm, - this, + lexicalForm, + this, "Invalid WKT format: " + e.getMessage() ); } @@ -76,24 +88,10 @@ public boolean isValid(String lexicalForm) { } } - /** - * Comparison logic for SPARQL FILTERs - * @param value1 - * @param value2 - * @return - */ - @Override - public boolean isEqual(LiteralLabel value1, LiteralLabel value2) { - // Parse both to Geometries and check equality (topological equality) - if (value1.getDatatype() == this && value2.getDatatype() == this) { - try { - Geometry g1 = (Geometry) value1.getValue(); - Geometry g2 = (Geometry) value2.getValue(); - return g1.equals(g2); // JTS topological equality - } catch (Exception e) { - return false; - } - } - return super.isEqual(value1, value2); - } + // NOTE: no isEqual override. An earlier version compared wktLiterals by JTS + // topological equality, which changes RDF *term* equality - sameTerm, HashSet + // membership, and the writer's dictionary deduplication would silently + // collapse lexically distinct geometries (and pay a JTS parse per equality + // check). Term equality stays lexical (the BaseDatatype default); geometric + // comparison belongs in filter functions like geof:sfIntersects. } diff --git a/src/main/java/com/ebremer/halcyon/hilbert/eRange.java b/src/main/java/com/ebremer/halcyon/hilbert/eRange.java deleted file mode 100644 index 5ac0fd29..00000000 --- a/src/main/java/com/ebremer/halcyon/hilbert/eRange.java +++ /dev/null @@ -1,74 +0,0 @@ -package com.ebremer.halcyon.hilbert; - -/** - * - * @author erich - */ -public class eRange implements Comparable { - private final long low; - private final long high; - - public eRange(long low, long high) { - this.low = Math.min(low, high); - this.high = Math.max(low, high); - } - - @Override - public int compareTo(eRange candidate) { - return (this.low < candidate.low() ? -1 : - (this.low == candidate.low() ? 0 : 1)); - } - - public static eRange create(long low, long high) { - return new eRange(low, high); - } - - public static eRange create(long value) { - return new eRange(value, value); - } - - public long low() { - return low; - } - - public long high() { - return high; - } - - public boolean contains(long value) { - return low <= value && value <= high; - } - - @Override - public String toString() { - return "Range [low=" + low + ", high=" + high + "]"; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + (int) (high ^ (high >>> 32)); - result = prime * result + (int) (low ^ (low >>> 32)); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - eRange other = (eRange) obj; - if (high != other.high) - return false; - return low == other.low; - } - - public eRange join(eRange range) { - return eRange.create(Math.min(low, range.low), Math.max(high, range.high)); - } - -} diff --git a/src/main/java/com/ebremer/halcyon/hilbert/hNode.java b/src/main/java/com/ebremer/halcyon/hilbert/hNode.java deleted file mode 100644 index 95ccb3ed..00000000 --- a/src/main/java/com/ebremer/halcyon/hilbert/hNode.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.ebremer.halcyon.hilbert; - -/** - * - * @author erich - */ -public class hNode { - public hNode left = null; - public hNode right = null; - -} diff --git a/src/main/java/com/ebremer/halcyon/hilbert/hTools.java b/src/main/java/com/ebremer/halcyon/hilbert/hTools.java deleted file mode 100644 index 65258159..00000000 --- a/src/main/java/com/ebremer/halcyon/hilbert/hTools.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.ebremer.halcyon.hilbert; - -import java.util.ArrayList; -import java.util.Collections; -import org.davidmoten.hilbert.Range; -import org.davidmoten.hilbert.Ranges; - -/** - * - * @author erich - */ -public class hTools { - - public static Ranges Tran(ArrayList src) { - ArrayList tmp = new ArrayList<>(); - for (Range r : src) { - tmp.add(new eRange(r.low(),r.high())); - } - Collections.sort(tmp); - Ranges rr = new Ranges(tmp.size()); - for (eRange ee : tmp) { - rr.add(ee.low(),ee.high()); - } - return rr; - } -} diff --git a/src/main/java/com/ebremer/halcyon/hilbert/hsPolygon.java b/src/main/java/com/ebremer/halcyon/hilbert/hsPolygon.java deleted file mode 100644 index 3f36eb08..00000000 --- a/src/main/java/com/ebremer/halcyon/hilbert/hsPolygon.java +++ /dev/null @@ -1,41 +0,0 @@ -package com.ebremer.halcyon.hilbert; - -import java.util.List; -import org.davidmoten.hilbert.Range; -import org.davidmoten.hilbert.Ranges; - -/** - * - * @author erich - */ -public class hsPolygon { - List ranges; - private long min; - private long max; - - public hsPolygon(Ranges r) { - ranges = r.toList(); - } - - private long getMin() { - return min; - } - - private long getMax() { - return max; - } - - private void FindMixMax() { - min = Long.MAX_VALUE; - max = Long.MIN_VALUE; - for (Range r : ranges) { - min = Math.min(min,r.low()); - max = Math.max(max,r.high()); - } - } - - public List getRanges() { - return ranges; - } - -} diff --git a/src/main/resources/META-INF/native-image/resource-config.json b/src/main/resources/META-INF/native-image/resource-config.json index 23d8e113..c78a3056 100644 --- a/src/main/resources/META-INF/native-image/resource-config.json +++ b/src/main/resources/META-INF/native-image/resource-config.json @@ -2,7 +2,9 @@ "resources": { "includes": [ { "pattern": "META-INF/sparql/.*" }, - { "pattern": "log4j2.yaml" }, + { "pattern": "beakgraph.png" }, + { "pattern": "beakgraph-version.properties" }, + { "pattern": "log4j2.yml" }, { "pattern": "org/apache/jena/ext/xerces/impl/msg/.*" }, { "pattern": "META-INF/services/.*" } ] diff --git a/src/main/resources/META-INF/sparql/beakgraph.png b/src/main/resources/META-INF/sparql/beakgraph.png deleted file mode 100644 index 57e7a442..00000000 Binary files a/src/main/resources/META-INF/sparql/beakgraph.png and /dev/null differ diff --git a/src/main/resources/log4j2.yml b/src/main/resources/log4j2.yml index 821b8f6f..7ab3d18f 100644 --- a/src/main/resources/log4j2.yml +++ b/src/main/resources/log4j2.yml @@ -23,6 +23,15 @@ Configuration: - ref: FileAppender Logger: + # BeakGraph's own progress reporting (quads loaded, dictionary/index build, + # skipped geometries) lives at INFO/WARN - the root stays at ERROR so + # third-party libraries remain quiet. + - name: com.ebremer.beakgraph + level: info + additivity: false + AppenderRef: + - ref: STDOUT + - ref: FileAppender - name: org.apache.jena.fuseki level: error additivity: false diff --git a/src/test/java/com/ebremer/beakgraph/BGIteratorConcreteSubjectTest.java b/src/test/java/com/ebremer/beakgraph/BGIteratorConcreteSubjectTest.java index 9f6fb858..890e56f2 100644 --- a/src/test/java/com/ebremer/beakgraph/BGIteratorConcreteSubjectTest.java +++ b/src/test/java/com/ebremer/beakgraph/BGIteratorConcreteSubjectTest.java @@ -35,6 +35,7 @@ class BGIteratorConcreteSubjectTest { @TempDir static Path dir; + static BeakGraph bg; static Dataset ds; @BeforeAll @@ -46,7 +47,14 @@ static void buildAndOpen() throws Exception { .setSource(ttl).setDestination(h5) .setSpatial(false).setFeatures(false) .build().write(); - ds = new BeakGraph(new HDF5Reader(h5)).getDataset(); + bg = new BeakGraph(new HDF5Reader(h5)); + ds = bg.getDataset(); + } + + @org.junit.jupiter.api.AfterAll + static void closeReader() { + // Release the mapped file: a leaked reader makes @TempDir cleanup flaky on Windows. + if (bg != null) bg.close(); } private static int count(String iri) { diff --git a/src/test/java/com/ebremer/beakgraph/BadPolygonResilienceTest.java b/src/test/java/com/ebremer/beakgraph/BadPolygonResilienceTest.java new file mode 100644 index 00000000..f2a87a0e --- /dev/null +++ b/src/test/java/com/ebremer/beakgraph/BadPolygonResilienceTest.java @@ -0,0 +1,139 @@ +package com.ebremer.beakgraph; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.ebremer.beakgraph.core.BeakGraph; +import com.ebremer.beakgraph.hdf5.readers.HDF5Reader; +import com.ebremer.beakgraph.hdf5.writers.HDF5Writer; +import com.ebremer.halcyon.hilbert.PolygonScaler; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import org.apache.jena.query.Dataset; +import org.apache.jena.query.QueryExecution; +import org.apache.jena.query.QueryFactory; +import org.apache.jena.query.ResultSet; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Regression tests for the bad-geometry policy: one problematic polygon must + * never abort a (possibly multi-hour) spatial build, and the failure modes are + * uniform - parse failure, snap-collapse and unfixable geometry all skip that + * geometry's spatial indexing (with a warning) while the source quad itself is + * still stored and every other geometry is indexed normally. + */ +class BadPolygonResilienceTest { + + // ex:degen is structurally invalid WKT (a two-point ring): JTS throws + // IllegalArgumentException for it, not ParseException - real pathology data + // contains these, and one of them must not abort the parse of a whole file. + private static final String TTL = """ + @prefix ex: . + @prefix geo: . + ex:good geo:asWKT "POLYGON((0 0,64 0,64 64,0 64,0 0))"^^geo:wktLiteral . + ex:tiny geo:asWKT "POLYGON((0.1 0.1,0.4 0.1,0.4 0.4,0.1 0.4,0.1 0.1))"^^geo:wktLiteral . + ex:bow geo:asWKT "POLYGON((0 0,10 10,10 0,0 10,0 0))"^^geo:wktLiteral . + ex:degen geo:asWKT "POLYGON((0 0,1 1))"^^geo:wktLiteral . + """; + + @TempDir + static Path dir; + static BeakGraph bg; + static Dataset ds; + + @BeforeAll + static void buildAndOpen() throws Exception { + File ttl = dir.resolve("badpoly.ttl").toFile(); + File h5 = dir.resolve("badpoly.ttl.h5").toFile(); + Files.write(ttl.toPath(), TTL.getBytes(StandardCharsets.UTF_8)); + // The collapsing and self-intersecting geometries must not abort this. + HDF5Writer.Builder() + .setSource(ttl).setDestination(h5) + .setSpatial(true).setFeatures(false) + .build().write(); + bg = new BeakGraph(new HDF5Reader(h5)); + ds = bg.getDataset(); + } + + @AfterAll + static void closeReader() { + if (bg != null) bg.close(); + } + + private static int count(String where) { + try (QueryExecution qe = QueryExecution.dataset(ds).query(QueryFactory.create( + "PREFIX ex: SELECT * WHERE { " + where + " }")).build()) { + ResultSet rs = qe.execSelect(); + int n = 0; + while (rs.hasNext()) { rs.next(); n++; } + return n; + } + } + + @Test + void goodGeometryIsIndexedDespiteBadNeighbors() { + assertTrue(count("GRAPH <" + Params.SPATIALSTRING + "> { ex:good ?p ?o }") > 0, + "the well-formed geometry must be spatially indexed"); + } + + @Test + void collapsedGeometryHasNoScaledPyramidButBuildSurvives() { + // Snapping collapses the sub-pixel polygon, so no scaled-WKT pyramid is + // produced for it (and - critically - the build did not abort). Its + // recall-safe index cells ARE still written: the geometry stays findable, + // and sfIntersects verification uses the precise original WKT. + assertEquals(0, count("GRAPH <" + Params.SPATIALSTRING + "> { ex:tiny ?o }")); + } + + @Test + void sourceQuadsSurviveRegardlessOfGeometryFate() { + assertEquals(4, count("?s ?o")); + } + + @Test + void structurallyInvalidWktIsKeptAsATermAndSkippedSpatially() { + // The two-point ring survives as an RDF term (lexical + datatype intact)... + assertEquals(1, count("ex:degen " + + "\"POLYGON((0 0,1 1))\"^^")); + // ...but contributes nothing to the spatial index. + assertEquals(0, count("GRAPH <" + Params.SPATIALSTRING + "> { ex:degen ?p ?o }")); + } + + @Test + void selfIntersectingStoredGeometryIsReturnedBySpatialQueries() { + // The bowtie is deliberately indexed at build time; the sfIntersects + // verification stage must repair it rather than throw (which made Jena + // drop the row - an indexed geometry no query could ever return). + // Region inside the bowtie's left lobe; it also overlaps ex:good. + assertEquals(1, count("?f ?w . " + + "FILTER(?f = ex:bow) " + + "FILTER((?w, " + + "\"POLYGON((1 4,2 4,2 6,1 6,1 4))\"^^))"), + "the self-intersecting geometry must be returned by a query over its real area"); + } + + @Test + void invalidQueryRegionStillAnswersInsteadOfEmptying() { + // A self-intersecting QUERY region must be repaired too, not silently + // turned into zero rows for every candidate. + assertEquals(1, count("ex:good ?w . " + + "FILTER((?w, " + + "\"POLYGON((0 0,20 20,20 0,0 20,0 0))\"^^))")); + } + + @Test + void unfixablePolygonReturnsNullInsteadOfThrowing() { + // Direct unit check of the policy seam: a geometry the fixer cannot repair + // must come back as "skip" (null), never as an exception that would abort + // the build. An empty polygon exercises the null/empty guard; the + // collapsing TTL geometry above exercises the snap-collapse path end-to-end. + assertNull(PolygonScaler.toPolygons( + new org.locationtech.jts.geom.GeometryFactory().createPolygon())); + } +} diff --git a/src/test/java/com/ebremer/beakgraph/BitPackedUnSignedLongBufferTest.java b/src/test/java/com/ebremer/beakgraph/BitPackedUnSignedLongBufferTest.java index 78196422..235704c0 100644 --- a/src/test/java/com/ebremer/beakgraph/BitPackedUnSignedLongBufferTest.java +++ b/src/test/java/com/ebremer/beakgraph/BitPackedUnSignedLongBufferTest.java @@ -35,6 +35,34 @@ void roundTripsAcrossSupportedWidths() { } } + @Test + void streamFailsLoudlyOnTruncatedBuffer() { + // A buffer shorter than its declared entry count is corrupt. The stream + // used to emit silent garbage (negative-shift artifacts) where the + // sequential reader threw; both must now fail loudly. + java.nio.ByteBuffer twoBytes = java.nio.ByteBuffer.allocate(2); + BitPackedUnSignedLongBuffer truncated = new BitPackedUnSignedLongBuffer(null, twoBytes, 10, 16); + assertThrows(java.nio.BufferUnderflowException.class, () -> truncated.stream().toArray()); + } + + @Test + void rejectsValuesThatWouldBeTruncated() { + // A value wider than the buffer's bit width used to be silently masked, + // corrupting the dictionary far from the cause. It must be rejected. + BitPackedUnSignedLongBuffer narrow = new BitPackedUnSignedLongBuffer(null, null, 0, 8); + assertDoesNotThrow(() -> narrow.writeLong(255)); + assertThrows(IllegalArgumentException.class, () -> narrow.writeLong(256)); + assertThrows(IllegalArgumentException.class, () -> narrow.writeLong(-1)); + assertThrows(IllegalArgumentException.class, () -> narrow.writeInteger(256)); + + // Widths 32 and 64 legitimately carry full two's-complement patterns + // (the int/long literal buffers): negatives must stay writable there. + BitPackedUnSignedLongBuffer w32 = new BitPackedUnSignedLongBuffer(null, null, 0, 32); + assertDoesNotThrow(() -> w32.writeInteger(-5)); + BitPackedUnSignedLongBuffer w64 = new BitPackedUnSignedLongBuffer(null, null, 0, 64); + assertDoesNotThrow(() -> w64.writeLong(Long.MIN_VALUE)); + } + @Test void rejectsUnsupportedBitWidths() { // 58..63 would overflow the 64-bit accumulator (value + <=7-bit sub-byte offset) and silently diff --git a/src/test/java/com/ebremer/beakgraph/DefaultGraphSentinelTest.java b/src/test/java/com/ebremer/beakgraph/DefaultGraphSentinelTest.java new file mode 100644 index 00000000..801289a7 --- /dev/null +++ b/src/test/java/com/ebremer/beakgraph/DefaultGraphSentinelTest.java @@ -0,0 +1,84 @@ +package com.ebremer.beakgraph; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.ebremer.beakgraph.core.BeakGraph; +import com.ebremer.beakgraph.core.lib.NodeComparator; +import com.ebremer.beakgraph.hdf5.readers.HDF5Reader; +import com.ebremer.beakgraph.hdf5.writers.HDF5Writer; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import org.apache.jena.graph.Node; +import org.apache.jena.graph.NodeFactory; +import org.apache.jena.query.Dataset; +import org.apache.jena.query.QueryExecution; +import org.apache.jena.query.QueryFactory; +import org.apache.jena.query.ResultSet; +import org.apache.jena.sparql.core.Quad; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * The two ARQ default-graph sentinel URIs are DISTINCT RDF terms. The + * comparator ranked both "default" and returned 0 for the pair, so a source + * that used urn:x-arq:DefaultGraphNode as a data term collapsed it onto the + * same dictionary id as urn:x-arq:DefaultGraph - queries then returned the + * wrong term or nothing. + */ +class DefaultGraphSentinelTest { + + @Test + void sentinelTermsAreDistinctAndDefaultGraphSortsFirst() { + NodeComparator cmp = NodeComparator.INSTANCE; + Node dg = Quad.defaultGraphIRI; + Node dgn = Quad.defaultGraphNodeGenerated; + assertTrue(cmp.compare(dg, dgn) != 0, "distinct sentinel terms must not compare equal"); + assertTrue(cmp.compare(dg, dgn) < 0, "urn:x-arq:DefaultGraph must keep the lowest rank"); + assertEquals(-Integer.signum(cmp.compare(dgn, dg)), Integer.signum(cmp.compare(dg, dgn))); + // Both sentinels still rank before ordinary terms. + Node uri = NodeFactory.createURI("http://ex.org/a"); + assertTrue(cmp.compare(dg, uri) < 0); + assertTrue(cmp.compare(dgn, uri) < 0); + assertEquals(0, cmp.compare(dg, dg)); + assertEquals(0, cmp.compare(dgn, dgn)); + } + + @TempDir + Path dir; + + @Test + void sentinelUriAsDataTermRoundTrips() throws Exception { + String ttl = """ + @prefix ex: . + ex:p . + ex:q "v" . + ex:normal ex:p ex:other . + """; + File src = dir.resolve("sentinel.ttl").toFile(); + File h5 = dir.resolve("sentinel.ttl.h5").toFile(); + Files.write(src.toPath(), ttl.getBytes(StandardCharsets.UTF_8)); + HDF5Writer.Builder().setSource(src).setDestination(h5) + .setSpatial(false).setFeatures(false).build().write(); + + try (BeakGraph bg = new BeakGraph(new HDF5Reader(h5))) { + Dataset ds = bg.getDataset(); + assertEquals(1, count(ds, "SELECT ?o WHERE { ex:p ?o }"), + "the sentinel-URI subject must be findable as its own term"); + assertEquals(1, count(ds, "SELECT ?s WHERE { ?s ex:q \"v\" }")); + assertEquals(3, count(ds, "SELECT * WHERE { ?s ?p ?o }")); + } + } + + private static int count(Dataset ds, String query) { + try (QueryExecution qe = QueryExecution.dataset(ds) + .query(QueryFactory.create("PREFIX ex: " + query)).build()) { + ResultSet rs = qe.execSelect(); + int n = 0; + while (rs.hasNext()) { rs.next(); n++; } + return n; + } + } +} diff --git a/src/test/java/com/ebremer/beakgraph/DistinctCorrectnessTest.java b/src/test/java/com/ebremer/beakgraph/DistinctCorrectnessTest.java new file mode 100644 index 00000000..80143943 --- /dev/null +++ b/src/test/java/com/ebremer/beakgraph/DistinctCorrectnessTest.java @@ -0,0 +1,116 @@ +package com.ebremer.beakgraph; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.ebremer.beakgraph.core.BeakGraph; +import com.ebremer.beakgraph.hdf5.readers.HDF5Reader; +import com.ebremer.beakgraph.hdf5.writers.HDF5Writer; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.Set; +import org.apache.jena.query.Dataset; +import org.apache.jena.query.QueryExecution; +import org.apache.jena.query.QueryFactory; +import org.apache.jena.query.ResultSet; +import org.apache.jena.rdf.model.RDFNode; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Regression tests for C2: SELECT DISTINCT over a single all-variable pattern + * previously streamed the file-global dictionary id lists instead of executing + * the query. Those lists span every named graph - including the always-written + * VoID metadata graph - so default-graph queries over-reported terms that have + * no default-graph triple, any FILTER around the pattern was silently dropped, + * and DISTINCT ?g included the default graph. DISTINCT must answer exactly + * what a normal execution answers. + */ +class DistinctCorrectnessTest { + + private static final String TTL = """ + @prefix ex: . + ex:s1 ex:p ex:o1 . + ex:s2 ex:q ex:o2 . + ex:s1 ex:q "lit" . + """; + + private static final String PREFIX = "PREFIX ex: "; + + @TempDir + static Path dir; + static BeakGraph bg; + static Dataset ds; + + @BeforeAll + static void buildAndOpen() throws Exception { + File ttl = dir.resolve("distinct.ttl").toFile(); + File h5 = dir.resolve("distinct.ttl.h5").toFile(); + Files.write(ttl.toPath(), TTL.getBytes(StandardCharsets.UTF_8)); + HDF5Writer.Builder() + .setSource(ttl).setDestination(h5) + .setSpatial(false).setFeatures(false) + .build().write(); + bg = new BeakGraph(new HDF5Reader(h5)); + ds = bg.getDataset(); + } + + @AfterAll + static void closeReader() { + if (bg != null) bg.close(); + } + + /** URIs as-is; literals as their lexical form. */ + private static Set select(String var, String query) { + Set values = new HashSet<>(); + try (QueryExecution qe = QueryExecution.dataset(ds) + .query(QueryFactory.create(PREFIX + query)).build()) { + ResultSet rs = qe.execSelect(); + while (rs.hasNext()) { + RDFNode n = rs.next().get(var); + if (n == null) continue; + values.add(n.isLiteral() ? n.asLiteral().getLexicalForm() : n.asResource().getURI()); + } + } + return values; + } + + @Test + void distinctPredicatesAreDefaultGraphOnly() { + // The VoID metadata graph contributes rdf:type, void:*, sd:* predicates - + // none of which occur in any default-graph triple. + assertEquals(Set.of("http://ex.org/p", "http://ex.org/q"), + select("p", "SELECT DISTINCT ?p WHERE { ?s ?p ?o }")); + } + + @Test + void distinctSubjectsAreDefaultGraphOnly() { + assertEquals(Set.of("http://ex.org/s1", "http://ex.org/s2"), + select("s", "SELECT DISTINCT ?s WHERE { ?s ?p ?o }")); + } + + @Test + void distinctObjectsAreDefaultGraphOnly() { + assertEquals(Set.of("http://ex.org/o1", "http://ex.org/o2", "lit"), + select("o", "SELECT DISTINCT ?o WHERE { ?s ?p ?o }")); + } + + @Test + void distinctRespectsFilters() { + assertEquals(Set.of("http://ex.org/q"), + select("p", "SELECT DISTINCT ?p WHERE { ?s ?p ?o FILTER(?p != ex:p) }")); + assertEquals(Set.of("http://ex.org/s1"), + select("s", "SELECT DISTINCT ?s WHERE { ?s ?p ?o FILTER(?p = ex:p) }")); + } + + @Test + void distinctGraphsExcludeDefaultGraph() { + // SPARQL: GRAPH ?g ranges over named graphs only. + assertEquals(Set.of(Params.VOIDSTRING), + select("g", "SELECT DISTINCT ?g WHERE { GRAPH ?g { ?s ?p ?o } }")); + } +} diff --git a/src/test/java/com/ebremer/beakgraph/DualRoleUriTest.java b/src/test/java/com/ebremer/beakgraph/DualRoleUriTest.java index c70f6641..5c01da3b 100644 --- a/src/test/java/com/ebremer/beakgraph/DualRoleUriTest.java +++ b/src/test/java/com/ebremer/beakgraph/DualRoleUriTest.java @@ -44,6 +44,7 @@ class DualRoleUriTest { @TempDir static Path dir; + static BeakGraph bg; static Dataset ds; static File h5; @@ -53,7 +54,14 @@ static void build() throws Exception { h5 = dir.resolve("dual.ttl.h5").toFile(); Files.write(ttl.toPath(), TTL.getBytes(StandardCharsets.UTF_8)); HDF5Writer.Builder().setSource(ttl).setDestination(h5).setSpatial(false).setFeatures(false).build().write(); - ds = new BeakGraph(new HDF5Reader(h5)).getDataset(); + bg = new BeakGraph(new HDF5Reader(h5)); + ds = bg.getDataset(); + } + + @org.junit.jupiter.api.AfterAll + static void closeReader() { + // Release the mapped file: a leaked reader makes @TempDir cleanup flaky on Windows. + if (bg != null) bg.close(); } private static String one(String var, String query) { diff --git a/src/test/java/com/ebremer/beakgraph/FeatureGenerationResilienceTest.java b/src/test/java/com/ebremer/beakgraph/FeatureGenerationResilienceTest.java new file mode 100644 index 00000000..ff5ac868 --- /dev/null +++ b/src/test/java/com/ebremer/beakgraph/FeatureGenerationResilienceTest.java @@ -0,0 +1,97 @@ +package com.ebremer.beakgraph; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.ebremer.beakgraph.core.BeakGraph; +import com.ebremer.beakgraph.hdf5.readers.HDF5Reader; +import com.ebremer.beakgraph.hdf5.writers.HDF5Writer; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import org.apache.jena.query.Dataset; +import org.apache.jena.query.QueryExecution; +import org.apache.jena.query.QueryFactory; +import org.apache.jena.query.ResultSet; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Regression test for the FEATURES pipeline's bad-geometry policy: a + * structurally invalid WKT (two-point ring - JTS throws + * IllegalArgumentException, not ParseException) must not abort the build + * through MajorMinor/Gen2DFeatures. One geometry's features are skipped with a + * warning; every other geometry still gets its features. + */ +class FeatureGenerationResilienceTest { + + private static final String TTL = """ + @prefix ex: . + @prefix geo: . + ex:good geo:asWKT "POLYGON((0 0,64 0,64 32,0 32,0 0))"^^geo:wktLiteral . + ex:degen geo:asWKT "POLYGON((0 0,1 1))"^^geo:wktLiteral . + ex:crs geo:asWKT " POLYGON((200 200,264 200,264 232,200 232,200 200))"^^geo:wktLiteral . + """; + + @TempDir + static Path dir; + static BeakGraph bg; + static Dataset ds; + + @BeforeAll + static void buildAndOpen() throws Exception { + File ttl = dir.resolve("features.ttl").toFile(); + File h5 = dir.resolve("features.ttl.h5").toFile(); + Files.write(ttl.toPath(), TTL.getBytes(StandardCharsets.UTF_8)); + // Pre-fix this aborted: MajorMinor.add let the IllegalArgumentException + // escape into the spatial task's Future, failing the whole write. + HDF5Writer.Builder().setSource(ttl).setDestination(h5) + .setSpatial(true).setFeatures(true).build().write(); + bg = new BeakGraph(new HDF5Reader(h5)); + ds = bg.getDataset(); + } + + @AfterAll + static void closeReader() { + if (bg != null) bg.close(); + } + + private static int count(String where) { + try (QueryExecution qe = QueryExecution.dataset(ds).query(QueryFactory.create( + "PREFIX ex: PREFIX hal: SELECT * WHERE { " + where + " }")).build()) { + ResultSet rs = qe.execSelect(); + int n = 0; + while (rs.hasNext()) { rs.next(); n++; } + return n; + } + } + + @Test + void goodGeometryGetsFeatures() { + assertEquals(1, count("ex:good hal:centroid ?c"), + "the well-formed geometry must get its derived features"); + } + + @Test + void degenerateGeometryIsSkippedWithoutFeatures() { + assertEquals(0, count("ex:degen hal:centroid ?c")); + } + + @Test + void crsPrefixedGeometryGetsFeatures() { + // The GeoSPARQL-standard " WKT" form: the spatial indexing path + // stripped the prefix but feature generation re-read the raw lexical, so + // JTS threw and CRS-prefixed geometries silently got no features. + assertEquals(1, count("ex:crs hal:centroid ?c"), + "a CRS-prefixed geometry must get the same derived features as a plain one"); + } + + @Test + void bothSourceQuadsSurvive() { + assertTrue(count("ex:good ?p ?o") >= 1); + assertTrue(count("ex:degen ?p ?o") >= 1); + } +} diff --git a/src/test/java/com/ebremer/beakgraph/FeatureMathTest.java b/src/test/java/com/ebremer/beakgraph/FeatureMathTest.java new file mode 100644 index 00000000..93d2709a --- /dev/null +++ b/src/test/java/com/ebremer/beakgraph/FeatureMathTest.java @@ -0,0 +1,49 @@ +package com.ebremer.beakgraph; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.ebremer.beakgraph.features.ShapeAnalysis; +import com.ebremer.halcyon.geometry.Point; +import com.ebremer.halcyon.geometry.Vector2D; +import java.awt.image.BufferedImage; +import org.junit.jupiter.api.Test; + +/** + * Regression tests for feature-math defects: + *

      + *
    • Vector2D.Magnitude(Point) computed sqrt(x*x + y+y) - addition instead + * of multiplication - and overflowed int for coordinates above ~46340.
      • + *
    • ShapeAnalysis.isEdge read the four neighbours without bounds checks, so + * a filled pixel on the image border threw ArrayIndexOutOfBoundsException + * out of Circumference.
      • + *
    + */ +class FeatureMathTest { + + @Test + void magnitudeUsesSquaresNotSums() { + assertEquals(5.0, Vector2D.Magnitude(new Point(3, 4)), 1e-9); + assertEquals(13.0, Vector2D.Magnitude(new Point(5, 12)), 1e-9); + } + + @Test + void magnitudeSurvivesLargeCoordinates() { + // 50000^2 overflows int; slide-scale coordinates routinely exceed 46340. + double expected = Math.sqrt(2.0 * 50000.0 * 50000.0); + assertEquals(expected, Vector2D.Magnitude(new Point(50000, 50000)), 1e-3); + } + + @Test + void circumferenceHandlesShapesTouchingTheBorder() { + // Every pixel filled: all are border/edge pixels. Previously isEdge read + // (x-1,y) etc. without bounds checks and threw on the first border pixel. + BufferedImage bi = new BufferedImage(3, 3, BufferedImage.TYPE_INT_ARGB); + for (int x = 0; x < 3; x++) { + for (int y = 0; y < 3; y++) { + bi.setRGB(x, y, 0xFFFFFFFF); + } + } + assertEquals(8, ShapeAnalysis.Circumference(bi), + "all pixels except the centre are edge pixels of the 3x3 block"); + } +} diff --git a/src/test/java/com/ebremer/beakgraph/HighSeverityFixesRoundTripTest.java b/src/test/java/com/ebremer/beakgraph/HighSeverityFixesRoundTripTest.java new file mode 100644 index 00000000..38065236 --- /dev/null +++ b/src/test/java/com/ebremer/beakgraph/HighSeverityFixesRoundTripTest.java @@ -0,0 +1,228 @@ +package com.ebremer.beakgraph; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.ebremer.beakgraph.core.BeakGraph; +import com.ebremer.beakgraph.hdf5.readers.HDF5Reader; +import com.ebremer.beakgraph.hdf5.writers.HDF5Writer; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.Set; +import org.apache.jena.graph.NodeFactory; +import org.apache.jena.query.Dataset; +import org.apache.jena.query.QueryExecution; +import org.apache.jena.query.QueryFactory; +import org.apache.jena.query.ResultSet; +import org.apache.jena.rdf.model.Literal; +import org.apache.jena.rdf.model.RDFNode; +import org.apache.jena.vocabulary.XSD; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Regression tests for the high-severity correctness fixes: + *
      + *
    • C3 - a concrete object term absent from the dictionary must yield no + * solutions (BGIteratorSO previously treated "not in dictionary" like an + * unbound variable and scanned the whole object range, so + * {@code ASK {

      }} answered true).

      • + *
    • C6a - {@code FILTER(?o <= X)} range pushdown must keep the boundary row + * (BGIteratorPOS subtracted 1 from an already-inclusive upperBound).
      • + *
    • H4 - {@code GRAPH } where x is an entity but not a graph must return + * empty instead of leaking id-0 padding rows (which crashed extract), and + * containsGraph must only report actual graphs.
      • + *
    • H1 - xsd:long values needing 58-63 bits (>= 2^56) must build instead of + * crashing on an unsupported bit width.
      • + *
    • H2 - sorted-adjacent strings sharing a high surrogate (emoji) must not + * be corrupted by the front-coded dictionary splitting the pair.
      • + *
    + */ +class HighSeverityFixesRoundTripTest { + + // p/q/r emoji pairs: each pair is sorted-adjacent and shares its prefix up to + // (and including) the high surrogate of the emoji, so front-coding would split + // the surrogate pair. Three pairs so at least two are front-coded within a + // block regardless of where the FCD block boundary (blockSize 16) falls. + private static final String TTL = """ + @prefix ex: . + @prefix xsd: . + ex:s1 ex:p ex:o1 . + ex:s1 ex:q "alpha" . + ex:v1 ex:value 10 . + ex:v2 ex:value 20 . + ex:v3 ex:value 30 . + ex:v4 ex:value 40 . + ex:t1 ex:stamp "100000000000000000"^^xsd:long . + ex:t2 ex:stamp "5"^^xsd:long . + ex:e1 ex:label "p😀" . + ex:e2 ex:label "p😁" . + ex:e3 ex:label "q😀" . + ex:e4 ex:label "q😁" . + ex:e5 ex:label "r😀" . + ex:e6 ex:label "r😁" . + """; + + private static final String PREFIX = + "PREFIX ex: PREFIX xsd: "; + + @TempDir + static Path dir; + static BeakGraph bg; + static Dataset ds; + + @BeforeAll + static void buildAndOpen() throws Exception { + File ttl = dir.resolve("highsev.ttl").toFile(); + File h5 = dir.resolve("highsev.ttl.h5").toFile(); + Files.write(ttl.toPath(), TTL.getBytes(StandardCharsets.UTF_8)); + HDF5Writer.Builder() + .setSource(ttl).setDestination(h5) + .setSpatial(false).setFeatures(false) + .build().write(); + bg = new BeakGraph(new HDF5Reader(h5)); + ds = bg.getDataset(); + } + + @AfterAll + static void closeReader() { + if (bg != null) bg.close(); + } + + private static boolean ask(String pattern) { + try (QueryExecution qe = QueryExecution.dataset(ds) + .query(QueryFactory.create(PREFIX + "ASK { " + pattern + " }")).build()) { + return qe.execAsk(); + } + } + + private static int count(String whereBody) { + try (QueryExecution qe = QueryExecution.dataset(ds) + .query(QueryFactory.create(PREFIX + "SELECT * WHERE { " + whereBody + " }")).build()) { + ResultSet rs = qe.execSelect(); + int n = 0; + while (rs.hasNext()) { rs.next(); n++; } + return n; + } + } + + private static Set selectURIs(String var, String query) { + Set uris = new HashSet<>(); + try (QueryExecution qe = QueryExecution.dataset(ds) + .query(QueryFactory.create(PREFIX + query)).build()) { + ResultSet rs = qe.execSelect(); + while (rs.hasNext()) { + RDFNode n = rs.next().get(var); + if (n != null && n.isURIResource()) uris.add(n.asResource().getURI()); + } + } + return uris; + } + + // --- C3: concrete object absent from the dictionary ------------------- + + @Test + void askWithAbsentConcreteObjectIsFalse() { + assertTrue(ask("ex:s1 ex:p ex:o1"), "control: existing triple must match"); + assertFalse(ask("ex:s1 ex:p ex:notInStore"), "URI object absent from the store"); + assertFalse(ask("ex:s1 ex:q \"not in store\""), "literal object absent from the store"); + assertFalse(ask("ex:s1 ex:p \"42\"^^xsd:int"), "literal absent from the store"); + } + + @Test + void selectWithAbsentConcreteObjectIsEmpty() { + assertEquals(0, count("ex:s1 ex:p ex:notInStore")); + assertEquals(0, count("ex:s1 ex:q \"not in store\"")); + } + + // --- C6a: FILTER upper-bound boundary row ------------------------------ + + @Test + void filterLessOrEqualKeepsBoundaryRow() { + assertEquals(Set.of("http://ex.org/v1", "http://ex.org/v2", "http://ex.org/v3"), + selectURIs("s", "SELECT ?s WHERE { ?s ex:value ?o FILTER(?o <= 30) }")); + } + + @Test + void filterLessThanKeepsLastIncludedRow() { + assertEquals(Set.of("http://ex.org/v1", "http://ex.org/v2", "http://ex.org/v3"), + selectURIs("s", "SELECT ?s WHERE { ?s ex:value ?o FILTER(?o < 40) }")); + } + + // --- H4: GRAPH on a non-graph entity ----------------------------------- + + @Test + void graphPatternOnNonGraphEntityIsEmpty() { + // ex:o1 exists in the entity dictionary (as an object) but is not a graph. + // Previously the GSPO padding rows (S=0) leaked through and extract(0) threw. + assertEquals(0, count("GRAPH ex:o1 { ?s ?p ?o }")); + } + + @Test + void graphPatternOnRealNamedGraphStillWorks() { + // The writer always embeds VoID metadata in its own named graph; the id-0 + // padding guard must not suppress real rows. + assertTrue(count("GRAPH <" + Params.VOIDSTRING + "> { ?s ?p ?o }") > 0, + "VoID named graph must still be queryable"); + } + + @Test + void containsGraphReportsOnlyActualGraphs() { + assertFalse(ds.asDatasetGraph().containsGraph(NodeFactory.createURI("http://ex.org/o1")), + "an entity that never occurs as a graph is not a graph"); + assertTrue(ds.asDatasetGraph().containsGraph(NodeFactory.createURI(Params.VOIDSTRING)), + "the VoID metadata graph is a real named graph"); + } + + // --- H1: xsd:long needing 58-63 bits ------------------------------------ + + @Test + void largeLongRoundTrips() { + // 1e17 > 2^56, so the value-derived width is 58 - previously rejected by + // BitPackedUnSignedLongBuffer (which supports 1..57 and 64 only), crashing + // the whole build in @BeforeAll. + try (QueryExecution qe = QueryExecution.dataset(ds) + .query(QueryFactory.create(PREFIX + "SELECT ?v WHERE { ex:t1 ex:stamp ?v }")).build()) { + ResultSet rs = qe.execSelect(); + assertTrue(rs.hasNext()); + Literal v = rs.next().getLiteral("v"); + assertEquals("100000000000000000", v.getLexicalForm()); + assertEquals(XSD.xlong.getURI(), v.getDatatypeURI()); + assertFalse(rs.hasNext()); + } + } + + // --- H2: surrogate-pair-safe front coding ------------------------------- + + @Test + void supplementaryPlaneStringsRoundTrip() { + String[][] cases = { + {"http://ex.org/e1", "p😀"}, + {"http://ex.org/e2", "p😁"}, + {"http://ex.org/e3", "q😀"}, + {"http://ex.org/e4", "q😁"}, + {"http://ex.org/e5", "r😀"}, + {"http://ex.org/e6", "r😁"}, + }; + for (String[] c : cases) { + // Extraction: the stored value must come back byte-identical. + try (QueryExecution qe = QueryExecution.dataset(ds) + .query(QueryFactory.create(PREFIX + "SELECT ?v WHERE { <" + c[0] + "> ex:label ?v }")).build()) { + ResultSet rs = qe.execSelect(); + assertTrue(rs.hasNext(), "no label for " + c[0]); + assertEquals(c[1], rs.next().getLiteral("v").getLexicalForm(), + "label for " + c[0] + " must survive front coding"); + } + // Lookup: the exact term must be locatable again. + assertEquals(Set.of(c[0]), + selectURIs("s", "SELECT ?s WHERE { ?s ex:label \"" + c[1] + "\" }"), + "term lookup for label of " + c[0]); + } + } +} diff --git a/src/test/java/com/ebremer/beakgraph/HilbertSpaceFixesTest.java b/src/test/java/com/ebremer/beakgraph/HilbertSpaceFixesTest.java new file mode 100644 index 00000000..fae8c44d --- /dev/null +++ b/src/test/java/com/ebremer/beakgraph/HilbertSpaceFixesTest.java @@ -0,0 +1,109 @@ +package com.ebremer.beakgraph; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.ebremer.beakgraph.hdf5.jena.HilbertPolygon; +import com.ebremer.halcyon.geometry.Point; +import com.ebremer.halcyon.hilbert.HilbertSpace; +import java.util.ArrayList; +import org.davidmoten.hilbert.Range; +import org.junit.jupiter.api.Test; + +/** + * Regression tests for two HilbertSpace bugs: + *
      + *
    • H8 - {@code fromWkt} did not strip the optional CRS prefix GeoSPARQL + * wktLiterals may carry and threw a bare {@code java.lang.Error} on any + * parse problem, crashing the spatial query path for standard-form data.
      • + *
    • H10 - both {@code inRange} overloads computed {@code contains(...)} in + * arrow-switch statements, discarded the result, and always returned + * false (breaking the {@code getPolygon} boundary walk).
      • + *
    + */ +class HilbertSpaceFixesTest { + + private static final String CRS = " "; + private static final String SQUARE = "POLYGON((0 0,8 0,8 8,0 8,0 0))"; + + // --- H8 ----------------------------------------------------------------- + + @Test + void fromWktAcceptsCrsPrefixedLiterals() { + org.locationtech.jts.geom.Polygon p = HilbertSpace.fromWkt(CRS + SQUARE); + assertNotNull(p); + assertEquals(64.0, p.getArea(), 1e-9); + // And the plain form keeps working. + assertEquals(64.0, HilbertSpace.fromWkt(SQUARE).getArea(), 1e-9); + } + + @Test + void polygon2HilbertAcceptsCrsPrefixedLiterals() { + ArrayList ranges = HilbertPolygon.Polygon2Hilbert(CRS + SQUARE, 0); + assertFalse(ranges.isEmpty(), "a real polygon must produce Hilbert ranges"); + } + + @Test + void fromWktRejectsGarbageWithCatchableException() { + // Not a java.lang.Error: callers must be able to catch and degrade. + try { + HilbertSpace.fromWkt("NOT A POLYGON AT ALL"); + } catch (RuntimeException expected) { + return; + } + throw new AssertionError("malformed WKT must raise a RuntimeException"); + } + + // --- Corner-only range relevance ---------------------------------------- + + @Test + void subCellPolygonProducesANonEmptyCover() { + // Smaller than one cell and away from every cell corner: the old + // relevance test only asked whether the polygon covers a range's two + // endpoint cell-corner POINTS, so this real polygon produced an EMPTY + // Hilbert cover - a silent false negative for every API consumer. + String subCell = "POLYGON((0.2 0.2,0.8 0.2,0.8 0.8,0.2 0.8,0.2 0.2))"; + ArrayList ranges = HilbertPolygon.Polygon2Hilbert(subCell, 0); + long cell00 = HilbertSpace.hc.index(0, 0); + assertFalse(ranges.isEmpty(), "a real polygon must produce a non-empty cover"); + assertTrue(ranges.stream().anyMatch(r -> r.low() <= cell00 && cell00 <= r.high()), + "the cover must contain the polygon's own cell"); + + // Same failure mode at a coarser scale (cells of 2^2 units). + ArrayList scaled = HilbertPolygon.Polygon2Hilbert(subCell, 2); + assertFalse(scaled.isEmpty(), "the scaled cover must not be empty either"); + } + + @Test + void cellInteriorOverlapKeepsTheRange() { + // Overlaps the interior of cell (5,4) without covering its corner point. + ArrayList ranges = HilbertPolygon.Polygon2Hilbert( + "POLYGON((5.2 4.2,5.8 4.2,5.8 4.8,5.2 4.8,5.2 4.2))", 0); + long cell = HilbertSpace.hc.index(5, 4); + assertTrue(ranges.stream().anyMatch(r -> r.low() <= cell && cell <= r.high()), + "a range whose cell the polygon overlaps must be kept"); + } + + // --- H10 ---------------------------------------------------------------- + + @Test + void inRangeReportsNeighborMembership() { + // A single-cell range at Hilbert index of (5,4). + long idx = HilbertSpace.hc.index(5, 4); + ArrayList ranges = new ArrayList<>(); + ranges.add(new Range(idx, idx)); + + // From (5,5): the N neighbour is (5,4) -> in range; S is (5,6) -> not. + assertTrue(HilbertSpace.inRange(ranges, new Point(5, 5), HilbertSpace.N), + "north neighbour (5,4) is in the range"); + assertFalse(HilbertSpace.inRange(ranges, new Point(5, 5), HilbertSpace.S), + "south neighbour (5,6) is not in the range"); + // From (4,4): the E neighbour is (5,4) -> in range; W is (3,4) -> not. + assertTrue(HilbertSpace.inRange(ranges, new Point(4, 4), HilbertSpace.E)); + assertFalse(HilbertSpace.inRange(ranges, new Point(4, 4), HilbertSpace.W)); + // Diagonals: from (6,5), NW is (5,4) -> in range. + assertTrue(HilbertSpace.inRange(ranges, new Point(6, 5), HilbertSpace.NW)); + } +} diff --git a/src/test/java/com/ebremer/beakgraph/InputBindingShapeTest.java b/src/test/java/com/ebremer/beakgraph/InputBindingShapeTest.java new file mode 100644 index 00000000..2e8ab935 --- /dev/null +++ b/src/test/java/com/ebremer/beakgraph/InputBindingShapeTest.java @@ -0,0 +1,126 @@ +package com.ebremer.beakgraph; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.ebremer.beakgraph.core.BeakGraph; +import com.ebremer.beakgraph.hdf5.readers.HDF5Reader; +import com.ebremer.beakgraph.hdf5.writers.HDF5Writer; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.apache.jena.graph.NodeFactory; +import org.apache.jena.query.Dataset; +import org.apache.jena.query.QueryExecution; +import org.apache.jena.query.QueryFactory; +import org.apache.jena.query.ResultSet; +import org.apache.jena.sparql.core.Var; +import org.apache.jena.sparql.engine.binding.Binding; +import org.apache.jena.sparql.exec.QueryExec; +import org.apache.jena.sparql.exec.RowSet; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Bindings flowing INTO a BeakGraph BGP (VALUES, BIND, joins) must come back + * out well-formed. SolverLibBeak.convert copies every input var into the + * BindingNodeId for id-level solving, and BindingBG also mounts the original + * binding as its BindingBase parent - before the fix each input var was + * therefore exposed twice: size() was inflated, vars() yielded duplicates, and + * DISTINCT could not merge such a row with an equal-valued row of normal shape + * (two identical rows came back). + */ +class InputBindingShapeTest { + + private static final String TTL = """ + @prefix ex: . + ex:s1 ex:p ex:o1 . + """; + + private static final String PREFIX = "PREFIX ex: "; + + @TempDir + static Path dir; + static BeakGraph bg; + static Dataset ds; + + @BeforeAll + static void buildAndOpen() throws Exception { + File ttl = dir.resolve("shape.ttl").toFile(); + File h5 = dir.resolve("shape.ttl.h5").toFile(); + Files.write(ttl.toPath(), TTL.getBytes(StandardCharsets.UTF_8)); + HDF5Writer.Builder().setSource(ttl).setDestination(h5) + .setSpatial(false).setFeatures(false).build().write(); + bg = new BeakGraph(new HDF5Reader(h5)); + ds = bg.getDataset(); + } + + @AfterAll + static void closeReader() { + if (bg != null) bg.close(); + } + + private static int countRows(String query) { + try (QueryExecution qe = QueryExecution.dataset(ds).query(QueryFactory.create(PREFIX + query)).build()) { + ResultSet rs = qe.execSelect(); + int n = 0; + while (rs.hasNext()) { rs.next(); n++; } + return n; + } + } + + @Test + void valuesSeededRowHasWellFormedShape() { + try (QueryExec qe = QueryExec.dataset(ds.asDatasetGraph()) + .query(PREFIX + "SELECT * WHERE { VALUES ?x { ex:s1 } ?x ex:p ?o }").build()) { + RowSet rows = qe.select(); + assertTrue(rows.hasNext()); + Binding b = rows.next(); + assertFalse(rows.hasNext(), "exactly one row expected"); + assertEquals(2, b.size(), "two variables must count as two, not re-count the input var"); + List vars = new ArrayList<>(); + b.vars().forEachRemaining(vars::add); + Set distinct = new HashSet<>(vars); + assertEquals(distinct.size(), vars.size(), "vars() must not yield duplicates: " + vars); + assertEquals(Set.of(Var.alloc("x"), Var.alloc("o")), distinct); + assertEquals(NodeFactory.createURI("http://ex.org/s1"), b.get(Var.alloc("x"))); + assertEquals(NodeFactory.createURI("http://ex.org/o1"), b.get(Var.alloc("o"))); + } + } + + @Test + void distinctMergesValuesSeededRowsWithScannedRows() { + assertEquals(1, countRows("SELECT DISTINCT ?x WHERE { { ?x ex:p ex:o1 } " + + "UNION { VALUES ?x { ex:s1 } ?x ex:p ex:o1 } }"), + "the seeded row and the scanned row bind the same term and must merge"); + } + + @Test + void distinctMergesBindSeededRowsWithScannedRows() { + assertEquals(1, countRows("SELECT DISTINCT ?x WHERE { { ?x ex:p ex:o1 } " + + "UNION { BIND(ex:s1 AS ?x) ?x ex:p ex:o1 } }")); + } + + @Test + void termNotInStoreStillRidesThroughToOutput() { + // ?y is not used by the pattern; its term does not exist in this store + // (a "does not exist" id) and must still come back bound to the + // original term via the parent binding. + try (QueryExec qe = QueryExec.dataset(ds.asDatasetGraph()) + .query(PREFIX + "SELECT * WHERE { VALUES ?y { ex:missing } ?s ex:p ?o }").build()) { + RowSet rows = qe.select(); + assertTrue(rows.hasNext()); + Binding b = rows.next(); + assertEquals(NodeFactory.createURI("http://ex.org/missing"), b.get(Var.alloc("y"))); + assertEquals(3, b.size()); + } + } +} diff --git a/src/test/java/com/ebremer/beakgraph/MalformedLiteralRoundTripTest.java b/src/test/java/com/ebremer/beakgraph/MalformedLiteralRoundTripTest.java new file mode 100644 index 00000000..71d3896a --- /dev/null +++ b/src/test/java/com/ebremer/beakgraph/MalformedLiteralRoundTripTest.java @@ -0,0 +1,122 @@ +package com.ebremer.beakgraph; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.ebremer.beakgraph.core.BeakGraph; +import com.ebremer.beakgraph.hdf5.readers.HDF5Reader; +import com.ebremer.beakgraph.hdf5.writers.HDF5Writer; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.Set; +import org.apache.jena.query.Dataset; +import org.apache.jena.query.QueryExecution; +import org.apache.jena.query.QueryFactory; +import org.apache.jena.query.ResultSet; +import org.apache.jena.rdf.model.Literal; +import org.apache.jena.rdf.model.RDFNode; +import org.apache.jena.vocabulary.XSD; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Regression tests for writer robustness against ill-typed literals: RDF + * permits literals whose lexical form is not valid for their datatype + * ("abc"^^xsd:int), and Jena's parsers accept them with a warning. The writer + * used to call getLiteralValue() unconditionally and abort the entire build on + * the resulting DatatypeFormatException. Such terms must instead be stored + * term-exact via the lexical strings path. + */ +class MalformedLiteralRoundTripTest { + + private static final String TTL = """ + @prefix ex: . + @prefix xsd: . + ex:a ex:v "abc"^^xsd:int . + ex:b ex:v "not-a-date"^^xsd:date . + ex:c ex:v 5 . + ex:d ex:v "5"^^xsd:int . + ex:e ex:v "9z9"^^xsd:long . + """; + + private static final String PREFIX = + "PREFIX ex: PREFIX xsd: "; + + @TempDir + static Path dir; + static BeakGraph bg; + static Dataset ds; + + @BeforeAll + static void buildAndOpen() throws Exception { + File ttl = dir.resolve("malformed.ttl").toFile(); + File h5 = dir.resolve("malformed.ttl.h5").toFile(); + Files.write(ttl.toPath(), TTL.getBytes(StandardCharsets.UTF_8)); + // Pre-fix this aborted with an IOException wrapping DatatypeFormatException. + HDF5Writer.Builder() + .setSource(ttl).setDestination(h5) + .setSpatial(false).setFeatures(false) + .build().write(); + bg = new BeakGraph(new HDF5Reader(h5)); + ds = bg.getDataset(); + } + + @AfterAll + static void closeReader() { + if (bg != null) bg.close(); + } + + private static Set subjects(String objectTerm) { + Set uris = new HashSet<>(); + try (QueryExecution qe = QueryExecution.dataset(ds) + .query(QueryFactory.create(PREFIX + "SELECT ?s WHERE { ?s ex:v " + objectTerm + " }")).build()) { + ResultSet rs = qe.execSelect(); + while (rs.hasNext()) { + RDFNode n = rs.next().get("s"); + if (n != null && n.isURIResource()) uris.add(n.asResource().getURI()); + } + } + return uris; + } + + private static Literal value(String subject) { + try (QueryExecution qe = QueryExecution.dataset(ds) + .query(QueryFactory.create(PREFIX + "SELECT ?v WHERE { " + subject + " ex:v ?v }")).build()) { + return qe.execSelect().next().getLiteral("v"); + } + } + + @Test + void illTypedLiteralsRoundTripTermExact() { + Literal a = value("ex:a"); + assertEquals("abc", a.getLexicalForm()); + assertEquals(XSD.xint.getURI(), a.getDatatypeURI()); + + Literal b = value("ex:b"); + assertEquals("not-a-date", b.getLexicalForm()); + assertEquals(XSD.date.getURI(), b.getDatatypeURI()); + + Literal e = value("ex:e"); + assertEquals("9z9", e.getLexicalForm()); + assertEquals(XSD.xlong.getURI(), e.getDatatypeURI()); + } + + @Test + void illTypedLiteralsAreFindableByTerm() { + assertEquals(Set.of("http://ex.org/a"), subjects("\"abc\"^^xsd:int")); + assertEquals(Set.of("http://ex.org/b"), subjects("\"not-a-date\"^^xsd:date")); + } + + @Test + void wellFormedNeighborsAreUnaffected() { + assertEquals(Set.of("http://ex.org/c"), subjects("5")); + assertEquals(Set.of("http://ex.org/d"), subjects("\"5\"^^xsd:int")); + Literal d = value("ex:d"); + assertEquals("5", d.getLexicalForm()); + assertEquals(XSD.xint.getURI(), d.getDatatypeURI()); + } +} diff --git a/src/test/java/com/ebremer/beakgraph/NonExistentBindingTest.java b/src/test/java/com/ebremer/beakgraph/NonExistentBindingTest.java index 203f4f35..8e44091b 100644 --- a/src/test/java/com/ebremer/beakgraph/NonExistentBindingTest.java +++ b/src/test/java/com/ebremer/beakgraph/NonExistentBindingTest.java @@ -38,6 +38,7 @@ class NonExistentBindingTest { @TempDir static Path dir; + static BeakGraph bg; static Dataset ds; @BeforeAll @@ -46,7 +47,14 @@ static void build() throws Exception { File h5 = dir.resolve("ne.ttl.h5").toFile(); Files.write(ttl.toPath(), TTL.getBytes(StandardCharsets.UTF_8)); HDF5Writer.Builder().setSource(ttl).setDestination(h5).setSpatial(false).setFeatures(false).build().write(); - ds = new BeakGraph(new HDF5Reader(h5)).getDataset(); + bg = new BeakGraph(new HDF5Reader(h5)); + ds = bg.getDataset(); + } + + @org.junit.jupiter.api.AfterAll + static void closeReader() { + // Release the mapped file: a leaked reader makes @TempDir cleanup flaky on Windows. + if (bg != null) bg.close(); } private static int count(String body) { diff --git a/src/test/java/com/ebremer/beakgraph/NumericCanonicalizationTest.java b/src/test/java/com/ebremer/beakgraph/NumericCanonicalizationTest.java new file mode 100644 index 00000000..077817f0 --- /dev/null +++ b/src/test/java/com/ebremer/beakgraph/NumericCanonicalizationTest.java @@ -0,0 +1,130 @@ +package com.ebremer.beakgraph; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.ebremer.beakgraph.core.BeakGraph; +import com.ebremer.beakgraph.hdf5.readers.HDF5Reader; +import com.ebremer.beakgraph.hdf5.writers.HDF5Writer; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.Set; +import org.apache.jena.query.Dataset; +import org.apache.jena.query.QueryExecution; +import org.apache.jena.query.QueryFactory; +import org.apache.jena.query.ResultSet; +import org.apache.jena.rdf.model.Literal; +import org.apache.jena.rdf.model.RDFNode; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Regression tests for H3: numeric literals (xsd:int / xsd:long / xsd:float / + * xsd:double) are stored by VALUE, and the reader regenerates the canonical + * lexical form. Two lexical variants of one value ("01" vs "1"^^xsd:int) used + * to become two term-distinct dictionary entries that both extract to the same + * canonical term - duplicate "equal" entries that break the strict ordering + * the dictionary binary search relies on, leaving some triples unreachable by + * term lookup. + *

    + * Policy: such literals are canonicalized at ingest (the value-typed storage + * never preserved their lexical form anyway), so all variants collapse onto + * one term and every triple is findable via the canonical form. + */ +class NumericCanonicalizationTest { + + private static final String TTL = """ + @prefix ex: . + @prefix xsd: . + ex:a ex:v "01"^^xsd:int . + ex:b ex:v "1"^^xsd:int . + ex:c ex:v "007"^^xsd:long . + ex:d ex:v "7"^^xsd:long . + ex:e ex:v "2.50"^^xsd:float . + ex:f ex:v "2.5"^^xsd:float . + ex:g ex:v "1.0E1"^^xsd:double . + ex:h ex:v "10.0"^^xsd:double . + ex:z ex:v "2"^^xsd:int . + """; + + private static final String PREFIX = + "PREFIX ex: PREFIX xsd: "; + + @TempDir + static Path dir; + static BeakGraph bg; + static Dataset ds; + + @BeforeAll + static void buildAndOpen() throws Exception { + File ttl = dir.resolve("canon.ttl").toFile(); + File h5 = dir.resolve("canon.ttl.h5").toFile(); + Files.write(ttl.toPath(), TTL.getBytes(StandardCharsets.UTF_8)); + HDF5Writer.Builder() + .setSource(ttl).setDestination(h5) + .setSpatial(false).setFeatures(false) + .build().write(); + bg = new BeakGraph(new HDF5Reader(h5)); + ds = bg.getDataset(); + } + + @AfterAll + static void closeReader() { + if (bg != null) bg.close(); + } + + private static Set subjects(String objectTerm) { + Set uris = new HashSet<>(); + try (QueryExecution qe = QueryExecution.dataset(ds) + .query(QueryFactory.create(PREFIX + "SELECT ?s WHERE { ?s ex:v " + objectTerm + " }")).build()) { + ResultSet rs = qe.execSelect(); + while (rs.hasNext()) { + RDFNode n = rs.next().get("s"); + if (n != null && n.isURIResource()) uris.add(n.asResource().getURI()); + } + } + return uris; + } + + @Test + void lexicalVariantsCollapseAndAllTriplesStayFindable() { + assertEquals(Set.of("http://ex.org/a", "http://ex.org/b"), subjects("\"1\"^^xsd:int")); + assertEquals(Set.of("http://ex.org/c", "http://ex.org/d"), subjects("\"7\"^^xsd:long")); + assertEquals(Set.of("http://ex.org/e", "http://ex.org/f"), subjects("\"2.5\"^^xsd:float")); + assertEquals(Set.of("http://ex.org/g", "http://ex.org/h"), subjects("\"10.0\"^^xsd:double")); + // Neighbouring entries are still findable (ordering is intact). + assertEquals(Set.of("http://ex.org/z"), subjects("\"2\"^^xsd:int")); + } + + @Test + void nonCanonicalQueryConstantMatchesNothing() { + // Documented policy: the store contains the canonical term only; "01"^^xsd:int + // is a different RDF term and (correctly, under term semantics) matches nothing. + assertEquals(Set.of(), subjects("\"01\"^^xsd:int")); + } + + @Test + void extractionReturnsCanonicalForm() { + try (QueryExecution qe = QueryExecution.dataset(ds) + .query(QueryFactory.create(PREFIX + "SELECT ?v WHERE { ex:a ex:v ?v }")).build()) { + ResultSet rs = qe.execSelect(); + Literal v = rs.next().getLiteral("v"); + assertEquals("1", v.getLexicalForm()); + } + } + + @Test + void distinctValuesCollapseToCanonicalTerms() { + Set values = new HashSet<>(); + try (QueryExecution qe = QueryExecution.dataset(ds) + .query(QueryFactory.create(PREFIX + "SELECT DISTINCT ?v WHERE { ?s ex:v ?v }")).build()) { + ResultSet rs = qe.execSelect(); + while (rs.hasNext()) values.add(rs.next().getLiteral("v").getLexicalForm()); + } + assertEquals(Set.of("1", "7", "2.5", "10.0", "2"), values); + } +} diff --git a/src/test/java/com/ebremer/beakgraph/PolygonScalerGridCellTest.java b/src/test/java/com/ebremer/beakgraph/PolygonScalerGridCellTest.java new file mode 100644 index 00000000..f47a09a1 --- /dev/null +++ b/src/test/java/com/ebremer/beakgraph/PolygonScalerGridCellTest.java @@ -0,0 +1,46 @@ +package com.ebremer.beakgraph; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.ebremer.halcyon.hilbert.GridCell; +import com.ebremer.halcyon.hilbert.PolygonScaler; +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.Test; +import org.locationtech.jts.geom.Envelope; +import org.locationtech.jts.geom.GeometryFactory; +import org.locationtech.jts.geom.Polygon; + +/** + * PolygonScaler.getGridCells must agree with the tile grid the writer + * persists: every scale level's tiles are GRIDTILESIZE units on a side in that + * level's own (already divided) coordinates - generateGridURNs stamps the + * stored per-tile graph URNs with exactly that formula. The old code + * multiplied the tile size by 2^scale ON TOP of the already-scaled polygon, so + * for scale >= 1 every computed cell disagreed with every stored tile. + */ +class PolygonScalerGridCellTest { + + @Test + void gridCellsMatchThePersistedTileGrid() { + GeometryFactory gf = new GeometryFactory(); + // Coordinates are scale-1 coordinates (base 2048..2060 halved). + Polygon atScale1 = (Polygon) gf.toGeometry(new Envelope(1024, 1030, 0, 10)); + List cells = PolygonScaler.getGridCells(new ArrayList<>(), atScale1, (short) 1); + assertEquals(1, cells.size()); + assertEquals(1, cells.get(0).scale); + assertEquals(2, cells.get(0).x, "x = floor(1024 / 512), the writer's tile column"); + assertEquals(0, cells.get(0).y); + } + + @Test + void scaleZeroCellsAreUnchanged() { + GeometryFactory gf = new GeometryFactory(); + Polygon atScale0 = (Polygon) gf.toGeometry(new Envelope(600, 610, 0, 10)); + List cells = PolygonScaler.getGridCells(new ArrayList<>(), atScale0, (short) 0); + assertEquals(1, cells.size()); + assertEquals(0, cells.get(0).scale); + assertEquals(1, cells.get(0).x, "x = floor(600 / 512)"); + assertEquals(0, cells.get(0).y); + } +} diff --git a/src/test/java/com/ebremer/beakgraph/QuadFormatIngestionTest.java b/src/test/java/com/ebremer/beakgraph/QuadFormatIngestionTest.java new file mode 100644 index 00000000..8c93f58d --- /dev/null +++ b/src/test/java/com/ebremer/beakgraph/QuadFormatIngestionTest.java @@ -0,0 +1,106 @@ +package com.ebremer.beakgraph; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.ebremer.beakgraph.core.BeakGraph; +import com.ebremer.beakgraph.hdf5.readers.HDF5Reader; +import com.ebremer.beakgraph.hdf5.writers.HDF5Writer; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import org.apache.jena.query.Dataset; +import org.apache.jena.query.QueryExecution; +import org.apache.jena.query.QueryFactory; +import org.apache.jena.query.ResultSet; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Regression tests for source-syntax handling: the parser language must be + * detected from the file name instead of being hardcoded to Turtle - BeakGraph + * is a quad store, and named graphs can only arrive through a quad-capable + * syntax (TriG, N-Quads). + */ +class QuadFormatIngestionTest { + + @TempDir + static Path dir; + + private static int count(Dataset ds, String query) { + try (QueryExecution qe = QueryExecution.dataset(ds) + .query(QueryFactory.create("PREFIX ex: " + query)).build()) { + ResultSet rs = qe.execSelect(); + int n = 0; + while (rs.hasNext()) { rs.next(); n++; } + return n; + } + } + + @Test + void trigSourceWithNamedGraphRoundTrips() throws Exception { + String trig = """ + @prefix ex: . + ex:s0 ex:p ex:o0 . + ex:g1 { + ex:s1 ex:p ex:o1 . + ex:s2 ex:p ex:o2 . + } + """; + File src = dir.resolve("data.trig").toFile(); + File h5 = dir.resolve("data.trig.h5").toFile(); + Files.write(src.toPath(), trig.getBytes(StandardCharsets.UTF_8)); + HDF5Writer.Builder().setSource(src).setDestination(h5) + .setSpatial(false).setFeatures(false).build().write(); + try (BeakGraph bg = new BeakGraph(new HDF5Reader(h5))) { + Dataset ds = bg.getDataset(); + assertEquals(2, count(ds, "SELECT * WHERE { GRAPH ex:g1 { ?s ?p ?o } }"), + "the TriG named graph must be stored and queryable"); + assertEquals(1, count(ds, "SELECT * WHERE { ?s ex:p ex:o0 }"), + "the TriG default graph must be stored"); + assertTrue(ds.asDatasetGraph().containsGraph( + org.apache.jena.graph.NodeFactory.createURI("http://ex.org/g1"))); + } + } + + @Test + void nquadsSourceRoundTrips() throws Exception { + String nq = """ + . + . + """; + File src = dir.resolve("data.nq").toFile(); + File h5 = dir.resolve("data.nq.h5").toFile(); + Files.write(src.toPath(), nq.getBytes(StandardCharsets.UTF_8)); + HDF5Writer.Builder().setSource(src).setDestination(h5) + .setSpatial(false).setFeatures(false).build().write(); + try (BeakGraph bg = new BeakGraph(new HDF5Reader(h5))) { + Dataset ds = bg.getDataset(); + assertEquals(1, count(ds, "SELECT * WHERE { GRAPH { ?s ?p ?o } }")); + assertEquals(1, count(ds, "SELECT * WHERE { ?s ex:p }")); + } + } + + @Test + void emptySourceProducesConsistentFile() throws Exception { + File src = dir.resolve("empty.ttl").toFile(); + File h5 = dir.resolve("empty.ttl.h5").toFile(); + Files.write(src.toPath(), "# nothing here\n".getBytes(StandardCharsets.UTF_8)); + HDF5Writer.Builder().setSource(src).setDestination(h5) + .setSpatial(false).setFeatures(false).build().write(); + try (BeakGraph bg = new BeakGraph(new HDF5Reader(h5))) { + Dataset ds = bg.getDataset(); + // The VoID metadata graph is always written; the columnar graph list + // must therefore exist too, or enumeration/union/contains disagree + // with direct GRAPH queries on the same file. + int direct = count(ds, "SELECT * WHERE { GRAPH <" + Params.VOIDSTRING + "> { ?s ?p ?o } }"); + assertTrue(direct > 0, "VoID metadata must be present"); + assertEquals(Boolean.TRUE, + ds.asDatasetGraph().containsGraph(org.apache.jena.graph.NodeFactory.createURI(Params.VOIDSTRING)), + "containsGraph must agree with the stored VoID graph"); + assertEquals(direct, count(ds, "SELECT * WHERE { GRAPH ?g { ?s ?p ?o } }"), + "graph enumeration must see the same rows as the direct query"); + } + } +} diff --git a/src/test/java/com/ebremer/beakgraph/ReaderLifecycleTest.java b/src/test/java/com/ebremer/beakgraph/ReaderLifecycleTest.java new file mode 100644 index 00000000..a472e66b --- /dev/null +++ b/src/test/java/com/ebremer/beakgraph/ReaderLifecycleTest.java @@ -0,0 +1,128 @@ +package com.ebremer.beakgraph; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.ebremer.beakgraph.core.BeakGraph; +import com.ebremer.beakgraph.hdf5.readers.HDF5Reader; +import com.ebremer.beakgraph.hdf5.writers.HDF5Writer; +import com.ebremer.beakgraph.pool.BeakGraphPool; +import io.jhdf.HdfFile; +import io.jhdf.WritableHdfFile; +import java.io.File; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import org.apache.jena.query.Dataset; +import org.apache.jena.query.QueryExecution; +import org.apache.jena.query.QueryFactory; +import org.apache.jena.query.ResultSet; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Regression tests for the reader lifecycle fixes: + *

      + *
    • H5 - opening a structurally valid HDF5 file that is not a BeakGraph + * file must fail with a clear exception AND release the underlying file + * mapping (a leaked HdfFile pins the file lock on Windows).
      • + *
    • H6 - closing a named-graph wrapper obtained from the dataset must not + * close the shared reader under the rest of the dataset, and the keyed + * pool must not re-issue an instance whose reader has been closed.
      • + *
    + */ +class ReaderLifecycleTest { + + private static final String TTL = """ + @prefix ex: . + ex:s ex:p ex:o . + ex:s ex:p "v" . + """; + + @TempDir + static Path dir; + static File h5; + + @BeforeAll + static void build() throws Exception { + File ttl = dir.resolve("life.ttl").toFile(); + h5 = dir.resolve("life.ttl.h5").toFile(); + Files.write(ttl.toPath(), TTL.getBytes(StandardCharsets.UTF_8)); + HDF5Writer.Builder() + .setSource(ttl).setDestination(h5) + .setSpatial(false).setFeatures(false) + .build().write(); + } + + @AfterAll + static void drainPool() { + BeakGraphPool.getPool().clear(h5.toURI()); + } + + private static int count(Dataset ds) { + try (QueryExecution qe = QueryExecution.dataset(ds) + .query(QueryFactory.create("SELECT * WHERE { ?s ?p ?o }")).build()) { + ResultSet rs = qe.execSelect(); + int n = 0; + while (rs.hasNext()) { rs.next(); n++; } + return n; + } + } + + // --- H5 --------------------------------------------------------------- + + @Test + void malformedFileIsRejectedAndReleased() throws Exception { + Path notBG = dir.resolve("notbeakgraph.h5"); + try (WritableHdfFile out = HdfFile.write(notBG)) { + out.putGroup("SomethingElse").putAttribute("hello", 1); + } + assertThrows(IllegalStateException.class, () -> new HDF5Reader(notBG.toFile()), + "a non-BeakGraph HDF5 file must be rejected explicitly"); + // The failed constructor must have closed the mapped file: on Windows an + // open HdfFile mapping makes this delete fail. + Files.delete(notBG); + } + + // --- H6: named-graph wrapper close ------------------------------------ + + @Test + void closingNamedGraphWrapperDoesNotKillDataset() throws Exception { + try (BeakGraph bg = new BeakGraph(new HDF5Reader(h5))) { + Dataset ds = bg.getDataset(); + // Routine Jena usage: obtain a named model, read it, close it - BEFORE + // anything else has touched the file, so a close that wrongly cascades + // to the shared reader cannot hide behind already-cached index buffers. + ds.getNamedModel(Params.VOIDSTRING).close(); + assertTrue(count(ds) > 0, + "closing a named-graph view must not close the shared reader"); + assertTrue(bg.getReader().isOpen(), "shared reader must remain open"); + } + } + + // --- H6: pool must not re-issue a closed instance ----------------------- + + @Test + void poolDoesNotReissueClosedInstance() throws Exception { + URI key = h5.toURI(); + BeakGraph first = BeakGraphPool.getPool().borrowObject(key); + assertNotNull(first); + // Poison the instance the way a buggy/abusive caller would, then return it. + first.close(); + BeakGraphPool.getPool().returnObject(key, first); + + BeakGraph second = BeakGraphPool.getPool().borrowObject(key); + try { + // With testOnBorrow active and a working validateObject, the poisoned + // instance is destroyed and a fresh one is created - so this works. + assertTrue(count(second.getDataset()) > 0, + "borrowed instance must be usable after a poisoned return"); + } finally { + BeakGraphPool.getPool().returnObject(key, second); + } + } +} diff --git a/src/test/java/com/ebremer/beakgraph/RegistryWiringTest.java b/src/test/java/com/ebremer/beakgraph/RegistryWiringTest.java new file mode 100644 index 00000000..58f7c4e9 --- /dev/null +++ b/src/test/java/com/ebremer/beakgraph/RegistryWiringTest.java @@ -0,0 +1,175 @@ +package com.ebremer.beakgraph; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.ebremer.beakgraph.core.BeakGraph; +import com.ebremer.beakgraph.hdf5.jena.OpExecutorBG; +import com.ebremer.beakgraph.hdf5.readers.HDF5Reader; +import com.ebremer.beakgraph.hdf5.writers.HDF5Writer; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.Set; +import org.apache.jena.query.ARQ; +import org.apache.jena.query.Dataset; +import org.apache.jena.query.QueryExecution; +import org.apache.jena.query.QueryExecutionFactory; +import org.apache.jena.query.QueryFactory; +import org.apache.jena.query.ResultSet; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.ModelFactory; +import org.apache.jena.rdf.model.RDFNode; +import org.apache.jena.rdf.model.Resource; +import org.apache.jena.sparql.engine.main.QC; +import org.apache.jena.sparql.pfunction.PropertyFunctionRegistry; +import org.apache.jena.vocabulary.RDF; +import org.apache.jena.vocabulary.RDFS; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Regression tests for H7: loading BeakGraph must not mutate global Jena + * semantics for other datasets in the same JVM. + *
      + *
    • The standard rdfs:member property function must stay registered + * globally (it used to be remove()d JVM-wide).
      • + *
    • BG stores treat rdfs:member as a plain stored predicate - excluded + * from property-function rewriting via a dataset-scoped registry, not by + * changing the global one.
      • + *
    • The BG OpExecutor factory is wired into the dataset's context, not the + * global ARQ context.
      • + *
    • The globally registered geof:sfIntersects must actually evaluate + * intersection (it used to return TRUE unconditionally).
      • + *
    + */ +class RegistryWiringTest { + + private static final String TTL = """ + @prefix ex: . + @prefix rdfs: . + ex:bag rdfs:member ex:item . + ex:s1 ex:p ex:o1 . + """; + + private static final String GEO_WKT = "http://www.opengis.net/ont/geosparql#wktLiteral"; + private static final String GEOF = "http://www.opengis.net/def/function/geosparql/"; + + @TempDir + static Path dir; + static BeakGraph bg; + static Dataset ds; + + @BeforeAll + static void buildAndOpen() throws Exception { + File ttl = dir.resolve("wiring.ttl").toFile(); + File h5 = dir.resolve("wiring.ttl.h5").toFile(); + Files.write(ttl.toPath(), TTL.getBytes(StandardCharsets.UTF_8)); + HDF5Writer.Builder() + .setSource(ttl).setDestination(h5) + .setSpatial(false).setFeatures(false) + .build().write(); + bg = new BeakGraph(new HDF5Reader(h5)); + ds = bg.getDataset(); + } + + @AfterAll + static void closeReader() { + if (bg != null) bg.close(); + } + + // --- global registries stay intact ------------------------------------- + + @Test + void rdfsMemberStaysRegisteredGlobally() { + assertNotNull(PropertyFunctionRegistry.get().get(RDFS.member.getURI()), + "loading BeakGraph must not remove the standard rdfs:member property function"); + } + + @Test + void rdfsMemberStillWorksOnPlainJenaModels() { + // rdf:_1 containership answered via the rdfs:member property function + // (which requires the subject to be typed as a container). + Model m = ModelFactory.createDefaultModel(); + Resource c = m.createResource("http://ex.org/c"); + Resource item = m.createResource("http://ex.org/i"); + c.addProperty(RDF.type, RDF.Seq); + c.addProperty(RDF.li(1), item); + try (QueryExecution qe = QueryExecutionFactory.create(QueryFactory.create( + "PREFIX rdfs: " + + "SELECT ?x WHERE { rdfs:member ?x }"), m)) { + ResultSet rs = qe.execSelect(); + assertTrue(rs.hasNext(), "container membership must be answered"); + assertEquals("http://ex.org/i", rs.next().getResource("x").getURI()); + assertFalse(rs.hasNext()); + } + } + + @Test + void globalOpExecutorFactoryIsNotHijacked() { + assertNotSame(OpExecutorBG.opExecFactoryBG, QC.getFactory(ARQ.getContext()), + "the BG OpExecutor factory must not be installed JVM-globally"); + } + + // --- per-dataset wiring -------------------------------------------------- + + @Test + void datasetContextCarriesBGExecutorAndScopedPropertyFunctions() { + assertSame(OpExecutorBG.opExecFactoryBG, QC.getFactory(ds.asDatasetGraph().getContext()), + "BG datasets must carry their executor in their own context"); + PropertyFunctionRegistry scoped = + PropertyFunctionRegistry.chooseRegistry(ds.asDatasetGraph().getContext()); + assertNull(scoped.get(RDFS.member.getURI()), + "BG datasets must not rewrite rdfs:member patterns into container membership"); + } + + @Test + void rdfsMemberIsAPlainPredicateInBGData() { + // The stored triple uses rdfs:member literally; the property function + // (which expands to rdf:_N lookups) would answer nothing here. + Set items = new HashSet<>(); + try (QueryExecution qe = QueryExecution.dataset(ds).query(QueryFactory.create( + "PREFIX rdfs: " + + "SELECT ?x WHERE { rdfs:member ?x }")).build()) { + ResultSet rs = qe.execSelect(); + while (rs.hasNext()) { + RDFNode n = rs.next().get("x"); + if (n != null && n.isURIResource()) items.add(n.asResource().getURI()); + } + } + assertEquals(Set.of("http://ex.org/item"), items); + } + + // --- sfIntersects fallback is sound -------------------------------------- + + private static boolean intersects(String wkt1, String wkt2) { + String q = "ASK { FILTER(<" + GEOF + "sfIntersects>(" + + "\"" + wkt1 + "\"^^<" + GEO_WKT + ">, \"" + wkt2 + "\"^^<" + GEO_WKT + ">)) }"; + try (QueryExecution qe = QueryExecutionFactory.create( + QueryFactory.create(q), ModelFactory.createDefaultModel())) { + return qe.execAsk(); + } + } + + @Test + void sfIntersectsEvaluatesGeometry() { + assertTrue(intersects("POLYGON((0 0,10 0,10 10,0 10,0 0))", "POINT(5 5)"), + "point inside polygon"); + assertTrue(intersects("POLYGON((0 0,10 0,10 10,0 10,0 0))", "POLYGON((5 5,15 5,15 15,5 15,5 5))"), + "overlapping polygons intersect even though neither is within the other"); + assertFalse(intersects("POLYGON((0 0,10 0,10 10,0 10,0 0))", "POINT(50 50)"), + "disjoint geometries must NOT intersect (previously always true)"); + assertTrue(intersects(" POINT(5 5)", + "POLYGON((0 0,10 0,10 10,0 10,0 0))"), + "CRS-prefixed wktLiteral must be handled"); + } +} diff --git a/src/test/java/com/ebremer/beakgraph/RepeatedVariablePatternTest.java b/src/test/java/com/ebremer/beakgraph/RepeatedVariablePatternTest.java new file mode 100644 index 00000000..ac14b122 --- /dev/null +++ b/src/test/java/com/ebremer/beakgraph/RepeatedVariablePatternTest.java @@ -0,0 +1,131 @@ +package com.ebremer.beakgraph; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.ebremer.beakgraph.core.BeakGraph; +import com.ebremer.beakgraph.hdf5.readers.HDF5Reader; +import com.ebremer.beakgraph.hdf5.writers.HDF5Writer; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.Set; +import org.apache.jena.query.Dataset; +import org.apache.jena.query.QueryExecution; +import org.apache.jena.query.QueryFactory; +import org.apache.jena.query.QuerySolution; +import org.apache.jena.query.ResultSet; +import org.apache.jena.rdf.model.RDFNode; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Regression tests for C1: a variable repeated within one triple pattern must + * only match rows where both positions hold the same term. BindingNodeId.put + * previously kept the first value and silently ignored the second, so + * {@code ?s ?p ?s} matched every triple. + */ +class RepeatedVariablePatternTest { + + // ex:loop is subject, predicate AND object of one triple (S=P=O); + // ex:a and ex:c are S=O with distinct predicates; (a p b) and (d q e) + // are ordinary triples that must NOT match repeated-variable patterns. + private static final String TTL = """ + @prefix ex: . + ex:loop ex:loop ex:loop . + ex:a ex:p ex:a . + ex:a ex:p ex:b . + ex:c ex:q ex:c . + ex:d ex:q ex:e . + """; + + private static final String PREFIX = "PREFIX ex: "; + + @TempDir + static Path dir; + static BeakGraph bg; + static Dataset ds; + + @BeforeAll + static void buildAndOpen() throws Exception { + File ttl = dir.resolve("repvar.ttl").toFile(); + File h5 = dir.resolve("repvar.ttl.h5").toFile(); + Files.write(ttl.toPath(), TTL.getBytes(StandardCharsets.UTF_8)); + HDF5Writer.Builder() + .setSource(ttl).setDestination(h5) + .setSpatial(false).setFeatures(false) + .build().write(); + bg = new BeakGraph(new HDF5Reader(h5)); + ds = bg.getDataset(); + } + + @AfterAll + static void closeReader() { + if (bg != null) bg.close(); + } + + private static Set select(String var, String query) { + Set uris = new HashSet<>(); + try (QueryExecution qe = QueryExecution.dataset(ds) + .query(QueryFactory.create(PREFIX + query)).build()) { + ResultSet rs = qe.execSelect(); + while (rs.hasNext()) { + QuerySolution qs = rs.next(); + RDFNode n = qs.get(var); + if (n != null && n.isURIResource()) uris.add(n.asResource().getURI()); + } + } + return uris; + } + + @Test + void subjectObjectRepeatMatchesOnlyEqualRows() { + // ?s ?p ?s : subject must equal object -> loop, a, c (NOT b/d/e rows). + assertEquals(Set.of("http://ex.org/loop", "http://ex.org/a", "http://ex.org/c"), + select("s", "SELECT ?s WHERE { ?s ?p ?s }")); + } + + @Test + void subjectObjectRepeatWithBoundPredicate() { + // ?x ex:p ?x : only (a p a); (a p b) must not match. + assertEquals(Set.of("http://ex.org/a"), + select("x", "SELECT ?x WHERE { ?x ex:p ?x }")); + assertEquals(Set.of("http://ex.org/c"), + select("x", "SELECT ?x WHERE { ?x ex:q ?x }")); + } + + @Test + void subjectPredicateRepeatMatchesOnlyEqualRows() { + // ?x ?x ?o : subject must equal predicate -> only the loop triple. + // (Subject ids live in the entity dictionary, predicate ids in their own + // dictionary, so this also exercises the cross-id-space comparison.) + assertEquals(Set.of("http://ex.org/loop"), + select("x", "SELECT ?x WHERE { ?x ?x ?o }")); + assertEquals(Set.of("http://ex.org/loop"), + select("o", "SELECT ?o WHERE { ?x ?x ?o }")); + } + + @Test + void tripleRepeatMatchesOnlyAllEqualRow() { + // ?x ?x ?x : S = P = O -> only the loop triple. + assertEquals(Set.of("http://ex.org/loop"), + select("x", "SELECT ?x WHERE { ?x ?x ?x }")); + } + + @Test + void askVariantsAgree() { + try (QueryExecution qe = QueryExecution.dataset(ds) + .query(QueryFactory.create(PREFIX + "ASK { ex:d ?p ex:d }")).build()) { + assertFalse(qe.execAsk(), "ex:d is never its own object"); + } + try (QueryExecution qe = QueryExecution.dataset(ds) + .query(QueryFactory.create(PREFIX + "ASK { ex:a ?p ex:a }")).build()) { + assertTrue(qe.execAsk()); + } + } +} diff --git a/src/test/java/com/ebremer/beakgraph/SpatialConcreteSubjectTest.java b/src/test/java/com/ebremer/beakgraph/SpatialConcreteSubjectTest.java new file mode 100644 index 00000000..0bc38da1 --- /dev/null +++ b/src/test/java/com/ebremer/beakgraph/SpatialConcreteSubjectTest.java @@ -0,0 +1,133 @@ +package com.ebremer.beakgraph; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.ebremer.beakgraph.core.BeakGraph; +import com.ebremer.beakgraph.hdf5.readers.HDF5Reader; +import com.ebremer.beakgraph.hdf5.writers.HDF5Writer; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import org.apache.jena.query.Dataset; +import org.apache.jena.query.QueryExecution; +import org.apache.jena.query.QueryFactory; +import org.apache.jena.query.ResultSet; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * geof:sfIntersects with a CONCRETE feature subject. The spatial index stores + * geometry SUBJECT ids, so with nothing to seed (no variable subject in the + * geometry triple) the accelerator must step aside and leave the answer to the + * pattern solve + sfIntersects verification. The old code seeded the geometry + * OBJECT variable with subject ids instead: every candidate row failed the + * triple pattern and any pinned-subject spatial query silently returned zero + * rows (ASK said false for a geometry that genuinely intersects). + */ +class SpatialConcreteSubjectTest { + + private static final String REGION = "POLYGON((100 100,300 100,300 300,100 300,100 100))"; + + private static final String TTL = """ + @prefix ex: . + @prefix geo: . + # entirely inside the region + ex:small geo:asWKT "POLYGON((150 150,160 150,160 160,150 160,150 150))"^^geo:wktLiteral . + # far away - disjoint from the region + ex:far geo:asWKT "POLYGON((5000 5000,5100 5000,5100 5100,5000 5100,5000 5000))"^^geo:wktLiteral . + """; + + private static final String PREFIXES = """ + PREFIX ex: + PREFIX geo: + PREFIX geof: + """; + + @TempDir + static Path dir; + static BeakGraph bg; + static Dataset ds; + + @BeforeAll + static void buildAndOpen() throws Exception { + File ttl = dir.resolve("concrete.ttl").toFile(); + File h5 = dir.resolve("concrete.ttl.h5").toFile(); + Files.write(ttl.toPath(), TTL.getBytes(StandardCharsets.UTF_8)); + HDF5Writer.Builder().setSource(ttl).setDestination(h5) + .setSpatial(true).setFeatures(false).build().write(); + bg = new BeakGraph(new HDF5Reader(h5)); + ds = bg.getDataset(); + } + + @AfterAll + static void closeReader() { + if (bg != null) bg.close(); + } + + private static List select(String query, String var) { + List out = new ArrayList<>(); + try (QueryExecution qe = QueryExecution.dataset(ds).query(QueryFactory.create(PREFIXES + query)).build()) { + ResultSet rs = qe.execSelect(); + while (rs.hasNext()) { + org.apache.jena.rdf.model.RDFNode n = rs.next().get(var); + out.add(n.isLiteral() ? n.asLiteral().getLexicalForm() : n.toString()); + } + } + return out; + } + + private static boolean ask(String query) { + try (QueryExecution qe = QueryExecution.dataset(ds).query(QueryFactory.create(PREFIXES + query)).build()) { + return qe.execAsk(); + } + } + + @Test + void concreteSubjectIntersectingReturnsItsGeometry() { + List rows = select( + "SELECT ?w WHERE { ex:small geo:asWKT ?w FILTER(geof:sfIntersects(?w, \"" + + REGION + "\"^^geo:wktLiteral)) }", "w"); + assertEquals(1, rows.size(), "pinned-subject spatial query must return its intersecting geometry"); + assertTrue(rows.get(0).startsWith("POLYGON((150 150"), "the bound WKT must be ex:small's geometry"); + } + + @Test + void concreteSubjectAskIsTrue() { + assertTrue(ask("ASK { ex:small geo:asWKT ?w FILTER(geof:sfIntersects(?w, \"" + + REGION + "\"^^geo:wktLiteral)) }")); + } + + @Test + void concreteSubjectDisjointReturnsNothing() { + assertEquals(List.of(), select( + "SELECT ?w WHERE { ex:far geo:asWKT ?w FILTER(geof:sfIntersects(?w, \"" + + REGION + "\"^^geo:wktLiteral)) }", "w")); + assertFalse(ask("ASK { ex:far geo:asWKT ?w FILTER(geof:sfIntersects(?w, \"" + + REGION + "\"^^geo:wktLiteral)) }")); + } + + @Test + void preBoundSubjectFromValuesStillMatches() { + // The subject var is seeded with index candidates while the incoming + // binding already pins it - putCompatible must keep the compatible row. + List rows = select( + "SELECT ?w WHERE { VALUES ?f { ex:small } ?f geo:asWKT ?w FILTER(geof:sfIntersects(?w, \"" + + REGION + "\"^^geo:wktLiteral)) }", "w"); + assertEquals(1, rows.size()); + } + + @Test + void variableSubjectControlStillWorks() { + List rows = select( + "SELECT ?f WHERE { ?f geo:asWKT ?w FILTER(geof:sfIntersects(?w, \"" + + REGION + "\"^^geo:wktLiteral)) }", "f"); + assertEquals(List.of("http://ex.org/small"), rows); + } +} diff --git a/src/test/java/com/ebremer/beakgraph/SpatialCrsIndexingTest.java b/src/test/java/com/ebremer/beakgraph/SpatialCrsIndexingTest.java new file mode 100644 index 00000000..0098cdfa --- /dev/null +++ b/src/test/java/com/ebremer/beakgraph/SpatialCrsIndexingTest.java @@ -0,0 +1,68 @@ +package com.ebremer.beakgraph; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.ebremer.beakgraph.core.BeakGraph; +import com.ebremer.beakgraph.hdf5.readers.HDF5Reader; +import com.ebremer.beakgraph.hdf5.writers.HDF5Writer; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import org.apache.jena.query.Dataset; +import org.apache.jena.query.QueryExecution; +import org.apache.jena.query.QueryFactory; +import org.apache.jena.query.ResultSet; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Regression test for the H8 write path: a wktLiteral carrying the optional + * GeoSPARQL CRS prefix must be spatially indexed. Previously the raw lexical + * form went straight into a JTS WKTReader, failed to parse, and the geometry + * was silently dropped from the index (classified "degenerate"). + */ +class SpatialCrsIndexingTest { + + private static final String TTL = """ + @prefix ex: . + @prefix geo: . + ex:f1 geo:asWKT " POLYGON((0 0,64 0,64 64,0 64,0 0))"^^geo:wktLiteral . + """; + + @TempDir + static Path dir; + static BeakGraph bg; + static Dataset ds; + + @BeforeAll + static void buildAndOpen() throws Exception { + File ttl = dir.resolve("crs.ttl").toFile(); + File h5 = dir.resolve("crs.ttl.h5").toFile(); + Files.write(ttl.toPath(), TTL.getBytes(StandardCharsets.UTF_8)); + HDF5Writer.Builder() + .setSource(ttl).setDestination(h5) + .setSpatial(true).setFeatures(false) + .build().write(); + bg = new BeakGraph(new HDF5Reader(h5)); + ds = bg.getDataset(); + } + + @AfterAll + static void closeReader() { + if (bg != null) bg.close(); + } + + @Test + void crsPrefixedGeometryIsIndexed() { + try (QueryExecution qe = QueryExecution.dataset(ds).query(QueryFactory.create( + "SELECT * WHERE { GRAPH <" + Params.SPATIALSTRING + "> { ?s ?p ?o } }")).build()) { + ResultSet rs = qe.execSelect(); + int n = 0; + while (rs.hasNext()) { rs.next(); n++; } + assertTrue(n > 0, "the spatial graph must contain index rows for a CRS-prefixed geometry"); + } + } +} diff --git a/src/test/java/com/ebremer/beakgraph/SpatialFidelityTest.java b/src/test/java/com/ebremer/beakgraph/SpatialFidelityTest.java new file mode 100644 index 00000000..43c8b40c --- /dev/null +++ b/src/test/java/com/ebremer/beakgraph/SpatialFidelityTest.java @@ -0,0 +1,114 @@ +package com.ebremer.beakgraph; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.ebremer.beakgraph.core.BeakGraph; +import com.ebremer.beakgraph.hdf5.readers.HDF5Reader; +import com.ebremer.beakgraph.hdf5.writers.HDF5Writer; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import org.apache.jena.query.Dataset; +import org.apache.jena.query.QueryExecution; +import org.apache.jena.query.QueryFactory; +import org.apache.jena.query.ResultSet; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Regression tests for spatial write fidelity: + *
      + *
    • polygon holes (donut annotations - lumens in pathology) must survive + * into the scaled WKT pyramid instead of being dropped;
      • + *
    • every part of a MULTIPOLYGON must be indexed, not just the first;
      • + *
    • POINT geometries must be indexed (via their envelope) instead of being + * silently skipped.
      • + *
    + */ +class SpatialFidelityTest { + + private static final String TTL = """ + @prefix ex: . + @prefix geo: . + ex:donut geo:asWKT "POLYGON((0 0,64 0,64 64,0 64,0 0),(16 16,48 16,48 48,16 48,16 16))"^^geo:wktLiteral . + ex:multi geo:asWKT "MULTIPOLYGON(((0 0,32 0,32 32,0 32,0 0)),((1000 1000,1032 1000,1032 1032,1000 1032,1000 1000)))"^^geo:wktLiteral . + ex:pt geo:asWKT "POINT(500 500)"^^geo:wktLiteral . + """; + + @TempDir + static Path dir; + static BeakGraph bg; + static Dataset ds; + + @BeforeAll + static void buildAndOpen() throws Exception { + File ttl = dir.resolve("fidelity.ttl").toFile(); + File h5 = dir.resolve("fidelity.ttl.h5").toFile(); + Files.write(ttl.toPath(), TTL.getBytes(StandardCharsets.UTF_8)); + HDF5Writer.Builder().setSource(ttl).setDestination(h5) + .setSpatial(true).setFeatures(false).build().write(); + bg = new BeakGraph(new HDF5Reader(h5)); + ds = bg.getDataset(); + } + + @AfterAll + static void closeReader() { + if (bg != null) bg.close(); + } + + private static int count(String where) { + try (QueryExecution qe = QueryExecution.dataset(ds).query(QueryFactory.create( + "PREFIX ex: PREFIX hal: SELECT * WHERE { " + where + " }")).build()) { + ResultSet rs = qe.execSelect(); + int n = 0; + while (rs.hasNext()) { rs.next(); n++; } + return n; + } + } + + private static String levelZeroWkt(String subject) { + try (QueryExecution qe = QueryExecution.dataset(ds).query(QueryFactory.create( + "PREFIX ex: PREFIX hal: " + + "SELECT ?w WHERE { GRAPH <" + Params.SPATIALSTRING + "> { " + subject + " hal:asWKT0 ?w } }")).build()) { + ResultSet rs = qe.execSelect(); + assertTrue(rs.hasNext(), "expected a level-0 scaled WKT for " + subject); + return rs.next().getLiteral("w").getLexicalForm(); + } + } + + @Test + void polygonHolesSurviveScaling() { + String wkt0 = levelZeroWkt("ex:donut"); + // A polygon with an interior ring serializes with two coordinate lists. + assertTrue(wkt0.indexOf('(') != wkt0.lastIndexOf('('), + "level-0 WKT must be a polygon"); + assertTrue(wkt0.contains("), ("), + "the donut's interior ring must survive into the scaled WKT, got: " + wkt0); + } + + @Test + void allMultiPolygonPartsAreIndexed() { + // A query region overlapping ONLY the second part must still find the + // geometry - first-part-only indexing left the other parts unfindable. + String q = "PREFIX ex: " + + "PREFIX geo: " + + "PREFIX geof: " + + "SELECT ?f WHERE { ?f geo:asWKT ?w FILTER(geof:sfIntersects(?w, " + + "\"POLYGON((1010 1010,1020 1010,1020 1020,1010 1020,1010 1010))\"^^geo:wktLiteral)) }"; + try (QueryExecution qe = QueryExecution.dataset(ds).query(QueryFactory.create(q)).build()) { + ResultSet rs = qe.execSelect(); + assertTrue(rs.hasNext(), "the second MULTIPOLYGON part must be findable"); + assertEquals("http://ex.org/multi", rs.next().get("f").asResource().getURI()); + } + } + + @Test + void pointGeometriesAreIndexed() { + assertTrue(count("GRAPH <" + Params.SPATIALSTRING + "> { ex:pt ?p ?o }") > 0, + "a POINT must be indexed via its envelope, not silently skipped"); + } +} diff --git a/src/test/java/com/ebremer/beakgraph/SpatialGeometryCollectionTest.java b/src/test/java/com/ebremer/beakgraph/SpatialGeometryCollectionTest.java new file mode 100644 index 00000000..420d1a71 --- /dev/null +++ b/src/test/java/com/ebremer/beakgraph/SpatialGeometryCollectionTest.java @@ -0,0 +1,94 @@ +package com.ebremer.beakgraph; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.ebremer.beakgraph.core.BeakGraph; +import com.ebremer.beakgraph.hdf5.readers.HDF5Reader; +import com.ebremer.beakgraph.hdf5.writers.HDF5Writer; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.Set; +import org.apache.jena.query.Dataset; +import org.apache.jena.query.QueryExecution; +import org.apache.jena.query.QueryFactory; +import org.apache.jena.query.ResultSet; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * EVERY member of a mixed GEOMETRYCOLLECTION must be spatially indexed. The + * envelope fallback for non-polygonal geometry used to fire only when NO + * polygonal part existed, so a point member sitting next to a polygon member + * was silently unindexed - invisible to every sfIntersects query. + */ +class SpatialGeometryCollectionTest { + + private static final String TTL = """ + @prefix ex: . + @prefix geo: . + ex:mixed geo:asWKT "GEOMETRYCOLLECTION(POLYGON((0 0,10 0,10 10,0 10,0 0)), POINT(500 500))"^^geo:wktLiteral . + """; + + private static final String PREFIXES = """ + PREFIX ex: + PREFIX geo: + PREFIX geof: + """; + + @TempDir + static Path dir; + static BeakGraph bg; + static Dataset ds; + + @BeforeAll + static void buildAndOpen() throws Exception { + File ttl = dir.resolve("mixed.ttl").toFile(); + File h5 = dir.resolve("mixed.ttl.h5").toFile(); + Files.write(ttl.toPath(), TTL.getBytes(StandardCharsets.UTF_8)); + HDF5Writer.Builder().setSource(ttl).setDestination(h5) + .setSpatial(true).setFeatures(false).build().write(); + bg = new BeakGraph(new HDF5Reader(h5)); + ds = bg.getDataset(); + } + + @AfterAll + static void closeReader() { + if (bg != null) bg.close(); + } + + private static Set intersecting(String regionWkt) { + Set hits = new HashSet<>(); + String q = PREFIXES + + "SELECT ?f WHERE { ?f geo:asWKT ?w FILTER(geof:sfIntersects(?w, \"" + regionWkt + "\"^^geo:wktLiteral)) }"; + try (QueryExecution qe = QueryExecution.dataset(ds).query(QueryFactory.create(q)).build()) { + ResultSet rs = qe.execSelect(); + while (rs.hasNext()) { + hits.add(rs.next().getResource("f").getURI()); + } + } + return hits; + } + + @Test + void pointMemberNextToAPolygonMemberIsFindable() { + assertEquals(Set.of("http://ex.org/mixed"), + intersecting("POLYGON((495 495,505 495,505 505,495 505,495 495))")); + } + + @Test + void polygonMemberIsStillFindable() { + assertEquals(Set.of("http://ex.org/mixed"), + intersecting("POLYGON((2 2,8 2,8 8,2 8,2 2))")); + } + + @Test + void regionTouchingNeitherMemberMatchesNothing() { + assertEquals(Set.of(), + intersecting("POLYGON((100 100,200 100,200 200,100 200,100 100))")); + } +} diff --git a/src/test/java/com/ebremer/beakgraph/SpatialHugeCoordinateTest.java b/src/test/java/com/ebremer/beakgraph/SpatialHugeCoordinateTest.java new file mode 100644 index 00000000..bb159448 --- /dev/null +++ b/src/test/java/com/ebremer/beakgraph/SpatialHugeCoordinateTest.java @@ -0,0 +1,103 @@ +package com.ebremer.beakgraph; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.ebremer.beakgraph.core.BeakGraph; +import com.ebremer.beakgraph.hdf5.readers.HDF5Reader; +import com.ebremer.beakgraph.hdf5.writers.HDF5Writer; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.Locale; +import java.util.Set; +import org.apache.jena.query.Dataset; +import org.apache.jena.query.QueryExecution; +import org.apache.jena.query.QueryFactory; +import org.apache.jena.query.ResultSet; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Coordinates at or beyond 2^31 exceed the 31-bit Hilbert curve's domain. The + * davidmoten curve silently MASKS such coordinates onto unrelated cells, so + * the written cover and the queried cover aliased differently and genuine + * intersections were silently lost. Both sides now clamp into the domain with + * the same monotone projection: out-of-domain geometry indexes coarsely at + * the domain-edge cells (recall-safe) and JTS verification does the rest. + */ +class SpatialHugeCoordinateTest { + + private static final long BASE = 1L << 31; + + private static final String PREFIXES = """ + PREFIX ex: + PREFIX geo: + PREFIX geof: + """; + + @TempDir + static Path dir; + static BeakGraph bg; + static Dataset ds; + + private static String square(long minX, long minY, long size) { + long maxX = minX + size, maxY = minY + size; + return String.format(Locale.ROOT, "POLYGON((%d %d,%d %d,%d %d,%d %d,%d %d))", + minX, minY, maxX, minY, maxX, maxY, minX, maxY, minX, minY); + } + + @BeforeAll + static void buildAndOpen() throws Exception { + String ttl = "@prefix ex: .\n" + + "@prefix geo: .\n" + + "ex:huge geo:asWKT \"" + square(BASE + 100, BASE + 100, 100) + "\"^^geo:wktLiteral .\n" + + "ex:small geo:asWKT \"" + square(500, 500, 60) + "\"^^geo:wktLiteral .\n"; + File src = dir.resolve("huge.ttl").toFile(); + File h5 = dir.resolve("huge.ttl.h5").toFile(); + Files.write(src.toPath(), ttl.getBytes(StandardCharsets.UTF_8)); + HDF5Writer.Builder().setSource(src).setDestination(h5) + .setSpatial(true).setFeatures(false).build().write(); + bg = new BeakGraph(new HDF5Reader(h5)); + ds = bg.getDataset(); + } + + @AfterAll + static void closeReader() { + if (bg != null) bg.close(); + } + + private static Set intersecting(String regionWkt) { + Set hits = new HashSet<>(); + String q = PREFIXES + + "SELECT ?f WHERE { ?f geo:asWKT ?w FILTER(geof:sfIntersects(?w, \"" + regionWkt + "\"^^geo:wktLiteral)) }"; + try (QueryExecution qe = QueryExecution.dataset(ds).query(QueryFactory.create(q)).build()) { + ResultSet rs = qe.execSelect(); + while (rs.hasNext()) { + hits.add(rs.next().getResource("f").getURI()); + } + } + return hits; + } + + @Test + void beyondDomainGeometryIsFoundByOverlappingRegion() { + assertEquals(Set.of("http://ex.org/huge"), + intersecting(square(BASE + 50, BASE + 50, 100))); + } + + @Test + void beyondDomainRegionWithNoOverlapMatchesNothing() { + // Both clamp onto the domain-edge cells (candidates!), so the JTS + // verification must reject the non-overlapping pair. + assertEquals(Set.of(), intersecting(square(BASE + 10_000, BASE + 10_000, 100))); + } + + @Test + void inDomainQueriesAreUnaffected() { + assertEquals(Set.of("http://ex.org/small"), intersecting(square(480, 480, 60))); + } +} diff --git a/src/test/java/com/ebremer/beakgraph/SpatialNegativeCoordinateTest.java b/src/test/java/com/ebremer/beakgraph/SpatialNegativeCoordinateTest.java new file mode 100644 index 00000000..a7e5b093 --- /dev/null +++ b/src/test/java/com/ebremer/beakgraph/SpatialNegativeCoordinateTest.java @@ -0,0 +1,119 @@ +package com.ebremer.beakgraph; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.ebremer.beakgraph.core.BeakGraph; +import com.ebremer.beakgraph.hdf5.readers.HDF5Reader; +import com.ebremer.beakgraph.hdf5.writers.HDF5Writer; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.Set; +import org.apache.jena.query.Dataset; +import org.apache.jena.query.QueryExecution; +import org.apache.jena.query.QueryFactory; +import org.apache.jena.query.ResultSet; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Spatial correctness for geometry and query regions in NEGATIVE coordinate + * space (real GeoSPARQL lon/lat data lives there). The Hilbert domain is + * non-negative, so both sides clamp their bboxes into it with the same + * monotone projection - overlap survives the clamp and the exact JTS + * verification removes the clamp's false positives. Before the fix, + * fully-negative geometry was skipped at build (unfindable by any query) and + * negative query regions bailed out to zero candidates: genuine intersections + * were silent false negatives. + */ +class SpatialNegativeCoordinateTest { + + private static final String TTL = """ + @prefix ex: . + @prefix geo: . + # entirely in negative space + ex:neg geo:asWKT "POLYGON((-100 -100,-50 -100,-50 -50,-100 -50,-100 -100))"^^geo:wktLiteral . + # straddles the origin + ex:strad geo:asWKT "POLYGON((-10 -10,10 -10,10 10,-10 10,-10 -10))"^^geo:wktLiteral . + # entirely positive, far from the origin + ex:pos geo:asWKT "POLYGON((500 500,560 500,560 560,500 560,500 500))"^^geo:wktLiteral . + """; + + private static final String PREFIXES = """ + PREFIX ex: + PREFIX geo: + PREFIX geof: + """; + + @TempDir + static Path dir; + static BeakGraph bg; + static Dataset ds; + + @BeforeAll + static void buildAndOpen() throws Exception { + File ttl = dir.resolve("negspace.ttl").toFile(); + File h5 = dir.resolve("negspace.ttl.h5").toFile(); + Files.write(ttl.toPath(), TTL.getBytes(StandardCharsets.UTF_8)); + HDF5Writer.Builder().setSource(ttl).setDestination(h5) + .setSpatial(true).setFeatures(false).build().write(); + bg = new BeakGraph(new HDF5Reader(h5)); + ds = bg.getDataset(); + } + + @AfterAll + static void closeReader() { + if (bg != null) bg.close(); + } + + private static Set intersecting(String regionWkt) { + Set hits = new HashSet<>(); + String q = PREFIXES + + "SELECT ?f WHERE { ?f geo:asWKT ?w FILTER(geof:sfIntersects(?w, \"" + regionWkt + "\"^^geo:wktLiteral)) }"; + try (QueryExecution qe = QueryExecution.dataset(ds).query(QueryFactory.create(q)).build()) { + ResultSet rs = qe.execSelect(); + while (rs.hasNext()) { + hits.add(rs.next().getResource("f").getURI()); + } + } + return hits; + } + + @Test + void negativeRegionFindsFullyNegativeGeometry() { + assertEquals(Set.of("http://ex.org/neg"), + intersecting("POLYGON((-90 -90,-60 -90,-60 -60,-90 -60,-90 -90))")); + } + + @Test + void negativeRegionFindsStraddlingGeometry() { + assertEquals(Set.of("http://ex.org/strad"), + intersecting("POLYGON((-8 -8,-2 -8,-2 -2,-8 -2,-8 -8))")); + } + + @Test + void straddlingRegionFindsBothNegativeAndStraddlingGeometry() { + assertEquals(Set.of("http://ex.org/neg", "http://ex.org/strad"), + intersecting("POLYGON((-60 -60,5 -60,5 5,-60 5,-60 -60))")); + } + + @Test + void positiveRegionIsUnaffectedByOriginClustering() { + // The clamped negative geometries cluster at the axis cells; a positive + // region far away must still return only its true intersections. + assertEquals(Set.of("http://ex.org/pos"), + intersecting("POLYGON((520 520,540 520,540 540,520 540,520 520))")); + } + + @Test + void disjointNegativeRegionMatchesNothing() { + // Clamping makes every negative-space geometry a CANDIDATE here; the + // JTS verification stage must reject them all. + assertEquals(Set.of(), + intersecting("POLYGON((-2000 -2000,-1900 -2000,-1900 -1900,-2000 -1900,-2000 -2000))")); + } +} diff --git a/src/test/java/com/ebremer/beakgraph/SpatialQueryCorrectnessTest.java b/src/test/java/com/ebremer/beakgraph/SpatialQueryCorrectnessTest.java new file mode 100644 index 00000000..b23d84f4 --- /dev/null +++ b/src/test/java/com/ebremer/beakgraph/SpatialQueryCorrectnessTest.java @@ -0,0 +1,136 @@ +package com.ebremer.beakgraph; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.ebremer.beakgraph.core.BeakGraph; +import com.ebremer.beakgraph.hdf5.readers.HDF5Reader; +import com.ebremer.beakgraph.hdf5.writers.HDF5Writer; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.apache.jena.query.Dataset; +import org.apache.jena.query.QueryExecution; +import org.apache.jena.query.QueryFactory; +import org.apache.jena.query.ResultSet; +import org.apache.jena.rdf.model.RDFNode; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * End-to-end correctness tests for the spatial index + geof:sfIntersects. + * The index must be a recall-safe PRE-FILTER (bbox cell-cover overlap - never a + * false negative) and the JTS-backed sfIntersects filter must stay in the plan + * as the verification stage (killing the index's false positives). The old + * corner-only design failed both ways: a geometry containing or straddling the + * query region never matched, candidates were never re-verified, and one match + * could appear up to four times. + */ +class SpatialQueryCorrectnessTest { + + // Query region: the square [100,300] x [100,300]. + private static final String REGION = "POLYGON((100 100,300 100,300 300,100 300,100 100))"; + + private static final String TTL = """ + @prefix ex: . + @prefix geo: . + # entirely inside the region + ex:small geo:asWKT "POLYGON((150 150,160 150,160 160,150 160,150 150))"^^geo:wktLiteral . + # CONTAINS the whole region - the classic corner-index false negative + ex:big geo:asWKT "POLYGON((0 0,1000 0,1000 1000,0 1000,0 0))"^^geo:wktLiteral . + # straddles the region's left edge - corners outside, overlap real + ex:strad geo:asWKT "POLYGON((50 120,150 120,150 200,50 200,50 120))"^^geo:wktLiteral . + # far away - bbox disjoint, excluded by the index + ex:far geo:asWKT "POLYGON((5000 5000,5100 5000,5100 5100,5000 5100,5000 5000))"^^geo:wktLiteral . + # bbox overlaps the region corner but the triangle itself does not + # (x + y >= 620 everywhere; the region maxes out at 600) - an index + # candidate that the JTS verification stage must reject + ex:trap geo:asWKT "POLYGON((260 360,360 260,370 370,260 360))"^^geo:wktLiteral . + # residual-filter sanity data + ex:n1 ex:name "axb" . + ex:n2 ex:name "ab" . + """; + + private static final String PREFIXES = """ + PREFIX ex: + PREFIX geo: + PREFIX geof: + """; + + @TempDir + static Path dir; + static BeakGraph bg; + static Dataset ds; + + @BeforeAll + static void buildAndOpen() throws Exception { + File ttl = dir.resolve("spatialq.ttl").toFile(); + File h5 = dir.resolve("spatialq.ttl.h5").toFile(); + Files.write(ttl.toPath(), TTL.getBytes(StandardCharsets.UTF_8)); + HDF5Writer.Builder().setSource(ttl).setDestination(h5) + .setSpatial(true).setFeatures(false).build().write(); + bg = new BeakGraph(new HDF5Reader(h5)); + ds = bg.getDataset(); + } + + @AfterAll + static void closeReader() { + if (bg != null) bg.close(); + } + + private static List intersecting(String regionWkt) { + List hits = new ArrayList<>(); + String q = PREFIXES + + "SELECT ?f WHERE { ?f geo:asWKT ?w FILTER(geof:sfIntersects(?w, \"" + regionWkt + "\"^^geo:wktLiteral)) }"; + try (QueryExecution qe = QueryExecution.dataset(ds).query(QueryFactory.create(q)).build()) { + ResultSet rs = qe.execSelect(); + while (rs.hasNext()) { + RDFNode n = rs.next().get("f"); + if (n != null && n.isURIResource()) hits.add(n.asResource().getURI()); + } + } + return hits; + } + + @Test + void intersectionMatchesGeometryNotCorners() { + List hits = intersecting(REGION); + Set distinct = new HashSet<>(hits); + assertEquals(Set.of("http://ex.org/small", "http://ex.org/big", "http://ex.org/strad"), distinct, + "containing and straddling geometries must match; disjoint and bbox-only-overlap must not"); + assertEquals(distinct.size(), hits.size(), + "each matching geometry must appear exactly once (no per-corner/per-range duplicates)"); + } + + @Test + void containingGeometryAloneIsFound() { + // A region buried deep inside only ex:big - zero stored corners or cells + // of ex:big lie anywhere near it; only a recall-safe cover finds it. + assertEquals(List.of("http://ex.org/big"), + intersecting("POLYGON((600 600,610 600,610 610,600 610,600 600))")); + } + + @Test + void disjointRegionMatchesNothing() { + assertEquals(List.of(), intersecting("POLYGON((8000 8000,8100 8000,8100 8100,8000 8100,8000 8000))")); + } + + @Test + void residualFiltersStillApplyToOrdinaryPatterns() { + // Sanity guard for the same plan machinery the verify stage rides on: + // a non-pushdownable FILTER (regex) must still be evaluated. + String q = "PREFIX ex: SELECT ?s WHERE { ?s ex:name ?n FILTER(regex(?n, \"x\")) }"; + List hits = new ArrayList<>(); + try (QueryExecution qe = QueryExecution.dataset(ds).query(QueryFactory.create(q)).build()) { + ResultSet rs = qe.execSelect(); + while (rs.hasNext()) hits.add(rs.next().getResource("s").getURI()); + } + assertEquals(List.of("http://ex.org/n1"), hits); + } +} diff --git a/src/test/java/com/ebremer/beakgraph/TemporalTotalOrderTest.java b/src/test/java/com/ebremer/beakgraph/TemporalTotalOrderTest.java new file mode 100644 index 00000000..7e3933eb --- /dev/null +++ b/src/test/java/com/ebremer/beakgraph/TemporalTotalOrderTest.java @@ -0,0 +1,212 @@ +package com.ebremer.beakgraph; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.ebremer.beakgraph.core.BeakGraph; +import com.ebremer.beakgraph.core.lib.NodeComparator; +import com.ebremer.beakgraph.hdf5.readers.HDF5Reader; +import com.ebremer.beakgraph.hdf5.writers.HDF5Writer; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.apache.jena.datatypes.xsd.XSDDatatype; +import org.apache.jena.graph.Node; +import org.apache.jena.graph.NodeFactory; +import org.apache.jena.query.Dataset; +import org.apache.jena.query.QueryExecution; +import org.apache.jena.query.QueryFactory; +import org.apache.jena.query.ResultSet; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * NodeComparator must be a strict total order - every sort and binary search in + * the store depends on it. NodeValue.compareAlways alone is NOT one: for + * XSD-indeterminate pairs (a timezone-less dateTime vs a timezoned one within + * +/-14h, a month-based duration vs a day-based one) it falls back to term + * order pairwise, which mixes two different orders and creates comparison + * cycles. Before the fix, a file with ordinary mixed local/UTC timestamps + * failed to build ("Cannot resolve Object (not in dictionary)") or, worse, + * built and then silently dropped rows on lookup. + */ +class TemporalTotalOrderTest { + + private static final NodeComparator CMP = NodeComparator.INSTANCE; + + private static Node dt(String lex) { return NodeFactory.createLiteralDT(lex, XSDDatatype.XSDdateTime); } + private static Node dur(String lex) { return NodeFactory.createLiteralDT(lex, XSDDatatype.XSDduration); } + + /** The reproduced 3-cycle: naive < +14:00 < Z < naive under the old comparator. */ + @Test + void reportedDateTimeCycleIsGone() { + assertTotallyOrdered(List.of( + dt("2020-01-02T00:00:00"), + dt("2020-01-02T08:00:00+14:00"), + dt("2020-01-01T20:00:00Z"))); + } + + @Test + void durationCycleIsGone() { + // P1Y < P13M by value, but P13M and P15D (and P15D and P1Y) were + // month-vs-day indeterminate and fell back to term order: a cycle. + assertTotallyOrdered(List.of(dur("P1Y"), dur("P13M"), dur("P15D"))); + } + + @Test + void comparatorIsTotalOverMixedLiteralSpaces() { + List nodes = new ArrayList<>(); + // dateTimes: naive, UTC, and offsets all within each other's +/-14h windows + for (int i = 0; i < 8; i++) { + nodes.add(dt(String.format("2020-01-0%dT0%d:00:00", 1 + i % 3, i))); + nodes.add(dt(String.format("2020-01-0%dT0%d:30:00Z", 1 + i % 3, i))); + nodes.add(dt(String.format("2020-01-0%dT0%d:15:00+0%d:00", 1 + i % 3, i, 1 + i % 9))); + nodes.add(dt(String.format("2020-01-0%dT0%d:45:00-0%d:30", 1 + i % 3, i, 1 + i % 9))); + } + // instant-equal, term-distinct dateTimes must order deterministically + nodes.add(dt("2020-06-01T12:00:00Z")); + nodes.add(dt("2020-06-01T12:00:00+00:00")); + nodes.add(dt("2020-06-01T14:00:00+02:00")); + // times and dates with and without timezones + for (int i = 0; i < 4; i++) { + nodes.add(NodeFactory.createLiteralDT(String.format("0%d:00:00", 2 * i), XSDDatatype.XSDtime)); + nodes.add(NodeFactory.createLiteralDT(String.format("0%d:00:00+1%d:00", 2 * i, i), XSDDatatype.XSDtime)); + nodes.add(NodeFactory.createLiteralDT(String.format("2020-01-1%d", i), XSDDatatype.XSDdate)); + nodes.add(NodeFactory.createLiteralDT(String.format("2020-01-1%d+14:00", i), XSDDatatype.XSDdate)); + nodes.add(NodeFactory.createLiteralDT(String.format("201%d", i), XSDDatatype.XSDgYear)); + nodes.add(NodeFactory.createLiteralDT(String.format("201%dZ", i), XSDDatatype.XSDgYear)); + } + // durations: month-based, day/time-based, mixed, value-equal pair (P1D/PT24H) + for (String d : new String[]{"P1Y", "P13M", "P15D", "P1M", "P100D", "P2M", "P30D", + "P1M10D", "PT24H", "P1D", "PT36H", "P400D", "P1Y1D", "-P1M", "-P20D"}) { + nodes.add(dur(d)); + } + // neighbors from other value spaces (space-ranked by compareAlways) + nodes.add(NodeFactory.createLiteralDT("2020-01-02T04:00:00", XSDDatatype.XSDstring)); + nodes.add(NodeFactory.createLiteralDT("zzz", XSDDatatype.XSDstring)); + nodes.add(NodeFactory.createLiteralDT("42", XSDDatatype.XSDinteger)); + nodes.add(NodeFactory.createLiteralDT("41.5", XSDDatatype.XSDdouble)); + nodes.add(NodeFactory.createLiteralDT("not-a-date", XSDDatatype.XSDdateTime)); // ill-formed + nodes.add(NodeFactory.createLiteral("plain")); + + assertTotallyOrdered(nodes); + + // The property that actually broke: sort, then find EVERY element again. + Node[] sorted = nodes.toArray(Node[]::new); + Arrays.sort(sorted, CMP); + for (Node n : nodes) { + assertTrue(Arrays.binarySearch(sorted, n, CMP) >= 0, + "binary search must find every sorted element, missed: " + n); + } + } + + private static void assertTotallyOrdered(List nodes) { + int n = nodes.size(); + for (int i = 0; i < n; i++) { + for (int j = 0; j < n; j++) { + int ij = Integer.signum(CMP.compare(nodes.get(i), nodes.get(j))); + int ji = Integer.signum(CMP.compare(nodes.get(j), nodes.get(i))); + assertEquals(-ji, ij, "antisymmetry violated for " + nodes.get(i) + " / " + nodes.get(j)); + if (i == j) continue; + for (int k = 0; k < n; k++) { + int jk = Integer.signum(CMP.compare(nodes.get(j), nodes.get(k))); + int ik = Integer.signum(CMP.compare(nodes.get(i), nodes.get(k))); + if (ij < 0 && jk < 0) { + assertTrue(ik < 0, "transitivity violated: " + nodes.get(i) + " < " + + nodes.get(j) + " < " + nodes.get(k) + " but not i < k"); + } + } + } + } + } + + @TempDir + Path dir; + + /** + * End-to-end: the exact failure mode - hundreds of mixed timezone-less and + * timezoned dateTimes in one file. Under the cyclic comparator this build + * threw "Cannot resolve Object (not in dictionary)" at index construction. + */ + @Test + void mixedTimezoneDateTimesRoundTrip() throws Exception { + int count = 300; + StringBuilder ttl = new StringBuilder("@prefix ex: .\n" + + "@prefix xsd: .\n"); + for (int i = 0; i < count; i++) { + int day = 1 + (i / 24) % 27; + int hour = i % 24; + int minute = (i * 7) % 60; + String lex = String.format("2020-01-%02dT%02d:%02d:00", day, hour, minute); + String form = switch (i % 3) { + case 0 -> lex; + case 1 -> lex + "Z"; + default -> lex + String.format("%s%02d:00", (i % 2 == 0 ? "+" : "-"), 1 + i % 13); + }; + ttl.append(String.format("ex:s%d ex:when \"%s\"^^xsd:dateTime .%n", i, form)); + } + + File src = dir.resolve("mixedtz.ttl").toFile(); + File h5 = dir.resolve("mixedtz.ttl.h5").toFile(); + Files.write(src.toPath(), ttl.toString().getBytes(StandardCharsets.UTF_8)); + HDF5Writer.Builder().setSource(src).setDestination(h5) + .setSpatial(false).setFeatures(false).build().write(); + + try (BeakGraph bg = new BeakGraph(new HDF5Reader(h5))) { + Dataset ds = bg.getDataset(); + assertEquals(count, countRows(ds, + "PREFIX ex: SELECT ?s ?o WHERE { ?s ex:when ?o }"), + "every mixed-timezone dateTime row must round-trip"); + // concrete-literal lookups exercise the read-side binary search on both forms + assertEquals(1, countRows(ds, "PREFIX ex: " + + "PREFIX xsd: " + + "SELECT ?s WHERE { ?s ex:when \"2020-01-01T00:00:00\"^^xsd:dateTime }")); + assertEquals(1, countRows(ds, "PREFIX ex: " + + "PREFIX xsd: " + + "SELECT ?s WHERE { ?s ex:when \"2020-01-01T01:07:00Z\"^^xsd:dateTime }")); + } + } + + @Test + void mixedDurationsRoundTrip() throws Exception { + String[] durations = {"P1Y", "P13M", "P15D", "P1M", "P100D", "P2M", "P30D", "P28D", + "P1M10D", "PT24H", "P1D", "PT36H", "P400D", "P1Y1D", "P9M", "P276D", + "-P1M", "-P20D", "P3D", "P20D", "PT240H", "P10D", "P1M9D", "P38D"}; + StringBuilder ttl = new StringBuilder("@prefix ex: .\n" + + "@prefix xsd: .\n"); + for (int i = 0; i < durations.length; i++) { + ttl.append(String.format("ex:d%d ex:lasts \"%s\"^^xsd:duration .%n", i, durations[i])); + } + + File src = dir.resolve("durations.ttl").toFile(); + File h5 = dir.resolve("durations.ttl.h5").toFile(); + Files.write(src.toPath(), ttl.toString().getBytes(StandardCharsets.UTF_8)); + HDF5Writer.Builder().setSource(src).setDestination(h5) + .setSpatial(false).setFeatures(false).build().write(); + + try (BeakGraph bg = new BeakGraph(new HDF5Reader(h5))) { + Dataset ds = bg.getDataset(); + assertEquals(durations.length, countRows(ds, + "PREFIX ex: SELECT ?s ?o WHERE { ?s ex:lasts ?o }")); + assertEquals(1, countRows(ds, "PREFIX ex: " + + "PREFIX xsd: " + + "SELECT ?s WHERE { ?s ex:lasts \"P13M\"^^xsd:duration }")); + } + } + + private static int countRows(Dataset ds, String query) { + try (QueryExecution qe = QueryExecution.dataset(ds).query(QueryFactory.create(query)).build()) { + ResultSet rs = qe.execSelect(); + int rows = 0; + while (rs.hasNext()) { + rs.next(); + rows++; + } + return rows; + } + } +} diff --git a/src/test/java/com/ebremer/beakgraph/TransactionStateTest.java b/src/test/java/com/ebremer/beakgraph/TransactionStateTest.java new file mode 100644 index 00000000..f50f03e7 --- /dev/null +++ b/src/test/java/com/ebremer/beakgraph/TransactionStateTest.java @@ -0,0 +1,82 @@ +package com.ebremer.beakgraph; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.ebremer.beakgraph.core.BeakGraph; +import com.ebremer.beakgraph.hdf5.readers.HDF5Reader; +import com.ebremer.beakgraph.hdf5.writers.HDF5Writer; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import org.apache.jena.query.ReadWrite; +import org.apache.jena.sparql.core.DatasetGraph; +import org.apache.jena.system.Txn; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * isInTransaction() must report REAL per-thread state. The old implementation + * unconditionally answered true - a contract lie that masked mispaired + * begin/end in callers (a skipped begin, an end without begin, and a nested + * begin all passed silently). + */ +class TransactionStateTest { + + @TempDir + static Path dir; + static BeakGraph bg; + static DatasetGraph dsg; + + @BeforeAll + static void buildAndOpen() throws Exception { + File ttl = dir.resolve("txn.ttl").toFile(); + File h5 = dir.resolve("txn.ttl.h5").toFile(); + Files.write(ttl.toPath(), + " .\n".getBytes(StandardCharsets.UTF_8)); + HDF5Writer.Builder().setSource(ttl).setDestination(h5) + .setSpatial(false).setFeatures(false).build().write(); + bg = new BeakGraph(new HDF5Reader(h5)); + dsg = bg.getDataset().asDatasetGraph(); + } + + @AfterAll + static void closeReader() { + if (bg != null) bg.close(); + } + + @Test + void transactionStateFollowsTheLifecycle() { + assertFalse(dsg.isInTransaction(), "no transaction has begun"); + dsg.begin(ReadWrite.READ); + assertTrue(dsg.isInTransaction()); + assertEquals(ReadWrite.READ, dsg.transactionMode()); + dsg.end(); + assertFalse(dsg.isInTransaction(), "end() must leave the transaction"); + + dsg.begin(ReadWrite.READ); + dsg.commit(); + dsg.end(); + assertFalse(dsg.isInTransaction()); + } + + @Test + void txnExecuteReadWorks() { + long[] count = {0}; + Txn.executeRead(dsg, () -> + dsg.getDefaultGraph().find().forEachRemaining(t -> count[0]++)); + assertTrue(count[0] >= 1); + assertFalse(dsg.isInTransaction()); + } + + @Test + void writeTransactionsAreRejected() { + assertThrows(UnsupportedOperationException.class, () -> dsg.begin(ReadWrite.WRITE)); + assertFalse(dsg.isInTransaction(), "a rejected begin must not leave txn state behind"); + } +} diff --git a/src/test/java/com/ebremer/beakgraph/UnionGraphAndDetachTest.java b/src/test/java/com/ebremer/beakgraph/UnionGraphAndDetachTest.java new file mode 100644 index 00000000..bd24fcf6 --- /dev/null +++ b/src/test/java/com/ebremer/beakgraph/UnionGraphAndDetachTest.java @@ -0,0 +1,163 @@ +package com.ebremer.beakgraph; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.ebremer.beakgraph.core.BeakGraph; +import com.ebremer.beakgraph.hdf5.jena.BindingBG; +import com.ebremer.beakgraph.hdf5.jena.BindingNodeId; +import com.ebremer.beakgraph.hdf5.jena.NodeId; +import com.ebremer.beakgraph.hdf5.readers.HDF5Reader; +import com.ebremer.beakgraph.hdf5.writers.HDF5Writer; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Iterator; +import org.apache.jena.graph.Node; +import org.apache.jena.graph.NodeFactory; +import org.apache.jena.query.Dataset; +import org.apache.jena.query.QueryExecution; +import org.apache.jena.query.QueryFactory; +import org.apache.jena.query.ResultSet; +import org.apache.jena.sparql.core.Quad; +import org.apache.jena.sparql.core.Var; +import org.apache.jena.sparql.engine.binding.Binding; +import org.apache.jena.sparql.engine.binding.BindingFactory; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Regression tests for three dataset/binding contract gaps: + *
      + *
    • {@code GRAPH } must query the union of all named + * graphs (it silently returned empty).
      • + *
    • {@code findNG} must return named-graph quads only (it delegated to + * {@code find}, which prepends the default graph).
      • + *
    • {@code BindingBG.detachWithNewParent} must materialize the + * NodeId-backed bindings (it threw UnsupportedOperationException, which + * breaks Jena's binding detach/copy paths).
      • + *
    + */ +class UnionGraphAndDetachTest { + + private static final String TTL = """ + @prefix ex: . + ex:s1 ex:p ex:o1 . + ex:s2 ex:p "v" . + """; + + private static final int DEFAULT_GRAPH_TRIPLES = 2; + // Use Jena's constant - the URI is urn:x-arq:UnionGraph (capital U), and a + // hand-typed variant silently names a non-existent graph. + private static final String UNION = Quad.unionGraph.getURI(); + + @TempDir + static Path dir; + static BeakGraph bg; + static Dataset ds; + + @BeforeAll + static void buildAndOpen() throws Exception { + File ttl = dir.resolve("union.ttl").toFile(); + File h5 = dir.resolve("union.ttl.h5").toFile(); + Files.write(ttl.toPath(), TTL.getBytes(StandardCharsets.UTF_8)); + HDF5Writer.Builder() + .setSource(ttl).setDestination(h5) + .setSpatial(false).setFeatures(false) + .build().write(); + bg = new BeakGraph(new HDF5Reader(h5)); + ds = bg.getDataset(); + } + + @AfterAll + static void closeReader() { + if (bg != null) bg.close(); + } + + private static int count(String query) { + try (QueryExecution qe = QueryExecution.dataset(ds) + .query(QueryFactory.create("PREFIX ex: " + query)).build()) { + ResultSet rs = qe.execSelect(); + int n = 0; + while (rs.hasNext()) { rs.next(); n++; } + return n; + } + } + + private static int voidGraphSize() { + return count("SELECT * WHERE { GRAPH <" + Params.VOIDSTRING + "> { ?s ?p ?o } }"); + } + + // --- union default graph ------------------------------------------------- + + @Test + void unionGraphQueriesAllNamedGraphs() { + int voidCount = voidGraphSize(); + assertTrue(voidCount > 0, "control: the VoID named graph is non-empty"); + assertEquals(voidCount, + count("SELECT * WHERE { GRAPH <" + UNION + "> { ?s ?p ?o } }"), + "the union graph must contain exactly the named-graph triples"); + } + + @Test + void unionGraphExcludesDefaultGraphTriples() { + try (QueryExecution qe = QueryExecution.dataset(ds).query(QueryFactory.create( + "PREFIX ex: ASK { GRAPH <" + UNION + "> { ex:s1 ex:p ex:o1 } }")).build()) { + assertFalse(qe.execAsk(), "default-graph triples are not part of the union of named graphs"); + } + } + + @Test + void getUnionModelWorks() { + assertEquals(voidGraphSize(), ds.getUnionModel().size()); + } + + // --- findNG --------------------------------------------------------------- + + @Test + void findNGReturnsNamedGraphQuadsOnly() { + Iterator it = ds.asDatasetGraph().findNG(Node.ANY, Node.ANY, Node.ANY, Node.ANY); + int n = 0; + while (it.hasNext()) { + Quad q = it.next(); + assertFalse(Quad.isDefaultGraph(q.getGraph()), + "findNG must not return default-graph quads, got: " + q); + n++; + } + assertEquals(voidGraphSize(), n); + } + + @Test + void findAnyStillIncludesDefaultGraph() { + Iterator it = ds.asDatasetGraph().find(Node.ANY, Node.ANY, Node.ANY, Node.ANY); + int n = 0; + while (it.hasNext()) { it.next(); n++; } + assertEquals(DEFAULT_GRAPH_TRIPLES + voidGraphSize(), n); + } + + // --- detach ---------------------------------------------------------------- + + @Test + void detachMaterializesNodeIdBindings() { + Node s1 = NodeFactory.createURI("http://ex.org/s1"); + NodeId id = bg.getReader().getNodeTable().getNodeIdForNode(s1); + assertFalse(NodeId.isDoesNotExist(id), "control: ex:s1 must be in the dictionary"); + + Node parentTerm = NodeFactory.createURI("http://ex.org/parent"); + Binding parent = BindingFactory.binding(Var.alloc("x"), parentTerm); + BindingNodeId bnid = new BindingNodeId(parent); + bnid.put(Var.alloc("s"), id); + + Binding detached = new BindingBG(bnid, bg).detach(); + assertFalse(detached instanceof BindingBG, + "detach must materialize away from the reader-backed binding"); + assertEquals(s1, detached.get(Var.alloc("s")), + "detached binding must carry the materialized node"); + assertEquals(parentTerm, detached.get(Var.alloc("x")), + "detached binding must keep parent bindings"); + } +} diff --git a/src/test/java/com/ebremer/beakgraph/ValueClusterFilterTest.java b/src/test/java/com/ebremer/beakgraph/ValueClusterFilterTest.java new file mode 100644 index 00000000..4b5cbc20 --- /dev/null +++ b/src/test/java/com/ebremer/beakgraph/ValueClusterFilterTest.java @@ -0,0 +1,128 @@ +package com.ebremer.beakgraph; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.ebremer.beakgraph.core.BeakGraph; +import com.ebremer.beakgraph.hdf5.readers.HDF5Reader; +import com.ebremer.beakgraph.hdf5.writers.HDF5Writer; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.Set; +import org.apache.jena.query.Dataset; +import org.apache.jena.query.QueryExecution; +import org.apache.jena.query.QueryFactory; +import org.apache.jena.query.ResultSet; +import org.apache.jena.rdf.model.RDFNode; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Regression tests for C6b: range-filter pushdown must enclose the whole + * cluster of value-equal but term-distinct literals. "5"^^xsd:int, 5 + * (xsd:integer) and "5.0"^^xsd:double compare value-equal and occupy distinct + * adjacent dictionary ids; deriving the scan bound from the exact-term + * insertion point landed inside that cluster and silently dropped qualifying + * rows at the boundary (e.g. FILTER(?o >= 5) missed "5"^^xsd:int). + */ +class ValueClusterFilterTest { + + private static final String TTL = """ + @prefix ex: . + @prefix xsd: . + ex:m1 ex:value "5"^^xsd:int . + ex:m2 ex:value 5 . + ex:m3 ex:value "5.0"^^xsd:double . + ex:m4 ex:value 4 . + ex:m5 ex:value 6 . + """; + + private static final String PREFIX = + "PREFIX ex: PREFIX xsd: "; + + private static final Set ALL_FIVES_AND_UP = + Set.of("http://ex.org/m1", "http://ex.org/m2", "http://ex.org/m3", "http://ex.org/m5"); + private static final Set ALL_FIVES_AND_DOWN = + Set.of("http://ex.org/m1", "http://ex.org/m2", "http://ex.org/m3", "http://ex.org/m4"); + + @TempDir + static Path dir; + static BeakGraph bg; + static Dataset ds; + + @BeforeAll + static void buildAndOpen() throws Exception { + File ttl = dir.resolve("cluster.ttl").toFile(); + File h5 = dir.resolve("cluster.ttl.h5").toFile(); + Files.write(ttl.toPath(), TTL.getBytes(StandardCharsets.UTF_8)); + HDF5Writer.Builder() + .setSource(ttl).setDestination(h5) + .setSpatial(false).setFeatures(false) + .build().write(); + bg = new BeakGraph(new HDF5Reader(h5)); + ds = bg.getDataset(); + } + + @AfterAll + static void closeReader() { + if (bg != null) bg.close(); + } + + private static Set subjects(String where) { + Set uris = new HashSet<>(); + try (QueryExecution qe = QueryExecution.dataset(ds) + .query(QueryFactory.create(PREFIX + "SELECT ?s WHERE { " + where + " }")).build()) { + ResultSet rs = qe.execSelect(); + while (rs.hasNext()) { + RDFNode n = rs.next().get("s"); + if (n != null && n.isURIResource()) uris.add(n.asResource().getURI()); + } + } + return uris; + } + + // --- bound predicate (GPOS / BGIteratorPOS route) ----------------------- + + @Test + void greaterOrEqualIncludesWholeValueCluster() { + assertEquals(ALL_FIVES_AND_UP, subjects("?s ex:value ?o FILTER(?o >= 5)")); + } + + @Test + void lessOrEqualIncludesWholeValueCluster() { + assertEquals(ALL_FIVES_AND_DOWN, subjects("?s ex:value ?o FILTER(?o <= 5)")); + } + + @Test + void strictBoundsExcludeWholeValueCluster() { + assertEquals(Set.of("http://ex.org/m5"), subjects("?s ex:value ?o FILTER(?o > 5)")); + assertEquals(Set.of("http://ex.org/m4"), subjects("?s ex:value ?o FILTER(?o < 5)")); + } + + @Test + void absentConstantInsideClusterRange() { + // 4.5 is not a stored term; the insertion point sits next to the 5-cluster. + assertEquals(ALL_FIVES_AND_UP, subjects("?s ex:value ?o FILTER(?o > 4.5)")); + assertEquals(Set.of("http://ex.org/m4"), subjects("?s ex:value ?o FILTER(?o < 4.5)")); + } + + @Test + void typedConstantVariantsAgree() { + // The same value written as a different term must produce identical results. + assertEquals(ALL_FIVES_AND_UP, subjects("?s ex:value ?o FILTER(?o >= \"5\"^^xsd:int)")); + assertEquals(ALL_FIVES_AND_UP, subjects("?s ex:value ?o FILTER(?o >= \"5.0\"^^xsd:double)")); + } + + // --- variable predicate (GSPO full scan / BGIteratorSPO_All route) ------ + + @Test + void fullScanRouteAgrees() { + assertEquals(ALL_FIVES_AND_UP, subjects("?s ?p ?o FILTER(?o >= 5)")); + assertEquals(ALL_FIVES_AND_DOWN, subjects("?s ?p ?o FILTER(?o <= 5)")); + assertEquals(Set.of("http://ex.org/m5"), subjects("?s ?p ?o FILTER(?o > 5)")); + } +} diff --git a/src/test/java/com/ebremer/beakgraph/VariableGraphScanTest.java b/src/test/java/com/ebremer/beakgraph/VariableGraphScanTest.java index 51a5931a..1162e1d4 100644 --- a/src/test/java/com/ebremer/beakgraph/VariableGraphScanTest.java +++ b/src/test/java/com/ebremer/beakgraph/VariableGraphScanTest.java @@ -78,4 +78,32 @@ void variableGraphScanBindsGraphAndCoversExactlyTheRealGraphs() throws Exception assertTrue(count >= 20, "should return at least the 20 default-graph triples; got " + count); } } + + @Test + void preBoundGraphVariableScansTheBoundGraph() throws Exception { + // The graph var is a VARIABLE in the quad but already bound in the + // BindingNodeId (BGIteratorMaster routes this shape to BGIteratorSPO_All + // when the predicate is unbound). SPO_All used to locate() the raw + // variable node, get -1, and silently yield nothing for a graph that + // exists - unlike the other three iterators behind the same dispatcher. + try (HDF5Reader reader = new HDF5Reader(h5)) { + PositionalDictionaryReader dict = (PositionalDictionaryReader) reader.getDictionary(); + NodeTable nt = reader.getNodeTable(); + Var g = Var.alloc("g"), s = Var.alloc("s"), p = Var.alloc("p"), o = Var.alloc("o"); + + long gid = dict.getGraphs().locate(Quad.defaultGraphIRI); + assertTrue(gid >= 1, "the default graph must exist in the dictionary"); + BindingNodeId bnid = new BindingNodeId(); + bnid.put(g, new NodeId(gid, com.ebremer.beakgraph.hdf5.jena.NodeType.GRAPH)); + + long count = 0; + Iterator it = new BGIteratorMaster( + reader, dict, bnid, new Quad(g, s, p, o), null, nt); + while (it.hasNext()) { + it.next(); + count++; + } + assertEquals(20, count, "the pre-bound default graph holds exactly the 20 source triples"); + } + } } diff --git a/src/test/java/com/ebremer/beakgraph/WriteErrorHandlingTest.java b/src/test/java/com/ebremer/beakgraph/WriteErrorHandlingTest.java index e1703fa9..1c96101d 100644 --- a/src/test/java/com/ebremer/beakgraph/WriteErrorHandlingTest.java +++ b/src/test/java/com/ebremer/beakgraph/WriteErrorHandlingTest.java @@ -1,7 +1,9 @@ package com.ebremer.beakgraph; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.ebremer.beakgraph.hdf5.writers.HDF5Writer; import java.io.File; @@ -31,4 +33,29 @@ void malformedRdfAbortsTheWriteInsteadOfSwallowing() throws Exception { // And no .h5 should have been produced for the failed input. assertFalse(h5.exists(), "a failed write must not leave a partial .h5 behind"); } + + @Test + void failedRebuildPreservesThePreviousGoodArtifact() throws Exception { + File ttl = dir.resolve("data.ttl").toFile(); + File h5 = dir.resolve("data.ttl.h5").toFile(); + Files.write(ttl.toPath(), + " .\n".getBytes(StandardCharsets.UTF_8)); + HDF5Writer.Builder().setSource(ttl).setDestination(h5) + .setSpatial(false).setFeatures(false).build().write(); + byte[] good = Files.readAllBytes(h5.toPath()); + + // Rebuild over the same destination with a broken source: the failure + // happens before anything is written, and the old cleanup deleted the + // previous, fully valid artifact anyway. + Files.write(ttl.toPath(), "no longer valid turtle @@@".getBytes(StandardCharsets.UTF_8)); + assertThrows(Exception.class, () -> + HDF5Writer.Builder().setSource(ttl).setDestination(h5) + .setSpatial(false).setFeatures(false).build().write()); + + assertTrue(h5.exists(), "a failed rebuild must not destroy the previous artifact"); + assertArrayEquals(good, Files.readAllBytes(h5.toPath()), + "the previous artifact must be byte-identical after a failed rebuild"); + assertFalse(dir.resolve("data.ttl.h5.tmp").toFile().exists(), + "the temp file must be cleaned up after a failed build"); + } } diff --git a/src/test/java/com/ebremer/beakgraph/cmdline/BeakGraphCLIConversionTest.java b/src/test/java/com/ebremer/beakgraph/cmdline/BeakGraphCLIConversionTest.java new file mode 100644 index 00000000..bf3eb119 --- /dev/null +++ b/src/test/java/com/ebremer/beakgraph/cmdline/BeakGraphCLIConversionTest.java @@ -0,0 +1,98 @@ +package com.ebremer.beakgraph.cmdline; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Conversion-run accounting. FileProcessor runs inside Futures that are never + * inspected, so before the fix any failure thrown outside its try block - + * including the NPE from a missing -dest - was swallowed by FutureTask: the + * run logged nothing, counted nothing, and reported every file as a + * successful conversion. + */ +class BeakGraphCLIConversionTest { + + @TempDir + Path dir; + + @Test + void failedConversionsAreCountedAndGoodFilesStillConvert() throws Exception { + Path src = Files.createDirectories(dir.resolve("src")); + Files.write(src.resolve("good.ttl"), + " .\n".getBytes(StandardCharsets.UTF_8)); + Files.write(src.resolve("bad.ttl"), + "this is not turtle @@@\n".getBytes(StandardCharsets.UTF_8)); + + Parameters p = new Parameters(); + p.src = src.toFile(); + p.dest = dir.resolve("out").toFile(); + + BeakGraphCLI cli = new BeakGraphCLI(p); + cli.traverse(); + + assertEquals(2, cli.getFileCounter().getRDFFileCount()); + assertEquals(1, cli.getFileCounter().getFailedConversionFileCount(), + "the unparseable source must be counted as a failed conversion"); + File goodH5 = dir.resolve("out").resolve("good.h5").toFile(); + assertTrue(goodH5.exists() && goodH5.length() > 0, + "the well-formed source must still convert"); + } + + @Test + void hugeFlagConvertsThroughDiskBasedWriter() throws Exception { + org.junit.jupiter.api.Assumptions.assumeTrue( + com.ebremer.beakgraph.huge.NativeHdf5File.isAvailable(), + "native HDF5 library unavailable"); + Path src = Files.createDirectories(dir.resolve("srchuge")); + Files.write(src.resolve("data.ttl"), + " .\n".getBytes(StandardCharsets.UTF_8)); + + Parameters p = new Parameters(); + p.src = src.toFile(); + p.dest = dir.resolve("outhuge").toFile(); + p.huge = true; + p.workdir = dir.resolve("workhuge").toFile(); // exercised even when absent: CLI creates it + + BeakGraphCLI cli = new BeakGraphCLI(p); + cli.traverse(); + + assertEquals(0, cli.getFileCounter().getFailedConversionFileCount(), + "the -huge conversion must succeed"); + File h5 = dir.resolve("outhuge").resolve("data.h5").toFile(); + assertTrue(h5.exists() && h5.length() > 0, "the -huge writer must produce the .h5 file"); + // The produced store must be readable by the standard reader stack. + try (com.ebremer.beakgraph.core.BeakGraph bg = new com.ebremer.beakgraph.core.BeakGraph( + new com.ebremer.beakgraph.hdf5.readers.HDF5Reader(h5))) { + assertTrue(bg.find(org.apache.jena.graph.NodeFactory.createURI("http://ex.org/a"), + org.apache.jena.graph.NodeFactory.createURI("http://ex.org/p"), + org.apache.jena.graph.NodeFactory.createURI("http://ex.org/b")).hasNext(), + "the converted triple must be queryable"); + } + } + + @Test + void missingDestinationFailsEveryFileLoudlyInsteadOfSilently() throws Exception { + // main() refuses to start without -dest; if a processor is ever reached + // without one anyway, the failure must land in the counter, not vanish + // inside a discarded Future. + Path src = Files.createDirectories(dir.resolve("src2")); + Files.write(src.resolve("a.ttl"), + " .\n".getBytes(StandardCharsets.UTF_8)); + + Parameters p = new Parameters(); + p.src = src.toFile(); + p.dest = null; + + BeakGraphCLI cli = new BeakGraphCLI(p); + cli.traverse(); + + assertEquals(1, cli.getFileCounter().getFailedConversionFileCount()); + } +} diff --git a/src/test/java/com/ebremer/beakgraph/cmdline/FileCounterTest.java b/src/test/java/com/ebremer/beakgraph/cmdline/FileCounterTest.java new file mode 100644 index 00000000..df92cc65 --- /dev/null +++ b/src/test/java/com/ebremer/beakgraph/cmdline/FileCounterTest.java @@ -0,0 +1,31 @@ +package com.ebremer.beakgraph.cmdline; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +class FileCounterTest { + + @Test + void successfulConversionsAreRdfMinusFailedOnly() { + FileCounter fc = new FileCounter(); + // The traverse filter rejects zero-length files BEFORE they count as RDF + // files, so the summary must not subtract them a second time (the old + // formula printed rdf - zero - failed, which understated the count and + // went negative when empties outnumbered conversions). + for (int i = 0; i < 7; i++) fc.incrementRDFFileCount(); + for (int i = 0; i < 3; i++) fc.incrementZeroLengthFileCount(); + for (int i = 0; i < 3; i++) fc.incrementFailedConversionFileCount(); + assertEquals(4, fc.getSuccessfulConversionCount()); + assertTrue(fc.toString().contains("Successful Conversions : 4"), fc.toString()); + } + + @Test + void manyEmptyFilesCannotDriveTheSummaryNegative() { + FileCounter fc = new FileCounter(); + for (int i = 0; i < 2; i++) fc.incrementRDFFileCount(); + for (int i = 0; i < 5; i++) fc.incrementZeroLengthFileCount(); + assertEquals(2, fc.getSuccessfulConversionCount()); + } +} diff --git a/src/test/java/com/ebremer/beakgraph/core/VoidStatsTest.java b/src/test/java/com/ebremer/beakgraph/core/VoidStatsTest.java new file mode 100644 index 00000000..6e2f0db4 --- /dev/null +++ b/src/test/java/com/ebremer/beakgraph/core/VoidStatsTest.java @@ -0,0 +1,51 @@ +package com.ebremer.beakgraph.core; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.ebremer.beakgraph.hdf5.readers.HDF5Reader; +import com.ebremer.beakgraph.hdf5.writers.HDF5Writer; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import org.apache.jena.graph.NodeFactory; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * The VoID description carries one void:propertyPartition per (graph, + * predicate) pair, so a predicate used in several graphs appears in several + * partitions. VoidStats must SUM them: a plain put() kept whichever partition + * HashMap iteration visited last, feeding the reorder cost model a per-graph + * count against the dataset-wide total. + */ +class VoidStatsTest { + + @TempDir + Path dir; + + @Test + void predicateCountsSumAcrossGraphPartitions() throws Exception { + String trig = """ + @prefix ex: . + ex:s1 ex:p ex:o1 . + ex:s2 ex:p ex:o2 . + ex:g1 { + ex:s3 ex:p ex:o3 . + ex:s4 ex:p ex:o4 . + ex:s5 ex:p ex:o5 . + } + """; + File src = dir.resolve("stats.trig").toFile(); + File h5 = dir.resolve("stats.trig.h5").toFile(); + Files.write(src.toPath(), trig.getBytes(StandardCharsets.UTF_8)); + HDF5Writer.Builder().setSource(src).setDestination(h5) + .setSpatial(false).setFeatures(false).build().write(); + + try (HDF5Reader reader = new HDF5Reader(h5)) { + VoidStats stats = VoidStats.load(reader); + assertEquals(5, stats.predicateCount(NodeFactory.createURI("http://ex.org/p")), + "ex:p appears in two graph partitions (2 + 3) and must sum to 5"); + } + } +} diff --git a/src/test/java/com/ebremer/beakgraph/core/fuseki/LWSStorageServletTest.java b/src/test/java/com/ebremer/beakgraph/core/fuseki/LWSStorageServletTest.java index df803986..31633add 100644 --- a/src/test/java/com/ebremer/beakgraph/core/fuseki/LWSStorageServletTest.java +++ b/src/test/java/com/ebremer/beakgraph/core/fuseki/LWSStorageServletTest.java @@ -71,6 +71,76 @@ void resolveWithinRejectsTraversalAndAbsolutePaths() throws Exception { assertNull(LWSStorageServlet.resolveWithin(null, "anything")); } + @Test + void escapeHtmlNeutralizesMarkupAndAttributeBreakouts() { + assertEquals("<script>alert(1)</script>", + LWSStorageServlet.escapeHtml("")); + assertEquals("a" onmouseover="x", LWSStorageServlet.escapeHtml("a\" onmouseover=\"x")); + assertEquals("a&b'c", LWSStorageServlet.escapeHtml("a&b'c")); + assertEquals("plain-name.h5", LWSStorageServlet.escapeHtml("plain-name.h5")); + } + + @Test + void encodeHrefPercentEncodesUnsafeCharactersButKeepsSlashes() { + assertEquals("/sub/a%20b.h5", LWSStorageServlet.encodeHref("/sub/a b.h5")); + assertEquals("/x%22y", LWSStorageServlet.encodeHref("/x\"y")); + assertEquals("/plain/file.h5", LWSStorageServlet.encodeHref("/plain/file.h5")); + } + + @Test + void toLiveUriMapsCanonicalUrisWithoutDoubledSlashes() { + String canonicalRoot = com.ebremer.beakgraph.lws.LWSMetadataGenerator.CANONICAL_BASE; + // Live bases end with '/'; canonical remainders start with '/': the old + // concatenation minted "http://host//name" URIs that 404 when followed. + assertEquals("http://example:9999/sub/file.h5", + LWSStorageServlet.toLiveUri(canonicalRoot + "/sub/file.h5", "http://example:9999/")); + assertEquals("http://example:9999/", + LWSStorageServlet.toLiveUri(canonicalRoot, "http://example:9999/")); + } + + @Test + void decodePathDecodesPercentEncodingAndRejectsGarbage() { + // getRequestURI() is undecoded; the metadata model and the filesystem + // hold raw names, so "a b.h5" must be reachable via its encoded link. + assertEquals("/a b.h5", LWSStorageServlet.decodePath("/a%20b.h5")); + assertEquals("/x/y.h5", LWSStorageServlet.decodePath("/x/y.h5")); + // '+' is a literal plus in a path (URLDecoder would have eaten it). + assertEquals("/a+b.h5", LWSStorageServlet.decodePath("/a+b.h5")); + assertNull(LWSStorageServlet.decodePath("/bad%zz")); + } + + @Test + void wildcardAcceptHeadersGetARepresentationNotA406() { + assertTrue(LWSStorageServlet.acceptsHtmlRepresentation("")); // no Accept at all + assertTrue(LWSStorageServlet.acceptsHtmlRepresentation("*/*")); // curl's default + assertTrue(LWSStorageServlet.acceptsHtmlRepresentation("text/*")); + assertTrue(LWSStorageServlet.acceptsHtmlRepresentation("text/html,application/xhtml+xml")); + assertFalse(LWSStorageServlet.acceptsHtmlRepresentation("application/ld+json")); + assertFalse(LWSStorageServlet.acceptsHtmlRepresentation("text/turtle")); + } + + @Test + void fileUrisAreNotExposableToClients() { + org.apache.jena.rdf.model.Model m = org.apache.jena.rdf.model.ModelFactory.createDefaultModel(); + assertFalse(LWSStorageServlet.exposableToClient( + m.createResource("file:///C:/secret/storage/data.h5")), "server paths must be redacted"); + assertTrue(LWSStorageServlet.exposableToClient(m.createResource("https://example.org/r"))); + assertTrue(LWSStorageServlet.exposableToClient(m.createLiteral("file:///looks-like-but-is-a-literal"))); + assertTrue(LWSStorageServlet.exposableToClient(m.createResource())); // bnode + } + + @Test + void sparqlBodyReadIsBounded() throws Exception { + String small = "SELECT * WHERE { ?s ?p ?o }"; + assertEquals(small, BGSparqlService.readBody( + new java.io.ByteArrayInputStream(small.getBytes(java.nio.charset.StandardCharsets.UTF_8)), 1024)); + byte[] huge = new byte[2048]; + java.util.Arrays.fill(huge, (byte) 'x'); + org.junit.jupiter.api.Assertions.assertThrows( + BGSparqlService.QueryBodyTooLargeException.class, + () -> BGSparqlService.readBody(new java.io.ByteArrayInputStream(huge), 1024)); + } + @Test void resolveWithinRejectsSymlinkEscape() throws Exception { Path root = Files.createTempDirectory("lws-sym").toRealPath(); diff --git a/src/test/java/com/ebremer/beakgraph/core/fuseki/RelativeIRIResolverTest.java b/src/test/java/com/ebremer/beakgraph/core/fuseki/RelativeIRIResolverTest.java index d5d5a0e1..3b90a2a3 100644 --- a/src/test/java/com/ebremer/beakgraph/core/fuseki/RelativeIRIResolverTest.java +++ b/src/test/java/com/ebremer/beakgraph/core/fuseki/RelativeIRIResolverTest.java @@ -6,6 +6,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.List; +import java.util.function.Predicate; import org.apache.jena.graph.Node; import org.apache.jena.graph.NodeFactory; import org.apache.jena.query.ResultSet; @@ -55,29 +56,61 @@ void storageToAbsoluteLeavesAbsoluteIrisUnchanged() { assertSame(snomed, t.apply(snomed)); } + /** A dictionary that holds only relative-form terms (no scheme) - the shape a BG store has. */ + private static final Predicate RELATIVE_FORMS_STORED = n -> !n.getURI().contains(":"); + @Test void absoluteToStorageRelativizesTheDocumentItself() { - NodeTransform t = new RelativeIRIResolver(BASE).absoluteToStorage(); + NodeTransform t = new RelativeIRIResolver(BASE).absoluteToStorage(RELATIVE_FORMS_STORED); assertEquals("", t.apply(NodeFactory.createURI(BASE)).getURI()); } @Test void absoluteToStorageRelativizesASibling() { - NodeTransform t = new RelativeIRIResolver(BASE).absoluteToStorage(); + NodeTransform t = new RelativeIRIResolver(BASE).absoluteToStorage(RELATIVE_FORMS_STORED); assertEquals("image.png", t.apply(NodeFactory.createURI(SIBLING)).getURI()); } @Test void absoluteToStorageLeavesUnrelatedIrisUnchanged() { - NodeTransform t = new RelativeIRIResolver(BASE).absoluteToStorage(); + NodeTransform t = new RelativeIRIResolver(BASE).absoluteToStorage(RELATIVE_FORMS_STORED); Node snomed = NodeFactory.createURI("http://snomed.info/id/123"); assertSame(snomed, t.apply(snomed)); } + @Test + void absoluteStoredIriIsNotRewrittenIntoAGuaranteedMiss() { + // Same host, different subtree: relativize yields the path-absolute form + // "/other/thing", which the writer never stores. The rewrite must not + // fire - the dictionary holds the ABSOLUTE term and the query must keep + // naming it (the old unconditional rewrite silently returned 0 rows). + String abs = "http://localhost:8888/other/thing"; + NodeTransform t = new RelativeIRIResolver(BASE) + .absoluteToStorage(n -> n.getURI().equals(abs)); + Node node = NodeFactory.createURI(abs); + assertSame(node, t.apply(node)); + } + + @Test + void parentDirectoryFormIsNeverStoredSoNoRewrite() { + // "../other.png" is relative per IRIx but not a form the writer produces; + // with nothing stored, the absolute name must pass through untouched. + Node up = NodeFactory.createURI("http://localhost:8888/HalcyonStorage/utah/other.png"); + NodeTransform t = new RelativeIRIResolver(BASE).absoluteToStorage(n -> false); + assertSame(up, t.apply(up)); + } + + @Test + void bothFormsStoredKeepsTheExactTermTheQueryNamed() { + NodeTransform t = new RelativeIRIResolver(BASE).absoluteToStorage(n -> true); + Node sibling = NodeFactory.createURI(SIBLING); + assertSame(sibling, t.apply(sibling)); + } + @Test void inputThenOutputRoundTripsToTheOriginalIri() { RelativeIRIResolver r = new RelativeIRIResolver(BASE); - NodeTransform in = r.absoluteToStorage(); + NodeTransform in = r.absoluteToStorage(RELATIVE_FORMS_STORED); NodeTransform out = r.storageToAbsolute(); for (String iri : new String[] { BASE, SIBLING }) { Node original = NodeFactory.createURI(iri); @@ -91,7 +124,7 @@ void nullBaseMakesResolverInactiveAndTransformsIdentity() { assertFalse(r.isActive()); Node rel = NodeFactory.createURI("image.png"); assertSame(rel, r.storageToAbsolute().apply(rel)); - assertSame(rel, r.absoluteToStorage().apply(rel)); + assertSame(rel, r.absoluteToStorage(n -> true).apply(rel)); } @Test diff --git a/src/test/java/com/ebremer/beakgraph/features/FeatureEdgeCasesTest.java b/src/test/java/com/ebremer/beakgraph/features/FeatureEdgeCasesTest.java new file mode 100644 index 00000000..8e59a7b8 --- /dev/null +++ b/src/test/java/com/ebremer/beakgraph/features/FeatureEdgeCasesTest.java @@ -0,0 +1,101 @@ +package com.ebremer.beakgraph.features; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.ebremer.beakgraph.features.pyradiomics.Gen2DFeatures; +import com.ebremer.ns.HAL; +import com.ebremer.ns.PYR; +import java.util.ArrayList; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.jena.graph.Node; +import org.apache.jena.graph.NodeFactory; +import org.apache.jena.sparql.core.Quad; +import org.junit.jupiter.api.Test; + +/** + * Feature-math edge cases: + *
      + *
    • a valid polygon thinner than ~0.5 units must still get features (the + * raster used to round to a 0-sized image and BufferedImage threw, + * skipping even the purely polygon-based features);
      • + *
    • zero-area geometry must not emit "INF"^^xsd:double literals;
      • + *
    • MajorMinor values come from exact AREA moments: holed polygons get an + * unbiased centroid (the vertex cloud double-weighted ring closures) and + * axis lengths follow the pyradiomics 4*sqrt(lambda) convention that + * PYR.MajorAxisLength uses.
      • + *
    + */ +class FeatureEdgeCasesTest { + + private static final Node F = NodeFactory.createURI("http://ex.org/f"); + + private static Map byPredicate(ArrayList quads) { + return quads.stream().collect(Collectors.toMap(Quad::getPredicate, + q -> q.getObject().getLiteralLexicalForm())); + } + + @Test + void thinPolygonStillGetsAllFeatures() { + ArrayList quads = new ArrayList<>(); + Gen2DFeatures.generate(quads, F, "POLYGON((0 0,100 0,100 0.4,0 0.4,0 0))"); + Map features = byPredicate(quads); + assertTrue(features.containsKey(PYR.MeshSurface.asNode()), + "polygon-based features must survive a sub-pixel raster height"); + assertEquals(40.0, Double.parseDouble(features.get(PYR.MeshSurface.asNode())), 1e-9); + assertTrue(features.containsKey(PYR.PixelSurface.asNode()), + "the raster features must be computed on the clamped 1px image"); + } + + @Test + void zeroAreaGeometryEmitsNoNonFiniteFeatureValues() { + // The bowtie's shoelace area is exactly 0: the ratio features divide by + // it and used to store "INF"^^xsd:double as real feature values. + ArrayList quads = new ArrayList<>(); + Gen2DFeatures.generate(quads, F, "POLYGON((0 0,10 10,10 0,0 10,0 0))"); + Map features = byPredicate(quads); + assertTrue(features.containsKey(PYR.Perimeter.asNode()), "finite features are still emitted"); + assertFalse(features.containsKey(PYR.PerimeterSurfaceRatio.asNode()), + "perimeter/area is infinite for zero area and must be skipped"); + assertFalse(features.containsKey(PYR.SphericalDisproportion.asNode())); + for (Map.Entry e : features.entrySet()) { + assertTrue(Double.isFinite(Double.parseDouble(e.getValue())), + "non-finite value emitted for " + e.getKey() + ": " + e.getValue()); + } + } + + @Test + void holedPolygonCentroidIsTheAreaCentroid() { + // Square shell with a centered square hole: the area centroid is exactly + // the center. The vertex cloud kept the shell's ring-closure vertex, so + // the old PCA centroid was biased toward it (~1.78 instead of 2). + ArrayList quads = new ArrayList<>(); + MajorMinor.add(quads, F, "POLYGON((0 0,4 0,4 4,0 4,0 0),(1 1,3 1,3 3,1 3,1 1))"); + Map features = byPredicate(quads); + assertEquals("POINT(2.0000 2.0000)", features.get(HAL.centroid.asNode())); + } + + @Test + void axisLengthsFollowThePyradiomicsAreaConvention() throws Exception { + // Rectangle 20x10: region variance along x is 20^2/12, so the pyradiomics + // major axis is 4*sqrt(400/12) = 23.094 - the SAME definition + // PYR.MajorAxisLength uses on the raster. The old vertex-cloud 2*sqrt + // drawn axis was mutually inconsistent with it. + ArrayList quads = new ArrayList<>(); + MajorMinor.add(quads, F, "POLYGON((0 0,20 0,20 10,0 10,0 0))"); + Map features = byPredicate(quads); + + org.locationtech.jts.geom.Geometry major = new org.locationtech.jts.io.WKTReader() + .read(features.get(HAL.majorAxis.asNode())); + org.locationtech.jts.geom.Geometry minor = new org.locationtech.jts.io.WKTReader() + .read(features.get(HAL.minorAxis.asNode())); + assertEquals(4 * Math.sqrt(400.0 / 12.0), major.getLength(), 1e-3); + assertEquals(4 * Math.sqrt(100.0 / 12.0), minor.getLength(), 1e-3); + // The major axis runs along x, centered on the area centroid. + org.locationtech.jts.geom.Coordinate[] c = major.getCoordinates(); + assertEquals(5.0, c[0].y, 1e-6); + assertEquals(5.0, c[1].y, 1e-6); + } +} diff --git a/src/test/java/com/ebremer/beakgraph/features/MajorMinorLocaleTest.java b/src/test/java/com/ebremer/beakgraph/features/MajorMinorLocaleTest.java new file mode 100644 index 00000000..8218cb4f --- /dev/null +++ b/src/test/java/com/ebremer/beakgraph/features/MajorMinorLocaleTest.java @@ -0,0 +1,51 @@ +package com.ebremer.beakgraph.features; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.ebremer.ns.HAL; +import java.util.ArrayList; +import java.util.Locale; +import org.apache.jena.graph.NodeFactory; +import org.apache.jena.sparql.core.Quad; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.locationtech.jts.io.WKTReader; + +/** + * Regression test: the WKT literals MajorMinor generates must use '.' as the + * decimal separator regardless of the JVM default locale. Under de_DE the old + * String.format calls produced "POINT(5,0000 2,5000)" - invalid WKT. + */ +class MajorMinorLocaleTest { + + private Locale saved; + + @BeforeEach + void forceCommaDecimalLocale() { + saved = Locale.getDefault(); + Locale.setDefault(Locale.GERMANY); + } + + @AfterEach + void restoreLocale() { + Locale.setDefault(saved); + } + + @Test + void wktUsesDotDecimalSeparatorUnderCommaLocale() throws Exception { + ArrayList quads = new ArrayList<>(); + MajorMinor.add(quads, NodeFactory.createURI("http://ex.org/f"), + "POLYGON((0 0, 10 0, 10 5, 0 5, 0 0))"); + assertEquals(3, quads.size(), "centroid + major axis + minor axis expected"); + + WKTReader reader = new WKTReader(); + String centroid = null; + for (Quad q : quads) { + String wkt = q.getObject().getLiteralLexicalForm(); + reader.read(wkt); // throws ParseException on locale-corrupted WKT + if (q.getPredicate().equals(HAL.centroid.asNode())) centroid = wkt; + } + assertEquals("POINT(5.0000 2.5000)", centroid); + } +} diff --git a/src/test/java/com/ebremer/beakgraph/huge/ExternalSorterTest.java b/src/test/java/com/ebremer/beakgraph/huge/ExternalSorterTest.java new file mode 100644 index 00000000..5e1a234b --- /dev/null +++ b/src/test/java/com/ebremer/beakgraph/huge/ExternalSorterTest.java @@ -0,0 +1,76 @@ +package com.ebremer.beakgraph.huge; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Random; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * External merge sort correctness: tiny in-RAM batches and a small fan-in force + * many spill runs and multi-level merges; the output must be a permutation of + * the input in exact comparator order, and every spill file must be gone + * afterwards. + */ +class ExternalSorterTest { + + @TempDir + Path dir; + + private static final ExternalSorter.Codec LONGS = new ExternalSorter.Codec<>() { + @Override public void write(DataOutput out, Long record) throws IOException { out.writeLong(record); } + @Override public Long read(DataInput in) throws IOException { return in.readLong(); } + }; + + @Test + void multiLevelMergeSortsAndCleansUp() throws Exception { + Random rnd = new Random(7); + int n = 100_000; + List expected = new ArrayList<>(n); + try (ExternalSorter sorter = new ExternalSorter<>(dir, "t", LONGS, Comparator.naturalOrder(), 1_000, 4)) { + for (int i = 0; i < n; i++) { + long v = rnd.nextLong(1_000_000); // duplicates likely + expected.add(v); + sorter.add(v); + } + expected.sort(Comparator.naturalOrder()); + assertEquals(n, sorter.size()); + List actual = new ArrayList<>(n); + try (ExternalSorter.SortedStream s = sorter.sorted()) { + while (s.hasNext()) { + actual.add(s.next()); + } + } + assertEquals(expected, actual, "sorted output must be the exact sorted multiset of the input"); + } + try (var files = Files.list(dir)) { + assertTrue(files.findAny().isEmpty(), "all spill runs must be deleted after consumption"); + } + } + + @Test + void allInMemoryPathSkipsDisk() throws Exception { + try (ExternalSorter sorter = new ExternalSorter<>(dir, "m", LONGS, Comparator.naturalOrder(), 1_000_000, 8)) { + for (long v : new long[]{5, 3, 9, 1, 3}) { + sorter.add(v); + } + List actual = new ArrayList<>(); + try (ExternalSorter.SortedStream s = sorter.sorted()) { + s.forEachRemaining(actual::add); + } + assertEquals(List.of(1L, 3L, 3L, 5L, 9L), actual); + } + try (var files = Files.list(dir)) { + assertFalse(files.findAny().isPresent(), "in-memory sort must not create files"); + } + } +} diff --git a/src/test/java/com/ebremer/beakgraph/huge/HugeWriterParityTest.java b/src/test/java/com/ebremer/beakgraph/huge/HugeWriterParityTest.java new file mode 100644 index 00000000..0c443be9 --- /dev/null +++ b/src/test/java/com/ebremer/beakgraph/huge/HugeWriterParityTest.java @@ -0,0 +1,277 @@ +package com.ebremer.beakgraph.huge; + +import com.ebremer.beakgraph.core.BeakGraph; +import com.ebremer.beakgraph.hdf5.readers.HDF5Reader; +import com.ebremer.beakgraph.hdf5.writers.HDF5Writer; +import io.jhdf.HdfFile; +import io.jhdf.api.Attribute; +import io.jhdf.api.Dataset; +import io.jhdf.api.Group; +import io.jhdf.api.Node; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import org.apache.jena.query.QueryExecution; +import org.apache.jena.query.QueryFactory; +import org.apache.jena.query.QuerySolution; +import org.apache.jena.query.ResultSet; +import org.apache.jena.rdf.model.Model; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * End-to-end parity: the disk-based writer must produce a store that reads + * back (through the UNMODIFIED jHDF-based readers) semantically identical to + * the RAM writer's - same graphs, isomorphic per-graph content (blank node + * labels legitimately differ), same query answers through both indexes, and + * structurally identical HDF5 metadata (same dataset tree, same numEntries / + * width / FCD attributes everywhere). + */ +class HugeWriterParityTest { + + @TempDir + static Path dir; + + @BeforeAll + static void requireNative() { + assumeTrue(NativeHdf5File.isAvailable(), + "native HDF5 library unavailable: " + NativeHdf5File.getUnavailableCause()); + } + + // ------------------------------------------------------------------ + // helpers + // ------------------------------------------------------------------ + + private static Path writeBoth(String name, String content, boolean spatial, boolean features) throws Exception { + File src = dir.resolve(name).toFile(); + Files.write(src.toPath(), content.getBytes(StandardCharsets.UTF_8)); + return writeBoth(name, src, spatial, features); + } + + /** Writes src with both writers; returns the directory holding {name}.ram.h5 / {name}.huge.h5. */ + private static Path writeBoth(String name, File src, boolean spatial, boolean features) throws Exception { + File ram = dir.resolve(name + ".ram.h5").toFile(); + File huge = dir.resolve(name + ".huge.h5").toFile(); + HDF5Writer.Builder().setSource(src).setDestination(ram) + .setSpatial(spatial).setFeatures(features).build().write(); + HugeHDF5Writer.Builder().setSource(src).setDestination(huge) + .setSpatial(spatial).setFeatures(features) + // deliberately tiny batches: force spill runs and multi-level merges + .setTermSpillBatch(512).setIdSpillBatch(1024).setMergeFanIn(4) + .build().write(); + return dir; + } + + private static Set graphNames(org.apache.jena.query.Dataset ds) { + Set names = new TreeSet<>(); + ds.asDatasetGraph().listGraphNodes().forEachRemaining(g -> names.add(g.toString())); + return names; + } + + private static Model graphModel(org.apache.jena.query.Dataset ds, String graphUri) { + String q = (graphUri == null) + ? "CONSTRUCT { ?s ?p ?o } WHERE { ?s ?p ?o }" + : "CONSTRUCT { ?s ?p ?o } WHERE { GRAPH <" + graphUri + "> { ?s ?p ?o } }"; + try (QueryExecution qe = QueryExecution.dataset(ds).query(QueryFactory.create(q)).build()) { + return qe.execConstruct(); + } + } + + private static java.util.List select(org.apache.jena.query.Dataset ds, String query) { + java.util.List rows = new java.util.ArrayList<>(); + try (QueryExecution qe = QueryExecution.dataset(ds).query(QueryFactory.create(query)).build()) { + ResultSet rs = qe.execSelect(); + while (rs.hasNext()) { + QuerySolution qs = rs.next(); + StringBuilder sb = new StringBuilder(); + rs.getResultVars().forEach(v -> sb.append(v).append('=').append(qs.get(v)).append('|')); + rows.add(sb.toString()); + } + } + rows.sort(String::compareTo); + return rows; + } + + /** Semantic equality of the two stores, graph by graph. */ + private static void assertStoresEquivalent(Path ramH5, Path hugeH5, String... probes) throws Exception { + try (BeakGraph a = new BeakGraph(new HDF5Reader(ramH5.toFile())); + BeakGraph b = new BeakGraph(new HDF5Reader(hugeH5.toFile()))) { + org.apache.jena.query.Dataset da = a.getDataset(); + org.apache.jena.query.Dataset db = b.getDataset(); + + Set graphsA = graphNames(da); + Set graphsB = graphNames(db); + assertEquals(graphsA, graphsB, "graph lists must match"); + + Model defA = graphModel(da, null); + Model defB = graphModel(db, null); + assertTrue(defA.isIsomorphicWith(defB), + "default graphs must be isomorphic (ram=" + defA.size() + ", huge=" + defB.size() + ")"); + for (String g : graphsA) { + if (!g.startsWith("http") && !g.startsWith("urn")) continue; // skip any non-URI form + Model ga = graphModel(da, g); + Model gb = graphModel(db, g); + assertTrue(ga.isIsomorphicWith(gb), + "graph <" + g + "> must be isomorphic (ram=" + ga.size() + ", huge=" + gb.size() + ")"); + } + for (String probe : probes) { + assertEquals(select(da, probe), select(db, probe), "probe results must match: " + probe); + } + } + } + + /** + * Structural parity of the HDF5 trees: identical group/dataset names and + * identical attribute values everywhere. Counts and bit widths are + * bnode-order independent, so this must hold exactly even though raw bytes + * may differ (blank node ranks differ between the writers). + */ + private static void assertSameStructure(Path ramH5, Path hugeH5) { + try (HdfFile ram = new HdfFile(ramH5); HdfFile huge = new HdfFile(hugeH5)) { + compareGroups("/", (Group) ram.getChild(".BG"), (Group) huge.getChild(".BG")); + } + } + + private static void compareGroups(String path, Group a, Group b) { + assertEquals(a != null, b != null, "group presence at " + path); + if (a == null) return; + compareAttributes(path, a, b); + Map ca = a.getChildren(); + Map cb = b.getChildren(); + assertEquals(new TreeSet<>(ca.keySet()), new TreeSet<>(cb.keySet()), "children of " + path); + for (String name : ca.keySet()) { + Node na = ca.get(name); + Node nb = cb.get(name); + assertEquals(na instanceof Group, nb instanceof Group, "node kind of " + path + name); + if (na instanceof Group ga) { + compareGroups(path + name + "/", ga, (Group) nb); + } else { + compareAttributes(path + name, na, nb); + assertEquals(((Dataset) na).getSize(), ((Dataset) nb).getSize(), + "dataset size of " + path + name); + } + } + } + + private static void compareAttributes(String path, Node a, Node b) { + Map attrsA = new HashMap<>(); + Map attrsB = new HashMap<>(); + for (Map.Entry e : a.getAttributes().entrySet()) { + attrsA.put(e.getKey(), e.getValue().getData()); + } + for (Map.Entry e : b.getAttributes().entrySet()) { + attrsB.put(e.getKey(), e.getValue().getData()); + } + assertEquals(attrsA, attrsB, "attributes of " + path); + } + + // ------------------------------------------------------------------ + // tests + // ------------------------------------------------------------------ + + @Test + void mixedTypesAndNamedGraphs() throws Exception { + String longStr = "y".repeat(150); + String trig = """ + @prefix ex: . + @prefix xsd: . + + ex:s0 ex:p0 ex:o0 . + ex:s0 ex:p0 ex:o0 . + ex:s0 ex:name "Alice" . + ex:s0 ex:name "Alice"@en . + ex:s0 ex:name "Alicia"@fr-CA . + ex:s0 ex:count "42"^^xsd:int . + ex:s0 ex:count "042"^^xsd:int . + ex:s0 ex:count2 "-7"^^xsd:int . + ex:s0 ex:big "9223372036854775806"^^xsd:long . + ex:s0 ex:neg "-9007199254740993"^^xsd:long . + ex:s0 ex:f "1.5"^^xsd:float . + ex:s0 ex:d "-2.25E8"^^xsd:double . + ex:s0 ex:unbounded "123456789012345678901234567890"^^xsd:integer . + ex:s0 ex:flag true . + ex:s0 ex:when "2024-05-06T07:08:09Z"^^xsd:dateTime . + ex:s0 ex:longstr "%s" . + ex:s0 ex:uni "h\\u00e9llo \\u00fcrld" . + ex:s0 ex:ill "abc"^^xsd:int . + <> ex:self . + _:b1 ex:p0 _:b2 . + _:b2 ex:knows ex:s0 . + ex:g1 { + ex:s1 ex:p0 ex:o0 . + ex:s1 ex:num "10"^^xsd:int . + _:b1 ex:inGraph ex:g1 . + } + ex:g2 { ex:s0 ex:p0 ex:s1 . } + """.formatted(longStr); + writeBoth("mixed.trig", trig, false, false); + Path ram = dir.resolve("mixed.trig.ram.h5"); + Path huge = dir.resolve("mixed.trig.huge.h5"); + assertStoresEquivalent(ram, huge, + "SELECT ?p ?o WHERE { ?p ?o }", + "SELECT ?s WHERE { ?s }", + "SELECT ?s ?o WHERE { GRAPH { ?s ?o } }", + "SELECT ?o WHERE { ?o }", + "SELECT ?g ?s WHERE { GRAPH ?g { ?s } }"); + assertSameStructure(ram, huge); + } + + @Test + void spatialAndFeaturesParity() throws Exception { + String trig = """ + @prefix ex: . + @prefix geo: . + + ex:geo1 geo:asWKT "POLYGON((0 0, 100 0, 100 100, 0 100, 0 0))"^^geo:wktLiteral . + ex:geo2 geo:asWKT "MULTIPOLYGON(((200 200, 300 200, 300 300, 200 200)),((600 600, 700 600, 700 700, 600 600)))"^^geo:wktLiteral . + ex:geo3 geo:asWKT "POINT(5000 6000)"^^geo:wktLiteral . + ex:geo4 geo:asWKT " POLYGON((10 10, 60 10, 60 60, 10 60, 10 10))"^^geo:wktLiteral . + ex:geo5 geo:asWKT "POLYGON((0 0, 1 0))"^^geo:wktLiteral . + ex:geo1 ex:label "region one" . + """; + writeBoth("spatial.trig", trig, true, true); + Path ram = dir.resolve("spatial.trig.ram.h5"); + Path huge = dir.resolve("spatial.trig.huge.h5"); + assertStoresEquivalent(ram, huge, + "SELECT ?s ?p ?o WHERE { GRAPH { ?s ?p ?o } }", + "SELECT ?g WHERE { GRAPH ?g { ?p ?o } }"); + assertSameStructure(ram, huge); + } + + @Test + void stressManySpillRunsMatchesRamWriter() throws Exception { + StringBuilder nq = new StringBuilder(1 << 22); + for (int i = 0; i < 30_000; i++) { + String s = ""; + String p = ""; + String o = switch (i % 4) { + case 0 -> ""; + case 1 -> "\"str" + (i % 1000) + "\""; + case 2 -> "\"" + (i % 1000 - 500) + "\"^^"; + default -> "\"" + ((i % 100) / 8.0) + "\"^^"; + }; + nq.append(s).append(' ').append(p).append(' ').append(o); + if (i % 3 != 0) { + nq.append(" '); + } + nq.append(" .\n"); + } + writeBoth("stress.nq", nq.toString(), false, false); + Path ram = dir.resolve("stress.nq.ram.h5"); + Path huge = dir.resolve("stress.nq.huge.h5"); + assertStoresEquivalent(ram, huge, + "SELECT ?s ?o WHERE { ?s ?o }", + "SELECT ?g (COUNT(*) AS ?n) WHERE { GRAPH ?g { ?s ?p ?o } } GROUP BY ?g ORDER BY ?g", + "SELECT ?s WHERE { GRAPH { ?s \"str101\" } }"); + assertSameStructure(ram, huge); + } +} diff --git a/src/test/java/com/ebremer/beakgraph/huge/NativeHdf5SmokeTest.java b/src/test/java/com/ebremer/beakgraph/huge/NativeHdf5SmokeTest.java new file mode 100644 index 00000000..7b16acaa --- /dev/null +++ b/src/test/java/com/ebremer/beakgraph/huge/NativeHdf5SmokeTest.java @@ -0,0 +1,79 @@ +package com.ebremer.beakgraph.huge; + +import io.jhdf.HdfFile; +import io.jhdf.api.Group; +import io.jhdf.api.dataset.ContiguousDataset; +import java.nio.ByteBuffer; +import java.nio.file.Path; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assumptions.assumeTrue; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * The reader-compatibility contract of the native backend, verified against the + * ACTUAL production reader stack (jHDF): contiguous-layout byte datasets + * written in several slabs, groups (including the dot-prefixed .BG name), and + * scalar attributes that come back as exactly Integer / Long - the boxed types + * the BeakGraph readers cast to. + */ +class NativeHdf5SmokeTest { + + @TempDir + Path dir; + + @Test + void nativeWrittenFileReadsBackThroughJhdf() throws Exception { + assumeTrue(NativeHdf5File.isAvailable(), + "native HDF5 library unavailable: " + NativeHdf5File.getUnavailableCause()); + Path h5 = dir.resolve("smoke.h5"); + + int len = 300_000; + byte[] pattern = new byte[len]; + for (int i = 0; i < len; i++) { + pattern[i] = (byte) (i * 31 + (i >> 8)); + } + + try (StreamingHdf5File f = NativeHdf5File.create(h5)) { + StreamingHdf5Group bg = f.rootGroup().putGroup(".BG"); + bg.putAttribute("numQuads", 42L); + bg.putAttribute("formatVersion", 3); + StreamingHdf5Group dict = bg.putGroup("dictionary"); + try (StreamingHdf5Dataset ds = dict.createByteDataset("offsets", len)) { + // three unequal slabs, one with a non-zero source offset + ds.write(pattern, 0, 100_000); + ds.write(pattern, 100_000, 150_000); + ds.write(pattern, 250_000, 50_000); + ds.putAttribute("width", 7); + ds.putAttribute("numEntries", 999L); + } + } + + try (HdfFile hdf = new HdfFile(h5)) { + Group bg = (Group) hdf.getChild(".BG"); + assertNotNull(bg, "the .BG group must exist"); + Object numQuads = bg.getAttribute("numQuads").getData(); + assertInstanceOf(Long.class, numQuads, "numQuads must read back as Long"); + assertEquals(42L, numQuads); + Object version = bg.getAttribute("formatVersion").getData(); + assertInstanceOf(Integer.class, version, "formatVersion must read back as Integer"); + assertEquals(3, version); + + Group dict = (Group) bg.getChild("dictionary"); + ContiguousDataset ds = (ContiguousDataset) dict.getChild("offsets"); + assertEquals(7, ds.getAttribute("width").getData(), "width attribute (Integer)"); + assertEquals(999L, ds.getAttribute("numEntries").getData(), "numEntries attribute (Long)"); + ByteBuffer buf = ds.getBuffer(); + assertEquals(len, buf.remaining(), "dataset length"); + byte[] readBack = new byte[len]; + buf.get(readBack); + for (int i = 0; i < len; i++) { + if (readBack[i] != pattern[i]) { + assertEquals(pattern[i], readBack[i], "byte mismatch at " + i); + } + } + } + } +} diff --git a/src/test/java/com/ebremer/beakgraph/huge/SpillBitPackedBufferTest.java b/src/test/java/com/ebremer/beakgraph/huge/SpillBitPackedBufferTest.java new file mode 100644 index 00000000..a563d6f7 --- /dev/null +++ b/src/test/java/com/ebremer/beakgraph/huge/SpillBitPackedBufferTest.java @@ -0,0 +1,71 @@ +package com.ebremer.beakgraph.huge; + +import com.ebremer.beakgraph.hdf5.BitPackedUnSignedLongBuffer; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Random; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * The spill bit packer must produce bytes the PRODUCTION reader + * ({@link BitPackedUnSignedLongBuffer} wrapping the dataset buffer, exactly as + * the HDF5 readers construct it) decodes back to the original values - for + * every supported width, including the signed 32/64 escape hatches. + */ +class SpillBitPackedBufferTest { + + @TempDir + Path dir; + + @Test + void roundTripsThroughProductionReaderAtEveryWidth() throws Exception { + Random rnd = new Random(42); + int[] widths = {1, 2, 5, 8, 13, 24, 32, 40, 57, 64}; + for (int w : widths) { + int n = 1000; + long[] values = new long[n]; + Path f = dir.resolve("w" + w); + try (SpillBitPackedBuffer spill = new SpillBitPackedBuffer(f, w)) { + for (int i = 0; i < n; i++) { + long v; + if (w == 64) { + v = rnd.nextLong(); // full pattern incl. negatives + } else if (w == 32) { + v = rnd.nextInt(); // negative ints allowed at width 32 + spill.writeInteger((int) v); + values[i] = ((int) v) & 0xFFFFFFFFL; // reader returns the unsigned pattern + continue; + } else { + v = rnd.nextLong() & ((1L << w) - 1); + } + values[i] = v; + spill.writeLong(v); + } + spill.complete(); + assertEquals(n, spill.getNumEntries()); + byte[] bytes = Files.readAllBytes(f); + assertEquals((n * (long) w + 7) / 8, bytes.length, "width " + w + ": packed length"); + BitPackedUnSignedLongBuffer reader = + new BitPackedUnSignedLongBuffer(null, ByteBuffer.wrap(bytes), n, w); + for (int i = 0; i < n; i++) { + assertEquals(values[i], reader.get(i), "width " + w + " index " + i); + } + } + } + } + + @Test + void rejectsOutOfRangeValuesAndBadWidths() throws Exception { + assertThrows(IllegalArgumentException.class, + () -> new SpillBitPackedBuffer(dir.resolve("bad"), 58)); + try (SpillBitPackedBuffer b = new SpillBitPackedBuffer(dir.resolve("w3"), 3)) { + b.writeLong(7); + assertThrows(IllegalArgumentException.class, () -> b.writeLong(8)); + assertThrows(IllegalArgumentException.class, () -> b.writeLong(-1)); + } + } +} diff --git a/src/test/java/com/ebremer/beakgraph/io/FfmReaderPathTest.java b/src/test/java/com/ebremer/beakgraph/io/FfmReaderPathTest.java new file mode 100644 index 00000000..e750fe0b --- /dev/null +++ b/src/test/java/com/ebremer/beakgraph/io/FfmReaderPathTest.java @@ -0,0 +1,121 @@ +package com.ebremer.beakgraph.io; + +import com.ebremer.beakgraph.core.BeakGraph; +import com.ebremer.beakgraph.hdf5.readers.HDF5Reader; +import com.ebremer.beakgraph.hdf5.writers.HDF5Writer; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.Set; +import org.apache.jena.query.QueryExecution; +import org.apache.jena.query.QueryFactory; +import org.apache.jena.query.QuerySolution; +import org.apache.jena.query.ResultSet; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.CleanupMode; +import org.junit.jupiter.api.io.TempDir; + +/** + * End-to-end proof that the whole reader stack works over FFM-mapped segments: + * the FFM threshold is forced to 0 so EVERY dataset - bit-packed ids and + * bitmaps, rank/select directories, FCD string buffers (compressed and raw + * fragments), float/double value stores, language tags - is read through + * {@link MemorySegmentBytes} instead of jHDF's ByteBuffer, and every query + * answer must match the default (ByteBuffer) path over the same file exactly + * (same file, same ids, so even blank node labels must agree). + */ +class FfmReaderPathTest { + + // NEVER + manual delete: auto-arena mappings unmap at GC, which on Windows + // can outlive JUnit's cleanup and fail the run over a locked temp file. + @TempDir(cleanup = CleanupMode.NEVER) + static Path dir; + + @AfterAll + static void tryCleanup() { + try (var files = Files.walk(dir)) { + files.sorted(java.util.Comparator.reverseOrder()).forEach(p -> { + p.toFile().deleteOnExit(); + try { Files.deleteIfExists(p); } catch (Exception ignored) {} + }); + } catch (Exception ignored) {} + } + + private static final String[] PROBES = { + "SELECT ?p ?o WHERE { ?p ?o }", + "SELECT ?s WHERE { ?s }", + "SELECT ?s ?o WHERE { GRAPH { ?s ?p ?o } }", + "SELECT ?o WHERE { ?o }", + "SELECT ?o WHERE { ?o }", + "SELECT ?o WHERE { ?o }", + "SELECT ?g ?s ?p ?o WHERE { GRAPH ?g { ?s ?p ?o } }", + "SELECT ?s ?p ?o WHERE { ?s ?p ?o }", + }; + + @Test + void allDatasetsReadIdenticallyThroughFfmMappings() throws Exception { + String trig = """ + @prefix ex: . + @prefix xsd: . + ex:s0 ex:p0 ex:o0 . + ex:s0 ex:name "Alice" . + ex:s0 ex:name "Alice"@en . + ex:s0 ex:count "42"^^xsd:int . + ex:s0 ex:big "9007199254740993"^^xsd:long . + ex:s0 ex:f "1.5"^^xsd:float . + ex:s0 ex:d "-2.25E8"^^xsd:double . + ex:s0 ex:when "2024-05-06T07:08:09Z"^^xsd:dateTime . + ex:s0 ex:longstr "%s" . + _:b1 ex:p0 _:b2 . + ex:g1 { ex:s1 ex:p0 ex:o0 . _:b1 ex:inGraph ex:g1 . } + """.formatted("z".repeat(200)); + File src = dir.resolve("ffm.trig").toFile(); + File h5 = dir.resolve("ffm.trig.h5").toFile(); + Files.write(src.toPath(), trig.getBytes(StandardCharsets.UTF_8)); + HDF5Writer.Builder().setSource(src).setDestination(h5) + .setSpatial(false).setFeatures(false).build().write(); + + // Baseline: today's ByteBuffer path. + Set[] expected = runProbes(h5); + + // Forced-FFM: every dataset larger than 0 bytes goes through MemorySegmentBytes. + long previous = DatasetBytes.setFfmThreshold(0); + try { + Set[] actual = runProbes(h5); + for (int i = 0; i < PROBES.length; i++) { + assertFalse(expected[i].isEmpty(), "probe must return rows: " + PROBES[i]); + assertEquals(expected[i], actual[i], + "FFM path must answer identically: " + PROBES[i]); + } + } finally { + DatasetBytes.setFfmThreshold(previous); + } + } + + @SuppressWarnings("unchecked") + private static Set[] runProbes(File h5) throws Exception { + Set[] results = new Set[PROBES.length]; + try (BeakGraph bg = new BeakGraph(new HDF5Reader(h5))) { + for (int i = 0; i < PROBES.length; i++) { + Set rows = new HashSet<>(); + try (QueryExecution qe = QueryExecution.dataset(bg.getDataset()) + .query(QueryFactory.create(PROBES[i])).build()) { + ResultSet rs = qe.execSelect(); + while (rs.hasNext()) { + QuerySolution qs = rs.next(); + StringBuilder sb = new StringBuilder(); + rs.getResultVars().forEach(v -> sb.append(v).append('=').append(qs.get(v)).append('|')); + rows.add(sb.toString()); + } + } + results[i] = rows; + } + } + return results; + } +} diff --git a/src/test/java/com/ebremer/beakgraph/io/RandomAccessBytesParityTest.java b/src/test/java/com/ebremer/beakgraph/io/RandomAccessBytesParityTest.java new file mode 100644 index 00000000..20d7f67d --- /dev/null +++ b/src/test/java/com/ebremer/beakgraph/io/RandomAccessBytesParityTest.java @@ -0,0 +1,110 @@ +package com.ebremer.beakgraph.io; + +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Random; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.CleanupMode; +import org.junit.jupiter.api.io.TempDir; + +/** + * The two RandomAccessBytes implementations must be observably identical: + * every read (byte, big-endian long/float/double, bulk) at every offset - + * including unaligned and boundary offsets - returns the same value, and + * out-of-range offsets fail the same way. The FFM implementation is exercised + * over a real file mapping, at offset zero and at a non-zero base offset (the + * dataset-address case). + */ +class RandomAccessBytesParityTest { + + // NEVER + manual delete: auto-arena mappings unmap at GC, which on Windows + // can outlive JUnit's cleanup and fail the run over a locked temp file. + @TempDir(cleanup = CleanupMode.NEVER) + static Path dir; + + @AfterAll + static void tryCleanup() { + try (var files = Files.walk(dir)) { + files.sorted(java.util.Comparator.reverseOrder()).forEach(p -> { + p.toFile().deleteOnExit(); + try { Files.deleteIfExists(p); } catch (Exception ignored) {} + }); + } catch (Exception ignored) {} + } + + @Test + void implementationsAgreeAtEveryOffset() throws Exception { + Random rnd = new Random(99); + byte[] data = new byte[64 * 1024 + 13]; // deliberately not power-of-two + rnd.nextBytes(data); + Path f = dir.resolve("parity.bin"); + Files.write(f, data); + + RandomAccessBytes bb = new ByteBufferBytes(ByteBuffer.wrap(data)); + RandomAccessBytes seg = MemorySegmentBytes.map(f, 0, data.length); + + assertEquals(data.length, bb.size()); + assertEquals(data.length, seg.size()); + + // dense sweep near both ends plus random interior offsets + for (int i = 0; i < 512; i++) { + checkAt(bb, seg, i, data.length); + checkAt(bb, seg, data.length - 1 - i, data.length); + } + for (int i = 0; i < 5_000; i++) { + checkAt(bb, seg, rnd.nextLong(data.length), data.length); + } + + // bulk reads, including a full-array read + byte[] a = new byte[1024], b = new byte[1024]; + for (int i = 0; i < 200; i++) { + long off = rnd.nextLong(data.length - a.length); + bb.get(off, a, 0, a.length); + seg.get(off, b, 0, b.length); + assertArrayEquals(a, b, "bulk @" + off); + } + byte[] whole = new byte[data.length]; + seg.get(0, whole, 0, whole.length); + assertArrayEquals(data, whole, "full bulk read through the segment"); + + // identical failure mode out of range + assertThrows(IndexOutOfBoundsException.class, () -> bb.get(data.length)); + assertThrows(IndexOutOfBoundsException.class, () -> seg.get(data.length)); + assertThrows(IndexOutOfBoundsException.class, () -> bb.getLong(data.length - 4)); + assertThrows(IndexOutOfBoundsException.class, () -> seg.getLong(data.length - 4)); + assertThrows(IndexOutOfBoundsException.class, () -> seg.get(-1)); + } + + @Test + void nonZeroBaseOffsetMapsTheRightWindow() throws Exception { + Random rnd = new Random(7); + byte[] data = new byte[4096]; + rnd.nextBytes(data); + Path f = dir.resolve("offset.bin"); + Files.write(f, data); + + int base = 1234; // like a dataset's file address + RandomAccessBytes seg = MemorySegmentBytes.map(f, base, data.length - base); + assertEquals(data.length - base, seg.size()); + for (int i = 0; i < seg.size(); i++) { + assertEquals(data[base + i], seg.get(i), "byte @" + i); + } + } + + private static void checkAt(RandomAccessBytes bb, RandomAccessBytes seg, long off, int len) { + if (off < 0 || off >= len) return; + assertEquals(bb.get(off), seg.get(off), "byte @" + off); + if (off + Long.BYTES <= len) { + assertEquals(bb.getLong(off), seg.getLong(off), "long @" + off); + assertEquals(bb.getDouble(off), seg.getDouble(off), "double @" + off); + } + if (off + Float.BYTES <= len) { + assertEquals(bb.getFloat(off), seg.getFloat(off), "float @" + off); + } + } +} diff --git a/src/test/java/com/ebremer/beakgraph/lws/LWSMetadataGeneratorTest.java b/src/test/java/com/ebremer/beakgraph/lws/LWSMetadataGeneratorTest.java new file mode 100644 index 00000000..bda18733 --- /dev/null +++ b/src/test/java/com/ebremer/beakgraph/lws/LWSMetadataGeneratorTest.java @@ -0,0 +1,36 @@ +package com.ebremer.beakgraph.lws; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import org.apache.jena.rdf.model.Model; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class LWSMetadataGeneratorTest { + + @TempDir + Path root; + + @Test + void regenerationDoesNotIndexTheMetadataCacheItself() throws Exception { + Files.write(root.resolve("data.h5"), new byte[]{1, 2, 3}); + // A previous generation's cache is on disk - the regeneration walk must + // not turn it into a listed DataResource: the raw model carries the + // owl:sameAs file:/// server paths the servlet redacts from clients. + Files.write(root.resolve(LWSMetadataGenerator.CACHE_FILE_NAME), + "stale cache".getBytes(StandardCharsets.UTF_8)); + + Model model = LWSMetadataGenerator.generateLWSModel(root); + + String base = LWSMetadataGenerator.CANONICAL_BASE; + assertTrue(model.containsResource(model.createResource(base + "/data.h5")), + "real files must be indexed"); + assertFalse(model.containsResource( + model.createResource(base + "/" + LWSMetadataGenerator.CACHE_FILE_NAME)), + "the metadata cache file must never index itself"); + } +} diff --git a/src/test/java/com/ebremer/beakgraph/pool/BeakGraphPoolValidationTest.java b/src/test/java/com/ebremer/beakgraph/pool/BeakGraphPoolValidationTest.java new file mode 100644 index 00000000..2d61563b --- /dev/null +++ b/src/test/java/com/ebremer/beakgraph/pool/BeakGraphPoolValidationTest.java @@ -0,0 +1,47 @@ +package com.ebremer.beakgraph.pool; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.ebremer.beakgraph.core.BeakGraph; +import com.ebremer.beakgraph.hdf5.writers.HDF5Writer; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import org.apache.commons.pool2.PooledObject; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Pool validation must actually probe the reader (testOnBorrow is on): an + * isOpen-only check re-issued a poisoned instance to every borrower forever, + * since a reader whose backing data went bad still reports "open". + */ +class BeakGraphPoolValidationTest { + + @TempDir + Path dir; + + @Test + void validateObjectProbesRealDataAndRejectsClosedReaders() throws Exception { + File ttl = dir.resolve("pool.ttl").toFile(); + File h5 = dir.resolve("pool.ttl.h5").toFile(); + Files.write(ttl.toPath(), + " .\n".getBytes(StandardCharsets.UTF_8)); + HDF5Writer.Builder().setSource(ttl).setDestination(h5) + .setSpatial(false).setFeatures(false).build().write(); + + BeakGraphPoolFactory factory = new BeakGraphPoolFactory(); + BeakGraph bg = factory.create(h5.toURI()); + PooledObject pooled = factory.wrap(bg); + try { + assertTrue(factory.validateObject(h5.toURI(), pooled), + "a healthy instance must validate (probe included)"); + } finally { + bg.close(); + } + assertFalse(factory.validateObject(h5.toURI(), pooled), + "a closed instance must fail validation and be discarded"); + } +}