From 867ecb60165b3ea31cd476fc62fa00e4cea64c15 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Fri, 30 Jan 2026 13:11:02 +0100 Subject: [PATCH 1/8] change int to long --- .../java/dev/zarr/zarrjava/core/Array.java | 18 +++++++-------- .../zarr/zarrjava/utils/IndexingUtils.java | 10 ++++----- .../v3/codec/core/ShardingIndexedCodec.java | 4 ++-- .../java/dev/zarr/zarrjava/TestUtils.java | 6 ++--- .../java/dev/zarr/zarrjava/ZarrV2Test.java | 14 ++++++------ .../java/dev/zarr/zarrjava/ZarrV3Test.java | 22 +++++++++---------- .../zarrjava/store/OnlineS3StoreTest.java | 4 ++-- 7 files changed, 39 insertions(+), 39 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/core/Array.java b/src/main/java/dev/zarr/zarrjava/core/Array.java index 9f4d5b2..4b5ea92 100644 --- a/src/main/java/dev/zarr/zarrjava/core/Array.java +++ b/src/main/java/dev/zarr/zarrjava/core/Array.java @@ -92,7 +92,7 @@ public void write(long[] offset, ucar.ma2.Array array, boolean parallel) { throw new IllegalArgumentException("'array' needs to have rank '" + metadata.ndim() + "'."); } - int[] shape = array.getShape(); + long[] shape = Utils.toLongArray(array.getShape()); final int[] chunkShape = metadata.chunkShape(); Stream chunkStream = Arrays.stream(IndexingUtils.computeChunkCoords(metadata.shape, chunkShape, offset, shape)); @@ -221,7 +221,7 @@ public void write(ucar.ma2.Array array, boolean parallel) { */ @Nonnull public ucar.ma2.Array read() throws ZarrException { - return read(new long[metadata().ndim()], Utils.toIntArray(metadata().shape)); + return read(new long[metadata().ndim()], metadata().shape); } /** @@ -233,7 +233,7 @@ public ucar.ma2.Array read() throws ZarrException { * @throws ZarrException throws ZarrException if the requested data is outside the array's domain or if the read fails */ @Nonnull - public ucar.ma2.Array read(final long[] offset, final int[] shape) throws ZarrException { + public ucar.ma2.Array read(final long[] offset, final long[] shape) throws ZarrException { return read(offset, shape, false); } @@ -245,7 +245,7 @@ public ucar.ma2.Array read(final long[] offset, final int[] shape) throws ZarrEx */ @Nonnull public ucar.ma2.Array read(final boolean parallel) throws ZarrException { - return read(new long[metadata().ndim()], Utils.toIntArray(metadata().shape), parallel); + return read(new long[metadata().ndim()], metadata().shape, parallel); } boolean chunkIsInArray(long[] chunkCoords) { @@ -268,7 +268,7 @@ boolean chunkIsInArray(long[] chunkCoords) { * @throws ZarrException throws ZarrException if the requested data is outside the array's domain or if the read fails */ @Nonnull - public ucar.ma2.Array read(final long[] offset, final int[] shape, final boolean parallel) throws ZarrException { + public ucar.ma2.Array read(final long[] offset, final long[] shape, final boolean parallel) throws ZarrException { ArrayMetadata metadata = metadata(); if (offset.length != metadata.ndim()) { throw new IllegalArgumentException("'offset' needs to have rank '" + metadata.ndim() + "'."); @@ -288,7 +288,7 @@ public ucar.ma2.Array read(final long[] offset, final int[] shape, final boolean } final ucar.ma2.Array outputArray = ucar.ma2.Array.factory(metadata.dataType().getMA2DataType(), - shape); + Utils.toIntArray(shape)); Stream chunkStream = Arrays.stream(IndexingUtils.computeChunkCoords(metadata.shape, chunkShape, offset, shape)); if (parallel) { chunkStream = chunkStream.parallel(); @@ -340,7 +340,7 @@ public static final class ArrayAccessor { @Nullable long[] offset; @Nullable - int[] shape; + long[] shape; @Nonnull Array array; @@ -357,13 +357,13 @@ public ArrayAccessor withOffset(@Nonnull long... offset) { @Nonnull public ArrayAccessor withShape(@Nonnull int... shape) { - this.shape = shape; + this.shape = Utils.toLongArray(shape); return this; } @Nonnull public ArrayAccessor withShape(@Nonnull long... shape) { - this.shape = Utils.toIntArray(shape); + this.shape = shape; return this; } diff --git a/src/main/java/dev/zarr/zarrjava/utils/IndexingUtils.java b/src/main/java/dev/zarr/zarrjava/utils/IndexingUtils.java index 9ff0764..c68a137 100644 --- a/src/main/java/dev/zarr/zarrjava/utils/IndexingUtils.java +++ b/src/main/java/dev/zarr/zarrjava/utils/IndexingUtils.java @@ -6,7 +6,7 @@ public class IndexingUtils { public static long[][] computeChunkCoords(long[] arrayShape, int[] chunkShape) { return computeChunkCoords(arrayShape, chunkShape, new long[arrayShape.length], - Utils.toIntArray(arrayShape)); + arrayShape); } public static long[][] computeChunkCoords(int[] arrayShape, int[] chunkShape) { @@ -14,7 +14,7 @@ public static long[][] computeChunkCoords(int[] arrayShape, int[] chunkShape) { } public static long[][] computeChunkCoords(long[] arrayShape, int[] chunkShape, long[] selOffset, - int[] selShape) { + long[] selShape) { final int ndim = arrayShape.length; long[] start = new long[ndim]; long[] end = new long[ndim]; @@ -54,14 +54,14 @@ public static ChunkProjection computeProjection(long[] chunkCoords, int[] arrayS public static ChunkProjection computeProjection(long[] chunkCoords, long[] arrayShape, int[] chunkShape) { return computeProjection(chunkCoords, arrayShape, chunkShape, new long[chunkCoords.length], - Utils.toIntArray(arrayShape) + arrayShape ); } public static ChunkProjection computeProjection( final long[] chunkCoords, final long[] arrayShape, final int[] chunkShape, final long[] selOffset, - final int[] selShape + final long[] selShape ) { final int ndim = chunkCoords.length; final int[] chunkOffset = new int[ndim]; @@ -141,7 +141,7 @@ public static boolean isFullChunk(final int[] selOffset, final int[] selShape, return true; } - public static boolean isSingleFullChunk(final long[] selOffset, final int[] selShape, + public static boolean isSingleFullChunk(final long[] selOffset, final long[] selShape, final int[] chunkShape) { if (selOffset.length != selShape.length) { throw new IllegalArgumentException("'selOffset' and 'selShape' need to have the same rank."); diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java index 733dd0e..de4092e 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java @@ -203,7 +203,7 @@ private Array decodeInternal( final Array shardIndexArray = indexCodecPipeline.decode(shardIndexBytes); long[][] allChunkCoords = IndexingUtils.computeChunkCoords(shardMetadata.shape, shardMetadata.chunkShape, offset, - shape); + Utils.toLongArray(shape)); Arrays.stream(allChunkCoords) // .parallel() @@ -217,7 +217,7 @@ private Array decodeInternal( Array chunkArray = null; final IndexingUtils.ChunkProjection chunkProjection = IndexingUtils.computeProjection(chunkCoords, shardMetadata.shape, - shardMetadata.chunkShape, offset, shape + shardMetadata.chunkShape, offset, Utils.toLongArray(shape) ); if (chunkByteOffset != -1 && chunkByteLength != -1) { final ByteBuffer chunkBytes = dataProvider.read(chunkByteOffset, chunkByteLength); diff --git a/src/test/java/dev/zarr/zarrjava/TestUtils.java b/src/test/java/dev/zarr/zarrjava/TestUtils.java index 582dfab..68a5777 100644 --- a/src/test/java/dev/zarr/zarrjava/TestUtils.java +++ b/src/test/java/dev/zarr/zarrjava/TestUtils.java @@ -34,7 +34,7 @@ public void testComputeChunkCoords(){ long[] arrayShape = new long[]{100, 100}; int[] chunkShape = new int[]{30, 30}; long[] selOffset = new long[]{50, 20}; - int[] selShape = new int[]{20, 1}; + long[] selShape = new long[]{20, 1}; long[][] chunkCoords = computeChunkCoords(arrayShape, chunkShape, selOffset, selShape); long[][] expectedChunkCoords = new long[][]{ {1, 0}, @@ -45,7 +45,7 @@ public void testComputeChunkCoords(){ arrayShape = new long[]{1, 52}; chunkShape = new int[]{1, 17}; selOffset = new long[]{0, 32}; - selShape = new int[]{1, 20}; + selShape = new long[]{1, 20}; chunkCoords = computeChunkCoords(arrayShape, chunkShape, selOffset, selShape); expectedChunkCoords = new long[][]{ {0, 1}, @@ -65,7 +65,7 @@ public void testComputeProjection(){ final long[] arrayShape = new long[]{1, 52}; final int[] chunkShape = new int[]{1, 17}; final long[] selOffset = new long[]{0, 32}; - final int[] selShape = new int[]{1, 20}; + final long[] selShape = new long[]{1, 20}; IndexingUtils.ChunkProjection projection = IndexingUtils.computeProjection( chunkCoords, arrayShape, chunkShape, selOffset, selShape diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java index 3bd3a88..c3e498a 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java @@ -51,7 +51,7 @@ public void testCreateBlosc(String cname, String shuffle, int clevel) throws IOE ); array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{8, 8})); - ucar.ma2.Array outArray = array.read(new long[]{2, 2}, new int[]{8, 8}); + ucar.ma2.Array outArray = array.read(new long[]{2, 2}, new long[]{8, 8}); Assertions.assertEquals(8 * 8, outArray.getSize()); Assertions.assertEquals(0, outArray.getByte(0)); } @@ -64,7 +64,7 @@ public void testReadBloscDetectTypesize(DataType dt) throws IOException, ZarrExc String arrayname = dt == DataType.BOOL ? "bool" : "double"; StoreHandle storeHandle = new FilesystemStore(TESTDATA).resolve("v2_sample", arrayname); Array array = Array.open(storeHandle); - array.read(new long[]{0, 0, 0}, new int[]{3, 4, 5}); + array.read(new long[]{0, 0, 0}, new long[]{3, 4, 5}); Assertions.assertEquals(dt, array.metadata().dataType); } @@ -83,7 +83,7 @@ public void testCreate() throws IOException, ZarrException { ); array.write(new long[]{2, 2}, ucar.ma2.Array.factory(dataType.getMA2DataType(), new int[]{8, 8})); - ucar.ma2.Array outArray = array.read(new long[]{2, 2}, new int[]{8, 8}); + ucar.ma2.Array outArray = array.read(new long[]{2, 2}, new long[]{8, 8}); Assertions.assertEquals(8 * 8, outArray.getSize()); Assertions.assertEquals(0, outArray.getByte(0)); } @@ -103,7 +103,7 @@ public void testCreateZlib(int level) throws IOException, ZarrException { ); array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{7, 6})); - ucar.ma2.Array outArray = array.read(new long[]{2, 2}, new int[]{7, 6}); + ucar.ma2.Array outArray = array.read(new long[]{2, 2}, new long[]{7, 6}); Assertions.assertEquals(7 * 6, outArray.getSize()); Assertions.assertEquals(0, outArray.getByte(0)); } @@ -123,7 +123,7 @@ public void testNoFillValue(DataType dataType) throws IOException, ZarrException ); Assertions.assertNull(array.metadata().fillValue); - ucar.ma2.Array outArray = array.read(new long[]{0, 0}, new int[]{1, 1}); + ucar.ma2.Array outArray = array.read(new long[]{0, 0}, new long[]{1, 1}); if (dataType == DataType.BOOL) { Assertions.assertFalse(outArray.getBoolean(0)); } else { @@ -351,10 +351,10 @@ public void testResizeArray() throws IOException, ZarrException { array = array.resize(new long[]{20, 15}); Assertions.assertArrayEquals(new int[]{20, 15}, array.read().getShape()); - ucar.ma2.Array data = array.read(new long[]{0, 0}, new int[]{10, 10}); + ucar.ma2.Array data = array.read(new long[]{0, 0}, new long[]{10, 10}); Assertions.assertArrayEquals(testData, (int[]) data.get1DJavaArray(ma2DataType)); - data = array.read(new long[]{10, 10}, new int[]{5, 5}); + data = array.read(new long[]{10, 10}, new long[]{5, 5}); int[] expectedData = new int[5 * 5]; Arrays.fill(expectedData, 1); Assertions.assertArrayEquals(expectedData, (int[]) data.get1DJavaArray(ma2DataType)); diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java index 1fcb8dd..e15401c 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java @@ -245,7 +245,7 @@ public void testCheckInvalidTransposeOrder(int[] transposeOrder) throws Exceptio public void testShardingReadCutout() throws IOException, ZarrException { Array array = Array.open(new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "1")); - ucar.ma2.Array outArray = array.read(new long[]{0, 3073, 3073, 513}, new int[]{1, 64, 64, 64}); + ucar.ma2.Array outArray = array.read(new long[]{0, 3073, 3073, 513}, new long[]{1, 64, 64, 64}); Assertions.assertEquals(64 * 64 * 64, outArray.getSize()); Assertions.assertEquals(-98, outArray.getByte(0)); } @@ -285,7 +285,7 @@ public void testShardingReadWrite(String indexLocation) throws IOException, Zarr @Test public void testCodecs() throws IOException, ZarrException { - int[] readShape = new int[]{1, 1, 1024, 1024}; + long[] readShape = new long[]{1, 1, 1024, 1024}; Array readArray = Array.open( new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "8-8-2")); ucar.ma2.Array readArrayContent = readArray.read(new long[4], readShape); @@ -363,7 +363,7 @@ public void testReadme1() throws IOException, ZarrException { Array array = (Array) color.get("1"); ucar.ma2.Array outArray = array.read( new long[]{0, 3073, 3073, 513}, // offset - new int[]{1, 64, 64, 64} // shape + new long[]{1, 64, 64, 64} // shape ); Assertions.assertEquals(64 * 64 * 64, outArray.getSize()); } @@ -385,7 +385,7 @@ public void testReadme2() throws IOException, ZarrException { new long[]{0, 0, 0, 0}, // offset data ); - ucar.ma2.Array output = array.read(new long[]{0, 0, 0, 0}, new int[]{1, 1, 2, 2}); + ucar.ma2.Array output = array.read(new long[]{0, 0, 0, 0}, new long[]{1, 1, 2, 2}); assert MultiArrayUtils.allValuesEqual(data, output); } @@ -401,8 +401,8 @@ public void testReadL4Sample(String mag) throws IOException, ZarrException { Assertions.assertArrayEquals(httpArray.metadata().shape, localArray.metadata().shape); Assertions.assertArrayEquals(httpArray.metadata().chunkShape(), localArray.metadata().chunkShape()); - ucar.ma2.Array httpData1 = httpArray.read(new long[]{0, 0, 0, 0}, new int[]{1, 64, 64, 64}); - ucar.ma2.Array localData1 = localArray.read(new long[]{0, 0, 0, 0}, new int[]{1, 64, 64, 64}); + ucar.ma2.Array httpData1 = httpArray.read(new long[]{0, 0, 0, 0}, new long[]{1, 64, 64, 64}); + ucar.ma2.Array localData1 = localArray.read(new long[]{0, 0, 0, 0}, new long[]{1, 64, 64, 64}); assert MultiArrayUtils.allValuesEqual(httpData1, localData1); @@ -415,8 +415,8 @@ public void testReadL4Sample(String mag) throws IOException, ZarrException { offset[i] = originalOffset[i] / (originalShape[i] / arrayShape[i]); } - ucar.ma2.Array httpData2 = httpArray.read(offset, new int[]{1, 64, 64, 64}); - ucar.ma2.Array localData2 = localArray.read(offset, new int[]{1, 64, 64, 64}); + ucar.ma2.Array httpData2 = httpArray.read(offset, new long[]{1, 64, 64, 64}); + ucar.ma2.Array localData2 = localArray.read(offset, new long[]{1, 64, 64, 64}); assert MultiArrayUtils.allValuesEqual(httpData2, localData2); } @@ -727,10 +727,10 @@ public void testResizeArray() throws IOException, ZarrException { array = array.resize(new long[]{20, 15}); Assertions.assertArrayEquals(new int[]{20, 15}, array.read().getShape()); - ucar.ma2.Array data = array.read(new long[]{0, 0}, new int[]{10, 10}); + ucar.ma2.Array data = array.read(new long[]{0, 0}, new long[]{10, 10}); Assertions.assertArrayEquals(testData, (int[]) data.get1DJavaArray(ma2DataType)); - data = array.read(new long[]{10, 10}, new int[]{5, 5}); + data = array.read(new long[]{10, 10}, new long[]{5, 5}); int[] expectedData = new int[5 * 5]; Arrays.fill(expectedData, 1); Assertions.assertArrayEquals(expectedData, (int[]) data.get1DJavaArray(ma2DataType)); @@ -770,7 +770,7 @@ public void testUnalignedArrayAccess(int arrayShape, int chunkShape, int accessS for (int i = 0; i < arrayShape; i += accessShape) { accessShape = Math.min(accessShape, arrayShape - i); - ucar.ma2.Array result = array.read(new long[]{i}, new int[]{accessShape}); + ucar.ma2.Array result = array.read(new long[]{i}, new long[]{accessShape}); int[] expectedData = Arrays.copyOfRange(testData, i, i + accessShape); Assertions.assertArrayEquals(expectedData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); } diff --git a/src/test/java/dev/zarr/zarrjava/store/OnlineS3StoreTest.java b/src/test/java/dev/zarr/zarrjava/store/OnlineS3StoreTest.java index c1600f2..051006c 100644 --- a/src/test/java/dev/zarr/zarrjava/store/OnlineS3StoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/store/OnlineS3StoreTest.java @@ -40,11 +40,11 @@ void createStore() { public void testOpen() throws IOException, ZarrException { Array arrayV3 = Array.open(storeHandle); Assertions.assertArrayEquals(new long[]{5, 1552, 2080}, arrayV3.metadata().shape); - Assertions.assertEquals(574, arrayV3.read(new long[]{0, 0, 0}, new int[]{1, 1, 1}).getInt(0)); + Assertions.assertEquals(574, arrayV3.read(new long[]{0, 0, 0}, new long[]{1, 1, 1}).getInt(0)); dev.zarr.zarrjava.core.Array arrayCore = dev.zarr.zarrjava.core.Array.open(storeHandle); Assertions.assertArrayEquals(new long[]{5, 1552, 2080}, arrayCore.metadata().shape); - Assertions.assertEquals(574, arrayCore.read(new long[]{0, 0, 0}, new int[]{1, 1, 1}).getInt(0)); + Assertions.assertEquals(574, arrayCore.read(new long[]{0, 0, 0}, new long[]{1, 1, 1}).getInt(0)); } From 7e8943461225d24097bba1820f8903e6e180090d Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Fri, 30 Jan 2026 13:11:51 +0100 Subject: [PATCH 2/8] Add path traversal validation to FilesystemStore --- .../zarr/zarrjava/store/FilesystemStore.java | 7 ++- .../zarrjava/store/FileSystemStoreTest.java | 43 +++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java b/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java index 02c552b..cf2c4eb 100644 --- a/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java @@ -30,7 +30,12 @@ Path resolveKeys(String[] keys) { for (String key : keys) { newPath = newPath.resolve(key); } - return newPath; + Path absoluteRoot = path.toAbsolutePath().normalize(); + Path absoluteTarget = newPath.toAbsolutePath().normalize(); + if (!absoluteTarget.startsWith(absoluteRoot)) { + throw new IllegalArgumentException("Key resolves outside of store root: " + absoluteTarget); + } + return newPath.normalize(); } @Override diff --git a/src/test/java/dev/zarr/zarrjava/store/FileSystemStoreTest.java b/src/test/java/dev/zarr/zarrjava/store/FileSystemStoreTest.java index dec204e..93e1802 100644 --- a/src/test/java/dev/zarr/zarrjava/store/FileSystemStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/store/FileSystemStoreTest.java @@ -7,7 +7,9 @@ import org.junit.jupiter.api.Test; import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.file.Files; +import java.nio.file.Path; import static dev.zarr.zarrjava.v3.Node.makeObjectMapper; @@ -69,4 +71,45 @@ public void testFileSystemStores() throws IOException, ZarrException { Store writableStore() { return new FilesystemStore(TESTOUTPUT.resolve("writableFSStore")); } + + @Test + public void testPathTraversal() throws IOException { + Path storeRoot = TESTOUTPUT.resolve("testPathTraversal").resolve("store"); + Files.createDirectories(storeRoot); + FilesystemStore store = new FilesystemStore(storeRoot); + + // Try to write outside the store directory + String[] maliciousKeys = {"..", "outside.txt"}; + ByteBuffer data = ByteBuffer.wrap("pwned".getBytes()); + + boolean exceptionThrown = false; + try { + store.set(maliciousKeys, data); + } catch (IllegalArgumentException e) { + exceptionThrown = true; + } catch (Exception e) { + // ignore other exceptions + } + + Assertions.assertTrue(exceptionThrown, "Should have thrown IllegalArgumentException for path traversal"); + + Path targetFile = TESTOUTPUT.resolve("testPathTraversal").resolve("outside.txt"); + Assertions.assertFalse(Files.exists(targetFile), "Path Traversal Vulnerability detected: File written outside store root!"); + } + + @Test + public void testValidTraversal() throws IOException { + Path storeRoot = TESTOUTPUT.resolve("testValidTraversal").resolve("store"); + Files.createDirectories(storeRoot); + FilesystemStore store = new FilesystemStore(storeRoot); + + // Valid traversal: subdirectory and back up, but still inside root + String[] validKeys = {"subdir", "..", "inside.txt"}; + ByteBuffer data = ByteBuffer.wrap("safe".getBytes()); + + store.set(validKeys, data); + + Path targetFile = storeRoot.resolve("inside.txt"); + Assertions.assertTrue(Files.exists(targetFile), "Valid traversal should be allowed"); + } } From 619d615641fd18598611c13840cd0be1c71b0cf6 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Fri, 30 Jan 2026 20:45:48 +0100 Subject: [PATCH 3/8] Enhance path validation and error handling in FilesystemStore and HttpStore; update chunk handling in IndexingUtils to support long values --- .../zarr/zarrjava/store/FilesystemStore.java | 40 +++++++++++++++-- .../dev/zarr/zarrjava/store/HttpStore.java | 27 +++++++++--- .../java/dev/zarr/zarrjava/store/S3Store.java | 20 +++++---- .../zarr/zarrjava/utils/IndexingUtils.java | 34 ++++++++++---- .../java/dev/zarr/zarrjava/TestUtils.java | 37 +++++++++++++++- .../java/dev/zarr/zarrjava/ZarrV3Test.java | 44 +++++++++++++++++++ 6 files changed, 174 insertions(+), 28 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java b/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java index cf2c4eb..6e430cc 100644 --- a/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java @@ -30,11 +30,43 @@ Path resolveKeys(String[] keys) { for (String key : keys) { newPath = newPath.resolve(key); } - Path absoluteRoot = path.toAbsolutePath().normalize(); - Path absoluteTarget = newPath.toAbsolutePath().normalize(); - if (!absoluteTarget.startsWith(absoluteRoot)) { - throw new IllegalArgumentException("Key resolves outside of store root: " + absoluteTarget); + + try { + // Use toRealPath() to resolve symlinks and verify path is within root + // For non-existent paths, validate the existing parent path + Path absoluteRoot = path.toAbsolutePath().normalize(); + Path targetPath = newPath.toAbsolutePath().normalize(); + + // Try to get real path if it exists (follows symlinks) + if (Files.exists(targetPath)) { + Path realTarget = targetPath.toRealPath(); + Path realRoot = absoluteRoot.toRealPath(); + if (!realTarget.startsWith(realRoot)) { + throw new IllegalArgumentException("Key resolves outside of store root: " + realTarget); + } + } else { + // For non-existent paths, check the normalized path + // and ensure existing parent doesn't escape via symlinks + Path parent = targetPath.getParent(); + if (parent != null && Files.exists(parent)) { + Path realParent = parent.toRealPath(); + Path realRoot = absoluteRoot.toRealPath(); + if (!realParent.startsWith(realRoot)) { + throw new IllegalArgumentException("Parent path resolves outside of store root: " + realParent); + } + } else if (!targetPath.startsWith(absoluteRoot)) { + throw new IllegalArgumentException("Key resolves outside of store root: " + targetPath); + } + } + } catch (IOException e) { + // If toRealPath() fails, fall back to normalized path check + Path absoluteRoot = path.toAbsolutePath().normalize(); + Path absoluteTarget = newPath.toAbsolutePath().normalize(); + if (!absoluteTarget.startsWith(absoluteRoot)) { + throw new IllegalArgumentException("Key resolves outside of store root: " + absoluteTarget); + } } + return newPath.normalize(); } diff --git a/src/main/java/dev/zarr/zarrjava/store/HttpStore.java b/src/main/java/dev/zarr/zarrjava/store/HttpStore.java index 31d5a0a..f047144 100644 --- a/src/main/java/dev/zarr/zarrjava/store/HttpStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/HttpStore.java @@ -22,11 +22,17 @@ public HttpStore(@Nonnull String uri) { } String resolveKeys(String[] keys) { - StringBuilder newUri = new StringBuilder(uri.replaceAll("\\/+$", "")); + HttpUrl url = HttpUrl.parse(uri); + if (url == null) { + throw new IllegalArgumentException("Invalid base URI: " + uri); + } + HttpUrl.Builder builder = url.newBuilder(); for (String key : keys) { - newUri.append("/").append(key); + for (String segment : key.split("/", -1)) { + builder.addPathSegment(segment); + } } - return newUri.toString(); + return builder.build().toString(); } @Nullable @@ -34,7 +40,13 @@ ByteBuffer get(Request request) { Call call = httpClient.newCall(request); try { Response response = call.execute(); + if (!response.isSuccessful()) { + return null; + } try (ResponseBody body = response.body()) { + if (body == null) { + return null; + } return ByteBuffer.wrap(body.bytes()); } } catch (IOException e) { @@ -65,7 +77,7 @@ public ByteBuffer get(String[] keys) { @Override public ByteBuffer get(String[] keys, long start) { Request request = new Request.Builder().url(resolveKeys(keys)).header( - "Range", start < 0 ? String.format("Bytes=%d", start) : String.format("Bytes=%d-", start)) + "Range", start < 0 ? String.format("bytes=%d", start) : String.format("bytes=%d-", start)) .build(); return get(request); @@ -78,7 +90,7 @@ public ByteBuffer get(String[] keys, long start, long end) { throw new IllegalArgumentException("Argument 'start' needs to be non-negative."); } Request request = new Request.Builder().url(resolveKeys(keys)).header( - "Range", String.format("Bytes=%d-%d", start, end - 1)).build(); + "Range", String.format("bytes=%d-%d", start, end - 1)).build(); return get(request); } @@ -110,10 +122,13 @@ public InputStream getInputStream(String[] keys, long start, long end) { throw new IllegalArgumentException("Argument 'start' needs to be non-negative."); } Request request = new Request.Builder().url(resolveKeys(keys)).header( - "Range", String.format("Bytes=%d-%d", start, end - 1)).build(); + "Range", String.format("bytes=%d-%d", start, end - 1)).build(); Call call = httpClient.newCall(request); try { Response response = call.execute(); + if (!response.isSuccessful()) { + return null; + } ResponseBody body = response.body(); if (body == null) return null; InputStream stream = body.byteStream(); diff --git a/src/main/java/dev/zarr/zarrjava/store/S3Store.java b/src/main/java/dev/zarr/zarrjava/store/S3Store.java index 7278856..d7d21ef 100644 --- a/src/main/java/dev/zarr/zarrjava/store/S3Store.java +++ b/src/main/java/dev/zarr/zarrjava/store/S3Store.java @@ -8,7 +8,6 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; @@ -89,12 +88,16 @@ public ByteBuffer get(String[] keys, long start, long end) { @Override public void set(String[] keys, ByteBuffer bytes) { - try (InputStream byteStream = new ByteArrayInputStream(Utils.toArray(bytes))) { - /*AWS SDK for Java v2 migration: When using InputStream to upload with S3Client, Content-Length should be specified and used with RequestBody.fromInputStream(). Otherwise, the entire stream will be buffered in memory. If content length must be unknown, we recommend using the CRT-based S3 client - https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/crt-based-s3-client.html*/ - s3client.putObject(PutObjectRequest.builder().bucket(bucketName).key(resolveKeys(keys)).build(), RequestBody.fromContentProvider(() -> byteStream, "application/octet-stream")); - } catch (IOException e) { - throw new RuntimeException(e); - } + // Convert ByteBuffer to byte array and use RequestBody.fromBytes() + // This properly sets Content-Length and avoids buffering the entire stream in memory + byte[] data = Utils.toArray(bytes); + s3client.putObject( + PutObjectRequest.builder() + .bucket(bucketName) + .key(resolveKeys(keys)) + .build(), + RequestBody.fromBytes(data) + ); } @Override @@ -178,8 +181,7 @@ public InputStream getInputStream(String[] keys, long start, long end) { .key(resolveKeys(keys)) .range(String.format("bytes=%d-%d", start, end - 1)) // S3 range is inclusive .build(); - ResponseInputStream responseInputStream = s3client.getObject(req); - return responseInputStream; + return s3client.getObject(req); } @Override diff --git a/src/main/java/dev/zarr/zarrjava/utils/IndexingUtils.java b/src/main/java/dev/zarr/zarrjava/utils/IndexingUtils.java index c68a137..e975a5c 100644 --- a/src/main/java/dev/zarr/zarrjava/utils/IndexingUtils.java +++ b/src/main/java/dev/zarr/zarrjava/utils/IndexingUtils.java @@ -18,7 +18,7 @@ public static long[][] computeChunkCoords(long[] arrayShape, int[] chunkShape, l final int ndim = arrayShape.length; long[] start = new long[ndim]; long[] end = new long[ndim]; - int numChunks = 1; + long numChunks = 1; for (int dimIdx = 0; dimIdx < ndim; dimIdx++) { final int staIdx = (int) (selOffset[dimIdx] / chunkShape[dimIdx]); final int endIdx = (int) ((selOffset[dimIdx] + selShape[dimIdx] - 1) / chunkShape[dimIdx]); @@ -27,7 +27,11 @@ public static long[][] computeChunkCoords(long[] arrayShape, int[] chunkShape, l end[dimIdx] = endIdx; } - final long[][] chunkCoords = new long[numChunks][]; + if (numChunks > Integer.MAX_VALUE) { + throw new ArithmeticException("Number of chunks exceeds Integer.MAX_VALUE"); + } + + final long[][] chunkCoords = new long[(int) numChunks][]; final long[] currentIdx = Arrays.copyOf(start, ndim); for (int i = 0; i < chunkCoords.length; i++) { @@ -78,10 +82,20 @@ public static ChunkProjection computeProjection( // selection starts before current chunk chunkOffset[dimIdx] = 0; // compute number of previous items, provides offset into output array - outOffset[dimIdx] = (int) (dimOffset - selOffset[dimIdx]); + long outOffsetValue = dimOffset - selOffset[dimIdx]; + if (outOffsetValue > Integer.MAX_VALUE) { + throw new ArithmeticException( + "Output offset exceeds Integer.MAX_VALUE at dimension " + dimIdx + ": " + outOffsetValue); + } + outOffset[dimIdx] = (int) outOffsetValue; } else { // selection starts within current chunk - chunkOffset[dimIdx] = (int) (selOffset[dimIdx] - dimOffset); + long chunkOffsetValue = selOffset[dimIdx] - dimOffset; + if (chunkOffsetValue > Integer.MAX_VALUE) { + throw new ArithmeticException( + "Chunk offset exceeds Integer.MAX_VALUE at dimension " + dimIdx + ": " + chunkOffsetValue); + } + chunkOffset[dimIdx] = (int) chunkOffsetValue; outOffset[dimIdx] = 0; } @@ -90,8 +104,12 @@ public static ChunkProjection computeProjection( shape[dimIdx] = chunkShape[dimIdx] - chunkOffset[dimIdx]; } else { // selection ends within current chunk - shape[dimIdx] = (int) (selOffset[dimIdx] + selShape[dimIdx] - dimOffset - - chunkOffset[dimIdx]); + long shapeValue = selOffset[dimIdx] + selShape[dimIdx] - dimOffset - chunkOffset[dimIdx]; + if (shapeValue > Integer.MAX_VALUE || shapeValue < 0) { + throw new ArithmeticException( + "Shape value exceeds Integer.MAX_VALUE or is negative at dimension " + dimIdx + ": " + shapeValue); + } + shape[dimIdx] = (int) shapeValue; } } @@ -112,8 +130,8 @@ public static long cOrderIndex(final long[] chunkCoords, final long[] arrayShape } public static long fOrderIndex(final long[] chunkCoords, final long[] arrayShape) { - int index = 0; - int multiplier = 1; + long index = 0; + long multiplier = 1; for (int i = 0; i < arrayShape.length; i++) { index += chunkCoords[i] * multiplier; diff --git a/src/test/java/dev/zarr/zarrjava/TestUtils.java b/src/test/java/dev/zarr/zarrjava/TestUtils.java index 68a5777..9745063 100644 --- a/src/test/java/dev/zarr/zarrjava/TestUtils.java +++ b/src/test/java/dev/zarr/zarrjava/TestUtils.java @@ -2,7 +2,7 @@ import dev.zarr.zarrjava.utils.IndexingUtils; -import org.junit.Test; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Assertions; import java.util.Arrays; @@ -76,5 +76,40 @@ public void testComputeProjection(){ Assertions.assertArrayEquals(new int[]{1, 17}, projection.shape); } + @Test + public void testFOrderIndexOverflow() { + // Create a shape that fits in long but would cause int overflow in fOrderIndex + // Shape: [2000, 2000, 1000] -> Total elements = 4,000,000,000 (exceeds Integer.MAX_VALUE 2.14B) + long[] arrayShape = {2000, 2000, 1000}; + + // Target coordinates near the end + long[] chunkCoords = {1999, 1999, 999}; + + // Expected index should be large (approx 4 billion) + long expectedIndex = 1999 + + 1999 * 2000L + + 999 * 2000L * 2000L; + + long actualIndex = IndexingUtils.fOrderIndex(chunkCoords, arrayShape); + + Assertions.assertEquals(expectedIndex, actualIndex, "fOrderIndex failed due to integer overflow"); + } + + @Test + public void testComputeChunkCoordsOverflow() { + // Shape: [100000, 100000] + long[] arrayShape = {100000, 100000}; + // Chunk: [1, 1] + int[] chunkShape = {1, 1}; + // Selection: Full array + long[] selOffset = {0, 0}; + long[] selShape = {100000, 100000}; + + // This should cause overflow: 100000 * 100000 = 10,000,000,000 > Integer.MAX_VALUE + Assertions.assertThrows(ArithmeticException.class, () -> { + IndexingUtils.computeChunkCoords(arrayShape, chunkShape, selOffset, selShape); + }); + } + } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java index e15401c..3ed3c10 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java @@ -822,4 +822,48 @@ public void testDefaultChunkShape() throws IOException, ZarrException { Assertions.assertTrue(mixedArray.metadata().chunkShape()[2] > 0); Assertions.assertTrue(mixedArray.metadata().chunkShape()[2] <= 2048); } + + @Test + public void testLargeArrayWithOffsetBeyondMaxInt() throws IOException, ZarrException { + // Create an array with second dimension exceeding Integer.MAX_VALUE + // Shape: [2, 3_000_000_000] - 3 billion elements in second dimension + // This array is mostly fillvalue, only write one small chunk + long largeSize = 3_000_000_000L; // 3 billion > Integer.MAX_VALUE (2.147B) + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("large_array_beyond_int"); + ArrayMetadata metadata = Array.metadataBuilder() + .withShape(largeSize, largeSize) + .withDataType(DataType.INT32) + .withChunkShape(1000, 1000) + .withFillValue(42) + .build(); + + Array array = Array.create(storeHandle, metadata); + + // Write a small chunk at position [0, 0] + int[] testData = new int[1000]; + Arrays.fill(testData, 100); + ucar.ma2.Array smallChunk = ucar.ma2.Array.factory(ucar.ma2.DataType.INT, new int[]{1, 1000}, testData); + array.write(new long[]{0, 0}, smallChunk); + + // Write a small chunk at position [1, Integer.MAX_VALUE + 1] + long beyondIntMax = (long)(Integer.MAX_VALUE) + 1; + long[] offset = new long[]{1, beyondIntMax}; + Arrays.fill(testData, 200); + smallChunk = ucar.ma2.Array.factory(ucar.ma2.DataType.INT, new int[]{1, 1000}, testData); + array.write(offset, smallChunk); + + // Read from the written region - should get our data + ucar.ma2.Array readStart = array.read(new long[]{0, 0}, new long[]{1, 100}); + Assertions.assertEquals(100, readStart.getInt(0), "Data at start should be written value"); + + // Read from position beyond Integer.MAX_VALUE - should get fillvalue + ucar.ma2.Array readFar = array.read(offset, new long[]{1, 100}); + Assertions.assertEquals(200, readFar.getInt(1), "Data beyond Integer.MAX_VALUE should be fillvalue"); + + // Verify metadata + Assertions.assertEquals(2, array.metadata().shape.length); + Assertions.assertEquals(largeSize, array.metadata().shape[0]); + Assertions.assertEquals(largeSize, array.metadata().shape[1]); + } } From 03c403798a99831b6e035486a6ba30ca06e85a3c Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Fri, 30 Jan 2026 21:24:40 +0100 Subject: [PATCH 4/8] better exceptions --- .../java/dev/zarr/zarrjava/core/Array.java | 10 +++- .../zarrjava/core/codec/CodecPipeline.java | 5 +- .../zarrjava/core/codec/core/BytesCodec.java | 2 +- .../zarr/zarrjava/store/BufferedZipStore.java | 15 ++++-- .../zarr/zarrjava/store/FilesystemStore.java | 47 +++++++++++++---- .../dev/zarr/zarrjava/store/HttpStore.java | 13 ++++- .../zarr/zarrjava/store/ReadOnlyZipStore.java | 10 +++- .../java/dev/zarr/zarrjava/store/S3Store.java | 52 +++++++++++++++---- .../zarr/zarrjava/store/StoreException.java | 44 ++++++++++++++++ src/main/java/dev/zarr/zarrjava/v3/Array.java | 6 +-- .../java/dev/zarr/zarrjava/v3/DataType.java | 2 +- src/main/java/dev/zarr/zarrjava/v3/Group.java | 8 ++- .../dev/zarr/zarrjava/v3/GroupMetadata.java | 5 +- 13 files changed, 178 insertions(+), 41 deletions(-) create mode 100644 src/main/java/dev/zarr/zarrjava/store/StoreException.java diff --git a/src/main/java/dev/zarr/zarrjava/core/Array.java b/src/main/java/dev/zarr/zarrjava/core/Array.java index 4b5ea92..884d4b3 100644 --- a/src/main/java/dev/zarr/zarrjava/core/Array.java +++ b/src/main/java/dev/zarr/zarrjava/core/Array.java @@ -122,8 +122,14 @@ public void write(long[] offset, ucar.ma2.Array array, boolean parallel) { ); } writeChunk(chunkCoords, chunkArray); - } catch (ZarrException | InvalidRangeException e) { - throw new RuntimeException(e); + } catch (ZarrException e) { + throw new RuntimeException( + "Failed to write chunk at coordinates " + Arrays.toString(chunkCoords) + + ": " + e.getMessage(), e); + } catch (InvalidRangeException e) { + throw new RuntimeException( + "Invalid array range when writing chunk at coordinates " + Arrays.toString(chunkCoords) + + ": " + e.getMessage(), e); } }); diff --git a/src/main/java/dev/zarr/zarrjava/core/codec/CodecPipeline.java b/src/main/java/dev/zarr/zarrjava/core/codec/CodecPipeline.java index 2c27d20..64c36d0 100644 --- a/src/main/java/dev/zarr/zarrjava/core/codec/CodecPipeline.java +++ b/src/main/java/dev/zarr/zarrjava/core/codec/CodecPipeline.java @@ -68,8 +68,9 @@ ArrayBytesCodec getArrayBytesCodec() { return (ArrayBytesCodec) codec; } } - throw new RuntimeException( - "Unreachable because the existence of exactly 1 ArrayBytes codec is asserted upon construction."); + throw new IllegalStateException( + "No ArrayBytesCodec found in codec pipeline. This should never happen as the existence " + + "of exactly 1 ArrayBytesCodec is validated during construction."); } BytesBytesCodec[] getBytesBytesCodecs() { diff --git a/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java b/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java index 3faf97a..41382b5 100644 --- a/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java +++ b/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java @@ -102,7 +102,7 @@ public ByteOrder getByteOrder() { case BIG: return ByteOrder.BIG_ENDIAN; default: - throw new RuntimeException("Unreachable"); + throw new IllegalStateException("Unknown endian type: " + this); } } } diff --git a/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java b/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java index bf47191..ab1afc8 100644 --- a/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java @@ -56,7 +56,10 @@ public BufferedZipStore(@Nonnull StoreHandle underlyingStore, @Nonnull Store.Lis try { loadBuffer(); } catch (IOException e) { - throw new RuntimeException("Failed to load buffer from underlying store", e); + throw StoreException.readFailed( + underlyingStore.toString(), + new String[]{}, + new IOException("Failed to load ZIP buffer from underlying store: " + underlyingStore, e)); } } @@ -285,7 +288,10 @@ public void set(String[] keys, ByteBuffer bytes) { try { writeBuffer(); } catch (IOException e) { - throw new RuntimeException("Failed to flush buffer to underlying store after set operation", e); + throw StoreException.writeFailed( + underlyingStore.toString(), + keys, + new IOException("Failed to flush ZIP buffer to underlying store after set operation", e)); } } } @@ -297,7 +303,10 @@ public void delete(String[] keys) { try { writeBuffer(); } catch (IOException e) { - throw new RuntimeException("Failed to flush buffer to underlying store after delete operation", e); + throw StoreException.deleteFailed( + underlyingStore.toString(), + keys, + new IOException("Failed to flush ZIP buffer to underlying store after delete operation", e)); } } } diff --git a/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java b/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java index 6e430cc..368e006 100644 --- a/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java @@ -133,7 +133,10 @@ public void set(String[] keys, ByteBuffer bytes) { try { Files.createDirectories(keyPath.getParent()); } catch (IOException e) { - throw new RuntimeException(e); + throw StoreException.writeFailed( + this.toString(), + keys, + new IOException("Failed to create parent directories for path: " + keyPath.getParent(), e)); } try (SeekableByteChannel channel = Files.newByteChannel(keyPath.toAbsolutePath(), StandardOpenOption.CREATE, @@ -142,18 +145,25 @@ public void set(String[] keys, ByteBuffer bytes) { )) { channel.write(bytes); } catch (IOException e) { - throw new RuntimeException(e); + throw StoreException.writeFailed( + this.toString(), + keys, + new IOException("Failed to write " + bytes.remaining() + " bytes to file: " + keyPath, e)); } } @Override public void delete(String[] keys) { + Path keyPath = resolveKeys(keys); try { - Files.delete(resolveKeys(keys)); + Files.delete(keyPath); } catch (NoSuchFileException e) { - // ignore + // ignore - file doesn't exist, which is the desired outcome } catch (IOException e) { - throw new RuntimeException(e); + throw StoreException.deleteFailed( + this.toString(), + keys, + new IOException("Failed to delete file: " + keyPath, e)); } } @@ -180,7 +190,10 @@ public Stream list(String[] prefix) { .filter(Files::isRegularFile) .map(path -> pathToKeyArray(rootPath, path, prefix)); } catch (IOException e) { - throw new RuntimeException("Failed to list store content", e); + throw StoreException.listFailed( + this.toString(), + prefix, + new IOException("Failed to walk directory tree at: " + rootPath, e)); } } @@ -193,7 +206,10 @@ public Stream listChildren(String[] prefix) { try { return Files.list(rootPath).map(path -> path.getFileName().toString()); } catch (IOException e) { - throw new RuntimeException("Failed to list store children", e); + throw StoreException.listFailed( + this.toString(), + prefix, + new IOException("Failed to list directory contents at: " + rootPath, e)); } } @@ -219,7 +235,8 @@ public InputStream getInputStream(String[] keys, long start, long end) { if (start > 0) { long skipped = inputStream.skip(start); if (skipped < start) { - throw new IOException("Unable to skip to the desired start position."); + throw new IOException("Unable to skip to position " + start + + ", only skipped " + skipped + " bytes in file: " + keyPath); } } if (end != -1) { @@ -229,17 +246,25 @@ public InputStream getInputStream(String[] keys, long start, long end) { return inputStream; } } catch (IOException e) { - throw new RuntimeException(e); + throw StoreException.readFailed( + this.toString(), + keys, + new IOException("Failed to open input stream for file: " + keyPath + + " (start: " + start + ", end: " + end + ")", e)); } } public long getSize(String[] keys) { + Path keyPath = resolveKeys(keys); try { - return Files.size(resolveKeys(keys)); + return Files.size(keyPath); } catch (NoSuchFileException e) { return -1; } catch (IOException e) { - throw new RuntimeException(e); + throw StoreException.readFailed( + this.toString(), + keys, + new IOException("Failed to get file size for: " + keyPath, e)); } } } diff --git a/src/main/java/dev/zarr/zarrjava/store/HttpStore.java b/src/main/java/dev/zarr/zarrjava/store/HttpStore.java index f047144..178899f 100644 --- a/src/main/java/dev/zarr/zarrjava/store/HttpStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/HttpStore.java @@ -148,11 +148,12 @@ public void close() throws IOException { @Override public long getSize(String[] keys) { + String url = resolveKeys(keys); // Explicitly request "identity" encoding to prevent OkHttp from adding "gzip" // and subsequently stripping the Content-Length header. Request request = new Request.Builder() .head() - .url(resolveKeys(keys)) + .url(url) .header("Accept-Encoding", "identity") .build(); @@ -168,8 +169,16 @@ public long getSize(String[] keys) { return Long.parseLong(contentLength); } return -1; + } catch (NumberFormatException e) { + throw StoreException.readFailed( + this.toString(), + keys, + new IOException("Invalid Content-Length header value from: " + url, e)); } catch (IOException e) { - throw new RuntimeException(e); + throw StoreException.readFailed( + this.toString(), + keys, + new IOException("Failed to get content length from HTTP HEAD request to: " + url, e)); } } } diff --git a/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java b/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java index f5ed18a..9bdc729 100644 --- a/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java @@ -268,7 +268,10 @@ public long getSize(String[] keys) { // if size is not in header/cache, we fallback to reading InputStream inputStream = underlyingStore.getInputStream(); if (inputStream == null) { - throw new RuntimeException(new IOException("Underlying store input stream is null")); + throw StoreException.readFailed( + underlyingStore.toString(), + keys, + new IOException("Cannot get size - underlying store input stream is null")); } try (ZipArchiveInputStream zis = new ZipArchiveInputStream(inputStream)) { ZipArchiveEntry entry; @@ -295,7 +298,10 @@ public long getSize(String[] keys) { } return -1; // file not found } catch (IOException e) { - throw new RuntimeException(e); + throw StoreException.readFailed( + underlyingStore.toString(), + keys, + new IOException("Failed to read ZIP entry size for key: " + String.join("/", keys), e)); } } } diff --git a/src/main/java/dev/zarr/zarrjava/store/S3Store.java b/src/main/java/dev/zarr/zarrjava/store/S3Store.java index d7d21ef..85db5f4 100644 --- a/src/main/java/dev/zarr/zarrjava/store/S3Store.java +++ b/src/main/java/dev/zarr/zarrjava/store/S3Store.java @@ -42,8 +42,20 @@ String resolveKeys(String[] keys) { ByteBuffer get(GetObjectRequest getObjectRequest) { try (ResponseInputStream inputStream = s3client.getObject(getObjectRequest)) { return Utils.asByteBuffer(inputStream); - } catch (IOException e) { + } catch (NoSuchKeyException e) { + // Key doesn't exist, return null as per Store contract return null; + } catch (S3Exception e) { + // Include S3-specific error details + throw StoreException.readFailed( + this.toString(), + new String[]{getObjectRequest.key()}, + new IOException("S3 error (code: " + e.statusCode() + "): " + e.awsErrorDetails().errorMessage(), e)); + } catch (IOException e) { + throw StoreException.readFailed( + this.toString(), + new String[]{getObjectRequest.key()}, + new IOException("Failed to read S3 object content", e)); } } @@ -91,19 +103,39 @@ public void set(String[] keys, ByteBuffer bytes) { // Convert ByteBuffer to byte array and use RequestBody.fromBytes() // This properly sets Content-Length and avoids buffering the entire stream in memory byte[] data = Utils.toArray(bytes); - s3client.putObject( - PutObjectRequest.builder() - .bucket(bucketName) - .key(resolveKeys(keys)) - .build(), - RequestBody.fromBytes(data) - ); + String key = resolveKeys(keys); + try { + s3client.putObject( + PutObjectRequest.builder() + .bucket(bucketName) + .key(key) + .build(), + RequestBody.fromBytes(data) + ); + } catch (S3Exception e) { + throw StoreException.writeFailed( + this.toString(), + keys, + new IOException("S3 putObject failed (code: " + e.statusCode() + ") for key '" + key + + "', bucket '" + bucketName + "': " + e.awsErrorDetails().errorMessage(), e)); + } } @Override public void delete(String[] keys) { - s3client.deleteObject(DeleteObjectRequest.builder().bucket(bucketName).key(resolveKeys(keys)) - .build()); + String key = resolveKeys(keys); + try { + s3client.deleteObject(DeleteObjectRequest.builder() + .bucket(bucketName) + .key(key) + .build()); + } catch (S3Exception e) { + throw StoreException.deleteFailed( + this.toString(), + keys, + new IOException("S3 deleteObject failed (code: " + e.statusCode() + ") for key '" + key + + "', bucket '" + bucketName + "': " + e.awsErrorDetails().errorMessage(), e)); + } } @Override diff --git a/src/main/java/dev/zarr/zarrjava/store/StoreException.java b/src/main/java/dev/zarr/zarrjava/store/StoreException.java new file mode 100644 index 0000000..356c982 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/store/StoreException.java @@ -0,0 +1,44 @@ +package dev.zarr.zarrjava.store; + +/** + * Exception thrown when store operations fail. + * Provides context about which store and operation failed. + */ +public class StoreException extends RuntimeException { + + public StoreException(String message) { + super(message); + } + + public StoreException(String message, Throwable cause) { + super(message, cause); + } + + public static StoreException readFailed(String storePath, String[] keys, Throwable cause) { + return new StoreException( + String.format("Failed to read from store '%s' at key '%s': %s", + storePath, String.join("/", keys), cause.getMessage()), + cause); + } + + public static StoreException writeFailed(String storePath, String[] keys, Throwable cause) { + return new StoreException( + String.format("Failed to write to store '%s' at key '%s': %s", + storePath, String.join("/", keys), cause.getMessage()), + cause); + } + + public static StoreException deleteFailed(String storePath, String[] keys, Throwable cause) { + return new StoreException( + String.format("Failed to delete from store '%s' at key '%s': %s", + storePath, String.join("/", keys), cause.getMessage()), + cause); + } + + public static StoreException listFailed(String storePath, String[] keys, Throwable cause) { + return new StoreException( + String.format("Failed to list store contents at '%s' under key '%s': %s", + storePath, String.join("/", keys), cause.getMessage()), + cause); + } +} diff --git a/src/main/java/dev/zarr/zarrjava/v3/Array.java b/src/main/java/dev/zarr/zarrjava/v3/Array.java index 72aad1e..ea029db 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Array.java @@ -143,9 +143,9 @@ public static Array create(StoreHandle storeHandle, ArrayMetadata arrayMetadata, throws IOException, ZarrException { StoreHandle metadataHandle = storeHandle.resolve(ZARR_JSON); if (!existsOk && metadataHandle.exists()) { - throw new RuntimeException( - "Trying to create a new array in " + storeHandle + ". But " + metadataHandle - + " already exists."); + throw new ZarrException( + "Cannot create array at " + storeHandle + " - metadata file " + metadataHandle + + " already exists. Use existsOk=true to overwrite."); } ObjectWriter objectWriter = makeObjectWriter(); ByteBuffer metadataBytes = ByteBuffer.wrap(objectWriter.writeValueAsBytes(arrayMetadata)); diff --git a/src/main/java/dev/zarr/zarrjava/v3/DataType.java b/src/main/java/dev/zarr/zarrjava/v3/DataType.java index 65ab28a..159663d 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/DataType.java +++ b/src/main/java/dev/zarr/zarrjava/v3/DataType.java @@ -63,7 +63,7 @@ public ucar.ma2.DataType getMA2DataType() { case FLOAT64: return ucar.ma2.DataType.DOUBLE; default: - throw new RuntimeException("Unreachable"); + throw new IllegalStateException("Unknown DataType: " + this); } } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/Group.java b/src/main/java/dev/zarr/zarrjava/v3/Group.java index 5df6d5b..1d3aa6b 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Group.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Group.java @@ -200,8 +200,12 @@ public Stream list() { return metadataKeys.map(key -> { try { return get(Arrays.copyOf(key, key.length - 1)); - } catch (Exception e) { - throw new RuntimeException(e); + } catch (IOException e) { + throw new RuntimeException( + "Failed to read node metadata for key '" + String.join("/", key) + "': " + e.getMessage(), e); + } catch (ZarrException e) { + throw new RuntimeException( + "Failed to parse node metadata for key '" + String.join("/", key) + "': " + e.getMessage(), e); } }); } diff --git a/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java b/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java index 281087d..7d3dd85 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java @@ -47,8 +47,9 @@ public static GroupMetadata defaultValue() { try { return new GroupMetadata(ZARR_FORMAT, NODE_TYPE, new Attributes()); } catch (ZarrException e) { - // This should never happen - throw new RuntimeException(e); + // This should never happen with default values + throw new IllegalStateException( + "Failed to create default GroupMetadata - this indicates a programming error", e); } } From a5c8a50d4838b276ea5926e725499fd976c55589 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 2 Feb 2026 10:27:18 +0100 Subject: [PATCH 5/8] remove redundant exist calls --- .../java/dev/zarr/zarrjava/core/Array.java | 11 +++--- .../zarr/zarrjava/store/FilesystemStore.java | 39 ++----------------- .../v3/codec/core/ShardingIndexedCodec.java | 5 ++- 3 files changed, 14 insertions(+), 41 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/core/Array.java b/src/main/java/dev/zarr/zarrjava/core/Array.java index 884d4b3..db48551 100644 --- a/src/main/java/dev/zarr/zarrjava/core/Array.java +++ b/src/main/java/dev/zarr/zarrjava/core/Array.java @@ -317,8 +317,6 @@ public ucar.ma2.Array read(final long[] offset, final long[] shape, final boolea final String[] chunkKeys = metadata.chunkKeyEncoding().encodeChunkKey(chunkCoords); final StoreHandle chunkHandle = storeHandle.resolve(chunkKeys); - if (!chunkHandle.exists()) return; - if (codecPipeline.supportsPartialDecode()) { final ucar.ma2.Array chunkArray = codecPipeline.decodePartial(chunkHandle, Utils.toLongArray(chunkProjection.chunkOffset), chunkProjection.shape); @@ -326,9 +324,12 @@ public ucar.ma2.Array read(final long[] offset, final long[] shape, final boolea chunkProjection.outOffset, chunkProjection.shape ); } else { - MultiArrayUtils.copyRegion(readChunk(chunkCoords), chunkProjection.chunkOffset, - outputArray, chunkProjection.outOffset, chunkProjection.shape - ); + ByteBuffer chunkBytes = chunkHandle.read(); + if (chunkBytes != null) { + MultiArrayUtils.copyRegion(codecPipeline.decode(chunkBytes), chunkProjection.chunkOffset, + outputArray, chunkProjection.outOffset, chunkProjection.shape + ); + } } } catch (ZarrException e) { diff --git a/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java b/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java index 368e006..4c29614 100644 --- a/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java @@ -30,43 +30,12 @@ Path resolveKeys(String[] keys) { for (String key : keys) { newPath = newPath.resolve(key); } + Path absRoot = path.toAbsolutePath().normalize(); + Path absTarget = newPath.toAbsolutePath().normalize(); - try { - // Use toRealPath() to resolve symlinks and verify path is within root - // For non-existent paths, validate the existing parent path - Path absoluteRoot = path.toAbsolutePath().normalize(); - Path targetPath = newPath.toAbsolutePath().normalize(); - - // Try to get real path if it exists (follows symlinks) - if (Files.exists(targetPath)) { - Path realTarget = targetPath.toRealPath(); - Path realRoot = absoluteRoot.toRealPath(); - if (!realTarget.startsWith(realRoot)) { - throw new IllegalArgumentException("Key resolves outside of store root: " + realTarget); - } - } else { - // For non-existent paths, check the normalized path - // and ensure existing parent doesn't escape via symlinks - Path parent = targetPath.getParent(); - if (parent != null && Files.exists(parent)) { - Path realParent = parent.toRealPath(); - Path realRoot = absoluteRoot.toRealPath(); - if (!realParent.startsWith(realRoot)) { - throw new IllegalArgumentException("Parent path resolves outside of store root: " + realParent); - } - } else if (!targetPath.startsWith(absoluteRoot)) { - throw new IllegalArgumentException("Key resolves outside of store root: " + targetPath); - } - } - } catch (IOException e) { - // If toRealPath() fails, fall back to normalized path check - Path absoluteRoot = path.toAbsolutePath().normalize(); - Path absoluteTarget = newPath.toAbsolutePath().normalize(); - if (!absoluteTarget.startsWith(absoluteRoot)) { - throw new IllegalArgumentException("Key resolves outside of store root: " + absoluteTarget); - } + if (!absTarget.startsWith(absRoot)) { + throw new IllegalArgumentException("Key resolves outside of store root: " + absTarget); } - return newPath.normalize(); } diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java index de4092e..085c677 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java @@ -198,7 +198,10 @@ private Array decodeInternal( throw new ZarrException("Only index_location \"start\" or \"end\" are supported."); } if (shardIndexBytes == null) { - throw new ZarrException("Could not read shard index."); + if (arrayMetadata.parsedFillValue != null) { + MultiArrayUtils.fill(outputArray, arrayMetadata.parsedFillValue); + } + return outputArray; } final Array shardIndexArray = indexCodecPipeline.decode(shardIndexBytes); long[][] allChunkCoords = IndexingUtils.computeChunkCoords(shardMetadata.shape, From f7ff2be40c5836933d80eca74c1ad97e26234e15 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 2 Feb 2026 10:48:56 +0100 Subject: [PATCH 6/8] returning null only on key not found --- .../zarr/zarrjava/store/FilesystemStore.java | 17 +++++++---- .../dev/zarr/zarrjava/store/HttpStore.java | 28 +++++++++++++------ .../zarr/zarrjava/store/ReadOnlyZipStore.java | 12 +++++--- 3 files changed, 39 insertions(+), 18 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java b/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java index 4c29614..3644121 100644 --- a/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java @@ -49,8 +49,10 @@ public boolean exists(String[] keys) { public ByteBuffer get(String[] keys) { try { return ByteBuffer.wrap(Files.readAllBytes(resolveKeys(keys))); - } catch (IOException e) { + } catch (NoSuchFileException e) { return null; + } catch (IOException e) { + throw StoreException.readFailed(this.toString(), keys, e); } } @@ -70,8 +72,10 @@ public ByteBuffer get(String[] keys, long start) { byteChannel.read(bytes); bytes.rewind(); return bytes; - } catch (IOException e) { + } catch (NoSuchFileException e) { return null; + } catch (IOException e) { + throw StoreException.readFailed(this.toString(), keys, e); } } @@ -90,8 +94,10 @@ public ByteBuffer get(String[] keys, long start, long end) { byteChannel.read(bytes); bytes.rewind(); return bytes; - } catch (IOException e) { + } catch (NoSuchFileException e) { return null; + } catch (IOException e) { + throw StoreException.readFailed(this.toString(), keys, e); } } @@ -197,9 +203,6 @@ public String toString() { public InputStream getInputStream(String[] keys, long start, long end) { Path keyPath = resolveKeys(keys); try { - if (!Files.exists(keyPath)) { - return null; - } InputStream inputStream = Files.newInputStream(keyPath); if (start > 0) { long skipped = inputStream.skip(start); @@ -214,6 +217,8 @@ public InputStream getInputStream(String[] keys, long start, long end) { } else { return inputStream; } + } catch (NoSuchFileException e) { + return null; } catch (IOException e) { throw StoreException.readFailed( this.toString(), diff --git a/src/main/java/dev/zarr/zarrjava/store/HttpStore.java b/src/main/java/dev/zarr/zarrjava/store/HttpStore.java index 178899f..c2cc568 100644 --- a/src/main/java/dev/zarr/zarrjava/store/HttpStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/HttpStore.java @@ -36,12 +36,18 @@ String resolveKeys(String[] keys) { } @Nullable - ByteBuffer get(Request request) { + ByteBuffer get(Request request, String[] keys) { Call call = httpClient.newCall(request); try { Response response = call.execute(); if (!response.isSuccessful()) { - return null; + if (response.code() == 404) { + return null; + } + throw StoreException.readFailed( + this.toString(), + keys, + new IOException("HTTP request failed with status code: " + response.code() + " " + response.message())); } try (ResponseBody body = response.body()) { if (body == null) { @@ -50,7 +56,7 @@ ByteBuffer get(Request request) { return ByteBuffer.wrap(body.bytes()); } } catch (IOException e) { - return null; + throw StoreException.readFailed(this.toString(), keys, e); } } @@ -70,7 +76,7 @@ public boolean exists(String[] keys) { @Override public ByteBuffer get(String[] keys) { Request request = new Request.Builder().url(resolveKeys(keys)).build(); - return get(request); + return get(request, keys); } @Nullable @@ -80,7 +86,7 @@ public ByteBuffer get(String[] keys, long start) { "Range", start < 0 ? String.format("bytes=%d", start) : String.format("bytes=%d-", start)) .build(); - return get(request); + return get(request, keys); } @Nullable @@ -91,7 +97,7 @@ public ByteBuffer get(String[] keys, long start, long end) { } Request request = new Request.Builder().url(resolveKeys(keys)).header( "Range", String.format("bytes=%d-%d", start, end - 1)).build(); - return get(request); + return get(request, keys); } @Override @@ -127,7 +133,13 @@ public InputStream getInputStream(String[] keys, long start, long end) { try { Response response = call.execute(); if (!response.isSuccessful()) { - return null; + if (response.code() == 404) { + return null; + } + throw StoreException.readFailed( + this.toString(), + keys, + new IOException("HTTP request failed with status code: " + response.code() + " " + response.message())); } ResponseBody body = response.body(); if (body == null) return null; @@ -142,7 +154,7 @@ public void close() throws IOException { } }; } catch (IOException e) { - return null; + throw StoreException.readFailed(this.toString(), keys, e); } } diff --git a/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java b/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java index 9bdc729..689db73 100644 --- a/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java @@ -64,7 +64,11 @@ private synchronized void ensureCache() { fileIndex.put(name, entry.getSize()); } } - } catch (IOException ignored) { + } catch (IOException e) { + throw StoreException.readFailed( + underlyingStore.toString(), + new String[]{}, + new IOException("Failed to read ZIP directory from underlying store", e)); } isCached = true; } @@ -140,7 +144,7 @@ public ByteBuffer get(String[] keys, long start, long end) { return ByteBuffer.wrap(bytes); } } catch (IOException e) { - return null; + throw StoreException.readFailed(underlyingStore.toString(), keys, e); } return null; } @@ -248,9 +252,9 @@ public InputStream getInputStream(String[] keys, long start, long end) { return new BoundedInputStream(zis, bytesToRead); } return null; - } catch (IOException ignored) { + } catch (IOException e) { + throw StoreException.readFailed(underlyingStore.toString(), keys, e); } - return null; } @Override From 8095b135d49d233687b124e6b7a40fbf6f0b9861 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 2 Feb 2026 11:06:31 +0100 Subject: [PATCH 7/8] remove unused partialDecode, partialEncode as decodePartial exists --- .../zarr/zarrjava/core/codec/CodecPipeline.java | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/core/codec/CodecPipeline.java b/src/main/java/dev/zarr/zarrjava/core/codec/CodecPipeline.java index 64c36d0..d06b316 100644 --- a/src/main/java/dev/zarr/zarrjava/core/codec/CodecPipeline.java +++ b/src/main/java/dev/zarr/zarrjava/core/codec/CodecPipeline.java @@ -159,18 +159,4 @@ public long computeEncodedSize(long inputByteLength, CoreArrayMetadata arrayMeta } return inputByteLength; } - - public Array partialDecode( - StoreHandle valueHandle, long[] offset, int[] shape, - CoreArrayMetadata arrayMetadata - ) { - return null; // TODO - } - - public ByteBuffer partialEncode( - StoreHandle oldValueHandle, Array array, long[] offset, int[] shape, - CoreArrayMetadata arrayMetadata - ) { - return null; // TODO - } } From c1af057da5a322073787d9daf8f94189617b74e4 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 2 Feb 2026 11:31:44 +0100 Subject: [PATCH 8/8] remove unused fOrderIndex and cOrderIndex --- .../zarr/zarrjava/utils/IndexingUtils.java | 25 ------------------- .../v3/codec/core/ShardingIndexedCodec.java | 2 -- .../java/dev/zarr/zarrjava/TestUtils.java | 19 -------------- 3 files changed, 46 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/utils/IndexingUtils.java b/src/main/java/dev/zarr/zarrjava/utils/IndexingUtils.java index e975a5c..404d314 100644 --- a/src/main/java/dev/zarr/zarrjava/utils/IndexingUtils.java +++ b/src/main/java/dev/zarr/zarrjava/utils/IndexingUtils.java @@ -116,31 +116,6 @@ public static ChunkProjection computeProjection( return new ChunkProjection(chunkCoords, chunkOffset, outOffset, shape); } - - public static long cOrderIndex(final long[] chunkCoords, final long[] arrayShape) { - long index = 0; - long multiplier = 1; - - for (int i = arrayShape.length - 1; i >= 0; i--) { - index += chunkCoords[i] * multiplier; - multiplier *= arrayShape[i]; - } - - return index; - } - - public static long fOrderIndex(final long[] chunkCoords, final long[] arrayShape) { - long index = 0; - long multiplier = 1; - - for (int i = 0; i < arrayShape.length; i++) { - index += chunkCoords[i] * multiplier; - multiplier *= arrayShape[i]; - } - - return index; - } - public static boolean isFullChunk(final int[] selOffset, final int[] selShape, final int[] chunkShape) { if (selOffset.length != selShape.length) { diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java index 085c677..73f86e1 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java @@ -119,8 +119,6 @@ public ByteBuffer encode(final Array shardArray) throws ZarrException { .forEach( chunkCoords -> { try { - final int i = - (int) IndexingUtils.cOrderIndex(chunkCoords, Utils.toLongArray(chunksPerShard)); final IndexingUtils.ChunkProjection chunkProjection = IndexingUtils.computeProjection(chunkCoords, shardMetadata.shape, shardMetadata.chunkShape diff --git a/src/test/java/dev/zarr/zarrjava/TestUtils.java b/src/test/java/dev/zarr/zarrjava/TestUtils.java index 9745063..ad02558 100644 --- a/src/test/java/dev/zarr/zarrjava/TestUtils.java +++ b/src/test/java/dev/zarr/zarrjava/TestUtils.java @@ -76,25 +76,6 @@ public void testComputeProjection(){ Assertions.assertArrayEquals(new int[]{1, 17}, projection.shape); } - @Test - public void testFOrderIndexOverflow() { - // Create a shape that fits in long but would cause int overflow in fOrderIndex - // Shape: [2000, 2000, 1000] -> Total elements = 4,000,000,000 (exceeds Integer.MAX_VALUE 2.14B) - long[] arrayShape = {2000, 2000, 1000}; - - // Target coordinates near the end - long[] chunkCoords = {1999, 1999, 999}; - - // Expected index should be large (approx 4 billion) - long expectedIndex = 1999 + - 1999 * 2000L + - 999 * 2000L * 2000L; - - long actualIndex = IndexingUtils.fOrderIndex(chunkCoords, arrayShape); - - Assertions.assertEquals(expectedIndex, actualIndex, "fOrderIndex failed due to integer overflow"); - } - @Test public void testComputeChunkCoordsOverflow() { // Shape: [100000, 100000]