From 8c28b18742c6dc1d1363ea513123c67565518ca3 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Fri, 16 Jan 2026 04:30:32 -0800 Subject: [PATCH 01/17] vectored read config changes --- .../hadoop/fs/azurebfs/AbfsConfiguration.java | 58 ++++++++++++++ .../azurebfs/constants/ConfigurationKeys.java | 19 +++++ .../constants/FileSystemConfigurations.java | 4 + .../azurebfs/enums/VectoredReadStrategy.java | 77 +++++++++++++++++++ .../fs/azurebfs/services/AbfsInputStream.java | 63 ++++++++++++++- 5 files changed, 217 insertions(+), 4 deletions(-) create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/enums/VectoredReadStrategy.java diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index e9dd94ff4ed88..78dc38a7489a4 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -51,6 +51,7 @@ import org.apache.hadoop.fs.azurebfs.diagnostics.LongConfigurationBasicValidator; import org.apache.hadoop.fs.azurebfs.diagnostics.StringConfigurationBasicValidator; import org.apache.hadoop.fs.azurebfs.enums.Trilean; +import org.apache.hadoop.fs.azurebfs.enums.VectoredReadStrategy; import org.apache.hadoop.fs.azurebfs.extensions.CustomTokenProviderAdaptee; import org.apache.hadoop.fs.azurebfs.extensions.EncryptionContextProvider; import org.apache.hadoop.fs.azurebfs.extensions.SASTokenProvider; @@ -679,6 +680,18 @@ public class AbfsConfiguration{ DefaultValue = DEFAULT_FS_AZURE_LOWEST_REQUEST_PRIORITY_VALUE) private int prefetchRequestPriorityValue; + @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_VECTORED_READ_STRATEGY, + DefaultValue = DEFAULT_FS_AZURE_VECTORED_READ_STRATEGY) + private String vectoredReadStrategy; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_MIN_SEEK_FOR_VECTORED_READS, + DefaultValue = DEFAULT_FS_AZURE_MIN_SEEK_FOR_VECTORED_READS) + private int minSeekForVectoredReads; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_MAX_SEEK_FOR_VECTORED_READS, + DefaultValue = DEFAULT_FS_AZURE_MAX_SEEK_FOR_VECTORED_READS) + private int maxSeekForVectoredReads; + private String clientProvidedEncryptionKey; private String clientProvidedEncryptionKeySHA; @@ -2246,4 +2259,49 @@ public int getTailLatencyAnalysisWindowGranularity() { public int getTailLatencyMaxRetryCount() { return tailLatencyMaxRetryCount; } + + /** + * Returns the configured vectored read strategy. + * + *

+ * The configuration value is parsed in a case-insensitive manner and may be + * specified either using the enum name (for example, + * {@code TPS_OPTIMIZED}) or the short name (for example, {@code TPS}). + *

+ * + * @return the resolved {@link VectoredReadStrategy} + * + * @throws IllegalArgumentException if the configured value is invalid + */ + public VectoredReadStrategy getVectoredReadStrategy() { + try { + return VectoredReadStrategy.fromString(vectoredReadStrategy); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException( + "Invalid value for " + FS_AZURE_VECTORED_READ_STRATEGY + + ": " + vectoredReadStrategy + + ". Expected one of: TPS, THROUGHPUT, TPS_OPTIMIZED, THROUGHPUT_OPTIMIZED", + e); + } + } + + /** + * Returns the minimum gap between adjacent read ranges that qualifies them + * for merging during vectored reads. + * + * @return minimum gap threshold for range merging + */ + public int getMinSeekForVectoredReads() { + return minSeekForVectoredReads; + } + + /** + * Returns the maximum gap between adjacent read ranges allowed when + * considering them for merging during vectored reads. + * + * @return maximum gap threshold for range merging + */ + public int getMaxSeekForVectoredReads() { + return maxSeekForVectoredReads; + } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java index c5eb9235fbb54..00576e70ec919 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java @@ -657,5 +657,24 @@ public static String containerProperty(String property, String fsName, String ac */ public static final String FS_AZURE_TAIL_LATENCY_MAX_RETRY_COUNT = "fs.azure.tail.latency.max.retry.count"; + /** + * Configuration key to control the vectored read strategy used by ABFS: {@value} + */ + public static final String FS_AZURE_VECTORED_READ_STRATEGY = "fs.azure.vectored.read.strategy"; + + /** + * Configuration key that defines the minimum gap between adjacent read ranges + * for merging ranges during vectored reads in ABFS: {@value}. + */ + public static final String FS_AZURE_MIN_SEEK_FOR_VECTORED_READS = + "fs.azure.min.seek.for.vectored.reads"; + + /** + * Configuration key that defines the maximum gap between adjacent read ranges + * for merging ranges during vectored reads in ABFS: {@value}. + */ + public static final String FS_AZURE_MAX_SEEK_FOR_VECTORED_READS = + "fs.azure.max.seek.for.vectored.reads"; + private ConfigurationKeys() {} } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java index 6f76f2e033c06..2e78d778ad924 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java @@ -20,6 +20,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.azurebfs.enums.VectoredReadStrategy; import org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.EMPTY_STRING; @@ -440,6 +441,9 @@ public final class FileSystemConfigurations { public static final int MIN_FS_AZURE_TAIL_LATENCY_ANALYSIS_WINDOW_GRANULARITY = 1; public static final int DEFAULT_FS_AZURE_TAIL_LATENCY_PERCENTILE_COMPUTATION_INTERVAL_MILLIS = 500; public static final int DEFAULT_FS_AZURE_TAIL_LATENCY_MAX_RETRY_COUNT = 1; + public static final String DEFAULT_FS_AZURE_VECTORED_READ_STRATEGY = "TPS"; + public static final int DEFAULT_FS_AZURE_MIN_SEEK_FOR_VECTORED_READS = ONE_MB; + public static final int DEFAULT_FS_AZURE_MAX_SEEK_FOR_VECTORED_READS = 4 * ONE_MB; private FileSystemConfigurations() {} } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/enums/VectoredReadStrategy.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/enums/VectoredReadStrategy.java new file mode 100644 index 0000000000000..d67499ddb9dd5 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/enums/VectoredReadStrategy.java @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.enums; + +/** + * Defines the strategy used for vectored reads in ABFS. + * + *

+ * The strategy controls how read ranges are planned and executed, trading off + * between request parallelism and per-request payload size. + *

+ */ +public enum VectoredReadStrategy { + + /** + * Optimizes for transactions per second (TPS). + */ + TPS_OPTIMIZED("TPS"), + + /** + * Optimizes for overall data throughput. + */ + THROUGHPUT_OPTIMIZED("THROUGHPUT"); + + /** Short name used for configuration and logging. */ + private final String name; + + /** + * Constructs a vectored read strategy with a short, user-friendly name. + * + * @param name short identifier for the strategy + */ + VectoredReadStrategy(String name) { + this.name = name; + } + + /** + * Returns the short name of the vectored read strategy. + * + * @return short strategy name + */ + public String getName() { + return name; + } + + /** + * Parses a configuration value into a {@link VectoredReadStrategy}. + * @param value configuration value + * @return matching vectored read strategy + * @throws IllegalArgumentException if the value is invalid + */ + public static VectoredReadStrategy fromString(String value) { + for (VectoredReadStrategy strategy : values()) { + if (strategy.name().equalsIgnoreCase(value) + || strategy.getName().equalsIgnoreCase(value)) { + return strategy; + } + } + throw new IllegalArgumentException("Invalid vectored read strategy: " + value); + } +} \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java index 31b6f0f073940..b541f1f80523b 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java @@ -22,12 +22,18 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.net.HttpURLConnection; +import java.nio.ByteBuffer; +import java.util.List; import java.util.UUID; +import java.util.function.IntFunction; +import java.util.concurrent.locks.ReentrantLock; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.fs.PositionedReadable; +import org.apache.hadoop.fs.FileRange; +import org.apache.hadoop.fs.azurebfs.AbfsConfiguration; import org.apache.hadoop.fs.azurebfs.constants.ReadType; +import org.apache.hadoop.fs.azurebfs.enums.VectoredReadStrategy; import org.apache.hadoop.fs.impl.BackReference; import org.apache.hadoop.util.Preconditions; @@ -133,6 +139,8 @@ public class AbfsInputStream extends FSInputStream implements CanUnbuffer, /** ABFS instance to be held by the input stream to avoid GC close. */ private final BackReference fsBackRef; private final ReadBufferManager readBufferManager; + private static volatile VectoredReadHandler vectoredReadHandler; + private static final ReentrantLock VECTORED_READ_HANDLER_LOCK = new ReentrantLock(); public AbfsInputStream( final AbfsClient client, @@ -316,6 +324,54 @@ public synchronized int read(final byte[] b, final int off, final int len) throw return totalReadBytes > 0 ? totalReadBytes : lastReadBytes; } + /** + * Returns the singleton {@link VectoredReadHandler} shared across all streams. + * + *

+ * The handler is lazily initialized using double-checked locking to ensure + * thread-safe, one-time creation with minimal synchronization overhead. + *

+ * + * @param readBufferManager shared read buffer manager + * @param abfsConfiguration ABFS configuration + * @return shared {@link VectoredReadHandler} + */ + static VectoredReadHandler getVectoredReadHandler( + ReadBufferManager readBufferManager, + AbfsConfiguration abfsConfiguration) { + + if (vectoredReadHandler == null) { + VECTORED_READ_HANDLER_LOCK.lock(); + try { + if (vectoredReadHandler == null) { + vectoredReadHandler = + new VectoredReadHandler( + readBufferManager, + abfsConfiguration.getVectoredReadStrategy()); + } + } finally { + VECTORED_READ_HANDLER_LOCK.unlock(); + } + } + return vectoredReadHandler; + } + + /** + * {@inheritDoc} + * Vectored read implementation for AbfsInputStream. + * + * @param ranges the byte ranges to read. + * @param allocate the function to allocate ByteBuffer. + * + * @throws IOException IOE if any. + */ + @Override + public void readVectored(List ranges, + IntFunction allocate) throws IOException { + VectoredReadHandler vectoredReadHandler = getVectoredReadHandler(readBufferManager, client.getAbfsConfiguration()); + vectoredReadHandler.readVectored(this, ranges, allocate); + } + private boolean shouldReadFully() { return this.firstRead && this.context.readSmallFilesCompletely() && this.contentLength <= this.bufferSize; @@ -943,12 +999,11 @@ ReadBufferManager getReadBufferManager() { @Override public int minSeekForVectorReads() { - return S_128K; + return client.getAbfsConfiguration().getMinSeekForVectoredReads(); } @Override public int maxReadSizeForVectorReads() { - return S_2M; + return client.getAbfsConfiguration().getMaxSeekForVectoredReads(); } - } From 08617b7d6a3a8a5aeedf89fc84141b438741968a Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Tue, 20 Jan 2026 03:33:27 -0800 Subject: [PATCH 02/17] Vectored read code --- .../hadoop/fs/azurebfs/AbfsConfiguration.java | 14 + .../azurebfs/constants/ConfigurationKeys.java | 7 + .../constants/FileSystemConfigurations.java | 1 + .../fs/azurebfs/constants/ReadType.java | 5 + .../hadoop/fs/azurebfs/enums/BufferType.java | 36 ++ .../fs/azurebfs/services/AbfsInputStream.java | 44 +-- .../fs/azurebfs/services/ReadBuffer.java | 86 ++++- .../azurebfs/services/ReadBufferManager.java | 24 ++ .../services/ReadBufferManagerV1.java | 155 +++++++- .../services/ReadBufferManagerV2.java | 145 +++++++- .../services/VectoredReadHandler.java | 334 ++++++++++++++++++ .../services/ITestReadBufferManager.java | 2 +- 12 files changed, 805 insertions(+), 48 deletions(-) create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/enums/BufferType.java create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VectoredReadHandler.java diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index 78dc38a7489a4..00ad6914128f1 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -692,6 +692,10 @@ public class AbfsConfiguration{ DefaultValue = DEFAULT_FS_AZURE_MAX_SEEK_FOR_VECTORED_READS) private int maxSeekForVectoredReads; + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_MAX_SEEK_FOR_VECTORED_READS_THROUGHPUT, + DefaultValue = DEFAULT_FS_AZURE_MAX_SEEK_FOR_VECTORED_READS_THROUGHPUT) + private int maxSeekForVectoredReadsThroughput; + private String clientProvidedEncryptionKey; private String clientProvidedEncryptionKeySHA; @@ -2304,4 +2308,14 @@ public int getMinSeekForVectoredReads() { public int getMaxSeekForVectoredReads() { return maxSeekForVectoredReads; } + + /** + * Returns the maximum gap between adjacent read ranges allowed when + * considering them for merging during vectored reads throughput optimized.. + * + * @return maximum gap threshold for range merging + */ + public int getMaxSeekForVectoredReadsThroughput() { + return maxSeekForVectoredReadsThroughput; + } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java index 00576e70ec919..d8dc4976a1b54 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java @@ -676,5 +676,12 @@ public static String containerProperty(String property, String fsName, String ac public static final String FS_AZURE_MAX_SEEK_FOR_VECTORED_READS = "fs.azure.max.seek.for.vectored.reads"; + /** + * Configuration key that defines the maximum gap between adjacent read ranges + * for merging ranges during vectored reads in ABFS throughput optimized: {@value}. + */ + public static final String FS_AZURE_MAX_SEEK_FOR_VECTORED_READS_THROUGHPUT = + "fs.azure.max.seek.for.vectored.reads.throughput"; + private ConfigurationKeys() {} } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java index 2e78d778ad924..f342e73a72bcc 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java @@ -444,6 +444,7 @@ public final class FileSystemConfigurations { public static final String DEFAULT_FS_AZURE_VECTORED_READ_STRATEGY = "TPS"; public static final int DEFAULT_FS_AZURE_MIN_SEEK_FOR_VECTORED_READS = ONE_MB; public static final int DEFAULT_FS_AZURE_MAX_SEEK_FOR_VECTORED_READS = 4 * ONE_MB; + public static final int DEFAULT_FS_AZURE_MAX_SEEK_FOR_VECTORED_READS_THROUGHPUT = 8 * ONE_MB; private FileSystemConfigurations() {} } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ReadType.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ReadType.java index 332a5a5ac56e2..184eabf883192 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ReadType.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ReadType.java @@ -48,6 +48,11 @@ public enum ReadType { * Only triggered when small file read optimization kicks in. */ SMALLFILE_READ("SR"), + /** + * Read multiple disjoint ranges from the storage service using vectored reads. + * Used to coalesce and execute non-contiguous reads efficiently. + */ + VECTORED_READ("VR"), /** * None of the above read types were applicable. */ diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/enums/BufferType.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/enums/BufferType.java new file mode 100644 index 0000000000000..85c6d1e2e7d86 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/enums/BufferType.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.enums; + +import org.apache.hadoop.fs.azurebfs.services.ReadBuffer; + +/** + * Enum for buffer types. + * Used in {@link ReadBuffer} to separate normal vs vectored read. + */ +public enum BufferType { + /** + * Normal read buffer. + */ + NORMAL, + /** + * Vectored read buffer. + */ + VECTORED +} diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java index b541f1f80523b..2640e13b2e515 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java @@ -139,8 +139,6 @@ public class AbfsInputStream extends FSInputStream implements CanUnbuffer, /** ABFS instance to be held by the input stream to avoid GC close. */ private final BackReference fsBackRef; private final ReadBufferManager readBufferManager; - private static volatile VectoredReadHandler vectoredReadHandler; - private static final ReentrantLock VECTORED_READ_HANDLER_LOCK = new ReentrantLock(); public AbfsInputStream( final AbfsClient client, @@ -196,7 +194,7 @@ public AbfsInputStream( readAheadBlockSize, client.getAbfsConfiguration()); readBufferManager = ReadBufferManagerV2.getBufferManager(client.getAbfsCounters()); } else { - ReadBufferManagerV1.setReadBufferManagerConfigs(readAheadBlockSize); + ReadBufferManagerV1.setReadBufferManagerConfigs(readAheadBlockSize, client.getAbfsConfiguration()); readBufferManager = ReadBufferManagerV1.getBufferManager(); } @@ -324,52 +322,18 @@ public synchronized int read(final byte[] b, final int off, final int len) throw return totalReadBytes > 0 ? totalReadBytes : lastReadBytes; } - /** - * Returns the singleton {@link VectoredReadHandler} shared across all streams. - * - *

- * The handler is lazily initialized using double-checked locking to ensure - * thread-safe, one-time creation with minimal synchronization overhead. - *

- * - * @param readBufferManager shared read buffer manager - * @param abfsConfiguration ABFS configuration - * @return shared {@link VectoredReadHandler} - */ - static VectoredReadHandler getVectoredReadHandler( - ReadBufferManager readBufferManager, - AbfsConfiguration abfsConfiguration) { - - if (vectoredReadHandler == null) { - VECTORED_READ_HANDLER_LOCK.lock(); - try { - if (vectoredReadHandler == null) { - vectoredReadHandler = - new VectoredReadHandler( - readBufferManager, - abfsConfiguration.getVectoredReadStrategy()); - } - } finally { - VECTORED_READ_HANDLER_LOCK.unlock(); - } - } - return vectoredReadHandler; - } - /** * {@inheritDoc} * Vectored read implementation for AbfsInputStream. * * @param ranges the byte ranges to read. * @param allocate the function to allocate ByteBuffer. - * - * @throws IOException IOE if any. */ @Override public void readVectored(List ranges, - IntFunction allocate) throws IOException { - VectoredReadHandler vectoredReadHandler = getVectoredReadHandler(readBufferManager, client.getAbfsConfiguration()); - vectoredReadHandler.readVectored(this, ranges, allocate); + IntFunction allocate) { + readBufferManager.getVectoredReadHandler() + .readVectored(this, ranges, allocate); } private boolean shouldReadFully() { diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBuffer.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBuffer.java index a6aa75f59d261..8fd1e26afa251 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBuffer.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBuffer.java @@ -19,26 +19,46 @@ package org.apache.hadoop.fs.azurebfs.services; import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.IntFunction; import org.apache.hadoop.fs.azurebfs.contracts.services.ReadBufferStatus; +import org.apache.hadoop.fs.azurebfs.enums.BufferType; import org.apache.hadoop.fs.azurebfs.utils.TracingContext; +import org.apache.hadoop.fs.impl.CombinedFileRange; import static org.apache.hadoop.fs.azurebfs.contracts.services.ReadBufferStatus.READ_FAILED; -class ReadBuffer { +public class ReadBuffer { private AbfsInputStream stream; + private String eTag; + private String path; // path of the file this buffer is for - private long offset; // offset within the file for the buffer - private int length; // actual length, set after the buffer is filles + + private long offset; + // offset within the file for the buffer + + private int length; + // actual length, set after the buffer is filles + private int requestedLength; // requested length of the read + private byte[] buffer; // the buffer itself - private int bufferindex = -1; // index in the buffers array in Buffer manager + + private int bufferindex = -1; + // index in the buffers array in Buffer manager + private ReadBufferStatus status; // status of the buffer - private CountDownLatch latch = null; // signaled when the buffer is done reading, so any client + + private CountDownLatch latch = null; + // signaled when the buffer is done reading, so any client + // waiting on this buffer gets unblocked private TracingContext tracingContext; @@ -48,6 +68,11 @@ class ReadBuffer { private boolean isLastByteConsumed = false; private boolean isAnyByteConsumed = false; private AtomicInteger refCount = new AtomicInteger(0); + private BufferType bufferType = BufferType.NORMAL; + // list of combined file ranges for vectored read. + private List vectoredUnits; + /* Allocator used for vectored fan-out; captured at queue time */ + private IntFunction allocator; private IOException errException = null; @@ -199,4 +224,55 @@ public void setAnyByteConsumed(boolean isAnyByteConsumed) { public boolean isFullyConsumed() { return isFirstByteConsumed() && isLastByteConsumed(); } + + void initVectoredUnits() { + if (vectoredUnits == null) { + vectoredUnits = new ArrayList<>(); + } + } + + void addVectoredUnit(CombinedFileRange u) { + vectoredUnits.add(u); + } + + List getVectoredUnits() { + return vectoredUnits; + } + + void clearVectoredUnits() { + if (vectoredUnits != null) { + vectoredUnits.clear(); + } + } + + public BufferType getBufferType() { + return bufferType; + } + + public void setBufferType(final BufferType bufferType) { + this.bufferType = bufferType; + } + + /** + * Set the allocator associated with this buffer. + * + *

The same allocator instance may be shared across multiple buffers + * belonging to a single vectored read operation. It is captured at + * queue time so it is available when the asynchronous read completes.

+ * + * @param allocator allocator used for vectored fan-out + */ + public void setAllocator(IntFunction allocator) { + this.allocator = allocator; + } + + /** + * Return the allocator associated with this buffer. + * + * @return allocator used for vectored fan-out, or {@code null} for + * non-vectored buffers + */ + public IntFunction getAllocator() { + return allocator; + } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManager.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManager.java index 712b04fb4999c..f4c1b28a57124 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManager.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManager.java @@ -19,19 +19,23 @@ package org.apache.hadoop.fs.azurebfs.services; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.Queue; import java.util.Stack; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.IntFunction; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.fs.azurebfs.contracts.services.ReadBufferStatus; +import org.apache.hadoop.fs.azurebfs.enums.VectoredReadStrategy; import org.apache.hadoop.fs.azurebfs.utils.TracingContext; +import org.apache.hadoop.fs.impl.CombinedFileRange; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_READ_AHEAD_BLOCK_SIZE; @@ -69,6 +73,18 @@ abstract void queueReadAhead(AbfsInputStream stream, int requestedLength, TracingContext tracingContext); + /** + * Queues a read-ahead request from {@link AbfsInputStream} + * for a given offset in file and given length. + * + * @param stream the input stream requesting the read-ahead + * @param unit buffer-sized vectored read unit to be queued + * @param tracingContext the tracing context for diagnostics + */ + abstract boolean queueVectoredRead(AbfsInputStream stream, + CombinedFileRange unit, + TracingContext tracingContext, IntFunction allocator); + /** * Gets a block of data from the prefetched data by ReadBufferManager. * {@link AbfsInputStream} calls this method read any bytes already available in a buffer (thereby saving a @@ -129,6 +145,14 @@ abstract void doneReading(ReadBuffer buffer, @VisibleForTesting abstract int getNumBuffers(); + abstract VectoredReadHandler getVectoredReadHandler(); + + abstract VectoredReadStrategy getVectoredReadStrategy(); + + abstract int getMaxSeekForVectoredReads(); + + abstract int getMaxSeekForVectoredReadsThroughput(); + /** * Attempts to evict buffers based on the eviction policy. */ diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV1.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV1.java index 240a618666621..764f07d6e3b86 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV1.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV1.java @@ -17,16 +17,23 @@ */ package org.apache.hadoop.fs.azurebfs.services; +import org.apache.hadoop.fs.azurebfs.AbfsConfiguration; +import org.apache.hadoop.fs.azurebfs.constants.ReadType; import org.apache.hadoop.fs.azurebfs.contracts.services.ReadBufferStatus; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; import java.util.LinkedList; import java.util.concurrent.CountDownLatch; +import java.util.function.IntFunction; +import org.apache.hadoop.fs.azurebfs.enums.BufferType; +import org.apache.hadoop.fs.azurebfs.enums.VectoredReadStrategy; import org.apache.hadoop.fs.azurebfs.utils.TracingContext; +import org.apache.hadoop.fs.impl.CombinedFileRange; import org.apache.hadoop.util.concurrent.SubjectInheritingThread; import org.apache.hadoop.classification.VisibleForTesting; @@ -43,21 +50,37 @@ public final class ReadBufferManagerV1 extends ReadBufferManager { private Thread[] threads = new Thread[NUM_THREADS]; private byte[][] buffers; private static ReadBufferManagerV1 bufferManager; + private final VectoredReadHandler vectoredReadHandler; + private static VectoredReadStrategy vectoredReadStrategy; + private static int maxSeekForVectoredReads; + private static int maxSeekForeVectoredReadsThroughput; // hide instance constructor private ReadBufferManagerV1() { + this.vectoredReadHandler = new VectoredReadHandler(this); LOGGER.trace("Creating readbuffer manager with HADOOP-18546 patch"); } + public VectoredReadHandler getVectoredReadHandler() { + return vectoredReadHandler; + } + /** * Sets the read buffer manager configurations. + * * @param readAheadBlockSize the size of the read-ahead block in bytes + * @param abfsConfiguration the configuration to set for the ReadBufferManagerV1. */ - public static void setReadBufferManagerConfigs(int readAheadBlockSize) { + public static void setReadBufferManagerConfigs(int readAheadBlockSize, + AbfsConfiguration abfsConfiguration) { if (bufferManager == null) { LOGGER.debug( "ReadBufferManagerV1 not initialized yet. Overriding readAheadBlockSize as {}", readAheadBlockSize); + vectoredReadStrategy = abfsConfiguration.getVectoredReadStrategy(); + maxSeekForVectoredReads = abfsConfiguration.getMaxSeekForVectoredReads(); + maxSeekForeVectoredReadsThroughput + = abfsConfiguration.getMaxSeekForVectoredReadsThroughput(); setReadAheadBlockSize(readAheadBlockSize); setThresholdAgeMilliseconds(DEFAULT_THRESHOLD_AGE_MILLISECONDS); } @@ -143,6 +166,102 @@ public void queueReadAhead(final AbfsInputStream stream, final long requestedOff } } + /** + * Queue a vectored read for a buffer-sized physical read unit. + * + *

The method first attempts to attach the logical unit to an already + * in-progress physical read for the same file and offset. If that is not + * possible, a free read buffer is acquired and a new backend read is + * queued.

+ * + * @param stream input stream for the file being read + * @param unit buffer-sized combined file range to be read + * @param tracingContext tracing context used for the backend read request + * @param allocator allocator used to create buffers for vectored fan-out + * @return {@code true} if the read was queued or attached to an existing + * in-progress buffer; {@code false} if no buffer was available + */ + boolean queueVectoredRead(AbfsInputStream stream, + CombinedFileRange unit, + TracingContext tracingContext, + IntFunction allocator) { + /* Create a child tracing context for vectored read-ahead requests */ + TracingContext readAheadTracingContext = + new TracingContext(tracingContext); + readAheadTracingContext.setPrimaryRequestID(); + readAheadTracingContext.setReadType(ReadType.VECTORED_READ); + + synchronized (this) { + /* + * Attempt to hitchhike on an existing in-progress physical read if it + * covers the requested logical range completely. + */ + if (isAlreadyQueued(stream, unit.getOffset())) { + ReadBuffer existing = getFromList(getInProgressList(), stream, unit.getOffset()); + if (existing != null && stream.getETag().equals(existing.getETag())) { + long end = existing.getOffset() + ( + existing.getStatus() == ReadBufferStatus.AVAILABLE + ? existing.getLength() + : existing.getRequestedLength()); + if (end >= unit.getOffset() + unit.getLength()) { + existing.initVectoredUnits(); + existing.addVectoredUnit(unit); + return true; + } + } + } + /* + * Ensure a free buffer is available, attempting best-effort recovery + * through memory upscaling or eviction if necessary. + */ + if (getFreeList().isEmpty() && !tryEvict()) { + return false; + } + /* + * Create a logical ReadBuffer descriptor without binding pooled memory. + * This captures metadata required to schedule the physical read. + */ + ReadBuffer buffer = new ReadBuffer(); + buffer.setStream(stream); + buffer.setETag(stream.getETag()); + buffer.setPath(stream.getPath()); + buffer.setOffset(unit.getOffset()); + buffer.setRequestedLength(unit.getLength()); + buffer.setBufferType(BufferType.VECTORED); + buffer.setStatus(ReadBufferStatus.NOT_AVAILABLE); + buffer.setLatch(new CountDownLatch(1)); + buffer.initVectoredUnits(); + buffer.addVectoredUnit(unit); + buffer.setAllocator(allocator); + /* + * Perform a final free-list check before consuming pooled memory to + * ensure buffer availability. + */ + if (getFreeList().isEmpty()) { + return false; + } + Integer bufferIndex = getFreeList().pop(); + if (bufferIndex >= buffers.length) { + /* Defensive guard; should never occur */ + return false; + } + /* + * Bind the physical buffer and queue the read for asynchronous + * execution. + */ + buffer.setBuffer(buffers[bufferIndex]); + buffer.setBufferindex(bufferIndex); + + getReadAheadQueue().add(buffer); + notifyAll(); + if (LOGGER.isTraceEnabled()) { + LOGGER.trace("Done q-ing readAhead for file {} offset {} buffer idx {}", + stream.getPath(), unit.getOffset(), buffer.getBufferindex()); + } + return true; + } + } + /** * {@inheritDoc} */ @@ -207,6 +326,24 @@ public void doneReading(final ReadBuffer buffer, final ReadBufferStatus result, LOGGER.trace("ReadBufferWorker completed read file {} for offset {} outcome {} bytes {}", buffer.getStream().getPath(), buffer.getOffset(), result, bytesActuallyRead); } + if (buffer.getBufferType() == BufferType.VECTORED) { + try { + if (buffer.getStatus() == ReadBufferStatus.AVAILABLE && bytesActuallyRead > 0) { + getVectoredReadHandler().fanOut(buffer, bytesActuallyRead); + } else { + throw new IOException( + "Vectored read failed for path: " + buffer.getPath() + + ", status=" + buffer.getStatus()); + } + } catch (Exception e) { + // Fail all logical FileRange futures + getVectoredReadHandler().failBufferFutures(buffer, e); + buffer.setStatus( ReadBufferStatus.READ_FAILED); + } finally { + // Must be cleared before publication / reuse + buffer.clearVectoredUnits(); + } + } synchronized (this) { // If this buffer has already been purged during // close of InputStream then we don't update the lists. @@ -610,4 +747,20 @@ void resetBufferManager() { private static void setBufferManager(ReadBufferManagerV1 manager) { bufferManager = manager; } + + + @VisibleForTesting + public VectoredReadStrategy getVectoredReadStrategy() { + return vectoredReadStrategy; + } + + @VisibleForTesting + public int getMaxSeekForVectoredReads() { + return maxSeekForVectoredReads; + } + + @VisibleForTesting + public int getMaxSeekForVectoredReadsThroughput() { + return maxSeekForeVectoredReadsThroughput; + } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV2.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV2.java index 7f276eb77d859..f6bdfcda0bceb 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV2.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV2.java @@ -19,9 +19,11 @@ import org.apache.hadoop.fs.azurebfs.AbfsConfiguration; import org.apache.hadoop.fs.azurebfs.JvmUniqueIdProvider; +import org.apache.hadoop.fs.azurebfs.constants.ReadType; import org.apache.hadoop.fs.azurebfs.contracts.services.ReadBufferStatus; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; @@ -38,10 +40,14 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.IntFunction; +import org.apache.hadoop.fs.azurebfs.enums.BufferType; +import org.apache.hadoop.fs.azurebfs.enums.VectoredReadStrategy; import org.apache.hadoop.fs.azurebfs.utils.ResourceUtilizationUtils; import org.apache.hadoop.fs.azurebfs.utils.TracingContext; import org.apache.hadoop.classification.VisibleForTesting; +import org.apache.hadoop.fs.impl.CombinedFileRange; import org.apache.hadoop.util.concurrent.SubjectInheritingThread; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.EMPTY_STRING; @@ -117,6 +123,10 @@ public final class ReadBufferManagerV2 extends ReadBufferManager { private volatile String lastScaleDirection = EMPTY_STRING; /* Maximum CPU utilization observed during the monitoring interval. */ private volatile long maxJvmCpuUtilization = 0L; + private final VectoredReadHandler vectoredReadHandler; + private static VectoredReadStrategy vectoredReadStrategy; + private static int maxSeekForVectoredReads; + private static int maxSeekForeVectoredReadsThroughput; /** * Private constructor to prevent instantiation as this needs to be singleton. @@ -126,6 +136,7 @@ public final class ReadBufferManagerV2 extends ReadBufferManager { private ReadBufferManagerV2(AbfsCounters abfsCounters) { this.abfsCounters = abfsCounters; readThreadPoolMetrics = abfsCounters.getAbfsReadResourceUtilizationMetrics(); + vectoredReadHandler = new VectoredReadHandler(this); printTraceLog("Creating Read Buffer Manager V2 with HADOOP-18546 patch"); } @@ -155,12 +166,16 @@ static ReadBufferManagerV2 getBufferManager(AbfsCounters abfsCounters) { return bufferManager; } + public VectoredReadHandler getVectoredReadHandler() { + return vectoredReadHandler; + } + /** * Set the ReadBufferManagerV2 configurations based on the provided before singleton initialization. * @param readAheadBlockSize the read-ahead block size to set for the ReadBufferManagerV2. * @param abfsConfiguration the configuration to set for the ReadBufferManagerV2. */ - public static void setReadBufferManagerConfigs(final int readAheadBlockSize, + public static void setReadBufferManagerConfigs(int readAheadBlockSize, final AbfsConfiguration abfsConfiguration) { // Set Configs only before initializations. if (bufferManager == null && !isConfigured.get()) { @@ -189,6 +204,9 @@ public static void setReadBufferManagerConfigs(final int readAheadBlockSize, abfsConfiguration.getReadAheadV2CachedBufferTTLMillis()); isDynamicScalingEnabled = abfsConfiguration.isReadAheadV2DynamicScalingEnabled(); + vectoredReadStrategy = abfsConfiguration.getVectoredReadStrategy(); + maxSeekForVectoredReads = abfsConfiguration.getMaxSeekForVectoredReads(); + maxSeekForeVectoredReadsThroughput = abfsConfiguration.getMaxSeekForVectoredReadsThroughput(); setReadAheadBlockSize(readAheadBlockSize); setIsConfigured(true); } @@ -333,6 +351,98 @@ public void queueReadAhead(final AbfsInputStream stream, } } + /** + * Queue a vectored read for a buffer-sized physical read unit. + * + *

The method first attempts to attach the logical unit to an already + * in-progress physical read for the same file and offset. If that is not + * possible, a free read buffer is acquired and a new backend read is + * queued.

+ * + * @param stream input stream for the file being read + * @param unit buffer-sized combined file range to be read + * @param tracingContext tracing context used for the backend read request + * @param allocator allocator used to create buffers for vectored fan-out + * @return {@code true} if the read was queued or attached to an existing + * in-progress buffer; {@code false} if no buffer was available + */ + boolean queueVectoredRead(AbfsInputStream stream, + CombinedFileRange unit, + TracingContext tracingContext, + IntFunction allocator) { + /* Create a child tracing context for vectored read-ahead requests */ + TracingContext readAheadTracingContext = + new TracingContext(tracingContext); + readAheadTracingContext.setPrimaryRequestID(); + readAheadTracingContext.setReadType(ReadType.VECTORED_READ); + + synchronized (this) { + /* + * Attempt to hitchhike on an existing in-progress physical read if it + * covers the requested logical range completely. + */ + if (isAlreadyQueued(stream.getETag(), unit.getOffset())) { + ReadBuffer existing = getFromList(getInProgressList(), stream.getPath(), unit.getOffset()); + if (existing != null && stream.getETag().equals(existing.getETag())) { + long end = existing.getOffset() + ( + existing.getStatus() == ReadBufferStatus.AVAILABLE + ? existing.getLength() + : existing.getRequestedLength()); + if (end >= unit.getOffset() + unit.getLength()) { + existing.initVectoredUnits(); + existing.addVectoredUnit(unit); + return true; + } + } + } + /* + * Ensure a free buffer is available, attempting best-effort recovery + * through memory upscaling or eviction if necessary. + */ + if (isFreeListEmpty() && !tryMemoryUpscale() && !tryEvict()) { + return false; + } + /* + * Create a logical ReadBuffer descriptor without binding pooled memory. + * This captures metadata required to schedule the physical read. + */ + ReadBuffer buffer = new ReadBuffer(); + buffer.setStream(stream); + buffer.setETag(stream.getETag()); + buffer.setPath(stream.getPath()); + buffer.setOffset(unit.getOffset()); + buffer.setRequestedLength(unit.getLength()); + buffer.setBufferType(BufferType.VECTORED); + buffer.setStatus(ReadBufferStatus.NOT_AVAILABLE); + buffer.setLatch(new CountDownLatch(1)); + buffer.initVectoredUnits(); + buffer.addVectoredUnit(unit); + buffer.setAllocator(allocator); + /* + * Perform a final free-list check before consuming pooled memory to + * ensure buffer availability. + */ + if (isFreeListEmpty()) { + return false; + } + Integer bufferIndex = popFromFreeList(); + if (bufferIndex >= bufferPool.length) { + /* Defensive guard; should never occur */ + return false; + } + /* + * Bind the physical buffer and queue the read for asynchronous + * execution. + */ + buffer.setBuffer(bufferPool[bufferIndex]); + buffer.setBufferindex(bufferIndex); + + getReadAheadQueue().add(buffer); + notifyAll(); + return true; + } + } + /** * {@link AbfsInputStream} calls this method read any bytes already available in a buffer (thereby saving a * remote read). This returns the bytes if the data already exists in buffer. If there is a buffer that is reading @@ -426,6 +536,24 @@ public void doneReading(final ReadBuffer buffer, "ReadBufferWorker completed prefetch for file: {} with eTag: {}, for offset: {}, queued by stream: {}, with status: {} and bytes read: {}", buffer.getPath(), buffer.getETag(), buffer.getOffset(), buffer.getStream().hashCode(), result, bytesActuallyRead); + if (buffer.getBufferType() == BufferType.VECTORED) { + try { + if (buffer.getStatus() == ReadBufferStatus.AVAILABLE && bytesActuallyRead > 0) { + getVectoredReadHandler().fanOut(buffer, bytesActuallyRead); + } else { + throw new IOException( + "Vectored read failed for path: " + buffer.getPath() + + ", status=" + buffer.getStatus()); + } + } catch (Exception e) { + // Fail all logical FileRange futures + getVectoredReadHandler().failBufferFutures(buffer, e); + buffer.setStatus( ReadBufferStatus.READ_FAILED); + } finally { + // Must be cleared before publication / reuse + buffer.clearVectoredUnits(); + } + } synchronized (this) { // If this buffer has already been purged during // close of InputStream then we don't update the lists. @@ -1087,6 +1215,21 @@ public ScheduledExecutorService getCpuMonitoringThread() { return cpuMonitorThread; } + @VisibleForTesting + public VectoredReadStrategy getVectoredReadStrategy() { + return vectoredReadStrategy; + } + + @VisibleForTesting + public int getMaxSeekForVectoredReads() { + return maxSeekForVectoredReads; + } + + @VisibleForTesting + public int getMaxSeekForVectoredReadsThroughput() { + return maxSeekForeVectoredReadsThroughput; + } + /** * Returns the maximum JVM CPU utilization observed during the current * monitoring interval or since the last reset. diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VectoredReadHandler.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VectoredReadHandler.java new file mode 100644 index 0000000000000..8ac72fd03fcd3 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VectoredReadHandler.java @@ -0,0 +1,334 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.services; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.function.IntFunction; + +import org.apache.hadoop.fs.FileRange; +import org.apache.hadoop.fs.azurebfs.enums.VectoredReadStrategy; +import org.apache.hadoop.fs.azurebfs.utils.ResourceUtilizationUtils; +import org.apache.hadoop.fs.impl.CombinedFileRange; + +public final class VectoredReadHandler { + private final ReadBufferManager readBufferManager; + private final VectoredReadStrategy strategy; + + public VectoredReadHandler(ReadBufferManager readBufferManager) { + this.readBufferManager = readBufferManager; + this.strategy = readBufferManager.getVectoredReadStrategy(); + } + + /** + * Perform a vectored read over multiple logical file ranges. + * + *

Logical ranges are first merged using a span-first strategy determined + * by the configured {@link VectoredReadStrategy}. The merged ranges are then + * split into buffer-sized physical read units and queued for asynchronous + * execution. If a pooled buffer is unavailable, the read falls back to a + * direct read path.

+ * + * @param stream input stream for the file being read + * @param ranges logical file ranges to read; each range will be completed + * with data or failure via its associated future + * @param allocator allocator used to create buffers for direct reads and + * vectored fan-out + */ + public void readVectored( + AbfsInputStream stream, + List ranges, + IntFunction allocator) { + + /* Initialize a future for each logical file range */ + for (FileRange r : ranges) { + r.setData(new CompletableFuture<>()); + } + + /* Select the maximum allowed merge span based on the configured strategy */ + int maxSpan = + (strategy == VectoredReadStrategy.TPS_OPTIMIZED) + ? readBufferManager.getMaxSeekForVectoredReads() + : readBufferManager.getMaxSeekForVectoredReadsThroughput(); + + /* Merge logical ranges using a span-first coalescing strategy */ + List merged = + mergeBySpanAndGap(ranges, maxSpan); + + /* Read buffer size acts as a hard upper bound for physical reads */ + int readBufferSize = ReadBufferManager.getReadAheadBlockSize(); + + /* Split merged ranges into buffer-sized chunks and queue each for read */ + for (CombinedFileRange unit : merged) { + List chunks = + splitByBufferSize(unit, readBufferSize); + + for (CombinedFileRange chunk : chunks) { + try { + boolean queued = + readBufferManager.queueVectoredRead( + stream, chunk, stream.getTracingContext(), allocator); + + if (!queued) { + /* Fall back to direct read if no buffer is available */ + directRead(stream, chunk, allocator); + } + } catch (Exception e) { + /* Propagate failure to all logical ranges in this unit */ + failUnit(chunk, e); + } + } + } + } + + + /** + * Split a merged logical range into buffer-sized physical read units. + * + *

The input {@link CombinedFileRange} may span more bytes than the + * configured read buffer size. This method divides it into multiple + * {@link CombinedFileRange} instances, each limited to {@code bufferSize} + * and containing only the logical {@link FileRange}s that intersect its span.

+ * + * @param unit merged logical range to be split + * @param bufferSize maximum size (in bytes) of each physical read unit + * @return a list of buffer-sized {@link CombinedFileRange} instances + */ + private List splitByBufferSize( + CombinedFileRange unit, + int bufferSize) { + + List parts = new ArrayList<>(); + + long unitStart = unit.getOffset(); + long unitEnd = unitStart + unit.getLength(); + long start = unitStart; + + /* Create buffer-sized slices covering the merged unit span */ + while (start < unitEnd) { + long partEnd = Math.min(start + bufferSize, unitEnd); + + /* Initialize a physical read unit for the span [start, partEnd) */ + CombinedFileRange part = + new CombinedFileRange(start, partEnd, + unit.getUnderlying().get(0)); + + /* Remove the constructor-added range and attach only overlapping ranges */ + part.getUnderlying().clear(); + + /* Attach logical ranges that intersect this physical read unit */ + for (FileRange r : unit.getUnderlying()) { + long rStart = r.getOffset(); + long rEnd = rStart + r.getLength(); + + if (rEnd > start && rStart < partEnd) { + part.getUnderlying().add(r); + } + } + + parts.add(part); + start = partEnd; + } + + return parts; + } + + /** + * Merge logical {@link FileRange}s into {@link CombinedFileRange}s using a + * span-first coalescing strategy. + * + *

Ranges are merged as long as the total span from the first offset to the + * end of the last range does not exceed {@code maxSpan}. Gaps between ranges + * are ignored.

+ * + * @param ranges logical file ranges to merge + * @param maxSpan maximum allowed span (in bytes) for a combined read + * @return merged {@link CombinedFileRange}s covering the input ranges + */ + private List mergeBySpanAndGap( + List ranges, + int maxSpan) { + + /* Sort ranges by starting offset for span-based merging */ + ranges.sort(Comparator.comparingLong(FileRange::getOffset)); + + List out = new ArrayList<>(); + CombinedFileRange current = null; + + for (FileRange r : ranges) { + long rOffset = r.getOffset(); + long rEnd = rOffset + r.getLength(); + + /* Initialize the first combined range */ + if (current == null) { + current = new CombinedFileRange(rOffset, rEnd, r); + continue; + } + + /* Check whether adding this range keeps the total span within the limit */ + long newSpan = rEnd - current.getOffset(); + + if (newSpan <= maxSpan) { + current.setLength((int) newSpan); + current.getUnderlying().add(r); + } else { + /* Span exceeded; finalize current range and start a new one */ + out.add(current); + current = new CombinedFileRange(rOffset, rEnd, r); + } + } + + /* Add the final combined range, if any */ + if (current != null) { + out.add(current); + } + + return out; + } + + + /** + * Fan out data from a completed physical read buffer to all logical + * {@link FileRange}s associated with the vectored read. + * + *

For each logical range, the corresponding slice of data is copied + * into a newly allocated {@link ByteBuffer} and the range's future is + * completed. Ranges whose futures are cancelled are skipped.

+ * + * @param buffer completed read buffer containing the physical data + * @param bytesRead number of bytes actually read into the buffer + */ + void fanOut(ReadBuffer buffer, int bytesRead) { + + List units = buffer.getVectoredUnits(); + if (units == null) { + return; + } + + /* Distribute buffer data to all logical ranges attached to this buffer */ + for (CombinedFileRange unit : units) { + for (FileRange r : unit.getUnderlying()) { + + /* Skip ranges whose futures have been cancelled */ + if (r.getData().isCancelled()) { + continue; + } + + try { + /* Compute offset of the logical range relative to the buffer */ + long rel = r.getOffset() - buffer.getOffset(); + + /* Determine how many bytes are available for this range */ + int available = + (int) Math.max( + 0, + Math.min(r.getLength(), bytesRead - rel)); + + /* Allocate output buffer and copy available data */ + ByteBuffer bb = buffer.getAllocator().apply(r.getLength()); + if (available > 0) { + bb.put(buffer.getBuffer(), (int) rel, available); + } + bb.flip(); + r.getData().complete(bb); + } catch (Exception e) { + /* Propagate failure to the affected logical range */ + r.getData().completeExceptionally(e); + } + } + } + } + + /** + * Fail all logical {@link FileRange}s associated with a single combined + * vectored read unit. + * + * @param unit combined file range whose logical ranges should be failed + * @param t failure cause to propagate to waiting futures + */ + private void failUnit(CombinedFileRange unit, Throwable t) { + for (FileRange r : unit.getUnderlying()) { + r.getData().completeExceptionally(t); + } + } + + + /** + * Completes all logical {@link FileRange} futures associated with a vectored + * {@link ReadBuffer} exceptionally when the backend read fails. + * + * @param buffer the vectored read buffer + * @param t the failure cause to propagate to waiting futures + */ + void failBufferFutures(ReadBuffer buffer, Throwable t) { + List units = buffer.getVectoredUnits(); + if (units == null) { + return; + } + + /* Propagate failure to all logical ranges attached to this buffer */ + for (CombinedFileRange unit : units) { + for (FileRange r : unit.getUnderlying()) { + CompletableFuture future = r.getData(); + if (future != null && !future.isDone()) { + future.completeExceptionally(t); + } + } + } + } + + /** + * Perform a synchronous direct read for a vectored unit when no pooled + * read buffer is available. + * + *

This method reads the required byte range directly from the backend + * and completes all associated logical {@link FileRange} futures. It is + * used as a fallback path when vectored buffering cannot be used.

+ * + * @param stream input stream for the file being read + * @param unit combined file range to read directly + * @param allocator allocator used to create output buffers for logical ranges + * @throws IOException if memory pressure is high or the backend read fails + */ + private void directRead( + AbfsInputStream stream, + CombinedFileRange unit, + IntFunction allocator) throws IOException { + /* Read the entire combined range into a temporary buffer */ + byte[] tmp = new byte[unit.getLength()]; + stream.readRemote(unit.getOffset(), tmp, 0, unit.getLength(), + stream.getTracingContext()); + + /* Fan out data to individual logical ranges */ + for (FileRange r : unit.getUnderlying()) { + ByteBuffer bb = allocator.apply(r.getLength()); + bb.put(tmp, + (int) (r.getOffset() - unit.getOffset()), + r.getLength()); + bb.flip(); + r.getData().complete(bb); + } + } + +} + diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestReadBufferManager.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestReadBufferManager.java index e663db9bbdd39..57a82ee7c4060 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestReadBufferManager.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestReadBufferManager.java @@ -174,7 +174,7 @@ private ReadBufferManager getBufferManager(AzureBlobFileSystem fs) { getConfiguration()); return ReadBufferManagerV2.getBufferManager(fs.getAbfsStore().getClient().getAbfsCounters()); } - ReadBufferManagerV1.setReadBufferManagerConfigs(blockSize); + ReadBufferManagerV1.setReadBufferManagerConfigs(blockSize, fs.getAbfsStore().getClient().getAbfsConfiguration()); return ReadBufferManagerV1.getBufferManager(); } } From 95cbb73f8338b1d8062f685963c7e3387db1caa0 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Tue, 20 Jan 2026 04:21:20 -0800 Subject: [PATCH 03/17] Fix tests --- .../services/ReadBufferManagerV1.java | 3 +- .../services/ReadBufferManagerV2.java | 3 +- .../azurebfs/services/ITestVectoredRead.java | 176 ++++++++++++++++++ 3 files changed, 180 insertions(+), 2 deletions(-) create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV1.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV1.java index 764f07d6e3b86..07d63f73749a5 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV1.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV1.java @@ -233,6 +233,7 @@ boolean queueVectoredRead(AbfsInputStream stream, buffer.initVectoredUnits(); buffer.addVectoredUnit(unit); buffer.setAllocator(allocator); + buffer.setTracingContext(tracingContext); /* * Perform a final free-list check before consuming pooled memory to * ensure buffer availability. @@ -328,7 +329,7 @@ public void doneReading(final ReadBuffer buffer, final ReadBufferStatus result, } if (buffer.getBufferType() == BufferType.VECTORED) { try { - if (buffer.getStatus() == ReadBufferStatus.AVAILABLE && bytesActuallyRead > 0) { + if (result == ReadBufferStatus.AVAILABLE && bytesActuallyRead > 0) { getVectoredReadHandler().fanOut(buffer, bytesActuallyRead); } else { throw new IOException( diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV2.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV2.java index f6bdfcda0bceb..f89b4efd4be73 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV2.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV2.java @@ -418,6 +418,7 @@ boolean queueVectoredRead(AbfsInputStream stream, buffer.initVectoredUnits(); buffer.addVectoredUnit(unit); buffer.setAllocator(allocator); + buffer.setTracingContext(tracingContext); /* * Perform a final free-list check before consuming pooled memory to * ensure buffer availability. @@ -538,7 +539,7 @@ public void doneReading(final ReadBuffer buffer, buffer.getStream().hashCode(), result, bytesActuallyRead); if (buffer.getBufferType() == BufferType.VECTORED) { try { - if (buffer.getStatus() == ReadBufferStatus.AVAILABLE && bytesActuallyRead > 0) { + if (result == ReadBufferStatus.AVAILABLE && bytesActuallyRead > 0) { getVectoredReadHandler().fanOut(buffer, bytesActuallyRead); } else { throw new IOException( diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java new file mode 100644 index 0000000000000..bf06ffa2df5e7 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java @@ -0,0 +1,176 @@ +package org.apache.hadoop.fs.azurebfs.services; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.function.IntFunction; + +import org.junit.jupiter.api.Test; + +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileRange; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest; +import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem; + +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.ONE_MB; +import static org.apache.hadoop.fs.contract.ContractTestUtils.validateVectoredReadResult; + +public class ITestVectoredRead extends AbstractAbfsIntegrationTest { + + public ITestVectoredRead() throws Exception { + } + + @Test + public void testDisjointRangesWithVectoredRead() throws Throwable { + int fileSize = ONE_MB; + final AzureBlobFileSystem fs = getFileSystem(); + String fileName = methodName.getMethodName() + 1; + byte[] fileContent = getRandomBytesArray(fileSize); + Path testFilePath = createFileWithContent(fs, fileName, fileContent); + + List rangeList = new ArrayList<>(); + rangeList.add(FileRange.createFileRange(100, 10000)); + rangeList.add(FileRange.createFileRange(15000, 27000)); + IntFunction allocate = ByteBuffer::allocate; + CompletableFuture builder = fs.openFile(testFilePath).build(); + + try (FSDataInputStream in = builder.get()) { + in.readVectored(rangeList, allocate); + byte[] readFullRes = new byte[(int)fileSize]; + in.readFully(0, readFullRes); + // Comparing vectored read results with read fully. + validateVectoredReadResult(rangeList, readFullRes, 0); + } + } + + @Test + public void testVectoredReadDisjointRanges() throws Exception { + final AzureBlobFileSystem fs = getFileSystem(); + String fileName = methodName.getMethodName() + 1; + byte[] fileContent = getRandomBytesArray(ONE_MB); + Path testFilePath = createFileWithContent(fs, fileName, fileContent); + List fileRanges = new ArrayList<>(); + fileRanges.add(FileRange.createFileRange(100, 10000)); + fileRanges.add(FileRange.createFileRange(15000, 12000)); + IntFunction allocate = ByteBuffer::allocate; + try (FSDataInputStream in = + fs.openFile(testFilePath).build().get()) { + + in.readVectored(fileRanges, allocate); + CompletableFuture[] futures = + new CompletableFuture[fileRanges.size()]; + int i = 0; + for (FileRange range : fileRanges) { + futures[i++] = range.getData(); + } + CompletableFuture.allOf(futures).get(); + validateVectoredReadResult(fileRanges, fileContent, 0); + } + } + + @Test + public void testMultipleDisjointRangesWithVectoredRead() throws Throwable { + int fileSize = ONE_MB; + final AzureBlobFileSystem fs = getFileSystem(); + String fileName = methodName.getMethodName() + 1; + byte[] fileContent = getRandomBytesArray(fileSize); + Path testFilePath = createFileWithContent(fs, fileName, fileContent); + + List rangeList = new ArrayList<>(); + rangeList.add(FileRange.createFileRange(100, 10000)); + rangeList.add(FileRange.createFileRange(15000, 27000)); + rangeList.add(FileRange.createFileRange(42500, 40000)); + IntFunction allocate = ByteBuffer::allocate; + CompletableFuture builder = fs.openFile(testFilePath).build(); + + try (FSDataInputStream in = builder.get()) { + in.readVectored(rangeList, allocate); + byte[] readFullRes = new byte[(int)fileSize]; + in.readFully(0, readFullRes); + // Comparing vectored read results with read fully. + validateVectoredReadResult(rangeList, readFullRes, 0); + } + } + + @Test + public void testMultipleRangesWithVectoredRead() throws Throwable { + int fileSize = ONE_MB; + final AzureBlobFileSystem fs = getFileSystem(); + String fileName = methodName.getMethodName() + 1; + byte[] fileContent = getRandomBytesArray(fileSize); + Path testFilePath = createFileWithContent(fs, fileName, fileContent); + + List rangeList = new ArrayList<>(); + rangeList.add(FileRange.createFileRange(100, 10000)); + rangeList.add(FileRange.createFileRange(15000, 27000)); + rangeList.add(FileRange.createFileRange(47500, 27000)); + + IntFunction allocate = ByteBuffer::allocate; + CompletableFuture builder = fs.openFile(testFilePath).build(); + + try (FSDataInputStream in = builder.get()) { + in.readVectored(rangeList, allocate); + byte[] readFullRes = new byte[(int)fileSize]; + in.readFully(0, readFullRes); + // Comparing vectored read results with read fully. + validateVectoredReadResult(rangeList, readFullRes, 0); + } + } + + @Test + public void testMergedRangesWithVectoredRead() throws Throwable { + int fileSize = ONE_MB; + final AzureBlobFileSystem fs = getFileSystem(); + String fileName = methodName.getMethodName() + 1; + byte[] fileContent = getRandomBytesArray(fileSize); + Path testFilePath = createFileWithContent(fs, fileName, fileContent); + + List rangeList = new ArrayList<>(); + rangeList.add(FileRange.createFileRange(100, 10000)); + rangeList.add(FileRange.createFileRange(12000, 27000)); + IntFunction allocate = ByteBuffer::allocate; + CompletableFuture builder = fs.openFile(testFilePath).build(); + + try (FSDataInputStream in = builder.get()) { + in.readVectored(rangeList, allocate); + byte[] readFullRes = new byte[(int)fileSize]; + in.readFully(0, readFullRes); + // Comparing vectored read results with read fully. + validateVectoredReadResult(rangeList, readFullRes, 0); + } + } + + @Test + public void test_045_vectoredIOHugeFile() throws Throwable { + int fileSize = 100 * ONE_MB; + final AzureBlobFileSystem fs = getFileSystem(); + String fileName = methodName.getMethodName() + 1; + byte[] fileContent = getRandomBytesArray(fileSize); + Path testFilePath = createFileWithContent(fs, fileName, fileContent); + + List rangeList = new ArrayList<>(); + rangeList.add(FileRange.createFileRange(5856368, 116770)); + rangeList.add(FileRange.createFileRange(3520861, 116770)); + rangeList.add(FileRange.createFileRange(8191913, 116770)); + rangeList.add(FileRange.createFileRange(1520861, 116770)); + rangeList.add(FileRange.createFileRange(2520861, 116770)); + rangeList.add(FileRange.createFileRange(9191913, 116770)); + rangeList.add(FileRange.createFileRange(2820861, 156770)); + IntFunction allocate = ByteBuffer::allocate; + + CompletableFuture builder = + fs.openFile(testFilePath).build(); + try (FSDataInputStream in = builder.get()) { + long timeMilli1 = System.currentTimeMillis(); + in.readVectored(rangeList, allocate); + byte[] readFullRes = new byte[(int)fileSize]; + in.readFully(0, readFullRes); + // Comparing vectored read results with read fully. + validateVectoredReadResult(rangeList, readFullRes, 0); + long timeMilli2 = System.currentTimeMillis(); + System.out.println("Time taken for the code to execute: " + (timeMilli2 - timeMilli1) + " milliseconds"); + } + } +} From d03d3cd38fcf9c26caaf93f40279d5812edb482e Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Thu, 22 Jan 2026 22:52:55 -0800 Subject: [PATCH 04/17] Made changes for inprogress list --- .../fs/azurebfs/services/AbfsInputStream.java | 15 +- .../fs/azurebfs/services/ReadBuffer.java | 44 +- .../azurebfs/services/ReadBufferManager.java | 86 ++++ .../services/ReadBufferManagerV1.java | 40 +- .../services/ReadBufferManagerV2.java | 34 +- .../services/VectoredReadHandler.java | 34 +- .../azurebfs/services/ITestVectoredRead.java | 380 +++++++++++++++--- 7 files changed, 494 insertions(+), 139 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java index 2640e13b2e515..30e949e03f924 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java @@ -61,8 +61,6 @@ import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.ONE_KB; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.STREAM_ID_LEN; import static org.apache.hadoop.fs.azurebfs.constants.InternalConstants.CAPABILITY_SAFE_READAHEAD; -import static org.apache.hadoop.io.Sizes.S_128K; -import static org.apache.hadoop.io.Sizes.S_2M; import static org.apache.hadoop.util.StringUtils.toLowerCase; /** @@ -130,6 +128,8 @@ public class AbfsInputStream extends FSInputStream implements CanUnbuffer, private final AbfsInputStreamContext context; private IOStatistics ioStatistics; private String filePathIdentifier; + private VectoredReadHandler vectoredReadHandler; + /** * This is the actual position within the object, used by * lazy seek to decide whether to seek on the next read or not. @@ -211,6 +211,14 @@ private String createInputStreamId() { return StringUtils.right(UUID.randomUUID().toString(), STREAM_ID_LEN); } + /** + * Retrieves the handler responsible for processing vectored read requests. + * @return the {@link VectoredReadHandler} instance associated with the buffer manager. + */ + VectoredReadHandler getVectoredReadHandler() { + return getReadBufferManager().getVectoredReadHandler(); + } + @Override public int read(long position, byte[] buffer, int offset, int length) throws IOException { @@ -332,8 +340,7 @@ public synchronized int read(final byte[] b, final int off, final int len) throw @Override public void readVectored(List ranges, IntFunction allocate) { - readBufferManager.getVectoredReadHandler() - .readVectored(this, ranges, allocate); + getVectoredReadHandler().readVectored(this, ranges, allocate); } private boolean shouldReadFully() { diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBuffer.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBuffer.java index 8fd1e26afa251..fc938ae2da1f0 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBuffer.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBuffer.java @@ -36,29 +36,15 @@ public class ReadBuffer { private AbfsInputStream stream; - private String eTag; - private String path; // path of the file this buffer is for - - private long offset; - // offset within the file for the buffer - - private int length; - // actual length, set after the buffer is filles - + private long offset; // offset within the file for the buffer + private int length; // actual length, set after the buffer is filles private int requestedLength; // requested length of the read - private byte[] buffer; // the buffer itself - - private int bufferindex = -1; - // index in the buffers array in Buffer manager - + private int bufferindex = -1; // index in the buffers array in Buffer manager private ReadBufferStatus status; // status of the buffer - - private CountDownLatch latch = null; - // signaled when the buffer is done reading, so any client - + private CountDownLatch latch = null; // signaled when the buffer is done reading, so any client // waiting on this buffer gets unblocked private TracingContext tracingContext; @@ -71,8 +57,10 @@ public class ReadBuffer { private BufferType bufferType = BufferType.NORMAL; // list of combined file ranges for vectored read. private List vectoredUnits; - /* Allocator used for vectored fan-out; captured at queue time */ + // Allocator used for vectored fan-out; captured at queue time */ private IntFunction allocator; + // Tracks whether fanOut has already been executed + private final AtomicInteger fanOutDone = new AtomicInteger(0); private IOException errException = null; @@ -275,4 +263,22 @@ public void setAllocator(IntFunction allocator) { public IntFunction getAllocator() { return allocator; } + + /** + * Attempt to execute vectored fan-out exactly once for this buffer. + * + * @return {@code true} if the caller should perform fan-out; {@code false} + * if fan-out has already been executed + */ + boolean tryFanOut() { + return fanOutDone.compareAndSet(0, 1); + } + + /** + * @return {@code true} if vectored fan-out has already been executed + * for this buffer; {@code false} otherwise + */ + boolean isFanOutDone() { + return fanOutDone.get() == 1; + } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManager.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManager.java index f4c1b28a57124..3056ba90cbef4 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManager.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManager.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Collection; import java.util.LinkedList; import java.util.List; import java.util.Queue; @@ -321,4 +322,89 @@ protected void testMimicFullUseAndAddFailedBuffer(ReadBuffer buf) { freeList.clear(); completedReadList.add(buf); } + + /** + * Finds an existing {@link ReadBuffer} for the given stream whose buffered + * range covers the specified logical offset. + * + *

The search is performed in the read-ahead queue, in-progress list, + * and completed-read list, in that order. + * + * @param stream the {@link AbfsInputStream} associated with the read request + * + * @return a matching {@link ReadBuffer} if one exists, or {@code null} otherwise + */ + ReadBuffer findQueuedBuffer(final AbfsInputStream stream, + long requestedOffset) { + ReadBuffer buffer; + buffer = findInList(getReadAheadQueue(), stream, requestedOffset); + if (buffer != null) { + return buffer; + } + buffer = findInList(getInProgressList(), stream, requestedOffset); + if (buffer != null) { + return buffer; + } + return findInList(getCompletedReadList(), stream, requestedOffset); + } + + /** + * Searches the given collection of {@link ReadBuffer}s for one that belongs + * to the specified stream and whose buffered range covers the given offset. + * + * @param buffers the collection of {@link ReadBuffer}s to search + * @param stream the {@link AbfsInputStream} associated with the read request + * + * @return the matching {@link ReadBuffer}, or {@code null} if none is found + */ + ReadBuffer findInList(final Collection buffers, + final AbfsInputStream stream, long requestedOffset) { + for (ReadBuffer buffer : buffers) { + if (buffer.getStream() == stream + && requestedOffset >= buffer.getOffset() + && requestedOffset < buffer.getOffset() + + buffer.getRequestedLength()) { + return buffer; + } + } + return null; + } + + /** + * Handle vectored-read completion for a buffer. + * + *

If the buffer participates in a vectored read, this method performs + * vectored fan-out exactly once when the physical read completes successfully, + * or fails all associated logical ranges on error. Vectored units are cleared + * after fan-out is finalized to allow safe publication or reuse of the buffer.

+ * + * @param buffer the read buffer whose physical read has completed + * @param result the completion status of the physical read + * @param bytesActuallyRead number of bytes read from the backend + */ + void handleVectoredCompletion( + ReadBuffer buffer, + ReadBufferStatus result, + int bytesActuallyRead) { + try { + if (result == ReadBufferStatus.AVAILABLE && bytesActuallyRead > 0) { + if (buffer.tryFanOut()) { + getVectoredReadHandler().fanOut(buffer, bytesActuallyRead); + } + } else { + throw new IOException( + "Vectored read failed for path: " + buffer.getPath() + + ", status=" + buffer.getStatus()); + } + } catch (Exception e) { + // Fail all logical FileRange futures + getVectoredReadHandler().failBufferFutures(buffer, e); + buffer.setStatus(ReadBufferStatus.READ_FAILED); + } finally { + if (buffer.isFanOutDone()) { + // Must be cleared before publication / reuse + buffer.clearVectoredUnits(); + } + } + } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV1.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV1.java index 07d63f73749a5..b5b8944b7a419 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV1.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV1.java @@ -17,25 +17,25 @@ */ package org.apache.hadoop.fs.azurebfs.services; -import org.apache.hadoop.fs.azurebfs.AbfsConfiguration; -import org.apache.hadoop.fs.azurebfs.constants.ReadType; -import org.apache.hadoop.fs.azurebfs.contracts.services.ReadBufferStatus; - import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; import java.util.LinkedList; +import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.function.IntFunction; +import org.apache.hadoop.classification.VisibleForTesting; +import org.apache.hadoop.fs.azurebfs.AbfsConfiguration; +import org.apache.hadoop.fs.azurebfs.constants.ReadType; +import org.apache.hadoop.fs.azurebfs.contracts.services.ReadBufferStatus; import org.apache.hadoop.fs.azurebfs.enums.BufferType; import org.apache.hadoop.fs.azurebfs.enums.VectoredReadStrategy; import org.apache.hadoop.fs.azurebfs.utils.TracingContext; import org.apache.hadoop.fs.impl.CombinedFileRange; import org.apache.hadoop.util.concurrent.SubjectInheritingThread; -import org.apache.hadoop.classification.VisibleForTesting; /** * The Read Buffer Manager for Rest AbfsClient. @@ -197,8 +197,9 @@ boolean queueVectoredRead(AbfsInputStream stream, * covers the requested logical range completely. */ if (isAlreadyQueued(stream, unit.getOffset())) { - ReadBuffer existing = getFromList(getInProgressList(), stream, unit.getOffset()); - if (existing != null && stream.getETag().equals(existing.getETag())) { + ReadBuffer existing = findQueuedBuffer(stream, unit.getOffset()); + if (existing != null && stream.getETag() + .equals(existing.getStream().getETag())) { long end = existing.getOffset() + ( existing.getStatus() == ReadBufferStatus.AVAILABLE ? existing.getLength() @@ -206,6 +207,11 @@ boolean queueVectoredRead(AbfsInputStream stream, if (end >= unit.getOffset() + unit.getLength()) { existing.initVectoredUnits(); existing.addVectoredUnit(unit); + existing.setAllocator(allocator); + if (existing.getStatus() == ReadBufferStatus.AVAILABLE) { + handleVectoredCompletion(existing, existing.getStatus(), + existing.getLength()); + } return true; } } @@ -327,23 +333,9 @@ public void doneReading(final ReadBuffer buffer, final ReadBufferStatus result, LOGGER.trace("ReadBufferWorker completed read file {} for offset {} outcome {} bytes {}", buffer.getStream().getPath(), buffer.getOffset(), result, bytesActuallyRead); } - if (buffer.getBufferType() == BufferType.VECTORED) { - try { - if (result == ReadBufferStatus.AVAILABLE && bytesActuallyRead > 0) { - getVectoredReadHandler().fanOut(buffer, bytesActuallyRead); - } else { - throw new IOException( - "Vectored read failed for path: " + buffer.getPath() - + ", status=" + buffer.getStatus()); - } - } catch (Exception e) { - // Fail all logical FileRange futures - getVectoredReadHandler().failBufferFutures(buffer, e); - buffer.setStatus( ReadBufferStatus.READ_FAILED); - } finally { - // Must be cleared before publication / reuse - buffer.clearVectoredUnits(); - } + List vectoredUnits = buffer.getVectoredUnits(); + if (buffer.getBufferType() == BufferType.VECTORED || (vectoredUnits != null && !vectoredUnits.isEmpty())) { + handleVectoredCompletion(buffer, result, bytesActuallyRead); } synchronized (this) { // If this buffer has already been purged during diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV2.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV2.java index f89b4efd4be73..c2c9b3bc49022 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV2.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV2.java @@ -166,7 +166,7 @@ static ReadBufferManagerV2 getBufferManager(AbfsCounters abfsCounters) { return bufferManager; } - public VectoredReadHandler getVectoredReadHandler() { +VectoredReadHandler getVectoredReadHandler() { return vectoredReadHandler; } @@ -375,15 +375,15 @@ boolean queueVectoredRead(AbfsInputStream stream, new TracingContext(tracingContext); readAheadTracingContext.setPrimaryRequestID(); readAheadTracingContext.setReadType(ReadType.VECTORED_READ); - synchronized (this) { /* * Attempt to hitchhike on an existing in-progress physical read if it * covers the requested logical range completely. */ if (isAlreadyQueued(stream.getETag(), unit.getOffset())) { - ReadBuffer existing = getFromList(getInProgressList(), stream.getPath(), unit.getOffset()); - if (existing != null && stream.getETag().equals(existing.getETag())) { + ReadBuffer existing = findQueuedBuffer(stream, unit.getOffset()); + if (existing != null && stream.getETag() + .equals(existing.getStream().getETag())) { long end = existing.getOffset() + ( existing.getStatus() == ReadBufferStatus.AVAILABLE ? existing.getLength() @@ -391,6 +391,11 @@ boolean queueVectoredRead(AbfsInputStream stream, if (end >= unit.getOffset() + unit.getLength()) { existing.initVectoredUnits(); existing.addVectoredUnit(unit); + existing.setAllocator(allocator); + if (existing.getStatus() == ReadBufferStatus.AVAILABLE) { + handleVectoredCompletion(existing, existing.getStatus(), + existing.getLength()); + } return true; } } @@ -537,23 +542,10 @@ public void doneReading(final ReadBuffer buffer, "ReadBufferWorker completed prefetch for file: {} with eTag: {}, for offset: {}, queued by stream: {}, with status: {} and bytes read: {}", buffer.getPath(), buffer.getETag(), buffer.getOffset(), buffer.getStream().hashCode(), result, bytesActuallyRead); - if (buffer.getBufferType() == BufferType.VECTORED) { - try { - if (result == ReadBufferStatus.AVAILABLE && bytesActuallyRead > 0) { - getVectoredReadHandler().fanOut(buffer, bytesActuallyRead); - } else { - throw new IOException( - "Vectored read failed for path: " + buffer.getPath() - + ", status=" + buffer.getStatus()); - } - } catch (Exception e) { - // Fail all logical FileRange futures - getVectoredReadHandler().failBufferFutures(buffer, e); - buffer.setStatus( ReadBufferStatus.READ_FAILED); - } finally { - // Must be cleared before publication / reuse - buffer.clearVectoredUnits(); - } + List vectoredUnits = buffer.getVectoredUnits(); + if (buffer.getBufferType() == BufferType.VECTORED || (vectoredUnits != null + && !vectoredUnits.isEmpty())) { + handleVectoredCompletion(buffer, result, bytesActuallyRead); } synchronized (this) { // If this buffer has already been purged during diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VectoredReadHandler.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VectoredReadHandler.java index 8ac72fd03fcd3..7ccb73745ab82 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VectoredReadHandler.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VectoredReadHandler.java @@ -26,12 +26,12 @@ import java.util.concurrent.CompletableFuture; import java.util.function.IntFunction; +import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.fs.FileRange; import org.apache.hadoop.fs.azurebfs.enums.VectoredReadStrategy; -import org.apache.hadoop.fs.azurebfs.utils.ResourceUtilizationUtils; import org.apache.hadoop.fs.impl.CombinedFileRange; -public final class VectoredReadHandler { +class VectoredReadHandler { private final ReadBufferManager readBufferManager; private final VectoredReadStrategy strategy; @@ -85,10 +85,7 @@ public void readVectored( for (CombinedFileRange chunk : chunks) { try { - boolean queued = - readBufferManager.queueVectoredRead( - stream, chunk, stream.getTracingContext(), allocator); - + boolean queued = queueVectoredRead(stream, chunk, allocator); if (!queued) { /* Fall back to direct read if no buffer is available */ directRead(stream, chunk, allocator); @@ -101,6 +98,22 @@ public void readVectored( } } + /** + * Queues a vectored read request with the buffer manager. + * @return true if successfully queued, false if the queue is full and fallback is required. + */ + @VisibleForTesting + boolean queueVectoredRead(AbfsInputStream stream, CombinedFileRange unit, IntFunction allocator) { + return getReadBufferManager().queueVectoredRead(stream, unit, stream.getTracingContext(), allocator); + } + + /** + * Accesses the shared manager responsible for coordinating asynchronous read buffers. + * @return the {@link ReadBufferManager} instance. + */ + public ReadBufferManager getReadBufferManager() { + return readBufferManager; + } /** * Split a merged logical range into buffer-sized physical read units. @@ -219,31 +232,25 @@ private List mergeBySpanAndGap( * @param bytesRead number of bytes actually read into the buffer */ void fanOut(ReadBuffer buffer, int bytesRead) { - List units = buffer.getVectoredUnits(); if (units == null) { return; } - /* Distribute buffer data to all logical ranges attached to this buffer */ for (CombinedFileRange unit : units) { for (FileRange r : unit.getUnderlying()) { - /* Skip ranges whose futures have been cancelled */ if (r.getData().isCancelled()) { continue; } - try { /* Compute offset of the logical range relative to the buffer */ long rel = r.getOffset() - buffer.getOffset(); - /* Determine how many bytes are available for this range */ int available = (int) Math.max( 0, Math.min(r.getLength(), bytesRead - rel)); - /* Allocate output buffer and copy available data */ ByteBuffer bb = buffer.getAllocator().apply(r.getLength()); if (available > 0) { @@ -310,7 +317,7 @@ void failBufferFutures(ReadBuffer buffer, Throwable t) { * @param allocator allocator used to create output buffers for logical ranges * @throws IOException if memory pressure is high or the backend read fails */ - private void directRead( + void directRead( AbfsInputStream stream, CombinedFileRange unit, IntFunction allocator) throws IOException { @@ -329,6 +336,5 @@ private void directRead( r.getData().complete(bb); } } - } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java index bf06ffa2df5e7..cfe62c3cbdc49 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java @@ -2,17 +2,26 @@ import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import java.util.function.IntFunction; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentMatchers; +import org.mockito.Mockito; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileRange; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest; import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem; +import org.apache.hadoop.fs.impl.CombinedFileRange; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.ONE_MB; import static org.apache.hadoop.fs.contract.ContractTestUtils.validateVectoredReadResult; @@ -34,11 +43,12 @@ public void testDisjointRangesWithVectoredRead() throws Throwable { rangeList.add(FileRange.createFileRange(100, 10000)); rangeList.add(FileRange.createFileRange(15000, 27000)); IntFunction allocate = ByteBuffer::allocate; - CompletableFuture builder = fs.openFile(testFilePath).build(); + CompletableFuture builder = fs.openFile(testFilePath) + .build(); try (FSDataInputStream in = builder.get()) { in.readVectored(rangeList, allocate); - byte[] readFullRes = new byte[(int)fileSize]; + byte[] readFullRes = new byte[(int) fileSize]; in.readFully(0, readFullRes); // Comparing vectored read results with read fully. validateVectoredReadResult(rangeList, readFullRes, 0); @@ -46,19 +56,33 @@ public void testDisjointRangesWithVectoredRead() throws Throwable { } @Test - public void testVectoredReadDisjointRanges() throws Exception { + public void testVectoredReadDisjointRangesExpectTwoBackendReads() + throws Exception { final AzureBlobFileSystem fs = getFileSystem(); - String fileName = methodName.getMethodName() + 1; - byte[] fileContent = getRandomBytesArray(ONE_MB); + String fileName = methodName.getMethodName(); + byte[] fileContent = getRandomBytesArray(16 * ONE_MB); Path testFilePath = createFileWithContent(fs, fileName, fileContent); List fileRanges = new ArrayList<>(); - fileRanges.add(FileRange.createFileRange(100, 10000)); - fileRanges.add(FileRange.createFileRange(15000, 12000)); + // 0.0 – 1.0 MB + fileRanges.add(FileRange.createFileRange(0L, (int) ONE_MB)); + // 1.2 – 2.0 MB + fileRanges.add( + FileRange.createFileRange((long) (1.2 * ONE_MB), (int) (0.8 * ONE_MB))); + // 3.1 – 4.0 MB + fileRanges.add( + FileRange.createFileRange((long) (3.1 * ONE_MB), (int) (0.9 * ONE_MB))); + // 4.1 – 6.0 MB + fileRanges.add( + FileRange.createFileRange((long) (4.1 * ONE_MB), (int) (1.9 * ONE_MB))); + // 6.2 – 8.0 MB + fileRanges.add( + FileRange.createFileRange((long) (6.2 * ONE_MB), (int) (1.8 * ONE_MB))); IntFunction allocate = ByteBuffer::allocate; try (FSDataInputStream in = fs.openFile(testFilePath).build().get()) { - - in.readVectored(fileRanges, allocate); + AbfsInputStream abfsIn = (AbfsInputStream) in.getWrappedStream(); + AbfsInputStream spyIn = Mockito.spy(abfsIn); + spyIn.readVectored(fileRanges, allocate); CompletableFuture[] futures = new CompletableFuture[fileRanges.size()]; int i = 0; @@ -67,60 +91,60 @@ public void testVectoredReadDisjointRanges() throws Exception { } CompletableFuture.allOf(futures).get(); validateVectoredReadResult(fileRanges, fileContent, 0); + Mockito.verify(spyIn, Mockito.times(2)) + .readRemote( + Mockito.anyLong(), + Mockito.any(byte[].class), + Mockito.anyInt(), + Mockito.anyInt(), + Mockito.any()); } } @Test - public void testMultipleDisjointRangesWithVectoredRead() throws Throwable { - int fileSize = ONE_MB; - final AzureBlobFileSystem fs = getFileSystem(); - String fileName = methodName.getMethodName() + 1; - byte[] fileContent = getRandomBytesArray(fileSize); - Path testFilePath = createFileWithContent(fs, fileName, fileContent); - - List rangeList = new ArrayList<>(); - rangeList.add(FileRange.createFileRange(100, 10000)); - rangeList.add(FileRange.createFileRange(15000, 27000)); - rangeList.add(FileRange.createFileRange(42500, 40000)); - IntFunction allocate = ByteBuffer::allocate; - CompletableFuture builder = fs.openFile(testFilePath).build(); - - try (FSDataInputStream in = builder.get()) { - in.readVectored(rangeList, allocate); - byte[] readFullRes = new byte[(int)fileSize]; - in.readFully(0, readFullRes); - // Comparing vectored read results with read fully. - validateVectoredReadResult(rangeList, readFullRes, 0); - } - } - - @Test - public void testMultipleRangesWithVectoredRead() throws Throwable { - int fileSize = ONE_MB; + public void testVectoredReadFallsBackToDirectReadWhenQueuingFails() + throws Exception { final AzureBlobFileSystem fs = getFileSystem(); - String fileName = methodName.getMethodName() + 1; - byte[] fileContent = getRandomBytesArray(fileSize); + String fileName = methodName.getMethodName(); + byte[] fileContent = getRandomBytesArray(4 * ONE_MB); Path testFilePath = createFileWithContent(fs, fileName, fileContent); - List rangeList = new ArrayList<>(); - rangeList.add(FileRange.createFileRange(100, 10000)); - rangeList.add(FileRange.createFileRange(15000, 27000)); - rangeList.add(FileRange.createFileRange(47500, 27000)); + List fileRanges = new ArrayList<>(); + fileRanges.add(FileRange.createFileRange(0, ONE_MB)); + fileRanges.add(FileRange.createFileRange(2 * ONE_MB, ONE_MB)); + IntFunction allocator = ByteBuffer::allocate; - IntFunction allocate = ByteBuffer::allocate; - CompletableFuture builder = fs.openFile(testFilePath).build(); + try (FSDataInputStream in = fs.openFile(testFilePath).build().get()) { + AbfsInputStream abfsIn = (AbfsInputStream) in.getWrappedStream(); + AbfsInputStream spyIn = Mockito.spy(abfsIn); + VectoredReadHandler realHandler = abfsIn.getVectoredReadHandler(); + VectoredReadHandler spyHandler = Mockito.spy(realHandler); + Mockito.doReturn(spyHandler).when(spyIn).getVectoredReadHandler(); + Mockito.doReturn(false) + .when(spyHandler) + .queueVectoredRead( + Mockito.any(AbfsInputStream.class), + Mockito.any(CombinedFileRange.class), + ArgumentMatchers.>any()); + spyIn.readVectored(fileRanges, allocator); + CompletableFuture[] futures + = new CompletableFuture[fileRanges.size()]; + for (int i = 0; i < fileRanges.size(); i++) { + futures[i] = fileRanges.get(i).getData(); + } + CompletableFuture.allOf(futures).get(); + Mockito.verify(spyHandler, Mockito.atLeastOnce()) + .directRead( + Mockito.any(AbfsInputStream.class), + Mockito.any(CombinedFileRange.class), + Mockito.eq(allocator)); - try (FSDataInputStream in = builder.get()) { - in.readVectored(rangeList, allocate); - byte[] readFullRes = new byte[(int)fileSize]; - in.readFully(0, readFullRes); - // Comparing vectored read results with read fully. - validateVectoredReadResult(rangeList, readFullRes, 0); + validateVectoredReadResult(fileRanges, fileContent, 0); } } @Test - public void testMergedRangesWithVectoredRead() throws Throwable { + public void testMultipleDisjointRangesWithVectoredRead() throws Throwable { int fileSize = ONE_MB; final AzureBlobFileSystem fs = getFileSystem(); String fileName = methodName.getMethodName() + 1; @@ -129,13 +153,15 @@ public void testMergedRangesWithVectoredRead() throws Throwable { List rangeList = new ArrayList<>(); rangeList.add(FileRange.createFileRange(100, 10000)); - rangeList.add(FileRange.createFileRange(12000, 27000)); + rangeList.add(FileRange.createFileRange(15000, 27000)); + rangeList.add(FileRange.createFileRange(42500, 40000)); IntFunction allocate = ByteBuffer::allocate; - CompletableFuture builder = fs.openFile(testFilePath).build(); + CompletableFuture builder = fs.openFile(testFilePath) + .build(); try (FSDataInputStream in = builder.get()) { in.readVectored(rangeList, allocate); - byte[] readFullRes = new byte[(int)fileSize]; + byte[] readFullRes = new byte[(int) fileSize]; in.readFully(0, readFullRes); // Comparing vectored read results with read fully. validateVectoredReadResult(rangeList, readFullRes, 0); @@ -163,14 +189,254 @@ public void test_045_vectoredIOHugeFile() throws Throwable { CompletableFuture builder = fs.openFile(testFilePath).build(); try (FSDataInputStream in = builder.get()) { - long timeMilli1 = System.currentTimeMillis(); in.readVectored(rangeList, allocate); - byte[] readFullRes = new byte[(int)fileSize]; + byte[] readFullRes = new byte[(int) fileSize]; in.readFully(0, readFullRes); // Comparing vectored read results with read fully. validateVectoredReadResult(rangeList, readFullRes, 0); - long timeMilli2 = System.currentTimeMillis(); - System.out.println("Time taken for the code to execute: " + (timeMilli2 - timeMilli1) + " milliseconds"); + } + } + + @Test + public void testSimultaneousPrefetchAndVectoredRead() throws Exception { + final AzureBlobFileSystem fs = getFileSystem(); + String fileName = methodName.getMethodName(); + byte[] fileContent = getRandomBytesArray(16 * ONE_MB); + Path testFilePath = createFileWithContent(fs, fileName, fileContent); + try (FSDataInputStream in = fs.openFile(testFilePath).build().get()) { + AbfsInputStream abfsIn = (AbfsInputStream) in.getWrappedStream(); + IntFunction allocator = ByteBuffer::allocate; + List vRanges = new ArrayList<>(); + vRanges.add(FileRange.createFileRange(10 * ONE_MB, (int) ONE_MB)); + vRanges.add(FileRange.createFileRange(12 * ONE_MB, (int) ONE_MB)); + byte[] seqBuffer = new byte[(int) ONE_MB]; + CountDownLatch latch = new CountDownLatch(1); + CompletableFuture vectoredTask = CompletableFuture.runAsync(() -> { + try { + latch.await(); + abfsIn.readVectored(vRanges, allocator); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + CompletableFuture sequentialTask = CompletableFuture.runAsync( + () -> { + try { + latch.await(); + abfsIn.read(0, seqBuffer, 0, (int) ONE_MB); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + latch.countDown(); + CompletableFuture.allOf(vectoredTask, sequentialTask).get(); + CompletableFuture[] vFutures = vRanges.stream() + .map(FileRange::getData) + .toArray(CompletableFuture[]::new); + CompletableFuture.allOf(vFutures).get(); + assertArrayEquals(Arrays.copyOfRange(fileContent, 0, (int) ONE_MB), + seqBuffer, "Sequential read data mismatch"); + validateVectoredReadResult(vRanges, fileContent, 0); + } + } + + @Test + public void testConcurrentStreamsOnDifferentFiles() throws Exception { + final AzureBlobFileSystem fs = getFileSystem(); + // Create two distinct files with random content + byte[] content1 = getRandomBytesArray(16 * ONE_MB); + byte[] content2 = getRandomBytesArray(16 * ONE_MB); + Path path1 = createFileWithContent(fs, "file1", content1); + Path path2 = createFileWithContent(fs, "file2", content2); + + // Open two separate input streams for concurrent access + try (FSDataInputStream in1 = fs.openFile(path1).build().get(); + FSDataInputStream in2 = fs.openFile(path2).build().get()) { + + AbfsInputStream streamVectored = (AbfsInputStream) in1.getWrappedStream(); + AbfsInputStream streamSequential + = (AbfsInputStream) in2.getWrappedStream(); + IntFunction allocator = ByteBuffer::allocate; + + // Define non-contiguous ranges for the vectored read on file 1 + List vRanges = new ArrayList<>(); + vRanges.add(FileRange.createFileRange(2 * ONE_MB, (int) ONE_MB)); + vRanges.add(FileRange.createFileRange(4 * ONE_MB, (int) ONE_MB)); + + // Use a latch to ensure both threads start their I/O at the same time + CountDownLatch latch = new CountDownLatch(1); + + // Thread 1: Perform asynchronous vectored reads on file 1 + CompletableFuture vectoredTask = CompletableFuture.runAsync(() -> { + try { + latch.await(); + streamVectored.readVectored(vRanges, allocator); + } catch (Exception e) { + throw new RuntimeException("Vectored read task failed", e); + } + }); + + // Thread 2: Perform multiple sequential reads on file 2 to trigger readahead + CompletableFuture sequentialTask = CompletableFuture.runAsync( + () -> { + try { + latch.await(); + for (int i = 0; i < 5; i++) { + byte[] tempBuf = new byte[(int) ONE_MB]; + streamSequential.read(i * ONE_MB, tempBuf, 0, (int) ONE_MB); + // Validate data integrity for file 2 immediately + assertArrayEquals(Arrays.copyOfRange(content2, i * (int) ONE_MB, + (i + 1) * (int) ONE_MB), tempBuf, + "Sequential read mismatch in file 2 at block " + i); + } + } catch (Exception e) { + throw new RuntimeException("Sequential read task failed", e); + } + }); + + // Trigger simultaneous execution + latch.countDown(); + + // Wait for both high-level tasks to finish + CompletableFuture.allOf(vectoredTask, sequentialTask).get(); + + // Explicitly wait for the vectored read futures to complete their data transfer + CompletableFuture[] vFutures = vRanges.stream() + .map(FileRange::getData) + .toArray(CompletableFuture[]::new); + CompletableFuture.allOf(vFutures).get(); + + // Final validation of vectored read content for file 1 + validateVectoredReadResult(vRanges, content1, 0); + } + } + + @Test + public void testVectoredReadHitchhikesOnExistingPrefetch() throws Exception { + final AzureBlobFileSystem fs = getFileSystem(); + String fileName = methodName.getMethodName(); + byte[] fileContent = getRandomBytesArray(8 * ONE_MB); + Path testFilePath = createFileWithContent(fs, fileName, fileContent); + + try (FSDataInputStream in = fs.openFile(testFilePath).build().get()) { + AbfsInputStream abfsIn = (AbfsInputStream) in.getWrappedStream(); + AbfsInputStream spyIn = Mockito.spy(abfsIn); + + // 1. Trigger a normal read to start the prefetch logic + // Reading the first byte often triggers a larger readahead (e.g., 4MB) + byte[] seqBuf = new byte[1]; + spyIn.read(seqBuf, 0, 1); + + // 2. Immediately queue a vectored read for an offset within that prefetch range + List vRanges = new ArrayList<>(); + // Using 1MB offset, which should be inside the initial readahead buffer + vRanges.add(FileRange.createFileRange(ONE_MB, (int) ONE_MB)); + + IntFunction allocator = ByteBuffer::allocate; + spyIn.readVectored(vRanges, allocator); + + // 3. Wait for the vectored read to complete + vRanges.get(0).getData().get(); + + // 4. Validate Data Integrity + validateVectoredReadResult(vRanges, fileContent, 0); + + // 5. THE CRITICAL VALIDATION: + // Even though we did a manual read and a vectored read, + // there should only be ONE remote call if hitchhiking worked. + Mockito.verify(spyIn, Mockito.atMost(spyIn.getReadAheadQueueDepth())) + .readRemote( + Mockito.anyLong(), + Mockito.any(byte[].class), + Mockito.anyInt(), + Mockito.anyInt(), + Mockito.any()); + } + } + + @Test + public void testMultipleReadsWhileBufferInProgressEventuallyComplete() + throws Exception { + final AzureBlobFileSystem fs = getFileSystem(); + String fileName = methodName.getMethodName(); + byte[] fileContent = getRandomBytesArray(8 * ONE_MB); + Path testFilePath = createFileWithContent(fs, fileName, fileContent); + + CountDownLatch blockCompletion = new CountDownLatch(1); + + try (FSDataInputStream in = fs.openFile(testFilePath).build().get()) { + AbfsInputStream spyIn = + Mockito.spy((AbfsInputStream) in.getWrappedStream()); + ReadBufferManager rbm = spyIn.getReadBufferManager(); + + /* Block completion so buffer stays in inProgressList */ + Mockito.doAnswer(invocation -> { + blockCompletion.await(); + return invocation.callRealMethod(); + }).when(spyIn).readRemote( + Mockito.anyLong(), + Mockito.any(byte[].class), + Mockito.anyInt(), + Mockito.anyInt(), + Mockito.any()); + + ExecutorService exec = Executors.newFixedThreadPool(3); + + /* 1. Start first normal read → creates in-progress buffer */ + Future r1 = exec.submit(() -> { + try { + spyIn.read(new byte[1], 0, 1); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + + /* 2. Explicitly validate buffer is in inProgressList */ + ReadBuffer inProgress = null; + for (int i = 0; i < 100; i++) { + synchronized (rbm) { + inProgress = rbm.findInList( + rbm.getInProgressList(), spyIn, 0); + } + if (inProgress != null) { + break; + } + Thread.sleep(10); + } + assertNotNull(inProgress, "Expected buffer to be in inProgressList while completion is blocked"); + + /* 3. Submit another normal read while buffer is in progress */ + Future r2 = exec.submit(() -> { + try { + spyIn.read(new byte[1], 0, 1); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + + /* 4. Submit vectored read while buffer is in progress */ + List ranges = new ArrayList<>(); + ranges.add(FileRange.createFileRange(ONE_MB, (int) ONE_MB)); + Future vr = exec.submit(() -> { + try { + spyIn.readVectored(ranges, ByteBuffer::allocate); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + + /* 5. Allow completion */ + blockCompletion.countDown(); + + /* 6. All reads must complete */ + r1.get(5, TimeUnit.SECONDS); + r2.get(5, TimeUnit.SECONDS); + vr.get(5, TimeUnit.SECONDS); + ranges.get(0).getData().get(5, TimeUnit.SECONDS); + + validateVectoredReadResult(ranges, fileContent, 0); + + exec.shutdownNow(); } } } From c4313e9f9c35af86537adfc3ffa902b0e8568abc Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Tue, 27 Jan 2026 03:14:22 -0800 Subject: [PATCH 05/17] Checkstyle fix --- .../org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index 0b3d0653e8dde..47bb066443773 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -2303,10 +2303,9 @@ public VectoredReadStrategy getVectoredReadStrategy() { return VectoredReadStrategy.fromString(vectoredReadStrategy); } catch (IllegalArgumentException e) { throw new IllegalArgumentException( - "Invalid value for " + FS_AZURE_VECTORED_READ_STRATEGY + - ": " + vectoredReadStrategy + - ". Expected one of: TPS, THROUGHPUT, TPS_OPTIMIZED, THROUGHPUT_OPTIMIZED", - e); + "Invalid value for " + FS_AZURE_VECTORED_READ_STRATEGY + + ": " + vectoredReadStrategy + + ". Expected one of: TPS, THROUGHPUT, TPS_OPTIMIZED, THROUGHPUT_OPTIMIZED", e); } } From 0106fc1a89f72cc6cf4717c9a1a41017df69018e Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Tue, 27 Jan 2026 03:29:55 -0800 Subject: [PATCH 06/17] Fix checkstyle --- .../constants/FileSystemConfigurations.java | 1 - .../fs/azurebfs/services/AbfsInputStream.java | 3 - .../services/VectoredReadHandler.java | 26 +- .../azurebfs/services/ITestVectoredRead.java | 620 ++++++++---------- 4 files changed, 306 insertions(+), 344 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java index e3f731c2332aa..e54cf493eff15 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java @@ -20,7 +20,6 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.fs.azurebfs.enums.VectoredReadStrategy; import org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory; import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_ADAPTIVE; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java index 99225beb36483..4f96bbb1b5420 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java @@ -26,14 +26,11 @@ import java.util.List; import java.util.UUID; import java.util.function.IntFunction; -import java.util.concurrent.locks.ReentrantLock; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.fs.FileRange; -import org.apache.hadoop.fs.azurebfs.AbfsConfiguration; import org.apache.hadoop.fs.azurebfs.constants.ReadType; -import org.apache.hadoop.fs.azurebfs.enums.VectoredReadStrategy; import org.apache.hadoop.fs.impl.BackReference; import org.apache.hadoop.util.Preconditions; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VectoredReadHandler.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VectoredReadHandler.java index 7ccb73745ab82..df011aa465002 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VectoredReadHandler.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VectoredReadHandler.java @@ -31,11 +31,35 @@ import org.apache.hadoop.fs.azurebfs.enums.VectoredReadStrategy; import org.apache.hadoop.fs.impl.CombinedFileRange; +/** + * Handles vectored read operations by coordinating with a ReadBufferManager + * and applying the configured VectoredReadStrategy. + * This class acts as the orchestration layer that decides how vectored reads + * are executed, while delegating buffer management and read behavior to + * dedicated components. + */ class VectoredReadHandler { + + /** + * Manages allocation, lifecycle, and reuse of read buffers + * used during vectored read operations. + */ private final ReadBufferManager readBufferManager; + + /** + * Strategy defining how vectored reads should be performed. + */ private final VectoredReadStrategy strategy; - public VectoredReadHandler(ReadBufferManager readBufferManager) { + /** + * Creates a VectoredReadHandler using the provided ReadBufferManager. + * The vectored read strategy is obtained from the manager to ensure + * consistent configuration across the read pipeline. + * + * @param readBufferManager manager responsible for buffer handling + * and providing the vectored read strategy + */ + VectoredReadHandler(ReadBufferManager readBufferManager) { this.readBufferManager = readBufferManager; this.strategy = readBufferManager.getVectoredReadStrategy(); } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java index f21239a4a768b..49009b8494681 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.fs.azurebfs.services; import java.nio.ByteBuffer; @@ -10,7 +28,6 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import java.util.function.IntFunction; import org.junit.jupiter.api.Test; import org.mockito.ArgumentMatchers; @@ -21,82 +38,121 @@ import org.apache.hadoop.fs.FileRange; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.azurebfs.AbfsConfiguration; import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest; import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem; import org.apache.hadoop.fs.impl.CombinedFileRange; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_READAHEAD_V2_MEMORY_USAGE_THRESHOLD_PERCENT; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_VECTORED_READ_STRATEGY; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.ONE_MB; import static org.apache.hadoop.fs.contract.ContractTestUtils.validateVectoredReadResult; public class ITestVectoredRead extends AbstractAbfsIntegrationTest { + private static final int FILE_1_MB = ONE_MB; + private static final int FILE_4_MB = 4 * ONE_MB; + private static final int FILE_8_MB = 8 * ONE_MB; + private static final int FILE_16_MB = 16 * ONE_MB; + private static final int FILE_32_MB = 32 * ONE_MB; + private static final int FILE_100_MB = 100 * ONE_MB; + + private static final int OFFSET_100_B = 100; + private static final int OFFSET_15K_B = 15_000; + private static final int OFFSET_42K_B = 42_500; + + private static final int LEN_10K_B = 10_000; + private static final int LEN_27K_B = 27_000; + private static final int LEN_40K_B = 40_000; + + private static final double MB_1_2 = 1.2; + private static final double MB_3_1 = 3.1; + private static final double MB_4_1 = 4.1; + private static final double MB_6_2 = 6.2; + + private static final double MB_0_8 = 0.8; + private static final double MB_0_9 = 0.9; + private static final double MB_1_8 = 1.8; + private static final double MB_1_9 = 1.9; + + private static final int HUGE_OFFSET_1 = 5_856_368; + private static final int HUGE_OFFSET_2 = 3_520_861; + private static final int HUGE_OFFSET_3 = 8_191_913; + private static final int HUGE_OFFSET_4 = 1_520_861; + private static final int HUGE_OFFSET_5 = 2_520_861; + private static final int HUGE_OFFSET_6 = 9_191_913; + private static final int HUGE_OFFSET_7 = 2_820_861; + + private static final int HUGE_RANGE = 116_770; + private static final int HUGE_RANGE_LARGE = 156_770; + + private static final int LOOKUP_RETRIES = 100; + private static final int EXEC_THREADS = 3; + private static final int SEQ_READ_ITERATIONS = 5; + private static final int FUTURE_TIMEOUT_SEC = 5; public ITestVectoredRead() throws Exception { } + /** + * Verifies basic vectored read functionality with two disjoint ranges. + * Ensures data read via readVectored matches data read via a full sequential read. + * Acts as a correctness baseline for simple non-overlapping ranges. + */ @Test public void testDisjointRangesWithVectoredRead() throws Throwable { - int fileSize = ONE_MB; final AzureBlobFileSystem fs = getFileSystem(); - String fileName = methodName.getMethodName() + 1; - byte[] fileContent = getRandomBytesArray(fileSize); - Path testFilePath = createFileWithContent(fs, fileName, fileContent); - - List rangeList = new ArrayList<>(); - rangeList.add(FileRange.createFileRange(100, 10000)); - rangeList.add(FileRange.createFileRange(15000, 27000)); - IntFunction allocate = ByteBuffer::allocate; - CompletableFuture builder = fs.openFile(testFilePath) - .build(); - - try (FSDataInputStream in = builder.get()) { - in.readVectored(rangeList, allocate); - byte[] readFullRes = new byte[(int) fileSize]; - in.readFully(0, readFullRes); - // Comparing vectored read results with read fully. - validateVectoredReadResult(rangeList, readFullRes, 0); + byte[] fileContent = getRandomBytesArray(FILE_1_MB); + Path path = createFileWithContent(fs, methodName.getMethodName(), fileContent); + + List ranges = new ArrayList<>(); + ranges.add(FileRange.createFileRange(OFFSET_100_B, LEN_10K_B)); + ranges.add(FileRange.createFileRange(OFFSET_15K_B, LEN_27K_B)); + + try (FSDataInputStream in = fs.openFile(path).build().get()) { + in.readVectored(ranges, ByteBuffer::allocate); + byte[] full = new byte[FILE_1_MB]; + in.readFully(0, full); + validateVectoredReadResult(ranges, full, 0); } } + /** + * Validates that disjoint vectored ranges are coalesced into the minimum + * number of backend reads. + * Ensures that only two remote reads are issued for the given range layout. + */ @Test public void testVectoredReadDisjointRangesExpectTwoBackendReads() throws Exception { + final AzureBlobFileSystem fs = getFileSystem(); - String fileName = methodName.getMethodName(); - byte[] fileContent = getRandomBytesArray(16 * ONE_MB); - Path testFilePath = createFileWithContent(fs, fileName, fileContent); - List fileRanges = new ArrayList<>(); - // 0.0 – 1.0 MB - fileRanges.add(FileRange.createFileRange(0L, (int) ONE_MB)); - // 1.2 – 2.0 MB - fileRanges.add( - FileRange.createFileRange((long) (1.2 * ONE_MB), (int) (0.8 * ONE_MB))); - // 3.1 – 4.0 MB - fileRanges.add( - FileRange.createFileRange((long) (3.1 * ONE_MB), (int) (0.9 * ONE_MB))); - // 4.1 – 6.0 MB - fileRanges.add( - FileRange.createFileRange((long) (4.1 * ONE_MB), (int) (1.9 * ONE_MB))); - // 6.2 – 8.0 MB - fileRanges.add( - FileRange.createFileRange((long) (6.2 * ONE_MB), (int) (1.8 * ONE_MB))); - IntFunction allocate = ByteBuffer::allocate; - try (FSDataInputStream in = - fs.openFile(testFilePath).build().get()) { - AbfsInputStream abfsIn = (AbfsInputStream) in.getWrappedStream(); - AbfsInputStream spyIn = Mockito.spy(abfsIn); - spyIn.readVectored(fileRanges, allocate); - CompletableFuture[] futures = - new CompletableFuture[fileRanges.size()]; - int i = 0; - for (FileRange range : fileRanges) { - futures[i++] = range.getData(); - } - CompletableFuture.allOf(futures).get(); - validateVectoredReadResult(fileRanges, fileContent, 0); - Mockito.verify(spyIn, Mockito.times(2)) + byte[] content = getRandomBytesArray(FILE_16_MB); + Path path = createFileWithContent(fs, methodName.getMethodName(), content); + + List ranges = new ArrayList<>(); + ranges.add(FileRange.createFileRange(0L, ONE_MB)); + ranges.add(FileRange.createFileRange((long) (MB_1_2 * ONE_MB), + (int) (MB_0_8 * ONE_MB))); + ranges.add(FileRange.createFileRange((long) (MB_3_1 * ONE_MB), + (int) (MB_0_9 * ONE_MB))); + ranges.add(FileRange.createFileRange((long) (MB_4_1 * ONE_MB), + (int) (MB_1_9 * ONE_MB))); + ranges.add(FileRange.createFileRange((long) (MB_6_2 * ONE_MB), + (int) (MB_1_8 * ONE_MB))); + + try (FSDataInputStream in = fs.openFile(path).build().get()) { + AbfsInputStream spy = + Mockito.spy((AbfsInputStream) in.getWrappedStream()); + + spy.readVectored(ranges, ByteBuffer::allocate); + + CompletableFuture.allOf( + ranges.stream() + .map(FileRange::getData) + .toArray(CompletableFuture[]::new)) + .get(); + + validateVectoredReadResult(ranges, content, 0); + + Mockito.verify(spy, Mockito.times(2)) .readRemote( Mockito.anyLong(), Mockito.any(byte[].class), @@ -106,385 +162,271 @@ public void testVectoredReadDisjointRangesExpectTwoBackendReads() } } + /** + * Simulates failure in vectored read queuing. + * Verifies that the system safely falls back to direct reads + * and still returns correct data. + */ @Test public void testVectoredReadFallsBackToDirectReadWhenQueuingFails() throws Exception { + final AzureBlobFileSystem fs = getFileSystem(); - String fileName = methodName.getMethodName(); - byte[] fileContent = getRandomBytesArray(4 * ONE_MB); - Path testFilePath = createFileWithContent(fs, fileName, fileContent); + byte[] content = getRandomBytesArray(FILE_4_MB); + Path path = createFileWithContent(fs, methodName.getMethodName(), content); - List fileRanges = new ArrayList<>(); - fileRanges.add(FileRange.createFileRange(0, ONE_MB)); - fileRanges.add(FileRange.createFileRange(2 * ONE_MB, ONE_MB)); - IntFunction allocator = ByteBuffer::allocate; + List ranges = List.of( + FileRange.createFileRange(0, ONE_MB), + FileRange.createFileRange(2 * ONE_MB, ONE_MB)); - try (FSDataInputStream in = fs.openFile(testFilePath).build().get()) { - AbfsInputStream abfsIn = (AbfsInputStream) in.getWrappedStream(); + try (FSDataInputStream in = fs.openFile(path).build().get()) { + AbfsInputStream abfsIn = + (AbfsInputStream) in.getWrappedStream(); AbfsInputStream spyIn = Mockito.spy(abfsIn); + VectoredReadHandler realHandler = abfsIn.getVectoredReadHandler(); VectoredReadHandler spyHandler = Mockito.spy(realHandler); + Mockito.doReturn(spyHandler).when(spyIn).getVectoredReadHandler(); Mockito.doReturn(false) .when(spyHandler) .queueVectoredRead( - Mockito.any(AbfsInputStream.class), + Mockito.any(), Mockito.any(CombinedFileRange.class), - ArgumentMatchers.>any()); - spyIn.readVectored(fileRanges, allocator); - CompletableFuture[] futures - = new CompletableFuture[fileRanges.size()]; - for (int i = 0; i < fileRanges.size(); i++) { - futures[i] = fileRanges.get(i).getData(); - } - CompletableFuture.allOf(futures).get(); + ArgumentMatchers.any()); + + spyIn.readVectored(ranges, ByteBuffer::allocate); + + CompletableFuture.allOf( + ranges.stream() + .map(FileRange::getData) + .toArray(CompletableFuture[]::new)) + .get(); + Mockito.verify(spyHandler, Mockito.atLeastOnce()) .directRead( - Mockito.any(AbfsInputStream.class), + Mockito.any(), Mockito.any(CombinedFileRange.class), - Mockito.eq(allocator)); + Mockito.any()); - validateVectoredReadResult(fileRanges, fileContent, 0); + validateVectoredReadResult(ranges, content, 0); } } + /** + * Tests vectored reads with multiple small disjoint ranges. + * Ensures correctness when several non-contiguous ranges are requested together. + */ @Test public void testMultipleDisjointRangesWithVectoredRead() throws Throwable { - int fileSize = ONE_MB; final AzureBlobFileSystem fs = getFileSystem(); - String fileName = methodName.getMethodName() + 1; - byte[] fileContent = getRandomBytesArray(fileSize); - Path testFilePath = createFileWithContent(fs, fileName, fileContent); - - List rangeList = new ArrayList<>(); - rangeList.add(FileRange.createFileRange(100, 10000)); - rangeList.add(FileRange.createFileRange(15000, 27000)); - rangeList.add(FileRange.createFileRange(42500, 40000)); - IntFunction allocate = ByteBuffer::allocate; - CompletableFuture builder = fs.openFile(testFilePath) - .build(); - - try (FSDataInputStream in = builder.get()) { - in.readVectored(rangeList, allocate); - byte[] readFullRes = new byte[(int) fileSize]; - in.readFully(0, readFullRes); - // Comparing vectored read results with read fully. - validateVectoredReadResult(rangeList, readFullRes, 0); + byte[] content = getRandomBytesArray(FILE_1_MB); + Path path = createFileWithContent(fs, methodName.getMethodName(), content); + + List ranges = List.of( + FileRange.createFileRange(OFFSET_100_B, LEN_10K_B), + FileRange.createFileRange(OFFSET_15K_B, LEN_27K_B), + FileRange.createFileRange(OFFSET_42K_B, LEN_40K_B)); + + try (FSDataInputStream in = fs.openFile(path).build().get()) { + in.readVectored(ranges, ByteBuffer::allocate); + byte[] full = new byte[FILE_1_MB]; + in.readFully(0, full); + validateVectoredReadResult(ranges, full, 0); } } + /** + * Validates vectored reads against a very large file with widely scattered ranges. + * Ensures correctness and stability under large-file and non-localized access patterns. + */ @Test - public void test_045_vectoredIOHugeFile() throws Throwable { - int fileSize = 100 * ONE_MB; + public void testVectoredIOHugeFile() throws Throwable { final AzureBlobFileSystem fs = getFileSystem(); - String fileName = methodName.getMethodName() + 1; - byte[] fileContent = getRandomBytesArray(fileSize); - Path testFilePath = createFileWithContent(fs, fileName, fileContent); - - List rangeList = new ArrayList<>(); - rangeList.add(FileRange.createFileRange(5856368, 116770)); - rangeList.add(FileRange.createFileRange(3520861, 116770)); - rangeList.add(FileRange.createFileRange(8191913, 116770)); - rangeList.add(FileRange.createFileRange(1520861, 116770)); - rangeList.add(FileRange.createFileRange(2520861, 116770)); - rangeList.add(FileRange.createFileRange(9191913, 116770)); - rangeList.add(FileRange.createFileRange(2820861, 156770)); - IntFunction allocate = ByteBuffer::allocate; - - CompletableFuture builder = - fs.openFile(testFilePath).build(); - try (FSDataInputStream in = builder.get()) { - in.readVectored(rangeList, allocate); - byte[] readFullRes = new byte[(int) fileSize]; - in.readFully(0, readFullRes); - // Comparing vectored read results with read fully. - validateVectoredReadResult(rangeList, readFullRes, 0); - } - } - - @Test - public void testSimultaneousPrefetchAndVectoredRead() throws Exception { - final AzureBlobFileSystem fs = getFileSystem(); - String fileName = methodName.getMethodName(); - byte[] fileContent = getRandomBytesArray(16 * ONE_MB); - Path testFilePath = createFileWithContent(fs, fileName, fileContent); - try (FSDataInputStream in = fs.openFile(testFilePath).build().get()) { - AbfsInputStream abfsIn = (AbfsInputStream) in.getWrappedStream(); - IntFunction allocator = ByteBuffer::allocate; - List vRanges = new ArrayList<>(); - vRanges.add(FileRange.createFileRange(10 * ONE_MB, (int) ONE_MB)); - vRanges.add(FileRange.createFileRange(12 * ONE_MB, (int) ONE_MB)); - byte[] seqBuffer = new byte[(int) ONE_MB]; - CountDownLatch latch = new CountDownLatch(1); - CompletableFuture vectoredTask = CompletableFuture.runAsync(() -> { - try { - latch.await(); - abfsIn.readVectored(vRanges, allocator); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - CompletableFuture sequentialTask = CompletableFuture.runAsync( - () -> { - try { - latch.await(); - abfsIn.read(0, seqBuffer, 0, (int) ONE_MB); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - latch.countDown(); - CompletableFuture.allOf(vectoredTask, sequentialTask).get(); - CompletableFuture[] vFutures = vRanges.stream() - .map(FileRange::getData) - .toArray(CompletableFuture[]::new); - CompletableFuture.allOf(vFutures).get(); - assertArrayEquals(Arrays.copyOfRange(fileContent, 0, (int) ONE_MB), - seqBuffer, "Sequential read data mismatch"); - validateVectoredReadResult(vRanges, fileContent, 0); + byte[] content = getRandomBytesArray(FILE_100_MB); + Path path = createFileWithContent(fs, methodName.getMethodName(), content); + + List ranges = List.of( + FileRange.createFileRange(HUGE_OFFSET_1, HUGE_RANGE), + FileRange.createFileRange(HUGE_OFFSET_2, HUGE_RANGE), + FileRange.createFileRange(HUGE_OFFSET_3, HUGE_RANGE), + FileRange.createFileRange(HUGE_OFFSET_4, HUGE_RANGE), + FileRange.createFileRange(HUGE_OFFSET_5, HUGE_RANGE), + FileRange.createFileRange(HUGE_OFFSET_6, HUGE_RANGE), + FileRange.createFileRange(HUGE_OFFSET_7, HUGE_RANGE_LARGE)); + + try (FSDataInputStream in = fs.openFile(path).build().get()) { + in.readVectored(ranges, ByteBuffer::allocate); + byte[] full = new byte[FILE_100_MB]; + in.readFully(0, full); + validateVectoredReadResult(ranges, full, 0); } } + /** + * Ensures vectored reads on one file do not interfere with + * sequential reads and readahead on a different file. + * Validates isolation across concurrent streams. + */ @Test public void testConcurrentStreamsOnDifferentFiles() throws Exception { final AzureBlobFileSystem fs = getFileSystem(); - // Create two distinct files with random content - byte[] content1 = getRandomBytesArray(16 * ONE_MB); - byte[] content2 = getRandomBytesArray(16 * ONE_MB); + + byte[] content1 = getRandomBytesArray(FILE_16_MB); + byte[] content2 = getRandomBytesArray(FILE_16_MB); + Path path1 = createFileWithContent(fs, "file1", content1); Path path2 = createFileWithContent(fs, "file2", content2); - // Open two separate input streams for concurrent access try (FSDataInputStream in1 = fs.openFile(path1).build().get(); FSDataInputStream in2 = fs.openFile(path2).build().get()) { - AbfsInputStream streamVectored = (AbfsInputStream) in1.getWrappedStream(); - AbfsInputStream streamSequential - = (AbfsInputStream) in2.getWrappedStream(); - IntFunction allocator = ByteBuffer::allocate; + AbfsInputStream vStream = + (AbfsInputStream) in1.getWrappedStream(); + AbfsInputStream sStream = + (AbfsInputStream) in2.getWrappedStream(); - // Define non-contiguous ranges for the vectored read on file 1 - List vRanges = new ArrayList<>(); - vRanges.add(FileRange.createFileRange(2 * ONE_MB, (int) ONE_MB)); - vRanges.add(FileRange.createFileRange(4 * ONE_MB, (int) ONE_MB)); + List ranges = List.of( + FileRange.createFileRange(2 * ONE_MB, ONE_MB), + FileRange.createFileRange(4 * ONE_MB, ONE_MB)); - // Use a latch to ensure both threads start their I/O at the same time CountDownLatch latch = new CountDownLatch(1); - // Thread 1: Perform asynchronous vectored reads on file 1 CompletableFuture vectoredTask = CompletableFuture.runAsync(() -> { try { latch.await(); - streamVectored.readVectored(vRanges, allocator); + vStream.readVectored(ranges, ByteBuffer::allocate); } catch (Exception e) { - throw new RuntimeException("Vectored read task failed", e); + throw new RuntimeException(e); } }); - // Thread 2: Perform multiple sequential reads on file 2 to trigger readahead - CompletableFuture sequentialTask = CompletableFuture.runAsync( - () -> { - try { - latch.await(); - for (int i = 0; i < 5; i++) { - byte[] tempBuf = new byte[(int) ONE_MB]; - streamSequential.read(i * ONE_MB, tempBuf, 0, (int) ONE_MB); - // Validate data integrity for file 2 immediately - assertArrayEquals(Arrays.copyOfRange(content2, i * (int) ONE_MB, - (i + 1) * (int) ONE_MB), tempBuf, - "Sequential read mismatch in file 2 at block " + i); - } - } catch (Exception e) { - throw new RuntimeException("Sequential read task failed", e); - } - }); - - // Trigger simultaneous execution - latch.countDown(); + CompletableFuture sequentialTask = CompletableFuture.runAsync(() -> { + try { + latch.await(); + for (int i = 0; i < SEQ_READ_ITERATIONS; i++) { + byte[] buf = new byte[ONE_MB]; + sStream.read(i * ONE_MB, buf, 0, ONE_MB); + assertArrayEquals( + Arrays.copyOfRange(content2, + i * ONE_MB, (i + 1) * ONE_MB), + buf); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + }); - // Wait for both high-level tasks to finish + latch.countDown(); CompletableFuture.allOf(vectoredTask, sequentialTask).get(); - // Explicitly wait for the vectored read futures to complete their data transfer - CompletableFuture[] vFutures = vRanges.stream() - .map(FileRange::getData) - .toArray(CompletableFuture[]::new); - CompletableFuture.allOf(vFutures).get(); + CompletableFuture.allOf( + ranges.stream().map(FileRange::getData) + .toArray(CompletableFuture[]::new)).get(); - // Final validation of vectored read content for file 1 - validateVectoredReadResult(vRanges, content1, 0); - } - } - - @Test - public void testVectoredReadHitchhikesOnExistingPrefetch() throws Exception { - final AzureBlobFileSystem fs = getFileSystem(); - String fileName = methodName.getMethodName(); - byte[] fileContent = getRandomBytesArray(8 * ONE_MB); - Path testFilePath = createFileWithContent(fs, fileName, fileContent); - - try (FSDataInputStream in = fs.openFile(testFilePath).build().get()) { - AbfsInputStream abfsIn = (AbfsInputStream) in.getWrappedStream(); - AbfsInputStream spyIn = Mockito.spy(abfsIn); - - // 1. Trigger a normal read to start the prefetch logic - // Reading the first byte often triggers a larger readahead (e.g., 4MB) - byte[] seqBuf = new byte[1]; - spyIn.read(seqBuf, 0, 1); - - // 2. Immediately queue a vectored read for an offset within that prefetch range - List vRanges = new ArrayList<>(); - // Using 1MB offset, which should be inside the initial readahead buffer - vRanges.add(FileRange.createFileRange(ONE_MB, (int) ONE_MB)); - - IntFunction allocator = ByteBuffer::allocate; - spyIn.readVectored(vRanges, allocator); - - // 3. Wait for the vectored read to complete - vRanges.get(0).getData().get(); - - // 4. Validate Data Integrity - validateVectoredReadResult(vRanges, fileContent, 0); - - // 5. THE CRITICAL VALIDATION: - // Even though we did a manual read and a vectored read, - // there should only be ONE remote call if hitchhiking worked. - Mockito.verify(spyIn, Mockito.atMost(spyIn.getReadAheadQueueDepth())) - .readRemote( - Mockito.anyLong(), - Mockito.any(byte[].class), - Mockito.anyInt(), - Mockito.anyInt(), - Mockito.any()); + validateVectoredReadResult(ranges, content1, 0); } } + /** + * Ensures multiple reads issued while a buffer is in progress + * properly wait and complete once the buffer finishes. + * Covers both sequential and vectored reads during in-flight I/O. + */ @Test public void testMultipleReadsWhileBufferInProgressEventuallyComplete() throws Exception { + final AzureBlobFileSystem fs = getFileSystem(); - String fileName = methodName.getMethodName(); - byte[] fileContent = getRandomBytesArray(8 * ONE_MB); - Path testFilePath = createFileWithContent(fs, fileName, fileContent); + byte[] content = getRandomBytesArray(FILE_8_MB); + Path path = createFileWithContent(fs, methodName.getMethodName(), content); - CountDownLatch blockCompletion = new CountDownLatch(1); + CountDownLatch block = new CountDownLatch(1); - try (FSDataInputStream in = fs.openFile(testFilePath).build().get()) { - AbfsInputStream spyIn = + try (FSDataInputStream in = fs.openFile(path).build().get()) { + AbfsInputStream spy = Mockito.spy((AbfsInputStream) in.getWrappedStream()); - ReadBufferManager rbm = spyIn.getReadBufferManager(); + ReadBufferManager rbm = spy.getReadBufferManager(); - /* Block completion so buffer stays in inProgressList */ - Mockito.doAnswer(invocation -> { - blockCompletion.await(); - return invocation.callRealMethod(); - }).when(spyIn).readRemote( + Mockito.doAnswer(inv -> { + block.await(); + return inv.callRealMethod(); + }).when(spy).readRemote( Mockito.anyLong(), Mockito.any(byte[].class), Mockito.anyInt(), Mockito.anyInt(), Mockito.any()); - ExecutorService exec = Executors.newFixedThreadPool(3); + ExecutorService exec = Executors.newFixedThreadPool(EXEC_THREADS); - /* 1. Start first normal read → creates in-progress buffer */ - Future r1 = exec.submit(() -> { - try { - spyIn.read(new byte[1], 0, 1); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); + exec.submit(() -> spy.read(new byte[1], 0, 1)); - /* 2. Explicitly validate buffer is in inProgressList */ ReadBuffer inProgress = null; - for (int i = 0; i < 100; i++) { + for (int i = 0; i < LOOKUP_RETRIES; i++) { synchronized (rbm) { inProgress = rbm.findInList( - rbm.getInProgressList(), spyIn, 0); + rbm.getInProgressList(), spy, 0); } if (inProgress != null) { break; } Thread.sleep(10); } - assertNotNull(inProgress, - "Expected buffer to be in inProgressList while completion is blocked"); - - /* 3. Submit another normal read while buffer is in progress */ - Future r2 = exec.submit(() -> { - try { - spyIn.read(new byte[1], 0, 1); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - - /* 4. Submit vectored read while buffer is in progress */ - List ranges = new ArrayList<>(); - ranges.add(FileRange.createFileRange(ONE_MB, (int) ONE_MB)); - Future vr = exec.submit(() -> { - try { - spyIn.readVectored(ranges, ByteBuffer::allocate); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - /* 5. Allow completion */ - blockCompletion.countDown(); + assertNotNull(inProgress); - /* 6. All reads must complete */ - r1.get(5, TimeUnit.SECONDS); - r2.get(5, TimeUnit.SECONDS); - vr.get(5, TimeUnit.SECONDS); - ranges.get(0).getData().get(5, TimeUnit.SECONDS); + Future vr = exec.submit(() -> + spy.readVectored( + List.of(FileRange.createFileRange(ONE_MB, ONE_MB)), + ByteBuffer::allocate)); - validateVectoredReadResult(ranges, fileContent, 0); + block.countDown(); + vr.get(FUTURE_TIMEOUT_SEC, TimeUnit.SECONDS); exec.shutdownNow(); } } + /** + * Verifies vectored reads using the throughput-optimized (TPS) strategy. + * Ensures expected backend read count and data correctness under TPS mode. + */ @Test public void testThroughputOptimizedReadVectored() throws Exception { - Configuration configuration = getRawConfiguration(); - configuration.set(FS_AZURE_VECTORED_READ_STRATEGY, "TPS"); - FileSystem fileSystem = FileSystem.newInstance(configuration); - try (AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem) { - String fileName = methodName.getMethodName(); - byte[] fileContent = getRandomBytesArray(32 * ONE_MB); - Path testFilePath = createFileWithContent(abfs, fileName, fileContent); - List fileRanges = new ArrayList<>(); - // 0.0 – 3.8 MB - fileRanges.add(FileRange.createFileRange(0L, (int) (3.8 * ONE_MB))); - // 4.0 – 7.2 MB - fileRanges.add(FileRange.createFileRange((long) (4.0 * ONE_MB), - (int) (3.2 * ONE_MB))); - // 8.0 – 10.0 MB - fileRanges.add(FileRange.createFileRange((long) (8.0 * ONE_MB), - (int) (2.0 * ONE_MB))); - // 12.0 – 16.0 MB - fileRanges.add(FileRange.createFileRange((long) (12.0 * ONE_MB), - (int) (4.0 * ONE_MB))); - // 16.0 – 18.0 MB - fileRanges.add(FileRange.createFileRange((long) (16.0 * ONE_MB), - (int) (2.0 * ONE_MB))); - IntFunction allocate = ByteBuffer::allocate; - try (FSDataInputStream in = - abfs.openFile(testFilePath).build().get()) { - AbfsInputStream abfsIn = (AbfsInputStream) in.getWrappedStream(); - AbfsInputStream spyIn = Mockito.spy(abfsIn); - spyIn.readVectored(fileRanges, allocate); - CompletableFuture[] futures = - new CompletableFuture[fileRanges.size()]; - int i = 0; - for (FileRange range : fileRanges) { - futures[i++] = range.getData(); - } - CompletableFuture.allOf(futures).get(); - validateVectoredReadResult(fileRanges, fileContent, 0); - Mockito.verify(spyIn, Mockito.times(5)) + Configuration conf = getRawConfiguration(); + conf.set(FS_AZURE_VECTORED_READ_STRATEGY, "TPS"); + + try (AzureBlobFileSystem fs = + (AzureBlobFileSystem) FileSystem.newInstance(conf)) { + + byte[] content = getRandomBytesArray(FILE_32_MB); + Path path = createFileWithContent(fs, methodName.getMethodName(), content); + + List ranges = List.of( + FileRange.createFileRange(0L, (int) (3.8 * ONE_MB)), + FileRange.createFileRange((long) (4.0 * ONE_MB), + (int) (3.2 * ONE_MB)), + FileRange.createFileRange((long) (8.0 * ONE_MB), + (int) (2.0 * ONE_MB)), + FileRange.createFileRange((long) (12.0 * ONE_MB), + (int) (4.0 * ONE_MB)), + FileRange.createFileRange((long) (16.0 * ONE_MB), + (int) (2.0 * ONE_MB))); + + try (FSDataInputStream in = fs.openFile(path).build().get()) { + AbfsInputStream spy = + Mockito.spy((AbfsInputStream) in.getWrappedStream()); + + spy.readVectored(ranges, ByteBuffer::allocate); + + CompletableFuture.allOf( + ranges.stream().map(FileRange::getData) + .toArray(CompletableFuture[]::new)).get(); + + validateVectoredReadResult(ranges, content, 0); + + Mockito.verify(spy, Mockito.times(5)) .readRemote( Mockito.anyLong(), Mockito.any(byte[].class), From 1430b4a07d44163e5afc196a10a54de41a8a493d Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Tue, 27 Jan 2026 04:17:00 -0800 Subject: [PATCH 07/17] Fix checkstyle --- .../services/VectoredReadHandler.java | 7 +- .../azurebfs/services/ITestVectoredRead.java | 535 +++++++++++------- 2 files changed, 321 insertions(+), 221 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VectoredReadHandler.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VectoredReadHandler.java index df011aa465002..72ee1db51794d 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VectoredReadHandler.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VectoredReadHandler.java @@ -205,14 +205,13 @@ private List splitByBufferSize( private List mergeBySpanAndGap( List ranges, int maxSpan) { - - /* Sort ranges by starting offset for span-based merging */ - ranges.sort(Comparator.comparingLong(FileRange::getOffset)); + List sortedRanges = new ArrayList<>(ranges); + sortedRanges.sort(Comparator.comparingLong(FileRange::getOffset)); List out = new ArrayList<>(); CombinedFileRange current = null; - for (FileRange r : ranges) { + for (FileRange r : sortedRanges) { long rOffset = r.getOffset(); long rEnd = rOffset + r.getLength(); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java index 49009b8494681..d862f62b3c932 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java @@ -1,21 +1,3 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package org.apache.hadoop.fs.azurebfs.services; import java.nio.ByteBuffer; @@ -28,6 +10,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.function.IntFunction; import org.junit.jupiter.api.Test; import org.mockito.ArgumentMatchers; @@ -40,6 +23,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest; import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem; +import org.apache.hadoop.fs.azurebfs.enums.VectoredReadStrategy; import org.apache.hadoop.fs.impl.CombinedFileRange; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_VECTORED_READ_STRATEGY; @@ -47,12 +31,14 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.validateVectoredReadResult; public class ITestVectoredRead extends AbstractAbfsIntegrationTest { - private static final int FILE_1_MB = ONE_MB; - private static final int FILE_4_MB = 4 * ONE_MB; - private static final int FILE_8_MB = 8 * ONE_MB; - private static final int FILE_16_MB = 16 * ONE_MB; - private static final int FILE_32_MB = 32 * ONE_MB; - private static final int FILE_100_MB = 100 * ONE_MB; + private static final int DATA_2_MB = 2 * ONE_MB; + private static final int DATA_4_MB = 4 * ONE_MB; + private static final int DATA_8_MB = 8 * ONE_MB; + private static final int DATA_10_MB = 10 * ONE_MB; + private static final int DATA_12_MB = 12 * ONE_MB; + private static final int DATA_16_MB = 16 * ONE_MB; + private static final int DATA_32_MB = 32 * ONE_MB; + private static final int DATA_100_MB = 100 * ONE_MB; private static final int OFFSET_100_B = 100; private static final int OFFSET_15K_B = 15_000; @@ -72,6 +58,14 @@ public class ITestVectoredRead extends AbstractAbfsIntegrationTest { private static final double MB_1_8 = 1.8; private static final double MB_1_9 = 1.9; + private static final double MB_3_8 = 3.8; + private static final double MB_4_0 = 4.0; + private static final double MB_3_2 = 3.2; + private static final double MB_8_0 = 8.0; + private static final double MB_2_0 = 2.0; + private static final double MB_12_0 = 12.0; + private static final double MB_16_0 = 16.0; + private static final int HUGE_OFFSET_1 = 5_856_368; private static final int HUGE_OFFSET_2 = 3_520_861; private static final int HUGE_OFFSET_3 = 8_191_913; @@ -87,46 +81,42 @@ public class ITestVectoredRead extends AbstractAbfsIntegrationTest { private static final int EXEC_THREADS = 3; private static final int SEQ_READ_ITERATIONS = 5; private static final int FUTURE_TIMEOUT_SEC = 5; + public static final int SLEEP_TIME = 10; public ITestVectoredRead() throws Exception { } - /** - * Verifies basic vectored read functionality with two disjoint ranges. - * Ensures data read via readVectored matches data read via a full sequential read. - * Acts as a correctness baseline for simple non-overlapping ranges. - */ @Test public void testDisjointRangesWithVectoredRead() throws Throwable { + int fileSize = ONE_MB; final AzureBlobFileSystem fs = getFileSystem(); - byte[] fileContent = getRandomBytesArray(FILE_1_MB); - Path path = createFileWithContent(fs, methodName.getMethodName(), fileContent); - - List ranges = new ArrayList<>(); - ranges.add(FileRange.createFileRange(OFFSET_100_B, LEN_10K_B)); - ranges.add(FileRange.createFileRange(OFFSET_15K_B, LEN_27K_B)); - - try (FSDataInputStream in = fs.openFile(path).build().get()) { - in.readVectored(ranges, ByteBuffer::allocate); - byte[] full = new byte[FILE_1_MB]; - in.readFully(0, full); - validateVectoredReadResult(ranges, full, 0); + String fileName = methodName.getMethodName() + 1; + byte[] fileContent = getRandomBytesArray(fileSize); + Path testFilePath = createFileWithContent(fs, fileName, fileContent); + + List rangeList = new ArrayList<>(); + rangeList.add(FileRange.createFileRange(OFFSET_100_B, LEN_10K_B)); + rangeList.add(FileRange.createFileRange(OFFSET_15K_B, LEN_27K_B)); + IntFunction allocate = ByteBuffer::allocate; + CompletableFuture builder = fs.openFile(testFilePath) + .build(); + + try (FSDataInputStream in = builder.get()) { + in.readVectored(rangeList, allocate); + byte[] readFullRes = new byte[(int) fileSize]; + in.readFully(0, readFullRes); + // Comparing vectored read results with read fully. + validateVectoredReadResult(rangeList, readFullRes, 0); } } - /** - * Validates that disjoint vectored ranges are coalesced into the minimum - * number of backend reads. - * Ensures that only two remote reads are issued for the given range layout. - */ @Test public void testVectoredReadDisjointRangesExpectTwoBackendReads() throws Exception { - final AzureBlobFileSystem fs = getFileSystem(); - byte[] content = getRandomBytesArray(FILE_16_MB); - Path path = createFileWithContent(fs, methodName.getMethodName(), content); - + String fileName = methodName.getMethodName(); + byte[] fileContent = getRandomBytesArray(DATA_16_MB); + Path testFilePath = createFileWithContent(fs, fileName, fileContent); List ranges = new ArrayList<>(); ranges.add(FileRange.createFileRange(0L, ONE_MB)); ranges.add(FileRange.createFileRange((long) (MB_1_2 * ONE_MB), @@ -137,22 +127,21 @@ public void testVectoredReadDisjointRangesExpectTwoBackendReads() (int) (MB_1_9 * ONE_MB))); ranges.add(FileRange.createFileRange((long) (MB_6_2 * ONE_MB), (int) (MB_1_8 * ONE_MB))); - - try (FSDataInputStream in = fs.openFile(path).build().get()) { - AbfsInputStream spy = - Mockito.spy((AbfsInputStream) in.getWrappedStream()); - - spy.readVectored(ranges, ByteBuffer::allocate); - - CompletableFuture.allOf( - ranges.stream() - .map(FileRange::getData) - .toArray(CompletableFuture[]::new)) - .get(); - - validateVectoredReadResult(ranges, content, 0); - - Mockito.verify(spy, Mockito.times(2)) + IntFunction allocate = ByteBuffer::allocate; + try (FSDataInputStream in = + fs.openFile(testFilePath).build().get()) { + AbfsInputStream abfsIn = (AbfsInputStream) in.getWrappedStream(); + AbfsInputStream spyIn = Mockito.spy(abfsIn); + spyIn.readVectored(ranges, allocate); + CompletableFuture[] futures = + new CompletableFuture[ranges.size()]; + int i = 0; + for (FileRange range : ranges) { + futures[i++] = range.getData(); + } + CompletableFuture.allOf(futures).get(); + validateVectoredReadResult(ranges, fileContent, 0); + Mockito.verify(spyIn, Mockito.times(2)) .readRemote( Mockito.anyLong(), Mockito.any(byte[].class), @@ -162,89 +151,80 @@ public void testVectoredReadDisjointRangesExpectTwoBackendReads() } } - /** - * Simulates failure in vectored read queuing. - * Verifies that the system safely falls back to direct reads - * and still returns correct data. - */ @Test public void testVectoredReadFallsBackToDirectReadWhenQueuingFails() throws Exception { - final AzureBlobFileSystem fs = getFileSystem(); - byte[] content = getRandomBytesArray(FILE_4_MB); - Path path = createFileWithContent(fs, methodName.getMethodName(), content); + String fileName = methodName.getMethodName(); + byte[] fileContent = getRandomBytesArray(DATA_4_MB); + Path testFilePath = createFileWithContent(fs, fileName, fileContent); List ranges = List.of( FileRange.createFileRange(0, ONE_MB), - FileRange.createFileRange(2 * ONE_MB, ONE_MB)); + FileRange.createFileRange(DATA_2_MB, ONE_MB)); + IntFunction allocator = ByteBuffer::allocate; - try (FSDataInputStream in = fs.openFile(path).build().get()) { - AbfsInputStream abfsIn = - (AbfsInputStream) in.getWrappedStream(); + try (FSDataInputStream in = fs.openFile(testFilePath).build().get()) { + AbfsInputStream abfsIn = (AbfsInputStream) in.getWrappedStream(); AbfsInputStream spyIn = Mockito.spy(abfsIn); - VectoredReadHandler realHandler = abfsIn.getVectoredReadHandler(); VectoredReadHandler spyHandler = Mockito.spy(realHandler); - Mockito.doReturn(spyHandler).when(spyIn).getVectoredReadHandler(); Mockito.doReturn(false) .when(spyHandler) .queueVectoredRead( - Mockito.any(), + Mockito.any(AbfsInputStream.class), Mockito.any(CombinedFileRange.class), - ArgumentMatchers.any()); - - spyIn.readVectored(ranges, ByteBuffer::allocate); - - CompletableFuture.allOf( - ranges.stream() - .map(FileRange::getData) - .toArray(CompletableFuture[]::new)) - .get(); - + ArgumentMatchers.>any()); + spyIn.readVectored(ranges, allocator); + CompletableFuture[] futures + = new CompletableFuture[ranges.size()]; + for (int i = 0; i < ranges.size(); i++) { + futures[i] = ranges.get(i).getData(); + } + CompletableFuture.allOf(futures).get(); Mockito.verify(spyHandler, Mockito.atLeastOnce()) .directRead( - Mockito.any(), + Mockito.any(AbfsInputStream.class), Mockito.any(CombinedFileRange.class), - Mockito.any()); + Mockito.eq(allocator)); - validateVectoredReadResult(ranges, content, 0); + validateVectoredReadResult(ranges, fileContent, 0); } } - /** - * Tests vectored reads with multiple small disjoint ranges. - * Ensures correctness when several non-contiguous ranges are requested together. - */ @Test public void testMultipleDisjointRangesWithVectoredRead() throws Throwable { + int fileSize = ONE_MB; final AzureBlobFileSystem fs = getFileSystem(); - byte[] content = getRandomBytesArray(FILE_1_MB); - Path path = createFileWithContent(fs, methodName.getMethodName(), content); + String fileName = methodName.getMethodName() + 1; + byte[] fileContent = getRandomBytesArray(fileSize); + Path testFilePath = createFileWithContent(fs, fileName, fileContent); List ranges = List.of( FileRange.createFileRange(OFFSET_100_B, LEN_10K_B), FileRange.createFileRange(OFFSET_15K_B, LEN_27K_B), FileRange.createFileRange(OFFSET_42K_B, LEN_40K_B)); - - try (FSDataInputStream in = fs.openFile(path).build().get()) { - in.readVectored(ranges, ByteBuffer::allocate); - byte[] full = new byte[FILE_1_MB]; - in.readFully(0, full); - validateVectoredReadResult(ranges, full, 0); + IntFunction allocate = ByteBuffer::allocate; + CompletableFuture builder = fs.openFile(testFilePath) + .build(); + + try (FSDataInputStream in = builder.get()) { + in.readVectored(ranges, allocate); + byte[] readFullRes = new byte[(int) fileSize]; + in.readFully(0, readFullRes); + // Comparing vectored read results with read fully. + validateVectoredReadResult(ranges, readFullRes, 0); } } - /** - * Validates vectored reads against a very large file with widely scattered ranges. - * Ensures correctness and stability under large-file and non-localized access patterns. - */ @Test - public void testVectoredIOHugeFile() throws Throwable { + public void test_045_vectoredIOHugeFile() throws Throwable { + int fileSize = DATA_100_MB; final AzureBlobFileSystem fs = getFileSystem(); - byte[] content = getRandomBytesArray(FILE_100_MB); - Path path = createFileWithContent(fs, methodName.getMethodName(), content); + String fileName = methodName.getMethodName() + 1; + byte[] fileContent = getRandomBytesArray(fileSize); + Path testFilePath = createFileWithContent(fs, fileName, fileContent); List ranges = List.of( FileRange.createFileRange(HUGE_OFFSET_1, HUGE_RANGE), @@ -254,104 +234,194 @@ public void testVectoredIOHugeFile() throws Throwable { FileRange.createFileRange(HUGE_OFFSET_5, HUGE_RANGE), FileRange.createFileRange(HUGE_OFFSET_6, HUGE_RANGE), FileRange.createFileRange(HUGE_OFFSET_7, HUGE_RANGE_LARGE)); + IntFunction allocate = ByteBuffer::allocate; + + CompletableFuture builder = + fs.openFile(testFilePath).build(); + try (FSDataInputStream in = builder.get()) { + in.readVectored(ranges, allocate); + byte[] readFullRes = new byte[(int) fileSize]; + in.readFully(0, readFullRes); + // Comparing vectored read results with read fully. + validateVectoredReadResult(ranges, readFullRes, 0); + } + } - try (FSDataInputStream in = fs.openFile(path).build().get()) { - in.readVectored(ranges, ByteBuffer::allocate); - byte[] full = new byte[FILE_100_MB]; - in.readFully(0, full); - validateVectoredReadResult(ranges, full, 0); + @Test + public void testSimultaneousPrefetchAndVectoredRead() throws Exception { + final AzureBlobFileSystem fs = getFileSystem(); + String fileName = methodName.getMethodName(); + byte[] fileContent = getRandomBytesArray(DATA_16_MB); + Path testFilePath = createFileWithContent(fs, fileName, fileContent); + try (FSDataInputStream in = fs.openFile(testFilePath).build().get()) { + AbfsInputStream abfsIn = (AbfsInputStream) in.getWrappedStream(); + IntFunction allocator = ByteBuffer::allocate; + List vRanges = new ArrayList<>(); + vRanges.add(FileRange.createFileRange(DATA_10_MB, (int) ONE_MB)); + vRanges.add(FileRange.createFileRange(DATA_12_MB, (int) ONE_MB)); + byte[] seqBuffer = new byte[(int) ONE_MB]; + CountDownLatch latch = new CountDownLatch(1); + CompletableFuture vectoredTask = CompletableFuture.runAsync(() -> { + try { + latch.await(); + abfsIn.readVectored(vRanges, allocator); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + CompletableFuture sequentialTask = CompletableFuture.runAsync( + () -> { + try { + latch.await(); + abfsIn.read(0, seqBuffer, 0, (int) ONE_MB); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + latch.countDown(); + CompletableFuture.allOf(vectoredTask, sequentialTask).get(); + CompletableFuture[] vFutures = vRanges.stream() + .map(FileRange::getData) + .toArray(CompletableFuture[]::new); + CompletableFuture.allOf(vFutures).get(); + assertArrayEquals(Arrays.copyOfRange(fileContent, 0, (int) ONE_MB), + seqBuffer, "Sequential read data mismatch"); + validateVectoredReadResult(vRanges, fileContent, 0); } } - /** - * Ensures vectored reads on one file do not interfere with - * sequential reads and readahead on a different file. - * Validates isolation across concurrent streams. - */ @Test public void testConcurrentStreamsOnDifferentFiles() throws Exception { final AzureBlobFileSystem fs = getFileSystem(); - - byte[] content1 = getRandomBytesArray(FILE_16_MB); - byte[] content2 = getRandomBytesArray(FILE_16_MB); - + // Create two distinct files with random content + byte[] content1 = getRandomBytesArray(DATA_16_MB); + byte[] content2 = getRandomBytesArray(DATA_16_MB); Path path1 = createFileWithContent(fs, "file1", content1); Path path2 = createFileWithContent(fs, "file2", content2); + // Open two separate input streams for concurrent access try (FSDataInputStream in1 = fs.openFile(path1).build().get(); FSDataInputStream in2 = fs.openFile(path2).build().get()) { - AbfsInputStream vStream = - (AbfsInputStream) in1.getWrappedStream(); - AbfsInputStream sStream = - (AbfsInputStream) in2.getWrappedStream(); + AbfsInputStream streamVectored = (AbfsInputStream) in1.getWrappedStream(); + AbfsInputStream streamSequential + = (AbfsInputStream) in2.getWrappedStream(); + IntFunction allocator = ByteBuffer::allocate; + // Define non-contiguous ranges for the vectored read on file 1 List ranges = List.of( - FileRange.createFileRange(2 * ONE_MB, ONE_MB), - FileRange.createFileRange(4 * ONE_MB, ONE_MB)); + FileRange.createFileRange(DATA_2_MB, ONE_MB), + FileRange.createFileRange(DATA_4_MB, ONE_MB)); + // Use a latch to ensure both threads start their I/O at the same time CountDownLatch latch = new CountDownLatch(1); + // Thread 1: Perform asynchronous vectored reads on file 1 CompletableFuture vectoredTask = CompletableFuture.runAsync(() -> { try { latch.await(); - vStream.readVectored(ranges, ByteBuffer::allocate); + streamVectored.readVectored(ranges, allocator); } catch (Exception e) { - throw new RuntimeException(e); - } - }); - - CompletableFuture sequentialTask = CompletableFuture.runAsync(() -> { - try { - latch.await(); - for (int i = 0; i < SEQ_READ_ITERATIONS; i++) { - byte[] buf = new byte[ONE_MB]; - sStream.read(i * ONE_MB, buf, 0, ONE_MB); - assertArrayEquals( - Arrays.copyOfRange(content2, - i * ONE_MB, (i + 1) * ONE_MB), - buf); - } - } catch (Exception e) { - throw new RuntimeException(e); + throw new RuntimeException("Vectored read task failed", e); } }); + // Thread 2: Perform multiple sequential reads on file 2 to trigger readahead + CompletableFuture sequentialTask = CompletableFuture.runAsync( + () -> { + try { + latch.await(); + for (int i = 0; i < SEQ_READ_ITERATIONS; i++) { + byte[] tempBuf = new byte[(int) ONE_MB]; + streamSequential.read(i * ONE_MB, tempBuf, 0, (int) ONE_MB); + // Validate data integrity for file 2 immediately + assertArrayEquals(Arrays.copyOfRange(content2, i * (int) ONE_MB, + (i + 1) * (int) ONE_MB), tempBuf, + "Sequential read mismatch in file 2 at block " + i); + } + } catch (Exception e) { + throw new RuntimeException("Sequential read task failed", e); + } + }); + // Trigger simultaneous execution latch.countDown(); + // Wait for both high-level tasks to finish CompletableFuture.allOf(vectoredTask, sequentialTask).get(); - CompletableFuture.allOf( - ranges.stream().map(FileRange::getData) - .toArray(CompletableFuture[]::new)).get(); + // Explicitly wait for the vectored read futures to complete their data transfer + CompletableFuture[] vFutures = ranges.stream() + .map(FileRange::getData) + .toArray(CompletableFuture[]::new); + CompletableFuture.allOf(vFutures).get(); + // Final validation of vectored read content for file 1 validateVectoredReadResult(ranges, content1, 0); } } - /** - * Ensures multiple reads issued while a buffer is in progress - * properly wait and complete once the buffer finishes. - * Covers both sequential and vectored reads during in-flight I/O. - */ + @Test + public void testVectoredReadHitchhikesOnExistingPrefetch() throws Exception { + final AzureBlobFileSystem fs = getFileSystem(); + String fileName = methodName.getMethodName(); + byte[] fileContent = getRandomBytesArray(DATA_8_MB); + Path testFilePath = createFileWithContent(fs, fileName, fileContent); + + try (FSDataInputStream in = fs.openFile(testFilePath).build().get()) { + AbfsInputStream abfsIn = (AbfsInputStream) in.getWrappedStream(); + AbfsInputStream spyIn = Mockito.spy(abfsIn); + + // 1. Trigger a normal read to start the prefetch logic + // Reading the first byte often triggers a larger readahead (e.g., 4MB) + byte[] seqBuf = new byte[1]; + spyIn.read(seqBuf, 0, 1); + + // 2. Immediately queue a vectored read for an offset within that prefetch range + List vRanges = new ArrayList<>(); + // Using 1MB offset, which should be inside the initial readahead buffer + vRanges.add(FileRange.createFileRange(ONE_MB, (int) ONE_MB)); + + IntFunction allocator = ByteBuffer::allocate; + spyIn.readVectored(vRanges, allocator); + + // 3. Wait for the vectored read to complete + vRanges.get(0).getData().get(); + + // 4. Validate Data Integrity + validateVectoredReadResult(vRanges, fileContent, 0); + + // 5. THE CRITICAL VALIDATION: + // Even though we did a manual read and a vectored read, + // there should only be ONE remote call if hitchhiking worked. + Mockito.verify(spyIn, Mockito.atMost(spyIn.getReadAheadQueueDepth())) + .readRemote( + Mockito.anyLong(), + Mockito.any(byte[].class), + Mockito.anyInt(), + Mockito.anyInt(), + Mockito.any()); + } + } + @Test public void testMultipleReadsWhileBufferInProgressEventuallyComplete() throws Exception { - final AzureBlobFileSystem fs = getFileSystem(); - byte[] content = getRandomBytesArray(FILE_8_MB); - Path path = createFileWithContent(fs, methodName.getMethodName(), content); + String fileName = methodName.getMethodName(); + byte[] fileContent = getRandomBytesArray(DATA_8_MB); + Path testFilePath = createFileWithContent(fs, fileName, fileContent); - CountDownLatch block = new CountDownLatch(1); + CountDownLatch blockCompletion = new CountDownLatch(1); - try (FSDataInputStream in = fs.openFile(path).build().get()) { - AbfsInputStream spy = + try (FSDataInputStream in = fs.openFile(testFilePath).build().get()) { + AbfsInputStream spyIn = Mockito.spy((AbfsInputStream) in.getWrappedStream()); - ReadBufferManager rbm = spy.getReadBufferManager(); + ReadBufferManager rbm = spyIn.getReadBufferManager(); - Mockito.doAnswer(inv -> { - block.await(); - return inv.callRealMethod(); - }).when(spy).readRemote( + /* Block completion so buffer stays in inProgressList */ + Mockito.doAnswer(invocation -> { + blockCompletion.await(); + return invocation.callRealMethod(); + }).when(spyIn).readRemote( Mockito.anyLong(), Mockito.any(byte[].class), Mockito.anyInt(), @@ -360,73 +430,104 @@ public void testMultipleReadsWhileBufferInProgressEventuallyComplete() ExecutorService exec = Executors.newFixedThreadPool(EXEC_THREADS); - exec.submit(() -> spy.read(new byte[1], 0, 1)); + /* 1. Start first normal read → creates in-progress buffer */ + Future r1 = exec.submit(() -> { + try { + spyIn.read(new byte[1], 0, 1); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + /* 2. Explicitly validate buffer is in inProgressList */ ReadBuffer inProgress = null; - for (int i = 0; i < LOOKUP_RETRIES; i++) { + for (int i = 0; i r2 = exec.submit(() -> { + try { + spyIn.read(new byte[1], 0, 1); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); - Future vr = exec.submit(() -> - spy.readVectored( - List.of(FileRange.createFileRange(ONE_MB, ONE_MB)), - ByteBuffer::allocate)); + /* 4. Submit vectored read while buffer is in progress */ + List ranges = new ArrayList<>(); + ranges.add(FileRange.createFileRange(ONE_MB, (int) ONE_MB)); + Future vr = exec.submit(() -> { + try { + spyIn.readVectored(ranges, ByteBuffer::allocate); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); - block.countDown(); + /* 5. Allow completion */ + blockCompletion.countDown(); + /* 6. All reads must complete */ + r1.get(FUTURE_TIMEOUT_SEC, TimeUnit.SECONDS); + r2.get(FUTURE_TIMEOUT_SEC, TimeUnit.SECONDS); vr.get(FUTURE_TIMEOUT_SEC, TimeUnit.SECONDS); + ranges.get(0).getData().get(FUTURE_TIMEOUT_SEC, TimeUnit.SECONDS); + + validateVectoredReadResult(ranges, fileContent, 0); + exec.shutdownNow(); } } - /** - * Verifies vectored reads using the throughput-optimized (TPS) strategy. - * Ensures expected backend read count and data correctness under TPS mode. - */ @Test public void testThroughputOptimizedReadVectored() throws Exception { - Configuration conf = getRawConfiguration(); - conf.set(FS_AZURE_VECTORED_READ_STRATEGY, "TPS"); - - try (AzureBlobFileSystem fs = - (AzureBlobFileSystem) FileSystem.newInstance(conf)) { - - byte[] content = getRandomBytesArray(FILE_32_MB); - Path path = createFileWithContent(fs, methodName.getMethodName(), content); - - List ranges = List.of( - FileRange.createFileRange(0L, (int) (3.8 * ONE_MB)), - FileRange.createFileRange((long) (4.0 * ONE_MB), - (int) (3.2 * ONE_MB)), - FileRange.createFileRange((long) (8.0 * ONE_MB), - (int) (2.0 * ONE_MB)), - FileRange.createFileRange((long) (12.0 * ONE_MB), - (int) (4.0 * ONE_MB)), - FileRange.createFileRange((long) (16.0 * ONE_MB), - (int) (2.0 * ONE_MB))); - - try (FSDataInputStream in = fs.openFile(path).build().get()) { - AbfsInputStream spy = - Mockito.spy((AbfsInputStream) in.getWrappedStream()); - - spy.readVectored(ranges, ByteBuffer::allocate); - - CompletableFuture.allOf( - ranges.stream().map(FileRange::getData) - .toArray(CompletableFuture[]::new)).get(); - - validateVectoredReadResult(ranges, content, 0); - - Mockito.verify(spy, Mockito.times(5)) + Configuration configuration = getRawConfiguration(); + configuration.set(FS_AZURE_VECTORED_READ_STRATEGY, VectoredReadStrategy.THROUGHPUT_OPTIMIZED.getName()); + FileSystem fileSystem = FileSystem.newInstance(configuration); + try (AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem) { + String fileName = methodName.getMethodName(); + byte[] fileContent = getRandomBytesArray(DATA_32_MB); + Path testFilePath = createFileWithContent(abfs, fileName, fileContent); + List fileRanges = new ArrayList<>(); + fileRanges.add( + FileRange.createFileRange(0L, (int) (MB_3_8 * ONE_MB))); + fileRanges.add( + FileRange.createFileRange( + (long) (MB_4_0 * ONE_MB), (int) (MB_3_2 * ONE_MB))); + fileRanges.add( + FileRange.createFileRange( + (long) (MB_8_0 * ONE_MB), (int) (MB_2_0 * ONE_MB))); + fileRanges.add( + FileRange.createFileRange( + (long) (MB_12_0 * ONE_MB), (int) (MB_4_0 * ONE_MB))); + fileRanges.add( + FileRange.createFileRange( + (long) (MB_16_0 * ONE_MB), (int) (MB_2_0 * ONE_MB))); + IntFunction allocate = ByteBuffer::allocate; + try (FSDataInputStream in = + abfs.openFile(testFilePath).build().get()) { + AbfsInputStream abfsIn = (AbfsInputStream) in.getWrappedStream(); + AbfsInputStream spyIn = Mockito.spy(abfsIn); + spyIn.readVectored(fileRanges, allocate); + CompletableFuture[] futures = + new CompletableFuture[fileRanges.size()]; + int i = 0; + for (FileRange range : fileRanges) { + futures[i++] = range.getData(); + } + CompletableFuture.allOf(futures).get(); + validateVectoredReadResult(fileRanges, fileContent, 0); + Mockito.verify(spyIn, Mockito.times(5)) .readRemote( Mockito.anyLong(), Mockito.any(byte[].class), From 0927ca1cb0b6fe1fdc3f0897a7239f72e4e113fa Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Tue, 27 Jan 2026 04:24:01 -0800 Subject: [PATCH 08/17] Add explanations --- .../azurebfs/services/ITestVectoredRead.java | 52 ++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java index d862f62b3c932..23e1a73f44119 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java @@ -28,6 +28,7 @@ import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_VECTORED_READ_STRATEGY; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.ONE_MB; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.ZERO; import static org.apache.hadoop.fs.contract.ContractTestUtils.validateVectoredReadResult; public class ITestVectoredRead extends AbstractAbfsIntegrationTest { @@ -86,6 +87,11 @@ public class ITestVectoredRead extends AbstractAbfsIntegrationTest { public ITestVectoredRead() throws Exception { } + /** + * Verifies basic correctness of vectored reads using simple disjoint ranges. + * Compares vectored read output against a full sequential read to ensure + * data integrity is preserved. + */ @Test public void testDisjointRangesWithVectoredRead() throws Throwable { int fileSize = ONE_MB; @@ -110,6 +116,11 @@ public void testDisjointRangesWithVectoredRead() throws Throwable { } } + /** + * Ensures disjoint but mergeable ranges result in fewer backend reads. + * Validates that vectored read coalescing reduces remote calls + * while still returning correct data. + */ @Test public void testVectoredReadDisjointRangesExpectTwoBackendReads() throws Exception { @@ -151,6 +162,11 @@ public void testVectoredReadDisjointRangesExpectTwoBackendReads() } } + /** + * Validates fallback behavior when vectored read queuing fails. + * Ensures the implementation switches to direct reads and still + * completes all requested ranges correctly. + */ @Test public void testVectoredReadFallsBackToDirectReadWhenQueuingFails() throws Exception { @@ -193,6 +209,11 @@ public void testVectoredReadFallsBackToDirectReadWhenQueuingFails() } } + /** + * Tests vectored read correctness with multiple non-contiguous ranges. + * Confirms that all ranges are read correctly even when more than two + * disjoint segments are requested. + */ @Test public void testMultipleDisjointRangesWithVectoredRead() throws Throwable { int fileSize = ONE_MB; @@ -218,6 +239,11 @@ public void testMultipleDisjointRangesWithVectoredRead() throws Throwable { } } + /** + * Exercises vectored reads on a large file with many scattered ranges. + * Ensures correctness and stability of vectored read logic under + * high-offset and large-file conditions. + */ @Test public void test_045_vectoredIOHugeFile() throws Throwable { int fileSize = DATA_100_MB; @@ -247,6 +273,10 @@ public void test_045_vectoredIOHugeFile() throws Throwable { } } + /** + * Verifies that vectored reads and sequential reads can execute concurrently. + * Ensures correct behavior when prefetch and vectored I/O overlap in time. + */ @Test public void testSimultaneousPrefetchAndVectoredRead() throws Exception { final AzureBlobFileSystem fs = getFileSystem(); @@ -290,6 +320,11 @@ public void testSimultaneousPrefetchAndVectoredRead() throws Exception { } } + /** + * Tests concurrent access using separate streams on different files. + * Ensures vectored reads on one file do not interfere with sequential + * reads and readahead on another file. + */ @Test public void testConcurrentStreamsOnDifferentFiles() throws Exception { final AzureBlobFileSystem fs = getFileSystem(); @@ -359,6 +394,11 @@ public void testConcurrentStreamsOnDifferentFiles() throws Exception { } } + /** + * Validates that vectored reads can reuse an in-progress prefetch buffer. + * Ensures no redundant backend read is issued when data is already + * available via readahead. + */ @Test public void testVectoredReadHitchhikesOnExistingPrefetch() throws Exception { final AzureBlobFileSystem fs = getFileSystem(); @@ -387,7 +427,7 @@ public void testVectoredReadHitchhikesOnExistingPrefetch() throws Exception { vRanges.get(0).getData().get(); // 4. Validate Data Integrity - validateVectoredReadResult(vRanges, fileContent, 0); + validateVectoredReadResult(vRanges, fileContent, ZERO); // 5. THE CRITICAL VALIDATION: // Even though we did a manual read and a vectored read, @@ -402,6 +442,11 @@ public void testVectoredReadHitchhikesOnExistingPrefetch() throws Exception { } } + /** + * Ensures multiple reads issued while a buffer is in progress eventually + * complete successfully. Verifies correct synchronization between + * in-progress buffers and new vectored read requests. + */ @Test public void testMultipleReadsWhileBufferInProgressEventuallyComplete() throws Exception { @@ -489,6 +534,11 @@ public void testMultipleReadsWhileBufferInProgressEventuallyComplete() } } + /** + * Verifies vectored read behavior under throughput-optimized strategy. + * Confirms that ranges are split as expected and that the number of + * backend reads matches the throughput-oriented execution model. + */ @Test public void testThroughputOptimizedReadVectored() throws Exception { Configuration configuration = getRawConfiguration(); From 08ca94aa1097e7fead85b587238a6005b938ad64 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Tue, 27 Jan 2026 04:34:27 -0800 Subject: [PATCH 09/17] Add debug log statements --- .../services/VectoredReadHandler.java | 41 +++++++++++++++++-- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VectoredReadHandler.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VectoredReadHandler.java index 72ee1db51794d..1093026d5a95b 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VectoredReadHandler.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VectoredReadHandler.java @@ -26,6 +26,9 @@ import java.util.concurrent.CompletableFuture; import java.util.function.IntFunction; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.fs.FileRange; import org.apache.hadoop.fs.azurebfs.enums.VectoredReadStrategy; @@ -40,6 +43,8 @@ */ class VectoredReadHandler { + private static final Logger LOG = LoggerFactory.getLogger(VectoredReadHandler.class); + /** * Manages allocation, lifecycle, and reuse of read buffers * used during vectored read operations. @@ -62,6 +67,7 @@ class VectoredReadHandler { VectoredReadHandler(ReadBufferManager readBufferManager) { this.readBufferManager = readBufferManager; this.strategy = readBufferManager.getVectoredReadStrategy(); + LOG.debug("VectoredReadHandler initialized with strategy={}", strategy); } /** @@ -83,6 +89,8 @@ public void readVectored( AbfsInputStream stream, List ranges, IntFunction allocator) { + LOG.debug("readVectored invoked: stream={}, ranges={}", + stream, ranges.size()); /* Initialize a future for each logical file range */ for (FileRange r : ranges) { @@ -95,10 +103,14 @@ public void readVectored( ? readBufferManager.getMaxSeekForVectoredReads() : readBufferManager.getMaxSeekForVectoredReadsThroughput(); + LOG.debug("Using maxSpan={} for strategy={}", maxSpan, strategy); + /* Merge logical ranges using a span-first coalescing strategy */ List merged = mergeBySpanAndGap(ranges, maxSpan); + LOG.debug("Merged logical ranges into {} combined ranges", merged.size()); + /* Read buffer size acts as a hard upper bound for physical reads */ int readBufferSize = ReadBufferManager.getReadAheadBlockSize(); @@ -107,14 +119,21 @@ public void readVectored( List chunks = splitByBufferSize(unit, readBufferSize); + LOG.debug("Combined range offset={}, length={} split into {} chunks", + unit.getOffset(), unit.getLength(), chunks.size()); + for (CombinedFileRange chunk : chunks) { try { boolean queued = queueVectoredRead(stream, chunk, allocator); if (!queued) { + LOG.debug("Queue failed; falling back to directRead for offset={}, length={}", + chunk.getOffset(), chunk.getLength()); /* Fall back to direct read if no buffer is available */ directRead(stream, chunk, allocator); } } catch (Exception e) { + LOG.debug("Exception during vectored read chunk offset={}, length={}", + chunk.getOffset(), chunk.getLength(), e); /* Propagate failure to all logical ranges in this unit */ failUnit(chunk, e); } @@ -128,6 +147,8 @@ public void readVectored( */ @VisibleForTesting boolean queueVectoredRead(AbfsInputStream stream, CombinedFileRange unit, IntFunction allocator) { + LOG.debug("queueVectoredRead offset={}, length={}", + unit.getOffset(), unit.getLength()); return getReadBufferManager().queueVectoredRead(stream, unit, stream.getTracingContext(), allocator); } @@ -154,6 +175,8 @@ public ReadBufferManager getReadBufferManager() { private List splitByBufferSize( CombinedFileRange unit, int bufferSize) { + LOG.debug("splitByBufferSize offset={}, length={}, bufferSize={}", + unit.getOffset(), unit.getLength(), bufferSize); List parts = new ArrayList<>(); @@ -186,7 +209,7 @@ private List splitByBufferSize( parts.add(part); start = partEnd; } - + LOG.debug("splitByBufferSize produced {} parts", parts.size()); return parts; } @@ -205,6 +228,8 @@ private List splitByBufferSize( private List mergeBySpanAndGap( List ranges, int maxSpan) { + LOG.debug("mergeBySpanAndGap ranges={}, maxSpan={}", + ranges.size(), maxSpan); List sortedRanges = new ArrayList<>(ranges); sortedRanges.sort(Comparator.comparingLong(FileRange::getOffset)); @@ -238,7 +263,7 @@ private List mergeBySpanAndGap( if (current != null) { out.add(current); } - + LOG.debug("mergeBySpanAndGap produced {} combined ranges", out.size()); return out; } @@ -259,6 +284,8 @@ void fanOut(ReadBuffer buffer, int bytesRead) { if (units == null) { return; } + LOG.debug("fanOut bufferOffset={}, bytesRead={}, units={}", + buffer.getOffset(), bytesRead, units.size()); /* Distribute buffer data to all logical ranges attached to this buffer */ for (CombinedFileRange unit : units) { for (FileRange r : unit.getUnderlying()) { @@ -281,7 +308,11 @@ void fanOut(ReadBuffer buffer, int bytesRead) { } bb.flip(); r.getData().complete(bb); + LOG.debug("fanOut completed logical range offset={}, length={}", + r.getOffset(), r.getLength()); } catch (Exception e) { + LOG.debug("fanOut failed for logical range offset={}", + r.getOffset(), e); /* Propagate failure to the affected logical range */ r.getData().completeExceptionally(e); } @@ -315,7 +346,7 @@ void failBufferFutures(ReadBuffer buffer, Throwable t) { if (units == null) { return; } - + LOG.debug("failBufferFutures bufferOffset={}", buffer.getOffset(), t); /* Propagate failure to all logical ranges attached to this buffer */ for (CombinedFileRange unit : units) { for (FileRange r : unit.getUnderlying()) { @@ -344,6 +375,8 @@ void directRead( AbfsInputStream stream, CombinedFileRange unit, IntFunction allocator) throws IOException { + LOG.debug("directRead offset={}, length={}", + unit.getOffset(), unit.getLength()); /* Read the entire combined range into a temporary buffer */ byte[] tmp = new byte[unit.getLength()]; stream.readRemote(unit.getOffset(), tmp, 0, unit.getLength(), @@ -358,6 +391,8 @@ void directRead( bb.flip(); r.getData().complete(bb); } + LOG.debug("directRead completed offset={}, length={}", + unit.getOffset(), unit.getLength()); } } From 02834ef4277c96d3a76d34c3c895cb86890bfc09 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Wed, 28 Jan 2026 03:20:41 -0800 Subject: [PATCH 10/17] Checkstyle fixes --- .../services/VectoredReadHandler.java | 6 +- .../azurebfs/services/ITestVectoredRead.java | 89 ++++++++++++++++++- 2 files changed, 92 insertions(+), 3 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VectoredReadHandler.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VectoredReadHandler.java index 1093026d5a95b..d7f1ebcc13c02 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VectoredReadHandler.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VectoredReadHandler.java @@ -31,7 +31,9 @@ import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.fs.FileRange; +import org.apache.hadoop.fs.azurebfs.constants.ReadType; import org.apache.hadoop.fs.azurebfs.enums.VectoredReadStrategy; +import org.apache.hadoop.fs.azurebfs.utils.TracingContext; import org.apache.hadoop.fs.impl.CombinedFileRange; /** @@ -149,7 +151,9 @@ public void readVectored( boolean queueVectoredRead(AbfsInputStream stream, CombinedFileRange unit, IntFunction allocator) { LOG.debug("queueVectoredRead offset={}, length={}", unit.getOffset(), unit.getLength()); - return getReadBufferManager().queueVectoredRead(stream, unit, stream.getTracingContext(), allocator); + TracingContext tracingContext = stream.getTracingContext(); + tracingContext.setReadType(ReadType.VECTORED_READ); + return getReadBufferManager().queueVectoredRead(stream, unit, tracingContext, allocator); } /** diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java index 23e1a73f44119..55edcd82906d0 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java @@ -1,9 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.fs.azurebfs.services; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Random; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; @@ -27,6 +46,7 @@ import org.apache.hadoop.fs.impl.CombinedFileRange; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_VECTORED_READ_STRATEGY; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.ONE_KB; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.ONE_MB; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.ZERO; import static org.apache.hadoop.fs.contract.ContractTestUtils.validateVectoredReadResult; @@ -245,7 +265,7 @@ public void testMultipleDisjointRangesWithVectoredRead() throws Throwable { * high-offset and large-file conditions. */ @Test - public void test_045_vectoredIOHugeFile() throws Throwable { + public void testVectoredIOHugeFile() throws Throwable { int fileSize = DATA_100_MB; final AzureBlobFileSystem fs = getFileSystem(); String fileName = methodName.getMethodName() + 1; @@ -486,7 +506,7 @@ public void testMultipleReadsWhileBufferInProgressEventuallyComplete() /* 2. Explicitly validate buffer is in inProgressList */ ReadBuffer inProgress = null; - for (int i = 0; i offsets = new ArrayList<>(); + Random rnd = new Random(12345L); + for (int i = 0; i < readCount; i++) { + long offset = Math.abs(rnd.nextLong()) + % (fileSize - readSize); + offsets.add(offset); + } + // Build vectored ranges from the same offsets + List vectoredRanges = new ArrayList<>(); + for (long offset : offsets) { + vectoredRanges.add( + FileRange.createFileRange(offset, readSize)); + } + try (FSDataInputStream in = fs.openFile(testPath).build().get()) { + /* ---------------------------------------------------- + * Phase 1: Random non-vectored reads + * ---------------------------------------------------- */ + byte[] buffer = new byte[readSize]; + long nonVectoredStartNs = System.nanoTime(); + for (long offset : offsets) { + in.seek(offset); + in.read(buffer, 0, readSize); + } + long nonVectoredTimeNs = + System.nanoTime() - nonVectoredStartNs; + /* ---------------------------------------------------- + * Phase 2: Vectored reads (after all non-vectored reads) + * ---------------------------------------------------- */ + long vectoredStartNs = System.nanoTime(); + in.readVectored(vectoredRanges, ByteBuffer::allocate); + CompletableFuture.allOf( + vectoredRanges.stream() + .map(FileRange::getData) + .toArray(CompletableFuture[]::new)) + .get(); + long vectoredTimeNs = + System.nanoTime() - vectoredStartNs; + assertTrue(vectoredTimeNs < nonVectoredTimeNs, + String.format("Vectored read time %d ns not faster than " + + "non-vectored time %d ns", + vectoredTimeNs, nonVectoredTimeNs)); + } + } } From 1ea571e55d3d3765fd34aa193776afc5ad4f8746 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Thu, 12 Mar 2026 05:31:06 -0700 Subject: [PATCH 11/17] fix null issue --- .../hadoop/fs/azurebfs/services/ReadBufferManagerV1.java | 4 ++-- .../hadoop/fs/azurebfs/services/ReadBufferManagerV2.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV1.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV1.java index 2cf5e69edae9d..fb33327e0d9e3 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV1.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV1.java @@ -200,7 +200,7 @@ boolean queueVectoredRead(AbfsInputStream stream, */ if (isAlreadyQueued(stream, unit.getOffset())) { ReadBuffer existing = findQueuedBuffer(stream, unit.getOffset()); - if (existing != null && stream.getETag() + if (existing != null && existing.getStream().getETag() != null && stream.getETag() .equals(existing.getStream().getETag())) { long end = existing.getOffset() + ( existing.getStatus() == ReadBufferStatus.AVAILABLE @@ -241,7 +241,7 @@ boolean queueVectoredRead(AbfsInputStream stream, buffer.initVectoredUnits(); buffer.addVectoredUnit(unit); buffer.setAllocator(allocator); - buffer.setTracingContext(tracingContext); + buffer.setTracingContext(readAheadTracingContext); /* * Perform a final free-list check before consuming pooled memory to * ensure buffer availability. diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV2.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV2.java index 8614993ac73ee..f1881225b4cd2 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV2.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV2.java @@ -387,7 +387,7 @@ boolean queueVectoredRead(AbfsInputStream stream, */ if (isAlreadyQueued(stream.getETag(), unit.getOffset())) { ReadBuffer existing = findQueuedBuffer(stream, unit.getOffset()); - if (existing != null && stream.getETag() + if (existing != null && existing.getStream().getETag() != null && stream.getETag() .equals(existing.getStream().getETag())) { long end = existing.getOffset() + ( existing.getStatus() == ReadBufferStatus.AVAILABLE @@ -428,7 +428,7 @@ boolean queueVectoredRead(AbfsInputStream stream, buffer.initVectoredUnits(); buffer.addVectoredUnit(unit); buffer.setAllocator(allocator); - buffer.setTracingContext(tracingContext); + buffer.setTracingContext(readAheadTracingContext); /* * Perform a final free-list check before consuming pooled memory to * ensure buffer availability. From 279e7f42f17521befec93f5471ff861acd8c439f Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Mon, 30 Mar 2026 05:39:23 -0700 Subject: [PATCH 12/17] Fix vectored read --- .../fs/azurebfs/constants/ReadType.java | 5 + .../fs/azurebfs/services/AbfsInputStream.java | 5 +- .../fs/azurebfs/services/ReadBuffer.java | 8 +- .../azurebfs/services/ReadBufferManager.java | 4 + .../services/ReadBufferManagerV1.java | 95 +++- .../services/ReadBufferManagerV2.java | 96 +++- .../services/VectoredReadHandler.java | 468 +++++++++++++----- .../utils/ITestAbfsDriverLatency.java | 217 ++++++++ 8 files changed, 749 insertions(+), 149 deletions(-) create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/ITestAbfsDriverLatency.java diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ReadType.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ReadType.java index 82355842d8a4e..056a0aba4988c 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ReadType.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ReadType.java @@ -53,6 +53,11 @@ public enum ReadType { * Used to coalesce and execute non-contiguous reads efficiently. */ VECTORED_READ("VR"), + /** + * Performs a vectored direct read by fetching multiple non-contiguous + * ranges in a single operation. + */ + VECTORED_DIRECT_READ("VDR"), /** * Reads from Random Input Stream with read ahead up to readAheadRange */ diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java index 4f96bbb1b5420..5d765d84ab6de 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java @@ -818,7 +818,10 @@ public synchronized void unbuffer() { @Override public boolean hasCapability(String capability) { - return StreamCapabilities.UNBUFFER.equals(toLowerCase(capability)); + return switch (toLowerCase(capability)) { + case StreamCapabilities.UNBUFFER, StreamCapabilities.VECTOREDIO -> true; + default -> false; + }; } /** diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBuffer.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBuffer.java index fc938ae2da1f0..b5809d97e07c0 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBuffer.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBuffer.java @@ -56,7 +56,7 @@ public class ReadBuffer { private AtomicInteger refCount = new AtomicInteger(0); private BufferType bufferType = BufferType.NORMAL; // list of combined file ranges for vectored read. - private List vectoredUnits; + private List vectoredUnits = new ArrayList<>(); // Allocator used for vectored fan-out; captured at queue time */ private IntFunction allocator; // Tracks whether fanOut has already been executed @@ -213,12 +213,6 @@ public boolean isFullyConsumed() { return isFirstByteConsumed() && isLastByteConsumed(); } - void initVectoredUnits() { - if (vectoredUnits == null) { - vectoredUnits = new ArrayList<>(); - } - } - void addVectoredUnit(CombinedFileRange u) { vectoredUnits.add(u); } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManager.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManager.java index 86b2bab038b3a..1151f9b5d363b 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManager.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManager.java @@ -379,6 +379,10 @@ void handleVectoredCompletion( getVectoredReadHandler().fanOut(buffer, bytesActuallyRead); } } else { + LOGGER.debug( + "Handling vectored completion for buffer with path: {}, offset: {}, length: {}, result: {}, bytesActuallyRead: {}", + buffer.getPath(), buffer.getOffset(), buffer.getRequestedLength(), + result, bytesActuallyRead); throw new IOException( "Vectored read failed for path: " + buffer.getPath() + ", status=" + buffer.getStatus()); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV1.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV1.java index fb33327e0d9e3..cf2ce1548892e 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV1.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV1.java @@ -194,26 +194,41 @@ boolean queueVectoredRead(AbfsInputStream stream, readAheadTracingContext.setReadType(ReadType.VECTORED_READ); synchronized (this) { - /* - * Attempt to hitchhike on an existing in-progress physical read if it - * covers the requested logical range completely. - */ if (isAlreadyQueued(stream, unit.getOffset())) { ReadBuffer existing = findQueuedBuffer(stream, unit.getOffset()); - if (existing != null && existing.getStream().getETag() != null && stream.getETag() - .equals(existing.getStream().getETag())) { + if (existing != null && existing.getStream().getETag() != null + && stream.getETag().equals(existing.getStream().getETag())) { + /* + * For AVAILABLE buffers use actual bytes read (getLength()) for + * coverage check. For READING_IN_PROGRESS buffers use + * requestedLength as an estimate — the short-read guard will be + * applied later in doneReading before dispatching completion. + */ long end = existing.getOffset() + ( existing.getStatus() == ReadBufferStatus.AVAILABLE ? existing.getLength() : existing.getRequestedLength()); if (end >= unit.getOffset() + unit.getLength()) { - existing.initVectoredUnits(); existing.addVectoredUnit(unit); existing.setAllocator(allocator); if (existing.getStatus() == ReadBufferStatus.AVAILABLE) { - handleVectoredCompletion(existing, existing.getStatus(), + /* + * Buffer is already AVAILABLE. Trigger completion immediately. + * Use getLength() (actual bytes) for coverage — redundant here + * since the outer check already used getLength() for AVAILABLE, + * but kept explicit for clarity. + */ + LOGGER.debug("Hitchhiking onto AVAILABLE buffer {}, length {}", + existing, existing.getLength()); + handleVectoredCompletion(existing, + existing.getStatus(), existing.getLength()); } + /* + * For READING_IN_PROGRESS: unit is attached and will be + * completed in doneReading once actual bytes are known. + * Short-read safety is enforced there via per-unit coverage check. + */ return true; } } @@ -238,7 +253,6 @@ boolean queueVectoredRead(AbfsInputStream stream, buffer.setBufferType(BufferType.VECTORED); buffer.setStatus(ReadBufferStatus.NOT_AVAILABLE); buffer.setLatch(new CountDownLatch(1)); - buffer.initVectoredUnits(); buffer.addVectoredUnit(unit); buffer.setAllocator(allocator); buffer.setTracingContext(readAheadTracingContext); @@ -333,12 +347,69 @@ public ReadBuffer getNextBlockToRead() throws InterruptedException { public void doneReading(final ReadBuffer buffer, final ReadBufferStatus result, final int bytesActuallyRead) { if (LOGGER.isTraceEnabled()) { LOGGER.trace("ReadBufferWorker completed read file {} for offset {} outcome {} bytes {}", - buffer.getStream().getPath(), buffer.getOffset(), result, bytesActuallyRead); + buffer.getStream().getPath(), buffer.getOffset(), result, bytesActuallyRead); } + List vectoredUnits = buffer.getVectoredUnits(); - if (buffer.getBufferType() == BufferType.VECTORED || (vectoredUnits != null && !vectoredUnits.isEmpty())) { - handleVectoredCompletion(buffer, result, bytesActuallyRead); + if (result == ReadBufferStatus.AVAILABLE + && (buffer.getBufferType() == BufferType.VECTORED && !vectoredUnits.isEmpty())) { + + /* + * Set length BEFORE handling vectored completion so that any + * hitchhiked units that call existing.getLength() see the correct + * actual value rather than 0. + */ + buffer.setLength(bytesActuallyRead); + + /* + * Guard against short reads: units hitchhiked while buffer was + * READING_IN_PROGRESS used requestedLength as coverage estimate. + * Now that actual bytes are known, fail any units not fully covered + * so their callers are not left hanging on the CompletableFuture. + */ + long actualEnd = buffer.getOffset() + bytesActuallyRead; + + /* + * Fast path: check if any unit exceeds actual bytes read before + * doing expensive stream/collect. Short reads are rare so this + * avoids unnecessary allocations in the common case. + */ + boolean hasUncovered = false; + for (CombinedFileRange u : vectoredUnits) { + if ((u.getOffset() + u.getLength()) > actualEnd) { + hasUncovered = true; + break; + } + } + + if (hasUncovered) { + /* + * Short read detected — fail uncovered units explicitly so callers + * are not left hanging on their CompletableFuture. + */ + Iterator it = vectoredUnits.iterator(); + while (it.hasNext()) { + CombinedFileRange u = it.next(); + if ((u.getOffset() + u.getLength()) > actualEnd) { + it.remove(); + LOGGER.debug( + "Vectored unit not covered by actual bytes read: unitEnd={} actualEnd={}, failing unit", + (u.getOffset() + u.getLength()), actualEnd); + u.getData().completeExceptionally(new IOException( + "Vectored read unit not covered by actual bytes read: " + + "unitEnd=" + (u.getOffset() + u.getLength()) + + " actualEnd=" + actualEnd)); + } + } + } + + if (!vectoredUnits.isEmpty()) { + LOGGER.debug("Entering vectored read completion with buffer {}, result {}, bytesActuallyRead {}", + buffer, result, bytesActuallyRead); + handleVectoredCompletion(buffer, result, bytesActuallyRead); + } } + synchronized (this) { // If this buffer has already been purged during // close of InputStream then we don't update the lists. diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV2.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV2.java index f1881225b4cd2..8e6ff6f718627 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV2.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV2.java @@ -387,20 +387,33 @@ boolean queueVectoredRead(AbfsInputStream stream, */ if (isAlreadyQueued(stream.getETag(), unit.getOffset())) { ReadBuffer existing = findQueuedBuffer(stream, unit.getOffset()); - if (existing != null && existing.getStream().getETag() != null && stream.getETag() + if (existing != null && existing.getStream().getETag() != null && stream.getETag() .equals(existing.getStream().getETag())) { long end = existing.getOffset() + ( existing.getStatus() == ReadBufferStatus.AVAILABLE ? existing.getLength() : existing.getRequestedLength()); if (end >= unit.getOffset() + unit.getLength()) { - existing.initVectoredUnits(); existing.addVectoredUnit(unit); existing.setAllocator(allocator); if (existing.getStatus() == ReadBufferStatus.AVAILABLE) { - handleVectoredCompletion(existing, existing.getStatus(), + /* + * Buffer is already AVAILABLE. Trigger completion immediately. + * Use getLength() (actual bytes) for coverage — redundant here + * since the outer check already used getLength() for AVAILABLE, + * but kept explicit for clarity. + */ + LOGGER.debug("Hitchhiking onto AVAILABLE buffer {}, length {}", + existing, existing.getLength()); + handleVectoredCompletion(existing, + existing.getStatus(), existing.getLength()); } + /* + * For READING_IN_PROGRESS: unit is attached and will be + * completed in doneReading once actual bytes are known. + * Short-read safety is enforced there via per-unit coverage check. + */ return true; } } @@ -425,7 +438,6 @@ boolean queueVectoredRead(AbfsInputStream stream, buffer.setBufferType(BufferType.VECTORED); buffer.setStatus(ReadBufferStatus.NOT_AVAILABLE); buffer.setLatch(new CountDownLatch(1)); - buffer.initVectoredUnits(); buffer.addVectoredUnit(unit); buffer.setAllocator(allocator); buffer.setTracingContext(readAheadTracingContext); @@ -543,14 +555,64 @@ public ReadBuffer getNextBlockToRead() throws InterruptedException { public void doneReading(final ReadBuffer buffer, final ReadBufferStatus result, final int bytesActuallyRead) { - printTraceLog( - "ReadBufferWorker completed prefetch for file: {} with eTag: {}, for offset: {}, queued by stream: {}, with status: {} and bytes read: {}", - buffer.getPath(), buffer.getETag(), buffer.getOffset(), - buffer.getStream().hashCode(), result, bytesActuallyRead); + if (LOGGER.isTraceEnabled()) { + LOGGER.trace("ReadBufferWorker completed read file {} for offset {} outcome {} bytes {}", + buffer.getStream().getPath(), buffer.getOffset(), result, bytesActuallyRead); + } List vectoredUnits = buffer.getVectoredUnits(); - if (buffer.getBufferType() == BufferType.VECTORED || (vectoredUnits != null - && !vectoredUnits.isEmpty())) { - handleVectoredCompletion(buffer, result, bytesActuallyRead); + if (result == ReadBufferStatus.AVAILABLE + && (buffer.getBufferType() == BufferType.VECTORED && !vectoredUnits.isEmpty())) { + + /* + * Set length BEFORE handling vectored completion so that any + * hitchhiked units that call existing.getLength() see the correct + * actual value rather than 0. + */ + buffer.setLength(bytesActuallyRead); + /* + * Guard against short reads: units hitchhiked while buffer was + * READING_IN_PROGRESS used requestedLength as coverage estimate. + * Now that actual bytes are known, fail any units not fully covered + * so their callers are not left hanging on the CompletableFuture. + */ + long actualEnd = buffer.getOffset() + bytesActuallyRead; + /* + * Fast path: check if any unit exceeds actual bytes read before + * doing expensive stream/collect. Short reads are rare so this + * avoids unnecessary allocations in the common case. + */ + boolean hasUncovered = false; + for (CombinedFileRange u : vectoredUnits) { + if ((u.getOffset() + u.getLength()) > actualEnd) { + hasUncovered = true; + break; + } + } + if (hasUncovered) { + /* + * Short read detected — fail uncovered units explicitly so callers + * are not left hanging on their CompletableFuture. + */ + Iterator it = vectoredUnits.iterator(); + while (it.hasNext()) { + CombinedFileRange u = it.next(); + if ((u.getOffset() + u.getLength()) > actualEnd) { + it.remove(); + LOGGER.debug( + "Vectored unit not covered by actual bytes read: unitEnd={} actualEnd={}, failing unit", + (u.getOffset() + u.getLength()), actualEnd); + u.getData().completeExceptionally(new IOException( + "Vectored read unit not covered by actual bytes read: " + + "unitEnd=" + (u.getOffset() + u.getLength()) + + " actualEnd=" + actualEnd)); + } + } + } + if (!vectoredUnits.isEmpty()) { + LOGGER.debug("Entering vectored read completion with buffer {}, result {}, bytesActuallyRead {}", + buffer, result, bytesActuallyRead); + handleVectoredCompletion(buffer, result, bytesActuallyRead); + } } synchronized (this) { // If this buffer has already been purged during @@ -559,16 +621,18 @@ public void doneReading(final ReadBuffer buffer, getInProgressList().remove(buffer); if (result == ReadBufferStatus.AVAILABLE && bytesActuallyRead > 0) { // Successful read, so update the buffer status and length - buffer.setStatus(ReadBufferStatus.AVAILABLE); - buffer.setLength(bytesActuallyRead); + if (!buffer.isFanOutDone()) { + buffer.setStatus(ReadBufferStatus.AVAILABLE); + buffer.setLength(bytesActuallyRead); + } } else { // Failed read, reuse buffer for next read, this buffer will be // evicted later based on eviction policy. pushToFreeList(buffer.getBufferindex()); + buffer.setStatus(result); } // completed list also contains FAILED read buffers // for sending exception message to clients. - buffer.setStatus(result); buffer.setTimeStamp(currentTimeMillis()); getCompletedReadList().add(buffer); } @@ -627,7 +691,7 @@ private ReadBuffer getFromList(final Collection list, final String eTag, final long requestedOffset) { for (ReadBuffer buffer : list) { - if (eTag.equals(buffer.getETag())) { + if (eTag != null && eTag.equals(buffer.getETag())) { if (buffer.getStatus() == ReadBufferStatus.AVAILABLE && requestedOffset >= buffer.getOffset() && requestedOffset < buffer.getOffset() + buffer.getLength()) { @@ -870,7 +934,7 @@ private ReadBuffer getBufferFromCompletedQueue(final String eTag, for (ReadBuffer buffer : getCompletedReadList()) { // Buffer is returned if the requestedOffset is at or above buffer's // offset but less than buffer's length or the actual requestedLength - if (eTag.equals(buffer.getETag()) + if (eTag != null && eTag.equals(buffer.getETag()) && (requestedOffset >= buffer.getOffset()) && ((requestedOffset < buffer.getOffset() + buffer.getLength()) || (requestedOffset diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VectoredReadHandler.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VectoredReadHandler.java index d7f1ebcc13c02..34c161eb903d2 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VectoredReadHandler.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VectoredReadHandler.java @@ -24,6 +24,8 @@ import java.util.Comparator; import java.util.List; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.IntFunction; import org.slf4j.Logger; @@ -58,6 +60,20 @@ class VectoredReadHandler { */ private final VectoredReadStrategy strategy; + /** + * Shared reconstruction buffers for ranges that span multiple chunks. + * Keyed by the original FileRange instance. + */ + private final ConcurrentHashMap partialBuffers = + new ConcurrentHashMap<>(); + + /** + * Tracks remaining bytes to be received for each logical range. + * Keyed by the original FileRange instance. + */ + private final ConcurrentHashMap pendingBytes = + new ConcurrentHashMap<>(); + /** * Creates a VectoredReadHandler using the provided ReadBufferManager. * The vectored read strategy is obtained from the manager to ensure @@ -91,8 +107,8 @@ public void readVectored( AbfsInputStream stream, List ranges, IntFunction allocator) { - LOG.debug("readVectored invoked: stream={}, ranges={}", - stream, ranges.size()); + LOG.debug("readVectored invoked: path={}, rangeCount={}", + stream.getPath(), ranges.size()); /* Initialize a future for each logical file range */ for (FileRange r : ranges) { @@ -105,38 +121,38 @@ public void readVectored( ? readBufferManager.getMaxSeekForVectoredReads() : readBufferManager.getMaxSeekForVectoredReadsThroughput(); - LOG.debug("Using maxSpan={} for strategy={}", maxSpan, strategy); + LOG.debug("readVectored: path={}, strategy={}, maxSpan={}", + stream.getPath(), strategy, maxSpan); /* Merge logical ranges using a span-first coalescing strategy */ - List merged = - mergeBySpanAndGap(ranges, maxSpan); + List merged = mergeBySpanAndGap(ranges, maxSpan); - LOG.debug("Merged logical ranges into {} combined ranges", merged.size()); + LOG.debug("readVectored: path={}, mergedRangeCount={}", + stream.getPath(), merged.size()); /* Read buffer size acts as a hard upper bound for physical reads */ int readBufferSize = ReadBufferManager.getReadAheadBlockSize(); /* Split merged ranges into buffer-sized chunks and queue each for read */ for (CombinedFileRange unit : merged) { - List chunks = - splitByBufferSize(unit, readBufferSize); + List chunks = splitByBufferSize(unit, readBufferSize); - LOG.debug("Combined range offset={}, length={} split into {} chunks", - unit.getOffset(), unit.getLength(), chunks.size()); + LOG.debug("readVectored: path={}, mergedOffset={}, mergedLength={}, chunkCount={}", + stream.getPath(), unit.getOffset(), unit.getLength(), chunks.size()); for (CombinedFileRange chunk : chunks) { try { boolean queued = queueVectoredRead(stream, chunk, allocator); if (!queued) { - LOG.debug("Queue failed; falling back to directRead for offset={}, length={}", - chunk.getOffset(), chunk.getLength()); - /* Fall back to direct read if no buffer is available */ + LOG.debug("readVectored: buffer pool exhausted, falling back to directRead:" + + " path={}, offset={}, length={}", + stream.getPath(), chunk.getOffset(), chunk.getLength()); directRead(stream, chunk, allocator); } } catch (Exception e) { - LOG.debug("Exception during vectored read chunk offset={}, length={}", - chunk.getOffset(), chunk.getLength(), e); - /* Propagate failure to all logical ranges in this unit */ + LOG.warn("readVectored: chunk read failed, failing underlying ranges:" + + " path={}, offset={}, length={}", + stream.getPath(), chunk.getOffset(), chunk.getLength(), e); failUnit(chunk, e); } } @@ -145,12 +161,14 @@ public void readVectored( /** * Queues a vectored read request with the buffer manager. + * * @return true if successfully queued, false if the queue is full and fallback is required. */ @VisibleForTesting - boolean queueVectoredRead(AbfsInputStream stream, CombinedFileRange unit, IntFunction allocator) { - LOG.debug("queueVectoredRead offset={}, length={}", - unit.getOffset(), unit.getLength()); + boolean queueVectoredRead(AbfsInputStream stream, CombinedFileRange unit, + IntFunction allocator) { + LOG.debug("queueVectoredRead: path={}, offset={}, length={}", + stream.getPath(), unit.getOffset(), unit.getLength()); TracingContext tracingContext = stream.getTracingContext(); tracingContext.setReadType(ReadType.VECTORED_READ); return getReadBufferManager().queueVectoredRead(stream, unit, tracingContext, allocator); @@ -158,6 +176,7 @@ boolean queueVectoredRead(AbfsInputStream stream, CombinedFileRange unit, IntFun /** * Accesses the shared manager responsible for coordinating asynchronous read buffers. + * * @return the {@link ReadBufferManager} instance. */ public ReadBufferManager getReadBufferManager() { @@ -165,59 +184,52 @@ public ReadBufferManager getReadBufferManager() { } /** - * Split a merged logical range into buffer-sized physical read units. + * Splits a merged logical {@link CombinedFileRange} into smaller + * buffer-sized physical read units. * - *

The input {@link CombinedFileRange} may span more bytes than the - * configured read buffer size. This method divides it into multiple - * {@link CombinedFileRange} instances, each limited to {@code bufferSize} - * and containing only the logical {@link FileRange}s that intersect its span.

+ *

Each resulting unit will have a maximum size equal to the provided + * {@code bufferSize}. Any handling of multi-chunk ranges or reassembly + * of underlying {@link FileRange} data is delegated to the fan-out logic.

* - * @param unit merged logical range to be split - * @param bufferSize maximum size (in bytes) of each physical read unit + * @param unit the combined logical range to be split + * @param bufferSize the maximum size (in bytes) of each physical read unit * @return a list of buffer-sized {@link CombinedFileRange} instances */ private List splitByBufferSize( CombinedFileRange unit, int bufferSize) { - LOG.debug("splitByBufferSize offset={}, length={}, bufferSize={}", + LOG.debug("splitByBufferSize: offset={}, length={}, bufferSize={}", unit.getOffset(), unit.getLength(), bufferSize); List parts = new ArrayList<>(); - long unitStart = unit.getOffset(); long unitEnd = unitStart + unit.getLength(); long start = unitStart; - /* Create buffer-sized slices covering the merged unit span */ while (start < unitEnd) { long partEnd = Math.min(start + bufferSize, unitEnd); - /* Initialize a physical read unit for the span [start, partEnd) */ CombinedFileRange part = - new CombinedFileRange(start, partEnd, - unit.getUnderlying().get(0)); - - /* Remove the constructor-added range and attach only overlapping ranges */ + new CombinedFileRange(start, partEnd, unit.getUnderlying().get(0)); part.getUnderlying().clear(); - /* Attach logical ranges that intersect this physical read unit */ for (FileRange r : unit.getUnderlying()) { long rStart = r.getOffset(); long rEnd = rStart + r.getLength(); - if (rEnd > start && rStart < partEnd) { part.getUnderlying().add(r); } } - parts.add(part); start = partEnd; } - LOG.debug("splitByBufferSize produced {} parts", parts.size()); + + LOG.debug("splitByBufferSize: offset={}, produced {} parts", + unit.getOffset(), parts.size()); return parts; } - /** + /** * Merge logical {@link FileRange}s into {@link CombinedFileRange}s using a * span-first coalescing strategy. * @@ -232,8 +244,8 @@ private List splitByBufferSize( private List mergeBySpanAndGap( List ranges, int maxSpan) { - LOG.debug("mergeBySpanAndGap ranges={}, maxSpan={}", - ranges.size(), maxSpan); + LOG.debug("mergeBySpanAndGap: rangeCount={}, maxSpan={}", ranges.size(), maxSpan); + List sortedRanges = new ArrayList<>(ranges); sortedRanges.sort(Comparator.comparingLong(FileRange::getOffset)); @@ -267,93 +279,211 @@ private List mergeBySpanAndGap( if (current != null) { out.add(current); } - LOG.debug("mergeBySpanAndGap produced {} combined ranges", out.size()); + + LOG.debug("mergeBySpanAndGap: produced {} combined ranges", out.size()); return out; } - /** - * Fan out data from a completed physical read buffer to all logical - * {@link FileRange}s associated with the vectored read. + * Distributes data from a physical read buffer into the corresponding + * logical {@link FileRange}s. + * + *

This method performs a "fan-out" operation where a single physical + * read (represented by {@link ReadBuffer}) may contain data for multiple + * logical ranges. The relevant portions are copied into per-range buffers + * and completed once fully populated.

+ * + *

Partial reads are accumulated using {@code partialBuffers} and + * {@code pendingBytes}. A range is only completed when all expected + * bytes have been received.

* - *

For each logical range, the corresponding slice of data is copied - * into a newly allocated {@link ByteBuffer} and the range's future is - * completed. Ranges whose futures are cancelled are skipped.

+ *

Thread safety: + *

    + *
  • Each logical range buffer is synchronized independently
    • + *
  • Writes use {@code System.arraycopy} directly into the backing array + * to avoid shared {@link ByteBuffer} position mutation
    • + *
+ *

* - * @param buffer completed read buffer containing the physical data - * @param bytesRead number of bytes actually read into the buffer + * @param buffer the physical read buffer containing merged data + * @param bytesRead number of valid bytes in the buffer */ void fanOut(ReadBuffer buffer, int bytesRead) { + LOG.debug("fanOut: path={}, bufferOffset={}, bytesRead={}", + buffer.getPath(), buffer.getOffset(), bytesRead); + List units = buffer.getVectoredUnits(); if (units == null) { + LOG.warn("fanOut: no vectored units found for path={}, offset={}", + buffer.getPath(), buffer.getOffset()); return; } - LOG.debug("fanOut bufferOffset={}, bytesRead={}, units={}", - buffer.getOffset(), bytesRead, units.size()); - /* Distribute buffer data to all logical ranges attached to this buffer */ + + long bufferStart = buffer.getOffset(); + long bufferEnd = bufferStart + bytesRead; + + /* Iterate over all combined logical units mapped to this buffer */ for (CombinedFileRange unit : units) { + /* Each unit may contain multiple logical FileRanges */ for (FileRange r : unit.getUnderlying()) { - /* Skip ranges whose futures have been cancelled */ - if (r.getData().isCancelled()) { + CompletableFuture future = r.getData(); + + /* Skip already completed or cancelled ranges */ + if (future.isCancelled()) { + LOG.debug("fanOut: range cancelled, cleaning up: path={}, rangeOffset={}", + buffer.getPath(), r.getOffset()); + RangeKey key = new RangeKey(r); + partialBuffers.remove(key); + pendingBytes.remove(key); + continue; + } + if (future.isDone()) { continue; } + try { - /* Compute offset of the logical range relative to the buffer */ - long rel = r.getOffset() - buffer.getOffset(); - /* Determine how many bytes are available for this range */ - int available = - (int) Math.max( - 0, - Math.min(r.getLength(), bytesRead - rel)); - /* Allocate output buffer and copy available data */ - ByteBuffer bb = buffer.getAllocator().apply(r.getLength()); - if (available > 0) { - bb.put(buffer.getBuffer(), (int) rel, available); + long rangeStart = r.getOffset(); + long rangeEnd = rangeStart + r.getLength(); + + /* Compute overlap between buffer and logical range */ + long overlapStart = Math.max(rangeStart, bufferStart); + long overlapEnd = Math.min(rangeEnd, bufferEnd); + + /* No overlap nothing to copy */ + if (overlapStart >= overlapEnd) { + LOG.debug("fanOut: no overlap for path={}, rangeOffset={}, bufferOffset={}", + buffer.getPath(), r.getOffset(), bufferStart); + continue; + } + + int srcOffset = (int) (overlapStart - bufferStart); + int destOffset = (int) (overlapStart - rangeStart); + int length = (int)(overlapEnd - overlapStart); + + LOG.debug("fanOut: copying path={}, rangeOffset={}, rangeLength={}," + + " bufferOffset={}, srcOffset={}, destOffset={}, length={}", + buffer.getPath(), r.getOffset(), r.getLength(), + bufferStart, srcOffset, destOffset, length); + + RangeKey key = new RangeKey(r); + + /* Allocate or reuse the full buffer for this logical range */ + ByteBuffer fullBuf = partialBuffers.computeIfAbsent( + key, k -> buffer.getAllocator().apply(r.getLength())); + + /* Track remaining bytes required to complete this range */ + AtomicInteger pending = pendingBytes.computeIfAbsent( + key, k -> new AtomicInteger(r.getLength())); + + synchronized (fullBuf) { + /* Double-check completion inside lock */ + if (future.isDone()) { + continue; + } + + ByteBuffer dst = fullBuf.duplicate(); + dst.position(destOffset); + dst.put(buffer.getBuffer(), srcOffset, length); + + int left = pending.addAndGet(-length); + + LOG.debug("fanOut: wrote chunk: path={}, rangeOffset={}, destOffset={}," + + " length={}, pendingBytes={}", + buffer.getPath(), r.getOffset(), destOffset, length, left); + + if (left < 0) { + LOG.error("fanOut: pending bytes went negative  possible duplicate write:" + + " path={}, rangeOffset={}, pending={}", + buffer.getPath(), r.getOffset(), left); + future.completeExceptionally(new IllegalStateException( + "Pending bytes negative for offset=" + r.getOffset())); + partialBuffers.remove(key); + pendingBytes.remove(key); + continue; + } + + /* Complete future once all bytes are received */ + if (left == 0 && !future.isDone()) { + /* + * Prepare buffer for reading. + * DO NOT use flip() because writes may arrive out-of-order. + * Instead explicitly expose the full buffer. + */ + fullBuf.position(0); + fullBuf.limit(fullBuf.capacity()); + + if (fullBuf.limit() != r.getLength()) { + LOG.warn("fanOut: buffer size mismatch: path={}, rangeOffset={}," + + " expected={}, actual={}", + buffer.getPath(), r.getOffset(), r.getLength(), fullBuf.limit()); + } + + future.complete(fullBuf); + partialBuffers.remove(key); + pendingBytes.remove(key); + + LOG.debug("fanOut: completed range: path={}, rangeOffset={}, rangeLength={}", + buffer.getPath(), r.getOffset(), r.getLength()); + } } - bb.flip(); - r.getData().complete(bb); - LOG.debug("fanOut completed logical range offset={}, length={}", - r.getOffset(), r.getLength()); } catch (Exception e) { - LOG.debug("fanOut failed for logical range offset={}", - r.getOffset(), e); - /* Propagate failure to the affected logical range */ - r.getData().completeExceptionally(e); + LOG.warn("fanOut: exception processing range: path={}, rangeOffset={}", + buffer.getPath(), r.getOffset(), e); + RangeKey key = new RangeKey(r); + partialBuffers.remove(key); + pendingBytes.remove(key); + if (!future.isDone()) { + future.completeExceptionally(e); + } } } } } /** - * Fail all logical {@link FileRange}s associated with a single combined - * vectored read unit. + * Fails all logical {@link FileRange}s associated with a given + * {@link CombinedFileRange}. + * + *

This method is invoked when a vectored read for the combined unit + * fails. It ensures that: + *

    + *
  • Any partially accumulated buffers are cleaned up
    • + *
  • Pending byte tracking state is removed
    • + *
  • All corresponding {@link CompletableFuture}s are completed exceptionally
    • + *
+ *

* - * @param unit combined file range whose logical ranges should be failed - * @param t failure cause to propagate to waiting futures + * @param unit the combined vectored read unit whose underlying ranges must be failed + * @param t the exception that caused the failure */ private void failUnit(CombinedFileRange unit, Throwable t) { for (FileRange r : unit.getUnderlying()) { - r.getData().completeExceptionally(t); + RangeKey key = new RangeKey(r); + partialBuffers.remove(key); + pendingBytes.remove(key); + CompletableFuture future = r.getData(); + if (future != null && !future.isDone()) { + future.completeExceptionally(t); + } } } - /** - * Completes all logical {@link FileRange} futures associated with a vectored - * {@link ReadBuffer} exceptionally when the backend read fails. + * Fails all {@link FileRange} futures associated with the given + * {@link ReadBuffer} and clears any partial state. * - * @param buffer the vectored read buffer - * @param t the failure cause to propagate to waiting futures + * @param buffer the read buffer whose ranges should be failed + * @param t the exception causing the failure */ void failBufferFutures(ReadBuffer buffer, Throwable t) { List units = buffer.getVectoredUnits(); - if (units == null) { - return; - } - LOG.debug("failBufferFutures bufferOffset={}", buffer.getOffset(), t); - /* Propagate failure to all logical ranges attached to this buffer */ + if (units == null) return; + for (CombinedFileRange unit : units) { for (FileRange r : unit.getUnderlying()) { + RangeKey key = new RangeKey(r); + partialBuffers.remove(key); + pendingBytes.remove(key); CompletableFuture future = r.getData(); if (future != null && !future.isDone()) { future.completeExceptionally(t); @@ -363,40 +493,152 @@ void failBufferFutures(ReadBuffer buffer, Throwable t) { } /** - * Perform a synchronous direct read for a vectored unit when no pooled - * read buffer is available. + * Performs a synchronous direct read for a {@link CombinedFileRange} + * when pooled buffering is not available. * - *

This method reads the required byte range directly from the backend - * and completes all associated logical {@link FileRange} futures. It is - * used as a fallback path when vectored buffering cannot be used.

+ *

Data is accumulated into per-range partial buffers shared with + * {@link #fanOut}, ensuring that ranges spanning multiple chunks are + * correctly reassembled regardless of whether individual chunks were + * served via the async queue or this direct fallback path.

* - * @param stream input stream for the file being read - * @param unit combined file range to read directly - * @param allocator allocator used to create output buffers for logical ranges - * @throws IOException if memory pressure is high or the backend read fails + * @param stream input stream to read from + * @param unit combined range to read + * @param allocator buffer allocator for logical ranges + * @throws IOException if the read fails */ void directRead( AbfsInputStream stream, CombinedFileRange unit, IntFunction allocator) throws IOException { - LOG.debug("directRead offset={}, length={}", - unit.getOffset(), unit.getLength()); - /* Read the entire combined range into a temporary buffer */ + + LOG.debug("directRead: path={}, offset={}, length={}", + stream.getPath(), unit.getOffset(), unit.getLength()); + + /* Read entire combined range into a temporary buffer */ byte[] tmp = new byte[unit.getLength()]; - stream.readRemote(unit.getOffset(), tmp, 0, unit.getLength(), - stream.getTracingContext()); + TracingContext tracingContext = new TracingContext(stream.getTracingContext()); + tracingContext.setReadType(ReadType.VECTORED_DIRECT_READ); + + int total = 0; + int requested = unit.getLength(); + while (total < requested) { + int n = stream.readRemote(unit.getOffset() + total, tmp, total, + requested - total, tracingContext); + if (n <= 0) { + throw new IOException( + "Unexpected end of stream during direct read: path=" + stream.getPath() + + ", offset=" + (unit.getOffset() + total) + + ", requested=" + requested); + } + total += n; + } + + LOG.debug("directRead: read complete: path={}, offset={}, bytesRead={}", + stream.getPath(), unit.getOffset(), total); - /* Fan out data to individual logical ranges */ + long unitStart = unit.getOffset(); + long unitEnd = unitStart + unit.getLength(); + + /* Distribute data to each logical FileRange */ for (FileRange r : unit.getUnderlying()) { - ByteBuffer bb = allocator.apply(r.getLength()); - bb.put(tmp, - (int) (r.getOffset() - unit.getOffset()), - r.getLength()); - bb.flip(); - r.getData().complete(bb); + CompletableFuture future = r.getData(); + if (future == null || future.isDone()) continue; + + long rangeStart = r.getOffset(); + long rangeEnd = rangeStart + r.getLength(); + + /* Compute overlap between unit and logical range */ + long overlapStart = Math.max(rangeStart, unitStart); + long overlapEnd = Math.min(rangeEnd, unitEnd); + if (overlapStart >= overlapEnd) continue; + + int srcOffset = (int) (overlapStart - unitStart); + int destOffset = (int) (overlapStart - rangeStart); + int length = (int) (overlapEnd - overlapStart); + + LOG.debug("directRead: copying: path={}, rangeOffset={}, rangeLength={}," + + " srcOffset={}, destOffset={}, length={}", + stream.getPath(), r.getOffset(), r.getLength(), + srcOffset, destOffset, length); + + RangeKey key = new RangeKey(r); + + /* + * Use the shared partialBuffers/pendingBytes maps so that ranges + * spanning multiple chunks are correctly reassembled even when some + * chunks are served via directRead and others via the async fanOut path. + */ + ByteBuffer fullBuf = partialBuffers.computeIfAbsent( + key, k -> allocator.apply(r.getLength())); + AtomicInteger pending = pendingBytes.computeIfAbsent( + key, k -> new AtomicInteger(r.getLength())); + + synchronized (fullBuf) { + /* Re-check inside lock in case another chunk already completed this range */ + if (future.isDone()) continue; + + System.arraycopy(tmp, srcOffset, + fullBuf.array(), fullBuf.arrayOffset() + destOffset, + length); + + int left = pending.addAndGet(-length); + + LOG.debug("directRead: wrote chunk: path={}, rangeOffset={}, destOffset={}," + + " length={}, pendingBytes={}", + stream.getPath(), r.getOffset(), destOffset, length, left); + + if (left < 0) { + LOG.error("directRead: pending bytes went negative  possible duplicate write:" + + " path={}, rangeOffset={}, pending={}", + stream.getPath(), r.getOffset(), left); + future.completeExceptionally(new IllegalStateException( + "Pending bytes negative in directRead for offset=" + r.getOffset())); + partialBuffers.remove(key); + pendingBytes.remove(key); + continue; + } + + if (left == 0) { + fullBuf.position(0); + fullBuf.limit(r.getLength()); + future.complete(fullBuf); + partialBuffers.remove(key); + pendingBytes.remove(key); + LOG.debug("directRead: completed range: path={}, rangeOffset={}, rangeLength={}", + stream.getPath(), r.getOffset(), r.getLength()); + } + } } - LOG.debug("directRead completed offset={}, length={}", - unit.getOffset(), unit.getLength()); } -} + /** + * Identity-based key wrapper for {@link FileRange}. + * + *

This class ensures that {@link FileRange} instances are compared + * using reference equality rather than logical equality. It is used as + * a key in maps where multiple ranges may have identical offsets and + * lengths but must be treated as distinct objects.

+ * + *

Equality and hash code are based on object identity + * ({@code ==}) via {@link System#identityHashCode(Object)}.

+ */ + static final class RangeKey { + private final FileRange range; + + RangeKey(FileRange range) { + this.range = range; + } + + @Override + public boolean equals(Object o) { + return this == o || + (o instanceof RangeKey && + this.range == ((RangeKey) o).range); + } + + @Override + public int hashCode() { + return System.identityHashCode(range); + } + } +} \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/ITestAbfsDriverLatency.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/ITestAbfsDriverLatency.java new file mode 100644 index 0000000000000..91f3e26322b4c --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/ITestAbfsDriverLatency.java @@ -0,0 +1,217 @@ +package org.apache.hadoop.fs.azurebfs.utils; + +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.*; + +/** + * Measures e2e read latency through the ABFS driver. + * + * Compare P50/P90/P99 from this test against AbfsCheckLatency + * (plain HTTP app) to isolate driver overhead from network latency. + * + * Run: + * mvn test -Dtest=ITestAbfsDriverLatency + * -Dlatency.jobs=10 + * -Dlatency.files=100 + * -Dlatency.chunkKB=64 + * -Dlatency.fileSizeMB=1024 + */ +public class ITestAbfsDriverLatency extends AbstractAbfsIntegrationTest { + + // ===== CONFIG ===== + private int NUM_JOBS; + private int FILES_PER_JOB; + private int CHUNK_SIZE_KB; + private int FILE_SIZE_MB; + + private int CHUNK_SIZE; + private long FILE_SIZE; + + // one buffer per thread — no allocations per chunk + private ThreadLocal threadBuffer; + + // all chunk latencies across all jobs and files + private final List readLatencies = + Collections.synchronizedList(new ArrayList<>()); + + // test file paths created during setup + private final List testFiles = new ArrayList<>(); + + // ===== CONSTRUCTOR ===== + public ITestAbfsDriverLatency() throws Exception { + super(); + } + + // ===== SETUP ===== + @BeforeEach + public void setUp() throws Exception { + NUM_JOBS = Integer.parseInt(System.getProperty("latency.jobs", "2")); + FILES_PER_JOB = Integer.parseInt(System.getProperty("latency.files", "5")); + CHUNK_SIZE_KB = Integer.parseInt(System.getProperty("latency.chunkKB", "64")); + FILE_SIZE_MB = Integer.parseInt(System.getProperty("latency.fileSizeMB", "10")); + + CHUNK_SIZE = CHUNK_SIZE_KB * 1024; + FILE_SIZE = (long) FILE_SIZE_MB * 1024 * 1024; + + threadBuffer = ThreadLocal.withInitial(() -> new byte[CHUNK_SIZE]); + + System.out.println("===== CONFIG ====="); + System.out.printf("Jobs: %d%n", NUM_JOBS); + System.out.printf("Files/job: %d%n", FILES_PER_JOB); + System.out.printf("Chunk size: %d KB%n", CHUNK_SIZE_KB); + System.out.printf("File size: %d MB%n", FILE_SIZE_MB); + System.out.println(); + + createTestFiles(); + + // warmup — not recorded + System.out.println("Warmup..."); + readFile(getFileSystem(), testFiles.get(0)); + readLatencies.clear(); + Thread.sleep(2000); + } + + // ===== TEARDOWN ===== + @AfterEach + public void tearDown() throws Exception { + FileSystem fs = getFileSystem(); + for (Path p : testFiles) { + try { fs.delete(p, false); } + catch (Exception ignored) {} + } + testFiles.clear(); + readLatencies.clear(); + } + + // ===== TEST ===== + @Test + public void testE2EReadLatency() throws Exception { + System.out.println("Starting benchmark...\n"); + long wallStart = System.currentTimeMillis(); + + List threads = new ArrayList<>(); + for (int j = 0; j < NUM_JOBS; j++) { + final int jobId = j; + Thread t = new Thread(() -> runJob(jobId)); + threads.add(t); + t.start(); + } + + for (Thread t : threads) t.join(); + long wallMs = System.currentTimeMillis() - wallStart; + + printResults(wallMs); + } + + // ===== CREATE TEST FILES ===== + private void createTestFiles() throws Exception { + FileSystem fs = getFileSystem(); + int total = NUM_JOBS * FILES_PER_JOB; + + System.out.println("Creating " + total + " test files of " + + FILE_SIZE_MB + " MB each..."); + + for (int i = 0; i < total; i++) { + Path path = new Path(getTestPath1(), "latency_test_file_" + i); + testFiles.add(path); + + try (FSDataOutputStream out = fs.create(path, true)) { + byte[] buf = new byte[4 * 1024 * 1024]; + long written = 0; + while (written < FILE_SIZE) { + int toWrite = (int) Math.min(buf.length, FILE_SIZE - written); + out.write(buf, 0, toWrite); + written += toWrite; + } + } + } + + System.out.println("Done.\n"); + } + + // ===== ONE JOB ===== + private void runJob(int jobId) { + try { + FileSystem fs = getFileSystem(); + + int start = jobId * FILES_PER_JOB; + int end = start + FILES_PER_JOB; + + for (int i = start; i < end; i++) { + try { + readFile(fs, testFiles.get(i)); + } catch (Exception e) { + System.err.println("Job " + jobId + + " failed on file " + i + ": " + e.getMessage()); + } + } + + System.out.println("Job " + jobId + " done"); + + } catch (Exception e) { + System.err.println("Job " + jobId + " failed: " + e.getMessage()); + } + } + + // ===== READ ONE FILE via ABFS driver ===== + private void readFile(FileSystem fs, Path path) throws Exception { + byte[] buf = threadBuffer.get(); + + try (FSDataInputStream in = fs.open(path)) { + int n; + while (true) { + long start = System.currentTimeMillis(); + n = in.read(buf, 0, CHUNK_SIZE); + long latency = System.currentTimeMillis() - start; + + if (n == -1) break; + + readLatencies.add(latency); + // discard — no processing + } + } + } + + // ===== HELPER ===== + private Path getTestPath1() throws IOException { + return new Path(getFileSystem().getUri().toString() + + "/latency-benchmark"); + } + + // ===== PRINT RESULTS ===== + private void printResults(long wallMs) { + int chunksPerFile = (int) Math.ceil((double) FILE_SIZE / CHUNK_SIZE); + + System.out.println("\n===== RESULTS ====="); + System.out.printf("Wall time: %d ms%n", wallMs); + System.out.printf("Total reads: %d%n", readLatencies.size()); + System.out.printf("Expected reads: %d%n", + NUM_JOBS * FILES_PER_JOB * chunksPerFile); + + System.out.println("\n===== READ LATENCY via ABFS DRIVER ====="); + System.out.printf("Min: %d ms%n", readLatencies.stream().mapToLong(l->l).min().orElse(0)); + System.out.printf("Avg: %.0f ms%n", readLatencies.stream().mapToLong(l->l).average().orElse(0)); + System.out.printf("P50: %d ms%n", percentile(50)); + System.out.printf("P90: %d ms%n", percentile(90)); + System.out.printf("P99: %d ms%n", percentile(99)); + System.out.printf("Max: %d ms%n", readLatencies.stream().mapToLong(l->l).max().orElse(0)); + } + + // ===== PERCENTILE ===== + private long percentile(double p) { + List sorted = new ArrayList<>(readLatencies); + Collections.sort(sorted); + int idx = (int) Math.ceil(p / 100.0 * sorted.size()) - 1; + return sorted.get(Math.max(idx, 0)); + } +} \ No newline at end of file From 25fa8217866412871ad68edd09e30a34375dfc8a Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Thu, 2 Apr 2026 05:25:46 -0700 Subject: [PATCH 13/17] range validation fixes --- .../fs/azurebfs/services/AbfsInputStream.java | 2 +- .../services/ReadBufferManagerV1.java | 1 + .../services/ReadBufferManagerV2.java | 1 + .../services/VectoredReadHandler.java | 30 ++++- .../azurebfs/services/ITestVectoredRead.java | 115 +++++++----------- 5 files changed, 74 insertions(+), 75 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java index 5d765d84ab6de..b39c35cf30691 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java @@ -350,7 +350,7 @@ public synchronized int read(final byte[] b, final int off, final int len) throw */ @Override public void readVectored(List ranges, - IntFunction allocate) { + IntFunction allocate) throws EOFException { getVectoredReadHandler().readVectored(this, ranges, allocate); } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV1.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV1.java index cf2ce1548892e..ce64daed6d71a 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV1.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV1.java @@ -209,6 +209,7 @@ boolean queueVectoredRead(AbfsInputStream stream, ? existing.getLength() : existing.getRequestedLength()); if (end >= unit.getOffset() + unit.getLength()) { + existing.setBufferType(BufferType.VECTORED); existing.addVectoredUnit(unit); existing.setAllocator(allocator); if (existing.getStatus() == ReadBufferStatus.AVAILABLE) { diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV2.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV2.java index 8e6ff6f718627..a4c0ac7720988 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV2.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV2.java @@ -394,6 +394,7 @@ boolean queueVectoredRead(AbfsInputStream stream, ? existing.getLength() : existing.getRequestedLength()); if (end >= unit.getOffset() + unit.getLength()) { + existing.setBufferType(BufferType.VECTORED); existing.addVectoredUnit(unit); existing.setAllocator(allocator); if (existing.getStatus() == ReadBufferStatus.AVAILABLE) { diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VectoredReadHandler.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VectoredReadHandler.java index 34c161eb903d2..8d90cb85c9888 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VectoredReadHandler.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VectoredReadHandler.java @@ -18,11 +18,14 @@ package org.apache.hadoop.fs.azurebfs.services; +import java.io.EOFException; +import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Comparator; import java.util.List; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; @@ -33,6 +36,7 @@ import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.fs.FileRange; +import org.apache.hadoop.fs.VectoredReadUtils; import org.apache.hadoop.fs.azurebfs.constants.ReadType; import org.apache.hadoop.fs.azurebfs.enums.VectoredReadStrategy; import org.apache.hadoop.fs.azurebfs.utils.TracingContext; @@ -106,13 +110,26 @@ class VectoredReadHandler { public void readVectored( AbfsInputStream stream, List ranges, - IntFunction allocator) { + IntFunction allocator) throws EOFException { LOG.debug("readVectored invoked: path={}, rangeCount={}", stream.getPath(), ranges.size()); /* Initialize a future for each logical file range */ + /* Initialize a future for each logical file range */ + long fileLength = stream.getContentLength(); + List validRanges = new ArrayList<>(); for (FileRange r : ranges) { + VectoredReadUtils.validateRangeRequest(r); r.setData(new CompletableFuture<>()); + long offset = r.getOffset(); + long length = r.getLength(); + + if (offset < 0 || length < 0 || offset > fileLength || length > fileLength - offset) { + r.getData().completeExceptionally(new EOFException( + "Invalid range: offset=" + offset + ", length=" + length + ", fileLength=" + fileLength)); + continue; + } + validRanges.add(r); } /* Select the maximum allowed merge span based on the configured strategy */ @@ -125,7 +142,7 @@ public void readVectored( stream.getPath(), strategy, maxSpan); /* Merge logical ranges using a span-first coalescing strategy */ - List merged = mergeBySpanAndGap(ranges, maxSpan); + List merged = mergeBySpanAndGap(validRanges, maxSpan, fileLength); LOG.debug("readVectored: path={}, mergedRangeCount={}", stream.getPath(), merged.size()); @@ -142,6 +159,7 @@ public void readVectored( for (CombinedFileRange chunk : chunks) { try { + VectoredReadUtils.validateRangeRequest(chunk); boolean queued = queueVectoredRead(stream, chunk, allocator); if (!queued) { LOG.debug("readVectored: buffer pool exhausted, falling back to directRead:" @@ -243,11 +261,11 @@ private List splitByBufferSize( */ private List mergeBySpanAndGap( List ranges, - int maxSpan) { - LOG.debug("mergeBySpanAndGap: rangeCount={}, maxSpan={}", ranges.size(), maxSpan); + int maxSpan, long fileLength) throws EOFException { - List sortedRanges = new ArrayList<>(ranges); - sortedRanges.sort(Comparator.comparingLong(FileRange::getOffset)); + LOG.debug("mergeBySpanAndGap: rangeCount={}, maxSpan={}", ranges.size(), maxSpan); + List sortedRanges = VectoredReadUtils.validateAndSortRanges( + ranges, Optional.of(fileLength)); List out = new ArrayList<>(); CombinedFileRange current = null; diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java index 55edcd82906d0..56d06a9f53282 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java @@ -21,8 +21,10 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Random; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; @@ -60,25 +62,20 @@ public class ITestVectoredRead extends AbstractAbfsIntegrationTest { private static final int DATA_16_MB = 16 * ONE_MB; private static final int DATA_32_MB = 32 * ONE_MB; private static final int DATA_100_MB = 100 * ONE_MB; - private static final int OFFSET_100_B = 100; private static final int OFFSET_15K_B = 15_000; private static final int OFFSET_42K_B = 42_500; - private static final int LEN_10K_B = 10_000; private static final int LEN_27K_B = 27_000; private static final int LEN_40K_B = 40_000; - private static final double MB_1_2 = 1.2; private static final double MB_3_1 = 3.1; private static final double MB_4_1 = 4.1; private static final double MB_6_2 = 6.2; - private static final double MB_0_8 = 0.8; private static final double MB_0_9 = 0.9; private static final double MB_1_8 = 1.8; private static final double MB_1_9 = 1.9; - private static final double MB_3_8 = 3.8; private static final double MB_4_0 = 4.0; private static final double MB_3_2 = 3.2; @@ -86,7 +83,6 @@ public class ITestVectoredRead extends AbstractAbfsIntegrationTest { private static final double MB_2_0 = 2.0; private static final double MB_12_0 = 12.0; private static final double MB_16_0 = 16.0; - private static final int HUGE_OFFSET_1 = 5_856_368; private static final int HUGE_OFFSET_2 = 3_520_861; private static final int HUGE_OFFSET_3 = 8_191_913; @@ -94,14 +90,12 @@ public class ITestVectoredRead extends AbstractAbfsIntegrationTest { private static final int HUGE_OFFSET_5 = 2_520_861; private static final int HUGE_OFFSET_6 = 9_191_913; private static final int HUGE_OFFSET_7 = 2_820_861; - private static final int HUGE_RANGE = 116_770; private static final int HUGE_RANGE_LARGE = 156_770; - private static final int LOOKUP_RETRIES = 100; private static final int EXEC_THREADS = 3; private static final int SEQ_READ_ITERATIONS = 5; - private static final int FUTURE_TIMEOUT_SEC = 5; + private static final int FUTURE_TIMEOUT_SEC = 50; public static final int SLEEP_TIME = 10; public ITestVectoredRead() throws Exception { @@ -462,40 +456,24 @@ public void testVectoredReadHitchhikesOnExistingPrefetch() throws Exception { } } - /** - * Ensures multiple reads issued while a buffer is in progress eventually - * complete successfully. Verifies correct synchronization between - * in-progress buffers and new vectored read requests. - */ @Test - public void testMultipleReadsWhileBufferInProgressEventuallyComplete() - throws Exception { + public void testMultipleReadsWhileBufferInProgressEventuallyComplete() throws Exception { final AzureBlobFileSystem fs = getFileSystem(); String fileName = methodName.getMethodName(); byte[] fileContent = getRandomBytesArray(DATA_8_MB); Path testFilePath = createFileWithContent(fs, fileName, fileContent); - CountDownLatch blockCompletion = new CountDownLatch(1); - try (FSDataInputStream in = fs.openFile(testFilePath).build().get()) { - AbfsInputStream spyIn = - Mockito.spy((AbfsInputStream) in.getWrappedStream()); + AbfsInputStream spyIn = Mockito.spy((AbfsInputStream) in.getWrappedStream()); ReadBufferManager rbm = spyIn.getReadBufferManager(); - - /* Block completion so buffer stays in inProgressList */ + AtomicBoolean firstCall = new AtomicBoolean(true); Mockito.doAnswer(invocation -> { - blockCompletion.await(); + if (firstCall.getAndSet(false)) { + blockCompletion.await(); + } return invocation.callRealMethod(); - }).when(spyIn).readRemote( - Mockito.anyLong(), - Mockito.any(byte[].class), - Mockito.anyInt(), - Mockito.anyInt(), - Mockito.any()); - + }).when(spyIn).readRemote(Mockito.anyLong(),Mockito.any(byte[].class),Mockito.anyInt(),Mockito.anyInt(),Mockito.any()); ExecutorService exec = Executors.newFixedThreadPool(EXEC_THREADS); - - /* 1. Start first normal read → creates in-progress buffer */ Future r1 = exec.submit(() -> { try { spyIn.read(new byte[1], 0, 1); @@ -503,23 +481,17 @@ public void testMultipleReadsWhileBufferInProgressEventuallyComplete() throw new RuntimeException(e); } }); - - /* 2. Explicitly validate buffer is in inProgressList */ ReadBuffer inProgress = null; - for (int i = 0; i r2 = exec.submit(() -> { try { spyIn.read(new byte[1], 0, 1); @@ -527,10 +499,10 @@ public void testMultipleReadsWhileBufferInProgressEventuallyComplete() throw new RuntimeException(e); } }); - - /* 4. Submit vectored read while buffer is in progress */ + long bufferOffset = inProgress.getOffset(); + int length = (int)Math.min(ONE_MB, DATA_8_MB - bufferOffset); List ranges = new ArrayList<>(); - ranges.add(FileRange.createFileRange(ONE_MB, (int) ONE_MB)); + ranges.add(FileRange.createFileRange(bufferOffset, length)); Future vr = exec.submit(() -> { try { spyIn.readVectored(ranges, ByteBuffer::allocate); @@ -538,18 +510,13 @@ public void testMultipleReadsWhileBufferInProgressEventuallyComplete() throw new RuntimeException(e); } }); - - /* 5. Allow completion */ + Thread.sleep(50); blockCompletion.countDown(); - - /* 6. All reads must complete */ r1.get(FUTURE_TIMEOUT_SEC, TimeUnit.SECONDS); r2.get(FUTURE_TIMEOUT_SEC, TimeUnit.SECONDS); vr.get(FUTURE_TIMEOUT_SEC, TimeUnit.SECONDS); ranges.get(0).getData().get(FUTURE_TIMEOUT_SEC, TimeUnit.SECONDS); - - validateVectoredReadResult(ranges, fileContent, 0); - + validateVectoredReadResult(ranges, fileContent, bufferOffset); exec.shutdownNow(); } } @@ -562,7 +529,8 @@ public void testMultipleReadsWhileBufferInProgressEventuallyComplete() @Test public void testThroughputOptimizedReadVectored() throws Exception { Configuration configuration = getRawConfiguration(); - configuration.set(FS_AZURE_VECTORED_READ_STRATEGY, VectoredReadStrategy.THROUGHPUT_OPTIMIZED.getName()); + configuration.set(FS_AZURE_VECTORED_READ_STRATEGY, + VectoredReadStrategy.THROUGHPUT_OPTIMIZED.getName()); FileSystem fileSystem = FileSystem.newInstance(configuration); try (AzureBlobFileSystem abfs = (AzureBlobFileSystem) fileSystem) { String fileName = methodName.getMethodName(); @@ -621,28 +589,34 @@ public void testThroughputOptimizedReadVectored() throws Exception { public void testRandomReadsNonVectoredThenVectoredPerformance() throws Exception { final AzureBlobFileSystem fs = getFileSystem(); - // File large enough to amplify performance differences final int fileSize = 128 * ONE_MB; final int readSize = 64 * ONE_KB; final int readCount = 512; byte[] fileContent = getRandomBytesArray(fileSize); Path testPath = createFileWithContent(fs, methodName.getMethodName(), fileContent); - // Generate deterministic random offsets + /* ---------------------------------------------------- + * Generate NON-overlapping offsets (shuffled) + * ---------------------------------------------------- */ List offsets = new ArrayList<>(); - Random rnd = new Random(12345L); - for (int i = 0; i < readCount; i++) { - long offset = Math.abs(rnd.nextLong()) - % (fileSize - readSize); + for (long offset = 0; offset + readSize <= fileSize; offset += readSize) { offsets.add(offset); } - // Build vectored ranges from the same offsets + // Shuffle to simulate randomness without overlap + Collections.shuffle(offsets, new Random(12345L)); + // Limit to readCount + offsets = offsets.subList(0, Math.min(readCount, offsets.size())); + + /* ---------------------------------------------------- + * Build vectored ranges + * ---------------------------------------------------- */ List vectoredRanges = new ArrayList<>(); for (long offset : offsets) { vectoredRanges.add( FileRange.createFileRange(offset, readSize)); } try (FSDataInputStream in = fs.openFile(testPath).build().get()) { + /* ---------------------------------------------------- * Phase 1: Random non-vectored reads * ---------------------------------------------------- */ @@ -655,21 +629,26 @@ public void testRandomReadsNonVectoredThenVectoredPerformance() long nonVectoredTimeNs = System.nanoTime() - nonVectoredStartNs; /* ---------------------------------------------------- - * Phase 2: Vectored reads (after all non-vectored reads) + * Phase 2: Vectored reads * ---------------------------------------------------- */ long vectoredStartNs = System.nanoTime(); in.readVectored(vectoredRanges, ByteBuffer::allocate); CompletableFuture.allOf( - vectoredRanges.stream() - .map(FileRange::getData) - .toArray(CompletableFuture[]::new)) - .get(); + vectoredRanges.stream() + .map(FileRange::getData) + .toArray(CompletableFuture[]::new) + ).get(); long vectoredTimeNs = System.nanoTime() - vectoredStartNs; - assertTrue(vectoredTimeNs < nonVectoredTimeNs, - String.format("Vectored read time %d ns not faster than " + - "non-vectored time %d ns", - vectoredTimeNs, nonVectoredTimeNs)); + /* ---------------------------------------------------- + * Assertion (less flaky) + * ---------------------------------------------------- */ + assertTrue( + vectoredTimeNs <= nonVectoredTimeNs * 1.2, + String.format( + "Vectored read slower: vectored=%d ns, non-vectored=%d ns", + vectoredTimeNs, nonVectoredTimeNs) + ); } } } From 5b2632a91239ca3a3ee62744de7fc8b4bf806e72 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Thu, 2 Apr 2026 05:46:04 -0700 Subject: [PATCH 14/17] fix issues --- .../hadoop/fs/azurebfs/enums/VectoredReadStrategy.java | 2 +- .../apache/hadoop/fs/azurebfs/services/AbfsInputStream.java | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/enums/VectoredReadStrategy.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/enums/VectoredReadStrategy.java index d67499ddb9dd5..5fc9700c15431 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/enums/VectoredReadStrategy.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/enums/VectoredReadStrategy.java @@ -74,4 +74,4 @@ public static VectoredReadStrategy fromString(String value) { } throw new IllegalArgumentException("Invalid vectored read strategy: " + value); } -} \ No newline at end of file +} diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java index b39c35cf30691..f03190a1e5418 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java @@ -74,6 +74,7 @@ public abstract class AbfsInputStream extends FSInputStream implements CanUnbuff private final AbfsClient client; private final Statistics statistics; private final String path; + private final long contentLength; private final int bufferSize; // default buffer size private final int footerReadSize; // default buffer size to read when reading footer @@ -95,7 +96,8 @@ public abstract class AbfsInputStream extends FSInputStream implements CanUnbuff // User configured size of read ahead. private final int readAheadRange; - private boolean firstRead = true; + private boolean firstRead = true; // to identify first read for optimizations + // SAS tokens can be re-used until they expire private CachedSASToken cachedSasToken; private byte[] buffer = null; // will be initialized on first use @@ -125,7 +127,6 @@ public abstract class AbfsInputStream extends FSInputStream implements CanUnbuff private final AbfsInputStreamContext context; private IOStatistics ioStatistics; private String filePathIdentifier; - private VectoredReadHandler vectoredReadHandler; /** * This is the actual position within the object, used by From 437ffc8e00e2dc1f16aad75cb8f02ecc741de8be Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Thu, 2 Apr 2026 10:38:58 -0700 Subject: [PATCH 15/17] Fix checkstyle --- .../services/VectoredReadHandler.java | 14 +- .../azurebfs/services/ITestVectoredRead.java | 17 +- .../utils/ITestAbfsDriverLatency.java | 217 ------------------ 3 files changed, 17 insertions(+), 231 deletions(-) delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/ITestAbfsDriverLatency.java diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VectoredReadHandler.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VectoredReadHandler.java index 8d90cb85c9888..e482f93c40cca 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VectoredReadHandler.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VectoredReadHandler.java @@ -376,7 +376,7 @@ void fanOut(ReadBuffer buffer, int bytesRead) { int srcOffset = (int) (overlapStart - bufferStart); int destOffset = (int) (overlapStart - rangeStart); - int length = (int)(overlapEnd - overlapStart); + int length = (int) (overlapEnd - overlapStart); LOG.debug("fanOut: copying path={}, rangeOffset={}, rangeLength={}," + " bufferOffset={}, srcOffset={}, destOffset={}, length={}", @@ -495,7 +495,7 @@ private void failUnit(CombinedFileRange unit, Throwable t) { */ void failBufferFutures(ReadBuffer buffer, Throwable t) { List units = buffer.getVectoredUnits(); - if (units == null) return; + if (units == null) {return;} for (CombinedFileRange unit : units) { for (FileRange r : unit.getUnderlying()) { @@ -560,7 +560,7 @@ void directRead( /* Distribute data to each logical FileRange */ for (FileRange r : unit.getUnderlying()) { CompletableFuture future = r.getData(); - if (future == null || future.isDone()) continue; + if (future == null || future.isDone()) {continue;} long rangeStart = r.getOffset(); long rangeEnd = rangeStart + r.getLength(); @@ -568,7 +568,7 @@ void directRead( /* Compute overlap between unit and logical range */ long overlapStart = Math.max(rangeStart, unitStart); long overlapEnd = Math.min(rangeEnd, unitEnd); - if (overlapStart >= overlapEnd) continue; + if (overlapStart >= overlapEnd) {continue;} int srcOffset = (int) (overlapStart - unitStart); int destOffset = (int) (overlapStart - rangeStart); @@ -593,7 +593,7 @@ void directRead( synchronized (fullBuf) { /* Re-check inside lock in case another chunk already completed this range */ - if (future.isDone()) continue; + if (future.isDone()) {continue;} System.arraycopy(tmp, srcOffset, fullBuf.array(), fullBuf.arrayOffset() + destOffset, @@ -649,9 +649,7 @@ static final class RangeKey { @Override public boolean equals(Object o) { - return this == o || - (o instanceof RangeKey && - this.range == ((RangeKey) o).range); + return this == o || (o instanceof RangeKey && this.range == ((RangeKey) o).range); } @Override diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java index 56d06a9f53282..02913693b795f 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java @@ -97,6 +97,8 @@ public class ITestVectoredRead extends AbstractAbfsIntegrationTest { private static final int SEQ_READ_ITERATIONS = 5; private static final int FUTURE_TIMEOUT_SEC = 50; public static final int SLEEP_TIME = 10; + private static final long SHUFFLE_SEED = 12345L; + private static final double PERFORMANCE_TOLERANCE_FACTOR = 1.2; public ITestVectoredRead() throws Exception { } @@ -472,7 +474,10 @@ public void testMultipleReadsWhileBufferInProgressEventuallyComplete() throws Ex blockCompletion.await(); } return invocation.callRealMethod(); - }).when(spyIn).readRemote(Mockito.anyLong(),Mockito.any(byte[].class),Mockito.anyInt(),Mockito.anyInt(),Mockito.any()); + }) + .when(spyIn) + .readRemote(Mockito.anyLong(), Mockito.any(byte[].class), + Mockito.anyInt(), Mockito.anyInt(), Mockito.any()); ExecutorService exec = Executors.newFixedThreadPool(EXEC_THREADS); Future r1 = exec.submit(() -> { try { @@ -491,7 +496,7 @@ public void testMultipleReadsWhileBufferInProgressEventuallyComplete() throws Ex } Thread.sleep(SLEEP_TIME); } - assertNotNull(inProgress,"Expected buffer to be in inProgressList while completion is blocked"); + assertNotNull(inProgress, "Expected buffer to be in inProgressList while completion is blocked"); Future r2 = exec.submit(() -> { try { spyIn.read(new byte[1], 0, 1); @@ -500,7 +505,7 @@ public void testMultipleReadsWhileBufferInProgressEventuallyComplete() throws Ex } }); long bufferOffset = inProgress.getOffset(); - int length = (int)Math.min(ONE_MB, DATA_8_MB - bufferOffset); + int length = (int) Math.min(ONE_MB, DATA_8_MB - bufferOffset); List ranges = new ArrayList<>(); ranges.add(FileRange.createFileRange(bufferOffset, length)); Future vr = exec.submit(() -> { @@ -510,7 +515,7 @@ public void testMultipleReadsWhileBufferInProgressEventuallyComplete() throws Ex throw new RuntimeException(e); } }); - Thread.sleep(50); + Thread.sleep(FUTURE_TIMEOUT_SEC); blockCompletion.countDown(); r1.get(FUTURE_TIMEOUT_SEC, TimeUnit.SECONDS); r2.get(FUTURE_TIMEOUT_SEC, TimeUnit.SECONDS); @@ -603,7 +608,7 @@ public void testRandomReadsNonVectoredThenVectoredPerformance() offsets.add(offset); } // Shuffle to simulate randomness without overlap - Collections.shuffle(offsets, new Random(12345L)); + Collections.shuffle(offsets, new Random(SHUFFLE_SEED)); // Limit to readCount offsets = offsets.subList(0, Math.min(readCount, offsets.size())); @@ -644,7 +649,7 @@ public void testRandomReadsNonVectoredThenVectoredPerformance() * Assertion (less flaky) * ---------------------------------------------------- */ assertTrue( - vectoredTimeNs <= nonVectoredTimeNs * 1.2, + vectoredTimeNs <= nonVectoredTimeNs * PERFORMANCE_TOLERANCE_FACTOR, String.format( "Vectored read slower: vectored=%d ns, non-vectored=%d ns", vectoredTimeNs, nonVectoredTimeNs) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/ITestAbfsDriverLatency.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/ITestAbfsDriverLatency.java deleted file mode 100644 index 91f3e26322b4c..0000000000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/ITestAbfsDriverLatency.java +++ /dev/null @@ -1,217 +0,0 @@ -package org.apache.hadoop.fs.azurebfs.utils; - -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.util.*; - -/** - * Measures e2e read latency through the ABFS driver. - * - * Compare P50/P90/P99 from this test against AbfsCheckLatency - * (plain HTTP app) to isolate driver overhead from network latency. - * - * Run: - * mvn test -Dtest=ITestAbfsDriverLatency - * -Dlatency.jobs=10 - * -Dlatency.files=100 - * -Dlatency.chunkKB=64 - * -Dlatency.fileSizeMB=1024 - */ -public class ITestAbfsDriverLatency extends AbstractAbfsIntegrationTest { - - // ===== CONFIG ===== - private int NUM_JOBS; - private int FILES_PER_JOB; - private int CHUNK_SIZE_KB; - private int FILE_SIZE_MB; - - private int CHUNK_SIZE; - private long FILE_SIZE; - - // one buffer per thread — no allocations per chunk - private ThreadLocal threadBuffer; - - // all chunk latencies across all jobs and files - private final List readLatencies = - Collections.synchronizedList(new ArrayList<>()); - - // test file paths created during setup - private final List testFiles = new ArrayList<>(); - - // ===== CONSTRUCTOR ===== - public ITestAbfsDriverLatency() throws Exception { - super(); - } - - // ===== SETUP ===== - @BeforeEach - public void setUp() throws Exception { - NUM_JOBS = Integer.parseInt(System.getProperty("latency.jobs", "2")); - FILES_PER_JOB = Integer.parseInt(System.getProperty("latency.files", "5")); - CHUNK_SIZE_KB = Integer.parseInt(System.getProperty("latency.chunkKB", "64")); - FILE_SIZE_MB = Integer.parseInt(System.getProperty("latency.fileSizeMB", "10")); - - CHUNK_SIZE = CHUNK_SIZE_KB * 1024; - FILE_SIZE = (long) FILE_SIZE_MB * 1024 * 1024; - - threadBuffer = ThreadLocal.withInitial(() -> new byte[CHUNK_SIZE]); - - System.out.println("===== CONFIG ====="); - System.out.printf("Jobs: %d%n", NUM_JOBS); - System.out.printf("Files/job: %d%n", FILES_PER_JOB); - System.out.printf("Chunk size: %d KB%n", CHUNK_SIZE_KB); - System.out.printf("File size: %d MB%n", FILE_SIZE_MB); - System.out.println(); - - createTestFiles(); - - // warmup — not recorded - System.out.println("Warmup..."); - readFile(getFileSystem(), testFiles.get(0)); - readLatencies.clear(); - Thread.sleep(2000); - } - - // ===== TEARDOWN ===== - @AfterEach - public void tearDown() throws Exception { - FileSystem fs = getFileSystem(); - for (Path p : testFiles) { - try { fs.delete(p, false); } - catch (Exception ignored) {} - } - testFiles.clear(); - readLatencies.clear(); - } - - // ===== TEST ===== - @Test - public void testE2EReadLatency() throws Exception { - System.out.println("Starting benchmark...\n"); - long wallStart = System.currentTimeMillis(); - - List threads = new ArrayList<>(); - for (int j = 0; j < NUM_JOBS; j++) { - final int jobId = j; - Thread t = new Thread(() -> runJob(jobId)); - threads.add(t); - t.start(); - } - - for (Thread t : threads) t.join(); - long wallMs = System.currentTimeMillis() - wallStart; - - printResults(wallMs); - } - - // ===== CREATE TEST FILES ===== - private void createTestFiles() throws Exception { - FileSystem fs = getFileSystem(); - int total = NUM_JOBS * FILES_PER_JOB; - - System.out.println("Creating " + total + " test files of " - + FILE_SIZE_MB + " MB each..."); - - for (int i = 0; i < total; i++) { - Path path = new Path(getTestPath1(), "latency_test_file_" + i); - testFiles.add(path); - - try (FSDataOutputStream out = fs.create(path, true)) { - byte[] buf = new byte[4 * 1024 * 1024]; - long written = 0; - while (written < FILE_SIZE) { - int toWrite = (int) Math.min(buf.length, FILE_SIZE - written); - out.write(buf, 0, toWrite); - written += toWrite; - } - } - } - - System.out.println("Done.\n"); - } - - // ===== ONE JOB ===== - private void runJob(int jobId) { - try { - FileSystem fs = getFileSystem(); - - int start = jobId * FILES_PER_JOB; - int end = start + FILES_PER_JOB; - - for (int i = start; i < end; i++) { - try { - readFile(fs, testFiles.get(i)); - } catch (Exception e) { - System.err.println("Job " + jobId - + " failed on file " + i + ": " + e.getMessage()); - } - } - - System.out.println("Job " + jobId + " done"); - - } catch (Exception e) { - System.err.println("Job " + jobId + " failed: " + e.getMessage()); - } - } - - // ===== READ ONE FILE via ABFS driver ===== - private void readFile(FileSystem fs, Path path) throws Exception { - byte[] buf = threadBuffer.get(); - - try (FSDataInputStream in = fs.open(path)) { - int n; - while (true) { - long start = System.currentTimeMillis(); - n = in.read(buf, 0, CHUNK_SIZE); - long latency = System.currentTimeMillis() - start; - - if (n == -1) break; - - readLatencies.add(latency); - // discard — no processing - } - } - } - - // ===== HELPER ===== - private Path getTestPath1() throws IOException { - return new Path(getFileSystem().getUri().toString() - + "/latency-benchmark"); - } - - // ===== PRINT RESULTS ===== - private void printResults(long wallMs) { - int chunksPerFile = (int) Math.ceil((double) FILE_SIZE / CHUNK_SIZE); - - System.out.println("\n===== RESULTS ====="); - System.out.printf("Wall time: %d ms%n", wallMs); - System.out.printf("Total reads: %d%n", readLatencies.size()); - System.out.printf("Expected reads: %d%n", - NUM_JOBS * FILES_PER_JOB * chunksPerFile); - - System.out.println("\n===== READ LATENCY via ABFS DRIVER ====="); - System.out.printf("Min: %d ms%n", readLatencies.stream().mapToLong(l->l).min().orElse(0)); - System.out.printf("Avg: %.0f ms%n", readLatencies.stream().mapToLong(l->l).average().orElse(0)); - System.out.printf("P50: %d ms%n", percentile(50)); - System.out.printf("P90: %d ms%n", percentile(90)); - System.out.printf("P99: %d ms%n", percentile(99)); - System.out.printf("Max: %d ms%n", readLatencies.stream().mapToLong(l->l).max().orElse(0)); - } - - // ===== PERCENTILE ===== - private long percentile(double p) { - List sorted = new ArrayList<>(readLatencies); - Collections.sort(sorted); - int idx = (int) Math.ceil(p / 100.0 * sorted.size()) - 1; - return sorted.get(Math.max(idx, 0)); - } -} \ No newline at end of file From 975bf739d2d8bc1f1f7acd86f0a3d14d9dae5341 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Thu, 2 Apr 2026 13:16:54 -0700 Subject: [PATCH 16/17] fix checkstyle --- .../services/VectoredReadHandler.java | 20 +- .../azurebfs/services/ITestVectoredRead.java | 176 +++++++++++------- 2 files changed, 124 insertions(+), 72 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VectoredReadHandler.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VectoredReadHandler.java index e482f93c40cca..8c6d2c7227d39 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VectoredReadHandler.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VectoredReadHandler.java @@ -19,11 +19,9 @@ package org.apache.hadoop.fs.azurebfs.services; import java.io.EOFException; -import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.Comparator; import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; @@ -495,7 +493,11 @@ private void failUnit(CombinedFileRange unit, Throwable t) { */ void failBufferFutures(ReadBuffer buffer, Throwable t) { List units = buffer.getVectoredUnits(); - if (units == null) {return;} + if (units == null) { + LOG.warn("fanOut: no vectored units found for path={}, offset={}", + buffer.getPath(), buffer.getOffset()); + return; + } for (CombinedFileRange unit : units) { for (FileRange r : unit.getUnderlying()) { @@ -560,7 +562,9 @@ void directRead( /* Distribute data to each logical FileRange */ for (FileRange r : unit.getUnderlying()) { CompletableFuture future = r.getData(); - if (future == null || future.isDone()) {continue;} + if (future == null || future.isDone()) { + continue; + } long rangeStart = r.getOffset(); long rangeEnd = rangeStart + r.getLength(); @@ -568,7 +572,9 @@ void directRead( /* Compute overlap between unit and logical range */ long overlapStart = Math.max(rangeStart, unitStart); long overlapEnd = Math.min(rangeEnd, unitEnd); - if (overlapStart >= overlapEnd) {continue;} + if (overlapStart >= overlapEnd) { + continue; + } int srcOffset = (int) (overlapStart - unitStart); int destOffset = (int) (overlapStart - rangeStart); @@ -593,7 +599,9 @@ void directRead( synchronized (fullBuf) { /* Re-check inside lock in case another chunk already completed this range */ - if (future.isDone()) {continue;} + if (future.isDone()) { + continue; + } System.arraycopy(tmp, srcOffset, fullBuf.array(), fullBuf.arrayOffset() + destOffset, diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java index 02913693b795f..df82a2c7e93c8 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java @@ -18,6 +18,8 @@ package org.apache.hadoop.fs.azurebfs.services; +import java.io.FilterInputStream; +import java.lang.reflect.Field; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; @@ -410,11 +412,6 @@ public void testConcurrentStreamsOnDifferentFiles() throws Exception { } } - /** - * Validates that vectored reads can reuse an in-progress prefetch buffer. - * Ensures no redundant backend read is issued when data is already - * available via readahead. - */ @Test public void testVectoredReadHitchhikesOnExistingPrefetch() throws Exception { final AzureBlobFileSystem fs = getFileSystem(); @@ -424,31 +421,47 @@ public void testVectoredReadHitchhikesOnExistingPrefetch() throws Exception { try (FSDataInputStream in = fs.openFile(testFilePath).build().get()) { AbfsInputStream abfsIn = (AbfsInputStream) in.getWrappedStream(); - AbfsInputStream spyIn = Mockito.spy(abfsIn); - // 1. Trigger a normal read to start the prefetch logic - // Reading the first byte often triggers a larger readahead (e.g., 4MB) + // Since client is final in AbfsInputStream, we cannot inject a spy into it. + // Instead, spy on abfsIn itself and stub readRemote to delegate to the real + // implementation while still being tracked by Mockito. + AbfsInputStream spyIn = Mockito.spy(abfsIn); + Mockito.doCallRealMethod().when(spyIn).readRemote( + Mockito.anyLong(), + Mockito.any(byte[].class), + Mockito.anyInt(), + Mockito.anyInt(), + Mockito.any()); + + // Replace the wrapped stream inside FSDataInputStream with our spy, + // so all subsequent calls go through spyIn. + Field wrappedField = FilterInputStream.class.getDeclaredField("in"); + wrappedField.setAccessible(true); + wrappedField.set(in, spyIn); + + // 1. Trigger sequential read → starts readahead covering [0, readAheadSize). byte[] seqBuf = new byte[1]; - spyIn.read(seqBuf, 0, 1); + in.read(seqBuf, 0, 1); - // 2. Immediately queue a vectored read for an offset within that prefetch range + // 2. Queue a vectored read fully inside the readahead window. List vRanges = new ArrayList<>(); - // Using 1MB offset, which should be inside the initial readahead buffer vRanges.add(FileRange.createFileRange(ONE_MB, (int) ONE_MB)); - IntFunction allocator = ByteBuffer::allocate; - spyIn.readVectored(vRanges, allocator); + in.readVectored(vRanges, allocator); - // 3. Wait for the vectored read to complete + // 3. Wait for completion. vRanges.get(0).getData().get(); - // 4. Validate Data Integrity + // 4. Validate data integrity. validateVectoredReadResult(vRanges, fileContent, ZERO); // 5. THE CRITICAL VALIDATION: - // Even though we did a manual read and a vectored read, - // there should only be ONE remote call if hitchhiking worked. - Mockito.verify(spyIn, Mockito.atMost(spyIn.getReadAheadQueueDepth())) + // Max 2 remote reads acceptable: + // - Read #1: readahead triggered by the sequential read + // - Read #2: only if vectored read just missed the prefetch window (race edge) + // 3+ means hitchhiking is broken — vectored read issued a redundant remote fetch. + final int MAX_EXPECTED_REMOTE_READS = 2; + Mockito.verify(spyIn, Mockito.atMost(MAX_EXPECTED_REMOTE_READS)) .readRemote( Mockito.anyLong(), Mockito.any(byte[].class), @@ -465,64 +478,95 @@ public void testMultipleReadsWhileBufferInProgressEventuallyComplete() throws Ex byte[] fileContent = getRandomBytesArray(DATA_8_MB); Path testFilePath = createFileWithContent(fs, fileName, fileContent); CountDownLatch blockCompletion = new CountDownLatch(1); + try (FSDataInputStream in = fs.openFile(testFilePath).build().get()) { - AbfsInputStream spyIn = Mockito.spy((AbfsInputStream) in.getWrappedStream()); + AbfsInputStream abfsIn = (AbfsInputStream) in.getWrappedStream(); + AbfsInputStream spyIn = Mockito.spy(abfsIn); + + // Inject spy into FSDataInputStream so all calls go through spyIn, + // including those from background threads spawned during read(). + Field wrappedField = FilterInputStream.class.getDeclaredField("in"); + wrappedField.setAccessible(true); + wrappedField.set(in, spyIn); + ReadBufferManager rbm = spyIn.getReadBufferManager(); AtomicBoolean firstCall = new AtomicBoolean(true); + Mockito.doAnswer(invocation -> { - if (firstCall.getAndSet(false)) { - blockCompletion.await(); - } - return invocation.callRealMethod(); - }) + if (firstCall.getAndSet(false)) { + blockCompletion.await(); + } + return invocation.callRealMethod(); + }) .when(spyIn) .readRemote(Mockito.anyLong(), Mockito.any(byte[].class), Mockito.anyInt(), Mockito.anyInt(), Mockito.any()); + ExecutorService exec = Executors.newFixedThreadPool(EXEC_THREADS); - Future r1 = exec.submit(() -> { - try { - spyIn.read(new byte[1], 0, 1); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - ReadBuffer inProgress = null; - for (int i = 0; i < LOOKUP_RETRIES; i++) { - synchronized (rbm) { - inProgress = rbm.findInList(rbm.getInProgressList(), spyIn, 0); + try { + // r1 triggers a readahead; the first readRemote call will block on + // blockCompletion, simulating an in-progress buffer. + Future r1 = exec.submit(() -> { + try { + in.read(new byte[1], 0, 1); // use 'in', not 'spyIn' directly + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + + // Poll until the buffer appears in the inProgressList. + ReadBuffer inProgress = null; + for (int i = 0; i < LOOKUP_RETRIES; i++) { + synchronized (rbm) { + inProgress = rbm.findInList(rbm.getInProgressList(), spyIn, 0); + } + if (inProgress != null) { + break; + } + Thread.sleep(SLEEP_TIME); } - if (inProgress != null) { - break; - } - Thread.sleep(SLEEP_TIME); + assertNotNull(inProgress, + "Expected buffer to be in inProgressList while completion is blocked"); + + // r2 reads the same offset — should wait on the in-progress buffer, + // not issue a new remote read. + Future r2 = exec.submit(() -> { + try { + in.read(new byte[1], 0, 1); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + + // Vectored read targeting the same in-progress buffer range — + // should hitchhike rather than issue a new remote read. + long bufferOffset = inProgress.getOffset(); + int length = (int) Math.min(ONE_MB, DATA_8_MB - bufferOffset); + List ranges = new ArrayList<>(); + ranges.add(FileRange.createFileRange(bufferOffset, length)); + Future vr = exec.submit(() -> { + try { + in.readVectored(ranges, ByteBuffer::allocate); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + + // Give r2 and vr time to reach their wait state before unblocking. + Thread.sleep(FUTURE_TIMEOUT_SEC); + blockCompletion.countDown(); + + r1.get(FUTURE_TIMEOUT_SEC, TimeUnit.SECONDS); + r2.get(FUTURE_TIMEOUT_SEC, TimeUnit.SECONDS); + vr.get(FUTURE_TIMEOUT_SEC, TimeUnit.SECONDS); + ranges.get(0).getData().get(FUTURE_TIMEOUT_SEC, TimeUnit.SECONDS); + + validateVectoredReadResult(ranges, fileContent, bufferOffset); + } finally { + exec.shutdownNow(); + // Restore original stream reference to avoid affecting shared state. + wrappedField.set(in, abfsIn); } - assertNotNull(inProgress, "Expected buffer to be in inProgressList while completion is blocked"); - Future r2 = exec.submit(() -> { - try { - spyIn.read(new byte[1], 0, 1); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - long bufferOffset = inProgress.getOffset(); - int length = (int) Math.min(ONE_MB, DATA_8_MB - bufferOffset); - List ranges = new ArrayList<>(); - ranges.add(FileRange.createFileRange(bufferOffset, length)); - Future vr = exec.submit(() -> { - try { - spyIn.readVectored(ranges, ByteBuffer::allocate); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - Thread.sleep(FUTURE_TIMEOUT_SEC); - blockCompletion.countDown(); - r1.get(FUTURE_TIMEOUT_SEC, TimeUnit.SECONDS); - r2.get(FUTURE_TIMEOUT_SEC, TimeUnit.SECONDS); - vr.get(FUTURE_TIMEOUT_SEC, TimeUnit.SECONDS); - ranges.get(0).getData().get(FUTURE_TIMEOUT_SEC, TimeUnit.SECONDS); - validateVectoredReadResult(ranges, fileContent, bufferOffset); - exec.shutdownNow(); } } From ba7c9ff404bf445cf1841c36f084e2e9da2a4ac2 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Fri, 3 Apr 2026 02:02:11 -0700 Subject: [PATCH 17/17] fix checkstyle failures --- hadoop-tools/hadoop-azure/pom.xml | 2 ++ .../apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml index 4311a4a2a8012..177fbdc23352d 100644 --- a/hadoop-tools/hadoop-azure/pom.xml +++ b/hadoop-tools/hadoop-azure/pom.xml @@ -456,6 +456,7 @@ **/azurebfs/ITestSmallWriteOptimization.java **/azurebfs/ITestAbfsStreamStatistics*.java **/azurebfs/services/ITestReadBufferManager.java + **/azurebfs/services/ITestVectoredRead.java **/azurebfs/commit/*.java @@ -497,6 +498,7 @@ **/azurebfs/ITestSmallWriteOptimization.java **/azurebfs/services/ITestReadBufferManager.java **/azurebfs/ITestAbfsStreamStatistics*.java + **/azurebfs/services/ITestVectoredRead.java **/azurebfs/commit/*.java diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java index df82a2c7e93c8..054fb9bbe37b2 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestVectoredRead.java @@ -460,8 +460,8 @@ public void testVectoredReadHitchhikesOnExistingPrefetch() throws Exception { // - Read #1: readahead triggered by the sequential read // - Read #2: only if vectored read just missed the prefetch window (race edge) // 3+ means hitchhiking is broken — vectored read issued a redundant remote fetch. - final int MAX_EXPECTED_REMOTE_READS = 2; - Mockito.verify(spyIn, Mockito.atMost(MAX_EXPECTED_REMOTE_READS)) + final int maxExpectedRemoteReads = 2; + Mockito.verify(spyIn, Mockito.atMost(maxExpectedRemoteReads)) .readRemote( Mockito.anyLong(), Mockito.any(byte[].class),