From bb27ec4d0ec51c01f5100a4919b0eb0f3fea4fd8 Mon Sep 17 00:00:00 2001 From: Anuj Modi Date: Sun, 28 Dec 2025 23:26:54 -0800 Subject: [PATCH 01/14] input stream refraction --- .../fs/azurebfs/AzureBlobFileSystemStore.java | 3 +- .../services/AbfsAdaptiveInputStream.java | 181 ++++++++++++++++++ .../fs/azurebfs/services/AbfsInputPolicy.java | 66 +++++++ .../fs/azurebfs/services/AbfsInputStream.java | 175 +++-------------- .../services/AbfsSequentialInputStream.java | 168 ++++++++++++++++ .../ITestAbfsInputStreamStatistics.java | 3 +- .../services/TestAbfsInputStream.java | 4 +- 7 files changed, 447 insertions(+), 153 deletions(-) create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputPolicy.java create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsSequentialInputStream.java diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index 5d7d0895d0223..1c543fb81f0e7 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -77,6 +77,7 @@ import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidUriException; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.TrileanConversionException; import org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode; +import org.apache.hadoop.fs.azurebfs.services.AbfsAdaptiveInputStream; import org.apache.hadoop.fs.azurebfs.services.ListResponseData; import org.apache.hadoop.fs.azurebfs.enums.Trilean; import org.apache.hadoop.fs.azurebfs.extensions.EncryptionContextProvider; @@ -947,7 +948,7 @@ public AbfsInputStream openFileForRead(Path path, perfInfo.registerSuccess(true); // Add statistics for InputStream - return new AbfsInputStream(getClient(), statistics, relativePath, + return new AbfsAdaptiveInputStream(getClient(), statistics, relativePath, contentLength, populateAbfsInputStreamContext( parameters.map(OpenFileParameters::getOptions), contextEncryptionAdapter), diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java new file mode 100644 index 0000000000000..1828aa5e17a80 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java @@ -0,0 +1,181 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.services; + +import java.io.IOException; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.azurebfs.constants.ReadType; +import org.apache.hadoop.fs.azurebfs.utils.TracingContext; + +import static java.lang.Math.max; + +public class AbfsAdaptiveInputStream extends AbfsInputStream { + + public AbfsAdaptiveInputStream( + final AbfsClient client, + final FileSystem.Statistics statistics, + final String path, + final long contentLength, + final AbfsInputStreamContext abfsInputStreamContext, + final String eTag, + TracingContext tracingContext) { + super(client, statistics, path, contentLength, + abfsInputStreamContext, eTag, tracingContext); + } + + @Override + public int read(long position, byte[] buffer, int offset, int length) + throws IOException { + return super.read(position, buffer, offset, length); + } + + @Override + public int read() throws IOException { + return super.read(); + } + + @Override + public synchronized int read(final byte[] b, final int off, final int len) throws IOException { + // check if buffer is null before logging the length + if (b != null) { + LOG.debug("read requested b.length = {} offset = {} len = {}", b.length, + off, len); + } else { + LOG.debug("read requested b = null offset = {} len = {}", off, len); + } + + int currentOff = off; + int currentLen = len; + int lastReadBytes; + int totalReadBytes = 0; + if (streamStatistics != null) { + streamStatistics.readOperationStarted(); + } + incrementReadOps(); + do { + + // limit is the maximum amount of data present in buffer. + // fCursor is the current file pointer. Thus maximum we can + // go back and read from buffer is fCursor - limit. + // There maybe case that we read less than requested data. + long filePosAtStartOfBuffer = fCursor - limit; + if (abfsReadFooterMetrics != null) { + abfsReadFooterMetrics.updateReadMetrics(filePathIdentifier, len, contentLength, nextReadPos); + } + if (nextReadPos >= filePosAtStartOfBuffer && nextReadPos <= fCursor) { + // Determining position in buffer from where data is to be read. + bCursor = (int) (nextReadPos - filePosAtStartOfBuffer); + + // When bCursor == limit, buffer will be filled again. + // So in this case we are not actually reading from buffer. + if (bCursor != limit && streamStatistics != null) { + streamStatistics.seekInBuffer(); + } + } else { + // Clearing the buffer and setting the file pointer + // based on previous seek() call. + fCursor = nextReadPos; + limit = 0; + bCursor = 0; + } + if (shouldReadFully()) { + lastReadBytes = readFileCompletely(b, currentOff, currentLen); + } else if (shouldReadLastBlock()) { + lastReadBytes = readLastBlock(b, currentOff, currentLen); + } else { + lastReadBytes = readOneBlock(b, currentOff, currentLen); + } + if (lastReadBytes > 0) { + currentOff += lastReadBytes; + currentLen -= lastReadBytes; + totalReadBytes += lastReadBytes; + } + if (currentLen <= 0 || currentLen > b.length - currentOff) { + break; + } + } while (lastReadBytes > 0); + return totalReadBytes > 0 ? totalReadBytes : lastReadBytes; + } + + @Override + protected int readOneBlock(final byte[] b, final int off, final int len) throws IOException { + if (len == 0) { + return 0; + } + if (!validate(b, off, len)) { + return -1; + } + //If buffer is empty, then fill the buffer. + if (bCursor == limit) { + //If EOF, then return -1 + if (fCursor >= contentLength) { + return -1; + } + + long bytesRead = 0; + //reset buffer to initial state - i.e., throw away existing data + bCursor = 0; + limit = 0; + if (buffer == null) { + LOG.debug("created new buffer size {}", bufferSize); + buffer = new byte[bufferSize]; + } + + // Reset Read Type back to normal and set again based on code flow. + tracingContext.setReadType(ReadType.NORMAL_READ); + if (alwaysReadBufferSize) { + bytesRead = readInternal(fCursor, buffer, 0, bufferSize, false); + } else { + // Enable readAhead when reading sequentially + if (-1 == fCursorAfterLastRead || fCursorAfterLastRead == fCursor || b.length >= bufferSize) { + LOG.debug("Sequential read with read ahead size of {}", bufferSize); + bytesRead = readInternal(fCursor, buffer, 0, bufferSize, false); + } else { + // Enabling read ahead for random reads as well to reduce number of remote calls. + int lengthWithReadAhead = Math.min(b.length + readAheadRange, bufferSize); + LOG.debug("Random read with read ahead size of {}", lengthWithReadAhead); + bytesRead = readInternal(fCursor, buffer, 0, lengthWithReadAhead, true); + } + } + if (firstRead) { + firstRead = false; + } + if (bytesRead == -1) { + return -1; + } + + limit += bytesRead; + fCursor += bytesRead; + fCursorAfterLastRead = fCursor; + } + return copyToUserBuffer(b, off, len); + } + + private boolean shouldReadFully() { + return this.firstRead && this.context.readSmallFilesCompletely() + && this.contentLength <= this.bufferSize; + } + + private boolean shouldReadLastBlock() { + long footerStart = max(0, this.contentLength - FOOTER_SIZE); + return this.firstRead && this.context.optimizeFooterRead() + && this.fCursor >= footerStart; + } +} diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputPolicy.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputPolicy.java new file mode 100644 index 0000000000000..8b98c9c8c5cf6 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputPolicy.java @@ -0,0 +1,66 @@ +package org.apache.hadoop.fs.azurebfs.services; + +import java.util.Locale; + +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_ADAPTIVE; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_AVRO; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_COLUMNAR; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_CSV; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_HBASE; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_JSON; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_ORC; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_PARQUET; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_RANDOM; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_VECTOR; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE; + +public enum AbfsInputPolicy { + + SEQUENTIAL(FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL), + RANDOM(FS_OPTION_OPENFILE_READ_POLICY_RANDOM), + VECTORED(FS_OPTION_OPENFILE_READ_POLICY_VECTOR), + LAYOUT("layout"), + ADAPTIVE(FS_OPTION_OPENFILE_READ_POLICY_ADAPTIVE); + + private final String policy; + + AbfsInputPolicy(String policy) { + this.policy = policy; + } + + @Override + public String toString() { + return policy; + } + + String getPolicy() { + return policy; + } + + public static AbfsInputPolicy getPolicy(String name, boolean isLayoutPresent) { + String trimmed = name.trim().toLowerCase(Locale.ENGLISH); + if (isLayoutPresent) { + return LAYOUT; + } + switch (trimmed) { + // all these options currently map to random IO. + case FS_OPTION_OPENFILE_READ_POLICY_HBASE: + case FS_OPTION_OPENFILE_READ_POLICY_RANDOM: + case FS_OPTION_OPENFILE_READ_POLICY_COLUMNAR: + case FS_OPTION_OPENFILE_READ_POLICY_ORC: + case FS_OPTION_OPENFILE_READ_POLICY_PARQUET: + return RANDOM; + + // handle the sequential formats. + case FS_OPTION_OPENFILE_READ_POLICY_AVRO: + case FS_OPTION_OPENFILE_READ_POLICY_CSV: + case FS_OPTION_OPENFILE_READ_POLICY_JSON: + case FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL: + case FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE: + return SEQUENTIAL; + default: + return ADAPTIVE; + } + } +} diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java index 31b6f0f073940..0730876bcfef7 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java @@ -26,7 +26,6 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.fs.PositionedReadable; import org.apache.hadoop.fs.azurebfs.constants.ReadType; import org.apache.hadoop.fs.impl.BackReference; import org.apache.hadoop.util.Preconditions; @@ -62,9 +61,9 @@ /** * The AbfsInputStream for AbfsClient. */ -public class AbfsInputStream extends FSInputStream implements CanUnbuffer, +public abstract class AbfsInputStream extends FSInputStream implements CanUnbuffer, StreamCapabilities, IOStatisticsSource { - private static final Logger LOG = LoggerFactory.getLogger(AbfsInputStream.class); + protected static final Logger LOG = LoggerFactory.getLogger(AbfsInputStream.class); // Footer size is set to qualify for both ORC and parquet files public static final int FOOTER_SIZE = 16 * ONE_KB; public static final int MAX_OPTIMIZED_READ_ATTEMPTS = 2; @@ -73,8 +72,8 @@ public class AbfsInputStream extends FSInputStream implements CanUnbuffer, private final AbfsClient client; private final Statistics statistics; private final String path; - private final long contentLength; - private final int bufferSize; // default buffer size + protected final long contentLength; + protected final int bufferSize; // default buffer size private final int footerReadSize; // default buffer size to read when reading footer private final int readAheadQueueDepth; // initialized in constructor private final String eTag; // eTag of the path when InputStream are created @@ -82,7 +81,7 @@ public class AbfsInputStream extends FSInputStream implements CanUnbuffer, private final boolean readAheadEnabled; // whether enable readAhead; private final boolean readAheadV2Enabled; // whether enable readAhead V2; private final String inputStreamId; - private final boolean alwaysReadBufferSize; + protected final boolean alwaysReadBufferSize; /* * By default the pread API will do a seek + read as in FSInputStream. * The read data will be kept in a buffer. When bufferedPreadDisabled is true, @@ -92,20 +91,20 @@ public class AbfsInputStream extends FSInputStream implements CanUnbuffer, */ private final boolean bufferedPreadDisabled; // User configured size of read ahead. - private final int readAheadRange; + protected final int readAheadRange; - private boolean firstRead = true; + protected boolean firstRead = true; // SAS tokens can be re-used until they expire private CachedSASToken cachedSasToken; - private byte[] buffer = null; // will be initialized on first use + protected byte[] buffer = null; // will be initialized on first use - private long fCursor = 0; // cursor of buffer within file - offset of next byte to read from remote server - private long fCursorAfterLastRead = -1; - private int bCursor = 0; // cursor of read within buffer - offset of next byte to be returned from buffer - private int limit = 0; // offset of next byte to be read into buffer from service (i.e., upper marker+1 + protected long fCursor = 0; // cursor of buffer within file - offset of next byte to read from remote server + protected long fCursorAfterLastRead = -1; + protected int bCursor = 0; // cursor of read within buffer - offset of next byte to be returned from buffer + protected int limit = 0; // offset of next byte to be read into buffer from service (i.e., upper marker+1 // of valid bytes in buffer) private boolean closed = false; - private TracingContext tracingContext; + protected TracingContext tracingContext; private final ContextEncryptionAdapter contextEncryptionAdapter; // Optimisations modify the pointer fields. @@ -115,20 +114,20 @@ public class AbfsInputStream extends FSInputStream implements CanUnbuffer, private int bCursorBkp; private long fCursorBkp; private long fCursorAfterLastReadBkp; - private final AbfsReadFooterMetrics abfsReadFooterMetrics; + protected final AbfsReadFooterMetrics abfsReadFooterMetrics; /** Stream statistics. */ - private final AbfsInputStreamStatistics streamStatistics; + protected final AbfsInputStreamStatistics streamStatistics; private long bytesFromReadAhead; // bytes read from readAhead; for testing private long bytesFromRemoteRead; // bytes read remotely; for testing private Listener listener; - private final AbfsInputStreamContext context; + protected final AbfsInputStreamContext context; private IOStatistics ioStatistics; - private String filePathIdentifier; + protected String filePathIdentifier; /** * This is the actual position within the object, used by * lazy seek to decide whether to seek on the next read or not. */ - private long nextReadPos; + protected long nextReadPos; /** ABFS instance to be held by the input stream to avoid GC close. */ private final BackReference fsBackRef; @@ -254,133 +253,11 @@ public int read() throws IOException { } @Override - public synchronized int read(final byte[] b, final int off, final int len) throws IOException { - // check if buffer is null before logging the length - if (b != null) { - LOG.debug("read requested b.length = {} offset = {} len = {}", b.length, - off, len); - } else { - LOG.debug("read requested b = null offset = {} len = {}", off, len); - } - - int currentOff = off; - int currentLen = len; - int lastReadBytes; - int totalReadBytes = 0; - if (streamStatistics != null) { - streamStatistics.readOperationStarted(); - } - incrementReadOps(); - do { - - // limit is the maximum amount of data present in buffer. - // fCursor is the current file pointer. Thus maximum we can - // go back and read from buffer is fCursor - limit. - // There maybe case that we read less than requested data. - long filePosAtStartOfBuffer = fCursor - limit; - if (abfsReadFooterMetrics != null) { - abfsReadFooterMetrics.updateReadMetrics(filePathIdentifier, len, contentLength, nextReadPos); - } - if (nextReadPos >= filePosAtStartOfBuffer && nextReadPos <= fCursor) { - // Determining position in buffer from where data is to be read. - bCursor = (int) (nextReadPos - filePosAtStartOfBuffer); - - // When bCursor == limit, buffer will be filled again. - // So in this case we are not actually reading from buffer. - if (bCursor != limit && streamStatistics != null) { - streamStatistics.seekInBuffer(); - } - } else { - // Clearing the buffer and setting the file pointer - // based on previous seek() call. - fCursor = nextReadPos; - limit = 0; - bCursor = 0; - } - if (shouldReadFully()) { - lastReadBytes = readFileCompletely(b, currentOff, currentLen); - } else if (shouldReadLastBlock()) { - lastReadBytes = readLastBlock(b, currentOff, currentLen); - } else { - lastReadBytes = readOneBlock(b, currentOff, currentLen); - } - if (lastReadBytes > 0) { - currentOff += lastReadBytes; - currentLen -= lastReadBytes; - totalReadBytes += lastReadBytes; - } - if (currentLen <= 0 || currentLen > b.length - currentOff) { - break; - } - } while (lastReadBytes > 0); - return totalReadBytes > 0 ? totalReadBytes : lastReadBytes; - } - - private boolean shouldReadFully() { - return this.firstRead && this.context.readSmallFilesCompletely() - && this.contentLength <= this.bufferSize; - } - - private boolean shouldReadLastBlock() { - long footerStart = max(0, this.contentLength - FOOTER_SIZE); - return this.firstRead && this.context.optimizeFooterRead() - && this.fCursor >= footerStart; - } + public abstract int read(final byte[] b, final int off, final int len) throws IOException; - private int readOneBlock(final byte[] b, final int off, final int len) throws IOException { - if (len == 0) { - return 0; - } - if (!validate(b, off, len)) { - return -1; - } - //If buffer is empty, then fill the buffer. - if (bCursor == limit) { - //If EOF, then return -1 - if (fCursor >= contentLength) { - return -1; - } - - long bytesRead = 0; - //reset buffer to initial state - i.e., throw away existing data - bCursor = 0; - limit = 0; - if (buffer == null) { - LOG.debug("created new buffer size {}", bufferSize); - buffer = new byte[bufferSize]; - } - - // Reset Read Type back to normal and set again based on code flow. - tracingContext.setReadType(ReadType.NORMAL_READ); - if (alwaysReadBufferSize) { - bytesRead = readInternal(fCursor, buffer, 0, bufferSize, false); - } else { - // Enable readAhead when reading sequentially - if (-1 == fCursorAfterLastRead || fCursorAfterLastRead == fCursor || b.length >= bufferSize) { - LOG.debug("Sequential read with read ahead size of {}", bufferSize); - bytesRead = readInternal(fCursor, buffer, 0, bufferSize, false); - } else { - // Enabling read ahead for random reads as well to reduce number of remote calls. - int lengthWithReadAhead = Math.min(b.length + readAheadRange, bufferSize); - LOG.debug("Random read with read ahead size of {}", lengthWithReadAhead); - bytesRead = readInternal(fCursor, buffer, 0, lengthWithReadAhead, true); - } - } - if (firstRead) { - firstRead = false; - } - if (bytesRead == -1) { - return -1; - } - - limit += bytesRead; - fCursor += bytesRead; - fCursorAfterLastRead = fCursor; - } - return copyToUserBuffer(b, off, len); - } + protected abstract int readOneBlock(final byte[] b, final int off, final int len) throws IOException; - private int readFileCompletely(final byte[] b, final int off, final int len) + protected int readFileCompletely(final byte[] b, final int off, final int len) throws IOException { if (len == 0) { return 0; @@ -397,7 +274,7 @@ private int readFileCompletely(final byte[] b, final int off, final int len) } // To do footer read of files when enabled. - private int readLastBlock(final byte[] b, final int off, final int len) + protected int readLastBlock(final byte[] b, final int off, final int len) throws IOException { if (len == 0) { return 0; @@ -472,7 +349,7 @@ private void restorePointerState() { this.bCursor = this.bCursorBkp; } - private boolean validate(final byte[] b, final int off, final int len) + protected boolean validate(final byte[] b, final int off, final int len) throws IOException { if (closed) { throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED); @@ -492,7 +369,7 @@ private boolean validate(final byte[] b, final int off, final int len) return true; } - private int copyToUserBuffer(byte[] b, int off, int len){ + protected int copyToUserBuffer(byte[] b, int off, int len){ //If there is anything in the buffer, then return lesser of (requested bytes) and (bytes in buffer) //(bytes returned may be less than requested) int bytesRemaining = limit - bCursor; @@ -511,7 +388,7 @@ private int copyToUserBuffer(byte[] b, int off, int len){ return bytesToRead; } - private int readInternal(final long position, final byte[] b, final int offset, final int length, + protected int readInternal(final long position, final byte[] b, final int offset, final int length, final boolean bypassReadAhead) throws IOException { if (isReadAheadEnabled() && !bypassReadAhead) { // try reading from read-ahead @@ -623,7 +500,7 @@ int readRemote(long position, byte[] b, int offset, int length, TracingContext t /** * Increment Read Operations. */ - private void incrementReadOps() { + protected void incrementReadOps() { if (statistics != null) { statistics.incrementReadOps(1); } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsSequentialInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsSequentialInputStream.java new file mode 100644 index 0000000000000..a142e9e0a0b05 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsSequentialInputStream.java @@ -0,0 +1,168 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.services; + +import java.io.IOException; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.azurebfs.constants.ReadType; +import org.apache.hadoop.fs.azurebfs.utils.TracingContext; + +import static java.lang.Math.max; + +public class AbfsSequentialInputStream extends AbfsInputStream { + + public AbfsSequentialInputStream( + final AbfsClient client, + final FileSystem.Statistics statistics, + final String path, + final long contentLength, + final AbfsInputStreamContext abfsInputStreamContext, + final String eTag, + TracingContext tracingContext) { + super(client, statistics, path, contentLength, + abfsInputStreamContext, eTag, tracingContext); + } + + @Override + public int read(long position, byte[] buffer, int offset, int length) + throws IOException { + return super.read(position, buffer, offset, length); + } + + @Override + public int read() throws IOException { + return super.read(); + } + + @Override + public synchronized int read(final byte[] b, final int off, final int len) throws IOException { + // check if buffer is null before logging the length + if (b != null) { + LOG.debug("read requested b.length = {} offset = {} len = {}", b.length, + off, len); + } else { + LOG.debug("read requested b = null offset = {} len = {}", off, len); + } + + int currentOff = off; + int currentLen = len; + int lastReadBytes; + int totalReadBytes = 0; + if (streamStatistics != null) { + streamStatistics.readOperationStarted(); + } + incrementReadOps(); + do { + + // limit is the maximum amount of data present in buffer. + // fCursor is the current file pointer. Thus maximum we can + // go back and read from buffer is fCursor - limit. + // There maybe case that we read less than requested data. + long filePosAtStartOfBuffer = fCursor - limit; + if (abfsReadFooterMetrics != null) { + abfsReadFooterMetrics.updateReadMetrics(filePathIdentifier, len, contentLength, nextReadPos); + } + if (nextReadPos >= filePosAtStartOfBuffer && nextReadPos <= fCursor) { + // Determining position in buffer from where data is to be read. + bCursor = (int) (nextReadPos - filePosAtStartOfBuffer); + + // When bCursor == limit, buffer will be filled again. + // So in this case we are not actually reading from buffer. + if (bCursor != limit && streamStatistics != null) { + streamStatistics.seekInBuffer(); + } + } else { + // Clearing the buffer and setting the file pointer + // based on previous seek() call. + fCursor = nextReadPos; + limit = 0; + bCursor = 0; + } + if (shouldReadFully()) { + lastReadBytes = readFileCompletely(b, currentOff, currentLen); + } else if (shouldReadLastBlock()) { + lastReadBytes = readLastBlock(b, currentOff, currentLen); + } else { + lastReadBytes = readOneBlock(b, currentOff, currentLen); + } + if (lastReadBytes > 0) { + currentOff += lastReadBytes; + currentLen -= lastReadBytes; + totalReadBytes += lastReadBytes; + } + if (currentLen <= 0 || currentLen > b.length - currentOff) { + break; + } + } while (lastReadBytes > 0); + return totalReadBytes > 0 ? totalReadBytes : lastReadBytes; + } + + @Override + protected int readOneBlock(final byte[] b, final int off, final int len) throws IOException { + if (len == 0) { + return 0; + } + if (!validate(b, off, len)) { + return -1; + } + //If buffer is empty, then fill the buffer. + if (bCursor == limit) { + //If EOF, then return -1 + if (fCursor >= contentLength) { + return -1; + } + + long bytesRead = 0; + //reset buffer to initial state - i.e., throw away existing data + bCursor = 0; + limit = 0; + if (buffer == null) { + LOG.debug("created new buffer size {}", bufferSize); + buffer = new byte[bufferSize]; + } + + // Reset Read Type back to normal and set again based on code flow. + tracingContext.setReadType(ReadType.NORMAL_READ); + bytesRead = readInternal(fCursor, buffer, 0, bufferSize, false); + if (firstRead) { + firstRead = false; + } + if (bytesRead == -1) { + return -1; + } + + limit += bytesRead; + fCursor += bytesRead; + fCursorAfterLastRead = fCursor; + } + return copyToUserBuffer(b, off, len); + } + + private boolean shouldReadFully() { + return this.firstRead && this.context.readSmallFilesCompletely() + && this.contentLength <= this.bufferSize; + } + + private boolean shouldReadLastBlock() { + long footerStart = max(0, this.contentLength - FOOTER_SIZE); + return this.firstRead && this.context.optimizeFooterRead() + && this.fCursor >= footerStart; + } +} diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsInputStreamStatistics.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsInputStreamStatistics.java index 6b87f1b73ef20..c215094eca0ce 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsInputStreamStatistics.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsInputStreamStatistics.java @@ -26,6 +26,7 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.azurebfs.services.AbfsAdaptiveInputStream; import org.apache.hadoop.fs.azurebfs.services.AbfsInputStream; import org.apache.hadoop.fs.azurebfs.services.AbfsInputStreamContext; import org.apache.hadoop.fs.azurebfs.services.AbfsInputStreamStatisticsImpl; @@ -275,7 +276,7 @@ public void testWithNullStreamStatistics() throws IOException { getTestTracingContext(fs, false), null); // AbfsInputStream with no StreamStatistics. - in = new AbfsInputStream(fs.getAbfsClient(), null, + in = new AbfsAdaptiveInputStream(fs.getAbfsClient(), null, nullStatFilePath.toUri().getPath(), ONE_KB, abfsInputStreamContext, abfsRestOperation.getResult().getResponseHeader("ETag"), getTestTracingContext(fs, false)); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java index 93df6529cb869..c2725420edb7e 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java @@ -148,7 +148,7 @@ AbfsInputStream getAbfsInputStream(AbfsClient mockAbfsClient, String fileName) throws IOException { AbfsInputStreamContext inputStreamContext = new AbfsInputStreamContext(-1); // Create AbfsInputStream with the client instance - AbfsInputStream inputStream = new AbfsInputStream( + AbfsInputStream inputStream = new AbfsAdaptiveInputStream( mockAbfsClient, null, FORWARD_SLASH + fileName, @@ -176,7 +176,7 @@ public AbfsInputStream getAbfsInputStream(AbfsClient abfsClient, int readAheadBlockSize) throws IOException { AbfsInputStreamContext inputStreamContext = new AbfsInputStreamContext(-1); // Create AbfsInputStream with the client instance - AbfsInputStream inputStream = new AbfsInputStream( + AbfsInputStream inputStream = new AbfsAdaptiveInputStream( abfsClient, null, FORWARD_SLASH + fileName, From 16253aa53ca290e26bb79fb3e934eca028d22d00 Mon Sep 17 00:00:00 2001 From: Anuj Modi Date: Mon, 29 Dec 2025 20:35:41 -0800 Subject: [PATCH 02/14] RandomIS --- .../services/AbfsAdaptiveInputStream.java | 77 +------- .../fs/azurebfs/services/AbfsInputPolicy.java | 25 ++- .../fs/azurebfs/services/AbfsInputStream.java | 73 +++++++- .../services/AbfsPrefetchInputStream.java | 100 +++++++++++ .../services/AbfsRandomInputStream.java | 104 +++++++++++ .../services/AbfsSequentialInputStream.java | 168 ------------------ 6 files changed, 301 insertions(+), 246 deletions(-) create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPrefetchInputStream.java create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRandomInputStream.java delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsSequentialInputStream.java diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java index 1828aa5e17a80..03c768c96e7de 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java @@ -53,67 +53,10 @@ public int read() throws IOException { @Override public synchronized int read(final byte[] b, final int off, final int len) throws IOException { - // check if buffer is null before logging the length - if (b != null) { - LOG.debug("read requested b.length = {} offset = {} len = {}", b.length, - off, len); - } else { - LOG.debug("read requested b = null offset = {} len = {}", off, len); - } - - int currentOff = off; - int currentLen = len; - int lastReadBytes; - int totalReadBytes = 0; - if (streamStatistics != null) { - streamStatistics.readOperationStarted(); - } - incrementReadOps(); - do { - - // limit is the maximum amount of data present in buffer. - // fCursor is the current file pointer. Thus maximum we can - // go back and read from buffer is fCursor - limit. - // There maybe case that we read less than requested data. - long filePosAtStartOfBuffer = fCursor - limit; - if (abfsReadFooterMetrics != null) { - abfsReadFooterMetrics.updateReadMetrics(filePathIdentifier, len, contentLength, nextReadPos); - } - if (nextReadPos >= filePosAtStartOfBuffer && nextReadPos <= fCursor) { - // Determining position in buffer from where data is to be read. - bCursor = (int) (nextReadPos - filePosAtStartOfBuffer); - - // When bCursor == limit, buffer will be filled again. - // So in this case we are not actually reading from buffer. - if (bCursor != limit && streamStatistics != null) { - streamStatistics.seekInBuffer(); - } - } else { - // Clearing the buffer and setting the file pointer - // based on previous seek() call. - fCursor = nextReadPos; - limit = 0; - bCursor = 0; - } - if (shouldReadFully()) { - lastReadBytes = readFileCompletely(b, currentOff, currentLen); - } else if (shouldReadLastBlock()) { - lastReadBytes = readLastBlock(b, currentOff, currentLen); - } else { - lastReadBytes = readOneBlock(b, currentOff, currentLen); - } - if (lastReadBytes > 0) { - currentOff += lastReadBytes; - currentLen -= lastReadBytes; - totalReadBytes += lastReadBytes; - } - if (currentLen <= 0 || currentLen > b.length - currentOff) { - break; - } - } while (lastReadBytes > 0); - return totalReadBytes > 0 ? totalReadBytes : lastReadBytes; + return super.read(b, off, len); } + @Override protected int readOneBlock(final byte[] b, final int off, final int len) throws IOException { if (len == 0) { @@ -148,7 +91,10 @@ protected int readOneBlock(final byte[] b, final int off, final int len) throws LOG.debug("Sequential read with read ahead size of {}", bufferSize); bytesRead = readInternal(fCursor, buffer, 0, bufferSize, false); } else { - // Enabling read ahead for random reads as well to reduce number of remote calls. + /* + * Disable queuing prefetches when random read pattern detected. + * Instead, read ahead only for readAheadRange above what is asked by caller. + */ int lengthWithReadAhead = Math.min(b.length + readAheadRange, bufferSize); LOG.debug("Random read with read ahead size of {}", lengthWithReadAhead); bytesRead = readInternal(fCursor, buffer, 0, lengthWithReadAhead, true); @@ -167,15 +113,4 @@ protected int readOneBlock(final byte[] b, final int off, final int len) throws } return copyToUserBuffer(b, off, len); } - - private boolean shouldReadFully() { - return this.firstRead && this.context.readSmallFilesCompletely() - && this.contentLength <= this.bufferSize; - } - - private boolean shouldReadLastBlock() { - long footerStart = max(0, this.contentLength - FOOTER_SIZE); - return this.firstRead && this.context.optimizeFooterRead() - && this.fCursor >= footerStart; - } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputPolicy.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputPolicy.java index 8b98c9c8c5cf6..3d7a8c8d17319 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputPolicy.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputPolicy.java @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.fs.azurebfs.services; import java.util.Locale; @@ -19,8 +37,6 @@ public enum AbfsInputPolicy { SEQUENTIAL(FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL), RANDOM(FS_OPTION_OPENFILE_READ_POLICY_RANDOM), - VECTORED(FS_OPTION_OPENFILE_READ_POLICY_VECTOR), - LAYOUT("layout"), ADAPTIVE(FS_OPTION_OPENFILE_READ_POLICY_ADAPTIVE); private final String policy; @@ -38,11 +54,8 @@ String getPolicy() { return policy; } - public static AbfsInputPolicy getPolicy(String name, boolean isLayoutPresent) { + public static AbfsInputPolicy getPolicy(String name) { String trimmed = name.trim().toLowerCase(Locale.ENGLISH); - if (isLayoutPresent) { - return LAYOUT; - } switch (trimmed) { // all these options currently map to random IO. case FS_OPTION_OPENFILE_READ_POLICY_HBASE: diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java index 0730876bcfef7..0a88c400b5308 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java @@ -253,10 +253,81 @@ public int read() throws IOException { } @Override - public abstract int read(final byte[] b, final int off, final int len) throws IOException; + public synchronized int read(final byte[] b, final int off, final int len) throws IOException { + // check if buffer is null before logging the length + if (b != null) { + LOG.debug("read requested b.length = {} offset = {} len = {}", b.length, + off, len); + } else { + LOG.debug("read requested b = null offset = {} len = {}", off, len); + } + + int currentOff = off; + int currentLen = len; + int lastReadBytes; + int totalReadBytes = 0; + if (streamStatistics != null) { + streamStatistics.readOperationStarted(); + } + incrementReadOps(); + do { + + // limit is the maximum amount of data present in buffer. + // fCursor is the current file pointer. Thus maximum we can + // go back and read from buffer is fCursor - limit. + // There maybe case that we read less than requested data. + long filePosAtStartOfBuffer = fCursor - limit; + if (abfsReadFooterMetrics != null) { + abfsReadFooterMetrics.updateReadMetrics(filePathIdentifier, len, contentLength, nextReadPos); + } + if (nextReadPos >= filePosAtStartOfBuffer && nextReadPos <= fCursor) { + // Determining position in buffer from where data is to be read. + bCursor = (int) (nextReadPos - filePosAtStartOfBuffer); + + // When bCursor == limit, buffer will be filled again. + // So in this case we are not actually reading from buffer. + if (bCursor != limit && streamStatistics != null) { + streamStatistics.seekInBuffer(); + } + } else { + // Clearing the buffer and setting the file pointer + // based on previous seek() call. + fCursor = nextReadPos; + limit = 0; + bCursor = 0; + } + if (shouldReadFully()) { + lastReadBytes = readFileCompletely(b, currentOff, currentLen); + } else if (shouldReadLastBlock()) { + lastReadBytes = readLastBlock(b, currentOff, currentLen); + } else { + lastReadBytes = readOneBlock(b, currentOff, currentLen); + } + if (lastReadBytes > 0) { + currentOff += lastReadBytes; + currentLen -= lastReadBytes; + totalReadBytes += lastReadBytes; + } + if (currentLen <= 0 || currentLen > b.length - currentOff) { + break; + } + } while (lastReadBytes > 0); + return totalReadBytes > 0 ? totalReadBytes : lastReadBytes; + } protected abstract int readOneBlock(final byte[] b, final int off, final int len) throws IOException; + private boolean shouldReadFully() { + return this.firstRead && this.context.readSmallFilesCompletely() + && this.contentLength <= this.bufferSize; + } + + private boolean shouldReadLastBlock() { + long footerStart = max(0, this.contentLength - FOOTER_SIZE); + return this.firstRead && this.context.optimizeFooterRead() + && this.fCursor >= footerStart; + } + protected int readFileCompletely(final byte[] b, final int off, final int len) throws IOException { if (len == 0) { diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPrefetchInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPrefetchInputStream.java new file mode 100644 index 0000000000000..d91794870e137 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPrefetchInputStream.java @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.services; + +import java.io.IOException; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.azurebfs.utils.TracingContext; + +import static java.lang.Math.max; + +public class AbfsPrefetchInputStream extends AbfsInputStream { + + public AbfsPrefetchInputStream( + final AbfsClient client, + final FileSystem.Statistics statistics, + final String path, + final long contentLength, + final AbfsInputStreamContext abfsInputStreamContext, + final String eTag, + TracingContext tracingContext) { + super(client, statistics, path, contentLength, + abfsInputStreamContext, eTag, tracingContext); + } + + @Override + public int read(long position, byte[] buffer, int offset, int length) + throws IOException { + return super.read(position, buffer, offset, length); + } + + @Override + public int read() throws IOException { + return super.read(); + } + + @Override + public synchronized int read(final byte[] b, final int off, final int len) throws IOException { + return super.read(b, off, len); + } + + @Override + protected int readOneBlock(final byte[] b, final int off, final int len) throws IOException { + if (len == 0) { + return 0; + } + if (!validate(b, off, len)) { + return -1; + } + //If buffer is empty, then fill the buffer. + if (bCursor == limit) { + //If EOF, then return -1 + if (fCursor >= contentLength) { + return -1; + } + + long bytesRead = 0; + //reset buffer to initial state - i.e., throw away existing data + bCursor = 0; + limit = 0; + if (buffer == null) { + LOG.debug("created new buffer size {}", bufferSize); + buffer = new byte[bufferSize]; + } + + /* + * Always start with Prefetch even from first read. + * Even if out of order seek comes, prefetches will be triggered for next set of blocks. + */ + bytesRead = readInternal(fCursor, buffer, 0, bufferSize, false); + if (firstRead) { + firstRead = false; + } + if (bytesRead == -1) { + return -1; + } + + limit += bytesRead; + fCursor += bytesRead; + fCursorAfterLastRead = fCursor; + } + return copyToUserBuffer(b, off, len); + } +} diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRandomInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRandomInputStream.java new file mode 100644 index 0000000000000..b088251645302 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRandomInputStream.java @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.services; + +import java.io.IOException; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.azurebfs.constants.ReadType; +import org.apache.hadoop.fs.azurebfs.utils.TracingContext; + +import static java.lang.Math.max; + +public class AbfsRandomInputStream extends AbfsInputStream { + + public AbfsRandomInputStream( + final AbfsClient client, + final FileSystem.Statistics statistics, + final String path, + final long contentLength, + final AbfsInputStreamContext abfsInputStreamContext, + final String eTag, + TracingContext tracingContext) { + super(client, statistics, path, contentLength, + abfsInputStreamContext, eTag, tracingContext); + } + + @Override + public int read(long position, byte[] buffer, int offset, int length) + throws IOException { + return super.read(position, buffer, offset, length); + } + + @Override + public int read() throws IOException { + return super.read(); + } + + @Override + public synchronized int read(final byte[] b, final int off, final int len) throws IOException { + return super.read(b, off, len); + } + + @Override + protected int readOneBlock(final byte[] b, final int off, final int len) throws IOException { + if (len == 0) { + return 0; + } + if (!validate(b, off, len)) { + return -1; + } + //If buffer is empty, then fill the buffer. + if (bCursor == limit) { + //If EOF, then return -1 + if (fCursor >= contentLength) { + return -1; + } + + long bytesRead = 0; + //reset buffer to initial state - i.e., throw away existing data + bCursor = 0; + limit = 0; + if (buffer == null) { + LOG.debug("created new buffer size {}", bufferSize); + buffer = new byte[bufferSize]; + } + + /* + * Disable queuing prefetches when random read pattern detected. + * Instead, read ahead only for readAheadRange above what is asked by caller. + */ + tracingContext.setReadType(ReadType.NORMAL_READ); + int lengthWithReadAhead = Math.min(b.length + readAheadRange, bufferSize); + LOG.debug("Random read with read ahead size of {}", lengthWithReadAhead); + bytesRead = readInternal(fCursor, buffer, 0, lengthWithReadAhead, true); + if (firstRead) { + firstRead = false; + } + if (bytesRead == -1) { + return -1; + } + + limit += bytesRead; + fCursor += bytesRead; + fCursorAfterLastRead = fCursor; + } + return copyToUserBuffer(b, off, len); + } +} diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsSequentialInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsSequentialInputStream.java deleted file mode 100644 index a142e9e0a0b05..0000000000000 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsSequentialInputStream.java +++ /dev/null @@ -1,168 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azurebfs.services; - -import java.io.IOException; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.azurebfs.constants.ReadType; -import org.apache.hadoop.fs.azurebfs.utils.TracingContext; - -import static java.lang.Math.max; - -public class AbfsSequentialInputStream extends AbfsInputStream { - - public AbfsSequentialInputStream( - final AbfsClient client, - final FileSystem.Statistics statistics, - final String path, - final long contentLength, - final AbfsInputStreamContext abfsInputStreamContext, - final String eTag, - TracingContext tracingContext) { - super(client, statistics, path, contentLength, - abfsInputStreamContext, eTag, tracingContext); - } - - @Override - public int read(long position, byte[] buffer, int offset, int length) - throws IOException { - return super.read(position, buffer, offset, length); - } - - @Override - public int read() throws IOException { - return super.read(); - } - - @Override - public synchronized int read(final byte[] b, final int off, final int len) throws IOException { - // check if buffer is null before logging the length - if (b != null) { - LOG.debug("read requested b.length = {} offset = {} len = {}", b.length, - off, len); - } else { - LOG.debug("read requested b = null offset = {} len = {}", off, len); - } - - int currentOff = off; - int currentLen = len; - int lastReadBytes; - int totalReadBytes = 0; - if (streamStatistics != null) { - streamStatistics.readOperationStarted(); - } - incrementReadOps(); - do { - - // limit is the maximum amount of data present in buffer. - // fCursor is the current file pointer. Thus maximum we can - // go back and read from buffer is fCursor - limit. - // There maybe case that we read less than requested data. - long filePosAtStartOfBuffer = fCursor - limit; - if (abfsReadFooterMetrics != null) { - abfsReadFooterMetrics.updateReadMetrics(filePathIdentifier, len, contentLength, nextReadPos); - } - if (nextReadPos >= filePosAtStartOfBuffer && nextReadPos <= fCursor) { - // Determining position in buffer from where data is to be read. - bCursor = (int) (nextReadPos - filePosAtStartOfBuffer); - - // When bCursor == limit, buffer will be filled again. - // So in this case we are not actually reading from buffer. - if (bCursor != limit && streamStatistics != null) { - streamStatistics.seekInBuffer(); - } - } else { - // Clearing the buffer and setting the file pointer - // based on previous seek() call. - fCursor = nextReadPos; - limit = 0; - bCursor = 0; - } - if (shouldReadFully()) { - lastReadBytes = readFileCompletely(b, currentOff, currentLen); - } else if (shouldReadLastBlock()) { - lastReadBytes = readLastBlock(b, currentOff, currentLen); - } else { - lastReadBytes = readOneBlock(b, currentOff, currentLen); - } - if (lastReadBytes > 0) { - currentOff += lastReadBytes; - currentLen -= lastReadBytes; - totalReadBytes += lastReadBytes; - } - if (currentLen <= 0 || currentLen > b.length - currentOff) { - break; - } - } while (lastReadBytes > 0); - return totalReadBytes > 0 ? totalReadBytes : lastReadBytes; - } - - @Override - protected int readOneBlock(final byte[] b, final int off, final int len) throws IOException { - if (len == 0) { - return 0; - } - if (!validate(b, off, len)) { - return -1; - } - //If buffer is empty, then fill the buffer. - if (bCursor == limit) { - //If EOF, then return -1 - if (fCursor >= contentLength) { - return -1; - } - - long bytesRead = 0; - //reset buffer to initial state - i.e., throw away existing data - bCursor = 0; - limit = 0; - if (buffer == null) { - LOG.debug("created new buffer size {}", bufferSize); - buffer = new byte[bufferSize]; - } - - // Reset Read Type back to normal and set again based on code flow. - tracingContext.setReadType(ReadType.NORMAL_READ); - bytesRead = readInternal(fCursor, buffer, 0, bufferSize, false); - if (firstRead) { - firstRead = false; - } - if (bytesRead == -1) { - return -1; - } - - limit += bytesRead; - fCursor += bytesRead; - fCursorAfterLastRead = fCursor; - } - return copyToUserBuffer(b, off, len); - } - - private boolean shouldReadFully() { - return this.firstRead && this.context.readSmallFilesCompletely() - && this.contentLength <= this.bufferSize; - } - - private boolean shouldReadLastBlock() { - long footerStart = max(0, this.contentLength - FOOTER_SIZE); - return this.firstRead && this.context.optimizeFooterRead() - && this.fCursor >= footerStart; - } -} From 68f905cb91ecc2868e1e61e35b76f0c7322edba7 Mon Sep 17 00:00:00 2001 From: Anuj Modi Date: Mon, 29 Dec 2025 21:12:15 -0800 Subject: [PATCH 03/14] Configuration --- .../hadoop/fs/azurebfs/AbfsConfiguration.java | 8 +++ .../fs/azurebfs/AzureBlobFileSystemStore.java | 59 ++++++++----------- .../azurebfs/constants/ConfigurationKeys.java | 7 +++ .../constants/FileSystemConfigurations.java | 2 + .../services/AbfsAdaptiveInputStream.java | 17 ------ .../fs/azurebfs/services/AbfsInputPolicy.java | 10 +--- .../services/AbfsPrefetchInputStream.java | 16 ----- .../services/AbfsRandomInputStream.java | 16 ----- 8 files changed, 45 insertions(+), 90 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index e7591292c919a..e29be85a4de9e 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -656,6 +656,10 @@ public class AbfsConfiguration{ DefaultValue = DEFAULT_FS_AZURE_LOWEST_REQUEST_PRIORITY_VALUE) private int prefetchRequestPriorityValue; + @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_READ_POLICY, + DefaultValue = DEFAULT_FS_AZURE_READ_POLICY) + private String abfsReadPolicy; + private String clientProvidedEncryptionKey; private String clientProvidedEncryptionKeySHA; @@ -1381,6 +1385,10 @@ public String getPrefetchRequestPriorityValue() { return Integer.toString(prefetchRequestPriorityValue); } + public String getAbfsReadPolicy() { + return abfsReadPolicy; + } + /** * Enum config to allow user to pick format of x-ms-client-request-id header * @return tracingContextFormat config if valid, else default ALL_ID_FORMAT diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index 1c543fb81f0e7..c346573e3029e 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -51,6 +51,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; +import org.apache.hadoop.fs.azurebfs.services.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -77,8 +78,6 @@ import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidUriException; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.TrileanConversionException; import org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode; -import org.apache.hadoop.fs.azurebfs.services.AbfsAdaptiveInputStream; -import org.apache.hadoop.fs.azurebfs.services.ListResponseData; import org.apache.hadoop.fs.azurebfs.enums.Trilean; import org.apache.hadoop.fs.azurebfs.extensions.EncryptionContextProvider; import org.apache.hadoop.fs.azurebfs.extensions.ExtensionHelper; @@ -90,32 +89,6 @@ import org.apache.hadoop.fs.azurebfs.security.ContextEncryptionAdapter; import org.apache.hadoop.fs.azurebfs.security.ContextProviderEncryptionAdapter; import org.apache.hadoop.fs.azurebfs.security.NoContextEncryptionAdapter; -import org.apache.hadoop.fs.azurebfs.services.AbfsAclHelper; -import org.apache.hadoop.fs.azurebfs.services.AbfsClient; -import org.apache.hadoop.fs.azurebfs.services.AbfsClientContext; -import org.apache.hadoop.fs.azurebfs.services.AbfsClientContextBuilder; -import org.apache.hadoop.fs.azurebfs.services.AbfsClientHandler; -import org.apache.hadoop.fs.azurebfs.services.AbfsClientRenameResult; -import org.apache.hadoop.fs.azurebfs.services.AbfsCounters; -import org.apache.hadoop.fs.azurebfs.services.AbfsHttpOperation; -import org.apache.hadoop.fs.azurebfs.services.AbfsInputStream; -import org.apache.hadoop.fs.azurebfs.services.AbfsInputStreamContext; -import org.apache.hadoop.fs.azurebfs.services.AbfsInputStreamStatisticsImpl; -import org.apache.hadoop.fs.azurebfs.services.AbfsLease; -import org.apache.hadoop.fs.azurebfs.services.AbfsOutputStream; -import org.apache.hadoop.fs.azurebfs.services.AbfsOutputStreamContext; -import org.apache.hadoop.fs.azurebfs.services.AbfsOutputStreamStatisticsImpl; -import org.apache.hadoop.fs.azurebfs.services.AbfsPerfInfo; -import org.apache.hadoop.fs.azurebfs.services.AbfsPerfTracker; -import org.apache.hadoop.fs.azurebfs.services.AbfsPermission; -import org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation; -import org.apache.hadoop.fs.azurebfs.services.AuthType; -import org.apache.hadoop.fs.azurebfs.services.ExponentialRetryPolicy; -import org.apache.hadoop.fs.azurebfs.services.ListingSupport; -import org.apache.hadoop.fs.azurebfs.services.SharedKeyCredentials; -import org.apache.hadoop.fs.azurebfs.services.StaticRetryPolicy; -import org.apache.hadoop.fs.azurebfs.services.TailLatencyRequestTimeoutRetryPolicy; -import org.apache.hadoop.fs.azurebfs.services.VersionedFileStatus; import org.apache.hadoop.fs.azurebfs.utils.Base64; import org.apache.hadoop.fs.azurebfs.utils.CRC64; import org.apache.hadoop.fs.azurebfs.utils.DateTimeUtils; @@ -947,12 +920,30 @@ public AbfsInputStream openFileForRead(Path path, perfInfo.registerSuccess(true); - // Add statistics for InputStream - return new AbfsAdaptiveInputStream(getClient(), statistics, relativePath, - contentLength, populateAbfsInputStreamContext( - parameters.map(OpenFileParameters::getOptions), - contextEncryptionAdapter), - eTag, tracingContext); + AbfsInputPolicy inputPolicy = AbfsInputPolicy.getPolicy(getAbfsConfiguration().getAbfsReadPolicy()); + switch (inputPolicy) { + case SEQUENTIAL: + return new AbfsPrefetchInputStream(getClient(), statistics, relativePath, + contentLength, populateAbfsInputStreamContext( + parameters.map(OpenFileParameters::getOptions), + contextEncryptionAdapter), + eTag, tracingContext); + + case RANDOM: + return new AbfsRandomInputStream(getClient(), statistics, relativePath, + contentLength, populateAbfsInputStreamContext( + parameters.map(OpenFileParameters::getOptions), + contextEncryptionAdapter), + eTag, tracingContext); + + case ADAPTIVE: + default: + return new AbfsAdaptiveInputStream(getClient(), statistics, relativePath, + contentLength, populateAbfsInputStreamContext( + parameters.map(OpenFileParameters::getOptions), + contextEncryptionAdapter), + eTag, tracingContext); + } } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java index 3de55adcdabf1..6b13273f6f15b 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java @@ -21,6 +21,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Options; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.DOT; @@ -215,6 +216,12 @@ public final class ConfigurationKeys { public static final String FS_AZURE_READ_AHEAD_QUEUE_DEPTH = "fs.azure.readaheadqueue.depth"; public static final String FS_AZURE_ALWAYS_READ_BUFFER_SIZE = "fs.azure.read.alwaysReadBufferSize"; public static final String FS_AZURE_READ_AHEAD_BLOCK_SIZE = "fs.azure.read.readahead.blocksize"; + /** + * Provides hint for the read workload pattern. + * Possible Values Exposed in {@link Options.OpenFileOptions} + */ + public static final String FS_AZURE_READ_POLICY = "fs.azure.read.policy"; + /** Provides a config control to enable or disable ABFS Flush operations - * HFlush and HSync. Default is true. **/ public static final String FS_AZURE_ENABLE_FLUSH = "fs.azure.enable.flush"; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java index fb336da51966d..d46b53847fe14 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java @@ -22,6 +22,7 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_ADAPTIVE; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.EMPTY_STRING; /** @@ -108,6 +109,7 @@ public final class FileSystemConfigurations { public static final long MAX_AZURE_BLOCK_SIZE = 256 * 1024 * 1024L; // changing default abfs blocksize to 256MB public static final String AZURE_BLOCK_LOCATION_HOST_DEFAULT = "localhost"; public static final int DEFAULT_AZURE_LIST_MAX_RESULTS = 5000; + public static final String DEFAULT_FS_AZURE_READ_POLICY = FS_OPTION_OPENFILE_READ_POLICY_ADAPTIVE; public static final String SERVER_SIDE_ENCRYPTION_ALGORITHM = "AES256"; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java index 03c768c96e7de..e4497dcbf6ab2 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java @@ -40,23 +40,6 @@ public AbfsAdaptiveInputStream( abfsInputStreamContext, eTag, tracingContext); } - @Override - public int read(long position, byte[] buffer, int offset, int length) - throws IOException { - return super.read(position, buffer, offset, length); - } - - @Override - public int read() throws IOException { - return super.read(); - } - - @Override - public synchronized int read(final byte[] b, final int off, final int len) throws IOException { - return super.read(b, off, len); - } - - @Override protected int readOneBlock(final byte[] b, final int off, final int len) throws IOException { if (len == 0) { diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputPolicy.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputPolicy.java index 3d7a8c8d17319..865e322d909fa 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputPolicy.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputPolicy.java @@ -21,16 +21,12 @@ import java.util.Locale; import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_ADAPTIVE; -import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_AVRO; import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_COLUMNAR; -import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_CSV; import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_HBASE; -import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_JSON; import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_ORC; import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_PARQUET; import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_RANDOM; import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL; -import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_VECTOR; import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE; public enum AbfsInputPolicy { @@ -66,12 +62,12 @@ public static AbfsInputPolicy getPolicy(String name) { return RANDOM; // handle the sequential formats. - case FS_OPTION_OPENFILE_READ_POLICY_AVRO: - case FS_OPTION_OPENFILE_READ_POLICY_CSV: - case FS_OPTION_OPENFILE_READ_POLICY_JSON: case FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL: case FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE: return SEQUENTIAL; + + // Everything else including ABFS Default Policy maps to Adaptive + case FS_OPTION_OPENFILE_READ_POLICY_ADAPTIVE: default: return ADAPTIVE; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPrefetchInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPrefetchInputStream.java index d91794870e137..5cb5ec5beba83 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPrefetchInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPrefetchInputStream.java @@ -39,22 +39,6 @@ public AbfsPrefetchInputStream( abfsInputStreamContext, eTag, tracingContext); } - @Override - public int read(long position, byte[] buffer, int offset, int length) - throws IOException { - return super.read(position, buffer, offset, length); - } - - @Override - public int read() throws IOException { - return super.read(); - } - - @Override - public synchronized int read(final byte[] b, final int off, final int len) throws IOException { - return super.read(b, off, len); - } - @Override protected int readOneBlock(final byte[] b, final int off, final int len) throws IOException { if (len == 0) { diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRandomInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRandomInputStream.java index b088251645302..beac642213977 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRandomInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRandomInputStream.java @@ -40,22 +40,6 @@ public AbfsRandomInputStream( abfsInputStreamContext, eTag, tracingContext); } - @Override - public int read(long position, byte[] buffer, int offset, int length) - throws IOException { - return super.read(position, buffer, offset, length); - } - - @Override - public int read() throws IOException { - return super.read(); - } - - @Override - public synchronized int read(final byte[] b, final int off, final int len) throws IOException { - return super.read(b, off, len); - } - @Override protected int readOneBlock(final byte[] b, final int off, final int len) throws IOException { if (len == 0) { From bb42e3c0d4636befa9a9f3ac7ff16005e0249ff2 Mon Sep 17 00:00:00 2001 From: Anuj Modi Date: Tue, 30 Dec 2025 04:29:32 -0800 Subject: [PATCH 04/14] Added Tests --- .../hadoop/fs/azurebfs/AbfsConfiguration.java | 5 + .../fs/azurebfs/constants/ReadType.java | 4 + .../services/AbfsAdaptiveInputStream.java | 1 + .../fs/azurebfs/services/AbfsInputPolicy.java | 6 - .../services/AbfsRandomInputStream.java | 2 +- .../services/ITestAbfsInputStream.java | 5 +- .../services/TestAbfsInputStream.java | 151 ++++++++++++++++-- 7 files changed, 154 insertions(+), 20 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index e29be85a4de9e..97ddda538ee76 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -2087,6 +2087,11 @@ public void setIsChecksumValidationEnabled(boolean isChecksumValidationEnabled) this.isChecksumValidationEnabled = isChecksumValidationEnabled; } + @VisibleForTesting + public void setAbfsReadPolicy(String readPolicy) { + abfsReadPolicy = readPolicy; + } + public boolean isFullBlobChecksumValidationEnabled() { return isFullBlobChecksumValidationEnabled; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ReadType.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ReadType.java index 332a5a5ac56e2..51391cc747740 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ReadType.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ReadType.java @@ -48,6 +48,10 @@ public enum ReadType { * Only triggered when small file read optimization kicks in. */ SMALLFILE_READ("SR"), + /** + * Reads from Random Input Stream with read ahead up to readAheadRange + */ + RANDOM_READ("RR"), /** * None of the above read types were applicable. */ diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java index e4497dcbf6ab2..048951ee44417 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java @@ -78,6 +78,7 @@ protected int readOneBlock(final byte[] b, final int off, final int len) throws * Disable queuing prefetches when random read pattern detected. * Instead, read ahead only for readAheadRange above what is asked by caller. */ + tracingContext.setReadType(ReadType.RANDOM_READ); int lengthWithReadAhead = Math.min(b.length + readAheadRange, bufferSize); LOG.debug("Random read with read ahead size of {}", lengthWithReadAhead); bytesRead = readInternal(fCursor, buffer, 0, lengthWithReadAhead, true); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputPolicy.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputPolicy.java index 865e322d909fa..6e854824969c1 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputPolicy.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputPolicy.java @@ -22,7 +22,6 @@ import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_ADAPTIVE; import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_COLUMNAR; -import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_HBASE; import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_ORC; import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_PARQUET; import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_RANDOM; @@ -46,15 +45,10 @@ public String toString() { return policy; } - String getPolicy() { - return policy; - } - public static AbfsInputPolicy getPolicy(String name) { String trimmed = name.trim().toLowerCase(Locale.ENGLISH); switch (trimmed) { // all these options currently map to random IO. - case FS_OPTION_OPENFILE_READ_POLICY_HBASE: case FS_OPTION_OPENFILE_READ_POLICY_RANDOM: case FS_OPTION_OPENFILE_READ_POLICY_COLUMNAR: case FS_OPTION_OPENFILE_READ_POLICY_ORC: diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRandomInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRandomInputStream.java index beac642213977..910c9cde4404d 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRandomInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRandomInputStream.java @@ -68,7 +68,7 @@ protected int readOneBlock(final byte[] b, final int off, final int len) throws * Disable queuing prefetches when random read pattern detected. * Instead, read ahead only for readAheadRange above what is asked by caller. */ - tracingContext.setReadType(ReadType.NORMAL_READ); + tracingContext.setReadType(ReadType.RANDOM_READ); int lengthWithReadAhead = Math.min(b.length + readAheadRange, bufferSize); LOG.debug("Random read with read ahead size of {}", lengthWithReadAhead); bytesRead = readInternal(fCursor, buffer, 0, lengthWithReadAhead, true); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsInputStream.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsInputStream.java index 938f5f4300ce9..03fdabe65f741 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsInputStream.java @@ -19,6 +19,7 @@ package org.apache.hadoop.fs.azurebfs.services; import java.io.IOException; +import java.io.InputStream; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; @@ -31,8 +32,10 @@ import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; +import static org.apache.hadoop.fs.Options.OpenFileOptions.*; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.ONE_MB; import static org.apache.hadoop.fs.azurebfs.services.AbfsInputStreamTestUtils.HUNDRED; +import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; @@ -117,7 +120,7 @@ public void testAzureBlobFileSystemBackReferenceInInputStream() FSDataInputStream in = getFileSystem().open(path)) { AbfsInputStream abfsInputStream = (AbfsInputStream) in.getWrappedStream(); - Assertions.assertThat(abfsInputStream.getFsBackRef().isNull()) + assertThat(abfsInputStream.getFsBackRef().isNull()) .describedAs("BackReference in input stream should not be null") .isFalse(); } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java index c2725420edb7e..503a1581f678f 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java @@ -19,6 +19,7 @@ package org.apache.hadoop.fs.azurebfs.services; import java.io.IOException; +import java.io.InputStream; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; @@ -57,17 +58,16 @@ import org.apache.hadoop.fs.azurebfs.utils.TracingHeaderVersion; import org.apache.hadoop.fs.impl.OpenFileParameters; +import static org.apache.hadoop.fs.Options.OpenFileOptions.*; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_AVRO; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.COLON; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.EMPTY_STRING; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.SPLIT_NO_LIMIT; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENABLE_PREFETCH_REQUEST_PRIORITY; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.ONE_MB; import static org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations.X_MS_REQUEST_PRIORITY; -import static org.apache.hadoop.fs.azurebfs.constants.ReadType.DIRECT_READ; -import static org.apache.hadoop.fs.azurebfs.constants.ReadType.FOOTER_READ; -import static org.apache.hadoop.fs.azurebfs.constants.ReadType.MISSEDCACHE_READ; -import static org.apache.hadoop.fs.azurebfs.constants.ReadType.NORMAL_READ; -import static org.apache.hadoop.fs.azurebfs.constants.ReadType.PREFETCH_READ; -import static org.apache.hadoop.fs.azurebfs.constants.ReadType.SMALLFILE_READ; +import static org.apache.hadoop.fs.azurebfs.constants.ReadType.*; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; @@ -92,8 +92,7 @@ /** * Unit test AbfsInputStream. */ -public class TestAbfsInputStream extends - AbstractAbfsIntegrationTest { +public class TestAbfsInputStream extends AbstractAbfsIntegrationTest { private static final int ONE_KB = 1 * 1024; private static final int TWO_KB = 2 * 1024; @@ -848,6 +847,7 @@ public void testReadTypeInTracingContextHeader() throws Exception { fileSize = 3 * ONE_MB; // To make sure multiple blocks are read with MR totalReadCalls += 3; // 3 block of 1MB. Mockito.doReturn(0).when(spiedConfig).getReadAheadQueueDepth(); + Mockito.doReturn(FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL).when(spiedConfig).getAbfsReadPolicy(); doReturn(true).when(spiedConfig).isReadAheadEnabled(); testReadTypeInTracingContextHeaderInternal(spiedFs, fileSize, MISSEDCACHE_READ, 3, totalReadCalls); @@ -881,6 +881,15 @@ public void testReadTypeInTracingContextHeader() throws Exception { doReturn(false).when(spiedConfig).optimizeFooterRead(); testReadTypeInTracingContextHeaderInternal(spiedFs, fileSize, SMALLFILE_READ, 1, totalReadCalls); + /* + * Test to verify Random Read Type. + * Settin Read Policy to Parquet ensures Random Read Type. + */ + fileSize = 3 * ONE_MB; // To make sure multiple blocks are read. + totalReadCalls += 3; // Full file will be read along with footer. + doReturn(FS_OPTION_OPENFILE_READ_POLICY_PARQUET).when(spiedConfig).getAbfsReadPolicy(); + testReadTypeInTracingContextHeaderInternal(spiedFs, fileSize, RANDOM_READ, 1, totalReadCalls); + /* * Test to verify Direct Read Type and a read from random position. * Separate AbfsInputStream method needs to be called. @@ -904,7 +913,7 @@ public void testReadTypeInTracingContextHeader() throws Exception { private void testReadTypeInTracingContextHeaderInternal(AzureBlobFileSystem fs, int fileSize, ReadType readType, int numOfReadCalls, int totalReadCalls) throws Exception { Path testPath = createTestFile(fs, fileSize); - readFile(fs, testPath, fileSize); + readFile(fs, testPath, fileSize, readType); assertReadTypeInClientRequestId(fs, numOfReadCalls, totalReadCalls, readType); } @@ -937,6 +946,111 @@ public void testPrefetchReadAddsPriorityHeaderWithDifferentConfigs() executePrefetchReadTest(tracingContext1, configuration1, false); } + /** + * Test to verify that the correct AbfsInputStream instance is created + * based on the read policy set in AbfsConfiguration. + */ + @Test + public void testAbfsInputStreamInstance() throws Exception { + AzureBlobFileSystem fs = getFileSystem(); + Path path = new Path("/testPath"); + fs.create(path).close(); + + // Assert that Sequential Read Policy uses Prefetch Input Stream + getAbfsStore(fs).getAbfsConfiguration().setAbfsReadPolicy(FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL); + InputStream stream = fs.open(path).getWrappedStream(); + assertThat(stream).isInstanceOf(AbfsPrefetchInputStream.class); + stream.close(); + + // Assert that Adaptive Read Policy uses Adaptive Input Stream + getAbfsStore(fs).getAbfsConfiguration().setAbfsReadPolicy(FS_OPTION_OPENFILE_READ_POLICY_ADAPTIVE); + stream = fs.open(path).getWrappedStream(); + assertThat(stream).isInstanceOf(AbfsAdaptiveInputStream.class); + stream.close(); + + // Assert that Parquet Read Policy uses Random Input Stream + getAbfsStore(fs).getAbfsConfiguration().setAbfsReadPolicy(FS_OPTION_OPENFILE_READ_POLICY_PARQUET); + stream = fs.open(path).getWrappedStream(); + assertThat(stream).isInstanceOf(AbfsRandomInputStream.class); + stream.close(); + + // Assert that Avro Read Policy uses Adaptive Input Stream + getAbfsStore(fs).getAbfsConfiguration().setAbfsReadPolicy(FS_OPTION_OPENFILE_READ_POLICY_AVRO); + stream = fs.open(path).getWrappedStream(); + assertThat(stream).isInstanceOf(AbfsAdaptiveInputStream.class); + stream.close(); + } + + @Test + public void testPrefetchInputStreamQueuesPrefetches() throws Exception { + AzureBlobFileSystem spiedFs = Mockito.spy(getFileSystem()); + AzureBlobFileSystemStore spiedStore = Mockito.spy(spiedFs.getAbfsStore()); + AbfsConfiguration spiedConfig = Mockito.spy(spiedStore.getAbfsConfiguration()); + AbfsClient spiedClient = Mockito.spy(spiedStore.getClient()); + Mockito.doReturn(ONE_MB).when(spiedConfig).getReadBufferSize(); + Mockito.doReturn(ONE_MB).when(spiedConfig).getReadAheadBlockSize(); + Mockito.doReturn(spiedClient).when(spiedStore).getClient(); + Mockito.doReturn(spiedStore).when(spiedFs).getAbfsStore(); + Mockito.doReturn(spiedConfig).when(spiedStore).getAbfsConfiguration(); + + int fileSize = 3 * ONE_MB; // To make sure multiple blocks are read. + int totalReadCalls = 3; + Mockito.doReturn(3).when(spiedConfig).getReadAheadQueueDepth(); + Mockito.doReturn(FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL).when(spiedConfig).getAbfsReadPolicy(); + testReadTypeInTracingContextHeaderInternal(spiedFs, fileSize, PREFETCH_READ, 3, totalReadCalls); + } + + @Test + public void testRandomInputStreamDoesNotQueuePrefetches() throws Exception { + AzureBlobFileSystem spiedFs = Mockito.spy(getFileSystem()); + AzureBlobFileSystemStore spiedStore = Mockito.spy(spiedFs.getAbfsStore()); + AbfsConfiguration spiedConfig = Mockito.spy(spiedStore.getAbfsConfiguration()); + AbfsClient spiedClient = Mockito.spy(spiedStore.getClient()); + Mockito.doReturn(ONE_MB).when(spiedConfig).getReadBufferSize(); + Mockito.doReturn(ONE_MB).when(spiedConfig).getReadAheadBlockSize(); + Mockito.doReturn(spiedClient).when(spiedStore).getClient(); + Mockito.doReturn(spiedStore).when(spiedFs).getAbfsStore(); + Mockito.doReturn(spiedConfig).when(spiedStore).getAbfsConfiguration(); + + int fileSize = 3 * ONE_MB; // To make sure multiple blocks are read. + int totalReadCalls = 3; + Mockito.doReturn(3).when(spiedConfig).getReadAheadQueueDepth(); + Mockito.doReturn(FS_OPTION_OPENFILE_READ_POLICY_PARQUET).when(spiedConfig).getAbfsReadPolicy(); + testReadTypeInTracingContextHeaderInternal(spiedFs, fileSize, RANDOM_READ, 3, totalReadCalls); + } + + @Test + public void testAdaptiveInputStream() throws Exception { + AzureBlobFileSystem spiedFs = Mockito.spy(getFileSystem()); + AzureBlobFileSystemStore spiedStore = Mockito.spy(spiedFs.getAbfsStore()); + AbfsConfiguration spiedConfig = Mockito.spy(spiedStore.getAbfsConfiguration()); + AbfsClient spiedClient = Mockito.spy(spiedStore.getClient()); + Mockito.doReturn(ONE_MB).when(spiedConfig).getReadBufferSize(); + Mockito.doReturn(ONE_MB).when(spiedConfig).getReadAheadBlockSize(); + Mockito.doReturn(ONE_KB).when(spiedConfig).getReadAheadRange(); + Mockito.doReturn(FS_OPTION_OPENFILE_READ_POLICY_ADAPTIVE).when(spiedConfig).getAbfsReadPolicy(); + Mockito.doReturn(spiedClient).when(spiedStore).getClient(); + Mockito.doReturn(spiedStore).when(spiedFs).getAbfsStore(); + Mockito.doReturn(spiedConfig).when(spiedStore).getAbfsConfiguration(); + + int fileSize = 10 * ONE_MB; + Path testPath = createTestFile(spiedFs, fileSize); + + try (FSDataInputStream iStream = spiedFs.open(testPath)) { + assertThat(iStream.getWrappedStream()).isInstanceOf(AbfsAdaptiveInputStream.class); + + // In order reads trigger prefetches in adaptive stream + int bytesRead = iStream.read(new byte[2 * ONE_MB], 0, 2 * ONE_MB); + assertReadTypeInClientRequestId(spiedFs, 3, 3, PREFETCH_READ); + assertThat(bytesRead).isEqualTo(2 * ONE_MB); + + // Out of order seek causes random read + iStream.seek(7 * ONE_MB); + bytesRead = iStream.read(new byte[ONE_MB/2], 0, ONE_MB/2); + assertReadTypeInClientRequestId(spiedFs, 1, 4, RANDOM_READ); + } + } + /* * Helper method to execute read and verify if priority header is added or not as expected */ @@ -1005,8 +1119,15 @@ private Path createTestFile(AzureBlobFileSystem fs, int fileSize) throws Excepti return testPath; } - private void readFile(AzureBlobFileSystem fs, Path testPath, int fileSize) throws Exception { + private void readFile(AzureBlobFileSystem fs, Path testPath, int fileSize, ReadType readType) throws Exception { try (FSDataInputStream iStream = fs.open(testPath)) { + if (readType == PREFETCH_READ || readType == MISSEDCACHE_READ) { + assertThat(iStream.getWrappedStream()).isInstanceOf(AbfsPrefetchInputStream.class); + } else if (readType == NORMAL_READ) { + assertThat(iStream.getWrappedStream()).isInstanceOf(AbfsAdaptiveInputStream.class); + } else if (readType == RANDOM_READ) { + assertThat(iStream.getWrappedStream()).isInstanceOf(AbfsRandomInputStream.class); + } int bytesRead = iStream.read(new byte[fileSize], 0, fileSize); assertThat(fileSize) @@ -1071,8 +1192,14 @@ private void verifyHeaderForReadTypeInTracingContextHeader(TracingContext tracin } assertThat(idList[OPERATION_INDEX]).describedAs("Operation Type Should Be Read") .isEqualTo(FSOperationType.READ.toString()); - assertThat(idList[READTYPE_INDEX]).describedAs("Read type in tracing context header should match") - .isEqualTo(readType.toString()); + if (readType == PREFETCH_READ) { + // For prefetch read, it might be missed cache as well. + assertThat(idList[READTYPE_INDEX]).describedAs("Read type in tracing context header should match") + .isIn(PREFETCH_READ.toString(), MISSEDCACHE_READ.toString()); + } else { + assertThat(idList[READTYPE_INDEX]).describedAs("Read type in tracing context header should match") + .isEqualTo(readType.toString()); + } } From 94d2336087bfeb09276c98ee5d49925f7c264694 Mon Sep 17 00:00:00 2001 From: Anuj Modi Date: Wed, 31 Dec 2025 02:58:39 -0800 Subject: [PATCH 05/14] PR Checks --- .../hadoop/fs/azurebfs/AbfsConfiguration.java | 8 +++++ .../fs/azurebfs/AzureBlobFileSystemStore.java | 32 ++++++++++++++++++- .../constants/FileSystemConfigurations.java | 2 +- .../services/AbfsAdaptiveInputStream.java | 9 ++++++ .../fs/azurebfs/services/AbfsInputPolicy.java | 9 ++++++ .../fs/azurebfs/services/AbfsInputStream.java | 24 +++++++------- .../services/AbfsPrefetchInputStream.java | 8 +++++ .../services/AbfsRandomInputStream.java | 7 ++-- .../services/ITestAbfsInputStream.java | 3 -- .../services/TestAbfsInputStream.java | 22 ++----------- 10 files changed, 86 insertions(+), 38 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index 97ddda538ee76..70c5e5261259a 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -1385,6 +1385,10 @@ public String getPrefetchRequestPriorityValue() { return Integer.toString(prefetchRequestPriorityValue); } + /** + * Get the ABFS read policy set by user. + * @return the ABFS read policy. + */ public String getAbfsReadPolicy() { return abfsReadPolicy; } @@ -2087,6 +2091,10 @@ public void setIsChecksumValidationEnabled(boolean isChecksumValidationEnabled) this.isChecksumValidationEnabled = isChecksumValidationEnabled; } + /** + * Sets the ABFS read policy for testing purposes. + * @param readPolicy the read policy to set. + */ @VisibleForTesting public void setAbfsReadPolicy(String readPolicy) { abfsReadPolicy = readPolicy; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index c346573e3029e..14a95182f7cf1 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -51,7 +51,6 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; -import org.apache.hadoop.fs.azurebfs.services.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -89,6 +88,37 @@ import org.apache.hadoop.fs.azurebfs.security.ContextEncryptionAdapter; import org.apache.hadoop.fs.azurebfs.security.ContextProviderEncryptionAdapter; import org.apache.hadoop.fs.azurebfs.security.NoContextEncryptionAdapter; +import org.apache.hadoop.fs.azurebfs.services.AbfsAclHelper; +import org.apache.hadoop.fs.azurebfs.services.AbfsAdaptiveInputStream; +import org.apache.hadoop.fs.azurebfs.services.AbfsClient; +import org.apache.hadoop.fs.azurebfs.services.AbfsClientContext; +import org.apache.hadoop.fs.azurebfs.services.AbfsClientContextBuilder; +import org.apache.hadoop.fs.azurebfs.services.AbfsClientHandler; +import org.apache.hadoop.fs.azurebfs.services.AbfsClientRenameResult; +import org.apache.hadoop.fs.azurebfs.services.AbfsCounters; +import org.apache.hadoop.fs.azurebfs.services.AbfsHttpOperation; +import org.apache.hadoop.fs.azurebfs.services.AbfsInputPolicy; +import org.apache.hadoop.fs.azurebfs.services.AbfsInputStream; +import org.apache.hadoop.fs.azurebfs.services.AbfsInputStreamContext; +import org.apache.hadoop.fs.azurebfs.services.AbfsInputStreamStatisticsImpl; +import org.apache.hadoop.fs.azurebfs.services.AbfsLease; +import org.apache.hadoop.fs.azurebfs.services.AbfsOutputStream; +import org.apache.hadoop.fs.azurebfs.services.AbfsOutputStreamContext; +import org.apache.hadoop.fs.azurebfs.services.AbfsOutputStreamStatisticsImpl; +import org.apache.hadoop.fs.azurebfs.services.AbfsPerfInfo; +import org.apache.hadoop.fs.azurebfs.services.AbfsPerfTracker; +import org.apache.hadoop.fs.azurebfs.services.AbfsPermission; +import org.apache.hadoop.fs.azurebfs.services.AbfsPrefetchInputStream; +import org.apache.hadoop.fs.azurebfs.services.AbfsRandomInputStream; +import org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation; +import org.apache.hadoop.fs.azurebfs.services.AuthType; +import org.apache.hadoop.fs.azurebfs.services.ExponentialRetryPolicy; +import org.apache.hadoop.fs.azurebfs.services.ListingSupport; +import org.apache.hadoop.fs.azurebfs.services.ListResponseData; +import org.apache.hadoop.fs.azurebfs.services.SharedKeyCredentials; +import org.apache.hadoop.fs.azurebfs.services.StaticRetryPolicy; +import org.apache.hadoop.fs.azurebfs.services.TailLatencyRequestTimeoutRetryPolicy; +import org.apache.hadoop.fs.azurebfs.services.VersionedFileStatus; import org.apache.hadoop.fs.azurebfs.utils.Base64; import org.apache.hadoop.fs.azurebfs.utils.CRC64; import org.apache.hadoop.fs.azurebfs.utils.DateTimeUtils; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java index d46b53847fe14..54fcb6092d3dc 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java @@ -418,7 +418,7 @@ public final class FileSystemConfigurations { public static final boolean DEFAULT_FS_AZURE_ENABLE_CREATE_BLOB_IDEMPOTENCY = true; - public static final boolean DEFAULT_FS_AZURE_ENABLE_PREFETCH_REQUEST_PRIORITY = true; + public static final boolean DEFAULT_FS_AZURE_ENABLE_PREFETCH_REQUEST_PRIORITY = false; // The default traffic request priority is 3 (from service side) // The lowest priority a request can get is 7 diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java index 048951ee44417..15234dbd3b947 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java @@ -26,6 +26,12 @@ import static java.lang.Math.max; +/** + * Input stream implementation optimized for adaptive read patterns. + * This is the default implementation used for cases where user does not specify any input policy. + * It switches between sequential and random read optimizations based on the detected read pattern. + * It also keeps footer read and small file optimizations enabled. + */ public class AbfsAdaptiveInputStream extends AbfsInputStream { public AbfsAdaptiveInputStream( @@ -40,6 +46,9 @@ public AbfsAdaptiveInputStream( abfsInputStreamContext, eTag, tracingContext); } + /** + * {@inheritDoc} + */ @Override protected int readOneBlock(final byte[] b, final int off, final int len) throws IOException { if (len == 0) { diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputPolicy.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputPolicy.java index 6e854824969c1..c9dc01be9b729 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputPolicy.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputPolicy.java @@ -28,6 +28,10 @@ import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL; import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE; +/** + * Enum for ABFS Input Policies. + * Each policy maps to a particular implementation of {@link AbfsInputStream} + */ public enum AbfsInputPolicy { SEQUENTIAL(FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL), @@ -45,6 +49,11 @@ public String toString() { return policy; } + /** + * Get the enum constant from the string name. + * @param name policy name as configured by user + * @return the corresponding AbsInputPolicy to be used + */ public static AbfsInputPolicy getPolicy(String name) { String trimmed = name.trim().toLowerCase(Locale.ENGLISH); switch (trimmed) { diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java index 0a88c400b5308..c0838ba89c808 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java @@ -114,20 +114,20 @@ public abstract class AbfsInputStream extends FSInputStream implements CanUnbuff private int bCursorBkp; private long fCursorBkp; private long fCursorAfterLastReadBkp; - protected final AbfsReadFooterMetrics abfsReadFooterMetrics; + private final AbfsReadFooterMetrics abfsReadFooterMetrics; /** Stream statistics. */ - protected final AbfsInputStreamStatistics streamStatistics; + private final AbfsInputStreamStatistics streamStatistics; private long bytesFromReadAhead; // bytes read from readAhead; for testing private long bytesFromRemoteRead; // bytes read remotely; for testing private Listener listener; - protected final AbfsInputStreamContext context; + private final AbfsInputStreamContext context; private IOStatistics ioStatistics; - protected String filePathIdentifier; + private String filePathIdentifier; /** * This is the actual position within the object, used by * lazy seek to decide whether to seek on the next read or not. */ - protected long nextReadPos; + private long nextReadPos; /** ABFS instance to be held by the input stream to avoid GC close. */ private final BackReference fsBackRef; @@ -315,20 +315,20 @@ public synchronized int read(final byte[] b, final int off, final int len) throw return totalReadBytes > 0 ? totalReadBytes : lastReadBytes; } - protected abstract int readOneBlock(final byte[] b, final int off, final int len) throws IOException; - private boolean shouldReadFully() { return this.firstRead && this.context.readSmallFilesCompletely() - && this.contentLength <= this.bufferSize; + && this.contentLength <= this.bufferSize; } private boolean shouldReadLastBlock() { long footerStart = max(0, this.contentLength - FOOTER_SIZE); return this.firstRead && this.context.optimizeFooterRead() - && this.fCursor >= footerStart; + && this.fCursor >= footerStart; } - protected int readFileCompletely(final byte[] b, final int off, final int len) + protected abstract int readOneBlock(final byte[] b, final int off, final int len) throws IOException; + + private int readFileCompletely(final byte[] b, final int off, final int len) throws IOException { if (len == 0) { return 0; @@ -345,7 +345,7 @@ protected int readFileCompletely(final byte[] b, final int off, final int len) } // To do footer read of files when enabled. - protected int readLastBlock(final byte[] b, final int off, final int len) + private int readLastBlock(final byte[] b, final int off, final int len) throws IOException { if (len == 0) { return 0; @@ -571,7 +571,7 @@ int readRemote(long position, byte[] b, int offset, int length, TracingContext t /** * Increment Read Operations. */ - protected void incrementReadOps() { + private void incrementReadOps() { if (statistics != null) { statistics.incrementReadOps(1); } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPrefetchInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPrefetchInputStream.java index 5cb5ec5beba83..a8692c2d6961b 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPrefetchInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPrefetchInputStream.java @@ -25,6 +25,11 @@ import static java.lang.Math.max; +/** + * Input stream implementation optimized for prefetching data. + * This implementation always prefetches data in advance if enabled + * to optimize for sequential read patterns. + */ public class AbfsPrefetchInputStream extends AbfsInputStream { public AbfsPrefetchInputStream( @@ -39,6 +44,9 @@ public AbfsPrefetchInputStream( abfsInputStreamContext, eTag, tracingContext); } + /** + * {@inheritDoc} + */ @Override protected int readOneBlock(final byte[] b, final int off, final int len) throws IOException { if (len == 0) { diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRandomInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRandomInputStream.java index 910c9cde4404d..ac645841f52f9 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRandomInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRandomInputStream.java @@ -24,8 +24,11 @@ import org.apache.hadoop.fs.azurebfs.constants.ReadType; import org.apache.hadoop.fs.azurebfs.utils.TracingContext; -import static java.lang.Math.max; - +/** + * Input stream implementation optimized for random read patterns. + * This implementation disables prefetching of data blocks instead only + * reads ahead for a small range beyond what is requested by the caller. + */ public class AbfsRandomInputStream extends AbfsInputStream { public AbfsRandomInputStream( diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsInputStream.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsInputStream.java index 03fdabe65f741..2d8629294f643 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsInputStream.java @@ -19,7 +19,6 @@ package org.apache.hadoop.fs.azurebfs.services; import java.io.IOException; -import java.io.InputStream; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; @@ -29,10 +28,8 @@ import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem; import org.apache.hadoop.fs.azurebfs.utils.TracingContext; -import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; -import static org.apache.hadoop.fs.Options.OpenFileOptions.*; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.ONE_MB; import static org.apache.hadoop.fs.azurebfs.services.AbfsInputStreamTestUtils.HUNDRED; import static org.assertj.core.api.Assertions.assertThat; diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java index 503a1581f678f..4b7a0d6a18dd9 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java @@ -981,25 +981,6 @@ public void testAbfsInputStreamInstance() throws Exception { stream.close(); } - @Test - public void testPrefetchInputStreamQueuesPrefetches() throws Exception { - AzureBlobFileSystem spiedFs = Mockito.spy(getFileSystem()); - AzureBlobFileSystemStore spiedStore = Mockito.spy(spiedFs.getAbfsStore()); - AbfsConfiguration spiedConfig = Mockito.spy(spiedStore.getAbfsConfiguration()); - AbfsClient spiedClient = Mockito.spy(spiedStore.getClient()); - Mockito.doReturn(ONE_MB).when(spiedConfig).getReadBufferSize(); - Mockito.doReturn(ONE_MB).when(spiedConfig).getReadAheadBlockSize(); - Mockito.doReturn(spiedClient).when(spiedStore).getClient(); - Mockito.doReturn(spiedStore).when(spiedFs).getAbfsStore(); - Mockito.doReturn(spiedConfig).when(spiedStore).getAbfsConfiguration(); - - int fileSize = 3 * ONE_MB; // To make sure multiple blocks are read. - int totalReadCalls = 3; - Mockito.doReturn(3).when(spiedConfig).getReadAheadQueueDepth(); - Mockito.doReturn(FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL).when(spiedConfig).getAbfsReadPolicy(); - testReadTypeInTracingContextHeaderInternal(spiedFs, fileSize, PREFETCH_READ, 3, totalReadCalls); - } - @Test public void testRandomInputStreamDoesNotQueuePrefetches() throws Exception { AzureBlobFileSystem spiedFs = Mockito.spy(getFileSystem()); @@ -1148,11 +1129,14 @@ private void assertReadTypeInClientRequestId(AzureBlobFileSystem fs, int numOfRe ArgumentCaptor captor8 = ArgumentCaptor.forClass(ContextEncryptionAdapter.class); ArgumentCaptor captor9 = ArgumentCaptor.forClass(TracingContext.class); + List paths = captor1.getAllValues(); + System.out.println(paths); verify(fs.getAbfsStore().getClient(), times(totalReadCalls)).read( captor1.capture(), captor2.capture(), captor3.capture(), captor4.capture(), captor5.capture(), captor6.capture(), captor7.capture(), captor8.capture(), captor9.capture()); List tracingContextList = captor9.getAllValues(); + System.out.println(tracingContextList); if (readType == PREFETCH_READ) { /* * For Prefetch Enabled, first read can be Normal or Missed Cache Read. From 46b3e18e952a2531a9644b7f7f0c708ae21129c5 Mon Sep 17 00:00:00 2001 From: Anuj Modi Date: Tue, 6 Jan 2026 03:41:27 -0800 Subject: [PATCH 06/14] Checkstyle --- .../hadoop/fs/azurebfs/AbfsConfiguration.java | 2 +- .../constants/FileSystemConfigurations.java | 2 +- .../services/AbfsAdaptiveInputStream.java | 12 +++++-- .../fs/azurebfs/services/AbfsInputStream.java | 33 +++++++++++++++++++ 4 files changed, 45 insertions(+), 4 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index 70c5e5261259a..56d02d831cf3d 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -657,7 +657,7 @@ public class AbfsConfiguration{ private int prefetchRequestPriorityValue; @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_READ_POLICY, - DefaultValue = DEFAULT_FS_AZURE_READ_POLICY) + DefaultValue = DEFAULT_AZURE_READ_POLICY) private String abfsReadPolicy; private String clientProvidedEncryptionKey; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java index 54fcb6092d3dc..126b86652718e 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java @@ -109,7 +109,7 @@ public final class FileSystemConfigurations { public static final long MAX_AZURE_BLOCK_SIZE = 256 * 1024 * 1024L; // changing default abfs blocksize to 256MB public static final String AZURE_BLOCK_LOCATION_HOST_DEFAULT = "localhost"; public static final int DEFAULT_AZURE_LIST_MAX_RESULTS = 5000; - public static final String DEFAULT_FS_AZURE_READ_POLICY = FS_OPTION_OPENFILE_READ_POLICY_ADAPTIVE; + public static final String DEFAULT_AZURE_READ_POLICY = FS_OPTION_OPENFILE_READ_POLICY_ADAPTIVE; public static final String SERVER_SIDE_ENCRYPTION_ALGORITHM = "AES256"; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java index 15234dbd3b947..7ddbdf529d6be 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java @@ -24,8 +24,6 @@ import org.apache.hadoop.fs.azurebfs.constants.ReadType; import org.apache.hadoop.fs.azurebfs.utils.TracingContext; -import static java.lang.Math.max; - /** * Input stream implementation optimized for adaptive read patterns. * This is the default implementation used for cases where user does not specify any input policy. @@ -34,6 +32,16 @@ */ public class AbfsAdaptiveInputStream extends AbfsInputStream { + /** + * Constructs AbfsAdaptiveInputStream + * @param client AbfsClient to be used for read operations + * @param statistics to recordinput stream statistics + * @param path file path + * @param contentLength file content length + * @param abfsInputStreamContext input stream context + * @param eTag file eTag + * @param tracingContext tracing context to trace the read operations + */ public AbfsAdaptiveInputStream( final AbfsClient client, final FileSystem.Statistics statistics, diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java index c0838ba89c808..7096818f8a820 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java @@ -326,6 +326,14 @@ private boolean shouldReadLastBlock() { && this.fCursor >= footerStart; } + /** + * Read one block of data into buffer. + * @param b buffer + * @param off offset + * @param len length + * @return number of bytes read + * @throws IOException if there is an error + */ protected abstract int readOneBlock(final byte[] b, final int off, final int len) throws IOException; private int readFileCompletely(final byte[] b, final int off, final int len) @@ -420,6 +428,14 @@ private void restorePointerState() { this.bCursor = this.bCursorBkp; } + /** + * Validate the read parameters. + * @param b buffer byte array + * @param off offset in buffer + * @param len length to read + * @return true if valid else false + * @throws IOException if there is an error + */ protected boolean validate(final byte[] b, final int off, final int len) throws IOException { if (closed) { @@ -440,6 +456,13 @@ protected boolean validate(final byte[] b, final int off, final int len) return true; } + /** + * Copy data from internal buffer to user buffer. + * @param b user buffer + * @param off offset + * @param len length + * @return number of bytes copied + */ protected int copyToUserBuffer(byte[] b, int off, int len){ //If there is anything in the buffer, then return lesser of (requested bytes) and (bytes in buffer) //(bytes returned may be less than requested) @@ -459,6 +482,16 @@ protected int copyToUserBuffer(byte[] b, int off, int len){ return bytesToRead; } + /** + * Internal read method which handles read-ahead logic. + * @param position to read from + * @param b buffer + * @param offset in buffer + * @param length to read + * @param bypassReadAhead whether to bypass read-ahead + * @return number of bytes read + * @throws IOException if there is an error + */ protected int readInternal(final long position, final byte[] b, final int offset, final int length, final boolean bypassReadAhead) throws IOException { if (isReadAheadEnabled() && !bypassReadAhead) { From 3560cc5a5232066789e07bf340cc019219c5609d Mon Sep 17 00:00:00 2001 From: Anuj Modi Date: Tue, 6 Jan 2026 10:59:50 -0800 Subject: [PATCH 07/14] RBM Improvements --- .../fs/azurebfs/AzureBlobFileSystemStore.java | 4 +- .../services/AbfsPrefetchInputStream.java | 10 ++ .../services/AbfsRandomInputStream.java | 118 ++++++++++-------- ...fsInputPolicy.java => AbfsReadPolicy.java} | 12 +- .../azurebfs/services/ReadBufferManager.java | 18 +-- .../services/ReadBufferManagerV1.java | 18 +++ .../services/ReadBufferManagerV2.java | 56 +++++---- .../services/TestAbfsInputStream.java | 16 ++- .../services/TestReadBufferManagerV2.java | 2 + 9 files changed, 150 insertions(+), 104 deletions(-) rename hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/{AbfsInputPolicy.java => AbfsReadPolicy.java} (92%) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index 14a95182f7cf1..4ee385e5ff379 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -97,7 +97,7 @@ import org.apache.hadoop.fs.azurebfs.services.AbfsClientRenameResult; import org.apache.hadoop.fs.azurebfs.services.AbfsCounters; import org.apache.hadoop.fs.azurebfs.services.AbfsHttpOperation; -import org.apache.hadoop.fs.azurebfs.services.AbfsInputPolicy; +import org.apache.hadoop.fs.azurebfs.services.AbfsReadPolicy; import org.apache.hadoop.fs.azurebfs.services.AbfsInputStream; import org.apache.hadoop.fs.azurebfs.services.AbfsInputStreamContext; import org.apache.hadoop.fs.azurebfs.services.AbfsInputStreamStatisticsImpl; @@ -950,7 +950,7 @@ public AbfsInputStream openFileForRead(Path path, perfInfo.registerSuccess(true); - AbfsInputPolicy inputPolicy = AbfsInputPolicy.getPolicy(getAbfsConfiguration().getAbfsReadPolicy()); + AbfsReadPolicy inputPolicy = AbfsReadPolicy.getAbfsReadPolicy(getAbfsConfiguration().getAbfsReadPolicy()); switch (inputPolicy) { case SEQUENTIAL: return new AbfsPrefetchInputStream(getClient(), statistics, relativePath, diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPrefetchInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPrefetchInputStream.java index a8692c2d6961b..013c8fbd90b20 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPrefetchInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPrefetchInputStream.java @@ -32,6 +32,16 @@ */ public class AbfsPrefetchInputStream extends AbfsInputStream { + /** + * Constructs AbfsPrefetchInputStream + * @param client AbfsClient to be used for read operations + * @param statistics to recordinput stream statistics + * @param path file path + * @param contentLength file content length + * @param abfsInputStreamContext input stream context + * @param eTag file eTag + * @param tracingContext tracing context to trace the read operations + */ public AbfsPrefetchInputStream( final AbfsClient client, final FileSystem.Statistics statistics, diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRandomInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRandomInputStream.java index ac645841f52f9..6870a2f4a69e1 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRandomInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRandomInputStream.java @@ -31,61 +31,75 @@ */ public class AbfsRandomInputStream extends AbfsInputStream { - public AbfsRandomInputStream( - final AbfsClient client, - final FileSystem.Statistics statistics, - final String path, - final long contentLength, - final AbfsInputStreamContext abfsInputStreamContext, - final String eTag, - TracingContext tracingContext) { - super(client, statistics, path, contentLength, - abfsInputStreamContext, eTag, tracingContext); - } + /** + * Constructs AbfsRandomInputStream + * @param client AbfsClient to be used for read operations + * @param statistics to record input stream statistics + * @param path file path + * @param contentLength file content length + * @param abfsInputStreamContext input stream context + * @param eTag file eTag + * @param tracingContext tracing context to trace the read operations + */ + public AbfsRandomInputStream( + final AbfsClient client, + final FileSystem.Statistics statistics, + final String path, + final long contentLength, + final AbfsInputStreamContext abfsInputStreamContext, + final String eTag, + TracingContext tracingContext) { + super(client, statistics, path, contentLength, + abfsInputStreamContext, eTag, tracingContext); + } - @Override - protected int readOneBlock(final byte[] b, final int off, final int len) throws IOException { - if (len == 0) { - return 0; - } - if (!validate(b, off, len)) { - return -1; - } - //If buffer is empty, then fill the buffer. - if (bCursor == limit) { - //If EOF, then return -1 - if (fCursor >= contentLength) { - return -1; - } + /** + * inheritDoc + */ + @Override + protected int readOneBlock(final byte[] b, final int off, final int len) + throws IOException { + if (len == 0) { + return 0; + } + if (!validate(b, off, len)) { + return -1; + } + //If buffer is empty, then fill the buffer. + if (bCursor == limit) { + //If EOF, then return -1 + if (fCursor >= contentLength) { + return -1; + } - long bytesRead = 0; - //reset buffer to initial state - i.e., throw away existing data - bCursor = 0; - limit = 0; - if (buffer == null) { - LOG.debug("created new buffer size {}", bufferSize); - buffer = new byte[bufferSize]; - } + long bytesRead = 0; + //reset buffer to initial state - i.e., throw away existing data + bCursor = 0; + limit = 0; + if (buffer == null) { + LOG.debug("created new buffer size {}", bufferSize); + buffer = new byte[bufferSize]; + } - /* - * Disable queuing prefetches when random read pattern detected. - * Instead, read ahead only for readAheadRange above what is asked by caller. - */ - tracingContext.setReadType(ReadType.RANDOM_READ); - int lengthWithReadAhead = Math.min(b.length + readAheadRange, bufferSize); - LOG.debug("Random read with read ahead size of {}", lengthWithReadAhead); - bytesRead = readInternal(fCursor, buffer, 0, lengthWithReadAhead, true); - if (firstRead) { - firstRead = false; - } - if (bytesRead == -1) { - return -1; - } + /* + * Disable queuing prefetches when random read pattern detected. + * Instead, read ahead only for readAheadRange above what is asked by caller. + */ + tracingContext.setReadType(ReadType.RANDOM_READ); + int lengthWithReadAhead = Math.min(b.length + readAheadRange, bufferSize); + LOG.debug("Random read with read ahead size of {}", lengthWithReadAhead); + bytesRead = readInternal(fCursor, buffer, 0, lengthWithReadAhead, true); + if (firstRead) { + firstRead = false; + } + if (bytesRead == -1) { + return -1; + } - limit += bytesRead; - fCursor += bytesRead; - fCursorAfterLastRead = fCursor; - } - return copyToUserBuffer(b, off, len); + limit += bytesRead; + fCursor += bytesRead; + fCursorAfterLastRead = fCursor; } + return copyToUserBuffer(b, off, len); + } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputPolicy.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsReadPolicy.java similarity index 92% rename from hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputPolicy.java rename to hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsReadPolicy.java index c9dc01be9b729..4df2839c0b386 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputPolicy.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsReadPolicy.java @@ -32,21 +32,21 @@ * Enum for ABFS Input Policies. * Each policy maps to a particular implementation of {@link AbfsInputStream} */ -public enum AbfsInputPolicy { +public enum AbfsReadPolicy { SEQUENTIAL(FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL), RANDOM(FS_OPTION_OPENFILE_READ_POLICY_RANDOM), ADAPTIVE(FS_OPTION_OPENFILE_READ_POLICY_ADAPTIVE); - private final String policy; + private final String readPolicy; - AbfsInputPolicy(String policy) { - this.policy = policy; + AbfsReadPolicy(String readPolicy) { + this.readPolicy = readPolicy; } @Override public String toString() { - return policy; + return readPolicy; } /** @@ -54,7 +54,7 @@ public String toString() { * @param name policy name as configured by user * @return the corresponding AbsInputPolicy to be used */ - public static AbfsInputPolicy getPolicy(String name) { + public static AbfsReadPolicy getAbfsReadPolicy(String name) { String trimmed = name.trim().toLowerCase(Locale.ENGLISH); switch (trimmed) { // all these options currently map to random IO. diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManager.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManager.java index 712b04fb4999c..48323862e6ea0 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManager.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManager.java @@ -45,7 +45,6 @@ public abstract class ReadBufferManager { private static int thresholdAgeMilliseconds; private static int blockSize = DEFAULT_READ_AHEAD_BLOCK_SIZE; // default block size for read-ahead in bytes - private Stack freeList = new Stack<>(); // indices in buffers[] array that are available private Queue readAheadQueue = new LinkedList<>(); // queue of requests that are not picked up by any worker thread yet private LinkedList inProgressList = new LinkedList<>(); // requests being processed by worker threads private LinkedList completedReadList = new LinkedList<>(); // buffers available for reading @@ -200,15 +199,6 @@ protected static void setReadAheadBlockSize(int readAheadBlockSize) { blockSize = readAheadBlockSize; } - /** - * Gets the stack of free buffer indices. - * - * @return the stack of free buffer indices - */ - Stack getFreeList() { - return freeList; - } - /** * Gets the queue of read-ahead requests. * @@ -243,9 +233,7 @@ LinkedList getCompletedReadList() { * @return a list of free buffer indices */ @VisibleForTesting - List getFreeListCopy() { - return new ArrayList<>(freeList); - } + abstract List getFreeListCopy(); /** * Gets a copy of the read-ahead queue. @@ -294,7 +282,9 @@ int getCompletedReadListSize() { */ @VisibleForTesting protected void testMimicFullUseAndAddFailedBuffer(ReadBuffer buf) { - freeList.clear(); + clearFreeList(); completedReadList.add(buf); } + + abstract void clearFreeList(); } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV1.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV1.java index 240a618666621..edc23a653956f 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV1.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV1.java @@ -22,8 +22,11 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.Iterator; import java.util.LinkedList; +import java.util.List; +import java.util.Stack; import java.util.concurrent.CountDownLatch; import org.apache.hadoop.fs.azurebfs.utils.TracingContext; @@ -42,6 +45,7 @@ public final class ReadBufferManagerV1 extends ReadBufferManager { private Thread[] threads = new Thread[NUM_THREADS]; private byte[][] buffers; + private Stack freeList = new Stack<>(); // indices in buffers[] array that are available private static ReadBufferManagerV1 bufferManager; // hide instance constructor @@ -607,7 +611,21 @@ void resetBufferManager() { setBufferManager(null); // reset the singleton instance } + @Override + protected List getFreeListCopy() { + return new ArrayList<>(freeList); + } + + private Stack getFreeList() { + return freeList; + } + private static void setBufferManager(ReadBufferManagerV1 manager) { bufferManager = manager; } + + @Override + protected void clearFreeList() { + getFreeList().clear(); + } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV2.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV2.java index 071c1b1684955..40287c7407edf 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV2.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV2.java @@ -24,10 +24,12 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Stack; +import java.util.concurrent.ConcurrentSkipListSet; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -100,7 +102,8 @@ public final class ReadBufferManagerV2 extends ReadBufferManager { private byte[][] bufferPool; - private final Stack removedBufferList = new Stack<>(); + private final ConcurrentSkipListSet removedBufferList = new ConcurrentSkipListSet<>(); + private ConcurrentSkipListSet freeList = new ConcurrentSkipListSet<>(); private ScheduledExecutorService memoryMonitorThread; @@ -209,7 +212,7 @@ void init() { // Start with just minimum number of buffers. bufferPool[i] = new byte[getReadAheadBlockSize()]; // same buffers are reused. The byte array never goes back to GC - getFreeList().add(i); + pushToFreeList(i); numberOfActiveBuffers.getAndIncrement(); } memoryMonitorThread = Executors.newSingleThreadScheduledExecutor( @@ -768,12 +771,17 @@ private synchronized boolean tryMemoryUpscale() { if (memoryLoad < memoryThreshold && getNumBuffers() < maxBufferPoolSize) { // Create and Add more buffers in getFreeList(). int nextIndx = getNumBuffers(); - if (removedBufferList.isEmpty() && nextIndx < bufferPool.length) { + if (removedBufferList.isEmpty()) { + if (nextIndx >= bufferPool.length) { + printTraceLog("Invalid next index: {}. Current buffer pool size: {}", + nextIndx, bufferPool.length); + return false; + } bufferPool[nextIndx] = new byte[getReadAheadBlockSize()]; pushToFreeList(nextIndx); } else { // Reuse a removed buffer index. - int freeIndex = removedBufferList.pop(); + int freeIndex = removedBufferList.pollFirst(); if (freeIndex >= bufferPool.length || bufferPool[freeIndex] != null) { printTraceLog("Invalid free index: {}. Current buffer pool size: {}", freeIndex, bufferPool.length); @@ -811,7 +819,7 @@ > getThresholdAgeMilliseconds()) { } double memoryLoad = ResourceUtilizationUtils.getMemoryLoad(); - if (isDynamicScalingEnabled && memoryLoad > memoryThreshold) { + if (isDynamicScalingEnabled && memoryLoad > memoryThreshold && getNumBuffers() > minBufferPoolSize) { synchronized (this) { if (isFreeListEmpty()) { printTraceLog( @@ -980,7 +988,7 @@ public void testResetReadBufferManager() { getReadAheadQueue().clear(); getInProgressList().clear(); getCompletedReadList().clear(); - getFreeList().clear(); + this.freeList.clear(); for (int i = 0; i < maxBufferPoolSize; i++) { bufferPool[i] = null; } @@ -1023,6 +1031,16 @@ void resetBufferManager() { setIsConfigured(false); } + @Override + protected List getFreeListCopy() { + return new ArrayList<>(freeList); + } + + @Override + protected void clearFreeList() { + freeList.clear(); + } + private static void setBufferManager(ReadBufferManagerV2 manager) { bufferManager = manager; } @@ -1062,6 +1080,11 @@ public int getMinBufferPoolSize() { return minBufferPoolSize; } + @VisibleForTesting + public void setMinBufferPoolSize(int size) { + this.minBufferPoolSize = size; + } + @VisibleForTesting public int getMaxBufferPoolSize() { return maxBufferPoolSize; @@ -1105,30 +1128,15 @@ public int getRequiredThreadPoolSize() { } private boolean isFreeListEmpty() { - LOCK.lock(); - try { - return getFreeList().isEmpty(); - } finally { - LOCK.unlock(); - } + return this.freeList.isEmpty(); } private Integer popFromFreeList() { - LOCK.lock(); - try { - return getFreeList().pop(); - } finally { - LOCK.unlock(); - } + return this.freeList.pollFirst(); } private void pushToFreeList(int idx) { - LOCK.lock(); - try { - getFreeList().push(idx); - } finally { - LOCK.unlock(); - } + this.freeList.add(idx); } private void incrementActiveBufferCount() { diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java index 4b7a0d6a18dd9..cd4a469e75298 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java @@ -58,16 +58,22 @@ import org.apache.hadoop.fs.azurebfs.utils.TracingHeaderVersion; import org.apache.hadoop.fs.impl.OpenFileParameters; -import static org.apache.hadoop.fs.Options.OpenFileOptions.*; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_ADAPTIVE; import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_AVRO; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_PARQUET; import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.COLON; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.EMPTY_STRING; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.SPLIT_NO_LIMIT; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENABLE_PREFETCH_REQUEST_PRIORITY; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.ONE_MB; import static org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations.X_MS_REQUEST_PRIORITY; -import static org.apache.hadoop.fs.azurebfs.constants.ReadType.*; +import static org.apache.hadoop.fs.azurebfs.constants.ReadType.DIRECT_READ; +import static org.apache.hadoop.fs.azurebfs.constants.ReadType.FOOTER_READ; +import static org.apache.hadoop.fs.azurebfs.constants.ReadType.MISSEDCACHE_READ; +import static org.apache.hadoop.fs.azurebfs.constants.ReadType.NORMAL_READ; +import static org.apache.hadoop.fs.azurebfs.constants.ReadType.PREFETCH_READ; +import static org.apache.hadoop.fs.azurebfs.constants.ReadType.RANDOM_READ; +import static org.apache.hadoop.fs.azurebfs.constants.ReadType.SMALLFILE_READ; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; @@ -883,7 +889,7 @@ public void testReadTypeInTracingContextHeader() throws Exception { /* * Test to verify Random Read Type. - * Settin Read Policy to Parquet ensures Random Read Type. + * Setting Read Policy to Parquet ensures Random Read Type. */ fileSize = 3 * ONE_MB; // To make sure multiple blocks are read. totalReadCalls += 3; // Full file will be read along with footer. @@ -1130,13 +1136,11 @@ private void assertReadTypeInClientRequestId(AzureBlobFileSystem fs, int numOfRe ArgumentCaptor captor9 = ArgumentCaptor.forClass(TracingContext.class); List paths = captor1.getAllValues(); - System.out.println(paths); verify(fs.getAbfsStore().getClient(), times(totalReadCalls)).read( captor1.capture(), captor2.capture(), captor3.capture(), captor4.capture(), captor5.capture(), captor6.capture(), captor7.capture(), captor8.capture(), captor9.capture()); List tracingContextList = captor9.getAllValues(); - System.out.println(tracingContextList); if (readType == PREFETCH_READ) { /* * For Prefetch Enabled, first read can be Normal or Missed Cache Read. diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestReadBufferManagerV2.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestReadBufferManagerV2.java index e94c535bd3900..a7cbc83f0c7c4 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestReadBufferManagerV2.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestReadBufferManagerV2.java @@ -268,6 +268,7 @@ public void testMemoryDownscaleIfMemoryAboveThreshold() throws Exception { ReadBufferManagerV2 bufferManagerV2 = ReadBufferManagerV2.getBufferManager(abfsClient.getAbfsCounters()); int initialBuffers = bufferManagerV2.getMinBufferPoolSize(); assertThat(bufferManagerV2.getNumBuffers()).isEqualTo(initialBuffers); + bufferManagerV2.setMinBufferPoolSize(initialBuffers - 5); // allow downscale running = true; Thread t = new Thread(() -> { while (running) { @@ -314,6 +315,7 @@ public void testReadMetricUpdation() throws Exception { bufferManagerV2.getCurrentStats(ResourceUtilizationUtils.getJvmCpuLoad()); int initialBuffers = bufferManagerV2.getMinBufferPoolSize(); assertThat(bufferManagerV2.getNumBuffers()).isEqualTo(initialBuffers); + bufferManagerV2.setMinBufferPoolSize(initialBuffers - 5); // allow downscale running = true; Thread t = new Thread(() -> { while (running) { From f3b6b577faa9209616ccd137dffbb9461851c6a4 Mon Sep 17 00:00:00 2001 From: Anuj Modi Date: Tue, 6 Jan 2026 22:55:34 -0800 Subject: [PATCH 08/14] javadoc erorr --- .../services/AbfsAdaptiveInputStream.java | 42 ++--- .../fs/azurebfs/services/AbfsInputStream.java | 174 ++++++++++++++++-- .../services/AbfsPrefetchInputStream.java | 28 ++- .../services/AbfsRandomInputStream.java | 30 +-- .../azurebfs/services/ReadBufferManager.java | 1 - .../services/ReadBufferManagerV1.java | 1 - .../services/ReadBufferManagerV2.java | 2 - 7 files changed, 210 insertions(+), 68 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java index 7ddbdf529d6be..8f2fe9ece2a54 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java @@ -66,51 +66,51 @@ protected int readOneBlock(final byte[] b, final int off, final int len) throws return -1; } //If buffer is empty, then fill the buffer. - if (bCursor == limit) { + if (getBCursor() == getLimit()) { //If EOF, then return -1 - if (fCursor >= contentLength) { + if (getFCursor() >= getContentLength()) { return -1; } long bytesRead = 0; //reset buffer to initial state - i.e., throw away existing data - bCursor = 0; - limit = 0; - if (buffer == null) { - LOG.debug("created new buffer size {}", bufferSize); - buffer = new byte[bufferSize]; + setBCursor(0); + setLimit(0); + if (getBuffer() == null) { + LOG.debug("created new buffer size {}", getBufferSize()); + setBuffer(new byte[getBufferSize()]); } // Reset Read Type back to normal and set again based on code flow. - tracingContext.setReadType(ReadType.NORMAL_READ); - if (alwaysReadBufferSize) { - bytesRead = readInternal(fCursor, buffer, 0, bufferSize, false); + getTracingContext().setReadType(ReadType.NORMAL_READ); + if (shouldAlwaysReadBufferSize()) { + bytesRead = readInternal(getFCursor(), getBuffer(), 0, getBufferSize(), false); } else { // Enable readAhead when reading sequentially - if (-1 == fCursorAfterLastRead || fCursorAfterLastRead == fCursor || b.length >= bufferSize) { - LOG.debug("Sequential read with read ahead size of {}", bufferSize); - bytesRead = readInternal(fCursor, buffer, 0, bufferSize, false); + if (-1 == getFCursorAfterLastRead() || getFCursorAfterLastRead() == getFCursor() || b.length >= getBufferSize()) { + LOG.debug("Sequential read with read ahead size of {}", getBufferSize()); + bytesRead = readInternal(getFCursor(), getBuffer(), 0, getBufferSize(), false); } else { /* * Disable queuing prefetches when random read pattern detected. * Instead, read ahead only for readAheadRange above what is asked by caller. */ - tracingContext.setReadType(ReadType.RANDOM_READ); - int lengthWithReadAhead = Math.min(b.length + readAheadRange, bufferSize); + getTracingContext().setReadType(ReadType.RANDOM_READ); + int lengthWithReadAhead = Math.min(b.length + getReadAheadRange(), getBufferSize()); LOG.debug("Random read with read ahead size of {}", lengthWithReadAhead); - bytesRead = readInternal(fCursor, buffer, 0, lengthWithReadAhead, true); + bytesRead = readInternal(getFCursor(), getBuffer(), 0, lengthWithReadAhead, true); } } - if (firstRead) { - firstRead = false; + if (isFirstRead()) { + setFirstRead(false); } if (bytesRead == -1) { return -1; } - limit += bytesRead; - fCursor += bytesRead; - fCursorAfterLastRead = fCursor; + setLimit(getLimit() + (int) bytesRead); + setFCursor(getFCursor() + bytesRead); + setFCursorAfterLastRead(getFCursor()); } return copyToUserBuffer(b, off, len); } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java index 7096818f8a820..be4ff81bca9a5 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java @@ -72,8 +72,9 @@ public abstract class AbfsInputStream extends FSInputStream implements CanUnbuff private final AbfsClient client; private final Statistics statistics; private final String path; - protected final long contentLength; - protected final int bufferSize; // default buffer size + + private final long contentLength; + private final int bufferSize; // default buffer size private final int footerReadSize; // default buffer size to read when reading footer private final int readAheadQueueDepth; // initialized in constructor private final String eTag; // eTag of the path when InputStream are created @@ -81,7 +82,7 @@ public abstract class AbfsInputStream extends FSInputStream implements CanUnbuff private final boolean readAheadEnabled; // whether enable readAhead; private final boolean readAheadV2Enabled; // whether enable readAhead V2; private final String inputStreamId; - protected final boolean alwaysReadBufferSize; + private final boolean alwaysReadBufferSize; /* * By default the pread API will do a seek + read as in FSInputStream. * The read data will be kept in a buffer. When bufferedPreadDisabled is true, @@ -91,20 +92,21 @@ public abstract class AbfsInputStream extends FSInputStream implements CanUnbuff */ private final boolean bufferedPreadDisabled; // User configured size of read ahead. - protected final int readAheadRange; + private final int readAheadRange; + + private boolean firstRead = true; - protected boolean firstRead = true; // SAS tokens can be re-used until they expire private CachedSASToken cachedSasToken; - protected byte[] buffer = null; // will be initialized on first use + private byte[] buffer = null; // will be initialized on first use - protected long fCursor = 0; // cursor of buffer within file - offset of next byte to read from remote server - protected long fCursorAfterLastRead = -1; - protected int bCursor = 0; // cursor of read within buffer - offset of next byte to be returned from buffer - protected int limit = 0; // offset of next byte to be read into buffer from service (i.e., upper marker+1 + private long fCursor = 0; // cursor of buffer within file - offset of next byte to read from remote server + private long fCursorAfterLastRead = -1; + private int bCursor = 0; // cursor of read within buffer - offset of next byte to be returned from buffer + private int limit = 0; // offset of next byte to be read into buffer from service (i.e., upper marker+1 // of valid bytes in buffer) private boolean closed = false; - protected TracingContext tracingContext; + private TracingContext tracingContext; private final ContextEncryptionAdapter contextEncryptionAdapter; // Optimisations modify the pointer fields. @@ -133,6 +135,16 @@ public abstract class AbfsInputStream extends FSInputStream implements CanUnbuff private final BackReference fsBackRef; private final ReadBufferManager readBufferManager; + /** + * Constructor for AbfsInputStream. + * @param client the ABFS client + * @param statistics the statistics + * @param path the file path + * @param contentLength the content length + * @param abfsInputStreamContext the input stream context + * @param eTag the eTag of the file + * @param tracingContext the tracing context + */ public AbfsInputStream( final AbfsClient client, final Statistics statistics, @@ -196,6 +208,10 @@ public AbfsInputStream( } } + /** + * Returns the path of file associated with this stream. + * @return the path of the file + */ public String getPath() { return path; } @@ -334,7 +350,7 @@ private boolean shouldReadLastBlock() { * @return number of bytes read * @throws IOException if there is an error */ - protected abstract int readOneBlock(final byte[] b, final int off, final int len) throws IOException; + protected abstract int readOneBlock(byte[] b, int off, int len) throws IOException; private int readFileCompletely(final byte[] b, final int off, final int len) throws IOException { @@ -709,6 +725,10 @@ public synchronized long getPos() throws IOException { return nextReadPos < 0 ? 0 : nextReadPos; } + /** + * Get the tracing context associated with this stream. + * @return the tracing context + */ public TracingContext getTracingContext() { return tracingContext; } @@ -782,6 +802,10 @@ byte[] getBuffer() { return buffer; } + protected void setBuffer(byte[] buffer) { + this.buffer = buffer; + } + /** * Checks if any version of read ahead is enabled. * If both are disabled, then skip read ahead logic. @@ -792,21 +816,38 @@ public boolean isReadAheadEnabled() { return (readAheadEnabled || readAheadV2Enabled) && getReadBufferManager() != null; } + /** + * Getter for user configured read ahead range. + * @return the read ahead range in int. + */ @VisibleForTesting public int getReadAheadRange() { return readAheadRange; } + /** + * Setter for cachedSasToken. + * @param cachedSasToken the cachedSasToken to set + */ @VisibleForTesting protected void setCachedSasToken(final CachedSASToken cachedSasToken) { this.cachedSasToken = cachedSasToken; } + /** + * Getter for inputStreamId. + * @return the inputStreamId + */ @VisibleForTesting public String getStreamID() { return inputStreamId; } + /** + * Getter for eTag. + * + * @return the eTag + */ public String getETag() { return eTag; } @@ -821,6 +862,10 @@ public AbfsInputStreamStatistics getStreamStatistics() { return streamStatistics; } + /** + * Register a listener for this stream. + * @param listener1 the listener to register + */ @VisibleForTesting public void registerListener(Listener listener1) { listener = listener1; @@ -847,26 +892,46 @@ public long getBytesFromRemoteRead() { return bytesFromRemoteRead; } + /** + * Getter for buffer size. + * @return the buffer size + */ @VisibleForTesting public int getBufferSize() { return bufferSize; } + /** + * Getter for footer read buffer size. + * @return the footer read buffer size + */ @VisibleForTesting protected int getFooterReadBufferSize() { return footerReadSize; } + /** + * Getter for read ahead queue depth. + * @return the read ahead queue depth + */ @VisibleForTesting public int getReadAheadQueueDepth() { return readAheadQueueDepth; } + /** + * Getter for alwaysReadBufferSize. + * @return the alwaysReadBufferSize + */ @VisibleForTesting public boolean shouldAlwaysReadBufferSize() { return alwaysReadBufferSize; } + /** + * Get the IOStatistics for the stream. + * @return IOStatistics + */ @Override public IOStatistics getIOStatistics() { return ioStatistics; @@ -888,48 +953,131 @@ public String toString() { return sb.toString(); } + /** + * Getter for bCursor. + * @return the bCursor + */ @VisibleForTesting int getBCursor() { return this.bCursor; } + /** + * Setter for bCursor. + * @param bCursor the bCursor to set + */ + protected void setBCursor(int bCursor) { + this.bCursor = bCursor; + } + + /** + * Getter for fCursor. + * @return the fCursor + */ @VisibleForTesting long getFCursor() { return this.fCursor; } + /** + * Setter for fCursor. + * @param fCursor the fCursor to set + */ + protected void setFCursor(long fCursor) { + this.fCursor = fCursor; + } + + /** + * Getter for fCursorAfterLastRead. + * @return the fCursorAfterLastRead + */ @VisibleForTesting long getFCursorAfterLastRead() { return this.fCursorAfterLastRead; } + /** + * Setter for fCursorAfterLastRead. + * @param fCursorAfterLastRead the fCursorAfterLastRead to set + */ + protected void setFCursorAfterLastRead(long fCursorAfterLastRead) { + this.fCursorAfterLastRead = fCursorAfterLastRead; + } + + /** + * Getter for limit. + * @return the limit + */ @VisibleForTesting - long getLimit() { + int getLimit() { return this.limit; } + /** + * Setter for limit. + * @param limit the limit to set + */ + protected void setLimit(int limit) { + this.limit = limit; + } + + /** + * Getter for firstRead. + * @return the firstRead + */ boolean isFirstRead() { return this.firstRead; } + /** + * Setter for firstRead. + * @param firstRead the firstRead to set + */ + protected void setFirstRead(boolean firstRead) { + this.firstRead = firstRead; + } + + /** + * Getter for fsBackRef. + * @return the fsBackRef + */ @VisibleForTesting BackReference getFsBackRef() { return fsBackRef; } + /** + * Getter for readBufferManager. + * @return the readBufferManager + */ @VisibleForTesting ReadBufferManager getReadBufferManager() { return readBufferManager; } + /** + * Minimum seek distance for vector reads. + * @return the minimum seek distance + */ @Override public int minSeekForVectorReads() { return S_128K; } + /** + * Maximum read size for vector reads. + * @return the maximum read size + */ @Override public int maxReadSizeForVectorReads() { return S_2M; } + /** + * Getter for contentLength. + * @return the contentLength + */ + protected long getContentLength() { + return contentLength; + } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPrefetchInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPrefetchInputStream.java index 013c8fbd90b20..840f631a076ab 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPrefetchInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPrefetchInputStream.java @@ -23,8 +23,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.azurebfs.utils.TracingContext; -import static java.lang.Math.max; - /** * Input stream implementation optimized for prefetching data. * This implementation always prefetches data in advance if enabled @@ -66,36 +64,36 @@ protected int readOneBlock(final byte[] b, final int off, final int len) throws return -1; } //If buffer is empty, then fill the buffer. - if (bCursor == limit) { + if (getBCursor() == getLimit()) { //If EOF, then return -1 - if (fCursor >= contentLength) { + if (getFCursor() >= getContentLength()) { return -1; } long bytesRead = 0; //reset buffer to initial state - i.e., throw away existing data - bCursor = 0; - limit = 0; - if (buffer == null) { - LOG.debug("created new buffer size {}", bufferSize); - buffer = new byte[bufferSize]; + setBCursor(0); + setLimit(0); + if (getBuffer() == null) { + LOG.debug("created new buffer size {}", getBufferSize()); + setBuffer(new byte[getBufferSize()]); } /* * Always start with Prefetch even from first read. * Even if out of order seek comes, prefetches will be triggered for next set of blocks. */ - bytesRead = readInternal(fCursor, buffer, 0, bufferSize, false); - if (firstRead) { - firstRead = false; + bytesRead = readInternal(getFCursor(), getBuffer(), 0, getBufferSize(), false); + if (isFirstRead()) { + setFirstRead(false); } if (bytesRead == -1) { return -1; } - limit += bytesRead; - fCursor += bytesRead; - fCursorAfterLastRead = fCursor; + setLimit(getLimit() + (int) bytesRead); + setFCursor(getFCursor() + bytesRead); + setFCursorAfterLastRead(getFCursor()); } return copyToUserBuffer(b, off, len); } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRandomInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRandomInputStream.java index 6870a2f4a69e1..4cf76caa8d995 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRandomInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRandomInputStream.java @@ -66,39 +66,39 @@ protected int readOneBlock(final byte[] b, final int off, final int len) return -1; } //If buffer is empty, then fill the buffer. - if (bCursor == limit) { + if (getBCursor() == getLimit()) { //If EOF, then return -1 - if (fCursor >= contentLength) { + if (getFCursor() >= getContentLength()) { return -1; } long bytesRead = 0; //reset buffer to initial state - i.e., throw away existing data - bCursor = 0; - limit = 0; - if (buffer == null) { - LOG.debug("created new buffer size {}", bufferSize); - buffer = new byte[bufferSize]; + setBCursor(0); + setLimit(0); + if (getBuffer() == null) { + LOG.debug("created new buffer size {}", getBufferSize()); + setBuffer(new byte[getBufferSize()]); } /* * Disable queuing prefetches when random read pattern detected. * Instead, read ahead only for readAheadRange above what is asked by caller. */ - tracingContext.setReadType(ReadType.RANDOM_READ); - int lengthWithReadAhead = Math.min(b.length + readAheadRange, bufferSize); + getTracingContext().setReadType(ReadType.RANDOM_READ); + int lengthWithReadAhead = Math.min(b.length + getReadAheadRange(), getBufferSize()); LOG.debug("Random read with read ahead size of {}", lengthWithReadAhead); - bytesRead = readInternal(fCursor, buffer, 0, lengthWithReadAhead, true); - if (firstRead) { - firstRead = false; + bytesRead = readInternal(getFCursor(), getBuffer(), 0, lengthWithReadAhead, true); + if (isFirstRead()) { + setFirstRead(false); } if (bytesRead == -1) { return -1; } - limit += bytesRead; - fCursor += bytesRead; - fCursorAfterLastRead = fCursor; + setLimit(getLimit() + (int) bytesRead); + setFCursor(getFCursor() + bytesRead); + setFCursorAfterLastRead(getFCursor()); } return copyToUserBuffer(b, off, len); } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManager.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManager.java index 48323862e6ea0..5b53d641a20df 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManager.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManager.java @@ -23,7 +23,6 @@ import java.util.LinkedList; import java.util.List; import java.util.Queue; -import java.util.Stack; import java.util.concurrent.locks.ReentrantLock; import org.slf4j.Logger; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV1.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV1.java index edc23a653956f..c034d85659603 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV1.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV1.java @@ -22,7 +22,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.Iterator; import java.util.LinkedList; import java.util.List; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV2.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV2.java index 40287c7407edf..023bf1860af68 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV2.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV2.java @@ -24,11 +24,9 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.Iterator; import java.util.LinkedList; import java.util.List; -import java.util.Stack; import java.util.concurrent.ConcurrentSkipListSet; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executors; From c53a82f3862c1d3af5cf536ee12c188f53e6a2da Mon Sep 17 00:00:00 2001 From: Anuj Modi Date: Thu, 8 Jan 2026 22:38:01 -0800 Subject: [PATCH 09/14] Addressed Comments --- .../fs/azurebfs/AzureBlobFileSystemStore.java | 59 +++++++++++-------- .../azurebfs/constants/ConfigurationKeys.java | 4 +- .../constants/FileSystemConfigurations.java | 2 +- .../exceptions/HttpResponseException.java | 10 ++++ .../services/AbfsAdaptiveInputStream.java | 12 ++-- .../fs/azurebfs/services/AbfsInputStream.java | 10 +++- .../fs/azurebfs/services/AbfsLease.java | 7 ++- .../services/AbfsPrefetchInputStream.java | 6 +- .../services/AbfsRandomInputStream.java | 6 +- .../fs/azurebfs/services/AbfsReadPolicy.java | 4 +- .../fs/azurebfs/services/AbfsRetryPolicy.java | 5 ++ .../services/ReadBufferManagerV2.java | 21 ++++++- .../services/TestAbfsInputStream.java | 11 +++- 13 files changed, 110 insertions(+), 47 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index 4ee385e5ff379..29898a7cd6f63 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -950,30 +950,41 @@ public AbfsInputStream openFileForRead(Path path, perfInfo.registerSuccess(true); - AbfsReadPolicy inputPolicy = AbfsReadPolicy.getAbfsReadPolicy(getAbfsConfiguration().getAbfsReadPolicy()); - switch (inputPolicy) { - case SEQUENTIAL: - return new AbfsPrefetchInputStream(getClient(), statistics, relativePath, - contentLength, populateAbfsInputStreamContext( - parameters.map(OpenFileParameters::getOptions), - contextEncryptionAdapter), - eTag, tracingContext); - - case RANDOM: - return new AbfsRandomInputStream(getClient(), statistics, relativePath, - contentLength, populateAbfsInputStreamContext( - parameters.map(OpenFileParameters::getOptions), - contextEncryptionAdapter), - eTag, tracingContext); - - case ADAPTIVE: - default: - return new AbfsAdaptiveInputStream(getClient(), statistics, relativePath, - contentLength, populateAbfsInputStreamContext( - parameters.map(OpenFileParameters::getOptions), - contextEncryptionAdapter), - eTag, tracingContext); - } + return getRelevantInputStream(statistics, relativePath, contentLength, + parameters, contextEncryptionAdapter, eTag, tracingContext); + } + } + + private AbfsInputStream getRelevantInputStream(final FileSystem.Statistics statistics, + final String relativePath, + final long contentLength, + final Optional parameters, + final ContextEncryptionAdapter contextEncryptionAdapter, + final String eTag, + TracingContext tracingContext) { + AbfsReadPolicy inputPolicy = AbfsReadPolicy.getAbfsReadPolicy(getAbfsConfiguration().getAbfsReadPolicy()); + switch (inputPolicy) { + case SEQUENTIAL: + return new AbfsPrefetchInputStream(getClient(), statistics, relativePath, + contentLength, populateAbfsInputStreamContext( + parameters.map(OpenFileParameters::getOptions), + contextEncryptionAdapter), + eTag, tracingContext); + + case RANDOM: + return new AbfsRandomInputStream(getClient(), statistics, relativePath, + contentLength, populateAbfsInputStreamContext( + parameters.map(OpenFileParameters::getOptions), + contextEncryptionAdapter), + eTag, tracingContext); + + case ADAPTIVE: + default: + return new AbfsAdaptiveInputStream(getClient(), statistics, relativePath, + contentLength, populateAbfsInputStreamContext( + parameters.map(OpenFileParameters::getOptions), + contextEncryptionAdapter), + eTag, tracingContext); } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java index 6b13273f6f15b..fd48fd759181f 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java @@ -21,7 +21,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Options; +import org.apache.hadoop.fs.Options.OpenFileOptions; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.DOT; @@ -218,7 +218,7 @@ public final class ConfigurationKeys { public static final String FS_AZURE_READ_AHEAD_BLOCK_SIZE = "fs.azure.read.readahead.blocksize"; /** * Provides hint for the read workload pattern. - * Possible Values Exposed in {@link Options.OpenFileOptions} + * Possible Values Exposed in {@link OpenFileOptions} */ public static final String FS_AZURE_READ_POLICY = "fs.azure.read.policy"; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java index 126b86652718e..f564ff4204439 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java @@ -418,7 +418,7 @@ public final class FileSystemConfigurations { public static final boolean DEFAULT_FS_AZURE_ENABLE_CREATE_BLOB_IDEMPOTENCY = true; - public static final boolean DEFAULT_FS_AZURE_ENABLE_PREFETCH_REQUEST_PRIORITY = false; + public static final boolean DEFAULT_FS_AZURE_ENABLE_PREFETCH_REQUEST_PRIORITY = true; // The default traffic request priority is 3 (from service side) // The lowest priority a request can get is 7 diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/HttpResponseException.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/HttpResponseException.java index c257309c8c9fb..76126049f077c 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/HttpResponseException.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/HttpResponseException.java @@ -28,12 +28,22 @@ */ public class HttpResponseException extends IOException { private final HttpResponse httpResponse; + + /** + * Constructor for HttpResponseException. + * @param s the exception message + * @param httpResponse the HttpResponse object + */ public HttpResponseException(final String s, final HttpResponse httpResponse) { super(s); Objects.requireNonNull(httpResponse, "httpResponse should be non-null"); this.httpResponse = httpResponse; } + /** + * Gets the HttpResponse associated with this exception. + * @return the HttpResponse + */ public HttpResponse getHttpResponse() { return httpResponse; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java index 8f2fe9ece2a54..25b4529aa0863 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java @@ -33,9 +33,9 @@ public class AbfsAdaptiveInputStream extends AbfsInputStream { /** - * Constructs AbfsAdaptiveInputStream - * @param client AbfsClient to be used for read operations - * @param statistics to recordinput stream statistics + * Constructs AbfsAdaptiveInputStream instance. + * @param client to be used for read operations + * @param statistics to record input stream statistics * @param path file path * @param contentLength file content length * @param abfsInputStreamContext input stream context @@ -65,15 +65,15 @@ protected int readOneBlock(final byte[] b, final int off, final int len) throws if (!validate(b, off, len)) { return -1; } - //If buffer is empty, then fill the buffer. + // If buffer is empty, then fill the buffer. if (getBCursor() == getLimit()) { - //If EOF, then return -1 + // If EOF, then return -1 if (getFCursor() >= getContentLength()) { return -1; } long bytesRead = 0; - //reset buffer to initial state - i.e., throw away existing data + // reset buffer to initial state - i.e., throw away existing data setBCursor(0); setLimit(0); if (getBuffer() == null) { diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java index be4ff81bca9a5..4a9880794a89b 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java @@ -94,7 +94,7 @@ public abstract class AbfsInputStream extends FSInputStream implements CanUnbuff // User configured size of read ahead. private final int readAheadRange; - private boolean firstRead = true; + private boolean firstRead = true; // to identify first read for optimizations // SAS tokens can be re-used until they expire private CachedSASToken cachedSasToken; @@ -798,10 +798,18 @@ public boolean hasCapability(String capability) { return StreamCapabilities.UNBUFFER.equals(toLowerCase(capability)); } + /** + * Getter for buffer. + * @return the buffer + */ byte[] getBuffer() { return buffer; } + /** + * Setter for buffer. + * @param buffer the buffer to set + */ protected void setBuffer(byte[] buffer) { this.buffer = buffer; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsLease.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsLease.java index fa4b0ac209de9..aab6f3d5510e8 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsLease.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsLease.java @@ -22,6 +22,7 @@ import java.util.Timer; import java.util.TimerTask; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -70,7 +71,7 @@ public final class AbfsLease { private volatile boolean leaseFreed; private volatile String leaseID = null; private volatile Throwable exception = null; - private volatile int acquireRetryCount = 0; + private AtomicInteger acquireRetryCount = new AtomicInteger(0); private volatile ListenableScheduledFuture future = null; private final long leaseRefreshDuration; private final int leaseRefreshDurationInSeconds; @@ -197,7 +198,7 @@ public void onFailure(Throwable throwable) { if (RetryPolicy.RetryAction.RetryDecision.RETRY == retryPolicy.shouldRetry(null, numRetries, 0, true).action) { LOG.debug("Failed to acquire lease on {}, retrying: {}", path, throwable); - acquireRetryCount++; + acquireRetryCount.incrementAndGet(); acquireLease(retryPolicy, numRetries + 1, retryInterval, retryInterval, eTag, tracingContext); } else { @@ -289,7 +290,7 @@ public String getLeaseID() { */ @VisibleForTesting public int getAcquireRetryCount() { - return acquireRetryCount; + return acquireRetryCount.get(); } /** diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPrefetchInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPrefetchInputStream.java index 840f631a076ab..c0343ca724e05 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPrefetchInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPrefetchInputStream.java @@ -63,15 +63,15 @@ protected int readOneBlock(final byte[] b, final int off, final int len) throws if (!validate(b, off, len)) { return -1; } - //If buffer is empty, then fill the buffer. + // If buffer is empty, then fill the buffer. if (getBCursor() == getLimit()) { - //If EOF, then return -1 + // If EOF, then return -1 if (getFCursor() >= getContentLength()) { return -1; } long bytesRead = 0; - //reset buffer to initial state - i.e., throw away existing data + // reset buffer to initial state - i.e., throw away existing data setBCursor(0); setLimit(0); if (getBuffer() == null) { diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRandomInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRandomInputStream.java index 4cf76caa8d995..b484cc6c84353 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRandomInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRandomInputStream.java @@ -65,15 +65,15 @@ protected int readOneBlock(final byte[] b, final int off, final int len) if (!validate(b, off, len)) { return -1; } - //If buffer is empty, then fill the buffer. + // If buffer is empty, then fill the buffer. if (getBCursor() == getLimit()) { - //If EOF, then return -1 + // If EOF, then return -1 if (getFCursor() >= getContentLength()) { return -1; } long bytesRead = 0; - //reset buffer to initial state - i.e., throw away existing data + // reset buffer to initial state - i.e., throw away existing data setBCursor(0); setLimit(0); if (getBuffer() == null) { diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsReadPolicy.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsReadPolicy.java index 4df2839c0b386..bdd895a39def6 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsReadPolicy.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsReadPolicy.java @@ -55,8 +55,8 @@ public String toString() { * @return the corresponding AbsInputPolicy to be used */ public static AbfsReadPolicy getAbfsReadPolicy(String name) { - String trimmed = name.trim().toLowerCase(Locale.ENGLISH); - switch (trimmed) { + String readPolicyStr = name.trim().toLowerCase(Locale.ENGLISH); + switch (readPolicyStr) { // all these options currently map to random IO. case FS_OPTION_OPENFILE_READ_POLICY_RANDOM: case FS_OPTION_OPENFILE_READ_POLICY_COLUMNAR: diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRetryPolicy.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRetryPolicy.java index f3e1e582f9dab..7164e55e90ce5 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRetryPolicy.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRetryPolicy.java @@ -41,6 +41,11 @@ public abstract class AbfsRetryPolicy { */ private final String retryPolicyAbbreviation; + /** + * Constructor to initialize max retry count and abbreviation + * @param maxRetryCount maximum retry count + * @param retryPolicyAbbreviation abbreviation for retry policy + */ protected AbfsRetryPolicy(final int maxRetryCount, final String retryPolicyAbbreviation) { this.maxRetryCount = maxRetryCount; this.retryPolicyAbbreviation = retryPolicyAbbreviation; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV2.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV2.java index 023bf1860af68..d4b5d6ec557b7 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV2.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManagerV2.java @@ -100,6 +100,10 @@ public final class ReadBufferManagerV2 extends ReadBufferManager { private byte[][] bufferPool; + /* + * List of buffer indexes that are currently free and can be assigned to new read-ahead requests. + * Using a thread safe data structure as multiple threads can access this concurrently. + */ private final ConcurrentSkipListSet removedBufferList = new ConcurrentSkipListSet<>(); private ConcurrentSkipListSet freeList = new ConcurrentSkipListSet<>(); @@ -986,7 +990,7 @@ public void testResetReadBufferManager() { getReadAheadQueue().clear(); getInProgressList().clear(); getCompletedReadList().clear(); - this.freeList.clear(); + clearFreeList(); for (int i = 0; i < maxBufferPoolSize; i++) { bufferPool[i] = null; } @@ -1088,6 +1092,10 @@ public int getMaxBufferPoolSize() { return maxBufferPoolSize; } + /** + * Gets the maximum buffer pool size. + * @return size of the maximum buffer pool + */ @VisibleForTesting public int getCurrentThreadPoolSize() { return workerRefs.size(); @@ -1103,6 +1111,10 @@ public int getMemoryMonitoringIntervalInMilliSec() { return memoryMonitoringIntervalInMilliSec; } + /** + * Returns the scheduled executor service used for CPU monitoring. + * @return the ScheduledExecutorService for CPU monitoring tasks + */ @VisibleForTesting public ScheduledExecutorService getCpuMonitoringThread() { return cpuMonitorThread; @@ -1119,6 +1131,13 @@ public long getMaxJvmCpuUtilization() { return maxJvmCpuUtilization; } + /** + * Calculates the required thread pool size based on the current + * read-ahead queue size and in-progress list size, applying a buffer + * to accommodate workload fluctuations. + * + * @return the calculated required thread pool size + */ public int getRequiredThreadPoolSize() { return (int) Math.ceil(THREAD_POOL_REQUIREMENT_BUFFER * (getReadAheadQueue().size() diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java index cd4a469e75298..5cf0bd473fc24 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java @@ -923,7 +923,7 @@ private void testReadTypeInTracingContextHeaderInternal(AzureBlobFileSystem fs, assertReadTypeInClientRequestId(fs, numOfReadCalls, totalReadCalls, readType); } - /* + /** * Test to verify that both conditions of prefetch read and respective config * enabled needs to be true for the priority header to be added */ @@ -987,6 +987,10 @@ public void testAbfsInputStreamInstance() throws Exception { stream.close(); } + /** + * Test to verify that Random Input Stream does not queue prefetches. + * @throws Exception if any error occurs during the test + */ @Test public void testRandomInputStreamDoesNotQueuePrefetches() throws Exception { AzureBlobFileSystem spiedFs = Mockito.spy(getFileSystem()); @@ -1006,6 +1010,11 @@ public void testRandomInputStreamDoesNotQueuePrefetches() throws Exception { testReadTypeInTracingContextHeaderInternal(spiedFs, fileSize, RANDOM_READ, 3, totalReadCalls); } + /** + * Test to verify that Adaptive Input Stream queues prefetches for in-order reads + * and performs random reads for out-of-order seeks. + * @throws Exception if any error occurs during the test + */ @Test public void testAdaptiveInputStream() throws Exception { AzureBlobFileSystem spiedFs = Mockito.spy(getFileSystem()); From ba7131c213ee20368130adaa2f8f0f69f9e8ea59 Mon Sep 17 00:00:00 2001 From: Manika Joshi Date: Wed, 21 Jan 2026 03:06:10 -0800 Subject: [PATCH 10/14] prod changes --- .../hadoop/fs/azurebfs/AbfsConfiguration.java | 8 + .../fs/azurebfs/AzureBlobFileSystemStore.java | 93 +-- .../azurebfs/constants/ConfigurationKeys.java | 6 + .../constants/FileSystemConfigurations.java | 6 + .../constants/HttpHeaderConfigurations.java | 1 + .../services/AbfsAdaptiveInputStream.java | 8 +- .../fs/azurebfs/services/AbfsBlobClient.java | 5 +- .../fs/azurebfs/services/AbfsDfsClient.java | 6 +- .../fs/azurebfs/services/AbfsInputStream.java | 121 +++- .../services/AbfsInputStreamContext.java | 12 + .../services/AbfsPrefetchInputStream.java | 21 +- .../services/AbfsRandomInputStream.java | 2 +- .../services/TestAbfsInputStream.java | 653 +++++++++++++++++- 13 files changed, 853 insertions(+), 89 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index 56d02d831cf3d..05df695fce46f 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -660,6 +660,10 @@ public class AbfsConfiguration{ DefaultValue = DEFAULT_AZURE_READ_POLICY) private String abfsReadPolicy; + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_RESTRICT_GPS_ON_OPENFILE, + DefaultValue = DEFAULT_FS_AZURE_RESTRICT_GPS_ON_OPENFILE) + private boolean restrictGpsOnOpenFile; + private String clientProvidedEncryptionKey; private String clientProvidedEncryptionKeySHA; @@ -1393,6 +1397,10 @@ public String getAbfsReadPolicy() { return abfsReadPolicy; } + public boolean shouldRestrictGpsOnOpenFile() { + return restrictGpsOnOpenFile; + } + /** * Enum config to allow user to pick format of x-ms-client-request-id header * @return tracingContextFormat config if valid, else default ALL_ID_FORMAT diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index 29898a7cd6f63..bff500cda6908 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -145,17 +145,7 @@ import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.METADATA_INCOMPLETE_RENAME_FAILURES; import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.RENAME_RECOVERY; -import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.CHAR_EQUALS; -import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.CHAR_FORWARD_SLASH; -import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.CHAR_HYPHEN; -import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.CHAR_PLUS; -import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.CHAR_STAR; -import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.CHAR_UNDERSCORE; -import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.DIRECTORY; -import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.FILE; -import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.ROOT_PATH; -import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.SINGLE_WHITE_SPACE; -import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.TOKEN_VERSION; +import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.*; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_ABFS_ENDPOINT; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_FOOTER_READ_BUFFER_SIZE; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_BUFFERED_PREAD_DISABLE; @@ -425,7 +415,7 @@ private synchronized boolean getNamespaceEnabledInformationFromServer( } try { LOG.debug("Get root ACL status"); - getClient(AbfsServiceType.DFS).getAclStatus(AbfsHttpConstants.ROOT_PATH, tracingContext); + getClient(AbfsServiceType.DFS).getAclStatus(ROOT_PATH, tracingContext); // If getAcl succeeds, namespace is enabled. setNamespaceEnabled(true); } catch (AbfsRestOperationException ex) { @@ -562,7 +552,7 @@ public Hashtable getPathStatus(final Path path, /** * Creates an object of {@link ContextEncryptionAdapter} - * from a file path. It calls {@link org.apache.hadoop.fs.azurebfs.services.AbfsClient + * from a file path. It calls {@link AbfsClient * #getPathStatus(String, boolean, TracingContext, EncryptionAdapter)} method to get * contextValue (x-ms-encryption-context) from the server. The contextValue is passed * to the constructor of EncryptionAdapter to create the required object of @@ -571,11 +561,11 @@ public Hashtable getPathStatus(final Path path, * @return
    *
  • * {@link NoContextEncryptionAdapter}: if encryptionType is not of type - * {@link org.apache.hadoop.fs.azurebfs.utils.EncryptionType#ENCRYPTION_CONTEXT}. + * {@link EncryptionType#ENCRYPTION_CONTEXT}. *
    • *
  • * new object of {@link ContextProviderEncryptionAdapter} containing required encryptionKeys for the give file: - * if encryptionType is of type {@link org.apache.hadoop.fs.azurebfs.utils.EncryptionType#ENCRYPTION_CONTEXT}. + * if encryptionType is of type {@link EncryptionType#ENCRYPTION_CONTEXT}. *
    • *
*/ @@ -886,8 +876,8 @@ public AbfsInputStream openFileForRead(Path path, FileStatus fileStatus = parameters.map(OpenFileParameters::getStatus) .orElse(null); String relativePath = getRelativePath(path); - String resourceType, eTag; - long contentLength; + String resourceType, eTag = EMPTY_STRING; + long contentLength = 0; ContextEncryptionAdapter contextEncryptionAdapter = NoContextEncryptionAdapter.getInstance(); /* * GetPathStatus API has to be called in case of: @@ -898,54 +888,68 @@ public AbfsInputStream openFileForRead(Path path, * ENCRYPTION_CONTEXT. */ if ((fileStatus instanceof VersionedFileStatus) && ( - getClient().getEncryptionType() != EncryptionType.ENCRYPTION_CONTEXT - || ((VersionedFileStatus) fileStatus).getEncryptionContext() - != null)) { + getClient().getEncryptionType() != EncryptionType.ENCRYPTION_CONTEXT + || ((VersionedFileStatus) fileStatus).getEncryptionContext() + != null)) { path = path.makeQualified(this.uri, path); Preconditions.checkArgument(fileStatus.getPath().equals(path), - String.format( - "Filestatus path [%s] does not match with given path [%s]", - fileStatus.getPath(), path)); + String.format( + "Filestatus path [%s] does not match with given path [%s]", + fileStatus.getPath(), path)); resourceType = fileStatus.isFile() ? FILE : DIRECTORY; contentLength = fileStatus.getLen(); eTag = ((VersionedFileStatus) fileStatus).getVersion(); final String encryptionContext - = ((VersionedFileStatus) fileStatus).getEncryptionContext(); + = ((VersionedFileStatus) fileStatus).getEncryptionContext(); if (getClient().getEncryptionType() == EncryptionType.ENCRYPTION_CONTEXT) { contextEncryptionAdapter = new ContextProviderEncryptionAdapter( - getClient().getEncryptionContextProvider(), getRelativePath(path), - encryptionContext.getBytes(StandardCharsets.UTF_8)); + getClient().getEncryptionContextProvider(), getRelativePath(path), + encryptionContext.getBytes(StandardCharsets.UTF_8)); } - } else { + if (parseIsDirectory(resourceType)) { + throw new AbfsRestOperationException( + AzureServiceErrorCode.PATH_NOT_FOUND.getStatusCode(), + AzureServiceErrorCode.PATH_NOT_FOUND.getErrorCode(), + "openFileForRead must be used with files and not directories", + null); + } + } + /* + * If file created with ENCRYPTION_CONTEXT, irrespective of whether isRestrictGpsOnOpenFile config is enabled or not, + * GetPathStatus API has to be called to get the encryptionContext from the response header + */ + else if (getClient().getEncryptionType() == EncryptionType.ENCRYPTION_CONTEXT + || !getAbfsConfiguration().shouldRestrictGpsOnOpenFile()) { + AbfsHttpOperation op = getClient().getPathStatus(relativePath, false, - tracingContext, null).getResult(); - resourceType = getClient().checkIsDir(op) ? DIRECTORY : FILE; - contentLength = extractContentLength(op); - eTag = op.getResponseHeader(HttpHeaderConfigurations.ETAG); + tracingContext, null).getResult(); /* * For file created with ENCRYPTION_CONTEXT, client shall receive * encryptionContext from header field: X_MS_ENCRYPTION_CONTEXT. */ if (getClient().getEncryptionType() == EncryptionType.ENCRYPTION_CONTEXT) { final String fileEncryptionContext = op.getResponseHeader( - HttpHeaderConfigurations.X_MS_ENCRYPTION_CONTEXT); + X_MS_ENCRYPTION_CONTEXT); if (fileEncryptionContext == null) { LOG.debug("EncryptionContext missing in GetPathStatus response"); throw new PathIOException(path.toString(), - "EncryptionContext not present in GetPathStatus response headers"); + "EncryptionContext not present in GetPathStatus response headers"); } contextEncryptionAdapter = new ContextProviderEncryptionAdapter( - getClient().getEncryptionContextProvider(), getRelativePath(path), - fileEncryptionContext.getBytes(StandardCharsets.UTF_8)); + getClient().getEncryptionContextProvider(), getRelativePath(path), + fileEncryptionContext.getBytes(StandardCharsets.UTF_8)); } - } + resourceType = getClient().checkIsDir(op) ? DIRECTORY : FILE; + contentLength = extractContentLength(op); + eTag = op.getResponseHeader(HttpHeaderConfigurations.ETAG); - if (parseIsDirectory(resourceType)) { - throw new AbfsRestOperationException( - AzureServiceErrorCode.PATH_NOT_FOUND.getStatusCode(), - AzureServiceErrorCode.PATH_NOT_FOUND.getErrorCode(), - "openFileForRead must be used with files and not directories", - null); + if (parseIsDirectory(resourceType)) { + throw new AbfsRestOperationException( + AzureServiceErrorCode.PATH_NOT_FOUND.getStatusCode(), + AzureServiceErrorCode.PATH_NOT_FOUND.getErrorCode(), + "openFileForRead must be used with files and not directories", + null); + } } perfInfo.registerSuccess(true); @@ -1011,6 +1015,7 @@ AZURE_FOOTER_READ_BUFFER_SIZE, getAbfsConfiguration().getFooterReadBufferSize()) .withStreamStatistics(new AbfsInputStreamStatisticsImpl()) .withShouldReadBufferSizeAlways(getAbfsConfiguration().shouldReadBufferSizeAlways()) .withReadAheadBlockSize(getAbfsConfiguration().getReadAheadBlockSize()) + .shouldRestrictGpsOnOpenFile(getAbfsConfiguration().shouldRestrictGpsOnOpenFile()) .withBufferedPreadDisabled(bufferedPreadDisabled) .withEncryptionAdapter(contextEncryptionAdapter) .withAbfsBackRef(fsBackRef) @@ -1348,7 +1353,7 @@ public String listStatus(final Path path, final String startFrom, // generate continuation token for xns account private String generateContinuationTokenForXns(final String firstEntryName) { Preconditions.checkArgument(!Strings.isNullOrEmpty(firstEntryName) - && !firstEntryName.startsWith(AbfsHttpConstants.ROOT_PATH), + && !firstEntryName.startsWith(ROOT_PATH), "startFrom must be a dir/file name and it can not be a full path"); StringBuilder sb = new StringBuilder(); @@ -1368,7 +1373,7 @@ private String generateContinuationTokenForXns(final String firstEntryName) { // generate continuation token for non-xns account private String generateContinuationTokenForNonXns(String path, final String firstEntryName) { Preconditions.checkArgument(!Strings.isNullOrEmpty(firstEntryName) - && !firstEntryName.startsWith(AbfsHttpConstants.ROOT_PATH), + && !firstEntryName.startsWith(ROOT_PATH), "startFrom must be a dir/file name and it can not be a full path"); // Notice: non-xns continuation token requires full path (first "/" is not included) for startFrom diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java index fd48fd759181f..f92b33cda2ea0 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java @@ -614,5 +614,11 @@ public static String containerProperty(String property, String fsName, String ac */ public static final String FS_AZURE_TAIL_LATENCY_MAX_RETRY_COUNT = "fs.azure.tail.latency.max.retry.count"; + /** + * If true, restricts GPS (Geo-Replication Secondary) access on openFile operations. + * Default: false + */ + public static final String FS_AZURE_RESTRICT_GPS_ON_OPENFILE = "fs.azure.restrict.gps.on.openfile"; + private ConfigurationKeys() {} } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java index f564ff4204439..a1edaf5553b86 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java @@ -436,5 +436,11 @@ public final class FileSystemConfigurations { public static final int DEFAULT_FS_AZURE_TAIL_LATENCY_PERCENTILE_COMPUTATION_INTERVAL_MILLIS = 500; public static final int DEFAULT_FS_AZURE_TAIL_LATENCY_MAX_RETRY_COUNT = 1; + /** + * If true, restricts GPS (getPathStatus) calls on openFile operations. + * Default: false + */ + public static final boolean DEFAULT_FS_AZURE_RESTRICT_GPS_ON_OPENFILE = false; + private FileSystemConfigurations() {} } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpHeaderConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpHeaderConfigurations.java index 9521518fa1f17..fd8c162fd99d6 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpHeaderConfigurations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpHeaderConfigurations.java @@ -39,6 +39,7 @@ public final class HttpHeaderConfigurations { public static final String CONTENT_MD5 = "Content-MD5"; public static final String CONTENT_TYPE = "Content-Type"; public static final String RANGE = "Range"; + public static final String CONTENT_RANGE = "Content-Range"; public static final String TRANSFER_ENCODING = "Transfer-Encoding"; public static final String USER_AGENT = "User-Agent"; public static final String X_HTTP_METHOD_OVERRIDE = "X-HTTP-Method-Override"; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java index 25b4529aa0863..9435472e52310 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java @@ -68,7 +68,7 @@ protected int readOneBlock(final byte[] b, final int off, final int len) throws // If buffer is empty, then fill the buffer. if (getBCursor() == getLimit()) { // If EOF, then return -1 - if (getFCursor() >= getContentLength()) { + if (!(shouldRestrictGpsOnOpenFile() && isFirstRead()) && getFCursor() >= getContentLength()) { return -1; } @@ -83,7 +83,11 @@ protected int readOneBlock(final byte[] b, final int off, final int len) throws // Reset Read Type back to normal and set again based on code flow. getTracingContext().setReadType(ReadType.NORMAL_READ); - if (shouldAlwaysReadBufferSize()) { + if(shouldRestrictGpsOnOpenFile() && isFirstRead()) { + LOG.debug("RestrictGpsOnOpenFile is enabled. Skip readahead."); + bytesRead = readInternal(getFCursor(), getBuffer(), 0, getBufferSize(), true); + } + else if (shouldAlwaysReadBufferSize()) { bytesRead = readInternal(getFCursor(), getBuffer(), 0, getBufferSize(), false); } else { // Enable readAhead when reading sequentially diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsBlobClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsBlobClient.java index 52fbd3182fdd5..cdf50535e0ee6 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsBlobClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsBlobClient.java @@ -1321,7 +1321,10 @@ public AbfsRestOperation read(final String path, AbfsHttpHeader rangeHeader = new AbfsHttpHeader(RANGE, String.format( "bytes=%d-%d", position, position + bufferLength - 1)); requestHeaders.add(rangeHeader); - requestHeaders.add(new AbfsHttpHeader(IF_MATCH, eTag)); + + // eTag won't be present for first read with restrict GPS on openFile config enabled + if (!eTag.isEmpty()) + requestHeaders.add(new AbfsHttpHeader(IF_MATCH, eTag)); // Add request priority header for prefetch reads addRequestPriorityForPrefetch(requestHeaders, tracingContext); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsDfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsDfsClient.java index 5ddb9770ac56e..40efa98354c7e 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsDfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsDfsClient.java @@ -1045,7 +1045,11 @@ public AbfsRestOperation read(final String path, AbfsHttpHeader rangeHeader = new AbfsHttpHeader(RANGE, String.format("bytes=%d-%d", position, position + bufferLength - 1)); requestHeaders.add(rangeHeader); - requestHeaders.add(new AbfsHttpHeader(IF_MATCH, eTag)); + + // eTag won't be present for first read with restrict GPS on openFile config enabled + if (!eTag.isEmpty()){ + requestHeaders.add(new AbfsHttpHeader(IF_MATCH, eTag)); + } // Add request header to fetch MD5 Hash of data returned by server. if (isChecksumValidationEnabled(requestHeaders, rangeHeader, bufferLength)) { diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java index 4a9880794a89b..616b6ca8c7f0f 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java @@ -22,22 +22,21 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.net.HttpURLConnection; +import java.util.Objects; import java.util.UUID; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.classification.VisibleForTesting; +import org.apache.hadoop.fs.*; import org.apache.hadoop.fs.azurebfs.constants.ReadType; +import org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode; import org.apache.hadoop.fs.impl.BackReference; import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hadoop.fs.CanUnbuffer; -import org.apache.hadoop.fs.FSExceptionMessages; -import org.apache.hadoop.fs.FSInputStream; import org.apache.hadoop.fs.FileSystem.Statistics; -import org.apache.hadoop.fs.StreamCapabilities; import org.apache.hadoop.fs.azurebfs.constants.FSOperationType; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException; @@ -51,8 +50,10 @@ import static java.lang.Math.max; import static java.lang.Math.min; +import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.*; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.ONE_KB; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.STREAM_ID_LEN; +import static org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations.X_MS_META_HDI_ISFOLDER; import static org.apache.hadoop.fs.azurebfs.constants.InternalConstants.CAPABILITY_SAFE_READAHEAD; import static org.apache.hadoop.io.Sizes.S_128K; import static org.apache.hadoop.io.Sizes.S_2M; @@ -73,11 +74,11 @@ public abstract class AbfsInputStream extends FSInputStream implements CanUnbuff private final Statistics statistics; private final String path; - private final long contentLength; + private long contentLength; private final int bufferSize; // default buffer size private final int footerReadSize; // default buffer size to read when reading footer private final int readAheadQueueDepth; // initialized in constructor - private final String eTag; // eTag of the path when InputStream are created + private String eTag; // eTag of the path when InputStream are created private final boolean tolerateOobAppends; // whether tolerate Oob Appends private final boolean readAheadEnabled; // whether enable readAhead; private final boolean readAheadV2Enabled; // whether enable readAhead V2; @@ -203,6 +204,8 @@ public AbfsInputStream( readBufferManager = ReadBufferManagerV1.getBufferManager(); } + // isRestrictGpsOnOpenFile = client.getAbfsConfiguration().isRestrictGpsOnOpenFile(); + if (streamStatistics != null) { ioStatistics = streamStatistics.getIOStatistics(); } @@ -293,9 +296,6 @@ public synchronized int read(final byte[] b, final int off, final int len) throw // go back and read from buffer is fCursor - limit. // There maybe case that we read less than requested data. long filePosAtStartOfBuffer = fCursor - limit; - if (abfsReadFooterMetrics != null) { - abfsReadFooterMetrics.updateReadMetrics(filePathIdentifier, len, contentLength, nextReadPos); - } if (nextReadPos >= filePosAtStartOfBuffer && nextReadPos <= fCursor) { // Determining position in buffer from where data is to be read. bCursor = (int) (nextReadPos - filePosAtStartOfBuffer); @@ -312,6 +312,7 @@ public synchronized int read(final byte[] b, final int off, final int len) throw limit = 0; bCursor = 0; } + if (shouldReadFully()) { lastReadBytes = readFileCompletely(b, currentOff, currentLen); } else if (shouldReadLastBlock()) { @@ -319,6 +320,11 @@ public synchronized int read(final byte[] b, final int off, final int len) throw } else { lastReadBytes = readOneBlock(b, currentOff, currentLen); } + + if (abfsReadFooterMetrics != null) { + abfsReadFooterMetrics.updateReadMetrics(filePathIdentifier, len, contentLength, nextReadPos); + } + if (lastReadBytes > 0) { currentOff += lastReadBytes; currentLen -= lastReadBytes; @@ -332,13 +338,13 @@ public synchronized int read(final byte[] b, final int off, final int len) throw } private boolean shouldReadFully() { - return this.firstRead && this.context.readSmallFilesCompletely() + return this.firstRead && !shouldRestrictGpsOnOpenFile() && this.context.readSmallFilesCompletely() && this.contentLength <= this.bufferSize; } private boolean shouldReadLastBlock() { long footerStart = max(0, this.contentLength - FOOTER_SIZE); - return this.firstRead && this.context.optimizeFooterRead() + return this.firstRead && !shouldRestrictGpsOnOpenFile() && this.context.optimizeFooterRead() && this.fCursor >= footerStart; } @@ -561,11 +567,30 @@ protected int readInternal(final long position, final byte[] b, final int offset } } + String getRelativePath(final Path path) { + Preconditions.checkNotNull(path, "path"); + String relPath = path.toUri().getPath(); + if (relPath.isEmpty()) { + // This means that path passed by user is absolute path of root without "/" at end. + relPath = ROOT_PATH; + } + return relPath; + } + + private IOException directoryReadException() { + return new AbfsRestOperationException( + AzureServiceErrorCode.PATH_NOT_FOUND.getStatusCode(), + AzureServiceErrorCode.PATH_NOT_FOUND.getErrorCode(), + "Read operation not permitted on a directory.", + null); + } + + int readRemote(long position, byte[] b, int offset, int length, TracingContext tracingContext) throws IOException { if (position < 0) { throw new IllegalArgumentException("attempting to read from negative offset"); } - if (position >= contentLength) { + if (!(shouldRestrictGpsOnOpenFile() && isFirstRead()) && position >= contentLength) { return -1; // Hadoop prefers -1 to EOFException } if (b == null) { @@ -591,6 +616,15 @@ int readRemote(long position, byte[] b, int offset, int length, TracingContext t op = client.read(path, position, b, offset, length, tolerateOobAppends ? "*" : eTag, cachedSasToken.get(), contextEncryptionAdapter, tracingContext); + + if (shouldRestrictGpsOnOpenFile() && isFirstRead()){ + String resourceType = op.getResult().getResponseHeader("x-ms-resource-type"); + if (Objects.equals(resourceType, DIRECTORY)){ + throw directoryReadException(); + } + contentLength = Long.parseLong(op.getResult().getResponseHeader("Content-Range").split("/")[1]); + eTag = op.getResult().getResponseHeader("ETag"); + } cachedSasToken.update(op.getSasToken()); LOG.debug("issuing HTTP GET request params position = {} b.length = {} " + "offset = {} length = {}", position, b.length, offset, length); @@ -599,9 +633,57 @@ int readRemote(long position, byte[] b, int offset, int length, TracingContext t } catch (AzureBlobFileSystemException ex) { if (ex instanceof AbfsRestOperationException) { AbfsRestOperationException ere = (AbfsRestOperationException) ex; - if (ere.getStatusCode() == HttpURLConnection.HTTP_NOT_FOUND) { + int status = ere.getStatusCode(); + if(ere.getErrorMessage().contains("Read operation not permitted on a directory.")){ + throw ere; + } + boolean isHnsEnabled = client.getIsNamespaceEnabled(); + + // Case: 404 NOT FOUND + if (status == HttpURLConnection.HTTP_NOT_FOUND) { + + // HNS account → plain FileNotFound + if (isHnsEnabled) { + throw new FileNotFoundException(ere.getMessage()); + } + + // FNS account → do GPS check + AbfsHttpOperation gpsOp = client.getPathStatus( + getRelativePath(new Path(path)), + false, + tracingContext, + contextEncryptionAdapter).getResult(); + + String resourceType = + gpsOp.getResponseHeaderIgnoreCase(X_MS_META_HDI_ISFOLDER); + + if (TRUE.equals(resourceType)) { + throw directoryReadException(); + } + + // Not a directory → plain FileNotFound throw new FileNotFoundException(ere.getMessage()); } + + // Case: 416 + non-namespace-enabled + //todo: convert to azureserviceerrorcode + if (!isHnsEnabled && status == 416) { + AbfsHttpOperation gpsOp = client.getPathStatus( + getRelativePath(new Path(path)), + false, + tracingContext, + contextEncryptionAdapter).getResult(); + + String resourceType = + gpsOp.getResponseHeaderIgnoreCase(X_MS_META_HDI_ISFOLDER); + + if (TRUE.equals(resourceType)) { + throw directoryReadException(); + } + } + + // Default: propagate original error + throw new IOException(ex); } throw new IOException(ex); } @@ -693,9 +775,12 @@ public synchronized int available() throws IOException { throw new IOException( FSExceptionMessages.STREAM_IS_CLOSED); } - final long remaining = this.contentLength - this.getPos(); - return remaining <= Integer.MAX_VALUE - ? (int) remaining : Integer.MAX_VALUE; + if (!(shouldRestrictGpsOnOpenFile() && isFirstRead())) { + final long remaining = this.contentLength - this.getPos(); + return remaining <= Integer.MAX_VALUE + ? (int) remaining : Integer.MAX_VALUE; + } + return Integer.MAX_VALUE; } /** @@ -1088,4 +1173,8 @@ public int maxReadSizeForVectorReads() { protected long getContentLength() { return contentLength; } + + public boolean shouldRestrictGpsOnOpenFile() { + return context.shouldRestrictGpsOnOpenFile(); + } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStreamContext.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStreamContext.java index cb51fa22900e4..ea80c66a46aee 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStreamContext.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStreamContext.java @@ -59,6 +59,8 @@ public class AbfsInputStreamContext extends AbfsStreamContext { private boolean bufferedPreadDisabled; + private boolean restrictGpsOnOpenFile; + /** A BackReference to the FS instance that created this OutputStream. */ private BackReference fsBackRef; @@ -254,6 +256,12 @@ public AbfsInputStreamContext withEncryptionAdapter( return this; } + public AbfsInputStreamContext shouldRestrictGpsOnOpenFile( + final boolean restrictGpsOnOpenFile) { + this.restrictGpsOnOpenFile = restrictGpsOnOpenFile; + return this; + } + /** * Finalizes and validates the context configuration. *

@@ -337,6 +345,10 @@ public int getReadAheadBlockSize() { return readAheadBlockSize; } + public boolean shouldRestrictGpsOnOpenFile() { + return this.restrictGpsOnOpenFile; + } + /** @return whether buffered pread is disabled. */ public boolean isBufferedPreadDisabled() { return bufferedPreadDisabled; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPrefetchInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPrefetchInputStream.java index c0343ca724e05..34172d7c644ad 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPrefetchInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPrefetchInputStream.java @@ -21,6 +21,7 @@ import java.io.IOException; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.azurebfs.constants.ReadType; import org.apache.hadoop.fs.azurebfs.utils.TracingContext; /** @@ -66,7 +67,7 @@ protected int readOneBlock(final byte[] b, final int off, final int len) throws // If buffer is empty, then fill the buffer. if (getBCursor() == getLimit()) { // If EOF, then return -1 - if (getFCursor() >= getContentLength()) { + if (!(shouldRestrictGpsOnOpenFile() && isFirstRead()) && getFCursor() >= getContentLength()) { return -1; } @@ -80,10 +81,22 @@ protected int readOneBlock(final byte[] b, final int off, final int len) throws } /* - * Always start with Prefetch even from first read. - * Even if out of order seek comes, prefetches will be triggered for next set of blocks. + Skips prefetch for the first read if isRestrictGpsOnOpenFile config is enabled. + This is required since contentLength is not available yet to determine prefetch block size. */ - bytesRead = readInternal(getFCursor(), getBuffer(), 0, getBufferSize(), false); + if(shouldRestrictGpsOnOpenFile() && isFirstRead()) { + getTracingContext().setReadType(ReadType.NORMAL_READ); + LOG.debug("RestrictGpsOnOpenFile is enabled. Skip readahead for first read even for sequential input policy."); + bytesRead = readInternal(getFCursor(), getBuffer(), 0, getBufferSize(), true); + } + else { + /* + * Always start with Prefetch even from first read UNLESS isRestrictGpsOnOpenFile config is enabled. + * Even if out of order seek comes, prefetches will be triggered for next set of blocks. + */ + bytesRead = readInternal(getFCursor(), getBuffer(), 0, getBufferSize(), false); + } + if (isFirstRead()) { setFirstRead(false); } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRandomInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRandomInputStream.java index b484cc6c84353..d676cfd49bf75 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRandomInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRandomInputStream.java @@ -68,7 +68,7 @@ protected int readOneBlock(final byte[] b, final int off, final int len) // If buffer is empty, then fill the buffer. if (getBCursor() == getLimit()) { // If EOF, then return -1 - if (getFCursor() >= getContentLength()) { + if (!(shouldRestrictGpsOnOpenFile() && isFirstRead()) && getFCursor() >= getContentLength()) { return -1; } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java index 5cf0bd473fc24..d9a0142b636dd 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java @@ -23,15 +23,19 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.URL; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Optional; -import java.util.Random; +import java.nio.charset.StandardCharsets; +import java.util.*; import java.util.concurrent.ExecutionException; import org.apache.hadoop.fs.azurebfs.AbfsConfiguration; import org.apache.hadoop.fs.azurebfs.AbfsCountersImpl; +import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys; +import org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations; +import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException; +import org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode; +import org.apache.hadoop.fs.azurebfs.extensions.EncryptionContextProvider; +import org.apache.hadoop.fs.azurebfs.security.ABFSKey; +import org.apache.hadoop.fs.azurebfs.utils.EncryptionType; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -74,26 +78,13 @@ import static org.apache.hadoop.fs.azurebfs.constants.ReadType.PREFETCH_READ; import static org.apache.hadoop.fs.azurebfs.constants.ReadType.RANDOM_READ; import static org.apache.hadoop.fs.azurebfs.constants.ReadType.SMALLFILE_READ; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.anyList; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.nullable; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +import static org.mockito.ArgumentMatchers.*; import static org.apache.hadoop.test.LambdaTestUtils.intercept; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.FORWARD_SLASH; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_READ_AHEAD_QUEUE_DEPTH; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.*; /** * Unit test AbfsInputStream. @@ -135,6 +126,18 @@ AbfsRestOperation getMockRestOp() { return op; } + AbfsRestOperation getMockRestOpWithMetadata() { + AbfsRestOperation op = mock(AbfsRestOperation.class); + AbfsHttpOperation httpOp = mock(AbfsHttpOperation.class); + when(httpOp.getBytesReceived()).thenReturn(1024L); + when(op.getResult()).thenReturn(httpOp); + when(op.getSasToken()).thenReturn(TestCachedSASToken.getTestCachedSASTokenInstance().get()); + when(op.getResult().getResponseHeader(HttpHeaderConfigurations.CONTENT_RANGE)).thenReturn("bytes 0-1023/1024"); + when(op.getResult().getResponseHeader(HttpHeaderConfigurations.ETAG)).thenReturn("etag"); + + return op; + } + AbfsClient getMockAbfsClient() throws URISyntaxException { // Mock failure for client.read() AbfsClient client = mock(AbfsClient.class); @@ -350,6 +353,614 @@ public void testOpenFileWithOptions() throws Exception { AbfsRestOperationType.GetPathStatus)); } +/** + * Mocks an {@link AbfsClient} to simulate encryption context behavior for testing. + * Sets up the client to return ENCRYPTION_CONTEXT as the encryption type and all the necessary + * mock responses to simulate reading a file with encryption context. + * + * @param encryptedClient the {@link AbfsClient} to mock + * @throws IOException if mocking fails + */ +private void mockClientForEncryptionContext(AbfsClient encryptedClient) throws IOException { + doReturn(EncryptionType.ENCRYPTION_CONTEXT) + .when(encryptedClient) + .getEncryptionType(); + + AbfsHttpOperation mockOp = mock(AbfsHttpOperation.class); + when(mockOp.getResponseHeader(HttpHeaderConfigurations.X_MS_ENCRYPTION_CONTEXT)) + .thenReturn(Base64.getEncoder() + .encodeToString("ctx".getBytes(StandardCharsets.UTF_8))); + when(mockOp.getResponseHeader(HttpHeaderConfigurations.CONTENT_LENGTH)) + .thenReturn("10"); + + AbfsRestOperation mockResult = mock(AbfsRestOperation.class); + when(mockResult.getResult()).thenReturn(mockOp); + + doReturn(mockResult) + .when(encryptedClient) + .getPathStatus(anyString(), anyBoolean(), any(), any()); + + doReturn(false) + .when(encryptedClient) + .checkIsDir(any()); + + EncryptionContextProvider provider = + mock(EncryptionContextProvider.class); + when(provider.getEncryptionKey(anyString(), any())) + .thenReturn(new ABFSKey(new byte[32])); + + doReturn(provider) + .when(encryptedClient) + .getEncryptionContextProvider(); +} + +/** + * Tests opening encrypted and non-encrypted files under different clients. + * Verifies that even with restrictGpsOnOpenFile enabled, for files created with ENCRYPTION_CONTEXT, the client invokes + * getPathStatus, while for non-encrypted files, getPathStatus is not called. + * + * @throws Exception if any error occurs during the test + */ +@Test +public void testMixedEncryptedAndNonEncryptedOpenUnderDifferentClients() + throws Exception { + + Configuration conf = getRawConfiguration(); + conf.setBoolean( + ConfigurationKeys.FS_AZURE_RESTRICT_GPS_ON_OPENFILE, + true); + + AzureBlobFileSystem fs = getFileSystem(conf); + + Path encryptedFile = new Path("/enc/file1"); + Path plainFile = new Path("/plain/file2"); + + fs.mkdirs(encryptedFile.getParent()); + fs.mkdirs(plainFile.getParent()); + + writeBufferToNewFile(encryptedFile, new byte[10]); + writeBufferToNewFile(plainFile, new byte[10]); + + TracingContext tracingContext = getTestTracingContext(fs, false); + + /* + * ========================= + * Client A — ENCRYPTED FILE + * ========================= + */ + AzureBlobFileSystemStore encryptedStore = getAbfsStore(fs); + + AbfsClient encryptedRealClient = + getAbfsClient(encryptedStore); + AbfsClient encryptedClient = + spy(encryptedRealClient); + + setAbfsClient(encryptedStore, encryptedClient); + mockClientForEncryptionContext(encryptedClient); + + encryptedStore.openFileForRead( + encryptedFile, Optional.empty(), null, tracingContext); + + // File created with ENCRYPTION_CONTEXT, so GPS should be invoked + verify(encryptedClient, times(1)) + .getPathStatus(anyString(), anyBoolean(), any(), isNull()); + + /* + * ============================= + * Client B — NON-ENCRYPTED FILE + * ============================= + */ + AzureBlobFileSystemStore plainStore = getAbfsStore(fs); + + AbfsClient plainRealClient = + getAbfsClient(plainStore); + AbfsClient plainClient = + spy(plainRealClient); + + setAbfsClient(plainStore, plainClient); + + doReturn(EncryptionType.NONE) + .when(plainClient) + .getEncryptionType(); + + plainStore.openFileForRead( + plainFile, Optional.empty(), null, tracingContext); + + verify(plainClient, never()) + .getPathStatus(anyString(), anyBoolean(), any(), any()); +} + + /** + * Verifies the prefetch behavior of the input stream by performing two reads and checking + * the number of times the client's read method is invoked after each read. + * + * @param fs the AzureBlobFileSystem instance + * @param store the AzureBlobFileSystemStore instance + * @param config the AbfsConfiguration instance + * @param file the file path to read from + * @param restrictGps whether to restrict GPS on open file + * @param readsAfterFirst expected number of client.read invocations after the first read + * @param readsAfterSecond expected number of client.read invocations after the second read + * @throws Exception if any error occurs during verification + */ + private void verifyPrefetchBehavior( + AzureBlobFileSystem fs, + AzureBlobFileSystemStore store, + AbfsConfiguration config, + Path file, + boolean restrictGps, + int readsAfterFirst, + int readsAfterSecond) throws Exception { + + AbfsClient realClient = store.getClient(); + AbfsClient spyClient = Mockito.spy(realClient); + Mockito.doReturn(spyClient).when(store).getClient(); + + Mockito.doReturn(restrictGps) + .when(config).shouldRestrictGpsOnOpenFile(); + + try (FSDataInputStream in = fs.open(file)) { + AbfsInputStream abfsIn = + (AbfsInputStream) in.getWrappedStream(); + + // First read. Sleep for a sec to get the readAhead threads to complete + abfsIn.read(new byte[ONE_MB]); + Thread.sleep(1000); + + verify(spyClient, times(readsAfterFirst)) + .read(anyString(), anyLong(), any(byte[].class), + anyInt(), anyInt(), + anyString(), nullable(String.class), + any(ContextEncryptionAdapter.class), + any(TracingContext.class)); + + // Second read. Sleep for a sec to get the readAhead threads to complete + abfsIn.read(ONE_MB, new byte[ONE_MB], 0, ONE_MB); + Thread.sleep(1000); + + verify(spyClient, times(readsAfterSecond)) + .read(anyString(), anyLong(), any(byte[].class), + anyInt(), anyInt(), + anyString(), nullable(String.class), + any(ContextEncryptionAdapter.class), + any(TracingContext.class)); + } + } + + /** + * Tests the prefetch behavior of the input stream when restrictGPSOnOpenFile is enabled. + * First read: only direct read is triggered. + * Second read: triggers readahead reads. + * Verifies the expected number of read invocations after each read operation + * + * @throws Exception if any error occurs during the test + */ + @Test + public void testPrefetchBehaviourWithRestrictGPSOnOpenFile() throws Exception { + AzureBlobFileSystem fs = Mockito.spy(getFileSystem()); + AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore()); + AbfsConfiguration config = Mockito.spy(store.getAbfsConfiguration()); + + Mockito.doReturn(ONE_MB).when(config).getReadBufferSize(); + Mockito.doReturn(ONE_MB).when(config).getReadAheadBlockSize(); + Mockito.doReturn(3).when(config).getReadAheadQueueDepth(); + Mockito.doReturn(true).when(config).isReadAheadEnabled(); + + Mockito.doReturn(store).when(fs).getAbfsStore(); + Mockito.doReturn(config).when(store).getAbfsConfiguration(); + + Path file = createTestFile(fs, 4 * ONE_MB); + + // restrictGPSOnOpenFile set as true + verifyPrefetchBehavior( + fs, store, config, file, + true, + 1, // only direct read + 4 // second read triggers readaheads + ); + } + + + private void assertReadType(AzureBlobFileSystem fs, int numOfReadCalls, + int totalReadCalls, ReadType readType, boolean isRandom) throws Exception { + ArgumentCaptor captor1 = ArgumentCaptor.forClass(String.class); + ArgumentCaptor captor2 = ArgumentCaptor.forClass(Long.class); + ArgumentCaptor captor3 = ArgumentCaptor.forClass(byte[].class); + ArgumentCaptor captor4 = ArgumentCaptor.forClass(Integer.class); + ArgumentCaptor captor5 = ArgumentCaptor.forClass(Integer.class); + ArgumentCaptor captor6 = ArgumentCaptor.forClass(String.class); + ArgumentCaptor captor7 = ArgumentCaptor.forClass(String.class); + ArgumentCaptor captor8 = ArgumentCaptor.forClass(ContextEncryptionAdapter.class); + ArgumentCaptor captor9 = ArgumentCaptor.forClass(TracingContext.class); + + verify(fs.getAbfsStore().getClient(), times(totalReadCalls)).read( + captor1.capture(), captor2.capture(), captor3.capture(), + captor4.capture(), captor5.capture(), captor6.capture(), + captor7.capture(), captor8.capture(), captor9.capture()); + List tracingContextList = captor9.getAllValues(); + + if(!isRandom){ + verifyHeaderForReadTypeInTracingContextHeader(tracingContextList.get(0), NORMAL_READ, 0); + } + + if (readType == PREFETCH_READ) { + /* + * For Prefetch Enabled, first read will be Normal or Missed Cache Read. + * So we will assert only for last 2 calls which should be Prefetched Read. + * Since calls are asynchronous, we can not guarantee the order of calls. + * Therefore, we cannot assert on exact position here. + */ + for (int i = tracingContextList.size() - (numOfReadCalls - 2); i < tracingContextList.size(); i++) { + verifyHeaderForReadTypeInTracingContextHeader(tracingContextList.get(i), readType, -1); + } + } else if (readType == DIRECT_READ) { + int expectedReadPos = ONE_MB/3; + for (int i = tracingContextList.size() - numOfReadCalls; i < tracingContextList.size(); i++) { + verifyHeaderForReadTypeInTracingContextHeader(tracingContextList.get(i), readType, expectedReadPos); + expectedReadPos += ONE_MB; + } + } else if (readType == MISSEDCACHE_READ) { + int expectedReadPos = ONE_MB; + for (int i = tracingContextList.size() - numOfReadCalls+1; i < tracingContextList.size(); i++) { + verifyHeaderForReadTypeInTracingContextHeader(tracingContextList.get(i), readType, expectedReadPos); + expectedReadPos += ONE_MB; + } + } + else { + //normal raed + int expectedReadPos = ONE_MB; + for (int i = tracingContextList.size() - numOfReadCalls+1; i < tracingContextList.size(); i++) { + verifyHeaderForReadTypeInTracingContextHeader(tracingContextList.get(i), readType, expectedReadPos); + expectedReadPos += ONE_MB; + } + } + } + + private void testReadTypeWithRestrictGPSOnOpenFile(AzureBlobFileSystem fs, + int fileSize, ReadType readType, Boolean isRandom, int numOfReadCalls, int totalReadCalls) throws Exception { + Path testPath = createTestFile(fs, fileSize); + try (FSDataInputStream iStream = fs.open(testPath)) { + int bytesRead = iStream.read(new byte[fileSize], 0, + fileSize); + assertThat(fileSize) + .describedAs("Read size should match file size") + .isEqualTo(bytesRead); + } + assertReadType(fs, numOfReadCalls, totalReadCalls, readType, isRandom); + } + + @Test + public void testReadTypeIn() throws Exception { + AzureBlobFileSystem spiedFs = Mockito.spy(getFileSystem()); + AzureBlobFileSystemStore spiedStore = Mockito.spy(spiedFs.getAbfsStore()); + AbfsConfiguration spiedConfig = Mockito.spy(spiedStore.getAbfsConfiguration()); + AbfsClient spiedClient = Mockito.spy(spiedStore.getClient()); + Mockito.doReturn(ONE_MB).when(spiedConfig).getReadBufferSize(); + Mockito.doReturn(true).when(spiedConfig).shouldRestrictGpsOnOpenFile(); + Mockito.doReturn(false).when(spiedConfig).isReadAheadV2Enabled(); + Mockito.doReturn(spiedClient).when(spiedStore).getClient(); + Mockito.doReturn(spiedStore).when(spiedFs).getAbfsStore(); + Mockito.doReturn(spiedConfig).when(spiedStore).getAbfsConfiguration(); + int totalReadCalls = 0; + int fileSize; + + //adaptive- prefetch- 1st NR, then PR/MR + //adaptive-no prefetch (prefetch DISABLED)- all NR (NO TEST REQD) + //adaptive- prefetch true, readahead depth=0, - first NR, then all MR + + //sequential- prefetch- 1st NR, then PR/MR + //sequential-no prefetch- 1st NR, then MR + + //random remains random + + /* + * Test to verify Prefetch Read Type. + * Setting read ahead depth to 2 with prefetch enabled ensures that prefetch is done. + * + * ADaptive-prefetch- gives NR for first read + */ + fileSize = 3 * ONE_MB; // To make sure multiple blocks are read. + totalReadCalls += 3; + doReturn(true).when(spiedConfig).isReadAheadEnabled(); + Mockito.doReturn(3).when(spiedConfig).getReadAheadQueueDepth(); + testReadTypeWithRestrictGPSOnOpenFile(spiedFs, fileSize, PREFETCH_READ, false, 3, totalReadCalls); + + //Adaptive- no prefetch queue- 1st NR, then MR + fileSize = 3 * ONE_MB; // To make sure multiple blocks are read. + totalReadCalls += 3; + doReturn(true).when(spiedConfig).isReadAheadEnabled(); + Mockito.doReturn(0).when(spiedConfig).getReadAheadQueueDepth(); + testReadTypeWithRestrictGPSOnOpenFile(spiedFs, fileSize, MISSEDCACHE_READ, false, 3, totalReadCalls); + + /* + * Test to verify Footer Read Type. + * Having file size less than footer read size and disabling small file opt + */ + fileSize = 8 * ONE_KB; + totalReadCalls += 1; // Full file will be read along with footer. + doReturn(false).when(spiedConfig).readSmallFilesCompletely(); + doReturn(true).when(spiedConfig).optimizeFooterRead(); + testReadTypeWithRestrictGPSOnOpenFile(spiedFs, fileSize, NORMAL_READ, false, 1, totalReadCalls); + + /* + * Test to verify Small File Read Type. + * Having file size less than block size and disabling footer read opt + */ + totalReadCalls += 1; // Full file will be read along with footer. + doReturn(true).when(spiedConfig).readSmallFilesCompletely(); + doReturn(false).when(spiedConfig).optimizeFooterRead(); + testReadTypeWithRestrictGPSOnOpenFile(spiedFs, fileSize, NORMAL_READ, false, 1, totalReadCalls); + + //SEquential-no prefetch (depth0)- should give NR for firstread and MR for remaining 2 + fileSize = 3 * ONE_MB; // To make sure multiple blocks are read. + totalReadCalls += 3; + Mockito.doReturn(0).when(spiedConfig).getReadAheadQueueDepth(); + Mockito.doReturn(FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL).when(spiedConfig).getAbfsReadPolicy(); + doReturn(true).when(spiedConfig).isReadAheadEnabled(); + testReadTypeWithRestrictGPSOnOpenFile(spiedFs, fileSize, MISSEDCACHE_READ, false, 3, totalReadCalls); +// +// //SEquential-yes prefetch- should give NR for firstread and PR/MR fr second for PR for last + fileSize = 3 * ONE_MB; // To make sure multiple blocks are read. + totalReadCalls += 3; + Mockito.doReturn(3).when(spiedConfig).getReadAheadQueueDepth(); + doReturn(true).when(spiedConfig).isReadAheadEnabled(); + testReadTypeWithRestrictGPSOnOpenFile(spiedFs, fileSize, PREFETCH_READ, false, 3, totalReadCalls); + + /* + * Test to verify Random Read Type. + * Setting Read Policy to Parquet ensures Random Read Type. + */ + fileSize = 3 * ONE_MB; // To make sure multiple blocks are read. + totalReadCalls += 3; // Full file will be read along with footer. + doReturn(FS_OPTION_OPENFILE_READ_POLICY_PARQUET).when(spiedConfig).getAbfsReadPolicy(); + testReadTypeWithRestrictGPSOnOpenFile(spiedFs, fileSize, RANDOM_READ, true, 3, totalReadCalls); + } + + /** + * Tests that for FNS accounts reading from a directory with FS_AZURE_RESTRICT_GPS_ON_OPENFILE enabled + * throws the expected exception with PATH_NOT_FOUND status code and respective error message. + * Verifies that getPathStatus is called for both explicit and implicit folders' confirmation. + * + * @throws Exception if any error occurs during the test + */ + @Test +public void testFNSExceptionOnDirReadWithRestrictGPSConfig() throws Exception { + assumeHnsDisabled(); + Configuration conf = getRawConfiguration(); + conf.setBoolean(ConfigurationKeys.FS_AZURE_RESTRICT_GPS_ON_OPENFILE, true); + + AzureBlobFileSystem fs = getFileSystem(conf); + AzureBlobFileSystemStore store = getAbfsStore(fs); + + AbfsClient realClient = getAbfsClient(store); + AbfsClient spyClient = spy(realClient); + setAbfsClient(store, spyClient); + + String explicitTestFolder = "/testExplFolderRestrictGps"; + fs.mkdirs(new Path(explicitTestFolder)); + + try (FSDataInputStream in = fs.open(new Path(explicitTestFolder))) { + AbfsInputStream abfsIn = (AbfsInputStream) in.getWrappedStream(); + byte[] buf = new byte[100]; + AbfsRestOperationException ex = Assertions.assertThrows(AbfsRestOperationException.class, () -> abfsIn.read(buf)); + assertThat(ex.getStatusCode()).isEqualTo(AzureServiceErrorCode.PATH_NOT_FOUND.getStatusCode()); + assertThat(ex.getMessage()).contains("Read operation not permitted on a directory."); + } + verify(spyClient, times(1)) + .getPathStatus( + anyString(), + anyBoolean(), + any(TracingContext.class), + any()); + + String implicitTestFolder = "/testImplFolderRestrictGps"; + createAzCopyFolder(new Path(implicitTestFolder)); + try (FSDataInputStream in2 = fs.open(new Path(implicitTestFolder))) { + AbfsInputStream abfsIn2 = (AbfsInputStream) in2.getWrappedStream(); + byte[] buf2 = new byte[100]; + AbfsRestOperationException ex2 = Assertions.assertThrows(AbfsRestOperationException.class, () -> abfsIn2.read(buf2)); + assertThat(ex2.getStatusCode()).isEqualTo(AzureServiceErrorCode.PATH_NOT_FOUND.getStatusCode()); + assertThat(ex2.getMessage()).contains("Read operation not permitted on a directory."); + } + verify(spyClient, times(2)) + .getPathStatus( + anyString(), + anyBoolean(), + any(TracingContext.class), + any()); +} + +/** + * Tests that for HNS accounts reading from a directory when + * FS_AZURE_RESTRICT_GPS_ON_OPENFILE is set to true, throws an AbfsRestOperationException + * with PATH_NOT_FOUND status code and the correct error message. + * Also verifies that getPathStatus is not called. + * + * @throws Exception if any error occurs during the test + */ +@Test +public void testHNSExceptionOnDirReadWithRestrictGPSConfig() throws Exception { + assumeHnsEnabled(); + Configuration conf = getRawConfiguration(); + conf.setBoolean(ConfigurationKeys.FS_AZURE_RESTRICT_GPS_ON_OPENFILE, true); + + AzureBlobFileSystem fs = getFileSystem(conf); + AzureBlobFileSystemStore store = getAbfsStore(fs); + + AbfsClient realClient = getAbfsClient(store); + AbfsClient spyClient = spy(realClient); + setAbfsClient(store, spyClient); + + String testFolder = "/testFolderRestrictGps"; + fs.mkdirs(new Path(testFolder)); + + try (FSDataInputStream in = fs.open(new Path(testFolder))) { + AbfsInputStream abfsIn = (AbfsInputStream) in.getWrappedStream(); + byte[] buf = new byte[100]; + AbfsRestOperationException ex = Assertions.assertThrows(AbfsRestOperationException.class, () -> abfsIn.read(buf)); + assertThat(ex.getStatusCode()).isEqualTo(AzureServiceErrorCode.PATH_NOT_FOUND.getStatusCode()); + assertThat(ex.getMessage()).contains("Read operation not permitted on a directory."); + + } + verify(spyClient, times(0)) + .getPathStatus( + anyString(), + anyBoolean(), + any(TracingContext.class), + any()); +} + + +/** + * Tests the behavior of openFileForRead with the FS_AZURE_RESTRICT_GPS_ON_OPENFILE configuration enabled. + * Verifies that irrespective of whether FileStatus is provided or not, getPathStatus is not invoked for read flow. + * + * @throws Exception if any error occurs during the test + */ +@Test +public void testOpenFileWithOptionsWithRestrictGpsOnOpenFile() throws Exception { + Configuration conf = getRawConfiguration(); + conf.setBoolean(ConfigurationKeys.FS_AZURE_RESTRICT_GPS_ON_OPENFILE, true); + + AzureBlobFileSystem fs = getFileSystem(conf); + Path fileWithFileStatus = new Path("/testFile0"); + Path fileWithoutFileStatus = new Path("/testFile1"); + + // Create and write to both files, but we'll be sending FileStatus only for one of them + writeBufferToNewFile(fileWithFileStatus, new byte[5]); + writeBufferToNewFile(fileWithoutFileStatus, new byte[5]); + + AzureBlobFileSystemStore abfsStore = getAbfsStore(fs); + AbfsClient mockClient = spy(getAbfsClient(abfsStore)); + setAbfsClient(abfsStore, mockClient); + TracingContext tracingContext = getTestTracingContext(fs, false); + + // NOTE: One call for GPS needs to come from openFileForWrite. If GPS were happening at openFileForRead, we would've seen 2 calls. + + // Case 1: FileStatus is provided + abfsStore.openFileForRead(fileWithFileStatus, Optional.ofNullable(new OpenFileParameters().withStatus(fs.getFileStatus(fileWithFileStatus))), null, tracingContext); + verify(mockClient, times(1).description("FileStatus provided, restrict GPS: getPathStatus should NOT be invoked")) + .getPathStatus(any(String.class), any(Boolean.class), any(TracingContext.class), nullable(ContextEncryptionAdapter.class)); + + // Case 2: FileStatus is not provided + abfsStore.openFileForRead(fileWithoutFileStatus, Optional.empty(), null, tracingContext); + verify(mockClient, times(1).description("FileStatus not provided, restrict GPS: getPathStatus should NOT be invoked")) + .getPathStatus(any(String.class), any(Boolean.class), any(TracingContext.class), nullable(ContextEncryptionAdapter.class)); +} + + /** + * Tests that when FS_AZURE_RESTRICT_GPS_ON_OPENFILE is enabled, + * the eTag and content length metadata are correctly initialized + * after the first read operation, and remain consistent for subsequent reads. + * + * @throws Exception if any error occurs during the test + */ + @Test + public void testMetadataFromReadForRestrictGpsOnOpenFile() throws Exception { + Configuration conf = getRawConfiguration(); + conf.setBoolean(org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_RESTRICT_GPS_ON_OPENFILE, true); + + AzureBlobFileSystem fs = getFileSystem(conf); + Path testFile = new Path("/testFile0"); + writeBufferToNewFile(testFile, new byte[250]); + + // Open the file and perform a read + try (FSDataInputStream in = fs.open(testFile)) { + AbfsInputStream abfsIn = (AbfsInputStream) in.getWrappedStream(); + + // Before first read, eTag and content length aren't initialized + String etagPreRead = abfsIn.getETag(); + long fileLengthPreRead = abfsIn.getContentLength(); + assertThat(etagPreRead).isEmpty(); + assertThat(fileLengthPreRead).isEqualTo(0); + + // Trigger the first read + byte[] buf = new byte[100]; + int n = abfsIn.read(buf); + assertThat(n).isGreaterThan(0); + + // After first read, eTag and content length should be set from the response + String etagFirstRead = abfsIn.getETag(); + long fileLengthFirstRead = abfsIn.getContentLength(); + assertThat(etagFirstRead).isNotNull(); + assertThat(fileLengthFirstRead).isEqualTo(250L); + + //Trigger the second read + n = abfsIn.read(100, buf, 0, 100); + assertThat(n).isGreaterThan(0); + + // eTag and content length should remain same as first read for second read onwards + String etagSecondRead = abfsIn.getETag(); + long fileLengthSecondRead = abfsIn.getContentLength(); + assertThat(etagSecondRead).isEqualTo(etagFirstRead); + assertThat(fileLengthSecondRead).isEqualTo(fileLengthFirstRead); + } + } + +/** + * Tests that when FS_AZURE_RESTRICT_GPS_ON_OPENFILE is enabled, + * metadata (eTag and content length) is not partially initialized if read requests fail with a timeout. + * Ensures that after failed reads, metadata remains unset, and only after a successful read + * are the metadata fields correctly initialized. + * + * @throws Exception if any error occurs during the test + */ +@Test +public void testMetadataNotPartiallyInitializedOnReadWithRestrictGpsOnOpenFile() + throws Exception { + + Configuration conf = getRawConfiguration(); + conf.setBoolean( + ConfigurationKeys.FS_AZURE_RESTRICT_GPS_ON_OPENFILE, true); + + AzureBlobFileSystem fs = spy(getFileSystem(conf)); + + AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore()); + Mockito.doReturn(store).when(fs).getAbfsStore(); + AbfsClient client = Mockito.spy(store.getClient()); + Mockito.doReturn(client).when(store).getClient(); + + AbfsRestOperation successOp = getMockRestOpWithMetadata(); + + doThrow(new TimeoutException("First-read-failure")) + .doThrow(new TimeoutException("Second-read-failure")) + .doReturn(successOp) + .when(client) + .read(any(String.class), any(Long.class), any(byte[].class), + any(Integer.class), any(Integer.class), any(String.class), + nullable(String.class), any(ContextEncryptionAdapter.class), any(TracingContext.class)); + + Path testFile = new Path("/testFile0"); + byte[] fileContent = new byte[ONE_KB]; + writeBufferToNewFile(testFile, fileContent); + + try (FSDataInputStream in = fs.open(testFile)) { + AbfsInputStream abfsIn = (AbfsInputStream) in.getWrappedStream(); + + // Metadata not initialized before read + assertThat(abfsIn.getETag()).isEmpty(); + assertThat(abfsIn.getContentLength()).isEqualTo(0); + + // First read fails- metadata should not be initialized + intercept(IOException.class, + () -> abfsIn.read(fileContent)); + assertThat(abfsIn.getETag()).isEmpty(); + assertThat(abfsIn.getContentLength()).isEqualTo(0); + + // Second read fails- metadata should not be initialized + intercept(IOException.class, + () -> abfsIn.read(fileContent)); + assertThat(abfsIn.getETag()).isEmpty(); + assertThat(abfsIn.getContentLength()).isEqualTo(0); + + // Third read succeeds- metadata should be initialized + abfsIn.read(fileContent); + assertThat(abfsIn.getETag()).isEqualTo("etag"); + assertThat(abfsIn.getContentLength()).isEqualTo(1024L); + } +} + + /** * This test expects AbfsInputStream to throw the exception that readAhead * thread received on read. The readAhead thread must be initiated from the @@ -487,6 +1098,7 @@ public void testOlderReadAheadFailure() throws Exception { * @throws Exception */ @Test + //TODO: HEREEEE public void testSuccessfulReadAhead() throws Exception { // Mock failure for client.read() AbfsClient client = getMockAbfsClient(); @@ -822,6 +1434,7 @@ public void testDefaultReadaheadQueueDepth() throws Exception { * @throws Exception if any error occurs during the test */ @Test + //todo: HEREREEE public void testReadTypeInTracingContextHeader() throws Exception { AzureBlobFileSystem spiedFs = Mockito.spy(getFileSystem()); AzureBlobFileSystemStore spiedStore = Mockito.spy(spiedFs.getAbfsStore()); From b935f8fff52df78644b2c31279e6fb0f7de351a3 Mon Sep 17 00:00:00 2001 From: Manika Joshi Date: Thu, 22 Jan 2026 02:30:00 -0800 Subject: [PATCH 11/14] corrections --- .../hadoop/fs/azurebfs/AbfsConfiguration.java | 4 + .../fs/azurebfs/AzureBlobFileSystemStore.java | 62 +- .../azurebfs/constants/ConfigurationKeys.java | 2 +- .../constants/FileSystemConfigurations.java | 5 - .../services/AzureServiceErrorCode.java | 2 + .../services/AbfsAdaptiveInputStream.java | 2 +- .../fs/azurebfs/services/AbfsBlobClient.java | 5 +- .../fs/azurebfs/services/AbfsDfsClient.java | 6 +- .../fs/azurebfs/services/AbfsInputStream.java | 115 +-- .../services/AbfsInputStreamContext.java | 7 + .../services/AbfsPrefetchInputStream.java | 4 +- .../services/TestAbfsInputStream.java | 686 ++++++++++-------- 12 files changed, 500 insertions(+), 400 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index 86691fb5726ed..85dce25db70f0 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -1449,6 +1449,10 @@ public String getAbfsReadPolicy() { return abfsReadPolicy; } +/** + * Indicates whether GPS restriction on open file is enabled. + * @return true if GPS restriction is enabled on open file, false otherwise. + */ public boolean shouldRestrictGpsOnOpenFile() { return restrictGpsOnOpenFile; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index 99dd603117ce2..b9e759312d8f3 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -145,7 +145,18 @@ import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.METADATA_INCOMPLETE_RENAME_FAILURES; import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.RENAME_RECOVERY; -import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.*; +import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.CHAR_EQUALS; +import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.CHAR_FORWARD_SLASH; +import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.CHAR_HYPHEN; +import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.CHAR_PLUS; +import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.CHAR_STAR; +import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.CHAR_UNDERSCORE; +import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.DIRECTORY; +import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.EMPTY_STRING; +import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.FILE; +import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.ROOT_PATH; +import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.SINGLE_WHITE_SPACE; +import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.TOKEN_VERSION; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_ABFS_ENDPOINT; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_FOOTER_READ_BUFFER_SIZE; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_BUFFERED_PREAD_DISABLE; @@ -417,7 +428,7 @@ private synchronized boolean getNamespaceEnabledInformationFromServer( } try { LOG.debug("Get root ACL status"); - getClient(AbfsServiceType.DFS).getAclStatus(ROOT_PATH, tracingContext); + getClient(AbfsServiceType.DFS).getAclStatus(AbfsHttpConstants.ROOT_PATH, tracingContext); // If getAcl succeeds, namespace is enabled. setNamespaceEnabled(true); } catch (AbfsRestOperationException ex) { @@ -554,7 +565,7 @@ public Hashtable getPathStatus(final Path path, /** * Creates an object of {@link ContextEncryptionAdapter} - * from a file path. It calls {@link AbfsClient + * from a file path. It calls {@link org.apache.hadoop.fs.azurebfs.services.AbfsClient * #getPathStatus(String, boolean, TracingContext, EncryptionAdapter)} method to get * contextValue (x-ms-encryption-context) from the server. The contextValue is passed * to the constructor of EncryptionAdapter to create the required object of @@ -563,11 +574,11 @@ public Hashtable getPathStatus(final Path path, * @return

    *
  • * {@link NoContextEncryptionAdapter}: if encryptionType is not of type - * {@link EncryptionType#ENCRYPTION_CONTEXT}. + * {@link org.apache.hadoop.fs.azurebfs.utils.EncryptionType#ENCRYPTION_CONTEXT}. *
    • *
  • * new object of {@link ContextProviderEncryptionAdapter} containing required encryptionKeys for the give file: - * if encryptionType is of type {@link EncryptionType#ENCRYPTION_CONTEXT}. + * if encryptionType is of type {@link org.apache.hadoop.fs.azurebfs.utils.EncryptionType#ENCRYPTION_CONTEXT}. *
    • *
*/ @@ -868,6 +879,20 @@ public AbfsInputStream openFileForRead(final Path path, tracingContext); } + /** + * Creates an exception indicating that openFileForRead was called on a directory. + * + * @return AbfsRestOperationException with PATH_NOT_FOUND error code and a message + * indicating that openFileForRead must be used with files and not directories. + */ + private AbfsRestOperationException openFileForReadDirectoryException() { + return new AbfsRestOperationException( + AzureServiceErrorCode.PATH_NOT_FOUND.getStatusCode(), + AzureServiceErrorCode.PATH_NOT_FOUND.getErrorCode(), + "openFileForRead must be used with files and not directories", + null); + } + public AbfsInputStream openFileForRead(Path path, final Optional parameters, final FileSystem.Statistics statistics, TracingContext tracingContext) @@ -885,8 +910,8 @@ public AbfsInputStream openFileForRead(Path path, ContextEncryptionAdapter contextEncryptionAdapter = NoContextEncryptionAdapter.getInstance(); /* * GetPathStatus API has to be called in case of: - * 1. fileStatus is null or not an object of VersionedFileStatus: as eTag - * would not be there in the fileStatus object. + * 1. restrictGpsOnOpenFile config is disabled AND fileStatus is null or not + * an object of VersionedFileStatus: as eTag would not be there in the fileStatus object. * 2. fileStatus is an object of VersionedFileStatus and the object doesn't * have encryptionContext field when client's encryptionType is * ENCRYPTION_CONTEXT. @@ -911,11 +936,7 @@ public AbfsInputStream openFileForRead(Path path, encryptionContext.getBytes(StandardCharsets.UTF_8)); } if (parseIsDirectory(resourceType)) { - throw new AbfsRestOperationException( - AzureServiceErrorCode.PATH_NOT_FOUND.getStatusCode(), - AzureServiceErrorCode.PATH_NOT_FOUND.getErrorCode(), - "openFileForRead must be used with files and not directories", - null); + throw openFileForReadDirectoryException(); } } /* @@ -948,13 +969,16 @@ else if (getClient().getEncryptionType() == EncryptionType.ENCRYPTION_CONTEXT eTag = op.getResponseHeader(HttpHeaderConfigurations.ETAG); if (parseIsDirectory(resourceType)) { - throw new AbfsRestOperationException( - AzureServiceErrorCode.PATH_NOT_FOUND.getStatusCode(), - AzureServiceErrorCode.PATH_NOT_FOUND.getErrorCode(), - "openFileForRead must be used with files and not directories", - null); + throw openFileForReadDirectoryException(); } } + /* The only remaining case is: + * - restrictGpsOnOpenFile config is enabled with null FileStatus and encryptionType not as ENCRYPTION_CONTEXT + * In this case, we don't need to call GetPathStatus API. + */ + else { + // do nothing + } perfInfo.registerSuccess(true); @@ -1357,7 +1381,7 @@ public String listStatus(final Path path, final String startFrom, // generate continuation token for xns account private String generateContinuationTokenForXns(final String firstEntryName) { Preconditions.checkArgument(!Strings.isNullOrEmpty(firstEntryName) - && !firstEntryName.startsWith(ROOT_PATH), + && !firstEntryName.startsWith(AbfsHttpConstants.ROOT_PATH), "startFrom must be a dir/file name and it can not be a full path"); StringBuilder sb = new StringBuilder(); @@ -1377,7 +1401,7 @@ private String generateContinuationTokenForXns(final String firstEntryName) { // generate continuation token for non-xns account private String generateContinuationTokenForNonXns(String path, final String firstEntryName) { Preconditions.checkArgument(!Strings.isNullOrEmpty(firstEntryName) - && !firstEntryName.startsWith(ROOT_PATH), + && !firstEntryName.startsWith(AbfsHttpConstants.ROOT_PATH), "startFrom must be a dir/file name and it can not be a full path"); // Notice: non-xns continuation token requires full path (first "/" is not included) for startFrom diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java index a1ad6ea2731df..a0612224a0804 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java @@ -665,7 +665,7 @@ public static String containerProperty(String property, String fsName, String ac public static final String FS_AZURE_TAIL_LATENCY_MAX_RETRY_COUNT = "fs.azure.tail.latency.max.retry.count"; /** - * If true, restricts GPS (Geo-Replication Secondary) access on openFile operations. + * If true, restricts GPS (getPathStatus) calls on openFileforRead * Default: false */ public static final String FS_AZURE_RESTRICT_GPS_ON_OPENFILE = "fs.azure.restrict.gps.on.openfile"; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java index c53648b03c5c1..b8f7da8204c4e 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java @@ -442,11 +442,6 @@ public final class FileSystemConfigurations { public static final int MIN_FS_AZURE_TAIL_LATENCY_ANALYSIS_WINDOW_GRANULARITY = 1; public static final int DEFAULT_FS_AZURE_TAIL_LATENCY_PERCENTILE_COMPUTATION_INTERVAL_MILLIS = 500; public static final int DEFAULT_FS_AZURE_TAIL_LATENCY_MAX_RETRY_COUNT = 1; - - /** - * If true, restricts GPS (getPathStatus) calls on openFile operations. - * Default: false - */ public static final boolean DEFAULT_FS_AZURE_RESTRICT_GPS_ON_OPENFILE = false; private FileSystemConfigurations() {} diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java index ce03d794ddb51..c38813a2e27d0 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java @@ -66,6 +66,8 @@ public enum AzureServiceErrorCode { INVALID_APPEND_OPERATION("InvalidAppendOperation", HttpURLConnection.HTTP_CONFLICT, null), UNAUTHORIZED_BLOB_OVERWRITE("UnauthorizedBlobOverwrite", HttpURLConnection.HTTP_FORBIDDEN, "This request is not authorized to perform blob overwrites."), + INVALID_RANGE("InvalidRange", 416, + "The range specified is invalid for the current size of the resource."), UNKNOWN(null, -1, null); private final String errorCode; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java index 9435472e52310..f50bfdd2e014d 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java @@ -84,7 +84,7 @@ protected int readOneBlock(final byte[] b, final int off, final int len) throws // Reset Read Type back to normal and set again based on code flow. getTracingContext().setReadType(ReadType.NORMAL_READ); if(shouldRestrictGpsOnOpenFile() && isFirstRead()) { - LOG.debug("RestrictGpsOnOpenFile is enabled. Skip readahead."); + LOG.debug("RestrictGpsOnOpenFile is enabled. Skip readahead for first read."); bytesRead = readInternal(getFCursor(), getBuffer(), 0, getBufferSize(), true); } else if (shouldAlwaysReadBufferSize()) { diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsBlobClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsBlobClient.java index cdf50535e0ee6..52fbd3182fdd5 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsBlobClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsBlobClient.java @@ -1321,10 +1321,7 @@ public AbfsRestOperation read(final String path, AbfsHttpHeader rangeHeader = new AbfsHttpHeader(RANGE, String.format( "bytes=%d-%d", position, position + bufferLength - 1)); requestHeaders.add(rangeHeader); - - // eTag won't be present for first read with restrict GPS on openFile config enabled - if (!eTag.isEmpty()) - requestHeaders.add(new AbfsHttpHeader(IF_MATCH, eTag)); + requestHeaders.add(new AbfsHttpHeader(IF_MATCH, eTag)); // Add request priority header for prefetch reads addRequestPriorityForPrefetch(requestHeaders, tracingContext); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsDfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsDfsClient.java index 40efa98354c7e..5ddb9770ac56e 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsDfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsDfsClient.java @@ -1045,11 +1045,7 @@ public AbfsRestOperation read(final String path, AbfsHttpHeader rangeHeader = new AbfsHttpHeader(RANGE, String.format("bytes=%d-%d", position, position + bufferLength - 1)); requestHeaders.add(rangeHeader); - - // eTag won't be present for first read with restrict GPS on openFile config enabled - if (!eTag.isEmpty()){ - requestHeaders.add(new AbfsHttpHeader(IF_MATCH, eTag)); - } + requestHeaders.add(new AbfsHttpHeader(IF_MATCH, eTag)); // Add request header to fetch MD5 Hash of data returned by server. if (isChecksumValidationEnabled(requestHeaders, rangeHeader, bufferLength)) { diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java index 616b6ca8c7f0f..3ef2c2248da0e 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java @@ -27,7 +27,13 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.fs.*; +import org.apache.hadoop.fs.CanUnbuffer; +import org.apache.hadoop.fs.FSExceptionMessages; +import org.apache.hadoop.fs.FSInputStream; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.StreamCapabilities; +import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants; +import org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations; import org.apache.hadoop.fs.azurebfs.constants.ReadType; import org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode; import org.apache.hadoop.fs.impl.BackReference; @@ -50,11 +56,15 @@ import static java.lang.Math.max; import static java.lang.Math.min; -import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.*; +import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.DIRECTORY; +import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.ROOT_PATH; +import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.TRUE; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.ONE_KB; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.STREAM_ID_LEN; import static org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations.X_MS_META_HDI_ISFOLDER; import static org.apache.hadoop.fs.azurebfs.constants.InternalConstants.CAPABILITY_SAFE_READAHEAD; +import static org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode.INVALID_RANGE; +import static org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode.UNAUTHORIZED_BLOB_OVERWRITE; import static org.apache.hadoop.io.Sizes.S_128K; import static org.apache.hadoop.io.Sizes.S_2M; import static org.apache.hadoop.util.StringUtils.toLowerCase; @@ -135,6 +145,7 @@ public abstract class AbfsInputStream extends FSInputStream implements CanUnbuff /** ABFS instance to be held by the input stream to avoid GC close. */ private final BackReference fsBackRef; private final ReadBufferManager readBufferManager; + private final String readOnDirectoryErrorMsg = "Read operation not permitted on a directory."; /** * Constructor for AbfsInputStream. @@ -313,6 +324,8 @@ public synchronized int read(final byte[] b, final int off, final int len) throw bCursor = 0; } + long nextReadPosForMetric = nextReadPos; + if (shouldReadFully()) { lastReadBytes = readFileCompletely(b, currentOff, currentLen); } else if (shouldReadLastBlock()) { @@ -322,7 +335,7 @@ public synchronized int read(final byte[] b, final int off, final int len) throw } if (abfsReadFooterMetrics != null) { - abfsReadFooterMetrics.updateReadMetrics(filePathIdentifier, len, contentLength, nextReadPos); + abfsReadFooterMetrics.updateReadMetrics(filePathIdentifier, len, contentLength, nextReadPosForMetric); } if (lastReadBytes > 0) { @@ -577,14 +590,39 @@ String getRelativePath(final Path path) { return relPath; } + /** + * Creates an exception indicating that a read operation was attempted on a directory. + * + * @return an {@link AbfsRestOperationException} indicating the operation is not permitted on a directory + */ private IOException directoryReadException() { return new AbfsRestOperationException( AzureServiceErrorCode.PATH_NOT_FOUND.getStatusCode(), AzureServiceErrorCode.PATH_NOT_FOUND.getErrorCode(), - "Read operation not permitted on a directory.", + readOnDirectoryErrorMsg, null); } + /** + * Checks if the current path is a directory (for both implicit and explicit) in FNS account. + * If the path is a directory, throws an exception indicating that read operations are not permitted. + * + * @throws IOException if the path is a directory or if there is an error accessing the path status + */ + private void checkIfDirPathInFNS() throws IOException { + AbfsHttpOperation gpsOp = client.getPathStatus( + getRelativePath(new Path(path)), + false, + tracingContext, + contextEncryptionAdapter).getResult(); + + String resourceType = + gpsOp.getResponseHeaderIgnoreCase(X_MS_META_HDI_ISFOLDER); + + if (TRUE.equals(resourceType)) { + throw directoryReadException(); + } + } int readRemote(long position, byte[] b, int offset, int length, TracingContext tracingContext) throws IOException { if (position < 0) { @@ -617,14 +655,17 @@ int readRemote(long position, byte[] b, int offset, int length, TracingContext t tolerateOobAppends ? "*" : eTag, cachedSasToken.get(), contextEncryptionAdapter, tracingContext); - if (shouldRestrictGpsOnOpenFile() && isFirstRead()){ - String resourceType = op.getResult().getResponseHeader("x-ms-resource-type"); - if (Objects.equals(resourceType, DIRECTORY)){ + // Update metadata on first read if restrictGpsOnOpenFile is enabled + if (shouldRestrictGpsOnOpenFile() && isFirstRead()) { + String resourceType = op.getResult().getResponseHeader(HttpHeaderConfigurations.X_MS_RESOURCE_TYPE); + if (Objects.equals(resourceType, DIRECTORY)) { throw directoryReadException(); } - contentLength = Long.parseLong(op.getResult().getResponseHeader("Content-Range").split("/")[1]); + contentLength = Long.parseLong(op.getResult().getResponseHeader(HttpHeaderConfigurations.CONTENT_RANGE). + split(AbfsHttpConstants.FORWARD_SLASH)[1]); eTag = op.getResult().getResponseHeader("ETag"); } + cachedSasToken.update(op.getSasToken()); LOG.debug("issuing HTTP GET request params position = {} b.length = {} " + "offset = {} length = {}", position, b.length, offset, length); @@ -634,52 +675,40 @@ int readRemote(long position, byte[] b, int offset, int length, TracingContext t if (ex instanceof AbfsRestOperationException) { AbfsRestOperationException ere = (AbfsRestOperationException) ex; int status = ere.getStatusCode(); - if(ere.getErrorMessage().contains("Read operation not permitted on a directory.")){ + if(ere.getErrorMessage().contains(readOnDirectoryErrorMsg)){ throw ere; } boolean isHnsEnabled = client.getIsNamespaceEnabled(); - // Case: 404 NOT FOUND + // Case-1: 404 NOT FOUND if (status == HttpURLConnection.HTTP_NOT_FOUND) { - - // HNS account → plain FileNotFound - if (isHnsEnabled) { + /* + * If HNS account or restrictGpsOnOpenFile disabled, + * we dont need any further checks + */ + if (isHnsEnabled || !shouldRestrictGpsOnOpenFile()) { throw new FileNotFoundException(ere.getMessage()); } - // FNS account → do GPS check - AbfsHttpOperation gpsOp = client.getPathStatus( - getRelativePath(new Path(path)), - false, - tracingContext, - contextEncryptionAdapter).getResult(); - - String resourceType = - gpsOp.getResponseHeaderIgnoreCase(X_MS_META_HDI_ISFOLDER); - - if (TRUE.equals(resourceType)) { - throw directoryReadException(); + // FNS account with restrictGpsOnOpenFile enabled + try { + // Need to rule out if the path is an implicit directory + checkIfDirPathInFNS(); + + } catch (AzureBlobFileSystemException gpsEx) { + AbfsRestOperationException gpsEre = (AbfsRestOperationException) gpsEx; + if(gpsEre.getErrorMessage().contains(readOnDirectoryErrorMsg)){ + throw gpsEre; + } + // The file does not exist + else throw new FileNotFoundException(gpsEre.getMessage()); } - - // Not a directory → plain FileNotFound - throw new FileNotFoundException(ere.getMessage()); } - // Case: 416 + non-namespace-enabled - //todo: convert to azureserviceerrorcode - if (!isHnsEnabled && status == 416) { - AbfsHttpOperation gpsOp = client.getPathStatus( - getRelativePath(new Path(path)), - false, - tracingContext, - contextEncryptionAdapter).getResult(); - - String resourceType = - gpsOp.getResponseHeaderIgnoreCase(X_MS_META_HDI_ISFOLDER); - - if (TRUE.equals(resourceType)) { - throw directoryReadException(); - } + // Case-2: 416 INVALID RANGE + if (!isHnsEnabled && INVALID_RANGE.equals(ere.getErrorCode())) { + // Need to rule out if the path is an explicit directory + checkIfDirPathInFNS(); } // Default: propagate original error diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStreamContext.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStreamContext.java index ea80c66a46aee..96531e4653360 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStreamContext.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStreamContext.java @@ -256,6 +256,12 @@ public AbfsInputStreamContext withEncryptionAdapter( return this; } +/** + * Sets restriction of GPS on open file. + * + * @param restrictGpsOnOpenFile whether to restrict GPS on open file. + * @return this instance. + */ public AbfsInputStreamContext shouldRestrictGpsOnOpenFile( final boolean restrictGpsOnOpenFile) { this.restrictGpsOnOpenFile = restrictGpsOnOpenFile; @@ -345,6 +351,7 @@ public int getReadAheadBlockSize() { return readAheadBlockSize; } + /** @return whether restrictGpsOnOpenFile is enabled. */ public boolean shouldRestrictGpsOnOpenFile() { return this.restrictGpsOnOpenFile; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPrefetchInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPrefetchInputStream.java index 34172d7c644ad..21be850a0b712 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPrefetchInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPrefetchInputStream.java @@ -81,7 +81,7 @@ protected int readOneBlock(final byte[] b, final int off, final int len) throws } /* - Skips prefetch for the first read if isRestrictGpsOnOpenFile config is enabled. + Skips prefetch for the first read if restrictGpsOnOpenFile config is enabled. This is required since contentLength is not available yet to determine prefetch block size. */ if(shouldRestrictGpsOnOpenFile() && isFirstRead()) { @@ -91,7 +91,7 @@ protected int readOneBlock(final byte[] b, final int off, final int len) throws } else { /* - * Always start with Prefetch even from first read UNLESS isRestrictGpsOnOpenFile config is enabled. + * Always start with Prefetch even from first read UNLESS restrictGpsOnOpenFile config is enabled. * Even if out of order seek comes, prefetches will be triggered for next set of blocks. */ bytesRead = readInternal(getFCursor(), getBuffer(), 0, getBufferSize(), false); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java index d9a0142b636dd..14f590d766819 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java @@ -24,7 +24,12 @@ import java.net.URISyntaxException; import java.net.URL; import java.nio.charset.StandardCharsets; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.List; +import java.util.Optional; +import java.util.Random; import java.util.concurrent.ExecutionException; import org.apache.hadoop.fs.azurebfs.AbfsConfiguration; @@ -78,13 +83,29 @@ import static org.apache.hadoop.fs.azurebfs.constants.ReadType.PREFETCH_READ; import static org.apache.hadoop.fs.azurebfs.constants.ReadType.RANDOM_READ; import static org.apache.hadoop.fs.azurebfs.constants.ReadType.SMALLFILE_READ; -import static org.mockito.ArgumentMatchers.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import static org.apache.hadoop.test.LambdaTestUtils.intercept; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.FORWARD_SLASH; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_READ_AHEAD_QUEUE_DEPTH; import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.*; /** * Unit test AbfsInputStream. @@ -394,92 +415,103 @@ private void mockClientForEncryptionContext(AbfsClient encryptedClient) throws I .getEncryptionContextProvider(); } -/** - * Tests opening encrypted and non-encrypted files under different clients. - * Verifies that even with restrictGpsOnOpenFile enabled, for files created with ENCRYPTION_CONTEXT, the client invokes - * getPathStatus, while for non-encrypted files, getPathStatus is not called. - * - * @throws Exception if any error occurs during the test - */ -@Test -public void testMixedEncryptedAndNonEncryptedOpenUnderDifferentClients() - throws Exception { + /** + * Returns an instance of {@link AzureBlobFileSystem} with the + * FS_AZURE_RESTRICT_GPS_ON_OPENFILE configuration enabled. + * This setting restricts the use of GetPathStatus on open file operations. + * + * @return an AzureBlobFileSystem with restrictGpsOnOpenFile enabled + * @throws Exception if the file system cannot be created + */ + private AzureBlobFileSystem getFileSystemWithRestrictGpsEnabled() throws Exception { + Configuration conf = getRawConfiguration(); + conf.setBoolean( + ConfigurationKeys.FS_AZURE_RESTRICT_GPS_ON_OPENFILE, + true); - Configuration conf = getRawConfiguration(); - conf.setBoolean( - ConfigurationKeys.FS_AZURE_RESTRICT_GPS_ON_OPENFILE, - true); + return getFileSystem(conf); + } - AzureBlobFileSystem fs = getFileSystem(conf); + /** + * Tests opening encrypted and non-encrypted files under different clients. + * Verifies that even with restrictGpsOnOpenFile enabled, for files created with ENCRYPTION_CONTEXT, the client invokes + * getPathStatus, while for non-encrypted files, getPathStatus is not called. + * + * @throws Exception if any error occurs during the test + */ + @Test + public void testEncryptedAndNonEncryptedOpenUnderDifferentClients() + throws Exception { + AzureBlobFileSystem fs = getFileSystemWithRestrictGpsEnabled(); - Path encryptedFile = new Path("/enc/file1"); - Path plainFile = new Path("/plain/file2"); + Path encryptedFile = new Path("/enc/file1"); + Path plainFile = new Path("/plain/file2"); - fs.mkdirs(encryptedFile.getParent()); - fs.mkdirs(plainFile.getParent()); + fs.mkdirs(encryptedFile.getParent()); + fs.mkdirs(plainFile.getParent()); - writeBufferToNewFile(encryptedFile, new byte[10]); - writeBufferToNewFile(plainFile, new byte[10]); + writeBufferToNewFile(encryptedFile, new byte[10]); + writeBufferToNewFile(plainFile, new byte[10]); - TracingContext tracingContext = getTestTracingContext(fs, false); + TracingContext tracingContext = getTestTracingContext(fs, false); - /* - * ========================= - * Client A — ENCRYPTED FILE - * ========================= - */ - AzureBlobFileSystemStore encryptedStore = getAbfsStore(fs); + /* + * ========================= + * Client A — ENCRYPTED FILE + * ========================= + */ + AzureBlobFileSystemStore encryptedStore = getAbfsStore(fs); - AbfsClient encryptedRealClient = - getAbfsClient(encryptedStore); - AbfsClient encryptedClient = - spy(encryptedRealClient); + AbfsClient encryptedRealClient = + getAbfsClient(encryptedStore); + AbfsClient encryptedClient = + spy(encryptedRealClient); - setAbfsClient(encryptedStore, encryptedClient); - mockClientForEncryptionContext(encryptedClient); + setAbfsClient(encryptedStore, encryptedClient); + mockClientForEncryptionContext(encryptedClient); - encryptedStore.openFileForRead( - encryptedFile, Optional.empty(), null, tracingContext); + encryptedStore.openFileForRead( + encryptedFile, Optional.empty(), null, tracingContext); - // File created with ENCRYPTION_CONTEXT, so GPS should be invoked - verify(encryptedClient, times(1)) - .getPathStatus(anyString(), anyBoolean(), any(), isNull()); + // File created with ENCRYPTION_CONTEXT, so GPS should be invoked + verify(encryptedClient, times(1)) + .getPathStatus(anyString(), anyBoolean(), any(), isNull()); - /* - * ============================= - * Client B — NON-ENCRYPTED FILE - * ============================= - */ - AzureBlobFileSystemStore plainStore = getAbfsStore(fs); + /* + * ============================= + * Client B — NON-ENCRYPTED FILE + * ============================= + */ + AzureBlobFileSystemStore plainStore = getAbfsStore(fs); - AbfsClient plainRealClient = - getAbfsClient(plainStore); - AbfsClient plainClient = - spy(plainRealClient); + AbfsClient plainRealClient = + getAbfsClient(plainStore); + AbfsClient plainClient = + spy(plainRealClient); - setAbfsClient(plainStore, plainClient); + setAbfsClient(plainStore, plainClient); - doReturn(EncryptionType.NONE) - .when(plainClient) - .getEncryptionType(); + doReturn(EncryptionType.NONE) + .when(plainClient) + .getEncryptionType(); - plainStore.openFileForRead( - plainFile, Optional.empty(), null, tracingContext); + plainStore.openFileForRead( + plainFile, Optional.empty(), null, tracingContext); - verify(plainClient, never()) - .getPathStatus(anyString(), anyBoolean(), any(), any()); -} + verify(plainClient, never()) + .getPathStatus(anyString(), anyBoolean(), any(), any()); + } /** * Verifies the prefetch behavior of the input stream by performing two reads and checking * the number of times the client's read method is invoked after each read. * - * @param fs the AzureBlobFileSystem instance - * @param store the AzureBlobFileSystemStore instance - * @param config the AbfsConfiguration instance - * @param file the file path to read from - * @param restrictGps whether to restrict GPS on open file - * @param readsAfterFirst expected number of client.read invocations after the first read + * @param fs the AzureBlobFileSystem instance + * @param store the AzureBlobFileSystemStore instance + * @param config the AbfsConfiguration instance + * @param file the file path to read from + * @param restrictGps whether to restrict GPS on open file + * @param readsAfterFirst expected number of client.read invocations after the first read * @param readsAfterSecond expected number of client.read invocations after the second read * @throws Exception if any error occurs during verification */ @@ -529,9 +561,9 @@ private void verifyPrefetchBehavior( /** * Tests the prefetch behavior of the input stream when restrictGPSOnOpenFile is enabled. - * First read: only direct read is triggered. - * Second read: triggers readahead reads. - * Verifies the expected number of read invocations after each read operation + * First read: only direct read is triggered. + * Second read: triggers readahead reads. + * Verifies the expected number of read invocations after each read operation * * @throws Exception if any error occurs during the test */ @@ -561,8 +593,18 @@ public void testPrefetchBehaviourWithRestrictGPSOnOpenFile() throws Exception { } - private void assertReadType(AzureBlobFileSystem fs, int numOfReadCalls, - int totalReadCalls, ReadType readType, boolean isRandom) throws Exception { + /** + * Asserts that the correct read types are present in the tracing context headers + * when restrictGpsOnOpenFile is enabled. + * + * @param fs the AzureBlobFileSystem instance + * @param numOfReadCalls the number of read calls to check for the specified read type + * @param totalReadCalls the total number of read calls expected + * @param readType the expected ReadType for the calls (e.g., PREFETCH_READ, MISSEDCACHE_READ) + * @throws Exception if verification fails + */ + private void assertReadTypeWithRestrictGpsOnOpenFileEnabled(AzureBlobFileSystem fs, int numOfReadCalls, + int totalReadCalls, ReadType readType) throws Exception { ArgumentCaptor captor1 = ArgumentCaptor.forClass(String.class); ArgumentCaptor captor2 = ArgumentCaptor.forClass(Long.class); ArgumentCaptor captor3 = ArgumentCaptor.forClass(byte[].class); @@ -579,63 +621,86 @@ private void assertReadType(AzureBlobFileSystem fs, int numOfReadCalls, captor7.capture(), captor8.capture(), captor9.capture()); List tracingContextList = captor9.getAllValues(); - if(!isRandom){ - verifyHeaderForReadTypeInTracingContextHeader(tracingContextList.get(0), NORMAL_READ, 0); - } + // Apart from random read policy, all policies will result in first read being normal read. + verifyHeaderForReadTypeInTracingContextHeader(tracingContextList.get(0), NORMAL_READ, 0); if (readType == PREFETCH_READ) { /* - * For Prefetch Enabled, first read will be Normal or Missed Cache Read. - * So we will assert only for last 2 calls which should be Prefetched Read. + * The first read was a normal read. Prefetching would have started from second read. + * Second read also could be a Normal or Missed Cache read, so we will assert for reads apart from these two. * Since calls are asynchronous, we can not guarantee the order of calls. * Therefore, we cannot assert on exact position here. */ for (int i = tracingContextList.size() - (numOfReadCalls - 2); i < tracingContextList.size(); i++) { verifyHeaderForReadTypeInTracingContextHeader(tracingContextList.get(i), readType, -1); } - } else if (readType == DIRECT_READ) { - int expectedReadPos = ONE_MB/3; - for (int i = tracingContextList.size() - numOfReadCalls; i < tracingContextList.size(); i++) { - verifyHeaderForReadTypeInTracingContextHeader(tracingContextList.get(i), readType, expectedReadPos); - expectedReadPos += ONE_MB; - } } else if (readType == MISSEDCACHE_READ) { int expectedReadPos = ONE_MB; - for (int i = tracingContextList.size() - numOfReadCalls+1; i < tracingContextList.size(); i++) { + for (int i = tracingContextList.size() - numOfReadCalls + 1; i < tracingContextList.size(); i++) { verifyHeaderForReadTypeInTracingContextHeader(tracingContextList.get(i), readType, expectedReadPos); expectedReadPos += ONE_MB; } - } - else { - //normal raed + } else { int expectedReadPos = ONE_MB; - for (int i = tracingContextList.size() - numOfReadCalls+1; i < tracingContextList.size(); i++) { + for (int i = tracingContextList.size() - numOfReadCalls + 1; i < tracingContextList.size(); i++) { verifyHeaderForReadTypeInTracingContextHeader(tracingContextList.get(i), readType, expectedReadPos); expectedReadPos += ONE_MB; } } } - private void testReadTypeWithRestrictGPSOnOpenFile(AzureBlobFileSystem fs, - int fileSize, ReadType readType, Boolean isRandom, int numOfReadCalls, int totalReadCalls) throws Exception { + /** + * Helper method to test the read type behavior with FS_AZURE_RESTRICT_GPS_ON_OPENFILE enabled. + *

+ * Creates a test file of the specified size, performs a read operation, and asserts that the number + * of bytes read matches the file size. Then verifies that the expected read types are present in the + * tracing context headers for the specified number of read calls. + * + * @param fs the AzureBlobFileSystem instance + * @param fileSize the size of the test file to create and read + * @param readType the expected ReadType for the calls (e.g., PREFETCH_READ, MISSEDCACHE_READ) + * @param numOfReadCalls the number of read calls to check for the specified read type + * @param totalReadCalls the total number of read calls expected + * @throws Exception if verification fails + */ + private void testReadTypeWithRestrictGpsOnOpenFile(AzureBlobFileSystem fs, + int fileSize, ReadType readType, int numOfReadCalls, int totalReadCalls) throws Exception { Path testPath = createTestFile(fs, fileSize); try (FSDataInputStream iStream = fs.open(testPath)) { int bytesRead = iStream.read(new byte[fileSize], 0, fileSize); + Thread.sleep(1000); //Sleep for a sec to get the readAhead threads to complete assertThat(fileSize) .describedAs("Read size should match file size") .isEqualTo(bytesRead); } - assertReadType(fs, numOfReadCalls, totalReadCalls, readType, isRandom); + assertReadTypeWithRestrictGpsOnOpenFileEnabled(fs, numOfReadCalls, totalReadCalls, readType); } + /** + * Test to verify the read type behavior when FS_AZURE_RESTRICT_GPS_ON_OPENFILE is enabled. + *

+ * This test covers multiple scenarios to ensure that the correct read type is set in the tracing context + * and that the expected number of client read calls are made for different configurations: + *

    + *
  • Prefetch Read Type with adaptive policy (read ahead enabled, depth 2)
    • + *
  • Missed Cache Read Type with adaptive policy (read ahead enabled, depth 0)
    • + *
  • Footer Read Type (file size less than footer read size, small file optimization disabled)
    • + *
  • Small File Read Type (file size less than block size, small file optimization enabled)
    • + *
  • Missed Cache Read Type with sequential policy (read ahead enabled, depth 0, sequential policy)
    • + *
  • Prefetch Read Type with sequential policy (read ahead enabled, depth 3, sequential policy)
    • + *
+ * + * @throws Exception if any error occurs during the test + */ @Test - public void testReadTypeIn() throws Exception { + public void testReadTypeWithRestrictGpsInOpenFileEnabled() throws Exception { AzureBlobFileSystem spiedFs = Mockito.spy(getFileSystem()); AzureBlobFileSystemStore spiedStore = Mockito.spy(spiedFs.getAbfsStore()); AbfsConfiguration spiedConfig = Mockito.spy(spiedStore.getAbfsConfiguration()); AbfsClient spiedClient = Mockito.spy(spiedStore.getClient()); Mockito.doReturn(ONE_MB).when(spiedConfig).getReadBufferSize(); + Mockito.doReturn(ONE_MB).when(spiedConfig).getReadAheadBlockSize(); Mockito.doReturn(true).when(spiedConfig).shouldRestrictGpsOnOpenFile(); Mockito.doReturn(false).when(spiedConfig).isReadAheadV2Enabled(); Mockito.doReturn(spiedClient).when(spiedStore).getClient(); @@ -644,33 +709,30 @@ public void testReadTypeIn() throws Exception { int totalReadCalls = 0; int fileSize; - //adaptive- prefetch- 1st NR, then PR/MR - //adaptive-no prefetch (prefetch DISABLED)- all NR (NO TEST REQD) - //adaptive- prefetch true, readahead depth=0, - first NR, then all MR - - //sequential- prefetch- 1st NR, then PR/MR - //sequential-no prefetch- 1st NR, then MR - - //random remains random - /* - * Test to verify Prefetch Read Type. + * Test to verify Prefetch Read Type with adaptive policy. * Setting read ahead depth to 2 with prefetch enabled ensures that prefetch is done. * - * ADaptive-prefetch- gives NR for first read + * Should give normal read for first read. */ - fileSize = 3 * ONE_MB; // To make sure multiple blocks are read. - totalReadCalls += 3; + fileSize = 4 * ONE_MB; // To make sure multiple blocks are read. + totalReadCalls += 4; doReturn(true).when(spiedConfig).isReadAheadEnabled(); - Mockito.doReturn(3).when(spiedConfig).getReadAheadQueueDepth(); - testReadTypeWithRestrictGPSOnOpenFile(spiedFs, fileSize, PREFETCH_READ, false, 3, totalReadCalls); + Mockito.doReturn(2).when(spiedConfig).getReadAheadQueueDepth(); + testReadTypeWithRestrictGpsOnOpenFile(spiedFs, fileSize, PREFETCH_READ, 4, totalReadCalls); - //Adaptive- no prefetch queue- 1st NR, then MR + /* + * Test to verify Missed Cache Read Type with adaptive policy. + * Setting read ahead depth to 0 ensure that nothing can be got from prefetch. + * In such a case Input Stream will do a sequential read with missed cache read type. + * + * Should give normal read for first read. + */ fileSize = 3 * ONE_MB; // To make sure multiple blocks are read. totalReadCalls += 3; doReturn(true).when(spiedConfig).isReadAheadEnabled(); Mockito.doReturn(0).when(spiedConfig).getReadAheadQueueDepth(); - testReadTypeWithRestrictGPSOnOpenFile(spiedFs, fileSize, MISSEDCACHE_READ, false, 3, totalReadCalls); + testReadTypeWithRestrictGpsOnOpenFile(spiedFs, fileSize, MISSEDCACHE_READ, 3, totalReadCalls); /* * Test to verify Footer Read Type. @@ -680,7 +742,7 @@ public void testReadTypeIn() throws Exception { totalReadCalls += 1; // Full file will be read along with footer. doReturn(false).when(spiedConfig).readSmallFilesCompletely(); doReturn(true).when(spiedConfig).optimizeFooterRead(); - testReadTypeWithRestrictGPSOnOpenFile(spiedFs, fileSize, NORMAL_READ, false, 1, totalReadCalls); + testReadTypeWithRestrictGpsOnOpenFile(spiedFs, fileSize, NORMAL_READ, 1, totalReadCalls); /* * Test to verify Small File Read Type. @@ -689,31 +751,32 @@ public void testReadTypeIn() throws Exception { totalReadCalls += 1; // Full file will be read along with footer. doReturn(true).when(spiedConfig).readSmallFilesCompletely(); doReturn(false).when(spiedConfig).optimizeFooterRead(); - testReadTypeWithRestrictGPSOnOpenFile(spiedFs, fileSize, NORMAL_READ, false, 1, totalReadCalls); + testReadTypeWithRestrictGpsOnOpenFile(spiedFs, fileSize, NORMAL_READ, 1, totalReadCalls); - //SEquential-no prefetch (depth0)- should give NR for firstread and MR for remaining 2 + /* + * Test to verify Missed Cache Read Type with sequential policy. + * Setting read ahead depth to 0 ensure that nothing can be got from prefetch. + * In such a case Input Stream will do a sequential read with missed cache read type. + * + * Should give normal read for first read. + */ fileSize = 3 * ONE_MB; // To make sure multiple blocks are read. totalReadCalls += 3; Mockito.doReturn(0).when(spiedConfig).getReadAheadQueueDepth(); Mockito.doReturn(FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL).when(spiedConfig).getAbfsReadPolicy(); doReturn(true).when(spiedConfig).isReadAheadEnabled(); - testReadTypeWithRestrictGPSOnOpenFile(spiedFs, fileSize, MISSEDCACHE_READ, false, 3, totalReadCalls); -// -// //SEquential-yes prefetch- should give NR for firstread and PR/MR fr second for PR for last - fileSize = 3 * ONE_MB; // To make sure multiple blocks are read. - totalReadCalls += 3; - Mockito.doReturn(3).when(spiedConfig).getReadAheadQueueDepth(); - doReturn(true).when(spiedConfig).isReadAheadEnabled(); - testReadTypeWithRestrictGPSOnOpenFile(spiedFs, fileSize, PREFETCH_READ, false, 3, totalReadCalls); + testReadTypeWithRestrictGpsOnOpenFile(spiedFs, fileSize, MISSEDCACHE_READ, 3, totalReadCalls); /* - * Test to verify Random Read Type. - * Setting Read Policy to Parquet ensures Random Read Type. + * Test to verify Prefetch Read Type with sequential policy. + * Setting read ahead depth to 3 with prefetch enabled ensures that prefetch is done. + * + * Should give normal read for first read. */ - fileSize = 3 * ONE_MB; // To make sure multiple blocks are read. - totalReadCalls += 3; // Full file will be read along with footer. - doReturn(FS_OPTION_OPENFILE_READ_POLICY_PARQUET).when(spiedConfig).getAbfsReadPolicy(); - testReadTypeWithRestrictGPSOnOpenFile(spiedFs, fileSize, RANDOM_READ, true, 3, totalReadCalls); + totalReadCalls += 3; + Mockito.doReturn(3).when(spiedConfig).getReadAheadQueueDepth(); + doReturn(true).when(spiedConfig).isReadAheadEnabled(); + testReadTypeWithRestrictGpsOnOpenFile(spiedFs, fileSize, PREFETCH_READ, 3, totalReadCalls); } /** @@ -724,105 +787,98 @@ public void testReadTypeIn() throws Exception { * @throws Exception if any error occurs during the test */ @Test -public void testFNSExceptionOnDirReadWithRestrictGPSConfig() throws Exception { - assumeHnsDisabled(); - Configuration conf = getRawConfiguration(); - conf.setBoolean(ConfigurationKeys.FS_AZURE_RESTRICT_GPS_ON_OPENFILE, true); - - AzureBlobFileSystem fs = getFileSystem(conf); - AzureBlobFileSystemStore store = getAbfsStore(fs); - - AbfsClient realClient = getAbfsClient(store); - AbfsClient spyClient = spy(realClient); - setAbfsClient(store, spyClient); - - String explicitTestFolder = "/testExplFolderRestrictGps"; - fs.mkdirs(new Path(explicitTestFolder)); - - try (FSDataInputStream in = fs.open(new Path(explicitTestFolder))) { - AbfsInputStream abfsIn = (AbfsInputStream) in.getWrappedStream(); - byte[] buf = new byte[100]; - AbfsRestOperationException ex = Assertions.assertThrows(AbfsRestOperationException.class, () -> abfsIn.read(buf)); - assertThat(ex.getStatusCode()).isEqualTo(AzureServiceErrorCode.PATH_NOT_FOUND.getStatusCode()); - assertThat(ex.getMessage()).contains("Read operation not permitted on a directory."); - } - verify(spyClient, times(1)) - .getPathStatus( - anyString(), - anyBoolean(), - any(TracingContext.class), - any()); - - String implicitTestFolder = "/testImplFolderRestrictGps"; - createAzCopyFolder(new Path(implicitTestFolder)); - try (FSDataInputStream in2 = fs.open(new Path(implicitTestFolder))) { - AbfsInputStream abfsIn2 = (AbfsInputStream) in2.getWrappedStream(); - byte[] buf2 = new byte[100]; - AbfsRestOperationException ex2 = Assertions.assertThrows(AbfsRestOperationException.class, () -> abfsIn2.read(buf2)); - assertThat(ex2.getStatusCode()).isEqualTo(AzureServiceErrorCode.PATH_NOT_FOUND.getStatusCode()); - assertThat(ex2.getMessage()).contains("Read operation not permitted on a directory."); + public void testFNSExceptionOnDirReadWithRestrictGpsConfig() throws Exception { + assumeHnsDisabled(); + AzureBlobFileSystem fs = getFileSystemWithRestrictGpsEnabled(); + AzureBlobFileSystemStore store = getAbfsStore(fs); + + AbfsClient realClient = getAbfsClient(store); + AbfsClient spyClient = spy(realClient); + setAbfsClient(store, spyClient); + + String explicitTestFolder = "/testExplFolderRestrictGps"; + fs.mkdirs(new Path(explicitTestFolder)); + + try (FSDataInputStream in = fs.open(new Path(explicitTestFolder))) { + AbfsInputStream abfsIn = (AbfsInputStream) in.getWrappedStream(); + byte[] buf = new byte[100]; + AbfsRestOperationException ex = Assertions.assertThrows(AbfsRestOperationException.class, () -> abfsIn.read(buf)); + assertThat(ex.getStatusCode()).isEqualTo(AzureServiceErrorCode.PATH_NOT_FOUND.getStatusCode()); + assertThat(ex.getMessage()).contains("Read operation not permitted on a directory."); + } + verify(spyClient, times(1)) + .getPathStatus( + anyString(), + anyBoolean(), + any(TracingContext.class), + any()); + + String implicitTestFolder = "/testImplFolderRestrictGps"; + createAzCopyFolder(new Path(implicitTestFolder)); + try (FSDataInputStream in2 = fs.open(new Path(implicitTestFolder))) { + AbfsInputStream abfsIn2 = (AbfsInputStream) in2.getWrappedStream(); + byte[] buf2 = new byte[100]; + AbfsRestOperationException ex2 = Assertions.assertThrows(AbfsRestOperationException.class, () -> abfsIn2.read(buf2)); + assertThat(ex2.getStatusCode()).isEqualTo(AzureServiceErrorCode.PATH_NOT_FOUND.getStatusCode()); + assertThat(ex2.getMessage()).contains("Read operation not permitted on a directory."); + } + verify(spyClient, times(2)) + .getPathStatus( + anyString(), + anyBoolean(), + any(TracingContext.class), + any()); } - verify(spyClient, times(2)) - .getPathStatus( - anyString(), - anyBoolean(), - any(TracingContext.class), - any()); -} -/** - * Tests that for HNS accounts reading from a directory when - * FS_AZURE_RESTRICT_GPS_ON_OPENFILE is set to true, throws an AbfsRestOperationException - * with PATH_NOT_FOUND status code and the correct error message. - * Also verifies that getPathStatus is not called. - * - * @throws Exception if any error occurs during the test - */ -@Test -public void testHNSExceptionOnDirReadWithRestrictGPSConfig() throws Exception { - assumeHnsEnabled(); - Configuration conf = getRawConfiguration(); - conf.setBoolean(ConfigurationKeys.FS_AZURE_RESTRICT_GPS_ON_OPENFILE, true); + /** + * Tests that for HNS accounts reading from a directory when + * FS_AZURE_RESTRICT_GPS_ON_OPENFILE is set to true, throws an AbfsRestOperationException + * with PATH_NOT_FOUND status code and the correct error message. + * Also verifies that getPathStatus is not called. + * + * @throws Exception if any error occurs during the test + */ + @Test + public void testHNSExceptionOnDirReadWithRestrictGpsConfig() throws Exception { + assumeHnsEnabled(); + AzureBlobFileSystem fs = getFileSystemWithRestrictGpsEnabled(); - AzureBlobFileSystem fs = getFileSystem(conf); - AzureBlobFileSystemStore store = getAbfsStore(fs); + AzureBlobFileSystemStore store = getAbfsStore(fs); - AbfsClient realClient = getAbfsClient(store); - AbfsClient spyClient = spy(realClient); - setAbfsClient(store, spyClient); + AbfsClient realClient = getAbfsClient(store); + AbfsClient spyClient = spy(realClient); + setAbfsClient(store, spyClient); - String testFolder = "/testFolderRestrictGps"; - fs.mkdirs(new Path(testFolder)); + String testFolder = "/testFolderRestrictGps"; + fs.mkdirs(new Path(testFolder)); - try (FSDataInputStream in = fs.open(new Path(testFolder))) { - AbfsInputStream abfsIn = (AbfsInputStream) in.getWrappedStream(); - byte[] buf = new byte[100]; - AbfsRestOperationException ex = Assertions.assertThrows(AbfsRestOperationException.class, () -> abfsIn.read(buf)); - assertThat(ex.getStatusCode()).isEqualTo(AzureServiceErrorCode.PATH_NOT_FOUND.getStatusCode()); - assertThat(ex.getMessage()).contains("Read operation not permitted on a directory."); + try (FSDataInputStream in = fs.open(new Path(testFolder))) { + AbfsInputStream abfsIn = (AbfsInputStream) in.getWrappedStream(); + byte[] buf = new byte[100]; + AbfsRestOperationException ex = Assertions.assertThrows(AbfsRestOperationException.class, () -> abfsIn.read(buf)); + assertThat(ex.getStatusCode()).isEqualTo(AzureServiceErrorCode.PATH_NOT_FOUND.getStatusCode()); + assertThat(ex.getMessage()).contains("Read operation not permitted on a directory."); + } + verify(spyClient, times(0)) + .getPathStatus( + anyString(), + anyBoolean(), + any(TracingContext.class), + any()); } - verify(spyClient, times(0)) - .getPathStatus( - anyString(), - anyBoolean(), - any(TracingContext.class), - any()); -} -/** - * Tests the behavior of openFileForRead with the FS_AZURE_RESTRICT_GPS_ON_OPENFILE configuration enabled. - * Verifies that irrespective of whether FileStatus is provided or not, getPathStatus is not invoked for read flow. - * - * @throws Exception if any error occurs during the test - */ -@Test -public void testOpenFileWithOptionsWithRestrictGpsOnOpenFile() throws Exception { - Configuration conf = getRawConfiguration(); - conf.setBoolean(ConfigurationKeys.FS_AZURE_RESTRICT_GPS_ON_OPENFILE, true); + /** + * Tests the behavior of openFileForRead with the FS_AZURE_RESTRICT_GPS_ON_OPENFILE configuration enabled. + * Verifies that irrespective of whether FileStatus is provided or not, getPathStatus is not invoked for read flow. + * + * @throws Exception if any error occurs during the test + */ + @Test + public void testOpenFileWithOptionsWithRestrictGpsOnOpenFile() throws Exception { + AzureBlobFileSystem fs = getFileSystemWithRestrictGpsEnabled(); - AzureBlobFileSystem fs = getFileSystem(conf); Path fileWithFileStatus = new Path("/testFile0"); Path fileWithoutFileStatus = new Path("/testFile1"); @@ -840,126 +896,118 @@ public void testOpenFileWithOptionsWithRestrictGpsOnOpenFile() throws Exception // Case 1: FileStatus is provided abfsStore.openFileForRead(fileWithFileStatus, Optional.ofNullable(new OpenFileParameters().withStatus(fs.getFileStatus(fileWithFileStatus))), null, tracingContext); verify(mockClient, times(1).description("FileStatus provided, restrict GPS: getPathStatus should NOT be invoked")) - .getPathStatus(any(String.class), any(Boolean.class), any(TracingContext.class), nullable(ContextEncryptionAdapter.class)); + .getPathStatus(any(String.class), any(Boolean.class), any(TracingContext.class), nullable(ContextEncryptionAdapter.class)); // Case 2: FileStatus is not provided abfsStore.openFileForRead(fileWithoutFileStatus, Optional.empty(), null, tracingContext); verify(mockClient, times(1).description("FileStatus not provided, restrict GPS: getPathStatus should NOT be invoked")) - .getPathStatus(any(String.class), any(Boolean.class), any(TracingContext.class), nullable(ContextEncryptionAdapter.class)); -} + .getPathStatus(any(String.class), any(Boolean.class), any(TracingContext.class), nullable(ContextEncryptionAdapter.class)); + } - /** - * Tests that when FS_AZURE_RESTRICT_GPS_ON_OPENFILE is enabled, - * the eTag and content length metadata are correctly initialized - * after the first read operation, and remain consistent for subsequent reads. - * - * @throws Exception if any error occurs during the test - */ - @Test - public void testMetadataFromReadForRestrictGpsOnOpenFile() throws Exception { - Configuration conf = getRawConfiguration(); - conf.setBoolean(org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_RESTRICT_GPS_ON_OPENFILE, true); - - AzureBlobFileSystem fs = getFileSystem(conf); - Path testFile = new Path("/testFile0"); - writeBufferToNewFile(testFile, new byte[250]); - - // Open the file and perform a read - try (FSDataInputStream in = fs.open(testFile)) { - AbfsInputStream abfsIn = (AbfsInputStream) in.getWrappedStream(); - - // Before first read, eTag and content length aren't initialized - String etagPreRead = abfsIn.getETag(); - long fileLengthPreRead = abfsIn.getContentLength(); - assertThat(etagPreRead).isEmpty(); - assertThat(fileLengthPreRead).isEqualTo(0); - - // Trigger the first read - byte[] buf = new byte[100]; - int n = abfsIn.read(buf); - assertThat(n).isGreaterThan(0); - - // After first read, eTag and content length should be set from the response - String etagFirstRead = abfsIn.getETag(); - long fileLengthFirstRead = abfsIn.getContentLength(); - assertThat(etagFirstRead).isNotNull(); - assertThat(fileLengthFirstRead).isEqualTo(250L); - - //Trigger the second read - n = abfsIn.read(100, buf, 0, 100); - assertThat(n).isGreaterThan(0); - - // eTag and content length should remain same as first read for second read onwards - String etagSecondRead = abfsIn.getETag(); - long fileLengthSecondRead = abfsIn.getContentLength(); - assertThat(etagSecondRead).isEqualTo(etagFirstRead); - assertThat(fileLengthSecondRead).isEqualTo(fileLengthFirstRead); - } + /** + * Tests that when FS_AZURE_RESTRICT_GPS_ON_OPENFILE is enabled, + * the eTag and content length metadata are correctly initialized + * after the first read operation, and remain consistent for subsequent reads. + * + * @throws Exception if any error occurs during the test + */ + @Test + public void testMetadataFromReadForRestrictGpsOnOpenFile() throws Exception { + AzureBlobFileSystem fs = getFileSystemWithRestrictGpsEnabled(); + + Path testFile = new Path("/testFile0"); + writeBufferToNewFile(testFile, new byte[250]); + + // Open the file and perform a read + try (FSDataInputStream in = fs.open(testFile)) { + AbfsInputStream abfsIn = (AbfsInputStream) in.getWrappedStream(); + + // Before first read, eTag and content length aren't initialized + String etagPreRead = abfsIn.getETag(); + long fileLengthPreRead = abfsIn.getContentLength(); + assertThat(etagPreRead).isEmpty(); + assertThat(fileLengthPreRead).isEqualTo(0); + + // Trigger the first read + byte[] buf = new byte[100]; + int n = abfsIn.read(buf); + assertThat(n).isGreaterThan(0); + + // After first read, eTag and content length should be set from the response + String etagFirstRead = abfsIn.getETag(); + long fileLengthFirstRead = abfsIn.getContentLength(); + assertThat(etagFirstRead).isNotNull(); + assertThat(fileLengthFirstRead).isEqualTo(250L); + + //Trigger the second read + n = abfsIn.read(100, buf, 0, 100); + assertThat(n).isGreaterThan(0); + + // eTag and content length should remain same as first read for second read onwards + String etagSecondRead = abfsIn.getETag(); + long fileLengthSecondRead = abfsIn.getContentLength(); + assertThat(etagSecondRead).isEqualTo(etagFirstRead); + assertThat(fileLengthSecondRead).isEqualTo(fileLengthFirstRead); } + } -/** - * Tests that when FS_AZURE_RESTRICT_GPS_ON_OPENFILE is enabled, - * metadata (eTag and content length) is not partially initialized if read requests fail with a timeout. - * Ensures that after failed reads, metadata remains unset, and only after a successful read - * are the metadata fields correctly initialized. - * - * @throws Exception if any error occurs during the test - */ -@Test -public void testMetadataNotPartiallyInitializedOnReadWithRestrictGpsOnOpenFile() - throws Exception { - - Configuration conf = getRawConfiguration(); - conf.setBoolean( - ConfigurationKeys.FS_AZURE_RESTRICT_GPS_ON_OPENFILE, true); - - AzureBlobFileSystem fs = spy(getFileSystem(conf)); + /** + * Tests that when FS_AZURE_RESTRICT_GPS_ON_OPENFILE is enabled, + * metadata (eTag and content length) is not partially initialized if read requests fail with a timeout. + * Ensures that after failed reads, metadata remains unset, and only after a successful read + * are the metadata fields correctly initialized. + * + * @throws Exception if any error occurs during the test + */ + @Test + public void testMetadataNotPartiallyInitializedOnReadWithRestrictGpsOnOpenFile() + throws Exception { + AzureBlobFileSystem fs = spy(getFileSystemWithRestrictGpsEnabled()); - AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore()); - Mockito.doReturn(store).when(fs).getAbfsStore(); - AbfsClient client = Mockito.spy(store.getClient()); - Mockito.doReturn(client).when(store).getClient(); + AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore()); + Mockito.doReturn(store).when(fs).getAbfsStore(); + AbfsClient client = Mockito.spy(store.getClient()); + Mockito.doReturn(client).when(store).getClient(); - AbfsRestOperation successOp = getMockRestOpWithMetadata(); + AbfsRestOperation successOp = getMockRestOpWithMetadata(); - doThrow(new TimeoutException("First-read-failure")) - .doThrow(new TimeoutException("Second-read-failure")) - .doReturn(successOp) - .when(client) - .read(any(String.class), any(Long.class), any(byte[].class), - any(Integer.class), any(Integer.class), any(String.class), - nullable(String.class), any(ContextEncryptionAdapter.class), any(TracingContext.class)); + doThrow(new TimeoutException("First-read-failure")) + .doThrow(new TimeoutException("Second-read-failure")) + .doReturn(successOp) + .when(client) + .read(any(String.class), any(Long.class), any(byte[].class), + any(Integer.class), any(Integer.class), any(String.class), + nullable(String.class), any(ContextEncryptionAdapter.class), any(TracingContext.class)); - Path testFile = new Path("/testFile0"); - byte[] fileContent = new byte[ONE_KB]; - writeBufferToNewFile(testFile, fileContent); + Path testFile = new Path("/testFile0"); + byte[] fileContent = new byte[ONE_KB]; + writeBufferToNewFile(testFile, fileContent); - try (FSDataInputStream in = fs.open(testFile)) { - AbfsInputStream abfsIn = (AbfsInputStream) in.getWrappedStream(); + try (FSDataInputStream in = fs.open(testFile)) { + AbfsInputStream abfsIn = (AbfsInputStream) in.getWrappedStream(); - // Metadata not initialized before read - assertThat(abfsIn.getETag()).isEmpty(); - assertThat(abfsIn.getContentLength()).isEqualTo(0); + // Metadata not initialized before read + assertThat(abfsIn.getETag()).isEmpty(); + assertThat(abfsIn.getContentLength()).isEqualTo(0); - // First read fails- metadata should not be initialized - intercept(IOException.class, - () -> abfsIn.read(fileContent)); - assertThat(abfsIn.getETag()).isEmpty(); - assertThat(abfsIn.getContentLength()).isEqualTo(0); + // First read fails- metadata should not be initialized + intercept(IOException.class, + () -> abfsIn.read(fileContent)); + assertThat(abfsIn.getETag()).isEmpty(); + assertThat(abfsIn.getContentLength()).isEqualTo(0); - // Second read fails- metadata should not be initialized - intercept(IOException.class, - () -> abfsIn.read(fileContent)); - assertThat(abfsIn.getETag()).isEmpty(); - assertThat(abfsIn.getContentLength()).isEqualTo(0); - - // Third read succeeds- metadata should be initialized - abfsIn.read(fileContent); - assertThat(abfsIn.getETag()).isEqualTo("etag"); - assertThat(abfsIn.getContentLength()).isEqualTo(1024L); + // Second read fails- metadata should not be initialized + intercept(IOException.class, + () -> abfsIn.read(fileContent)); + assertThat(abfsIn.getETag()).isEmpty(); + assertThat(abfsIn.getContentLength()).isEqualTo(0); + + // Third read succeeds- metadata should be initialized + abfsIn.read(fileContent); + assertThat(abfsIn.getETag()).isEqualTo("etag"); + assertThat(abfsIn.getContentLength()).isEqualTo(1024L); + } } -} - /** * This test expects AbfsInputStream to throw the exception that readAhead @@ -1098,7 +1146,6 @@ public void testOlderReadAheadFailure() throws Exception { * @throws Exception */ @Test - //TODO: HEREEEE public void testSuccessfulReadAhead() throws Exception { // Mock failure for client.read() AbfsClient client = getMockAbfsClient(); @@ -1434,7 +1481,6 @@ public void testDefaultReadaheadQueueDepth() throws Exception { * @throws Exception if any error occurs during the test */ @Test - //todo: HEREREEE public void testReadTypeInTracingContextHeader() throws Exception { AzureBlobFileSystem spiedFs = Mockito.spy(getFileSystem()); AzureBlobFileSystemStore spiedStore = Mockito.spy(spiedFs.getAbfsStore()); From b173e07589f49b3853e78120875e481dcfb81c0e Mon Sep 17 00:00:00 2001 From: Manika Joshi Date: Tue, 24 Mar 2026 21:02:47 -0700 Subject: [PATCH 12/14] comments --- .../fs/azurebfs/AzureBlobFileSystem.java | 15 ++++ .../fs/azurebfs/AzureBlobFileSystemStore.java | 74 +++++++++++++------ .../services/AbfsAdaptiveInputStream.java | 5 +- .../fs/azurebfs/services/AbfsBlobClient.java | 2 + .../fs/azurebfs/services/AbfsDfsClient.java | 2 +- .../fs/azurebfs/services/AbfsInputStream.java | 48 ++++++++---- .../services/AbfsPrefetchInputStream.java | 2 +- .../hadoop-azure/src/site/markdown/index.md | 27 +++++++ .../services/TestAbfsInputStream.java | 40 ++++++---- 9 files changed, 158 insertions(+), 57 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index 9ed20251043d2..a24f3303270f1 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -402,6 +402,21 @@ public FSDataInputStream open(final Path path, final int bufferSize) throws IOEx return open(path, Optional.empty()); } + /** + * Open a file for reading and return an {@link FSDataInputStream} that wraps + * the underlying {@link InputStream}. + * + * Note: when the filesystem is configured with `restrictGpsOnOpenFile` enabled + * (its disabled by default), existence check for the file path will be deferred + * and will not occur during this open call; it will happen when the first read + * is attempted on the returned stream. + * + * @param path the location of the file to open + * @param parameters optional {@link OpenFileParameters} which can include + * FileStatus, configuration, buffer size and mandatory keys + * @return an {@link FSDataInputStream} wrapping the opened InputStream + * @throws IOException if an I/O error occurs while opening the file + */ private FSDataInputStream open(final Path path, final Optional parameters) throws IOException { statIncrement(CALL_OPEN); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index b9e759312d8f3..526eab6993dcd 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -893,6 +893,39 @@ private AbfsRestOperationException openFileForReadDirectoryException() { null); } + /** + * Opens a file for read and returns an {@link AbfsInputStream}. + * + *

+ * The method decides whether to call the server's GetPathStatus based on: + *

    + *
  • the supplied {@code parameters} (if it contains a {@link VersionedFileStatus} + * with a valid encryption context when required),
    • + *
  • the client's encryption type ({@link EncryptionType#ENCRYPTION_CONTEXT}), and
    • + *
  • the configuration flag returned by {@link AbfsConfiguration#shouldRestrictGpsOnOpenFile()}.
    • + *
+ * If the encryption type is {@code ENCRYPTION_CONTEXT} the server-supplied + * X-MS-ENCRYPTION-CONTEXT header will be required and used to construct a + * {@link ContextProviderEncryptionAdapter}. If that header is missing a + * {@link PathIOException} is thrown. + *

+ * + *

+ * Note: when {@link AbfsConfiguration#shouldRestrictGpsOnOpenFile()} is enabled, + * the implementation won't do the GetPathStatus call. In that case, if the file does not + * actually exist or read is attempted on a directory, {@code openFileForRead} will not fail immediately. + * It will only be detected when the returned stream performs its first read, at which point an appropriate error will be raised. + *

+ * + * @param path the path to open (may be unqualified) + * @param parameters optional {@link OpenFileParameters} that may include a {@link FileStatus} + * (possibly a {@link VersionedFileStatus}) and other open parameters + * @param statistics filesystem statistics to associate with the returned stream + * @param tracingContext tracing context for remote calls + * @return an {@link AbfsInputStream} for reading the file + * @throws IOException on IO or server errors. A {@link PathIOException} is thrown when + * an expected encryption context header is missing. + */ public AbfsInputStream openFileForRead(Path path, final Optional parameters, final FileSystem.Statistics statistics, TracingContext tracingContext) @@ -905,7 +938,7 @@ public AbfsInputStream openFileForRead(Path path, FileStatus fileStatus = parameters.map(OpenFileParameters::getStatus) .orElse(null); String relativePath = getRelativePath(path); - String resourceType, eTag = EMPTY_STRING; + String resourceType = EMPTY_STRING, eTag = EMPTY_STRING; long contentLength = 0; ContextEncryptionAdapter contextEncryptionAdapter = NoContextEncryptionAdapter.getInstance(); /* @@ -917,26 +950,23 @@ public AbfsInputStream openFileForRead(Path path, * ENCRYPTION_CONTEXT. */ if ((fileStatus instanceof VersionedFileStatus) && ( - getClient().getEncryptionType() != EncryptionType.ENCRYPTION_CONTEXT - || ((VersionedFileStatus) fileStatus).getEncryptionContext() - != null)) { + getClient().getEncryptionType() != EncryptionType.ENCRYPTION_CONTEXT + || ((VersionedFileStatus) fileStatus).getEncryptionContext() + != null)) { path = path.makeQualified(this.uri, path); Preconditions.checkArgument(fileStatus.getPath().equals(path), - String.format( - "Filestatus path [%s] does not match with given path [%s]", - fileStatus.getPath(), path)); + String.format( + "Filestatus path [%s] does not match with given path [%s]", + fileStatus.getPath(), path)); resourceType = fileStatus.isFile() ? FILE : DIRECTORY; contentLength = fileStatus.getLen(); eTag = ((VersionedFileStatus) fileStatus).getVersion(); final String encryptionContext - = ((VersionedFileStatus) fileStatus).getEncryptionContext(); + = ((VersionedFileStatus) fileStatus).getEncryptionContext(); if (getClient().getEncryptionType() == EncryptionType.ENCRYPTION_CONTEXT) { contextEncryptionAdapter = new ContextProviderEncryptionAdapter( - getClient().getEncryptionContextProvider(), getRelativePath(path), - encryptionContext.getBytes(StandardCharsets.UTF_8)); - } - if (parseIsDirectory(resourceType)) { - throw openFileForReadDirectoryException(); + getClient().getEncryptionContextProvider(), getRelativePath(path), + encryptionContext.getBytes(StandardCharsets.UTF_8)); } } /* @@ -954,32 +984,32 @@ else if (getClient().getEncryptionType() == EncryptionType.ENCRYPTION_CONTEXT */ if (getClient().getEncryptionType() == EncryptionType.ENCRYPTION_CONTEXT) { final String fileEncryptionContext = op.getResponseHeader( - X_MS_ENCRYPTION_CONTEXT); + HttpHeaderConfigurations.X_MS_ENCRYPTION_CONTEXT); if (fileEncryptionContext == null) { LOG.debug("EncryptionContext missing in GetPathStatus response"); throw new PathIOException(path.toString(), - "EncryptionContext not present in GetPathStatus response headers"); + "EncryptionContext not present in GetPathStatus response headers"); } contextEncryptionAdapter = new ContextProviderEncryptionAdapter( - getClient().getEncryptionContextProvider(), getRelativePath(path), - fileEncryptionContext.getBytes(StandardCharsets.UTF_8)); + getClient().getEncryptionContextProvider(), getRelativePath(path), + fileEncryptionContext.getBytes(StandardCharsets.UTF_8)); } resourceType = getClient().checkIsDir(op) ? DIRECTORY : FILE; contentLength = extractContentLength(op); eTag = op.getResponseHeader(HttpHeaderConfigurations.ETAG); - - if (parseIsDirectory(resourceType)) { - throw openFileForReadDirectoryException(); - } } /* The only remaining case is: - * - restrictGpsOnOpenFile config is enabled with null FileStatus and encryptionType not as ENCRYPTION_CONTEXT + * - restrictGpsOnOpenFile config is enabled with null/wrong FileStatus and encryptionType not as ENCRYPTION_CONTEXT * In this case, we don't need to call GetPathStatus API. */ else { // do nothing } + if (parseIsDirectory(resourceType)) { + throw openFileForReadDirectoryException(); + } + perfInfo.registerSuccess(true); return getRelevantInputStream(statistics, relativePath, contentLength, diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java index f50bfdd2e014d..466d593ec3c5b 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java @@ -83,7 +83,10 @@ protected int readOneBlock(final byte[] b, final int off, final int len) throws // Reset Read Type back to normal and set again based on code flow. getTracingContext().setReadType(ReadType.NORMAL_READ); - if(shouldRestrictGpsOnOpenFile() && isFirstRead()) { + + // If restrictGpsOnOpenFile config is enabled, skip prefetch for the first read since contentLength + // is not available yet to determine prefetch block size. + if (shouldRestrictGpsOnOpenFile() && isFirstRead()) { LOG.debug("RestrictGpsOnOpenFile is enabled. Skip readahead for first read."); bytesRead = readInternal(getFCursor(), getBuffer(), 0, getBufferSize(), true); } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsBlobClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsBlobClient.java index 52fbd3182fdd5..f93d1f3a4c757 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsBlobClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsBlobClient.java @@ -502,6 +502,7 @@ public AbfsRestOperation createPath(final String path, AbfsRestOperation statusOp = null; try { // Check if the file already exists by calling GetPathStatus + //TODO: GBP statusOp = getPathStatus(path, tracingContext, null, false); } catch (AbfsRestOperationException ex) { // If the path does not exist, continue with file creation @@ -2259,6 +2260,7 @@ private AbfsRestOperation createFile(final String path, // If the overwrite flag is true, we must verify whether an empty directory exists at the specified path. // However, if overwrite is false, we can skip this validation and proceed with blob creation, // which will fail with a conflict error if a file or directory already exists at the path. + //todo: GBP if (overwrite && isEmptyDirectory(path, tracingContext, false)) { throw new AbfsRestOperationException(HTTP_CONFLICT, AzureServiceErrorCode.PATH_CONFLICT.getErrorCode(), diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsDfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsDfsClient.java index 5ddb9770ac56e..9e7dbdd89e823 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsDfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsDfsClient.java @@ -1347,7 +1347,7 @@ public AbfsRestOperation checkAccess(String path, public boolean checkIsDir(AbfsHttpOperation result) { String resourceType = result.getResponseHeader( HttpHeaderConfigurations.X_MS_RESOURCE_TYPE); - return StringUtils.equalsIgnoreCase(resourceType, DIRECTORY); + return resourceType != null && StringUtils.equalsIgnoreCase(resourceType, DIRECTORY); } /** diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java index 7d1a7f8540e63..58dafb587c326 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java @@ -56,6 +56,7 @@ import static java.lang.Math.max; import static java.lang.Math.min; +import static org.apache.hadoop.fs.azurebfs.AzureBlobFileSystemStore.extractEtagHeader; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.DIRECTORY; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.ROOT_PATH; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.TRUE; @@ -88,7 +89,7 @@ public abstract class AbfsInputStream extends FSInputStream implements CanUnbuff private final int bufferSize; // default buffer size private final int footerReadSize; // default buffer size to read when reading footer private final int readAheadQueueDepth; // initialized in constructor - private String eTag; // eTag of the path when InputStream are created + private String eTag; // eTag of the path when InputStream is created private final boolean tolerateOobAppends; // whether tolerate Oob Appends private final boolean readAheadEnabled; // whether enable readAhead; private final boolean readAheadV2Enabled; // whether enable readAhead V2; @@ -578,7 +579,16 @@ protected int readInternal(final long position, final byte[] b, final int offset } } - String getRelativePath(final Path path) { + /** + * Convert a {@link Path} to the relative path string used by ABFS. + * + *

This returns the URI path component of the supplied {@code path}. If the + * resulting path is empty, this method returns {@code ROOT_PATH}. + * + * @param path the {@link Path} to convert; must not be null + * @return the relative path as a {@link String}; never null + */ + String getRelativePath(final Path path) { Preconditions.checkNotNull(path, "path"); String relPath = path.toUri().getPath(); if (relPath.isEmpty()) { @@ -614,14 +624,23 @@ private void checkIfDirPathInFNS() throws IOException { tracingContext, contextEncryptionAdapter).getResult(); - String resourceType = - gpsOp.getResponseHeaderIgnoreCase(X_MS_META_HDI_ISFOLDER); - - if (TRUE.equals(resourceType)) { + if (client.checkIsDir(gpsOp)) { throw directoryReadException(); } } + private long extractContentLength(AbfsHttpOperation op) { + // We need to use content range header instead of content length to take care of partial reads + String contentRange = op.getResponseHeader(HttpHeaderConfigurations.CONTENT_RANGE); + if (!StringUtils.isEmpty(contentRange)) { + contentLength = Long.parseLong(contentRange.split(AbfsHttpConstants.FORWARD_SLASH)[1]); + } + else { + contentLength = 0; + } + return contentLength; + } + int readRemote(long position, byte[] b, int offset, int length, TracingContext tracingContext) throws IOException { if (position < 0) { throw new IllegalArgumentException("attempting to read from negative offset"); @@ -655,13 +674,11 @@ int readRemote(long position, byte[] b, int offset, int length, TracingContext t // Update metadata on first read if restrictGpsOnOpenFile is enabled if (shouldRestrictGpsOnOpenFile() && isFirstRead()) { - String resourceType = op.getResult().getResponseHeader(HttpHeaderConfigurations.X_MS_RESOURCE_TYPE); - if (Objects.equals(resourceType, DIRECTORY)) { + if (client.checkIsDir(op.getResult())) { throw directoryReadException(); } - contentLength = Long.parseLong(op.getResult().getResponseHeader(HttpHeaderConfigurations.CONTENT_RANGE). - split(AbfsHttpConstants.FORWARD_SLASH)[1]); - eTag = op.getResult().getResponseHeader("ETag"); + contentLength = extractContentLength(op.getResult()); + eTag = extractEtagHeader(op.getResult()); } cachedSasToken.update(op.getSasToken()); @@ -673,7 +690,7 @@ int readRemote(long position, byte[] b, int offset, int length, TracingContext t if (ex instanceof AbfsRestOperationException) { AbfsRestOperationException ere = (AbfsRestOperationException) ex; int status = ere.getStatusCode(); - if(ere.getErrorMessage().contains(readOnDirectoryErrorMsg)){ + if (ere.getErrorMessage().contains(readOnDirectoryErrorMsg)) { throw ere; } boolean isHnsEnabled = client.getIsNamespaceEnabled(); @@ -695,7 +712,7 @@ int readRemote(long position, byte[] b, int offset, int length, TracingContext t } catch (AzureBlobFileSystemException gpsEx) { AbfsRestOperationException gpsEre = (AbfsRestOperationException) gpsEx; - if(gpsEre.getErrorMessage().contains(readOnDirectoryErrorMsg)){ + if (gpsEre.getErrorMessage().contains(readOnDirectoryErrorMsg)) { throw gpsEre; } // The file does not exist @@ -708,12 +725,11 @@ int readRemote(long position, byte[] b, int offset, int length, TracingContext t // Need to rule out if the path is an explicit directory checkIfDirPathInFNS(); } - - // Default: propagate original error - throw new IOException(ex); } + // Default: propagate original error throw new IOException(ex); } + long bytesRead = op.getResult().getBytesReceived(); if (streamStatistics != null) { streamStatistics.remoteBytesRead(bytesRead); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPrefetchInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPrefetchInputStream.java index 21be850a0b712..0002f3bdf5089 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPrefetchInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPrefetchInputStream.java @@ -84,7 +84,7 @@ protected int readOneBlock(final byte[] b, final int off, final int len) throws Skips prefetch for the first read if restrictGpsOnOpenFile config is enabled. This is required since contentLength is not available yet to determine prefetch block size. */ - if(shouldRestrictGpsOnOpenFile() && isFirstRead()) { + if (shouldRestrictGpsOnOpenFile() && isFirstRead()) { getTracingContext().setReadType(ReadType.NORMAL_READ); LOG.debug("RestrictGpsOnOpenFile is enabled. Skip readahead for first read even for sequential input policy."); bytesRead = readInternal(getFCursor(), getBuffer(), 0, getBufferSize(), true); diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/index.md b/hadoop-tools/hadoop-azure/src/site/markdown/index.md index c482e7864456e..74720882196b4 100644 --- a/hadoop-tools/hadoop-azure/src/site/markdown/index.md +++ b/hadoop-tools/hadoop-azure/src/site/markdown/index.md @@ -1195,6 +1195,33 @@ when there are too many writes from the same process. `fs.azure.analysis.period`: The time after which sleep duration is recomputed after analyzing metrics. The default value for the same is 10 seconds. +### Metadata Options + +The following configurations are related to metadata operations. + +`fs.azure.restrict.gps.on.openfile`: Controls whether the GetPathStatus (GPS) API call +is restricted when opening a file for read. When enabled, this configuration reduces +metadata overhead by skipping the GPS call during file open operations. The file +existence checks are also delayed until the first read operation occurs. + +**Default:** `false` (disabled) + +**Behavior when enabled:** +* The GetPathStatus call is skipped when opening a file, reducing metadata call overhead +* File existence validation is deferred until the first read operation +* Small file full read optimizations are not available +* Footer read optimizations are not available +* The first read operation will not be able to initiate prefetch + +**Exception:** If the file was created with an encryption context, the GetPathStatus call +will still be performed even when this configuration is enabled, as the encryption metadata +is required. + +**Recommended Alternative:** To reduce metadata calls while maintaining optimal read +performance, provide the `FileStatus` object when opening the file using the +`openFile()` builder pattern with the `.withFileStatus()` option. This approach avoids +the GPS call while preserving all read optimizations. + ### Security Options `fs.azure.always.use.https`: Enforces to use HTTPS instead of HTTP when the flag is made true. Irrespective of the flag, `AbfsClient` will use HTTPS if the secure diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java index 14f590d766819..e49c01a78c8fa 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java @@ -868,10 +868,10 @@ public void testHNSExceptionOnDirReadWithRestrictGpsConfig() throws Exception { any()); } - /** * Tests the behavior of openFileForRead with the FS_AZURE_RESTRICT_GPS_ON_OPENFILE configuration enabled. - * Verifies that irrespective of whether FileStatus is provided or not, getPathStatus is not invoked for read flow. + * Verifies that irrespective of whether FileStatus is provided (correct or incorrect status type) or not, + * getPathStatus is not invoked for read flow. * * @throws Exception if any error occurs during the test */ @@ -891,16 +891,23 @@ public void testOpenFileWithOptionsWithRestrictGpsOnOpenFile() throws Exception setAbfsClient(abfsStore, mockClient); TracingContext tracingContext = getTestTracingContext(fs, false); - // NOTE: One call for GPS needs to come from openFileForWrite. If GPS were happening at openFileForRead, we would've seen 2 calls. + // Case 1: FileStatus is not provided + abfsStore.openFileForRead(fileWithoutFileStatus, Optional.empty(), null, tracingContext); + verify(mockClient, times(0).description("FileStatus not provided, restrict GPS: getPathStatus should NOT be invoked")) + .getPathStatus(any(String.class), any(Boolean.class), any(TracingContext.class), nullable(ContextEncryptionAdapter.class)); - // Case 1: FileStatus is provided - abfsStore.openFileForRead(fileWithFileStatus, Optional.ofNullable(new OpenFileParameters().withStatus(fs.getFileStatus(fileWithFileStatus))), null, tracingContext); - verify(mockClient, times(1).description("FileStatus provided, restrict GPS: getPathStatus should NOT be invoked")) + // NOTE: One call for GPS will come from getFileStatus for both cases below. + // If GPS were happening at openFileForRead, we would've seen more than 2 calls. + + // Case 2: FileStatus is provided (of wrong status type AbfsLocatedFileStatus) + abfsStore.openFileForRead(fileWithFileStatus, Optional.ofNullable(new OpenFileParameters().withStatus( + new AbfsLocatedFileStatus(fs.getFileStatus(fileWithFileStatus), null))), null, tracingContext); + verify(mockClient, times(1).description("Wrong FileStatus type provided, restrict GPS: getPathStatus still should NOT be invoked")) .getPathStatus(any(String.class), any(Boolean.class), any(TracingContext.class), nullable(ContextEncryptionAdapter.class)); - // Case 2: FileStatus is not provided - abfsStore.openFileForRead(fileWithoutFileStatus, Optional.empty(), null, tracingContext); - verify(mockClient, times(1).description("FileStatus not provided, restrict GPS: getPathStatus should NOT be invoked")) + // Case 3: FileStatus is provided (correct status type VersionedFileStatus) + abfsStore.openFileForRead(fileWithFileStatus, Optional.ofNullable(new OpenFileParameters().withStatus(fs.getFileStatus(fileWithFileStatus))), null, tracingContext); + verify(mockClient, times(2).description("Correct type FileStatus provided, restrict GPS: getPathStatus should NOT be invoked")) .getPathStatus(any(String.class), any(Boolean.class), any(TracingContext.class), nullable(ContextEncryptionAdapter.class)); } @@ -916,7 +923,7 @@ public void testMetadataFromReadForRestrictGpsOnOpenFile() throws Exception { AzureBlobFileSystem fs = getFileSystemWithRestrictGpsEnabled(); Path testFile = new Path("/testFile0"); - writeBufferToNewFile(testFile, new byte[250]); + writeBufferToNewFile(testFile, new byte[6 * ONE_MB]); // Open the file and perform a read try (FSDataInputStream in = fs.open(testFile)) { @@ -929,19 +936,19 @@ public void testMetadataFromReadForRestrictGpsOnOpenFile() throws Exception { assertThat(fileLengthPreRead).isEqualTo(0); // Trigger the first read - byte[] buf = new byte[100]; - int n = abfsIn.read(buf); - assertThat(n).isGreaterThan(0); + byte[] buf = new byte[6 * ONE_MB]; + int n = abfsIn.read(0, buf, 0, 2 * ONE_MB); + assertThat(n).isEqualTo(2 * ONE_MB); // After first read, eTag and content length should be set from the response String etagFirstRead = abfsIn.getETag(); long fileLengthFirstRead = abfsIn.getContentLength(); assertThat(etagFirstRead).isNotNull(); - assertThat(fileLengthFirstRead).isEqualTo(250L); + assertThat(fileLengthFirstRead).isEqualTo(6 * ONE_MB); //Trigger the second read - n = abfsIn.read(100, buf, 0, 100); - assertThat(n).isGreaterThan(0); + n = abfsIn.read(2 * ONE_MB, buf, 2 * ONE_MB, 4 * ONE_MB); + assertThat(n).isEqualTo(4 * ONE_MB); // eTag and content length should remain same as first read for second read onwards String etagSecondRead = abfsIn.getETag(); @@ -1458,6 +1465,7 @@ public void testDiffReadRequestSizeAndRAHBlockSize() throws Exception { true, SIXTEEN_KB); testReadAheads(inputStream, FORTY_EIGHT_KB, SIXTEEN_KB); + resetReadBufferManager(FOUR_MB, REDUCED_READ_BUFFER_AGE_THRESHOLD); //reset for next set of tests } @Test From 1cf2643df8cb3b58f85b86067222c7b0c6e572c9 Mon Sep 17 00:00:00 2001 From: Manika Joshi Date: Tue, 31 Mar 2026 22:13:30 -0700 Subject: [PATCH 13/14] checkstyle --- .../fs/azurebfs/AzureBlobFileSystemStore.java | 13 ++- .../fs/azurebfs/services/AbfsBlobClient.java | 2 - .../fs/azurebfs/services/AbfsErrors.java | 4 + .../fs/azurebfs/services/AbfsInputStream.java | 59 ++++------ .../services/TestAbfsInputStream.java | 106 +++++++++++++----- 5 files changed, 110 insertions(+), 74 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index 526eab6993dcd..1e14479b75b15 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -164,6 +164,7 @@ import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.INFINITE_LEASE_DURATION; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.ABFS_BLOB_DOMAIN_NAME; import static org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations.X_MS_ENCRYPTION_CONTEXT; +import static org.apache.hadoop.fs.azurebfs.services.AbfsErrors.ERR_OPENFILE_ON_DIRECTORY; import static org.apache.hadoop.fs.azurebfs.utils.UriUtils.isKeyForDirectorySet; /** @@ -889,7 +890,7 @@ private AbfsRestOperationException openFileForReadDirectoryException() { return new AbfsRestOperationException( AzureServiceErrorCode.PATH_NOT_FOUND.getStatusCode(), AzureServiceErrorCode.PATH_NOT_FOUND.getErrorCode(), - "openFileForRead must be used with files and not directories", + ERR_OPENFILE_ON_DIRECTORY, null); } @@ -951,13 +952,13 @@ public AbfsInputStream openFileForRead(Path path, */ if ((fileStatus instanceof VersionedFileStatus) && ( getClient().getEncryptionType() != EncryptionType.ENCRYPTION_CONTEXT - || ((VersionedFileStatus) fileStatus).getEncryptionContext() - != null)) { + || ((VersionedFileStatus) fileStatus).getEncryptionContext() + != null)) { path = path.makeQualified(this.uri, path); Preconditions.checkArgument(fileStatus.getPath().equals(path), String.format( - "Filestatus path [%s] does not match with given path [%s]", - fileStatus.getPath(), path)); + "Filestatus path [%s] does not match with given path [%s]", + fileStatus.getPath(), path)); resourceType = fileStatus.isFile() ? FILE : DIRECTORY; contentLength = fileStatus.getLen(); eTag = ((VersionedFileStatus) fileStatus).getVersion(); @@ -1926,7 +1927,7 @@ private AbfsClientContext populateAbfsClientContext() { .build(); } - public String getRelativePath(final Path path) { + public static String getRelativePath(final Path path) { Preconditions.checkNotNull(path, "path"); String relPath = path.toUri().getPath(); if (relPath.isEmpty()) { diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsBlobClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsBlobClient.java index f93d1f3a4c757..52fbd3182fdd5 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsBlobClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsBlobClient.java @@ -502,7 +502,6 @@ public AbfsRestOperation createPath(final String path, AbfsRestOperation statusOp = null; try { // Check if the file already exists by calling GetPathStatus - //TODO: GBP statusOp = getPathStatus(path, tracingContext, null, false); } catch (AbfsRestOperationException ex) { // If the path does not exist, continue with file creation @@ -2260,7 +2259,6 @@ private AbfsRestOperation createFile(final String path, // If the overwrite flag is true, we must verify whether an empty directory exists at the specified path. // However, if overwrite is false, we can skip this validation and proceed with blob creation, // which will fail with a conflict error if a file or directory already exists at the path. - //todo: GBP if (overwrite && isEmptyDirectory(path, tracingContext, false)) { throw new AbfsRestOperationException(HTTP_CONFLICT, AzureServiceErrorCode.PATH_CONFLICT.getErrorCode(), diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsErrors.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsErrors.java index fe7f3b5cb1b21..70dd05c19b1e3 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsErrors.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsErrors.java @@ -59,6 +59,10 @@ public final class AbfsErrors { + "and cannot be appended to by the Azure Data Lake Storage Service API"; public static final String CONDITION_NOT_MET = "The condition specified using " + "HTTP conditional header(s) is not met."; + public static final String ERR_READ_ON_DIRECTORY = "Read operation not permitted on a directory."; + public static final String ERR_OPENFILE_ON_DIRECTORY = "openFileForRead must be used with files and not directories"; + + /** * Exception message on filesystem init if token-provider-auth-type configs are provided */ diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java index 58dafb587c326..b3f35c6767394 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java @@ -22,7 +22,6 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.net.HttpURLConnection; -import java.util.Objects; import java.util.UUID; import org.apache.commons.lang3.StringUtils; @@ -57,15 +56,12 @@ import static java.lang.Math.min; import static org.apache.hadoop.fs.azurebfs.AzureBlobFileSystemStore.extractEtagHeader; -import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.DIRECTORY; -import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.ROOT_PATH; -import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.TRUE; +import static org.apache.hadoop.fs.azurebfs.AzureBlobFileSystemStore.getRelativePath; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.ONE_KB; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.STREAM_ID_LEN; -import static org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations.X_MS_META_HDI_ISFOLDER; import static org.apache.hadoop.fs.azurebfs.constants.InternalConstants.CAPABILITY_SAFE_READAHEAD; import static org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode.INVALID_RANGE; -import static org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode.UNAUTHORIZED_BLOB_OVERWRITE; +import static org.apache.hadoop.fs.azurebfs.services.AbfsErrors.ERR_READ_ON_DIRECTORY; import static org.apache.hadoop.io.Sizes.S_128K; import static org.apache.hadoop.io.Sizes.S_2M; import static org.apache.hadoop.util.StringUtils.toLowerCase; @@ -85,7 +81,7 @@ public abstract class AbfsInputStream extends FSInputStream implements CanUnbuff private final Statistics statistics; private final String path; - private long contentLength; + private volatile long contentLength; private final int bufferSize; // default buffer size private final int footerReadSize; // default buffer size to read when reading footer private final int readAheadQueueDepth; // initialized in constructor @@ -146,7 +142,6 @@ public abstract class AbfsInputStream extends FSInputStream implements CanUnbuff /** ABFS instance to be held by the input stream to avoid GC close. */ private final BackReference fsBackRef; private final ReadBufferManager readBufferManager; - private final String readOnDirectoryErrorMsg = "Read operation not permitted on a directory."; /** * Constructor for AbfsInputStream. @@ -579,25 +574,6 @@ protected int readInternal(final long position, final byte[] b, final int offset } } - /** - * Convert a {@link Path} to the relative path string used by ABFS. - * - *

This returns the URI path component of the supplied {@code path}. If the - * resulting path is empty, this method returns {@code ROOT_PATH}. - * - * @param path the {@link Path} to convert; must not be null - * @return the relative path as a {@link String}; never null - */ - String getRelativePath(final Path path) { - Preconditions.checkNotNull(path, "path"); - String relPath = path.toUri().getPath(); - if (relPath.isEmpty()) { - // This means that path passed by user is absolute path of root without "/" at end. - relPath = ROOT_PATH; - } - return relPath; - } - /** * Creates an exception indicating that a read operation was attempted on a directory. * @@ -607,7 +583,7 @@ private IOException directoryReadException() { return new AbfsRestOperationException( AzureServiceErrorCode.PATH_NOT_FOUND.getStatusCode(), AzureServiceErrorCode.PATH_NOT_FOUND.getErrorCode(), - readOnDirectoryErrorMsg, + ERR_READ_ON_DIRECTORY, null); } @@ -629,15 +605,21 @@ private void checkIfDirPathInFNS() throws IOException { } } +/** + * Extracts the content length from the HTTP response headers. + * Uses the Content-Range header to determine the total file size, which is necessary + * for handling partial reads correctly. + * + * @param op the ABFS HTTP operation containing the response headers + * @return the content length of the file + */ private long extractContentLength(AbfsHttpOperation op) { // We need to use content range header instead of content length to take care of partial reads String contentRange = op.getResponseHeader(HttpHeaderConfigurations.CONTENT_RANGE); + contentLength = 0; if (!StringUtils.isEmpty(contentRange)) { contentLength = Long.parseLong(contentRange.split(AbfsHttpConstants.FORWARD_SLASH)[1]); } - else { - contentLength = 0; - } return contentLength; } @@ -690,7 +672,7 @@ int readRemote(long position, byte[] b, int offset, int length, TracingContext t if (ex instanceof AbfsRestOperationException) { AbfsRestOperationException ere = (AbfsRestOperationException) ex; int status = ere.getStatusCode(); - if (ere.getErrorMessage().contains(readOnDirectoryErrorMsg)) { + if (ere.getErrorMessage().contains(ERR_READ_ON_DIRECTORY)) { throw ere; } boolean isHnsEnabled = client.getIsNamespaceEnabled(); @@ -705,18 +687,21 @@ int readRemote(long position, byte[] b, int offset, int length, TracingContext t throw new FileNotFoundException(ere.getMessage()); } - // FNS account with restrictGpsOnOpenFile enabled try { - // Need to rule out if the path is an implicit directory + /* + * For FNS account with restrictGpsOnOpenFile enabled, + * need to rule out if the path is an implicit directory + */ checkIfDirPathInFNS(); - } catch (AzureBlobFileSystemException gpsEx) { AbfsRestOperationException gpsEre = (AbfsRestOperationException) gpsEx; - if (gpsEre.getErrorMessage().contains(readOnDirectoryErrorMsg)) { + if (gpsEre.getErrorMessage().contains(ERR_READ_ON_DIRECTORY)) { throw gpsEre; } // The file does not exist - else throw new FileNotFoundException(gpsEre.getMessage()); + else { + throw new FileNotFoundException(gpsEre.getMessage()); + } } } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java index e49c01a78c8fa..d85cd35ce0c54 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java @@ -74,7 +74,9 @@ import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.COLON; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.EMPTY_STRING; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.SPLIT_NO_LIMIT; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_READ_OPTIMIZE_FOOTER_READ; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENABLE_PREFETCH_REQUEST_PRIORITY; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_RESTRICT_GPS_ON_OPENFILE; import static org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations.X_MS_REQUEST_PRIORITY; import static org.apache.hadoop.fs.azurebfs.constants.ReadType.DIRECT_READ; import static org.apache.hadoop.fs.azurebfs.constants.ReadType.FOOTER_READ; @@ -129,6 +131,8 @@ public class TestAbfsInputStream extends AbstractAbfsIntegrationTest { private static final int POSITION_INDEX = 9; private static final int OPERATION_INDEX = 6; private static final int READTYPE_INDEX = 11; + private static final int ENCRYPTION_KEY_SIZE = 32; + private static final int SMALL_BUFFER_SIZE = 100; @AfterEach @@ -321,6 +325,37 @@ private void checkGetPathStatusCalls(Path testFile, FileStatus fileStatus, Mockito.reset(mockClient); //clears invocation count for next test case } + @Test + public void testReadFile() throws Exception { + Configuration conf = getRawConfiguration(); + //conf.set(AZURE_READ_OPTIMIZE_FOOTER_READ, "false"); + conf.set(FS_AZURE_RESTRICT_GPS_ON_OPENFILE, "true"); + AzureBlobFileSystem fs = getFileSystem(conf); + Path testFile = new Path("/readFileTest"); + byte[] data = "hello world".getBytes(StandardCharsets.UTF_8); //11bytes + writeBufferToNewFile(testFile, data); + int fileSize = Math.toIntExact(getFileSystem().getFileStatus(testFile).getLen()); + + try (FSDataInputStream in = fs.open(testFile)) { + byte[] buf = new byte[data.length]; + + //todo: check for available in validate() + int bytesRead = in.read(13, buf, 0, fileSize); // pos >=contentlength -> checks earlier itself (-1) + +// in.seek(12); +// int bytesRead = in.read(buf); // fcursor >=contentlength -> fails earlier at seek (EOFException) + + //read(long position, byte[] buffer, int offset, int length) + // int bytesRead = in.read(13, buf, 0, fileSize); // pos >=contentlength -> checks earlier itself (-1) + //int bytesRead = in.read(buf, 0, fileSize+2); // len requested >=contentlength -> fails earlier itself (indexoutofbounds) + + //int bytesRead = in.read(); + + Assertions.assertEquals(fileSize, bytesRead); + Assertions.assertArrayEquals(data, buf); + } + } + @Test public void testOpenFileWithOptions() throws Exception { AzureBlobFileSystem fs = getFileSystem(); @@ -408,7 +443,7 @@ private void mockClientForEncryptionContext(AbfsClient encryptedClient) throws I EncryptionContextProvider provider = mock(EncryptionContextProvider.class); when(provider.getEncryptionKey(anyString(), any())) - .thenReturn(new ABFSKey(new byte[32])); + .thenReturn(new ABFSKey(new byte[ENCRYPTION_KEY_SIZE])); doReturn(provider) .when(encryptedClient) @@ -801,7 +836,7 @@ public void testFNSExceptionOnDirReadWithRestrictGpsConfig() throws Exception { try (FSDataInputStream in = fs.open(new Path(explicitTestFolder))) { AbfsInputStream abfsIn = (AbfsInputStream) in.getWrappedStream(); - byte[] buf = new byte[100]; + byte[] buf = new byte[SMALL_BUFFER_SIZE]; AbfsRestOperationException ex = Assertions.assertThrows(AbfsRestOperationException.class, () -> abfsIn.read(buf)); assertThat(ex.getStatusCode()).isEqualTo(AzureServiceErrorCode.PATH_NOT_FOUND.getStatusCode()); assertThat(ex.getMessage()).contains("Read operation not permitted on a directory."); @@ -817,7 +852,7 @@ public void testFNSExceptionOnDirReadWithRestrictGpsConfig() throws Exception { createAzCopyFolder(new Path(implicitTestFolder)); try (FSDataInputStream in2 = fs.open(new Path(implicitTestFolder))) { AbfsInputStream abfsIn2 = (AbfsInputStream) in2.getWrappedStream(); - byte[] buf2 = new byte[100]; + byte[] buf2 = new byte[SMALL_BUFFER_SIZE]; AbfsRestOperationException ex2 = Assertions.assertThrows(AbfsRestOperationException.class, () -> abfsIn2.read(buf2)); assertThat(ex2.getStatusCode()).isEqualTo(AzureServiceErrorCode.PATH_NOT_FOUND.getStatusCode()); assertThat(ex2.getMessage()).contains("Read operation not permitted on a directory."); @@ -854,7 +889,7 @@ public void testHNSExceptionOnDirReadWithRestrictGpsConfig() throws Exception { try (FSDataInputStream in = fs.open(new Path(testFolder))) { AbfsInputStream abfsIn = (AbfsInputStream) in.getWrappedStream(); - byte[] buf = new byte[100]; + byte[] buf = new byte[SMALL_BUFFER_SIZE]; AbfsRestOperationException ex = Assertions.assertThrows(AbfsRestOperationException.class, () -> abfsIn.read(buf)); assertThat(ex.getStatusCode()).isEqualTo(AzureServiceErrorCode.PATH_NOT_FOUND.getStatusCode()); assertThat(ex.getMessage()).contains("Read operation not permitted on a directory."); @@ -893,22 +928,34 @@ public void testOpenFileWithOptionsWithRestrictGpsOnOpenFile() throws Exception // Case 1: FileStatus is not provided abfsStore.openFileForRead(fileWithoutFileStatus, Optional.empty(), null, tracingContext); - verify(mockClient, times(0).description("FileStatus not provided, restrict GPS: getPathStatus should NOT be invoked")) - .getPathStatus(any(String.class), any(Boolean.class), any(TracingContext.class), nullable(ContextEncryptionAdapter.class)); + verify(mockClient, times(0) + .description("FileStatus not provided, restrict GPS: getPathStatus should NOT be invoked")) + .getPathStatus(any(String.class), any(Boolean.class), any(TracingContext.class), + nullable(ContextEncryptionAdapter.class)); // NOTE: One call for GPS will come from getFileStatus for both cases below. // If GPS were happening at openFileForRead, we would've seen more than 2 calls. // Case 2: FileStatus is provided (of wrong status type AbfsLocatedFileStatus) - abfsStore.openFileForRead(fileWithFileStatus, Optional.ofNullable(new OpenFileParameters().withStatus( - new AbfsLocatedFileStatus(fs.getFileStatus(fileWithFileStatus), null))), null, tracingContext); - verify(mockClient, times(1).description("Wrong FileStatus type provided, restrict GPS: getPathStatus still should NOT be invoked")) - .getPathStatus(any(String.class), any(Boolean.class), any(TracingContext.class), nullable(ContextEncryptionAdapter.class)); + abfsStore.openFileForRead(fileWithFileStatus, + Optional.ofNullable(new OpenFileParameters().withStatus( + new AbfsLocatedFileStatus(fs.getFileStatus(fileWithFileStatus), null))), + null, tracingContext); + verify(mockClient, times(1) + .description("Wrong FileStatus type provided, restrict GPS: getPathStatus still should NOT be invoked")) + .getPathStatus(any(String.class), any(Boolean.class), any(TracingContext.class), + nullable(ContextEncryptionAdapter.class)); // Case 3: FileStatus is provided (correct status type VersionedFileStatus) - abfsStore.openFileForRead(fileWithFileStatus, Optional.ofNullable(new OpenFileParameters().withStatus(fs.getFileStatus(fileWithFileStatus))), null, tracingContext); - verify(mockClient, times(2).description("Correct type FileStatus provided, restrict GPS: getPathStatus should NOT be invoked")) - .getPathStatus(any(String.class), any(Boolean.class), any(TracingContext.class), nullable(ContextEncryptionAdapter.class)); + abfsStore.openFileForRead(fileWithFileStatus, + Optional.ofNullable(new OpenFileParameters() + .withStatus(fs.getFileStatus(fileWithFileStatus))), + null, tracingContext); + verify(mockClient, times(2).description( + "Correct type FileStatus provided, restrict GPS: " + + "getPathStatus should NOT be invoked")) + .getPathStatus(any(String.class), any(Boolean.class), + any(TracingContext.class), nullable(ContextEncryptionAdapter.class)); } /** @@ -1163,7 +1210,6 @@ public void testSuccessfulReadAhead() throws Exception { // Stub : // Pass all readAheads and fail the post eviction request to // prove ReadAhead buffer is used - // for post eviction check, fail all read aheads doReturn(op) .doReturn(op) .doReturn(op) @@ -1405,26 +1451,28 @@ public void testReadAheadManagerForSuccessfulReadAhead() throws Exception { any(String.class), any(), any(TracingContext.class)); AbfsInputStream inputStream = getAbfsInputStream(client, "testSuccessfulReadAhead.txt"); + int beforeReadCompletedListSize = getBufferManager().getCompletedReadListSize(); - queueReadAheads(inputStream); - - // AbfsInputStream Read would have waited for the read-ahead for the requested offset - // as we are testing from ReadAheadManager directly, sleep for a sec to - // get the read ahead threads to complete - Thread.sleep(1000); + // First read request that triggers readAheads. + inputStream.read(new byte[ONE_KB]); // Only the 3 readAhead threads should have triggered client.read verifyReadCallCount(client, 3); + int newAdditionsToCompletedRead = + getBufferManager().getCompletedReadListSize() + - beforeReadCompletedListSize; + // read buffer might be dumped if the ReadBufferManager getblock preceded + // the action of buffer being picked for reading from readaheadqueue, so that + // inputstream can proceed with read and not be blocked on readahead thread + // availability. So the count of buffers in completedReadQueue for the stream + // can be same or lesser than the requests triggered to queue readahead. + assertThat(newAdditionsToCompletedRead) + .describedAs( + "New additions to completed reads should be same or less than as number of readaheads") + .isLessThanOrEqualTo(3); - // getBlock for a new read should return the buffer read-ahead - int bytesRead = getBufferManager().getBlock( - inputStream, - ONE_KB, - ONE_KB, - new byte[ONE_KB]); - - Assertions.assertTrue(bytesRead > 0, "bytesRead should be non-zero from the " - + "buffer that was read-ahead"); + // Another read request whose requested data is already read ahead. + inputStream.read(ONE_KB, new byte[ONE_KB], 0, ONE_KB); // Once created, mock will remember all interactions. // As the above read should not have triggered any server calls, total From 63f7ac1d0f54a9e9f6151f8bdb06cdf5af712676 Mon Sep 17 00:00:00 2001 From: Manika Joshi Date: Thu, 2 Apr 2026 00:57:12 -0700 Subject: [PATCH 14/14] suggestions --- .../azurebfs/constants/AbfsHttpConstants.java | 1 + .../services/AzureServiceErrorCode.java | 4 +- .../services/AbfsAdaptiveInputStream.java | 2 +- .../fs/azurebfs/services/AbfsInputStream.java | 14 +++-- .../services/TestAbfsInputStream.java | 58 ++++++++++--------- 5 files changed, 45 insertions(+), 34 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/AbfsHttpConstants.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/AbfsHttpConstants.java index aa1d410298314..9c31936a72219 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/AbfsHttpConstants.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/AbfsHttpConstants.java @@ -175,6 +175,7 @@ public final class AbfsHttpConstants { * @see "https://learn.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-to-use-vm-token#error-handling" */ public static final int HTTP_TOO_MANY_REQUESTS = 429; + public static final int HTTP_INVALID_RANGE = 416; public static final char CHAR_FORWARD_SLASH = '/'; public static final char CHAR_EXCLAMATION_POINT = '!'; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java index c38813a2e27d0..777460cde8446 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java @@ -21,6 +21,8 @@ import java.net.HttpURLConnection; import java.util.ArrayList; import java.util.List; + +import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -66,7 +68,7 @@ public enum AzureServiceErrorCode { INVALID_APPEND_OPERATION("InvalidAppendOperation", HttpURLConnection.HTTP_CONFLICT, null), UNAUTHORIZED_BLOB_OVERWRITE("UnauthorizedBlobOverwrite", HttpURLConnection.HTTP_FORBIDDEN, "This request is not authorized to perform blob overwrites."), - INVALID_RANGE("InvalidRange", 416, + INVALID_RANGE("InvalidRange", AbfsHttpConstants.HTTP_INVALID_RANGE, "The range specified is invalid for the current size of the resource."), UNKNOWN(null, -1, null); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java index 466d593ec3c5b..2b2daa8a95199 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAdaptiveInputStream.java @@ -85,7 +85,7 @@ protected int readOneBlock(final byte[] b, final int off, final int len) throws getTracingContext().setReadType(ReadType.NORMAL_READ); // If restrictGpsOnOpenFile config is enabled, skip prefetch for the first read since contentLength - // is not available yet to determine prefetch block size. + // is not available yet. if (shouldRestrictGpsOnOpenFile() && isFirstRead()) { LOG.debug("RestrictGpsOnOpenFile is enabled. Skip readahead for first read."); bytesRead = readInternal(getFCursor(), getBuffer(), 0, getBufferSize(), true); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java index b3f35c6767394..eba5b09337775 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java @@ -577,14 +577,16 @@ protected int readInternal(final long position, final byte[] b, final int offset /** * Creates an exception indicating that a read operation was attempted on a directory. * - * @return an {@link AbfsRestOperationException} indicating the operation is not permitted on a directory + * @return an {@link UnsupportedOperationException} indicating the operation is not permitted on a directory */ - private IOException directoryReadException() { - return new AbfsRestOperationException( - AzureServiceErrorCode.PATH_NOT_FOUND.getStatusCode(), - AzureServiceErrorCode.PATH_NOT_FOUND.getErrorCode(), + private UnsupportedOperationException directoryReadException() { + return new UnsupportedOperationException( ERR_READ_ON_DIRECTORY, - null); + new AbfsRestOperationException( + AzureServiceErrorCode.PATH_NOT_FOUND.getStatusCode(), + AzureServiceErrorCode.PATH_NOT_FOUND.getErrorCode(), + ERR_READ_ON_DIRECTORY, + null)); } /** diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java index fbeed9db61255..40c3440ef25c5 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java @@ -74,9 +74,7 @@ import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.COLON; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.EMPTY_STRING; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.SPLIT_NO_LIMIT; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_READ_OPTIMIZE_FOOTER_READ; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENABLE_PREFETCH_REQUEST_PRIORITY; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_RESTRICT_GPS_ON_OPENFILE; import static org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations.X_MS_REQUEST_PRIORITY; import static org.apache.hadoop.fs.azurebfs.constants.ReadType.DIRECT_READ; import static org.apache.hadoop.fs.azurebfs.constants.ReadType.FOOTER_READ; @@ -151,6 +149,12 @@ AbfsRestOperation getMockRestOp() { return op; } + /** + * Creates a mock AbfsRestOperation with metadata headers for testing. + * The mock includes Content-Range and ETag headers in the response. + * + * @return a mocked AbfsRestOperation with response metadata + */ AbfsRestOperation getMockRestOpWithMetadata() { AbfsRestOperation op = mock(AbfsRestOperation.class); AbfsHttpOperation httpOp = mock(AbfsHttpOperation.class); @@ -806,8 +810,9 @@ public void testFNSExceptionOnDirReadWithRestrictGpsConfig() throws Exception { try (FSDataInputStream in = fs.open(new Path(explicitTestFolder))) { AbfsInputStream abfsIn = (AbfsInputStream) in.getWrappedStream(); byte[] buf = new byte[SMALL_BUFFER_SIZE]; - AbfsRestOperationException ex = Assertions.assertThrows(AbfsRestOperationException.class, () -> abfsIn.read(buf)); - assertThat(ex.getStatusCode()).isEqualTo(AzureServiceErrorCode.PATH_NOT_FOUND.getStatusCode()); + UnsupportedOperationException ex = Assertions.assertThrows(UnsupportedOperationException.class, () -> abfsIn.read(buf)); + AbfsRestOperationException cause = (AbfsRestOperationException) ex.getCause(); + assertThat(cause.getStatusCode()).isEqualTo(AzureServiceErrorCode.PATH_NOT_FOUND.getStatusCode()); assertThat(ex.getMessage()).contains("Read operation not permitted on a directory."); } verify(spyClient, times(1)) @@ -822,8 +827,9 @@ public void testFNSExceptionOnDirReadWithRestrictGpsConfig() throws Exception { try (FSDataInputStream in2 = fs.open(new Path(implicitTestFolder))) { AbfsInputStream abfsIn2 = (AbfsInputStream) in2.getWrappedStream(); byte[] buf2 = new byte[SMALL_BUFFER_SIZE]; - AbfsRestOperationException ex2 = Assertions.assertThrows(AbfsRestOperationException.class, () -> abfsIn2.read(buf2)); - assertThat(ex2.getStatusCode()).isEqualTo(AzureServiceErrorCode.PATH_NOT_FOUND.getStatusCode()); + UnsupportedOperationException ex2 = Assertions.assertThrows(UnsupportedOperationException.class, () -> abfsIn2.read(buf2)); + AbfsRestOperationException cause = (AbfsRestOperationException) ex2.getCause(); + assertThat(cause.getStatusCode()).isEqualTo(AzureServiceErrorCode.PATH_NOT_FOUND.getStatusCode()); assertThat(ex2.getMessage()).contains("Read operation not permitted on a directory."); } verify(spyClient, times(2)) @@ -859,11 +865,12 @@ public void testHNSExceptionOnDirReadWithRestrictGpsConfig() throws Exception { try (FSDataInputStream in = fs.open(new Path(testFolder))) { AbfsInputStream abfsIn = (AbfsInputStream) in.getWrappedStream(); byte[] buf = new byte[SMALL_BUFFER_SIZE]; - AbfsRestOperationException ex = Assertions.assertThrows(AbfsRestOperationException.class, () -> abfsIn.read(buf)); - assertThat(ex.getStatusCode()).isEqualTo(AzureServiceErrorCode.PATH_NOT_FOUND.getStatusCode()); + UnsupportedOperationException ex = Assertions.assertThrows(UnsupportedOperationException.class, () -> abfsIn.read(buf)); + AbfsRestOperationException cause = (AbfsRestOperationException) ex.getCause(); + assertThat(cause.getStatusCode()).isEqualTo(AzureServiceErrorCode.PATH_NOT_FOUND.getStatusCode()); assertThat(ex.getMessage()).contains("Read operation not permitted on a directory."); - } + verify(spyClient, times(0)) .getPathStatus( anyString(), @@ -1179,6 +1186,7 @@ public void testSuccessfulReadAhead() throws Exception { // Stub : // Pass all readAheads and fail the post eviction request to // prove ReadAhead buffer is used + // for post eviction check, fail all read aheads doReturn(op) .doReturn(op) .doReturn(op) @@ -1420,28 +1428,26 @@ public void testReadAheadManagerForSuccessfulReadAhead() throws Exception { any(String.class), any(), any(TracingContext.class)); AbfsInputStream inputStream = getAbfsInputStream(client, "testSuccessfulReadAhead.txt"); - int beforeReadCompletedListSize = getBufferManager().getCompletedReadListSize(); - // First read request that triggers readAheads. - inputStream.read(new byte[ONE_KB]); + queueReadAheads(inputStream); + + // AbfsInputStream Read would have waited for the read-ahead for the requested offset + // as we are testing from ReadAheadManager directly, sleep for a sec to + // get the read ahead threads to complete + Thread.sleep(1000); // Only the 3 readAhead threads should have triggered client.read verifyReadCallCount(client, 3); - int newAdditionsToCompletedRead = - getBufferManager().getCompletedReadListSize() - - beforeReadCompletedListSize; - // read buffer might be dumped if the ReadBufferManager getblock preceded - // the action of buffer being picked for reading from readaheadqueue, so that - // inputstream can proceed with read and not be blocked on readahead thread - // availability. So the count of buffers in completedReadQueue for the stream - // can be same or lesser than the requests triggered to queue readahead. - assertThat(newAdditionsToCompletedRead) - .describedAs( - "New additions to completed reads should be same or less than as number of readaheads") - .isLessThanOrEqualTo(3); - // Another read request whose requested data is already read ahead. - inputStream.read(ONE_KB, new byte[ONE_KB], 0, ONE_KB); + // getBlock for a new read should return the buffer read-ahead + int bytesRead = getBufferManager().getBlock( + inputStream, + ONE_KB, + ONE_KB, + new byte[ONE_KB]); + + Assertions.assertTrue(bytesRead > 0, "bytesRead should be non-zero from the " + + "buffer that was read-ahead"); // Once created, mock will remember all interactions. // As the above read should not have triggered any server calls, total