diff --git a/core/aws-core/src/main/java/software/amazon/awssdk/awscore/retry/AwsRetryStrategy.java b/core/aws-core/src/main/java/software/amazon/awssdk/awscore/retry/AwsRetryStrategy.java index a68b95e503ac..93f9aec6bbc5 100644 --- a/core/aws-core/src/main/java/software/amazon/awssdk/awscore/retry/AwsRetryStrategy.java +++ b/core/aws-core/src/main/java/software/amazon/awssdk/awscore/retry/AwsRetryStrategy.java @@ -18,9 +18,11 @@ import software.amazon.awssdk.annotations.SdkPublicApi; import software.amazon.awssdk.awscore.exception.AwsServiceException; import software.amazon.awssdk.awscore.internal.AwsErrorCode; +import software.amazon.awssdk.core.exception.SdkException; import software.amazon.awssdk.core.internal.retry.RetryPolicyAdapter; import software.amazon.awssdk.core.internal.retry.SdkDefaultRetryStrategy; import software.amazon.awssdk.core.retry.RetryMode; +import software.amazon.awssdk.core.retry.RetryUtils; import software.amazon.awssdk.retries.AdaptiveRetryStrategy; import software.amazon.awssdk.retries.DefaultRetryStrategy; import software.amazon.awssdk.retries.LegacyRetryStrategy; @@ -135,7 +137,7 @@ public static StandardRetryStrategy standardRetryStrategy() { */ public static StandardRetryStrategy standardRetryStrategy(boolean newRetries2026Enabled) { StandardRetryStrategy.Builder builder = SdkDefaultRetryStrategy.standardRetryStrategyBuilder(newRetries2026Enabled); - return configure(builder).build(); + return configure(builder, newRetries2026Enabled).build(); } /** @@ -167,7 +169,7 @@ public static AdaptiveRetryStrategy adaptiveRetryStrategy() { */ public static AdaptiveRetryStrategy adaptiveRetryStrategy(boolean newRetries2026Enabled) { AdaptiveRetryStrategy.Builder builder = SdkDefaultRetryStrategy.adaptiveRetryStrategyBuilder(newRetries2026Enabled); - return configure(builder) + return configure(builder, newRetries2026Enabled) .build(); } @@ -179,7 +181,22 @@ public static AdaptiveRetryStrategy adaptiveRetryStrategy(boolean newRetries2026 * @return The given builder */ public static > T configure(T builder) { + return configure(builder, false); + } + + /** + * Configures a retry strategy using its builder to add AWS-specific retry exceptions. + * + * @param builder The builder to add the AWS-specific retry exceptions + * @param The type of the builder extending {@link RetryStrategy.Builder} + * @return The given builder + */ + private static > T configure(T builder, boolean newRetries2026Enabled) { builder.retryOnException(AwsRetryStrategy::retryOnAwsRetryableErrors); + if (newRetries2026Enabled) { + builder.retryOnException(AwsRetryStrategy::isLimitExceededErrorCode); + builder.treatAsThrottling(AwsRetryStrategy::treatAsThrottlingV21); + } markDefaultsAdded(builder); return builder; } @@ -205,6 +222,25 @@ private static boolean retryOnAwsRetryableErrors(Throwable ex) { return false; } + /** + * Additionally, check for LimitExceededException as it was not previously treated as a throttling exception. + */ + private static boolean treatAsThrottlingV21(Throwable ex) { + if (!(ex instanceof SdkException)) { + return false; + } + + SdkException sdkException = (SdkException) ex; + + return RetryUtils.isThrottlingException(sdkException) + || isLimitExceededErrorCode(sdkException); + } + + private static boolean isLimitExceededErrorCode(Throwable ex) { + return ex instanceof AwsServiceException + && "LimitExceededException".equals(((AwsServiceException) ex).awsErrorDetails().errorCode()); + } + /** * Returns a {@link RetryStrategy} that implements the legacy {@link RetryMode#ADAPTIVE} mode. * diff --git a/core/aws-core/src/test/java/software/amazon/awssdk/awscore/retry/AwsRetryStrategyTest.java b/core/aws-core/src/test/java/software/amazon/awssdk/awscore/retry/AwsRetryStrategyTest.java new file mode 100644 index 000000000000..12e4f86020e5 --- /dev/null +++ b/core/aws-core/src/test/java/software/amazon/awssdk/awscore/retry/AwsRetryStrategyTest.java @@ -0,0 +1,98 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.awscore.retry; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.google.common.base.Supplier; +import java.time.Duration; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import software.amazon.awssdk.awscore.exception.AwsErrorDetails; +import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.retries.StandardRetryStrategy; +import software.amazon.awssdk.retries.api.AcquireInitialTokenRequest; +import software.amazon.awssdk.retries.api.RefreshRetryTokenRequest; +import software.amazon.awssdk.retries.api.RetryToken; +import software.amazon.awssdk.retries.api.TokenAcquisitionFailedException; +import software.amazon.awssdk.retries.internal.DefaultRetryToken; + +public class AwsRetryStrategyTest { + + @ParameterizedTest + @CsvSource({"true", "false"}) + void standardRetryStrategy_limitExceededException_retryBehaviorCorrect(boolean newRetries2026Enabled) { + StandardRetryStrategy strategy = AwsRetryStrategy.standardRetryStrategy(newRetries2026Enabled); + + RetryToken token = strategy.acquireInitialToken(AcquireInitialTokenRequest.create("test")).token(); + RefreshRetryTokenRequest refresh = RefreshRetryTokenRequest.builder() + .failure(createTestException("LimitExceededException")) + .token(token) + .build(); + + if (newRetries2026Enabled) { + assertThat(strategy.refreshRetryToken(refresh).delay()).isGreaterThanOrEqualTo(Duration.ZERO); + } else { + assertThatThrownBy(() -> strategy.refreshRetryToken(refresh)) + .isInstanceOf(TokenAcquisitionFailedException.class) + .matches(e -> { + TokenAcquisitionFailedException acquireException = (TokenAcquisitionFailedException) e; + DefaultRetryToken exceptionToken = (DefaultRetryToken) acquireException.token(); + return exceptionToken.state() == DefaultRetryToken.TokenState.NON_RETRYABLE_EXCEPTION; + }); + } + } + + @ParameterizedTest + @CsvSource({"Throttling", + "ThrottlingException", + "ThrottledException", + "RequestThrottledException", + "TooManyRequestsException", + "ProvisionedThroughputExceededException", + "TransactionInProgressException", + "RequestLimitExceeded", + "BandwidthLimitExceeded", + "LimitExceededException", + "RequestThrottled", + "SlowDown", + "PriorRequestNotComplete", + "EC2ThrottledException"}) + void standardRetryStrategy_retry21_throttlingBehaviorCorrect(String errorCode) { + AwsServiceException exception = createTestException(errorCode); + + for (int i = 0; i < 128; ++i) { + StandardRetryStrategy strategy = AwsRetryStrategy.standardRetryStrategy(true); + + RetryToken token = strategy.acquireInitialToken(AcquireInitialTokenRequest.create("test")).token(); + RefreshRetryTokenRequest refresh = RefreshRetryTokenRequest.builder() + .token(token) + .failure(exception) + .build(); + Duration delay = strategy.refreshRetryToken(refresh).delay(); + + assertThat(delay).isBetween(Duration.ZERO, Duration.ofMillis(1000)); + } + } + + private static AwsServiceException createTestException(String errorCode) { + AwsErrorDetails details = AwsErrorDetails.builder() + .errorCode(errorCode) + .build(); + return AwsServiceException.builder().awsErrorDetails(details).build(); + } +} diff --git a/core/profiles/src/main/java/software/amazon/awssdk/profiles/ProfileProperty.java b/core/profiles/src/main/java/software/amazon/awssdk/profiles/ProfileProperty.java index fc1fbc32fe3f..069359c37567 100644 --- a/core/profiles/src/main/java/software/amazon/awssdk/profiles/ProfileProperty.java +++ b/core/profiles/src/main/java/software/amazon/awssdk/profiles/ProfileProperty.java @@ -111,6 +111,12 @@ public final class ProfileProperty { */ public static final String RETRY_MODE = "retry_mode"; + /** + * How many HTTP requests an SDK should make for a single SDK operation invocation before giving up. See the JavaDocs for + * {@code SdkSystemSetting.AWS_MAX_ATTEMPTS} and {@code RetryStrategy.maxAttempts()} for more information. + */ + public static final String MAX_ATTEMPTS = "max_attempts"; + /** * The "defaults mode" to be used for clients created using the currently-configured profile. Defaults mode determins how SDK * default configuration should be resolved. See the {@code DefaultsMode} class JavaDoc for more diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/retry/MaxAttemptsResolver.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/retry/MaxAttemptsResolver.java new file mode 100644 index 000000000000..accb4192b995 --- /dev/null +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/retry/MaxAttemptsResolver.java @@ -0,0 +1,73 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.retry; + +import java.util.Optional; +import java.util.function.Supplier; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.SdkSystemSetting; +import software.amazon.awssdk.profiles.ProfileFile; +import software.amazon.awssdk.profiles.ProfileFileSystemSetting; +import software.amazon.awssdk.profiles.ProfileProperty; +import software.amazon.awssdk.utils.OptionalUtils; + +/** + * Resolves the retry max attempts from {@link SdkSystemSetting#AWS_MAX_ATTEMPTS} and {@link ProfileProperty#MAX_ATTEMPTS}. + */ +@SdkInternalApi +public class MaxAttemptsResolver { + private Supplier profileFile; + private String profileName; + + /** + * Configure the profile file that should be used when determining the max attempts. The supplier is only consulted + * if a higher-priority determinant (e.g. environment variables) does not find the setting. + */ + public MaxAttemptsResolver profileFile(Supplier profileFile) { + this.profileFile = profileFile; + return this; + } + + /** + * Configure the profile file name should be used when determining the max attempts. + */ + public MaxAttemptsResolver profileName(String profileName) { + this.profileName = profileName; + return this; + } + + /** + * Resolve the max attempts based on the configured values. If not configured, returns {@code null}. + */ + public Integer resolve() { + return OptionalUtils.firstPresent(fromSystemSettings(), () -> fromProfileFile(profileFile, profileName)) + .orElse(null); + } + + + private static Optional fromSystemSettings() { + return SdkSystemSetting.AWS_MAX_ATTEMPTS.getIntegerValue(); + } + + private static Optional fromProfileFile(Supplier profileFile, String profileName) { + profileFile = profileFile != null ? profileFile : ProfileFile::defaultProfileFile; + profileName = profileName != null ? profileName : ProfileFileSystemSetting.AWS_PROFILE.getStringValueOrThrow(); + return profileFile.get() + .profile(profileName) + .flatMap(p -> p.property(ProfileProperty.MAX_ATTEMPTS)) + .map(Integer::parseInt); + } +} diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/retry/SdkDefaultRetryStrategy.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/retry/SdkDefaultRetryStrategy.java index ea4aae5e1869..c31de0a7d4db 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/retry/SdkDefaultRetryStrategy.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/retry/SdkDefaultRetryStrategy.java @@ -70,9 +70,19 @@ public static RetryStrategy defaultRetryStrategy() { * @return the appropriate retry strategy for the retry mode with AWS-specific conditions added. */ public static RetryStrategy forRetryMode(RetryMode mode) { + return forRetryMode(mode, false); + } + + /** + * Retrieve the appropriate retry strategy for the retry mode with AWS-specific conditions added. + * + * @param mode The retry mode for which we want the retry strategy + * @return the appropriate retry strategy for the retry mode with AWS-specific conditions added. + */ + public static RetryStrategy forRetryMode(RetryMode mode, boolean newRetries2026Enabled) { switch (mode) { case STANDARD: - return standardRetryStrategy(); + return standardRetryStrategy(newRetries2026Enabled); case ADAPTIVE: return legacyAdaptiveRetryStrategy(); case ADAPTIVE_V2: @@ -115,6 +125,10 @@ public static StandardRetryStrategy standardRetryStrategy() { return standardRetryStrategyBuilder().build(); } + public static StandardRetryStrategy standardRetryStrategy(boolean newRetries2026Enabled) { + return standardRetryStrategyBuilder(newRetries2026Enabled).build(); + } + /** * Returns a {@link LegacyRetryStrategy} with generic SDK retry conditions. * @@ -149,7 +163,7 @@ public static StandardRetryStrategy.Builder standardRetryStrategyBuilder() { */ public static StandardRetryStrategy.Builder standardRetryStrategyBuilder(boolean newRetries2026Enabled) { StandardRetryStrategy.Builder builder = DefaultRetryStrategy.standardStrategyBuilder(newRetries2026Enabled); - return configure(builder); + return configure(builder, newRetries2026Enabled); } @@ -179,7 +193,7 @@ public static AdaptiveRetryStrategy.Builder adaptiveRetryStrategyBuilder() { */ public static AdaptiveRetryStrategy.Builder adaptiveRetryStrategyBuilder(boolean newRetries2026Enabled) { AdaptiveRetryStrategy.Builder builder = DefaultRetryStrategy.adaptiveStrategyBuilder(newRetries2026Enabled); - return configure(builder); + return configure(builder, newRetries2026Enabled); } /** @@ -190,13 +204,17 @@ public static AdaptiveRetryStrategy.Builder adaptiveRetryStrategyBuilder(boolean * @return The given builder */ public static > T configure(T builder) { + return configure(builder, false); + } + + private static > T configure(T builder, boolean newRetries2026Enabled) { builder.retryOnException(SdkDefaultRetryStrategy::retryOnRetryableException) .retryOnException(SdkDefaultRetryStrategy::retryOnStatusCodes) .retryOnException(SdkDefaultRetryStrategy::retryOnClockSkewException) .retryOnException(SdkDefaultRetryStrategy::retryOnThrottlingCondition); SdkDefaultRetrySetting.RETRYABLE_EXCEPTIONS.forEach(builder::retryOnExceptionOrCauseInstanceOf); builder.treatAsThrottling(SdkDefaultRetryStrategy::treatAsThrottling); - Integer maxAttempts = SdkSystemSetting.AWS_MAX_ATTEMPTS.getIntegerValue().orElse(null); + Integer maxAttempts = resolveMaxAttempts(newRetries2026Enabled); if (maxAttempts != null) { builder.maxAttempts(maxAttempts); } @@ -281,5 +299,13 @@ private static void markDefaultsAdded(RetryStrategy.Builder builder) { } } + static Integer resolveMaxAttempts(boolean newRetries2026Enabled) { + if (newRetries2026Enabled) { + return new MaxAttemptsResolver().resolve(); + } + + // pre 2.1 changes, we never looked at the profile file + return SdkSystemSetting.AWS_MAX_ATTEMPTS.getIntegerValue().orElse(null); + } } diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/retry/RetryStrategyMaxRetriesTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/retry/RetryStrategyMaxRetriesTest.java index b712a9ce4c4c..bc07c151bc76 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/retry/RetryStrategyMaxRetriesTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/retry/RetryStrategyMaxRetriesTest.java @@ -18,6 +18,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import com.google.common.base.Supplier; import java.nio.file.Files; import java.nio.file.Paths; import java.util.Arrays; @@ -31,6 +32,7 @@ import software.amazon.awssdk.core.SdkSystemSetting; import software.amazon.awssdk.core.internal.retry.SdkDefaultRetryStrategy; import software.amazon.awssdk.profiles.ProfileFileSystemSetting; +import software.amazon.awssdk.retries.api.RetryStrategy; import software.amazon.awssdk.testutils.EnvironmentVariableHelper; import software.amazon.awssdk.utils.Validate; @@ -49,6 +51,7 @@ public static Collection data() { new TestData(null, null, null, null, null, 4), new TestData(null, null, null, null, "PropertyNotSet", 4), + // Test precedence new TestData("9", "2", "standard", "standard", "PropertySetToStandard", 9), @@ -60,6 +63,9 @@ public static Collection data() { "PropertySetToStandard", 3), new TestData(null, null, null, null, "PropertySetToStandard", 3), + // pre v2.1, we didn't look at the max_attempts profile file property + new TestData(null, null, null, null, + "PropertySetMaxAttempts10", 4), // Test invalid values new TestData("wrongValue", null, null, null, null, null), @@ -68,6 +74,31 @@ public static Collection data() { new TestData(null, null, null, "wrongValue", null, null), new TestData(null, null, null, null, "PropertySetToUnsupportedValue", null), + + // v2.1 + + // defaults + new TestData(true, null, null, null, null, null, 3), + new TestData(true, null, null, null, null, "PropertyNotSet", 3), + + // precedence + new TestData(true, "9", null, null, null, + "PropertySetMaxAttempts10", 9), + new TestData(true, null, "8", null, null, + "PropertySetMaxAttempts10", 8), + new TestData(true, "9", "8", null, null, + "PropertySetMaxAttempts10", 9), + new TestData(true, null, null, null, null, + "PropertySetMaxAttempts10", 10), + + // invalid values + new TestData(true, "wrongValue", null, null, null, null, null), + new TestData(true, null, "wrongValue", null, null, null, null), + new TestData(true, null, null, "wrongValue", null, null, null), + new TestData(true, null, null, null, "wrongValue", null, null), + new TestData(true, null, null, null, null, + "PropertySetToUnsupportedValue", null), + }); } @@ -85,10 +116,15 @@ public void methodSetup() { System.clearProperty(SdkSystemSetting.AWS_RETRY_MODE.property()); System.clearProperty(ProfileFileSystemSetting.AWS_PROFILE.property()); System.clearProperty(ProfileFileSystemSetting.AWS_CONFIG_FILE.property()); + System.clearProperty(SdkSystemSetting.AWS_NEW_RETRIES_2026.property()); } @Test public void differentCombinationOfConfigs_shouldResolveCorrectly() { + if (testData.newRetries2026Enabled != null) { + System.setProperty(SdkSystemSetting.AWS_NEW_RETRIES_2026.property(), testData.newRetries2026Enabled.toString()); + } + if (testData.attemptCountEnvVarValue != null) { ENVIRONMENT_VARIABLE_HELPER.set(SdkSystemSetting.AWS_MAX_ATTEMPTS.environmentVariable(), testData.attemptCountEnvVarValue); @@ -113,10 +149,19 @@ public void differentCombinationOfConfigs_shouldResolveCorrectly() { System.setProperty(ProfileFileSystemSetting.AWS_CONFIG_FILE.property(), diskLocationForFile); } + Supplier retryStrategySupplier; + + if (testData.newRetries2026Enabled != null) { + retryStrategySupplier = () -> SdkDefaultRetryStrategy.forRetryMode(RetryMode.defaultRetryMode(), + testData.newRetries2026Enabled); + } else { + retryStrategySupplier = () -> SdkDefaultRetryStrategy.forRetryMode(RetryMode.defaultRetryMode()); + } + if (testData.expected == null) { - assertThatThrownBy(() -> SdkDefaultRetryStrategy.forRetryMode(RetryMode.defaultRetryMode())).isInstanceOf(RuntimeException.class); + assertThatThrownBy(retryStrategySupplier::get).isInstanceOf(RuntimeException.class); } else { - assertThat(SdkDefaultRetryStrategy.forRetryMode(RetryMode.defaultRetryMode()).maxAttempts()).isEqualTo(testData.expected); + assertThat(retryStrategySupplier.get().maxAttempts()).isEqualTo(testData.expected); } } @@ -125,6 +170,7 @@ private String diskLocationForConfig(String configFileName) { } private static class TestData { + private final Boolean newRetries2026Enabled; private final String attemptCountSystemProperty; private final String attemptCountEnvVarValue; private final String envVarValue; @@ -138,6 +184,23 @@ private static class TestData { String retryModeEnvVarValue, String configFile, Integer expected) { + this(null, + attemptCountSystemProperty, + attemptCountEnvVarValue, + retryModeSystemProperty, + retryModeEnvVarValue, + configFile, + expected); + } + + TestData(Boolean newRetries2026Enabled, + String attemptCountSystemProperty, + String attemptCountEnvVarValue, + String retryModeSystemProperty, + String retryModeEnvVarValue, + String configFile, + Integer expected) { + this.newRetries2026Enabled = newRetries2026Enabled; this.attemptCountSystemProperty = attemptCountSystemProperty; this.attemptCountEnvVarValue = attemptCountEnvVarValue; this.envVarValue = retryModeEnvVarValue;