From 38158547e63f159aee55ebf6afd53790daf4788a Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Mon, 8 Dec 2025 14:50:05 +0300 Subject: [PATCH 01/23] fix: added limiters for retry --- .../logic/PaymentExternalServiceImpl.kt | 124 ++++++++---------- 1 file changed, 55 insertions(+), 69 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index 598992345..46683b7f7 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -3,12 +3,7 @@ package ru.quipy.payments.logic import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.registerKotlinModule import io.micrometer.core.instrument.MeterRegistry -import kotlinx.coroutines.CoroutineDispatcher -import kotlinx.coroutines.CoroutineExceptionHandler -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.SupervisorJob -import kotlinx.coroutines.asCoroutineDispatcher -import kotlinx.coroutines.launch +import kotlinx.coroutines.* import kotlinx.coroutines.sync.Semaphore import org.slf4j.LoggerFactory import ru.quipy.common.utils.NamedThreadFactory @@ -37,8 +32,7 @@ class PaymentExternalSystemAdapterImpl( private val paymentProviderHostPort: String, private val token: String, meterRegistry: MeterRegistry, - private val parallelLimiter: Semaphore, - private val ioDispatcher: CoroutineDispatcher = Executors.newFixedThreadPool( + private val parallelLimiter: Semaphore, ioDispatcher: CoroutineDispatcher = Executors.newFixedThreadPool( Runtime.getRuntime().availableProcessors() * 2, NamedThreadFactory("payment-io-") ).asCoroutineDispatcher() @@ -118,24 +112,6 @@ class PaymentExternalSystemAdapterImpl( throw TooManyRequestsException(retryAfterMs) } - val timeBeforeCall = now() - - if (!tryAcquire(now(), deadline - now())) { - val retryAfterMs = (requestAverageProcessingTime.toMillis() / parallelRequests * 10).coerceIn( - 10, - 100 - ) + Random.nextLong(10) - - throw TooManyRequestsException(retryAfterMs) - } - - if (!rateLimit.tickBlockingWithTimeout(deadline - now())) { - parallelLimiter.release() - - val retryAfterMs = (1000L / rateLimitPerSec * 10).coerceIn(10, 100) + Random.nextLong(10) - throw TooManyRequestsException(retryAfterMs) - } - val requestTimeout = minOf( deadline - now(), requestAverageProcessingTime.toMillis() * 2 @@ -156,7 +132,7 @@ class PaymentExternalSystemAdapterImpl( val retryCount = 0L - completeAction(retryCount, request, paymentId, transactionId, timeBeforeCall, deadline) + completeAction(retryCount, request, paymentId, transactionId, deadline) } override fun price() = properties.price @@ -180,54 +156,64 @@ class PaymentExternalSystemAdapterImpl( request: HttpRequest, paymentId: UUID, transactionId: UUID, - timeBeforeCall: Long, deadline: Long ) { + val timeBeforeCall = now() + + if (!tryAcquire(now(), deadline - now())) { + val retryAfterMs = (requestAverageProcessingTime.toMillis() / parallelRequests * 10).coerceIn( + 10, 100 + ) + Random.nextLong(10) + + throw TooManyRequestsException(retryAfterMs) + } + + if (!rateLimit.tickBlockingWithTimeout(deadline - now())) { + parallelLimiter.release() + + val retryAfterMs = (1000L / rateLimitPerSec * 10).coerceIn(10, 100) + Random.nextLong(10) + throw TooManyRequestsException(retryAfterMs) + } httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .whenComplete { response, throwable -> scope.launch { - try { - if (throwable != null) { - val e = throwable.cause - when (throwable.cause) { - is SocketTimeoutException -> { - logger.warn("[$accountName] attempt ${retryCount + 1} timeout: $paymentId", e) - } + if (throwable != null) { + val e = throwable.cause + when (throwable.cause) { + is SocketTimeoutException -> { + logger.warn("[$accountName] attempt ${retryCount + 1} timeout: $paymentId", e) + } - is InterruptedIOException -> { - logger.warn("[$accountName] interrupted: $paymentId", e) - } + is InterruptedIOException -> { + logger.warn("[$accountName] interrupted: $paymentId", e) + } - else -> { - logger.warn("[$accountName] io error: $paymentId", e) - } + else -> { + logger.warn("[$accountName] io error: $paymentId", e) } + } - if (retryCount + 1 >= 3) { + if (retryCount + 1 >= 3) { + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "Max attempts reached") + } + } else { + val backoff = ((2.0.pow(retryCount.toDouble()) * 25).toLong() + Random.nextLong(10)) + val capped = backoff.coerceAtMost(deadline - now() - 5) + if (capped <= 0) { paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "Max attempts reached") + it.logProcessing(false, now(), transactionId, "Deadline expired") } } else { - val backoff = ((2.0.pow(retryCount.toDouble()) * 25).toLong() + Random.nextLong(10)) - val capped = backoff.coerceAtMost(deadline - now() - 5) - if (capped <= 0) { - paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "Deadline expired") - } - } else { - scheduleRetry( - retryCount, - request, - paymentId, - transactionId, - timeBeforeCall, - deadline, - capped - ) - return@launch - } + parallelLimiter.release() + scheduleRetry( + retryCount, request, paymentId, transactionId, deadline, capped + ) + return@launch } - } else { + } + } else { + try { logger.warn("Free space in semaphore: {}", parallelLimiter.availablePermits) logger.info( "success in callback for payment: {}, retry count: {}, in time: {}", @@ -246,12 +232,12 @@ class PaymentExternalSystemAdapterImpl( it.logProcessing(parsed.result, now(), transactionId, parsed.message) } timer.record(now() - timeBeforeCall, TimeUnit.MILLISECONDS) + } catch (e: Exception) { + logger.error("[$accountName] Error processing payment $paymentId", e) + } finally { + startedRequests.increment() + parallelLimiter.release() } - } catch (e: Exception) { - logger.error("[$accountName] Error processing payment $paymentId", e) - } finally { - startedRequests.increment() - parallelLimiter.release() } } } @@ -259,7 +245,7 @@ class PaymentExternalSystemAdapterImpl( private fun scheduleRetry( retryCount: Long, request: HttpRequest, paymentId: UUID, - transactionId: UUID, timeBeforeCall: Long, deadline: Long, delay: Long + transactionId: UUID, deadline: Long, delay: Long ) { retryScheduler.schedule({ val remainingTime = deadline - now() @@ -286,7 +272,7 @@ class PaymentExternalSystemAdapterImpl( .POST(HttpRequest.BodyPublishers.noBody()) .build() - completeAction(retryCount + 1, newRequest, paymentId, transactionId, timeBeforeCall, deadline) + completeAction(retryCount + 1, newRequest, paymentId, transactionId, deadline) }, delay, TimeUnit.MILLISECONDS) } } From c48992f6f4ad2d47762c38fa7ca5c76b148f155e Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Mon, 8 Dec 2025 15:07:56 +0300 Subject: [PATCH 02/23] fix: fixed retry, removed from tick logic Thread.sleep in corutines --- .../logic/PaymentExternalServiceImpl.kt | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index 46683b7f7..fe3a795a5 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -5,6 +5,7 @@ import com.fasterxml.jackson.module.kotlin.registerKotlinModule import io.micrometer.core.instrument.MeterRegistry import kotlinx.coroutines.* import kotlinx.coroutines.sync.Semaphore +import okio.EOFException import org.slf4j.LoggerFactory import ru.quipy.common.utils.NamedThreadFactory import ru.quipy.common.utils.SlidingWindowRateLimiter @@ -168,7 +169,7 @@ class PaymentExternalSystemAdapterImpl( throw TooManyRequestsException(retryAfterMs) } - if (!rateLimit.tickBlockingWithTimeout(deadline - now())) { + if (!rateLimit.tick()) { parallelLimiter.release() val retryAfterMs = (1000L / rateLimitPerSec * 10).coerceIn(10, 100) + Random.nextLong(10) @@ -179,6 +180,7 @@ class PaymentExternalSystemAdapterImpl( scope.launch { if (throwable != null) { val e = throwable.cause + var isRetriable = true when (throwable.cause) { is SocketTimeoutException -> { logger.warn("[$accountName] attempt ${retryCount + 1} timeout: $paymentId", e) @@ -188,8 +190,13 @@ class PaymentExternalSystemAdapterImpl( logger.warn("[$accountName] interrupted: $paymentId", e) } + is EOFException -> { + logger.warn("[$accountName] eof exception in: $paymentId", e) + } + else -> { logger.warn("[$accountName] io error: $paymentId", e) + isRetriable = false } } @@ -205,11 +212,17 @@ class PaymentExternalSystemAdapterImpl( it.logProcessing(false, now(), transactionId, "Deadline expired") } } else { - parallelLimiter.release() - scheduleRetry( - retryCount, request, paymentId, transactionId, deadline, capped - ) - return@launch + if (isRetriable) { + parallelLimiter.release() + scheduleRetry( + retryCount, request, paymentId, transactionId, deadline, capped + ) + return@launch + } else { + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "Non-retriable exception") + } + } } } } else { From f7d396ff15d6fc087f675669f8e9b4e0e2ce979f Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Mon, 8 Dec 2025 19:19:16 +0300 Subject: [PATCH 03/23] hotfix --- .../kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index fe3a795a5..376cfe720 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -272,7 +272,6 @@ class PaymentExternalSystemAdapterImpl( logger.error("[$accountName] Failed to record retry failure for $paymentId", e) } finally { startedRequests.increment() - parallelLimiter.release() } } return@schedule From 58a49aaffa5d4faa17ed1f40b98cd04425609e71 Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Mon, 8 Dec 2025 19:23:49 +0300 Subject: [PATCH 04/23] hotfix --- .../ru/quipy/payments/logic/PaymentExternalServiceImpl.kt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index 376cfe720..286270eb2 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -131,9 +131,9 @@ class PaymentExternalSystemAdapterImpl( .POST(HttpRequest.BodyPublishers.noBody()) .build() - val retryCount = 0L - completeAction(retryCount, request, paymentId, transactionId, deadline) + + completeAction(0, request, paymentId, transactionId, deadline) } override fun price() = properties.price @@ -222,6 +222,7 @@ class PaymentExternalSystemAdapterImpl( paymentESService.update(paymentId) { it.logProcessing(false, now(), transactionId, "Non-retriable exception") } + parallelLimiter.release() } } } From f33c69c915bd6c30f59b316b4e12179585b055e2 Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Mon, 8 Dec 2025 19:30:23 +0300 Subject: [PATCH 05/23] hotfix --- .../payments/logic/PaymentExternalServiceImpl.kt | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index 286270eb2..6bca771e6 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -160,7 +160,10 @@ class PaymentExternalSystemAdapterImpl( deadline: Long ) { val timeBeforeCall = now() - + if (!rateLimit.tick()) { + val retryAfterMs = (1000L / rateLimitPerSec * 10).coerceIn(10, 100) + Random.nextLong(10) + throw TooManyRequestsException(retryAfterMs) + } if (!tryAcquire(now(), deadline - now())) { val retryAfterMs = (requestAverageProcessingTime.toMillis() / parallelRequests * 10).coerceIn( 10, 100 @@ -168,13 +171,6 @@ class PaymentExternalSystemAdapterImpl( throw TooManyRequestsException(retryAfterMs) } - - if (!rateLimit.tick()) { - parallelLimiter.release() - - val retryAfterMs = (1000L / rateLimitPerSec * 10).coerceIn(10, 100) + Random.nextLong(10) - throw TooManyRequestsException(retryAfterMs) - } httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .whenComplete { response, throwable -> scope.launch { From 3c36bb0e05536ef1a48f5fa62909ef0228978dcf Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Mon, 8 Dec 2025 19:43:01 +0300 Subject: [PATCH 06/23] hotfix --- .../logic/PaymentExternalServiceImpl.kt | 54 ++++++++----------- 1 file changed, 22 insertions(+), 32 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index 6bca771e6..e4859f60e 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -74,10 +74,6 @@ class PaymentExternalSystemAdapterImpl( .version(HttpClient.Version.HTTP_2) .build() - private val retryScheduler = Executors.newScheduledThreadPool( - Runtime.getRuntime().availableProcessors() - ) - override fun performPaymentAsync(paymentId: UUID, amount: Int, paymentStartedAt: Long, deadline: Long) { logger.warn("[$accountName] Submitting payment request for payment $paymentId") val transactionId = UUID.randomUUID() @@ -133,7 +129,7 @@ class PaymentExternalSystemAdapterImpl( - completeAction(0, request, paymentId, transactionId, deadline) + scope.launch { completeAction(0, request, paymentId, transactionId, deadline) } } override fun price() = properties.price @@ -142,17 +138,17 @@ class PaymentExternalSystemAdapterImpl( override fun name() = properties.accountName - fun tryAcquire(startedAt: Long, remaining: Long): Boolean { + suspend fun tryAcquire(startedAt: Long, remaining: Long): Boolean { var isAcquired = parallelLimiter.tryAcquire() while (!isAcquired && now() - startedAt < remaining) { isAcquired = parallelLimiter.tryAcquire() - Thread.sleep(1) + delay(2) } return isAcquired } - fun completeAction( + suspend fun completeAction( retryCount: Long, request: HttpRequest, paymentId: UUID, @@ -173,7 +169,6 @@ class PaymentExternalSystemAdapterImpl( } httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .whenComplete { response, throwable -> - scope.launch { if (throwable != null) { val e = throwable.cause var isRetriable = true @@ -213,7 +208,6 @@ class PaymentExternalSystemAdapterImpl( scheduleRetry( retryCount, request, paymentId, transactionId, deadline, capped ) - return@launch } else { paymentESService.update(paymentId) { it.logProcessing(false, now(), transactionId, "Non-retriable exception") @@ -249,7 +243,6 @@ class PaymentExternalSystemAdapterImpl( parallelLimiter.release() } } - } } } @@ -257,32 +250,29 @@ class PaymentExternalSystemAdapterImpl( retryCount: Long, request: HttpRequest, paymentId: UUID, transactionId: UUID, deadline: Long, delay: Long ) { - retryScheduler.schedule({ + scope.launch { + delay(delay) val remainingTime = deadline - now() if (remainingTime < requestAverageProcessingTime.toMillis()) { - scope.launch { - try { - paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "Not enough time for retry") - } - } catch (e: Exception) { - logger.error("[$accountName] Failed to record retry failure for $paymentId", e) - } finally { - startedRequests.increment() + try { + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "Not enough time for retry") } + } catch (e: Exception) { + logger.error("[$accountName] Failed to record retry failure for $paymentId", e) + } finally { + startedRequests.increment() } - return@schedule + } else { + val newRequestTimeout = remainingTime.coerceIn(100, requestAverageProcessingTime.toMillis() * 2) + val newRequest = HttpRequest.newBuilder() + .uri(request.uri()) + .timeout(Duration.ofMillis(newRequestTimeout)) + .POST(HttpRequest.BodyPublishers.noBody()) + .build() + completeAction(retryCount + 1, newRequest, paymentId, transactionId, deadline) } - - val newRequestTimeout = remainingTime.coerceIn(100, requestAverageProcessingTime.toMillis() * 2) - val newRequest = HttpRequest.newBuilder() - .uri(request.uri()) - .timeout(Duration.ofMillis(newRequestTimeout)) - .POST(HttpRequest.BodyPublishers.noBody()) - .build() - - completeAction(retryCount + 1, newRequest, paymentId, transactionId, deadline) - }, delay, TimeUnit.MILLISECONDS) + } } } From b728d7183795c854607b24995492bc036ddae830 Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Mon, 8 Dec 2025 19:56:12 +0300 Subject: [PATCH 07/23] hotfix --- .../ru/quipy/payments/logic/PaymentExternalServiceImpl.kt | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index e4859f60e..d21cf308c 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -63,7 +63,7 @@ class PaymentExternalSystemAdapterImpl( private val rateLimit: SlidingWindowRateLimiter by lazy { SlidingWindowRateLimiter( - rate = (rateLimitPerSec * 0.95).toLong(), + rate = rateLimitPerSec.toLong(), window = Duration.ofMillis(1000), ) } @@ -116,7 +116,6 @@ class PaymentExternalSystemAdapterImpl( if (requestTimeout < requestAverageProcessingTime.toMillis()) { logger.warn("[$accountName] Timeout too short for payment $paymentId: ${requestTimeout}ms") - parallelLimiter.release() val retryAfterMs = requestAverageProcessingTime.toMillis() - requestTimeout + Random.nextLong(100) throw TooManyRequestsException(retryAfterMs) } @@ -167,6 +166,7 @@ class PaymentExternalSystemAdapterImpl( throw TooManyRequestsException(retryAfterMs) } + startedRequests.increment() httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .whenComplete { response, throwable -> if (throwable != null) { @@ -195,6 +195,7 @@ class PaymentExternalSystemAdapterImpl( paymentESService.update(paymentId) { it.logProcessing(false, now(), transactionId, "Max attempts reached") } + parallelLimiter.release() } else { val backoff = ((2.0.pow(retryCount.toDouble()) * 25).toLong() + Random.nextLong(10)) val capped = backoff.coerceAtMost(deadline - now() - 5) @@ -239,7 +240,6 @@ class PaymentExternalSystemAdapterImpl( } catch (e: Exception) { logger.error("[$accountName] Error processing payment $paymentId", e) } finally { - startedRequests.increment() parallelLimiter.release() } } @@ -260,8 +260,6 @@ class PaymentExternalSystemAdapterImpl( } } catch (e: Exception) { logger.error("[$accountName] Failed to record retry failure for $paymentId", e) - } finally { - startedRequests.increment() } } else { val newRequestTimeout = remainingTime.coerceIn(100, requestAverageProcessingTime.toMillis() * 2) From e5a7053e256bcb4054c605bbd35e557885d929b6 Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Mon, 8 Dec 2025 20:07:05 +0300 Subject: [PATCH 08/23] hotfix --- .../ru/quipy/payments/logic/PaymentExternalServiceImpl.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index d21cf308c..2b7ea2fa7 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -99,7 +99,7 @@ class PaymentExternalSystemAdapterImpl( logger.info("[$accountName] Submit: $paymentId , txId: $transactionId") val remaining = deadline - now() - val minRequiredTime = requestAverageProcessingTime.toMillis() * 2 + val minRequiredTime = requestAverageProcessingTime.toMillis() if (remaining < minRequiredTime) { logger.warn("[$accountName] Not enough time for payment $paymentId: ${remaining}ms remaining, need ${minRequiredTime}ms") paymentESService.update(paymentId) { @@ -111,7 +111,7 @@ class PaymentExternalSystemAdapterImpl( val requestTimeout = minOf( deadline - now(), - requestAverageProcessingTime.toMillis() * 2 + requestAverageProcessingTime.toMillis() ).coerceAtLeast(100) if (requestTimeout < requestAverageProcessingTime.toMillis()) { From fa252a5d29d36a42f6b6252827069e0941870c9f Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Fri, 12 Dec 2025 18:03:13 +0300 Subject: [PATCH 09/23] hotfix --- .../kotlin/ru/quipy/OnlineShopApplication.kt | 4 +- .../apigateway/GlobalExceptionHandler.kt | 3 +- .../payments/config/PaymentAccountsConfig.kt | 15 +++++- .../ru/quipy/payments/logic/OrderPayer.kt | 32 ++++++++--- .../logic/PaymentExternalServiceImpl.kt | 54 ++++++------------- 5 files changed, 57 insertions(+), 51 deletions(-) diff --git a/src/main/kotlin/ru/quipy/OnlineShopApplication.kt b/src/main/kotlin/ru/quipy/OnlineShopApplication.kt index 1fcd3f89c..bb8d5464e 100644 --- a/src/main/kotlin/ru/quipy/OnlineShopApplication.kt +++ b/src/main/kotlin/ru/quipy/OnlineShopApplication.kt @@ -14,10 +14,10 @@ class OnlineShopApplication { val log: Logger = LoggerFactory.getLogger(OnlineShopApplication::class.java) companion object { - val appExecutor = Executors.newFixedThreadPool(20_000, NamedThreadFactory("main-app-executor")).asCoroutineDispatcher() + val appExecutor = Executors.newFixedThreadPool(64, NamedThreadFactory("main-app-executor")).asCoroutineDispatcher() } } -suspend fun main(args: Array) { +fun main(args: Array) { runApplication(*args) } diff --git a/src/main/kotlin/ru/quipy/apigateway/GlobalExceptionHandler.kt b/src/main/kotlin/ru/quipy/apigateway/GlobalExceptionHandler.kt index ea33b9e32..7d2c38fe8 100644 --- a/src/main/kotlin/ru/quipy/apigateway/GlobalExceptionHandler.kt +++ b/src/main/kotlin/ru/quipy/apigateway/GlobalExceptionHandler.kt @@ -29,10 +29,9 @@ class GlobalExceptionHandler( @ExceptionHandler(TooManyRequestsException::class) fun handleTooManyRequestsRetriable(exception: TooManyRequestsException): ResponseEntity { - val retryAfterSeconds = (exception.retryAfterMs / 1000.0).coerceAtLeast(0.1) return ResponseEntity .status(HttpStatus.TOO_MANY_REQUESTS) - .header("Retry-After", retryAfterSeconds.toString()) + .header("Retry-After", "10") .body("too many requests") } } \ No newline at end of file diff --git a/src/main/kotlin/ru/quipy/payments/config/PaymentAccountsConfig.kt b/src/main/kotlin/ru/quipy/payments/config/PaymentAccountsConfig.kt index 8012d1ee7..68981f838 100644 --- a/src/main/kotlin/ru/quipy/payments/config/PaymentAccountsConfig.kt +++ b/src/main/kotlin/ru/quipy/payments/config/PaymentAccountsConfig.kt @@ -4,10 +4,10 @@ import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule import com.fasterxml.jackson.module.kotlin.registerKotlinModule import io.micrometer.core.instrument.MeterRegistry -import kotlinx.coroutines.sync.Semaphore import org.springframework.beans.factory.annotation.Value import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration +import ru.quipy.common.utils.SlidingWindowRateLimiter import ru.quipy.core.EventSourcingService import ru.quipy.payments.api.PaymentAggregate import ru.quipy.payments.logic.PaymentAccountProperties @@ -18,7 +18,9 @@ import java.net.URI import java.net.http.HttpClient import java.net.http.HttpRequest import java.net.http.HttpResponse +import java.time.Duration import java.util.* +import java.util.concurrent.Semaphore @Configuration @@ -40,10 +42,18 @@ class PaymentAccountsConfig { @Value("#{'\${payment.accounts}'.split(',')}") lateinit var allowedAccounts: List + @Bean + fun rateLimit(): SlidingWindowRateLimiter { + return SlidingWindowRateLimiter( + rate = 1100L, + window = Duration.ofMillis(1000), + ) + } @Bean fun accountAdapters( paymentService: EventSourcingService, meterRegistry: MeterRegistry, + rateLimiter: SlidingWindowRateLimiter ): List { val request = HttpRequest.newBuilder() .uri(URI("http://${paymentProviderHostPort}/external/accounts?serviceName=$serviceName&token=$token")) @@ -67,7 +77,8 @@ class PaymentAccountsConfig { paymentProviderHostPort, token, meterRegistry, - Semaphore(it.parallelRequests) + Semaphore(it.parallelRequests), + rateLimiter ) } } diff --git a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt index e2bd9e203..50b1f81bf 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt @@ -5,12 +5,19 @@ import org.slf4j.Logger import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service +import ru.quipy.common.utils.NamedThreadFactory +import ru.quipy.common.utils.SlidingWindowRateLimiter import ru.quipy.core.EventSourcingService +import ru.quipy.exceptions.TooManyRequestsException import ru.quipy.payments.api.PaymentAggregate import java.util.* +import java.util.concurrent.LinkedBlockingQueue +import java.util.concurrent.ThreadPoolExecutor +import java.util.concurrent.TimeUnit +import kotlin.random.Random @Service -class OrderPayer(meterRegistry: MeterRegistry) { +class OrderPayer(val rateLimiter : SlidingWindowRateLimiter, meterRegistry: MeterRegistry) { companion object { val logger: Logger = LoggerFactory.getLogger(OrderPayer::class.java) @@ -18,6 +25,14 @@ class OrderPayer(meterRegistry: MeterRegistry) { private val plannedRequests = meterRegistry.counter("payment.processing.planned", "accountName", "acc-12") + private val paymentExecutor = ThreadPoolExecutor( + 50, + 50, + 100L, + TimeUnit.MILLISECONDS, + LinkedBlockingQueue(40_000), + NamedThreadFactory("payment-submission-executor") + ) @Autowired private lateinit var paymentESService: EventSourcingService @@ -28,8 +43,13 @@ class OrderPayer(meterRegistry: MeterRegistry) { fun processPayment(orderId: UUID, amount: Int, paymentId: UUID, deadline: Long): Long { val createdAt = System.currentTimeMillis() - plannedRequests.increment() - val createdEvent = paymentESService.create { + if (!rateLimiter.tick()) { + val retryAfterMs = (1000L / 1100 * 10).coerceIn(10, 100) + Random.nextLong(10) + throw TooManyRequestsException(retryAfterMs) + } + paymentExecutor.submit { + plannedRequests.increment() + val createdEvent = paymentESService.create { it.create( paymentId, orderId, @@ -37,10 +57,10 @@ class OrderPayer(meterRegistry: MeterRegistry) { ) } - logger.trace("Payment {} for order {} created.", createdEvent.paymentId, orderId) - - paymentService.submitPaymentRequest(paymentId, amount, createdAt, deadline) + logger.trace("Payment {} for order {} created.", createdEvent.paymentId, orderId) + paymentService.submitPaymentRequest(paymentId, amount, createdAt, deadline) + } return createdAt } } \ No newline at end of file diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index 2b7ea2fa7..a9a92b465 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -4,10 +4,8 @@ import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.registerKotlinModule import io.micrometer.core.instrument.MeterRegistry import kotlinx.coroutines.* -import kotlinx.coroutines.sync.Semaphore import okio.EOFException import org.slf4j.LoggerFactory -import ru.quipy.common.utils.NamedThreadFactory import ru.quipy.common.utils.SlidingWindowRateLimiter import ru.quipy.core.EventSourcingService import ru.quipy.exceptions.TooManyRequestsException @@ -21,6 +19,7 @@ import java.net.http.HttpResponse import java.time.Duration import java.util.* import java.util.concurrent.Executors +import java.util.concurrent.Semaphore import java.util.concurrent.TimeUnit import kotlin.math.pow import kotlin.random.Random @@ -33,10 +32,8 @@ class PaymentExternalSystemAdapterImpl( private val paymentProviderHostPort: String, private val token: String, meterRegistry: MeterRegistry, - private val parallelLimiter: Semaphore, ioDispatcher: CoroutineDispatcher = Executors.newFixedThreadPool( - Runtime.getRuntime().availableProcessors() * 2, - NamedThreadFactory("payment-io-") - ).asCoroutineDispatcher() + private val parallelLimiter: Semaphore, + private val rateLimiter: SlidingWindowRateLimiter ) : PaymentExternalSystemAdapter { companion object { val logger = LoggerFactory.getLogger(PaymentExternalSystemAdapter::class.java) @@ -47,8 +44,6 @@ class PaymentExternalSystemAdapterImpl( logger.error("[$accountName] Unhandled exception in payment adapter coroutine", throwable) } - private val scope = CoroutineScope(SupervisorJob() + ioDispatcher + exceptionHandler) - // 2025-11-20T20:30:35.780+03:00 INFO 56644 --- [alhost:1234/...] ru.quipy.core.EventSourcingService : Optimistic lock exception. Failed to save event records id: [7dca693e-e811-4b7f-8bce-23e13d952c04-4] private val startedRequests = @@ -61,12 +56,7 @@ class PaymentExternalSystemAdapterImpl( private val rateLimitPerSec = properties.rateLimitPerSec private val parallelRequests = properties.parallelRequests - private val rateLimit: SlidingWindowRateLimiter by lazy { - SlidingWindowRateLimiter( - rate = rateLimitPerSec.toLong(), - window = Duration.ofMillis(1000), - ) - } + private val httpClient = HttpClient .newBuilder() @@ -80,7 +70,6 @@ class PaymentExternalSystemAdapterImpl( // Вне зависимости от исхода оплаты важно отметить что она была отправлена. // Это требуется сделать ВО ВСЕХ СЛУЧАЯХ, поскольку эта информация используется сервисом тестирования. - scope.launch { try { paymentESService.update(paymentId) { it.logSubmission( @@ -94,7 +83,6 @@ class PaymentExternalSystemAdapterImpl( } catch (e: Exception) { logger.error("[$accountName] Failed to record log submission for $paymentId", e) } - } logger.info("[$accountName] Submit: $paymentId , txId: $transactionId") @@ -126,9 +114,7 @@ class PaymentExternalSystemAdapterImpl( .POST(HttpRequest.BodyPublishers.noBody()) .build() - - - scope.launch { completeAction(0, request, paymentId, transactionId, deadline) } + completeAction(0, request, paymentId, transactionId, deadline) } override fun price() = properties.price @@ -137,17 +123,7 @@ class PaymentExternalSystemAdapterImpl( override fun name() = properties.accountName - suspend fun tryAcquire(startedAt: Long, remaining: Long): Boolean { - var isAcquired = parallelLimiter.tryAcquire() - while (!isAcquired && now() - startedAt < remaining) { - isAcquired = parallelLimiter.tryAcquire() - delay(2) - } - - return isAcquired - } - - suspend fun completeAction( + fun completeAction( retryCount: Long, request: HttpRequest, paymentId: UUID, @@ -155,11 +131,8 @@ class PaymentExternalSystemAdapterImpl( deadline: Long ) { val timeBeforeCall = now() - if (!rateLimit.tick()) { - val retryAfterMs = (1000L / rateLimitPerSec * 10).coerceIn(10, 100) + Random.nextLong(10) - throw TooManyRequestsException(retryAfterMs) - } - if (!tryAcquire(now(), deadline - now())) { + + if (!parallelLimiter.tryAcquire(deadline - now(), TimeUnit.MILLISECONDS)) { val retryAfterMs = (requestAverageProcessingTime.toMillis() / parallelRequests * 10).coerceIn( 10, 100 ) + Random.nextLong(10) @@ -219,7 +192,7 @@ class PaymentExternalSystemAdapterImpl( } } else { try { - logger.warn("Free space in semaphore: {}", parallelLimiter.availablePermits) + logger.warn("Free space in semaphore: {}", parallelLimiter.availablePermits()) logger.info( "success in callback for payment: {}, retry count: {}, in time: {}", paymentId, @@ -250,8 +223,8 @@ class PaymentExternalSystemAdapterImpl( retryCount: Long, request: HttpRequest, paymentId: UUID, transactionId: UUID, deadline: Long, delay: Long ) { - scope.launch { - delay(delay) + + Thread.sleep(delay) val remainingTime = deadline - now() if (remainingTime < requestAverageProcessingTime.toMillis()) { try { @@ -268,8 +241,11 @@ class PaymentExternalSystemAdapterImpl( .timeout(Duration.ofMillis(newRequestTimeout)) .POST(HttpRequest.BodyPublishers.noBody()) .build() + if (!rateLimiter.tick()) { + val retryAfterMs = (1000L / rateLimitPerSec * 10).coerceIn(10, 100) + Random.nextLong(10) + throw TooManyRequestsException(retryAfterMs) + } completeAction(retryCount + 1, newRequest, paymentId, transactionId, deadline) - } } } } From 661900bc8ad9a2fcfef5e8d1595db80df6e8edd8 Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Fri, 12 Dec 2025 18:03:54 +0300 Subject: [PATCH 10/23] hotfix --- src/main/kotlin/ru/quipy/OnlineShopApplication.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/ru/quipy/OnlineShopApplication.kt b/src/main/kotlin/ru/quipy/OnlineShopApplication.kt index bb8d5464e..f311c6fdf 100644 --- a/src/main/kotlin/ru/quipy/OnlineShopApplication.kt +++ b/src/main/kotlin/ru/quipy/OnlineShopApplication.kt @@ -6,6 +6,7 @@ import org.slf4j.LoggerFactory import org.springframework.boot.autoconfigure.SpringBootApplication import org.springframework.boot.runApplication import ru.quipy.common.utils.NamedThreadFactory +import java.util.concurrent.ExecutorService import java.util.concurrent.Executors @@ -14,7 +15,7 @@ class OnlineShopApplication { val log: Logger = LoggerFactory.getLogger(OnlineShopApplication::class.java) companion object { - val appExecutor = Executors.newFixedThreadPool(64, NamedThreadFactory("main-app-executor")).asCoroutineDispatcher() + val appExecutor: ExecutorService = Executors.newFixedThreadPool(64, NamedThreadFactory("main-app-executor")) } } From ba2768127e6daf597190c39dfd05691af9a7c5e5 Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Fri, 12 Dec 2025 18:14:43 +0300 Subject: [PATCH 11/23] hotfix --- src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt index 50b1f81bf..36818491e 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt @@ -26,11 +26,11 @@ class OrderPayer(val rateLimiter : SlidingWindowRateLimiter, meterRegistry: Mete private val plannedRequests = meterRegistry.counter("payment.processing.planned", "accountName", "acc-12") private val paymentExecutor = ThreadPoolExecutor( - 50, - 50, + 1000, + 1000, 100L, TimeUnit.MILLISECONDS, - LinkedBlockingQueue(40_000), + LinkedBlockingQueue(100_000), NamedThreadFactory("payment-submission-executor") ) From e7f47b749fbdae688d1df101d9c0a4844ca38421 Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Fri, 12 Dec 2025 18:39:22 +0300 Subject: [PATCH 12/23] hotfix --- src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt index 36818491e..e70e67a76 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt @@ -26,8 +26,8 @@ class OrderPayer(val rateLimiter : SlidingWindowRateLimiter, meterRegistry: Mete private val plannedRequests = meterRegistry.counter("payment.processing.planned", "accountName", "acc-12") private val paymentExecutor = ThreadPoolExecutor( - 1000, - 1000, + 2000, + 2000, 100L, TimeUnit.MILLISECONDS, LinkedBlockingQueue(100_000), From ca635e3cdbc4a1b5e961e65bb897dd87f75c4340 Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Fri, 12 Dec 2025 18:48:14 +0300 Subject: [PATCH 13/23] hotfix --- src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt | 4 ++-- .../ru/quipy/payments/logic/PaymentExternalServiceImpl.kt | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt index e70e67a76..36818491e 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt @@ -26,8 +26,8 @@ class OrderPayer(val rateLimiter : SlidingWindowRateLimiter, meterRegistry: Mete private val plannedRequests = meterRegistry.counter("payment.processing.planned", "accountName", "acc-12") private val paymentExecutor = ThreadPoolExecutor( - 2000, - 2000, + 1000, + 1000, 100L, TimeUnit.MILLISECONDS, LinkedBlockingQueue(100_000), diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index a9a92b465..990a86c4e 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -40,6 +40,7 @@ class PaymentExternalSystemAdapterImpl( val mapper = ObjectMapper().registerKotlinModule() } + private val time_95_percentile = 20_000 private val exceptionHandler = CoroutineExceptionHandler { _, throwable -> logger.error("[$accountName] Unhandled exception in payment adapter coroutine", throwable) } @@ -98,7 +99,7 @@ class PaymentExternalSystemAdapterImpl( } val requestTimeout = minOf( - deadline - now(), + time_95_percentile.toLong(), requestAverageProcessingTime.toMillis() ).coerceAtLeast(100) From 1ca0e9d7f1d69d4559673869e212209d6f46ebfe Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Fri, 12 Dec 2025 19:06:22 +0300 Subject: [PATCH 14/23] hotfix --- src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt index 36818491e..036ed4121 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt @@ -26,11 +26,11 @@ class OrderPayer(val rateLimiter : SlidingWindowRateLimiter, meterRegistry: Mete private val plannedRequests = meterRegistry.counter("payment.processing.planned", "accountName", "acc-12") private val paymentExecutor = ThreadPoolExecutor( - 1000, - 1000, + 3600, + 3600, 100L, TimeUnit.MILLISECONDS, - LinkedBlockingQueue(100_000), + LinkedBlockingQueue(200_000), NamedThreadFactory("payment-submission-executor") ) From 8453d1447f7932ba94a64f65883b64dd7d5d5396 Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Fri, 12 Dec 2025 19:20:21 +0300 Subject: [PATCH 15/23] hotfix --- .../ru/quipy/payments/logic/PaymentExternalServiceImpl.kt | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index 990a86c4e..d14208328 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -3,7 +3,6 @@ package ru.quipy.payments.logic import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.registerKotlinModule import io.micrometer.core.instrument.MeterRegistry -import kotlinx.coroutines.* import okio.EOFException import org.slf4j.LoggerFactory import ru.quipy.common.utils.SlidingWindowRateLimiter @@ -41,9 +40,6 @@ class PaymentExternalSystemAdapterImpl( } private val time_95_percentile = 20_000 - private val exceptionHandler = CoroutineExceptionHandler { _, throwable -> - logger.error("[$accountName] Unhandled exception in payment adapter coroutine", throwable) - } // 2025-11-20T20:30:35.780+03:00 INFO 56644 --- [alhost:1234/...] ru.quipy.core.EventSourcingService : Optimistic lock exception. Failed to save event records id: [7dca693e-e811-4b7f-8bce-23e13d952c04-4] @@ -224,9 +220,9 @@ class PaymentExternalSystemAdapterImpl( retryCount: Long, request: HttpRequest, paymentId: UUID, transactionId: UUID, deadline: Long, delay: Long ) { - + parallelLimiter.release() Thread.sleep(delay) - val remainingTime = deadline - now() + val remainingTime = deadline - now() if (remainingTime < requestAverageProcessingTime.toMillis()) { try { paymentESService.update(paymentId) { From f1f589843b6bd33376d5fa72bf73b4d57cdc37ec Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Fri, 12 Dec 2025 19:39:30 +0300 Subject: [PATCH 16/23] hotfix --- .../dashboards/ServicesStatistic.json | 207 ++++++++++++++++++ .../logic/PaymentExternalServiceImpl.kt | 6 + 2 files changed, 213 insertions(+) diff --git a/grafana/provisioning/dashboards/ServicesStatistic.json b/grafana/provisioning/dashboards/ServicesStatistic.json index eb6f028d2..e30ee04a3 100644 --- a/grafana/provisioning/dashboards/ServicesStatistic.json +++ b/grafana/provisioning/dashboards/ServicesStatistic.json @@ -3727,6 +3727,213 @@ ], "title": "Payment Processing Latency", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Количество повторных запросов к платежной системе в секунду", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Запросов/сек", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 109, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(payment_request_retried_total{accountName=~\".*\"}[1m])", + "legendFormat": "Повторные запросы: {{accountName}}", + "range": true, + "refId": "A" + } + ], + "title": "Payment Request Retries Rate (per second)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Общее количество повторных запросов к платежной системе", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Количество", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 100 + }, + { + "color": "red", + "value": 500 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 110, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "payment_request_retried_total{accountName=~\".*\"}", + "legendFormat": "Всего повторов: {{accountName}}", + "range": true, + "refId": "A" + } + ], + "title": "Payment Request Retries Total Count", + "type": "timeseries" }], "preload": false, "refresh": "5s", diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index d14208328..46c9920de 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -20,6 +20,7 @@ import java.util.* import java.util.concurrent.Executors import java.util.concurrent.Semaphore import java.util.concurrent.TimeUnit +import kotlin.math.log import kotlin.math.pow import kotlin.random.Random @@ -42,9 +43,12 @@ class PaymentExternalSystemAdapterImpl( private val time_95_percentile = 20_000 + // 2025-11-20T20:30:35.780+03:00 INFO 56644 --- [alhost:1234/...] ru.quipy.core.EventSourcingService : Optimistic lock exception. Failed to save event records id: [7dca693e-e811-4b7f-8bce-23e13d952c04-4] private val startedRequests = meterRegistry.counter("payment.processing.started", "accountName", properties.accountName) + private val requestsRetried = + meterRegistry.counter("payment.request.retried", "accountName", properties.accountName) private val timer = meterRegistry.timer("payment.external.system.request.latency", "accountName", properties.accountName) private val serviceName = properties.serviceName @@ -220,6 +224,8 @@ class PaymentExternalSystemAdapterImpl( retryCount: Long, request: HttpRequest, paymentId: UUID, transactionId: UUID, deadline: Long, delay: Long ) { + requestsRetried.increment() + logger.info("Completing retry. All retry count - {}", requestsRetried.count()) parallelLimiter.release() Thread.sleep(delay) val remainingTime = deadline - now() From f7af64e68bc83d32c8fd277a67717e61b0a88f44 Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Fri, 12 Dec 2025 19:49:52 +0300 Subject: [PATCH 17/23] hotfix --- src/main/resources/application.properties | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index bdbe25f61..ab2efd0cc 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -29,4 +29,11 @@ payment.token=${PAYMENT_TOKEN} payment.accounts=${PAYMENT_ACCOUNTS:acc-12} # payment.accounts=${PAYMENT_ACCOUNTS:acc-18} payment.hostPort=${PAYMENT_HOST:localhost}:${PAYMENT_PORT:1234} +spring.datasource.hikari.maximum-pool-size=20_000 +spring.datasource.hikari.minimum-idle=50 +spring.datasource.hikari.connection-timeout=5000 +server.tomcat.threads.max=20000 +server.tomcat.threads.min-spare=100 +server.tomcat.accept-count=2000 +server.tomcat.max-connections=20000 From 8864f7841824a63ea6db7c49cef7fd80805a0045 Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Fri, 12 Dec 2025 19:51:56 +0300 Subject: [PATCH 18/23] hotfix --- src/main/resources/application.properties | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index ab2efd0cc..d95ec6218 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -32,8 +32,11 @@ payment.hostPort=${PAYMENT_HOST:localhost}:${PAYMENT_PORT:1234} spring.datasource.hikari.maximum-pool-size=20_000 spring.datasource.hikari.minimum-idle=50 spring.datasource.hikari.connection-timeout=5000 -server.tomcat.threads.max=20000 -server.tomcat.threads.min-spare=100 -server.tomcat.accept-count=2000 -server.tomcat.max-connections=20000 +server.jetty.threads.min=100 +server.jetty.threads.max=20000 +server.jetty.threads.max-queue-capacity=100000 +server.jetty.threads.acceptors=16 +server.jetty.threads.selectors=32 +server.jetty.connection-idle-timeout=60000 + From 745946e76460f0d7afd9db2497648833d8cd7837 Mon Sep 17 00:00:00 2001 From: RuReVange Date: Sat, 20 Dec 2025 14:35:17 +0300 Subject: [PATCH 19/23] fix: delete toomanyrequests --- .../kotlin/ru/quipy/apigateway/APIController.kt | 4 ---- .../logic/PaymentExternalServiceImpl.kt | 17 ----------------- 2 files changed, 21 deletions(-) diff --git a/src/main/kotlin/ru/quipy/apigateway/APIController.kt b/src/main/kotlin/ru/quipy/apigateway/APIController.kt index 251ee553e..124b8f9d6 100644 --- a/src/main/kotlin/ru/quipy/apigateway/APIController.kt +++ b/src/main/kotlin/ru/quipy/apigateway/APIController.kt @@ -60,10 +60,6 @@ class APIController( @PostMapping("/orders/{orderId}/payment") fun payOrder(@PathVariable orderId: UUID, @RequestParam deadline: Long): PaymentSubmissionDto { - if (!rateLimiter.tick()) { - val retryAfterMs = 10L + Random.nextLong(10) - throw TooManyRequestsException(retryAfterMs) - } val paymentId = UUID.randomUUID() val order = orderRepository.findById(orderId)?.let { orderRepository.save(it.copy(status = OrderStatus.PAYMENT_IN_PROGRESS)) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index 46c9920de..b9e0378b2 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -103,12 +103,6 @@ class PaymentExternalSystemAdapterImpl( requestAverageProcessingTime.toMillis() ).coerceAtLeast(100) - if (requestTimeout < requestAverageProcessingTime.toMillis()) { - logger.warn("[$accountName] Timeout too short for payment $paymentId: ${requestTimeout}ms") - val retryAfterMs = requestAverageProcessingTime.toMillis() - requestTimeout + Random.nextLong(100) - throw TooManyRequestsException(retryAfterMs) - } - val request = HttpRequest.newBuilder() .uri(URI("http://$paymentProviderHostPort/external/process?serviceName=$serviceName&token=$token&accountName=$accountName&transactionId=$transactionId&paymentId=$paymentId&amount=$amount")) .timeout(Duration.ofMillis(requestTimeout)) @@ -133,13 +127,6 @@ class PaymentExternalSystemAdapterImpl( ) { val timeBeforeCall = now() - if (!parallelLimiter.tryAcquire(deadline - now(), TimeUnit.MILLISECONDS)) { - val retryAfterMs = (requestAverageProcessingTime.toMillis() / parallelRequests * 10).coerceIn( - 10, 100 - ) + Random.nextLong(10) - - throw TooManyRequestsException(retryAfterMs) - } startedRequests.increment() httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .whenComplete { response, throwable -> @@ -244,10 +231,6 @@ class PaymentExternalSystemAdapterImpl( .timeout(Duration.ofMillis(newRequestTimeout)) .POST(HttpRequest.BodyPublishers.noBody()) .build() - if (!rateLimiter.tick()) { - val retryAfterMs = (1000L / rateLimitPerSec * 10).coerceIn(10, 100) + Random.nextLong(10) - throw TooManyRequestsException(retryAfterMs) - } completeAction(retryCount + 1, newRequest, paymentId, transactionId, deadline) } } From 6a070ea0a5cb90491a44bf20e7b67c64b0b575cb Mon Sep 17 00:00:00 2001 From: RuReVange Date: Sat, 20 Dec 2025 14:42:12 +0300 Subject: [PATCH 20/23] fix: fix --- src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt index 036ed4121..55586f9e9 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt @@ -8,13 +8,11 @@ import org.springframework.stereotype.Service import ru.quipy.common.utils.NamedThreadFactory import ru.quipy.common.utils.SlidingWindowRateLimiter import ru.quipy.core.EventSourcingService -import ru.quipy.exceptions.TooManyRequestsException import ru.quipy.payments.api.PaymentAggregate import java.util.* import java.util.concurrent.LinkedBlockingQueue import java.util.concurrent.ThreadPoolExecutor import java.util.concurrent.TimeUnit -import kotlin.random.Random @Service class OrderPayer(val rateLimiter : SlidingWindowRateLimiter, meterRegistry: MeterRegistry) { @@ -43,10 +41,6 @@ class OrderPayer(val rateLimiter : SlidingWindowRateLimiter, meterRegistry: Mete fun processPayment(orderId: UUID, amount: Int, paymentId: UUID, deadline: Long): Long { val createdAt = System.currentTimeMillis() - if (!rateLimiter.tick()) { - val retryAfterMs = (1000L / 1100 * 10).coerceIn(10, 100) + Random.nextLong(10) - throw TooManyRequestsException(retryAfterMs) - } paymentExecutor.submit { plannedRequests.increment() val createdEvent = paymentESService.create { From 999fde5108c0c8444ae019c39d764c92adafeba2 Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Fri, 26 Dec 2025 18:46:31 +0300 Subject: [PATCH 21/23] feature: added scheduled retry and returned rateLimiters --- .../ru/quipy/apigateway/APIController.kt | 6 ++++- .../ru/quipy/payments/logic/OrderPayer.kt | 5 ++++ .../logic/PaymentExternalServiceImpl.kt | 25 ++++++++++--------- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/src/main/kotlin/ru/quipy/apigateway/APIController.kt b/src/main/kotlin/ru/quipy/apigateway/APIController.kt index 124b8f9d6..c8399ad5f 100644 --- a/src/main/kotlin/ru/quipy/apigateway/APIController.kt +++ b/src/main/kotlin/ru/quipy/apigateway/APIController.kt @@ -9,9 +9,9 @@ import ru.quipy.common.utils.RateLimiter import ru.quipy.exceptions.TooManyRequestsException import ru.quipy.orders.repository.OrderRepository import ru.quipy.payments.logic.OrderPayer +import ru.quipy.payments.logic.now import java.time.Duration import java.util.* -import kotlin.random.Random @RestController class APIController( @@ -60,6 +60,10 @@ class APIController( @PostMapping("/orders/{orderId}/payment") fun payOrder(@PathVariable orderId: UUID, @RequestParam deadline: Long): PaymentSubmissionDto { + while (!rateLimiter.tick() || now() < deadline) { + } + if (now() >= deadline) + throw TooManyRequestsException(10); val paymentId = UUID.randomUUID() val order = orderRepository.findById(orderId)?.let { orderRepository.save(it.copy(status = OrderStatus.PAYMENT_IN_PROGRESS)) diff --git a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt index 55586f9e9..0dc3e0c78 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt @@ -8,6 +8,7 @@ import org.springframework.stereotype.Service import ru.quipy.common.utils.NamedThreadFactory import ru.quipy.common.utils.SlidingWindowRateLimiter import ru.quipy.core.EventSourcingService +import ru.quipy.exceptions.TooManyRequestsException import ru.quipy.payments.api.PaymentAggregate import java.util.* import java.util.concurrent.LinkedBlockingQueue @@ -41,6 +42,10 @@ class OrderPayer(val rateLimiter : SlidingWindowRateLimiter, meterRegistry: Mete fun processPayment(orderId: UUID, amount: Int, paymentId: UUID, deadline: Long): Long { val createdAt = System.currentTimeMillis() + while (!rateLimiter.tick() || now() < deadline) { + } + if (now() >= deadline) + throw TooManyRequestsException(10) paymentExecutor.submit { plannedRequests.increment() val createdEvent = paymentESService.create { diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index b9e0378b2..d934c1fdc 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -18,9 +18,9 @@ import java.net.http.HttpResponse import java.time.Duration import java.util.* import java.util.concurrent.Executors +import java.util.concurrent.ScheduledExecutorService import java.util.concurrent.Semaphore import java.util.concurrent.TimeUnit -import kotlin.math.log import kotlin.math.pow import kotlin.random.Random @@ -42,7 +42,7 @@ class PaymentExternalSystemAdapterImpl( private val time_95_percentile = 20_000 - + private val retryExecutor: ScheduledExecutorService = Executors.newScheduledThreadPool(properties.parallelRequests) // 2025-11-20T20:30:35.780+03:00 INFO 56644 --- [alhost:1234/...] ru.quipy.core.EventSourcingService : Optimistic lock exception. Failed to save event records id: [7dca693e-e811-4b7f-8bce-23e13d952c04-4] private val startedRequests = @@ -130,6 +130,7 @@ class PaymentExternalSystemAdapterImpl( startedRequests.increment() httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .whenComplete { response, throwable -> + parallelLimiter.release() if (throwable != null) { val e = throwable.cause var isRetriable = true @@ -156,7 +157,6 @@ class PaymentExternalSystemAdapterImpl( paymentESService.update(paymentId) { it.logProcessing(false, now(), transactionId, "Max attempts reached") } - parallelLimiter.release() } else { val backoff = ((2.0.pow(retryCount.toDouble()) * 25).toLong() + Random.nextLong(10)) val capped = backoff.coerceAtMost(deadline - now() - 5) @@ -166,7 +166,6 @@ class PaymentExternalSystemAdapterImpl( } } else { if (isRetriable) { - parallelLimiter.release() scheduleRetry( retryCount, request, paymentId, transactionId, deadline, capped ) @@ -174,7 +173,6 @@ class PaymentExternalSystemAdapterImpl( paymentESService.update(paymentId) { it.logProcessing(false, now(), transactionId, "Non-retriable exception") } - parallelLimiter.release() } } } @@ -200,8 +198,6 @@ class PaymentExternalSystemAdapterImpl( timer.record(now() - timeBeforeCall, TimeUnit.MILLISECONDS) } catch (e: Exception) { logger.error("[$accountName] Error processing payment $paymentId", e) - } finally { - parallelLimiter.release() } } } @@ -212,10 +208,14 @@ class PaymentExternalSystemAdapterImpl( transactionId: UUID, deadline: Long, delay: Long ) { requestsRetried.increment() - logger.info("Completing retry. All retry count - {}", requestsRetried.count()) - parallelLimiter.release() - Thread.sleep(delay) - val remainingTime = deadline - now() + retryExecutor.scheduleWithFixedDelay({ + while (!rateLimiter.tick() || now() < deadline) { + } + if (now() >= deadline) + throw TooManyRequestsException(10) + + logger.info("Completing retry. All retry count - {}", requestsRetried.count()) + val remainingTime = deadline - now() if (remainingTime < requestAverageProcessingTime.toMillis()) { try { paymentESService.update(paymentId) { @@ -232,7 +232,8 @@ class PaymentExternalSystemAdapterImpl( .POST(HttpRequest.BodyPublishers.noBody()) .build() completeAction(retryCount + 1, newRequest, paymentId, transactionId, deadline) - } + } + }, delay, delay, TimeUnit.MILLISECONDS) } } From 98d67b94b9ba4db73cf519be6c89b5e86f20d588 Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Fri, 26 Dec 2025 18:52:07 +0300 Subject: [PATCH 22/23] hotfix --- src/main/kotlin/ru/quipy/apigateway/APIController.kt | 2 +- src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt | 2 +- .../ru/quipy/payments/logic/PaymentExternalServiceImpl.kt | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/ru/quipy/apigateway/APIController.kt b/src/main/kotlin/ru/quipy/apigateway/APIController.kt index c8399ad5f..67c18a290 100644 --- a/src/main/kotlin/ru/quipy/apigateway/APIController.kt +++ b/src/main/kotlin/ru/quipy/apigateway/APIController.kt @@ -60,7 +60,7 @@ class APIController( @PostMapping("/orders/{orderId}/payment") fun payOrder(@PathVariable orderId: UUID, @RequestParam deadline: Long): PaymentSubmissionDto { - while (!rateLimiter.tick() || now() < deadline) { + while (!rateLimiter.tick() && now() < deadline) { } if (now() >= deadline) throw TooManyRequestsException(10); diff --git a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt index 0dc3e0c78..61995cdf2 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt @@ -42,7 +42,7 @@ class OrderPayer(val rateLimiter : SlidingWindowRateLimiter, meterRegistry: Mete fun processPayment(orderId: UUID, amount: Int, paymentId: UUID, deadline: Long): Long { val createdAt = System.currentTimeMillis() - while (!rateLimiter.tick() || now() < deadline) { + while (!rateLimiter.tick() && now() < deadline) { } if (now() >= deadline) throw TooManyRequestsException(10) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index d934c1fdc..14d1e7bc9 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -126,7 +126,7 @@ class PaymentExternalSystemAdapterImpl( deadline: Long ) { val timeBeforeCall = now() - + parallelLimiter.acquire() startedRequests.increment() httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .whenComplete { response, throwable -> @@ -209,7 +209,7 @@ class PaymentExternalSystemAdapterImpl( ) { requestsRetried.increment() retryExecutor.scheduleWithFixedDelay({ - while (!rateLimiter.tick() || now() < deadline) { + while (!rateLimiter.tick() && now() < deadline) { } if (now() >= deadline) throw TooManyRequestsException(10) From f8f6b05c64a2979bffae3dd77f9ba3cffdbac082 Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Fri, 26 Dec 2025 20:23:59 +0300 Subject: [PATCH 23/23] hotfix --- src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt | 2 +- .../ru/quipy/payments/logic/PaymentExternalServiceImpl.kt | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt index 61995cdf2..41326b06d 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt @@ -29,7 +29,7 @@ class OrderPayer(val rateLimiter : SlidingWindowRateLimiter, meterRegistry: Mete 3600, 100L, TimeUnit.MILLISECONDS, - LinkedBlockingQueue(200_000), + LinkedBlockingQueue(100_000), NamedThreadFactory("payment-submission-executor") ) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index 14d1e7bc9..3584b840d 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -40,7 +40,7 @@ class PaymentExternalSystemAdapterImpl( val mapper = ObjectMapper().registerKotlinModule() } - private val time_95_percentile = 20_000 + private val time_95_percentile = 20_000L private val retryExecutor: ScheduledExecutorService = Executors.newScheduledThreadPool(properties.parallelRequests) @@ -99,7 +99,7 @@ class PaymentExternalSystemAdapterImpl( } val requestTimeout = minOf( - time_95_percentile.toLong(), + time_95_percentile, requestAverageProcessingTime.toMillis() ).coerceAtLeast(100) @@ -225,7 +225,7 @@ class PaymentExternalSystemAdapterImpl( logger.error("[$accountName] Failed to record retry failure for $paymentId", e) } } else { - val newRequestTimeout = remainingTime.coerceIn(100, requestAverageProcessingTime.toMillis() * 2) + val newRequestTimeout = remainingTime.coerceIn(time_95_percentile, requestAverageProcessingTime.toMillis() * 2) val newRequest = HttpRequest.newBuilder() .uri(request.uri()) .timeout(Duration.ofMillis(newRequestTimeout)) @@ -233,7 +233,7 @@ class PaymentExternalSystemAdapterImpl( .build() completeAction(retryCount + 1, newRequest, paymentId, transactionId, deadline) } - }, delay, delay, TimeUnit.MILLISECONDS) + }, 0, delay, TimeUnit.MILLISECONDS) } }