diff --git a/grafana/provisioning/dashboards/ServicesStatistic.json b/grafana/provisioning/dashboards/ServicesStatistic.json index eb6f028d2..e30ee04a3 100644 --- a/grafana/provisioning/dashboards/ServicesStatistic.json +++ b/grafana/provisioning/dashboards/ServicesStatistic.json @@ -3727,6 +3727,213 @@ ], "title": "Payment Processing Latency", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Количество повторных запросов к платежной системе в секунду", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Запросов/сек", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 109, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(payment_request_retried_total{accountName=~\".*\"}[1m])", + "legendFormat": "Повторные запросы: {{accountName}}", + "range": true, + "refId": "A" + } + ], + "title": "Payment Request Retries Rate (per second)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Общее количество повторных запросов к платежной системе", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Количество", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 100 + }, + { + "color": "red", + "value": 500 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 110, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "payment_request_retried_total{accountName=~\".*\"}", + "legendFormat": "Всего повторов: {{accountName}}", + "range": true, + "refId": "A" + } + ], + "title": "Payment Request Retries Total Count", + "type": "timeseries" }], "preload": false, "refresh": "5s", diff --git a/src/main/kotlin/ru/quipy/OnlineShopApplication.kt b/src/main/kotlin/ru/quipy/OnlineShopApplication.kt index 1fcd3f89c..f311c6fdf 100644 --- a/src/main/kotlin/ru/quipy/OnlineShopApplication.kt +++ b/src/main/kotlin/ru/quipy/OnlineShopApplication.kt @@ -6,6 +6,7 @@ import org.slf4j.LoggerFactory import org.springframework.boot.autoconfigure.SpringBootApplication import org.springframework.boot.runApplication import ru.quipy.common.utils.NamedThreadFactory +import java.util.concurrent.ExecutorService import java.util.concurrent.Executors @@ -14,10 +15,10 @@ class OnlineShopApplication { val log: Logger = LoggerFactory.getLogger(OnlineShopApplication::class.java) companion object { - val appExecutor = Executors.newFixedThreadPool(20_000, NamedThreadFactory("main-app-executor")).asCoroutineDispatcher() + val appExecutor: ExecutorService = Executors.newFixedThreadPool(64, NamedThreadFactory("main-app-executor")) } } -suspend fun main(args: Array) { +fun main(args: Array) { runApplication(*args) } diff --git a/src/main/kotlin/ru/quipy/apigateway/APIController.kt b/src/main/kotlin/ru/quipy/apigateway/APIController.kt index 251ee553e..67c18a290 100644 --- a/src/main/kotlin/ru/quipy/apigateway/APIController.kt +++ b/src/main/kotlin/ru/quipy/apigateway/APIController.kt @@ -9,9 +9,9 @@ import ru.quipy.common.utils.RateLimiter import ru.quipy.exceptions.TooManyRequestsException import ru.quipy.orders.repository.OrderRepository import ru.quipy.payments.logic.OrderPayer +import ru.quipy.payments.logic.now import java.time.Duration import java.util.* -import kotlin.random.Random @RestController class APIController( @@ -60,10 +60,10 @@ class APIController( @PostMapping("/orders/{orderId}/payment") fun payOrder(@PathVariable orderId: UUID, @RequestParam deadline: Long): PaymentSubmissionDto { - if (!rateLimiter.tick()) { - val retryAfterMs = 10L + Random.nextLong(10) - throw TooManyRequestsException(retryAfterMs) + while (!rateLimiter.tick() && now() < deadline) { } + if (now() >= deadline) + throw TooManyRequestsException(10); val paymentId = UUID.randomUUID() val order = orderRepository.findById(orderId)?.let { orderRepository.save(it.copy(status = OrderStatus.PAYMENT_IN_PROGRESS)) diff --git a/src/main/kotlin/ru/quipy/apigateway/GlobalExceptionHandler.kt b/src/main/kotlin/ru/quipy/apigateway/GlobalExceptionHandler.kt index ea33b9e32..7d2c38fe8 100644 --- a/src/main/kotlin/ru/quipy/apigateway/GlobalExceptionHandler.kt +++ b/src/main/kotlin/ru/quipy/apigateway/GlobalExceptionHandler.kt @@ -29,10 +29,9 @@ class GlobalExceptionHandler( @ExceptionHandler(TooManyRequestsException::class) fun handleTooManyRequestsRetriable(exception: TooManyRequestsException): ResponseEntity { - val retryAfterSeconds = (exception.retryAfterMs / 1000.0).coerceAtLeast(0.1) return ResponseEntity .status(HttpStatus.TOO_MANY_REQUESTS) - .header("Retry-After", retryAfterSeconds.toString()) + .header("Retry-After", "10") .body("too many requests") } } \ No newline at end of file diff --git a/src/main/kotlin/ru/quipy/payments/config/PaymentAccountsConfig.kt b/src/main/kotlin/ru/quipy/payments/config/PaymentAccountsConfig.kt index 8012d1ee7..68981f838 100644 --- a/src/main/kotlin/ru/quipy/payments/config/PaymentAccountsConfig.kt +++ b/src/main/kotlin/ru/quipy/payments/config/PaymentAccountsConfig.kt @@ -4,10 +4,10 @@ import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule import com.fasterxml.jackson.module.kotlin.registerKotlinModule import io.micrometer.core.instrument.MeterRegistry -import kotlinx.coroutines.sync.Semaphore import org.springframework.beans.factory.annotation.Value import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration +import ru.quipy.common.utils.SlidingWindowRateLimiter import ru.quipy.core.EventSourcingService import ru.quipy.payments.api.PaymentAggregate import ru.quipy.payments.logic.PaymentAccountProperties @@ -18,7 +18,9 @@ import java.net.URI import java.net.http.HttpClient import java.net.http.HttpRequest import java.net.http.HttpResponse +import java.time.Duration import java.util.* +import java.util.concurrent.Semaphore @Configuration @@ -40,10 +42,18 @@ class PaymentAccountsConfig { @Value("#{'\${payment.accounts}'.split(',')}") lateinit var allowedAccounts: List + @Bean + fun rateLimit(): SlidingWindowRateLimiter { + return SlidingWindowRateLimiter( + rate = 1100L, + window = Duration.ofMillis(1000), + ) + } @Bean fun accountAdapters( paymentService: EventSourcingService, meterRegistry: MeterRegistry, + rateLimiter: SlidingWindowRateLimiter ): List { val request = HttpRequest.newBuilder() .uri(URI("http://${paymentProviderHostPort}/external/accounts?serviceName=$serviceName&token=$token")) @@ -67,7 +77,8 @@ class PaymentAccountsConfig { paymentProviderHostPort, token, meterRegistry, - Semaphore(it.parallelRequests) + Semaphore(it.parallelRequests), + rateLimiter ) } } diff --git a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt index e2bd9e203..41326b06d 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt @@ -5,12 +5,18 @@ import org.slf4j.Logger import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service +import ru.quipy.common.utils.NamedThreadFactory +import ru.quipy.common.utils.SlidingWindowRateLimiter import ru.quipy.core.EventSourcingService +import ru.quipy.exceptions.TooManyRequestsException import ru.quipy.payments.api.PaymentAggregate import java.util.* +import java.util.concurrent.LinkedBlockingQueue +import java.util.concurrent.ThreadPoolExecutor +import java.util.concurrent.TimeUnit @Service -class OrderPayer(meterRegistry: MeterRegistry) { +class OrderPayer(val rateLimiter : SlidingWindowRateLimiter, meterRegistry: MeterRegistry) { companion object { val logger: Logger = LoggerFactory.getLogger(OrderPayer::class.java) @@ -18,6 +24,14 @@ class OrderPayer(meterRegistry: MeterRegistry) { private val plannedRequests = meterRegistry.counter("payment.processing.planned", "accountName", "acc-12") + private val paymentExecutor = ThreadPoolExecutor( + 3600, + 3600, + 100L, + TimeUnit.MILLISECONDS, + LinkedBlockingQueue(100_000), + NamedThreadFactory("payment-submission-executor") + ) @Autowired private lateinit var paymentESService: EventSourcingService @@ -28,8 +42,13 @@ class OrderPayer(meterRegistry: MeterRegistry) { fun processPayment(orderId: UUID, amount: Int, paymentId: UUID, deadline: Long): Long { val createdAt = System.currentTimeMillis() - plannedRequests.increment() - val createdEvent = paymentESService.create { + while (!rateLimiter.tick() && now() < deadline) { + } + if (now() >= deadline) + throw TooManyRequestsException(10) + paymentExecutor.submit { + plannedRequests.increment() + val createdEvent = paymentESService.create { it.create( paymentId, orderId, @@ -37,10 +56,10 @@ class OrderPayer(meterRegistry: MeterRegistry) { ) } - logger.trace("Payment {} for order {} created.", createdEvent.paymentId, orderId) - - paymentService.submitPaymentRequest(paymentId, amount, createdAt, deadline) + logger.trace("Payment {} for order {} created.", createdEvent.paymentId, orderId) + paymentService.submitPaymentRequest(paymentId, amount, createdAt, deadline) + } return createdAt } } \ No newline at end of file diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index 598992345..3584b840d 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -3,15 +3,8 @@ package ru.quipy.payments.logic import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.registerKotlinModule import io.micrometer.core.instrument.MeterRegistry -import kotlinx.coroutines.CoroutineDispatcher -import kotlinx.coroutines.CoroutineExceptionHandler -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.SupervisorJob -import kotlinx.coroutines.asCoroutineDispatcher -import kotlinx.coroutines.launch -import kotlinx.coroutines.sync.Semaphore +import okio.EOFException import org.slf4j.LoggerFactory -import ru.quipy.common.utils.NamedThreadFactory import ru.quipy.common.utils.SlidingWindowRateLimiter import ru.quipy.core.EventSourcingService import ru.quipy.exceptions.TooManyRequestsException @@ -25,6 +18,8 @@ import java.net.http.HttpResponse import java.time.Duration import java.util.* import java.util.concurrent.Executors +import java.util.concurrent.ScheduledExecutorService +import java.util.concurrent.Semaphore import java.util.concurrent.TimeUnit import kotlin.math.pow import kotlin.random.Random @@ -38,26 +33,22 @@ class PaymentExternalSystemAdapterImpl( private val token: String, meterRegistry: MeterRegistry, private val parallelLimiter: Semaphore, - private val ioDispatcher: CoroutineDispatcher = Executors.newFixedThreadPool( - Runtime.getRuntime().availableProcessors() * 2, - NamedThreadFactory("payment-io-") - ).asCoroutineDispatcher() + private val rateLimiter: SlidingWindowRateLimiter ) : PaymentExternalSystemAdapter { companion object { val logger = LoggerFactory.getLogger(PaymentExternalSystemAdapter::class.java) val mapper = ObjectMapper().registerKotlinModule() } - private val exceptionHandler = CoroutineExceptionHandler { _, throwable -> - logger.error("[$accountName] Unhandled exception in payment adapter coroutine", throwable) - } - - private val scope = CoroutineScope(SupervisorJob() + ioDispatcher + exceptionHandler) + private val time_95_percentile = 20_000L + private val retryExecutor: ScheduledExecutorService = Executors.newScheduledThreadPool(properties.parallelRequests) // 2025-11-20T20:30:35.780+03:00 INFO 56644 --- [alhost:1234/...] ru.quipy.core.EventSourcingService : Optimistic lock exception. Failed to save event records id: [7dca693e-e811-4b7f-8bce-23e13d952c04-4] private val startedRequests = meterRegistry.counter("payment.processing.started", "accountName", properties.accountName) + private val requestsRetried = + meterRegistry.counter("payment.request.retried", "accountName", properties.accountName) private val timer = meterRegistry.timer("payment.external.system.request.latency", "accountName", properties.accountName) private val serviceName = properties.serviceName @@ -66,12 +57,7 @@ class PaymentExternalSystemAdapterImpl( private val rateLimitPerSec = properties.rateLimitPerSec private val parallelRequests = properties.parallelRequests - private val rateLimit: SlidingWindowRateLimiter by lazy { - SlidingWindowRateLimiter( - rate = (rateLimitPerSec * 0.95).toLong(), - window = Duration.ofMillis(1000), - ) - } + private val httpClient = HttpClient .newBuilder() @@ -79,17 +65,12 @@ class PaymentExternalSystemAdapterImpl( .version(HttpClient.Version.HTTP_2) .build() - private val retryScheduler = Executors.newScheduledThreadPool( - Runtime.getRuntime().availableProcessors() - ) - override fun performPaymentAsync(paymentId: UUID, amount: Int, paymentStartedAt: Long, deadline: Long) { logger.warn("[$accountName] Submitting payment request for payment $paymentId") val transactionId = UUID.randomUUID() // Вне зависимости от исхода оплаты важно отметить что она была отправлена. // Это требуется сделать ВО ВСЕХ СЛУЧАЯХ, поскольку эта информация используется сервисом тестирования. - scope.launch { try { paymentESService.update(paymentId) { it.logSubmission( @@ -103,12 +84,11 @@ class PaymentExternalSystemAdapterImpl( } catch (e: Exception) { logger.error("[$accountName] Failed to record log submission for $paymentId", e) } - } logger.info("[$accountName] Submit: $paymentId , txId: $transactionId") val remaining = deadline - now() - val minRequiredTime = requestAverageProcessingTime.toMillis() * 2 + val minRequiredTime = requestAverageProcessingTime.toMillis() if (remaining < minRequiredTime) { logger.warn("[$accountName] Not enough time for payment $paymentId: ${remaining}ms remaining, need ${minRequiredTime}ms") paymentESService.update(paymentId) { @@ -118,45 +98,18 @@ class PaymentExternalSystemAdapterImpl( throw TooManyRequestsException(retryAfterMs) } - val timeBeforeCall = now() - - if (!tryAcquire(now(), deadline - now())) { - val retryAfterMs = (requestAverageProcessingTime.toMillis() / parallelRequests * 10).coerceIn( - 10, - 100 - ) + Random.nextLong(10) - - throw TooManyRequestsException(retryAfterMs) - } - - if (!rateLimit.tickBlockingWithTimeout(deadline - now())) { - parallelLimiter.release() - - val retryAfterMs = (1000L / rateLimitPerSec * 10).coerceIn(10, 100) + Random.nextLong(10) - throw TooManyRequestsException(retryAfterMs) - } - val requestTimeout = minOf( - deadline - now(), - requestAverageProcessingTime.toMillis() * 2 + time_95_percentile, + requestAverageProcessingTime.toMillis() ).coerceAtLeast(100) - if (requestTimeout < requestAverageProcessingTime.toMillis()) { - logger.warn("[$accountName] Timeout too short for payment $paymentId: ${requestTimeout}ms") - parallelLimiter.release() - val retryAfterMs = requestAverageProcessingTime.toMillis() - requestTimeout + Random.nextLong(100) - throw TooManyRequestsException(retryAfterMs) - } - val request = HttpRequest.newBuilder() .uri(URI("http://$paymentProviderHostPort/external/process?serviceName=$serviceName&token=$token&accountName=$accountName&transactionId=$transactionId&paymentId=$paymentId&amount=$amount")) .timeout(Duration.ofMillis(requestTimeout)) .POST(HttpRequest.BodyPublishers.noBody()) .build() - val retryCount = 0L - - completeAction(retryCount, request, paymentId, transactionId, timeBeforeCall, deadline) + completeAction(0, request, paymentId, transactionId, deadline) } override fun price() = properties.price @@ -165,70 +118,67 @@ class PaymentExternalSystemAdapterImpl( override fun name() = properties.accountName - fun tryAcquire(startedAt: Long, remaining: Long): Boolean { - var isAcquired = parallelLimiter.tryAcquire() - while (!isAcquired && now() - startedAt < remaining) { - isAcquired = parallelLimiter.tryAcquire() - Thread.sleep(1) - } - - return isAcquired - } - fun completeAction( retryCount: Long, request: HttpRequest, paymentId: UUID, transactionId: UUID, - timeBeforeCall: Long, deadline: Long ) { + val timeBeforeCall = now() + parallelLimiter.acquire() + startedRequests.increment() httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .whenComplete { response, throwable -> - scope.launch { - try { - if (throwable != null) { - val e = throwable.cause - when (throwable.cause) { - is SocketTimeoutException -> { - logger.warn("[$accountName] attempt ${retryCount + 1} timeout: $paymentId", e) - } + parallelLimiter.release() + if (throwable != null) { + val e = throwable.cause + var isRetriable = true + when (throwable.cause) { + is SocketTimeoutException -> { + logger.warn("[$accountName] attempt ${retryCount + 1} timeout: $paymentId", e) + } - is InterruptedIOException -> { - logger.warn("[$accountName] interrupted: $paymentId", e) - } + is InterruptedIOException -> { + logger.warn("[$accountName] interrupted: $paymentId", e) + } - else -> { - logger.warn("[$accountName] io error: $paymentId", e) - } + is EOFException -> { + logger.warn("[$accountName] eof exception in: $paymentId", e) } - if (retryCount + 1 >= 3) { + else -> { + logger.warn("[$accountName] io error: $paymentId", e) + isRetriable = false + } + } + + if (retryCount + 1 >= 3) { + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "Max attempts reached") + } + } else { + val backoff = ((2.0.pow(retryCount.toDouble()) * 25).toLong() + Random.nextLong(10)) + val capped = backoff.coerceAtMost(deadline - now() - 5) + if (capped <= 0) { paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "Max attempts reached") + it.logProcessing(false, now(), transactionId, "Deadline expired") } } else { - val backoff = ((2.0.pow(retryCount.toDouble()) * 25).toLong() + Random.nextLong(10)) - val capped = backoff.coerceAtMost(deadline - now() - 5) - if (capped <= 0) { - paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "Deadline expired") - } - } else { + if (isRetriable) { scheduleRetry( - retryCount, - request, - paymentId, - transactionId, - timeBeforeCall, - deadline, - capped + retryCount, request, paymentId, transactionId, deadline, capped ) - return@launch + } else { + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "Non-retriable exception") + } } } - } else { - logger.warn("Free space in semaphore: {}", parallelLimiter.availablePermits) + } + } else { + try { + logger.warn("Free space in semaphore: {}", parallelLimiter.availablePermits()) logger.info( "success in callback for payment: {}, retry count: {}, in time: {}", paymentId, @@ -246,48 +196,44 @@ class PaymentExternalSystemAdapterImpl( it.logProcessing(parsed.result, now(), transactionId, parsed.message) } timer.record(now() - timeBeforeCall, TimeUnit.MILLISECONDS) + } catch (e: Exception) { + logger.error("[$accountName] Error processing payment $paymentId", e) } - } catch (e: Exception) { - logger.error("[$accountName] Error processing payment $paymentId", e) - } finally { - startedRequests.increment() - parallelLimiter.release() } - } } } private fun scheduleRetry( retryCount: Long, request: HttpRequest, paymentId: UUID, - transactionId: UUID, timeBeforeCall: Long, deadline: Long, delay: Long + transactionId: UUID, deadline: Long, delay: Long ) { - retryScheduler.schedule({ + requestsRetried.increment() + retryExecutor.scheduleWithFixedDelay({ + while (!rateLimiter.tick() && now() < deadline) { + } + if (now() >= deadline) + throw TooManyRequestsException(10) + + logger.info("Completing retry. All retry count - {}", requestsRetried.count()) val remainingTime = deadline - now() if (remainingTime < requestAverageProcessingTime.toMillis()) { - scope.launch { - try { - paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "Not enough time for retry") - } - } catch (e: Exception) { - logger.error("[$accountName] Failed to record retry failure for $paymentId", e) - } finally { - startedRequests.increment() - parallelLimiter.release() + try { + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "Not enough time for retry") } + } catch (e: Exception) { + logger.error("[$accountName] Failed to record retry failure for $paymentId", e) } - return@schedule + } else { + val newRequestTimeout = remainingTime.coerceIn(time_95_percentile, requestAverageProcessingTime.toMillis() * 2) + val newRequest = HttpRequest.newBuilder() + .uri(request.uri()) + .timeout(Duration.ofMillis(newRequestTimeout)) + .POST(HttpRequest.BodyPublishers.noBody()) + .build() + completeAction(retryCount + 1, newRequest, paymentId, transactionId, deadline) } - - val newRequestTimeout = remainingTime.coerceIn(100, requestAverageProcessingTime.toMillis() * 2) - val newRequest = HttpRequest.newBuilder() - .uri(request.uri()) - .timeout(Duration.ofMillis(newRequestTimeout)) - .POST(HttpRequest.BodyPublishers.noBody()) - .build() - - completeAction(retryCount + 1, newRequest, paymentId, transactionId, timeBeforeCall, deadline) - }, delay, TimeUnit.MILLISECONDS) + }, 0, delay, TimeUnit.MILLISECONDS) } } diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index bdbe25f61..d95ec6218 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -29,4 +29,14 @@ payment.token=${PAYMENT_TOKEN} payment.accounts=${PAYMENT_ACCOUNTS:acc-12} # payment.accounts=${PAYMENT_ACCOUNTS:acc-18} payment.hostPort=${PAYMENT_HOST:localhost}:${PAYMENT_PORT:1234} +spring.datasource.hikari.maximum-pool-size=20_000 +spring.datasource.hikari.minimum-idle=50 +spring.datasource.hikari.connection-timeout=5000 +server.jetty.threads.min=100 +server.jetty.threads.max=20000 +server.jetty.threads.max-queue-capacity=100000 +server.jetty.threads.acceptors=16 +server.jetty.threads.selectors=32 +server.jetty.connection-idle-timeout=60000 +