From 0fb2c721f0484ad9137878dac3e28b8c136a169c Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Thu, 13 Nov 2025 20:12:45 +0300 Subject: [PATCH 01/61] fix: try to fix client timeout --- .gitignore | 2 +- Dockerfile | 2 +- README.md | 15 +- .../dashboards/ServicesStatistic.json | 691 ++++++++++++------ pom.xml | 22 +- prometheus/prometheus.yml | 8 +- .../ru/quipy/apigateway/APIController.kt | 24 +- .../apigateway/GlobalExceptionHandler.kt | 53 ++ .../common/utils/SlidingWindowRateLimiter.kt | 14 +- .../exceptions/TooLongRequestEcexption.kt | 4 + .../exceptions/TooManyRequestsException.kt | 3 + .../TooManyRequestsRetriableException.kt | 4 + .../orders/subscribers/PaymentSubscriber.kt | 3 + .../payments/config/PaymentAccountsConfig.kt | 16 +- .../ru/quipy/payments/logic/OrderPayer.kt | 8 +- .../logic/PaymentExternalServiceImpl.kt | 106 ++- src/main/resources/application.properties | 17 +- test-local-run.http | 2 +- test-on-prem-run.http | 8 +- 19 files changed, 721 insertions(+), 281 deletions(-) create mode 100644 src/main/kotlin/ru/quipy/apigateway/GlobalExceptionHandler.kt create mode 100644 src/main/kotlin/ru/quipy/exceptions/TooLongRequestEcexption.kt create mode 100644 src/main/kotlin/ru/quipy/exceptions/TooManyRequestsException.kt create mode 100644 src/main/kotlin/ru/quipy/exceptions/TooManyRequestsRetriableException.kt diff --git a/.gitignore b/.gitignore index 259113f73..ea59400a3 100644 --- a/.gitignore +++ b/.gitignore @@ -32,4 +32,4 @@ build/ !**/src/test/**/build/ ### VS Code ### -.vscode/ +.vscode/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index cc6f2e042..3d9587cd0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ RUN mvn dependency:go-offline COPY src src RUN mvn package -FROM openjdk:17-jdk-slim +FROM eclipse-temurin:17-alpine-3.22 COPY --from=build /app/target/*.jar /high-load-course.jar diff --git a/README.md b/README.md index 56d36970f..bc19c4280 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,9 @@ # Template for the HighLoad course + This project is based on [Tiny Event Sourcing library](https://github.com/andrsuh/tiny-event-sourcing) ### Run PostgreSql + This example uses Postgres as an implementation of the Event store. You can see it in `pom.xml`: ``` @@ -12,7 +14,8 @@ This example uses Postgres as an implementation of the Event store. You can see ``` -Thus, you have to run Postgres in order to test this example. Postgres service is included in `docker-compose` file that we have in the root of the project. +Thus, you have to run Postgres in order to test this example. Postgres service is included in `docker-compose` file +that we have in the root of the project. # More comprehensive information about the course, project, how to run tests is here: @@ -21,19 +24,25 @@ https://andrsuh.notion.site/2595d535059281d8a815c2cb3875c376?source=copy_link https://andrsuh.notion.site/2625d5350592801aaf88c7c95302d10c?source=copy_link ### Run the infrastructure + Set of the services you need to start developing and testing process is following: -- Bombardier - service that is in charge of emulation the store's clients activity (creates the incoming load). Also serves as a third-party payment system. + +- Bombardier - service that is in charge of emulation the store's clients activity (creates the incoming load). Also + serves as a third-party payment system. - Postgres DBMS - Prometheus + Grafana - metrics collection and visualization services You can run all beforementioned services by the following command: + ``` docker compose -f docker-compose.yml up ``` ### Run the application -To make the application run you can start the main class `OnlineShopApplication`. It is not being launched as a docker contained to simplify and speed up the devevopment process as it is easier for you to refactor the application and re-run it immediately in the IDE. +To make the application run you can start the main class `OnlineShopApplication`. It is not being launched as a docker +contained to simplify and speed up the devevopment process as it is easier for you to refactor the application and +re-run it immediately in the IDE. ### If you want to pull changes from the main repository into your fork diff --git a/grafana/provisioning/dashboards/ServicesStatistic.json b/grafana/provisioning/dashboards/ServicesStatistic.json index 684b97269..eb6f028d2 100644 --- a/grafana/provisioning/dashboards/ServicesStatistic.json +++ b/grafana/provisioning/dashboards/ServicesStatistic.json @@ -1345,7 +1345,6 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "exemplar": true, "expr": "rate(stage_duration_ok_sum{service=~\"$service\", result=\"success\"}[1m]) / rate(stage_duration_ok_count{service=~\"$service\", result=\"success\"}[1m])", "interval": "", "legendFormat": "{{stage}}", @@ -1785,7 +1784,7 @@ "exemplar": true, "expr": "rate(external_sys_duration_count{service=~\"$service\"}[30s])", "hide": false, - "interval": "", + "instant": false, "legendFormat": "{{accountName}} - {{outcome}}", "range": true, "refId": "C" @@ -1923,7 +1922,7 @@ "exemplar": true, "expr": "rate(external_sys_duration_sum{service=~\"$service\"}[1m]) / rate(external_sys_duration_count{service=~\"$service\"}[1m])", "hide": false, - "interval": "", + "instant": false, "legendFormat": "{{accountName}} - {{outcome}}", "range": true, "refId": "C" @@ -2024,37 +2023,13 @@ } } ] - }, - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "payOrder - " - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] } ] }, "gridPos": { "h": 8, "w": 12, - "x": 12, + "x": 0, "y": 108 }, "id": 43, @@ -2065,9 +2040,7 @@ ], "displayMode": "table", "placement": "right", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true + "showLegend": true }, "tooltip": { "mode": "single", @@ -2082,7 +2055,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "rate(http_request_latent_sum{service=~\"$service\"}[15s]) / rate(http_request_latent_count{service=~\"$service\"}[15s])", + "expr": "sum(rate(http_request_latent_sum{service=~\"$service\"}[15s]) / rate(http_request_latent_count{service=~\"$service\"}[15s]))", "hide": false, "instant": false, "legendFormat": "{{method}} - {{result}}", @@ -2390,7 +2363,6 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "exemplar": true, "expr": "sum by (name) (executor_active_threads{})", "interval": "", "legendFormat": "{{name}}", @@ -2669,20 +2641,18 @@ "h": 8, "w": 12, "x": 0, - "y": 38 + "y": 46 }, - "id": 95, + "id": 41, "options": { "legend": { - "calcs": [ - "max" - ], + "calcs": [], "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { - "mode": "multi", + "mode": "single", "sort": "none" } }, @@ -2694,34 +2664,34 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum by (job) (system_cpu_usage{service=~\"$service\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{job}} - System CPU Usage", + "expr": "sum by (job) (process_files_open_files{})", + "instant": false, + "legendFormat": "{{job}}", "range": true, "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sum by (job) (process_cpu_usage{service=~\"$service\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{job}} -Process CPU Usage", - "range": true, - "refId": "B" } ], - "title": "CPU Usage", + "title": "Open files", "type": "timeseries" - }, + } + ], + "title": "JVM metrics", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 97, + "panels": [ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "P1C574E4B1E20B3B3" }, "fieldConfig": { "defaults": { @@ -2737,8 +2707,8 @@ "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 17, - "gradientMode": "opacity", + "fillOpacity": 0, + "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, @@ -2768,23 +2738,29 @@ { "color": "green", "value": null + }, + { + "color": "red", + "value": 80 } ] - }, - "unit": "s" + } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 38 + "x": 0, + "y": 7 }, - "id": 42, + "id": 98, + "maxDataPoints": 100, "options": { "legend": { - "calcs": [], + "calcs": [ + "lastNotNull" + ], "displayMode": "table", "placement": "right", "showLegend": true @@ -2799,23 +2775,25 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "P1C574E4B1E20B3B3" }, "editorMode": "code", - "expr": "sum(rate(jvm_gc_pause_seconds_sum{}[5m])) by (job)", - "instant": false, + "expr": "sum by (job) (jetty_threads_config_max{})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, "legendFormat": "{{job}}", "range": true, "refId": "A" } ], - "title": "GC delay", + "title": "Max Threads", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "P1C574E4B1E20B3B3" }, "fieldConfig": { "defaults": { @@ -2831,8 +2809,8 @@ "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 17, - "gradientMode": "opacity", + "fillOpacity": 0, + "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, @@ -2862,23 +2840,29 @@ { "color": "green", "value": null + }, + { + "color": "red", + "value": 80 } ] - }, - "unit": "short" + } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 46 + "x": 12, + "y": 7 }, - "id": 41, + "id": 99, + "maxDataPoints": 100, "options": { "legend": { - "calcs": [], + "calcs": [ + "lastNotNull" + ], "displayMode": "table", "placement": "right", "showLegend": true @@ -2893,37 +2877,25 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "P1C574E4B1E20B3B3" }, "editorMode": "code", - "expr": "sum by (job) (process_files_open_files{})", - "instant": false, + "expr": "sum by (job) (jetty_threads_config_min{service=~\"$service\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, "legendFormat": "{{job}}", "range": true, "refId": "A" } ], - "title": "Open files", + "title": "Thread Config Min", "type": "timeseries" - } - ], - "title": "JVM metrics", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 6 - }, - "id": 97, - "panels": [ + }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "P1C574E4B1E20B3B3" }, "fieldConfig": { "defaults": { @@ -2939,7 +2911,7 @@ "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, @@ -2953,7 +2925,7 @@ "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", + "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", @@ -2963,6 +2935,7 @@ "mode": "off" } }, + "links": [], "mappings": [], "thresholds": { "mode": "absolute", @@ -2976,18 +2949,18 @@ "value": 80 } ] - } + }, + "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, - "w": 12, + "w": 24, "x": 0, - "y": 7 + "y": 15 }, - "id": 98, - "maxDataPoints": 100, + "id": 24, "options": { "legend": { "calcs": [ @@ -2998,7 +2971,7 @@ "showLegend": true }, "tooltip": { - "mode": "single", + "mode": "multi", "sort": "none" } }, @@ -3010,16 +2983,52 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum by (job) (jetty_threads_config_max{})", + "expr": "sum by (job) (jetty_threads_busy{})", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{job}}", + "legendFormat": "{{job}} - busy ", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (job) (jetty_threads_idle{})", + "interval": "", + "legendFormat": "{{job}} - idle", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (job) (jetty_threads_current{})", + "interval": "", + "legendFormat": "{{job}} - current", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (job) (jetty_threads_jobs{})", + "interval": "", + "legendFormat": "{{job}} - jobs", + "range": true, + "refId": "D" } ], - "title": "Max Threads", + "title": "Jetty Threads", "type": "timeseries" }, { @@ -3041,7 +3050,7 @@ "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, @@ -3055,16 +3064,17 @@ "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", + "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, + "links": [], "mappings": [], "thresholds": { "mode": "absolute", @@ -3078,29 +3088,27 @@ "value": 80 } ] - } + }, + "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 7 + "x": 0, + "y": 23 }, - "id": 99, - "maxDataPoints": 100, + "id": 4, "options": { "legend": { - "calcs": [ - "lastNotNull" - ], + "calcs": [], "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { - "mode": "single", + "mode": "multi", "sort": "none" } }, @@ -3112,16 +3120,15 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum by (job) (jetty_threads_config_min{service=~\"$service\"})", + "expr": "irate(http_server_requests_seconds_count{service=~\"$service\", uri!~\".*actuator.*\"}[5m])", "format": "time_series", - "interval": "", "intervalFactor": 1, - "legendFormat": "{{job}}", + "legendFormat": "{{method}} [{{status}}] - {{uri}}", "range": true, "refId": "A" } ], - "title": "Thread Config Min", + "title": "Request Count", "type": "timeseries" }, { @@ -3161,13 +3168,12 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, - "links": [], "mappings": [], "thresholds": { "mode": "absolute", @@ -3188,15 +3194,15 @@ }, "gridPos": { "h": 8, - "w": 24, - "x": 0, - "y": 15 + "w": 12, + "x": 12, + "y": 23 }, - "id": 24, + "id": 96, "options": { "legend": { "calcs": [ - "lastNotNull" + "max" ], "displayMode": "table", "placement": "right", @@ -3215,52 +3221,125 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum by (job) (jetty_threads_busy{})", + "expr": "sum by (job, method, status, uri) (irate(http_server_requests_seconds_sum{service=~\"$service\", exception=\"None\", uri!~\".*actuator.*\"}[5m]) / irate(http_server_requests_seconds_count{service=~\"$service\", exception=\"None\", uri!~\".*actuator.*\"}[5m]))", "format": "time_series", - "interval": "", "intervalFactor": 1, - "legendFormat": "{{job}} - busy ", + "legendFormat": "{{method}} [{{status}}] - {{uri}} - {{job}}", "range": true, "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + } + ], + "title": "Response Time", + "type": "timeseries" + } + ], + "title": "Jetty Statistics", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 101, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "История изменения количества задач в очереди", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "editorMode": "code", - "expr": "sum by (job) (jetty_threads_idle{})", - "interval": "", - "legendFormat": "{{job}} - idle", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Задачи в очереди", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, - "editorMode": "code", - "expr": "sum by (job) (jetty_threads_current{})", - "interval": "", - "legendFormat": "{{job}} - current", - "range": true, - "refId": "C" + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 16, + "x": 8, + "y": 8 + }, + "id": 103, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.0", + "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "editorMode": "code", - "expr": "sum by (job) (jetty_threads_jobs{})", - "interval": "", - "legendFormat": "{{job}} - jobs", - "range": true, - "refId": "D" + "expr": "payment_processing_planned_total - payment_processing_started_total", + "legendFormat": "{{accountName}}", + "refId": "A" } ], - "title": "Jetty Threads", + "title": "История задач в очереди", "type": "timeseries" }, { @@ -3268,6 +3347,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "Скорость входа в ожидание и выхода из ожидания (requests/sec)", "fieldConfig": { "defaults": { "color": { @@ -3277,10 +3357,9 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "", + "axisLabel": "Запросов/сек", "axisPlacement": "auto", "barAlignment": 0, - "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -3290,8 +3369,8 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, + "lineInterpolation": "smooth", + "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" @@ -3306,7 +3385,6 @@ "mode": "off" } }, - "links": [], "mappings": [], "thresholds": { "mode": "absolute", @@ -3314,29 +3392,60 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "none" + "unit": "reqps" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Скорость увеличения очереди" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Скорость числа задач в обработке" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] }, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 23 + "y": 16 }, - "id": 4, + "id": 104, "options": { "legend": { - "calcs": [], + "calcs": [ + "mean", + "lastNotNull", + "max" + ], "displayMode": "table", - "placement": "right", + "placement": "bottom", "showLegend": true }, "tooltip": { @@ -3344,23 +3453,37 @@ "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "9.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "editorMode": "code", - "expr": "irate(http_server_requests_seconds_count{service=~\"$service\", uri!~\".*actuator.*\"}[5m])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{method}} [{{status}}] - {{uri}}", - "range": true, + "expr": "rate(payment_processing_planned_total[1m])", + "legendFormat": "Попало в очередь: {{accountName}}", "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "rate(payment_processing_started_total[1m])", + "legendFormat": "Приняты в обработку (вышли из очереди): {{accountName}}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "rate(payment_processing_completed_total[1m])", + "legendFormat": "Обработаны: {{accountName}}", + "refId": "C" } ], - "title": "Request Count", + "title": "Throughput (в очереди vs в обработке vs обработаны)", "type": "timeseries" }, { @@ -3368,6 +3491,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "Если значение > 0, задачи накапливаются быстрее, чем обрабатываются", "fieldConfig": { "defaults": { "color": { @@ -3375,23 +3499,22 @@ }, "custom": { "axisBorderShow": false, - "axisCenteredZero": false, + "axisCenteredZero": true, "axisColorMode": "text", - "axisLabel": "", + "axisLabel": "Δ запросов/сек", "axisPlacement": "auto", "barAlignment": 0, - "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", + "fillOpacity": 30, + "gradientMode": "scheme", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, + "lineInterpolation": "smooth", + "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" @@ -3403,10 +3526,9 @@ "mode": "none" }, "thresholdsStyle": { - "mode": "off" + "mode": "area" } }, - "links": [], "mappings": [], "thresholds": { "mode": "absolute", @@ -3417,11 +3539,11 @@ }, { "color": "red", - "value": 80 + "value": 0.1 } ] }, - "unit": "s" + "unit": "short" }, "overrides": [] }, @@ -3429,16 +3551,17 @@ "h": 8, "w": 12, "x": 12, - "y": 23 + "y": 16 }, - "id": 96, + "id": 105, "options": { "legend": { "calcs": [ - "max" + "mean", + "lastNotNull" ], "displayMode": "table", - "placement": "right", + "placement": "bottom", "showLegend": true }, "tooltip": { @@ -3446,30 +3569,165 @@ "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "9.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "editorMode": "code", - "expr": "sum by (job, method, status, uri) (irate(http_server_requests_seconds_sum{service=~\"$service\", exception=\"None\", uri!~\".*actuator.*\"}[5m]) / irate(http_server_requests_seconds_count{service=~\"$service\", exception=\"None\", uri!~\".*actuator.*\"}[5m]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{method}} [{{status}}] - {{uri}} - {{job}}", - "range": true, + "expr": "rate(payment_processing_planned_total[1m]) - rate(payment_processing_started_total[1m])", + "legendFormat": "{{accountName}}", "refId": "A" } ], - "title": "Response Time", + "title": "Скорость накопления задач (увеличения очереди)", "type": "timeseries" } ], - "title": "Jetty Statistics", + "title": "Payment Processing - Active Tasks", "type": "row" - } - ], + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 108, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.75, sum(rate(payment_external_system_request_latency_seconds_bucket[1m])) by (le, accountName))", + "legendFormat": "p75 {{accountName}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.80, sum(rate(payment_external_system_request_latency_seconds_bucket[1m])) by (le, accountName))", + "legendFormat": "p80 {{accountName}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.85, sum(rate(payment_external_system_request_latency_seconds_bucket[1m])) by (le, accountName))", + "legendFormat": "p85 {{accountName}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(rate(payment_external_system_request_latency_seconds_bucket[1m])) by (le, accountName))", + "hide": false, + "legendFormat": "p90 {{accountName}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(payment_external_system_request_latency_seconds_bucket[1m])) by (le, accountName))", + "hide": false, + "legendFormat": "p95 {{accountName}}", + "range": true, + "refId": "E" + } + ], + "title": "Payment Processing Latency", + "type": "timeseries" + }], "preload": false, "refresh": "5s", "schemaVersion": 40, @@ -3516,4 +3774,5 @@ "uid": "KVr-Vmpnz", "version": 4, "weekStart": "" -} \ No newline at end of file +} + diff --git a/pom.xml b/pom.xml index 5724b2568..e244cb6aa 100644 --- a/pom.xml +++ b/pom.xml @@ -1,5 +1,5 @@ - 4.0.0 @@ -15,7 +15,7 @@ OnlineShop Application for resilience and highly-loaded applications course - + 2.2.0 1.9.0 4.12.0 @@ -25,7 +25,7 @@ 3.1.8 - + io.github.resilience4j resilience4j-ratelimiter @@ -67,15 +67,15 @@ ${jetty.version} - - - - + + + + - - - - + + + + com.fasterxml.jackson.module diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml index 936c7edd9..ce045546f 100644 --- a/prometheus/prometheus.yml +++ b/prometheus/prometheus.yml @@ -6,16 +6,16 @@ scrape_configs: - job_name: 'bombardier-docker-network-job' metrics_path: '/actuator/prometheus' static_configs: - - targets: ['bombardier:1234'] + - targets: [ 'bombardier:1234' ] - job_name: 'bombardier-host-job' metrics_path: '/actuator/prometheus' static_configs: - - targets: ['host.docker.internal:1234'] + - targets: [ 'host.docker.internal:1234' ] - job_name: 'online-store-job' metrics_path: '/actuator/prometheus' static_configs: - - targets: ['host.docker.internal:8081'] + - targets: [ 'host.docker.internal:8081' ] - job_name: 'online-shop-job' metrics_path: '/actuator/prometheus' static_configs: - - targets: ['host.docker.internal:18081'] \ No newline at end of file + - targets: [ 'host.docker.internal:18081' ] \ No newline at end of file diff --git a/src/main/kotlin/ru/quipy/apigateway/APIController.kt b/src/main/kotlin/ru/quipy/apigateway/APIController.kt index 6f23fa18d..fff5ac386 100644 --- a/src/main/kotlin/ru/quipy/apigateway/APIController.kt +++ b/src/main/kotlin/ru/quipy/apigateway/APIController.kt @@ -2,23 +2,27 @@ package ru.quipy.apigateway import org.slf4j.Logger import org.slf4j.LoggerFactory -import org.springframework.beans.factory.annotation.Autowired +import org.springframework.beans.factory.annotation.Qualifier import org.springframework.web.bind.annotation.* +import ru.quipy.common.utils.LeakingBucketRateLimiter +import ru.quipy.common.utils.RateLimiter +import ru.quipy.exceptions.TooManyRequestsException +import ru.quipy.exceptions.TooManyRequestsRetriableException import ru.quipy.orders.repository.OrderRepository import ru.quipy.payments.logic.OrderPayer +import java.time.Duration import java.util.* @RestController -class APIController { +class APIController( + private val orderRepository: OrderRepository, + private val orderPayer: OrderPayer, + @field:Qualifier("parallelLimiter") + private val rateLimiter: RateLimiter = LeakingBucketRateLimiter(8, Duration.ofSeconds(1), 38) +) { val logger: Logger = LoggerFactory.getLogger(APIController::class.java) - @Autowired - private lateinit var orderRepository: OrderRepository - - @Autowired - private lateinit var orderPayer: OrderPayer - @PostMapping("/users") fun createUser(@RequestBody req: CreateUserRequest): User { return User(UUID.randomUUID(), req.name) @@ -56,13 +60,15 @@ class APIController { @PostMapping("/orders/{orderId}/payment") fun payOrder(@PathVariable orderId: UUID, @RequestParam deadline: Long): PaymentSubmissionDto { + if (!rateLimiter.tick()) { + throw TooManyRequestsException(deadline) + } val paymentId = UUID.randomUUID() val order = orderRepository.findById(orderId)?.let { orderRepository.save(it.copy(status = OrderStatus.PAYMENT_IN_PROGRESS)) it } ?: throw IllegalArgumentException("No such order $orderId") - val createdAt = orderPayer.processPayment(orderId, order.price, paymentId, deadline) return PaymentSubmissionDto(createdAt, paymentId) } diff --git a/src/main/kotlin/ru/quipy/apigateway/GlobalExceptionHandler.kt b/src/main/kotlin/ru/quipy/apigateway/GlobalExceptionHandler.kt new file mode 100644 index 000000000..d6c824805 --- /dev/null +++ b/src/main/kotlin/ru/quipy/apigateway/GlobalExceptionHandler.kt @@ -0,0 +1,53 @@ +package ru.quipy.apigateway + +import org.slf4j.LoggerFactory +import org.springframework.http.HttpStatus +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.ExceptionHandler +import org.springframework.web.bind.annotation.ResponseStatus +import org.springframework.web.bind.annotation.RestControllerAdvice +import ru.quipy.exceptions.TooManyRequestsRetriableException +import ru.quipy.exceptions.TooLongRequestException +import ru.quipy.exceptions.TooManyRequestsException +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicLong + +@RestControllerAdvice +class GlobalExceptionHandler( +) { + companion object { + val logger = LoggerFactory.getLogger(GlobalExceptionHandler::class.java) + private var currentRetryAfterSeconds = 1 + private var exp_base = 2; + private const val MAX_RETRY_AFTER_MS = 256 + private const val RESET_WINDOW_MS = 30000L + } + + private val rejectedRequestsCount = AtomicInteger(0) + private val lastRejectionTime = AtomicLong(0) + @ExceptionHandler(TooLongRequestException::class) + fun handleTooManyRequests(exception: TooLongRequestException): ResponseEntity { + return ResponseEntity.status(200).body("your request very long, i am so sorry") + } + @ExceptionHandler(TooManyRequestsException::class) + fun handleTooManyRequestsRetriable(exception: TooManyRequestsException): ResponseEntity { + logger.warn("to many request") + val currentTime = System.currentTimeMillis() + val lastRejection = lastRejectionTime.get() + + if (currentTime - lastRejection > RESET_WINDOW_MS) { + rejectedRequestsCount.set(0) + currentRetryAfterSeconds = 1 + } + lastRejectionTime.set(currentTime) + + if (rejectedRequestsCount.get() < 4 && exception.deadline < System.currentTimeMillis() + 1200) { + return ResponseEntity + .status(HttpStatus.TOO_MANY_REQUESTS) + .header("Retry-After", MAX_RETRY_AFTER_MS.coerceAtLeast(currentRetryAfterSeconds).toString()) + .build() + } + + return ResponseEntity.status(200).build() + } +} \ No newline at end of file diff --git a/src/main/kotlin/ru/quipy/common/utils/SlidingWindowRateLimiter.kt b/src/main/kotlin/ru/quipy/common/utils/SlidingWindowRateLimiter.kt index 6ff3092ab..c5bd36a0a 100644 --- a/src/main/kotlin/ru/quipy/common/utils/SlidingWindowRateLimiter.kt +++ b/src/main/kotlin/ru/quipy/common/utils/SlidingWindowRateLimiter.kt @@ -1,5 +1,7 @@ package ru.quipy.common.utils +import com.github.dockerjava.zerodep.shaded.org.apache.hc.core5.util.Deadline +import com.github.dockerjava.zerodep.shaded.org.apache.hc.core5.util.Timeout import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.asCoroutineDispatcher import kotlinx.coroutines.delay @@ -10,8 +12,6 @@ import java.time.Duration import java.util.concurrent.Executors import java.util.concurrent.PriorityBlockingQueue import java.util.concurrent.atomic.AtomicLong -import java.util.concurrent.locks.ReentrantLock -import kotlin.concurrent.withLock class SlidingWindowRateLimiter( private val rate: Long, @@ -39,6 +39,15 @@ class SlidingWindowRateLimiter( } } + fun tickBlocking(timeout: Long): Boolean { + val timeStarted = System.currentTimeMillis() + while (System.currentTimeMillis()-timeStarted < timeout && !tick()) { + Thread.sleep(2) + } + return System.currentTimeMillis()-timeStarted < timeout + } + + data class Measure( val value: Long, val timestamp: Long @@ -64,6 +73,7 @@ class SlidingWindowRateLimiter( queue.take() } }.invokeOnCompletion { th -> if (th != null) logger.error("Rate limiter release job completed", th) } + companion object { private val logger: Logger = LoggerFactory.getLogger(SlidingWindowRateLimiter::class.java) } diff --git a/src/main/kotlin/ru/quipy/exceptions/TooLongRequestEcexption.kt b/src/main/kotlin/ru/quipy/exceptions/TooLongRequestEcexption.kt new file mode 100644 index 000000000..c50d9bc48 --- /dev/null +++ b/src/main/kotlin/ru/quipy/exceptions/TooLongRequestEcexption.kt @@ -0,0 +1,4 @@ +package ru.quipy.exceptions + +class TooLongRequestException() : RuntimeException() { +} \ No newline at end of file diff --git a/src/main/kotlin/ru/quipy/exceptions/TooManyRequestsException.kt b/src/main/kotlin/ru/quipy/exceptions/TooManyRequestsException.kt new file mode 100644 index 000000000..cacd67810 --- /dev/null +++ b/src/main/kotlin/ru/quipy/exceptions/TooManyRequestsException.kt @@ -0,0 +1,3 @@ +package ru.quipy.exceptions + +class TooManyRequestsException(val deadline: Long) : RuntimeException() \ No newline at end of file diff --git a/src/main/kotlin/ru/quipy/exceptions/TooManyRequestsRetriableException.kt b/src/main/kotlin/ru/quipy/exceptions/TooManyRequestsRetriableException.kt new file mode 100644 index 000000000..4c95b4b30 --- /dev/null +++ b/src/main/kotlin/ru/quipy/exceptions/TooManyRequestsRetriableException.kt @@ -0,0 +1,4 @@ +package ru.quipy.exceptions + +class TooManyRequestsRetriableException(val deadline: Long) : RuntimeException(){ +} \ No newline at end of file diff --git a/src/main/kotlin/ru/quipy/orders/subscribers/PaymentSubscriber.kt b/src/main/kotlin/ru/quipy/orders/subscribers/PaymentSubscriber.kt index 767f23b3e..a4c312eaf 100644 --- a/src/main/kotlin/ru/quipy/orders/subscribers/PaymentSubscriber.kt +++ b/src/main/kotlin/ru/quipy/orders/subscribers/PaymentSubscriber.kt @@ -1,5 +1,6 @@ package ru.quipy.orders.subscribers +import io.micrometer.core.instrument.Metrics import jakarta.annotation.PostConstruct import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -19,6 +20,7 @@ class PaymentSubscriber { val logger: Logger = LoggerFactory.getLogger(PaymentSubscriber::class.java) + val paymentSucceededCounter = Metrics.counter("succeeded.payments", "account", "acc-7") @Autowired lateinit var subscriptionsManager: AggregateSubscriptionsManager @@ -42,6 +44,7 @@ class PaymentSubscriber { ).toSeconds() }, spent in queue: ${event.spentInQueueDuration.toSeconds()}" ) + paymentSucceededCounter.increment() } } } diff --git a/src/main/kotlin/ru/quipy/payments/config/PaymentAccountsConfig.kt b/src/main/kotlin/ru/quipy/payments/config/PaymentAccountsConfig.kt index eceb90cff..e10bd805a 100644 --- a/src/main/kotlin/ru/quipy/payments/config/PaymentAccountsConfig.kt +++ b/src/main/kotlin/ru/quipy/payments/config/PaymentAccountsConfig.kt @@ -3,17 +3,22 @@ package ru.quipy.payments.config import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule import com.fasterxml.jackson.module.kotlin.registerKotlinModule +import io.micrometer.core.instrument.MeterRegistry import org.springframework.beans.factory.annotation.Value import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration import ru.quipy.core.EventSourcingService import ru.quipy.payments.api.PaymentAggregate -import ru.quipy.payments.logic.* +import ru.quipy.payments.logic.PaymentAccountProperties +import ru.quipy.payments.logic.PaymentAggregateState +import ru.quipy.payments.logic.PaymentExternalSystemAdapter +import ru.quipy.payments.logic.PaymentExternalSystemAdapterImpl import java.net.URI import java.net.http.HttpClient import java.net.http.HttpRequest import java.net.http.HttpResponse import java.util.* +import java.util.concurrent.Semaphore @Configuration @@ -36,7 +41,10 @@ class PaymentAccountsConfig { lateinit var allowedAccounts: List @Bean - fun accountAdapters(paymentService: EventSourcingService): List { + fun accountAdapters( + paymentService: EventSourcingService, + meterRegistry: MeterRegistry, + ): List { val request = HttpRequest.newBuilder() .uri(URI("http://${paymentProviderHostPort}/external/accounts?serviceName=$serviceName&token=$token")) .GET() @@ -57,7 +65,9 @@ class PaymentAccountsConfig { it, paymentService, paymentProviderHostPort, - token + token, + meterRegistry, + Semaphore(it.parallelRequests) ) } } diff --git a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt index a5909b85b..d31fcfff5 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt @@ -27,11 +27,11 @@ class OrderPayer { private lateinit var paymentService: PaymentService private val paymentExecutor = ThreadPoolExecutor( - 16, - 16, - 0L, + 30, + 50, + 100L, TimeUnit.MILLISECONDS, - LinkedBlockingQueue(8_000), + LinkedBlockingQueue(40), NamedThreadFactory("payment-submission-executor"), CallerBlockingRejectedExecutionHandler() ) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index 5cb12106a..b8f38757c 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -2,15 +2,23 @@ package ru.quipy.payments.logic import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.registerKotlinModule +import io.micrometer.core.instrument.MeterRegistry +import java.util.concurrent.Semaphore import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.RequestBody +import okio.IOException import org.slf4j.LoggerFactory +import ru.quipy.common.utils.SlidingWindowRateLimiter import ru.quipy.core.EventSourcingService +import ru.quipy.exceptions.TooManyRequestsException import ru.quipy.payments.api.PaymentAggregate +import java.io.InterruptedIOException import java.net.SocketTimeoutException import java.time.Duration import java.util.* +import java.util.concurrent.TimeUnit +import kotlin.math.pow // Advice: always treat time as a Duration @@ -19,6 +27,8 @@ class PaymentExternalSystemAdapterImpl( private val paymentESService: EventSourcingService, private val paymentProviderHostPort: String, private val token: String, + private val meterRegistry: MeterRegistry, + private val parallelLimiter: Semaphore ) : PaymentExternalSystemAdapter { companion object { @@ -28,13 +38,21 @@ class PaymentExternalSystemAdapterImpl( val mapper = ObjectMapper().registerKotlinModule() } + private val timer = meterRegistry.timer("payment.external.system.request.latency", "accountName", properties.accountName) private val serviceName = properties.serviceName private val accountName = properties.accountName private val requestAverageProcessingTime = properties.averageProcessingTime private val rateLimitPerSec = properties.rateLimitPerSec private val parallelRequests = properties.parallelRequests + private val rateLimit: SlidingWindowRateLimiter by lazy { + SlidingWindowRateLimiter( + rate = rateLimitPerSec.toLong(), + window = Duration.ofMillis(1000), + ) + } - private val client = OkHttpClient.Builder().build() + private val client = OkHttpClient.Builder() + .callTimeout(1000, TimeUnit.MILLISECONDS).build() override fun performPaymentAsync(paymentId: UUID, amount: Int, paymentStartedAt: Long, deadline: Long) { logger.warn("[$accountName] Submitting payment request for payment $paymentId") @@ -49,26 +67,71 @@ class PaymentExternalSystemAdapterImpl( logger.info("[$accountName] Submit: $paymentId , txId: $transactionId") + if (!rateLimit.tickBlocking(timeToDead(deadline))) { + throw TooManyRequestsException(deadline) + } + parallelLimiter.acquire() + val timeBeforeCall = now() try { val request = Request.Builder().run { url("http://$paymentProviderHostPort/external/process?serviceName=$serviceName&token=$token&accountName=$accountName&transactionId=$transactionId&paymentId=$paymentId&amount=$amount") post(emptyBody) }.build() - client.newCall(request).execute().use { response -> - val body = try { - mapper.readValue(response.body?.string(), ExternalSysResponse::class.java) - } catch (e: Exception) { - logger.error("[$accountName] [ERROR] Payment processed for txId: $transactionId, payment: $paymentId, result code: ${response.code}, reason: ${response.body?.string()}") - ExternalSysResponse(transactionId.toString(), paymentId.toString(),false, e.message) + var isCompletedRequest = false + var retryCount = 0 + + if (timeToDead(deadline) < 0) { + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "deadline was expired") } + return + } + while (!isCompletedRequest && now() < deadline) { + if (timeToDead(deadline) < 0) { + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "deadline was expired") + } + return + } + client.newCall(request).execute().use { response -> + val body = try { + mapper.readValue(response.body?.string(), ExternalSysResponse::class.java) + } catch (e: Exception) { + logger.error("[$accountName] [ERROR] Payment processed for txId: $transactionId, payment: $paymentId, result code: ${response.code}, reason: ${response.body?.string()}") + ExternalSysResponse( + transactionId.toString(), + paymentId.toString(), + false, + e.message + ) + } + isCompletedRequest = if (!body.result && !(response.code >= 500 || response.code == 429)) { + if (retryCount < 3) { + retryCount++ + val backoffTime = (2.0.pow(retryCount.toDouble()) * 10 + Random().nextLong(0, 10)).toLong() + Thread.sleep(backoffTime) + continue + } else { + true + } + } else { + true + } - logger.warn("[$accountName] Payment processed for txId: $transactionId, payment: $paymentId, succeeded: ${body.result}, message: ${body.message}") + logger.warn("[$accountName] Payment processed for txId: $transactionId, payment: $paymentId, succeeded: ${body.result}, message: ${body.message}") - // Здесь мы обновляем состояние оплаты в зависимости от результата в базе данных оплат. - // Это требуется сделать ВО ВСЕХ ИСХОДАХ (успешная оплата / неуспешная / ошибочная ситуация) + // Здесь мы обновляем состояние оплаты в зависимости от результата в базе данных оплат. + // Это требуется сделать ВО ВСЕХ ИСХОДАХ (успешная оплата / неуспешная / ошибочная ситуация) + paymentESService.update(paymentId) { + it.logProcessing(body.result, now(), transactionId, reason = body.message) + } + } + } + + if (timeToDead(deadline) < 0) { paymentESService.update(paymentId) { - it.logProcessing(body.result, now(), transactionId, reason = body.message) + it.logProcessing(false, now(), transactionId, "deadline was expired") } } } catch (e: Exception) { @@ -79,7 +142,18 @@ class PaymentExternalSystemAdapterImpl( it.logProcessing(false, now(), transactionId, reason = "Request timeout.") } } - + is InterruptedIOException -> { + logger.error("[$accountName] Server timeout for txId: $transactionId, payment: $paymentId", e) + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, reason = "Server timeout") + } + } + is IOException -> { + logger.error("[$accountName] Server timeout for txId: $transactionId, payment: $paymentId", e) + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, reason = "Server timeout") + } + } else -> { logger.error("[$accountName] Payment failed for txId: $transactionId, payment: $paymentId", e) @@ -88,6 +162,9 @@ class PaymentExternalSystemAdapterImpl( } } } + } finally { + timer.record(now()-timeBeforeCall, TimeUnit.MILLISECONDS) + parallelLimiter.release() } } @@ -96,7 +173,8 @@ class PaymentExternalSystemAdapterImpl( override fun isEnabled() = properties.enabled override fun name() = properties.accountName - + fun timeToDead(deadline: Long): Long { + return deadline - now() - requestAverageProcessingTime.toMillis() - 200 + } } - public fun now() = System.currentTimeMillis() \ No newline at end of file diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 33d51a58b..3170b6dad 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -2,29 +2,30 @@ server.address=0.0.0.0 server.port=8081 server.http2.enabled=true spring.main.allow-bean-definition-overriding=true - # MongoDB properties spring.data.mongodb.host=localhost spring.data.mongodb.port=27017 spring.data.mongodb.database=online-shop - # Tiny event sourcing library properties event.sourcing.auto-scan-enabled=true event.sourcing.scan-package=ru.quipy event.sourcing.snapshots-enabled=false event.sourcing.sagas-enabled=false - # Postgres event store properties spring.datasource.hikari.jdbc-url=jdbc:postgresql://${POSTGRES_ADDRESS:localhost}:${POSTGRES_PORT:65432}/postgres spring.datasource.hikari.username=tiny_es spring.datasource.hikari.password=tiny_es -spring.datasource.hikari.leak-detection-threshold=2000 +spring.datasource.hikari.leak-detection-threshold=2000 management.metrics.web.server.request.autotime.percentiles=0.95 management.metrics.export.prometheus.enabled=true -management.endpoints.web.exposure.include=info,health,prometheus,metrics - +management.endpoints.web.exposure.include=prometheus,health,info +management.metrics.distribution.percentiles-histogram.http.server.requests=true +management.metrics.distribution.percentiles-histogram.payment.external.system.request.latency=true payment.service-name=${PAYMENT_SERVICE_NAME} payment.token=${PAYMENT_TOKEN} -payment.accounts=${PAYMENT_ACCOUNTS:acc-12,acc-20} -payment.hostPort=${PAYMENT_HOST:localhost}:${PAYMENT_PORT:1234} \ No newline at end of file +# payment.accounts=${PAYMENT_ACCOUNTS:acc-12,acc-20} +# payment.accounts=${PAYMENT_ACCOUNTS:acc-3} +payment.accounts=${PAYMENT_ACCOUNTS:acc-7} +# payment.accounts=${PAYMENT_ACCOUNTS:acc-18} +payment.hostPort=${PAYMENT_HOST:localhost}:${PAYMENT_PORT:1234} diff --git a/test-local-run.http b/test-local-run.http index 7be0e4f73..dc8dbeedf 100644 --- a/test-local-run.http +++ b/test-local-run.http @@ -12,4 +12,4 @@ Content-Type: application/json ### Stop running test to save time and resources # @timeout 120 -POST http://localhost:1234/test/stop/{{serviceName}} \ No newline at end of file +POST http://localhost:4321/test/stop/"{{serviceName}}" \ No newline at end of file diff --git a/test-on-prem-run.http b/test-on-prem-run.http index 584edc0b5..dd5765987 100644 --- a/test-on-prem-run.http +++ b/test-on-prem-run.http @@ -7,13 +7,13 @@ Content-Type: application/json "serviceName": "{{serviceName}}", "token": "{{token}}", "branch": "main", - "accounts": "acc-3", - "ratePerSecond": 11, - "testCount": 1200, + "accounts": "acc-12,acc-20", + "ratePerSecond": 2, + "testCount": 10, "processingTimeMillis": 80000, "onPremises": true } ### Stop running test to save credits # @timeout 120 -POST http://77.234.215.138:31234/test/stop/{{serviceName}} \ No newline at end of file +POST http://77.234.215.138:31234/test/stop/"{{serviceName}}" \ No newline at end of file From cb3b5c72808a30500c0c21fdabc1edf0dcaf7d9d Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Thu, 13 Nov 2025 21:13:11 +0300 Subject: [PATCH 02/61] fix: try to fix client timeout --- .../ru/quipy/apigateway/APIController.kt | 3 +-- .../apigateway/GlobalExceptionHandler.kt | 22 +----------------- .../ru/quipy/payments/logic/OrderPayer.kt | 21 +++++++++++++---- .../logic/PaymentExternalServiceImpl.kt | 23 ++++++++----------- 4 files changed, 28 insertions(+), 41 deletions(-) diff --git a/src/main/kotlin/ru/quipy/apigateway/APIController.kt b/src/main/kotlin/ru/quipy/apigateway/APIController.kt index fff5ac386..c07d96b7a 100644 --- a/src/main/kotlin/ru/quipy/apigateway/APIController.kt +++ b/src/main/kotlin/ru/quipy/apigateway/APIController.kt @@ -7,7 +7,6 @@ import org.springframework.web.bind.annotation.* import ru.quipy.common.utils.LeakingBucketRateLimiter import ru.quipy.common.utils.RateLimiter import ru.quipy.exceptions.TooManyRequestsException -import ru.quipy.exceptions.TooManyRequestsRetriableException import ru.quipy.orders.repository.OrderRepository import ru.quipy.payments.logic.OrderPayer import java.time.Duration @@ -18,7 +17,7 @@ class APIController( private val orderRepository: OrderRepository, private val orderPayer: OrderPayer, @field:Qualifier("parallelLimiter") - private val rateLimiter: RateLimiter = LeakingBucketRateLimiter(8, Duration.ofSeconds(1), 38) + private val rateLimiter: RateLimiter = LeakingBucketRateLimiter(8, Duration.ofSeconds(1), 30) ) { val logger: Logger = LoggerFactory.getLogger(APIController::class.java) diff --git a/src/main/kotlin/ru/quipy/apigateway/GlobalExceptionHandler.kt b/src/main/kotlin/ru/quipy/apigateway/GlobalExceptionHandler.kt index d6c824805..5679ea372 100644 --- a/src/main/kotlin/ru/quipy/apigateway/GlobalExceptionHandler.kt +++ b/src/main/kotlin/ru/quipy/apigateway/GlobalExceptionHandler.kt @@ -4,9 +4,7 @@ import org.slf4j.LoggerFactory import org.springframework.http.HttpStatus import org.springframework.http.ResponseEntity import org.springframework.web.bind.annotation.ExceptionHandler -import org.springframework.web.bind.annotation.ResponseStatus import org.springframework.web.bind.annotation.RestControllerAdvice -import ru.quipy.exceptions.TooManyRequestsRetriableException import ru.quipy.exceptions.TooLongRequestException import ru.quipy.exceptions.TooManyRequestsException import java.util.concurrent.atomic.AtomicInteger @@ -19,7 +17,6 @@ class GlobalExceptionHandler( val logger = LoggerFactory.getLogger(GlobalExceptionHandler::class.java) private var currentRetryAfterSeconds = 1 private var exp_base = 2; - private const val MAX_RETRY_AFTER_MS = 256 private const val RESET_WINDOW_MS = 30000L } @@ -31,23 +28,6 @@ class GlobalExceptionHandler( } @ExceptionHandler(TooManyRequestsException::class) fun handleTooManyRequestsRetriable(exception: TooManyRequestsException): ResponseEntity { - logger.warn("to many request") - val currentTime = System.currentTimeMillis() - val lastRejection = lastRejectionTime.get() - - if (currentTime - lastRejection > RESET_WINDOW_MS) { - rejectedRequestsCount.set(0) - currentRetryAfterSeconds = 1 - } - lastRejectionTime.set(currentTime) - - if (rejectedRequestsCount.get() < 4 && exception.deadline < System.currentTimeMillis() + 1200) { - return ResponseEntity - .status(HttpStatus.TOO_MANY_REQUESTS) - .header("Retry-After", MAX_RETRY_AFTER_MS.coerceAtLeast(currentRetryAfterSeconds).toString()) - .build() - } - - return ResponseEntity.status(200).build() + return ResponseEntity.status(HttpStatus.TOO_MANY_REQUESTS).header("Retry-After", "1").body("to many requests") } } \ No newline at end of file diff --git a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt index d31fcfff5..359aa0a59 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt @@ -6,8 +6,12 @@ import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service import ru.quipy.common.utils.CallerBlockingRejectedExecutionHandler import ru.quipy.common.utils.NamedThreadFactory +import ru.quipy.common.utils.SlidingWindowRateLimiter import ru.quipy.core.EventSourcingService +import ru.quipy.exceptions.RateLimitWasBreached +import ru.quipy.exceptions.TooManyRequestsException import ru.quipy.payments.api.PaymentAggregate +import java.time.Duration import java.util.* import java.util.concurrent.LinkedBlockingQueue import java.util.concurrent.ThreadPoolExecutor @@ -19,6 +23,12 @@ class OrderPayer { companion object { val logger: Logger = LoggerFactory.getLogger(OrderPayer::class.java) } + private val rateLimit: SlidingWindowRateLimiter by lazy { + SlidingWindowRateLimiter( + rate = 8, + window = Duration.ofMillis(1000), + ) + } @Autowired private lateinit var paymentESService: EventSourcingService @@ -27,17 +37,20 @@ class OrderPayer { private lateinit var paymentService: PaymentService private val paymentExecutor = ThreadPoolExecutor( - 30, - 50, - 100L, + 16, + 16, + 0, TimeUnit.MILLISECONDS, - LinkedBlockingQueue(40), + LinkedBlockingQueue(8_000), NamedThreadFactory("payment-submission-executor"), CallerBlockingRejectedExecutionHandler() ) fun processPayment(orderId: UUID, amount: Int, paymentId: UUID, deadline: Long): Long { val createdAt = System.currentTimeMillis() + if (!rateLimit.tickBlocking(deadline- System.currentTimeMillis())) { + throw TooManyRequestsException(deadline) + } paymentExecutor.submit { val createdEvent = paymentESService.create { it.create( diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index b8f38757c..fd368f079 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -3,7 +3,6 @@ package ru.quipy.payments.logic import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.registerKotlinModule import io.micrometer.core.instrument.MeterRegistry -import java.util.concurrent.Semaphore import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.RequestBody @@ -11,12 +10,12 @@ import okio.IOException import org.slf4j.LoggerFactory import ru.quipy.common.utils.SlidingWindowRateLimiter import ru.quipy.core.EventSourcingService -import ru.quipy.exceptions.TooManyRequestsException import ru.quipy.payments.api.PaymentAggregate import java.io.InterruptedIOException import java.net.SocketTimeoutException import java.time.Duration import java.util.* +import java.util.concurrent.Semaphore import java.util.concurrent.TimeUnit import kotlin.math.pow @@ -27,7 +26,7 @@ class PaymentExternalSystemAdapterImpl( private val paymentESService: EventSourcingService, private val paymentProviderHostPort: String, private val token: String, - private val meterRegistry: MeterRegistry, + meterRegistry: MeterRegistry, private val parallelLimiter: Semaphore ) : PaymentExternalSystemAdapter { @@ -44,15 +43,9 @@ class PaymentExternalSystemAdapterImpl( private val requestAverageProcessingTime = properties.averageProcessingTime private val rateLimitPerSec = properties.rateLimitPerSec private val parallelRequests = properties.parallelRequests - private val rateLimit: SlidingWindowRateLimiter by lazy { - SlidingWindowRateLimiter( - rate = rateLimitPerSec.toLong(), - window = Duration.ofMillis(1000), - ) - } private val client = OkHttpClient.Builder() - .callTimeout(1000, TimeUnit.MILLISECONDS).build() + .callTimeout(1100, TimeUnit.MILLISECONDS).build() override fun performPaymentAsync(paymentId: UUID, amount: Int, paymentStartedAt: Long, deadline: Long) { logger.warn("[$accountName] Submitting payment request for payment $paymentId") @@ -67,10 +60,12 @@ class PaymentExternalSystemAdapterImpl( logger.info("[$accountName] Submit: $paymentId , txId: $transactionId") - if (!rateLimit.tickBlocking(timeToDead(deadline))) { - throw TooManyRequestsException(deadline) + if (!parallelLimiter.tryAcquire(timeToDead(deadline), TimeUnit.MILLISECONDS)) { + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "deadline was expired") + } + return } - parallelLimiter.acquire() val timeBeforeCall = now() try { val request = Request.Builder().run { @@ -174,7 +169,7 @@ class PaymentExternalSystemAdapterImpl( override fun name() = properties.accountName fun timeToDead(deadline: Long): Long { - return deadline - now() - requestAverageProcessingTime.toMillis() - 200 + return deadline - now() } } public fun now() = System.currentTimeMillis() \ No newline at end of file From 76511a37560356fb4f32539cbaae2916bd84f8bc Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Thu, 13 Nov 2025 21:17:22 +0300 Subject: [PATCH 03/61] fix: try to fix client timeout --- Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3d9587cd0..c8b23fe7f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM maven:3.9.9-eclipse-temurin-17 AS build +FROM eclipse-temurin:17-alpine-3.22 AS build WORKDIR /app COPY pom.xml . @@ -6,8 +6,6 @@ RUN mvn dependency:go-offline COPY src src RUN mvn package -FROM eclipse-temurin:17-alpine-3.22 - COPY --from=build /app/target/*.jar /high-load-course.jar CMD ["java", "-jar", "/high-load-course.jar"] From f25a3a5a312df765418d0ba430b3551bd645ef41 Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Thu, 13 Nov 2025 21:19:44 +0300 Subject: [PATCH 04/61] fix: try to fix client timeout --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c8b23fe7f..3d9587cd0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM eclipse-temurin:17-alpine-3.22 AS build +FROM maven:3.9.9-eclipse-temurin-17 AS build WORKDIR /app COPY pom.xml . @@ -6,6 +6,8 @@ RUN mvn dependency:go-offline COPY src src RUN mvn package +FROM eclipse-temurin:17-alpine-3.22 + COPY --from=build /app/target/*.jar /high-load-course.jar CMD ["java", "-jar", "/high-load-course.jar"] From 6a90fe47630dc018f433565b90fb77342a66a446 Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Thu, 13 Nov 2025 21:22:45 +0300 Subject: [PATCH 05/61] fix: try to fix client timeout --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3d9587cd0..de31b9fe7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,4 +11,3 @@ FROM eclipse-temurin:17-alpine-3.22 COPY --from=build /app/target/*.jar /high-load-course.jar CMD ["java", "-jar", "/high-load-course.jar"] - From 524e42babcba02b3f6a6733a7c74b7a9a3023fca Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Thu, 13 Nov 2025 21:23:39 +0300 Subject: [PATCH 06/61] fix: try to fix client timeout --- src/main/kotlin/ru/quipy/exceptions/RateLimitWasBreached.kt | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 src/main/kotlin/ru/quipy/exceptions/RateLimitWasBreached.kt diff --git a/src/main/kotlin/ru/quipy/exceptions/RateLimitWasBreached.kt b/src/main/kotlin/ru/quipy/exceptions/RateLimitWasBreached.kt new file mode 100644 index 000000000..8469ea9a3 --- /dev/null +++ b/src/main/kotlin/ru/quipy/exceptions/RateLimitWasBreached.kt @@ -0,0 +1,4 @@ +package ru.quipy.exceptions + +class RateLimitWasBreached : RuntimeException() { +} \ No newline at end of file From b184e9c0c3c1d6a47d93a9c80fbf3f63c58d2fdd Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Fri, 14 Nov 2025 15:30:13 +0300 Subject: [PATCH 07/61] fix: try to fix client timeout --- .../logic/PaymentExternalServiceImpl.kt | 163 +++++++++--------- 1 file changed, 83 insertions(+), 80 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index fd368f079..f0e84098c 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -8,7 +8,6 @@ import okhttp3.Request import okhttp3.RequestBody import okio.IOException import org.slf4j.LoggerFactory -import ru.quipy.common.utils.SlidingWindowRateLimiter import ru.quipy.core.EventSourcingService import ru.quipy.payments.api.PaymentAggregate import java.io.InterruptedIOException @@ -60,106 +59,110 @@ class PaymentExternalSystemAdapterImpl( logger.info("[$accountName] Submit: $paymentId , txId: $transactionId") - if (!parallelLimiter.tryAcquire(timeToDead(deadline), TimeUnit.MILLISECONDS)) { - paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "deadline was expired") + var retryCount = 0 + while (true) { + val remaining = deadline - now() + if (remaining <= 0) { + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "deadline expired") + } + return } - return - } - val timeBeforeCall = now() - try { - val request = Request.Builder().run { - url("http://$paymentProviderHostPort/external/process?serviceName=$serviceName&token=$token&accountName=$accountName&transactionId=$transactionId&paymentId=$paymentId&amount=$amount") - post(emptyBody) - }.build() - - var isCompletedRequest = false - var retryCount = 0 - if (timeToDead(deadline) < 0) { + if (!parallelLimiter.tryAcquire(remaining, TimeUnit.MILLISECONDS)) { paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "deadline was expired") + it.logProcessing(false, now(), transactionId, "parallel limiter timeout") } return } - while (!isCompletedRequest && now() < deadline) { - if (timeToDead(deadline) < 0) { - paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "deadline was expired") - } - return - } - client.newCall(request).execute().use { response -> - val body = try { - mapper.readValue(response.body?.string(), ExternalSysResponse::class.java) - } catch (e: Exception) { - logger.error("[$accountName] [ERROR] Payment processed for txId: $transactionId, payment: $paymentId, result code: ${response.code}, reason: ${response.body?.string()}") - ExternalSysResponse( - transactionId.toString(), - paymentId.toString(), - false, - e.message - ) - } - isCompletedRequest = if (!body.result && !(response.code >= 500 || response.code == 429)) { - if (retryCount < 3) { - retryCount++ - val backoffTime = (2.0.pow(retryCount.toDouble()) * 10 + Random().nextLong(0, 10)).toLong() - Thread.sleep(backoffTime) - continue - } else { - true - } - } else { - true + + val timeBeforeCall = now() + var shouldRetry = false + try { + val perCallTimeoutMs = remaining.coerceAtMost(1100) + val request = Request.Builder() + .url( + "http://$paymentProviderHostPort/external/process" + + "?serviceName=$serviceName&token=$token&accountName=$accountName" + + "&transactionId=$transactionId&paymentId=$paymentId&amount=$amount" + ) + .post(emptyBody) + .build() + + val call = client.newCall(request) + call.timeout().timeout(perCallTimeoutMs, TimeUnit.MILLISECONDS) + + call.execute().use { response -> + val rawBody = response.body?.string() + val parsed = try { + mapper.readValue(rawBody, ExternalSysResponse::class.java) + } catch (ex: Exception) { + ExternalSysResponse(transactionId.toString(), paymentId.toString(), false, ex.message) } - logger.warn("[$accountName] Payment processed for txId: $transactionId, payment: $paymentId, succeeded: ${body.result}, message: ${body.message}") + shouldRetry = !parsed.result && (response.code == 429 || response.code >= 500) - // Здесь мы обновляем состояние оплаты в зависимости от результата в базе данных оплат. - // Это требуется сделать ВО ВСЕХ ИСХОДАХ (успешная оплата / неуспешная / ошибочная ситуация) paymentESService.update(paymentId) { - it.logProcessing(body.result, now(), transactionId, reason = body.message) + it.logProcessing(parsed.result, now(), transactionId, parsed.message) + } + + if (parsed.result) { + return } } - } + } catch (e: SocketTimeoutException) { + logger.warn("[$accountName] attempt ${retryCount + 1} timeout: $paymentId", e) + shouldRetry = true + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "socket timeout") + } + } catch (e: InterruptedIOException) { + logger.warn("[$accountName] interrupted: $paymentId", e) + shouldRetry = true - if (timeToDead(deadline) < 0) { + // Здесь мы обновляем состояние оплаты в зависимости от результата в базе данных оплат. + // Это требуется сделать ВО ВСЕХ ИСХОДАХ (успешная оплата / неуспешная / ошибочная ситуация) paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "deadline was expired") + it.logProcessing(false, now(), transactionId, "interrupted IO") } - } - } catch (e: Exception) { - when (e) { - is SocketTimeoutException -> { - logger.error("[$accountName] Payment timeout for txId: $transactionId, payment: $paymentId", e) - paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, reason = "Request timeout.") - } + } catch (e: IOException) { + logger.warn("[$accountName] io error: $paymentId", e) + shouldRetry = true + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "io exception") } - is InterruptedIOException -> { - logger.error("[$accountName] Server timeout for txId: $transactionId, payment: $paymentId", e) - paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, reason = "Server timeout") - } + } catch (e: Exception) { + logger.error("[$accountName] non-retriable error: $paymentId", e) + shouldRetry = false + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, e.message) } - is IOException -> { - logger.error("[$accountName] Server timeout for txId: $transactionId, payment: $paymentId", e) - paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, reason = "Server timeout") - } + } finally { + timer.record(now() - timeBeforeCall, TimeUnit.MILLISECONDS) + parallelLimiter.release() + } + + if (!shouldRetry) { + return + } + + retryCount++ + if (retryCount >= 3) { + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "Max attempts reached") } - else -> { - logger.error("[$accountName] Payment failed for txId: $transactionId, payment: $paymentId", e) + return + } - paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, reason = e.message) - } + val backoff = ((2.0.pow(retryCount.toDouble()) * 25).toLong() + kotlin.random.Random.nextLong(10)) + val capped = backoff.coerceAtMost(deadline - now() - 5) + if (capped <= 0) { + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "Deadline expired") } + return } - } finally { - timer.record(now()-timeBeforeCall, TimeUnit.MILLISECONDS) - parallelLimiter.release() + Thread.sleep(capped) } } From 148a40e0d426b027db648b46cc66afbb9dce0d26 Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Thu, 4 Dec 2025 15:37:52 +0300 Subject: [PATCH 08/61] feature: added corutines and callback logic --- .../kotlin/ru/quipy/OnlineShopApplication.kt | 5 +- .../ru/quipy/apigateway/APIController.kt | 4 +- .../orders/subscribers/PaymentSubscriber.kt | 2 +- .../ru/quipy/payments/logic/OrderPayer.kt | 22 ++-- .../quipy/payments/logic/PaymentCallback.kt | 90 ++++++++++++++ .../logic/PaymentExternalServiceImpl.kt | 112 +++++------------- .../ru/quipy/payments/logic/PaymentService.kt | 2 +- .../payments/logic/PaymentServiceImpl.kt | 10 +- src/main/resources/application.properties | 2 +- 9 files changed, 134 insertions(+), 115 deletions(-) create mode 100644 src/main/kotlin/ru/quipy/payments/logic/PaymentCallback.kt diff --git a/src/main/kotlin/ru/quipy/OnlineShopApplication.kt b/src/main/kotlin/ru/quipy/OnlineShopApplication.kt index 9ac22e094..1fcd3f89c 100644 --- a/src/main/kotlin/ru/quipy/OnlineShopApplication.kt +++ b/src/main/kotlin/ru/quipy/OnlineShopApplication.kt @@ -1,5 +1,6 @@ package ru.quipy +import kotlinx.coroutines.asCoroutineDispatcher import org.slf4j.Logger import org.slf4j.LoggerFactory import org.springframework.boot.autoconfigure.SpringBootApplication @@ -13,10 +14,10 @@ class OnlineShopApplication { val log: Logger = LoggerFactory.getLogger(OnlineShopApplication::class.java) companion object { - val appExecutor = Executors.newFixedThreadPool(64, NamedThreadFactory("main-app-executor")) + val appExecutor = Executors.newFixedThreadPool(20_000, NamedThreadFactory("main-app-executor")).asCoroutineDispatcher() } } -fun main(args: Array) { +suspend fun main(args: Array) { runApplication(*args) } diff --git a/src/main/kotlin/ru/quipy/apigateway/APIController.kt b/src/main/kotlin/ru/quipy/apigateway/APIController.kt index c07d96b7a..3720312da 100644 --- a/src/main/kotlin/ru/quipy/apigateway/APIController.kt +++ b/src/main/kotlin/ru/quipy/apigateway/APIController.kt @@ -17,7 +17,7 @@ class APIController( private val orderRepository: OrderRepository, private val orderPayer: OrderPayer, @field:Qualifier("parallelLimiter") - private val rateLimiter: RateLimiter = LeakingBucketRateLimiter(8, Duration.ofSeconds(1), 30) + private val rateLimiter: RateLimiter = LeakingBucketRateLimiter(1100, Duration.ofSeconds(1), 4400) ) { val logger: Logger = LoggerFactory.getLogger(APIController::class.java) @@ -58,7 +58,7 @@ class APIController( } @PostMapping("/orders/{orderId}/payment") - fun payOrder(@PathVariable orderId: UUID, @RequestParam deadline: Long): PaymentSubmissionDto { + suspend fun payOrder(@PathVariable orderId: UUID, @RequestParam deadline: Long): PaymentSubmissionDto { if (!rateLimiter.tick()) { throw TooManyRequestsException(deadline) } diff --git a/src/main/kotlin/ru/quipy/orders/subscribers/PaymentSubscriber.kt b/src/main/kotlin/ru/quipy/orders/subscribers/PaymentSubscriber.kt index a4c312eaf..0b7560804 100644 --- a/src/main/kotlin/ru/quipy/orders/subscribers/PaymentSubscriber.kt +++ b/src/main/kotlin/ru/quipy/orders/subscribers/PaymentSubscriber.kt @@ -36,7 +36,7 @@ class PaymentSubscriber { retryConf = RetryConf(1, RetryFailedStrategy.SKIP_EVENT) ) { `when`(PaymentProcessedEvent::class) { event -> - appExecutor.submit { + appExecutor.run { logger.trace( "Payment results. OrderId ${event.orderId}, succeeded: ${event.success}, txId: ${event.transactionId}, reason: ${event.reason}, duration: ${ Duration.ofMillis( diff --git a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt index 359aa0a59..4835a1d26 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt @@ -36,33 +36,25 @@ class OrderPayer { @Autowired private lateinit var paymentService: PaymentService - private val paymentExecutor = ThreadPoolExecutor( - 16, - 16, - 0, - TimeUnit.MILLISECONDS, - LinkedBlockingQueue(8_000), - NamedThreadFactory("payment-submission-executor"), - CallerBlockingRejectedExecutionHandler() - ) - fun processPayment(orderId: UUID, amount: Int, paymentId: UUID, deadline: Long): Long { + suspend fun processPayment(orderId: UUID, amount: Int, paymentId: UUID, deadline: Long): Long { val createdAt = System.currentTimeMillis() if (!rateLimit.tickBlocking(deadline- System.currentTimeMillis())) { throw TooManyRequestsException(deadline) } - paymentExecutor.submit { - val createdEvent = paymentESService.create { + + val createdEvent = paymentESService.create { it.create( paymentId, orderId, amount ) } - logger.trace("Payment ${createdEvent.paymentId} for order $orderId created.") - paymentService.submitPaymentRequest(paymentId, amount, createdAt, deadline) - } + logger.trace("Payment ${createdEvent.paymentId} for order $orderId created.") + + paymentService.submitPaymentRequest(paymentId, amount, createdAt, deadline) + return createdAt } } \ No newline at end of file diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentCallback.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentCallback.kt new file mode 100644 index 000000000..3b37418c6 --- /dev/null +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentCallback.kt @@ -0,0 +1,90 @@ +package ru.quipy.payments.logic + +import io.micrometer.core.instrument.Timer +import kotlinx.coroutines.sync.Semaphore +import okhttp3.Call +import okhttp3.Callback +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.Response +import ru.quipy.core.EventSourcingService +import ru.quipy.payments.api.PaymentAggregate +import ru.quipy.payments.logic.PaymentExternalSystemAdapterImpl.Companion.logger +import ru.quipy.payments.logic.PaymentExternalSystemAdapterImpl.Companion.mapper +import java.io.InterruptedIOException +import java.net.SocketTimeoutException +import java.util.UUID +import java.util.concurrent.TimeUnit +import kotlin.math.pow + + +class PaymentCallback(val semaphore: Semaphore, val accountName: String, val retryCount: Int, val paymentId: UUID, val transactionId: UUID, val paymentESService: EventSourcingService, val client: OkHttpClient, val request: Request, val timer: Timer, val deadline: Long, val timeBeforeCall: Long) : Callback { + override fun onFailure(call: Call, e: java.io.IOException) { + + logger.debug("fail in callback for payment: {}, retry count: {}, deadline: {}, in time: {}", paymentId, retryCount, deadline, now(), e) + when (e) { + is SocketTimeoutException -> { + logger.warn("[$accountName] attempt ${retryCount + 1} timeout: $paymentId", e) + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "socket timeout") + } + } + + is InterruptedIOException -> { + logger.warn("[$accountName] interrupted: $paymentId", e) + + // Здесь мы обновляем состояние оплаты в зависимости от результата в базе данных оплат. + // Это требуется сделать ВО ВСЕХ ИСХОДАХ (успешная оплата / неуспешная / ошибочная ситуация) + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "interrupted IO") + } + } + + else -> { + logger.warn("[$accountName] io error: $paymentId", e) + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "io exception") + } + } + } + + if (retryCount + 1 >= 3) { + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "Max attempts reached") + } + return + } + + val backoff = ((2.0.pow(retryCount.toDouble()) * 25).toLong() + kotlin.random.Random.nextLong(10)) + val capped = backoff.coerceAtMost(deadline - now() - 5) + if (capped <= 0) { + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "Deadline expired") + } + return + } + val nCall = client.newCall(request) + nCall.enqueue(PaymentCallback(semaphore, accountName, retryCount + 1, paymentId, transactionId, paymentESService, client, request, timer, deadline, now())) + timer.record(now() - timeBeforeCall, TimeUnit.MILLISECONDS) + semaphore.release() + + } + + override fun onResponse(call: Call, response: Response) { + + logger.debug("success in callback for payment: {}, retry count: {}, deadline: {}, in time: {}", paymentId, retryCount, deadline, now()) + + val rawBody = response.body?.string() + val parsed = try { + mapper.readValue(rawBody, ExternalSysResponse::class.java) + } catch (ex: Exception) { + ExternalSysResponse(transactionId.toString(), paymentId.toString(), false, ex.message) + } + + paymentESService.update(paymentId) { + it.logProcessing(parsed.result, now(), transactionId, parsed.message) + } + } + + fun now() = System.currentTimeMillis() +} \ No newline at end of file diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index f0e84098c..a1489a699 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -3,11 +3,17 @@ package ru.quipy.payments.logic import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.registerKotlinModule import io.micrometer.core.instrument.MeterRegistry +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.callbackFlow +import okhttp3.Call +import okhttp3.Callback import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.RequestBody +import okhttp3.Response import okio.IOException import org.slf4j.LoggerFactory +import org.springframework.boot.autoconfigure.integration.IntegrationProperties import ru.quipy.core.EventSourcingService import ru.quipy.payments.api.PaymentAggregate import java.io.InterruptedIOException @@ -35,7 +41,7 @@ class PaymentExternalSystemAdapterImpl( val emptyBody = RequestBody.create(null, ByteArray(0)) val mapper = ObjectMapper().registerKotlinModule() } - + // 2025-11-20T20:30:35.780+03:00 INFO 56644 --- [alhost:1234/...] ru.quipy.core.EventSourcingService : Optimistic lock exception. Failed to save event records id: [7dca693e-e811-4b7f-8bce-23e13d952c04-4] private val timer = meterRegistry.timer("payment.external.system.request.latency", "accountName", properties.accountName) private val serviceName = properties.serviceName private val accountName = properties.accountName @@ -78,91 +84,29 @@ class PaymentExternalSystemAdapterImpl( val timeBeforeCall = now() var shouldRetry = false - try { - val perCallTimeoutMs = remaining.coerceAtMost(1100) - val request = Request.Builder() - .url( - "http://$paymentProviderHostPort/external/process" + + val perCallTimeoutMs = remaining.coerceAtMost(1100) + val request = Request.Builder() + .url( + "http://$paymentProviderHostPort/external/process" + "?serviceName=$serviceName&token=$token&accountName=$accountName" + "&transactionId=$transactionId&paymentId=$paymentId&amount=$amount" - ) - .post(emptyBody) - .build() - - val call = client.newCall(request) - call.timeout().timeout(perCallTimeoutMs, TimeUnit.MILLISECONDS) - - call.execute().use { response -> - val rawBody = response.body?.string() - val parsed = try { - mapper.readValue(rawBody, ExternalSysResponse::class.java) - } catch (ex: Exception) { - ExternalSysResponse(transactionId.toString(), paymentId.toString(), false, ex.message) - } - - shouldRetry = !parsed.result && (response.code == 429 || response.code >= 500) - - paymentESService.update(paymentId) { - it.logProcessing(parsed.result, now(), transactionId, parsed.message) - } - - if (parsed.result) { - return - } - } - } catch (e: SocketTimeoutException) { - logger.warn("[$accountName] attempt ${retryCount + 1} timeout: $paymentId", e) - shouldRetry = true - paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "socket timeout") - } - } catch (e: InterruptedIOException) { - logger.warn("[$accountName] interrupted: $paymentId", e) - shouldRetry = true - - // Здесь мы обновляем состояние оплаты в зависимости от результата в базе данных оплат. - // Это требуется сделать ВО ВСЕХ ИСХОДАХ (успешная оплата / неуспешная / ошибочная ситуация) - paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "interrupted IO") - } - } catch (e: IOException) { - logger.warn("[$accountName] io error: $paymentId", e) - shouldRetry = true - paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "io exception") - } - } catch (e: Exception) { - logger.error("[$accountName] non-retriable error: $paymentId", e) - shouldRetry = false - paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, e.message) - } - } finally { - timer.record(now() - timeBeforeCall, TimeUnit.MILLISECONDS) - parallelLimiter.release() - } - - if (!shouldRetry) { - return - } - - retryCount++ - if (retryCount >= 3) { - paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "Max attempts reached") - } - return - } - - val backoff = ((2.0.pow(retryCount.toDouble()) * 25).toLong() + kotlin.random.Random.nextLong(10)) - val capped = backoff.coerceAtMost(deadline - now() - 5) - if (capped <= 0) { - paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "Deadline expired") - } - return - } - Thread.sleep(capped) + ) + .post(emptyBody) + .build() + + val call = client.newCall(request).enqueue(PaymentCallback( + kotlinx.coroutines.sync.Semaphore(parallelRequests), + accountName, + retryCount, + paymentId, + transactionId, + paymentESService, + client, + request, + timer, + deadline, + timeBeforeCall + )) } } diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentService.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentService.kt index 255db77dd..848f40474 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentService.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentService.kt @@ -7,7 +7,7 @@ interface PaymentService { /** * Submit payment request to some external service. */ - fun submitPaymentRequest(paymentId: UUID, amount: Int, paymentStartedAt: Long, deadline: Long) + suspend fun submitPaymentRequest(paymentId: UUID, amount: Int, paymentStartedAt: Long, deadline: Long) } /** diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentServiceImpl.kt index 1c24e5a72..3fb881462 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentServiceImpl.kt @@ -1,16 +1,8 @@ package ru.quipy.payments.logic import org.slf4j.LoggerFactory -import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service -import ru.quipy.common.utils.NamedThreadFactory -import ru.quipy.core.EventSourcingService -import ru.quipy.payments.api.PaymentAggregate -import java.time.Duration import java.util.* -import java.util.concurrent.Executors -import java.util.concurrent.locks.ReentrantLock -import kotlin.concurrent.withLock @Service @@ -21,7 +13,7 @@ class PaymentSystemImpl( val logger = LoggerFactory.getLogger(PaymentSystemImpl::class.java) } - override fun submitPaymentRequest(paymentId: UUID, amount: Int, paymentStartedAt: Long, deadline: Long) { + override suspend fun submitPaymentRequest(paymentId: UUID, amount: Int, paymentStartedAt: Long, deadline: Long) { for (account in paymentAccounts) { account.performPaymentAsync(paymentId, amount, paymentStartedAt, deadline) } diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 3170b6dad..c06d8d823 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -26,6 +26,6 @@ payment.service-name=${PAYMENT_SERVICE_NAME} payment.token=${PAYMENT_TOKEN} # payment.accounts=${PAYMENT_ACCOUNTS:acc-12,acc-20} # payment.accounts=${PAYMENT_ACCOUNTS:acc-3} -payment.accounts=${PAYMENT_ACCOUNTS:acc-7} +payment.accounts=${PAYMENT_ACCOUNTS:acc-9} # payment.accounts=${PAYMENT_ACCOUNTS:acc-18} payment.hostPort=${PAYMENT_HOST:localhost}:${PAYMENT_PORT:1234} From 3f76471a2041eed34cd9d9f0852d9ec99aa099f0 Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Thu, 4 Dec 2025 19:37:06 +0300 Subject: [PATCH 09/61] fix: fixed semaphore acquiring and releasing logic --- .../payments/config/PaymentAccountsConfig.kt | 2 +- .../quipy/payments/logic/PaymentCallback.kt | 39 +++++++++------- .../logic/PaymentExternalServiceImpl.kt | 45 +++++++------------ src/main/resources/application.properties | 4 +- 4 files changed, 44 insertions(+), 46 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/config/PaymentAccountsConfig.kt b/src/main/kotlin/ru/quipy/payments/config/PaymentAccountsConfig.kt index e10bd805a..8012d1ee7 100644 --- a/src/main/kotlin/ru/quipy/payments/config/PaymentAccountsConfig.kt +++ b/src/main/kotlin/ru/quipy/payments/config/PaymentAccountsConfig.kt @@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule import com.fasterxml.jackson.module.kotlin.registerKotlinModule import io.micrometer.core.instrument.MeterRegistry +import kotlinx.coroutines.sync.Semaphore import org.springframework.beans.factory.annotation.Value import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration @@ -18,7 +19,6 @@ import java.net.http.HttpClient import java.net.http.HttpRequest import java.net.http.HttpResponse import java.util.* -import java.util.concurrent.Semaphore @Configuration diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentCallback.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentCallback.kt index 3b37418c6..f1962a973 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentCallback.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentCallback.kt @@ -19,6 +19,7 @@ import kotlin.math.pow class PaymentCallback(val semaphore: Semaphore, val accountName: String, val retryCount: Int, val paymentId: UUID, val transactionId: UUID, val paymentESService: EventSourcingService, val client: OkHttpClient, val request: Request, val timer: Timer, val deadline: Long, val timeBeforeCall: Long) : Callback { + private val remaining: Long = 10_000L //ms override fun onFailure(call: Call, e: java.io.IOException) { logger.debug("fail in callback for payment: {}, retry count: {}, deadline: {}, in time: {}", paymentId, retryCount, deadline, now(), e) @@ -52,6 +53,7 @@ class PaymentCallback(val semaphore: Semaphore, val accountName: String, val ret paymentESService.update(paymentId) { it.logProcessing(false, now(), transactionId, "Max attempts reached") } + semaphore.release() return } @@ -61,30 +63,37 @@ class PaymentCallback(val semaphore: Semaphore, val accountName: String, val ret paymentESService.update(paymentId) { it.logProcessing(false, now(), transactionId, "Deadline expired") } + semaphore.release() return } val nCall = client.newCall(request) nCall.enqueue(PaymentCallback(semaphore, accountName, retryCount + 1, paymentId, transactionId, paymentESService, client, request, timer, deadline, now())) timer.record(now() - timeBeforeCall, TimeUnit.MILLISECONDS) - semaphore.release() - } override fun onResponse(call: Call, response: Response) { + try { + logger.warn("Free space in semaphore: {}", semaphore.availablePermits) + logger.info( + "success in callback for payment: {}, retry count: {}, deadline: {}, in time: {}", + paymentId, + retryCount, + deadline, + now() + ) + val rawBody = response.body?.string() + val parsed = try { + mapper.readValue(rawBody, ExternalSysResponse::class.java) + } catch (ex: Exception) { + ExternalSysResponse(transactionId.toString(), paymentId.toString(), false, ex.message) + } - logger.debug("success in callback for payment: {}, retry count: {}, deadline: {}, in time: {}", paymentId, retryCount, deadline, now()) - - val rawBody = response.body?.string() - val parsed = try { - mapper.readValue(rawBody, ExternalSysResponse::class.java) - } catch (ex: Exception) { - ExternalSysResponse(transactionId.toString(), paymentId.toString(), false, ex.message) - } - - paymentESService.update(paymentId) { - it.logProcessing(parsed.result, now(), transactionId, parsed.message) + paymentESService.update(paymentId) { + it.logProcessing(parsed.result, now(), transactionId, parsed.message) + } + } finally { + semaphore.release() + timer.record(now() - timeBeforeCall, TimeUnit.MILLISECONDS) } } - - fun now() = System.currentTimeMillis() } \ No newline at end of file diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index a1489a699..a0cc37859 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -3,26 +3,16 @@ package ru.quipy.payments.logic import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.registerKotlinModule import io.micrometer.core.instrument.MeterRegistry -import kotlinx.coroutines.delay -import kotlinx.coroutines.flow.callbackFlow -import okhttp3.Call -import okhttp3.Callback +import kotlinx.coroutines.sync.Semaphore import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.RequestBody -import okhttp3.Response -import okio.IOException import org.slf4j.LoggerFactory -import org.springframework.boot.autoconfigure.integration.IntegrationProperties import ru.quipy.core.EventSourcingService import ru.quipy.payments.api.PaymentAggregate -import java.io.InterruptedIOException -import java.net.SocketTimeoutException import java.time.Duration import java.util.* -import java.util.concurrent.Semaphore import java.util.concurrent.TimeUnit -import kotlin.math.pow // Advice: always treat time as a Duration @@ -50,11 +40,14 @@ class PaymentExternalSystemAdapterImpl( private val parallelRequests = properties.parallelRequests private val client = OkHttpClient.Builder() - .callTimeout(1100, TimeUnit.MILLISECONDS).build() + .callTimeout(30_000, TimeUnit.MILLISECONDS).build() override fun performPaymentAsync(paymentId: UUID, amount: Int, paymentStartedAt: Long, deadline: Long) { logger.warn("[$accountName] Submitting payment request for payment $paymentId") + + + fun now() = System.currentTimeMillis() val transactionId = UUID.randomUUID() // Вне зависимости от исхода оплаты важно отметить что она была отправлена. @@ -66,7 +59,7 @@ class PaymentExternalSystemAdapterImpl( logger.info("[$accountName] Submit: $paymentId , txId: $transactionId") var retryCount = 0 - while (true) { + val remaining = deadline - now() if (remaining <= 0) { paymentESService.update(paymentId) { @@ -75,16 +68,9 @@ class PaymentExternalSystemAdapterImpl( return } - if (!parallelLimiter.tryAcquire(remaining, TimeUnit.MILLISECONDS)) { - paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "parallel limiter timeout") - } - return - } - val timeBeforeCall = now() - var shouldRetry = false - val perCallTimeoutMs = remaining.coerceAtMost(1100) + remaining.coerceAtMost(30_000L) + tryAcquire(now(), remaining) val request = Request.Builder() .url( "http://$paymentProviderHostPort/external/process" + @@ -94,8 +80,8 @@ class PaymentExternalSystemAdapterImpl( .post(emptyBody) .build() - val call = client.newCall(request).enqueue(PaymentCallback( - kotlinx.coroutines.sync.Semaphore(parallelRequests), + client.newCall(request).enqueue(PaymentCallback( + parallelLimiter, accountName, retryCount, paymentId, @@ -108,15 +94,18 @@ class PaymentExternalSystemAdapterImpl( timeBeforeCall )) } - } override fun price() = properties.price override fun isEnabled() = properties.enabled override fun name() = properties.accountName - fun timeToDead(deadline: Long): Long { - return deadline - now() + + fun tryAcquire(startedAt: Long, remaining: Long): Boolean { + while (!parallelLimiter.tryAcquire() && now()-startedAt < remaining) { } + return parallelLimiter.tryAcquire() } + } -public fun now() = System.currentTimeMillis() \ No newline at end of file + +fun now() = System.currentTimeMillis() \ No newline at end of file diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index c06d8d823..4a60c6257 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -15,7 +15,7 @@ event.sourcing.sagas-enabled=false spring.datasource.hikari.jdbc-url=jdbc:postgresql://${POSTGRES_ADDRESS:localhost}:${POSTGRES_PORT:65432}/postgres spring.datasource.hikari.username=tiny_es spring.datasource.hikari.password=tiny_es - +logger.level.ru.quipy.payments.logic.PaymentCallback=DEBUG spring.datasource.hikari.leak-detection-threshold=2000 management.metrics.web.server.request.autotime.percentiles=0.95 management.metrics.export.prometheus.enabled=true @@ -26,6 +26,6 @@ payment.service-name=${PAYMENT_SERVICE_NAME} payment.token=${PAYMENT_TOKEN} # payment.accounts=${PAYMENT_ACCOUNTS:acc-12,acc-20} # payment.accounts=${PAYMENT_ACCOUNTS:acc-3} -payment.accounts=${PAYMENT_ACCOUNTS:acc-9} +payment.accounts=${PAYMENT_ACCOUNTS:acc-12} # payment.accounts=${PAYMENT_ACCOUNTS:acc-18} payment.hostPort=${PAYMENT_HOST:localhost}:${PAYMENT_PORT:1234} From 6aff9b2e856f3e3d5518736fa2d5a5a0c7ff2509 Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Thu, 4 Dec 2025 20:39:13 +0300 Subject: [PATCH 10/61] fix: fixed rate limiter in order payer and httpCall dispatcher --- .../ru/quipy/payments/logic/OrderPayer.kt | 12 ++++--- .../quipy/payments/logic/PaymentCallback.kt | 7 ++-- .../logic/PaymentExternalServiceImpl.kt | 34 ++++++++++--------- .../ru/quipy/payments/logic/PaymentService.kt | 2 +- 4 files changed, 31 insertions(+), 24 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt index 4835a1d26..a929ed6be 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt @@ -1,5 +1,6 @@ package ru.quipy.payments.logic +import io.micrometer.core.instrument.MeterRegistry import org.slf4j.Logger import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired @@ -18,17 +19,19 @@ import java.util.concurrent.ThreadPoolExecutor import java.util.concurrent.TimeUnit @Service -class OrderPayer { +class OrderPayer(meterRegistry: MeterRegistry) { companion object { val logger: Logger = LoggerFactory.getLogger(OrderPayer::class.java) } private val rateLimit: SlidingWindowRateLimiter by lazy { SlidingWindowRateLimiter( - rate = 8, + rate = 1100, window = Duration.ofMillis(1000), ) } + private val plannedRequests = meterRegistry.counter("payment.processing.planned", "accountName", "acc-12") + @Autowired private lateinit var paymentESService: EventSourcingService @@ -39,11 +42,12 @@ class OrderPayer { suspend fun processPayment(orderId: UUID, amount: Int, paymentId: UUID, deadline: Long): Long { val createdAt = System.currentTimeMillis() + plannedRequests.increment() if (!rateLimit.tickBlocking(deadline- System.currentTimeMillis())) { throw TooManyRequestsException(deadline) } - val createdEvent = paymentESService.create { + val createdEvent = paymentESService.create { it.create( paymentId, orderId, @@ -51,7 +55,7 @@ class OrderPayer { ) } - logger.trace("Payment ${createdEvent.paymentId} for order $orderId created.") + logger.trace("Payment {} for order {} created.", createdEvent.paymentId, orderId) paymentService.submitPaymentRequest(paymentId, amount, createdAt, deadline) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentCallback.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentCallback.kt index f1962a973..92bb53906 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentCallback.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentCallback.kt @@ -1,6 +1,7 @@ package ru.quipy.payments.logic import io.micrometer.core.instrument.Timer +import io.micrometer.core.instrument.Counter import kotlinx.coroutines.sync.Semaphore import okhttp3.Call import okhttp3.Callback @@ -18,8 +19,7 @@ import java.util.concurrent.TimeUnit import kotlin.math.pow -class PaymentCallback(val semaphore: Semaphore, val accountName: String, val retryCount: Int, val paymentId: UUID, val transactionId: UUID, val paymentESService: EventSourcingService, val client: OkHttpClient, val request: Request, val timer: Timer, val deadline: Long, val timeBeforeCall: Long) : Callback { - private val remaining: Long = 10_000L //ms +class PaymentCallback(val startedRequestsCounter: Counter, val semaphore: Semaphore, val accountName: String, val retryCount: Int, val paymentId: UUID, val transactionId: UUID, val paymentESService: EventSourcingService, val client: OkHttpClient, val request: Request, val timer: Timer, val deadline: Long, val timeBeforeCall: Long) : Callback { override fun onFailure(call: Call, e: java.io.IOException) { logger.debug("fail in callback for payment: {}, retry count: {}, deadline: {}, in time: {}", paymentId, retryCount, deadline, now(), e) @@ -67,11 +67,12 @@ class PaymentCallback(val semaphore: Semaphore, val accountName: String, val ret return } val nCall = client.newCall(request) - nCall.enqueue(PaymentCallback(semaphore, accountName, retryCount + 1, paymentId, transactionId, paymentESService, client, request, timer, deadline, now())) + nCall.enqueue(PaymentCallback(startedRequestsCounter, semaphore, accountName, retryCount + 1, paymentId, transactionId, paymentESService, client, request, timer, deadline, now())) timer.record(now() - timeBeforeCall, TimeUnit.MILLISECONDS) } override fun onResponse(call: Call, response: Response) { + startedRequestsCounter.increment() try { logger.warn("Free space in semaphore: {}", semaphore.availablePermits) logger.info( diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index a0cc37859..2c593db00 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -4,9 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.registerKotlinModule import io.micrometer.core.instrument.MeterRegistry import kotlinx.coroutines.sync.Semaphore -import okhttp3.OkHttpClient -import okhttp3.Request -import okhttp3.RequestBody +import okhttp3.* import org.slf4j.LoggerFactory import ru.quipy.core.EventSourcingService import ru.quipy.payments.api.PaymentAggregate @@ -24,7 +22,6 @@ class PaymentExternalSystemAdapterImpl( meterRegistry: MeterRegistry, private val parallelLimiter: Semaphore ) : PaymentExternalSystemAdapter { - companion object { val logger = LoggerFactory.getLogger(PaymentExternalSystemAdapter::class.java) @@ -32,6 +29,7 @@ class PaymentExternalSystemAdapterImpl( val mapper = ObjectMapper().registerKotlinModule() } // 2025-11-20T20:30:35.780+03:00 INFO 56644 --- [alhost:1234/...] ru.quipy.core.EventSourcingService : Optimistic lock exception. Failed to save event records id: [7dca693e-e811-4b7f-8bce-23e13d952c04-4] + private val startedRequests = meterRegistry.counter("payment.processing.started", "accountName", properties.accountName) private val timer = meterRegistry.timer("payment.external.system.request.latency", "accountName", properties.accountName) private val serviceName = properties.serviceName private val accountName = properties.accountName @@ -39,15 +37,20 @@ class PaymentExternalSystemAdapterImpl( private val rateLimitPerSec = properties.rateLimitPerSec private val parallelRequests = properties.parallelRequests - private val client = OkHttpClient.Builder() - .callTimeout(30_000, TimeUnit.MILLISECONDS).build() - - override fun performPaymentAsync(paymentId: UUID, amount: Int, paymentStartedAt: Long, deadline: Long) { - logger.warn("[$accountName] Submitting payment request for payment $paymentId") + private val dispatcher = Dispatcher().apply { + maxRequestsPerHost = parallelRequests + maxRequests = parallelRequests * 2 + } + private val client = OkHttpClient.Builder() + .dispatcher(dispatcher) + .connectionPool(ConnectionPool(parallelRequests, 6, TimeUnit.MINUTES)) + .callTimeout(30_000, TimeUnit.MILLISECONDS) + .build() - fun now() = System.currentTimeMillis() + override suspend fun performPaymentAsync(paymentId: UUID, amount: Int, paymentStartedAt: Long, deadline: Long) { + logger.warn("[$accountName] Submitting payment request for payment $paymentId") val transactionId = UUID.randomUUID() // Вне зависимости от исхода оплаты важно отметить что она была отправлена. @@ -57,9 +60,6 @@ class PaymentExternalSystemAdapterImpl( } logger.info("[$accountName] Submit: $paymentId , txId: $transactionId") - - var retryCount = 0 - val remaining = deadline - now() if (remaining <= 0) { paymentESService.update(paymentId) { @@ -80,10 +80,12 @@ class PaymentExternalSystemAdapterImpl( .post(emptyBody) .build() - client.newCall(request).enqueue(PaymentCallback( + logger.info("Client connections {}. Semaphore was locked: {} ", client.connectionPool.connectionCount(), parallelRequests-parallelLimiter.availablePermits) + client.newCall(request).enqueue(PaymentCallback( + startedRequests, parallelLimiter, accountName, - retryCount, + 0, paymentId, transactionId, paymentESService, @@ -106,6 +108,6 @@ class PaymentExternalSystemAdapterImpl( return parallelLimiter.tryAcquire() } -} +} fun now() = System.currentTimeMillis() \ No newline at end of file diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentService.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentService.kt index 848f40474..d67aebbf0 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentService.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentService.kt @@ -17,7 +17,7 @@ interface PaymentService { */ interface PaymentExternalSystemAdapter { - fun performPaymentAsync(paymentId: UUID, amount: Int, paymentStartedAt: Long, deadline: Long) + suspend fun performPaymentAsync(paymentId: UUID, amount: Int, paymentStartedAt: Long, deadline: Long) fun name(): String From e7fc62c452a31971a3ee34afa8a8cc65efccab05 Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Thu, 4 Dec 2025 21:07:22 +0300 Subject: [PATCH 11/61] fix: fixed rate limiter in order payer and httpCall dispatcher --- src/main/kotlin/ru/quipy/payments/logic/PaymentCallback.kt | 2 ++ src/main/resources/application.properties | 1 + 2 files changed, 3 insertions(+) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentCallback.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentCallback.kt index 92bb53906..87e6f4103 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentCallback.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentCallback.kt @@ -53,6 +53,7 @@ class PaymentCallback(val startedRequestsCounter: Counter, val semaphore: Semaph paymentESService.update(paymentId) { it.logProcessing(false, now(), transactionId, "Max attempts reached") } + startedRequestsCounter.increment() semaphore.release() return } @@ -63,6 +64,7 @@ class PaymentCallback(val startedRequestsCounter: Counter, val semaphore: Semaph paymentESService.update(paymentId) { it.logProcessing(false, now(), transactionId, "Deadline expired") } + startedRequestsCounter.increment() semaphore.release() return } diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 4a60c6257..bdbe25f61 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -29,3 +29,4 @@ payment.token=${PAYMENT_TOKEN} payment.accounts=${PAYMENT_ACCOUNTS:acc-12} # payment.accounts=${PAYMENT_ACCOUNTS:acc-18} payment.hostPort=${PAYMENT_HOST:localhost}:${PAYMENT_PORT:1234} + From d1d6e88b5566d2443087c169dcd61bacd461f468 Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Thu, 4 Dec 2025 21:49:33 +0300 Subject: [PATCH 12/61] fix: fixed timeout --- .../ru/quipy/payments/logic/PaymentExternalServiceImpl.kt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index 2c593db00..f780addf3 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -28,6 +28,8 @@ class PaymentExternalSystemAdapterImpl( val emptyBody = RequestBody.create(null, ByteArray(0)) val mapper = ObjectMapper().registerKotlinModule() } + + private val remaining = 50_000L // 2025-11-20T20:30:35.780+03:00 INFO 56644 --- [alhost:1234/...] ru.quipy.core.EventSourcingService : Optimistic lock exception. Failed to save event records id: [7dca693e-e811-4b7f-8bce-23e13d952c04-4] private val startedRequests = meterRegistry.counter("payment.processing.started", "accountName", properties.accountName) private val timer = meterRegistry.timer("payment.external.system.request.latency", "accountName", properties.accountName) @@ -45,7 +47,7 @@ class PaymentExternalSystemAdapterImpl( private val client = OkHttpClient.Builder() .dispatcher(dispatcher) .connectionPool(ConnectionPool(parallelRequests, 6, TimeUnit.MINUTES)) - .callTimeout(30_000, TimeUnit.MILLISECONDS) + .callTimeout(remaining, TimeUnit.MILLISECONDS) .build() @@ -69,7 +71,7 @@ class PaymentExternalSystemAdapterImpl( } val timeBeforeCall = now() - remaining.coerceAtMost(30_000L) + remaining.coerceAtMost(remaining) tryAcquire(now(), remaining) val request = Request.Builder() .url( From f24804d964b4bccd5e9ca6910d51dcddb1c7a343 Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Fri, 5 Dec 2025 16:39:51 +0300 Subject: [PATCH 13/61] fix: added http2 client --- .../logic/PaymentExternalServiceImpl.kt | 144 ++++++++++++------ 1 file changed, 99 insertions(+), 45 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index f780addf3..07b348927 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -4,12 +4,19 @@ import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.registerKotlinModule import io.micrometer.core.instrument.MeterRegistry import kotlinx.coroutines.sync.Semaphore -import okhttp3.* +import okhttp3.RequestBody import org.slf4j.LoggerFactory import ru.quipy.core.EventSourcingService import ru.quipy.payments.api.PaymentAggregate +import java.io.InterruptedIOException +import java.net.SocketTimeoutException +import java.net.URI +import java.net.http.HttpClient +import java.net.http.HttpRequest +import java.net.http.HttpResponse import java.time.Duration import java.util.* +import java.util.concurrent.Executors import java.util.concurrent.TimeUnit @@ -39,15 +46,10 @@ class PaymentExternalSystemAdapterImpl( private val rateLimitPerSec = properties.rateLimitPerSec private val parallelRequests = properties.parallelRequests - private val dispatcher = Dispatcher().apply { - maxRequestsPerHost = parallelRequests - maxRequests = parallelRequests * 2 - } - - private val client = OkHttpClient.Builder() - .dispatcher(dispatcher) - .connectionPool(ConnectionPool(parallelRequests, 6, TimeUnit.MINUTES)) - .callTimeout(remaining, TimeUnit.MILLISECONDS) + private val httpClient = HttpClient + .newBuilder() + .executor(Executors.newFixedThreadPool(parallelRequests)) + .version(HttpClient.Version.HTTP_2) .build() @@ -62,43 +64,28 @@ class PaymentExternalSystemAdapterImpl( } logger.info("[$accountName] Submit: $paymentId , txId: $transactionId") - val remaining = deadline - now() - if (remaining <= 0) { - paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "deadline expired") - } - return + val remaining = deadline - now() + if (remaining <= 0) { + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "deadline expired") } - - val timeBeforeCall = now() - remaining.coerceAtMost(remaining) - tryAcquire(now(), remaining) - val request = Request.Builder() - .url( - "http://$paymentProviderHostPort/external/process" + - "?serviceName=$serviceName&token=$token&accountName=$accountName" + - "&transactionId=$transactionId&paymentId=$paymentId&amount=$amount" - ) - .post(emptyBody) - .build() - - logger.info("Client connections {}. Semaphore was locked: {} ", client.connectionPool.connectionCount(), parallelRequests-parallelLimiter.availablePermits) - client.newCall(request).enqueue(PaymentCallback( - startedRequests, - parallelLimiter, - accountName, - 0, - paymentId, - transactionId, - paymentESService, - client, - request, - timer, - deadline, - timeBeforeCall - )) + return } + val timeBeforeCall = now() + remaining.coerceAtMost(remaining) + tryAcquire(now(), remaining) + val request = HttpRequest.newBuilder() + .uri(URI("http://$paymentProviderHostPort/external/process?serviceName=$serviceName&token=$token&accountName=$accountName&transactionId=$transactionId&paymentId=$paymentId&amount=$amount")) + .timeout(Duration.ofMillis(requestAverageProcessingTime.toMillis())) + .POST(HttpRequest.BodyPublishers.noBody()) + .build() + + val retryCount = 0L; + + completeAction(retryCount, request, paymentId, transactionId, timeBeforeCall) + } + override fun price() = properties.price override fun isEnabled() = properties.enabled @@ -110,6 +97,73 @@ class PaymentExternalSystemAdapterImpl( return parallelLimiter.tryAcquire() } + fun completeAction(retryCount: Long, request: HttpRequest, paymentId: UUID, transactionId: UUID, timeBeforeCall: Long) { + httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString()) + .whenComplete { response, throwable -> + if (throwable != null) { + val e = throwable.cause + when (throwable.cause) { + is SocketTimeoutException -> { + logger.warn("[$accountName] attempt ${retryCount + 1} timeout: $paymentId", e) + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "socket timeout") + } + } + + is InterruptedIOException -> { + logger.warn("[$accountName] interrupted: $paymentId", e) + + // Здесь мы обновляем состояние оплаты в зависимости от результата в базе данных оплат. + // Это требуется сделать ВО ВСЕХ ИСХОДАХ (успешная оплата / неуспешная / ошибочная ситуация) + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "interrupted IO") + } + } + + else -> { + logger.warn("[$accountName] io error: $paymentId", e) + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "io exception") + } + } + } + + if (retryCount + 1 >= 3) { + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "Max attempts reached") + } + startedRequests.increment() + parallelLimiter.release() + } else { + completeAction(retryCount + 1, request, paymentId, transactionId, timeBeforeCall) + } + } else { + startedRequests.increment() + try { + logger.warn("Free space in semaphore: {}", parallelLimiter.availablePermits) + logger.info( + "success in callback for payment: {}, retry count: {}, deadline: {}, in time: {}", + paymentId, + retryCount, + now() + ) + val rawBody = response.body() + val parsed = try { + mapper.readValue(rawBody, ExternalSysResponse::class.java) + } catch (ex: Exception) { + ExternalSysResponse(transactionId.toString(), paymentId.toString(), false, ex.message) + } + + paymentESService.update(paymentId) { + it.logProcessing(parsed.result, now(), transactionId, parsed.message) + } + } finally { + parallelLimiter.release() + timer.record(now() - timeBeforeCall, TimeUnit.MILLISECONDS) + } + } + } + } + } -} fun now() = System.currentTimeMillis() \ No newline at end of file From 9a87128fc15bfa7bd454632437f4be7365a6bb8e Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Fri, 5 Dec 2025 17:02:43 +0300 Subject: [PATCH 14/61] fix: fixed timeouts --- .../ru/quipy/common/utils/SlidingWindowRateLimiter.kt | 4 ++-- .../ru/quipy/payments/logic/PaymentExternalServiceImpl.kt | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/ru/quipy/common/utils/SlidingWindowRateLimiter.kt b/src/main/kotlin/ru/quipy/common/utils/SlidingWindowRateLimiter.kt index c5bd36a0a..4537b42a8 100644 --- a/src/main/kotlin/ru/quipy/common/utils/SlidingWindowRateLimiter.kt +++ b/src/main/kotlin/ru/quipy/common/utils/SlidingWindowRateLimiter.kt @@ -39,10 +39,10 @@ class SlidingWindowRateLimiter( } } - fun tickBlocking(timeout: Long): Boolean { + suspend fun tickBlocking(timeout: Long): Boolean { val timeStarted = System.currentTimeMillis() while (System.currentTimeMillis()-timeStarted < timeout && !tick()) { - Thread.sleep(2) + delay(2) } return System.currentTimeMillis()-timeStarted < timeout } diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index 07b348927..92590fffd 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -36,7 +36,7 @@ class PaymentExternalSystemAdapterImpl( val mapper = ObjectMapper().registerKotlinModule() } - private val remaining = 50_000L + private val remaining = 20_000L // 2025-11-20T20:30:35.780+03:00 INFO 56644 --- [alhost:1234/...] ru.quipy.core.EventSourcingService : Optimistic lock exception. Failed to save event records id: [7dca693e-e811-4b7f-8bce-23e13d952c04-4] private val startedRequests = meterRegistry.counter("payment.processing.started", "accountName", properties.accountName) private val timer = meterRegistry.timer("payment.external.system.request.latency", "accountName", properties.accountName) @@ -73,11 +73,11 @@ class PaymentExternalSystemAdapterImpl( } val timeBeforeCall = now() - remaining.coerceAtMost(remaining) + remaining.coerceAtMost(this.remaining) tryAcquire(now(), remaining) val request = HttpRequest.newBuilder() .uri(URI("http://$paymentProviderHostPort/external/process?serviceName=$serviceName&token=$token&accountName=$accountName&transactionId=$transactionId&paymentId=$paymentId&amount=$amount")) - .timeout(Duration.ofMillis(requestAverageProcessingTime.toMillis())) + .timeout(Duration.ofMillis(30_000)) .POST(HttpRequest.BodyPublishers.noBody()) .build() @@ -142,7 +142,7 @@ class PaymentExternalSystemAdapterImpl( try { logger.warn("Free space in semaphore: {}", parallelLimiter.availablePermits) logger.info( - "success in callback for payment: {}, retry count: {}, deadline: {}, in time: {}", + "success in callback for payment: {}, retry count: {}, in time: {}", paymentId, retryCount, now() From b3517ee59408ae454d3db7404d1bb7769d259400 Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Fri, 5 Dec 2025 17:40:30 +0300 Subject: [PATCH 15/61] fix: fixed timeouts and rate limiting --- .../ru/quipy/apigateway/APIController.kt | 4 --- .../ru/quipy/payments/logic/OrderPayer.kt | 20 +-------------- .../logic/PaymentExternalServiceImpl.kt | 25 ++++++++++++++----- 3 files changed, 20 insertions(+), 29 deletions(-) diff --git a/src/main/kotlin/ru/quipy/apigateway/APIController.kt b/src/main/kotlin/ru/quipy/apigateway/APIController.kt index 3720312da..e81d98b98 100644 --- a/src/main/kotlin/ru/quipy/apigateway/APIController.kt +++ b/src/main/kotlin/ru/quipy/apigateway/APIController.kt @@ -6,7 +6,6 @@ import org.springframework.beans.factory.annotation.Qualifier import org.springframework.web.bind.annotation.* import ru.quipy.common.utils.LeakingBucketRateLimiter import ru.quipy.common.utils.RateLimiter -import ru.quipy.exceptions.TooManyRequestsException import ru.quipy.orders.repository.OrderRepository import ru.quipy.payments.logic.OrderPayer import java.time.Duration @@ -59,9 +58,6 @@ class APIController( @PostMapping("/orders/{orderId}/payment") suspend fun payOrder(@PathVariable orderId: UUID, @RequestParam deadline: Long): PaymentSubmissionDto { - if (!rateLimiter.tick()) { - throw TooManyRequestsException(deadline) - } val paymentId = UUID.randomUUID() val order = orderRepository.findById(orderId)?.let { orderRepository.save(it.copy(status = OrderStatus.PAYMENT_IN_PROGRESS)) diff --git a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt index a929ed6be..8861311a1 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt @@ -5,18 +5,9 @@ import org.slf4j.Logger import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service -import ru.quipy.common.utils.CallerBlockingRejectedExecutionHandler -import ru.quipy.common.utils.NamedThreadFactory -import ru.quipy.common.utils.SlidingWindowRateLimiter import ru.quipy.core.EventSourcingService -import ru.quipy.exceptions.RateLimitWasBreached -import ru.quipy.exceptions.TooManyRequestsException import ru.quipy.payments.api.PaymentAggregate -import java.time.Duration import java.util.* -import java.util.concurrent.LinkedBlockingQueue -import java.util.concurrent.ThreadPoolExecutor -import java.util.concurrent.TimeUnit @Service class OrderPayer(meterRegistry: MeterRegistry) { @@ -24,12 +15,7 @@ class OrderPayer(meterRegistry: MeterRegistry) { companion object { val logger: Logger = LoggerFactory.getLogger(OrderPayer::class.java) } - private val rateLimit: SlidingWindowRateLimiter by lazy { - SlidingWindowRateLimiter( - rate = 1100, - window = Duration.ofMillis(1000), - ) - } + private val plannedRequests = meterRegistry.counter("payment.processing.planned", "accountName", "acc-12") @@ -43,10 +29,6 @@ class OrderPayer(meterRegistry: MeterRegistry) { suspend fun processPayment(orderId: UUID, amount: Int, paymentId: UUID, deadline: Long): Long { val createdAt = System.currentTimeMillis() plannedRequests.increment() - if (!rateLimit.tickBlocking(deadline- System.currentTimeMillis())) { - throw TooManyRequestsException(deadline) - } - val createdEvent = paymentESService.create { it.create( paymentId, diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index 92590fffd..f4e209509 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -4,9 +4,10 @@ import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.registerKotlinModule import io.micrometer.core.instrument.MeterRegistry import kotlinx.coroutines.sync.Semaphore -import okhttp3.RequestBody import org.slf4j.LoggerFactory +import ru.quipy.common.utils.SlidingWindowRateLimiter import ru.quipy.core.EventSourcingService +import ru.quipy.exceptions.TooManyRequestsException import ru.quipy.payments.api.PaymentAggregate import java.io.InterruptedIOException import java.net.SocketTimeoutException @@ -31,8 +32,6 @@ class PaymentExternalSystemAdapterImpl( ) : PaymentExternalSystemAdapter { companion object { val logger = LoggerFactory.getLogger(PaymentExternalSystemAdapter::class.java) - - val emptyBody = RequestBody.create(null, ByteArray(0)) val mapper = ObjectMapper().registerKotlinModule() } @@ -46,6 +45,13 @@ class PaymentExternalSystemAdapterImpl( private val rateLimitPerSec = properties.rateLimitPerSec private val parallelRequests = properties.parallelRequests + private val rateLimit: SlidingWindowRateLimiter by lazy { + SlidingWindowRateLimiter( + rate = rateLimitPerSec.toLong(), + window = Duration.ofMillis(1000), + ) + } + private val httpClient = HttpClient .newBuilder() .executor(Executors.newFixedThreadPool(parallelRequests)) @@ -75,13 +81,16 @@ class PaymentExternalSystemAdapterImpl( val timeBeforeCall = now() remaining.coerceAtMost(this.remaining) tryAcquire(now(), remaining) + if (!rateLimit.tickBlocking(deadline- now())) { + throw TooManyRequestsException(deadline) + } val request = HttpRequest.newBuilder() .uri(URI("http://$paymentProviderHostPort/external/process?serviceName=$serviceName&token=$token&accountName=$accountName&transactionId=$transactionId&paymentId=$paymentId&amount=$amount")) .timeout(Duration.ofMillis(30_000)) .POST(HttpRequest.BodyPublishers.noBody()) .build() - val retryCount = 0L; + val retryCount = 0L completeAction(retryCount, request, paymentId, transactionId, timeBeforeCall) } @@ -93,8 +102,12 @@ class PaymentExternalSystemAdapterImpl( override fun name() = properties.accountName fun tryAcquire(startedAt: Long, remaining: Long): Boolean { - while (!parallelLimiter.tryAcquire() && now()-startedAt < remaining) { } - return parallelLimiter.tryAcquire() + var isAcquired = parallelLimiter.tryAcquire() + while (!isAcquired && now()-startedAt < remaining) { + isAcquired = parallelLimiter.tryAcquire() + } + + return isAcquired } fun completeAction(retryCount: Long, request: HttpRequest, paymentId: UUID, transactionId: UUID, timeBeforeCall: Long) { From f3a52b90326a290930213c36cca4e013303ab93a Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Fri, 5 Dec 2025 20:40:57 +0300 Subject: [PATCH 16/61] fix: added backoff --- .../ru/quipy/apigateway/APIController.kt | 10 ++++--- .../logic/PaymentExternalServiceImpl.kt | 26 ++++++++++++++++--- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/ru/quipy/apigateway/APIController.kt b/src/main/kotlin/ru/quipy/apigateway/APIController.kt index e81d98b98..6a3ab4966 100644 --- a/src/main/kotlin/ru/quipy/apigateway/APIController.kt +++ b/src/main/kotlin/ru/quipy/apigateway/APIController.kt @@ -6,6 +6,7 @@ import org.springframework.beans.factory.annotation.Qualifier import org.springframework.web.bind.annotation.* import ru.quipy.common.utils.LeakingBucketRateLimiter import ru.quipy.common.utils.RateLimiter +import ru.quipy.exceptions.TooManyRequestsException import ru.quipy.orders.repository.OrderRepository import ru.quipy.payments.logic.OrderPayer import java.time.Duration @@ -16,13 +17,13 @@ class APIController( private val orderRepository: OrderRepository, private val orderPayer: OrderPayer, @field:Qualifier("parallelLimiter") - private val rateLimiter: RateLimiter = LeakingBucketRateLimiter(1100, Duration.ofSeconds(1), 4400) + private val rateLimiter: RateLimiter = LeakingBucketRateLimiter(1100, Duration.ofSeconds(1), 3300) ) { val logger: Logger = LoggerFactory.getLogger(APIController::class.java) @PostMapping("/users") - fun createUser(@RequestBody req: CreateUserRequest): User { + suspend fun createUser(@RequestBody req: CreateUserRequest): User { return User(UUID.randomUUID(), req.name) } @@ -31,7 +32,7 @@ class APIController( data class User(val id: UUID, val name: String) @PostMapping("/orders") - fun createOrder(@RequestParam userId: UUID, @RequestParam price: Int): Order { + suspend fun createOrder(@RequestParam userId: UUID, @RequestParam price: Int): Order { val order = Order( UUID.randomUUID(), userId, @@ -58,6 +59,9 @@ class APIController( @PostMapping("/orders/{orderId}/payment") suspend fun payOrder(@PathVariable orderId: UUID, @RequestParam deadline: Long): PaymentSubmissionDto { + if (!rateLimiter.tick()) { + throw TooManyRequestsException(deadline) + } val paymentId = UUID.randomUUID() val order = orderRepository.findById(orderId)?.let { orderRepository.save(it.copy(status = OrderStatus.PAYMENT_IN_PROGRESS)) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index f4e209509..0f9475fc2 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -2,7 +2,9 @@ package ru.quipy.payments.logic import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.registerKotlinModule +import com.github.dockerjava.api.model.Link import io.micrometer.core.instrument.MeterRegistry +import kotlinx.coroutines.delay import kotlinx.coroutines.sync.Semaphore import org.slf4j.LoggerFactory import ru.quipy.common.utils.SlidingWindowRateLimiter @@ -19,6 +21,7 @@ import java.time.Duration import java.util.* import java.util.concurrent.Executors import java.util.concurrent.TimeUnit +import kotlin.math.pow // Advice: always treat time as a Duration @@ -92,7 +95,7 @@ class PaymentExternalSystemAdapterImpl( val retryCount = 0L - completeAction(retryCount, request, paymentId, transactionId, timeBeforeCall) + completeAction(retryCount, request, paymentId, transactionId, timeBeforeCall, deadline) } override fun price() = properties.price @@ -110,7 +113,7 @@ class PaymentExternalSystemAdapterImpl( return isAcquired } - fun completeAction(retryCount: Long, request: HttpRequest, paymentId: UUID, transactionId: UUID, timeBeforeCall: Long) { + fun completeAction(retryCount: Long, request: HttpRequest, paymentId: UUID, transactionId: UUID, timeBeforeCall: Long, deadline: Long) { httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .whenComplete { response, throwable -> if (throwable != null) { @@ -148,7 +151,24 @@ class PaymentExternalSystemAdapterImpl( startedRequests.increment() parallelLimiter.release() } else { - completeAction(retryCount + 1, request, paymentId, transactionId, timeBeforeCall) + val backoff = ((2.0.pow(retryCount.toDouble()) * 25).toLong() + kotlin.random.Random.nextLong(10)) + val capped = backoff.coerceAtMost(deadline - now() - 5) + if (capped <= 0) { + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "Deadline expired") + } + startedRequests.increment() + } else { + Thread.sleep(backoff) + completeAction( + retryCount + 1, + request, + paymentId, + transactionId, + timeBeforeCall, + deadline + ) + } } } else { startedRequests.increment() From dd8540c44e9a5317cf69045ff9045b823f2b3b69 Mon Sep 17 00:00:00 2001 From: RuReVange Date: Sun, 7 Dec 2025 12:26:35 +0300 Subject: [PATCH 17/61] fix: functionality without suspend --- .../ru/quipy/apigateway/APIController.kt | 6 +++--- .../common/utils/SlidingWindowRateLimiter.kt | 18 ++++++++++++------ .../ru/quipy/payments/logic/OrderPayer.kt | 2 +- .../logic/PaymentExternalServiceImpl.kt | 11 ++++------- .../ru/quipy/payments/logic/PaymentService.kt | 4 ++-- .../quipy/payments/logic/PaymentServiceImpl.kt | 2 +- 6 files changed, 23 insertions(+), 20 deletions(-) diff --git a/src/main/kotlin/ru/quipy/apigateway/APIController.kt b/src/main/kotlin/ru/quipy/apigateway/APIController.kt index 6a3ab4966..9c4d48218 100644 --- a/src/main/kotlin/ru/quipy/apigateway/APIController.kt +++ b/src/main/kotlin/ru/quipy/apigateway/APIController.kt @@ -23,7 +23,7 @@ class APIController( val logger: Logger = LoggerFactory.getLogger(APIController::class.java) @PostMapping("/users") - suspend fun createUser(@RequestBody req: CreateUserRequest): User { + fun createUser(@RequestBody req: CreateUserRequest): User { return User(UUID.randomUUID(), req.name) } @@ -32,7 +32,7 @@ class APIController( data class User(val id: UUID, val name: String) @PostMapping("/orders") - suspend fun createOrder(@RequestParam userId: UUID, @RequestParam price: Int): Order { + fun createOrder(@RequestParam userId: UUID, @RequestParam price: Int): Order { val order = Order( UUID.randomUUID(), userId, @@ -58,7 +58,7 @@ class APIController( } @PostMapping("/orders/{orderId}/payment") - suspend fun payOrder(@PathVariable orderId: UUID, @RequestParam deadline: Long): PaymentSubmissionDto { + fun payOrder(@PathVariable orderId: UUID, @RequestParam deadline: Long): PaymentSubmissionDto { if (!rateLimiter.tick()) { throw TooManyRequestsException(deadline) } diff --git a/src/main/kotlin/ru/quipy/common/utils/SlidingWindowRateLimiter.kt b/src/main/kotlin/ru/quipy/common/utils/SlidingWindowRateLimiter.kt index 4537b42a8..4f3f64050 100644 --- a/src/main/kotlin/ru/quipy/common/utils/SlidingWindowRateLimiter.kt +++ b/src/main/kotlin/ru/quipy/common/utils/SlidingWindowRateLimiter.kt @@ -1,7 +1,5 @@ package ru.quipy.common.utils -import com.github.dockerjava.zerodep.shaded.org.apache.hc.core5.util.Deadline -import com.github.dockerjava.zerodep.shaded.org.apache.hc.core5.util.Timeout import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.asCoroutineDispatcher import kotlinx.coroutines.delay @@ -33,18 +31,26 @@ class SlidingWindowRateLimiter( } } - fun tickBlocking() { + fun tickBlockingAsync() { while (!tick()) { Thread.sleep(10) } } - suspend fun tickBlocking(timeout: Long): Boolean { + fun tickBlockingWithTimeout(timeout: Long): Boolean { val timeStarted = System.currentTimeMillis() - while (System.currentTimeMillis()-timeStarted < timeout && !tick()) { + while (System.currentTimeMillis() - timeStarted < timeout && !tick()) { + Thread.sleep(2) + } + return System.currentTimeMillis() - timeStarted < timeout + } + + suspend fun tickBlockingAsync(timeout: Long): Boolean { + val timeStarted = System.currentTimeMillis() + while (System.currentTimeMillis() - timeStarted < timeout && !tick()) { delay(2) } - return System.currentTimeMillis()-timeStarted < timeout + return System.currentTimeMillis() - timeStarted < timeout } diff --git a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt index 8861311a1..e2bd9e203 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt @@ -26,7 +26,7 @@ class OrderPayer(meterRegistry: MeterRegistry) { private lateinit var paymentService: PaymentService - suspend fun processPayment(orderId: UUID, amount: Int, paymentId: UUID, deadline: Long): Long { + fun processPayment(orderId: UUID, amount: Int, paymentId: UUID, deadline: Long): Long { val createdAt = System.currentTimeMillis() plannedRequests.increment() val createdEvent = paymentESService.create { diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index 0f9475fc2..df99db920 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -2,9 +2,7 @@ package ru.quipy.payments.logic import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.registerKotlinModule -import com.github.dockerjava.api.model.Link import io.micrometer.core.instrument.MeterRegistry -import kotlinx.coroutines.delay import kotlinx.coroutines.sync.Semaphore import org.slf4j.LoggerFactory import ru.quipy.common.utils.SlidingWindowRateLimiter @@ -61,8 +59,7 @@ class PaymentExternalSystemAdapterImpl( .version(HttpClient.Version.HTTP_2) .build() - - override suspend fun performPaymentAsync(paymentId: UUID, amount: Int, paymentStartedAt: Long, deadline: Long) { + override fun performPaymentAsync(paymentId: UUID, amount: Int, paymentStartedAt: Long, deadline: Long) { logger.warn("[$accountName] Submitting payment request for payment $paymentId") val transactionId = UUID.randomUUID() @@ -84,7 +81,7 @@ class PaymentExternalSystemAdapterImpl( val timeBeforeCall = now() remaining.coerceAtMost(this.remaining) tryAcquire(now(), remaining) - if (!rateLimit.tickBlocking(deadline- now())) { + if (!rateLimit.tickBlockingWithTimeout(deadline - now())) { throw TooManyRequestsException(deadline) } val request = HttpRequest.newBuilder() @@ -106,7 +103,7 @@ class PaymentExternalSystemAdapterImpl( fun tryAcquire(startedAt: Long, remaining: Long): Boolean { var isAcquired = parallelLimiter.tryAcquire() - while (!isAcquired && now()-startedAt < remaining) { + while (!isAcquired && now() - startedAt < remaining) { isAcquired = parallelLimiter.tryAcquire() } @@ -159,7 +156,7 @@ class PaymentExternalSystemAdapterImpl( } startedRequests.increment() } else { - Thread.sleep(backoff) + Thread.sleep(capped) completeAction( retryCount + 1, request, diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentService.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentService.kt index d67aebbf0..255db77dd 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentService.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentService.kt @@ -7,7 +7,7 @@ interface PaymentService { /** * Submit payment request to some external service. */ - suspend fun submitPaymentRequest(paymentId: UUID, amount: Int, paymentStartedAt: Long, deadline: Long) + fun submitPaymentRequest(paymentId: UUID, amount: Int, paymentStartedAt: Long, deadline: Long) } /** @@ -17,7 +17,7 @@ interface PaymentService { */ interface PaymentExternalSystemAdapter { - suspend fun performPaymentAsync(paymentId: UUID, amount: Int, paymentStartedAt: Long, deadline: Long) + fun performPaymentAsync(paymentId: UUID, amount: Int, paymentStartedAt: Long, deadline: Long) fun name(): String diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentServiceImpl.kt index 3fb881462..7aad9b847 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentServiceImpl.kt @@ -13,7 +13,7 @@ class PaymentSystemImpl( val logger = LoggerFactory.getLogger(PaymentSystemImpl::class.java) } - override suspend fun submitPaymentRequest(paymentId: UUID, amount: Int, paymentStartedAt: Long, deadline: Long) { + override fun submitPaymentRequest(paymentId: UUID, amount: Int, paymentStartedAt: Long, deadline: Long) { for (account in paymentAccounts) { account.performPaymentAsync(paymentId, amount, paymentStartedAt, deadline) } From 58ca47ca3f4c6e847ea03ccbf79c4a14bc70e4b3 Mon Sep 17 00:00:00 2001 From: RuReVange Date: Sun, 7 Dec 2025 13:46:56 +0300 Subject: [PATCH 18/61] fix: correct retry after logic, semaphore release --- .../ru/quipy/apigateway/APIController.kt | 4 ++- .../apigateway/GlobalExceptionHandler.kt | 7 ++++- .../exceptions/TooManyRequestsException.kt | 2 +- .../logic/PaymentExternalServiceImpl.kt | 26 +++++++++++++++---- 4 files changed, 31 insertions(+), 8 deletions(-) diff --git a/src/main/kotlin/ru/quipy/apigateway/APIController.kt b/src/main/kotlin/ru/quipy/apigateway/APIController.kt index 9c4d48218..251ee553e 100644 --- a/src/main/kotlin/ru/quipy/apigateway/APIController.kt +++ b/src/main/kotlin/ru/quipy/apigateway/APIController.kt @@ -11,6 +11,7 @@ import ru.quipy.orders.repository.OrderRepository import ru.quipy.payments.logic.OrderPayer import java.time.Duration import java.util.* +import kotlin.random.Random @RestController class APIController( @@ -60,7 +61,8 @@ class APIController( @PostMapping("/orders/{orderId}/payment") fun payOrder(@PathVariable orderId: UUID, @RequestParam deadline: Long): PaymentSubmissionDto { if (!rateLimiter.tick()) { - throw TooManyRequestsException(deadline) + val retryAfterMs = 10L + Random.nextLong(10) + throw TooManyRequestsException(retryAfterMs) } val paymentId = UUID.randomUUID() val order = orderRepository.findById(orderId)?.let { diff --git a/src/main/kotlin/ru/quipy/apigateway/GlobalExceptionHandler.kt b/src/main/kotlin/ru/quipy/apigateway/GlobalExceptionHandler.kt index 5679ea372..ea33b9e32 100644 --- a/src/main/kotlin/ru/quipy/apigateway/GlobalExceptionHandler.kt +++ b/src/main/kotlin/ru/quipy/apigateway/GlobalExceptionHandler.kt @@ -26,8 +26,13 @@ class GlobalExceptionHandler( fun handleTooManyRequests(exception: TooLongRequestException): ResponseEntity { return ResponseEntity.status(200).body("your request very long, i am so sorry") } + @ExceptionHandler(TooManyRequestsException::class) fun handleTooManyRequestsRetriable(exception: TooManyRequestsException): ResponseEntity { - return ResponseEntity.status(HttpStatus.TOO_MANY_REQUESTS).header("Retry-After", "1").body("to many requests") + val retryAfterSeconds = (exception.retryAfterMs / 1000.0).coerceAtLeast(0.1) + return ResponseEntity + .status(HttpStatus.TOO_MANY_REQUESTS) + .header("Retry-After", retryAfterSeconds.toString()) + .body("too many requests") } } \ No newline at end of file diff --git a/src/main/kotlin/ru/quipy/exceptions/TooManyRequestsException.kt b/src/main/kotlin/ru/quipy/exceptions/TooManyRequestsException.kt index cacd67810..3f063849a 100644 --- a/src/main/kotlin/ru/quipy/exceptions/TooManyRequestsException.kt +++ b/src/main/kotlin/ru/quipy/exceptions/TooManyRequestsException.kt @@ -1,3 +1,3 @@ package ru.quipy.exceptions -class TooManyRequestsException(val deadline: Long) : RuntimeException() \ No newline at end of file +class TooManyRequestsException(val retryAfterMs: Long) : RuntimeException() \ No newline at end of file diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index df99db920..8ee7edf32 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -20,6 +20,7 @@ import java.util.* import java.util.concurrent.Executors import java.util.concurrent.TimeUnit import kotlin.math.pow +import kotlin.random.Random // Advice: always treat time as a Duration @@ -36,7 +37,7 @@ class PaymentExternalSystemAdapterImpl( val mapper = ObjectMapper().registerKotlinModule() } - private val remaining = 20_000L + private val maxRequestTimeout = 20_000L // 2025-11-20T20:30:35.780+03:00 INFO 56644 --- [alhost:1234/...] ru.quipy.core.EventSourcingService : Optimistic lock exception. Failed to save event records id: [7dca693e-e811-4b7f-8bce-23e13d952c04-4] private val startedRequests = meterRegistry.counter("payment.processing.started", "accountName", properties.accountName) private val timer = meterRegistry.timer("payment.external.system.request.latency", "accountName", properties.accountName) @@ -79,14 +80,28 @@ class PaymentExternalSystemAdapterImpl( } val timeBeforeCall = now() - remaining.coerceAtMost(this.remaining) - tryAcquire(now(), remaining) + + if (!tryAcquire(now(), deadline - now())) { + val retryAfterMs = (requestAverageProcessingTime.toMillis() / parallelRequests * 10).coerceIn(10, 100) + Random.nextLong(10) + + throw TooManyRequestsException(retryAfterMs) + } + if (!rateLimit.tickBlockingWithTimeout(deadline - now())) { - throw TooManyRequestsException(deadline) + parallelLimiter.release() + + val retryAfterMs = (1000L / rateLimitPerSec * 10).coerceIn(10, 100) + Random.nextLong(10) + throw TooManyRequestsException(retryAfterMs) } + + val requestTimeout = minOf( + deadline - now(), + requestAverageProcessingTime.toMillis() * 2 + ).coerceAtLeast(100) + val request = HttpRequest.newBuilder() .uri(URI("http://$paymentProviderHostPort/external/process?serviceName=$serviceName&token=$token&accountName=$accountName&transactionId=$transactionId&paymentId=$paymentId&amount=$amount")) - .timeout(Duration.ofMillis(30_000)) + .timeout(Duration.ofMillis(requestTimeout)) .POST(HttpRequest.BodyPublishers.noBody()) .build() @@ -155,6 +170,7 @@ class PaymentExternalSystemAdapterImpl( it.logProcessing(false, now(), transactionId, "Deadline expired") } startedRequests.increment() + parallelLimiter.release() } else { Thread.sleep(capped) completeAction( From ffccdc33de7b866a59b8b63d31fb8ae605ee08bb Mon Sep 17 00:00:00 2001 From: RuReVange Date: Sun, 7 Dec 2025 14:06:33 +0300 Subject: [PATCH 19/61] fix: logging duplicate --- .../payments/logic/PaymentExternalServiceImpl.kt | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index 8ee7edf32..190614df6 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -37,7 +37,6 @@ class PaymentExternalSystemAdapterImpl( val mapper = ObjectMapper().registerKotlinModule() } - private val maxRequestTimeout = 20_000L // 2025-11-20T20:30:35.780+03:00 INFO 56644 --- [alhost:1234/...] ru.quipy.core.EventSourcingService : Optimistic lock exception. Failed to save event records id: [7dca693e-e811-4b7f-8bce-23e13d952c04-4] private val startedRequests = meterRegistry.counter("payment.processing.started", "accountName", properties.accountName) private val timer = meterRegistry.timer("payment.external.system.request.latency", "accountName", properties.accountName) @@ -120,6 +119,7 @@ class PaymentExternalSystemAdapterImpl( var isAcquired = parallelLimiter.tryAcquire() while (!isAcquired && now() - startedAt < remaining) { isAcquired = parallelLimiter.tryAcquire() + Thread.sleep(1) } return isAcquired @@ -133,26 +133,14 @@ class PaymentExternalSystemAdapterImpl( when (throwable.cause) { is SocketTimeoutException -> { logger.warn("[$accountName] attempt ${retryCount + 1} timeout: $paymentId", e) - paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "socket timeout") - } } is InterruptedIOException -> { logger.warn("[$accountName] interrupted: $paymentId", e) - - // Здесь мы обновляем состояние оплаты в зависимости от результата в базе данных оплат. - // Это требуется сделать ВО ВСЕХ ИСХОДАХ (успешная оплата / неуспешная / ошибочная ситуация) - paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "interrupted IO") - } } else -> { logger.warn("[$accountName] io error: $paymentId", e) - paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "io exception") - } } } From d68f49c3cb79de1d2a906be951c7e1d3f71a88eb Mon Sep 17 00:00:00 2001 From: RuReVange Date: Sun, 7 Dec 2025 14:17:58 +0300 Subject: [PATCH 20/61] feat: retry with new timeout for request --- .../quipy/payments/logic/PaymentExternalServiceImpl.kt | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index 190614df6..22f7923cd 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -161,9 +161,17 @@ class PaymentExternalSystemAdapterImpl( parallelLimiter.release() } else { Thread.sleep(capped) + + val newRequestTimeout = (deadline - now()).coerceIn(100, requestAverageProcessingTime.toMillis() * 2) + val newRequest = HttpRequest.newBuilder() + .uri(request.uri()) + .timeout(Duration.ofMillis(newRequestTimeout)) + .POST(HttpRequest.BodyPublishers.noBody()) + .build() + completeAction( retryCount + 1, - request, + newRequest, paymentId, transactionId, timeBeforeCall, From d9a15ca13a78f1760a6f7c4b050af71eb064e430 Mon Sep 17 00:00:00 2001 From: RuReVange Date: Sun, 7 Dec 2025 14:26:17 +0300 Subject: [PATCH 21/61] fix: softer rate limiter --- .../ru/quipy/payments/logic/PaymentExternalServiceImpl.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index 22f7923cd..c597b9765 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -48,7 +48,7 @@ class PaymentExternalSystemAdapterImpl( private val rateLimit: SlidingWindowRateLimiter by lazy { SlidingWindowRateLimiter( - rate = rateLimitPerSec.toLong(), + rate = (rateLimitPerSec * 0.95).toLong(), window = Duration.ofMillis(1000), ) } From 060407be3da8393ec00b28430f9d7237560370bc Mon Sep 17 00:00:00 2001 From: RuReVange Date: Sun, 7 Dec 2025 14:39:51 +0300 Subject: [PATCH 22/61] fix: additional retry after --- .../logic/PaymentExternalServiceImpl.kt | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index c597b9765..5bd3a308c 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -48,7 +48,7 @@ class PaymentExternalSystemAdapterImpl( private val rateLimit: SlidingWindowRateLimiter by lazy { SlidingWindowRateLimiter( - rate = (rateLimitPerSec * 0.95).toLong(), + rate = (rateLimitPerSec * 0.9).toLong(), window = Duration.ofMillis(1000), ) } @@ -70,12 +70,16 @@ class PaymentExternalSystemAdapterImpl( } logger.info("[$accountName] Submit: $paymentId , txId: $transactionId") + val remaining = deadline - now() - if (remaining <= 0) { + val minRequiredTime = requestAverageProcessingTime.toMillis() + 5000 + if (remaining < minRequiredTime) { + logger.warn("[$accountName] Not enough time for payment $paymentId: ${remaining}ms remaining, need ${minRequiredTime}ms") paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "deadline expired") + it.logProcessing(false, now(), transactionId, "not enough time") } - return + val retryAfterMs = minRequiredTime - remaining + Random.nextLong(100) + throw TooManyRequestsException(retryAfterMs) } val timeBeforeCall = now() @@ -98,6 +102,13 @@ class PaymentExternalSystemAdapterImpl( requestAverageProcessingTime.toMillis() * 2 ).coerceAtLeast(100) + if (requestTimeout < requestAverageProcessingTime.toMillis()) { + logger.warn("[$accountName] Timeout too short for payment $paymentId: ${requestTimeout}ms") + parallelLimiter.release() + val retryAfterMs = requestAverageProcessingTime.toMillis() - requestTimeout + Random.nextLong(100) + throw TooManyRequestsException(retryAfterMs) + } + val request = HttpRequest.newBuilder() .uri(URI("http://$paymentProviderHostPort/external/process?serviceName=$serviceName&token=$token&accountName=$accountName&transactionId=$transactionId&paymentId=$paymentId&amount=$amount")) .timeout(Duration.ofMillis(requestTimeout)) From c3eaad4f76428b01f83c465739e21e13233a5936 Mon Sep 17 00:00:00 2001 From: RuReVange Date: Sun, 7 Dec 2025 14:47:59 +0300 Subject: [PATCH 23/61] fix: minRequiredTime --- .../ru/quipy/payments/logic/PaymentExternalServiceImpl.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index 5bd3a308c..446b90bbe 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -72,7 +72,7 @@ class PaymentExternalSystemAdapterImpl( logger.info("[$accountName] Submit: $paymentId , txId: $transactionId") val remaining = deadline - now() - val minRequiredTime = requestAverageProcessingTime.toMillis() + 5000 + val minRequiredTime = requestAverageProcessingTime.toMillis() * 2 if (remaining < minRequiredTime) { logger.warn("[$accountName] Not enough time for payment $paymentId: ${remaining}ms remaining, need ${minRequiredTime}ms") paymentESService.update(paymentId) { From 586ff6c269650cc6eb87c57faea786a8cc0deb3c Mon Sep 17 00:00:00 2001 From: RuReVange Date: Sun, 7 Dec 2025 14:57:23 +0300 Subject: [PATCH 24/61] feat: sheduler for retry --- .../logic/PaymentExternalServiceImpl.kt | 49 ++++++++++++------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index 446b90bbe..4f1d559f0 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -59,6 +59,10 @@ class PaymentExternalSystemAdapterImpl( .version(HttpClient.Version.HTTP_2) .build() + private val retryScheduler = Executors.newScheduledThreadPool( + Runtime.getRuntime().availableProcessors() + ) + override fun performPaymentAsync(paymentId: UUID, amount: Int, paymentStartedAt: Long, deadline: Long) { logger.warn("[$accountName] Submitting payment request for payment $paymentId") val transactionId = UUID.randomUUID() @@ -171,23 +175,34 @@ class PaymentExternalSystemAdapterImpl( startedRequests.increment() parallelLimiter.release() } else { - Thread.sleep(capped) - - val newRequestTimeout = (deadline - now()).coerceIn(100, requestAverageProcessingTime.toMillis() * 2) - val newRequest = HttpRequest.newBuilder() - .uri(request.uri()) - .timeout(Duration.ofMillis(newRequestTimeout)) - .POST(HttpRequest.BodyPublishers.noBody()) - .build() - - completeAction( - retryCount + 1, - newRequest, - paymentId, - transactionId, - timeBeforeCall, - deadline - ) + retryScheduler.schedule({ + val remainingTime = deadline - now() + + if (remainingTime < requestAverageProcessingTime.toMillis()) { + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "Not enough time for retry") + } + startedRequests.increment() + parallelLimiter.release() + return@schedule + } + + val newRequestTimeout = remainingTime.coerceIn(100, requestAverageProcessingTime.toMillis() * 2) + val newRequest = HttpRequest.newBuilder() + .uri(request.uri()) + .timeout(Duration.ofMillis(newRequestTimeout)) + .POST(HttpRequest.BodyPublishers.noBody()) + .build() + + completeAction( + retryCount + 1, + newRequest, + paymentId, + transactionId, + timeBeforeCall, + deadline + ) + }, capped, TimeUnit.MILLISECONDS) } } } else { From 92ef40493aa5607bf12a7e022ce1796d1af9ef93 Mon Sep 17 00:00:00 2001 From: 21092004Goda Date: Sun, 7 Dec 2025 23:45:34 +0300 Subject: [PATCH 25/61] -- draft: Test async logic in payment external service. --- .../logic/PaymentExternalServiceImpl.kt | 175 ++++++++++++------ 1 file changed, 114 insertions(+), 61 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index 4f1d559f0..b67d86e9a 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -3,6 +3,12 @@ package ru.quipy.payments.logic import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.registerKotlinModule import io.micrometer.core.instrument.MeterRegistry +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.CoroutineExceptionHandler +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.asCoroutineDispatcher +import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Semaphore import org.slf4j.LoggerFactory import ru.quipy.common.utils.SlidingWindowRateLimiter @@ -30,16 +36,26 @@ class PaymentExternalSystemAdapterImpl( private val paymentProviderHostPort: String, private val token: String, meterRegistry: MeterRegistry, - private val parallelLimiter: Semaphore + private val parallelLimiter: Semaphore, + private val ioDispatcher: CoroutineDispatcher = Executors.newFixedThreadPool(10).asCoroutineDispatcher() ) : PaymentExternalSystemAdapter { companion object { val logger = LoggerFactory.getLogger(PaymentExternalSystemAdapter::class.java) val mapper = ObjectMapper().registerKotlinModule() } + private val exceptionHandler = CoroutineExceptionHandler { _, throwable -> + logger.error("[$accountName] Unhandled exception in payment adapter coroutine", throwable) + } + + private val scope = CoroutineScope(SupervisorJob() + ioDispatcher + exceptionHandler) + + // 2025-11-20T20:30:35.780+03:00 INFO 56644 --- [alhost:1234/...] ru.quipy.core.EventSourcingService : Optimistic lock exception. Failed to save event records id: [7dca693e-e811-4b7f-8bce-23e13d952c04-4] - private val startedRequests = meterRegistry.counter("payment.processing.started", "accountName", properties.accountName) - private val timer = meterRegistry.timer("payment.external.system.request.latency", "accountName", properties.accountName) + private val startedRequests = + meterRegistry.counter("payment.processing.started", "accountName", properties.accountName) + private val timer = + meterRegistry.timer("payment.external.system.request.latency", "accountName", properties.accountName) private val serviceName = properties.serviceName private val accountName = properties.accountName private val requestAverageProcessingTime = properties.averageProcessingTime @@ -69,8 +85,20 @@ class PaymentExternalSystemAdapterImpl( // Вне зависимости от исхода оплаты важно отметить что она была отправлена. // Это требуется сделать ВО ВСЕХ СЛУЧАЯХ, поскольку эта информация используется сервисом тестирования. - paymentESService.update(paymentId) { - it.logSubmission(success = true, transactionId, now(), Duration.ofMillis(now() - paymentStartedAt)) + scope.launch { + try { + paymentESService.update(paymentId) { + it.logSubmission( + success = true, + transactionId, + now(), + Duration.ofMillis(now() - paymentStartedAt) + ) + } + logger.info("[$accountName] Log submission recorded for $paymentId") + } catch (e: Exception) { + logger.error("[$accountName] Failed to record log submission for $paymentId", e) + } } logger.info("[$accountName] Submit: $paymentId , txId: $transactionId") @@ -89,7 +117,10 @@ class PaymentExternalSystemAdapterImpl( val timeBeforeCall = now() if (!tryAcquire(now(), deadline - now())) { - val retryAfterMs = (requestAverageProcessingTime.toMillis() / parallelRequests * 10).coerceIn(10, 100) + Random.nextLong(10) + val retryAfterMs = (requestAverageProcessingTime.toMillis() / parallelRequests * 10).coerceIn( + 10, + 100 + ) + Random.nextLong(10) throw TooManyRequestsException(retryAfterMs) } @@ -140,74 +171,59 @@ class PaymentExternalSystemAdapterImpl( return isAcquired } - fun completeAction(retryCount: Long, request: HttpRequest, paymentId: UUID, transactionId: UUID, timeBeforeCall: Long, deadline: Long) { + fun completeAction( + retryCount: Long, + request: HttpRequest, + paymentId: UUID, + transactionId: UUID, + timeBeforeCall: Long, + deadline: Long + ) { httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .whenComplete { response, throwable -> - if (throwable != null) { - val e = throwable.cause - when (throwable.cause) { - is SocketTimeoutException -> { - logger.warn("[$accountName] attempt ${retryCount + 1} timeout: $paymentId", e) - } + scope.launch { + try { + if (throwable != null) { + val e = throwable.cause + when (throwable.cause) { + is SocketTimeoutException -> { + logger.warn("[$accountName] attempt ${retryCount + 1} timeout: $paymentId", e) + } - is InterruptedIOException -> { - logger.warn("[$accountName] interrupted: $paymentId", e) - } + is InterruptedIOException -> { + logger.warn("[$accountName] interrupted: $paymentId", e) + } - else -> { - logger.warn("[$accountName] io error: $paymentId", e) + else -> { + logger.warn("[$accountName] io error: $paymentId", e) + } } - } - if (retryCount + 1 >= 3) { - paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "Max attempts reached") - } - startedRequests.increment() - parallelLimiter.release() - } else { - val backoff = ((2.0.pow(retryCount.toDouble()) * 25).toLong() + kotlin.random.Random.nextLong(10)) - val capped = backoff.coerceAtMost(deadline - now() - 5) - if (capped <= 0) { + if (retryCount + 1 >= 3) { paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "Deadline expired") + it.logProcessing(false, now(), transactionId, "Max attempts reached") } - startedRequests.increment() - parallelLimiter.release() } else { - retryScheduler.schedule({ - val remainingTime = deadline - now() - - if (remainingTime < requestAverageProcessingTime.toMillis()) { - paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "Not enough time for retry") - } - startedRequests.increment() - parallelLimiter.release() - return@schedule + val backoff = ((2.0.pow(retryCount.toDouble()) * 25).toLong() + Random.nextLong(10)) + val capped = backoff.coerceAtMost(deadline - now() - 5) + if (capped <= 0) { + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "Deadline expired") } - - val newRequestTimeout = remainingTime.coerceIn(100, requestAverageProcessingTime.toMillis() * 2) - val newRequest = HttpRequest.newBuilder() - .uri(request.uri()) - .timeout(Duration.ofMillis(newRequestTimeout)) - .POST(HttpRequest.BodyPublishers.noBody()) - .build() - - completeAction( - retryCount + 1, - newRequest, + } else { + scheduleRetry( + retryCount, + request, paymentId, transactionId, timeBeforeCall, - deadline + deadline, + capped ) - }, capped, TimeUnit.MILLISECONDS) + return@launch + } } - } - } else { - startedRequests.increment() - try { + } else { logger.warn("Free space in semaphore: {}", parallelLimiter.availablePermits) logger.info( "success in callback for payment: {}, retry count: {}, in time: {}", @@ -225,13 +241,50 @@ class PaymentExternalSystemAdapterImpl( paymentESService.update(paymentId) { it.logProcessing(parsed.result, now(), transactionId, parsed.message) } - } finally { - parallelLimiter.release() timer.record(now() - timeBeforeCall, TimeUnit.MILLISECONDS) } + } catch (e: Exception) { + logger.error("[$accountName] Error processing payment $paymentId", e) + } finally { + startedRequests.increment() + parallelLimiter.release() } } } } + private fun scheduleRetry( + retryCount: Long, request: HttpRequest, paymentId: UUID, + transactionId: UUID, timeBeforeCall: Long, deadline: Long, delay: Long + ) { + retryScheduler.schedule({ + val remainingTime = deadline - now() + if (remainingTime < requestAverageProcessingTime.toMillis()) { + scope.launch { + try { + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "Not enough time for retry") + } + } catch (e: Exception) { + logger.error("[$accountName] Failed to record retry failure for $paymentId", e) + } finally { + startedRequests.increment() + parallelLimiter.release() + } + } + return@schedule + } + + val newRequestTimeout = remainingTime.coerceIn(100, requestAverageProcessingTime.toMillis() * 2) + val newRequest = HttpRequest.newBuilder() + .uri(request.uri()) + .timeout(Duration.ofMillis(newRequestTimeout)) + .POST(HttpRequest.BodyPublishers.noBody()) + .build() + + completeAction(retryCount + 1, newRequest, paymentId, transactionId, timeBeforeCall, deadline) + }, delay, TimeUnit.MILLISECONDS) + } +} + fun now() = System.currentTimeMillis() \ No newline at end of file From 873d83ead9cb50624de0b02a15b5a84b303adc70 Mon Sep 17 00:00:00 2001 From: 21092004Goda Date: Sun, 7 Dec 2025 23:45:34 +0300 Subject: [PATCH 26/61] -- draft: Test async logic in payment external service. --- .../logic/PaymentExternalServiceImpl.kt | 179 ++++++++++++------ 1 file changed, 118 insertions(+), 61 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index 4f1d559f0..aa1b6f07d 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -3,8 +3,15 @@ package ru.quipy.payments.logic import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.registerKotlinModule import io.micrometer.core.instrument.MeterRegistry +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.CoroutineExceptionHandler +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.asCoroutineDispatcher +import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Semaphore import org.slf4j.LoggerFactory +import ru.quipy.common.utils.NamedThreadFactory import ru.quipy.common.utils.SlidingWindowRateLimiter import ru.quipy.core.EventSourcingService import ru.quipy.exceptions.TooManyRequestsException @@ -30,16 +37,29 @@ class PaymentExternalSystemAdapterImpl( private val paymentProviderHostPort: String, private val token: String, meterRegistry: MeterRegistry, - private val parallelLimiter: Semaphore + private val parallelLimiter: Semaphore, + private val ioDispatcher: CoroutineDispatcher = Executors.newFixedThreadPool( + Runtime.getRuntime().availableProcessors() * 4, + NamedThreadFactory("payment-io-") + ).asCoroutineDispatcher() ) : PaymentExternalSystemAdapter { companion object { val logger = LoggerFactory.getLogger(PaymentExternalSystemAdapter::class.java) val mapper = ObjectMapper().registerKotlinModule() } + private val exceptionHandler = CoroutineExceptionHandler { _, throwable -> + logger.error("[$accountName] Unhandled exception in payment adapter coroutine", throwable) + } + + private val scope = CoroutineScope(SupervisorJob() + ioDispatcher + exceptionHandler) + + // 2025-11-20T20:30:35.780+03:00 INFO 56644 --- [alhost:1234/...] ru.quipy.core.EventSourcingService : Optimistic lock exception. Failed to save event records id: [7dca693e-e811-4b7f-8bce-23e13d952c04-4] - private val startedRequests = meterRegistry.counter("payment.processing.started", "accountName", properties.accountName) - private val timer = meterRegistry.timer("payment.external.system.request.latency", "accountName", properties.accountName) + private val startedRequests = + meterRegistry.counter("payment.processing.started", "accountName", properties.accountName) + private val timer = + meterRegistry.timer("payment.external.system.request.latency", "accountName", properties.accountName) private val serviceName = properties.serviceName private val accountName = properties.accountName private val requestAverageProcessingTime = properties.averageProcessingTime @@ -69,8 +89,20 @@ class PaymentExternalSystemAdapterImpl( // Вне зависимости от исхода оплаты важно отметить что она была отправлена. // Это требуется сделать ВО ВСЕХ СЛУЧАЯХ, поскольку эта информация используется сервисом тестирования. - paymentESService.update(paymentId) { - it.logSubmission(success = true, transactionId, now(), Duration.ofMillis(now() - paymentStartedAt)) + scope.launch { + try { + paymentESService.update(paymentId) { + it.logSubmission( + success = true, + transactionId, + now(), + Duration.ofMillis(now() - paymentStartedAt) + ) + } + logger.info("[$accountName] Log submission recorded for $paymentId") + } catch (e: Exception) { + logger.error("[$accountName] Failed to record log submission for $paymentId", e) + } } logger.info("[$accountName] Submit: $paymentId , txId: $transactionId") @@ -89,7 +121,10 @@ class PaymentExternalSystemAdapterImpl( val timeBeforeCall = now() if (!tryAcquire(now(), deadline - now())) { - val retryAfterMs = (requestAverageProcessingTime.toMillis() / parallelRequests * 10).coerceIn(10, 100) + Random.nextLong(10) + val retryAfterMs = (requestAverageProcessingTime.toMillis() / parallelRequests * 10).coerceIn( + 10, + 100 + ) + Random.nextLong(10) throw TooManyRequestsException(retryAfterMs) } @@ -140,74 +175,59 @@ class PaymentExternalSystemAdapterImpl( return isAcquired } - fun completeAction(retryCount: Long, request: HttpRequest, paymentId: UUID, transactionId: UUID, timeBeforeCall: Long, deadline: Long) { + fun completeAction( + retryCount: Long, + request: HttpRequest, + paymentId: UUID, + transactionId: UUID, + timeBeforeCall: Long, + deadline: Long + ) { httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .whenComplete { response, throwable -> - if (throwable != null) { - val e = throwable.cause - when (throwable.cause) { - is SocketTimeoutException -> { - logger.warn("[$accountName] attempt ${retryCount + 1} timeout: $paymentId", e) - } + scope.launch { + try { + if (throwable != null) { + val e = throwable.cause + when (throwable.cause) { + is SocketTimeoutException -> { + logger.warn("[$accountName] attempt ${retryCount + 1} timeout: $paymentId", e) + } - is InterruptedIOException -> { - logger.warn("[$accountName] interrupted: $paymentId", e) - } + is InterruptedIOException -> { + logger.warn("[$accountName] interrupted: $paymentId", e) + } - else -> { - logger.warn("[$accountName] io error: $paymentId", e) + else -> { + logger.warn("[$accountName] io error: $paymentId", e) + } } - } - if (retryCount + 1 >= 3) { - paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "Max attempts reached") - } - startedRequests.increment() - parallelLimiter.release() - } else { - val backoff = ((2.0.pow(retryCount.toDouble()) * 25).toLong() + kotlin.random.Random.nextLong(10)) - val capped = backoff.coerceAtMost(deadline - now() - 5) - if (capped <= 0) { + if (retryCount + 1 >= 3) { paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "Deadline expired") + it.logProcessing(false, now(), transactionId, "Max attempts reached") } - startedRequests.increment() - parallelLimiter.release() } else { - retryScheduler.schedule({ - val remainingTime = deadline - now() - - if (remainingTime < requestAverageProcessingTime.toMillis()) { - paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "Not enough time for retry") - } - startedRequests.increment() - parallelLimiter.release() - return@schedule + val backoff = ((2.0.pow(retryCount.toDouble()) * 25).toLong() + Random.nextLong(10)) + val capped = backoff.coerceAtMost(deadline - now() - 5) + if (capped <= 0) { + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "Deadline expired") } - - val newRequestTimeout = remainingTime.coerceIn(100, requestAverageProcessingTime.toMillis() * 2) - val newRequest = HttpRequest.newBuilder() - .uri(request.uri()) - .timeout(Duration.ofMillis(newRequestTimeout)) - .POST(HttpRequest.BodyPublishers.noBody()) - .build() - - completeAction( - retryCount + 1, - newRequest, + } else { + scheduleRetry( + retryCount, + request, paymentId, transactionId, timeBeforeCall, - deadline + deadline, + capped ) - }, capped, TimeUnit.MILLISECONDS) + return@launch + } } - } - } else { - startedRequests.increment() - try { + } else { logger.warn("Free space in semaphore: {}", parallelLimiter.availablePermits) logger.info( "success in callback for payment: {}, retry count: {}, in time: {}", @@ -225,13 +245,50 @@ class PaymentExternalSystemAdapterImpl( paymentESService.update(paymentId) { it.logProcessing(parsed.result, now(), transactionId, parsed.message) } - } finally { - parallelLimiter.release() timer.record(now() - timeBeforeCall, TimeUnit.MILLISECONDS) } + } catch (e: Exception) { + logger.error("[$accountName] Error processing payment $paymentId", e) + } finally { + startedRequests.increment() + parallelLimiter.release() } } } } + private fun scheduleRetry( + retryCount: Long, request: HttpRequest, paymentId: UUID, + transactionId: UUID, timeBeforeCall: Long, deadline: Long, delay: Long + ) { + retryScheduler.schedule({ + val remainingTime = deadline - now() + if (remainingTime < requestAverageProcessingTime.toMillis()) { + scope.launch { + try { + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "Not enough time for retry") + } + } catch (e: Exception) { + logger.error("[$accountName] Failed to record retry failure for $paymentId", e) + } finally { + startedRequests.increment() + parallelLimiter.release() + } + } + return@schedule + } + + val newRequestTimeout = remainingTime.coerceIn(100, requestAverageProcessingTime.toMillis() * 2) + val newRequest = HttpRequest.newBuilder() + .uri(request.uri()) + .timeout(Duration.ofMillis(newRequestTimeout)) + .POST(HttpRequest.BodyPublishers.noBody()) + .build() + + completeAction(retryCount + 1, newRequest, paymentId, transactionId, timeBeforeCall, deadline) + }, delay, TimeUnit.MILLISECONDS) + } +} + fun now() = System.currentTimeMillis() \ No newline at end of file From 3a4c8e9901b5d1fa236588153c249ebd12a20a87 Mon Sep 17 00:00:00 2001 From: 21092004Goda Date: Mon, 8 Dec 2025 00:24:55 +0300 Subject: [PATCH 27/61] Merge remote-tracking branch 'origin/hw-9-io-kuro' into hw-9-io-kuro # Conflicts: # src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt --- .../ru/quipy/payments/logic/PaymentExternalServiceImpl.kt | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index a150dcffc..c9c25ccf8 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -39,7 +39,7 @@ class PaymentExternalSystemAdapterImpl( meterRegistry: MeterRegistry, private val parallelLimiter: Semaphore, private val ioDispatcher: CoroutineDispatcher = Executors.newFixedThreadPool( - Runtime.getRuntime().availableProcessors() * 8, + (properties.parallelRequests / 20).coerceIn(100, 1000), NamedThreadFactory("payment-io-") ).asCoroutineDispatcher() ) : PaymentExternalSystemAdapter { @@ -48,6 +48,8 @@ class PaymentExternalSystemAdapterImpl( val mapper = ObjectMapper().registerKotlinModule() } + private val coroutineLimiter = Semaphore(5000) + private val exceptionHandler = CoroutineExceptionHandler { _, throwable -> logger.error("[$accountName] Unhandled exception in payment adapter coroutine", throwable) } @@ -87,6 +89,10 @@ class PaymentExternalSystemAdapterImpl( logger.warn("[$accountName] Submitting payment request for payment $paymentId") val transactionId = UUID.randomUUID() + if (!coroutineLimiter.tryAcquire()) { + // Слишком много активных корутин - отклонить + throw TooManyRequestsException(100L) + } // Вне зависимости от исхода оплаты важно отметить что она была отправлена. // Это требуется сделать ВО ВСЕХ СЛУЧАЯХ, поскольку эта информация используется сервисом тестирования. scope.launch { From 9bd1887e1a1a58c135b74c85bdfa0bb4035ae632 Mon Sep 17 00:00:00 2001 From: 21092004Goda Date: Mon, 8 Dec 2025 00:30:00 +0300 Subject: [PATCH 28/61] Merge remote-tracking branch 'origin/hw-9-io-kuro' into hw-9-io-kuro # Conflicts: # src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt --- .../ru/quipy/payments/logic/PaymentExternalServiceImpl.kt | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index c9c25ccf8..086cf25dc 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -39,7 +39,7 @@ class PaymentExternalSystemAdapterImpl( meterRegistry: MeterRegistry, private val parallelLimiter: Semaphore, private val ioDispatcher: CoroutineDispatcher = Executors.newFixedThreadPool( - (properties.parallelRequests / 20).coerceIn(100, 1000), + Runtime.getRuntime().availableProcessors() * 2, NamedThreadFactory("payment-io-") ).asCoroutineDispatcher() ) : PaymentExternalSystemAdapter { @@ -48,8 +48,6 @@ class PaymentExternalSystemAdapterImpl( val mapper = ObjectMapper().registerKotlinModule() } - private val coroutineLimiter = Semaphore(5000) - private val exceptionHandler = CoroutineExceptionHandler { _, throwable -> logger.error("[$accountName] Unhandled exception in payment adapter coroutine", throwable) } @@ -89,10 +87,6 @@ class PaymentExternalSystemAdapterImpl( logger.warn("[$accountName] Submitting payment request for payment $paymentId") val transactionId = UUID.randomUUID() - if (!coroutineLimiter.tryAcquire()) { - // Слишком много активных корутин - отклонить - throw TooManyRequestsException(100L) - } // Вне зависимости от исхода оплаты важно отметить что она была отправлена. // Это требуется сделать ВО ВСЕХ СЛУЧАЯХ, поскольку эта информация используется сервисом тестирования. scope.launch { From 7f6250ff9f2c221fbe25077a19f77fb962dfbe77 Mon Sep 17 00:00:00 2001 From: 21092004Goda Date: Mon, 8 Dec 2025 00:35:29 +0300 Subject: [PATCH 29/61] Merge remote-tracking branch 'origin/hw-9-io-kuro' into hw-9-io-kuro # Conflicts: # src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt --- .../ru/quipy/payments/logic/PaymentExternalServiceImpl.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index 086cf25dc..35d1055e5 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -39,7 +39,7 @@ class PaymentExternalSystemAdapterImpl( meterRegistry: MeterRegistry, private val parallelLimiter: Semaphore, private val ioDispatcher: CoroutineDispatcher = Executors.newFixedThreadPool( - Runtime.getRuntime().availableProcessors() * 2, + Runtime.getRuntime().availableProcessors(), NamedThreadFactory("payment-io-") ).asCoroutineDispatcher() ) : PaymentExternalSystemAdapter { From 47d7c7a9c877c82d78ef67c2cda9ccb698599658 Mon Sep 17 00:00:00 2001 From: 21092004Goda Date: Mon, 8 Dec 2025 00:47:39 +0300 Subject: [PATCH 30/61] Merge remote-tracking branch 'origin/hw-9-io-kuro' into hw-9-io-kuro # Conflicts: # src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt --- .../ru/quipy/payments/logic/PaymentExternalServiceImpl.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index 35d1055e5..67e72355a 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -39,7 +39,7 @@ class PaymentExternalSystemAdapterImpl( meterRegistry: MeterRegistry, private val parallelLimiter: Semaphore, private val ioDispatcher: CoroutineDispatcher = Executors.newFixedThreadPool( - Runtime.getRuntime().availableProcessors(), + (Runtime.getRuntime().availableProcessors() * 1.5).toInt(), NamedThreadFactory("payment-io-") ).asCoroutineDispatcher() ) : PaymentExternalSystemAdapter { From ccd9f041b55392b21073d52de0fce8f8e222867b Mon Sep 17 00:00:00 2001 From: 21092004Goda Date: Mon, 8 Dec 2025 00:50:14 +0300 Subject: [PATCH 31/61] Merge remote-tracking branch 'origin/hw-9-io-kuro' into hw-9-io-kuro # Conflicts: # src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt --- .../ru/quipy/payments/logic/PaymentExternalServiceImpl.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index 67e72355a..086cf25dc 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -39,7 +39,7 @@ class PaymentExternalSystemAdapterImpl( meterRegistry: MeterRegistry, private val parallelLimiter: Semaphore, private val ioDispatcher: CoroutineDispatcher = Executors.newFixedThreadPool( - (Runtime.getRuntime().availableProcessors() * 1.5).toInt(), + Runtime.getRuntime().availableProcessors() * 2, NamedThreadFactory("payment-io-") ).asCoroutineDispatcher() ) : PaymentExternalSystemAdapter { From d96a1859fd646d3beb14866e33d11ef3b7c8f7c7 Mon Sep 17 00:00:00 2001 From: 21092004Goda Date: Mon, 8 Dec 2025 00:55:52 +0300 Subject: [PATCH 32/61] Merge remote-tracking branch 'origin/hw-9-io-kuro' into hw-9-io-kuro # Conflicts: # src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt --- .../logic/PaymentExternalServiceImpl.kt | 162 ++++++++++-------- 1 file changed, 86 insertions(+), 76 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index 086cf25dc..89502dc6d 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -3,12 +3,7 @@ package ru.quipy.payments.logic import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.registerKotlinModule import io.micrometer.core.instrument.MeterRegistry -import kotlinx.coroutines.CoroutineDispatcher -import kotlinx.coroutines.CoroutineExceptionHandler -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.SupervisorJob -import kotlinx.coroutines.asCoroutineDispatcher -import kotlinx.coroutines.launch +import kotlinx.coroutines.* import kotlinx.coroutines.sync.Semaphore import org.slf4j.LoggerFactory import ru.quipy.common.utils.NamedThreadFactory @@ -25,12 +20,11 @@ import java.net.http.HttpResponse import java.time.Duration import java.util.* import java.util.concurrent.Executors +import java.util.concurrent.ScheduledExecutorService import java.util.concurrent.TimeUnit import kotlin.math.pow import kotlin.random.Random - -// Advice: always treat time as a Duration class PaymentExternalSystemAdapterImpl( private val properties: PaymentAccountProperties, private val paymentESService: EventSourcingService, @@ -38,24 +32,26 @@ class PaymentExternalSystemAdapterImpl( private val token: String, meterRegistry: MeterRegistry, private val parallelLimiter: Semaphore, - private val ioDispatcher: CoroutineDispatcher = Executors.newFixedThreadPool( - Runtime.getRuntime().availableProcessors() * 2, - NamedThreadFactory("payment-io-") - ).asCoroutineDispatcher() -) : PaymentExternalSystemAdapter { +) : PaymentExternalSystemAdapter, AutoCloseable { + companion object { val logger = LoggerFactory.getLogger(PaymentExternalSystemAdapter::class.java) val mapper = ObjectMapper().registerKotlinModule() } + private val sharedScheduler: ScheduledExecutorService = Executors.newScheduledThreadPool( + Runtime.getRuntime().availableProcessors() * 2, + NamedThreadFactory("payment-shared-${properties.accountName}-") + ) + + private val ioDispatcher: CoroutineDispatcher = sharedScheduler.asCoroutineDispatcher() + private val exceptionHandler = CoroutineExceptionHandler { _, throwable -> logger.error("[$accountName] Unhandled exception in payment adapter coroutine", throwable) } private val scope = CoroutineScope(SupervisorJob() + ioDispatcher + exceptionHandler) - - // 2025-11-20T20:30:35.780+03:00 INFO 56644 --- [alhost:1234/...] ru.quipy.core.EventSourcingService : Optimistic lock exception. Failed to save event records id: [7dca693e-e811-4b7f-8bce-23e13d952c04-4] private val startedRequests = meterRegistry.counter("payment.processing.started", "accountName", properties.accountName) private val timer = @@ -75,20 +71,14 @@ class PaymentExternalSystemAdapterImpl( private val httpClient = HttpClient .newBuilder() - .executor(Executors.newFixedThreadPool(parallelRequests)) + .executor(sharedScheduler) .version(HttpClient.Version.HTTP_2) .build() - private val retryScheduler = Executors.newScheduledThreadPool( - Runtime.getRuntime().availableProcessors() - ) - override fun performPaymentAsync(paymentId: UUID, amount: Int, paymentStartedAt: Long, deadline: Long) { logger.warn("[$accountName] Submitting payment request for payment $paymentId") val transactionId = UUID.randomUUID() - // Вне зависимости от исхода оплаты важно отметить что она была отправлена. - // Это требуется сделать ВО ВСЕХ СЛУЧАЯХ, поскольку эта информация используется сервисом тестирования. scope.launch { try { paymentESService.update(paymentId) { @@ -111,9 +101,12 @@ class PaymentExternalSystemAdapterImpl( val minRequiredTime = requestAverageProcessingTime.toMillis() * 2 if (remaining < minRequiredTime) { logger.warn("[$accountName] Not enough time for payment $paymentId: ${remaining}ms remaining, need ${minRequiredTime}ms") - paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "not enough time") + runBlocking { + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "not enough time") + } } + val retryAfterMs = minRequiredTime - remaining + Random.nextLong(100) throw TooManyRequestsException(retryAfterMs) } @@ -185,74 +178,79 @@ class PaymentExternalSystemAdapterImpl( ) { httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .whenComplete { response, throwable -> - scope.launch { - try { - if (throwable != null) { - val e = throwable.cause - when (throwable.cause) { - is SocketTimeoutException -> { - logger.warn("[$accountName] attempt ${retryCount + 1} timeout: $paymentId", e) - } + try { + if (throwable != null) { + val e = throwable.cause + when (throwable.cause) { + is SocketTimeoutException -> { + logger.warn("[$accountName] attempt ${retryCount + 1} timeout: $paymentId", e) + } - is InterruptedIOException -> { - logger.warn("[$accountName] interrupted: $paymentId", e) - } + is InterruptedIOException -> { + logger.warn("[$accountName] interrupted: $paymentId", e) + } - else -> { - logger.warn("[$accountName] io error: $paymentId", e) - } + else -> { + logger.warn("[$accountName] io error: $paymentId", e) } + } - if (retryCount + 1 >= 3) { + if (retryCount + 1 >= 3) { + runBlocking { paymentESService.update(paymentId) { it.logProcessing(false, now(), transactionId, "Max attempts reached") } - } else { - val backoff = ((2.0.pow(retryCount.toDouble()) * 25).toLong() + Random.nextLong(10)) - val capped = backoff.coerceAtMost(deadline - now() - 5) - if (capped <= 0) { + } + } else { + val backoff = ((2.0.pow(retryCount.toDouble()) * 25).toLong() + Random.nextLong(10)) + val capped = backoff.coerceAtMost(deadline - now() - 5) + if (capped <= 0) { + runBlocking { paymentESService.update(paymentId) { it.logProcessing(false, now(), transactionId, "Deadline expired") } - } else { - scheduleRetry( - retryCount, - request, - paymentId, - transactionId, - timeBeforeCall, - deadline, - capped - ) - return@launch } + } else { + scheduleRetry( + retryCount, + request, + paymentId, + transactionId, + timeBeforeCall, + deadline, + capped + ) + return@whenComplete } - } else { - logger.warn("Free space in semaphore: {}", parallelLimiter.availablePermits) - logger.info( - "success in callback for payment: {}, retry count: {}, in time: {}", - paymentId, - retryCount, - now() - ) - val rawBody = response.body() - val parsed = try { - mapper.readValue(rawBody, ExternalSysResponse::class.java) - } catch (ex: Exception) { - ExternalSysResponse(transactionId.toString(), paymentId.toString(), false, ex.message) - } + } + } else { + logger.warn("Free space in semaphore: {}", parallelLimiter.availablePermits) + logger.info( + "success in callback for payment: {}, retry count: {}, in time: {}", + paymentId, + retryCount, + now() + ) + val rawBody = response.body() + val parsed = try { + mapper.readValue(rawBody, ExternalSysResponse::class.java) + } catch (ex: Exception) { + ExternalSysResponse(transactionId.toString(), paymentId.toString(), false, ex.message) + } + runBlocking { paymentESService.update(paymentId) { it.logProcessing(parsed.result, now(), transactionId, parsed.message) } - timer.record(now() - timeBeforeCall, TimeUnit.MILLISECONDS) } - } catch (e: Exception) { - logger.error("[$accountName] Error processing payment $paymentId", e) - } finally { - startedRequests.increment() - parallelLimiter.release() + + timer.record(now() - timeBeforeCall, TimeUnit.MILLISECONDS) } + } catch (e: Exception) { + logger.error("[$accountName] Error processing payment $paymentId", e) + } finally { + startedRequests.increment() + parallelLimiter.release() } } } @@ -261,10 +259,10 @@ class PaymentExternalSystemAdapterImpl( retryCount: Long, request: HttpRequest, paymentId: UUID, transactionId: UUID, timeBeforeCall: Long, deadline: Long, delay: Long ) { - retryScheduler.schedule({ + sharedScheduler.schedule({ val remainingTime = deadline - now() if (remainingTime < requestAverageProcessingTime.toMillis()) { - scope.launch { + runBlocking { try { paymentESService.update(paymentId) { it.logProcessing(false, now(), transactionId, "Not enough time for retry") @@ -289,6 +287,18 @@ class PaymentExternalSystemAdapterImpl( completeAction(retryCount + 1, newRequest, paymentId, transactionId, timeBeforeCall, deadline) }, delay, TimeUnit.MILLISECONDS) } + + override fun close() { + sharedScheduler.shutdown() + try { + if (!sharedScheduler.awaitTermination(5, TimeUnit.SECONDS)) { + sharedScheduler.shutdownNow() + } + } catch (e: InterruptedException) { + sharedScheduler.shutdownNow() + Thread.currentThread().interrupt() + } + } } fun now() = System.currentTimeMillis() \ No newline at end of file From 9c1a210599c7e5a5267810ac3cd9c7cbd30fb261 Mon Sep 17 00:00:00 2001 From: 21092004Goda Date: Mon, 8 Dec 2025 01:07:22 +0300 Subject: [PATCH 33/61] Merge remote-tracking branch 'origin/hw-9-io-kuro' into hw-9-io-kuro # Conflicts: # src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt --- .../logic/PaymentExternalServiceImpl.kt | 280 ++++++++++-------- 1 file changed, 160 insertions(+), 120 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index 89502dc6d..463b7cf88 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -22,6 +22,7 @@ import java.util.* import java.util.concurrent.Executors import java.util.concurrent.ScheduledExecutorService import java.util.concurrent.TimeUnit +import java.util.concurrent.TimeoutException import kotlin.math.pow import kotlin.random.Random @@ -35,8 +36,8 @@ class PaymentExternalSystemAdapterImpl( ) : PaymentExternalSystemAdapter, AutoCloseable { companion object { - val logger = LoggerFactory.getLogger(PaymentExternalSystemAdapter::class.java) - val mapper = ObjectMapper().registerKotlinModule() + private val logger = LoggerFactory.getLogger(PaymentExternalSystemAdapter::class.java) + private val mapper = ObjectMapper().registerKotlinModule() } private val sharedScheduler: ScheduledExecutorService = Executors.newScheduledThreadPool( @@ -47,15 +48,14 @@ class PaymentExternalSystemAdapterImpl( private val ioDispatcher: CoroutineDispatcher = sharedScheduler.asCoroutineDispatcher() private val exceptionHandler = CoroutineExceptionHandler { _, throwable -> - logger.error("[$accountName] Unhandled exception in payment adapter coroutine", throwable) + logger.error("[${properties.accountName}] Unhandled exception in payment adapter coroutine", throwable) } private val scope = CoroutineScope(SupervisorJob() + ioDispatcher + exceptionHandler) - private val startedRequests = - meterRegistry.counter("payment.processing.started", "accountName", properties.accountName) - private val timer = - meterRegistry.timer("payment.external.system.request.latency", "accountName", properties.accountName) + private val startedRequests = meterRegistry.counter("payment.processing.started", "accountName", properties.accountName) + private val timer = meterRegistry.timer("payment.external.system.request.latency", "accountName", properties.accountName) + private val serviceName = properties.serviceName private val accountName = properties.accountName private val requestAverageProcessingTime = properties.averageProcessingTime @@ -76,7 +76,6 @@ class PaymentExternalSystemAdapterImpl( .build() override fun performPaymentAsync(paymentId: UUID, amount: Int, paymentStartedAt: Long, deadline: Long) { - logger.warn("[$accountName] Submitting payment request for payment $paymentId") val transactionId = UUID.randomUUID() scope.launch { @@ -101,7 +100,7 @@ class PaymentExternalSystemAdapterImpl( val minRequiredTime = requestAverageProcessingTime.toMillis() * 2 if (remaining < minRequiredTime) { logger.warn("[$accountName] Not enough time for payment $paymentId: ${remaining}ms remaining, need ${minRequiredTime}ms") - runBlocking { + scope.launch { paymentESService.update(paymentId) { it.logProcessing(false, now(), transactionId, "not enough time") } @@ -111,158 +110,185 @@ class PaymentExternalSystemAdapterImpl( throw TooManyRequestsException(retryAfterMs) } - val timeBeforeCall = now() - - if (!tryAcquire(now(), deadline - now())) { - val retryAfterMs = (requestAverageProcessingTime.toMillis() / parallelRequests * 10).coerceIn( - 10, - 100 - ) + Random.nextLong(10) + var acquired = false + try { + if (!tryAcquire(now(), deadline - now())) { + val retryAfterMs = (requestAverageProcessingTime.toMillis() * 2).coerceAtLeast(100) + Random.nextLong(50) + throw TooManyRequestsException(retryAfterMs) + } + acquired = true - throw TooManyRequestsException(retryAfterMs) - } + if (!rateLimit.tickBlockingWithTimeout(deadline - now())) { + val retryAfterMs = (1100L / rateLimitPerSec).coerceAtLeast(100) + Random.nextLong(50) + throw TooManyRequestsException(retryAfterMs) + } - if (!rateLimit.tickBlockingWithTimeout(deadline - now())) { - parallelLimiter.release() + val requestTimeout = minOf( + deadline - now(), + requestAverageProcessingTime.toMillis() * 2 + ).coerceAtLeast(100) - val retryAfterMs = (1000L / rateLimitPerSec * 10).coerceIn(10, 100) + Random.nextLong(10) - throw TooManyRequestsException(retryAfterMs) - } + if (requestTimeout < requestAverageProcessingTime.toMillis()) { + logger.warn("[$accountName] Timeout too short for payment $paymentId: ${requestTimeout}ms") + val retryAfterMs = requestAverageProcessingTime.toMillis() - requestTimeout + Random.nextLong(100) + throw TooManyRequestsException(retryAfterMs) + } - val requestTimeout = minOf( - deadline - now(), - requestAverageProcessingTime.toMillis() * 2 - ).coerceAtLeast(100) + val request = HttpRequest.newBuilder() + .uri(URI("http://$paymentProviderHostPort/external/process?serviceName=$serviceName&token=$token&accountName=$accountName&transactionId=$transactionId&paymentId=$paymentId&amount=$amount")) + .timeout(Duration.ofMillis(requestTimeout)) + .POST(HttpRequest.BodyPublishers.noBody()) + .build() - if (requestTimeout < requestAverageProcessingTime.toMillis()) { - logger.warn("[$accountName] Timeout too short for payment $paymentId: ${requestTimeout}ms") - parallelLimiter.release() - val retryAfterMs = requestAverageProcessingTime.toMillis() - requestTimeout + Random.nextLong(100) - throw TooManyRequestsException(retryAfterMs) + completeAction( + retryCount = 0, + request = request, + paymentId = paymentId, + transactionId = transactionId, + timeBeforeCall = now(), + deadline = deadline, + acquired = true + ) + + } catch (e: TooManyRequestsException) { + if (acquired) { + parallelLimiter.release() + acquired = false + } + throw e + } catch (e: Exception) { + if (acquired) { + parallelLimiter.release() + acquired = false + } + logger.error("[$accountName] Unexpected error in performPaymentAsync for $paymentId", e) + scope.launch { + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "unexpected error: ${e.message}") + } + } + throw e } - - val request = HttpRequest.newBuilder() - .uri(URI("http://$paymentProviderHostPort/external/process?serviceName=$serviceName&token=$token&accountName=$accountName&transactionId=$transactionId&paymentId=$paymentId&amount=$amount")) - .timeout(Duration.ofMillis(requestTimeout)) - .POST(HttpRequest.BodyPublishers.noBody()) - .build() - - val retryCount = 0L - - completeAction(retryCount, request, paymentId, transactionId, timeBeforeCall, deadline) } - override fun price() = properties.price - - override fun isEnabled() = properties.enabled - - override fun name() = properties.accountName - - fun tryAcquire(startedAt: Long, remaining: Long): Boolean { + private fun tryAcquire(startedAt: Long, remaining: Long): Boolean { var isAcquired = parallelLimiter.tryAcquire() while (!isAcquired && now() - startedAt < remaining) { - isAcquired = parallelLimiter.tryAcquire() Thread.sleep(1) + isAcquired = parallelLimiter.tryAcquire() } - return isAcquired } - fun completeAction( + private fun completeAction( retryCount: Long, request: HttpRequest, paymentId: UUID, transactionId: UUID, timeBeforeCall: Long, - deadline: Long + deadline: Long, + acquired: Boolean ) { - httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString()) - .whenComplete { response, throwable -> - try { - if (throwable != null) { - val e = throwable.cause - when (throwable.cause) { - is SocketTimeoutException -> { - logger.warn("[$accountName] attempt ${retryCount + 1} timeout: $paymentId", e) - } + val requestTimeout = request.timeout().orElse(Duration.ofSeconds(10)).toMillis() + val future = httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString()) + .orTimeout(requestTimeout, TimeUnit.MILLISECONDS) - is InterruptedIOException -> { - logger.warn("[$accountName] interrupted: $paymentId", e) - } + future.whenComplete { response, throwable -> + try { + if (throwable != null) { + val cause = throwable.cause + val isTimeout = cause is SocketTimeoutException || throwable is TimeoutException + val isInterrupted = cause is InterruptedIOException + + val errorType = when { + isTimeout -> "timeout" + isInterrupted -> "interrupted" + else -> "io_error" + } + logger.warn("[$accountName] attempt ${retryCount + 1} $errorType: $paymentId", throwable) - else -> { - logger.warn("[$accountName] io error: $paymentId", e) + if (retryCount >= 2) { + scope.launch { + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "Max attempts reached") } } - - if (retryCount + 1 >= 3) { - runBlocking { + } else { + val backoff = ((2.0.pow(retryCount.toDouble()) * 25).toLong() + Random.nextLong(10)) + val capped = backoff.coerceAtMost(deadline - now() - 10) + if (capped <= 0) { + scope.launch { paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "Max attempts reached") + it.logProcessing(false, now(), transactionId, "Deadline expired") } } } else { - val backoff = ((2.0.pow(retryCount.toDouble()) * 25).toLong() + Random.nextLong(10)) - val capped = backoff.coerceAtMost(deadline - now() - 5) - if (capped <= 0) { - runBlocking { - paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "Deadline expired") - } - } - } else { - scheduleRetry( - retryCount, - request, - paymentId, - transactionId, - timeBeforeCall, - deadline, - capped - ) - return@whenComplete - } - } - } else { - logger.warn("Free space in semaphore: {}", parallelLimiter.availablePermits) - logger.info( - "success in callback for payment: {}, retry count: {}, in time: {}", - paymentId, - retryCount, - now() - ) - val rawBody = response.body() - val parsed = try { - mapper.readValue(rawBody, ExternalSysResponse::class.java) - } catch (ex: Exception) { - ExternalSysResponse(transactionId.toString(), paymentId.toString(), false, ex.message) + scheduleRetry( + retryCount = retryCount + 1, + request = request, + paymentId = paymentId, + transactionId = transactionId, + timeBeforeCall = timeBeforeCall, + deadline = deadline, + delay = capped, + acquired = acquired + ) + return@whenComplete } + } + } else { + logger.debug("[$accountName] Free space in semaphore: ${parallelLimiter.availablePermits}") + logger.info( + "[$accountName] success for payment: {}, retry count: {}, in time: {}", + paymentId, + retryCount, + now() + ) + val rawBody = response.body() + val parsed = try { + mapper.readValue(rawBody, ExternalSysResponse::class.java) + } catch (ex: Exception) { + ExternalSysResponse(transactionId.toString(), paymentId.toString(), false, ex.message) + } - runBlocking { - paymentESService.update(paymentId) { - it.logProcessing(parsed.result, now(), transactionId, parsed.message) - } + scope.launch { + paymentESService.update(paymentId) { + it.logProcessing(parsed.result, now(), transactionId, parsed.message) } + } - timer.record(now() - timeBeforeCall, TimeUnit.MILLISECONDS) + timer.record(now() - timeBeforeCall, TimeUnit.MILLISECONDS) + } + } catch (e: Exception) { + logger.error("[$accountName] Error in whenComplete for payment $paymentId", e) + scope.launch { + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "callback error: ${e.message}") } - } catch (e: Exception) { - logger.error("[$accountName] Error processing payment $paymentId", e) - } finally { - startedRequests.increment() + } + } finally { + startedRequests.increment() + if (acquired) { parallelLimiter.release() } } + } } private fun scheduleRetry( - retryCount: Long, request: HttpRequest, paymentId: UUID, - transactionId: UUID, timeBeforeCall: Long, deadline: Long, delay: Long + retryCount: Long, + request: HttpRequest, + paymentId: UUID, + transactionId: UUID, + timeBeforeCall: Long, + deadline: Long, + delay: Long, + acquired: Boolean ) { sharedScheduler.schedule({ val remainingTime = deadline - now() if (remainingTime < requestAverageProcessingTime.toMillis()) { - runBlocking { + scope.launch { try { paymentESService.update(paymentId) { it.logProcessing(false, now(), transactionId, "Not enough time for retry") @@ -271,23 +297,37 @@ class PaymentExternalSystemAdapterImpl( logger.error("[$accountName] Failed to record retry failure for $paymentId", e) } finally { startedRequests.increment() - parallelLimiter.release() + if (acquired) parallelLimiter.release() } } return@schedule } - val newRequestTimeout = remainingTime.coerceIn(100, requestAverageProcessingTime.toMillis() * 2) + val newRequestTimeout = minOf(remainingTime, requestAverageProcessingTime.toMillis() * 2).coerceAtLeast(100) val newRequest = HttpRequest.newBuilder() .uri(request.uri()) .timeout(Duration.ofMillis(newRequestTimeout)) .POST(HttpRequest.BodyPublishers.noBody()) .build() - completeAction(retryCount + 1, newRequest, paymentId, transactionId, timeBeforeCall, deadline) + completeAction( + retryCount = retryCount, + request = newRequest, + paymentId = paymentId, + transactionId = transactionId, + timeBeforeCall = timeBeforeCall, + deadline = deadline, + acquired = acquired + ) }, delay, TimeUnit.MILLISECONDS) } + override fun price() = properties.price + + override fun isEnabled() = properties.enabled + + override fun name() = properties.accountName + override fun close() { sharedScheduler.shutdown() try { From df2df2901a836213c36a9ac82c5374331f9980ba Mon Sep 17 00:00:00 2001 From: 21092004Goda Date: Mon, 8 Dec 2025 01:14:48 +0300 Subject: [PATCH 34/61] Merge remote-tracking branch 'origin/hw-9-io-kuro' into hw-9-io-kuro # Conflicts: # src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt --- .../ru/quipy/payments/logic/PaymentExternalServiceImpl.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index 463b7cf88..cd9417bcb 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -36,8 +36,8 @@ class PaymentExternalSystemAdapterImpl( ) : PaymentExternalSystemAdapter, AutoCloseable { companion object { - private val logger = LoggerFactory.getLogger(PaymentExternalSystemAdapter::class.java) - private val mapper = ObjectMapper().registerKotlinModule() + val logger = LoggerFactory.getLogger(PaymentExternalSystemAdapter::class.java) + val mapper = ObjectMapper().registerKotlinModule() } private val sharedScheduler: ScheduledExecutorService = Executors.newScheduledThreadPool( From 2a3c72ea007a6e1ec648e47424b22730dbeaf57b Mon Sep 17 00:00:00 2001 From: 21092004Goda Date: Mon, 8 Dec 2025 01:18:45 +0300 Subject: [PATCH 35/61] Merge remote-tracking branch 'origin/hw-9-io-kuro' into hw-9-io-kuro # Conflicts: # src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt --- .../logic/PaymentExternalServiceImpl.kt | 326 ++++++++---------- 1 file changed, 138 insertions(+), 188 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index cd9417bcb..98898467c 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -3,7 +3,12 @@ package ru.quipy.payments.logic import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.registerKotlinModule import io.micrometer.core.instrument.MeterRegistry -import kotlinx.coroutines.* +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.CoroutineExceptionHandler +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.asCoroutineDispatcher +import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Semaphore import org.slf4j.LoggerFactory import ru.quipy.common.utils.NamedThreadFactory @@ -20,12 +25,12 @@ import java.net.http.HttpResponse import java.time.Duration import java.util.* import java.util.concurrent.Executors -import java.util.concurrent.ScheduledExecutorService import java.util.concurrent.TimeUnit -import java.util.concurrent.TimeoutException import kotlin.math.pow import kotlin.random.Random + +// Advice: always treat time as a Duration class PaymentExternalSystemAdapterImpl( private val properties: PaymentAccountProperties, private val paymentESService: EventSourcingService, @@ -33,29 +38,28 @@ class PaymentExternalSystemAdapterImpl( private val token: String, meterRegistry: MeterRegistry, private val parallelLimiter: Semaphore, -) : PaymentExternalSystemAdapter, AutoCloseable { - + private val ioDispatcher: CoroutineDispatcher = Executors.newFixedThreadPool( + (Runtime.getRuntime().availableProcessors() * 2.5).toInt(), + NamedThreadFactory("payment-io-") + ).asCoroutineDispatcher() +) : PaymentExternalSystemAdapter { companion object { val logger = LoggerFactory.getLogger(PaymentExternalSystemAdapter::class.java) val mapper = ObjectMapper().registerKotlinModule() } - private val sharedScheduler: ScheduledExecutorService = Executors.newScheduledThreadPool( - Runtime.getRuntime().availableProcessors() * 2, - NamedThreadFactory("payment-shared-${properties.accountName}-") - ) - - private val ioDispatcher: CoroutineDispatcher = sharedScheduler.asCoroutineDispatcher() - private val exceptionHandler = CoroutineExceptionHandler { _, throwable -> - logger.error("[${properties.accountName}] Unhandled exception in payment adapter coroutine", throwable) + logger.error("[$accountName] Unhandled exception in payment adapter coroutine", throwable) } private val scope = CoroutineScope(SupervisorJob() + ioDispatcher + exceptionHandler) - private val startedRequests = meterRegistry.counter("payment.processing.started", "accountName", properties.accountName) - private val timer = meterRegistry.timer("payment.external.system.request.latency", "accountName", properties.accountName) + // 2025-11-20T20:30:35.780+03:00 INFO 56644 --- [alhost:1234/...] ru.quipy.core.EventSourcingService : Optimistic lock exception. Failed to save event records id: [7dca693e-e811-4b7f-8bce-23e13d952c04-4] + private val startedRequests = + meterRegistry.counter("payment.processing.started", "accountName", properties.accountName) + private val timer = + meterRegistry.timer("payment.external.system.request.latency", "accountName", properties.accountName) private val serviceName = properties.serviceName private val accountName = properties.accountName private val requestAverageProcessingTime = properties.averageProcessingTime @@ -71,13 +75,20 @@ class PaymentExternalSystemAdapterImpl( private val httpClient = HttpClient .newBuilder() - .executor(sharedScheduler) + .executor(Executors.newFixedThreadPool(parallelRequests)) .version(HttpClient.Version.HTTP_2) .build() + private val retryScheduler = Executors.newScheduledThreadPool( + Runtime.getRuntime().availableProcessors() + ) + override fun performPaymentAsync(paymentId: UUID, amount: Int, paymentStartedAt: Long, deadline: Long) { + logger.warn("[$accountName] Submitting payment request for payment $paymentId") val transactionId = UUID.randomUUID() + // Вне зависимости от исхода оплаты важно отметить что она была отправлена. + // Это требуется сделать ВО ВСЕХ СЛУЧАЯХ, поскольку эта информация используется сервисом тестирования. scope.launch { try { paymentESService.update(paymentId) { @@ -100,192 +111,157 @@ class PaymentExternalSystemAdapterImpl( val minRequiredTime = requestAverageProcessingTime.toMillis() * 2 if (remaining < minRequiredTime) { logger.warn("[$accountName] Not enough time for payment $paymentId: ${remaining}ms remaining, need ${minRequiredTime}ms") - scope.launch { - paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "not enough time") - } + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "not enough time") } - val retryAfterMs = minRequiredTime - remaining + Random.nextLong(100) throw TooManyRequestsException(retryAfterMs) } - var acquired = false - try { - if (!tryAcquire(now(), deadline - now())) { - val retryAfterMs = (requestAverageProcessingTime.toMillis() * 2).coerceAtLeast(100) + Random.nextLong(50) - throw TooManyRequestsException(retryAfterMs) - } - acquired = true + val timeBeforeCall = now() - if (!rateLimit.tickBlockingWithTimeout(deadline - now())) { - val retryAfterMs = (1100L / rateLimitPerSec).coerceAtLeast(100) + Random.nextLong(50) - throw TooManyRequestsException(retryAfterMs) - } + if (!tryAcquire(now(), deadline - now())) { + val retryAfterMs = (requestAverageProcessingTime.toMillis() / parallelRequests * 10).coerceIn( + 10, + 100 + ) + Random.nextLong(10) - val requestTimeout = minOf( - deadline - now(), - requestAverageProcessingTime.toMillis() * 2 - ).coerceAtLeast(100) + throw TooManyRequestsException(retryAfterMs) + } - if (requestTimeout < requestAverageProcessingTime.toMillis()) { - logger.warn("[$accountName] Timeout too short for payment $paymentId: ${requestTimeout}ms") - val retryAfterMs = requestAverageProcessingTime.toMillis() - requestTimeout + Random.nextLong(100) - throw TooManyRequestsException(retryAfterMs) - } + if (!rateLimit.tickBlockingWithTimeout(deadline - now())) { + parallelLimiter.release() - val request = HttpRequest.newBuilder() - .uri(URI("http://$paymentProviderHostPort/external/process?serviceName=$serviceName&token=$token&accountName=$accountName&transactionId=$transactionId&paymentId=$paymentId&amount=$amount")) - .timeout(Duration.ofMillis(requestTimeout)) - .POST(HttpRequest.BodyPublishers.noBody()) - .build() + val retryAfterMs = (1000L / rateLimitPerSec * 10).coerceIn(10, 100) + Random.nextLong(10) + throw TooManyRequestsException(retryAfterMs) + } - completeAction( - retryCount = 0, - request = request, - paymentId = paymentId, - transactionId = transactionId, - timeBeforeCall = now(), - deadline = deadline, - acquired = true - ) - - } catch (e: TooManyRequestsException) { - if (acquired) { - parallelLimiter.release() - acquired = false - } - throw e - } catch (e: Exception) { - if (acquired) { - parallelLimiter.release() - acquired = false - } - logger.error("[$accountName] Unexpected error in performPaymentAsync for $paymentId", e) - scope.launch { - paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "unexpected error: ${e.message}") - } - } - throw e + val requestTimeout = minOf( + deadline - now(), + requestAverageProcessingTime.toMillis() * 2 + ).coerceAtLeast(100) + + if (requestTimeout < requestAverageProcessingTime.toMillis()) { + logger.warn("[$accountName] Timeout too short for payment $paymentId: ${requestTimeout}ms") + parallelLimiter.release() + val retryAfterMs = requestAverageProcessingTime.toMillis() - requestTimeout + Random.nextLong(100) + throw TooManyRequestsException(retryAfterMs) } + + val request = HttpRequest.newBuilder() + .uri(URI("http://$paymentProviderHostPort/external/process?serviceName=$serviceName&token=$token&accountName=$accountName&transactionId=$transactionId&paymentId=$paymentId&amount=$amount")) + .timeout(Duration.ofMillis(requestTimeout)) + .POST(HttpRequest.BodyPublishers.noBody()) + .build() + + val retryCount = 0L + + completeAction(retryCount, request, paymentId, transactionId, timeBeforeCall, deadline) } - private fun tryAcquire(startedAt: Long, remaining: Long): Boolean { + override fun price() = properties.price + + override fun isEnabled() = properties.enabled + + override fun name() = properties.accountName + + fun tryAcquire(startedAt: Long, remaining: Long): Boolean { var isAcquired = parallelLimiter.tryAcquire() while (!isAcquired && now() - startedAt < remaining) { - Thread.sleep(1) isAcquired = parallelLimiter.tryAcquire() + Thread.sleep(1) } + return isAcquired } - private fun completeAction( + fun completeAction( retryCount: Long, request: HttpRequest, paymentId: UUID, transactionId: UUID, timeBeforeCall: Long, - deadline: Long, - acquired: Boolean + deadline: Long ) { - val requestTimeout = request.timeout().orElse(Duration.ofSeconds(10)).toMillis() - val future = httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString()) - .orTimeout(requestTimeout, TimeUnit.MILLISECONDS) + httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString()) + .whenComplete { response, throwable -> + scope.launch { + try { + if (throwable != null) { + val e = throwable.cause + when (throwable.cause) { + is SocketTimeoutException -> { + logger.warn("[$accountName] attempt ${retryCount + 1} timeout: $paymentId", e) + } - future.whenComplete { response, throwable -> - try { - if (throwable != null) { - val cause = throwable.cause - val isTimeout = cause is SocketTimeoutException || throwable is TimeoutException - val isInterrupted = cause is InterruptedIOException - - val errorType = when { - isTimeout -> "timeout" - isInterrupted -> "interrupted" - else -> "io_error" - } - logger.warn("[$accountName] attempt ${retryCount + 1} $errorType: $paymentId", throwable) + is InterruptedIOException -> { + logger.warn("[$accountName] interrupted: $paymentId", e) + } - if (retryCount >= 2) { - scope.launch { - paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "Max attempts reached") + else -> { + logger.warn("[$accountName] io error: $paymentId", e) + } } - } - } else { - val backoff = ((2.0.pow(retryCount.toDouble()) * 25).toLong() + Random.nextLong(10)) - val capped = backoff.coerceAtMost(deadline - now() - 10) - if (capped <= 0) { - scope.launch { + + if (retryCount + 1 >= 3) { paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "Deadline expired") + it.logProcessing(false, now(), transactionId, "Max attempts reached") + } + } else { + val backoff = ((2.0.pow(retryCount.toDouble()) * 25).toLong() + Random.nextLong(10)) + val capped = backoff.coerceAtMost(deadline - now() - 5) + if (capped <= 0) { + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "Deadline expired") + } + } else { + scheduleRetry( + retryCount, + request, + paymentId, + transactionId, + timeBeforeCall, + deadline, + capped + ) + return@launch } } } else { - scheduleRetry( - retryCount = retryCount + 1, - request = request, - paymentId = paymentId, - transactionId = transactionId, - timeBeforeCall = timeBeforeCall, - deadline = deadline, - delay = capped, - acquired = acquired + logger.warn("Free space in semaphore: {}", parallelLimiter.availablePermits) + logger.info( + "success in callback for payment: {}, retry count: {}, in time: {}", + paymentId, + retryCount, + now() ) - return@whenComplete - } - } - } else { - logger.debug("[$accountName] Free space in semaphore: ${parallelLimiter.availablePermits}") - logger.info( - "[$accountName] success for payment: {}, retry count: {}, in time: {}", - paymentId, - retryCount, - now() - ) - val rawBody = response.body() - val parsed = try { - mapper.readValue(rawBody, ExternalSysResponse::class.java) - } catch (ex: Exception) { - ExternalSysResponse(transactionId.toString(), paymentId.toString(), false, ex.message) - } + val rawBody = response.body() + val parsed = try { + mapper.readValue(rawBody, ExternalSysResponse::class.java) + } catch (ex: Exception) { + ExternalSysResponse(transactionId.toString(), paymentId.toString(), false, ex.message) + } - scope.launch { - paymentESService.update(paymentId) { - it.logProcessing(parsed.result, now(), transactionId, parsed.message) + paymentESService.update(paymentId) { + it.logProcessing(parsed.result, now(), transactionId, parsed.message) + } + timer.record(now() - timeBeforeCall, TimeUnit.MILLISECONDS) } - } - - timer.record(now() - timeBeforeCall, TimeUnit.MILLISECONDS) - } - } catch (e: Exception) { - logger.error("[$accountName] Error in whenComplete for payment $paymentId", e) - scope.launch { - paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "callback error: ${e.message}") + } catch (e: Exception) { + logger.error("[$accountName] Error processing payment $paymentId", e) + } finally { + startedRequests.increment() + parallelLimiter.release() } } - } finally { - startedRequests.increment() - if (acquired) { - parallelLimiter.release() - } } - } } private fun scheduleRetry( - retryCount: Long, - request: HttpRequest, - paymentId: UUID, - transactionId: UUID, - timeBeforeCall: Long, - deadline: Long, - delay: Long, - acquired: Boolean + retryCount: Long, request: HttpRequest, paymentId: UUID, + transactionId: UUID, timeBeforeCall: Long, deadline: Long, delay: Long ) { - sharedScheduler.schedule({ + retryScheduler.schedule({ val remainingTime = deadline - now() if (remainingTime < requestAverageProcessingTime.toMillis()) { scope.launch { @@ -297,48 +273,22 @@ class PaymentExternalSystemAdapterImpl( logger.error("[$accountName] Failed to record retry failure for $paymentId", e) } finally { startedRequests.increment() - if (acquired) parallelLimiter.release() + parallelLimiter.release() } } return@schedule } - val newRequestTimeout = minOf(remainingTime, requestAverageProcessingTime.toMillis() * 2).coerceAtLeast(100) + val newRequestTimeout = remainingTime.coerceIn(100, requestAverageProcessingTime.toMillis() * 2) val newRequest = HttpRequest.newBuilder() .uri(request.uri()) .timeout(Duration.ofMillis(newRequestTimeout)) .POST(HttpRequest.BodyPublishers.noBody()) .build() - completeAction( - retryCount = retryCount, - request = newRequest, - paymentId = paymentId, - transactionId = transactionId, - timeBeforeCall = timeBeforeCall, - deadline = deadline, - acquired = acquired - ) + completeAction(retryCount + 1, newRequest, paymentId, transactionId, timeBeforeCall, deadline) }, delay, TimeUnit.MILLISECONDS) } - - override fun price() = properties.price - - override fun isEnabled() = properties.enabled - - override fun name() = properties.accountName - - override fun close() { - sharedScheduler.shutdown() - try { - if (!sharedScheduler.awaitTermination(5, TimeUnit.SECONDS)) { - sharedScheduler.shutdownNow() - } - } catch (e: InterruptedException) { - sharedScheduler.shutdownNow() - Thread.currentThread().interrupt() - } - } } fun now() = System.currentTimeMillis() \ No newline at end of file From 824c90ecd165259a2c295c695586a9b15590ee94 Mon Sep 17 00:00:00 2001 From: 21092004Goda Date: Mon, 8 Dec 2025 01:27:12 +0300 Subject: [PATCH 36/61] Merge remote-tracking branch 'origin/hw-9-io-kuro' into hw-9-io-kuro # Conflicts: # src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt --- .../ru/quipy/payments/logic/PaymentExternalServiceImpl.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index 98898467c..086cf25dc 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -39,7 +39,7 @@ class PaymentExternalSystemAdapterImpl( meterRegistry: MeterRegistry, private val parallelLimiter: Semaphore, private val ioDispatcher: CoroutineDispatcher = Executors.newFixedThreadPool( - (Runtime.getRuntime().availableProcessors() * 2.5).toInt(), + Runtime.getRuntime().availableProcessors() * 2, NamedThreadFactory("payment-io-") ).asCoroutineDispatcher() ) : PaymentExternalSystemAdapter { From 765e50f84e82ffb2534cc788065a50979ab3b9ce Mon Sep 17 00:00:00 2001 From: 21092004Goda Date: Mon, 8 Dec 2025 01:33:02 +0300 Subject: [PATCH 37/61] Merge remote-tracking branch 'origin/hw-9-io-kuro' into hw-9-io-kuro # Conflicts: # src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt --- .../logic/PaymentExternalServiceImpl.kt | 381 ++++++++++-------- 1 file changed, 213 insertions(+), 168 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index 086cf25dc..e92d13475 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -3,13 +3,10 @@ package ru.quipy.payments.logic import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.registerKotlinModule import io.micrometer.core.instrument.MeterRegistry -import kotlinx.coroutines.CoroutineDispatcher -import kotlinx.coroutines.CoroutineExceptionHandler -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.* import kotlinx.coroutines.asCoroutineDispatcher -import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.sync.withPermit import org.slf4j.LoggerFactory import ru.quipy.common.utils.NamedThreadFactory import ru.quipy.common.utils.SlidingWindowRateLimiter @@ -25,12 +22,12 @@ import java.net.http.HttpResponse import java.time.Duration import java.util.* import java.util.concurrent.Executors +import java.util.concurrent.ScheduledFuture import java.util.concurrent.TimeUnit import kotlin.math.pow import kotlin.random.Random +import kotlin.time.Duration.Companion.milliseconds - -// Advice: always treat time as a Duration class PaymentExternalSystemAdapterImpl( private val properties: PaymentAccountProperties, private val paymentESService: EventSourcingService, @@ -54,12 +51,17 @@ class PaymentExternalSystemAdapterImpl( private val scope = CoroutineScope(SupervisorJob() + ioDispatcher + exceptionHandler) - - // 2025-11-20T20:30:35.780+03:00 INFO 56644 --- [alhost:1234/...] ru.quipy.core.EventSourcingService : Optimistic lock exception. Failed to save event records id: [7dca693e-e811-4b7f-8bce-23e13d952c04-4] private val startedRequests = meterRegistry.counter("payment.processing.started", "accountName", properties.accountName) private val timer = meterRegistry.timer("payment.external.system.request.latency", "accountName", properties.accountName) + private val successCounter = + meterRegistry.counter("payment.processing.success", "accountName", properties.accountName) + private val failureCounter = + meterRegistry.counter("payment.processing.failure", "accountName", properties.accountName) + private val timeoutCounter = + meterRegistry.counter("payment.processing.timeout", "accountName", properties.accountName) + private val serviceName = properties.serviceName private val accountName = properties.accountName private val requestAverageProcessingTime = properties.averageProcessingTime @@ -77,27 +79,30 @@ class PaymentExternalSystemAdapterImpl( .newBuilder() .executor(Executors.newFixedThreadPool(parallelRequests)) .version(HttpClient.Version.HTTP_2) + .connectTimeout(Duration.ofSeconds(10)) .build() private val retryScheduler = Executors.newScheduledThreadPool( Runtime.getRuntime().availableProcessors() ) + private val pendingRequests = Collections.synchronizedMap(mutableMapOf>()) + override fun performPaymentAsync(paymentId: UUID, amount: Int, paymentStartedAt: Long, deadline: Long) { - logger.warn("[$accountName] Submitting payment request for payment $paymentId") + logger.info("[$accountName] Submitting payment request for payment $paymentId") val transactionId = UUID.randomUUID() - // Вне зависимости от исхода оплаты важно отметить что она была отправлена. - // Это требуется сделать ВО ВСЕХ СЛУЧАЯХ, поскольку эта информация используется сервисом тестирования. scope.launch { try { - paymentESService.update(paymentId) { - it.logSubmission( - success = true, - transactionId, - now(), - Duration.ofMillis(now() - paymentStartedAt) - ) + withTimeout(deadline - System.currentTimeMillis()) { + paymentESService.update(paymentId) { + it.logSubmission( + success = true, + transactionId, + now(), + Duration.ofMillis(now() - paymentStartedAt) + ) + } } logger.info("[$accountName] Log submission recorded for $paymentId") } catch (e: Exception) { @@ -105,189 +110,229 @@ class PaymentExternalSystemAdapterImpl( } } - logger.info("[$accountName] Submit: $paymentId , txId: $transactionId") - - val remaining = deadline - now() - val minRequiredTime = requestAverageProcessingTime.toMillis() * 2 - if (remaining < minRequiredTime) { - logger.warn("[$accountName] Not enough time for payment $paymentId: ${remaining}ms remaining, need ${minRequiredTime}ms") - paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "not enough time") + scope.launch { + try { + executePaymentWithRetries(paymentId, transactionId, amount, deadline) + } catch (e: Exception) { + logger.error("[$accountName] Failed to process payment $paymentId", e) + handlePaymentFailure(paymentId, transactionId, e) } - val retryAfterMs = minRequiredTime - remaining + Random.nextLong(100) - throw TooManyRequestsException(retryAfterMs) } + } + + private suspend fun executePaymentWithRetries( + paymentId: UUID, + transactionId: UUID, + amount: Int, + deadline: Long + ) { + var retryCount = 0 + val maxRetries = 3 - val timeBeforeCall = now() + while (retryCount <= maxRetries) { + try { + checkDeadline(deadline, paymentId) + + val result = executePaymentAttempt(paymentId, transactionId, amount, deadline) + + if (result) { + successCounter.increment() + return + } else { + retryCount++ + if (retryCount <= maxRetries) { + val backoff = calculateBackoff(retryCount, deadline) + delay(backoff) + } + } + } catch (e: TooManyRequestsException) { + throw e + } catch (e: CancellationException) { + logger.warn("[$accountName] Payment $paymentId was cancelled") + throw e + } catch (e: Exception) { + logger.warn("[$accountName] Attempt ${retryCount + 1} failed for payment $paymentId", e) + retryCount++ - if (!tryAcquire(now(), deadline - now())) { - val retryAfterMs = (requestAverageProcessingTime.toMillis() / parallelRequests * 10).coerceIn( - 10, - 100 - ) + Random.nextLong(10) + if (retryCount > maxRetries) { + throw e + } - throw TooManyRequestsException(retryAfterMs) + val backoff = calculateBackoff(retryCount, deadline) + delay(backoff) + } } - if (!rateLimit.tickBlockingWithTimeout(deadline - now())) { - parallelLimiter.release() + throw RuntimeException("All retry attempts exhausted for payment $paymentId") + } - val retryAfterMs = (1000L / rateLimitPerSec * 10).coerceIn(10, 100) + Random.nextLong(10) - throw TooManyRequestsException(retryAfterMs) - } + private suspend fun executePaymentAttempt( + paymentId: UUID, + transactionId: UUID, + amount: Int, + deadline: Long + ): Boolean = withTimeoutOrNull(deadline - System.currentTimeMillis()) { + try { + val semaphoreAcquired = withTimeoutOrNull(deadline - System.currentTimeMillis()) { + parallelLimiter.tryAcquire() + if (!parallelLimiter.tryAcquire()) { + val waitTime = minOf(deadline - System.currentTimeMillis(), 100L) + if (waitTime > 0) { + delay(waitTime) + } + parallelLimiter.tryAcquire() + } else { + true + } + } ?: false - val requestTimeout = minOf( - deadline - now(), - requestAverageProcessingTime.toMillis() * 2 - ).coerceAtLeast(100) + if (!semaphoreAcquired) { + val retryAfter = (requestAverageProcessingTime.toMillis() / parallelRequests).coerceIn(10, 100) + throw TooManyRequestsException(retryAfter + Random.nextLong(10)) + } + + val rateLimitAcquired = withTimeoutOrNull(deadline - System.currentTimeMillis()) { + rateLimit.tickBlockingWithTimeout(deadline - System.currentTimeMillis()) + } ?: false + + if (!rateLimitAcquired) { + parallelLimiter.release() + val retryAfter = (1000L / rateLimitPerSec).coerceIn(10, 100) + throw TooManyRequestsException(retryAfter + Random.nextLong(10)) + } + + val requestTimeout = calculateRequestTimeout(deadline) - if (requestTimeout < requestAverageProcessingTime.toMillis()) { - logger.warn("[$accountName] Timeout too short for payment $paymentId: ${requestTimeout}ms") - parallelLimiter.release() - val retryAfterMs = requestAverageProcessingTime.toMillis() - requestTimeout + Random.nextLong(100) - throw TooManyRequestsException(retryAfterMs) + val response = executeHttpRequest(paymentId, transactionId, amount, requestTimeout) + + val processingResult = processHttpResponse(response, paymentId, transactionId) + + if (processingResult) { + paymentESService.update(paymentId) { + it.logProcessing(true, now(), transactionId, "Success") + } + } + + return@withTimeoutOrNull processingResult + + } finally { + if (parallelLimiter.availablePermits < parallelRequests) { + parallelLimiter.release() + } + startedRequests.increment() } + } ?: run { + timeoutCounter.increment() + logger.warn("[$accountName] Payment $paymentId timed out") + false + } + private suspend fun executeHttpRequest( + paymentId: UUID, + transactionId: UUID, + amount: Int, + requestTimeout: Long + ): HttpResponse = withContext(Dispatchers.IO) { val request = HttpRequest.newBuilder() .uri(URI("http://$paymentProviderHostPort/external/process?serviceName=$serviceName&token=$token&accountName=$accountName&transactionId=$transactionId&paymentId=$paymentId&amount=$amount")) .timeout(Duration.ofMillis(requestTimeout)) .POST(HttpRequest.BodyPublishers.noBody()) .build() - val retryCount = 0L - - completeAction(retryCount, request, paymentId, transactionId, timeBeforeCall, deadline) + try { + val timeBeforeCall = now() + val response = httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString()).get() + timer.record(now() - timeBeforeCall, TimeUnit.MILLISECONDS) + response + } catch (e: Exception) { + logger.error("[$accountName] HTTP request failed for $paymentId", e) + throw e + } } - override fun price() = properties.price + private fun processHttpResponse( + response: HttpResponse, + paymentId: UUID, + transactionId: UUID + ): Boolean { + return try { + val rawBody = response.body() + val parsed = mapper.readValue(rawBody, ExternalSysResponse::class.java) + + if (!parsed.result) { + logger.warn("[$accountName] External system returned failure for $paymentId: ${parsed.message}") + failureCounter.increment() + } - override fun isEnabled() = properties.enabled + parsed.result + } catch (ex: Exception) { + logger.error("[$accountName] Failed to parse response for $paymentId", ex) + failureCounter.increment() + false + } + } - override fun name() = properties.accountName + private fun checkDeadline(deadline: Long, paymentId: UUID) { + val remaining = deadline - System.currentTimeMillis() + if (remaining <= 0) { + throw CancellationException("Deadline expired for payment $paymentId") + } - fun tryAcquire(startedAt: Long, remaining: Long): Boolean { - var isAcquired = parallelLimiter.tryAcquire() - while (!isAcquired && now() - startedAt < remaining) { - isAcquired = parallelLimiter.tryAcquire() - Thread.sleep(1) + val minRequiredTime = requestAverageProcessingTime.toMillis() * 2 + if (remaining < minRequiredTime) { + logger.warn("[$accountName] Not enough time for payment $paymentId: ${remaining}ms remaining, need ${minRequiredTime}ms") + throw TooManyRequestsException(minRequiredTime - remaining + Random.nextLong(100)) } + } - return isAcquired + private fun calculateRequestTimeout(deadline: Long): Long { + val remaining = deadline - System.currentTimeMillis() + return minOf( + remaining * 2 / 3, + requestAverageProcessingTime.toMillis() * 3 + ).coerceIn(100, 30000L) } - fun completeAction( - retryCount: Long, - request: HttpRequest, - paymentId: UUID, - transactionId: UUID, - timeBeforeCall: Long, - deadline: Long - ) { - httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString()) - .whenComplete { response, throwable -> - scope.launch { - try { - if (throwable != null) { - val e = throwable.cause - when (throwable.cause) { - is SocketTimeoutException -> { - logger.warn("[$accountName] attempt ${retryCount + 1} timeout: $paymentId", e) - } - - is InterruptedIOException -> { - logger.warn("[$accountName] interrupted: $paymentId", e) - } - - else -> { - logger.warn("[$accountName] io error: $paymentId", e) - } - } - - if (retryCount + 1 >= 3) { - paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "Max attempts reached") - } - } else { - val backoff = ((2.0.pow(retryCount.toDouble()) * 25).toLong() + Random.nextLong(10)) - val capped = backoff.coerceAtMost(deadline - now() - 5) - if (capped <= 0) { - paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "Deadline expired") - } - } else { - scheduleRetry( - retryCount, - request, - paymentId, - transactionId, - timeBeforeCall, - deadline, - capped - ) - return@launch - } - } - } else { - logger.warn("Free space in semaphore: {}", parallelLimiter.availablePermits) - logger.info( - "success in callback for payment: {}, retry count: {}, in time: {}", - paymentId, - retryCount, - now() - ) - val rawBody = response.body() - val parsed = try { - mapper.readValue(rawBody, ExternalSysResponse::class.java) - } catch (ex: Exception) { - ExternalSysResponse(transactionId.toString(), paymentId.toString(), false, ex.message) - } - - paymentESService.update(paymentId) { - it.logProcessing(parsed.result, now(), transactionId, parsed.message) - } - timer.record(now() - timeBeforeCall, TimeUnit.MILLISECONDS) - } - } catch (e: Exception) { - logger.error("[$accountName] Error processing payment $paymentId", e) - } finally { - startedRequests.increment() - parallelLimiter.release() - } - } - } + private fun calculateBackoff(retryCount: Int, deadline: Long): Long { + val baseDelay = (2.0.pow(retryCount.toDouble()) * 25).toLong() + val jitter = Random.nextLong(10) + val totalDelay = baseDelay + jitter + + val remaining = deadline - System.currentTimeMillis() + val safeDelay = minOf(totalDelay, remaining / 2) + + return safeDelay.coerceAtLeast(0) } - private fun scheduleRetry( - retryCount: Long, request: HttpRequest, paymentId: UUID, - transactionId: UUID, timeBeforeCall: Long, deadline: Long, delay: Long - ) { - retryScheduler.schedule({ - val remainingTime = deadline - now() - if (remainingTime < requestAverageProcessingTime.toMillis()) { - scope.launch { - try { - paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "Not enough time for retry") - } - } catch (e: Exception) { - logger.error("[$accountName] Failed to record retry failure for $paymentId", e) - } finally { - startedRequests.increment() - parallelLimiter.release() + private fun handlePaymentFailure(paymentId: UUID, transactionId: UUID, cause: Exception) { + scope.launch { + try { + paymentESService.update(paymentId) { + val message = when (cause) { + is TooManyRequestsException -> "Rate limited" + is TimeoutCancellationException -> "Timeout" + is CancellationException -> "Cancelled" + else -> cause.message ?: "Unknown error" } + it.logProcessing(false, now(), transactionId, message) } - return@schedule + } catch (e: Exception) { + logger.error("[$accountName] Failed to record failure for payment $paymentId", e) } + } + } - val newRequestTimeout = remainingTime.coerceIn(100, requestAverageProcessingTime.toMillis() * 2) - val newRequest = HttpRequest.newBuilder() - .uri(request.uri()) - .timeout(Duration.ofMillis(newRequestTimeout)) - .POST(HttpRequest.BodyPublishers.noBody()) - .build() + override fun price() = properties.price + + override fun isEnabled() = properties.enabled + + override fun name() = properties.accountName - completeAction(retryCount + 1, newRequest, paymentId, transactionId, timeBeforeCall, deadline) - }, delay, TimeUnit.MILLISECONDS) + fun shutdown() { + logger.info("[$accountName] Shutting down payment adapter") + scope.cancel() + retryScheduler.shutdown() + (httpClient.executor().orElse(null) as? java.util.concurrent.ExecutorService)?.shutdown() } } From 3a925b2683dbed49606be4ce62c9c0ee7ca571ef Mon Sep 17 00:00:00 2001 From: 21092004Goda Date: Mon, 8 Dec 2025 01:36:52 +0300 Subject: [PATCH 38/61] Merge remote-tracking branch 'origin/hw-9-io-kuro' into hw-9-io-kuro # Conflicts: # src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt --- .../logic/PaymentExternalServiceImpl.kt | 381 ++++++++---------- 1 file changed, 168 insertions(+), 213 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index e92d13475..086cf25dc 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -3,10 +3,13 @@ package ru.quipy.payments.logic import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.registerKotlinModule import io.micrometer.core.instrument.MeterRegistry -import kotlinx.coroutines.* +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.CoroutineExceptionHandler +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.asCoroutineDispatcher +import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Semaphore -import kotlinx.coroutines.sync.withPermit import org.slf4j.LoggerFactory import ru.quipy.common.utils.NamedThreadFactory import ru.quipy.common.utils.SlidingWindowRateLimiter @@ -22,12 +25,12 @@ import java.net.http.HttpResponse import java.time.Duration import java.util.* import java.util.concurrent.Executors -import java.util.concurrent.ScheduledFuture import java.util.concurrent.TimeUnit import kotlin.math.pow import kotlin.random.Random -import kotlin.time.Duration.Companion.milliseconds + +// Advice: always treat time as a Duration class PaymentExternalSystemAdapterImpl( private val properties: PaymentAccountProperties, private val paymentESService: EventSourcingService, @@ -51,17 +54,12 @@ class PaymentExternalSystemAdapterImpl( private val scope = CoroutineScope(SupervisorJob() + ioDispatcher + exceptionHandler) + + // 2025-11-20T20:30:35.780+03:00 INFO 56644 --- [alhost:1234/...] ru.quipy.core.EventSourcingService : Optimistic lock exception. Failed to save event records id: [7dca693e-e811-4b7f-8bce-23e13d952c04-4] private val startedRequests = meterRegistry.counter("payment.processing.started", "accountName", properties.accountName) private val timer = meterRegistry.timer("payment.external.system.request.latency", "accountName", properties.accountName) - private val successCounter = - meterRegistry.counter("payment.processing.success", "accountName", properties.accountName) - private val failureCounter = - meterRegistry.counter("payment.processing.failure", "accountName", properties.accountName) - private val timeoutCounter = - meterRegistry.counter("payment.processing.timeout", "accountName", properties.accountName) - private val serviceName = properties.serviceName private val accountName = properties.accountName private val requestAverageProcessingTime = properties.averageProcessingTime @@ -79,30 +77,27 @@ class PaymentExternalSystemAdapterImpl( .newBuilder() .executor(Executors.newFixedThreadPool(parallelRequests)) .version(HttpClient.Version.HTTP_2) - .connectTimeout(Duration.ofSeconds(10)) .build() private val retryScheduler = Executors.newScheduledThreadPool( Runtime.getRuntime().availableProcessors() ) - private val pendingRequests = Collections.synchronizedMap(mutableMapOf>()) - override fun performPaymentAsync(paymentId: UUID, amount: Int, paymentStartedAt: Long, deadline: Long) { - logger.info("[$accountName] Submitting payment request for payment $paymentId") + logger.warn("[$accountName] Submitting payment request for payment $paymentId") val transactionId = UUID.randomUUID() + // Вне зависимости от исхода оплаты важно отметить что она была отправлена. + // Это требуется сделать ВО ВСЕХ СЛУЧАЯХ, поскольку эта информация используется сервисом тестирования. scope.launch { try { - withTimeout(deadline - System.currentTimeMillis()) { - paymentESService.update(paymentId) { - it.logSubmission( - success = true, - transactionId, - now(), - Duration.ofMillis(now() - paymentStartedAt) - ) - } + paymentESService.update(paymentId) { + it.logSubmission( + success = true, + transactionId, + now(), + Duration.ofMillis(now() - paymentStartedAt) + ) } logger.info("[$accountName] Log submission recorded for $paymentId") } catch (e: Exception) { @@ -110,229 +105,189 @@ class PaymentExternalSystemAdapterImpl( } } - scope.launch { - try { - executePaymentWithRetries(paymentId, transactionId, amount, deadline) - } catch (e: Exception) { - logger.error("[$accountName] Failed to process payment $paymentId", e) - handlePaymentFailure(paymentId, transactionId, e) - } - } - } - - private suspend fun executePaymentWithRetries( - paymentId: UUID, - transactionId: UUID, - amount: Int, - deadline: Long - ) { - var retryCount = 0 - val maxRetries = 3 - - while (retryCount <= maxRetries) { - try { - checkDeadline(deadline, paymentId) - - val result = executePaymentAttempt(paymentId, transactionId, amount, deadline) - - if (result) { - successCounter.increment() - return - } else { - retryCount++ - if (retryCount <= maxRetries) { - val backoff = calculateBackoff(retryCount, deadline) - delay(backoff) - } - } - } catch (e: TooManyRequestsException) { - throw e - } catch (e: CancellationException) { - logger.warn("[$accountName] Payment $paymentId was cancelled") - throw e - } catch (e: Exception) { - logger.warn("[$accountName] Attempt ${retryCount + 1} failed for payment $paymentId", e) - retryCount++ - - if (retryCount > maxRetries) { - throw e - } + logger.info("[$accountName] Submit: $paymentId , txId: $transactionId") - val backoff = calculateBackoff(retryCount, deadline) - delay(backoff) + val remaining = deadline - now() + val minRequiredTime = requestAverageProcessingTime.toMillis() * 2 + if (remaining < minRequiredTime) { + logger.warn("[$accountName] Not enough time for payment $paymentId: ${remaining}ms remaining, need ${minRequiredTime}ms") + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "not enough time") } + val retryAfterMs = minRequiredTime - remaining + Random.nextLong(100) + throw TooManyRequestsException(retryAfterMs) } - throw RuntimeException("All retry attempts exhausted for payment $paymentId") - } + val timeBeforeCall = now() - private suspend fun executePaymentAttempt( - paymentId: UUID, - transactionId: UUID, - amount: Int, - deadline: Long - ): Boolean = withTimeoutOrNull(deadline - System.currentTimeMillis()) { - try { - val semaphoreAcquired = withTimeoutOrNull(deadline - System.currentTimeMillis()) { - parallelLimiter.tryAcquire() - if (!parallelLimiter.tryAcquire()) { - val waitTime = minOf(deadline - System.currentTimeMillis(), 100L) - if (waitTime > 0) { - delay(waitTime) - } - parallelLimiter.tryAcquire() - } else { - true - } - } ?: false - - if (!semaphoreAcquired) { - val retryAfter = (requestAverageProcessingTime.toMillis() / parallelRequests).coerceIn(10, 100) - throw TooManyRequestsException(retryAfter + Random.nextLong(10)) - } - - val rateLimitAcquired = withTimeoutOrNull(deadline - System.currentTimeMillis()) { - rateLimit.tickBlockingWithTimeout(deadline - System.currentTimeMillis()) - } ?: false - - if (!rateLimitAcquired) { - parallelLimiter.release() - val retryAfter = (1000L / rateLimitPerSec).coerceIn(10, 100) - throw TooManyRequestsException(retryAfter + Random.nextLong(10)) - } - - val requestTimeout = calculateRequestTimeout(deadline) + if (!tryAcquire(now(), deadline - now())) { + val retryAfterMs = (requestAverageProcessingTime.toMillis() / parallelRequests * 10).coerceIn( + 10, + 100 + ) + Random.nextLong(10) - val response = executeHttpRequest(paymentId, transactionId, amount, requestTimeout) + throw TooManyRequestsException(retryAfterMs) + } - val processingResult = processHttpResponse(response, paymentId, transactionId) + if (!rateLimit.tickBlockingWithTimeout(deadline - now())) { + parallelLimiter.release() - if (processingResult) { - paymentESService.update(paymentId) { - it.logProcessing(true, now(), transactionId, "Success") - } - } + val retryAfterMs = (1000L / rateLimitPerSec * 10).coerceIn(10, 100) + Random.nextLong(10) + throw TooManyRequestsException(retryAfterMs) + } - return@withTimeoutOrNull processingResult + val requestTimeout = minOf( + deadline - now(), + requestAverageProcessingTime.toMillis() * 2 + ).coerceAtLeast(100) - } finally { - if (parallelLimiter.availablePermits < parallelRequests) { - parallelLimiter.release() - } - startedRequests.increment() + if (requestTimeout < requestAverageProcessingTime.toMillis()) { + logger.warn("[$accountName] Timeout too short for payment $paymentId: ${requestTimeout}ms") + parallelLimiter.release() + val retryAfterMs = requestAverageProcessingTime.toMillis() - requestTimeout + Random.nextLong(100) + throw TooManyRequestsException(retryAfterMs) } - } ?: run { - timeoutCounter.increment() - logger.warn("[$accountName] Payment $paymentId timed out") - false - } - private suspend fun executeHttpRequest( - paymentId: UUID, - transactionId: UUID, - amount: Int, - requestTimeout: Long - ): HttpResponse = withContext(Dispatchers.IO) { val request = HttpRequest.newBuilder() .uri(URI("http://$paymentProviderHostPort/external/process?serviceName=$serviceName&token=$token&accountName=$accountName&transactionId=$transactionId&paymentId=$paymentId&amount=$amount")) .timeout(Duration.ofMillis(requestTimeout)) .POST(HttpRequest.BodyPublishers.noBody()) .build() - try { - val timeBeforeCall = now() - val response = httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString()).get() - timer.record(now() - timeBeforeCall, TimeUnit.MILLISECONDS) - response - } catch (e: Exception) { - logger.error("[$accountName] HTTP request failed for $paymentId", e) - throw e - } - } - - private fun processHttpResponse( - response: HttpResponse, - paymentId: UUID, - transactionId: UUID - ): Boolean { - return try { - val rawBody = response.body() - val parsed = mapper.readValue(rawBody, ExternalSysResponse::class.java) - - if (!parsed.result) { - logger.warn("[$accountName] External system returned failure for $paymentId: ${parsed.message}") - failureCounter.increment() - } + val retryCount = 0L - parsed.result - } catch (ex: Exception) { - logger.error("[$accountName] Failed to parse response for $paymentId", ex) - failureCounter.increment() - false - } + completeAction(retryCount, request, paymentId, transactionId, timeBeforeCall, deadline) } - private fun checkDeadline(deadline: Long, paymentId: UUID) { - val remaining = deadline - System.currentTimeMillis() - if (remaining <= 0) { - throw CancellationException("Deadline expired for payment $paymentId") - } - - val minRequiredTime = requestAverageProcessingTime.toMillis() * 2 - if (remaining < minRequiredTime) { - logger.warn("[$accountName] Not enough time for payment $paymentId: ${remaining}ms remaining, need ${minRequiredTime}ms") - throw TooManyRequestsException(minRequiredTime - remaining + Random.nextLong(100)) - } - } + override fun price() = properties.price - private fun calculateRequestTimeout(deadline: Long): Long { - val remaining = deadline - System.currentTimeMillis() - return minOf( - remaining * 2 / 3, - requestAverageProcessingTime.toMillis() * 3 - ).coerceIn(100, 30000L) - } + override fun isEnabled() = properties.enabled - private fun calculateBackoff(retryCount: Int, deadline: Long): Long { - val baseDelay = (2.0.pow(retryCount.toDouble()) * 25).toLong() - val jitter = Random.nextLong(10) - val totalDelay = baseDelay + jitter + override fun name() = properties.accountName - val remaining = deadline - System.currentTimeMillis() - val safeDelay = minOf(totalDelay, remaining / 2) + fun tryAcquire(startedAt: Long, remaining: Long): Boolean { + var isAcquired = parallelLimiter.tryAcquire() + while (!isAcquired && now() - startedAt < remaining) { + isAcquired = parallelLimiter.tryAcquire() + Thread.sleep(1) + } - return safeDelay.coerceAtLeast(0) + return isAcquired } - private fun handlePaymentFailure(paymentId: UUID, transactionId: UUID, cause: Exception) { - scope.launch { - try { - paymentESService.update(paymentId) { - val message = when (cause) { - is TooManyRequestsException -> "Rate limited" - is TimeoutCancellationException -> "Timeout" - is CancellationException -> "Cancelled" - else -> cause.message ?: "Unknown error" + fun completeAction( + retryCount: Long, + request: HttpRequest, + paymentId: UUID, + transactionId: UUID, + timeBeforeCall: Long, + deadline: Long + ) { + httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString()) + .whenComplete { response, throwable -> + scope.launch { + try { + if (throwable != null) { + val e = throwable.cause + when (throwable.cause) { + is SocketTimeoutException -> { + logger.warn("[$accountName] attempt ${retryCount + 1} timeout: $paymentId", e) + } + + is InterruptedIOException -> { + logger.warn("[$accountName] interrupted: $paymentId", e) + } + + else -> { + logger.warn("[$accountName] io error: $paymentId", e) + } + } + + if (retryCount + 1 >= 3) { + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "Max attempts reached") + } + } else { + val backoff = ((2.0.pow(retryCount.toDouble()) * 25).toLong() + Random.nextLong(10)) + val capped = backoff.coerceAtMost(deadline - now() - 5) + if (capped <= 0) { + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "Deadline expired") + } + } else { + scheduleRetry( + retryCount, + request, + paymentId, + transactionId, + timeBeforeCall, + deadline, + capped + ) + return@launch + } + } + } else { + logger.warn("Free space in semaphore: {}", parallelLimiter.availablePermits) + logger.info( + "success in callback for payment: {}, retry count: {}, in time: {}", + paymentId, + retryCount, + now() + ) + val rawBody = response.body() + val parsed = try { + mapper.readValue(rawBody, ExternalSysResponse::class.java) + } catch (ex: Exception) { + ExternalSysResponse(transactionId.toString(), paymentId.toString(), false, ex.message) + } + + paymentESService.update(paymentId) { + it.logProcessing(parsed.result, now(), transactionId, parsed.message) + } + timer.record(now() - timeBeforeCall, TimeUnit.MILLISECONDS) + } + } catch (e: Exception) { + logger.error("[$accountName] Error processing payment $paymentId", e) + } finally { + startedRequests.increment() + parallelLimiter.release() } - it.logProcessing(false, now(), transactionId, message) } - } catch (e: Exception) { - logger.error("[$accountName] Failed to record failure for payment $paymentId", e) } - } } - override fun price() = properties.price - - override fun isEnabled() = properties.enabled + private fun scheduleRetry( + retryCount: Long, request: HttpRequest, paymentId: UUID, + transactionId: UUID, timeBeforeCall: Long, deadline: Long, delay: Long + ) { + retryScheduler.schedule({ + val remainingTime = deadline - now() + if (remainingTime < requestAverageProcessingTime.toMillis()) { + scope.launch { + try { + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "Not enough time for retry") + } + } catch (e: Exception) { + logger.error("[$accountName] Failed to record retry failure for $paymentId", e) + } finally { + startedRequests.increment() + parallelLimiter.release() + } + } + return@schedule + } - override fun name() = properties.accountName + val newRequestTimeout = remainingTime.coerceIn(100, requestAverageProcessingTime.toMillis() * 2) + val newRequest = HttpRequest.newBuilder() + .uri(request.uri()) + .timeout(Duration.ofMillis(newRequestTimeout)) + .POST(HttpRequest.BodyPublishers.noBody()) + .build() - fun shutdown() { - logger.info("[$accountName] Shutting down payment adapter") - scope.cancel() - retryScheduler.shutdown() - (httpClient.executor().orElse(null) as? java.util.concurrent.ExecutorService)?.shutdown() + completeAction(retryCount + 1, newRequest, paymentId, transactionId, timeBeforeCall, deadline) + }, delay, TimeUnit.MILLISECONDS) } } From c5041fdb9267e94760065eedcd76a7fc70610a5c Mon Sep 17 00:00:00 2001 From: 21092004Goda Date: Mon, 8 Dec 2025 01:46:50 +0300 Subject: [PATCH 39/61] Merge remote-tracking branch 'origin/hw-9-io-kuro' into hw-9-io-kuro # Conflicts: # src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt --- .../logic/PaymentExternalServiceImpl.kt | 256 ++++++++---------- 1 file changed, 118 insertions(+), 138 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index 086cf25dc..79694a7cc 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -3,12 +3,7 @@ package ru.quipy.payments.logic import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.registerKotlinModule import io.micrometer.core.instrument.MeterRegistry -import kotlinx.coroutines.CoroutineDispatcher -import kotlinx.coroutines.CoroutineExceptionHandler -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.SupervisorJob -import kotlinx.coroutines.asCoroutineDispatcher -import kotlinx.coroutines.launch +import kotlinx.coroutines.* import kotlinx.coroutines.sync.Semaphore import org.slf4j.LoggerFactory import ru.quipy.common.utils.NamedThreadFactory @@ -26,11 +21,10 @@ import java.time.Duration import java.util.* import java.util.concurrent.Executors import java.util.concurrent.TimeUnit -import kotlin.math.pow +import kotlin.math.min import kotlin.random.Random -// Advice: always treat time as a Duration class PaymentExternalSystemAdapterImpl( private val properties: PaymentAccountProperties, private val paymentESService: EventSourcingService, @@ -39,43 +33,44 @@ class PaymentExternalSystemAdapterImpl( meterRegistry: MeterRegistry, private val parallelLimiter: Semaphore, private val ioDispatcher: CoroutineDispatcher = Executors.newFixedThreadPool( - Runtime.getRuntime().availableProcessors() * 2, + Runtime.getRuntime().availableProcessors() * 4, NamedThreadFactory("payment-io-") ).asCoroutineDispatcher() ) : PaymentExternalSystemAdapter { + companion object { val logger = LoggerFactory.getLogger(PaymentExternalSystemAdapter::class.java) val mapper = ObjectMapper().registerKotlinModule() } private val exceptionHandler = CoroutineExceptionHandler { _, throwable -> - logger.error("[$accountName] Unhandled exception in payment adapter coroutine", throwable) + logger.error("[${properties.accountName}] Unhandled exception", throwable) } private val scope = CoroutineScope(SupervisorJob() + ioDispatcher + exceptionHandler) - - // 2025-11-20T20:30:35.780+03:00 INFO 56644 --- [alhost:1234/...] ru.quipy.core.EventSourcingService : Optimistic lock exception. Failed to save event records id: [7dca693e-e811-4b7f-8bce-23e13d952c04-4] private val startedRequests = meterRegistry.counter("payment.processing.started", "accountName", properties.accountName) private val timer = meterRegistry.timer("payment.external.system.request.latency", "accountName", properties.accountName) + private val serviceName = properties.serviceName private val accountName = properties.accountName - private val requestAverageProcessingTime = properties.averageProcessingTime + private val requestAvg = properties.averageProcessingTime private val rateLimitPerSec = properties.rateLimitPerSec private val parallelRequests = properties.parallelRequests + private val rateLimit: SlidingWindowRateLimiter by lazy { SlidingWindowRateLimiter( - rate = (rateLimitPerSec * 0.9).toLong(), - window = Duration.ofMillis(1000), + rate = rateLimitPerSec.toLong(), + window = Duration.ofSeconds(1) ) } private val httpClient = HttpClient .newBuilder() - .executor(Executors.newFixedThreadPool(parallelRequests)) + .executor(Executors.newFixedThreadPool(parallelRequests * 2)) .version(HttpClient.Version.HTTP_2) .build() @@ -84,98 +79,77 @@ class PaymentExternalSystemAdapterImpl( ) override fun performPaymentAsync(paymentId: UUID, amount: Int, paymentStartedAt: Long, deadline: Long) { - logger.warn("[$accountName] Submitting payment request for payment $paymentId") val transactionId = UUID.randomUUID() - // Вне зависимости от исхода оплаты важно отметить что она была отправлена. - // Это требуется сделать ВО ВСЕХ СЛУЧАЯХ, поскольку эта информация используется сервисом тестирования. scope.launch { try { paymentESService.update(paymentId) { - it.logSubmission( - success = true, - transactionId, - now(), - Duration.ofMillis(now() - paymentStartedAt) - ) + it.logSubmission(true, transactionId, now(), Duration.ofMillis(now() - paymentStartedAt)) } - logger.info("[$accountName] Log submission recorded for $paymentId") - } catch (e: Exception) { - logger.error("[$accountName] Failed to record log submission for $paymentId", e) - } + } catch (_: Exception) {} } - logger.info("[$accountName] Submit: $paymentId , txId: $transactionId") + val startRemaining = deadline - now() + val minRequired = requestAvg.toMillis() * 2 - val remaining = deadline - now() - val minRequiredTime = requestAverageProcessingTime.toMillis() * 2 - if (remaining < minRequiredTime) { - logger.warn("[$accountName] Not enough time for payment $paymentId: ${remaining}ms remaining, need ${minRequiredTime}ms") - paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "not enough time") - } - val retryAfterMs = minRequiredTime - remaining + Random.nextLong(100) - throw TooManyRequestsException(retryAfterMs) + if (startRemaining < minRequired) { + val retryAfter = minRequired - startRemaining + Random.nextLong(20) + throw TooManyRequestsException(retryAfter) } - val timeBeforeCall = now() - - if (!tryAcquire(now(), deadline - now())) { - val retryAfterMs = (requestAverageProcessingTime.toMillis() / parallelRequests * 10).coerceIn( - 10, - 100 - ) + Random.nextLong(10) - - throw TooManyRequestsException(retryAfterMs) + if (!tryAcquire(deadline)) { + val retryAfter = (requestAvg.toMillis() / parallelRequests * 5) + .coerceIn(20, 120) + Random.nextLong(20) + throw TooManyRequestsException(retryAfter) } if (!rateLimit.tickBlockingWithTimeout(deadline - now())) { parallelLimiter.release() - - val retryAfterMs = (1000L / rateLimitPerSec * 10).coerceIn(10, 100) + Random.nextLong(10) - throw TooManyRequestsException(retryAfterMs) + val retryAfter = (1000L / rateLimitPerSec * 5).coerceIn(20, 100) + Random.nextLong(20) + throw TooManyRequestsException(retryAfter) } - val requestTimeout = minOf( + val reqTimeout = min( deadline - now(), - requestAverageProcessingTime.toMillis() * 2 - ).coerceAtLeast(100) - - if (requestTimeout < requestAverageProcessingTime.toMillis()) { - logger.warn("[$accountName] Timeout too short for payment $paymentId: ${requestTimeout}ms") - parallelLimiter.release() - val retryAfterMs = requestAverageProcessingTime.toMillis() - requestTimeout + Random.nextLong(100) - throw TooManyRequestsException(retryAfterMs) - } + requestAvg.toMillis() * 2 + ).coerceAtLeast(150) val request = HttpRequest.newBuilder() - .uri(URI("http://$paymentProviderHostPort/external/process?serviceName=$serviceName&token=$token&accountName=$accountName&transactionId=$transactionId&paymentId=$paymentId&amount=$amount")) - .timeout(Duration.ofMillis(requestTimeout)) + .uri( + URI( + "http://$paymentProviderHostPort/external/process" + + "?serviceName=$serviceName&token=$token" + + "&accountName=$accountName&transactionId=$transactionId" + + "&paymentId=$paymentId&amount=$amount" + ) + ) + .timeout(Duration.ofMillis(reqTimeout)) .POST(HttpRequest.BodyPublishers.noBody()) .build() - val retryCount = 0L - - completeAction(retryCount, request, paymentId, transactionId, timeBeforeCall, deadline) + completeAction( + retryCount = 0, + request = request, + paymentId = paymentId, + transactionId = transactionId, + timeBeforeCall = now(), + deadline = deadline + ) } override fun price() = properties.price - override fun isEnabled() = properties.enabled + override fun name() = accountName - override fun name() = properties.accountName - - fun tryAcquire(startedAt: Long, remaining: Long): Boolean { - var isAcquired = parallelLimiter.tryAcquire() - while (!isAcquired && now() - startedAt < remaining) { - isAcquired = parallelLimiter.tryAcquire() + private fun tryAcquire(deadline: Long): Boolean { + while (now() < deadline) { + if (parallelLimiter.tryAcquire()) return true Thread.sleep(1) } - - return isAcquired + return false } - fun completeAction( + private fun completeAction( retryCount: Long, request: HttpRequest, paymentId: UUID, @@ -185,70 +159,61 @@ class PaymentExternalSystemAdapterImpl( ) { httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .whenComplete { response, throwable -> + scope.launch { try { if (throwable != null) { - val e = throwable.cause - when (throwable.cause) { - is SocketTimeoutException -> { - logger.warn("[$accountName] attempt ${retryCount + 1} timeout: $paymentId", e) - } + val cause = throwable.cause - is InterruptedIOException -> { - logger.warn("[$accountName] interrupted: $paymentId", e) - } + val maxRetries = 3 + val nextRetry = retryCount + 1 - else -> { - logger.warn("[$accountName] io error: $paymentId", e) + if (nextRetry >= maxRetries) { + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "max retries reached") } + return@launch } - if (retryCount + 1 >= 3) { + val remaining = deadline - now() + val need = requestAvg.toMillis() + + if (remaining < need) { paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "Max attempts reached") - } - } else { - val backoff = ((2.0.pow(retryCount.toDouble()) * 25).toLong() + Random.nextLong(10)) - val capped = backoff.coerceAtMost(deadline - now() - 5) - if (capped <= 0) { - paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "Deadline expired") - } - } else { - scheduleRetry( - retryCount, - request, - paymentId, - transactionId, - timeBeforeCall, - deadline, - capped - ) - return@launch + it.logProcessing(false, now(), transactionId, "deadline expired") } + return@launch } - } else { - logger.warn("Free space in semaphore: {}", parallelLimiter.availablePermits) - logger.info( - "success in callback for payment: {}, retry count: {}, in time: {}", - paymentId, - retryCount, - now() + + val delayMs = (25L * (1L shl retryCount.toInt())).coerceAtMost(80L) + + scheduleRetry( + retryCount = nextRetry, + request = request, + paymentId = paymentId, + transactionId = transactionId, + timeBeforeCall = timeBeforeCall, + deadline = deadline, + delay = delayMs ) - val rawBody = response.body() - val parsed = try { - mapper.readValue(rawBody, ExternalSysResponse::class.java) - } catch (ex: Exception) { - ExternalSysResponse(transactionId.toString(), paymentId.toString(), false, ex.message) - } - paymentESService.update(paymentId) { - it.logProcessing(parsed.result, now(), transactionId, parsed.message) - } - timer.record(now() - timeBeforeCall, TimeUnit.MILLISECONDS) + return@launch } - } catch (e: Exception) { - logger.error("[$accountName] Error processing payment $paymentId", e) + + // успех + val body = response.body() + val parsed = try { + mapper.readValue(body, ExternalSysResponse::class.java) + } catch (ex: Exception) { + ExternalSysResponse(transactionId.toString(), paymentId.toString(), false, ex.message) + } + + paymentESService.update(paymentId) { + it.logProcessing(parsed.result, now(), transactionId, parsed.message) + } + + timer.record(now() - timeBeforeCall, TimeUnit.MILLISECONDS) + } finally { startedRequests.increment() parallelLimiter.release() @@ -258,19 +223,25 @@ class PaymentExternalSystemAdapterImpl( } private fun scheduleRetry( - retryCount: Long, request: HttpRequest, paymentId: UUID, - transactionId: UUID, timeBeforeCall: Long, deadline: Long, delay: Long + retryCount: Long, + request: HttpRequest, + paymentId: UUID, + transactionId: UUID, + timeBeforeCall: Long, + deadline: Long, + delay: Long ) { retryScheduler.schedule({ - val remainingTime = deadline - now() - if (remainingTime < requestAverageProcessingTime.toMillis()) { + + val remaining = deadline - now() + val need = requestAvg.toMillis() + + if (remaining < need) { scope.launch { try { paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "Not enough time for retry") + it.logProcessing(false, now(), transactionId, "deadline expired retry") } - } catch (e: Exception) { - logger.error("[$accountName] Failed to record retry failure for $paymentId", e) } finally { startedRequests.increment() parallelLimiter.release() @@ -279,16 +250,25 @@ class PaymentExternalSystemAdapterImpl( return@schedule } - val newRequestTimeout = remainingTime.coerceIn(100, requestAverageProcessingTime.toMillis() * 2) - val newRequest = HttpRequest.newBuilder() + val timeout = min(remaining, requestAvg.toMillis() * 2).coerceAtLeast(100) + + val newReq = HttpRequest.newBuilder() .uri(request.uri()) - .timeout(Duration.ofMillis(newRequestTimeout)) + .timeout(Duration.ofMillis(timeout)) .POST(HttpRequest.BodyPublishers.noBody()) .build() - completeAction(retryCount + 1, newRequest, paymentId, transactionId, timeBeforeCall, deadline) + completeAction( + retryCount = retryCount, + request = newReq, + paymentId = paymentId, + transactionId = transactionId, + timeBeforeCall = timeBeforeCall, + deadline = deadline + ) + }, delay, TimeUnit.MILLISECONDS) } } -fun now() = System.currentTimeMillis() \ No newline at end of file +fun now() = System.currentTimeMillis() From e139c497a8d3f436d0e37979d848603fd900f475 Mon Sep 17 00:00:00 2001 From: 21092004Goda Date: Mon, 8 Dec 2025 01:50:09 +0300 Subject: [PATCH 40/61] Merge remote-tracking branch 'origin/hw-9-io-kuro' into hw-9-io-kuro # Conflicts: # src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt --- .../logic/PaymentExternalServiceImpl.kt | 256 ++++++++++-------- 1 file changed, 138 insertions(+), 118 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index 79694a7cc..598992345 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -3,7 +3,12 @@ package ru.quipy.payments.logic import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.registerKotlinModule import io.micrometer.core.instrument.MeterRegistry -import kotlinx.coroutines.* +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.CoroutineExceptionHandler +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.asCoroutineDispatcher +import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Semaphore import org.slf4j.LoggerFactory import ru.quipy.common.utils.NamedThreadFactory @@ -21,10 +26,11 @@ import java.time.Duration import java.util.* import java.util.concurrent.Executors import java.util.concurrent.TimeUnit -import kotlin.math.min +import kotlin.math.pow import kotlin.random.Random +// Advice: always treat time as a Duration class PaymentExternalSystemAdapterImpl( private val properties: PaymentAccountProperties, private val paymentESService: EventSourcingService, @@ -33,44 +39,43 @@ class PaymentExternalSystemAdapterImpl( meterRegistry: MeterRegistry, private val parallelLimiter: Semaphore, private val ioDispatcher: CoroutineDispatcher = Executors.newFixedThreadPool( - Runtime.getRuntime().availableProcessors() * 4, + Runtime.getRuntime().availableProcessors() * 2, NamedThreadFactory("payment-io-") ).asCoroutineDispatcher() ) : PaymentExternalSystemAdapter { - companion object { val logger = LoggerFactory.getLogger(PaymentExternalSystemAdapter::class.java) val mapper = ObjectMapper().registerKotlinModule() } private val exceptionHandler = CoroutineExceptionHandler { _, throwable -> - logger.error("[${properties.accountName}] Unhandled exception", throwable) + logger.error("[$accountName] Unhandled exception in payment adapter coroutine", throwable) } private val scope = CoroutineScope(SupervisorJob() + ioDispatcher + exceptionHandler) + + // 2025-11-20T20:30:35.780+03:00 INFO 56644 --- [alhost:1234/...] ru.quipy.core.EventSourcingService : Optimistic lock exception. Failed to save event records id: [7dca693e-e811-4b7f-8bce-23e13d952c04-4] private val startedRequests = meterRegistry.counter("payment.processing.started", "accountName", properties.accountName) private val timer = meterRegistry.timer("payment.external.system.request.latency", "accountName", properties.accountName) - private val serviceName = properties.serviceName private val accountName = properties.accountName - private val requestAvg = properties.averageProcessingTime + private val requestAverageProcessingTime = properties.averageProcessingTime private val rateLimitPerSec = properties.rateLimitPerSec private val parallelRequests = properties.parallelRequests - private val rateLimit: SlidingWindowRateLimiter by lazy { SlidingWindowRateLimiter( - rate = rateLimitPerSec.toLong(), - window = Duration.ofSeconds(1) + rate = (rateLimitPerSec * 0.95).toLong(), + window = Duration.ofMillis(1000), ) } private val httpClient = HttpClient .newBuilder() - .executor(Executors.newFixedThreadPool(parallelRequests * 2)) + .executor(Executors.newFixedThreadPool(parallelRequests)) .version(HttpClient.Version.HTTP_2) .build() @@ -79,77 +84,98 @@ class PaymentExternalSystemAdapterImpl( ) override fun performPaymentAsync(paymentId: UUID, amount: Int, paymentStartedAt: Long, deadline: Long) { + logger.warn("[$accountName] Submitting payment request for payment $paymentId") val transactionId = UUID.randomUUID() + // Вне зависимости от исхода оплаты важно отметить что она была отправлена. + // Это требуется сделать ВО ВСЕХ СЛУЧАЯХ, поскольку эта информация используется сервисом тестирования. scope.launch { try { paymentESService.update(paymentId) { - it.logSubmission(true, transactionId, now(), Duration.ofMillis(now() - paymentStartedAt)) + it.logSubmission( + success = true, + transactionId, + now(), + Duration.ofMillis(now() - paymentStartedAt) + ) } - } catch (_: Exception) {} + logger.info("[$accountName] Log submission recorded for $paymentId") + } catch (e: Exception) { + logger.error("[$accountName] Failed to record log submission for $paymentId", e) + } } - val startRemaining = deadline - now() - val minRequired = requestAvg.toMillis() * 2 + logger.info("[$accountName] Submit: $paymentId , txId: $transactionId") - if (startRemaining < minRequired) { - val retryAfter = minRequired - startRemaining + Random.nextLong(20) - throw TooManyRequestsException(retryAfter) + val remaining = deadline - now() + val minRequiredTime = requestAverageProcessingTime.toMillis() * 2 + if (remaining < minRequiredTime) { + logger.warn("[$accountName] Not enough time for payment $paymentId: ${remaining}ms remaining, need ${minRequiredTime}ms") + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "not enough time") + } + val retryAfterMs = minRequiredTime - remaining + Random.nextLong(100) + throw TooManyRequestsException(retryAfterMs) } - if (!tryAcquire(deadline)) { - val retryAfter = (requestAvg.toMillis() / parallelRequests * 5) - .coerceIn(20, 120) + Random.nextLong(20) - throw TooManyRequestsException(retryAfter) + val timeBeforeCall = now() + + if (!tryAcquire(now(), deadline - now())) { + val retryAfterMs = (requestAverageProcessingTime.toMillis() / parallelRequests * 10).coerceIn( + 10, + 100 + ) + Random.nextLong(10) + + throw TooManyRequestsException(retryAfterMs) } if (!rateLimit.tickBlockingWithTimeout(deadline - now())) { parallelLimiter.release() - val retryAfter = (1000L / rateLimitPerSec * 5).coerceIn(20, 100) + Random.nextLong(20) - throw TooManyRequestsException(retryAfter) + + val retryAfterMs = (1000L / rateLimitPerSec * 10).coerceIn(10, 100) + Random.nextLong(10) + throw TooManyRequestsException(retryAfterMs) } - val reqTimeout = min( + val requestTimeout = minOf( deadline - now(), - requestAvg.toMillis() * 2 - ).coerceAtLeast(150) + requestAverageProcessingTime.toMillis() * 2 + ).coerceAtLeast(100) + + if (requestTimeout < requestAverageProcessingTime.toMillis()) { + logger.warn("[$accountName] Timeout too short for payment $paymentId: ${requestTimeout}ms") + parallelLimiter.release() + val retryAfterMs = requestAverageProcessingTime.toMillis() - requestTimeout + Random.nextLong(100) + throw TooManyRequestsException(retryAfterMs) + } val request = HttpRequest.newBuilder() - .uri( - URI( - "http://$paymentProviderHostPort/external/process" + - "?serviceName=$serviceName&token=$token" + - "&accountName=$accountName&transactionId=$transactionId" + - "&paymentId=$paymentId&amount=$amount" - ) - ) - .timeout(Duration.ofMillis(reqTimeout)) + .uri(URI("http://$paymentProviderHostPort/external/process?serviceName=$serviceName&token=$token&accountName=$accountName&transactionId=$transactionId&paymentId=$paymentId&amount=$amount")) + .timeout(Duration.ofMillis(requestTimeout)) .POST(HttpRequest.BodyPublishers.noBody()) .build() - completeAction( - retryCount = 0, - request = request, - paymentId = paymentId, - transactionId = transactionId, - timeBeforeCall = now(), - deadline = deadline - ) + val retryCount = 0L + + completeAction(retryCount, request, paymentId, transactionId, timeBeforeCall, deadline) } override fun price() = properties.price + override fun isEnabled() = properties.enabled - override fun name() = accountName - private fun tryAcquire(deadline: Long): Boolean { - while (now() < deadline) { - if (parallelLimiter.tryAcquire()) return true + override fun name() = properties.accountName + + fun tryAcquire(startedAt: Long, remaining: Long): Boolean { + var isAcquired = parallelLimiter.tryAcquire() + while (!isAcquired && now() - startedAt < remaining) { + isAcquired = parallelLimiter.tryAcquire() Thread.sleep(1) } - return false + + return isAcquired } - private fun completeAction( + fun completeAction( retryCount: Long, request: HttpRequest, paymentId: UUID, @@ -159,61 +185,70 @@ class PaymentExternalSystemAdapterImpl( ) { httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .whenComplete { response, throwable -> - scope.launch { try { if (throwable != null) { - val cause = throwable.cause + val e = throwable.cause + when (throwable.cause) { + is SocketTimeoutException -> { + logger.warn("[$accountName] attempt ${retryCount + 1} timeout: $paymentId", e) + } - val maxRetries = 3 - val nextRetry = retryCount + 1 + is InterruptedIOException -> { + logger.warn("[$accountName] interrupted: $paymentId", e) + } - if (nextRetry >= maxRetries) { - paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "max retries reached") + else -> { + logger.warn("[$accountName] io error: $paymentId", e) } - return@launch } - val remaining = deadline - now() - val need = requestAvg.toMillis() - - if (remaining < need) { + if (retryCount + 1 >= 3) { paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "deadline expired") + it.logProcessing(false, now(), transactionId, "Max attempts reached") + } + } else { + val backoff = ((2.0.pow(retryCount.toDouble()) * 25).toLong() + Random.nextLong(10)) + val capped = backoff.coerceAtMost(deadline - now() - 5) + if (capped <= 0) { + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "Deadline expired") + } + } else { + scheduleRetry( + retryCount, + request, + paymentId, + transactionId, + timeBeforeCall, + deadline, + capped + ) + return@launch } - return@launch } - - val delayMs = (25L * (1L shl retryCount.toInt())).coerceAtMost(80L) - - scheduleRetry( - retryCount = nextRetry, - request = request, - paymentId = paymentId, - transactionId = transactionId, - timeBeforeCall = timeBeforeCall, - deadline = deadline, - delay = delayMs + } else { + logger.warn("Free space in semaphore: {}", parallelLimiter.availablePermits) + logger.info( + "success in callback for payment: {}, retry count: {}, in time: {}", + paymentId, + retryCount, + now() ) + val rawBody = response.body() + val parsed = try { + mapper.readValue(rawBody, ExternalSysResponse::class.java) + } catch (ex: Exception) { + ExternalSysResponse(transactionId.toString(), paymentId.toString(), false, ex.message) + } - return@launch - } - - // успех - val body = response.body() - val parsed = try { - mapper.readValue(body, ExternalSysResponse::class.java) - } catch (ex: Exception) { - ExternalSysResponse(transactionId.toString(), paymentId.toString(), false, ex.message) - } - - paymentESService.update(paymentId) { - it.logProcessing(parsed.result, now(), transactionId, parsed.message) + paymentESService.update(paymentId) { + it.logProcessing(parsed.result, now(), transactionId, parsed.message) + } + timer.record(now() - timeBeforeCall, TimeUnit.MILLISECONDS) } - - timer.record(now() - timeBeforeCall, TimeUnit.MILLISECONDS) - + } catch (e: Exception) { + logger.error("[$accountName] Error processing payment $paymentId", e) } finally { startedRequests.increment() parallelLimiter.release() @@ -223,25 +258,19 @@ class PaymentExternalSystemAdapterImpl( } private fun scheduleRetry( - retryCount: Long, - request: HttpRequest, - paymentId: UUID, - transactionId: UUID, - timeBeforeCall: Long, - deadline: Long, - delay: Long + retryCount: Long, request: HttpRequest, paymentId: UUID, + transactionId: UUID, timeBeforeCall: Long, deadline: Long, delay: Long ) { retryScheduler.schedule({ - - val remaining = deadline - now() - val need = requestAvg.toMillis() - - if (remaining < need) { + val remainingTime = deadline - now() + if (remainingTime < requestAverageProcessingTime.toMillis()) { scope.launch { try { paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "deadline expired retry") + it.logProcessing(false, now(), transactionId, "Not enough time for retry") } + } catch (e: Exception) { + logger.error("[$accountName] Failed to record retry failure for $paymentId", e) } finally { startedRequests.increment() parallelLimiter.release() @@ -250,25 +279,16 @@ class PaymentExternalSystemAdapterImpl( return@schedule } - val timeout = min(remaining, requestAvg.toMillis() * 2).coerceAtLeast(100) - - val newReq = HttpRequest.newBuilder() + val newRequestTimeout = remainingTime.coerceIn(100, requestAverageProcessingTime.toMillis() * 2) + val newRequest = HttpRequest.newBuilder() .uri(request.uri()) - .timeout(Duration.ofMillis(timeout)) + .timeout(Duration.ofMillis(newRequestTimeout)) .POST(HttpRequest.BodyPublishers.noBody()) .build() - completeAction( - retryCount = retryCount, - request = newReq, - paymentId = paymentId, - transactionId = transactionId, - timeBeforeCall = timeBeforeCall, - deadline = deadline - ) - + completeAction(retryCount + 1, newRequest, paymentId, transactionId, timeBeforeCall, deadline) }, delay, TimeUnit.MILLISECONDS) } } -fun now() = System.currentTimeMillis() +fun now() = System.currentTimeMillis() \ No newline at end of file From 38158547e63f159aee55ebf6afd53790daf4788a Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Mon, 8 Dec 2025 14:50:05 +0300 Subject: [PATCH 41/61] fix: added limiters for retry --- .../logic/PaymentExternalServiceImpl.kt | 124 ++++++++---------- 1 file changed, 55 insertions(+), 69 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index 598992345..46683b7f7 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -3,12 +3,7 @@ package ru.quipy.payments.logic import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.registerKotlinModule import io.micrometer.core.instrument.MeterRegistry -import kotlinx.coroutines.CoroutineDispatcher -import kotlinx.coroutines.CoroutineExceptionHandler -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.SupervisorJob -import kotlinx.coroutines.asCoroutineDispatcher -import kotlinx.coroutines.launch +import kotlinx.coroutines.* import kotlinx.coroutines.sync.Semaphore import org.slf4j.LoggerFactory import ru.quipy.common.utils.NamedThreadFactory @@ -37,8 +32,7 @@ class PaymentExternalSystemAdapterImpl( private val paymentProviderHostPort: String, private val token: String, meterRegistry: MeterRegistry, - private val parallelLimiter: Semaphore, - private val ioDispatcher: CoroutineDispatcher = Executors.newFixedThreadPool( + private val parallelLimiter: Semaphore, ioDispatcher: CoroutineDispatcher = Executors.newFixedThreadPool( Runtime.getRuntime().availableProcessors() * 2, NamedThreadFactory("payment-io-") ).asCoroutineDispatcher() @@ -118,24 +112,6 @@ class PaymentExternalSystemAdapterImpl( throw TooManyRequestsException(retryAfterMs) } - val timeBeforeCall = now() - - if (!tryAcquire(now(), deadline - now())) { - val retryAfterMs = (requestAverageProcessingTime.toMillis() / parallelRequests * 10).coerceIn( - 10, - 100 - ) + Random.nextLong(10) - - throw TooManyRequestsException(retryAfterMs) - } - - if (!rateLimit.tickBlockingWithTimeout(deadline - now())) { - parallelLimiter.release() - - val retryAfterMs = (1000L / rateLimitPerSec * 10).coerceIn(10, 100) + Random.nextLong(10) - throw TooManyRequestsException(retryAfterMs) - } - val requestTimeout = minOf( deadline - now(), requestAverageProcessingTime.toMillis() * 2 @@ -156,7 +132,7 @@ class PaymentExternalSystemAdapterImpl( val retryCount = 0L - completeAction(retryCount, request, paymentId, transactionId, timeBeforeCall, deadline) + completeAction(retryCount, request, paymentId, transactionId, deadline) } override fun price() = properties.price @@ -180,54 +156,64 @@ class PaymentExternalSystemAdapterImpl( request: HttpRequest, paymentId: UUID, transactionId: UUID, - timeBeforeCall: Long, deadline: Long ) { + val timeBeforeCall = now() + + if (!tryAcquire(now(), deadline - now())) { + val retryAfterMs = (requestAverageProcessingTime.toMillis() / parallelRequests * 10).coerceIn( + 10, 100 + ) + Random.nextLong(10) + + throw TooManyRequestsException(retryAfterMs) + } + + if (!rateLimit.tickBlockingWithTimeout(deadline - now())) { + parallelLimiter.release() + + val retryAfterMs = (1000L / rateLimitPerSec * 10).coerceIn(10, 100) + Random.nextLong(10) + throw TooManyRequestsException(retryAfterMs) + } httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .whenComplete { response, throwable -> scope.launch { - try { - if (throwable != null) { - val e = throwable.cause - when (throwable.cause) { - is SocketTimeoutException -> { - logger.warn("[$accountName] attempt ${retryCount + 1} timeout: $paymentId", e) - } + if (throwable != null) { + val e = throwable.cause + when (throwable.cause) { + is SocketTimeoutException -> { + logger.warn("[$accountName] attempt ${retryCount + 1} timeout: $paymentId", e) + } - is InterruptedIOException -> { - logger.warn("[$accountName] interrupted: $paymentId", e) - } + is InterruptedIOException -> { + logger.warn("[$accountName] interrupted: $paymentId", e) + } - else -> { - logger.warn("[$accountName] io error: $paymentId", e) - } + else -> { + logger.warn("[$accountName] io error: $paymentId", e) } + } - if (retryCount + 1 >= 3) { + if (retryCount + 1 >= 3) { + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "Max attempts reached") + } + } else { + val backoff = ((2.0.pow(retryCount.toDouble()) * 25).toLong() + Random.nextLong(10)) + val capped = backoff.coerceAtMost(deadline - now() - 5) + if (capped <= 0) { paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "Max attempts reached") + it.logProcessing(false, now(), transactionId, "Deadline expired") } } else { - val backoff = ((2.0.pow(retryCount.toDouble()) * 25).toLong() + Random.nextLong(10)) - val capped = backoff.coerceAtMost(deadline - now() - 5) - if (capped <= 0) { - paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "Deadline expired") - } - } else { - scheduleRetry( - retryCount, - request, - paymentId, - transactionId, - timeBeforeCall, - deadline, - capped - ) - return@launch - } + parallelLimiter.release() + scheduleRetry( + retryCount, request, paymentId, transactionId, deadline, capped + ) + return@launch } - } else { + } + } else { + try { logger.warn("Free space in semaphore: {}", parallelLimiter.availablePermits) logger.info( "success in callback for payment: {}, retry count: {}, in time: {}", @@ -246,12 +232,12 @@ class PaymentExternalSystemAdapterImpl( it.logProcessing(parsed.result, now(), transactionId, parsed.message) } timer.record(now() - timeBeforeCall, TimeUnit.MILLISECONDS) + } catch (e: Exception) { + logger.error("[$accountName] Error processing payment $paymentId", e) + } finally { + startedRequests.increment() + parallelLimiter.release() } - } catch (e: Exception) { - logger.error("[$accountName] Error processing payment $paymentId", e) - } finally { - startedRequests.increment() - parallelLimiter.release() } } } @@ -259,7 +245,7 @@ class PaymentExternalSystemAdapterImpl( private fun scheduleRetry( retryCount: Long, request: HttpRequest, paymentId: UUID, - transactionId: UUID, timeBeforeCall: Long, deadline: Long, delay: Long + transactionId: UUID, deadline: Long, delay: Long ) { retryScheduler.schedule({ val remainingTime = deadline - now() @@ -286,7 +272,7 @@ class PaymentExternalSystemAdapterImpl( .POST(HttpRequest.BodyPublishers.noBody()) .build() - completeAction(retryCount + 1, newRequest, paymentId, transactionId, timeBeforeCall, deadline) + completeAction(retryCount + 1, newRequest, paymentId, transactionId, deadline) }, delay, TimeUnit.MILLISECONDS) } } From c48992f6f4ad2d47762c38fa7ca5c76b148f155e Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Mon, 8 Dec 2025 15:07:56 +0300 Subject: [PATCH 42/61] fix: fixed retry, removed from tick logic Thread.sleep in corutines --- .../logic/PaymentExternalServiceImpl.kt | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index 46683b7f7..fe3a795a5 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -5,6 +5,7 @@ import com.fasterxml.jackson.module.kotlin.registerKotlinModule import io.micrometer.core.instrument.MeterRegistry import kotlinx.coroutines.* import kotlinx.coroutines.sync.Semaphore +import okio.EOFException import org.slf4j.LoggerFactory import ru.quipy.common.utils.NamedThreadFactory import ru.quipy.common.utils.SlidingWindowRateLimiter @@ -168,7 +169,7 @@ class PaymentExternalSystemAdapterImpl( throw TooManyRequestsException(retryAfterMs) } - if (!rateLimit.tickBlockingWithTimeout(deadline - now())) { + if (!rateLimit.tick()) { parallelLimiter.release() val retryAfterMs = (1000L / rateLimitPerSec * 10).coerceIn(10, 100) + Random.nextLong(10) @@ -179,6 +180,7 @@ class PaymentExternalSystemAdapterImpl( scope.launch { if (throwable != null) { val e = throwable.cause + var isRetriable = true when (throwable.cause) { is SocketTimeoutException -> { logger.warn("[$accountName] attempt ${retryCount + 1} timeout: $paymentId", e) @@ -188,8 +190,13 @@ class PaymentExternalSystemAdapterImpl( logger.warn("[$accountName] interrupted: $paymentId", e) } + is EOFException -> { + logger.warn("[$accountName] eof exception in: $paymentId", e) + } + else -> { logger.warn("[$accountName] io error: $paymentId", e) + isRetriable = false } } @@ -205,11 +212,17 @@ class PaymentExternalSystemAdapterImpl( it.logProcessing(false, now(), transactionId, "Deadline expired") } } else { - parallelLimiter.release() - scheduleRetry( - retryCount, request, paymentId, transactionId, deadline, capped - ) - return@launch + if (isRetriable) { + parallelLimiter.release() + scheduleRetry( + retryCount, request, paymentId, transactionId, deadline, capped + ) + return@launch + } else { + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "Non-retriable exception") + } + } } } } else { From f7d396ff15d6fc087f675669f8e9b4e0e2ce979f Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Mon, 8 Dec 2025 19:19:16 +0300 Subject: [PATCH 43/61] hotfix --- .../kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index fe3a795a5..376cfe720 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -272,7 +272,6 @@ class PaymentExternalSystemAdapterImpl( logger.error("[$accountName] Failed to record retry failure for $paymentId", e) } finally { startedRequests.increment() - parallelLimiter.release() } } return@schedule From 58a49aaffa5d4faa17ed1f40b98cd04425609e71 Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Mon, 8 Dec 2025 19:23:49 +0300 Subject: [PATCH 44/61] hotfix --- .../ru/quipy/payments/logic/PaymentExternalServiceImpl.kt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index 376cfe720..286270eb2 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -131,9 +131,9 @@ class PaymentExternalSystemAdapterImpl( .POST(HttpRequest.BodyPublishers.noBody()) .build() - val retryCount = 0L - completeAction(retryCount, request, paymentId, transactionId, deadline) + + completeAction(0, request, paymentId, transactionId, deadline) } override fun price() = properties.price @@ -222,6 +222,7 @@ class PaymentExternalSystemAdapterImpl( paymentESService.update(paymentId) { it.logProcessing(false, now(), transactionId, "Non-retriable exception") } + parallelLimiter.release() } } } From f33c69c915bd6c30f59b316b4e12179585b055e2 Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Mon, 8 Dec 2025 19:30:23 +0300 Subject: [PATCH 45/61] hotfix --- .../payments/logic/PaymentExternalServiceImpl.kt | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index 286270eb2..6bca771e6 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -160,7 +160,10 @@ class PaymentExternalSystemAdapterImpl( deadline: Long ) { val timeBeforeCall = now() - + if (!rateLimit.tick()) { + val retryAfterMs = (1000L / rateLimitPerSec * 10).coerceIn(10, 100) + Random.nextLong(10) + throw TooManyRequestsException(retryAfterMs) + } if (!tryAcquire(now(), deadline - now())) { val retryAfterMs = (requestAverageProcessingTime.toMillis() / parallelRequests * 10).coerceIn( 10, 100 @@ -168,13 +171,6 @@ class PaymentExternalSystemAdapterImpl( throw TooManyRequestsException(retryAfterMs) } - - if (!rateLimit.tick()) { - parallelLimiter.release() - - val retryAfterMs = (1000L / rateLimitPerSec * 10).coerceIn(10, 100) + Random.nextLong(10) - throw TooManyRequestsException(retryAfterMs) - } httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .whenComplete { response, throwable -> scope.launch { From 3c36bb0e05536ef1a48f5fa62909ef0228978dcf Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Mon, 8 Dec 2025 19:43:01 +0300 Subject: [PATCH 46/61] hotfix --- .../logic/PaymentExternalServiceImpl.kt | 54 ++++++++----------- 1 file changed, 22 insertions(+), 32 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index 6bca771e6..e4859f60e 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -74,10 +74,6 @@ class PaymentExternalSystemAdapterImpl( .version(HttpClient.Version.HTTP_2) .build() - private val retryScheduler = Executors.newScheduledThreadPool( - Runtime.getRuntime().availableProcessors() - ) - override fun performPaymentAsync(paymentId: UUID, amount: Int, paymentStartedAt: Long, deadline: Long) { logger.warn("[$accountName] Submitting payment request for payment $paymentId") val transactionId = UUID.randomUUID() @@ -133,7 +129,7 @@ class PaymentExternalSystemAdapterImpl( - completeAction(0, request, paymentId, transactionId, deadline) + scope.launch { completeAction(0, request, paymentId, transactionId, deadline) } } override fun price() = properties.price @@ -142,17 +138,17 @@ class PaymentExternalSystemAdapterImpl( override fun name() = properties.accountName - fun tryAcquire(startedAt: Long, remaining: Long): Boolean { + suspend fun tryAcquire(startedAt: Long, remaining: Long): Boolean { var isAcquired = parallelLimiter.tryAcquire() while (!isAcquired && now() - startedAt < remaining) { isAcquired = parallelLimiter.tryAcquire() - Thread.sleep(1) + delay(2) } return isAcquired } - fun completeAction( + suspend fun completeAction( retryCount: Long, request: HttpRequest, paymentId: UUID, @@ -173,7 +169,6 @@ class PaymentExternalSystemAdapterImpl( } httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .whenComplete { response, throwable -> - scope.launch { if (throwable != null) { val e = throwable.cause var isRetriable = true @@ -213,7 +208,6 @@ class PaymentExternalSystemAdapterImpl( scheduleRetry( retryCount, request, paymentId, transactionId, deadline, capped ) - return@launch } else { paymentESService.update(paymentId) { it.logProcessing(false, now(), transactionId, "Non-retriable exception") @@ -249,7 +243,6 @@ class PaymentExternalSystemAdapterImpl( parallelLimiter.release() } } - } } } @@ -257,32 +250,29 @@ class PaymentExternalSystemAdapterImpl( retryCount: Long, request: HttpRequest, paymentId: UUID, transactionId: UUID, deadline: Long, delay: Long ) { - retryScheduler.schedule({ + scope.launch { + delay(delay) val remainingTime = deadline - now() if (remainingTime < requestAverageProcessingTime.toMillis()) { - scope.launch { - try { - paymentESService.update(paymentId) { - it.logProcessing(false, now(), transactionId, "Not enough time for retry") - } - } catch (e: Exception) { - logger.error("[$accountName] Failed to record retry failure for $paymentId", e) - } finally { - startedRequests.increment() + try { + paymentESService.update(paymentId) { + it.logProcessing(false, now(), transactionId, "Not enough time for retry") } + } catch (e: Exception) { + logger.error("[$accountName] Failed to record retry failure for $paymentId", e) + } finally { + startedRequests.increment() } - return@schedule + } else { + val newRequestTimeout = remainingTime.coerceIn(100, requestAverageProcessingTime.toMillis() * 2) + val newRequest = HttpRequest.newBuilder() + .uri(request.uri()) + .timeout(Duration.ofMillis(newRequestTimeout)) + .POST(HttpRequest.BodyPublishers.noBody()) + .build() + completeAction(retryCount + 1, newRequest, paymentId, transactionId, deadline) } - - val newRequestTimeout = remainingTime.coerceIn(100, requestAverageProcessingTime.toMillis() * 2) - val newRequest = HttpRequest.newBuilder() - .uri(request.uri()) - .timeout(Duration.ofMillis(newRequestTimeout)) - .POST(HttpRequest.BodyPublishers.noBody()) - .build() - - completeAction(retryCount + 1, newRequest, paymentId, transactionId, deadline) - }, delay, TimeUnit.MILLISECONDS) + } } } From b728d7183795c854607b24995492bc036ddae830 Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Mon, 8 Dec 2025 19:56:12 +0300 Subject: [PATCH 47/61] hotfix --- .../ru/quipy/payments/logic/PaymentExternalServiceImpl.kt | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index e4859f60e..d21cf308c 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -63,7 +63,7 @@ class PaymentExternalSystemAdapterImpl( private val rateLimit: SlidingWindowRateLimiter by lazy { SlidingWindowRateLimiter( - rate = (rateLimitPerSec * 0.95).toLong(), + rate = rateLimitPerSec.toLong(), window = Duration.ofMillis(1000), ) } @@ -116,7 +116,6 @@ class PaymentExternalSystemAdapterImpl( if (requestTimeout < requestAverageProcessingTime.toMillis()) { logger.warn("[$accountName] Timeout too short for payment $paymentId: ${requestTimeout}ms") - parallelLimiter.release() val retryAfterMs = requestAverageProcessingTime.toMillis() - requestTimeout + Random.nextLong(100) throw TooManyRequestsException(retryAfterMs) } @@ -167,6 +166,7 @@ class PaymentExternalSystemAdapterImpl( throw TooManyRequestsException(retryAfterMs) } + startedRequests.increment() httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .whenComplete { response, throwable -> if (throwable != null) { @@ -195,6 +195,7 @@ class PaymentExternalSystemAdapterImpl( paymentESService.update(paymentId) { it.logProcessing(false, now(), transactionId, "Max attempts reached") } + parallelLimiter.release() } else { val backoff = ((2.0.pow(retryCount.toDouble()) * 25).toLong() + Random.nextLong(10)) val capped = backoff.coerceAtMost(deadline - now() - 5) @@ -239,7 +240,6 @@ class PaymentExternalSystemAdapterImpl( } catch (e: Exception) { logger.error("[$accountName] Error processing payment $paymentId", e) } finally { - startedRequests.increment() parallelLimiter.release() } } @@ -260,8 +260,6 @@ class PaymentExternalSystemAdapterImpl( } } catch (e: Exception) { logger.error("[$accountName] Failed to record retry failure for $paymentId", e) - } finally { - startedRequests.increment() } } else { val newRequestTimeout = remainingTime.coerceIn(100, requestAverageProcessingTime.toMillis() * 2) From e5a7053e256bcb4054c605bbd35e557885d929b6 Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Mon, 8 Dec 2025 20:07:05 +0300 Subject: [PATCH 48/61] hotfix --- .../ru/quipy/payments/logic/PaymentExternalServiceImpl.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index d21cf308c..2b7ea2fa7 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -99,7 +99,7 @@ class PaymentExternalSystemAdapterImpl( logger.info("[$accountName] Submit: $paymentId , txId: $transactionId") val remaining = deadline - now() - val minRequiredTime = requestAverageProcessingTime.toMillis() * 2 + val minRequiredTime = requestAverageProcessingTime.toMillis() if (remaining < minRequiredTime) { logger.warn("[$accountName] Not enough time for payment $paymentId: ${remaining}ms remaining, need ${minRequiredTime}ms") paymentESService.update(paymentId) { @@ -111,7 +111,7 @@ class PaymentExternalSystemAdapterImpl( val requestTimeout = minOf( deadline - now(), - requestAverageProcessingTime.toMillis() * 2 + requestAverageProcessingTime.toMillis() ).coerceAtLeast(100) if (requestTimeout < requestAverageProcessingTime.toMillis()) { From fa252a5d29d36a42f6b6252827069e0941870c9f Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Fri, 12 Dec 2025 18:03:13 +0300 Subject: [PATCH 49/61] hotfix --- .../kotlin/ru/quipy/OnlineShopApplication.kt | 4 +- .../apigateway/GlobalExceptionHandler.kt | 3 +- .../payments/config/PaymentAccountsConfig.kt | 15 +++++- .../ru/quipy/payments/logic/OrderPayer.kt | 32 ++++++++--- .../logic/PaymentExternalServiceImpl.kt | 54 ++++++------------- 5 files changed, 57 insertions(+), 51 deletions(-) diff --git a/src/main/kotlin/ru/quipy/OnlineShopApplication.kt b/src/main/kotlin/ru/quipy/OnlineShopApplication.kt index 1fcd3f89c..bb8d5464e 100644 --- a/src/main/kotlin/ru/quipy/OnlineShopApplication.kt +++ b/src/main/kotlin/ru/quipy/OnlineShopApplication.kt @@ -14,10 +14,10 @@ class OnlineShopApplication { val log: Logger = LoggerFactory.getLogger(OnlineShopApplication::class.java) companion object { - val appExecutor = Executors.newFixedThreadPool(20_000, NamedThreadFactory("main-app-executor")).asCoroutineDispatcher() + val appExecutor = Executors.newFixedThreadPool(64, NamedThreadFactory("main-app-executor")).asCoroutineDispatcher() } } -suspend fun main(args: Array) { +fun main(args: Array) { runApplication(*args) } diff --git a/src/main/kotlin/ru/quipy/apigateway/GlobalExceptionHandler.kt b/src/main/kotlin/ru/quipy/apigateway/GlobalExceptionHandler.kt index ea33b9e32..7d2c38fe8 100644 --- a/src/main/kotlin/ru/quipy/apigateway/GlobalExceptionHandler.kt +++ b/src/main/kotlin/ru/quipy/apigateway/GlobalExceptionHandler.kt @@ -29,10 +29,9 @@ class GlobalExceptionHandler( @ExceptionHandler(TooManyRequestsException::class) fun handleTooManyRequestsRetriable(exception: TooManyRequestsException): ResponseEntity { - val retryAfterSeconds = (exception.retryAfterMs / 1000.0).coerceAtLeast(0.1) return ResponseEntity .status(HttpStatus.TOO_MANY_REQUESTS) - .header("Retry-After", retryAfterSeconds.toString()) + .header("Retry-After", "10") .body("too many requests") } } \ No newline at end of file diff --git a/src/main/kotlin/ru/quipy/payments/config/PaymentAccountsConfig.kt b/src/main/kotlin/ru/quipy/payments/config/PaymentAccountsConfig.kt index 8012d1ee7..68981f838 100644 --- a/src/main/kotlin/ru/quipy/payments/config/PaymentAccountsConfig.kt +++ b/src/main/kotlin/ru/quipy/payments/config/PaymentAccountsConfig.kt @@ -4,10 +4,10 @@ import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule import com.fasterxml.jackson.module.kotlin.registerKotlinModule import io.micrometer.core.instrument.MeterRegistry -import kotlinx.coroutines.sync.Semaphore import org.springframework.beans.factory.annotation.Value import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration +import ru.quipy.common.utils.SlidingWindowRateLimiter import ru.quipy.core.EventSourcingService import ru.quipy.payments.api.PaymentAggregate import ru.quipy.payments.logic.PaymentAccountProperties @@ -18,7 +18,9 @@ import java.net.URI import java.net.http.HttpClient import java.net.http.HttpRequest import java.net.http.HttpResponse +import java.time.Duration import java.util.* +import java.util.concurrent.Semaphore @Configuration @@ -40,10 +42,18 @@ class PaymentAccountsConfig { @Value("#{'\${payment.accounts}'.split(',')}") lateinit var allowedAccounts: List + @Bean + fun rateLimit(): SlidingWindowRateLimiter { + return SlidingWindowRateLimiter( + rate = 1100L, + window = Duration.ofMillis(1000), + ) + } @Bean fun accountAdapters( paymentService: EventSourcingService, meterRegistry: MeterRegistry, + rateLimiter: SlidingWindowRateLimiter ): List { val request = HttpRequest.newBuilder() .uri(URI("http://${paymentProviderHostPort}/external/accounts?serviceName=$serviceName&token=$token")) @@ -67,7 +77,8 @@ class PaymentAccountsConfig { paymentProviderHostPort, token, meterRegistry, - Semaphore(it.parallelRequests) + Semaphore(it.parallelRequests), + rateLimiter ) } } diff --git a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt index e2bd9e203..50b1f81bf 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt @@ -5,12 +5,19 @@ import org.slf4j.Logger import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service +import ru.quipy.common.utils.NamedThreadFactory +import ru.quipy.common.utils.SlidingWindowRateLimiter import ru.quipy.core.EventSourcingService +import ru.quipy.exceptions.TooManyRequestsException import ru.quipy.payments.api.PaymentAggregate import java.util.* +import java.util.concurrent.LinkedBlockingQueue +import java.util.concurrent.ThreadPoolExecutor +import java.util.concurrent.TimeUnit +import kotlin.random.Random @Service -class OrderPayer(meterRegistry: MeterRegistry) { +class OrderPayer(val rateLimiter : SlidingWindowRateLimiter, meterRegistry: MeterRegistry) { companion object { val logger: Logger = LoggerFactory.getLogger(OrderPayer::class.java) @@ -18,6 +25,14 @@ class OrderPayer(meterRegistry: MeterRegistry) { private val plannedRequests = meterRegistry.counter("payment.processing.planned", "accountName", "acc-12") + private val paymentExecutor = ThreadPoolExecutor( + 50, + 50, + 100L, + TimeUnit.MILLISECONDS, + LinkedBlockingQueue(40_000), + NamedThreadFactory("payment-submission-executor") + ) @Autowired private lateinit var paymentESService: EventSourcingService @@ -28,8 +43,13 @@ class OrderPayer(meterRegistry: MeterRegistry) { fun processPayment(orderId: UUID, amount: Int, paymentId: UUID, deadline: Long): Long { val createdAt = System.currentTimeMillis() - plannedRequests.increment() - val createdEvent = paymentESService.create { + if (!rateLimiter.tick()) { + val retryAfterMs = (1000L / 1100 * 10).coerceIn(10, 100) + Random.nextLong(10) + throw TooManyRequestsException(retryAfterMs) + } + paymentExecutor.submit { + plannedRequests.increment() + val createdEvent = paymentESService.create { it.create( paymentId, orderId, @@ -37,10 +57,10 @@ class OrderPayer(meterRegistry: MeterRegistry) { ) } - logger.trace("Payment {} for order {} created.", createdEvent.paymentId, orderId) - - paymentService.submitPaymentRequest(paymentId, amount, createdAt, deadline) + logger.trace("Payment {} for order {} created.", createdEvent.paymentId, orderId) + paymentService.submitPaymentRequest(paymentId, amount, createdAt, deadline) + } return createdAt } } \ No newline at end of file diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index 2b7ea2fa7..a9a92b465 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -4,10 +4,8 @@ import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.registerKotlinModule import io.micrometer.core.instrument.MeterRegistry import kotlinx.coroutines.* -import kotlinx.coroutines.sync.Semaphore import okio.EOFException import org.slf4j.LoggerFactory -import ru.quipy.common.utils.NamedThreadFactory import ru.quipy.common.utils.SlidingWindowRateLimiter import ru.quipy.core.EventSourcingService import ru.quipy.exceptions.TooManyRequestsException @@ -21,6 +19,7 @@ import java.net.http.HttpResponse import java.time.Duration import java.util.* import java.util.concurrent.Executors +import java.util.concurrent.Semaphore import java.util.concurrent.TimeUnit import kotlin.math.pow import kotlin.random.Random @@ -33,10 +32,8 @@ class PaymentExternalSystemAdapterImpl( private val paymentProviderHostPort: String, private val token: String, meterRegistry: MeterRegistry, - private val parallelLimiter: Semaphore, ioDispatcher: CoroutineDispatcher = Executors.newFixedThreadPool( - Runtime.getRuntime().availableProcessors() * 2, - NamedThreadFactory("payment-io-") - ).asCoroutineDispatcher() + private val parallelLimiter: Semaphore, + private val rateLimiter: SlidingWindowRateLimiter ) : PaymentExternalSystemAdapter { companion object { val logger = LoggerFactory.getLogger(PaymentExternalSystemAdapter::class.java) @@ -47,8 +44,6 @@ class PaymentExternalSystemAdapterImpl( logger.error("[$accountName] Unhandled exception in payment adapter coroutine", throwable) } - private val scope = CoroutineScope(SupervisorJob() + ioDispatcher + exceptionHandler) - // 2025-11-20T20:30:35.780+03:00 INFO 56644 --- [alhost:1234/...] ru.quipy.core.EventSourcingService : Optimistic lock exception. Failed to save event records id: [7dca693e-e811-4b7f-8bce-23e13d952c04-4] private val startedRequests = @@ -61,12 +56,7 @@ class PaymentExternalSystemAdapterImpl( private val rateLimitPerSec = properties.rateLimitPerSec private val parallelRequests = properties.parallelRequests - private val rateLimit: SlidingWindowRateLimiter by lazy { - SlidingWindowRateLimiter( - rate = rateLimitPerSec.toLong(), - window = Duration.ofMillis(1000), - ) - } + private val httpClient = HttpClient .newBuilder() @@ -80,7 +70,6 @@ class PaymentExternalSystemAdapterImpl( // Вне зависимости от исхода оплаты важно отметить что она была отправлена. // Это требуется сделать ВО ВСЕХ СЛУЧАЯХ, поскольку эта информация используется сервисом тестирования. - scope.launch { try { paymentESService.update(paymentId) { it.logSubmission( @@ -94,7 +83,6 @@ class PaymentExternalSystemAdapterImpl( } catch (e: Exception) { logger.error("[$accountName] Failed to record log submission for $paymentId", e) } - } logger.info("[$accountName] Submit: $paymentId , txId: $transactionId") @@ -126,9 +114,7 @@ class PaymentExternalSystemAdapterImpl( .POST(HttpRequest.BodyPublishers.noBody()) .build() - - - scope.launch { completeAction(0, request, paymentId, transactionId, deadline) } + completeAction(0, request, paymentId, transactionId, deadline) } override fun price() = properties.price @@ -137,17 +123,7 @@ class PaymentExternalSystemAdapterImpl( override fun name() = properties.accountName - suspend fun tryAcquire(startedAt: Long, remaining: Long): Boolean { - var isAcquired = parallelLimiter.tryAcquire() - while (!isAcquired && now() - startedAt < remaining) { - isAcquired = parallelLimiter.tryAcquire() - delay(2) - } - - return isAcquired - } - - suspend fun completeAction( + fun completeAction( retryCount: Long, request: HttpRequest, paymentId: UUID, @@ -155,11 +131,8 @@ class PaymentExternalSystemAdapterImpl( deadline: Long ) { val timeBeforeCall = now() - if (!rateLimit.tick()) { - val retryAfterMs = (1000L / rateLimitPerSec * 10).coerceIn(10, 100) + Random.nextLong(10) - throw TooManyRequestsException(retryAfterMs) - } - if (!tryAcquire(now(), deadline - now())) { + + if (!parallelLimiter.tryAcquire(deadline - now(), TimeUnit.MILLISECONDS)) { val retryAfterMs = (requestAverageProcessingTime.toMillis() / parallelRequests * 10).coerceIn( 10, 100 ) + Random.nextLong(10) @@ -219,7 +192,7 @@ class PaymentExternalSystemAdapterImpl( } } else { try { - logger.warn("Free space in semaphore: {}", parallelLimiter.availablePermits) + logger.warn("Free space in semaphore: {}", parallelLimiter.availablePermits()) logger.info( "success in callback for payment: {}, retry count: {}, in time: {}", paymentId, @@ -250,8 +223,8 @@ class PaymentExternalSystemAdapterImpl( retryCount: Long, request: HttpRequest, paymentId: UUID, transactionId: UUID, deadline: Long, delay: Long ) { - scope.launch { - delay(delay) + + Thread.sleep(delay) val remainingTime = deadline - now() if (remainingTime < requestAverageProcessingTime.toMillis()) { try { @@ -268,8 +241,11 @@ class PaymentExternalSystemAdapterImpl( .timeout(Duration.ofMillis(newRequestTimeout)) .POST(HttpRequest.BodyPublishers.noBody()) .build() + if (!rateLimiter.tick()) { + val retryAfterMs = (1000L / rateLimitPerSec * 10).coerceIn(10, 100) + Random.nextLong(10) + throw TooManyRequestsException(retryAfterMs) + } completeAction(retryCount + 1, newRequest, paymentId, transactionId, deadline) - } } } } From 661900bc8ad9a2fcfef5e8d1595db80df6e8edd8 Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Fri, 12 Dec 2025 18:03:54 +0300 Subject: [PATCH 50/61] hotfix --- src/main/kotlin/ru/quipy/OnlineShopApplication.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/ru/quipy/OnlineShopApplication.kt b/src/main/kotlin/ru/quipy/OnlineShopApplication.kt index bb8d5464e..f311c6fdf 100644 --- a/src/main/kotlin/ru/quipy/OnlineShopApplication.kt +++ b/src/main/kotlin/ru/quipy/OnlineShopApplication.kt @@ -6,6 +6,7 @@ import org.slf4j.LoggerFactory import org.springframework.boot.autoconfigure.SpringBootApplication import org.springframework.boot.runApplication import ru.quipy.common.utils.NamedThreadFactory +import java.util.concurrent.ExecutorService import java.util.concurrent.Executors @@ -14,7 +15,7 @@ class OnlineShopApplication { val log: Logger = LoggerFactory.getLogger(OnlineShopApplication::class.java) companion object { - val appExecutor = Executors.newFixedThreadPool(64, NamedThreadFactory("main-app-executor")).asCoroutineDispatcher() + val appExecutor: ExecutorService = Executors.newFixedThreadPool(64, NamedThreadFactory("main-app-executor")) } } From ba2768127e6daf597190c39dfd05691af9a7c5e5 Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Fri, 12 Dec 2025 18:14:43 +0300 Subject: [PATCH 51/61] hotfix --- src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt index 50b1f81bf..36818491e 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt @@ -26,11 +26,11 @@ class OrderPayer(val rateLimiter : SlidingWindowRateLimiter, meterRegistry: Mete private val plannedRequests = meterRegistry.counter("payment.processing.planned", "accountName", "acc-12") private val paymentExecutor = ThreadPoolExecutor( - 50, - 50, + 1000, + 1000, 100L, TimeUnit.MILLISECONDS, - LinkedBlockingQueue(40_000), + LinkedBlockingQueue(100_000), NamedThreadFactory("payment-submission-executor") ) From e7f47b749fbdae688d1df101d9c0a4844ca38421 Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Fri, 12 Dec 2025 18:39:22 +0300 Subject: [PATCH 52/61] hotfix --- src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt index 36818491e..e70e67a76 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt @@ -26,8 +26,8 @@ class OrderPayer(val rateLimiter : SlidingWindowRateLimiter, meterRegistry: Mete private val plannedRequests = meterRegistry.counter("payment.processing.planned", "accountName", "acc-12") private val paymentExecutor = ThreadPoolExecutor( - 1000, - 1000, + 2000, + 2000, 100L, TimeUnit.MILLISECONDS, LinkedBlockingQueue(100_000), From ca635e3cdbc4a1b5e961e65bb897dd87f75c4340 Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Fri, 12 Dec 2025 18:48:14 +0300 Subject: [PATCH 53/61] hotfix --- src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt | 4 ++-- .../ru/quipy/payments/logic/PaymentExternalServiceImpl.kt | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt index e70e67a76..36818491e 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt @@ -26,8 +26,8 @@ class OrderPayer(val rateLimiter : SlidingWindowRateLimiter, meterRegistry: Mete private val plannedRequests = meterRegistry.counter("payment.processing.planned", "accountName", "acc-12") private val paymentExecutor = ThreadPoolExecutor( - 2000, - 2000, + 1000, + 1000, 100L, TimeUnit.MILLISECONDS, LinkedBlockingQueue(100_000), diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index a9a92b465..990a86c4e 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -40,6 +40,7 @@ class PaymentExternalSystemAdapterImpl( val mapper = ObjectMapper().registerKotlinModule() } + private val time_95_percentile = 20_000 private val exceptionHandler = CoroutineExceptionHandler { _, throwable -> logger.error("[$accountName] Unhandled exception in payment adapter coroutine", throwable) } @@ -98,7 +99,7 @@ class PaymentExternalSystemAdapterImpl( } val requestTimeout = minOf( - deadline - now(), + time_95_percentile.toLong(), requestAverageProcessingTime.toMillis() ).coerceAtLeast(100) From 1ca0e9d7f1d69d4559673869e212209d6f46ebfe Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Fri, 12 Dec 2025 19:06:22 +0300 Subject: [PATCH 54/61] hotfix --- src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt index 36818491e..036ed4121 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt @@ -26,11 +26,11 @@ class OrderPayer(val rateLimiter : SlidingWindowRateLimiter, meterRegistry: Mete private val plannedRequests = meterRegistry.counter("payment.processing.planned", "accountName", "acc-12") private val paymentExecutor = ThreadPoolExecutor( - 1000, - 1000, + 3600, + 3600, 100L, TimeUnit.MILLISECONDS, - LinkedBlockingQueue(100_000), + LinkedBlockingQueue(200_000), NamedThreadFactory("payment-submission-executor") ) From 8453d1447f7932ba94a64f65883b64dd7d5d5396 Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Fri, 12 Dec 2025 19:20:21 +0300 Subject: [PATCH 55/61] hotfix --- .../ru/quipy/payments/logic/PaymentExternalServiceImpl.kt | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index 990a86c4e..d14208328 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -3,7 +3,6 @@ package ru.quipy.payments.logic import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.registerKotlinModule import io.micrometer.core.instrument.MeterRegistry -import kotlinx.coroutines.* import okio.EOFException import org.slf4j.LoggerFactory import ru.quipy.common.utils.SlidingWindowRateLimiter @@ -41,9 +40,6 @@ class PaymentExternalSystemAdapterImpl( } private val time_95_percentile = 20_000 - private val exceptionHandler = CoroutineExceptionHandler { _, throwable -> - logger.error("[$accountName] Unhandled exception in payment adapter coroutine", throwable) - } // 2025-11-20T20:30:35.780+03:00 INFO 56644 --- [alhost:1234/...] ru.quipy.core.EventSourcingService : Optimistic lock exception. Failed to save event records id: [7dca693e-e811-4b7f-8bce-23e13d952c04-4] @@ -224,9 +220,9 @@ class PaymentExternalSystemAdapterImpl( retryCount: Long, request: HttpRequest, paymentId: UUID, transactionId: UUID, deadline: Long, delay: Long ) { - + parallelLimiter.release() Thread.sleep(delay) - val remainingTime = deadline - now() + val remainingTime = deadline - now() if (remainingTime < requestAverageProcessingTime.toMillis()) { try { paymentESService.update(paymentId) { From f1f589843b6bd33376d5fa72bf73b4d57cdc37ec Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Fri, 12 Dec 2025 19:39:30 +0300 Subject: [PATCH 56/61] hotfix --- .../dashboards/ServicesStatistic.json | 207 ++++++++++++++++++ .../logic/PaymentExternalServiceImpl.kt | 6 + 2 files changed, 213 insertions(+) diff --git a/grafana/provisioning/dashboards/ServicesStatistic.json b/grafana/provisioning/dashboards/ServicesStatistic.json index eb6f028d2..e30ee04a3 100644 --- a/grafana/provisioning/dashboards/ServicesStatistic.json +++ b/grafana/provisioning/dashboards/ServicesStatistic.json @@ -3727,6 +3727,213 @@ ], "title": "Payment Processing Latency", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Количество повторных запросов к платежной системе в секунду", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Запросов/сек", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 109, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(payment_request_retried_total{accountName=~\".*\"}[1m])", + "legendFormat": "Повторные запросы: {{accountName}}", + "range": true, + "refId": "A" + } + ], + "title": "Payment Request Retries Rate (per second)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Общее количество повторных запросов к платежной системе", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Количество", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 100 + }, + { + "color": "red", + "value": 500 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 110, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "payment_request_retried_total{accountName=~\".*\"}", + "legendFormat": "Всего повторов: {{accountName}}", + "range": true, + "refId": "A" + } + ], + "title": "Payment Request Retries Total Count", + "type": "timeseries" }], "preload": false, "refresh": "5s", diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index d14208328..46c9920de 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -20,6 +20,7 @@ import java.util.* import java.util.concurrent.Executors import java.util.concurrent.Semaphore import java.util.concurrent.TimeUnit +import kotlin.math.log import kotlin.math.pow import kotlin.random.Random @@ -42,9 +43,12 @@ class PaymentExternalSystemAdapterImpl( private val time_95_percentile = 20_000 + // 2025-11-20T20:30:35.780+03:00 INFO 56644 --- [alhost:1234/...] ru.quipy.core.EventSourcingService : Optimistic lock exception. Failed to save event records id: [7dca693e-e811-4b7f-8bce-23e13d952c04-4] private val startedRequests = meterRegistry.counter("payment.processing.started", "accountName", properties.accountName) + private val requestsRetried = + meterRegistry.counter("payment.request.retried", "accountName", properties.accountName) private val timer = meterRegistry.timer("payment.external.system.request.latency", "accountName", properties.accountName) private val serviceName = properties.serviceName @@ -220,6 +224,8 @@ class PaymentExternalSystemAdapterImpl( retryCount: Long, request: HttpRequest, paymentId: UUID, transactionId: UUID, deadline: Long, delay: Long ) { + requestsRetried.increment() + logger.info("Completing retry. All retry count - {}", requestsRetried.count()) parallelLimiter.release() Thread.sleep(delay) val remainingTime = deadline - now() From f7af64e68bc83d32c8fd277a67717e61b0a88f44 Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Fri, 12 Dec 2025 19:49:52 +0300 Subject: [PATCH 57/61] hotfix --- src/main/resources/application.properties | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index bdbe25f61..ab2efd0cc 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -29,4 +29,11 @@ payment.token=${PAYMENT_TOKEN} payment.accounts=${PAYMENT_ACCOUNTS:acc-12} # payment.accounts=${PAYMENT_ACCOUNTS:acc-18} payment.hostPort=${PAYMENT_HOST:localhost}:${PAYMENT_PORT:1234} +spring.datasource.hikari.maximum-pool-size=20_000 +spring.datasource.hikari.minimum-idle=50 +spring.datasource.hikari.connection-timeout=5000 +server.tomcat.threads.max=20000 +server.tomcat.threads.min-spare=100 +server.tomcat.accept-count=2000 +server.tomcat.max-connections=20000 From 8864f7841824a63ea6db7c49cef7fd80805a0045 Mon Sep 17 00:00:00 2001 From: vaycheslav Date: Fri, 12 Dec 2025 19:51:56 +0300 Subject: [PATCH 58/61] hotfix --- src/main/resources/application.properties | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index ab2efd0cc..d95ec6218 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -32,8 +32,11 @@ payment.hostPort=${PAYMENT_HOST:localhost}:${PAYMENT_PORT:1234} spring.datasource.hikari.maximum-pool-size=20_000 spring.datasource.hikari.minimum-idle=50 spring.datasource.hikari.connection-timeout=5000 -server.tomcat.threads.max=20000 -server.tomcat.threads.min-spare=100 -server.tomcat.accept-count=2000 -server.tomcat.max-connections=20000 +server.jetty.threads.min=100 +server.jetty.threads.max=20000 +server.jetty.threads.max-queue-capacity=100000 +server.jetty.threads.acceptors=16 +server.jetty.threads.selectors=32 +server.jetty.connection-idle-timeout=60000 + From 5923fec41f7fe13cb2b8fe5bf207daec6d8f6e46 Mon Sep 17 00:00:00 2001 From: 21092004Goda Date: Thu, 26 Feb 2026 21:43:59 +0300 Subject: [PATCH 59/61] =?UTF-8?q?=D0=A1=D1=82=D0=B0=D1=80=D1=82=20=D0=B2?= =?UTF-8?q?=D1=81=D0=B5=D0=B3=D0=BE=20=D1=81=D1=83=D1=89=D0=B5=D0=B3=D0=BE?= =?UTF-8?q?:=20"=D0=94=D0=BE=D0=BB=D0=B3=D0=B0=D1=8F=20=D0=BC=D0=BE=D0=BB?= =?UTF-8?q?=D0=B8=D1=82=D0=B2=D0=B0..."?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/resources/application.properties | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index d95ec6218..7c2e843a0 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -26,7 +26,8 @@ payment.service-name=${PAYMENT_SERVICE_NAME} payment.token=${PAYMENT_TOKEN} # payment.accounts=${PAYMENT_ACCOUNTS:acc-12,acc-20} # payment.accounts=${PAYMENT_ACCOUNTS:acc-3} -payment.accounts=${PAYMENT_ACCOUNTS:acc-12} +#payment.accounts=${PAYMENT_ACCOUNTS:acc-12} +payment.accounts=${PAYMENT_ACCOUNTS:acc-13} # payment.accounts=${PAYMENT_ACCOUNTS:acc-18} payment.hostPort=${PAYMENT_HOST:localhost}:${PAYMENT_PORT:1234} spring.datasource.hikari.maximum-pool-size=20_000 From 64379a5fb7f228d9f237ce5a1fbdc7778d4e63ea Mon Sep 17 00:00:00 2001 From: 21092004Goda Date: Thu, 26 Feb 2026 21:53:38 +0300 Subject: [PATCH 60/61] =?UTF-8?q?=D0=9D=D1=83=20=D0=BF=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B5=D1=80=D0=B8=D0=BC)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ru/quipy/payments/config/PaymentAccountsConfig.kt | 2 +- src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/config/PaymentAccountsConfig.kt b/src/main/kotlin/ru/quipy/payments/config/PaymentAccountsConfig.kt index 68981f838..dcc94330f 100644 --- a/src/main/kotlin/ru/quipy/payments/config/PaymentAccountsConfig.kt +++ b/src/main/kotlin/ru/quipy/payments/config/PaymentAccountsConfig.kt @@ -45,7 +45,7 @@ class PaymentAccountsConfig { @Bean fun rateLimit(): SlidingWindowRateLimiter { return SlidingWindowRateLimiter( - rate = 1100L, + rate = 4000L, window = Duration.ofMillis(1000), ) } diff --git a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt index 036ed4121..fd38a5c68 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt @@ -23,11 +23,11 @@ class OrderPayer(val rateLimiter : SlidingWindowRateLimiter, meterRegistry: Mete val logger: Logger = LoggerFactory.getLogger(OrderPayer::class.java) } - private val plannedRequests = meterRegistry.counter("payment.processing.planned", "accountName", "acc-12") + private val plannedRequests = meterRegistry.counter("payment.processing.planned", "accountName", "acc-13") private val paymentExecutor = ThreadPoolExecutor( - 3600, - 3600, + 4000, + 4000, 100L, TimeUnit.MILLISECONDS, LinkedBlockingQueue(200_000), From 129ec44aff87a219ba3de1f208b42a5bac15f462 Mon Sep 17 00:00:00 2001 From: 21092004Goda Date: Thu, 26 Feb 2026 23:14:24 +0300 Subject: [PATCH 61/61] =?UTF-8?q?=D0=9D=D1=83=20=D0=BF=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B5=D1=80=D0=B8=D0=BC)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../payments/config/PaymentAccountsConfig.kt | 2 +- .../kotlin/ru/quipy/payments/logic/OrderPayer.kt | 2 +- .../payments/logic/PaymentExternalServiceImpl.kt | 15 ++++++--------- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/main/kotlin/ru/quipy/payments/config/PaymentAccountsConfig.kt b/src/main/kotlin/ru/quipy/payments/config/PaymentAccountsConfig.kt index dcc94330f..17814f09f 100644 --- a/src/main/kotlin/ru/quipy/payments/config/PaymentAccountsConfig.kt +++ b/src/main/kotlin/ru/quipy/payments/config/PaymentAccountsConfig.kt @@ -45,7 +45,7 @@ class PaymentAccountsConfig { @Bean fun rateLimit(): SlidingWindowRateLimiter { return SlidingWindowRateLimiter( - rate = 4000L, + rate = 5000L, window = Duration.ofMillis(1000), ) } diff --git a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt index fd38a5c68..ac1e4bd1b 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/OrderPayer.kt @@ -44,7 +44,7 @@ class OrderPayer(val rateLimiter : SlidingWindowRateLimiter, meterRegistry: Mete fun processPayment(orderId: UUID, amount: Int, paymentId: UUID, deadline: Long): Long { val createdAt = System.currentTimeMillis() if (!rateLimiter.tick()) { - val retryAfterMs = (1000L / 1100 * 10).coerceIn(10, 100) + Random.nextLong(10) + val retryAfterMs = 10L + Random.nextLong(10) throw TooManyRequestsException(retryAfterMs) } paymentExecutor.submit { diff --git a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt index 46c9920de..89943d358 100644 --- a/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt +++ b/src/main/kotlin/ru/quipy/payments/logic/PaymentExternalServiceImpl.kt @@ -20,7 +20,6 @@ import java.util.* import java.util.concurrent.Executors import java.util.concurrent.Semaphore import java.util.concurrent.TimeUnit -import kotlin.math.log import kotlin.math.pow import kotlin.random.Random @@ -40,7 +39,7 @@ class PaymentExternalSystemAdapterImpl( val mapper = ObjectMapper().registerKotlinModule() } - private val time_95_percentile = 20_000 + private val time_95_percentile = 1_000 @@ -98,14 +97,12 @@ class PaymentExternalSystemAdapterImpl( throw TooManyRequestsException(retryAfterMs) } - val requestTimeout = minOf( - time_95_percentile.toLong(), - requestAverageProcessingTime.toMillis() - ).coerceAtLeast(100) + // Таймаут = оставшееся время до дедлайна, но не больше time_95_percentile + val requestTimeout = minOf(remaining, time_95_percentile.toLong()).coerceAtLeast(100) - if (requestTimeout < requestAverageProcessingTime.toMillis()) { - logger.warn("[$accountName] Timeout too short for payment $paymentId: ${requestTimeout}ms") - val retryAfterMs = requestAverageProcessingTime.toMillis() - requestTimeout + Random.nextLong(100) + if (!rateLimiter.tick()) { + val retryAfterMs = (1000L / rateLimitPerSec * 10).coerceIn(10, 100) + Random.nextLong(10) + logger.warn("[$accountName] Rate limit exceeded for payment $paymentId, retry after ${retryAfterMs}ms") throw TooManyRequestsException(retryAfterMs) }