From f33c1faa35bb68f6b0e84f11b7dde5c7371b7f74 Mon Sep 17 00:00:00 2001 From: tmcgroul Date: Fri, 26 Jun 2026 17:19:31 +0300 Subject: [PATCH] collect metrics about rpc calls usage --- .../rpc-client/master_2026-06-26-14-18.json | 10 +++ .../master_2026-06-26-14-18.json | 10 +++ util/rpc-client/src/client.ts | 26 ++++---- util/rpc-client/src/method-metrics.ts | 64 +++++++++++++++++++ util/util-internal-dump-cli/src/prometheus.ts | 23 +++++-- 5 files changed, 116 insertions(+), 17 deletions(-) create mode 100644 common/changes/@subsquid/rpc-client/master_2026-06-26-14-18.json create mode 100644 common/changes/@subsquid/util-internal-dump-cli/master_2026-06-26-14-18.json create mode 100644 util/rpc-client/src/method-metrics.ts diff --git a/common/changes/@subsquid/rpc-client/master_2026-06-26-14-18.json b/common/changes/@subsquid/rpc-client/master_2026-06-26-14-18.json new file mode 100644 index 000000000..25521657f --- /dev/null +++ b/common/changes/@subsquid/rpc-client/master_2026-06-26-14-18.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@subsquid/rpc-client", + "comment": "", + "type": "none" + } + ], + "packageName": "@subsquid/rpc-client" +} \ No newline at end of file diff --git a/common/changes/@subsquid/util-internal-dump-cli/master_2026-06-26-14-18.json b/common/changes/@subsquid/util-internal-dump-cli/master_2026-06-26-14-18.json new file mode 100644 index 000000000..a56dbbbd9 --- /dev/null +++ b/common/changes/@subsquid/util-internal-dump-cli/master_2026-06-26-14-18.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@subsquid/util-internal-dump-cli", + "comment": "", + "type": "none" + } + ], + "packageName": "@subsquid/util-internal-dump-cli" +} \ No newline at end of file diff --git a/util/rpc-client/src/client.ts b/util/rpc-client/src/client.ts index bc3766233..9a15c2723 100644 --- a/util/rpc-client/src/client.ts +++ b/util/rpc-client/src/client.ts @@ -6,6 +6,7 @@ import assert from 'assert' import {RetryError, RpcConnectionError, RpcError} from './errors' import {Connection, HttpHeaders, RpcCall, RpcErrorInfo, RpcNotification, RpcRequest, RpcResponse} from './interfaces' import {RateMeter} from './rate' +import {MethodMetrics} from './method-metrics' import {Subscription, SubscriptionHandle, Subscriptions} from './subscriptions' import {HttpConnection} from './transport/http' import {WsConnection} from './transport/ws' @@ -62,16 +63,6 @@ export interface RpcClientOptions { log?: Logger | null } -// Add interface for RPC metrics -export interface RpcMetrics { - url: string - requestsServed: number - connectionErrors: number - notificationsReceived: number - avg_response_time: number -} - - export interface RpcMetrics { url: string requestsServed: number @@ -130,6 +121,7 @@ export class RpcClient { private connectionErrorsInRow = 0 private connectionErrors = 0 private requestsServed = 0 + private requestsByMethod = new MethodMetrics() private notificationsReceived = 0 private totalResponseTime = 0 private backoffEpoch = 0 @@ -204,12 +196,14 @@ export class RpcClient { requestsServed: this.requestsServed, connectionErrors: this.connectionErrors, notificationsReceived: this.notificationsReceived, - // FIXME: only one of these metrics should remain; decide which to keep - avg_response_time: this.requestsServed > 0 ? this.totalResponseTime / this.requestsServed : 0, avgResponseTime: this.requestsServed > 0 ? this.totalResponseTime / this.requestsServed : 0, } } + getMethodMetrics() { + return this.requestsByMethod.getMethodMetrics() + } + private onNotification(msg: RpcNotification): void { this.notificationsReceived += 1 this.log?.debug({rpcMsg: msg}, 'rpc notification') @@ -391,17 +385,25 @@ export class RpcClient { let call = req.call this.log?.debug({rpcBatchId: [call[0].id, last(call).id]}, 'rpc send') promise = this.con.batchCall(call, req.timeout).then(res => { + this.requestsByMethod.recordBatch(call, res) let result = new Array(res.length) for (let i = 0; i < res.length; i++) { result[i] = this.receiveResult(call[i], res[i], req.validateResult, req.validateError) } return result + }, err => { + this.requestsByMethod.recordBatchError(call, err) + throw err }) } else { let call = req.call this.log?.debug({rpcId: call.id}, 'rpc send') promise = this.con.call(call, req.timeout).then(res => { + this.requestsByMethod.record(call.method, res) return this.receiveResult(call, res, req.validateResult, req.validateError) + }, err => { + this.requestsByMethod.recordError(call, err) + throw err }) } promise.then(result => { diff --git a/util/rpc-client/src/method-metrics.ts b/util/rpc-client/src/method-metrics.ts new file mode 100644 index 000000000..4d661f385 --- /dev/null +++ b/util/rpc-client/src/method-metrics.ts @@ -0,0 +1,64 @@ +import {HttpError} from '@subsquid/http-client' +import {RpcError, RpcProtocolError} from './errors' +import {RpcRequest, RpcResponse} from './interfaces' + + +/** + * Per-method request tally counted by batch element, keyed by a triplet of + * labels: RPC method name, HTTP status code, and JSON-RPC error code. + */ +export class MethodMetrics { + private items: Record>> = {} + + private inc(method: string, httpCode: string, rpcCode: string): void { + let byHttp = this.items[method] + if (!byHttp) byHttp = this.items[method] = {} + let byRpc = byHttp[httpCode] + if (!byRpc) byRpc = byHttp[httpCode] = {} + byRpc[rpcCode] = (byRpc[rpcCode] || 0) + 1 + } + + record(method: string, res: RpcResponse): void { + let rpcCode = res.error ? res.error.code.toString() : '' + this.inc(method, '200', rpcCode) + } + + recordBatch(calls: RpcRequest[], responses: RpcResponse[]): void { + for (let i = 0; i < calls.length; i++) { + this.record(calls[i].method, responses[i]) + } + } + + recordError(call: RpcRequest, err: unknown): void { + let {httpCode, rpcCode} = extractTransportLabels(err) + this.inc(call.method, httpCode, rpcCode) + } + + recordBatchError(calls: RpcRequest[], err: unknown): void { + let {httpCode, rpcCode} = extractTransportLabels(err) + for (let i = 0; i < calls.length; i++) { + this.inc(calls[i].method, httpCode, rpcCode) + } + } + + *getMethodMetrics() { + for (let [method, byHttp] of Object.entries(this.items)) { + for (let [httpCode, byRpc] of Object.entries(byHttp)) { + for (let [rpcCode, count] of Object.entries(byRpc)) { + yield {method, httpCode, rpcCode, count} + } + } + } + } +} + + +function extractTransportLabels(err: unknown) { + if (err instanceof HttpError) { + return {httpCode: err.response.status.toString(), rpcCode: ''} + } + if (err instanceof RpcError || err instanceof RpcProtocolError) { + return {httpCode: '200', rpcCode: err.code.toString()} + } + return {httpCode: '', rpcCode: ''} +} diff --git a/util/util-internal-dump-cli/src/prometheus.ts b/util/util-internal-dump-cli/src/prometheus.ts index 80bfe7a5a..a94987ae7 100644 --- a/util/util-internal-dump-cli/src/prometheus.ts +++ b/util/util-internal-dump-cli/src/prometheus.ts @@ -12,6 +12,7 @@ export class PrometheusServer { private rpcRequestsServedTotal: Counter private rpcAvgResponseTimeSeconds: Gauge private rpcConnectionErrorsTotal: Counter + private rpcMethodCallsTotal: Counter private s3RequestsCounter: Counter private latestReceivedBlockNumberGauge: Gauge private latestReceivedBlockTimestampGauge: Gauge @@ -126,11 +127,6 @@ export class PrometheusServer { registers: [this.registry], collect() { const metrics = rpc.getMetrics() - - this.set({ - url: metrics.url, - }, metrics.avg_response_time) - this.set({ url: metrics.url, }, metrics.avgResponseTime) @@ -150,6 +146,23 @@ export class PrometheusServer { } }); + this.rpcMethodCallsTotal = new Counter({ + name: 'sqd_rpc_method_calls_total', + help: 'Total number of RPC requests by method, HTTP code, and RPC code (counted by batch element)', + labelNames: ['method', 'http_code', 'rpc_code'], + registers: [this.registry], + collect() { + this.reset() + for (let item of rpc.getMethodMetrics()) { + this.inc({ + method: item.method, + http_code: item.httpCode, + rpc_code: item.rpcCode + }, item.count) + } + } + }); + this.s3RequestsCounter = new Counter({ name: 'sqd_s3_request_count', help: 'Number of s3 requests made',