From 0392cab9a450abb57af0a8a86f3015aefe48b713 Mon Sep 17 00:00:00 2001 From: simonredfern Date: Sat, 23 May 2026 15:43:33 +0200 Subject: [PATCH 1/3] Adding consent_reference_id to Metrics and Metrics Archive. Also in gRPC. --- .../src/main/protobuf/metrics_stream.proto | 4 + .../SwaggerDefinitionsJSON.scala | 3 +- .../main/scala/code/api/util/APIUtil.scala | 4 +- .../main/scala/code/api/util/ApiSession.scala | 10 +- .../scala/code/api/util/ConsentUtil.scala | 13 ++- .../main/scala/code/api/util/OBPParam.scala | 1 + .../scala/code/api/util/WriteMetricUtil.scala | 17 +-- .../code/api/util/migration/Migration.scala | 13 +++ .../MigrationOfMetricConsentReferenceId.scala | 101 ++++++++++++++++++ .../scala/code/api/v6_0_0/Http4s600.scala | 2 + .../code/api/v6_0_0/JSONFactory6.0.0.scala | 6 +- .../main/scala/code/metrics/APIMetrics.scala | 7 +- .../code/metrics/ElasticsearchMetrics.scala | 5 +- .../scala/code/metrics/MappedMetrics.scala | 32 ++++-- .../code/metrics/MetricBatchWriter.scala | 12 ++- .../MetricsStreamServiceImpl.scala | 6 +- .../grpc/metricsstream/api/MetricEvent.scala | 21 +++- .../api/MetricsStreamProto.scala | 2 + .../api/StreamMetricsRequest.scala | 21 +++- .../scheduler/MetricsArchiveScheduler.scala | 3 +- 20 files changed, 240 insertions(+), 43 deletions(-) create mode 100644 obp-api/src/main/scala/code/api/util/migration/MigrationOfMetricConsentReferenceId.scala diff --git a/obp-api/src/main/protobuf/metrics_stream.proto b/obp-api/src/main/protobuf/metrics_stream.proto index cf9c254dd3..a05a6a79d3 100644 --- a/obp-api/src/main/protobuf/metrics_stream.proto +++ b/obp-api/src/main/protobuf/metrics_stream.proto @@ -11,6 +11,7 @@ message StreamMetricsRequest { string url_substring = 4; string implemented_by_partial_function = 5; string app_name = 6; + string consent_reference_id = 7; } // Per-REST-call metric record, mirrors APIMetrics.saveMetric args and @@ -42,6 +43,9 @@ message MetricEvent { string api_instance_id = 16; // OBP operation id, e.g. "OBPv6.0.0-getBanks". Matches MetricJsonV600.operation_id. string operation_id = 17; + // Reference id of the consent (if any) that authorised the request. + // Mirrors MetricJsonV600.consent_reference_id (REST v6.0.0+). + string consent_reference_id = 18; } // Live tail of API metrics as they are written. diff --git a/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala b/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala index 6c9fd929fb..a6ec60c4da 100644 --- a/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala +++ b/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala @@ -3193,7 +3193,8 @@ object SwaggerDefinitionsJSON { response_body = json.parse("""{"code":401,"message":"OBP-20001: User not logged in. Authentication is required!"}"""), status_code = 401, operation_id = "OBPv4.0.0-getBanks", - api_instance_id = "obp_node_a" + api_instance_id = "obp_node_a", + consent_reference_id = Some(ExampleValue.consentReferenceIdExample.value) ) lazy val metricsJsonV600 = MetricsJsonV600( metrics = List(metricJsonV600) diff --git a/obp-api/src/main/scala/code/api/util/APIUtil.scala b/obp-api/src/main/scala/code/api/util/APIUtil.scala index 5e5e3e1838..3cb264cf40 100644 --- a/obp-api/src/main/scala/code/api/util/APIUtil.scala +++ b/obp-api/src/main/scala/code/api/util/APIUtil.scala @@ -1199,6 +1199,7 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ case "azp" => Full(OBPAzp(values.head)) case "iss" => Full(OBPIss(values.head)) case "consent_id" => Full(OBPConsentId(values.head)) + case "consent_reference_id" => Full(OBPConsentReferenceId(values.head)) case "user_id" => Full(OBPUserId(values.head)) case "provider_provider_id" => Full(ProviderProviderId(values.head)) case "bank_id" => Full(OBPBankId(values.head)) @@ -1333,6 +1334,7 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ val iss = getHttpRequestUrlParam(httpRequestUrl,"iss") val azp = getHttpRequestUrlParam(httpRequestUrl,"azp") val consentId = getHttpRequestUrlParam(httpRequestUrl,"consent_id") + val consentReferenceId = getHttpRequestUrlParam(httpRequestUrl,"consent_reference_id") val userId = getHttpRequestUrlParam(httpRequestUrl, "user_id") val providerProviderId = getHttpRequestUrlParam(httpRequestUrl, "provider_provider_id") val bankId = getHttpRequestUrlParam(httpRequestUrl, "bank_id") @@ -1368,7 +1370,7 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ Full(List( HTTPParam("sort_by",sortBy), HTTPParam("sort_direction",sortDirection), HTTPParam("from_date",fromDate), HTTPParam("to_date", toDate), HTTPParam("limit",limit), HTTPParam("offset",offset), - HTTPParam("anon", anon), HTTPParam("status", status), HTTPParam("consumer_id", consumerId), HTTPParam("azp", azp), HTTPParam("iss", iss), HTTPParam("consent_id", consentId), HTTPParam("user_id", userId), HTTPParam("provider_provider_id", providerProviderId), HTTPParam("url", url), HTTPParam("app_name", appName), + HTTPParam("anon", anon), HTTPParam("status", status), HTTPParam("consumer_id", consumerId), HTTPParam("azp", azp), HTTPParam("iss", iss), HTTPParam("consent_id", consentId), HTTPParam("consent_reference_id", consentReferenceId), HTTPParam("user_id", userId), HTTPParam("provider_provider_id", providerProviderId), HTTPParam("url", url), HTTPParam("app_name", appName), HTTPParam("implemented_by_partial_function",implementedByPartialFunction), HTTPParam("implemented_in_version",implementedInVersion), HTTPParam("verb", verb), HTTPParam("correlation_id", correlationId), HTTPParam("duration", duration), HTTPParam("exclude_app_names", excludeAppNames), HTTPParam("exclude_url_patterns", excludeUrlPattern),HTTPParam("exclude_implemented_by_partial_functions", excludeImplementedByPartialfunctions), diff --git a/obp-api/src/main/scala/code/api/util/ApiSession.scala b/obp-api/src/main/scala/code/api/util/ApiSession.scala index 4de0ce75b7..37e36b7ac3 100644 --- a/obp-api/src/main/scala/code/api/util/ApiSession.scala +++ b/obp-api/src/main/scala/code/api/util/ApiSession.scala @@ -58,7 +58,9 @@ case class CallContext( bank: Option[Bank] = None, bankAccount: Option[BankAccount] = None, view: Option[View] = None, - counterparty: Option[CounterpartyTrait] = None + counterparty: Option[CounterpartyTrait] = None, + // Set when the request is authenticated via a consent. Persisted on metric rows for search/audit. + consentReferenceId: Option[String] = None ) extends MdcLoggable { override def toString: String = SecureLogging.maskSensitive( s"${this.getClass.getSimpleName}(${this.productIterator.mkString(", ")})" @@ -144,7 +146,8 @@ case class CallContext( xRateLimitRemaining = this.xRateLimitRemaining, xRateLimitReset = this.xRateLimitReset, paginationOffset = this.paginationOffset, - paginationLimit = this.paginationLimit + paginationLimit = this.paginationLimit, + consentReferenceId = this.consentReferenceId ) } @@ -210,7 +213,8 @@ case class CallContextLight(gatewayLoginRequestPayload: Option[PayloadOfJwtJSON] xRateLimitRemaining : Long = -1, xRateLimitReset : Long = -1, paginationOffset : Option[String] = None, - paginationLimit : Option[String] = None + paginationLimit : Option[String] = None, + consentReferenceId: Option[String] = None ) trait LoginParam diff --git a/obp-api/src/main/scala/code/api/util/ConsentUtil.scala b/obp-api/src/main/scala/code/api/util/ConsentUtil.scala index ea6e99ca97..d6e88a3c04 100644 --- a/obp-api/src/main/scala/code/api/util/ConsentUtil.scala +++ b/obp-api/src/main/scala/code/api/util/ConsentUtil.scala @@ -451,12 +451,17 @@ object Consent extends MdcLoggable { def applyConsentRules(consent: ConsentJWT): Future[(Box[User], Option[CallContext])] = { val temp = callContext // updated context if createdByUserId is present - val cc = if (consent.createdByUserId.nonEmpty) { + val ccWithOnBehalf = if (consent.createdByUserId.nonEmpty) { val onBehalfOfUser = Users.users.vend.getUserByUserId(consent.createdByUserId) temp.copy(onBehalfOfUser = onBehalfOfUser.toOption) } else { temp } + // Stamp the consent_reference_id on the CallContext so the metric writer can record it. + val cc = Consents.consentProvider.vend.getConsentByConsentId(consent.jti) match { + case Full(mc) => ccWithOnBehalf.copy(consentReferenceId = Some(mc.consentReferenceId)) + case _ => ccWithOnBehalf + } if (cc.onBehalfOfUser.nonEmpty && APIUtil.getPropsAsBoolValue(nameOfProperty = "experimental_become_user_that_created_consent", defaultValue = false)) { logger.info("experimental_become_user_that_created_consent = true") @@ -552,7 +557,11 @@ object Consent extends MdcLoggable { implicit val dateFormats = CustomJsonFormats.formats def applyConsentRules(consent: ConsentJWT, callContext: CallContext): Future[(Box[User], Option[CallContext])] = { - val cc = callContext + // Stamp the consent_reference_id on the CallContext so the metric writer can record it. + val cc = Consents.consentProvider.vend.getConsentByConsentId(consent.jti) match { + case Full(mc) => callContext.copy(consentReferenceId = Some(mc.consentReferenceId)) + case _ => callContext + } // 1. Get or Create a User getOrCreateUser(consent.sub, consent.iss, Some(consent.toConsent().consentId), None, None) map { case (Full(user), newUser) => diff --git a/obp-api/src/main/scala/code/api/util/OBPParam.scala b/obp-api/src/main/scala/code/api/util/OBPParam.scala index 2e20724939..74f3ea4f16 100644 --- a/obp-api/src/main/scala/code/api/util/OBPParam.scala +++ b/obp-api/src/main/scala/code/api/util/OBPParam.scala @@ -30,6 +30,7 @@ case class OBPSortBy(value: String) extends OBPQueryParam case class OBPAzp(value: String) extends OBPQueryParam case class OBPIss(value: String) extends OBPQueryParam case class OBPConsentId(value: String) extends OBPQueryParam +case class OBPConsentReferenceId(value: String) extends OBPQueryParam case class OBPUserId(value: String) extends OBPQueryParam case class ProviderProviderId(value: String) extends OBPQueryParam case class OBPStatus(value: String) extends OBPQueryParam diff --git a/obp-api/src/main/scala/code/api/util/WriteMetricUtil.scala b/obp-api/src/main/scala/code/api/util/WriteMetricUtil.scala index 209a87f396..0f472095fb 100644 --- a/obp-api/src/main/scala/code/api/util/WriteMetricUtil.scala +++ b/obp-api/src/main/scala/code/api/util/WriteMetricUtil.scala @@ -83,11 +83,13 @@ object WriteMetricUtil extends MdcLoggable { responseBodyToWrite, sourceIp, targetIp, - code.api.Constant.ApiInstanceId + code.api.Constant.ApiInstanceId, + cc.consentReferenceId.orNull ) publishMetricEvent(userId, cc.url, cc.startTime.getOrElse(null), duration, userName, appName, developerEmail, consumerId, implementedByPartialFunction, cc.implementedInVersion, cc.verb, - cc.httpCode, cc.correlationId, sourceIp, targetIp, cc.operationId.getOrElse("")) + cc.httpCode, cc.correlationId, sourceIp, targetIp, cc.operationId.getOrElse(""), + cc.consentReferenceId.orNull) } } case _ => @@ -175,11 +177,12 @@ object WriteMetricUtil extends MdcLoggable { "Not enabled for old style endpoints", sourceIp, targetIp, - code.api.Constant.ApiInstanceId + code.api.Constant.ApiInstanceId, + null // Old-style endpoints don't thread consent_reference_id through S.session yet. ) publishMetricEvent(userId, url, date, duration, userName, appName, developerEmail, consumerId, implementedByPartialFunction, implementedInVersion, verb, None, correlationId, sourceIp, targetIp, - rd.map(_.operationId).getOrElse("")) + rd.map(_.operationId).getOrElse(""), null) } } @@ -207,7 +210,8 @@ object WriteMetricUtil extends MdcLoggable { correlationId: String, sourceIp: String, targetIp: String, - operationId: String): Unit = { + operationId: String, + consentReferenceId: String): Unit = { if (!MetricsEventBus.isEnabled) return try { implicit val fmts = metricFormats @@ -231,7 +235,8 @@ object WriteMetricUtil extends MdcLoggable { "source_ip" -> Option(sourceIp).getOrElse(""), "target_ip" -> Option(targetIp).getOrElse(""), "api_instance_id" -> code.api.Constant.ApiInstanceId, - "operation_id" -> Option(operationId).getOrElse("") + "operation_id" -> Option(operationId).getOrElse(""), + "consent_reference_id" -> Option(consentReferenceId).getOrElse("") )) MetricsEventBus.publish(payload) } catch { diff --git a/obp-api/src/main/scala/code/api/util/migration/Migration.scala b/obp-api/src/main/scala/code/api/util/migration/Migration.scala index 698a48edf8..26feb8bf43 100644 --- a/obp-api/src/main/scala/code/api/util/migration/Migration.scala +++ b/obp-api/src/main/scala/code/api/util/migration/Migration.scala @@ -119,6 +119,7 @@ object Migration extends MdcLoggable { migrateChatRoomIsOpenRoom() migrateChatRoomCreatedByAndLastMessageSender() migrateConsentReferenceIdToUuid(startedBeforeSchemifier) + migrateMetricConsentReferenceId(startedBeforeSchemifier) } private def dummyScript(): Boolean = { @@ -649,6 +650,18 @@ object Migration extends MdcLoggable { } } + private def migrateMetricConsentReferenceId(startedBeforeSchemifier: Boolean): Boolean = { + if(startedBeforeSchemifier == true) { + logger.warn(s"Migration.database.migrateMetricConsentReferenceId(true) cannot be run before Schemifier.") + true + } else { + val name = nameOf(migrateMetricConsentReferenceId(startedBeforeSchemifier)) + runOnce(name) { + MigrationOfMetricConsentReferenceId.migrate(name) + } + } + } + private def addAccountAccessWithViewsView(startedBeforeSchemifier: Boolean): Boolean = { if(startedBeforeSchemifier == true) { logger.warn(s"Migration.database.addAccountAccessWithViewsView(true) cannot be run before Schemifier.") diff --git a/obp-api/src/main/scala/code/api/util/migration/MigrationOfMetricConsentReferenceId.scala b/obp-api/src/main/scala/code/api/util/migration/MigrationOfMetricConsentReferenceId.scala new file mode 100644 index 0000000000..af75ea92df --- /dev/null +++ b/obp-api/src/main/scala/code/api/util/migration/MigrationOfMetricConsentReferenceId.scala @@ -0,0 +1,101 @@ +package code.api.util.migration + +import code.api.util.APIUtil +import code.api.util.migration.Migration.{DbFunction, saveLog} +import code.metrics.{MappedMetric, MetricArchive} +import net.liftweb.mapper.Schemifier + +/** + * Migration: add `consent_reference_id VARCHAR(36)` to both the live `Metric` table and + * the `metricarchive` table, plus an index on each for efficient search-by-consent. + * + * Backs up the live `Metric` table only (not metricarchive — the archive is itself a + * long-term backup of metrics, so duplicating it would be wasteful). + * + * No backfill: historical rows legitimately have no consent reference; nullable column. + * + * Lift's Schemifier auto-creates the column on fresh deploys from the updated model; + * this migration handles existing deploys. + */ +object MigrationOfMetricConsentReferenceId { + + def migrate(name: String): Boolean = { + DbFunction.tableExists(MappedMetric) match { + case true => + val startDate = System.currentTimeMillis() + val commitId: String = APIUtil.gitCommit + val dbDriver = APIUtil.getPropsValue("db.driver") openOr "org.h2.Driver" + val isMssql = dbDriver.contains("com.microsoft.sqlserver.jdbc.SQLServerDriver") + var isSuccessful = false + val sqlLog = new StringBuilder() + + try { + // 1. Backup of the live metric table (NOT the archive — it's already a long-term snapshot). + // Although MappedMetric.dbTableName is "Metric", Lift's Schemifier emits unquoted DDL, + // so Postgres folds the name to lowercase `metric`. Every other SQL site in the codebase + // (MetricBatchWriter INSERT, DoobieMetricsQueries, MigrationOfMetricView) references it + // as lowercase unquoted `metric` — mirror that here. MSSQL is case-insensitive by default. + val backupMetric = if (isMssql) { + "SELECT * INTO backup_2026_05_metric FROM metric;" + } else { + "CREATE TABLE backup_2026_05_metric AS SELECT * FROM metric;" + } + sqlLog.append(DbFunction.maybeWrite(true, Schemifier.infoF _)(() => backupMetric)).append("\n") + + // 2. Add the new column to the live metric table. + val addColumnMetric = if (isMssql) { + "ALTER TABLE metric ADD consent_reference_id VARCHAR(36) NULL;" + } else { + "ALTER TABLE metric ADD COLUMN IF NOT EXISTS consent_reference_id VARCHAR(36);" + } + sqlLog.append(DbFunction.maybeWrite(true, Schemifier.infoF _)(() => addColumnMetric)).append("\n") + + // 3. Add the new column to the archive table. + val addColumnArchive = if (isMssql) { + "ALTER TABLE metricarchive ADD consent_reference_id VARCHAR(36) NULL;" + } else { + "ALTER TABLE metricarchive ADD COLUMN IF NOT EXISTS consent_reference_id VARCHAR(36);" + } + sqlLog.append(DbFunction.maybeWrite(true, Schemifier.infoF _)(() => addColumnArchive)).append("\n") + + // 4. Index for search-by-consent on both tables. + val indexMetric = if (isMssql) { + "CREATE INDEX idx_metric_consent_reference_id ON metric(consent_reference_id);" + } else { + "CREATE INDEX IF NOT EXISTS idx_metric_consent_reference_id ON metric(consent_reference_id);" + } + sqlLog.append(DbFunction.maybeWrite(true, Schemifier.infoF _)(() => indexMetric)).append("\n") + + val indexArchive = if (isMssql) { + "CREATE INDEX idx_metricarchive_consent_reference_id ON metricarchive(consent_reference_id);" + } else { + "CREATE INDEX IF NOT EXISTS idx_metricarchive_consent_reference_id ON metricarchive(consent_reference_id);" + } + sqlLog.append(DbFunction.maybeWrite(true, Schemifier.infoF _)(() => indexArchive)).append("\n") + + isSuccessful = true + } catch { + case e: Exception => + isSuccessful = false + sqlLog.append(s"\nException: ${e.getMessage}\n") + } + + val endDate = System.currentTimeMillis() + val comment: String = + s"""Executed SQL: + |$sqlLog + |""".stripMargin + saveLog(name, commitId, isSuccessful, startDate, endDate, comment) + isSuccessful + + case false => + val startDate = System.currentTimeMillis() + val commitId: String = APIUtil.gitCommit + val isSuccessful = false + val endDate = System.currentTimeMillis() + val comment: String = s"""${MappedMetric._dbTableNameLC} table does not exist""".stripMargin + saveLog(name, commitId, isSuccessful, startDate, endDate, comment) + isSuccessful + } + } +} diff --git a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala index 5d00fb7c6e..910cf9ddcb 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala @@ -7227,6 +7227,8 @@ object Http4s600 { | |16 duration (if null ignore) - Returns calls where duration > specified value (in milliseconds). Use this to find slow API calls. eg: duration=5000 returns calls taking more than 5 seconds | + |17 consent_reference_id (if null ignore) - Returns calls authenticated via the consent with this reference id. eg: consent_reference_id=fd13b9af-4f74-4d52-a7f1-7c2c12f3aa11 + | """.stripMargin, EmptyBody, metricsJsonV600, diff --git a/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala b/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala index bb7c3f1172..9e6515011e 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala @@ -448,7 +448,8 @@ case class MetricJsonV600( response_body: net.liftweb.json.JValue, status_code: Int, operation_id: String, - api_instance_id: String + api_instance_id: String, + consent_reference_id: Option[String] ) case class MetricsJsonV600(metrics: List[MetricJsonV600]) @@ -1687,7 +1688,8 @@ object JSONFactory600 extends CustomJsonFormats with MdcLoggable { response_body = net.liftweb.json.parseOpt(metric.getResponseBody()).getOrElse(net.liftweb.json.JString("Not enabled")), status_code = metric.getHttpCode(), operation_id = operationId, - api_instance_id = metric.getApiInstanceId() + api_instance_id = metric.getApiInstanceId(), + consent_reference_id = Option(metric.getConsentReferenceId()).filter(_.nonEmpty) ) } diff --git a/obp-api/src/main/scala/code/metrics/APIMetrics.scala b/obp-api/src/main/scala/code/metrics/APIMetrics.scala index 392cd23eb9..3376aa161d 100644 --- a/obp-api/src/main/scala/code/metrics/APIMetrics.scala +++ b/obp-api/src/main/scala/code/metrics/APIMetrics.scala @@ -55,7 +55,8 @@ trait APIMetrics { responseBody: String, sourceIp: String, targetIp: String, - apiInstanceId: String): Unit + apiInstanceId: String, + consentReferenceId: String): Unit def saveMetricsArchive(primaryKey: Long, userId: String, @@ -74,7 +75,8 @@ trait APIMetrics { responseBody: String, sourceIp: String, targetIp: String, - apiInstanceId: String + apiInstanceId: String, + consentReferenceId: String ): Unit // //TODO: ordering of list? should this be by date? currently not enforced @@ -124,6 +126,7 @@ trait APIMetric { def getSourceIp(): String def getTargetIp(): String def getApiInstanceId(): String + def getConsentReferenceId(): String } diff --git a/obp-api/src/main/scala/code/metrics/ElasticsearchMetrics.scala b/obp-api/src/main/scala/code/metrics/ElasticsearchMetrics.scala index 6e7b4a0d87..daf0d95163 100644 --- a/obp-api/src/main/scala/code/metrics/ElasticsearchMetrics.scala +++ b/obp-api/src/main/scala/code/metrics/ElasticsearchMetrics.scala @@ -14,7 +14,7 @@ object ElasticsearchMetrics extends APIMetrics { lazy val es = new elasticsearchMetrics override def saveMetric(userId: String, url: String, date: Date, duration: Long, userName: String, appName: String, developerEmail: String, consumerId: String, implementedByPartialFunction: String, implementedInVersion: String, verb: String, httpCode: Option[Int], correlationId: String, - responseBody: String, sourceIp: String, targetIp: String, apiInstanceId: String): Unit = { + responseBody: String, sourceIp: String, targetIp: String, apiInstanceId: String, consentReferenceId: String): Unit = { if (APIUtil.getPropsAsBoolValue("allow_elasticsearch", false) && APIUtil.getPropsAsBoolValue("allow_elasticsearch_metrics", false) ) { //TODO ,need to be fixed now add more parameters es.indexMetric(userId, url, date, duration, userName, appName, developerEmail, correlationId, apiInstanceId) @@ -24,7 +24,8 @@ object ElasticsearchMetrics extends APIMetrics { responseBody: String, sourceIp: String, targetIp: String, - apiInstanceId: String): Unit = ??? + apiInstanceId: String, + consentReferenceId: String): Unit = ??? // override def getAllGroupedByUserId(): Map[String, List[APIMetric]] = { // //TODO: replace the following with valid ES query diff --git a/obp-api/src/main/scala/code/metrics/MappedMetrics.scala b/obp-api/src/main/scala/code/metrics/MappedMetrics.scala index 3c78d70809..f52986de21 100644 --- a/obp-api/src/main/scala/code/metrics/MappedMetrics.scala +++ b/obp-api/src/main/scala/code/metrics/MappedMetrics.scala @@ -113,7 +113,7 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ } override def saveMetric(userId: String, url: String, date: Date, duration: Long, userName: String, appName: String, developerEmail: String, consumerId: String, implementedByPartialFunction: String, implementedInVersion: String, verb: String, httpCode: Option[Int], correlationId: String, - responseBody: String, sourceIp: String, targetIp: String, apiInstanceId: String): Unit = { + responseBody: String, sourceIp: String, targetIp: String, apiInstanceId: String, consentReferenceId: String): Unit = { MetricBatchWriter.enqueue( MetricBatchWriter.MetricRow( userId = userId, @@ -132,7 +132,8 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ responseBody = responseBody, sourceIp = sourceIp, targetIp = targetIp, - apiInstanceId = apiInstanceId + apiInstanceId = apiInstanceId, + consentReferenceId = consentReferenceId ) ) } @@ -142,7 +143,7 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ implementedByPartialFunction: String, implementedInVersion: String, verb: String, httpCode: Option[Int], correlationId: String, responseBody: String, sourceIp: String, targetIp: String, - apiInstanceId: String): Unit = { + apiInstanceId: String, consentReferenceId: String): Unit = { val metric = MetricArchive.find(By(MetricArchive.id, primaryKey)).getOrElse(MetricArchive.create) metric .metricId(primaryKey) @@ -162,6 +163,7 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ .sourceIp(sourceIp) .targetIp(targetIp) .apiInstanceId(apiInstanceId) + .consentReferenceId(consentReferenceId) httpCode match { case Some(code) => metric.httpCode(code) @@ -261,6 +263,7 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ val correlationId = queryParams.collect { case OBPCorrelationId(value) => By(MappedMetric.correlationId, value) }.headOption val duration = queryParams.collect { case OBPDuration(value) => By_>(MappedMetric.duration, value) }.headOption val httpStatusCode = queryParams.collect { case OBPHttpStatusCode(value) => By(MappedMetric.httpCode, value) }.headOption + val consentReferenceId = queryParams.collect { case OBPConsentReferenceId(value) => By(MappedMetric.consentReferenceId, value) }.headOption val anon = queryParams.collect { case OBPAnon(true) => By(MappedMetric.userId, "null") case OBPAnon(false) => NotBy(MappedMetric.userId, "null") @@ -287,6 +290,7 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ correlationId.toSeq, duration.toSeq, httpStatusCode.toSeq, + consentReferenceId.toSeq, anon.toSeq, excludeAppNames.toSeq.flatten ).flatten @@ -644,6 +648,11 @@ class MappedMetric extends APIMetric with LongKeyedMapper[MappedMetric] with IdP object sourceIp extends MappedString(this, 64) object targetIp extends MappedString(this, 64) object apiInstanceId extends MappedString(this, 255) + // Set when the request was authenticated via a consent. Null otherwise. + object consentReferenceId extends MappedString(this, 36) { + override def dbColumnName = "consent_reference_id" + override def defaultValue = null + } override def getMetricId(): Long = id.get override def getUrl(): String = url.get @@ -663,16 +672,17 @@ class MappedMetric extends APIMetric with LongKeyedMapper[MappedMetric] with IdP override def getSourceIp(): String = sourceIp.get override def getTargetIp(): String = targetIp.get override def getApiInstanceId(): String = apiInstanceId.get + override def getConsentReferenceId(): String = consentReferenceId.get } object MappedMetric extends MappedMetric with LongKeyedMetaMapper[MappedMetric] { // Please note that the old table name was "MappedMetric" // Renaming implications: // - at an existing sandbox the table "MappedMetric" still exists with rows until this change is deployed at it - // and new rows are stored in the table "Metric" + // and new rows are stored in the table "Metric" // - at a fresh sandbox there is no the table "MappedMetric", only "Metric" is present override def dbTableName = "Metric" // define the DB table name - override def dbIndexes = Index(date) :: Index(consumerId) :: super.dbIndexes + override def dbIndexes = Index(date) :: Index(consumerId) :: Index(consentReferenceId) :: super.dbIndexes } @@ -704,6 +714,11 @@ class MetricArchive extends APIMetric with LongKeyedMapper[MetricArchive] with I object sourceIp extends MappedString(this, 64) object targetIp extends MappedString(this, 64) object apiInstanceId extends MappedString(this, 255) + // Set when the request was authenticated via a consent. Null otherwise. + object consentReferenceId extends MappedString(this, 36) { + override def dbColumnName = "consent_reference_id" + override def defaultValue = null + } override def getMetricId(): Long = metricId.get @@ -724,9 +739,10 @@ class MetricArchive extends APIMetric with LongKeyedMapper[MetricArchive] with I override def getSourceIp(): String = sourceIp.get override def getTargetIp(): String = targetIp.get override def getApiInstanceId(): String = apiInstanceId.get + override def getConsentReferenceId(): String = consentReferenceId.get } object MetricArchive extends MetricArchive with LongKeyedMetaMapper[MetricArchive] { - override def dbIndexes = - Index(userId) :: Index(consumerId) :: Index(url) :: Index(date) :: Index(userName) :: - Index(appName) :: Index(developerEmail) :: super.dbIndexes + override def dbIndexes = + Index(userId) :: Index(consumerId) :: Index(url) :: Index(date) :: Index(userName) :: + Index(appName) :: Index(developerEmail) :: Index(consentReferenceId) :: super.dbIndexes } diff --git a/obp-api/src/main/scala/code/metrics/MetricBatchWriter.scala b/obp-api/src/main/scala/code/metrics/MetricBatchWriter.scala index ff13b02d4b..d65a7e8b41 100644 --- a/obp-api/src/main/scala/code/metrics/MetricBatchWriter.scala +++ b/obp-api/src/main/scala/code/metrics/MetricBatchWriter.scala @@ -40,7 +40,8 @@ object MetricBatchWriter extends MdcLoggable { responseBody: String, sourceIp: String, targetIp: String, - apiInstanceId: String + apiInstanceId: String, + consentReferenceId: String ) private val queue = new ConcurrentLinkedQueue[MetricRow]() @@ -102,8 +103,8 @@ object MetricBatchWriter extends MdcLoggable { userid, url, date_c, duration, username, appname, developeremail, consumerid, implementedbypartialfunction, implementedinversion, verb, httpcode, correlationid, - responsebody, sourceip, targetip, apiinstanceid - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + responsebody, sourceip, targetip, apiinstanceid, consent_reference_id + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """ // Use Option[String] so Doobie handles nullable fields via Put[Option[String]] @@ -112,7 +113,7 @@ object MetricBatchWriter extends MdcLoggable { (Option[String], Option[String], Timestamp, Long, Option[String], Option[String], Option[String], Option[String], Option[String], Option[String], Option[String], Int, Option[String], - Option[String], Option[String], Option[String], Option[String]) + Option[String], Option[String], Option[String], Option[String], Option[String]) ](insertSql) val values = rows.map { r => @@ -121,7 +122,8 @@ object MetricBatchWriter extends MdcLoggable { r.duration, Option(r.userName), Option(r.appName), Option(r.developerEmail), Option(r.consumerId), Option(r.implementedByPartialFunction), Option(r.implementedInVersion), Option(r.verb), r.httpCode, Option(r.correlationId), - Option(r.responseBody), Option(r.sourceIp), Option(r.targetIp), Option(r.apiInstanceId) + Option(r.responseBody), Option(r.sourceIp), Option(r.targetIp), Option(r.apiInstanceId), + Option(r.consentReferenceId) ) } diff --git a/obp-api/src/main/scala/code/obp/grpc/metricsstream/MetricsStreamServiceImpl.scala b/obp-api/src/main/scala/code/obp/grpc/metricsstream/MetricsStreamServiceImpl.scala index f70f9c3d5f..502037d7a7 100644 --- a/obp-api/src/main/scala/code/obp/grpc/metricsstream/MetricsStreamServiceImpl.scala +++ b/obp-api/src/main/scala/code/obp/grpc/metricsstream/MetricsStreamServiceImpl.scala @@ -91,7 +91,8 @@ object MetricsStreamServiceImpl extends MetricsStreamServiceGrpc.MetricsStreamSe matchExact(req.verb, (jv \ "verb").extractOrElse[String]("")) && matchSubstring(req.urlSubstring, (jv \ "url").extractOrElse[String]("")) && matchExact(req.implementedByPartialFunction, (jv \ "implemented_by_partial_function").extractOrElse[String]("")) && - matchExact(req.appName, (jv \ "app_name").extractOrElse[String]("")) + matchExact(req.appName, (jv \ "app_name").extractOrElse[String]("")) && + matchExact(req.consentReferenceId, (jv \ "consent_reference_id").extractOrElse[String]("")) } private def jsonToMetricEvent(jv: JValue): MetricEvent = { @@ -112,7 +113,8 @@ object MetricsStreamServiceImpl extends MetricsStreamServiceGrpc.MetricsStreamSe sourceIp = (jv \ "source_ip").extractOrElse[String](""), targetIp = (jv \ "target_ip").extractOrElse[String](""), apiInstanceId = (jv \ "api_instance_id").extractOrElse[String](""), - operationId = (jv \ "operation_id").extractOrElse[String]("") + operationId = (jv \ "operation_id").extractOrElse[String](""), + consentReferenceId = (jv \ "consent_reference_id").extractOrElse[String]("") ) } } diff --git a/obp-api/src/main/scala/code/obp/grpc/metricsstream/api/MetricEvent.scala b/obp-api/src/main/scala/code/obp/grpc/metricsstream/api/MetricEvent.scala index 779043c206..391a2d0ec9 100644 --- a/obp-api/src/main/scala/code/obp/grpc/metricsstream/api/MetricEvent.scala +++ b/obp-api/src/main/scala/code/obp/grpc/metricsstream/api/MetricEvent.scala @@ -23,7 +23,8 @@ final case class MetricEvent( sourceIp: _root_.scala.Predef.String = "", targetIp: _root_.scala.Predef.String = "", apiInstanceId: _root_.scala.Predef.String = "", - operationId: _root_.scala.Predef.String = "" + operationId: _root_.scala.Predef.String = "", + consentReferenceId: _root_.scala.Predef.String = "" ) extends scalapb.GeneratedMessage with scalapb.Message[MetricEvent] with scalapb.lenses.Updatable[MetricEvent] { @transient private[this] var __serializedSizeCachedValue: _root_.scala.Int = 0 @@ -46,6 +47,7 @@ final case class MetricEvent( if (targetIp != "") { __size += _root_.com.google.protobuf.CodedOutputStream.computeStringSize(15, targetIp) } if (apiInstanceId != "") { __size += _root_.com.google.protobuf.CodedOutputStream.computeStringSize(16, apiInstanceId) } if (operationId != "") { __size += _root_.com.google.protobuf.CodedOutputStream.computeStringSize(17, operationId) } + if (consentReferenceId != "") { __size += _root_.com.google.protobuf.CodedOutputStream.computeStringSize(18, consentReferenceId) } __size } final override def serializedSize: _root_.scala.Int = { @@ -74,6 +76,7 @@ final case class MetricEvent( { val __v = targetIp; if (__v != "") _output__.writeString(15, __v) }; { val __v = apiInstanceId; if (__v != "") _output__.writeString(16, __v) }; { val __v = operationId; if (__v != "") _output__.writeString(17, __v) }; + { val __v = consentReferenceId; if (__v != "") _output__.writeString(18, __v) }; } def mergeFrom(`_input__`: _root_.com.google.protobuf.CodedInputStream): code.obp.grpc.metricsstream.api.MetricEvent = { var __url = this.url @@ -93,6 +96,7 @@ final case class MetricEvent( var __targetIp = this.targetIp var __apiInstanceId = this.apiInstanceId var __operationId = this.operationId + var __consentReferenceId = this.consentReferenceId var _done__ = false while (!_done__) { val _tag__ = _input__.readTag() @@ -115,6 +119,7 @@ final case class MetricEvent( case 122 => __targetIp = _input__.readString() case 130 => __apiInstanceId = _input__.readString() case 138 => __operationId = _input__.readString() + case 146 => __consentReferenceId = _input__.readString() case tag => _input__.skipField(tag) } } @@ -135,7 +140,8 @@ final case class MetricEvent( sourceIp = __sourceIp, targetIp = __targetIp, apiInstanceId = __apiInstanceId, - operationId = __operationId + operationId = __operationId, + consentReferenceId = __consentReferenceId ) } def withUrl(__v: _root_.scala.Predef.String): MetricEvent = copy(url = __v) @@ -155,6 +161,7 @@ final case class MetricEvent( def withTargetIp(__v: _root_.scala.Predef.String): MetricEvent = copy(targetIp = __v) def withApiInstanceId(__v: _root_.scala.Predef.String): MetricEvent = copy(apiInstanceId = __v) def withOperationId(__v: _root_.scala.Predef.String): MetricEvent = copy(operationId = __v) + def withConsentReferenceId(__v: _root_.scala.Predef.String): MetricEvent = copy(consentReferenceId = __v) def getFieldByNumber(__fieldNumber: _root_.scala.Int): scala.Any = { (__fieldNumber: @_root_.scala.unchecked) match { case 1 => { val __t = url; if (__t != "") __t else null } @@ -174,6 +181,7 @@ final case class MetricEvent( case 15 => { val __t = targetIp; if (__t != "") __t else null } case 16 => { val __t = apiInstanceId; if (__t != "") __t else null } case 17 => { val __t = operationId; if (__t != "") __t else null } + case 18 => { val __t = consentReferenceId; if (__t != "") __t else null } } } def getField(__field: _root_.scalapb.descriptors.FieldDescriptor): _root_.scalapb.descriptors.PValue = { @@ -196,6 +204,7 @@ final case class MetricEvent( case 15 => _root_.scalapb.descriptors.PString(targetIp) case 16 => _root_.scalapb.descriptors.PString(apiInstanceId) case 17 => _root_.scalapb.descriptors.PString(operationId) + case 18 => _root_.scalapb.descriptors.PString(consentReferenceId) } } def toProtoString: _root_.scala.Predef.String = _root_.scalapb.TextFormat.printToUnicodeString(this) @@ -224,7 +233,8 @@ object MetricEvent extends scalapb.GeneratedMessageCompanion[code.obp.grpc.metri __fieldsMap.getOrElse(__fields.get(13), "").asInstanceOf[_root_.scala.Predef.String], __fieldsMap.getOrElse(__fields.get(14), "").asInstanceOf[_root_.scala.Predef.String], __fieldsMap.getOrElse(__fields.get(15), "").asInstanceOf[_root_.scala.Predef.String], - __fieldsMap.getOrElse(__fields.get(16), "").asInstanceOf[_root_.scala.Predef.String] + __fieldsMap.getOrElse(__fields.get(16), "").asInstanceOf[_root_.scala.Predef.String], + __fieldsMap.getOrElse(__fields.get(17), "").asInstanceOf[_root_.scala.Predef.String] ) } implicit def messageReads: _root_.scalapb.descriptors.Reads[code.obp.grpc.metricsstream.api.MetricEvent] = _root_.scalapb.descriptors.Reads{ @@ -247,7 +257,8 @@ object MetricEvent extends scalapb.GeneratedMessageCompanion[code.obp.grpc.metri __fieldsMap.get(scalaDescriptor.findFieldByNumber(14).get).map(_.as[_root_.scala.Predef.String]).getOrElse(""), __fieldsMap.get(scalaDescriptor.findFieldByNumber(15).get).map(_.as[_root_.scala.Predef.String]).getOrElse(""), __fieldsMap.get(scalaDescriptor.findFieldByNumber(16).get).map(_.as[_root_.scala.Predef.String]).getOrElse(""), - __fieldsMap.get(scalaDescriptor.findFieldByNumber(17).get).map(_.as[_root_.scala.Predef.String]).getOrElse("") + __fieldsMap.get(scalaDescriptor.findFieldByNumber(17).get).map(_.as[_root_.scala.Predef.String]).getOrElse(""), + __fieldsMap.get(scalaDescriptor.findFieldByNumber(18).get).map(_.as[_root_.scala.Predef.String]).getOrElse("") ) case _ => throw new RuntimeException("Expected PMessage") } @@ -275,6 +286,7 @@ object MetricEvent extends scalapb.GeneratedMessageCompanion[code.obp.grpc.metri def targetIp: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Predef.String] = field(_.targetIp)((c_, f_) => c_.copy(targetIp = f_)) def apiInstanceId: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Predef.String] = field(_.apiInstanceId)((c_, f_) => c_.copy(apiInstanceId = f_)) def operationId: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Predef.String] = field(_.operationId)((c_, f_) => c_.copy(operationId = f_)) + def consentReferenceId: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Predef.String] = field(_.consentReferenceId)((c_, f_) => c_.copy(consentReferenceId = f_)) } final val URL_FIELD_NUMBER = 1 final val DATE_FIELD_NUMBER = 2 @@ -293,4 +305,5 @@ object MetricEvent extends scalapb.GeneratedMessageCompanion[code.obp.grpc.metri final val TARGET_IP_FIELD_NUMBER = 15 final val API_INSTANCE_ID_FIELD_NUMBER = 16 final val OPERATION_ID_FIELD_NUMBER = 17 + final val CONSENT_REFERENCE_ID_FIELD_NUMBER = 18 } diff --git a/obp-api/src/main/scala/code/obp/grpc/metricsstream/api/MetricsStreamProto.scala b/obp-api/src/main/scala/code/obp/grpc/metricsstream/api/MetricsStreamProto.scala index 0b75ed51ae..9e8f0e5d40 100644 --- a/obp-api/src/main/scala/code/obp/grpc/metricsstream/api/MetricsStreamProto.scala +++ b/obp-api/src/main/scala/code/obp/grpc/metricsstream/api/MetricsStreamProto.scala @@ -23,6 +23,7 @@ object MetricsStreamProto { .addField(stringField("url_substring", 4)) .addField(stringField("implemented_by_partial_function", 5)) .addField(stringField("app_name", 6)) + .addField(stringField("consent_reference_id", 7)) ) // MetricEvent .addMessageType(DescriptorProto.newBuilder() @@ -44,6 +45,7 @@ object MetricsStreamProto { .addField(stringField("target_ip", 15)) .addField(stringField("api_instance_id", 16)) .addField(stringField("operation_id", 17)) + .addField(stringField("consent_reference_id", 18)) ) // MetricsStreamService .addService(ServiceDescriptorProto.newBuilder() diff --git a/obp-api/src/main/scala/code/obp/grpc/metricsstream/api/StreamMetricsRequest.scala b/obp-api/src/main/scala/code/obp/grpc/metricsstream/api/StreamMetricsRequest.scala index edf1c27415..2e5b38a0cb 100644 --- a/obp-api/src/main/scala/code/obp/grpc/metricsstream/api/StreamMetricsRequest.scala +++ b/obp-api/src/main/scala/code/obp/grpc/metricsstream/api/StreamMetricsRequest.scala @@ -13,7 +13,8 @@ final case class StreamMetricsRequest( verb: _root_.scala.Predef.String = "", urlSubstring: _root_.scala.Predef.String = "", implementedByPartialFunction: _root_.scala.Predef.String = "", - appName: _root_.scala.Predef.String = "" + appName: _root_.scala.Predef.String = "", + consentReferenceId: _root_.scala.Predef.String = "" ) extends scalapb.GeneratedMessage with scalapb.Message[StreamMetricsRequest] with scalapb.lenses.Updatable[StreamMetricsRequest] { @transient private[this] var __serializedSizeCachedValue: _root_.scala.Int = 0 @@ -25,6 +26,7 @@ final case class StreamMetricsRequest( if (urlSubstring != "") { __size += _root_.com.google.protobuf.CodedOutputStream.computeStringSize(4, urlSubstring) } if (implementedByPartialFunction != "") { __size += _root_.com.google.protobuf.CodedOutputStream.computeStringSize(5, implementedByPartialFunction) } if (appName != "") { __size += _root_.com.google.protobuf.CodedOutputStream.computeStringSize(6, appName) } + if (consentReferenceId != "") { __size += _root_.com.google.protobuf.CodedOutputStream.computeStringSize(7, consentReferenceId) } __size } final override def serializedSize: _root_.scala.Int = { @@ -42,6 +44,7 @@ final case class StreamMetricsRequest( { val __v = urlSubstring; if (__v != "") _output__.writeString(4, __v) }; { val __v = implementedByPartialFunction; if (__v != "") _output__.writeString(5, __v) }; { val __v = appName; if (__v != "") _output__.writeString(6, __v) }; + { val __v = consentReferenceId; if (__v != "") _output__.writeString(7, __v) }; } def mergeFrom(`_input__`: _root_.com.google.protobuf.CodedInputStream): code.obp.grpc.metricsstream.api.StreamMetricsRequest = { var __consumerId = this.consumerId @@ -50,6 +53,7 @@ final case class StreamMetricsRequest( var __urlSubstring = this.urlSubstring var __implementedByPartialFunction = this.implementedByPartialFunction var __appName = this.appName + var __consentReferenceId = this.consentReferenceId var _done__ = false while (!_done__) { val _tag__ = _input__.readTag() @@ -61,6 +65,7 @@ final case class StreamMetricsRequest( case 34 => __urlSubstring = _input__.readString() case 42 => __implementedByPartialFunction = _input__.readString() case 50 => __appName = _input__.readString() + case 58 => __consentReferenceId = _input__.readString() case tag => _input__.skipField(tag) } } @@ -70,7 +75,8 @@ final case class StreamMetricsRequest( verb = __verb, urlSubstring = __urlSubstring, implementedByPartialFunction = __implementedByPartialFunction, - appName = __appName + appName = __appName, + consentReferenceId = __consentReferenceId ) } def withConsumerId(__v: _root_.scala.Predef.String): StreamMetricsRequest = copy(consumerId = __v) @@ -79,6 +85,7 @@ final case class StreamMetricsRequest( def withUrlSubstring(__v: _root_.scala.Predef.String): StreamMetricsRequest = copy(urlSubstring = __v) def withImplementedByPartialFunction(__v: _root_.scala.Predef.String): StreamMetricsRequest = copy(implementedByPartialFunction = __v) def withAppName(__v: _root_.scala.Predef.String): StreamMetricsRequest = copy(appName = __v) + def withConsentReferenceId(__v: _root_.scala.Predef.String): StreamMetricsRequest = copy(consentReferenceId = __v) def getFieldByNumber(__fieldNumber: _root_.scala.Int): scala.Any = { (__fieldNumber: @_root_.scala.unchecked) match { case 1 => { val __t = consumerId; if (__t != "") __t else null } @@ -87,6 +94,7 @@ final case class StreamMetricsRequest( case 4 => { val __t = urlSubstring; if (__t != "") __t else null } case 5 => { val __t = implementedByPartialFunction; if (__t != "") __t else null } case 6 => { val __t = appName; if (__t != "") __t else null } + case 7 => { val __t = consentReferenceId; if (__t != "") __t else null } } } def getField(__field: _root_.scalapb.descriptors.FieldDescriptor): _root_.scalapb.descriptors.PValue = { @@ -98,6 +106,7 @@ final case class StreamMetricsRequest( case 4 => _root_.scalapb.descriptors.PString(urlSubstring) case 5 => _root_.scalapb.descriptors.PString(implementedByPartialFunction) case 6 => _root_.scalapb.descriptors.PString(appName) + case 7 => _root_.scalapb.descriptors.PString(consentReferenceId) } } def toProtoString: _root_.scala.Predef.String = _root_.scalapb.TextFormat.printToUnicodeString(this) @@ -115,7 +124,8 @@ object StreamMetricsRequest extends scalapb.GeneratedMessageCompanion[code.obp.g __fieldsMap.getOrElse(__fields.get(2), "").asInstanceOf[_root_.scala.Predef.String], __fieldsMap.getOrElse(__fields.get(3), "").asInstanceOf[_root_.scala.Predef.String], __fieldsMap.getOrElse(__fields.get(4), "").asInstanceOf[_root_.scala.Predef.String], - __fieldsMap.getOrElse(__fields.get(5), "").asInstanceOf[_root_.scala.Predef.String] + __fieldsMap.getOrElse(__fields.get(5), "").asInstanceOf[_root_.scala.Predef.String], + __fieldsMap.getOrElse(__fields.get(6), "").asInstanceOf[_root_.scala.Predef.String] ) } implicit def messageReads: _root_.scalapb.descriptors.Reads[code.obp.grpc.metricsstream.api.StreamMetricsRequest] = _root_.scalapb.descriptors.Reads{ @@ -127,7 +137,8 @@ object StreamMetricsRequest extends scalapb.GeneratedMessageCompanion[code.obp.g __fieldsMap.get(scalaDescriptor.findFieldByNumber(3).get).map(_.as[_root_.scala.Predef.String]).getOrElse(""), __fieldsMap.get(scalaDescriptor.findFieldByNumber(4).get).map(_.as[_root_.scala.Predef.String]).getOrElse(""), __fieldsMap.get(scalaDescriptor.findFieldByNumber(5).get).map(_.as[_root_.scala.Predef.String]).getOrElse(""), - __fieldsMap.get(scalaDescriptor.findFieldByNumber(6).get).map(_.as[_root_.scala.Predef.String]).getOrElse("") + __fieldsMap.get(scalaDescriptor.findFieldByNumber(6).get).map(_.as[_root_.scala.Predef.String]).getOrElse(""), + __fieldsMap.get(scalaDescriptor.findFieldByNumber(7).get).map(_.as[_root_.scala.Predef.String]).getOrElse("") ) case _ => throw new RuntimeException("Expected PMessage") } @@ -144,6 +155,7 @@ object StreamMetricsRequest extends scalapb.GeneratedMessageCompanion[code.obp.g def urlSubstring: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Predef.String] = field(_.urlSubstring)((c_, f_) => c_.copy(urlSubstring = f_)) def implementedByPartialFunction: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Predef.String] = field(_.implementedByPartialFunction)((c_, f_) => c_.copy(implementedByPartialFunction = f_)) def appName: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Predef.String] = field(_.appName)((c_, f_) => c_.copy(appName = f_)) + def consentReferenceId: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Predef.String] = field(_.consentReferenceId)((c_, f_) => c_.copy(consentReferenceId = f_)) } final val CONSUMER_ID_FIELD_NUMBER = 1 final val USER_ID_FIELD_NUMBER = 2 @@ -151,4 +163,5 @@ object StreamMetricsRequest extends scalapb.GeneratedMessageCompanion[code.obp.g final val URL_SUBSTRING_FIELD_NUMBER = 4 final val IMPLEMENTED_BY_PARTIAL_FUNCTION_FIELD_NUMBER = 5 final val APP_NAME_FIELD_NUMBER = 6 + final val CONSENT_REFERENCE_ID_FIELD_NUMBER = 7 } diff --git a/obp-api/src/main/scala/code/scheduler/MetricsArchiveScheduler.scala b/obp-api/src/main/scala/code/scheduler/MetricsArchiveScheduler.scala index 053e963e40..f25c3847fb 100644 --- a/obp-api/src/main/scala/code/scheduler/MetricsArchiveScheduler.scala +++ b/obp-api/src/main/scala/code/scheduler/MetricsArchiveScheduler.scala @@ -133,7 +133,8 @@ object MetricsArchiveScheduler extends MdcLoggable { i.getResponseBody(), i.getSourceIp(), i.getTargetIp(), - i.getApiInstanceId() + i.getApiInstanceId(), + i.getConsentReferenceId() ) } From ec14e34b08cb77a652690ef43087bb092a954c0c Mon Sep 17 00:00:00 2001 From: simonredfern Date: Sat, 23 May 2026 17:32:46 +0200 Subject: [PATCH 2/3] Fix metrics test --- .../test/scala/code/metrics/MetricsTest.scala | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/obp-api/src/test/scala/code/metrics/MetricsTest.scala b/obp-api/src/test/scala/code/metrics/MetricsTest.scala index cec7f255ea..ff0c6d4f12 100644 --- a/obp-api/src/test/scala/code/metrics/MetricsTest.scala +++ b/obp-api/src/test/scala/code/metrics/MetricsTest.scala @@ -65,7 +65,7 @@ class MetricsTest extends ServerSetup with WipeMetrics { scenario("We save a new API metric") { metrics.saveMetric(testUserId,testUrl1, day1, -1L, testUserName, testAppName, testDeveloperEmail, testConsumerId, testImplementedByPartialFunction, - testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId) + testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null) MetricBatchWriter.flush() val byUrl = metrics.getAllMetrics(List(OBPLimit(limit))).groupBy(_.getUrl()) @@ -83,16 +83,16 @@ class MetricsTest extends ServerSetup with WipeMetrics { scenario("Group all metrics by url") { metrics.saveMetric(testUserId, testUrl1, day1, -1L, testUserName, testAppName, testDeveloperEmail, testConsumerId, testImplementedByPartialFunction, - testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId) + testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null) metrics.saveMetric(testUserId, testUrl1, day1, -1L, testUserName, testAppName, testDeveloperEmail, testConsumerId, testImplementedByPartialFunction, - testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId) + testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null) metrics.saveMetric(testUserId, testUrl1, day2, -1L, testUserName, testAppName, testDeveloperEmail, testConsumerId, testImplementedByPartialFunction, - testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId) + testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null) metrics.saveMetric(testUserId, testUrl2, day2, -1L, testUserName, testAppName, testDeveloperEmail, testConsumerId, testImplementedByPartialFunction, - testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId) + testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null) MetricBatchWriter.flush() val byUrl = metrics.getAllMetrics(List(OBPLimit(limit1))).groupBy(_.getUrl()) @@ -113,16 +113,16 @@ class MetricsTest extends ServerSetup with WipeMetrics { scenario("Group all metrics by day") { metrics.saveMetric(testUserId, testUrl1, day1, -1L, testUserName, testAppName, testDeveloperEmail, testConsumerId, testImplementedByPartialFunction, - testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId) + testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null) metrics.saveMetric(testUserId, testUrl1, day1, -1L, testUserName, testAppName, testDeveloperEmail, testConsumerId, testImplementedByPartialFunction, - testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId) + testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null) metrics.saveMetric(testUserId, testUrl1, day2, -1L, testUserName, testAppName, testDeveloperEmail, testConsumerId, testImplementedByPartialFunction, - testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId) + testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null) metrics.saveMetric(testUserId, testUrl2, day2, -1L, testUserName, testAppName, testDeveloperEmail, testConsumerId, testImplementedByPartialFunction, - testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId) + testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null) MetricBatchWriter.flush() val byDay = metrics.getAllMetrics(List(OBPLimit(limit2))).groupBy(APIMetrics.getMetricDay) From 2b0cafb2c739db324924c6617b4f6bde3ee8d2c7 Mon Sep 17 00:00:00 2001 From: simonredfern Date: Sat, 23 May 2026 20:00:41 +0200 Subject: [PATCH 3/3] Adding GET /obp/v7.0.0/consents/config so Clients / Consumers can determine the max TTL they should create Consents with. (saves getting errors if asking for Consents with TTLS greater than the max on the OBP API server) --- .../scala/code/api/util/ConsentUtil.scala | 1 + .../scala/code/api/v7_0_0/Http4s700.scala | 34 +++++++++++++++++++ .../code/api/v7_0_0/JSONFactory7.0.0.scala | 14 ++++++++ 3 files changed, 49 insertions(+) diff --git a/obp-api/src/main/scala/code/api/util/ConsentUtil.scala b/obp-api/src/main/scala/code/api/util/ConsentUtil.scala index d6e88a3c04..0b1ade51ab 100644 --- a/obp-api/src/main/scala/code/api/util/ConsentUtil.scala +++ b/obp-api/src/main/scala/code/api/util/ConsentUtil.scala @@ -464,6 +464,7 @@ object Consent extends MdcLoggable { } if (cc.onBehalfOfUser.nonEmpty && APIUtil.getPropsAsBoolValue(nameOfProperty = "experimental_become_user_that_created_consent", defaultValue = false)) { + logger.warn("WARNING: experimental_become_user_that_created_consent is DEPRECATED and will be removed soon. Please unset this property.") logger.info("experimental_become_user_that_created_consent = true") logger.info(s"${cc.onBehalfOfUser.map(_.userId).getOrElse("")} is logged on instead of Consent user") Future(cc.onBehalfOfUser, Some(cc)) // Just propagate on behalf of user back diff --git a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala index 816d3f6e1a..95de480546 100644 --- a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala +++ b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala @@ -734,6 +734,40 @@ object Http4s700 { http4sPartialFunction = Some(getFeatures) ) + // Route: GET /obp/v7.0.0/consents/config + // Anonymous: operator-published policy that TPPs/agents need to know before issuing a consent. + val getConsentsConfig: HttpRoutes[IO] = HttpRoutes.of[IO] { + case req @ GET -> `prefixPath` / "consents" / "config" => + EndpointHelpers.executeAndRespond(req) { _ => + Future.successful(JSONFactory700.ConsentsConfigJsonV700( + consents_allowed = APIUtil.getPropsAsBoolValue("consents.allowed", false), + max_time_to_live_in_seconds = APIUtil.getPropsAsIntValue("consents.max_time_to_live", code.api.Constant.DEFAULT_CONSENT_TTL), + sca_enabled = APIUtil.getPropsAsBoolValue("consents.sca.enabled", true) + )) + } + } + + resourceDocs += ResourceDoc( + null, + implementedInApiVersion, + nameOf(getConsentsConfig), + "GET", + "/consents/config", + "Get Consents Configuration", + """Returns the operator-configured consent policy for this OBP instance: + | + |* `consents_allowed` — whether consent issuance is enabled at all. + |* `max_time_to_live_in_seconds` — the cap enforced when a client supplies `time_to_live` on consent creation. Exceeding this triggers `OBP-35020`. + |* `sca_enabled` — whether Strong Customer Authentication is required for consent activation. + | + |No Authentication is Required — clients need these values before they hold credentials.""", + EmptyBody, + JSONFactory700.consentsConfigJsonV700Example, + List(UnknownError), + apiTagConsent :: apiTagApi :: Nil, + http4sPartialFunction = Some(getConsentsConfig) + ) + // Route: GET /obp/v7.0.0/api/versions val getScannedApiVersions: HttpRoutes[IO] = HttpRoutes.of[IO] { case req @ GET -> `prefixPath` / "api" / "versions" => diff --git a/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala b/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala index c68c6b9bc8..bc664cda84 100644 --- a/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala +++ b/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala @@ -1034,4 +1034,18 @@ object JSONFactory700 extends MdcLoggable with code.api.util.CustomJsonFormats { lazy val coreAccountsJsonV700Example = CoreAccountsJsonV700(accounts = List(coreAccountJsonV700Example)) + + // ─── Consents config — operator-published policy clients need before issuing a consent ── + + case class ConsentsConfigJsonV700( + consents_allowed: Boolean, + max_time_to_live_in_seconds: Int, + sca_enabled: Boolean + ) + + lazy val consentsConfigJsonV700Example = ConsentsConfigJsonV700( + consents_allowed = true, + max_time_to_live_in_seconds = code.api.Constant.DEFAULT_CONSENT_TTL, + sca_enabled = true + ) }