Adding consent_reference_id to Metrics

obp-api/src/main/protobuf/metrics_stream.proto

@@ -11,6 +11,7 @@ message StreamMetricsRequest {
   string url_substring                    = 4;
   string implemented_by_partial_function  = 5;
   string app_name                         = 6;
+  string consent_reference_id             = 7;
 }
 
 // Per-REST-call metric record, mirrors APIMetrics.saveMetric args and
@@ -42,6 +43,9 @@ message MetricEvent {
   string api_instance_id                  = 16;
   // OBP operation id, e.g. "OBPv6.0.0-getBanks". Matches MetricJsonV600.operation_id.
   string operation_id                     = 17;
+  // Reference id of the consent (if any) that authorised the request.
+  // Mirrors MetricJsonV600.consent_reference_id (REST v6.0.0+).
+  string consent_reference_id             = 18;
 }
 
 // Live tail of API metrics as they are written.

obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala

@@ -3193,7 +3193,8 @@ object SwaggerDefinitionsJSON {
     response_body = json.parse("""{"code":401,"message":"OBP-20001: User not logged in. Authentication is required!"}"""),
     status_code = 401,
     operation_id = "OBPv4.0.0-getBanks",
-    api_instance_id = "obp_node_a"
+    api_instance_id = "obp_node_a",
+    consent_reference_id = Some(ExampleValue.consentReferenceIdExample.value)
   )
   lazy val metricsJsonV600 = MetricsJsonV600(
     metrics = List(metricJsonV600)

obp-api/src/main/scala/code/api/util/APIUtil.scala

@@ -1199,6 +1199,7 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{
         case "azp" => Full(OBPAzp(values.head))
         case "iss" => Full(OBPIss(values.head))
         case "consent_id" => Full(OBPConsentId(values.head))
+        case "consent_reference_id" => Full(OBPConsentReferenceId(values.head))
         case "user_id" => Full(OBPUserId(values.head))
         case "provider_provider_id" => Full(ProviderProviderId(values.head))
         case "bank_id" => Full(OBPBankId(values.head))
@@ -1333,6 +1334,7 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{
     val iss =  getHttpRequestUrlParam(httpRequestUrl,"iss")
     val azp =  getHttpRequestUrlParam(httpRequestUrl,"azp")
     val consentId =  getHttpRequestUrlParam(httpRequestUrl,"consent_id")
+    val consentReferenceId =  getHttpRequestUrlParam(httpRequestUrl,"consent_reference_id")
     val userId =  getHttpRequestUrlParam(httpRequestUrl, "user_id")
     val providerProviderId =  getHttpRequestUrlParam(httpRequestUrl, "provider_provider_id")
     val bankId =  getHttpRequestUrlParam(httpRequestUrl, "bank_id")
@@ -1368,7 +1370,7 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{
 
     Full(List(
       HTTPParam("sort_by",sortBy), HTTPParam("sort_direction",sortDirection), HTTPParam("from_date",fromDate), HTTPParam("to_date", toDate), HTTPParam("limit",limit), HTTPParam("offset",offset),
-      HTTPParam("anon", anon), HTTPParam("status", status), HTTPParam("consumer_id", consumerId), HTTPParam("azp", azp), HTTPParam("iss", iss), HTTPParam("consent_id", consentId), HTTPParam("user_id", userId), HTTPParam("provider_provider_id", providerProviderId), HTTPParam("url", url), HTTPParam("app_name", appName),
+      HTTPParam("anon", anon), HTTPParam("status", status), HTTPParam("consumer_id", consumerId), HTTPParam("azp", azp), HTTPParam("iss", iss), HTTPParam("consent_id", consentId), HTTPParam("consent_reference_id", consentReferenceId), HTTPParam("user_id", userId), HTTPParam("provider_provider_id", providerProviderId), HTTPParam("url", url), HTTPParam("app_name", appName),
       HTTPParam("implemented_by_partial_function",implementedByPartialFunction), HTTPParam("implemented_in_version",implementedInVersion), HTTPParam("verb", verb),
       HTTPParam("correlation_id", correlationId), HTTPParam("duration", duration), HTTPParam("exclude_app_names", excludeAppNames),
       HTTPParam("exclude_url_patterns", excludeUrlPattern),HTTPParam("exclude_implemented_by_partial_functions", excludeImplementedByPartialfunctions), 

obp-api/src/main/scala/code/api/util/ApiSession.scala

@@ -58,7 +58,9 @@ case class CallContext(
                         bank: Option[Bank] = None,
                         bankAccount: Option[BankAccount] = None,
                         view: Option[View] = None,
-                        counterparty: Option[CounterpartyTrait] = None
+                        counterparty: Option[CounterpartyTrait] = None,
+                        // Set when the request is authenticated via a consent. Persisted on metric rows for search/audit.
+                        consentReferenceId: Option[String] = None
                       ) extends MdcLoggable {
   override def toString: String = SecureLogging.maskSensitive(
     s"${this.getClass.getSimpleName}(${this.productIterator.mkString(", ")})"
@@ -144,7 +146,8 @@ case class CallContext(
       xRateLimitRemaining = this.xRateLimitRemaining,
       xRateLimitReset = this.xRateLimitReset,
       paginationOffset = this.paginationOffset,
-      paginationLimit = this.paginationLimit
+      paginationLimit = this.paginationLimit,
+      consentReferenceId = this.consentReferenceId
     )
   }
 
@@ -210,7 +213,8 @@ case class CallContextLight(gatewayLoginRequestPayload: Option[PayloadOfJwtJSON]
                             xRateLimitRemaining : Long = -1,
                             xRateLimitReset : Long = -1,
                             paginationOffset : Option[String] = None,
-                            paginationLimit : Option[String] = None
+                            paginationLimit : Option[String] = None,
+                            consentReferenceId: Option[String] = None
                            )
 
 trait LoginParam

obp-api/src/main/scala/code/api/util/ConsentUtil.scala

@@ -451,14 +451,20 @@ object Consent extends MdcLoggable {
     def applyConsentRules(consent: ConsentJWT): Future[(Box[User], Option[CallContext])] = {
       val temp = callContext
       // updated context if createdByUserId is present
-      val cc = if (consent.createdByUserId.nonEmpty) {
+      val ccWithOnBehalf = if (consent.createdByUserId.nonEmpty) {
         val onBehalfOfUser = Users.users.vend.getUserByUserId(consent.createdByUserId)
         temp.copy(onBehalfOfUser = onBehalfOfUser.toOption)
       } else {
         temp
       }
+      // Stamp the consent_reference_id on the CallContext so the metric writer can record it.
+      val cc = Consents.consentProvider.vend.getConsentByConsentId(consent.jti) match {
+        case Full(mc) => ccWithOnBehalf.copy(consentReferenceId = Some(mc.consentReferenceId))
+        case _        => ccWithOnBehalf
+      }
       if (cc.onBehalfOfUser.nonEmpty &&
         APIUtil.getPropsAsBoolValue(nameOfProperty = "experimental_become_user_that_created_consent", defaultValue = false)) {
+        logger.warn("WARNING: experimental_become_user_that_created_consent is DEPRECATED and will be removed soon. Please unset this property.")
         logger.info("experimental_become_user_that_created_consent = true")
         logger.info(s"${cc.onBehalfOfUser.map(_.userId).getOrElse("")} is logged on instead of Consent user")
         Future(cc.onBehalfOfUser, Some(cc)) // Just propagate on behalf of user back
@@ -552,7 +558,11 @@ object Consent extends MdcLoggable {
     implicit val dateFormats = CustomJsonFormats.formats
 
     def applyConsentRules(consent: ConsentJWT, callContext: CallContext): Future[(Box[User], Option[CallContext])] = {
-      val cc = callContext
+      // Stamp the consent_reference_id on the CallContext so the metric writer can record it.
+      val cc = Consents.consentProvider.vend.getConsentByConsentId(consent.jti) match {
+        case Full(mc) => callContext.copy(consentReferenceId = Some(mc.consentReferenceId))
+        case _        => callContext
+      }
       // 1. Get or Create a User
       getOrCreateUser(consent.sub, consent.iss, Some(consent.toConsent().consentId), None, None) map {
         case (Full(user), newUser) =>

obp-api/src/main/scala/code/api/util/OBPParam.scala

@@ -30,6 +30,7 @@ case class OBPSortBy(value: String) extends OBPQueryParam
 case class OBPAzp(value: String) extends OBPQueryParam
 case class OBPIss(value: String) extends OBPQueryParam
 case class OBPConsentId(value: String) extends OBPQueryParam
+case class OBPConsentReferenceId(value: String) extends OBPQueryParam
 case class OBPUserId(value: String) extends OBPQueryParam
 case class ProviderProviderId(value: String) extends OBPQueryParam
 case class OBPStatus(value: String) extends OBPQueryParam

obp-api/src/main/scala/code/api/util/WriteMetricUtil.scala

@@ -83,11 +83,13 @@ object WriteMetricUtil extends MdcLoggable {
               responseBodyToWrite,
               sourceIp,
               targetIp,
-              code.api.Constant.ApiInstanceId
+              code.api.Constant.ApiInstanceId,
+              cc.consentReferenceId.orNull
             )
             publishMetricEvent(userId, cc.url, cc.startTime.getOrElse(null), duration, userName, appName,
               developerEmail, consumerId, implementedByPartialFunction, cc.implementedInVersion, cc.verb,
-              cc.httpCode, cc.correlationId, sourceIp, targetIp, cc.operationId.getOrElse(""))
+              cc.httpCode, cc.correlationId, sourceIp, targetIp, cc.operationId.getOrElse(""),
+              cc.consentReferenceId.orNull)
           }
         }
       case _ =>
@@ -175,11 +177,12 @@ object WriteMetricUtil extends MdcLoggable {
           "Not enabled for old style endpoints",
           sourceIp,
           targetIp,
-          code.api.Constant.ApiInstanceId
+          code.api.Constant.ApiInstanceId,
+          null  // Old-style endpoints don't thread consent_reference_id through S.session yet.
         )
         publishMetricEvent(userId, url, date, duration, userName, appName, developerEmail, consumerId,
           implementedByPartialFunction, implementedInVersion, verb, None, correlationId, sourceIp, targetIp,
-          rd.map(_.operationId).getOrElse(""))
+          rd.map(_.operationId).getOrElse(""), null)
       }
 
     }
@@ -207,7 +210,8 @@ object WriteMetricUtil extends MdcLoggable {
                                  correlationId: String,
                                  sourceIp: String,
                                  targetIp: String,
-                                 operationId: String): Unit = {
+                                 operationId: String,
+                                 consentReferenceId: String): Unit = {
     if (!MetricsEventBus.isEnabled) return
     try {
       implicit val fmts = metricFormats
@@ -231,7 +235,8 @@ object WriteMetricUtil extends MdcLoggable {
         "source_ip"                       -> Option(sourceIp).getOrElse(""),
         "target_ip"                       -> Option(targetIp).getOrElse(""),
         "api_instance_id"                 -> code.api.Constant.ApiInstanceId,
-        "operation_id"                    -> Option(operationId).getOrElse("")
+        "operation_id"                    -> Option(operationId).getOrElse(""),
+        "consent_reference_id"            -> Option(consentReferenceId).getOrElse("")
       ))
       MetricsEventBus.publish(payload)
     } catch {

obp-api/src/main/scala/code/api/util/migration/Migration.scala

@@ -119,6 +119,7 @@ object Migration extends MdcLoggable {
       migrateChatRoomIsOpenRoom()
       migrateChatRoomCreatedByAndLastMessageSender()
       migrateConsentReferenceIdToUuid(startedBeforeSchemifier)
+      migrateMetricConsentReferenceId(startedBeforeSchemifier)
     }
 
     private def dummyScript(): Boolean = {
@@ -649,6 +650,18 @@ object Migration extends MdcLoggable {
       }
     }
 
+    private def migrateMetricConsentReferenceId(startedBeforeSchemifier: Boolean): Boolean = {
+      if(startedBeforeSchemifier == true) {
+        logger.warn(s"Migration.database.migrateMetricConsentReferenceId(true) cannot be run before Schemifier.")
+        true
+      } else {
+        val name = nameOf(migrateMetricConsentReferenceId(startedBeforeSchemifier))
+        runOnce(name) {
+          MigrationOfMetricConsentReferenceId.migrate(name)
+        }
+      }
+    }
+
     private def addAccountAccessWithViewsView(startedBeforeSchemifier: Boolean): Boolean = {
       if(startedBeforeSchemifier == true) {
         logger.warn(s"Migration.database.addAccountAccessWithViewsView(true) cannot be run before Schemifier.")

obp-api/src/main/scala/code/api/util/migration/MigrationOfMetricConsentReferenceId.scala

@@ -0,0 +1,101 @@
+package code.api.util.migration
+
+import code.api.util.APIUtil
+import code.api.util.migration.Migration.{DbFunction, saveLog}
+import code.metrics.{MappedMetric, MetricArchive}
+import net.liftweb.mapper.Schemifier
+
+/**
+ * Migration: add `consent_reference_id VARCHAR(36)` to both the live `Metric` table and
+ * the `metricarchive` table, plus an index on each for efficient search-by-consent.
+ *
+ * Backs up the live `Metric` table only (not metricarchive — the archive is itself a
+ * long-term backup of metrics, so duplicating it would be wasteful).
+ *
+ * No backfill: historical rows legitimately have no consent reference; nullable column.
+ *
+ * Lift's Schemifier auto-creates the column on fresh deploys from the updated model;
+ * this migration handles existing deploys.
+ */
+object MigrationOfMetricConsentReferenceId {
+
+  def migrate(name: String): Boolean = {
+    DbFunction.tableExists(MappedMetric) match {
+      case true =>
+        val startDate = System.currentTimeMillis()
+        val commitId: String = APIUtil.gitCommit
+        val dbDriver = APIUtil.getPropsValue("db.driver") openOr "org.h2.Driver"
+        val isMssql = dbDriver.contains("com.microsoft.sqlserver.jdbc.SQLServerDriver")
+        var isSuccessful = false
+        val sqlLog = new StringBuilder()
+
+        try {
+          // 1. Backup of the live metric table (NOT the archive — it's already a long-term snapshot).
+          //    Although MappedMetric.dbTableName is "Metric", Lift's Schemifier emits unquoted DDL,
+          //    so Postgres folds the name to lowercase `metric`. Every other SQL site in the codebase
+          //    (MetricBatchWriter INSERT, DoobieMetricsQueries, MigrationOfMetricView) references it
+          //    as lowercase unquoted `metric` — mirror that here. MSSQL is case-insensitive by default.
+          val backupMetric = if (isMssql) {
+            "SELECT * INTO backup_2026_05_metric FROM metric;"
+          } else {
+            "CREATE TABLE backup_2026_05_metric AS SELECT * FROM metric;"
+          }
+          sqlLog.append(DbFunction.maybeWrite(true, Schemifier.infoF _)(() => backupMetric)).append("\n")
+
+          // 2. Add the new column to the live metric table.
+          val addColumnMetric = if (isMssql) {
+            "ALTER TABLE metric ADD consent_reference_id VARCHAR(36) NULL;"
+          } else {
+            "ALTER TABLE metric ADD COLUMN IF NOT EXISTS consent_reference_id VARCHAR(36);"
+          }
+          sqlLog.append(DbFunction.maybeWrite(true, Schemifier.infoF _)(() => addColumnMetric)).append("\n")
+
+          // 3. Add the new column to the archive table.
+          val addColumnArchive = if (isMssql) {
+            "ALTER TABLE metricarchive ADD consent_reference_id VARCHAR(36) NULL;"
+          } else {
+            "ALTER TABLE metricarchive ADD COLUMN IF NOT EXISTS consent_reference_id VARCHAR(36);"
+          }
+          sqlLog.append(DbFunction.maybeWrite(true, Schemifier.infoF _)(() => addColumnArchive)).append("\n")
+
+          // 4. Index for search-by-consent on both tables.
+          val indexMetric = if (isMssql) {
+            "CREATE INDEX idx_metric_consent_reference_id ON metric(consent_reference_id);"
+          } else {
+            "CREATE INDEX IF NOT EXISTS idx_metric_consent_reference_id ON metric(consent_reference_id);"
+          }
+          sqlLog.append(DbFunction.maybeWrite(true, Schemifier.infoF _)(() => indexMetric)).append("\n")
+
+          val indexArchive = if (isMssql) {
+            "CREATE INDEX idx_metricarchive_consent_reference_id ON metricarchive(consent_reference_id);"
+          } else {
+            "CREATE INDEX IF NOT EXISTS idx_metricarchive_consent_reference_id ON metricarchive(consent_reference_id);"
+          }
+          sqlLog.append(DbFunction.maybeWrite(true, Schemifier.infoF _)(() => indexArchive)).append("\n")
+
+          isSuccessful = true
+        } catch {
+          case e: Exception =>
+            isSuccessful = false
+            sqlLog.append(s"\nException: ${e.getMessage}\n")
+        }
+
+        val endDate = System.currentTimeMillis()
+        val comment: String =
+          s"""Executed SQL:
+             |$sqlLog
+             |""".stripMargin
+        saveLog(name, commitId, isSuccessful, startDate, endDate, comment)
+        isSuccessful
+
+      case false =>
+        val startDate = System.currentTimeMillis()
+        val commitId: String = APIUtil.gitCommit
+        val isSuccessful = false
+        val endDate = System.currentTimeMillis()
+        val comment: String = s"""${MappedMetric._dbTableNameLC} table does not exist""".stripMargin
+        saveLog(name, commitId, isSuccessful, startDate, endDate, comment)
+        isSuccessful
+    }
+  }
+}

obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala

@@ -7227,6 +7227,8 @@ object Http4s600 {
            |
            |16 duration (if null ignore) - Returns calls where duration > specified value (in milliseconds). Use this to find slow API calls. eg: duration=5000 returns calls taking more than 5 seconds
            |
+           |17 consent_reference_id (if null ignore) - Returns calls authenticated via the consent with this reference id. eg: consent_reference_id=fd13b9af-4f74-4d52-a7f1-7c2c12f3aa11
+           |
         """.stripMargin,
         EmptyBody,
         metricsJsonV600,

obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala

@@ -448,7 +448,8 @@
     response_body: net.liftweb.json.JValue,
     status_code: Int,
     operation_id: String,
-    api_instance_id: String
+    api_instance_id: String,
+    consent_reference_id: Option[String]
 )
 case class MetricsJsonV600(metrics: List[MetricJsonV600])
 
@@ -1687,7 +1688,8 @@
       response_body = net.liftweb.json.parseOpt(metric.getResponseBody()).getOrElse(net.liftweb.json.JString("Not enabled")),
       status_code = metric.getHttpCode(),
       operation_id = operationId,
-      api_instance_id = metric.getApiInstanceId()
+      api_instance_id = metric.getApiInstanceId(),
+      consent_reference_id = Option(metric.getConsentReferenceId()).filter(_.nonEmpty)
     )
   }
 

obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala

@@ -734,6 +734,40 @@ object Http4s700 {
       http4sPartialFunction = Some(getFeatures)
     )
 
+    // Route: GET /obp/v7.0.0/consents/config
+    // Anonymous: operator-published policy that TPPs/agents need to know before issuing a consent.
+    val getConsentsConfig: HttpRoutes[IO] = HttpRoutes.of[IO] {
+      case req @ GET -> `prefixPath` / "consents" / "config" =>
+        EndpointHelpers.executeAndRespond(req) { _ =>
+          Future.successful(JSONFactory700.ConsentsConfigJsonV700(
+            consents_allowed            = APIUtil.getPropsAsBoolValue("consents.allowed", false),
+            max_time_to_live_in_seconds = APIUtil.getPropsAsIntValue("consents.max_time_to_live", code.api.Constant.DEFAULT_CONSENT_TTL),
+            sca_enabled                 = APIUtil.getPropsAsBoolValue("consents.sca.enabled", true)
+          ))
+        }
+    }
+
+    resourceDocs += ResourceDoc(
+      null,
+      implementedInApiVersion,
+      nameOf(getConsentsConfig),
+      "GET",
+      "/consents/config",
+      "Get Consents Configuration",
+      """Returns the operator-configured consent policy for this OBP instance:
+        |
+        |* `consents_allowed` — whether consent issuance is enabled at all.
+        |* `max_time_to_live_in_seconds` — the cap enforced when a client supplies `time_to_live` on consent creation. Exceeding this triggers `OBP-35020`.
+        |* `sca_enabled` — whether Strong Customer Authentication is required for consent activation.
+        |
+        |No Authentication is Required — clients need these values before they hold credentials.""",
+      EmptyBody,
+      JSONFactory700.consentsConfigJsonV700Example,
+      List(UnknownError),
+      apiTagConsent :: apiTagApi :: Nil,
+      http4sPartialFunction = Some(getConsentsConfig)
+    )
+
     // Route: GET /obp/v7.0.0/api/versions
     val getScannedApiVersions: HttpRoutes[IO] = HttpRoutes.of[IO] {
       case req @ GET -> `prefixPath` / "api" / "versions" =>