From 1e07655e1e3e8ed4640dc986156c1a447a077811 Mon Sep 17 00:00:00 2001 From: ansh-meesho Date: Tue, 18 Nov 2025 14:54:51 +0530 Subject: [PATCH 01/21] feat: sync the internal code into horizon --- horizon/cmd/horizon/main.go | 49 +- horizon/go.mod | 136 +- horizon/go.sum | 700 +-- .../application/controller/controller.go | 90 + .../application/handler/application.go | 114 + .../internal/application/handler/config.go | 7 + horizon/internal/application/handler/init.go | 14 + .../internal/application/handler/models.go | 50 + horizon/internal/application/route/router.go | 34 + horizon/internal/configs/app_config.go | 109 + .../connectionconfig/controller/controller.go | 98 + .../connectionconfig/handler/config.go | 7 + .../handler/connection_config.go | 233 + .../internal/connectionconfig/handler/init.go | 14 + .../connectionconfig/handler/models.go | 90 + .../internal/connectionconfig/route/router.go | 34 + horizon/internal/constant/api.go | 7 + horizon/internal/constant/repository.go | 6 + .../deployable/controller/controller.go | 279 + horizon/internal/deployable/handler/config.go | 16 + .../internal/deployable/handler/deployable.go | 250 + horizon/internal/deployable/handler/init.go | 16 + .../deployable/handler/modelhandler.go | 64 + horizon/internal/deployable/handler/models.go | 68 + .../deployable/handler/predatorhandler.go | 322 + horizon/internal/deployable/router/router.go | 33 + .../externalcall/feature_validation_client.go | 288 + horizon/internal/externalcall/gcs_client.go | 634 ++ horizon/internal/externalcall/init.go | 14 + .../internal/externalcall/pricing_client.go | 153 + .../externalcall/prometheus_client.go | 246 + .../externalcall/ring_master_client.go | 473 ++ horizon/internal/externalcall/slack_client.go | 197 + .../feature_store/controller/fs_config.go | 31 + .../feature_store/handler/fs_config.go | 121 + .../internal/feature_store/handler/model.go | 75 + .../internal/feature_store/route/router.go | 28 + .../inferflow/controller/controller.go | 367 ++ horizon/internal/inferflow/etcd/config.go | 8 + horizon/internal/inferflow/etcd/etcd.go | 82 + horizon/internal/inferflow/etcd/models.go | 142 + horizon/internal/inferflow/handler/adaptor.go | 797 +++ horizon/internal/inferflow/handler/config.go | 19 + .../inferflow/handler/config_builder.go | 1481 +++++ .../internal/inferflow/handler/inferflow.go | 1035 ++++ horizon/internal/inferflow/handler/init.go | 14 + horizon/internal/inferflow/handler/models.go | 435 ++ .../inferflow/handler/proto/inferflow.proto | 38 + .../handler/proto/protogen/inferflow.pb.go | 422 ++ .../proto/protogen/inferflow_grpc.pb.go | 121 + horizon/internal/inferflow/init.go | 22 + horizon/internal/inferflow/route/router.go | 53 + horizon/internal/init.go | 20 + horizon/internal/jobs/bulk_delete.go | 91 + .../jobs/bulkdeletestrategy/config.go | 7 + .../bulkdeletestrategy/inferflow_service.go | 132 + .../bulkdeletestrategy/numerix_service.go | 105 + .../bulkdeletestrategy/predator_service.go | 227 + .../bulkdeletestrategy/strategy_selector.go | 81 + horizon/internal/middleware/middleware.go | 234 + .../resolver/application_resolver.go | 33 + .../internal/middleware/resolver/config.go | 25 + .../resolver/deployable_resolver.go | 55 + .../middleware/resolver/inferflow_resolver.go | 57 + .../middleware/resolver/numerix_resolver.go | 46 + .../middleware/resolver/predator_resolver.go | 54 + .../middleware/resolver/resolver_registry.go | 35 + horizon/internal/middlewares/middlewares.go | 117 - .../internal/numerix/controller/controller.go | 320 + horizon/internal/numerix/etcd/config.go | 7 + horizon/internal/numerix/etcd/etcd.go | 65 + horizon/internal/numerix/etcd/models.go | 9 + horizon/internal/numerix/handler/config.go | 15 + horizon/internal/numerix/handler/init.go | 14 + horizon/internal/numerix/handler/models.go | 185 + horizon/internal/numerix/handler/numerix.go | 637 ++ horizon/internal/numerix/init.go | 22 + horizon/internal/numerix/proto/numerix.proto | 52 + .../numerix/proto/protogen/numerix.pb.go | 593 ++ .../numerix/proto/protogen/numerix_grpc.pb.go | 121 + horizon/internal/numerix/route/router.go | 57 + .../numerix/util/expression_parser.go | 46 + .../numerix/util/expression_parser_test.go | 215 + .../numerix/util/expression_validator.go | 237 + .../numerix/util/expression_validator_test.go | 923 +++ .../predator/controller/controller.go | 462 ++ horizon/internal/predator/handler/config.go | 28 + horizon/internal/predator/handler/init.go | 23 + horizon/internal/predator/handler/model.go | 270 + .../predator/handler/model_config.pb.go | 4567 ++++++++++++++ horizon/internal/predator/handler/models.go | 78 + horizon/internal/predator/handler/predator.go | 3964 ++++++++++++ horizon/internal/predator/init.go | 29 + .../predator/proto/model_config.proto | 401 ++ .../proto/tritonproto/gpmc_service.proto | 1736 ++++++ .../predator/proto/tritonproto/health.proto | 77 + .../proto/tritonproto/model_config.proto | 92 + .../tritonproto/protogen/gpmc_service.pb.go | 5365 +++++++++++++++++ .../protogen/gpmc_service_grpc.pb.go | 1090 ++++ .../tritonproto/protogen/model_config.pb.go | 504 ++ horizon/internal/predator/route/router.go | 55 + .../repositories/sql/apiresolver/sql.go | 45 + .../repositories/sql/apiresolver/table.go | 33 + .../sql/application/repository.go | 54 + .../repositories/sql/application/table.go | 37 + .../repositories/sql/circuitbreaker/sql.go | 54 + .../repositories/sql/circuitbreaker/table.go | 50 + .../sql/connectionconfig/repository.go | 54 + .../sql/connectionconfig/table.go | 92 + .../internal/repositories/sql/counter/sql.go | 65 + .../repositories/sql/counter/table.go | 32 + .../sql/deployablemetadata/sql.go | 46 + .../sql/deployablemetadata/table.go | 31 + .../repositories/sql/discoveryconfig/sql.go | 104 + .../repositories/sql/discoveryconfig/table.go | 43 + .../sql/inferflow/config/repository.go | 121 + .../sql/inferflow/config/table.go | 43 + .../repositories/sql/inferflow/models.go | 226 + .../sql/inferflow/request/repository.go | 128 + .../sql/inferflow/request/table.go | 44 + .../sql/numerix/config/repository.go | 147 + .../repositories/sql/numerix/config/table.go | 62 + .../sql/numerix/request/repository.go | 125 + .../repositories/sql/numerix/request/table.go | 69 + .../repositories/sql/predatorconfig/sql.go | 151 + .../repositories/sql/predatorconfig/table.go | 39 + .../repositories/sql/predatorrequest/sql.go | 157 + .../repositories/sql/predatorrequest/table.go | 41 + .../repositories/sql/rolepermission/sql.go | 63 + .../repositories/sql/rolepermission/table.go | 34 + .../internal/repositories/sql/schedule/sql.go | 36 + .../repositories/sql/schedule/table.go | 28 + .../repositories/sql/serviceconfig/sql.go | 73 + .../repositories/sql/serviceconfig/table.go | 44 + .../sql/servicedeployableconfig/sql.go | 100 + .../sql/servicedeployableconfig/table.go | 44 + .../sql/validationjob/repository.go | 169 + .../repositories/sql/validationjob/table.go | 38 + .../sql/validationlock/repository.go | 199 + .../repositories/sql/validationlock/table.go | 23 + horizon/pkg/etcd/etcd.go | 10 +- horizon/pkg/etcd/init.go | 48 +- horizon/pkg/etcd/v1.go | 113 +- horizon/pkg/grpc/grpc.go | 27 + horizon/pkg/random/random_generator.go | 318 + horizon/pkg/scheduler/init.go | 37 + horizon/pkg/serializer/serializer.go | 601 ++ horizon/pkg/zookeeper/model.go | 24 + horizon/pkg/zookeeper/zookeeper.go | 259 + 149 files changed, 37374 insertions(+), 721 deletions(-) create mode 100644 horizon/internal/application/controller/controller.go create mode 100644 horizon/internal/application/handler/application.go create mode 100644 horizon/internal/application/handler/config.go create mode 100644 horizon/internal/application/handler/init.go create mode 100644 horizon/internal/application/handler/models.go create mode 100644 horizon/internal/application/route/router.go create mode 100644 horizon/internal/configs/app_config.go create mode 100644 horizon/internal/connectionconfig/controller/controller.go create mode 100644 horizon/internal/connectionconfig/handler/config.go create mode 100644 horizon/internal/connectionconfig/handler/connection_config.go create mode 100644 horizon/internal/connectionconfig/handler/init.go create mode 100644 horizon/internal/connectionconfig/handler/models.go create mode 100644 horizon/internal/connectionconfig/route/router.go create mode 100644 horizon/internal/constant/api.go create mode 100644 horizon/internal/constant/repository.go create mode 100644 horizon/internal/deployable/controller/controller.go create mode 100644 horizon/internal/deployable/handler/config.go create mode 100644 horizon/internal/deployable/handler/deployable.go create mode 100644 horizon/internal/deployable/handler/init.go create mode 100644 horizon/internal/deployable/handler/modelhandler.go create mode 100644 horizon/internal/deployable/handler/models.go create mode 100644 horizon/internal/deployable/handler/predatorhandler.go create mode 100644 horizon/internal/deployable/router/router.go create mode 100644 horizon/internal/externalcall/feature_validation_client.go create mode 100644 horizon/internal/externalcall/gcs_client.go create mode 100644 horizon/internal/externalcall/init.go create mode 100644 horizon/internal/externalcall/pricing_client.go create mode 100644 horizon/internal/externalcall/prometheus_client.go create mode 100644 horizon/internal/externalcall/ring_master_client.go create mode 100644 horizon/internal/externalcall/slack_client.go create mode 100644 horizon/internal/feature_store/controller/fs_config.go create mode 100644 horizon/internal/feature_store/handler/fs_config.go create mode 100644 horizon/internal/feature_store/handler/model.go create mode 100644 horizon/internal/feature_store/route/router.go create mode 100644 horizon/internal/inferflow/controller/controller.go create mode 100644 horizon/internal/inferflow/etcd/config.go create mode 100644 horizon/internal/inferflow/etcd/etcd.go create mode 100644 horizon/internal/inferflow/etcd/models.go create mode 100644 horizon/internal/inferflow/handler/adaptor.go create mode 100644 horizon/internal/inferflow/handler/config.go create mode 100644 horizon/internal/inferflow/handler/config_builder.go create mode 100644 horizon/internal/inferflow/handler/inferflow.go create mode 100644 horizon/internal/inferflow/handler/init.go create mode 100644 horizon/internal/inferflow/handler/models.go create mode 100644 horizon/internal/inferflow/handler/proto/inferflow.proto create mode 100644 horizon/internal/inferflow/handler/proto/protogen/inferflow.pb.go create mode 100644 horizon/internal/inferflow/handler/proto/protogen/inferflow_grpc.pb.go create mode 100644 horizon/internal/inferflow/init.go create mode 100644 horizon/internal/inferflow/route/router.go create mode 100644 horizon/internal/init.go create mode 100644 horizon/internal/jobs/bulk_delete.go create mode 100644 horizon/internal/jobs/bulkdeletestrategy/config.go create mode 100644 horizon/internal/jobs/bulkdeletestrategy/inferflow_service.go create mode 100644 horizon/internal/jobs/bulkdeletestrategy/numerix_service.go create mode 100644 horizon/internal/jobs/bulkdeletestrategy/predator_service.go create mode 100644 horizon/internal/jobs/bulkdeletestrategy/strategy_selector.go create mode 100644 horizon/internal/middleware/middleware.go create mode 100644 horizon/internal/middleware/resolver/application_resolver.go create mode 100644 horizon/internal/middleware/resolver/config.go create mode 100644 horizon/internal/middleware/resolver/deployable_resolver.go create mode 100644 horizon/internal/middleware/resolver/inferflow_resolver.go create mode 100644 horizon/internal/middleware/resolver/numerix_resolver.go create mode 100644 horizon/internal/middleware/resolver/predator_resolver.go create mode 100644 horizon/internal/middleware/resolver/resolver_registry.go delete mode 100644 horizon/internal/middlewares/middlewares.go create mode 100644 horizon/internal/numerix/controller/controller.go create mode 100644 horizon/internal/numerix/etcd/config.go create mode 100644 horizon/internal/numerix/etcd/etcd.go create mode 100644 horizon/internal/numerix/etcd/models.go create mode 100644 horizon/internal/numerix/handler/config.go create mode 100644 horizon/internal/numerix/handler/init.go create mode 100644 horizon/internal/numerix/handler/models.go create mode 100644 horizon/internal/numerix/handler/numerix.go create mode 100644 horizon/internal/numerix/init.go create mode 100644 horizon/internal/numerix/proto/numerix.proto create mode 100644 horizon/internal/numerix/proto/protogen/numerix.pb.go create mode 100644 horizon/internal/numerix/proto/protogen/numerix_grpc.pb.go create mode 100644 horizon/internal/numerix/route/router.go create mode 100644 horizon/internal/numerix/util/expression_parser.go create mode 100644 horizon/internal/numerix/util/expression_parser_test.go create mode 100644 horizon/internal/numerix/util/expression_validator.go create mode 100644 horizon/internal/numerix/util/expression_validator_test.go create mode 100644 horizon/internal/predator/controller/controller.go create mode 100644 horizon/internal/predator/handler/config.go create mode 100644 horizon/internal/predator/handler/init.go create mode 100644 horizon/internal/predator/handler/model.go create mode 100644 horizon/internal/predator/handler/model_config.pb.go create mode 100644 horizon/internal/predator/handler/models.go create mode 100644 horizon/internal/predator/handler/predator.go create mode 100644 horizon/internal/predator/init.go create mode 100644 horizon/internal/predator/proto/model_config.proto create mode 100644 horizon/internal/predator/proto/tritonproto/gpmc_service.proto create mode 100644 horizon/internal/predator/proto/tritonproto/health.proto create mode 100644 horizon/internal/predator/proto/tritonproto/model_config.proto create mode 100644 horizon/internal/predator/proto/tritonproto/protogen/gpmc_service.pb.go create mode 100644 horizon/internal/predator/proto/tritonproto/protogen/gpmc_service_grpc.pb.go create mode 100644 horizon/internal/predator/proto/tritonproto/protogen/model_config.pb.go create mode 100644 horizon/internal/predator/route/router.go create mode 100644 horizon/internal/repositories/sql/apiresolver/sql.go create mode 100644 horizon/internal/repositories/sql/apiresolver/table.go create mode 100644 horizon/internal/repositories/sql/application/repository.go create mode 100644 horizon/internal/repositories/sql/application/table.go create mode 100644 horizon/internal/repositories/sql/circuitbreaker/sql.go create mode 100644 horizon/internal/repositories/sql/circuitbreaker/table.go create mode 100644 horizon/internal/repositories/sql/connectionconfig/repository.go create mode 100644 horizon/internal/repositories/sql/connectionconfig/table.go create mode 100644 horizon/internal/repositories/sql/counter/sql.go create mode 100644 horizon/internal/repositories/sql/counter/table.go create mode 100644 horizon/internal/repositories/sql/deployablemetadata/sql.go create mode 100644 horizon/internal/repositories/sql/deployablemetadata/table.go create mode 100644 horizon/internal/repositories/sql/discoveryconfig/sql.go create mode 100644 horizon/internal/repositories/sql/discoveryconfig/table.go create mode 100644 horizon/internal/repositories/sql/inferflow/config/repository.go create mode 100644 horizon/internal/repositories/sql/inferflow/config/table.go create mode 100644 horizon/internal/repositories/sql/inferflow/models.go create mode 100644 horizon/internal/repositories/sql/inferflow/request/repository.go create mode 100644 horizon/internal/repositories/sql/inferflow/request/table.go create mode 100644 horizon/internal/repositories/sql/numerix/config/repository.go create mode 100644 horizon/internal/repositories/sql/numerix/config/table.go create mode 100644 horizon/internal/repositories/sql/numerix/request/repository.go create mode 100644 horizon/internal/repositories/sql/numerix/request/table.go create mode 100644 horizon/internal/repositories/sql/predatorconfig/sql.go create mode 100644 horizon/internal/repositories/sql/predatorconfig/table.go create mode 100644 horizon/internal/repositories/sql/predatorrequest/sql.go create mode 100644 horizon/internal/repositories/sql/predatorrequest/table.go create mode 100644 horizon/internal/repositories/sql/rolepermission/sql.go create mode 100644 horizon/internal/repositories/sql/rolepermission/table.go create mode 100644 horizon/internal/repositories/sql/schedule/sql.go create mode 100644 horizon/internal/repositories/sql/schedule/table.go create mode 100644 horizon/internal/repositories/sql/serviceconfig/sql.go create mode 100644 horizon/internal/repositories/sql/serviceconfig/table.go create mode 100644 horizon/internal/repositories/sql/servicedeployableconfig/sql.go create mode 100644 horizon/internal/repositories/sql/servicedeployableconfig/table.go create mode 100644 horizon/internal/repositories/sql/validationjob/repository.go create mode 100644 horizon/internal/repositories/sql/validationjob/table.go create mode 100644 horizon/internal/repositories/sql/validationlock/repository.go create mode 100644 horizon/internal/repositories/sql/validationlock/table.go create mode 100644 horizon/pkg/grpc/grpc.go create mode 100644 horizon/pkg/random/random_generator.go create mode 100644 horizon/pkg/scheduler/init.go create mode 100644 horizon/pkg/serializer/serializer.go create mode 100644 horizon/pkg/zookeeper/model.go create mode 100644 horizon/pkg/zookeeper/zookeeper.go diff --git a/horizon/cmd/horizon/main.go b/horizon/cmd/horizon/main.go index 99e835de..91ebc09a 100644 --- a/horizon/cmd/horizon/main.go +++ b/horizon/cmd/horizon/main.go @@ -3,27 +3,70 @@ package main import ( "strconv" + applicationRouter "github.com/Meesho/BharatMLStack/horizon/internal/application/route" authRouter "github.com/Meesho/BharatMLStack/horizon/internal/auth/router" - "github.com/Meesho/BharatMLStack/horizon/internal/middlewares" + "github.com/Meesho/BharatMLStack/horizon/internal/configs" + connectionConfigRouter "github.com/Meesho/BharatMLStack/horizon/internal/connectionconfig/route" + deployableRouter "github.com/Meesho/BharatMLStack/horizon/internal/deployable/router" + featureStoreRouter "github.com/Meesho/BharatMLStack/horizon/internal/feature_store/route" + inferflowConfig "github.com/Meesho/BharatMLStack/horizon/internal/inferflow/etcd" + inferflowRouter "github.com/Meesho/BharatMLStack/horizon/internal/inferflow/route" + "github.com/Meesho/BharatMLStack/horizon/internal/middleware" + numerixConfig "github.com/Meesho/BharatMLStack/horizon/internal/numerix/etcd" + numerixRouter "github.com/Meesho/BharatMLStack/horizon/internal/numerix/route" ofsConfig "github.com/Meesho/BharatMLStack/horizon/internal/online-feature-store/config" ofsRouter "github.com/Meesho/BharatMLStack/horizon/internal/online-feature-store/router" + predatorRouter "github.com/Meesho/BharatMLStack/horizon/internal/predator/route" "github.com/Meesho/BharatMLStack/horizon/pkg/config" "github.com/Meesho/BharatMLStack/horizon/pkg/etcd" "github.com/Meesho/BharatMLStack/horizon/pkg/httpframework" "github.com/Meesho/BharatMLStack/horizon/pkg/infra" "github.com/Meesho/BharatMLStack/horizon/pkg/logger" "github.com/Meesho/BharatMLStack/horizon/pkg/metric" + "github.com/Meesho/BharatMLStack/horizon/pkg/scheduler" + cacConfig "github.com/Meesho/go-core/config" + pricingclient "github.com/Meesho/price-aggregator-go/pricingfeatureretrieval/client" "github.com/spf13/viper" ) +type AppConfig struct { + Configs configs.Configs + DynamicConfigs configs.DynamicConfigs +} + +func (cfg *AppConfig) GetStaticConfig() interface{} { + return &cfg.Configs +} + +func (cfg *AppConfig) GetDynamicConfig() interface{} { + return &cfg.DynamicConfigs +} + +var ( + appConfig AppConfig +) + func main() { + cacConfig.InitGlobalConfig(&appConfig) config.InitEnv() infra.InitDBConnectors() logger.Init() metric.Init() - httpframework.Init(middlewares.NewMiddleware().GetMiddleWares()...) - etcd.InitFromAppName(&ofsConfig.FeatureRegistry{}, viper.GetString("ONLINE_FEATURE_STORE_APP_NAME")) + httpframework.Init(middleware.NewMiddleware().GetMiddleWares()...) + etcd.InitFromAppName(&ofsConfig.FeatureRegistry{}, appConfig.Configs.OnlineFeatureStoreAppName, appConfig.Configs) + etcd.InitFromAppName(&numerixConfig.NumerixConfigRegistery{}, appConfig.Configs.NumerixAppName, appConfig.Configs) + etcd.InitMPEtcdFromRegistry(&inferflowConfig.ModelConfigRegistery{}, appConfig.Configs) + etcd.InitFromAppName(&inferflowConfig.HorizonRegistry{}, appConfig.Configs.HorizonAppName, appConfig.Configs) + deployableRouter.Init() + inferflowRouter.Init() + numerixRouter.Init() + applicationRouter.Init() + connectionConfigRouter.Init() + predatorRouter.Init() authRouter.Init() ofsRouter.Init() + featureStoreRouter.Init(appConfig.Configs) + scheduler.Init(appConfig.Configs) + pricingclient.Init() httpframework.Instance().Run(":" + strconv.Itoa(viper.GetInt("APP_PORT"))) } diff --git a/horizon/go.mod b/horizon/go.mod index 179ae0ae..ba64a6ed 100644 --- a/horizon/go.mod +++ b/horizon/go.mod @@ -1,80 +1,128 @@ module github.com/Meesho/BharatMLStack/horizon -go 1.22.0 +go 1.24.4 + +toolchain go1.24.10 require ( - github.com/DataDog/datadog-go/v5 v5.5.0 + cloud.google.com/go/storage v1.57.2 + github.com/DataDog/datadog-go/v5 v5.6.0 + github.com/Meesho/go-core v1.30.20 + github.com/Meesho/helix-clients v0.8.1 + github.com/Meesho/price-aggregator-go/pricingfeatureretrieval/client v0.0.12 + github.com/deckarep/golang-set/v2 v2.8.0 github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/gin-contrib/cors v1.7.3 - github.com/gin-gonic/gin v1.10.0 + github.com/gin-gonic/gin v1.10.1 + github.com/go-zookeeper/zk v1.0.4 github.com/gocql/gocql v1.7.0 - github.com/rs/zerolog v1.33.0 - github.com/spf13/viper v1.16.0 - github.com/stretchr/testify v1.9.0 + github.com/google/uuid v1.6.0 + github.com/klauspost/compress v1.18.0 + github.com/robfig/cron/v3 v3.0.1 + github.com/rs/zerolog v1.34.0 + github.com/spf13/viper v1.20.1 + github.com/stretchr/testify v1.10.0 github.com/x448/float16 v0.8.4 - go.etcd.io/etcd/client/v3 v3.5.9 - golang.org/x/crypto v0.33.0 + go.etcd.io/etcd/client/v3 v3.5.12 + golang.org/x/crypto v0.43.0 + google.golang.org/api v0.256.0 + google.golang.org/grpc v1.76.0 + google.golang.org/protobuf v1.36.10 gorm.io/driver/mysql v1.5.6 gorm.io/gorm v1.25.10 ) require ( + cel.dev/expr v0.24.0 // indirect + cloud.google.com/go v0.121.6 // indirect + cloud.google.com/go/auth v0.17.0 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect + cloud.google.com/go/iam v1.5.2 // indirect + cloud.google.com/go/monitoring v1.24.2 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect - github.com/bytedance/sonic v1.12.6 // indirect - github.com/bytedance/sonic/loader v0.2.1 // indirect - github.com/cloudwego/base64x v0.1.4 // indirect - github.com/cloudwego/iasm v0.2.0 // indirect + github.com/bits-and-blooms/bitset v1.22.0 // indirect + github.com/bytedance/sonic v1.13.3 // indirect + github.com/bytedance/sonic/loader v0.3.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cloudwego/base64x v0.1.5 // indirect + github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/fsnotify/fsnotify v1.6.0 // indirect - github.com/gabriel-vasile/mimetype v1.4.7 // indirect - github.com/gin-contrib/sse v0.1.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect + github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect + github.com/failsafe-go/failsafe-go v0.6.9 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.9 // indirect + github.com/gin-contrib/sse v1.1.0 // indirect + github.com/go-jose/go-jose/v4 v4.1.2 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.23.0 // indirect + github.com/go-playground/validator/v10 v10.27.0 // indirect github.com/go-sql-driver/mysql v1.7.0 // indirect - github.com/goccy/go-json v0.10.4 // indirect + github.com/go-viper/mapstructure/v2 v2.4.0 // indirect + github.com/goccy/go-json v0.10.5 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/golang/snappy v0.0.4 // indirect + github.com/golang/snappy v1.0.0 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.7 // indirect + github.com/googleapis/gax-go/v2 v2.15.0 // indirect github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect - github.com/hashicorp/hcl v1.0.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/cpuid/v2 v2.2.9 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect - github.com/magiconair/properties v1.8.7 // indirect - github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/pelletier/go-toml/v2 v2.2.3 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/spf13/afero v1.9.5 // indirect - github.com/spf13/cast v1.5.1 // indirect - github.com/spf13/jwalterweatherman v1.1.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/sagikazarmark/locafero v0.9.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.14.0 // indirect + github.com/spf13/cast v1.9.2 // indirect + github.com/spf13/pflag v1.0.6 // indirect + github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect github.com/stretchr/objx v0.5.2 // indirect - github.com/subosito/gotenv v1.4.2 // indirect + github.com/subosito/gotenv v1.6.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect - github.com/ugorji/go/codec v1.2.12 // indirect - go.etcd.io/etcd/api/v3 v3.5.9 // indirect - go.etcd.io/etcd/client/pkg/v3 v3.5.9 // indirect + github.com/ugorji/go/codec v1.3.0 // indirect + github.com/zeebo/errs v1.4.0 // indirect + go.etcd.io/etcd/api/v3 v3.5.12 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.5.12 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.36.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.62.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 // indirect + go.opentelemetry.io/otel v1.37.0 // indirect + go.opentelemetry.io/otel/metric v1.37.0 // indirect + go.opentelemetry.io/otel/sdk v1.37.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.37.0 // indirect + go.opentelemetry.io/otel/trace v1.37.0 // indirect go.uber.org/atomic v1.9.0 // indirect - go.uber.org/multierr v1.8.0 // indirect + go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.21.0 // indirect - golang.org/x/arch v0.12.0 // indirect - golang.org/x/net v0.33.0 // indirect - golang.org/x/sys v0.30.0 // indirect - golang.org/x/text v0.22.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect - google.golang.org/grpc v1.67.1 // indirect - google.golang.org/protobuf v1.36.1 // indirect + golang.org/x/arch v0.19.0 // indirect + golang.org/x/net v0.46.0 // indirect + golang.org/x/oauth2 v0.33.0 // indirect + golang.org/x/sync v0.18.0 // indirect + golang.org/x/sys v0.37.0 // indirect + golang.org/x/text v0.30.0 // indirect + golang.org/x/time v0.14.0 // indirect + google.golang.org/genproto v0.0.0-20250715232539-7130f93afb79 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250818200422-3122310a409c // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251103181224-f26f9409b101 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/horizon/go.sum b/horizon/go.sum index 07af712e..b7b01bca 100644 --- a/horizon/go.sum +++ b/horizon/go.sum @@ -1,45 +1,41 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI6LDrKU= -github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= +cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY= +cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= +cloud.google.com/go v0.121.6 h1:waZiuajrI28iAf40cWgycWNgaXPO06dupuS+sgibK6c= +cloud.google.com/go v0.121.6/go.mod h1:coChdst4Ea5vUpiALcYKXEpR1S9ZgXbhEzzMcMR66vI= +cloud.google.com/go/auth v0.17.0 h1:74yCm7hCj2rUyyAocqnFzsAYXgJhrG26XCFimrc/Kz4= +cloud.google.com/go/auth v0.17.0/go.mod h1:6wv/t5/6rOPAX4fJiRjKkJCvswLwdet7G8+UGXt7nCQ= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +cloud.google.com/go/iam v1.5.2 h1:qgFRAGEmd8z6dJ/qyEchAuL9jpswyODjA2lS+w234g8= +cloud.google.com/go/iam v1.5.2/go.mod h1:SE1vg0N81zQqLzQEwxL2WI6yhetBdbNQuTvIKCSkUHE= +cloud.google.com/go/logging v1.13.0 h1:7j0HgAp0B94o1YRDqiqm26w4q1rDMH7XNRU34lJXHYc= +cloud.google.com/go/logging v1.13.0/go.mod h1:36CoKh6KA/M0PbhPKMq6/qety2DCAErbhXT62TuXALA= +cloud.google.com/go/longrunning v0.7.0 h1:FV0+SYF1RIj59gyoWDRi45GiYUMM3K1qO51qoboQT1E= +cloud.google.com/go/longrunning v0.7.0/go.mod h1:ySn2yXmjbK9Ba0zsQqunhDkYi0+9rlXIwnoAf+h+TPY= +cloud.google.com/go/monitoring v1.24.2 h1:5OTsoJ1dXYIiMiuL+sYscLc9BumrL3CarVLL7dd7lHM= +cloud.google.com/go/monitoring v1.24.2/go.mod h1:x7yzPWcgDRnPEv3sI+jJGBkwl5qINf+6qY4eq0I9B4U= +cloud.google.com/go/storage v1.57.2 h1:sVlym3cHGYhrp6XZKkKb+92I1V42ks2qKKpB0CF5Mb4= +cloud.google.com/go/storage v1.57.2/go.mod h1:n5ijg4yiRXXpCu0sJTD6k+eMf7GRrJmPyr9YxLXGHOk= +cloud.google.com/go/trace v1.11.6 h1:2O2zjPzqPYAHrn3OKl029qlqG6W8ZdYaOWRyr8NgMT4= +cloud.google.com/go/trace v1.11.6/go.mod h1:GA855OeDEBiBMzcckLPE2kDunIpC72N+Pq8WFieFjnI= +github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEUqFvRDw= +github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 h1:UQUsRi8WTzhZntp5313l+CHIAT95ojUI2lpP/ExlZa4= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0/go.mod h1:Cz6ft6Dkn3Et6l2v2a9/RpN7epQ1GtDlO6lj8bEcOvw= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 h1:owcC2UnmsZycprQ5RfRgjydWhuoxg71LUfyiQdijZuM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0/go.mod h1:ZPpqegjbE99EPKsu3iUWV22A04wzGPcAY/ziSIQEEgs= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.53.0 h1:4LP6hvB4I5ouTbGgWtixJhgED6xdf67twf9PoY96Tbg= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.53.0/go.mod h1:jUZ5LYlw40WMd07qxcQJD5M40aUxrfwqQX1g7zxYnrQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 h1:Ron4zCA/yk6U7WOBXhTJcDpsUBG9npumK6xw2auFltQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0/go.mod h1:cSgYe11MCNYunTnRXrKiR/tHc0eoKjICUuWpNZoVCOo= +github.com/Meesho/go-core v1.30.20 h1:ijkKm9objd09ZMXGgLMVZpj8Q4+0rvMFY/2hSFFIoMI= +github.com/Meesho/go-core v1.30.20/go.mod h1:Ftn5QRPrCwy/c/m0Mp8zoX0dWcW4PZvtPwyi/qFL6lc= +github.com/Meesho/helix-clients v0.8.1 h1:oCmxw62WyXjqMcLacbzU41NQkMyeYGF1V1UQUrAszK8= +github.com/Meesho/helix-clients v0.8.1/go.mod h1:S1mVOB7X3Mp6TkoLHPkxba5892WQiA794oVw339yiRE= +github.com/Meesho/price-aggregator-go/pricingfeatureretrieval/client v0.0.12 h1:PmYPWRThLACd5mOGAA2O5qBc7/Ygb4IszXq9TJpdgYw= +github.com/Meesho/price-aggregator-go/pricingfeatureretrieval/client v0.0.12/go.mod h1:6fvt8B1VeZElMKNotGs11qkdNjWQoTzJrxPVw4g1pI4= github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= @@ -47,158 +43,126 @@ github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLj github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY= github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= +github.com/bits-and-blooms/bitset v1.22.0 h1:Tquv9S8+SGaS3EhyA+up3FXzmkhxPGjQQCkcs2uw7w4= +github.com/bits-and-blooms/bitset v1.22.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= -github.com/bytedance/sonic v1.12.6 h1:/isNmCUF2x3Sh8RAp/4mh4ZGkcFAX/hLrzrK3AvpRzk= -github.com/bytedance/sonic v1.12.6/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= +github.com/bytedance/sonic v1.13.3 h1:MS8gmaH16Gtirygw7jV91pDCN33NyMrPbN7qiYhEsF0= +github.com/bytedance/sonic v1.13.3/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= -github.com/bytedance/sonic/loader v0.2.1 h1:1GgorWTqf12TA8mma4DDSbaQigE2wOgQo7iCjjJv3+E= -github.com/bytedance/sonic/loader v0.2.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= -github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= -github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZwvZJyqeA= +github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= +github.com/cenkalti/backoff/v5 v5.0.2 h1:rIfFVxEf1QsI7E1ZHfp/B4DF/6QBAUhmgkxc0H7Zss8= +github.com/cenkalti/backoff/v5 v5.0.2/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= +github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 h1:aQ3y1lwWyqYPiWZThqv1aFbZMiM9vblcSArJRf2Irls= +github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deckarep/golang-set/v2 v2.8.0 h1:swm0rlPCmdWn9mESxKOjWk8hXSqoxOp+ZlfuyaAdFlQ= +github.com/deckarep/golang-set/v2 v2.8.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= -github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= -github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= -github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/gabriel-vasile/mimetype v1.4.7 h1:SKFKl7kD0RiPdbht0s7hFtjl489WcQ1VyPW8ZzUMYCA= -github.com/gabriel-vasile/mimetype v1.4.7/go.mod h1:GDlAgAyIRT27BhFl53XNAFtfjzOkLaF35JdEG0P7LtU= +github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M= +github.com/envoyproxy/go-control-plane v0.13.4/go.mod h1:kDfuBlDVsSj2MjrLEtRWtHlsWIFcGyB2RMO44Dc5GZA= +github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A= +github.com/envoyproxy/go-control-plane/envoy v1.32.4/go.mod h1:Gzjc5k8JcJswLjAx1Zm+wSYE20UrLtt7JZMWiWQXQEw= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= +github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= +github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= +github.com/failsafe-go/failsafe-go v0.6.9 h1:7HWEzOlFOjNerxgWd8onWA2j/aEuqyAtuX6uWya/364= +github.com/failsafe-go/failsafe-go v0.6.9/go.mod h1:zb7xfp1/DJ7Mn4xJhVSZ9F2qmmMEGvYHxEOHYK5SIm0= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/gabriel-vasile/mimetype v1.4.9 h1:5k+WDwEsD9eTLL8Tz3L0VnmVh9QxGjRmjBvAG7U/oYY= +github.com/gabriel-vasile/mimetype v1.4.9/go.mod h1:WnSQhFKJuBlRyLiKohA/2DtIlPFAbguNaG7QCHcyGok= github.com/gin-contrib/cors v1.7.3 h1:hV+a5xp8hwJoTw7OY+a70FsL8JkVVFTXw9EcfrYUdns= github.com/gin-contrib/cors v1.7.3/go.mod h1:M3bcKZhxzsvI+rlRSkkxHyljJt1ESd93COUvemZ79j4= -github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= -github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= -github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w= +github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= +github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ= +github.com/gin-gonic/gin v1.10.1/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/go-jose/go-jose/v4 v4.1.2 h1:TK/7NqRQZfgAh+Td8AlsrvtPoUyiHh0LqVvokh+1vHI= +github.com/go-jose/go-jose/v4 v4.1.2/go.mod h1:22cg9HWM1pOlnRiY+9cQYJ9XHmya1bYW8OeDM6Ku6Oo= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.23.0 h1:/PwmTwZhS0dPkav3cdK9kV1FsAmrL8sThn8IHr/sO+o= -github.com/go-playground/validator/v10 v10.23.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/go-playground/validator/v10 v10.27.0 h1:w8+XrWVMhGkxOaaowyKH35gFydVHOvC0/uWoy2Fzwn4= +github.com/go-playground/validator/v10 v10.27.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= -github.com/goccy/go-json v0.10.4 h1:JSwxQzIqKfmFX1swYPpUThQZp/Ka4wzJdK0LWVytLPM= -github.com/goccy/go-json v0.10.4/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= +github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/go-zookeeper/zk v1.0.4 h1:DPzxraQx7OrPyXq2phlGlNSIyWEsAox0RJmjTseMV6I= +github.com/go-zookeeper/zk v1.0.4/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gocql/gocql v1.7.0 h1:O+7U7/1gSN7QTEAaMEsJc1Oq2QHXvCWoF3DFK9HDHus= github.com/gocql/gocql v1.7.0/go.mod h1:vnlvXyFZeLBF0Wy+RS8hrOdbn0UWsWtdg07XJnFxZ+4= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.7 h1:zrn2Ee/nWmHulBx5sAVrGgAa0f2/R35S4DJwfFaUPFQ= +github.com/googleapis/enterprise-certificate-proxy v0.3.7/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= +github.com/googleapis/gax-go/v2 v2.15.0 h1:SyjDc1mGgZU5LncH8gimWo9lW1DtIfPibOG81vgd/bo= +github.com/googleapis/gax-go/v2 v2.15.0/go.mod h1:zVVkkxAQHa1RQpg9z2AUCMnKhi0Qld9rcmyfL1OZhoc= +github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 h1:X5VWvz21y3gzm9Nw/kaUeku/1+uBhcekkmy4IkffJww= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1/go.mod h1:Zanoh4+gvIgluNqcfMVTJueD4wSS5hT7zTt4Mrutd90= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= -github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -208,47 +172,50 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= -github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= -github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= -github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= -github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= +github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= +github.com/sagikazarmark/locafero v0.9.0 h1:GbgQGNtTrEmddYDSAH9QLRyfAHY12md+8YFTqyMTC9k= +github.com/sagikazarmark/locafero v0.9.0/go.mod h1:UBUyz37V+EdMS3hDF3QWIiVr/2dPrx49OMO0Bn0hJqk= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= -github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= -github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= -github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.16.0 h1:rGGH0XDZhdUOryiDWjmIvUSWpbNqisK8Wk0Vyefw8hc= -github.com/spf13/viper v1.16.0/go.mod h1:yg78JgCJcbrQOvV9YLXgkLaZqUidkY9K+Dd1FofRzQg= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spf13/afero v1.14.0 h1:9tH6MapGnn/j0eb0yIXiLjERO8RB6xIVZRDCX7PtqWA= +github.com/spf13/afero v1.14.0/go.mod h1:acJQ8t0ohCGuMN3O+Pv0V0hgMxNYDlvdk+VTfyZmbYo= +github.com/spf13/cast v1.9.2 h1:SsGfm7M8QOFtEzumm7UZrZdLLquNdzFYfIbEXntcFbE= +github.com/spf13/cast v1.9.2/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4= +github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4= +github.com/spiffe/go-spiffe/v2 v2.5.0 h1:N2I01KCUkv1FAjZXJMwh95KK1ZIQLYbPfhaxw8WS0hE= +github.com/spiffe/go-spiffe/v2 v2.5.0/go.mod h1:P+NxobPc6wXhVtINNtFjNWGBTreew1GBUCwT2wPmb7g= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -256,369 +223,146 @@ github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= -github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= -github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= -github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/ugorji/go/codec v1.3.0 h1:Qd2W2sQawAfG8XSvzwhBeoGq71zXOC/Q1E9y/wUcsUA= +github.com/ugorji/go/codec v1.3.0/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.etcd.io/etcd/api/v3 v3.5.9 h1:4wSsluwyTbGGmyjJktOf3wFQoTBIURXHnq9n/G/JQHs= -go.etcd.io/etcd/api/v3 v3.5.9/go.mod h1:uyAal843mC8uUVSLWz6eHa/d971iDGnCRpmKd2Z+X8k= -go.etcd.io/etcd/client/pkg/v3 v3.5.9 h1:oidDC4+YEuSIQbsR94rY9gur91UPL6DnxDCIYd2IGsE= -go.etcd.io/etcd/client/pkg/v3 v3.5.9/go.mod h1:y+CzeSmkMpWN2Jyu1npecjB9BBnABxGM4pN8cGuJeL4= -go.etcd.io/etcd/client/v3 v3.5.9 h1:r5xghnU7CwbUxD/fbUtRyJGaYNfDun8sp/gTr1hew6E= -go.etcd.io/etcd/client/v3 v3.5.9/go.mod h1:i/Eo5LrZ5IKqpbtpPDuaUnDOUv471oDg8cjQaUr2MbA= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM= +github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= +go.etcd.io/etcd/api/v3 v3.5.12 h1:W4sw5ZoU2Juc9gBWuLk5U6fHfNVyY1WC5g9uiXZio/c= +go.etcd.io/etcd/api/v3 v3.5.12/go.mod h1:Ot+o0SWSyT6uHhA56al1oCED0JImsRiU9Dc26+C2a+4= +go.etcd.io/etcd/client/pkg/v3 v3.5.12 h1:EYDL6pWwyOsylrQyLp2w+HkQ46ATiOvoEdMarindU2A= +go.etcd.io/etcd/client/pkg/v3 v3.5.12/go.mod h1:seTzl2d9APP8R5Y2hFL3NVlD6qC/dOT+3kvrqPyTas4= +go.etcd.io/etcd/client/v3 v3.5.12 h1:v5lCPXn1pf1Uu3M4laUE2hp/geOTc5uPcYYsNe1lDxg= +go.etcd.io/etcd/client/v3 v3.5.12/go.mod h1:tSbBCakoWmmddL+BKVAJHa9km+O/E+bumDe9mSbPiqw= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/detectors/gcp v1.36.0 h1:F7q2tNlCaHY9nMKHR6XH9/qkp8FktLnIcy6jJNyOCQw= +go.opentelemetry.io/contrib/detectors/gcp v1.36.0/go.mod h1:IbBN8uAIIx734PTonTPxAxnjc2pQTxWNkwfstZ+6H2k= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.62.0 h1:rbRJ8BBoVMsQShESYZ0FkvcITu8X8QNwJogcLUmDNNw= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.62.0/go.mod h1:ru6KHrNtNHxM4nD/vd6QrLVWgKhxPYgblq4VAtNawTQ= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 h1:Hf9xI/XLML9ElpiHVDNwvqI0hIFlzV8dgIr35kV1kRU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0/go.mod h1:NfchwuyNoMcZ5MLHwPrODwUF1HWCXWrL31s8gSAdIKY= +go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= +go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 h1:Ahq7pZmv87yiyn3jeFz/LekZmPLLdKejuO3NcK9MssM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0/go.mod h1:MJTqhM0im3mRLw1i8uGHnCvUEeS7VwRyxlLC78PA18M= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0 h1:EtFWSnwW9hGObjkIdmlnWSydO+Qs8OwzfzXLUPg4xOc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0/go.mod h1:QjUEoiGCPkvFZ/MjK6ZZfNOS6mfVEVKYE99dFhuN2LI= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0 h1:rixTyDGXFxRy1xzhKrotaHy3/KXdPhlWARrCgK+eqUY= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0/go.mod h1:dowW6UsM9MKbJq5JTz2AMVp3/5iW5I/TStsk8S+CfHw= +go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= +go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= +go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= +go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= +go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= +go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= +go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz09os= +go.opentelemetry.io/proto/otlp v1.7.0/go.mod h1:fSKjH6YJ7HDlwzltzyMj036AJ3ejJLCgCSHGj4efDDo= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= -go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= -golang.org/x/arch v0.12.0 h1:UsYJhbzPYGsT0HbEdmYcqtCv8UNGvnaL561NnIUvaKg= -golang.org/x/arch v0.12.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/arch v0.19.0 h1:LmbDQUodHThXE+htjrnmVD73M//D9GTH6wFZjyDkjyU= +golang.org/x/arch v0.19.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= -golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= +golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= -golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4= +golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= +golang.org/x/oauth2 v0.33.0 h1:4Q+qn+E5z8gPRJfmRy7C2gGG3T4jIprK6aSYgTXGRpo= +golang.org/x/oauth2 v0.33.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= -golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= +golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= -golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= +golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 h1:wKguEg1hsxI2/L3hUYrpo1RVi48K+uTyzKqprwLXsb8= -google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142/go.mod h1:d6be+8HhtEtucleCbxpPW9PA9XwISACu8nvpPqF0BVo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 h1:e7S5W7MGGLaSu8j3YjdezkZ+m1/Nm0uRVRMEMGk26Xs= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= -google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= -google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/api v0.256.0 h1:u6Khm8+F9sxbCTYNoBHg6/Hwv0N/i+V94MvkOSor6oI= +google.golang.org/api v0.256.0/go.mod h1:KIgPhksXADEKJlnEoRa9qAII4rXcy40vfI8HRqcU964= +google.golang.org/genproto v0.0.0-20250715232539-7130f93afb79 h1:Nt6z9UHqSlIdIGJdz6KhTIs2VRx/iOsA5iE8bmQNcxs= +google.golang.org/genproto v0.0.0-20250715232539-7130f93afb79/go.mod h1:kTmlBHMPqR5uCZPBvwa2B18mvubkjyY3CRLI0c6fj0s= +google.golang.org/genproto/googleapis/api v0.0.0-20250818200422-3122310a409c h1:AtEkQdl5b6zsybXcbz00j1LwNodDuH6hVifIaNqk7NQ= +google.golang.org/genproto/googleapis/api v0.0.0-20250818200422-3122310a409c/go.mod h1:ea2MjsO70ssTfCjiwHgI0ZFqcw45Ksuk2ckf9G468GA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251103181224-f26f9409b101 h1:tRPGkdGHuewF4UisLzzHHr1spKw92qLM98nIzxbC0wY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251103181224-f26f9409b101/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= +google.golang.org/grpc v1.76.0 h1:UnVkv1+uMLYXoIz6o7chp59WfQUYA2ex/BXQ9rHZu7A= +google.golang.org/grpc v1.76.0/go.mod h1:Ju12QI8M6iQJtbcsV+awF5a4hfJMLi4X0JLo94ULZ6c= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= -gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= @@ -631,14 +375,4 @@ gorm.io/driver/mysql v1.5.6/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkD gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= gorm.io/gorm v1.25.10 h1:dQpO+33KalOA+aFYGlK+EfxcI5MbO7EP2yYygwh9h+s= gorm.io/gorm v1.25.10/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/horizon/internal/application/controller/controller.go b/horizon/internal/application/controller/controller.go new file mode 100644 index 00000000..61c0fd39 --- /dev/null +++ b/horizon/internal/application/controller/controller.go @@ -0,0 +1,90 @@ +package controller + +import ( + "sync" + + "github.com/Meesho/BharatMLStack/horizon/internal/application/handler" + "github.com/Meesho/BharatMLStack/horizon/pkg/api" + "github.com/gin-gonic/gin" +) + +type Config interface { + Onboard(ctx *gin.Context) + GetAll(ctx *gin.Context) + Edit(ctx *gin.Context) +} + +var ( + configController Config + once sync.Once + emptyResponse = "" +) + +type V1 struct { + Config handler.Config +} + +func NewConfigController() Config { + if configController == nil { + once.Do(func() { + configController = &V1{ + Config: handler.NewConfigHandler(1), + } + }) + } + return configController +} + +func (c *V1) Onboard(ctx *gin.Context) { + var request handler.OnboardRequest + if err := ctx.ShouldBindJSON(&request); err != nil { + ctx.JSON(api.NewBadRequestError(err.Error()).StatusCode, handler.Response{ + Error: err.Error(), + Data: handler.Message{Message: emptyResponse}, + }) + return + } + response, err := c.Config.Onboard(request) + if err != nil { + ctx.JSON(api.NewInternalServerError(err.Error()).StatusCode, handler.Response{ + Error: err.Error(), + Data: handler.Message{Message: emptyResponse}, + }) + return + } + ctx.JSON(200, response) +} + +func (c *V1) GetAll(ctx *gin.Context) { + response, err := c.Config.GetAll() + if err != nil { + ctx.JSON(api.NewInternalServerError(err.Error()).StatusCode, handler.Response{ + Error: err.Error(), + Data: handler.Message{Message: emptyResponse}, + }) + return + } + ctx.JSON(200, response) +} + +func (c *V1) Edit(ctx *gin.Context) { + var request handler.EditRequest + token := ctx.Param("token") + if err := ctx.ShouldBindJSON(&request); err != nil { + ctx.JSON(api.NewBadRequestError(err.Error()).StatusCode, handler.Response{ + Error: err.Error(), + Data: handler.Message{Message: emptyResponse}, + }) + return + } + request.Payload.AppToken = token + response, err := c.Config.Edit(request) + if err != nil { + ctx.JSON(api.NewInternalServerError(err.Error()).StatusCode, handler.Response{ + Error: err.Error(), + Data: handler.Message{Message: emptyResponse}, + }) + return + } + ctx.JSON(200, response) +} diff --git a/horizon/internal/application/handler/application.go b/horizon/internal/application/handler/application.go new file mode 100644 index 00000000..954d84ca --- /dev/null +++ b/horizon/internal/application/handler/application.go @@ -0,0 +1,114 @@ +package handler + +import ( + "fmt" + "time" + + "github.com/Meesho/BharatMLStack/horizon/internal/repositories/sql/application" + "github.com/Meesho/BharatMLStack/horizon/pkg/infra" + "github.com/google/uuid" + "github.com/rs/zerolog/log" +) + +var ( + emptyResponse = "" + activeTrue = true +) + +type applicationConfig struct { + ApplicationRepo application.ApplicationRepository +} + +func InitV1ConfigHandler() Config { + if config == nil { + conn, err := infra.SQL.GetConnection() + if err != nil { + log.Fatal().Err(err).Msg("Failed to get SQL connection") + } + sqlConn := conn.(*infra.SQLConnection) + + ApplicationRepo, err := application.Repository(sqlConn) + if err != nil { + log.Fatal().Err(err).Msg("Failed to create config repository") + } + + config = &applicationConfig{ + ApplicationRepo: ApplicationRepo, + } + } + return config +} + +func (c *applicationConfig) Onboard(request OnboardRequest) (Response, error) { + appToken := uuid.New().String() + + application := application.Application{ + AppToken: appToken, + Bu: request.Payload.Bu, + Team: request.Payload.Team, + Service: request.Payload.Service, + Active: activeTrue, + CreatedBy: request.CreatedBy, + } + + err := c.ApplicationRepo.Create(&application) + if err != nil { + return Response{ + Error: err.Error(), + Data: Message{Message: emptyResponse}, + }, err + } + return Response{ + Error: emptyResponse, + Data: Message{Message: fmt.Sprintf("Application onboarded successfully with token: %s", appToken)}, + }, nil +} + +func (c *applicationConfig) GetAll() (GetAllResponse, error) { + applications, err := c.ApplicationRepo.GetAll() + if err != nil { + return GetAllResponse{}, err + } + + response := []ApplicationConfig{} + + for _, application := range applications { + response = append(response, ApplicationConfig{ + AppToken: application.AppToken, + Active: application.Active, + Bu: application.Bu, + Team: application.Team, + Service: application.Service, + CreatedBy: application.CreatedBy, + UpdatedBy: application.UpdatedBy, + CreatedAt: application.CreatedAt.Format(time.RFC3339), + UpdatedAt: application.UpdatedAt.Format(time.RFC3339), + }) + } + return GetAllResponse{ + Data: response, + }, nil +} + +func (c *applicationConfig) Edit(request EditRequest) (Response, error) { + application := application.Application{ + AppToken: request.Payload.AppToken, + Bu: request.Payload.Bu, + Team: request.Payload.Team, + Service: request.Payload.Service, + UpdatedBy: request.CreatedBy, + } + + err := c.ApplicationRepo.Update(&application) + if err != nil { + return Response{ + Error: err.Error(), + Data: Message{Message: emptyResponse}, + }, err + } + + return Response{ + Error: emptyResponse, + Data: Message{Message: fmt.Sprintf("Application updated successfully with token: %s", request.Payload.AppToken)}, + }, nil +} diff --git a/horizon/internal/application/handler/config.go b/horizon/internal/application/handler/config.go new file mode 100644 index 00000000..59b594e6 --- /dev/null +++ b/horizon/internal/application/handler/config.go @@ -0,0 +1,7 @@ +package handler + +type Config interface { + Onboard(OnboardRequest) (Response, error) + GetAll() (GetAllResponse, error) + Edit(EditRequest) (Response, error) +} diff --git a/horizon/internal/application/handler/init.go b/horizon/internal/application/handler/init.go new file mode 100644 index 00000000..05cd5ff6 --- /dev/null +++ b/horizon/internal/application/handler/init.go @@ -0,0 +1,14 @@ +package handler + +var ( + config Config +) + +func NewConfigHandler(version int) Config { + switch version { + case 1: + return InitV1ConfigHandler() + default: + return nil + } +} diff --git a/horizon/internal/application/handler/models.go b/horizon/internal/application/handler/models.go new file mode 100644 index 00000000..20ddc04e --- /dev/null +++ b/horizon/internal/application/handler/models.go @@ -0,0 +1,50 @@ +package handler + +type OnboardRequestPayload struct { + Bu string `json:"bu"` + Team string `json:"team"` + Service string `json:"service"` +} + +type OnboardRequest struct { + Payload OnboardRequestPayload `json:"payload"` + CreatedBy string `json:"created_by"` +} + +type Message struct { + Message string `json:"message"` +} + +type Response struct { + Error string `json:"error"` + Data Message `json:"data"` +} + +type ApplicationConfig struct { + AppToken string `json:"app_token"` + Active bool `json:"active"` + Bu string `json:"bu"` + Team string `json:"team"` + Service string `json:"service"` + CreatedBy string `json:"created_by"` + UpdatedBy string `json:"updated_by"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` +} + +type GetAllResponse struct { + Data []ApplicationConfig `json:"data"` +} + +type EditRequestPayload struct { + AppToken string `json:"app_token"` + Bu string `json:"bu"` + Team string `json:"team"` + Service string `json:"service"` + UpdatedBy string `json:"updated_by"` +} + +type EditRequest struct { + Payload EditRequestPayload `json:"payload"` + CreatedBy string `json:"created_by"` +} diff --git a/horizon/internal/application/route/router.go b/horizon/internal/application/route/router.go new file mode 100644 index 00000000..12d65c66 --- /dev/null +++ b/horizon/internal/application/route/router.go @@ -0,0 +1,34 @@ +package route + +import ( + "sync" + + "github.com/Meesho/BharatMLStack/horizon/internal/application/controller" + "github.com/Meesho/BharatMLStack/horizon/pkg/httpframework" +) + +var initApplicationRouterOnce sync.Once + +func Init() { + initApplicationRouterOnce.Do(func() { + api := httpframework.Instance().Group("/api") + { + v1 := api.Group("/v1/horizon") + { + + // Application Config Registry/Discovery routes + registry := v1.Group("/application-registry/applications") + { + registry.POST("", controller.NewConfigController().Onboard) + registry.PUT("/:token", controller.NewConfigController().Edit) + } + + discovery := v1.Group("/application-discovery/applications") + { + discovery.GET("", controller.NewConfigController().GetAll) + } + + } + } + }) +} diff --git a/horizon/internal/configs/app_config.go b/horizon/internal/configs/app_config.go new file mode 100644 index 00000000..122a4073 --- /dev/null +++ b/horizon/internal/configs/app_config.go @@ -0,0 +1,109 @@ +package configs + +type Configs struct { + // App configuration + AppName string `mapstructure:"app_name"` + AppEnv string `mapstructure:"app_env"` + AppGcPercentage int `mapstructure:"app_gc_percentage"` + AppLogLevel string `mapstructure:"app_log_level"` + AppMetricSamplingRate float64 `mapstructure:"app_metric_sampling_rate"` + AppPort int `mapstructure:"app_port"` + + // MySQL configuration + MysqlDbName string `mapstructure:"mysql_db_name"` + MysqlMasterHost string `mapstructure:"mysql_master_host"` + MysqlMasterMaxPoolSize string `mapstructure:"mysql_master_max_pool_size"` + MysqlMasterMinPoolSize string `mapstructure:"mysql_master_min_pool_size"` + MysqlMasterPassword string `mapstructure:"mysql_master_password"` + MysqlMasterPort int `mapstructure:"mysql_master_port"` + MysqlMasterUsername string `mapstructure:"mysql_master_username"` + MysqlSlaveHost string `mapstructure:"mysql_slave_host"` + MysqlSlaveMaxPoolSize string `mapstructure:"mysql_slave_max_pool_size"` + MysqlSlaveMinPoolSize string `mapstructure:"mysql_slave_min_pool_size"` + MysqlSlavePassword string `mapstructure:"mysql_slave_password"` + MysqlSlavePort int `mapstructure:"mysql_slave_port"` + MysqlSlaveUsername string `mapstructure:"mysql_slave_username"` + + // Etcd configuration + EtcdPassword string `mapstructure:"etcd_password"` + EtcdServer string `mapstructure:"etcd_server"` + EtcdUsername string `mapstructure:"etcd_username"` + EtcdWatcherEnabled bool `mapstructure:"etcd_watcher_enabled"` + + // Ringmaster configuration + RingmasterApiKey string `mapstructure:"ringmaster_api_key"` + RingmasterAuthorization string `mapstructure:"ringmaster_authorization"` + RingmasterBaseUrl string `mapstructure:"ringmaster_base_url"` + RingmasterEnvironment string `mapstructure:"ringmaster_environment"` + RingmasterMiscSession string `mapstructure:"ringmaster_misc_session"` + + // Slack configuration + SlackCcTags string `mapstructure:"slack_cc_tags"` + SlackChannel string `mapstructure:"slack_channel"` + SlackInactiveDays int `mapstructure:"slack_inactive_days"` + SlackWebhookUrl string `mapstructure:"slack_webhook_url"` + + // Vmselect configuration + VmselectApiKey string `mapstructure:"vmselect_api_key"` + VmselectBaseUrl string `mapstructure:"vmselect_base_url"` + VmselectStartDaysAgo int `mapstructure:"vmselect_start_days_ago"` + + // Zookeeper configuration + ZookeeperBasePath string `mapstructure:"zookeeper_base_path"` + ZookeeperServer string `mapstructure:"zookeeper_server"` + ZookeeperWatcher string `mapstructure:"zookeeper_watcher"` + + // Horizon configuration + HorizonAppName string `mapstructure:"horizon_app_name"` + HorizonPort string `mapstructure:"horizon_port"` + HorizonServer string `mapstructure:"horizon_server"` + + // Other configurations + DefaultCpuThreshold string `mapstructure:"default_cpu_threshold"` + DefaultGpuThreshold string `mapstructure:"default_gpu_threshold"` + DefaultModelPath string `mapstructure:"default_model_path"` + + FeatureGroupDataTypeMappingUrl string `mapstructure:"feature_group_data_type_mapping_url"` + + GcsModelBucket string `mapstructure:"gcs_model_bucket"` + GcsModelBasePath string `mapstructure:"gcs_model_base_path"` + + GrafanaBaseUrl string `mapstructure:"grafana_base_url"` + + HostUrlSuffix string `mapstructure:"host_url_suffix"` + + NumerixAppName string `mapstructure:"numerix_app_name"` + NumerixMonitoringUrl string `mapstructure:"numerix_monitoring_url"` + + MaxNumerixInactiveAge int `mapstructure:"max_numerix_inactive_age"` + MaxInferflowInactiveAge int `mapstructure:"max_inferflow_inactive_age"` + MaxPredatorInactiveAge int `mapstructure:"max_predator_inactive_age"` + + InferflowAppName string `mapstructure:"inferflow_app_name"` + + OnlineFeatureMappingUrl string `mapstructure:"online_feature_mapping_url"` + + PhoenixServerBaseUrl string `mapstructure:"phoenix_server_base_url"` + + PredatorMonitoringUrl string `mapstructure:"predator_monitoring_url"` + + ScheduledCronExpression string `mapstructure:"scheduled_cron_expression"` + + TestDeployableID int `mapstructure:"test_deployable_id"` + TestGpuDeployableID int `mapstructure:"test_gpu_deployable_id"` + + // Pricing Feature Retrieval Service configuration + PricingFeatureRetrievalBatchSize string `mapstructure:"pricing_feature_retrieval_batch_size"` + PricingFeatureRetrievalDialTimeout string `mapstructure:"pricing_feature_retrieval_dial_timeout_ms"` + PricingFeatureRetrievalHost string `mapstructure:"pricing_feature_retrieval_host"` + PricingFeatureRetrievalIdleConnTimeout string `mapstructure:"pricing_feature_retrieval_idle_conn_timeout_ms"` + PricingFeatureRetrievalMaxIdleConns string `mapstructure:"pricing_feature_retrieval_max_idle_conns"` + PricingFeatureRetrievalMaxIdleConnsPerHost string `mapstructure:"pricing_feature_retrieval_max_idle_conns_per_host"` + PricingFeatureRetrievalPort string `mapstructure:"pricing_feature_retrieval_port"` + PricingFeatureRetrievalGrpcPlainText bool `mapstructure:"pricing_feature_retrieval_grpc_plain_text"` + PricingFeatureRetrievalTimeoutMs string `mapstructure:"pricing_feature_retrieval_timeout_in_ms"` + + OnlineFeatureStoreAppName string `mapstructure:"online_feature_store_app_name"` +} + +type DynamicConfigs struct{} diff --git a/horizon/internal/connectionconfig/controller/controller.go b/horizon/internal/connectionconfig/controller/controller.go new file mode 100644 index 00000000..5df396ef --- /dev/null +++ b/horizon/internal/connectionconfig/controller/controller.go @@ -0,0 +1,98 @@ +package controller + +import ( + "strconv" + "sync" + + "github.com/Meesho/BharatMLStack/horizon/internal/connectionconfig/handler" + "github.com/Meesho/BharatMLStack/horizon/pkg/api" + "github.com/gin-gonic/gin" +) + +type Config interface { + Onboard(ctx *gin.Context) + GetAll(ctx *gin.Context) + Edit(ctx *gin.Context) +} + +var ( + configController Config + once sync.Once + emptyResponse = "" +) + +type V1 struct { + Config handler.Config +} + +func NewConfigController() Config { + if configController == nil { + once.Do(func() { + configController = &V1{ + Config: handler.NewConfigHandler(1), + } + }) + } + return configController +} + +func (c *V1) Onboard(ctx *gin.Context) { + var request handler.OnboardRequest + if err := ctx.ShouldBindJSON(&request); err != nil { + ctx.JSON(api.NewBadRequestError(err.Error()).StatusCode, handler.Response{ + Error: err.Error(), + Data: handler.Message{Message: emptyResponse}, + }) + return + } + response, err := c.Config.Onboard(request) + if err != nil { + ctx.JSON(api.NewInternalServerError(err.Error()).StatusCode, handler.Response{ + Error: err.Error(), + Data: handler.Message{Message: emptyResponse}, + }) + return + } + ctx.JSON(200, response) +} + +func (c *V1) GetAll(ctx *gin.Context) { + response, err := c.Config.GetAll() + if err != nil { + ctx.JSON(api.NewInternalServerError(err.Error()).StatusCode, handler.Response{ + Error: err.Error(), + Data: handler.Message{Message: emptyResponse}, + }) + return + } + ctx.JSON(200, response) +} + +func (c *V1) Edit(ctx *gin.Context) { + var request handler.EditRequest + id, err := strconv.Atoi(ctx.Param("id")) + if err != nil { + ctx.JSON(api.NewBadRequestError(err.Error()).StatusCode, handler.Response{ + Error: err.Error(), + Data: handler.Message{Message: emptyResponse}, + }) + return + } + if err := ctx.ShouldBindJSON(&request); err != nil { + ctx.JSON(api.NewBadRequestError(err.Error()).StatusCode, handler.Response{ + Error: err.Error(), + Data: handler.Message{Message: emptyResponse}, + }) + return + } + request.Payload.Id = uint(id) + response, err := c.Config.Edit(request) + if err != nil { + ctx.JSON(api.NewInternalServerError(err.Error()).StatusCode, handler.Response{ + Error: err.Error(), + Data: handler.Message{Message: emptyResponse}, + }) + return + } + ctx.JSON(200, response) +} diff --git a/horizon/internal/connectionconfig/handler/config.go b/horizon/internal/connectionconfig/handler/config.go new file mode 100644 index 00000000..59b594e6 --- /dev/null +++ b/horizon/internal/connectionconfig/handler/config.go @@ -0,0 +1,7 @@ +package handler + +type Config interface { + Onboard(OnboardRequest) (Response, error) + GetAll() (GetAllResponse, error) + Edit(EditRequest) (Response, error) +} diff --git a/horizon/internal/connectionconfig/handler/connection_config.go b/horizon/internal/connectionconfig/handler/connection_config.go new file mode 100644 index 00000000..f2ca2979 --- /dev/null +++ b/horizon/internal/connectionconfig/handler/connection_config.go @@ -0,0 +1,233 @@ +package handler + +import ( + "fmt" + "time" + + connection_config "github.com/Meesho/BharatMLStack/horizon/internal/repositories/sql/connectionconfig" + "github.com/Meesho/BharatMLStack/horizon/pkg/infra" + "github.com/rs/zerolog/log" +) + +var ( + emptyResponse = "" + activeTrue = true + grpcConfig = "grpc" + httpConfig = "http" +) + +type ConnectionConfig struct { + connectionConfigRepo connection_config.ConnectionConfigRepository +} + +func InitV1ConfigHandler() Config { + if config == nil { + conn, err := infra.SQL.GetConnection() + if err != nil { + log.Fatal().Err(err).Msg("Failed to get SQL connection") + } + sqlConn := conn.(*infra.SQLConnection) + + connectionConfigRepo, err := connection_config.Repository(sqlConn) + if err != nil { + log.Fatal().Err(err).Msg("Failed to create config repository") + } + + config = &ConnectionConfig{ + connectionConfigRepo: connectionConfigRepo, + } + } + return config +} + +func (c *ConnectionConfig) Onboard(request OnboardRequest) (Response, error) { + var application connection_config.ConnectionConfig + + if request.Payload.ConnProtocol == grpcConfig { + grpcCfg := connection_config.GrpcConfig{ + Deadline: request.Payload.Deadline, + PlainText: request.Payload.PlainText, + GrpcChannelAlgorithm: request.Payload.GrpcChannelAlgorithm, + ChannelThreadPoolSize: request.Payload.ChannelThreadPoolSize, + BoundedQueueSize: request.Payload.BoundedQueueSize, + } + + application = connection_config.ConnectionConfig{ + Service: request.Payload.Service, + ConnProtocol: request.Payload.ConnProtocol, + Default: request.Payload.Default, + GrpcConfig: grpcCfg, + Active: activeTrue, + CreatedBy: request.CreatedBy, + } + } else if request.Payload.ConnProtocol == httpConfig { + httpCfg := connection_config.HttpConfig{ + Timeout: request.Payload.Timeout, + MaxIdleConnection: request.Payload.MaxIdleConnection, + MaxConnectionPerHost: request.Payload.MaxConnectionPerHost, + IdleConnectionTimeout: request.Payload.IdleConnectionTimeout, + KeepAliveTime: request.Payload.KeepAliveTime, + } + + application = connection_config.ConnectionConfig{ + Service: request.Payload.Service, + ConnProtocol: request.Payload.ConnProtocol, + Default: request.Payload.Default, + HttpConfig: httpCfg, + Active: activeTrue, + CreatedBy: request.CreatedBy, + } + } + + err := c.connectionConfigRepo.Create(&application) + if err != nil { + return Response{ + Error: err.Error(), + Data: Message{Message: emptyResponse}, + }, err + } + return Response{ + Error: emptyResponse, + Data: Message{Message: fmt.Sprintf("Application onboarded successfully with id: %d", application.Id)}, + }, nil +} + +func (c *ConnectionConfig) GetAll() (GetAllResponse, error) { + connectionConfigs, err := c.connectionConfigRepo.GetAll() + if err != nil { + return GetAllResponse{}, err + } + + response := []Configs{} + + for _, connectionConfig := range connectionConfigs { + + if connectionConfig.ConnProtocol == grpcConfig { + response = append(response, Configs{ + Id: connectionConfig.Id, + Default: connectionConfig.Default, + Service: connectionConfig.Service, + ConnProtocol: connectionConfig.ConnProtocol, + Config: ConnConfig{ + GrpcConfig: GrpcConfig{ + Deadline: connectionConfig.GrpcConfig.Deadline, + PlainText: connectionConfig.GrpcConfig.PlainText, + GrpcChannelAlgorithm: connectionConfig.GrpcConfig.GrpcChannelAlgorithm, + ChannelThreadPoolSize: connectionConfig.GrpcConfig.ChannelThreadPoolSize, + BoundedQueueSize: connectionConfig.GrpcConfig.BoundedQueueSize, + }, + }, + CreatedBy: connectionConfig.CreatedBy, + UpdatedBy: connectionConfig.UpdatedBy, + CreatedAt: connectionConfig.CreatedAt.Format(time.RFC3339), + UpdatedAt: connectionConfig.UpdatedAt.Format(time.RFC3339), + }) + } else if connectionConfig.ConnProtocol == httpConfig { + response = append(response, Configs{ + Id: connectionConfig.Id, + Default: connectionConfig.Default, + Service: connectionConfig.Service, + ConnProtocol: connectionConfig.ConnProtocol, + Config: ConnConfig{ + HttpConfig: HttpConfig{ + Timeout: connectionConfig.HttpConfig.Timeout, + MaxIdleConnection: connectionConfig.HttpConfig.MaxIdleConnection, + MaxConnectionPerHost: connectionConfig.HttpConfig.MaxConnectionPerHost, + IdleConnectionTimeout: connectionConfig.HttpConfig.IdleConnectionTimeout, + KeepAliveTime: connectionConfig.HttpConfig.KeepAliveTime, + }, + }, + UpdatedBy: connectionConfig.UpdatedBy, + CreatedAt: connectionConfig.CreatedAt.Format(time.RFC3339), + UpdatedAt: connectionConfig.UpdatedAt.Format(time.RFC3339), + }) + } + } + return GetAllResponse{ + Data: response, + }, nil +} + +func (c *ConnectionConfig) Edit(request EditRequest) (Response, error) { + var connectionConfig connection_config.ConnectionConfig + + // First check if the connection exists + configs, err := c.connectionConfigRepo.GetAll() + if err != nil { + return Response{ + Error: err.Error(), + Data: Message{Message: emptyResponse}, + }, err + } + + found := false + for _, config := range configs { + if config.Id == request.Payload.Id { + found = true + break + } + } + + if !found { + return Response{ + Error: "Connection config not found", + Data: Message{Message: emptyResponse}, + }, fmt.Errorf("connection config with id %d not found", request.Payload.Id) + } + + if request.Payload.ConnProtocol == grpcConfig { + grpcCfg := connection_config.GrpcConfig{ + Deadline: request.Payload.Deadline, + PlainText: request.Payload.PlainText, + GrpcChannelAlgorithm: request.Payload.GrpcChannelAlgorithm, + ChannelThreadPoolSize: request.Payload.ChannelThreadPoolSize, + BoundedQueueSize: request.Payload.BoundedQueueSize, + } + + connectionConfig = connection_config.ConnectionConfig{ + Id: request.Payload.Id, + Default: request.Payload.Default, + Service: request.Payload.Service, + ConnProtocol: request.Payload.ConnProtocol, + GrpcConfig: grpcCfg, + Active: activeTrue, + UpdatedBy: request.UpdatedBy, + } + } else if request.Payload.ConnProtocol == httpConfig { + httpCfg := connection_config.HttpConfig{ + Timeout: request.Payload.Timeout, + MaxIdleConnection: request.Payload.MaxIdleConnection, + MaxConnectionPerHost: request.Payload.MaxConnectionPerHost, + IdleConnectionTimeout: request.Payload.IdleConnectionTimeout, + KeepAliveTime: request.Payload.KeepAliveTime, + } + + connectionConfig = connection_config.ConnectionConfig{ + Id: request.Payload.Id, + Default: request.Payload.Default, + Service: request.Payload.Service, + ConnProtocol: request.Payload.ConnProtocol, + HttpConfig: httpCfg, + Active: activeTrue, + UpdatedBy: request.UpdatedBy, + } + } else { + return Response{ + Error: "Invalid connection protocol", + Data: Message{Message: emptyResponse}, + }, fmt.Errorf("invalid connection protocol: %s", request.Payload.ConnProtocol) + } + + err = c.connectionConfigRepo.Update(&connectionConfig) + if err != nil { + return Response{ + Error: err.Error(), + Data: Message{Message: emptyResponse}, + }, err + } + + return Response{ + Error: emptyResponse, + Data: Message{Message: fmt.Sprintf("Connection config updated successfully with id: %d", connectionConfig.Id)}, + }, nil +} diff --git a/horizon/internal/connectionconfig/handler/init.go b/horizon/internal/connectionconfig/handler/init.go new file mode 100644 index 00000000..05cd5ff6 --- /dev/null +++ b/horizon/internal/connectionconfig/handler/init.go @@ -0,0 +1,14 @@ +package handler + +var ( + config Config +) + +func NewConfigHandler(version int) Config { + switch version { + case 1: + return InitV1ConfigHandler() + default: + return nil + } +} diff --git a/horizon/internal/connectionconfig/handler/models.go b/horizon/internal/connectionconfig/handler/models.go new file mode 100644 index 00000000..2d80499d --- /dev/null +++ b/horizon/internal/connectionconfig/handler/models.go @@ -0,0 +1,90 @@ +package handler + +type OnboardRequestPayload struct { + Service string `json:"service" binding:"required"` + Default bool `json:"default"` + ConnProtocol string `json:"conn_protocol" binding:"required"` + GrpcChannelAlgorithm string `json:"grpc_channel_algorithm,omitempty"` + PlainText bool `json:"plain_text,omitempty"` + Deadline int `json:"deadline,omitempty"` + Timeout int `json:"timeout,omitempty"` + ChannelThreadPoolSize int `json:"channel_thread_pool_size,omitempty"` + BoundedQueueSize int `json:"bounded_queue_size,omitempty"` + MaxConnectionPerHost int `json:"max_connection_per_host,omitempty"` + IdleConnectionTimeout int `json:"idle_connection_timeout,omitempty"` + MaxIdleConnection int `json:"max_idle_connection,omitempty"` + KeepAliveTime int `json:"keep_alive_time,omitempty"` +} + +type OnboardRequest struct { + Payload OnboardRequestPayload `json:"payload" binding:"required"` + CreatedBy string `json:"created_by" binding:"required"` +} + +type Message struct { + Message string `json:"message"` +} + +type Response struct { + Error string `json:"error"` + Data Message `json:"data"` +} + +type GrpcConfig struct { + Deadline int `json:"deadline"` + PlainText bool `json:"plain_text"` + GrpcChannelAlgorithm string `json:"grpc_channel_algorithm"` + ChannelThreadPoolSize int `json:"channel_thread_pool_size"` + BoundedQueueSize int `json:"bounded_queue_size"` +} + +type HttpConfig struct { + Timeout int `json:"timeout"` + MaxIdleConnection int `json:"max_idle_connection"` + MaxConnectionPerHost int `json:"max_connection_per_host"` + IdleConnectionTimeout int `json:"idle_connection_timeout"` + KeepAliveTime int `json:"keep_alive_time"` +} + +type ConnConfig struct { + GrpcConfig GrpcConfig `json:"grpc_config,omitempty"` + HttpConfig HttpConfig `json:"http_config,omitempty"` +} + +type Configs struct { + Id uint `json:"id"` + Default bool `json:"default"` + Service string `json:"service"` + ConnProtocol string `json:"conn_protocol"` + Config ConnConfig `json:"config"` + CreatedBy string `json:"created_by"` + UpdatedBy string `json:"updated_by"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` +} + +type GetAllResponse struct { + Data []Configs `json:"data"` +} + +type EditRequestPayload struct { + Id uint `json:"id" binding:"required"` + Service string `json:"service" binding:"required"` + Default bool `json:"default"` + ConnProtocol string `json:"conn_protocol" binding:"required"` + GrpcChannelAlgorithm string `json:"grpc_channel_algorithm,omitempty"` + PlainText bool `json:"plain_text,omitempty"` + Deadline int `json:"deadline,omitempty"` + Timeout int `json:"timeout,omitempty"` + ChannelThreadPoolSize int `json:"channel_thread_pool_size,omitempty"` + BoundedQueueSize int `json:"bounded_queue_size,omitempty"` + MaxConnectionPerHost int `json:"max_connection_per_host,omitempty"` + IdleConnectionTimeout int `json:"idle_connection_timeout,omitempty"` + MaxIdleConnection int `json:"max_idle_connection,omitempty"` + KeepAliveTime int `json:"keep_alive_time,omitempty"` +} + +type EditRequest struct { + Payload EditRequestPayload `json:"payload" binding:"required"` + UpdatedBy string `json:"updated_by" binding:"required"` +} diff --git a/horizon/internal/connectionconfig/route/router.go b/horizon/internal/connectionconfig/route/router.go new file mode 100644 index 00000000..8628a61b --- /dev/null +++ b/horizon/internal/connectionconfig/route/router.go @@ -0,0 +1,34 @@ +package route + +import ( + "sync" + + "github.com/Meesho/BharatMLStack/horizon/internal/connectionconfig/controller" + "github.com/Meesho/BharatMLStack/horizon/pkg/httpframework" +) + +var initConnectionConfigRouterOnce sync.Once + +func Init() { + initConnectionConfigRouterOnce.Do(func() { + api := httpframework.Instance().Group("/api") + { + v1 := api.Group("/v1/horizon") + { + + // Application Config Registry/Discovery routes + registry := v1.Group("/service-connection-registry/connections") + { + registry.POST("", controller.NewConfigController().Onboard) + registry.PUT("/:id", controller.NewConfigController().Edit) + } + + discovery := v1.Group("/service-connection-discovery/connections") + { + discovery.GET("", controller.NewConfigController().GetAll) + } + + } + } + }) +} diff --git a/horizon/internal/constant/api.go b/horizon/internal/constant/api.go new file mode 100644 index 00000000..9ccb3665 --- /dev/null +++ b/horizon/internal/constant/api.go @@ -0,0 +1,7 @@ +package constant + +const ( + Error = "error" + Data = "data" + EmptyString = "" +) diff --git a/horizon/internal/constant/repository.go b/horizon/internal/constant/repository.go new file mode 100644 index 00000000..949fae40 --- /dev/null +++ b/horizon/internal/constant/repository.go @@ -0,0 +1,6 @@ +package constant + +const ( + CreatedAt = "CreatedAt" + UpdatedAt = "UpdatedAt" +) diff --git a/horizon/internal/deployable/controller/controller.go b/horizon/internal/deployable/controller/controller.go new file mode 100644 index 00000000..c541655b --- /dev/null +++ b/horizon/internal/deployable/controller/controller.go @@ -0,0 +1,279 @@ +package controller + +import ( + "encoding/json" + "fmt" + "net/http" + "sync" + + "github.com/Meesho/BharatMLStack/horizon/internal/constant" + "github.com/Meesho/BharatMLStack/horizon/internal/deployable/handler" + "github.com/Meesho/BharatMLStack/horizon/internal/repositories/sql/servicedeployableconfig" + "github.com/gin-gonic/gin" +) + +type Config interface { + GetMetaData(ctx *gin.Context) + CreateDeployable(ctx *gin.Context) + UpdateDeployable(ctx *gin.Context) + GetDeployablesByService(ctx *gin.Context) + RefreshDeployable(ctx *gin.Context) + TuneThresholds(ctx *gin.Context) +} + +var ( + deployable Config + deployableInitOnce sync.Once +) + +type V1 struct { + config handler.Config +} + +func NewConfigController() Config { + if deployable == nil { + deployableInitOnce.Do(func() { + config, err := handler.NewDeployable(1) + if err != nil { + panic(fmt.Sprintf("Failed to initialize deployable config: %v", err)) + } + if config == nil { + panic("Deployable config is nil after initialization") + } + deployable = &V1{ + config: config, + } + }) + } + return deployable +} + +func (d *V1) GetMetaData(ctx *gin.Context) { + metaData, err := d.config.GetMetaData() + if err != nil { + ctx.JSON(http.StatusInternalServerError, gin.H{constant.Error: "Failed to fetch metadata"}) + return + } + ctx.JSON(http.StatusOK, metaData) +} + +func (d *V1) CreateDeployable(ctx *gin.Context) { + var request handler.DeployableRequest + if err := ctx.ShouldBindJSON(&request); err != nil { + ctx.JSON(http.StatusBadRequest, gin.H{ + "error": "Invalid request format", + "data": nil, + }) + return + } + + if err := d.config.CreateDeployable(&request); err != nil { + ctx.JSON(http.StatusInternalServerError, gin.H{ + "error": fmt.Sprintf("Error registering deployable: %v", err), + "data": nil, + }) + return + } + + ctx.JSON(http.StatusOK, gin.H{ + "error": nil, + "data": gin.H{ + "message": "Deployable registered is in progress.", + }, + }) +} + +func (d *V1) UpdateDeployable(ctx *gin.Context) { + var request handler.DeployableRequest // Using same request structure as create + if err := ctx.ShouldBindJSON(&request); err != nil { + ctx.JSON(http.StatusBadRequest, gin.H{ + "error": "Invalid request format", + "data": nil, + }) + return + } + + if err := d.config.UpdateDeployable(&request); err != nil { + ctx.JSON(http.StatusInternalServerError, gin.H{ + "error": fmt.Sprintf("Error updating deployable: %v", err), + "data": nil, + }) + return + } + + ctx.JSON(http.StatusOK, gin.H{ + "error": nil, + "data": gin.H{ + "message": "Deployable update is in progress.", + }, + }) +} + +func (d *V1) GetDeployablesByService(ctx *gin.Context) { + serviceName := ctx.Query("service_name") + if serviceName == "" { + ctx.JSON(http.StatusBadRequest, gin.H{ + "error": "service_name query parameter is required", + "data": nil, + }) + return + } + + deployables, err := d.config.GetDeployablesByService(serviceName) + if err != nil { + ctx.JSON(http.StatusInternalServerError, gin.H{ + "error": fmt.Sprintf("Error fetching deployables: %v", err), + "data": nil, + }) + return + } + + if len(deployables) == 0 { + ctx.JSON(http.StatusOK, gin.H{ + "error": nil, + "data": []gin.H{}, + }) + return + } + + // Use channels for concurrent processing + type deployableResult struct { + index int + data gin.H + err error + } + + results := make(chan deployableResult, len(deployables)) + var wg sync.WaitGroup + + // Process each deployable concurrently + for i, deployable := range deployables { + wg.Add(1) + go func(idx int, dep servicedeployableconfig.ServiceDeployableConfig) { + defer wg.Done() + + // Parse config JSON + var configMap map[string]interface{} + if err := json.Unmarshal(dep.Config, &configMap); err != nil { + results <- deployableResult{index: idx, err: fmt.Errorf("error parsing deployable config: %w", err)} + return + } + + // Create base response + deployableResponse := gin.H{ + "id": dep.ID, + "name": dep.Name, + "host": dep.Host, + "service": dep.Service, + "active": dep.Active, + "created_by": dep.CreatedBy, + "updated_by": dep.UpdatedBy, + "created_at": dep.CreatedAt, + "updated_at": dep.UpdatedAt, + "monitoring_url": dep.MonitoringUrl, + "deployable_workflow_id": dep.DeployableWorkFlowId, + "deployment_run_id": dep.DeploymentRunID, + "deployable_health": dep.DeployableHealth, + "workflow_status": dep.WorkFlowStatus, + } + + // Add config fields (excluding replica fields) + for key, value := range configMap { + if key != "min_replica" && key != "max_replica" { + deployableResponse[key] = value + } + } + + // Get Ring Master config if needed + if dep.DeployableWorkFlowId != "" && dep.DeploymentRunID != "" { + ringMasterConfig := d.config.GetRingMasterConfig(dep.Name, dep.DeployableWorkFlowId, dep.DeploymentRunID) + deployableResponse["min_replica"] = ringMasterConfig.MinReplica + deployableResponse["max_replica"] = ringMasterConfig.MaxReplica + deployableResponse["deployable_running_status"] = ringMasterConfig.RunningStatus == "true" + } + + results <- deployableResult{index: idx, data: deployableResponse} + }(i, deployable) + } + + // Close results channel when all goroutines are done + go func() { + wg.Wait() + close(results) + }() + + // Collect results and handle errors + response := make([]gin.H, len(deployables)) + for result := range results { + if result.err != nil { + ctx.JSON(http.StatusInternalServerError, gin.H{ + "error": result.err.Error(), + "data": nil, + }) + return + } + response[result.index] = result.data + } + + ctx.JSON(http.StatusOK, gin.H{ + "error": nil, + "data": response, + }) +} + +func (d *V1) RefreshDeployable(ctx *gin.Context) { + appName := ctx.Query("app_name") + serviceType := ctx.Query("service_type") + + if appName == "" || serviceType == "" { + ctx.JSON(http.StatusBadRequest, gin.H{ + "error": "app_name and service_type query parameters are required", + "data": nil, + }) + return + } + + deployable, err := d.config.RefreshDeployable(appName, serviceType) + if err != nil { + ctx.JSON(http.StatusInternalServerError, gin.H{ + "error": fmt.Sprintf("Error refreshing deployable: %v", err), + "data": nil, + }) + return + } + + ctx.JSON(http.StatusOK, gin.H{ + "error": nil, + "data": gin.H{ + "deployable_running_status": deployable.DeployableRunningStatus, + "deployable_health": deployable.DeployableHealth, + "workflow_status": deployable.WorkFlowStatus, + }, + }) +} + +func (d *V1) TuneThresholds(ctx *gin.Context) { + var request handler.TuneThresholdsRequest + if err := ctx.ShouldBindJSON(&request); err != nil { + ctx.JSON(http.StatusBadRequest, gin.H{ + "error": "Invalid request format", + "data": nil, + }) + return + } + + if err := d.config.TuneThresholds(&request); err != nil { + ctx.JSON(http.StatusInternalServerError, gin.H{ + "error": fmt.Sprintf("Error updating deployable thresholds: %v", err), + "data": nil, + }) + return + } + + ctx.JSON(http.StatusOK, gin.H{ + "error": nil, + "data": gin.H{ + "message": "Deployable Thresholds Updated Started.", + }, + }) +} diff --git a/horizon/internal/deployable/handler/config.go b/horizon/internal/deployable/handler/config.go new file mode 100644 index 00000000..008f5a9a --- /dev/null +++ b/horizon/internal/deployable/handler/config.go @@ -0,0 +1,16 @@ +package handler + +import ( + "github.com/Meesho/BharatMLStack/horizon/internal/externalcall" + "github.com/Meesho/BharatMLStack/horizon/internal/repositories/sql/servicedeployableconfig" +) + +type Config interface { + GetMetaData() (map[string][]string, error) + CreateDeployable(request *DeployableRequest) error + UpdateDeployable(request *DeployableRequest) error + GetDeployablesByService(serviceName string) ([]servicedeployableconfig.ServiceDeployableConfig, error) + RefreshDeployable(appName, serviceType string) (*servicedeployableconfig.ServiceDeployableConfig, error) + GetRingMasterConfig(appName, workflowID, runID string) externalcall.Config + TuneThresholds(request *TuneThresholdsRequest) error +} diff --git a/horizon/internal/deployable/handler/deployable.go b/horizon/internal/deployable/handler/deployable.go new file mode 100644 index 00000000..ef019c48 --- /dev/null +++ b/horizon/internal/deployable/handler/deployable.go @@ -0,0 +1,250 @@ +package handler + +import ( + "encoding/json" + "fmt" + "sync" + + "github.com/Meesho/BharatMLStack/horizon/internal/externalcall" + mainHandler "github.com/Meesho/BharatMLStack/horizon/internal/externalcall" + "github.com/Meesho/BharatMLStack/horizon/internal/repositories/sql/deployablemetadata" + "github.com/Meesho/BharatMLStack/horizon/internal/repositories/sql/serviceconfig" + "github.com/Meesho/BharatMLStack/horizon/internal/repositories/sql/servicedeployableconfig" + "github.com/Meesho/BharatMLStack/horizon/pkg/infra" + "github.com/rs/zerolog/log" +) + +const ( + ErrFailedToFetchDeployable = "failed to fetch service deployable" +) + +var ( + deployableOnce sync.Once +) + +type Deployable struct { + repo servicedeployableconfig.ServiceDeployableRepository + deployableMetaDataRepo deployablemetadata.DeployableMetadataRepository + serviceConfigRepo serviceconfig.ServiceConfigRepository + ringMasterClient mainHandler.RingmasterClient +} + +func InitV1ConfigHandler() Config { + var handler Config + deployableOnce.Do(func() { + ringMasterClient := mainHandler.GetRingmasterClient() + connection, err := infra.SQL.GetConnection() + if err != nil { + log.Panic().Err(err).Msg("Failed to get SQL connection") + } + sqlConn, ok := connection.(*infra.SQLConnection) + if !ok { + log.Panic().Msg("Failed to cast connection to SQLConnection") + } + repo, err := servicedeployableconfig.NewRepository(sqlConn) + if err != nil { + log.Panic().Err(err).Msg("Failed to create service deployable repository") + } + deployableMetaDataRepo, err := deployablemetadata.NewRepository(sqlConn) + if err != nil { + log.Panic().Err(err).Msg("Failed to create deployable metadata repository") + } + serviceConfigRepo, err := serviceconfig.NewRepository(sqlConn) + if err != nil { + log.Panic().Err(err).Msg("Failed to create service config repository") + } + + handler = &Deployable{ + repo: repo, + deployableMetaDataRepo: deployableMetaDataRepo, + serviceConfigRepo: serviceConfigRepo, + ringMasterClient: ringMasterClient, + } + }) + + return handler +} + +func (d *Deployable) GetMetaData() (map[string][]string, error) { + deployablesMetaDataReponse, err := d.deployableMetaDataRepo.GetGroupedActiveMetadata() + if err != nil { + log.Error().Err(err).Msg("Failed to get deployable metadata") + return nil, fmt.Errorf("%s: %w", ErrFailedToFetchDeployable, err) + } + + return deployablesMetaDataReponse, nil +} + +func (d *Deployable) CreateDeployable(request *DeployableRequest) error { + switch request.ServiceName { + case "predator": + handler := NewHandler(d.repo, d.serviceConfigRepo, d.ringMasterClient) + return handler.CreateDeployable(request) + case "inferflow": + handler := NewInferflowHandler(d.repo) + return handler.CreateDeployable(request) + default: + return fmt.Errorf("unsupported service type: %s", request.ServiceName) + } +} + +func (d *Deployable) UpdateDeployable(request *DeployableRequest) error { + switch request.ServiceName { + case "predator": + handler := NewHandler(d.repo, d.serviceConfigRepo, d.ringMasterClient) + return handler.UpdateDeployable(request) + case "inferflow": + handler := NewInferflowHandler(d.repo) + return handler.UpdateDeployable(request) + default: + return fmt.Errorf("unsupported service type: %s", request.ServiceName) + } +} + +func (d *Deployable) GetDeployablesByService(serviceName string) ([]servicedeployableconfig.ServiceDeployableConfig, error) { + deployables, err := d.repo.GetByService(serviceName) + if err != nil { + log.Error().Err(err).Msg("Failed to get deployables by service") + return nil, fmt.Errorf("%s: %w", ErrFailedToFetchDeployable, err) + } + return deployables, nil +} + +func (d *Deployable) RefreshDeployable(appName, serviceType string) (*servicedeployableconfig.ServiceDeployableConfig, error) { + // Get deployable config + deployable, err := d.repo.GetByNameAndService(appName, serviceType) + if err != nil { + log.Error().Err(err).Msg("Failed to get deployable by name and service") + return nil, fmt.Errorf("%s: %w", ErrFailedToFetchDeployable, err) + } + + if deployable == nil { + return nil, fmt.Errorf("deployable not found with name: %s and service: %s", appName, serviceType) + } + + result, err := d.ringMasterClient.GetResourceDetail(appName) + if err != nil { + log.Error().Err(err).Msg("Failed to get workflow result from Ring Master") + return nil, fmt.Errorf("failed to get workflow result: %w", err) + } + + // Check if there are activities + if len(result.Nodes) > 0 { + healthyPodCount := 0 + for _, node := range result.Nodes { + if node.Kind == "Pod" && node.Health.Status == "Healthy" { + for _, info := range node.Info { + if info.Value == "Running" { + healthyPodCount += 1 + } + } + } + } + + if healthyPodCount > 0 { + deployable.DeployableHealth = "DEPLOYMENT_REASON_ARGO_APP_HEALTHY" + deployable.WorkFlowStatus = "WORKFLOW_COMPLETED" + deployable.DeployableRunningStatus = true + } + } + + // Update in database + if err := d.repo.Update(deployable); err != nil { + log.Error().Err(err).Msg("Failed to update deployable status") + return nil, fmt.Errorf("failed to update deployable status: %w", err) + } + + return deployable, nil +} + +func (d *Deployable) GetRingMasterConfig(appName, workflowID, runID string) externalcall.Config { + return d.ringMasterClient.GetConfig(appName, workflowID, runID) +} + +func (d *Deployable) TuneThresholds(request *TuneThresholdsRequest) error { + // Make a copy of request for goroutine safety + req := *request + + go func() { + deployable, err := d.repo.GetByNameAndService(req.AppName, req.ServiceName) + if err != nil { + log.Error().Err(err).Msg("Failed to get deployable by name and service") + return + } + + if deployable == nil { + log.Error().Msgf("Deployable config not found for app: %s", req.AppName) + return + } + + var config DeployableConfigPayload + if err := json.Unmarshal(deployable.Config, &config); err != nil { + log.Error().Err(err).Msg("Failed to unmarshal existing config") + return + } + + if req.MachineType == "GPU" { + if req.CPUThreshold != "" { + if err := d.ringMasterClient.UpdateCPUThreshold(req.AppName, req.CPUThreshold); err != nil { + log.Error().Err(err).Msg("Failed to update CPU threshold") + return + } + config.CPUThreshold = req.CPUThreshold + } + + if req.GPUThreshold != "" { + if err := d.ringMasterClient.UpdateGPUThreshold(req.AppName, req.GPUThreshold); err != nil { + log.Error().Err(err).Msg("Failed to update GPU threshold") + return + } + config.GPUThreshold = req.GPUThreshold + } + } else { + if req.CPUThreshold != "" { + if err := d.ringMasterClient.UpdateCPUThreshold(req.AppName, req.CPUThreshold); err != nil { + log.Error().Err(err).Msg("Failed to update CPU threshold") + return + } + config.CPUThreshold = req.CPUThreshold + } + } + + configJSON, err := json.Marshal(map[string]interface{}{ + "machine_type": config.MachineType, + "cpuRequest": config.CPURequest, + "cpuRequestUnit": config.CPURequestUnit, + "cpuLimit": config.CPULimit, + "cpuLimitUnit": config.CPULimitUnit, + "memoryRequest": config.MemoryRequest, + "memoryRequestUnit": config.MemoryRequestUnit, + "memoryLimit": config.MemoryLimit, + "memoryLimitUnit": config.MemoryLimitUnit, + "gpu_request": config.GPURequest, + "gpu_limit": config.GPULimit, + "min_replica": config.MinReplica, + "max_replica": config.MaxReplica, + "gcs_bucket_path": config.GCSBucketPath, + "triton_image_tag": config.TritonImageTag, + "serviceAccount": config.ServiceAccount, + "nodeSelectorValue": config.NodeSelectorValue, + "cpu_threshold": config.CPUThreshold, + "gpu_threshold": config.GPUThreshold, + }) + if err != nil { + log.Error().Err(err).Msg("Failed to marshal updated config") + return + } + + deployable.Config = configJSON + + if err := d.repo.Update(deployable); err != nil { + log.Error().Err(err).Msg("Failed to update deployable config") + return + } + + log.Info().Msg("Successfully updated thresholds and config") + }() + + // Return immediately, while processing happens in background + return nil +} diff --git a/horizon/internal/deployable/handler/init.go b/horizon/internal/deployable/handler/init.go new file mode 100644 index 00000000..85c824ce --- /dev/null +++ b/horizon/internal/deployable/handler/init.go @@ -0,0 +1,16 @@ +package handler + +import ( + "fmt" + "github.com/rs/zerolog/log" +) + +func NewDeployable(version int) (Config, error) { + switch version { + case 1: + return InitV1ConfigHandler(), nil + default: + log.Error().Msg("Invalid version for deployable handler") + return nil, fmt.Errorf("invalid version: %d", version) + } +} diff --git a/horizon/internal/deployable/handler/modelhandler.go b/horizon/internal/deployable/handler/modelhandler.go new file mode 100644 index 00000000..1a34cb52 --- /dev/null +++ b/horizon/internal/deployable/handler/modelhandler.go @@ -0,0 +1,64 @@ +package handler + +import ( + "encoding/json" + "fmt" + + "github.com/Meesho/BharatMLStack/horizon/internal/repositories/sql/servicedeployableconfig" +) + +type InferflowHandler struct { + repo servicedeployableconfig.ServiceDeployableRepository +} + +func NewInferflowHandler( + repo servicedeployableconfig.ServiceDeployableRepository, +) *InferflowHandler { + return &InferflowHandler{ + repo: repo, + } +} + +func (h *InferflowHandler) CreateDeployable(request *DeployableRequest) error { + configJSON, err := json.Marshal(map[string]interface{}{ + "cpuRequest": request.CPURequest, + "cpuRequestUnit": request.CPURequestUnit, + "cpuLimit": request.CPULimit, + "cpuLimitUnit": request.CPULimitUnit, + "memoryRequest": request.MemoryRequest, + "memoryRequestUnit": request.MemoryRequestUnit, + "memoryLimit": request.MemoryLimit, + "memoryLimitUnit": request.MemoryLimitUnit, + "gpu_request": request.GPURequest, + "gpu_limit": request.GPULimit, + "min_replica": request.MinReplica, + "max_replica": request.MaxReplica, + }) + if err != nil { + return fmt.Errorf("failed to marshal config: %w", err) + } + + deployableConfig := &servicedeployableconfig.ServiceDeployableConfig{ + Name: request.AppName, + Service: request.ServiceName, + Host: request.AppName + "." + hostUrlSuffix, + Active: true, + CreatedBy: request.CreatedBy, + Config: configJSON, + DeployableRunningStatus: false, + DeployableHealth: "DEPLOYMENT_REASON_ARGO_APP_HEALTHY", + DeployableWorkFlowId: "", + MonitoringUrl: fmt.Sprintf("%s/d/dQ1Gux-Vk/model-proxy-service?orgId=1&refresh=1m&var-service=%s", + grafanaBaseUrl, request.AppName), + WorkFlowStatus: "WORKFLOW_COMPLETED", + } + + if err := h.repo.Create(deployableConfig); err != nil { + return fmt.Errorf("failed to create deployable config: %w", err) + } + return nil +} + +func (h *InferflowHandler) UpdateDeployable(request *DeployableRequest) error { + return fmt.Errorf("UpdateDeployable is not implemented for InferflowHandler, use UpdateDeployableDBOnly instead") +} diff --git a/horizon/internal/deployable/handler/models.go b/horizon/internal/deployable/handler/models.go new file mode 100644 index 00000000..bc84d42a --- /dev/null +++ b/horizon/internal/deployable/handler/models.go @@ -0,0 +1,68 @@ +package handler + +// DeployableRequest represents the request body for creating a new deployable +type DeployableRequest struct { + AppName string `json:"appName"` + ServiceName string `json:"service_name"` + MachineType string `json:"machine_type"` + CPURequest string `json:"cpuRequest"` + CPURequestUnit string `json:"cpuRequestUnit"` + CPULimit string `json:"cpuLimit"` + CPULimitUnit string `json:"cpuLimitUnit"` + MemoryRequest string `json:"memoryRequest"` + MemoryRequestUnit string `json:"memoryRequestUnit"` + MemoryLimit string `json:"memoryLimit"` + MemoryLimitUnit string `json:"memoryLimitUnit"` + GPURequest string `json:"gpu_request"` + GPULimit string `json:"gpu_limit"` + MinReplica string `json:"min_replica"` + MaxReplica string `json:"max_replica"` + GCSBucketPath string `json:"gcs_bucket_path"` + GCSTritonPath string `json:"gcs_triton_path"` + TritonImageTag string `json:"triton_image_tag"` + CreatedBy string `json:"created_by"` + ServiceAccount string `json:"serviceAccount"` + NodeSelectorValue string `json:"nodeSelectorValue"` + DeploymentStrategy string `json:"deploymentStrategy"` +} + +// OnboardResponse represents the response from ringmaster onboarding API +type OnboardResponse struct { + DeploymentTriggered bool `json:"deployment-triggered"` + DeploymentRunID string `json:"deploymentRunID"` + DeploymentWorkflowID string `json:"deploymentWorkflowID"` + Onboarded bool `json:"onboarded"` + ServiceAccountBound bool `json:"serviceAccount-bound"` +} + +type DeployableConfigPayload struct { + MachineType string `json:"machine_type"` + CPURequest string `json:"cpuRequest"` + CPURequestUnit string `json:"cpuRequestUnit"` + CPULimit string `json:"cpuLimit"` + CPULimitUnit string `json:"cpuLimitUnit"` + MemoryRequest string `json:"memoryRequest"` + MemoryRequestUnit string `json:"memoryRequestUnit"` + MemoryLimit string `json:"memoryLimit"` + MemoryLimitUnit string `json:"memoryLimitUnit"` + GPURequest string `json:"gpu_request"` + GPULimit string `json:"gpu_limit"` + MinReplica string `json:"min_replica"` + MaxReplica string `json:"max_replica"` + GCSBucketPath string `json:"gcs_bucket_path"` + TritonImageTag string `json:"triton_image_tag"` + ServiceAccount string `json:"serviceAccount"` + NodeSelectorValue string `json:"nodeSelectorValue"` + GCSTritonPath string `json:"gcs_triton_path"` + CPUThreshold string `json:"cpu_threshold"` + GPUThreshold string `json:"gpu_threshold"` + DeploymentStrategy string `json:"deploymentStrategy"` +} + +type TuneThresholdsRequest struct { + AppName string `json:"appName" binding:"required"` + ServiceName string `json:"service_name" binding:"required"` + MachineType string `json:"machine_type" binding:"required"` + CPUThreshold string `json:"cpu_threshold"` + GPUThreshold string `json:"gpu_threshold"` +} diff --git a/horizon/internal/deployable/handler/predatorhandler.go b/horizon/internal/deployable/handler/predatorhandler.go new file mode 100644 index 00000000..5279d4bf --- /dev/null +++ b/horizon/internal/deployable/handler/predatorhandler.go @@ -0,0 +1,322 @@ +package handler + +import ( + "encoding/json" + "fmt" + "sync" + "time" + + "github.com/Meesho/BharatMLStack/horizon/internal/configs" + + "github.com/Meesho/BharatMLStack/horizon/internal/externalcall" + "github.com/Meesho/BharatMLStack/horizon/internal/repositories/sql/serviceconfig" + "github.com/Meesho/BharatMLStack/horizon/internal/repositories/sql/servicedeployableconfig" + "github.com/rs/zerolog/log" +) + +var ( + initOnce sync.Once + defaultCPUThreshold string + defaultGPUThreshold string + hostUrlSuffix string + grafanaBaseUrl string +) + +func Init(config configs.Configs) { + initOnce.Do(func() { + defaultCPUThreshold = config.DefaultCpuThreshold + defaultGPUThreshold = config.DefaultGpuThreshold + hostUrlSuffix = config.HostUrlSuffix + grafanaBaseUrl = config.GrafanaBaseUrl + }) +} + +type Handler struct { + repo servicedeployableconfig.ServiceDeployableRepository + serviceConfigRepo serviceconfig.ServiceConfigRepository + ringMasterClient externalcall.RingmasterClient +} + +func NewHandler( + repo servicedeployableconfig.ServiceDeployableRepository, + serviceConfigRepo serviceconfig.ServiceConfigRepository, + ringMasterClient externalcall.RingmasterClient, +) *Handler { + return &Handler{ + repo: repo, + serviceConfigRepo: serviceConfigRepo, + ringMasterClient: ringMasterClient, + } +} + +func (h *Handler) CreateDeployable(request *DeployableRequest) error { + // 1. Get service config + serviceConfig, err := h.serviceConfigRepo.GetByName(request.ServiceName) + if err != nil { + return fmt.Errorf("failed to get service config: %w", err) + } + + // 2. Create initial DB entry + configJSON, err := json.Marshal(map[string]interface{}{ + "machine_type": request.MachineType, + "cpuRequest": request.CPURequest, + "cpuRequestUnit": request.CPURequestUnit, + "cpuLimit": request.CPULimit, + "cpuLimitUnit": request.CPULimitUnit, + "memoryRequest": request.MemoryRequest, + "memoryRequestUnit": request.MemoryRequestUnit, + "memoryLimit": request.MemoryLimit, + "memoryLimitUnit": request.MemoryLimitUnit, + "gpu_request": request.GPURequest, + "gpu_limit": request.GPULimit, + "min_replica": request.MinReplica, + "max_replica": request.MaxReplica, + "deploymentStrategy": request.DeploymentStrategy, + "cpu_threshold": defaultCPUThreshold, + "gpu_threshold": defaultGPUThreshold, + "gcs_bucket_path": request.GCSBucketPath, + "gcs_triton_path": request.GCSTritonPath, + "triton_image_tag": request.TritonImageTag, + "serviceAccount": request.ServiceAccount, + "nodeSelectorValue": request.NodeSelectorValue, + }) + if err != nil { + return fmt.Errorf("failed to marshal config: %w", err) + } + + deployableConfig := &servicedeployableconfig.ServiceDeployableConfig{ + Name: request.AppName, + Service: request.ServiceName, + Host: request.AppName + "." + hostUrlSuffix, + Active: true, + CreatedBy: request.CreatedBy, + Config: configJSON, + DeployableRunningStatus: false, + DeployableHealth: "DEPLOYMENT_REASON_ARGO_APP_HEALTH_DEGRADED", + DeployableWorkFlowId: "", + MonitoringUrl: fmt.Sprintf("%s/d/a2605923-52c4-4834-bdae-97570966b765/model-inference-service?orgId=1&var-service=%s&var-query0=", + grafanaBaseUrl, request.AppName), + WorkFlowStatus: "WORKFLOW_NOT_STARTED", + } + + if err := h.repo.Create(deployableConfig); err != nil { + return fmt.Errorf("failed to create deployable config: %w", err) + } + + // 3. Prepare ringmaster payload + ringmasterPayload := map[string]interface{}{ + "appName": request.AppName, + "primaryOwner": serviceConfig.PrimaryOwner, + "repoName": serviceConfig.RepoName, + "branchName": serviceConfig.BranchName, + "secondaryOwner": serviceConfig.SecondaryOwner, + "healthCheck": serviceConfig.HealthCheck, + "host": request.AppName + "." + hostUrlSuffix, + "appPort": serviceConfig.AppPort, + "team": serviceConfig.Team, + "bu": serviceConfig.BU, + "priorityV2": serviceConfig.PriorityV2, + "module": serviceConfig.Module, + "appType": serviceConfig.AppType, + "ingress_class": serviceConfig.IngressClass, + "machine_type": request.MachineType, + "cpuRequest": request.CPURequest, + "cpuRequestUnit": request.CPURequestUnit, + "cpuLimit": request.CPULimit, + "cpuLimitUnit": request.CPULimitUnit, + "memoryRequest": request.MemoryRequest, + "memoryRequestUnit": request.MemoryRequestUnit, + "memoryLimit": request.MemoryLimit, + "memoryLimitUnit": request.MemoryLimitUnit, + "gpu_request": request.GPURequest, + "gpu_limit": request.GPULimit, + "as_min": request.MinReplica, + "as_max": request.MaxReplica, + "gcs_bucket_path": request.GCSBucketPath, + "buildNo": serviceConfig.BuildNo, + "triton_image_tag": request.TritonImageTag, + "created_by": request.CreatedBy, + } + + // Add service type config + var serviceTypeConfig map[string]string + if err := json.Unmarshal(serviceConfig.Config, &serviceTypeConfig); err != nil { + return fmt.Errorf("failed to unmarshal service type config: %w", err) + } + for k, v := range serviceTypeConfig { + ringmasterPayload[k] = v + } + + // Add remaining fields + ringmasterPayload["serviceAccount"] = request.ServiceAccount + ringmasterPayload["nodeSelectorValue"] = request.NodeSelectorValue + ringmasterPayload["deploymentStrategy"] = request.DeploymentStrategy + ringmasterPayload["gcs_triton_path"] = request.GCSTritonPath + + // 4. Make API calls in sequence + go func() { + + var rawBody []byte + sleepTime := 30 + for attempt := 1; attempt <= 3; attempt++ { + rawBody, err = h.ringMasterClient.CreateDeployable(ringmasterPayload) + time.Sleep(time.Duration(sleepTime) * time.Second) + sleepTime += 30 + } + + if err != nil { + log.Error().Err(err).Msg("failed to create deployable in ringmaster") + return + } + + var onboardResp OnboardResponse + if err := json.Unmarshal(rawBody, &onboardResp); err != nil { + log.Error().Err(err).Msg("failed to parse ringmaster response") + } + + // Update deployable with workflow IDs + deployableConfig.DeploymentRunID = onboardResp.DeploymentRunID + deployableConfig.DeployableWorkFlowId = onboardResp.DeploymentWorkflowID + if err := h.repo.Update(deployableConfig); err != nil { + log.Error().Err(err).Msg("failed to update deployable with workflow IDs") + return + } + }() + + return nil +} + +func (h *Handler) UpdateDeployable(request *DeployableRequest) error { + // 1. Get service config + serviceConfig, err := h.serviceConfigRepo.GetByName(request.ServiceName) + if err != nil { + return fmt.Errorf("failed to get service config: %w", err) + } + + // 2. Get existing deployable configs by service + existingConfig, err := h.repo.GetByNameAndService(request.AppName, request.ServiceName) + if err != nil { + return fmt.Errorf("failed to get existing deployable configs: %w", err) + } + + if existingConfig == nil { + return fmt.Errorf("deployable config not found for app: %s", request.AppName) + } + + var config DeployableConfigPayload + if err := json.Unmarshal(existingConfig.Config, &config); err != nil { + return fmt.Errorf("failed to unmarshal existing config: %w", err) + } + + // 3. Update DB entry + configJSON, err := json.Marshal(map[string]interface{}{ + "machine_type": request.MachineType, + "cpuRequest": request.CPURequest, + "cpuRequestUnit": request.CPURequestUnit, + "cpuLimit": request.CPULimit, + "cpuLimitUnit": request.CPULimitUnit, + "memoryRequest": request.MemoryRequest, + "memoryRequestUnit": request.MemoryRequestUnit, + "memoryLimit": request.MemoryLimit, + "memoryLimitUnit": request.MemoryLimitUnit, + "gpu_request": request.GPURequest, + "gpu_limit": request.GPULimit, + "min_replica": request.MinReplica, + "max_replica": request.MaxReplica, + "gcs_bucket_path": request.GCSBucketPath, + "triton_image_tag": request.TritonImageTag, + "serviceAccount": request.ServiceAccount, + "nodeSelectorValue": request.NodeSelectorValue, + "cpu_threshold": config.CPUThreshold, + "deploymentStrategy": request.DeploymentStrategy, + "gpu_threshold": config.GPUThreshold, + "gcs_triton_path": request.GCSTritonPath, + }) + if err != nil { + return fmt.Errorf("failed to marshal config: %w", err) + } + + existingConfig.Config = configJSON + existingConfig.CreatedBy = request.CreatedBy + existingConfig.WorkFlowStatus = "WORKFLOW_NOT_STARTED" + + if err := h.repo.Update(existingConfig); err != nil { + return fmt.Errorf("failed to update deployable config: %w", err) + } + + // 4. Prepare ringmaster payload + ringmasterPayload := map[string]interface{}{ + "appName": request.AppName, + "primaryOwner": serviceConfig.PrimaryOwner, + "repoName": serviceConfig.RepoName, + "branchName": serviceConfig.BranchName, + "secondaryOwner": serviceConfig.SecondaryOwner, + "healthCheck": serviceConfig.HealthCheck, + "appPort": serviceConfig.AppPort, + "team": serviceConfig.Team, + "bu": serviceConfig.BU, + "priorityV2": serviceConfig.PriorityV2, + "module": serviceConfig.Module, + "appType": serviceConfig.AppType, + "ingress_class": serviceConfig.IngressClass, + "machine_type": request.MachineType, + "cpuRequest": request.CPURequest, + "cpuRequestUnit": request.CPURequestUnit, + "cpuLimit": request.CPULimit, + "cpuLimitUnit": request.CPULimitUnit, + "memoryRequest": request.MemoryRequest, + "memoryRequestUnit": request.MemoryRequestUnit, + "memoryLimit": request.MemoryLimit, + "memoryLimitUnit": request.MemoryLimitUnit, + "gpu_request": request.GPURequest, + "gpu_limit": request.GPULimit, + "as_min": request.MinReplica, + "as_max": request.MaxReplica, + "gcs_bucket_path": request.GCSBucketPath, + "host": request.AppName + "." + hostUrlSuffix, + "buildNo": serviceConfig.BuildNo, + "triton_image_tag": request.TritonImageTag, + "created_by": request.CreatedBy, + } + + // Add service type config + var serviceTypeConfig map[string]string + if err := json.Unmarshal(serviceConfig.Config, &serviceTypeConfig); err != nil { + return fmt.Errorf("failed to unmarshal service type config: %w", err) + } + for k, v := range serviceTypeConfig { + ringmasterPayload[k] = v + } + + // Add remaining fields + ringmasterPayload["serviceAccount"] = request.ServiceAccount + ringmasterPayload["nodeSelectorValue"] = request.NodeSelectorValue + ringmasterPayload["deploymentStrategy"] = request.DeploymentStrategy + ringmasterPayload["gcs_triton_path"] = request.GCSTritonPath + + // 5. Make API calls in sequence + go func() { + // Update deployable using same create API as per requirement + rawBody, err := h.ringMasterClient.CreateDeployable(ringmasterPayload) + if err != nil { + log.Error().Err(err).Msg("failed to create deployable in ringmaster") + return + } + + var onboardResp OnboardResponse + if err := json.Unmarshal(rawBody, &onboardResp); err != nil { + log.Error().Err(err).Msg("failed to parse ringmaster response") + return + } + + // Update deployable with workflow IDs + existingConfig.DeploymentRunID = onboardResp.DeploymentRunID + existingConfig.DeployableWorkFlowId = onboardResp.DeploymentWorkflowID + if err := h.repo.Update(existingConfig); err != nil { + log.Error().Err(err).Msg("failed to update deployable with workflow IDs") + return + } + }() + + return nil +} diff --git a/horizon/internal/deployable/router/router.go b/horizon/internal/deployable/router/router.go new file mode 100644 index 00000000..2ca028ca --- /dev/null +++ b/horizon/internal/deployable/router/router.go @@ -0,0 +1,33 @@ +package router + +import ( + "sync" + + "github.com/Meesho/BharatMLStack/horizon/internal/deployable/controller" + "github.com/Meesho/BharatMLStack/horizon/pkg/httpframework" +) + +var initDeployableRouterOnce sync.Once + +// Init expects http framework to be initialized before calling this function +func Init() { + initDeployableRouterOnce.Do(func() { + deployableRegistryApi := httpframework.Instance().Group("/api/v1/horizon/deployable-registry/deployables") + { + deployableRegistryApi.POST("", controller.NewConfigController().CreateDeployable) + deployableRegistryApi.PUT("", controller.NewConfigController().UpdateDeployable) + deployableRegistryApi.POST("/refresh", controller.NewConfigController().RefreshDeployable) + } + + deployableDiscoveryApi := httpframework.Instance().Group("/api/v1/horizon/deployable-discovery/deployables") + { + deployableDiscoveryApi.GET("/metadata", controller.NewConfigController().GetMetaData) + deployableDiscoveryApi.GET("", controller.NewConfigController().GetDeployablesByService) + + } + deployableRegistry := httpframework.Instance().Group("/api/v1/horizon/deployable-registry") + { + deployableRegistry.PUT("/deployables/tune-thresholds", controller.NewConfigController().TuneThresholds) + } + }) +} diff --git a/horizon/internal/externalcall/feature_validation_client.go b/horizon/internal/externalcall/feature_validation_client.go new file mode 100644 index 00000000..dc70c007 --- /dev/null +++ b/horizon/internal/externalcall/feature_validation_client.go @@ -0,0 +1,288 @@ +package externalcall + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/Meesho/BharatMLStack/horizon/internal/configs" +) + +type FeatureValidationClient interface { + ValidateOnlineFeatures(entity string, token string) (*OnlineValidationResponse, error) + ValidateOfflineFeatures(offlineFeatures []string, token string) (*OfflineValidationResponse, error) +} + +type featureValidationClientImpl struct { + HTTPClient *http.Client + BaseURL string +} + +var ( + Client FeatureValidationClient + horizonBaseURL string + onlineFeatureMappingURL string + featureGroupDataTypeURL string +) + +// InitFeatureValidationClient initializes the feature validation client with config-based URLs +func InitFeatureValidationClient(config configs.Configs) { + // Build horizon base URL from config + if config.HorizonServer != "" { + if config.HorizonPort != "" && config.HorizonPort != "80" { + horizonBaseURL = fmt.Sprintf("http://%s:%s", config.HorizonServer, config.HorizonPort) + } else { + horizonBaseURL = fmt.Sprintf("http://%s", config.HorizonServer) + } + } else { + panic("horizon server is not set") + } + + onlineFeatureMappingURL = config.OnlineFeatureMappingUrl + featureGroupDataTypeURL = config.FeatureGroupDataTypeMappingUrl + + // Initialize the client directly + Client = &featureValidationClientImpl{ + HTTPClient: &http.Client{ + Timeout: 30 * time.Second, + }, + BaseURL: horizonBaseURL, + } +} + +// OnlineValidationResponse represents the response from online feature validation +type OnlineValidationResponse []struct { + EntityLabel string `json:"entity-label"` + FeatureGroupLabel string `json:"feature-group-label"` + ID int `json:"id"` + ActiveVersion string `json:"active-version"` + Features map[string]struct { + Labels interface{} `json:"labels"` + } `json:"features"` + StoreID string `json:"store-id"` + DataType string `json:"data-type"` + TTLInSeconds int `json:"ttl-in-seconds"` + TTLInEpoch int `json:"ttl-in-epoch"` + JobID string `json:"job-id"` + InMemoryCacheEnabled bool `json:"in-memory-cache-enabled"` + DistributedCacheEnabled bool `json:"distributed-cache-enabled"` + LayoutVersion int `json:"layout-version"` +} + +// OfflineValidationResponse represents the response from offline feature validation +type OfflineValidationResponse struct { + Error string `json:"error"` + Data map[string]string `json:"data"` +} + +func (f *featureValidationClientImpl) ValidateOnlineFeatures(entity string, token string) (*OnlineValidationResponse, error) { + // Use configured URL or fallback to default path + var url string + if featureGroupDataTypeURL != "" { + url = fmt.Sprintf("%s/%s?entity=%s", f.BaseURL, featureGroupDataTypeURL, entity) + } else { + url = fmt.Sprintf("%s/api/v1/orion/retrieve-feature-groups?entity=%s", f.BaseURL, entity) + } + + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("failed to create online validation request: %w", err) + } + + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json, text/plain, */*") + req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36") + + resp, err := f.HTTPClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to call online validation API: %w", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read online validation response: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("online validation API failed with status %d: %s", resp.StatusCode, string(body)) + } + + var response OnlineValidationResponse + if err := json.Unmarshal(body, &response); err != nil { + return nil, fmt.Errorf("failed to unmarshal online validation response: %w", err) + } + + return &response, nil +} + +func (f *featureValidationClientImpl) ValidateOfflineFeatures(offlineFeatures []string, token string) (*OfflineValidationResponse, error) { + // Use configured URL or fallback to default path + var url string + if onlineFeatureMappingURL != "" { + url = fmt.Sprintf("%s/%s", f.BaseURL, onlineFeatureMappingURL) + } else { + url = fmt.Sprintf("%s/api/v1/orion/get-online-features-mapping", f.BaseURL) + } + + requestBody := map[string][]string{ + "offline-feature-list": offlineFeatures, + } + + jsonBody, err := json.Marshal(requestBody) + if err != nil { + return nil, fmt.Errorf("failed to marshal offline validation request: %w", err) + } + + req, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(jsonBody)) + if err != nil { + return nil, fmt.Errorf("failed to create offline validation request: %w", err) + } + + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + req.Header.Set("Content-Type", "application/json") + + resp, err := f.HTTPClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to call offline validation API: %w", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read offline validation response: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("offline validation API failed with status %d: %s", resp.StatusCode, string(body)) + } + + var response OfflineValidationResponse + if err := json.Unmarshal(body, &response); err != nil { + return nil, fmt.Errorf("failed to unmarshal offline validation response: %w", err) + } + + return &response, nil +} + +// ParseFeatureString parses feature strings and determines validation requirements +// Returns featureType, entity, gf (entity:featureGroup), featureName, isValid +// +// Feature types and validation rules: +// 1. PARENT_OFFLINE_FEATURE|FEATURE -> validate as OFFLINE_FEATURE +// 2. OFFLINE_FEATURE|FEATURE -> validate as OFFLINE_FEATURE +// 3. ONLINE_FEATURE|FEATURE -> validate as ONLINE_FEATURE +// 4. PARENT_ONLINE_FEATURE|FEATURE -> validate as ONLINE_FEATURE +// 5. DEFAULT_FEATURE|FEATURE -> no validation needed +// 6. PARENT_DEFAULT_FEATURE|FEATURE -> no validation needed +// 7. MODEL_FEATURE|FEATURE -> no validation needed +// 8. CALIBRATION|FEATURE -> no validation needed +// 9. RTP_FEATURE|ENTITY:FEATURE_GROUP:FEATURE -> validate using pricing service +// 10. PARENT_RTP_FEATURE|ENTITY:FEATURE_GROUP:FEATURE -> validate using pricing service +func ParseFeatureString(feature string) (featureType, entity, gf, featureName string, isValid bool) { + parts := strings.Split(feature, "|") + if len(parts) < 2 { + return "", "", "", "", false + } + + featureType = strings.TrimSpace(parts[0]) + featureData := strings.TrimSpace(parts[1]) + + switch featureType { + case "PARENT_OFFLINE_FEATURE": + // Add PARENT| prefix for validation but keep original feature name + remainingParts := strings.Join(parts[1:], "|") + validationFeature := "PARENT|" + remainingParts + return featureType, "", validationFeature, remainingParts, true + + case "OFFLINE_FEATURE": + // These need offline validation - return everything after the first pipe + remainingParts := strings.Join(parts[1:], "|") + return featureType, "", remainingParts, remainingParts, true + + case "ONLINE_FEATURE", "PARENT_ONLINE_FEATURE": + // These need online validation - extract entity from e:fg:f format + featureParts := strings.Split(featureData, ":") + if len(featureParts) == 3 { + entity := featureParts[0] // e + gf := featureParts[1] + ":" + featureParts[2] // fg:f + featureName := featureParts[2] // f + return featureType, entity, gf, featureName, true + } + return featureType, "", featureData, featureData, false + + case "DEFAULT_FEATURE", "PARENT_DEFAULT_FEATURE", "MODEL_FEATURE", "CALIBRATION": + // These don't need API validation - they are correct by default + // Return everything after the first pipe + remainingParts := strings.Join(parts[1:], "|") + return featureType, "", remainingParts, remainingParts, true + + case "RTP_FEATURE", "PARENT_RTP_FEATURE": + // These need pricing service validation - extract entity from e:fg:f format + featureData := strings.TrimSpace(parts[1]) + featureParts := strings.Split(featureData, ":") + if len(featureParts) == 3 { + entity := featureParts[0] // e + gf := featureParts[1] + ":" + featureParts[2] // fg:f + featureName := featureParts[2] // f + return featureType, entity, gf, featureName, true + } + return featureType, "", featureData, featureData, false + + default: + // Unknown feature type + return "", "", "", "", false + } +} + +// ValidateFeatureExists checks if a feature exists in the validation response +func ValidateFeatureExists(featureName string, response *OnlineValidationResponse) bool { + if response == nil { + return false + } + featureParts := strings.Split(featureName, ":") + + fg := featureParts[0] + feature := featureParts[1] + + for _, featureGroup := range *response { + if featureGroup.FeatureGroupLabel != fg { + continue + } + + // Iterate through all versions in features (e.g., "2", "3", etc.) + for _, featureVersion := range featureGroup.Features { + // Handle the labels field which can be a comma-separated string + switch labels := featureVersion.Labels.(type) { + case string: + // Labels is a comma-separated string - split and check + labelList := strings.Split(labels, ",") + for _, label := range labelList { + if strings.TrimSpace(label) == feature { + return true + } + } + case []string: + // Labels is an array of strings + for _, label := range labels { + if strings.TrimSpace(label) == feature { + return true + } + } + case []interface{}: + // Labels is an array of interfaces (convert to strings) + for _, labelInterface := range labels { + if labelStr, ok := labelInterface.(string); ok && strings.TrimSpace(labelStr) == feature { + return true + } + } + } + } + } + return false +} diff --git a/horizon/internal/externalcall/gcs_client.go b/horizon/internal/externalcall/gcs_client.go new file mode 100644 index 00000000..bf4a140e --- /dev/null +++ b/horizon/internal/externalcall/gcs_client.go @@ -0,0 +1,634 @@ +package externalcall + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "path" + "path/filepath" + "strings" + "sync" + "time" + + "cloud.google.com/go/storage" + "github.com/rs/zerolog/log" + "google.golang.org/api/iterator" +) + +type GCSClientInterface interface { + ReadFile(bucket, objectPath string) ([]byte, error) + TransferFolder(srcBucket, srcPath, srcModelName, destBucket, destPath, destModelName string) error + TransferAndDeleteFolder(srcBucket, srcPath, srcModelName, destBucket, destPath, destModelName string) error + DeleteFolder(bucket, modelPath, modelName string) error + ListFolders(bucket, prefix string) ([]string, error) + UploadFile(bucket, objectPath string, data []byte) error + CheckFileExists(bucket, objectPath string) (bool, error) + CheckFolderExists(bucket, folderPrefix string) (bool, error) + UploadFolderFromLocal(srcFolderPath, bucket, destPath string) error + GetFolderInfo(bucket, folderPrefix string) (*GCSFolderInfo, error) + ListFoldersWithTimestamp(bucket, prefix string) ([]GCSFolderInfo, error) + FindFileWithSuffix(bucket, folderPath, suffix string) (bool, string, error) +} + +const ( + maxRetries = 3 + initialBackoffTime = 2 * time.Second + maxConcurrentFiles = 10 // Maximum number of files to transfer concurrently +) + +type GCSFolderInfo struct { + Name string `json:"name"` + Path string `json:"path"` + Created time.Time `json:"created"` + Updated time.Time `json:"updated"` + Size int64 `json:"size"` + FileCount int `json:"file_count"` +} + +type GCSClient struct { + client *storage.Client + ctx context.Context +} + +func CreateGCSClient() GCSClientInterface { + ctx := context.Background() + client, err := storage.NewClient(ctx) + if err != nil { + log.Error().Err(err).Msg("Failed to create GCS client") + return &GCSClient{ + client: nil, + ctx: ctx, + } + } + return &GCSClient{ + client: client, + ctx: ctx, + } +} + +func (g *GCSClient) ReadFile(bucket, objectPath string) ([]byte, error) { + rc, err := g.client.Bucket(bucket).Object(objectPath).NewReader(g.ctx) + if err != nil { + return nil, fmt.Errorf("failed to create object reader: %w", err) + } + defer rc.Close() + + buf := new(bytes.Buffer) + if _, err := io.Copy(buf, rc); err != nil { + return nil, fmt.Errorf("failed to read object into buffer: %w", err) + } + return buf.Bytes(), nil +} + +func (g *GCSClient) TransferFolder(srcBucket, srcPath, srcModelName, destBucket, destPath, destModelName string) error { + prefix := path.Join(srcPath, srcModelName) + if !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + + // Phase 1: Collect all objects and separate config files from regular files + var regularFiles []storage.ObjectAttrs + var configFiles []storage.ObjectAttrs + + it := g.client.Bucket(srcBucket).Objects(g.ctx, &storage.Query{Prefix: prefix}) + for { + objAttrs, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + return fmt.Errorf("failed to list source bucket: %w", err) + } + + if strings.HasSuffix(objAttrs.Name, "/") { + continue + } + + if strings.HasSuffix(objAttrs.Name, "config.pbtxt") { + configFiles = append(configFiles, *objAttrs) + } else { + regularFiles = append(regularFiles, *objAttrs) + } + } + + if len(regularFiles) == 0 && len(configFiles) == 0 { + log.Info().Msg("No objects found to transfer") + return nil + } + + log.Info().Msgf("Starting two-phase transfer: %d regular files, %d config files", len(regularFiles), len(configFiles)) + + // Phase 1: Fast direct transfer of regular files (no local download) + if len(regularFiles) > 0 { + if err := g.transferRegularFiles(regularFiles, srcBucket, destBucket, destPath, destModelName, prefix); err != nil { + return fmt.Errorf("failed to transfer regular files: %w", err) + } + } + + // Phase 2: Handle config files with model name replacement + if len(configFiles) > 0 { + if err := g.transferConfigFiles(configFiles, srcBucket, destBucket, destPath, destModelName, prefix); err != nil { + return fmt.Errorf("failed to transfer config files: %w", err) + } + } + + log.Info().Msgf("Successfully completed two-phase transfer") + return nil +} + +// transferRegularFiles performs fast direct GCS-to-GCS transfers for regular files +func (g *GCSClient) transferRegularFiles(files []storage.ObjectAttrs, srcBucket, destBucket, destPath, destModelName, prefix string) error { + log.Info().Msgf("Phase 1: Starting fast direct transfer of %d regular files", len(files)) + + // Use worker pool for concurrent direct transfers + semaphore := make(chan struct{}, maxConcurrentFiles) + var wg sync.WaitGroup + var mu sync.Mutex + var transferErrors []error + + for _, objAttrs := range files { + wg.Add(1) + go func(obj storage.ObjectAttrs) { + defer wg.Done() + semaphore <- struct{}{} // Acquire semaphore + defer func() { <-semaphore }() // Release semaphore + + if err := g.transferSingleRegularFile(obj, srcBucket, destBucket, destPath, destModelName, prefix); err != nil { + mu.Lock() + transferErrors = append(transferErrors, fmt.Errorf("failed to transfer %s: %w", obj.Name, err)) + mu.Unlock() + } + }(objAttrs) + } + + wg.Wait() + + if len(transferErrors) > 0 { + return fmt.Errorf("regular file transfer completed with %d errors: %v", len(transferErrors), transferErrors[0]) + } + + log.Info().Msgf("Phase 1 completed: Successfully transferred %d regular files", len(files)) + return nil +} + +// transferSingleRegularFile performs direct GCS-to-GCS copy for a single regular file +func (g *GCSClient) transferSingleRegularFile(objAttrs storage.ObjectAttrs, srcBucket, destBucket, destPath, destModelName, prefix string) error { + // Calculate destination path + relPath := strings.TrimPrefix(objAttrs.Name, prefix) + relPath = strings.TrimPrefix(relPath, "/") + destObjectPath := path.Join(destPath, destModelName, relPath) + + // Use direct GCS-to-GCS copy operation + srcObj := g.client.Bucket(srcBucket).Object(objAttrs.Name) + destObj := g.client.Bucket(destBucket).Object(destObjectPath) + + _, err := destObj.CopierFrom(srcObj).Run(g.ctx) + if err != nil { + return fmt.Errorf("failed to copy object directly: %w", err) + } + + log.Debug().Msgf("Direct transfer completed: %s -> %s", objAttrs.Name, destObjectPath) + return nil +} + +// transferConfigFiles handles config.pbtxt files with model name replacement +func (g *GCSClient) transferConfigFiles(files []storage.ObjectAttrs, srcBucket, destBucket, destPath, destModelName, prefix string) error { + log.Info().Msgf("Phase 2: Processing %d config files with model name replacement", len(files)) + + for _, objAttrs := range files { + if err := g.transferSingleConfigFile(objAttrs, srcBucket, destBucket, destPath, destModelName, prefix); err != nil { + return fmt.Errorf("failed to transfer config file %s: %w", objAttrs.Name, err) + } + } + + log.Info().Msgf("Phase 2 completed: Successfully processed %d config files", len(files)) + return nil +} + +// transferSingleConfigFile handles a single config.pbtxt file with model name replacement +func (g *GCSClient) transferSingleConfigFile(objAttrs storage.ObjectAttrs, srcBucket, destBucket, destPath, destModelName, prefix string) error { + // Calculate destination path + relPath := strings.TrimPrefix(objAttrs.Name, prefix) + relPath = strings.TrimPrefix(relPath, "/") + destObjectPath := path.Join(destPath, destModelName, relPath) + + // Download config file content + srcReader, err := g.client.Bucket(srcBucket).Object(objAttrs.Name).NewReader(g.ctx) + if err != nil { + return fmt.Errorf("failed to create source reader for config: %w", err) + } + defer srcReader.Close() + + // Read config content (config files are typically small) + content, err := io.ReadAll(srcReader) + if err != nil { + return fmt.Errorf("failed to read config file: %w", err) + } + + // Replace model name + log.Info().Msgf("Processing config.pbtxt file: %s -> %s", objAttrs.Name, destObjectPath) + modified := replaceModelNameInConfig(content, destModelName) + + // Upload modified content + destWriter := g.client.Bucket(destBucket).Object(destObjectPath).NewWriter(g.ctx) + defer destWriter.Close() + + _, err = destWriter.Write(modified) + if err != nil { + return fmt.Errorf("failed to write modified config: %w", err) + } + + log.Debug().Msgf("Config file processed: %s -> %s", objAttrs.Name, destObjectPath) + return nil +} + +func (g *GCSClient) DeleteFolder(bucket, modelPath, modelName string) error { + // Ensure the prefix ends with "/" to avoid matching partial directory names + prefix := path.Join(modelPath, modelName) + + if prefix != "" && !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + + log.Debug().Msgf("Deleting objects with prefix: %s", prefix) + it := g.client.Bucket(bucket).Objects(g.ctx, &storage.Query{Prefix: prefix}) + + for { + objAttrs, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + return fmt.Errorf("failed to list objects for deletion: %w", err) + } + + err = g.client.Bucket(bucket).Object(objAttrs.Name).Delete(g.ctx) + if err != nil { + log.Printf("failed to delete object %s: %v", objAttrs.Name, err) + continue + } + log.Debug().Msgf("deleted object: %s", objAttrs.Name) + } + return nil +} + +func (g *GCSClient) TransferAndDeleteFolder(srcBucket, srcPath, srcModelName, destBucket, destPath, destModelName string) error { + err := g.TransferFolder(srcBucket, srcPath, srcModelName, destBucket, destPath, destModelName) + if err != nil { + log.Error().Err(err).Msg("Failed to transfer folder") + return fmt.Errorf("failed to transfer folder: %w", err) + } + + backoffTime := initialBackoffTime + for i := 0; i < maxRetries; i++ { + err = g.DeleteFolder(srcBucket, srcPath, srcModelName) + if err == nil { + break + } + log.Error().Err(err).Msgf("Attempt %d: Failed to delete source folder, retrying...", i+1) + time.Sleep(backoffTime) + backoffTime *= 2 + } + + if err != nil { + log.Error().Err(err).Msg("Failed to delete source folder after retries, attempting rollback") + rollbackErr := g.TransferFolder(destBucket, destPath, destModelName, srcBucket, srcPath, srcModelName) + if rollbackErr != nil { + log.Error().Err(rollbackErr).Msg("Rollback failed") + return fmt.Errorf("failed to rollback the transfer after deletion failure: %w", rollbackErr) + } + log.Info().Msg("Rollback successful, but deletion still failed") + return fmt.Errorf("failed to delete source folder after successful transfer: %w", err) + } + log.Info().Msg("Successfully transferred and deleted folder") + return nil +} + +// replaceModelNameInConfig modifies the `name:` field in config.pbtxt content +func replaceModelNameInConfig(data []byte, destModelName string) []byte { + lines := strings.Split(string(data), "\n") + originalName := "" + for i, line := range lines { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "name:") { + originalName = line + lines[i] = fmt.Sprintf(`name: "%s"`, destModelName) + log.Info().Msgf("Replacing model name in config.pbtxt: '%s' -> 'name: \"%s\"'", originalName, destModelName) + break + } + } + return []byte(strings.Join(lines, "\n")) +} + +func (g *GCSClient) ListFolders(bucket, prefix string) ([]string, error) { + if g.client == nil { + return nil, fmt.Errorf("GCS client not initialized properly") + } + + var folders []string + seenFolders := make(map[string]bool) + + // Ensure prefix ends with a trailing slash + if !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + + log.Info().Msgf("Listing folders in GCS bucket %s with prefix %s", bucket, prefix) + + it := g.client.Bucket(bucket).Objects(g.ctx, &storage.Query{ + Prefix: prefix, + // Do NOT set Delimiter here + }) + + for { + attrs, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + return nil, fmt.Errorf("failed to list objects: %w", err) + } + + // Extract folder name after the prefix + if attrs.Name != "" { + trimmed := strings.TrimPrefix(attrs.Name, prefix) + parts := strings.SplitN(trimmed, "/", 2) + if len(parts) > 1 { + folderName := parts[0] + if !seenFolders[folderName] { + folders = append(folders, folderName) + seenFolders[folderName] = true + } + } + } + } + + return folders, nil +} + +func (g *GCSClient) UploadFile(bucket, objectPath string, data []byte) error { + if g.client == nil { + return fmt.Errorf("GCS client not initialized properly") + } + + writer := g.client.Bucket(bucket).Object(objectPath).NewWriter(g.ctx) + if _, err := io.Copy(writer, bytes.NewReader(data)); err != nil { + writer.Close() + return fmt.Errorf("failed to write file to GCS: %w", err) + } + + if err := writer.Close(); err != nil { + return fmt.Errorf("failed to close writer: %w", err) + } + + return nil +} + +func (g *GCSClient) CheckFileExists(bucket, objectPath string) (bool, error) { + if g.client == nil { + return false, fmt.Errorf("GCS client not initialized properly") + } + + _, err := g.client.Bucket(bucket).Object(objectPath).Attrs(g.ctx) + if err != nil { + if err == storage.ErrObjectNotExist { + return false, nil + } + return false, fmt.Errorf("failed to check file existence: %w", err) + } + return true, nil +} + +func (g *GCSClient) CheckFolderExists(bucket, folderPrefix string) (bool, error) { + if g.client == nil { + return false, fmt.Errorf("GCS client not initialized properly") + } + + // Ensure folderPrefix ends with "/" + if !strings.HasSuffix(folderPrefix, "/") { + folderPrefix += "/" + } + + it := g.client.Bucket(bucket).Objects(g.ctx, &storage.Query{ + Prefix: folderPrefix, + }) + + // Check if at least one object exists with this prefix + _, err := it.Next() + if err == iterator.Done { + return false, nil // No objects found with this prefix + } + if err != nil { + return false, fmt.Errorf("failed to check folder existence: %w", err) + } + + return true, nil +} + +func (g *GCSClient) UploadFolderFromLocal(srcFolderPath, bucket, destPath string) error { + if g.client == nil { + return fmt.Errorf("GCS client not initialized properly") + } + + // Walk through the source folder and upload all files + return filepath.Walk(srcFolderPath, func(filePath string, info os.FileInfo, err error) error { + if err != nil { + return fmt.Errorf("error walking path %s: %w", filePath, err) + } + + // Skip directories + if info.IsDir() { + return nil + } + + // Read file content + data, err := os.ReadFile(filePath) + if err != nil { + return fmt.Errorf("failed to read file %s: %w", filePath, err) + } + + // Calculate relative path from source folder + relPath, err := filepath.Rel(srcFolderPath, filePath) + if err != nil { + return fmt.Errorf("failed to calculate relative path: %w", err) + } + + // Convert to forward slashes for GCS + relPath = strings.ReplaceAll(relPath, "\\", "/") + + // Upload to GCS + gcsPath := path.Join(destPath, relPath) + if err := g.UploadFile(bucket, gcsPath, data); err != nil { + return fmt.Errorf("failed to upload file %s to GCS: %w", relPath, err) + } + + log.Info().Msgf("Uploaded file: %s to gs://%s/%s", relPath, bucket, gcsPath) + return nil + }) +} + +func (g *GCSClient) GetFolderInfo(bucket, folderPrefix string) (*GCSFolderInfo, error) { + if g.client == nil { + return nil, fmt.Errorf("GCS client not initialized properly") + } + + // Ensure folderPrefix ends with "/" + if !strings.HasSuffix(folderPrefix, "/") { + folderPrefix += "/" + } + + it := g.client.Bucket(bucket).Objects(g.ctx, &storage.Query{ + Prefix: folderPrefix, + }) + + var folderInfo GCSFolderInfo + folderInfo.Name = strings.TrimSuffix(path.Base(folderPrefix), "/") + folderInfo.Path = fmt.Sprintf("gs://%s/%s", bucket, strings.TrimSuffix(folderPrefix, "/")) + folderInfo.Created = time.Now() // Will be updated with actual earliest file + folderInfo.Updated = time.Time{} // Will be updated with actual latest file + + for { + attrs, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + return nil, fmt.Errorf("failed to list objects: %w", err) + } + + // Update folder stats + folderInfo.FileCount++ + folderInfo.Size += attrs.Size + + // Track earliest creation time + if attrs.Created.Before(folderInfo.Created) { + folderInfo.Created = attrs.Created + } + + // Track latest update time + if attrs.Updated.After(folderInfo.Updated) { + folderInfo.Updated = attrs.Updated + } + } + + if folderInfo.FileCount == 0 { + return nil, fmt.Errorf("folder not found or empty") + } + + return &folderInfo, nil +} + +func (g *GCSClient) ListFoldersWithTimestamp(bucket, prefix string) ([]GCSFolderInfo, error) { + if g.client == nil { + return nil, fmt.Errorf("GCS client not initialized properly") + } + + var folders []GCSFolderInfo + seenFolders := make(map[string]*GCSFolderInfo) + + // Ensure prefix ends with a trailing slash + if !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + + log.Info().Msgf("Listing folders with timestamps in GCS bucket %s with prefix %s", bucket, prefix) + + it := g.client.Bucket(bucket).Objects(g.ctx, &storage.Query{ + Prefix: prefix, + }) + + for { + attrs, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + return nil, fmt.Errorf("failed to list objects: %w", err) + } + + // Extract folder name after the prefix + if attrs.Name != "" { + trimmed := strings.TrimPrefix(attrs.Name, prefix) + parts := strings.SplitN(trimmed, "/", 2) + if len(parts) > 1 { + folderName := parts[0] + + // Initialize or update folder info + if folderInfo, exists := seenFolders[folderName]; !exists { + seenFolders[folderName] = &GCSFolderInfo{ + Name: folderName, + Path: fmt.Sprintf("gs://%s/%s%s", bucket, prefix, folderName), + Created: attrs.Created, + Updated: attrs.Updated, + Size: attrs.Size, + FileCount: 1, + } + } else { + // Update existing folder info + folderInfo.FileCount++ + folderInfo.Size += attrs.Size + + // Track earliest creation time + if attrs.Created.Before(folderInfo.Created) { + folderInfo.Created = attrs.Created + } + + // Track latest update time + if attrs.Updated.After(folderInfo.Updated) { + folderInfo.Updated = attrs.Updated + } + } + } + } + } + + // Convert map to slice + for _, folderInfo := range seenFolders { + folders = append(folders, *folderInfo) + } + + return folders, nil +} + +// FindFileWithSuffix finds the first file with the specified suffix in the given folder path +// Returns (exists, filename, error) +func (g *GCSClient) FindFileWithSuffix(bucket, folderPath, suffix string) (bool, string, error) { + if g.client == nil { + return false, "", fmt.Errorf("GCS client not initialized properly") + } + + // Ensure folderPath ends with "/" + if !strings.HasSuffix(folderPath, "/") { + folderPath += "/" + } + + log.Info().Msgf("Searching for files with suffix '%s' in GCS bucket %s with prefix %s", suffix, bucket, folderPath) + + it := g.client.Bucket(bucket).Objects(g.ctx, &storage.Query{ + Prefix: folderPath, + }) + + for { + attrs, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + return false, "", fmt.Errorf("failed to list objects: %w", err) + } + + // Get the filename from the full object path + fileName := path.Base(attrs.Name) + + // Check if the file ends with the specified suffix + if strings.HasSuffix(fileName, suffix) { + log.Info().Msgf("Found file with suffix '%s': %s", suffix, fileName) + return true, fileName, nil + } + } + + log.Info().Msgf("No file found with suffix '%s' in %s/%s", suffix, bucket, folderPath) + return false, "", nil +} diff --git a/horizon/internal/externalcall/init.go b/horizon/internal/externalcall/init.go new file mode 100644 index 00000000..99275dc0 --- /dev/null +++ b/horizon/internal/externalcall/init.go @@ -0,0 +1,14 @@ +package externalcall + +import ( + "github.com/Meesho/BharatMLStack/horizon/internal/configs" +) + +func Init(config configs.Configs) { + InitPrometheusClient(config.VmselectStartDaysAgo, config.VmselectBaseUrl, config.VmselectApiKey) + InitSlackClient(config.SlackWebhookUrl, config.SlackChannel, config.SlackCcTags, config.DefaultModelPath) + InitRingmasterClient(config.RingmasterBaseUrl, config.RingmasterMiscSession, config.RingmasterAuthorization, config.RingmasterEnvironment, config.RingmasterApiKey) + // Initialize feature validation client with config-based URLs + InitFeatureValidationClient(config) + // Pricing client is initialized in main.go +} diff --git a/horizon/internal/externalcall/pricing_client.go b/horizon/internal/externalcall/pricing_client.go new file mode 100644 index 00000000..0e37bc4b --- /dev/null +++ b/horizon/internal/externalcall/pricing_client.go @@ -0,0 +1,153 @@ +package externalcall + +import ( + "context" + "fmt" + "strings" + + pricingclient "github.com/Meesho/price-aggregator-go/pricingfeatureretrieval/client" + "github.com/Meesho/price-aggregator-go/pricingfeatureretrieval/client/models" + "github.com/rs/zerolog/log" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// PricingFeatureClient interface for pricing feature validation +type PricingFeatureClient interface { + GetDataTypes(entity string) (*PricingDataTypesResponse, error) +} + +// PricingDataTypesResponse represents the response from pricing feature service +type PricingDataTypesResponse struct { + Entities []struct { + Entity string `json:"entity"` + FeatureGroups []struct { + Label string `json:"label"` + Features []string `json:"features"` + DataType string `json:"dataType"` + } `json:"featureGroups"` + } `json:"entities"` +} + +type pricingFeatureClientImpl struct{} + +var PricingClient PricingFeatureClient = &pricingFeatureClientImpl{} + +// GetDataTypes calls the pricing service to get data types for features +func (p *pricingFeatureClientImpl) GetDataTypes(entity string) (*PricingDataTypesResponse, error) { + log.Info().Msgf("Calling pricing service for entity: %s", entity) + + // Create request for the pricing client (no entity parameter needed based on your requirement) + clientInstance := pricingclient.Instance(1) + + // Call the real pricing client + pricingResponse, err := clientInstance.GetDataTypes(context.Background(), &models.GetDataTypesRequest{}, map[string]string{}) + if err != nil { + log.Error().Err(err).Msgf("Failed to call pricing service for entity: %s", entity) + + // Check if it's a gRPC connection error and provide fallback + if st, ok := status.FromError(err); ok && st.Code() == codes.Unavailable { + log.Warn().Msgf("Pricing service unavailable for entity: %s, returning fallback response", entity) + return p.getFallbackResponse(entity), nil + } + + return nil, fmt.Errorf("failed to call pricing service: %w", err) + } + + // Convert the pricing client response to our response format + response := &PricingDataTypesResponse{ + Entities: make([]struct { + Entity string `json:"entity"` + FeatureGroups []struct { + Label string `json:"label"` + Features []string `json:"features"` + DataType string `json:"dataType"` + } `json:"featureGroups"` + }, len(pricingResponse.Entities)), + } + + // Map the response from pricing client to our format + for i, pricingEntity := range pricingResponse.Entities { + response.Entities[i].Entity = pricingEntity.Entity + response.Entities[i].FeatureGroups = make([]struct { + Label string `json:"label"` + Features []string `json:"features"` + DataType string `json:"dataType"` + }, len(pricingEntity.FeatureGroups)) + + for j, pricingFeatureGroup := range pricingEntity.FeatureGroups { + response.Entities[i].FeatureGroups[j].Label = pricingFeatureGroup.Label + response.Entities[i].FeatureGroups[j].Features = pricingFeatureGroup.Features + response.Entities[i].FeatureGroups[j].DataType = pricingFeatureGroup.DataType + } + } + + log.Info().Msgf("Successfully retrieved pricing data for entity %s with %d entities", entity, len(response.Entities)) + return response, nil +} + +// ValidatePricingFeatureExists checks if a pricing feature exists in the response +func ValidatePricingFeatureExists(featureName string, response *PricingDataTypesResponse) bool { + if response == nil { + return false + } + + // Parse feature name: entity:feature_group:feature + featureParts := strings.Split(featureName, ":") + if len(featureParts) != 3 { + return false + } + + expectedEntity := featureParts[0] // entity (e.g., "user_product") + expectedFG := featureParts[1] // feature group (e.g., "real_time_product_pricing") + expectedFeature := featureParts[2] // feature name (e.g., "rto_charge") + + // Search through entities and feature groups + for _, entity := range response.Entities { + // Check if this is the correct entity + if entity.Entity == expectedEntity { + for _, featureGroup := range entity.FeatureGroups { + // Check if this is the correct feature group + if featureGroup.Label == expectedFG { + // Check if feature exists in this feature group + for _, f := range featureGroup.Features { + if f == expectedFeature { + return true + } + } + } + } + } + } + + return false +} + +// getFallbackResponse provides a fallback response when pricing service is unavailable +func (p *pricingFeatureClientImpl) getFallbackResponse(entity string) *PricingDataTypesResponse { + return &PricingDataTypesResponse{ + Entities: []struct { + Entity string `json:"entity"` + FeatureGroups []struct { + Label string `json:"label"` + Features []string `json:"features"` + DataType string `json:"dataType"` + } `json:"featureGroups"` + }{ + { + Entity: entity, + FeatureGroups: []struct { + Label string `json:"label"` + Features []string `json:"features"` + DataType string `json:"dataType"` + }{ + { + Label: "fallback_feature_group", + Features: []string{"fallback_feature"}, + DataType: "DataTypeString", + }, + }, + }, + }, + } +} diff --git a/horizon/internal/externalcall/prometheus_client.go b/horizon/internal/externalcall/prometheus_client.go new file mode 100644 index 00000000..d62a0bd2 --- /dev/null +++ b/horizon/internal/externalcall/prometheus_client.go @@ -0,0 +1,246 @@ +package externalcall + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "sync" + "time" +) + +type PrometheusClient interface { + GetModelNames(serviceName string) ([]string, error) + GetInferflowConfigNames(serviceName string) ([]string, error) + GetNumerixConfigNames() ([]string, error) +} + +type prometheusClientImpl struct { + BaseURL string + HTTPClient *http.Client + APIKey string +} + +var ( + prometheusOnce sync.Once + prometheusInstance PrometheusClient + vmselectStartDaysAgo int + vmselectBaseUrl string + vmselectApiKey string + initPrometheusOnce sync.Once +) + +func InitPrometheusClient(VmselectStartDaysAgo int, VmselectBaseUrl string, VmselectApiKey string) { + initPrometheusOnce.Do(func() { + vmselectStartDaysAgo = VmselectStartDaysAgo + vmselectBaseUrl = VmselectBaseUrl + vmselectApiKey = VmselectApiKey + }) +} + +func GetPrometheusClient() PrometheusClient { + prometheusOnce.Do(func() { + prometheusInstance = &prometheusClientImpl{ + BaseURL: vmselectBaseUrl, + HTTPClient: &http.Client{ + Timeout: 10 * time.Second, + }, + APIKey: vmselectApiKey, + } + }) + return prometheusInstance +} + +type prometheusResponse struct { + Status string `json:"status"` + IsPartial bool `json:"isPartial"` + Data struct { + ResultType string `json:"resultType"` + Result []struct { + Metric struct { + ModelName string `json:"model_name"` + } `json:"metric"` + } `json:"result"` + } `json:"data"` +} + +type prometheusInferflowConfigResponse struct { + Status string `json:"status"` + IsPartial bool `json:"isPartial"` + Data struct { + ResultType string `json:"resultType"` + Result []struct { + Metric struct { + ModelID string `json:"model_id"` + } `json:"metric"` + } `json:"result"` + } `json:"data"` +} + +type prometheusNumerixConfigResponse struct { + Status string `json:"status"` + IsPartial bool `json:"isPartial"` + Data struct { + ResultType string `json:"resultType"` + Result []struct { + Metric struct { + ComputeID string `json:"compute_id"` + } `json:"metric"` + } `json:"result"` + } `json:"data"` +} + +func (p *prometheusClientImpl) GetModelNames(serviceName string) ([]string, error) { + end := time.Now().Unix() + daysAgo := vmselectStartDaysAgo + start := end - int64(daysAgo*24*60*60) + step := "1h40m" + + query := fmt.Sprintf( + "sum by (model_name)(rate(modelinferenceorchestrator_retrievemodelinferencescore_request_total_value{service=\"%s\"}[1m]))", + serviceName, + ) + + url := fmt.Sprintf("%s/select/100/prometheus/api/v1/query_range?query=%s&start=%d&end=%d&step=%s", + p.BaseURL, + escapePrometheusQuery(query), + start, + end, + step, + ) + + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("api-key", p.APIKey) + + resp, err := p.HTTPClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to call Prometheus: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + bodyBytes, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("Prometheus call failed, status: %d, body: %s", resp.StatusCode, string(bodyBytes)) + } + + var pr prometheusResponse + if err := json.NewDecoder(resp.Body).Decode(&pr); err != nil { + return nil, fmt.Errorf("failed to decode Prometheus response: %w", err) + } + + var modelNames []string + for _, item := range pr.Data.Result { + modelNames = append(modelNames, item.Metric.ModelName) + } + + return modelNames, nil +} + +func (p *prometheusClientImpl) GetInferflowConfigNames(serviceName string) ([]string, error) { + end := time.Now().Unix() + daysAgo := vmselectStartDaysAgo + start := end - int64(daysAgo*24*60*60) + step := "1h40m" + + query := fmt.Sprintf( + "sum by (model_id)(rate(inferflow_retrievemodelscore_request_total_value{service=\"%s\"}[1m]))", + serviceName, + ) + + url := fmt.Sprintf("%s/select/100/prometheus/api/v1/query_range?query=%s&start=%d&end=%d&step=%s", + p.BaseURL, + escapePrometheusQuery(query), + start, + end, + step, + ) + + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("api-key", p.APIKey) + + resp, err := p.HTTPClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to call Prometheus: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + bodyBytes, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("Prometheus call failed, status: %d, body: %s", resp.StatusCode, string(bodyBytes)) + } + + var pr prometheusInferflowConfigResponse + if err := json.NewDecoder(resp.Body).Decode(&pr); err != nil { + return nil, fmt.Errorf("failed to decode Prometheus response: %w", err) + } + + var modelNames []string + for _, item := range pr.Data.Result { + modelNames = append(modelNames, item.Metric.ModelID) + } + + return modelNames, nil +} + +func (p *prometheusClientImpl) GetNumerixConfigNames() ([]string, error) { + end := time.Now().Unix() + daysAgo := vmselectStartDaysAgo + start := end - int64(daysAgo*24*60*60) + step := "1h40m" + + query := fmt.Sprintf( + "sum by (compute_id)(rate(numerix_numerix_computation_request_total_value[1m]))", + ) + + url := fmt.Sprintf("%s/select/100/prometheus/api/v1/query_range?query=%s&start=%d&end=%d&step=%s", + p.BaseURL, + escapePrometheusQuery(query), + start, + end, + step, + ) + + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("api-key", p.APIKey) + + resp, err := p.HTTPClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to call Prometheus: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + bodyBytes, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("Prometheus call failed, status: %d, body: %s", resp.StatusCode, string(bodyBytes)) + } + + var pr prometheusNumerixConfigResponse + if err := json.NewDecoder(resp.Body).Decode(&pr); err != nil { + return nil, fmt.Errorf("failed to decode Prometheus response: %w", err) + } + + var computeIDs []string + for _, item := range pr.Data.Result { + computeIDs = append(computeIDs, item.Metric.ComputeID) + } + + return computeIDs, nil +} + +func escapePrometheusQuery(query string) string { + // Spaces, quotes, braces and brackets are URL-encoded + return url.QueryEscape(query) +} diff --git a/horizon/internal/externalcall/ring_master_client.go b/horizon/internal/externalcall/ring_master_client.go new file mode 100644 index 00000000..bd88482d --- /dev/null +++ b/horizon/internal/externalcall/ring_master_client.go @@ -0,0 +1,473 @@ +package externalcall + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "sync" + "time" + + "github.com/rs/zerolog/log" + + "github.com/Meesho/BharatMLStack/horizon/internal/repositories/sql/servicedeployableconfig" +) + +var ( + initRingmasterOnce sync.Once + ringmasterBaseUrl string + ringmasterMiscSession string + ringmasterAuthorization string + ringmasterEnvironment string + ringmasterApiKey string +) + +func InitRingmasterClient(RingmasterBaseUrl string, RingmasterMiscSession string, RingmasterAuthorization string, RingmasterEnvironment string, RingmasterApiKey string) { + initRingmasterOnce.Do(func() { + ringmasterBaseUrl = RingmasterBaseUrl + ringmasterMiscSession = RingmasterMiscSession + ringmasterAuthorization = RingmasterAuthorization + ringmasterEnvironment = RingmasterEnvironment + ringmasterApiKey = RingmasterApiKey + }) +} + +type RingmasterClient interface { + GetConfig(serviceName, workflowID, runID string) Config + RestartDeployable(sd *servicedeployableconfig.ServiceDeployableConfig) error + CreateDeployable(payload map[string]interface{}) ([]byte, error) + UpdateDeployable(name string, payload map[string]interface{}) error + UpdateCPUThreshold(appName string, cpuThreshold string) error + UpdateGPUThreshold(appName string, gpuThreshold string) error + GetResourceDetail(serviceName string) (*ResourceDetail, error) +} + +type Config struct { + MinReplica string `json:"min_replica"` + MaxReplica string `json:"max_replica"` + RunningStatus string `json:"running_status"` +} + +type Activity struct { + Name string `json:"name"` + Status string `json:"status"` + Error string `json:"error,omitempty"` +} + +type ResourceDetail struct { + Nodes []Node `json:"nodes"` +} + +var respBody struct { + Min int `json:"min"` + Max int `json:"max"` + Desired int `json:"desired"` + Current int `json:"current"` +} + +type DeployableConfig struct { + DeploymentStrategy string `json:"deploymentStrategy"` +} + +type Node struct { + Kind string `json:"kind"` + Name string `json:"name"` + Health Health `json:"health"` + Info []InfoItem `json:"info"` +} + +type Health struct { + Status string `json:"status"` +} + +type InfoItem struct { + Value string `json:"value"` +} + +type WorkflowResultRequest struct { + ApplicationName string `json:"applicationName"` + WorkflowID string `json:"workflowId"` + RunID string `json:"runId"` +} + +type ringmasterClientImpl struct { + BaseURL string + HTTPClient *http.Client + MiscSession string + Authorization string + Environment string +} + +var ( + clientInstance RingmasterClient + ringmasterOnce sync.Once +) + +// Helper function to read raw response +func readRawResponse(resp *http.Response) ([]byte, error) { + defer resp.Body.Close() + return io.ReadAll(resp.Body) +} + +func GetRingmasterClient() RingmasterClient { + ringmasterOnce.Do(func() { + clientInstance = &ringmasterClientImpl{ + BaseURL: ringmasterBaseUrl, + HTTPClient: &http.Client{ + Timeout: 200 * time.Second, + }, + MiscSession: ringmasterMiscSession, + Authorization: ringmasterAuthorization, + Environment: ringmasterEnvironment, + } + }) + return clientInstance +} + +func (r *ringmasterClientImpl) GetConfig(serviceName, workflowID, runID string) Config { + parts := strings.Split(r.Environment, "_") + url := fmt.Sprintf("%s/api/v1/mlp/application/resource/%s-%s/HorizontalPodAutoscaler?workingEnv=%s", + r.BaseURL, parts[1], serviceName, r.Environment) + + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + log.Error().Msgf("failed to create request: %s", err) + return Config{ + MinReplica: "0", + MaxReplica: "0", + RunningStatus: "false", + } + } + + r.setCommonHeaders(req) + + resp, err := r.HTTPClient.Do(req) + if err != nil { + log.Error().Msgf("failed to call ringmaster GetConfig: %s", err) + return Config{ + MinReplica: "0", + MaxReplica: "0", + RunningStatus: "false", + } + } + + rawBody, err := readRawResponse(resp) + if err != nil { + log.Error().Msgf("failed to read response body: %s", err) + return Config{ + MinReplica: "0", + MaxReplica: "0", + RunningStatus: "false", + } + } + + if resp.StatusCode != http.StatusOK { + log.Error().Msgf("ringmaster GetConfig failed, status: %d, body: %s", resp.StatusCode, string(rawBody)) + return Config{ + MinReplica: "0", + MaxReplica: "0", + RunningStatus: "false", + } + } + + if err := json.Unmarshal(rawBody, &respBody); err != nil { + log.Error().Msgf("failed to decode response body: %s, raw response: %s", err, string(rawBody)) + return Config{ + MinReplica: "0", + MaxReplica: "0", + RunningStatus: "false", + } + } + + // Get workflow result to determine running status + result, err := r.GetResourceDetail(serviceName) + healthyPodCount := 0 + for _, node := range result.Nodes { + if node.Kind == "Pod" && node.Health.Status == "Healthy" { + for _, info := range node.Info { + if info.Value == "Running" { + healthyPodCount += 1 + } + } + } + } + + if healthyPodCount == 0 { + log.Error().Msgf("No healthy pods found for service: %s", serviceName) + return Config{ + MinReplica: strconv.Itoa(respBody.Min), + MaxReplica: strconv.Itoa(respBody.Max), + RunningStatus: "false", + } + } + + return Config{ + MinReplica: strconv.Itoa(respBody.Min), + MaxReplica: strconv.Itoa(respBody.Max), + RunningStatus: "true", + } +} + +func (r *ringmasterClientImpl) RestartDeployable(sd *servicedeployableconfig.ServiceDeployableConfig) error { + var deployableConfig DeployableConfig + + if err := json.Unmarshal(sd.Config, &deployableConfig); err != nil { + return fmt.Errorf("failed to unmarshal deployable config: %w", err) + } + + parts := strings.Split(r.Environment, "_") + if len(parts) < 2 { + return fmt.Errorf("invalid environment format: %s", r.Environment) + } + + url := fmt.Sprintf("%s/api/v1/mlp/application/%s-%s/resource/deployment/restart?workingEnv=%s", + r.BaseURL, parts[1], sd.Name, r.Environment) + + // Always send isCanary: "true" or "false" + isCanary := "false" + if deployableConfig.DeploymentStrategy == "canary" { + isCanary = "true" + } + + body, err := json.Marshal(map[string]string{ + "isCanary": isCanary, + }) + if err != nil { + return fmt.Errorf("failed to marshal request body: %w", err) + } + + req, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(body)) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + + r.setCommonHeaders(req) + req.Header.Set("Content-Type", "application/json") + + resp, err := r.HTTPClient.Do(req) + if err != nil { + return fmt.Errorf("failed to call ringmaster RestartDeployable: %w", err) + } + + rawBody, err := readRawResponse(resp) + if err != nil { + return fmt.Errorf("failed to read response: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("ringmaster RestartDeployable failed, status: %d, body: %s", resp.StatusCode, string(rawBody)) + } + + return nil +} + +func (r *ringmasterClientImpl) setCommonHeaders(req *http.Request) { + req.Header.Set("misc_session", r.MiscSession) + req.Header.Set("Authorization", r.Authorization) +} + +func (r *ringmasterClientImpl) CreateDeployable(payload map[string]interface{}) ([]byte, error) { + url := fmt.Sprintf("%s/api/v1/mlp/gpu/cicd/onboarding?workingEnv=%s", r.BaseURL, r.Environment) + + bodyBytes, err := json.Marshal(payload) + if err != nil { + return nil, fmt.Errorf("failed to marshal payload: %w", err) + } + + req, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(bodyBytes)) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + r.setCommonHeaders(req) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("api-key", ringmasterApiKey) + + resp, err := r.HTTPClient.Do(req) + if err != nil { + return nil, fmt.Errorf("ringmaster CreateDeployable request failed: %w", err) + } + defer resp.Body.Close() + + rawBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("ringmaster CreateDeployable failed, status: %d, body: %s", resp.StatusCode, string(rawBody)) + } + + return rawBody, nil +} + +func (r *ringmasterClientImpl) UpdateDeployable(name string, payload map[string]interface{}) error { + parts := strings.Split(r.Environment, "_") + var url string + + if _, ok := payload["cpuThreshold"]; ok { + url = fmt.Sprintf("%s/api/v1/mlp/application/%s-%s/resource/hpa/cpu?workingEnv=%s", + r.BaseURL, parts[1], name, r.Environment) + } else if _, ok := payload["gpuThreshold"]; ok { + url = fmt.Sprintf("%s/api/v1/mlp/application/%s-%s/resource/hpa/gpu?workingEnv=%s", + r.BaseURL, parts[1], name, r.Environment) + } else { + return fmt.Errorf("invalid threshold update payload") + } + + bodyBytes, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("failed to marshal update deployable payload: %w", err) + } + + req, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(bodyBytes)) + if err != nil { + return fmt.Errorf("failed to create update request: %w", err) + } + + r.setCommonHeaders(req) + req.Header.Set("Content-Type", "application/json") + + resp, err := r.HTTPClient.Do(req) + if err != nil { + return fmt.Errorf("failed to call ringmaster UpdateDeployable: %w", err) + } + + rawBody, err := readRawResponse(resp) + if err != nil { + return fmt.Errorf("failed to read response: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("ringmaster UpdateDeployable failed, status: %d, body: %s", resp.StatusCode, string(rawBody)) + } + + return nil +} + +func (r *ringmasterClientImpl) UpdateCPUThreshold(appName string, cpuThreshold string) error { + parts := strings.Split(r.Environment, "_") + url := fmt.Sprintf("%s/api/v1/mlp/application/%s-%s/resource/hpa/cpu?workingEnv=%s", + r.BaseURL, parts[1], appName, r.Environment) + + payload := map[string]string{ + "cpuThreshold": cpuThreshold, + } + + bodyBytes, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("failed to marshal CPU threshold payload: %w", err) + } + + req, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(bodyBytes)) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + + r.setCommonHeaders(req) + req.Header.Set("Content-Type", "application/json") + + resp, err := r.HTTPClient.Do(req) + if err != nil { + return fmt.Errorf("failed to update CPU threshold: %w", err) + } + + rawBody, err := readRawResponse(resp) + if err != nil { + return fmt.Errorf("failed to read response: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("update CPU threshold failed, status: %d, body: %s", resp.StatusCode, string(rawBody)) + } + + return nil +} + +func (r *ringmasterClientImpl) UpdateGPUThreshold(appName string, gpuThreshold string) error { + parts := strings.Split(r.Environment, "_") + url := fmt.Sprintf("%s/api/v1/mlp/application/%s-%s/resource/hpa/gpu?workingEnv=%s", + r.BaseURL, parts[1], appName, r.Environment) + + payload := map[string]string{ + "gpuThreshold": gpuThreshold, + } + + bodyBytes, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("failed to marshal GPU threshold payload: %w", err) + } + + req, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(bodyBytes)) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + + r.setCommonHeaders(req) + req.Header.Set("Content-Type", "application/json") + + resp, err := r.HTTPClient.Do(req) + if err != nil { + return fmt.Errorf("failed to update GPU threshold: %w", err) + } + + rawBody, err := readRawResponse(resp) + if err != nil { + return fmt.Errorf("failed to read response: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("update GPU threshold failed, status: %d, body: %s", resp.StatusCode, string(rawBody)) + } + + return nil +} + +func (r *ringmasterClientImpl) GetResourceDetail(serviceName string) (*ResourceDetail, error) { + env := r.Environment + parts := strings.Split(env, "_") + var servicePrefix string + if len(parts) == 2 { + servicePrefix = parts[1] + "-" + } + + url := fmt.Sprintf("%s/api/v1/mlp/application/resource-detail?appName=%s&workingEnv=%s", + r.BaseURL, servicePrefix+serviceName, r.Environment) + + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + // Set headers + r.setCommonHeaders(req) + + resp, err := r.HTTPClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to call ringmaster GetResourceDetail: %w", err) + } + + defer func() { + _ = resp.Body.Close() // safe deferred close + }() + + rawBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("ringmaster GetResourceDetail failed, status: %d, body: %s", resp.StatusCode, string(rawBody)) + } + + var detail ResourceDetail + if err := json.Unmarshal(rawBody, &detail); err != nil { + return nil, fmt.Errorf("failed to decode resource detail: %w\nraw response:\n%s", err, string(rawBody)) + } + + return &detail, nil +} diff --git a/horizon/internal/externalcall/slack_client.go b/horizon/internal/externalcall/slack_client.go new file mode 100644 index 00000000..8e767963 --- /dev/null +++ b/horizon/internal/externalcall/slack_client.go @@ -0,0 +1,197 @@ +package externalcall + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "strings" + "sync" + "time" +) + +var ( + initSlackOnce sync.Once + slackWebhookUrl string + slackChannel string + slackCcTags string + defaultModelPath string +) + +func InitSlackClient(SlackWebhookUrl string, SlackChannel string, SlackCcTags string, DefaultModelPath string) { + initSlackOnce.Do(func() { + slackWebhookUrl = SlackWebhookUrl + slackChannel = SlackChannel + slackCcTags = SlackCcTags + defaultModelPath = DefaultModelPath + }) +} + +type SlackClient interface { + SendCleanupNotification(deployableName string, deletedModels []string) error + SendNumerixConfigCleanupNotification(deployableName string, deletedConfigs []string) error + SendInferflowConfigCleanupNotification(deployableName string, deletedConfigs []string) error +} + +type slackClientImpl struct { + WebhookURL string + Channel string + BackupPath string + CCTags string + InactiveDays int + HTTPClient *http.Client +} + +var ( + slackOnce sync.Once + slackInstance SlackClient +) + +func GetSlackClient() SlackClient { + slackOnce.Do(func() { + slackInstance = &slackClientImpl{ + WebhookURL: slackWebhookUrl, + Channel: slackChannel, + BackupPath: defaultModelPath, + CCTags: slackCcTags, + InactiveDays: vmselectStartDaysAgo, + HTTPClient: &http.Client{ + Timeout: 5 * time.Second, + }, + } + }) + return slackInstance +} + +type slackPayload struct { + Username string `json:"username"` + IconEmoji string `json:"icon_emoji"` + Channel string `json:"channel"` + Text string `json:"text"` +} + +func (s *slackClientImpl) SendCleanupNotification(deployableName string, deletedModels []string) error { + modelsText := strings.Join(deletedModels, "\n") + ccTags := strings.Join(strings.Fields(s.CCTags), "\n") // handles space-separated tags + message := fmt.Sprintf(`Prod Expt Models cleanup for %s + +Following models are deleted from prod experiment GCS repo and moved to backup folder (%s) as they aren't receiving any traffic in last %d days: +%s + +cc: +%s`, deployableName, s.BackupPath, s.InactiveDays, modelsText, ccTags) + + payload := slackPayload{ + Username: "Models Monitoring", + IconEmoji: ":computer:", + Channel: s.Channel, + Text: message, + } + + bodyBytes, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("failed to marshal slack payload: %w", err) + } + + req, err := http.NewRequest(http.MethodPost, s.WebhookURL, bytes.NewBuffer(bodyBytes)) + if err != nil { + return fmt.Errorf("failed to create slack request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + + resp, err := s.HTTPClient.Do(req) + if err != nil { + return fmt.Errorf("failed to send slack request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("slack webhook returned non-OK status: %d", resp.StatusCode) + } + + return nil +} + +func (s *slackClientImpl) SendNumerixConfigCleanupNotification(deployableName string, deletedConfigs []string) error { + configsText := strings.Join(deletedConfigs, "\n") + ccTags := strings.Join(strings.Fields(s.CCTags), "\n") // handles space-separated tags + message := fmt.Sprintf(`Prod Numerix Config cleanup for %s + +Following Numerix configs are deleted from prod Numerix config registry as they aren't receiving any traffic in last %d days: +%s + +cc: +%s`, deployableName, s.InactiveDays, configsText, ccTags) + + payload := slackPayload{ + Username: "Numerix Config Monitoring", + IconEmoji: ":computer:", + Channel: s.Channel, + Text: message, + } + + bodyBytes, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("failed to marshal slack payload: %w", err) + } + + req, err := http.NewRequest(http.MethodPost, s.WebhookURL, bytes.NewBuffer(bodyBytes)) + if err != nil { + return fmt.Errorf("failed to create slack request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + + resp, err := s.HTTPClient.Do(req) + if err != nil { + return fmt.Errorf("failed to send slack request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("slack webhook returned non-OK status: %d", resp.StatusCode) + } + + return nil +} + +func (s *slackClientImpl) SendInferflowConfigCleanupNotification(deployableName string, deletedConfigs []string) error { + configsText := strings.Join(deletedConfigs, "\n") + ccTags := strings.Join(strings.Fields(s.CCTags), "\n") // handles space-separated tags + message := fmt.Sprintf(`Prod Inferflow Config cleanup for %s + +Following Inferflow configs are deleted from prod Inferflow config registry as they aren't receiving any traffic in last %d days: +%s + +cc: +%s`, deployableName, s.InactiveDays, configsText, ccTags) + + payload := slackPayload{ + Username: "Inferflow Config Monitoring", + IconEmoji: ":computer:", + Channel: s.Channel, + Text: message, + } + + bodyBytes, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("failed to marshal slack payload: %w", err) + } + + req, err := http.NewRequest(http.MethodPost, s.WebhookURL, bytes.NewBuffer(bodyBytes)) + if err != nil { + return fmt.Errorf("failed to create slack request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + + resp, err := s.HTTPClient.Do(req) + if err != nil { + return fmt.Errorf("failed to send slack request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("slack webhook returned non-OK status: %d", resp.StatusCode) + } + + return nil +} diff --git a/horizon/internal/feature_store/controller/fs_config.go b/horizon/internal/feature_store/controller/fs_config.go new file mode 100644 index 00000000..71e51646 --- /dev/null +++ b/horizon/internal/feature_store/controller/fs_config.go @@ -0,0 +1,31 @@ +package controller + +import ( + feature_registry "github.com/Meesho/BharatMLStack/horizon/internal/feature_store/handler" + "github.com/gin-gonic/gin" + "github.com/rs/zerolog/log" +) + +type FsConfigController interface { + GetFsConfigs(ctx *gin.Context) +} + +type FsConfigControllerV1 struct { + FsConfigHandler feature_registry.FsConfigHandler +} + +func NewFsConfigControllerGenerator(handler feature_registry.FsConfigHandler) *FsConfigControllerV1 { + if handler == nil { + log.Panic().Msg("application config handler cannot be nil") + } + return &FsConfigControllerV1{ + FsConfigHandler: handler, + } +} + +func (f *FsConfigControllerV1) GetFsConfigs(ctx *gin.Context) { + if apiError := f.FsConfigHandler.GetFsConfigs(ctx); apiError != nil { + ctx.Error(apiError) + return + } +} diff --git a/horizon/internal/feature_store/handler/fs_config.go b/horizon/internal/feature_store/handler/fs_config.go new file mode 100644 index 00000000..ed818585 --- /dev/null +++ b/horizon/internal/feature_store/handler/fs_config.go @@ -0,0 +1,121 @@ +package handler + +import ( + "encoding/json" + "fmt" + "sync" + + "github.com/Meesho/BharatMLStack/horizon/internal/configs" + "github.com/Meesho/BharatMLStack/horizon/pkg/api" + "github.com/Meesho/BharatMLStack/horizon/pkg/zookeeper" + "github.com/gin-gonic/gin" + "github.com/klauspost/compress/zstd" + "github.com/rs/zerolog/log" +) + +type FsConfigHandler interface { + GetFsConfigs(ctx *gin.Context) *api.Error +} + +type FsConfig struct { + fsConfigMap map[string][]byte + zkConfig *Conf +} + +var Mutex sync.Mutex + +func NewFsConfigHandlerGenerator() *FsConfig { + return &FsConfig{ + fsConfigMap: make(map[string][]byte), + zkConfig: &Conf{}, + } +} +func (f *FsConfig) GetFsConfigs(ctx *gin.Context) *api.Error { + data, ok := f.fsConfigMap["fs_config"] + if !ok { + return api.NewInternalServerError("Failed to retrieve configs") + } + response := &FsConfigResponse{ + Data: data, + } + decoder, err := zstd.NewReader(nil) + if err != nil { + log.Error().Err(err).Msg("Failed to create zstd decoder") + + } + decompressedBytes, err := decoder.DecodeAll(data, nil) + if err != nil { + log.Error().Err(err).Msg("Failed to decompress bytes") + + } + + zkconfig1 := Conf{} + // Unmarshal the decompressed bytes into zkConfig + err = json.Unmarshal(decompressedBytes, &zkconfig1) + ctx.JSON(200, response) + return nil +} + +func (f *FsConfig) InitFsConfig(config configs.Configs) { + zookeeper.InitZKConnection(config) + zookeeper.WatchZkNode() + go f.ListenZk() + decoder, err := zstd.NewReader(nil) + if err != nil { + log.Error().Err(err).Msg("Failed to create zstd decoder") + return + } + decompressedBytes, err := decoder.DecodeAll(f.fsConfigMap["fs_config"], nil) + if err != nil { + log.Error().Err(err).Msg("Failed to decompress bytes") + return + } + + zkconfig1 := Conf{} + // Unmarshal the decompressed bytes into zkConfig + err = json.Unmarshal(decompressedBytes, &zkconfig1) + + log.Info().Msg("Initializing FS Config Handler") +} + +func (f *FsConfig) ListenZk() { + for { + select { + case data := <-zookeeper.ZKChannel: + var localZkConf *Conf + var compressedBytes []byte + err := json.Unmarshal(data, &localZkConf) + if err != nil { + log.Error().Err(err).Msg("Failed to unmarshal JSON") + } else { + Mutex.Lock() + f.zkConfig = localZkConf + + // Convert ZkConf to byte array + zkConfBytes, err := json.Marshal(f.zkConfig) + if err != nil { + log.Error().Err(err).Msg("Failed to marshal ZkConf") + } else { + // Print size before compression + fmt.Printf("Size before compression: %d bytes\n", len(zkConfBytes)) + + // Apply zstd compression + encoder, err := zstd.NewWriter(nil) + if err != nil { + log.Error().Err(err).Msg("Failed to create zstd encoder") + } + compressedBytes = encoder.EncodeAll(zkConfBytes, nil) + + // Print size after compression + fmt.Printf("Size after compression: %d bytes\n", len(compressedBytes)) + } + Mutex.Unlock() + + } + + if len(compressedBytes) != 0 { + f.fsConfigMap["fs_config"] = compressedBytes + } + } + } +} diff --git a/horizon/internal/feature_store/handler/model.go b/horizon/internal/feature_store/handler/model.go new file mode 100644 index 00000000..a710277a --- /dev/null +++ b/horizon/internal/feature_store/handler/model.go @@ -0,0 +1,75 @@ +package handler + +type Conf struct { + EntityConf map[string]EntityConf `json:"entities"` + Storage Storage `json:"storage"` + InMemory map[string]InMemoryConf `json:"in-memory"` + McacheEnabled ExperimentControl `json:"mcache-enabled"` //for all clusters + ErrorLoggingPercentage string `json:"error-logging-percentage"` +} + +type InMemoryConf struct { + FeatureGroupConf map[string]string `json:"feature-groups"` +} + +type EntityConf struct { + Label string `json:"label"` + Keys map[int]Keys `json:"keys"` + FeatureGroupConf map[string]FeatureGroupConf `json:"feature-groups"` + InMemoryEnabled string `json:"in-memory-enabled"` + DistributedCacheEnabled string `json:"distributed-cache-enabled"` + RedisCacheTTL string `json:"redis-cache-ttl"` + DistributeCacheInfraVersion string `json:"distributed-cache-infra-version"` + DistributedCacheWritePercentage string `json:"distributed-cache-write-percentage"` +} + +type Keys struct { + Sequence string `json:"sequence"` + EntityLabel string `json:"entity-label"` + ColumnLabel string `json:"column-label"` +} + +type FeatureGroupConf struct { + FeatureColumnConf map[string]FeatureColumnConf `json:"columns"` + StoreId string `json:"storeId"` + Label string `json:"label"` + DataType string `json:"data-type"` + Ttl string `json:"ttl"` + JobId string `json:"jobId"` + MaxSizeByte string `json:"max-size-byte"` +} + +type FeatureColumnConf struct { + ColumnLabel string `json:"column-label"` + ActiveConfig string `json:"active-config"` + CurrentSizeByte string `json:"current-size-byte"` + FeatureConf map[string]FeatureConf `json:"config"` +} + +type FeatureConf struct { + FeatureLabels string `json:"feature-labels"` + FeatureDefaultValues string `json:"feature-default-values"` +} + +type Storage struct { + RateLimiter RateLimiter `json:"rate-limiter"` + RedisCacheTTL string `json:"redis-cache-ttl"` + Stores map[string]string `json:"stores"` + DefaultScyllaPercentage string `json:"default-scylla-percentage"` +} + +type RateLimiter struct { + PermitsPerSecond string `json:"permits-per-second"` + WarmupPeriod string `json:"warmup-period"` +} + +type ExperimentControl struct { + Enabled string `json:"enabled"` + PercentageEnabled string `json:"percentage-enabled"` + Deadline string `json:"deadline"` + JitterPercentage string `json:"jitter-percentage"` +} + +type FsConfigResponse struct { + Data []byte `json:"data"` +} diff --git a/horizon/internal/feature_store/route/router.go b/horizon/internal/feature_store/route/router.go new file mode 100644 index 00000000..a99e0446 --- /dev/null +++ b/horizon/internal/feature_store/route/router.go @@ -0,0 +1,28 @@ +package route + +import ( + "sync" + + "github.com/Meesho/BharatMLStack/horizon/internal/configs" + "github.com/Meesho/BharatMLStack/horizon/internal/feature_store/controller" + feature_registry "github.com/Meesho/BharatMLStack/horizon/internal/feature_store/handler" + "github.com/Meesho/BharatMLStack/horizon/pkg/httpframework" +) + +var initFeatureStoreRouterOnce sync.Once + +func Init(config configs.Configs) { + initFeatureStoreRouterOnce.Do(func() { + api := httpframework.Instance().Group("/api") + { + v1 := api.Group("/1.0") + + fsConfigHandler := feature_registry.NewFsConfigHandlerGenerator() + fsConfigHandler.InitFsConfig(config) + fsConfig := v1.Group("/fs-config") + { + fsConfig.GET("/fetch", controller.NewFsConfigControllerGenerator(fsConfigHandler).GetFsConfigs) + } + } + }) +} diff --git a/horizon/internal/inferflow/controller/controller.go b/horizon/internal/inferflow/controller/controller.go new file mode 100644 index 00000000..0d62160f --- /dev/null +++ b/horizon/internal/inferflow/controller/controller.go @@ -0,0 +1,367 @@ +package controller + +import ( + "sync" + + "strings" + + handler "github.com/Meesho/BharatMLStack/horizon/internal/inferflow/handler" + "github.com/Meesho/BharatMLStack/horizon/pkg/api" + "github.com/gin-gonic/gin" +) + +type Config interface { + Onboard(ctx *gin.Context) + Promote(ctx *gin.Context) + Edit(ctx *gin.Context) + Clone(ctx *gin.Context) + Delete(ctx *gin.Context) + ScaleUp(ctx *gin.Context) + Cancel(ctx *gin.Context) + Review(ctx *gin.Context) + GetAll(ctx *gin.Context) + GetAllRequests(ctx *gin.Context) + ValidateRequest(ctx *gin.Context) + GenerateFunctionalTestRequest(ctx *gin.Context) + ExecuteFuncitonalTestRequest(ctx *gin.Context) + GetLatestRequest(ctx *gin.Context) + GetLoggingTTL(ctx *gin.Context) +} + +var ( + configController Config + once sync.Once +) + +type V1 struct { + Config handler.Config +} + +func NewConfigController() Config { + if configController == nil { + once.Do(func() { + configController = &V1{ + Config: handler.NewConfigHandler(1), + } + }) + } + return configController +} + +func (c *V1) Onboard(ctx *gin.Context) { + var request handler.InferflowOnboardRequest + var emptyResponse string + if err := ctx.ShouldBindJSON(&request); err != nil { + ctx.JSON(api.NewBadRequestError(err.Error()).StatusCode, handler.Response{ + Error: err.Error(), + Data: handler.Message{Message: emptyResponse}, + }) + return + } + + token := ctx.GetHeader("Authorization") + token = strings.TrimPrefix(token, "Bearer ") + response, err := c.Config.Onboard(request, token) + if err != nil { + ctx.JSON(api.NewBadRequestError(err.Error()).StatusCode, handler.Response{ + Error: err.Error(), + Data: handler.Message{Message: emptyResponse}, + }) + return + } + ctx.JSON(200, response) +} + +func (c *V1) Promote(ctx *gin.Context) { + var request handler.PromoteConfigRequest + var emptyResponse string + if err := ctx.ShouldBindJSON(&request); err != nil { + ctx.JSON(api.NewBadRequestError(err.Error()).StatusCode, handler.Response{ + Error: err.Error(), + Data: handler.Message{Message: emptyResponse}, + }) + return + } + + response, err := c.Config.Promote(request) + if err != nil { + ctx.JSON(api.NewBadRequestError(err.Error()).StatusCode, handler.Response{ + Error: err.Error(), + Data: handler.Message{Message: emptyResponse}, + }) + return + } + ctx.JSON(200, response) +} + +func (c *V1) Edit(ctx *gin.Context) { + var request handler.EditConfigOrCloneConfigRequest + var emptyResponse string + if err := ctx.ShouldBindJSON(&request); err != nil { + ctx.JSON(api.NewBadRequestError(err.Error()).StatusCode, handler.Response{ + Error: err.Error(), + Data: handler.Message{Message: emptyResponse}, + }) + return + } + + token := ctx.GetHeader("Authorization") + token = strings.TrimPrefix(token, "Bearer ") + response, err := c.Config.Edit(request, token) + if err != nil { + ctx.JSON(api.NewBadRequestError(err.Error()).StatusCode, handler.Response{ + Error: err.Error(), + Data: handler.Message{Message: emptyResponse}, + }) + return + } + ctx.JSON(200, response) +} + +func (c *V1) Clone(ctx *gin.Context) { + var request handler.EditConfigOrCloneConfigRequest + var emptyResponse string + if err := ctx.ShouldBindJSON(&request); err != nil { + ctx.JSON(api.NewBadRequestError(err.Error()).StatusCode, handler.Response{ + Error: err.Error(), + Data: handler.Message{Message: emptyResponse}, + }) + return + } + + token := ctx.GetHeader("Authorization") + token = strings.TrimPrefix(token, "Bearer ") + response, err := c.Config.Clone(request, token) + if err != nil { + ctx.JSON(api.NewBadRequestError(err.Error()).StatusCode, handler.Response{ + Error: err.Error(), + Data: handler.Message{Message: emptyResponse}, + }) + return + } + ctx.JSON(200, response) +} + +func (c *V1) Delete(ctx *gin.Context) { + var request handler.DeleteConfigRequest + var emptyResponse string + id := ctx.Query("id") + request.ConfigID = id + request.CreatedBy = ctx.GetString("email") + if err := ctx.ShouldBindJSON(&request); err != nil { + ctx.JSON(api.NewBadRequestError(err.Error()).StatusCode, handler.Response{ + Error: err.Error(), + Data: handler.Message{Message: emptyResponse}, + }) + return + } + + response, err := c.Config.Delete(request) + if err != nil { + ctx.JSON(api.NewBadRequestError(err.Error()).StatusCode, handler.Response{ + Error: err.Error(), + Data: handler.Message{Message: emptyResponse}, + }) + return + } + ctx.JSON(200, response) +} + +func (c *V1) ScaleUp(ctx *gin.Context) { + var request handler.ScaleUpConfigRequest + var emptyResponse string + if err := ctx.ShouldBindJSON(&request); err != nil { + ctx.JSON(api.NewBadRequestError(err.Error()).StatusCode, handler.Response{ + Error: err.Error(), + Data: handler.Message{Message: emptyResponse}, + }) + return + } + + response, err := c.Config.ScaleUp(request) + if err != nil { + ctx.JSON(api.NewBadRequestError(err.Error()).StatusCode, handler.Response{ + Error: err.Error(), + Data: handler.Message{Message: emptyResponse}, + }) + return + } + ctx.JSON(200, response) +} + +func (c *V1) Cancel(ctx *gin.Context) { + var request handler.CancelConfigRequest + var emptyResponse string + if err := ctx.ShouldBindJSON(&request); err != nil { + ctx.JSON(api.NewBadRequestError(err.Error()).StatusCode, handler.Response{ + Error: err.Error(), + Data: handler.Message{Message: emptyResponse}, + }) + return + } + + response, err := c.Config.Cancel(request) + if err != nil { + ctx.JSON(api.NewInternalServerError(err.Error()).StatusCode, handler.Response{ + Error: err.Error(), + Data: handler.Message{Message: emptyResponse}, + }) + return + } + + ctx.JSON(200, response) +} + +func (c *V1) GetAll(ctx *gin.Context) { + var emptyResponse []handler.ConfigTable + response, err := c.Config.GetAll() + if err != nil { + ctx.JSON(api.NewInternalServerError(err.Error()).StatusCode, handler.GetAllResponse{ + Error: err.Error(), + Data: emptyResponse, + }) + return + } + + ctx.JSON(200, response) +} + +func (c *V1) Review(ctx *gin.Context) { + var request handler.ReviewRequest + var emptyResponse string + if err := ctx.ShouldBindJSON(&request); err != nil { + ctx.JSON(api.NewBadRequestError(err.Error()).StatusCode, handler.Response{ + Error: err.Error(), + Data: handler.Message{Message: emptyResponse}, + }) + return + } + + response, err := c.Config.Review(request) + if err != nil { + ctx.JSON(api.NewBadRequestError(err.Error()).StatusCode, handler.Response{ + Error: err.Error(), + Data: handler.Message{Message: emptyResponse}, + }) + return + } + ctx.JSON(200, response) +} + +func (c *V1) GetAllRequests(ctx *gin.Context) { + role := ctx.GetString("role") + email := ctx.GetString("email") + + var emptyResponse []handler.RequestConfig + + request := handler.GetAllRequestConfigsRequest{ + Email: email, + Role: role, + } + + response, err := c.Config.GetAllRequests(request) + if err != nil { + ctx.JSON(api.NewBadRequestError(err.Error()).StatusCode, handler.GetAllRequestConfigsResponse{ + Error: err.Error(), + Data: emptyResponse, + }) + return + } + ctx.JSON(200, response) +} + +func (c *V1) ValidateRequest(ctx *gin.Context) { + + var emptyResponse string + requestID := ctx.Param("request_id") + if requestID == "" { + ctx.JSON(api.NewBadRequestError("request_id is required").StatusCode, handler.Response{ + Error: "request_id is required", + Data: handler.Message{Message: emptyResponse}, + }) + return + } + + var request handler.ValidateRequest = handler.ValidateRequest{ + ConfigID: requestID, + } + + token := ctx.GetHeader("Authorization") + token = strings.TrimPrefix(token, "Bearer ") + response, err := c.Config.ValidateRequest(request, token) + if err != nil { + ctx.JSON(api.NewBadRequestError(err.Error()).StatusCode, handler.Response{ + Error: err.Error(), + Data: handler.Message{Message: emptyResponse}, + }) + return + } + ctx.JSON(200, response) +} + +func (c *V1) GenerateFunctionalTestRequest(ctx *gin.Context) { + var request handler.GenerateRequestFunctionalTestingRequest + + if err := ctx.ShouldBindJSON(&request); err != nil { + ctx.JSON(api.NewBadRequestError(err.Error()).StatusCode, handler.GenerateRequestFunctionalTestingResponse{ + Error: err.Error(), + }) + return + } + + response, err := c.Config.GenerateFunctionalTestRequest(request) + if err != nil { + ctx.JSON(api.NewBadRequestError(err.Error()).StatusCode, response) + } + ctx.JSON(200, response) +} + +func (c *V1) ExecuteFuncitonalTestRequest(ctx *gin.Context) { + var request handler.ExecuteRequestFunctionalTestingRequest + + if err := ctx.ShouldBindJSON(&request); err != nil { + ctx.JSON(api.NewBadRequestError(err.Error()).StatusCode, handler.ExecuteRequestFunctionalTestingResponse{ + Error: err.Error(), + }) + return + } + + response, err := c.Config.ExecuteFuncitonalTestRequest(request) + if err != nil { + ctx.JSON(api.NewBadRequestError(err.Error()).StatusCode, response) + return + } + ctx.JSON(200, response) +} + +func (c *V1) GetLatestRequest(ctx *gin.Context) { + var emptyResponse string + requestID := ctx.Param("config_id") + if requestID == "" { + ctx.JSON(api.NewBadRequestError("config_id is required").StatusCode, handler.Response{ + Error: "config_id is required", + Data: handler.Message{Message: emptyResponse}, + }) + return + } + response, err := c.Config.GetLatestRequest(requestID) + if err != nil { + ctx.JSON(api.NewBadRequestError(err.Error()).StatusCode, handler.Response{ + Error: err.Error(), + Data: handler.Message{Message: emptyResponse}, + }) + return + } + ctx.JSON(200, response) +} + +func (c *V1) GetLoggingTTL(ctx *gin.Context) { + var emptyResponse []int + response, err := c.Config.GetLoggingTTL() + if err != nil { + ctx.JSON(api.NewBadRequestError(err.Error()).StatusCode, handler.GetLoggingTTLResponse{ + Data: emptyResponse, + }) + } + ctx.JSON(200, response) +} diff --git a/horizon/internal/inferflow/etcd/config.go b/horizon/internal/inferflow/etcd/config.go new file mode 100644 index 00000000..40807c87 --- /dev/null +++ b/horizon/internal/inferflow/etcd/config.go @@ -0,0 +1,8 @@ +package etcd + +type Manager interface { + GetComponentData(componentName string) *ComponentData + CreateConfig(serviceName string, ConfigId string, InferflowConfig InferflowConfig) error + UpdateConfig(serviceName string, ConfigId string, InferflowConfig InferflowConfig) error + DeleteConfig(serviceName string, ConfigId string) error +} diff --git a/horizon/internal/inferflow/etcd/etcd.go b/horizon/internal/inferflow/etcd/etcd.go new file mode 100644 index 00000000..30eb7b73 --- /dev/null +++ b/horizon/internal/inferflow/etcd/etcd.go @@ -0,0 +1,82 @@ +package etcd + +import ( + "encoding/json" + "fmt" + + "github.com/Meesho/BharatMLStack/horizon/internal/inferflow" + + "github.com/Meesho/BharatMLStack/horizon/pkg/etcd" + "github.com/rs/zerolog/log" +) + +type Etcd struct { + inferflowInstance etcd.Etcd + horizonInstance etcd.Etcd + appName string + env string +} + +func NewEtcdInstance() *Etcd { + return &Etcd{ + inferflowInstance: etcd.Instance()[inferflow.InferflowAppName], + horizonInstance: etcd.Instance()[inferflow.HorizonAppName], + appName: inferflow.InferflowAppName, + env: inferflow.AppEnv, + } +} + +func (e *Etcd) GetInferflowEtcdInstance() *ModelConfigRegistery { + instance, ok := e.inferflowInstance.GetConfigInstance().(*ModelConfigRegistery) + if !ok { + log.Panic().Msg("invalid etcd instance") + } + return instance +} + +func (e *Etcd) GetHorizonEtcdInstance() *HorizonRegistry { + instance, ok := e.horizonInstance.GetConfigInstance().(*HorizonRegistry) + if !ok { + log.Panic().Msg("invalid etcd instanc e") + } + return instance +} + +func (e *Etcd) GetComponentData(componentName string) *ComponentData { + registry := e.GetHorizonEtcdInstance() + if registry == nil { + log.Error().Msg("GetComponentData called on nil registry") + return nil + } + + component, ok := registry.Inferflow.InferflowComponents[componentName] + if !ok { + log.Error().Msgf("component data for '%s' not found in registry", componentName) + return nil + } + + return &component +} + +func (e *Etcd) CreateConfig(serviceName string, ConfigId string, InferflowConfig InferflowConfig) error { + // Marshal the struct directly to preserve field order as defined in the struct + configJson, err := json.Marshal(InferflowConfig) + if err != nil { + return err + } + + return e.inferflowInstance.CreateNode(fmt.Sprintf("/config/%s/%s/model-config/config-map/%s", e.appName, serviceName, ConfigId), string(configJson)) +} + +func (e *Etcd) UpdateConfig(serviceName string, ConfigId string, InferflowConfig InferflowConfig) error { + // Marshal the struct directly to preserve field order as defined in the struct + configJson, err := json.Marshal(InferflowConfig) + if err != nil { + return err + } + return e.inferflowInstance.SetValue(fmt.Sprintf("/config/%s/%s/model-config/config-map/%s", e.appName, serviceName, ConfigId), string(configJson)) +} + +func (e *Etcd) DeleteConfig(serviceName string, ConfigId string) error { + return e.inferflowInstance.DeleteNode(fmt.Sprintf("/config/%s/%s/model-config/config-map/%s", e.appName, serviceName, ConfigId)) +} diff --git a/horizon/internal/inferflow/etcd/models.go b/horizon/internal/inferflow/etcd/models.go new file mode 100644 index 00000000..70d03935 --- /dev/null +++ b/horizon/internal/inferflow/etcd/models.go @@ -0,0 +1,142 @@ +package etcd + +type ModelConfigRegistery struct { + InferflowConfig map[string]InferflowConfigs `json:"inferflow-config"` +} + +type HorizonRegistry struct { + Inferflow InferflowComponents `json:"inferflow"` +} + +type InferflowComponents struct { + InferflowComponents map[string]ComponentData `json:"inferflow-components"` +} + +type InferflowConfigs struct { + ModelConfig ModelConfigData `json:"model-config"` +} + +type ModelConfigData struct { + ConfigMap map[string]InferflowConfig `json:"config-map"` +} + +type NumerixComponent struct { + Component string `json:"component"` + ComponentID string `json:"component_id"` + ScoreCol string `json:"score_col"` + ComputeID string `json:"compute_id"` + ScoreMapping map[string]string `json:"score_mapping"` + DataType string `json:"data_type"` +} + +type PredatorInput struct { + Name string `json:"name"` + Features []string `json:"features"` + Dims []int `json:"shape"` + DataType string `json:"data_type"` +} + +type PredatorOutput struct { + Name string `json:"name"` + ModelScores []string `json:"model_scores"` + ModelScoresDims [][]int `json:"model_scores_dims"` + DataType string `json:"data_type"` +} + +type PredatorComponent struct { + Component string `json:"component"` + ComponentID string `json:"component_id"` + ModelName string `json:"model_name"` + Calibration string `json:"calibration,omitempty"` + ModelEndPoint string `json:"model_end_point"` + Deadline int `json:"deadline"` + BatchSize int `json:"batch_size"` + Inputs []PredatorInput `json:"inputs"` + Outputs []PredatorOutput `json:"outputs"` +} + +type RTPComponent struct { + Component string `json:"component"` + ComponentID string `json:"component_id"` + CompositeID bool `json:"composite_id"` + FSKeys []FSKey `json:"fs_keys"` + FSRequest *FSRequest `json:"fs_request"` + FSFlattenRespKeys []string `json:"fs_flatten_resp_keys"` + ColNamePrefix string `json:"col_name_prefix"` + CompCacheEnabled bool `json:"comp_cache_enabled"` +} + +type FinalResponseConfig struct { + LoggingPerc int `json:"logging_perc"` + ModelSchemaPerc int `json:"model_schema_features_perc"` + Features []string `json:"features"` + LogSelectiveFeatures bool `json:"log_features"` + LogBatchSize int `json:"log_batch_size"` +} + +type FSKey struct { + Schema string `json:"schema"` + Col string `json:"col"` +} + +type FSFeatureGroup struct { + Label string `json:"label"` + Features []string `json:"features"` + DataType string `json:"data_type"` +} + +type FSRequest struct { + Label string `json:"label"` + FeatureGroups []FSFeatureGroup `json:"featureGroups"` +} + +type FeatureComponent struct { + Component string `json:"component"` + ComponentID string `json:"component_id"` + ColNamePrefix string `json:"col_name_prefix,omitempty"` + CompCacheEnabled bool `json:"comp_cache_enabled"` + CompCacheTTL int `json:"comp_cache_ttl,omitempty"` + CompositeID bool `json:"composite_id,omitempty"` + FSKeys []FSKey `json:"fs_keys"` + FSRequest *FSRequest `json:"fs_request"` + FSFlattenRespKeys []string `json:"fs_flatten_resp_keys"` +} + +type ComponentConfig struct { + CacheEnabled bool `json:"cache_enabled"` + CacheTTL int `json:"cache_ttl"` + CacheVersion int `json:"cache_version"` + FeatureComponents []FeatureComponent `json:"feature_components"` + RTPComponents []RTPComponent `json:"real_time_pricing_feature_components"` + PredatorComponents []PredatorComponent `json:"predator_components"` + NumerixComponents []NumerixComponent `json:"numerix_components"` +} + +type DagExecutionConfig struct { + ComponentDependency map[string][]string `json:"component_dependency"` +} + +type InferflowConfig struct { + DagExecutionConfig DagExecutionConfig `json:"dag_execution_config"` + ComponentConfig ComponentConfig `json:"component_config"` + ResponseConfig FinalResponseConfig `json:"response_config"` +} + +type ComponentData struct { + ComponentID string `json:"component-id"` + CompositeID bool `json:"composite-id"` + ExecutionDependency string `json:"execution-dependency"` + FSFlattenResKeys map[string]string `json:"fs-flatten-res-keys"` + FSIdSchemaToValueColumns map[string]FSIdSchemaToValueColumnPair `json:"fs-id-schema-to-value-columns"` + Overridecomponent map[string]OverrideComponent `json:"override-component"` +} + +type OverrideComponent struct { + ComponentId string `json:"component-id"` +} + +type FSIdSchemaToValueColumnPair struct { + Schema string `json:"schema"` + ValueCol string `json:"value-col"` + DataType string `json:"data-type"` +} diff --git a/horizon/internal/inferflow/handler/adaptor.go b/horizon/internal/inferflow/handler/adaptor.go new file mode 100644 index 00000000..6f264ee5 --- /dev/null +++ b/horizon/internal/inferflow/handler/adaptor.go @@ -0,0 +1,797 @@ +package handler + +import ( + etcdModel "github.com/Meesho/BharatMLStack/horizon/internal/inferflow/etcd" + dbModel "github.com/Meesho/BharatMLStack/horizon/internal/repositories/sql/inferflow" + "github.com/rs/zerolog/log" +) + +type ConfigMapProvider interface { + GetConfigMapping() dbModel.ConfigMapping +} + +func AdaptOnboardRequestToDBPayload(req interface{}, inferflowConfig InferflowConfig, onboardPayload OnboardPayload) (dbModel.Payload, error) { + var payload dbModel.Payload + + payload.ConfigMapping = AdaptToDBConfig(req) + + dbResponseConfig := AdaptToDBResponseConfig(inferflowConfig) + + dbPredatorComponents := AdaptToDBPredatorComponent(inferflowConfig) + + dbNumerixComponents := AdaptToDBNumerixComponent(inferflowConfig) + + dbRTPComponents := AdaptToDBRTPComponent(inferflowConfig) + + featureComponents := AdaptToDBFeatureComponent(inferflowConfig) + + dbComponentConfig := AdaptToDBComponentConfig(inferflowConfig, featureComponents, dbNumerixComponents, dbPredatorComponents, dbRTPComponents) + + dbDagExecutionConfig := AdaptToDBDagExecutionConfig(inferflowConfig) + + payload.ConfigValue = AdaptToDBConfigValue(dbDagExecutionConfig, dbComponentConfig, dbResponseConfig) + payload.RequestPayload = AdaptToDBOnboardPayload(onboardPayload) + + return payload, nil +} + +func AdaptEditRequestToDBPayload(req interface{}, inferflowConfig InferflowConfig, onboardPayload OnboardPayload) (dbModel.Payload, error) { + var payload dbModel.Payload + + payload.ConfigMapping = AdaptToDBConfig(req) + + dbResponseConfig := AdaptToDBResponseConfig(inferflowConfig) + + dbPredatorComponents := AdaptToDBPredatorComponent(inferflowConfig) + + dbNumerixComponents := AdaptToDBNumerixComponent(inferflowConfig) + + dbRTPComponents := AdaptToDBRTPComponent(inferflowConfig) + + featureComponents := AdaptToDBFeatureComponent(inferflowConfig) + + dbComponentConfig := AdaptToDBComponentConfig(inferflowConfig, featureComponents, dbNumerixComponents, dbPredatorComponents, dbRTPComponents) + + dbDagExecutionConfig := AdaptToDBDagExecutionConfig(inferflowConfig) + + payload.ConfigValue = AdaptToDBConfigValue(dbDagExecutionConfig, dbComponentConfig, dbResponseConfig) + + payload.ConfigValue.ComponentConfig.CacheVersion = payload.ConfigValue.ComponentConfig.CacheVersion + 1 + payload.RequestPayload = AdaptToDBOnboardPayload(onboardPayload) + + return payload, nil +} + +func AdaptCloneConfigRequestToDBPayload(req interface{}, inferflowConfig InferflowConfig, onboardPayload OnboardPayload) (dbModel.Payload, error) { + var payload dbModel.Payload + + payload.ConfigMapping = AdaptToDBConfig(req) + + dbResponseConfig := AdaptToDBResponseConfig(inferflowConfig) + + dbPredatorComponents := AdaptToDBPredatorComponent(inferflowConfig) + + dbNumerixComponents := AdaptToDBNumerixComponent(inferflowConfig) + + dbRTPComponents := AdaptToDBRTPComponent(inferflowConfig) + + featureComponents := AdaptToDBFeatureComponent(inferflowConfig) + + dbComponentConfig := AdaptToDBComponentConfig(inferflowConfig, featureComponents, dbNumerixComponents, dbPredatorComponents, dbRTPComponents) + + dbDagExecutionConfig := AdaptToDBDagExecutionConfig(inferflowConfig) + + payload.ConfigValue = AdaptToDBConfigValue(dbDagExecutionConfig, dbComponentConfig, dbResponseConfig) + payload.RequestPayload = AdaptToDBOnboardPayload(onboardPayload) + + return payload, nil +} + +func AdaptPromoteRequestToDBPayload(req interface{}, requestPayload RequestConfig) (dbModel.Payload, error) { + var payload dbModel.Payload + + payload.ConfigMapping = AdaptToDBConfig(req) + + inferflowConfig := req.(PromoteConfigRequest).Payload.ConfigValue + + dbResponseConfig := AdaptToDBResponseConfig(inferflowConfig) + + dbPredatorComponents := AdaptToDBPredatorComponent(inferflowConfig) + + dbNumerixComponents := AdaptToDBNumerixComponent(inferflowConfig) + + dbRTPComponents := AdaptToDBRTPComponent(inferflowConfig) + + featureComponents := AdaptToDBFeatureComponent(inferflowConfig) + + dbComponentConfig := AdaptToDBComponentConfig(inferflowConfig, featureComponents, dbNumerixComponents, dbPredatorComponents, dbRTPComponents) + + dbDagExecutionConfig := AdaptToDBDagExecutionConfig(inferflowConfig) + + payload.ConfigValue = AdaptToDBConfigValue(dbDagExecutionConfig, dbComponentConfig, dbResponseConfig) + + payload.RequestPayload = AdaptToDBOnboardPayload(requestPayload.Payload.RequestPayload) + + return payload, nil +} + +func AdaptScaleUpRequestToDBPayload(req interface{}) (dbModel.Payload, error) { + var payload dbModel.Payload + + payload.ConfigMapping = AdaptToDBConfig(req) + + inferflowConfig := req.(ScaleUpConfigRequest).Payload.ConfigValue + + dbResponseConfig := AdaptToDBResponseConfig(inferflowConfig) + + dbPredatorComponents := AdaptToDBPredatorComponent(inferflowConfig) + + dbNumerixComponents := AdaptToDBNumerixComponent(inferflowConfig) + + dbRTPComponents := AdaptToDBRTPComponent(inferflowConfig) + + featureComponents := AdaptToDBFeatureComponent(inferflowConfig) + + dbComponentConfig := AdaptToDBComponentConfig(inferflowConfig, featureComponents, dbNumerixComponents, dbPredatorComponents, dbRTPComponents) + + dbDagExecutionConfig := AdaptToDBDagExecutionConfig(inferflowConfig) + + payload.ConfigValue = AdaptToDBConfigValue(dbDagExecutionConfig, dbComponentConfig, dbResponseConfig) + + return payload, nil +} + +func AdaptToDBConfig(req interface{}) dbModel.ConfigMapping { + + if provider, ok := req.(ConfigMapProvider); ok { + return provider.GetConfigMapping() + } else { + log.Warn().Msgf("AdaptToDBConfig received a type that does not implement ConfigMapProvider: %T", req) + return dbModel.ConfigMapping{} + } +} + +func AdaptToDBResponseConfig(inferflowConfig InferflowConfig) dbModel.ResponseConfig { + return dbModel.ResponseConfig{ + LoggingPerc: inferflowConfig.ResponseConfig.LoggingPerc, + ModelSchemaPerc: inferflowConfig.ResponseConfig.ModelSchemaPerc, + Features: inferflowConfig.ResponseConfig.Features, + LogSelectiveFeatures: inferflowConfig.ResponseConfig.LogSelectiveFeatures, + LogBatchSize: inferflowConfig.ResponseConfig.LogBatchSize, + } +} + +func AdaptToDBPredatorComponent(inferflowConfig InferflowConfig) []dbModel.PredatorComponent { + var predatorComponents []dbModel.PredatorComponent + + for _, ranker := range inferflowConfig.ComponentConfig.PredatorComponents { + dbInputs := make([]dbModel.PredatorInput, len(ranker.Inputs)) + for i, input := range ranker.Inputs { + dbInputs[i] = dbModel.PredatorInput{ + Name: input.Name, + Features: input.Features, + Dims: input.Dims, + DataType: input.DataType, + } + } + + dbOutputs := make([]dbModel.PredatorOutput, len(ranker.Outputs)) + for i, output := range ranker.Outputs { + dbOutputs[i] = dbModel.PredatorOutput{ + Name: output.Name, + ModelScores: output.ModelScores, + ModelScoresDims: output.ModelScoresDims, + DataType: output.DataType, + } + } + + predatorComp := dbModel.PredatorComponent{ + Component: ranker.Component, + ComponentID: ranker.ComponentID, + Calibration: ranker.Calibration, + ModelName: ranker.ModelName, + ModelEndPoint: ranker.ModelEndPoint, + Deadline: ranker.Deadline, + BatchSize: ranker.BatchSize, + Inputs: dbInputs, + Outputs: dbOutputs, + } + predatorComponents = append(predatorComponents, predatorComp) + } + + return predatorComponents +} + +func AdaptToDBNumerixComponent(inferflowConfig InferflowConfig) []dbModel.NumerixComponent { + var NumerixComponents []dbModel.NumerixComponent + + for _, reRanker := range inferflowConfig.ComponentConfig.NumerixComponents { + NumerixComp := dbModel.NumerixComponent{ + Component: reRanker.Component, + ComponentID: reRanker.ComponentID, + ScoreCol: reRanker.ScoreCol, + ComputeID: reRanker.ComputeID, + ScoreMapping: reRanker.ScoreMapping, + DataType: reRanker.DataType, + } + NumerixComponents = append(NumerixComponents, NumerixComp) + } + + return NumerixComponents +} + +func AdaptToDBRTPComponent(inferflowConfig InferflowConfig) []dbModel.RTPComponent { + var rtpComponents []dbModel.RTPComponent + for _, rtpComponent := range inferflowConfig.ComponentConfig.RTPComponents { + fsKeys := make([]dbModel.FSKey, len(rtpComponent.FSKeys)) + for i, key := range rtpComponent.FSKeys { + fsKeys[i] = dbModel.FSKey{ + Schema: key.Schema, + Col: key.Col, + } + } + fsFeatureGroups := make([]dbModel.FSFeatureGroup, len(rtpComponent.FeatureRequest.FeatureGroups)) + for i, grp := range rtpComponent.FeatureRequest.FeatureGroups { + fsFeatureGroups[i] = dbModel.FSFeatureGroup{ + Label: grp.Label, + Features: grp.Features, + DataType: grp.DataType, + } + } + fsRequest := dbModel.FSRequest{ + Label: rtpComponent.FeatureRequest.Label, + FeatureGroups: fsFeatureGroups, + } + dbRTPComponent := dbModel.RTPComponent{ + Component: rtpComponent.Component, + ComponentID: rtpComponent.ComponentID, + CompositeID: rtpComponent.CompositeID, + FSKeys: fsKeys, + FSRequest: &fsRequest, + FSFlattenRespKeys: rtpComponent.FSFlattenRespKeys, + ColNamePrefix: rtpComponent.ColNamePrefix, + CompCacheEnabled: rtpComponent.CompCacheEnabled, + } + rtpComponents = append(rtpComponents, dbRTPComponent) + } + return rtpComponents +} + +func AdaptToDBFeatureComponent(inferflowConfig InferflowConfig) []dbModel.FeatureComponent { + var featureComponents []dbModel.FeatureComponent + + for _, fc := range inferflowConfig.ComponentConfig.FeatureComponents { + fsKeys := make([]dbModel.FSKey, len(fc.FSKeys)) + for i, key := range fc.FSKeys { + fsKeys[i] = dbModel.FSKey{ + Schema: key.Schema, + Col: key.Col, + } + } + + fsFeatureGroups := make([]dbModel.FSFeatureGroup, len(fc.FSRequest.FeatureGroups)) + for i, grp := range fc.FSRequest.FeatureGroups { + fsFeatureGroups[i] = dbModel.FSFeatureGroup{ + Label: grp.Label, + Features: grp.Features, + DataType: grp.DataType, + } + } + + fsRequest := dbModel.FSRequest{ + Label: fc.FSRequest.Label, + FeatureGroups: fsFeatureGroups, + } + + comp := dbModel.FeatureComponent{ + Component: fc.Component, + ComponentID: fc.ComponentID, + ColNamePrefix: fc.ColNamePrefix, + CompCacheEnabled: fc.CompCacheEnabled, + CompositeID: fc.CompositeID, + FSKeys: fsKeys, + FSRequest: &fsRequest, + FSFlattenRespKeys: fc.FSFlattenRespKeys, + } + featureComponents = append(featureComponents, comp) + } + + return featureComponents +} + +func AdaptToDBComponentConfig(inferflowConfig InferflowConfig, featureComponents []dbModel.FeatureComponent, NumerixComponents []dbModel.NumerixComponent, predatorComponents []dbModel.PredatorComponent, rtpComponents []dbModel.RTPComponent) dbModel.ComponentConfig { + return dbModel.ComponentConfig{ + CacheEnabled: inferflowConfig.ComponentConfig.CacheEnabled, + CacheTTL: inferflowConfig.ComponentConfig.CacheTTL, + CacheVersion: inferflowConfig.ComponentConfig.CacheVersion, + FeatureComponents: featureComponents, + PredatorComponents: predatorComponents, + NumerixComponents: NumerixComponents, + RTPComponents: rtpComponents, + } +} + +func AdaptToDBDagExecutionConfig(inferflowConfig InferflowConfig) dbModel.DagExecutionConfig { + return dbModel.DagExecutionConfig{ + ComponentDependency: inferflowConfig.DagExecutionConfig.ComponentDependency, + } +} + +func AdaptToDBConfigValue(dagExecutionConfig dbModel.DagExecutionConfig, componentConfig dbModel.ComponentConfig, responseConfig dbModel.ResponseConfig) dbModel.InferflowConfig { + return dbModel.InferflowConfig{ + DagExecutionConfig: dagExecutionConfig, + ComponentConfig: componentConfig, + ResponseConfig: responseConfig, + } +} + +func AdaptToDBOnboardPayload(onboardPayload OnboardPayload) dbModel.OnboardPayload { + dbOnboardPayload := dbModel.OnboardPayload{ + RealEstate: onboardPayload.RealEstate, + Tenant: onboardPayload.Tenant, + ConfigIdentifier: onboardPayload.ConfigIdentifier, + Rankers: make([]dbModel.OnboardRanker, len(onboardPayload.Rankers)), + ReRankers: make([]dbModel.OnboardReRanker, len(onboardPayload.ReRankers)), + Response: dbModel.ResponseConfig{ + LoggingPerc: onboardPayload.Response.PrismLoggingPerc, + ModelSchemaPerc: onboardPayload.Response.RankerSchemaFeaturesInResponsePerc, + Features: onboardPayload.Response.ResponseFeatures, + LogSelectiveFeatures: onboardPayload.Response.LogSelectiveFeatures, + LogBatchSize: onboardPayload.Response.LogBatchSize, + }, + ConfigMapping: dbModel.ConfigMapping{ + AppToken: onboardPayload.ConfigMapping.AppToken, + ConnectionConfigID: onboardPayload.ConfigMapping.ConnectionConfigID, + DeployableID: onboardPayload.ConfigMapping.DeployableID, + ResponseDefaultValues: onboardPayload.ConfigMapping.ResponseDefaultValues, + }, + } + + for i, ranker := range onboardPayload.Rankers { + dbOnboardPayload.Rankers[i] = dbModel.OnboardRanker{ + ModelName: ranker.ModelName, + Calibration: ranker.Calibration, + EndPoint: ranker.EndPoint, + EntityID: ranker.EntityID, + Inputs: make([]dbModel.PredatorInput, len(ranker.Inputs)), + Outputs: make([]dbModel.PredatorOutput, len(ranker.Outputs)), + } + for j, input := range ranker.Inputs { + dbOnboardPayload.Rankers[i].Inputs[j] = dbModel.PredatorInput{ + Name: input.Name, + Features: input.Features, + Dims: input.Dims, + DataType: input.DataType, + } + } + for j, output := range ranker.Outputs { + dbOnboardPayload.Rankers[i].Outputs[j] = dbModel.PredatorOutput{ + Name: output.Name, + ModelScores: output.ModelScores, + ModelScoresDims: output.ModelScoresDims, + DataType: output.DataType, + } + } + } + for i, reRanker := range onboardPayload.ReRankers { + dbOnboardPayload.ReRankers[i] = dbModel.OnboardReRanker{ + Score: reRanker.Score, + EqID: reRanker.EqID, + EqVariables: reRanker.EqVariables, + DataType: reRanker.DataType, + EntityID: reRanker.EntityID, + } + } + return dbOnboardPayload +} + +func AdaptFromDbToInferFlowConfig(dbConfig dbModel.InferflowConfig) InferflowConfig { + return InferflowConfig{ + DagExecutionConfig: AdaptFromDbToDagExecutionConfig(dbConfig.DagExecutionConfig), + ComponentConfig: AdaptFromDbToComponentConfig(dbConfig.ComponentConfig), + ResponseConfig: AdaptFromDbToResponseConfig(dbConfig.ResponseConfig), + } +} + +func AdaptFromDbToOnboardPayload(dbOnboardPayload dbModel.OnboardPayload) OnboardPayload { + onboardPayload := OnboardPayload{ + RealEstate: dbOnboardPayload.RealEstate, + Tenant: dbOnboardPayload.Tenant, + ConfigIdentifier: dbOnboardPayload.ConfigIdentifier, + Rankers: []Ranker{}, + ReRankers: []ReRanker{}, + Response: ResponseConfig{ + PrismLoggingPerc: dbOnboardPayload.Response.LoggingPerc, + RankerSchemaFeaturesInResponsePerc: dbOnboardPayload.Response.ModelSchemaPerc, + ResponseFeatures: dbOnboardPayload.Response.Features, + LogSelectiveFeatures: dbOnboardPayload.Response.LogSelectiveFeatures, + LogBatchSize: dbOnboardPayload.Response.LogBatchSize, + }, + ConfigMapping: ConfigMapping{ + AppToken: dbOnboardPayload.ConfigMapping.AppToken, + ConnectionConfigID: dbOnboardPayload.ConfigMapping.ConnectionConfigID, + DeployableID: dbOnboardPayload.ConfigMapping.DeployableID, + ResponseDefaultValues: dbOnboardPayload.ConfigMapping.ResponseDefaultValues, + }, + } + + for i, predatorComponent := range dbOnboardPayload.Rankers { + onboardPayload.Rankers = append(onboardPayload.Rankers, Ranker{ + ModelName: predatorComponent.ModelName, + Calibration: predatorComponent.Calibration, + EndPoint: predatorComponent.EndPoint, + Inputs: make([]Input, len(predatorComponent.Inputs)), + Outputs: make([]Output, len(predatorComponent.Outputs)), + EntityID: predatorComponent.EntityID, + }) + for j, input := range predatorComponent.Inputs { + onboardPayload.Rankers[i].Inputs[j] = Input{ + Name: input.Name, + Features: input.Features, + Dims: input.Dims, + DataType: input.DataType, + } + } + for j, output := range predatorComponent.Outputs { + onboardPayload.Rankers[i].Outputs[j] = Output{ + Name: output.Name, + ModelScores: output.ModelScores, + ModelScoresDims: output.ModelScoresDims, + DataType: output.DataType, + } + } + } + + for _, reRanker := range dbOnboardPayload.ReRankers { + onboardPayload.ReRankers = append(onboardPayload.ReRankers, ReRanker{ + Score: reRanker.Score, + EqID: reRanker.EqID, + EqVariables: reRanker.EqVariables, + DataType: reRanker.DataType, + EntityID: reRanker.EntityID, + }) + } + + return onboardPayload +} + +func AdaptFromDbToComponentConfig(dbComponentConfig dbModel.ComponentConfig) *ComponentConfig { + return &ComponentConfig{ + CacheEnabled: dbComponentConfig.CacheEnabled, + CacheTTL: dbComponentConfig.CacheTTL, + CacheVersion: dbComponentConfig.CacheVersion, + FeatureComponents: AdaptFromDbToFeatureComponent(dbComponentConfig.FeatureComponents), + PredatorComponents: AdaptFromDbToPredatorComponent(dbComponentConfig.PredatorComponents), + NumerixComponents: AdaptFromDbToNumerixComponent(dbComponentConfig.NumerixComponents), + RTPComponents: AdaptFromDbToRTPComponent(dbComponentConfig.RTPComponents), + } +} + +func AdaptFromDbToResponseConfig(dbResponseConfig dbModel.ResponseConfig) *FinalResponseConfig { + return &FinalResponseConfig{ + LoggingPerc: dbResponseConfig.LoggingPerc, + ModelSchemaPerc: dbResponseConfig.ModelSchemaPerc, + Features: dbResponseConfig.Features, + LogSelectiveFeatures: dbResponseConfig.LogSelectiveFeatures, + LogBatchSize: dbResponseConfig.LogBatchSize, + } +} + +func AdaptFromDbToDagExecutionConfig(dbDagExecutionConfig dbModel.DagExecutionConfig) *DagExecutionConfig { + return &DagExecutionConfig{ + ComponentDependency: dbDagExecutionConfig.ComponentDependency, + } +} + +func AdaptFromDbToPredatorComponent(dbPredatorComponents []dbModel.PredatorComponent) []PredatorComponent { + + var predatorComponents []PredatorComponent + for _, predatorComponent := range dbPredatorComponents { + dbInputs := make([]PredatorInput, len(predatorComponent.Inputs)) + for i, input := range predatorComponent.Inputs { + dbInputs[i] = PredatorInput{ + Name: input.Name, + Features: input.Features, + Dims: input.Dims, + DataType: input.DataType, + } + } + + dbOutputs := make([]PredatorOutput, len(predatorComponent.Outputs)) + for i, output := range predatorComponent.Outputs { + dbOutputs[i] = PredatorOutput{ + Name: output.Name, + ModelScores: output.ModelScores, + ModelScoresDims: output.ModelScoresDims, + DataType: output.DataType, + } + } + + predatorComponent := PredatorComponent{ + Component: predatorComponent.Component, + ComponentID: predatorComponent.ComponentID, + ModelName: predatorComponent.ModelName, + ModelEndPoint: predatorComponent.ModelEndPoint, + Calibration: predatorComponent.Calibration, + Deadline: predatorComponent.Deadline, + BatchSize: predatorComponent.BatchSize, + Inputs: dbInputs, + Outputs: dbOutputs, + } + predatorComponents = append(predatorComponents, predatorComponent) + } + return predatorComponents +} + +func AdaptFromDbToNumerixComponent(dbNumerixComponents []dbModel.NumerixComponent) []NumerixComponent { + + var NumerixComponents []NumerixComponent + for _, numerixComponent := range dbNumerixComponents { + numerixComponent := NumerixComponent{ + Component: numerixComponent.Component, + ComponentID: numerixComponent.ComponentID, + ScoreCol: numerixComponent.ScoreCol, + ComputeID: numerixComponent.ComputeID, + ScoreMapping: numerixComponent.ScoreMapping, + DataType: numerixComponent.DataType, + } + NumerixComponents = append(NumerixComponents, numerixComponent) + } + return NumerixComponents +} + +func AdaptFromDbToRTPComponent(dbRTPComponents []dbModel.RTPComponent) []RTPComponent { + + var rtpComponents []RTPComponent + for _, rtpComponent := range dbRTPComponents { + fsKeys := make([]FSKey, len(rtpComponent.FSKeys)) + for i, key := range rtpComponent.FSKeys { + fsKeys[i] = FSKey{ + Schema: key.Schema, + Col: key.Col, + } + } + fsFeatureGroups := make([]FSFeatureGroup, len(rtpComponent.FSRequest.FeatureGroups)) + for i, grp := range rtpComponent.FSRequest.FeatureGroups { + fsFeatureGroups[i] = FSFeatureGroup{ + Label: grp.Label, + Features: grp.Features, + DataType: grp.DataType, + } + } + fsRequest := FSRequest{ + Label: rtpComponent.FSRequest.Label, + FeatureGroups: fsFeatureGroups, + } + rtpComponents = append(rtpComponents, RTPComponent{ + Component: rtpComponent.Component, + ComponentID: rtpComponent.ComponentID, + CompositeID: rtpComponent.CompositeID, + FSKeys: fsKeys, + FeatureRequest: &fsRequest, + FSFlattenRespKeys: rtpComponent.FSFlattenRespKeys, + ColNamePrefix: rtpComponent.ColNamePrefix, + CompCacheEnabled: rtpComponent.CompCacheEnabled, + }) + } + return rtpComponents +} + +func AdaptFromDbToFeatureComponent(dbFeatureComponents []dbModel.FeatureComponent) []FeatureComponent { + var featureComponents []FeatureComponent + for _, fc := range dbFeatureComponents { + fsKeys := make([]FSKey, len(fc.FSKeys)) + for i, key := range fc.FSKeys { + fsKeys[i] = FSKey{ + Schema: key.Schema, + Col: key.Col, + } + } + + fsFeatureGroups := make([]FSFeatureGroup, len(fc.FSRequest.FeatureGroups)) + for i, grp := range fc.FSRequest.FeatureGroups { + fsFeatureGroups[i] = FSFeatureGroup{ + Label: grp.Label, + Features: grp.Features, + DataType: grp.DataType, + } + } + + fsRequest := FSRequest{ + Label: fc.FSRequest.Label, + FeatureGroups: fsFeatureGroups, + } + + comp := FeatureComponent{ + Component: fc.Component, + ComponentID: fc.ComponentID, + ColNamePrefix: fc.ColNamePrefix, + CompCacheEnabled: fc.CompCacheEnabled, + CompositeID: fc.CompositeID, + FSKeys: fsKeys, + FSRequest: &fsRequest, + FSFlattenRespKeys: fc.FSFlattenRespKeys, + } + featureComponents = append(featureComponents, comp) + } + return featureComponents +} + +func AdaptFromDbToConfigMapping(dbMapping dbModel.ConfigMapping) ConfigMapping { + return ConfigMapping{ + AppToken: dbMapping.AppToken, + ConnectionConfigID: dbMapping.ConnectionConfigID, + DeployableID: dbMapping.DeployableID, + ResponseDefaultValues: dbMapping.ResponseDefaultValues, + } +} + +func AdaptToEtcdInferFlowConfig(dpConfig dbModel.InferflowConfig) etcdModel.InferflowConfig { + return etcdModel.InferflowConfig{ + DagExecutionConfig: AdaptToEtcdDagExecutionConfig(dpConfig.DagExecutionConfig), + ComponentConfig: AdaptToEtcdComponentConfig(dpConfig.ComponentConfig), + ResponseConfig: AdaptToEtcdResponseConfig(dpConfig.ResponseConfig), + } +} + +func AdaptToEtcdComponentConfig(dbComponentConfig dbModel.ComponentConfig) etcdModel.ComponentConfig { + return etcdModel.ComponentConfig{ + CacheEnabled: dbComponentConfig.CacheEnabled, + CacheTTL: dbComponentConfig.CacheTTL, + CacheVersion: dbComponentConfig.CacheVersion, + FeatureComponents: AdaptToEtcdFeatureComponent(dbComponentConfig.FeatureComponents), + PredatorComponents: AdaptToEtcdPredatorComponent(dbComponentConfig.PredatorComponents), + NumerixComponents: AdaptToEtcdNumerixComponent(dbComponentConfig.NumerixComponents), + RTPComponents: AdaptToEtcdRTPComponent(dbComponentConfig.RTPComponents), + } +} + +func AdaptToEtcdResponseConfig(dbResponseConfig dbModel.ResponseConfig) etcdModel.FinalResponseConfig { + return etcdModel.FinalResponseConfig{ + LoggingPerc: dbResponseConfig.LoggingPerc, + ModelSchemaPerc: dbResponseConfig.ModelSchemaPerc, + Features: dbResponseConfig.Features, + LogSelectiveFeatures: dbResponseConfig.LogSelectiveFeatures, + LogBatchSize: dbResponseConfig.LogBatchSize, + } +} + +func AdaptToEtcdDagExecutionConfig(dbDagExecutionConfig dbModel.DagExecutionConfig) etcdModel.DagExecutionConfig { + return etcdModel.DagExecutionConfig{ + ComponentDependency: dbDagExecutionConfig.ComponentDependency, + } +} + +func AdaptToEtcdPredatorComponent(dbPredatorComponents []dbModel.PredatorComponent) []etcdModel.PredatorComponent { + + var predatorComponents []etcdModel.PredatorComponent + for _, predatorComponent := range dbPredatorComponents { + dbInputs := make([]etcdModel.PredatorInput, len(predatorComponent.Inputs)) + for i, input := range predatorComponent.Inputs { + dbInputs[i] = etcdModel.PredatorInput{ + Name: input.Name, + Features: input.Features, + Dims: input.Dims, + DataType: input.DataType, + } + } + + dbOutputs := make([]etcdModel.PredatorOutput, len(predatorComponent.Outputs)) + for i, output := range predatorComponent.Outputs { + dbOutputs[i] = etcdModel.PredatorOutput{ + Name: output.Name, + ModelScores: output.ModelScores, + ModelScoresDims: output.ModelScoresDims, + DataType: output.DataType, + } + } + + predatorComponent := etcdModel.PredatorComponent{ + Component: predatorComponent.Component, + ComponentID: predatorComponent.ComponentID, + Calibration: predatorComponent.Calibration, + ModelName: predatorComponent.ModelName, + ModelEndPoint: predatorComponent.ModelEndPoint, + Deadline: predatorComponent.Deadline, + BatchSize: predatorComponent.BatchSize, + Inputs: dbInputs, + Outputs: dbOutputs, + } + predatorComponents = append(predatorComponents, predatorComponent) + } + return predatorComponents +} + +func AdaptToEtcdNumerixComponent(dbNumerixComponents []dbModel.NumerixComponent) []etcdModel.NumerixComponent { + + var NumerixComponents []etcdModel.NumerixComponent + for _, NumerixComponent := range dbNumerixComponents { + NumerixComponent := etcdModel.NumerixComponent{ + Component: NumerixComponent.Component, + ComponentID: NumerixComponent.ComponentID, + ScoreCol: NumerixComponent.ScoreCol, + ComputeID: NumerixComponent.ComputeID, + ScoreMapping: NumerixComponent.ScoreMapping, + DataType: NumerixComponent.DataType, + } + NumerixComponents = append(NumerixComponents, NumerixComponent) + } + return NumerixComponents +} + +func AdaptToEtcdRTPComponent(dbRTPComponents []dbModel.RTPComponent) []etcdModel.RTPComponent { + + var rtpComponents []etcdModel.RTPComponent + for _, rtpComponent := range dbRTPComponents { + fsKeys := make([]etcdModel.FSKey, len(rtpComponent.FSKeys)) + for i, key := range rtpComponent.FSKeys { + fsKeys[i] = etcdModel.FSKey{ + Schema: key.Schema, + Col: key.Col, + } + } + fsFeatureGroups := make([]etcdModel.FSFeatureGroup, len(rtpComponent.FSRequest.FeatureGroups)) + for i, grp := range rtpComponent.FSRequest.FeatureGroups { + fsFeatureGroups[i] = etcdModel.FSFeatureGroup{ + Label: grp.Label, + Features: grp.Features, + DataType: grp.DataType, + } + } + fsRequest := etcdModel.FSRequest{ + Label: rtpComponent.FSRequest.Label, + FeatureGroups: fsFeatureGroups, + } + rtpComponents = append(rtpComponents, etcdModel.RTPComponent{ + Component: rtpComponent.Component, + ComponentID: rtpComponent.ComponentID, + CompositeID: rtpComponent.CompositeID, + FSKeys: fsKeys, + FSRequest: &fsRequest, + FSFlattenRespKeys: rtpComponent.FSFlattenRespKeys, + ColNamePrefix: rtpComponent.ColNamePrefix, + CompCacheEnabled: rtpComponent.CompCacheEnabled, + }) + } + return rtpComponents +} + +func AdaptToEtcdFeatureComponent(dbFeatureComponents []dbModel.FeatureComponent) []etcdModel.FeatureComponent { + var featureComponents []etcdModel.FeatureComponent + for _, fc := range dbFeatureComponents { + fsKeys := make([]etcdModel.FSKey, len(fc.FSKeys)) + for i, key := range fc.FSKeys { + fsKeys[i] = etcdModel.FSKey{ + Schema: key.Schema, + Col: key.Col, + } + } + + fsFeatureGroups := make([]etcdModel.FSFeatureGroup, len(fc.FSRequest.FeatureGroups)) + for i, grp := range fc.FSRequest.FeatureGroups { + fsFeatureGroups[i] = etcdModel.FSFeatureGroup{ + Label: grp.Label, + Features: grp.Features, + DataType: grp.DataType, + } + } + + fsRequest := etcdModel.FSRequest{ + Label: fc.FSRequest.Label, + FeatureGroups: fsFeatureGroups, + } + + comp := etcdModel.FeatureComponent{ + Component: fc.Component, + ComponentID: fc.ComponentID, + ColNamePrefix: fc.ColNamePrefix, + CompCacheEnabled: fc.CompCacheEnabled, + CompositeID: fc.CompositeID, + FSKeys: fsKeys, + FSRequest: &fsRequest, + FSFlattenRespKeys: fc.FSFlattenRespKeys, + } + featureComponents = append(featureComponents, comp) + } + return featureComponents +} diff --git a/horizon/internal/inferflow/handler/config.go b/horizon/internal/inferflow/handler/config.go new file mode 100644 index 00000000..829ae74e --- /dev/null +++ b/horizon/internal/inferflow/handler/config.go @@ -0,0 +1,19 @@ +package handler + +type Config interface { + Onboard(request InferflowOnboardRequest, token string) (Response, error) + Review(request ReviewRequest) (Response, error) + Promote(request PromoteConfigRequest) (Response, error) + Edit(request EditConfigOrCloneConfigRequest, token string) (Response, error) + Clone(request EditConfigOrCloneConfigRequest, token string) (Response, error) + Delete(request DeleteConfigRequest) (Response, error) + ScaleUp(request ScaleUpConfigRequest) (Response, error) + Cancel(request CancelConfigRequest) (Response, error) + GetAll() (GetAllResponse, error) + GetAllRequests(request GetAllRequestConfigsRequest) (GetAllRequestConfigsResponse, error) + ValidateRequest(request ValidateRequest, token string) (Response, error) + GenerateFunctionalTestRequest(request GenerateRequestFunctionalTestingRequest) (GenerateRequestFunctionalTestingResponse, error) + ExecuteFuncitonalTestRequest(request ExecuteRequestFunctionalTestingRequest) (ExecuteRequestFunctionalTestingResponse, error) + GetLatestRequest(requestID string) (GetLatestRequestResponse, error) + GetLoggingTTL() (GetLoggingTTLResponse, error) +} diff --git a/horizon/internal/inferflow/handler/config_builder.go b/horizon/internal/inferflow/handler/config_builder.go new file mode 100644 index 00000000..b77f4899 --- /dev/null +++ b/horizon/internal/inferflow/handler/config_builder.go @@ -0,0 +1,1481 @@ +package handler + +import ( + "context" + "fmt" + "sort" + "strconv" + "strings" + + ofsHandler "github.com/Meesho/BharatMLStack/horizon/internal/online-feature-store/handler" + + etcd "github.com/Meesho/BharatMLStack/horizon/internal/inferflow/etcd" + "github.com/Meesho/price-aggregator-go/pricingfeatureretrieval/client" + "github.com/Meesho/price-aggregator-go/pricingfeatureretrieval/client/models" + mapset "github.com/deckarep/golang-set/v2" +) + +const ( + PARENT = "PARENT" + DEFAULT_FEATURE = "DEFAULT" + ONLINE_FEATURE = "ONLINE" + MODEL_FEATURE = "MODEL" + OFFLINE_FEATURE = "OFFLINE" + CALIBRATION = "CALIBRATION" + RTP_FEATURE = "RTP" + PCTR_CALIBRATION = "PCTR_CALIBRATION" + PCVR_CALIBRATION = "PCVR_CALIBRATION" + PIPE_DELIMITER = "|" + UNDERSCORE_DELIMITER = "_" + COLON_DELIMITER = ":" + COMMA_DELIMITER = "," + featureClassOffline = "offline" + featureClassOnline = "online" + featureClassDefault = "default" + featureClassRtp = "rtp" + featureClassPCVRCalibration = "pcvr_calibration" + featureClassPCTRCalibration = "pctr_calibration" + featureClassInvalid = "invalid" + COMPONENT_NAME_PREFIX = "composite_key_gen_" + FEATURE_INITIALIZER = "feature_initializer" + rtpClientVersion = 1 +) + +func (m *InferFlow) GetInferflowConfig(request InferflowOnboardRequest, token string) (InferflowConfig, error) { + client.Init() + entityIDs := extractEntityIDs(request) + + featureList, featureToDataType, rtpFeatures, pcvrCalibrationFeatures, pctrCalibrationFeatures, predatorAndNumerixOutputsToDataType, offlineToOnlineMapping, err := GetFeatureList(request, m.EtcdConfig, token, entityIDs) + if err != nil { + return InferflowConfig{}, err + } + + predatorComponents, err := GetPredatorComponents(request, offlineToOnlineMapping) + if err != nil { + return InferflowConfig{}, err + } + + NumerixComponents, err := GetNumerixComponents(request, offlineToOnlineMapping, predatorAndNumerixOutputsToDataType, featureToDataType) + if err != nil { + return InferflowConfig{}, err + } + + responseConfig, err := GetResponseConfigs(&request) + if err != nil { + return InferflowConfig{}, err + } + + rtpComponents, err := GetRTPComponents(request, rtpFeatures, featureToDataType, m.EtcdConfig, token) + if err != nil { + return InferflowConfig{}, err + } + + featureComponents, err := GetFeatureComponents(request, featureList, featureToDataType, pcvrCalibrationFeatures, pctrCalibrationFeatures, m.EtcdConfig, token, entityIDs) + if err != nil { + return InferflowConfig{}, err + } + + componentConfig, err := GetComponentConfig(featureComponents, rtpComponents, NumerixComponents, predatorComponents) + if err != nil { + return InferflowConfig{}, err + } + + dagExecutionConfig, err := GetDagExecutionConfig(request, featureComponents, rtpComponents, NumerixComponents, predatorComponents, m.EtcdConfig) + if err != nil { + return InferflowConfig{}, err + } + + mpConfig := InferflowConfig{ + DagExecutionConfig: dagExecutionConfig, + ComponentConfig: componentConfig, + ResponseConfig: responseConfig, + } + + return mpConfig, nil +} + +// GetFeatureList extracts features from the request and fetches the component features from the etcd config +// and returns a set of features, a map of feature to data type, a map of offline feature to online feature +// and an error if any. +func GetFeatureList(request InferflowOnboardRequest, etcdConfig etcd.Manager, token string, entityIDs map[string]bool) (mapset.Set[string], map[string]string, mapset.Set[string], mapset.Set[string], mapset.Set[string], map[string]string, map[string]string, error) { + initialFeatures, featureToDataType, predatorAndNumerixOutputsToDataType := extractFeatures(request, entityIDs) + + offlineFeatures, onlineFeatures, _, rtpFeatures, pctrCalibrationFeatures, pcvrCalibrationFeatures, newFeatureToDataType, err := classifyFeatures(initialFeatures, featureToDataType) + if err != nil { + return nil, nil, nil, nil, nil, nil, nil, err + } + + offlineToOnlineMapping, err := mapOfflineFeatures(offlineFeatures, token) + if err != nil { + return nil, nil, nil, nil, nil, nil, nil, err + } + for f, dtype := range newFeatureToDataType { + featureToDataType[f] = dtype + } + + features := mapset.NewSet[string]() + for offlineFeature, onlineFeature := range offlineToOnlineMapping { + features.Add(onlineFeature) + featureToDataType[onlineFeature] = featureToDataType[offlineFeature] + delete(featureToDataType, offlineFeature) + } + + for _, f := range onlineFeatures.ToSlice() { + features.Add(f) + } + + rtpComponentFeatures, rtpComponentFeatureToDataType, err := fetchRTPComponentFeatures(rtpFeatures, etcdConfig) + if err != nil { + return nil, nil, nil, nil, nil, nil, nil, err + } + for _, f := range rtpComponentFeatures.ToSlice() { + features.Add(f) + } + + for f, dtype := range rtpComponentFeatureToDataType { + featureToDataType[f] = dtype + } + + componentFeatures, newfeatureToDataType, err := fetchComponentFeatures(features, pctrCalibrationFeatures, pcvrCalibrationFeatures, etcdConfig, request.Payload.RealEstate, token) + if err != nil { + return nil, nil, nil, nil, nil, nil, nil, err + } + for _, f := range componentFeatures.ToSlice() { + features.Add(f) + } + + for f, dtype := range newfeatureToDataType { + featureToDataType[f] = dtype + } + + // for _, f := range defaultFeatures.ToSlice() { + // features.Add(f) + // } + + if err := fetchMissingDatatypes(featureToDataType, rtpFeatures, pctrCalibrationFeatures, pcvrCalibrationFeatures, onlineFeatures, token); err != nil { + return nil, nil, nil, nil, nil, nil, nil, err + } + + return features, featureToDataType, rtpFeatures, pctrCalibrationFeatures, pcvrCalibrationFeatures, predatorAndNumerixOutputsToDataType, offlineToOnlineMapping, nil +} + +func extractEntityIDs(request InferflowOnboardRequest) map[string]bool { + entityIDs := make(map[string]bool) + for _, ranker := range request.Payload.Rankers { + entityID := strings.Join(ranker.EntityID, COLON_DELIMITER) + entityIDs[entityID] = false + } + for _, reRanker := range request.Payload.ReRankers { + entityID := strings.Join(reRanker.EntityID, COLON_DELIMITER) + entityIDs[entityID] = false + } + return entityIDs +} + +// extractFeatures extracts features from the request and returns a set of features and a map of +// feature to data type, the features in it will have atleast two parts after split by PIPE_DELIMITER +// and the feature should not be the entity_id +func extractFeatures(request InferflowOnboardRequest, entityIDs map[string]bool) (mapset.Set[string], map[string]string, map[string]string) { + features := mapset.NewSet[string]() + featureToDataType := make(map[string]string) + outputFeatures := mapset.NewSet[string]() + predatorOutputs := mapset.NewSet[string]() + predatorAndNumerixOutputsToDataType := make(map[string]string) + + for _, ranker := range request.Payload.Rankers { + for _, output := range ranker.Outputs { + for _, ms := range output.ModelScores { + predatorOutputs.Add(ms) + predatorAndNumerixOutputsToDataType[ms] = output.DataType + } + } + } + + addFeature := func(feature, dtype string) { + if _, ok := entityIDs[feature]; ok { + return + } + if parts := strings.Split(feature, PIPE_DELIMITER); len(parts) >= 2 && !predatorOutputs.Contains(feature) { + features.Add(feature) + featureToDataType[feature] = dtype + } + } + + for _, ranker := range request.Payload.Rankers { + for _, input := range ranker.Inputs { + for _, feature := range input.Features { + addFeature(feature, input.DataType) + } + } + + for _, output := range ranker.Outputs { + outputFeatures.Add(output.Name) + } + } + + for _, reRanker := range request.Payload.ReRankers { + for _, feature := range reRanker.EqVariables { + if _, ok := entityIDs[feature]; ok { + continue + } + parts := strings.Split(feature, PIPE_DELIMITER) + if len(parts) < 2 { + continue + } + featureName := parts[1] + if predatorOutputs.Contains(feature) || predatorOutputs.Contains(featureName) { + continue + } + features.Add(feature) + featureToDataType[feature] = "" + } + predatorAndNumerixOutputsToDataType[reRanker.Score] = reRanker.DataType + } + + return features, featureToDataType, predatorAndNumerixOutputsToDataType +} + +func fetchMissingDatatypes( + featureToDataType map[string]string, + rtpFeatures mapset.Set[string], + pctrCalibrationFeatures mapset.Set[string], + pcvrCalibrationFeatures mapset.Set[string], + onlineFeatures mapset.Set[string], + token string, +) error { + hasEmptyDatatype := false + for _, dtype := range featureToDataType { + if dtype == "" { + hasEmptyDatatype = true + break + } + } + if !hasEmptyDatatype { + return nil + } + + horizonFeatures := make(map[string]struct{ label, group string }) + rtpFeaturesToFetch := mapset.NewSet[string]() + + for feature, dtype := range featureToDataType { + if dtype != "" { + continue + } + + parts := strings.Split(feature, COLON_DELIMITER) + + // Calibration features + if pctrCalibrationFeatures.Contains(feature) { + if len(parts) >= 3 { + label := parts[1] + group := parts[2] + horizonFeatures[feature] = struct{ label, group string }{label, group} + } + continue + } + if pcvrCalibrationFeatures.Contains(feature) { + if len(parts) >= 3 { + label := parts[1] + group := parts[2] + horizonFeatures[feature] = struct{ label, group string }{label, group} + } + continue + } + + // Check if it's an RTP feature + if rtpFeatures.Contains(feature) { + rtpFeaturesToFetch.Add(feature) + continue + } + + // Check if it's an online feature + if onlineFeatures.Contains(feature) { + if len(parts) >= 2 { + if strings.HasPrefix(feature, "parent:") && len(parts) == 2 { + continue + } + label := parts[0] + group := parts[1] + horizonFeatures[feature] = struct{ label, group string }{label, group} + } + continue + } + } + + // Query RTP API once for all RTP datatypes + if rtpFeaturesToFetch.Cardinality() > 0 { + rtpDataTypeMap, err := GetRTPFeatureGroupDataTypeMap(token) + if err == nil { + for _, feature := range rtpFeaturesToFetch.ToSlice() { + if dataType, exists := rtpDataTypeMap[feature]; exists { + featureToDataType[feature] = dataType + continue + } + + parts := strings.Split(feature, COLON_DELIMITER) + if len(parts) == 4 { + withoutPrefix := strings.Join(parts[1:], COLON_DELIMITER) + if dataType, exists := rtpDataTypeMap[withoutPrefix]; exists { + featureToDataType[feature] = dataType + } + } + } + } + } + + // Query Horizon API for remaining features grouped by label + if len(horizonFeatures) > 0 { + labelToGroups := make(map[string]mapset.Set[string]) + + for _, labelGroup := range horizonFeatures { + if labelToGroups[labelGroup.label] == nil { + labelToGroups[labelGroup.label] = mapset.NewSet[string]() + } + labelToGroups[labelGroup.label].Add(labelGroup.group) + } + + labelToGroupDataType := make(map[string]map[string]string) + for label := range labelToGroups { + featureGroupDataTypeMap, err := GetFeatureGroupDataTypeMap(label, token) + if err != nil { + continue + } + labelToGroupDataType[label] = featureGroupDataTypeMap + } + + for feature, labelGroup := range horizonFeatures { + if featureToDataType[feature] != "" { + continue + } + + if groupMap, exists := labelToGroupDataType[labelGroup.label]; exists { + if dataType, exists := groupMap[labelGroup.group]; exists { + featureToDataType[feature] = dataType + } + } + } + } + + return nil +} + +// classifyFeatures classifies features into offline, online and default features +// and returns a set of features for each class and a map of feature to data type +func classifyFeatures( + featureList mapset.Set[string], + featureDataTypes map[string]string, +) (mapset.Set[string], mapset.Set[string], mapset.Set[string], mapset.Set[string], mapset.Set[string], mapset.Set[string], map[string]string, error) { + defaultFeatures := mapset.NewSet[string]() + onlineFeatures := mapset.NewSet[string]() + offlineFeatures := mapset.NewSet[string]() + rtpFeatures := mapset.NewSet[string]() + pctrCalibrationFeatures := mapset.NewSet[string]() + pcvrCalibrationFeatures := mapset.NewSet[string]() + newFeatureToDataType := make(map[string]string) + + add := func(targetSet mapset.Set[string], name, originalFeature string) { + targetSet.Add(name) + newFeatureToDataType[name] = featureDataTypes[originalFeature] + } + + for feature := range featureList.Iter() { + transformedFeature, featureType, err := transformFeature(feature) + if err != nil { + return nil, nil, nil, nil, nil, nil, nil, err + } + + switch featureType { + case featureClassOffline: + add(offlineFeatures, transformedFeature, feature) + case featureClassOnline: + add(onlineFeatures, transformedFeature, feature) + case featureClassDefault: + add(defaultFeatures, transformedFeature, feature) + case featureClassRtp: + add(rtpFeatures, transformedFeature, feature) + case featureClassPCVRCalibration: + add(pcvrCalibrationFeatures, transformedFeature, feature) + case featureClassPCTRCalibration: + add(pctrCalibrationFeatures, transformedFeature, feature) + } + } + + return offlineFeatures, onlineFeatures, defaultFeatures, rtpFeatures, pctrCalibrationFeatures, pcvrCalibrationFeatures, newFeatureToDataType, nil +} + +// transformFeature transforms the feature to either online, offline or default feature +// and returns the transformed feature and the feature type. The feature can be of these given types: +// 1. PARENT_OFFLINE_FEATURE|FEATURE +// 2. DEFAULT_FEATURE|FEATURE +// 3. ONLINE_FEATURE|FEATURE +// 4. OFFLINE_FEATURE|FEATURE +// 5. PARENT_DEFAULT_FEATURE|FEATURE +// 6. PARENT_ONLINE_FEATURE|FEATURE +// 7. MODEL_FEATURE|FEATURE +// 8. CALIBRATION|FEATURE +// 9. RTP_FEATURE|FEATURE +// 10. PARENT_RTP_FEATURE|FEATURE +func transformFeature(feature string) (string, string, error) { + parts := strings.Split(feature, PIPE_DELIMITER) + if len(parts) < 2 { + return "", featureClassInvalid, fmt.Errorf("feature %s is invalid", feature) + } + + featureTypes := strings.Split(parts[0], UNDERSCORE_DELIMITER) + featureName := parts[1] + + if parts[0] == PCTR_CALIBRATION { + return strings.ToLower(PCTR_CALIBRATION) + COLON_DELIMITER + featureName, featureClassPCTRCalibration, nil + } + if parts[0] == PCVR_CALIBRATION { + return strings.ToLower(PCVR_CALIBRATION) + COLON_DELIMITER + featureName, featureClassPCVRCalibration, nil + } + + if featureTypes[0] == PARENT { + if len(featureTypes) > 1 { + newFeature := strings.ToLower(featureTypes[0]) + COLON_DELIMITER + featureName + switch featureTypes[1] { + case DEFAULT_FEATURE: + return newFeature, featureClassDefault, nil + case ONLINE_FEATURE, CALIBRATION: + return newFeature, featureClassOnline, nil + case OFFLINE_FEATURE: + return newFeature, featureClassOffline, nil + case RTP_FEATURE: + return newFeature, featureClassRtp, nil + case PCVR_CALIBRATION: + return newFeature, featureClassPCVRCalibration, nil + case PCTR_CALIBRATION: + return newFeature, featureClassPCTRCalibration, nil + } + } + } + + switch featureTypes[0] { + case DEFAULT_FEATURE: + return featureName, featureClassDefault, nil + case ONLINE_FEATURE, CALIBRATION: + return featureName, featureClassOnline, nil + case OFFLINE_FEATURE: + return featureName, featureClassOffline, nil + case RTP_FEATURE: + return featureName, featureClassRtp, nil + case PCVR_CALIBRATION: + return featureName, featureClassPCVRCalibration, nil + case PCTR_CALIBRATION: + return featureName, featureClassPCTRCalibration, nil + } + + return featureName, featureClassDefault, nil +} + +// mapOfflineFeatures maps the offline features to the online features +// and returns a map of offline feature to online feature +func mapOfflineFeatures(offlineFeatureList mapset.Set[string], token string) (map[string]string, error) { + return GetOnlineFeatureMapping(offlineFeatureList, token) +} + +func fetchRTPComponentFeatures(rtpFeatures mapset.Set[string], etcdConfig etcd.Manager) (mapset.Set[string], map[string]string, error) { + componentList := getComponentList(rtpFeatures, nil, nil) + componentFeatures := mapset.NewSet[string]() + featureToDataType := make(map[string]string) + + for _, component := range componentList.ToSlice() { + componentData := etcdConfig.GetComponentData(component) + if componentData == nil { + return nil, nil, fmt.Errorf("RTP Component: componentData for '%s' not found in registry", component) + } + + for _, pair := range componentData.FSIdSchemaToValueColumns { + if strings.Contains(pair.ValueCol, COLON_DELIMITER) { + componentFeatures.Add(pair.ValueCol) + featureToDataType[pair.ValueCol] = pair.DataType + } + } + } + + return componentFeatures, featureToDataType, nil +} + +// fetchComponentFeatures fetches the component features from the etcd config +// and returns a set of component features and a map of component feature to data type +func fetchComponentFeatures(features mapset.Set[string], pctrCalibrationFeatures mapset.Set[string], pcvrCalibrationFeatures mapset.Set[string], etcdConfig etcd.Manager, realEstate string, token string) (mapset.Set[string], map[string]string, error) { + componentList := getComponentList(features, pctrCalibrationFeatures, pcvrCalibrationFeatures) + componentFeatures := mapset.NewSet[string]() + featureToDataType := make(map[string]string) + + for _, component := range componentList.ToSlice() { + componentData := etcdConfig.GetComponentData(component) + if componentData == nil { + return nil, nil, fmt.Errorf("Component Data: ComponentData for '%s' not found in registry.\nPlease Contact MLP Team to onboard the component.", component) + } + for _, pair := range componentData.FSIdSchemaToValueColumns { + if strings.Contains(pair.ValueCol, COLON_DELIMITER) { + componentFeatures.Add(pair.ValueCol) + featureToDataType[pair.ValueCol] = pair.DataType + } + } + + if override, hasOverride := componentData.Overridecomponent[realEstate]; hasOverride { + componentFeatures.Add(override.ComponentId) + parts := strings.Split(override.ComponentId, COLON_DELIMITER) + var label, group string + + if len(parts) == 3 { + label, group = parts[0], parts[1] + } else if len(parts) == 4 { + label, group = parts[1], parts[2] + } else { + return nil, nil, fmt.Errorf("Component Data: invalid override component id: %s", override.ComponentId) + } + + featureGroupDataTypeMap, err := GetFeatureGroupDataTypeMap(label, token) + if err != nil { + return nil, nil, fmt.Errorf("Component Data: error getting feature group data type map: %w", err) + } + + if dataType, exists := featureGroupDataTypeMap[group]; exists { + featureToDataType[override.ComponentId] = dataType + } else { + return nil, nil, fmt.Errorf("Component Data: feature group data type not found for %s: %s", override.ComponentId, group) + } + } + } + + return componentFeatures, featureToDataType, nil +} + +// getComponentList gets the component list from the features +// and returns a set of component names. The component name can be of these given types: +// 1. parent_