From 2f73c700aaed36917502e1149b6e2641ec539039 Mon Sep 17 00:00:00 2001 From: Schulz Date: Wed, 11 Feb 2026 16:20:50 +0100 Subject: [PATCH 01/23] initial commit --- core2/pkg/foundation/00_module.go | 3 + core2/pkg/foundation/policies.go | 157 ++++++++++++++++++ .../policies/restrict_applications.yaml | 28 ++++ .../policies/restrict_cut_and_paste.yaml | 21 +++ .../policies/restrict_downloads.yaml | 15 ++ .../restrict_integrated_applications.yaml | 25 +++ .../policies/restrict_internet_access.yaml | 18 ++ .../restrict_organizations_members.yaml | 18 ++ .../restrict_provider_file_transfers.yaml | 21 +++ .../policies/restrict_public_ips.yaml | 12 ++ .../policies/restrict_public_links.yaml | 12 ++ .../policies/restrict_source_ip_range.yaml | 20 +++ core2/pkg/foundation/policy_templates.go | 78 +++++++++ core2/pkg/migrations/00_module.go | 1 + core2/pkg/migrations/policies.go | 23 +++ 15 files changed, 452 insertions(+) create mode 100644 core2/pkg/foundation/policies.go create mode 100644 core2/pkg/foundation/policies/restrict_applications.yaml create mode 100644 core2/pkg/foundation/policies/restrict_cut_and_paste.yaml create mode 100644 core2/pkg/foundation/policies/restrict_downloads.yaml create mode 100644 core2/pkg/foundation/policies/restrict_integrated_applications.yaml create mode 100644 core2/pkg/foundation/policies/restrict_internet_access.yaml create mode 100644 core2/pkg/foundation/policies/restrict_organizations_members.yaml create mode 100644 core2/pkg/foundation/policies/restrict_provider_file_transfers.yaml create mode 100644 core2/pkg/foundation/policies/restrict_public_ips.yaml create mode 100644 core2/pkg/foundation/policies/restrict_public_links.yaml create mode 100644 core2/pkg/foundation/policies/restrict_source_ip_range.yaml create mode 100644 core2/pkg/foundation/policy_templates.go create mode 100644 core2/pkg/migrations/policies.go diff --git a/core2/pkg/foundation/00_module.go b/core2/pkg/foundation/00_module.go index 1e916d05ad..4d138bd4ed 100644 --- a/core2/pkg/foundation/00_module.go +++ b/core2/pkg/foundation/00_module.go @@ -44,5 +44,8 @@ func Init() { initAuthOidc() times["Oidc"] = t.Mark() + initPolicies() + times["Policies"] = t.Mark() + coreutil.PrintStartupTimes("Foundation", times) } diff --git a/core2/pkg/foundation/policies.go b/core2/pkg/foundation/policies.go new file mode 100644 index 0000000000..e103105756 --- /dev/null +++ b/core2/pkg/foundation/policies.go @@ -0,0 +1,157 @@ +package foundation + +import ( + "encoding/json" + "net/http" + + "gopkg.in/yaml.v3" + "ucloud.dk/shared/pkg/cfgutil" + db "ucloud.dk/shared/pkg/database" + fndapi "ucloud.dk/shared/pkg/foundation" + "ucloud.dk/shared/pkg/log" + "ucloud.dk/shared/pkg/rpc" + "ucloud.dk/shared/pkg/util" +) + +var policySchemas map[string]fndapi.PolicySchema + +func initPolicies() { + readPolicies() + + loadPoliciesFromDB() + + fndapi.PoliciesRetrieve.Handler(func(info rpc.RequestInfo, request util.Empty) (map[string]fndapi.Policy, *util.HttpError) { + return retrievePolicies(info.Actor) + }) + + fndapi.PoliciesUpdate.Handler(func(info rpc.RequestInfo, request fndapi.PoliciesUpdateRequest) (util.Empty, *util.HttpError) { + return updatePolicies(info.Actor, request) + }) +} + +func readPolicies() { + policies := pullProjectPolicies() + policySchemas = make(map[string]fndapi.PolicySchema, len(policies)) + for _, policy := range policies { + var document yaml.Node + success := true + + err := yaml.Unmarshal(policy.Bytes, &document) + if err != nil { + log.Fatal("Error loading policy document ", policy.PolicyName, ": ", err) + } + + var policySchema fndapi.PolicySchema + + cfgutil.Decode("", &document, &policySchema, &success) + + if !success { + log.Fatal("Error decoding policy document ", policy.PolicyName, ": ", err) + } + policySchemas[policySchema.Name] = policySchema + } +} + +func loadPoliciesFromDB() { + db.NewTx0(func(tx *db.Transaction) { + db.Select[struct { + ProjectID string + Policy string + }]( + tx, + ` + select * + from project.policies + order by project_id + `, + db.Params{}, + ) + + //TODO(chaching) + }) +} + +func retrievePolicies(actor rpc.Actor) (map[string]fndapi.Policy, *util.HttpError) { + if !actor.Project.Present { + return nil, util.HttpErr(http.StatusBadRequest, "Polices only applicable to projects") + } + if !actor.Membership[actor.Project.Value].Satisfies(rpc.ProjectRoleAdmin) { + return nil, util.HttpErr(http.StatusForbidden, "Only PIs and Admins may list the policies") + } + return nil, nil +} + +type SimplePolicyProperty struct { + PropertyType fndapi.PolicyPropertyType `yaml:"property_type"` + PropertyValue any `yaml:"property_value"` +} + +func updatePolicies(actor rpc.Actor, request fndapi.PoliciesUpdateRequest) (util.Empty, *util.HttpError) { + if !actor.Project.Present { + return util.Empty{}, util.HttpErr(http.StatusBadRequest, "Polices only applicable to projects") + } + if !actor.Membership[actor.Project.Value].Satisfies(rpc.ProjectRolePI) { + return util.Empty{}, util.HttpErr(http.StatusForbidden, "Only PIs may update the policies") + } + + db.NewTx0(func(tx *db.Transaction) { + b := db.BatchNew(tx) + for _, specification := range request.UpdatedPolicies { + _, ok := policySchemas[specification.Schema] + //When trying to update a schema that does not exist, we just skip it + if !ok { + log.Debug("Unknown Schema ", specification.Schema) + continue + } + projectId := specification.Project + + isEnabled := false + for _, property := range specification.Properties { + if property.Name == "enabled" { + isEnabled = property.Bool + } + } + + //If the policy is not enabled we need to delete it form the DB. + //Else we need to insert or update already existing project policy + if !isEnabled { + db.BatchExec( + b, + ` + delete from project.policies + where project_id = :project_id and policy_name = :policy_name + `, + db.Params{ + "project_id": projectId, + "policy_name": specification.Schema, + }, + ) + } else { + properties, err := json.Marshal(specification.Properties) + if err != nil { + log.Debug("Error marshalling policy document ", specification.Schema, " ", specification.Properties) + continue + } + db.BatchExec( + b, + ` + insert into project.policies (policy_name, policy_property, project_id) + values (:policy_name, :policy_properties, :project_id) + on conflict (policy_name, project_id) do + update set policy_property = excluded.policy_property, + `, + db.Params{ + "policy_name": specification.Schema, + "policy_properties": properties, + "project_id": projectId, + }, + ) + } + } + db.BatchSend(b) + + //TODO( DO caching update) + }) + + return util.Empty{}, nil +} diff --git a/core2/pkg/foundation/policies/restrict_applications.yaml b/core2/pkg/foundation/policies/restrict_applications.yaml new file mode 100644 index 0000000000..dd616b5714 --- /dev/null +++ b/core2/pkg/foundation/policies/restrict_applications.yaml @@ -0,0 +1,28 @@ +name: "RestrictApplications" +title: "Restrict applications available for use" + +description: > + Restricts which applications are available for use in the project. + +configuration: + - name: "enabled" + type: "Bool" + title: "Enabled" + description: > + If enabled, the project will only be able to run applications that are listed along this policy. + If disabled the all apps in the app catalog is available (within the regular limitations of products needed). + - name: "applications" + type: "TextList" + title: "Applications" + description: > + List of applications which the project should be restricted to + use. This refers to the canonical application name. This name + can be found in the UI by copying the canonical name of a concrete + _flavor_. + + An empty list indicates that there are no apps available within the project. + Any additions to the list will cause the project to be able to run these applications. + + This configuration option does not affect provider registered + applications. This option cannot be used to control Syncthing or + other integrated applications. \ No newline at end of file diff --git a/core2/pkg/foundation/policies/restrict_cut_and_paste.yaml b/core2/pkg/foundation/policies/restrict_cut_and_paste.yaml new file mode 100644 index 0000000000..b4385d0c59 --- /dev/null +++ b/core2/pkg/foundation/policies/restrict_cut_and_paste.yaml @@ -0,0 +1,21 @@ +name: "CutAndPaste" +title: "Disable copy & paste in interactive applications" + +description: > + This option will turn off copy & paste in interactive applications. This is + implemented by forcing access to all interactive applications to be done + through a remote desktop. This option is insufficient if Internet access + is allowed. Copy & paste will still be possible internally in the + application, if supported, but it will not be available on the client + machine. + + This feature will automatically disable the "Open terminal" feature on all + jobs. + +configuration: + - name: "enabled" + type: "Bool" + title: "Enabled" + description: > + If enabled, copy & paste will be turned off. If not enabled, + copy & paste will be available if the application supports it. \ No newline at end of file diff --git a/core2/pkg/foundation/policies/restrict_downloads.yaml b/core2/pkg/foundation/policies/restrict_downloads.yaml new file mode 100644 index 0000000000..9cdf9fa15d --- /dev/null +++ b/core2/pkg/foundation/policies/restrict_downloads.yaml @@ -0,0 +1,15 @@ +name: "RestrictDownloads" +title: "Restrict downloads" + +description: > + Controls whether users are allowed to download data from the portal + to their local machine. Note that this policy may be ineffective if + the provider allows direct access to the data. + +configuration: + - name: "enabled" + type: "Bool" + title: "Enabled" + description: > + If enabled, downloads to the client machine are blocked. + If disabled, downloads are allowed. \ No newline at end of file diff --git a/core2/pkg/foundation/policies/restrict_integrated_applications.yaml b/core2/pkg/foundation/policies/restrict_integrated_applications.yaml new file mode 100644 index 0000000000..7e6df2365f --- /dev/null +++ b/core2/pkg/foundation/policies/restrict_integrated_applications.yaml @@ -0,0 +1,25 @@ +name: "RestrictIntegratedApplications" +title: "Restrict integrated applications (e.g. Syncthing and terminal)" + +description: > + Restricts applications from running which are registered by the provider. + This includes applications such as Syncthing and the terminal, which can + be opened from the file browser. + +configuration: + - name: "enabled" + type: Bool + title: "Enabled" + description: > + If this option is checked, then restrictions will apply and only + applications found on the allow-list can be run. If it is not + checked then all integrated applications can be run and the + allow-list is ignored. + - name: "allowList" + type: TextList + title: "Allow-list" + description: > + List of applications which are explicitly allowed. + options: + - "syncthing" + - "terminal" \ No newline at end of file diff --git a/core2/pkg/foundation/policies/restrict_internet_access.yaml b/core2/pkg/foundation/policies/restrict_internet_access.yaml new file mode 100644 index 0000000000..44293a83f6 --- /dev/null +++ b/core2/pkg/foundation/policies/restrict_internet_access.yaml @@ -0,0 +1,18 @@ +name: "RestrictInternetAccess" +title: "Restrict Internet access" +description: > + Limits outbound Internet connectivity from workloads belonging to the project. + +configuration: + - name: "enabled" + type: "Bool" + title: "Enabled" + description: > + If enabled, restriction of internet access will be activated and a limited predefined + subnet range is enforced. + - name: "allowedSubnets" + type: "Subnet" + title: "Allowed subnets" + description: > + Only the specified destination subnets are reachable. An empty list means + no Internet access is allowed. \ No newline at end of file diff --git a/core2/pkg/foundation/policies/restrict_organizations_members.yaml b/core2/pkg/foundation/policies/restrict_organizations_members.yaml new file mode 100644 index 0000000000..fd52477be1 --- /dev/null +++ b/core2/pkg/foundation/policies/restrict_organizations_members.yaml @@ -0,0 +1,18 @@ +name: "RestrictOrganizationMembers" +title: "Restrict members to organization" + +description: > + Ensures that all project members belong to the same organization. This does not remove existing project members that + are not a part of the organization. + +configuration: + - name: "enabled" + type: "Bool" + title: "Enabled" + description: > + If enabled, invitations to users outside the specified organization are rejected. + - name: "organizations" + type: "TextList" + title: "Organizations" + description: > + The list of organizations which are allowed in the project. \ No newline at end of file diff --git a/core2/pkg/foundation/policies/restrict_provider_file_transfers.yaml b/core2/pkg/foundation/policies/restrict_provider_file_transfers.yaml new file mode 100644 index 0000000000..937a9d98a9 --- /dev/null +++ b/core2/pkg/foundation/policies/restrict_provider_file_transfers.yaml @@ -0,0 +1,21 @@ +name: "RestrictProviderTransfers" +title: "Restrict file transfers between providers" + +description: > + Controls which providers may participate in data transfers for this project. + +configuration: + - name: "enabled" + type: "Bool" + title: "Enabled" + description: > + If enabled, users will only be able to transfer files between specific providers. + If disabled, users can move files between any provider they have access to. + - name: "allowedProviders" + type: "Providers" + title: "Allowed providers" + description: > + Only the listed providers may be used as source or destination for file + transfers. This option does not guarantee that data transfer between + selected providers is possible, that is, providers may overrule this + with their own transfer policies. \ No newline at end of file diff --git a/core2/pkg/foundation/policies/restrict_public_ips.yaml b/core2/pkg/foundation/policies/restrict_public_ips.yaml new file mode 100644 index 0000000000..cc5343e535 --- /dev/null +++ b/core2/pkg/foundation/policies/restrict_public_ips.yaml @@ -0,0 +1,12 @@ +name: "RestrictPublicIPs" +title: "Restrict public IPs" +description: > + Controls whether workloads in the project may be assigned public IP addresses. + +configuration: + - name: "enabled" + type: "Bool" + title: "Enabled" + description: > + If enabled, public IP creation and use is disabled. Existing public + IPs are _not_ deleted. \ No newline at end of file diff --git a/core2/pkg/foundation/policies/restrict_public_links.yaml b/core2/pkg/foundation/policies/restrict_public_links.yaml new file mode 100644 index 0000000000..21bd890fbb --- /dev/null +++ b/core2/pkg/foundation/policies/restrict_public_links.yaml @@ -0,0 +1,12 @@ +name: "RestrictPublicLinks" +title: "Restrict public links" +description: > + Controls whether public (unauthenticated) links to project data are allowed. + +configuration: + - name: "enabled" + type: "Bool" + title: "Enabled" + description: > + If enabled, creation and use of public links is disabled. Existing + public links are _not_ deleted. \ No newline at end of file diff --git a/core2/pkg/foundation/policies/restrict_source_ip_range.yaml b/core2/pkg/foundation/policies/restrict_source_ip_range.yaml new file mode 100644 index 0000000000..d798aa5e40 --- /dev/null +++ b/core2/pkg/foundation/policies/restrict_source_ip_range.yaml @@ -0,0 +1,20 @@ +name: "RestrictSourceIPRange" +title: "Restrict client source IP range" + +description: > + Limits which client IP ranges are allowed to access the project. + +configuration: + - name: "enabled" + type: "Bool" + title: "Enabled" + description: > + If enabled, users will only be able to access the project from specified subnets. + If disabled, users can access the project from any IP in the world. + + - name: "allowedClientSubnets" + type: "Subnet" + title: "Allowed client subnets" + description: > + Only clients originating from these subnets are permitted. An empty + list blocks all external access. \ No newline at end of file diff --git a/core2/pkg/foundation/policy_templates.go b/core2/pkg/foundation/policy_templates.go new file mode 100644 index 0000000000..984dce4445 --- /dev/null +++ b/core2/pkg/foundation/policy_templates.go @@ -0,0 +1,78 @@ +package foundation + +import _ "embed" + +// Generated by mailtpl/generate.sh. Please run the script from the mailtpl folder. + +//go:embed policies/restrict_applications.yaml +var restrictApplications []byte + +//go:embed policies/restrict_cut_and_paste.yaml +var restrictCutAndPast []byte + +//go:embed policies/restrict_downloads.yaml +var restrictDownloads []byte + +//go:embed policies/restrict_integrated_applications.yaml +var restrictIntegratedApplications []byte + +//go:embed policies/restrict_internet_access.yaml +var restrictInternetAccess []byte + +//go:embed policies/restrict_organizations_members.yaml +var restrictOrganizationsMembers []byte + +//go:embed policies/restrict_provider_file_transfers.yaml +var restrictProviderTransfers []byte + +//go:embed policies/restrict_public_ips.yaml +var restrictPublicIPs []byte + +//go:embed policies/restrict_public_links.yaml +var restrictPublicLinks []byte + +//go:embed policies/restrict_source_ip_range.yaml +var restrictSourceIpRange []byte + +type LoadedPolicy struct { + PolicyName string + Bytes []byte +} + +func pullProjectPolicies() []LoadedPolicy { + policies := []LoadedPolicy{ + { + PolicyName: "restrictApplications", + Bytes: restrictApplications, + }, + { + PolicyName: "restrictCutAndPast", + Bytes: restrictCutAndPast, + }, { + PolicyName: "restrictDownloads", + Bytes: restrictDownloads, + }, { + PolicyName: "restrictIntegratedApplications", + Bytes: restrictIntegratedApplications, + }, { + PolicyName: "restrictInternetAccess", + Bytes: restrictInternetAccess, + }, { + PolicyName: "restrictOrganizationsMembers", + Bytes: restrictOrganizationsMembers, + }, { + PolicyName: "restrictProviderTransfers", + Bytes: restrictProviderTransfers, + }, { + PolicyName: "restrictPublicIPs", + Bytes: restrictPublicIPs, + }, { + PolicyName: "restrictPublicLinks", + Bytes: restrictPublicLinks, + }, { + PolicyName: "restrictSourceIpRange", + Bytes: restrictSourceIpRange, + }, + } + return policies +} diff --git a/core2/pkg/migrations/00_module.go b/core2/pkg/migrations/00_module.go index e8b768daaa..dfa4a44f6d 100644 --- a/core2/pkg/migrations/00_module.go +++ b/core2/pkg/migrations/00_module.go @@ -27,4 +27,5 @@ func Init() { db.AddMigration(authV3()) db.AddMigration(grantV1()) db.AddMigration(projectsV4()) + db.AddMigration(policiesV1()) } diff --git a/core2/pkg/migrations/policies.go b/core2/pkg/migrations/policies.go new file mode 100644 index 0000000000..973f531e04 --- /dev/null +++ b/core2/pkg/migrations/policies.go @@ -0,0 +1,23 @@ +package migrations + +import db "ucloud.dk/shared/pkg/database" + +func policiesV1() db.MigrationScript { + return db.MigrationScript{ + Id: "policiesV1", + Execute: func(tx *db.Transaction) { + db.Exec( + tx, + ` + create table if not exists project.policies ( + policy_name text not null, + policy_property jsonb not null, + project_id text not null references project.projects(id) on delete cascade, + primary key (project_id, policy_name) + ) + `, + db.Params{}, + ) + }, + } +} From 56c7cbbe0b000b2425ef715c336dabae3e1a3366 Mon Sep 17 00:00:00 2001 From: Schulz Date: Wed, 11 Feb 2026 22:54:17 +0100 Subject: [PATCH 02/23] made caching --- core2/pkg/foundation/policies.go | 91 ++++++++++++++++++++++++++++---- 1 file changed, 80 insertions(+), 11 deletions(-) diff --git a/core2/pkg/foundation/policies.go b/core2/pkg/foundation/policies.go index e103105756..068e195c4b 100644 --- a/core2/pkg/foundation/policies.go +++ b/core2/pkg/foundation/policies.go @@ -3,6 +3,7 @@ package foundation import ( "encoding/json" "net/http" + "sync" "gopkg.in/yaml.v3" "ucloud.dk/shared/pkg/cfgutil" @@ -13,12 +14,20 @@ import ( "ucloud.dk/shared/pkg/util" ) +var projectPolicies struct { + Mu sync.RWMutex + PoliciesByProject map[string]*AssociatedPolicies +} + +type AssociatedPolicies struct { + EnabledPolices map[string]fndapi.PolicySpecification +} + var policySchemas map[string]fndapi.PolicySchema func initPolicies() { - readPolicies() - - loadPoliciesFromDB() + populatePolicySchemas() + loadProjectPoliciesFromDB() fndapi.PoliciesRetrieve.Handler(func(info rpc.RequestInfo, request util.Empty) (map[string]fndapi.Policy, *util.HttpError) { return retrievePolicies(info.Actor) @@ -29,7 +38,7 @@ func initPolicies() { }) } -func readPolicies() { +func populatePolicySchemas() { policies := pullProjectPolicies() policySchemas = make(map[string]fndapi.PolicySchema, len(policies)) for _, policy := range policies { @@ -52,11 +61,12 @@ func readPolicies() { } } -func loadPoliciesFromDB() { +func loadProjectPoliciesFromDB() { db.NewTx0(func(tx *db.Transaction) { - db.Select[struct { - ProjectID string - Policy string + rows := db.Select[struct { + ProjectId string + PolicyName string + PolicyProperty string }]( tx, ` @@ -67,7 +77,23 @@ func loadPoliciesFromDB() { db.Params{}, ) - //TODO(chaching) + projectPolicies.Mu.Lock() + + for _, row := range rows { + projectId := row.ProjectId + policies, ok := projectPolicies.PoliciesByProject[projectId] + if !ok { + policies = &AssociatedPolicies{EnabledPolices: map[string]fndapi.PolicySpecification{}} + } + specification := fndapi.PolicySpecification{} + err := json.Unmarshal([]byte(row.PolicyProperty), &specification) + if err != nil { + log.Fatal("Error loading policy document ", row.PolicyProperty, ": ", err) + } + policies.EnabledPolices[row.PolicyName] = specification + } + + projectPolicies.Mu.Unlock() }) } @@ -78,7 +104,24 @@ func retrievePolicies(actor rpc.Actor) (map[string]fndapi.Policy, *util.HttpErro if !actor.Membership[actor.Project.Value].Satisfies(rpc.ProjectRoleAdmin) { return nil, util.HttpErr(http.StatusForbidden, "Only PIs and Admins may list the policies") } - return nil, nil + + result := make(map[string]fndapi.Policy, len(policySchemas)) + + projectPolicies.Mu.RLock() + policies := projectPolicies.PoliciesByProject[string(actor.Project.Value)].EnabledPolices + projectPolicies.Mu.RUnlock() + for name, schema := range policySchemas { + + specification, ok := policies[name] + if !ok { + specification = fndapi.PolicySpecification{} + } + result[name] = fndapi.Policy{ + Schema: schema, + Specification: specification, + } + } + return result, nil } type SimplePolicyProperty struct { @@ -150,7 +193,33 @@ func updatePolicies(actor rpc.Actor, request fndapi.PoliciesUpdateRequest) (util } db.BatchSend(b) - //TODO( DO caching update) + //Updating cache + projectPolicies.Mu.Lock() + for _, specification := range request.UpdatedPolicies { + isEnabled := false + for _, property := range specification.Properties { + if property.Name == "enabled" { + isEnabled = property.Bool + } + } + projectId := string(specification.Project) + if !isEnabled { + policies, ok := projectPolicies.PoliciesByProject[projectId] + //If no policies are enabled for the project then just skip the deletion + if !ok { + continue + } + policies.EnabledPolices[specification.Schema] = fndapi.PolicySpecification{} + } else { + policies, ok := projectPolicies.PoliciesByProject[projectId] + if !ok { + policies = &AssociatedPolicies{EnabledPolices: map[string]fndapi.PolicySpecification{}} + projectPolicies.PoliciesByProject[projectId] = policies + } + policies.EnabledPolices[specification.Schema] = specification + } + } + projectPolicies.Mu.Unlock() }) return util.Empty{}, nil From 2e834125ebfbc29b261c2bbe5fa96049cb86517c Mon Sep 17 00:00:00 2001 From: Schulz Date: Mon, 16 Feb 2026 10:21:54 +0100 Subject: [PATCH 03/23] finished provider notification --- .../pkg/accounting/provider_notifications.go | 68 +++++++- core2/pkg/coreutil/00_module.go | 30 ++++ core2/pkg/foundation/policies.go | 154 ++++++++++-------- .../restrict_integrated_applications.yaml | 2 +- core2/pkg/foundation/policy_templates.go | 2 - core2/pkg/migrations/policies.go | 28 +++- .../shared/pkg/foundation/policies.go | 5 +- 7 files changed, 200 insertions(+), 89 deletions(-) diff --git a/core2/pkg/accounting/provider_notifications.go b/core2/pkg/accounting/provider_notifications.go index 95858f20bc..6947bcbe75 100644 --- a/core2/pkg/accounting/provider_notifications.go +++ b/core2/pkg/accounting/provider_notifications.go @@ -23,9 +23,28 @@ import ( var providerWalletNotifications = make(chan AccWalletId, 1024*1024) var providerNotifications struct { - Mu sync.Mutex + Mu sync.Mutex + + // All of these follow providerId -> sessionId -> channel + ProjectChannelsByProvider map[string]map[string]chan *fndapi.Project WalletsByProvider map[string]map[string]chan *accapi.WalletV2 + PoliciesByProvider map[string]map[string]chan policiesForProject +} + +type policiesForProject struct { + PoliciesByName map[string]*fndapi.PolicySpecification +} + +func retrieveRelevantProviders(projectId string) map[string]util.Empty { + projectWallets := internalRetrieveWallets(time.Now(), projectId, walletFilter{RequireActive: true}) + relevantProviders := map[string]util.Empty{} + + for _, w := range projectWallets { + relevantProviders[w.PaysFor.Provider] = util.Empty{} + } + + return relevantProviders } func initProviderNotifications() { @@ -37,6 +56,7 @@ func initProviderNotifications() { // operations. The payload is either the project or group ID which triggered the update. projectUpdates := db.Listen(context.Background(), "project_updates") groupUpdates := db.Listen(context.Background(), "project_group_updates") + policyUpdates := db.Listen(context.Background(), "policy_updates") for { var project fndapi.Project @@ -45,6 +65,9 @@ func initProviderNotifications() { var walletId AccWalletId var walletOk bool + var policySpecifications map[string]*fndapi.PolicySpecification + var policiesOk bool + select { case projectId := <-projectUpdates: db.NewTx0(func(tx *db.Transaction) { @@ -58,15 +81,14 @@ func initProviderNotifications() { case walletId = <-providerWalletNotifications: walletOk = true + case projectId := <-policyUpdates: + db.NewTx0(func(tx *db.Transaction) { + policySpecifications, policiesOk = coreutil.PolicySpecificationsRetrieveFromDatabase(tx, projectId) + }) } if projectOk { - projectWallets := internalRetrieveWallets(time.Now(), project.Id, walletFilter{RequireActive: true}) - relevantProviders := map[string]util.Empty{} - - for _, w := range projectWallets { - relevantProviders[w.PaysFor.Provider] = util.Empty{} - } + relevantProviders := retrieveRelevantProviders(project.Id) var allChannels []chan *fndapi.Project @@ -109,6 +131,29 @@ func initProviderNotifications() { } } } + } else if policiesOk { + relevantProviders := retrieveRelevantProviders(project.Id) + var allChannels []chan policiesForProject + + providerNotifications.Mu.Lock() + + for provider := range relevantProviders { + channels, ok := providerNotifications.PoliciesByProvider[provider] + if ok { + for _, ch := range channels { + allChannels = append(allChannels, ch) + } + } + } + + providerNotifications.Mu.Unlock() + + for _, ch := range allChannels { + select { + case ch <- policiesForProject{policySpecifications}: + case <-time.After(200 * time.Millisecond): + } + } } } }() @@ -195,6 +240,7 @@ func providerNotificationHandleClient(conn *ws.Conn) { sessionId := util.RandomTokenNoTs(32) projectUpdates := make(chan *fndapi.Project, 128) walletUpdates := make(chan *accapi.WalletV2, 128) + providerUpdates := make(chan policiesForProject, 128) { providerNotifications.Mu.Lock() @@ -213,6 +259,13 @@ func providerNotificationHandleClient(conn *ws.Conn) { } pmap[sessionId] = projectUpdates + polmap, ok := providerNotifications.PoliciesByProvider[providerId] + if !ok { + polmap = map[string]chan policiesForProject{} + providerNotifications.PoliciesByProvider[providerId] = polmap + } + polmap[sessionId] = providerUpdates + providerNotifications.Mu.Unlock() } @@ -222,6 +275,7 @@ func providerNotificationHandleClient(conn *ws.Conn) { providerNotifications.Mu.Lock() delete(providerNotifications.WalletsByProvider[providerId], sessionId) delete(providerNotifications.ProjectChannelsByProvider[providerId], sessionId) + delete(providerNotifications.PoliciesByProvider[providerId], sessionId) providerNotifications.Mu.Unlock() }() diff --git a/core2/pkg/coreutil/00_module.go b/core2/pkg/coreutil/00_module.go index 8a1c03c986..570f711e48 100644 --- a/core2/pkg/coreutil/00_module.go +++ b/core2/pkg/coreutil/00_module.go @@ -168,6 +168,36 @@ func ProjectRetrieveFromDatabaseViaGroupId(tx *db.Transaction, groupId string) ( } } +func PolicySpecificationsRetrieveFromDatabase(tx *db.Transaction, projectId string) (map[string]*fndapi.PolicySpecification, bool) { + + rows := db.Select[struct { + ProjectId string + PolicyName string + PolicyProperty string + }]( + tx, + ` + select * + from project.policies + where projet_id = :id + `, + db.Params{ + "id": projectId, + }, + ) + var policies map[string]*fndapi.PolicySpecification + for _, row := range rows { + specification := fndapi.PolicySpecification{} + err := json.Unmarshal([]byte(row.PolicyProperty), &specification) + if err != nil { + log.Debug("Error unmarshalling policy properties on update") + return nil, false + } + policies[specification.Schema] = &specification + } + return policies, true +} + func ProjectsListUpdatedAfter(timestamp time.Time) []rpc.ProjectId { return db.NewTx(func(tx *db.Transaction) []rpc.ProjectId { rows := db.Select[struct{ Id string }]( diff --git a/core2/pkg/foundation/policies.go b/core2/pkg/foundation/policies.go index 068e195c4b..fa055bdbde 100644 --- a/core2/pkg/foundation/policies.go +++ b/core2/pkg/foundation/policies.go @@ -5,6 +5,7 @@ import ( "net/http" "sync" + "golang.org/x/exp/maps" "gopkg.in/yaml.v3" "ucloud.dk/shared/pkg/cfgutil" db "ucloud.dk/shared/pkg/database" @@ -26,19 +27,19 @@ type AssociatedPolicies struct { var policySchemas map[string]fndapi.PolicySchema func initPolicies() { - populatePolicySchemas() + policyPopulateSchemaCache() loadProjectPoliciesFromDB() fndapi.PoliciesRetrieve.Handler(func(info rpc.RequestInfo, request util.Empty) (map[string]fndapi.Policy, *util.HttpError) { - return retrievePolicies(info.Actor) + return policiesRetrieve(info.Actor) }) fndapi.PoliciesUpdate.Handler(func(info rpc.RequestInfo, request fndapi.PoliciesUpdateRequest) (util.Empty, *util.HttpError) { - return updatePolicies(info.Actor, request) + return policiesUpdate(info.Actor, request) }) } -func populatePolicySchemas() { +func policyPopulateSchemaCache() { policies := pullProjectPolicies() policySchemas = make(map[string]fndapi.PolicySchema, len(policies)) for _, policy := range policies { @@ -64,13 +65,13 @@ func populatePolicySchemas() { func loadProjectPoliciesFromDB() { db.NewTx0(func(tx *db.Transaction) { rows := db.Select[struct { - ProjectId string - PolicyName string - PolicyProperty string + ProjectId string + PolicyName string + PolicyProperties string }]( tx, ` - select * + select project_id, policy_name, policy_properties from project.policies order by project_id `, @@ -86,9 +87,9 @@ func loadProjectPoliciesFromDB() { policies = &AssociatedPolicies{EnabledPolices: map[string]fndapi.PolicySpecification{}} } specification := fndapi.PolicySpecification{} - err := json.Unmarshal([]byte(row.PolicyProperty), &specification) + err := json.Unmarshal([]byte(row.PolicyProperties), &specification) if err != nil { - log.Fatal("Error loading policy document ", row.PolicyProperty, ": ", err) + log.Fatal("Error loading policy document %v : %v", row.PolicyProperties, err) } policies.EnabledPolices[row.PolicyName] = specification } @@ -97,7 +98,7 @@ func loadProjectPoliciesFromDB() { }) } -func retrievePolicies(actor rpc.Actor) (map[string]fndapi.Policy, *util.HttpError) { +func policiesRetrieve(actor rpc.Actor) (map[string]fndapi.Policy, *util.HttpError) { if !actor.Project.Present { return nil, util.HttpErr(http.StatusBadRequest, "Polices only applicable to projects") } @@ -108,7 +109,7 @@ func retrievePolicies(actor rpc.Actor) (map[string]fndapi.Policy, *util.HttpErro result := make(map[string]fndapi.Policy, len(policySchemas)) projectPolicies.Mu.RLock() - policies := projectPolicies.PoliciesByProject[string(actor.Project.Value)].EnabledPolices + policies := maps.Clone(projectPolicies.PoliciesByProject[string(actor.Project.Value)].EnabledPolices) projectPolicies.Mu.RUnlock() for name, schema := range policySchemas { @@ -129,7 +130,7 @@ type SimplePolicyProperty struct { PropertyValue any `yaml:"property_value"` } -func updatePolicies(actor rpc.Actor, request fndapi.PoliciesUpdateRequest) (util.Empty, *util.HttpError) { +func policiesUpdate(actor rpc.Actor, request fndapi.PoliciesUpdateRequest) (util.Empty, *util.HttpError) { if !actor.Project.Present { return util.Empty{}, util.HttpErr(http.StatusBadRequest, "Polices only applicable to projects") } @@ -137,86 +138,95 @@ func updatePolicies(actor rpc.Actor, request fndapi.PoliciesUpdateRequest) (util return util.Empty{}, util.HttpErr(http.StatusForbidden, "Only PIs may update the policies") } + filteredUpdates := map[string]map[string]fndapi.PolicySpecification{} + for _, specification := range request.UpdatedPolicies { + projectId := string(specification.Project) + filteredUpdates[projectId][specification.Schema] = specification + } + db.NewTx0(func(tx *db.Transaction) { b := db.BatchNew(tx) - for _, specification := range request.UpdatedPolicies { - _, ok := policySchemas[specification.Schema] - //When trying to update a schema that does not exist, we just skip it - if !ok { - log.Debug("Unknown Schema ", specification.Schema) - continue - } - projectId := specification.Project + for projectId, updates := range filteredUpdates { + for _, specification := range updates { + _, ok := policySchemas[specification.Schema] + //When trying to update a schema that does not exist, we just skip it + if !ok { + log.Debug("Unknown Schema ", specification.Schema) + continue + } - isEnabled := false - for _, property := range specification.Properties { - if property.Name == "enabled" { - isEnabled = property.Bool + isEnabled := false + for _, property := range specification.Properties { + if property.Name == "enabled" { + isEnabled = property.Bool + } } - } - //If the policy is not enabled we need to delete it form the DB. - //Else we need to insert or update already existing project policy - if !isEnabled { - db.BatchExec( - b, - ` + //If the policy is not enabled we need to delete it form the DB. + //Else we need to insert or update already existing project policy + if !isEnabled { + db.BatchExec( + b, + ` delete from project.policies where project_id = :project_id and policy_name = :policy_name `, - db.Params{ - "project_id": projectId, - "policy_name": specification.Schema, - }, - ) - } else { - properties, err := json.Marshal(specification.Properties) - if err != nil { - log.Debug("Error marshalling policy document ", specification.Schema, " ", specification.Properties) - continue - } - db.BatchExec( - b, - ` + db.Params{ + "project_id": projectId, + "policy_name": specification.Schema, + }, + ) + } else { + properties, err := json.Marshal(specification.Properties) + if err != nil { + log.Debug("Error marshalling policy document ", specification.Schema, " ", specification.Properties) + continue + } + db.BatchExec( + b, + ` insert into project.policies (policy_name, policy_property, project_id) values (:policy_name, :policy_properties, :project_id) on conflict (policy_name, project_id) do update set policy_property = excluded.policy_property, `, - db.Params{ - "policy_name": specification.Schema, - "policy_properties": properties, - "project_id": projectId, - }, - ) + db.Params{ + "policy_name": specification.Schema, + "policy_properties": properties, + "project_id": projectId, + }, + ) + } } } db.BatchSend(b) //Updating cache projectPolicies.Mu.Lock() - for _, specification := range request.UpdatedPolicies { - isEnabled := false - for _, property := range specification.Properties { - if property.Name == "enabled" { - isEnabled = property.Bool - } - } - projectId := string(specification.Project) - if !isEnabled { - policies, ok := projectPolicies.PoliciesByProject[projectId] - //If no policies are enabled for the project then just skip the deletion - if !ok { - continue + for _, updates := range filteredUpdates { + for _, specification := range updates { + isEnabled := false + for _, property := range specification.Properties { + if property.Name == "enabled" { + isEnabled = property.Bool + } } - policies.EnabledPolices[specification.Schema] = fndapi.PolicySpecification{} - } else { - policies, ok := projectPolicies.PoliciesByProject[projectId] - if !ok { - policies = &AssociatedPolicies{EnabledPolices: map[string]fndapi.PolicySpecification{}} - projectPolicies.PoliciesByProject[projectId] = policies + projectId := string(specification.Project) + if !isEnabled { + policies, ok := projectPolicies.PoliciesByProject[projectId] + //If no policies are enabled for the project then just skip the deletion + if !ok { + continue + } + policies.EnabledPolices[specification.Schema] = fndapi.PolicySpecification{} + } else { + policies, ok := projectPolicies.PoliciesByProject[projectId] + if !ok { + policies = &AssociatedPolicies{EnabledPolices: map[string]fndapi.PolicySpecification{}} + projectPolicies.PoliciesByProject[projectId] = policies + } + policies.EnabledPolices[specification.Schema] = specification } - policies.EnabledPolices[specification.Schema] = specification } } projectPolicies.Mu.Unlock() diff --git a/core2/pkg/foundation/policies/restrict_integrated_applications.yaml b/core2/pkg/foundation/policies/restrict_integrated_applications.yaml index 7e6df2365f..be27ea2512 100644 --- a/core2/pkg/foundation/policies/restrict_integrated_applications.yaml +++ b/core2/pkg/foundation/policies/restrict_integrated_applications.yaml @@ -16,7 +16,7 @@ configuration: checked then all integrated applications can be run and the allow-list is ignored. - name: "allowList" - type: TextList + type: EnumSet title: "Allow-list" description: > List of applications which are explicitly allowed. diff --git a/core2/pkg/foundation/policy_templates.go b/core2/pkg/foundation/policy_templates.go index 984dce4445..277e9365ed 100644 --- a/core2/pkg/foundation/policy_templates.go +++ b/core2/pkg/foundation/policy_templates.go @@ -2,8 +2,6 @@ package foundation import _ "embed" -// Generated by mailtpl/generate.sh. Please run the script from the mailtpl folder. - //go:embed policies/restrict_applications.yaml var restrictApplications []byte diff --git a/core2/pkg/migrations/policies.go b/core2/pkg/migrations/policies.go index 973f531e04..59c05f1a9d 100644 --- a/core2/pkg/migrations/policies.go +++ b/core2/pkg/migrations/policies.go @@ -6,18 +6,36 @@ func policiesV1() db.MigrationScript { return db.MigrationScript{ Id: "policiesV1", Execute: func(tx *db.Transaction) { - db.Exec( - tx, + statements := []string{ ` create table if not exists project.policies ( policy_name text not null, - policy_property jsonb not null, + policy_properties jsonb not null, project_id text not null references project.projects(id) on delete cascade, primary key (project_id, policy_name) ) `, - db.Params{}, - ) + ` + create or replace function project.notify_policy_change() + returns trigger as $$ + begin + perform pg_notify('policy_updates', new.project_id::text); + return new; + end; + $$ language plpgsql; + `, + ` + create trigger policy_update_trigger + after insert or update or delete + on project.policies + for each row + execute function project.notify_policy_change(); + `, + } + + for _, statement := range statements { + db.Exec(tx, statement, db.Params{}) + } }, } } diff --git a/provider-integration/shared/pkg/foundation/policies.go b/provider-integration/shared/pkg/foundation/policies.go index 2555f18495..f0a9d8a2f6 100644 --- a/provider-integration/shared/pkg/foundation/policies.go +++ b/provider-integration/shared/pkg/foundation/policies.go @@ -23,7 +23,7 @@ type PolicyProperty struct { Title string `json:"title"` Description string `json:"description"` - Options []string `json:"options,omitempty"` // Enum + Options []string `json:"options,omitempty"` // Enum, EnumSet } type PolicyPropertyType string @@ -37,6 +37,7 @@ const ( PolicyPropertyProviders PolicyPropertyType = "Providers" PolicyPropertyBool PolicyPropertyType = "Bool" PolicyPropertyTextList PolicyPropertyType = "TextList" + PolicyPropertyEnumSet PolicyPropertyType = "EnumSet" ) type PolicySpecification struct { @@ -54,7 +55,7 @@ type PolicyPropertyValue struct { Int int `json:"int,omitempty"` // Int Float float64 `json:"float,omitempty"` // Float Bool bool `json:"bool,omitempty"` // Bool - TextElements []string `json:"textElements,omitempty"` // TextList + TextElements []string `json:"textElements,omitempty"` // TextList, EnumSet } type Policy struct { From 6c01d89d3cdf0d423c0b53231ec5d77ad053ffd9 Mon Sep 17 00:00:00 2001 From: Schulz Date: Mon, 16 Feb 2026 11:01:05 +0100 Subject: [PATCH 04/23] added policy cache for accountiong and orchestrator --- core2/pkg/accounting/policy_cache.go | 44 ++++++++++++ .../pkg/accounting/provider_notifications.go | 3 + core2/pkg/orchestrator/00_module.go | 3 + core2/pkg/orchestrator/policy_cache.go | 67 +++++++++++++++++++ 4 files changed, 117 insertions(+) create mode 100644 core2/pkg/accounting/policy_cache.go create mode 100644 core2/pkg/orchestrator/policy_cache.go diff --git a/core2/pkg/accounting/policy_cache.go b/core2/pkg/accounting/policy_cache.go new file mode 100644 index 0000000000..3653e23b83 --- /dev/null +++ b/core2/pkg/accounting/policy_cache.go @@ -0,0 +1,44 @@ +package accounting + +import ( + "sync" + + "ucloud.dk/core/pkg/coreutil" + db "ucloud.dk/shared/pkg/database" + fndapi "ucloud.dk/shared/pkg/foundation" + "ucloud.dk/shared/pkg/log" +) + +// policyCache is a mapping of projectId -> map[schemaName] -> PolicySpecification +var policyCache struct { + Mu sync.RWMutex + PoliciesByProject map[string]map[string]*fndapi.PolicySpecification +} + +// policiesByProject returns mapping of [schema Name] => PolicySpecification. If no policy is cached for the project it +// will attempt to retrieve it from DB. This is also how it is populated. +func policiesByProject(projectId string) map[string]*fndapi.PolicySpecification { + projectPolicies := map[string]*fndapi.PolicySpecification{} + policyCache.Mu.Lock() + projectPolicies, ok := policyCache.PoliciesByProject[projectId] + if !ok { + log.Debug("No policies for project %v", projectId) + db.NewTx0(func(tx *db.Transaction) { + policySpecifications, policiesOk := coreutil.PolicySpecificationsRetrieveFromDatabase(tx, projectId) + if policiesOk { + policyCache.PoliciesByProject[projectId] = policySpecifications + } else { + log.Debug("No policies for project %v found in DB", projectId) + } + }) + } + policyCache.Mu.Unlock() + + return projectPolicies +} + +func updatePolicyCacheForProject(projectId string, policySpecifications map[string]*fndapi.PolicySpecification) { + policyCache.Mu.Lock() + policyCache.PoliciesByProject[projectId] = policySpecifications + policyCache.Mu.Unlock() +} diff --git a/core2/pkg/accounting/provider_notifications.go b/core2/pkg/accounting/provider_notifications.go index 99daac2fc6..f1ca35b4cf 100644 --- a/core2/pkg/accounting/provider_notifications.go +++ b/core2/pkg/accounting/provider_notifications.go @@ -81,6 +81,7 @@ func initProviderNotifications() { case walletId = <-providerWalletNotifications: walletOk = true + case projectId := <-policyUpdates: db.NewTx0(func(tx *db.Transaction) { policySpecifications, policiesOk = coreutil.PolicySpecificationsRetrieveFromDatabase(tx, projectId) @@ -148,6 +149,8 @@ func initProviderNotifications() { providerNotifications.Mu.Unlock() + updatePolicyCacheForProject(project.Id, policySpecifications) + for _, ch := range allChannels { select { case ch <- policiesForProject{policySpecifications}: diff --git a/core2/pkg/orchestrator/00_module.go b/core2/pkg/orchestrator/00_module.go index 6061347f36..03e9e4ae24 100644 --- a/core2/pkg/orchestrator/00_module.go +++ b/core2/pkg/orchestrator/00_module.go @@ -35,6 +35,9 @@ func Init() { initApiTokens() times["ApiTokens"] = t.Mark() + initPolicySubscriptions() + times["PolicySubscriptions"] = t.Mark() + // Storage //================================================================================================================== diff --git a/core2/pkg/orchestrator/policy_cache.go b/core2/pkg/orchestrator/policy_cache.go new file mode 100644 index 0000000000..e3680a48e1 --- /dev/null +++ b/core2/pkg/orchestrator/policy_cache.go @@ -0,0 +1,67 @@ +package orchestrator + +import ( + "context" + "sync" + + "ucloud.dk/core/pkg/coreutil" + db "ucloud.dk/shared/pkg/database" + fndapi "ucloud.dk/shared/pkg/foundation" + "ucloud.dk/shared/pkg/log" +) + +// policyCache is a mapping of projectId -> map[schemaName] -> PolicySpecification +var policyCache struct { + Mu sync.RWMutex + PoliciesByProject map[string]map[string]*fndapi.PolicySpecification +} + +func initPolicySubscriptions() { + go func() { + policyUpdates := db.Listen(context.Background(), "policy_updates") + + var policySpecifications map[string]*fndapi.PolicySpecification + var policiesOk bool + + for { + projectId := <-policyUpdates + + db.NewTx0(func(tx *db.Transaction) { + policySpecifications, policiesOk = coreutil.PolicySpecificationsRetrieveFromDatabase(tx, projectId) + }) + + if policiesOk { + updatePolicyCacheForProject(projectId, policySpecifications) + } + } + + }() +} + +// policiesByProject returns mapping of [schema Name] => PolicySpecification. If no policy is cached for the project it +// will attempt to retrieve it from DB. This is also how it is populated. +func policiesByProject(projectId string) map[string]*fndapi.PolicySpecification { + projectPolicies := map[string]*fndapi.PolicySpecification{} + policyCache.Mu.Lock() + projectPolicies, ok := policyCache.PoliciesByProject[projectId] + if !ok { + log.Debug("No policies for project %v", projectId) + db.NewTx0(func(tx *db.Transaction) { + policySpecifications, policiesOk := coreutil.PolicySpecificationsRetrieveFromDatabase(tx, projectId) + if policiesOk { + policyCache.PoliciesByProject[projectId] = policySpecifications + } else { + log.Debug("No policies for project %v found in DB", projectId) + } + }) + } + policyCache.Mu.Unlock() + + return projectPolicies +} + +func updatePolicyCacheForProject(projectId string, policySpecifications map[string]*fndapi.PolicySpecification) { + policyCache.Mu.Lock() + policyCache.PoliciesByProject[projectId] = policySpecifications + policyCache.Mu.Unlock() +} From 369ae1e1207848a602ebcd387898624c24e21942 Mon Sep 17 00:00:00 2001 From: Schulz Date: Mon, 16 Feb 2026 12:38:56 +0100 Subject: [PATCH 05/23] Added restriction names to api and restricted Downloads. Fixes #5310 --- core2/pkg/orchestrator/files.go | 7 +++++++ .../shared/pkg/foundation/policies.go | 20 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/core2/pkg/orchestrator/files.go b/core2/pkg/orchestrator/files.go index dc3d3ebb36..6cc9a27c03 100644 --- a/core2/pkg/orchestrator/files.go +++ b/core2/pkg/orchestrator/files.go @@ -380,6 +380,13 @@ func FilesCreateDownload( actor rpc.Actor, request fndapi.BulkRequest[fndapi.FindByStringId], ) (fndapi.BulkResponse[orcapi.FilesCreateDownloadResponse], *util.HttpError) { + if actor.Project.Present { + _, isRestricted := policiesByProject(string(actor.Project.Value))[fndapi.RestrictDownloads.String()] + if isRestricted { + return fndapi.BulkResponse[orcapi.FilesCreateDownloadResponse]{}, util.HttpErr(http.StatusForbidden, "This project does not allow downloads") + } + } + var result fndapi.BulkResponse[orcapi.FilesCreateDownloadResponse] var paths []string for _, reqItem := range request.Items { diff --git a/provider-integration/shared/pkg/foundation/policies.go b/provider-integration/shared/pkg/foundation/policies.go index f0a9d8a2f6..cfd2ea97ac 100644 --- a/provider-integration/shared/pkg/foundation/policies.go +++ b/provider-integration/shared/pkg/foundation/policies.go @@ -40,6 +40,26 @@ const ( PolicyPropertyEnumSet PolicyPropertyType = "EnumSet" ) +// Policy "Name"s. Remember to edit here when adding or editing policies +type PoliciesType string + +const ( + RestrictApplications PoliciesType = "RestrictApplications" + CutAndPaste PoliciesType = "CutAndPaste" + RestrictDownloads PoliciesType = "RestrictDownloads" + RestrictIntegratedApplications PoliciesType = "RestrictIntegratedApplications" + RestrictInternetAccess PoliciesType = "RestrictInternetAccess" + RestrictOrganizationMembers PoliciesType = "RestrictOrganizationMembers" + RestrictProviderTransfers PoliciesType = "RestrictProviderTransfers" + RestrictPublicIPs PoliciesType = "RestrictPublicIPs" + RestrictPublicLinks PoliciesType = "RestrictPublicLinks" + RestrictSourceIPRange PoliciesType = "RestrictSourceIPRange" +) + +func (t PoliciesType) String() string { + return string(t) +} + type PolicySpecification struct { Schema string `json:"schema"` Project rpc.ProjectId `json:"project"` From 8cc7b3fcd59cd363ae7b320d52532630b47df2b7 Mon Sep 17 00:00:00 2001 From: Schulz Date: Wed, 18 Feb 2026 13:46:56 +0100 Subject: [PATCH 06/23] clear up failing entiries and added check for files and publicLins --- .../pkg/accounting/provider_notifications.go | 5 + core2/pkg/coreutil/00_module.go | 22 ++-- core2/pkg/foundation/policies.go | 10 +- core2/pkg/orchestrator/ingress.go | 6 + core2/pkg/orchestrator/policy_cache.go | 6 + .../webclient/app/UCloud/FilesApi.tsx | 108 ++++++++++++------ 6 files changed, 110 insertions(+), 47 deletions(-) diff --git a/core2/pkg/accounting/provider_notifications.go b/core2/pkg/accounting/provider_notifications.go index f1ca35b4cf..303a27ca62 100644 --- a/core2/pkg/accounting/provider_notifications.go +++ b/core2/pkg/accounting/provider_notifications.go @@ -50,6 +50,11 @@ func retrieveRelevantProviders(projectId string) map[string]util.Empty { func initProviderNotifications() { providerNotifications.ProjectChannelsByProvider = map[string]map[string]chan *fndapi.Project{} providerNotifications.WalletsByProvider = map[string]map[string]chan *accapi.WalletV2{} + providerNotifications.PoliciesByProvider = map[string]map[string]chan policiesForProject{} + + policyCache.Mu.Lock() + policyCache.PoliciesByProject = make(map[string]map[string]*fndapi.PolicySpecification) + policyCache.Mu.Unlock() go func() { // NOTE(Dan): These two channels receive events from database triggers set on the relevant insert/update/delete diff --git a/core2/pkg/coreutil/00_module.go b/core2/pkg/coreutil/00_module.go index 570f711e48..742edc122e 100644 --- a/core2/pkg/coreutil/00_module.go +++ b/core2/pkg/coreutil/00_module.go @@ -171,28 +171,34 @@ func ProjectRetrieveFromDatabaseViaGroupId(tx *db.Transaction, groupId string) ( func PolicySpecificationsRetrieveFromDatabase(tx *db.Transaction, projectId string) (map[string]*fndapi.PolicySpecification, bool) { rows := db.Select[struct { - ProjectId string - PolicyName string - PolicyProperty string + ProjectId string + PolicyName string + PolicyProperties string }]( tx, ` - select * + select project_id, policy_name, policy_properties from project.policies - where projet_id = :id + where project_id = :id `, db.Params{ "id": projectId, }, ) - var policies map[string]*fndapi.PolicySpecification + var policies = make(map[string]*fndapi.PolicySpecification) for _, row := range rows { - specification := fndapi.PolicySpecification{} - err := json.Unmarshal([]byte(row.PolicyProperty), &specification) + properties := []fndapi.PolicyPropertyValue{} + err := json.Unmarshal([]byte(row.PolicyProperties), &properties) if err != nil { log.Debug("Error unmarshalling policy properties on update") return nil, false } + specification := fndapi.PolicySpecification{ + Schema: row.PolicyName, + Project: rpc.ProjectId(row.ProjectId), + Properties: properties, + } + policies[specification.Schema] = &specification } return policies, true diff --git a/core2/pkg/foundation/policies.go b/core2/pkg/foundation/policies.go index fa055bdbde..fad6099a70 100644 --- a/core2/pkg/foundation/policies.go +++ b/core2/pkg/foundation/policies.go @@ -86,11 +86,17 @@ func loadProjectPoliciesFromDB() { if !ok { policies = &AssociatedPolicies{EnabledPolices: map[string]fndapi.PolicySpecification{}} } - specification := fndapi.PolicySpecification{} - err := json.Unmarshal([]byte(row.PolicyProperties), &specification) + properties := []fndapi.PolicyPropertyValue{} + err := json.Unmarshal([]byte(row.PolicyProperties), &properties) if err != nil { log.Fatal("Error loading policy document %v : %v", row.PolicyProperties, err) } + specification := fndapi.PolicySpecification{ + Schema: row.PolicyName, + Project: rpc.ProjectId(projectId), + Properties: properties, + } + policies.EnabledPolices[row.PolicyName] = specification } diff --git a/core2/pkg/orchestrator/ingress.go b/core2/pkg/orchestrator/ingress.go index 41e411094a..8d4787a117 100644 --- a/core2/pkg/orchestrator/ingress.go +++ b/core2/pkg/orchestrator/ingress.go @@ -79,6 +79,12 @@ func initIngresses() { hostnamePartRegex := regexp.MustCompile(`^[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?$`) orcapi.IngressesCreate.Handler(func(info rpc.RequestInfo, request fndapi.BulkRequest[orcapi.IngressSpecification]) (fndapi.BulkResponse[fndapi.FindByStringId], *util.HttpError) { + if info.Actor.Project.Present { + _, restricted := policiesByProject(string(info.Actor.Project.Value))[fndapi.RestrictPublicLinks.String()] + if restricted { + return fndapi.BulkResponse[fndapi.FindByStringId]{}, util.HttpErr(http.StatusForbidden, "Project does not allow creation of public links") + } + } var result []fndapi.FindByStringId for _, item := range request.Items { supp, ok := SupportByProduct[orcapi.IngressSupport](ingressType, item.Product) diff --git a/core2/pkg/orchestrator/policy_cache.go b/core2/pkg/orchestrator/policy_cache.go index e3680a48e1..dc45f80ceb 100644 --- a/core2/pkg/orchestrator/policy_cache.go +++ b/core2/pkg/orchestrator/policy_cache.go @@ -17,6 +17,11 @@ var policyCache struct { } func initPolicySubscriptions() { + + policyCache.Mu.Lock() + policyCache.PoliciesByProject = make(map[string]map[string]*fndapi.PolicySpecification) + policyCache.Mu.Unlock() + go func() { policyUpdates := db.Listen(context.Background(), "policy_updates") @@ -50,6 +55,7 @@ func policiesByProject(projectId string) map[string]*fndapi.PolicySpecification policySpecifications, policiesOk := coreutil.PolicySpecificationsRetrieveFromDatabase(tx, projectId) if policiesOk { policyCache.PoliciesByProject[projectId] = policySpecifications + projectPolicies = policySpecifications } else { log.Debug("No policies for project %v found in DB", projectId) } diff --git a/frontend-web/webclient/app/UCloud/FilesApi.tsx b/frontend-web/webclient/app/UCloud/FilesApi.tsx index 2015c12bb2..1469fdf50b 100644 --- a/frontend-web/webclient/app/UCloud/FilesApi.tsx +++ b/frontend-web/webclient/app/UCloud/FilesApi.tsx @@ -9,7 +9,21 @@ import { } from "@/UCloud/ResourceApi"; import {BulkRequest, BulkResponse, PageV2} from "@/UCloud/index"; import FileCollectionsApi, {FileCollection, FileCollectionSupport} from "@/UCloud/FileCollectionsApi"; -import {Box, Button, Card, ExternalLink, Flex, FtIcon, Icon, MainContainer, Markdown, Select, Text, TextArea, Truncate} from "@/ui-components"; +import { + Box, + Button, + Card, + ExternalLink, + Flex, + FtIcon, + Icon, + MainContainer, + Markdown, + Select, + Text, + TextArea, + Truncate +} from "@/ui-components"; import * as React from "react"; import {useCallback, useEffect, useMemo, useState} from "react"; import {fileName, getParentPath, readableUnixMode, sizeToString} from "@/Utilities/FileUtilities"; @@ -20,7 +34,7 @@ import { errorMessageOrDefault, extensionFromPath, ExtensionType, - extensionType, + extensionType, extractErrorMessage, inDevEnvironment, onDevSite, prettierString, @@ -32,7 +46,7 @@ import * as Heading from "@/ui-components/Heading"; import {Operation, ShortcutKey} from "@/ui-components/Operation"; import {dialogStore} from "@/Dialog/DialogStore"; import {ItemRenderer} from "@/ui-components/Browse"; -import {PrettyFilePath, prettyFilePath, usePrettyFilePath} from "@/Files/FilePath"; +import {prettyFilePath, usePrettyFilePath} from "@/Files/FilePath"; import {OpenWithBrowser} from "@/Applications/OpenWith"; import {addStandardDialog, addStandardInputDialog} from "@/UtilityComponents"; import {ProductStorage} from "@/Accounting"; @@ -201,8 +215,7 @@ class FilesApi extends ResourceApi & ExtraFileCallbacks> = { - }; + renderer: ItemRenderer & ExtraFileCallbacks> = {}; private defaultRetrieveFlags: Partial = { includeMetadata: true, @@ -213,7 +226,7 @@ class FilesApi extends ResourceApi { - const {id} = useParams<{id?: string}>(); + const {id} = useParams<{ id?: string }>(); const [fileData, fetchFile] = useCloudAPI({noop: true}, null); @@ -230,10 +243,11 @@ class FilesApi extends ResourceApiMissing file id.} />; - if (!file) return File not found. Click to go to drives.} />; + if (!id) return Missing file id.}/>; + if (!file) return File not found. Click to go to drives.}/>; - return + return } public retrieveOperations(): Operation & ExtraFileCallbacks>[] { @@ -318,7 +332,7 @@ class FilesApi extends ResourceApi selected.length === 1 && cb.collection != null, onClick: (selected) => { dialogStore.addDialog( - , + , doNothing, true, this.fileSelectorModalStyle, @@ -403,7 +417,7 @@ class FilesApi extends ResourceApi, + }}/>, doNothing, true, this.fileSelectorModalStyle @@ -739,7 +753,10 @@ class FilesApi extends ResourceApi ({id})), )) - ); + ).catch(err => { + snackbarStore.addFailure(extractErrorMessage(err), false); + }); + ; const responses = result?.responses ?? []; for (const {endpoint} of responses) { @@ -783,7 +800,7 @@ class FilesApi extends ResourceApi, + }}/>, doNothing, true, this.fileSelectorModalStyle @@ -823,7 +840,7 @@ class FilesApi extends ResourceApi, + }}/>, doNothing, true, this.fileSelectorModalStyle @@ -841,8 +858,9 @@ function handleSyncthingWarning(files: UFile[], cb: ExtraFileCallbacks, op: () = {operationText} the folder(s) will break Syncthing synchronization for this folder. {(["Moving", "Renaming"] as typeof operationText[]).includes(operationText) ?
-
- To learn how to move a folder or rename a folder with Syncthing, click here. +
+ To learn how to move a folder or rename a folder with Syncthing, click here.
: null} , onConfirm: op, @@ -1083,10 +1101,10 @@ export async function addFileSensitivityDialog(file: UFile, invokeCommand: Invok dialogStore.addDialog( <> - Sensitive files not supported + Sensitive files not supported

- This provider () has declared + This provider () has declared that they do not support sensitive data. This means that you cannot/should not:

    @@ -1114,7 +1132,7 @@ export async function addFileSensitivityDialog(file: UFile, invokeCommand: Invok } dialogStore.addDialog(, () => undefined, true); + onUpdated={onUpdated}/>, () => undefined, true); } const api = new FilesApi(); @@ -1168,7 +1186,7 @@ export function FilePreview({initialFile}: { }) }, []); - const mediaFileMetadata: null | {type: ExtensionType, data: string, error: string | null} = useMemo(() => { + const mediaFileMetadata: null | { type: ExtensionType, data: string, error: string | null } = useMemo(() => { let [file, contentBuffer] = openFile; const isSvg = extensionFromPath(file) === "svg"; @@ -1259,15 +1277,17 @@ export function FilePreview({initialFile}: { return null; case "image": // Note(Jonas): extensions like .HEIC will fall back to just showing the alt. - return {elementKey} + return {elementKey} case "audio": - return