Support multiple remote root servers

cmd/tesseract/README.md

@@ -39,10 +39,11 @@ mechanisms to add roots, and one to reject roots:
 
 1. Manually, via a PEM file. Use the `root_pem_file` flag to configure its path.
 Roots from this file are read once at startup, and remain trusted thereafter.
-2. Automatically, from a remote endpoint like [CCADB's](https://ccadb.my.salesforce-sites.com/ccadb/RootCACertificatesIncludedByRSReportCSV).
-The URL of that endpoint is set via `roots_remote_fetch_url`. Roots are first
+2. Automatically, from one or more remote endpoints like [CCADB's](https://ccadb.my.salesforce-sites.com/ccadb/RootCACertificatesIncludedByRSReportCSV).
+The URL of each endpoint is set via `roots_remote_fetch_url`. This flag
+accepts a single URL, and can be specified multiple times. Roots are first
 fetched at startup, and then every `roots_remote_fetch_interval`. Each time
-roots are fetched from this remote endpoint, newly found roots become trusted,
+roots are fetched from these remote endpoints, newly found roots become trusted,
 if not rejected with `roots_reject_fingerprints`.
 Newly found roots are backed up in the log's storage, under `roots/`. Roots are
 never removed from this directory. Roots in the `roots/` directory are loaded

cmd/tesseract/aws/main.go

@@ -51,6 +51,7 @@ func init() {
 	flag.Float64Var(&dedupRL, "rate_limit_dedup", 100, "Rate limit for resolving duplicate submissions, in requests per second - i.e. duplicate requests for already integrated entries, which need to be fetched from the log storage by TesseraCT to extract their timestamp. When 0, all duplicate submissions are rejected. When negative, no rate limit is applied.")
 	// DEPRECATED: will be removed shortly
 	flag.Float64Var(&dedupRL, "pushback_max_dedupe_in_flight", 100, "DEPRECATED: use rate_limit_dedup. Maximum number of in-flight duplicate add requests - i.e. the number of requests matching entries that have already been integrated, but need to be fetched by the client to retrieve their timestamp. When 0, duplicate entries are always pushed back.")
+	flag.Var(&rootsRemoteFetchURLs, "roots_remote_fetch_url", "URL to fetch additional trusted roots from. May be specified multiple times.")
 }
 
 // Global flags that affect all log instances.
@@ -66,7 +67,7 @@ var (
 	origin                   = flag.String("origin", "", "Origin of the log, for checkpoints. This MUST match the log's submission prefix as per https://c2sp.org/static-ct-api.")
 	pathPrefix               = flag.String("path_prefix", "", "Prefix to use on endpoints URL paths: HOST:PATH_PREFIX/ct/v1/ENDPOINT.")
 	rootsPemFile             = flag.String("roots_pem_file", "", "Path to the file containing root certificates that are acceptable to the log.")
-	rootsRemoteFetchURL      = flag.String("roots_remote_fetch_url", "https://ccadb.my.salesforce-sites.com/ccadb/RootCACertificatesIncludedByRSReportCSV", "URL to fetch additional trusted roots from. Leave empty to disable.")
+	rootsRemoteFetchURLs     multiStringFlag
 	rootsRemoteFetchInterval = flag.Duration("roots_remote_fetch_interval", time.Duration(0), "Interval between two fetches from roots_fetch_url, e.g. \"1h\". Set to \"0s\" to disable.")
 	rejectExpired            = flag.Bool("reject_expired", false, "If true then the certificate validity period will be checked against the current time during the validation of submissions. This will cause expired certificates to be rejected.")
 	rejectUnexpired          = flag.Bool("reject_unexpired", false, "If true then TesseraCT rejects certificates that are either currently valid or not yet valid.")
@@ -140,9 +141,13 @@ func main() {
 		klog.Exitf("failed to initialize S3 backup storage for remotely fetched roots: %v", err)
 	}
 
+	if len(rootsRemoteFetchURLs) == 0 {
+		rootsRemoteFetchURLs = []string{"https://ccadb.my.salesforce-sites.com/ccadb/RootCACertificatesIncludedByRSReportCSV"}
+	}
+
 	chainValidationConfig := tesseract.ChainValidationConfig{
 		RootsPEMFile:             *rootsPemFile,
-		RootsRemoteFetchURL:      *rootsRemoteFetchURL,
+		RootsRemoteFetchURLs:     rootsRemoteFetchURLs,
 		RootsRemoteFetchInterval: *rootsRemoteFetchInterval,
 		RootsRemoteFetchBackup:   fetchedRootsBackupStorage,
 		RejectExpired:            *rejectExpired,

cmd/tesseract/gcp/main.go

@@ -58,6 +58,7 @@ func init() {
 	flag.Float64Var(&dedupRL, "rate_limit_dedup", 100, "Rate limit for resolving duplicate submissions, in requests per second - i.e. duplicate requests for already integrated entries, which need to be fetched from the log storage by TesseraCT to extract their timestamp. When 0, all duplicate submissions are rejected. When negative, no rate limit is applied.")
 	// DEPRECATED: will be removed shortly
 	flag.Float64Var(&dedupRL, "pushback_max_dedupe_in_flight", 100, "DEPRECATED: use rate_limit_dedup. Maximum number of in-flight duplicate add requests - i.e. the number of requests matching entries that have already been integrated, but need to be fetched by the client to retrieve their timestamp. When 0, duplicate entries are always pushed back.")
+	flag.Var(&rootsRemoteFetchURLs, "roots_remote_fetch_url", "URL to fetch additional trusted roots from. May be specified multiple times.")
 }
 
 // Global flags that affect all log instances.
@@ -73,7 +74,7 @@ var (
 	origin                   = flag.String("origin", "", "Origin of the log, for checkpoints. This MUST match the log's submission prefix as per https://c2sp.org/static-ct-api.")
 	pathPrefix               = flag.String("path_prefix", "", "Prefix to use on endpoints URL paths: HOST:PATH_PREFIX/ct/v1/ENDPOINT.")
 	rootsPemFile             = flag.String("roots_pem_file", "", "Path to the file containing root certificates that are acceptable to the log.")
-	rootsRemoteFetchURL      = flag.String("roots_remote_fetch_url", "https://ccadb.my.salesforce-sites.com/ccadb/RootCACertificatesIncludedByRSReportCSV", "URL to fetch additional trusted roots from.")
+	rootsRemoteFetchURLs     multiStringFlag
 	rootsRemoteFetchInterval = flag.Duration("roots_remote_fetch_interval", time.Duration(0), "Interval between two fetches from roots_fetch_url, e.g. \"1h\".")
 	rootsRejectFingerprints  multiStringFlag
 	rejectExpired            = flag.Bool("reject_expired", false, "If true then the certificate validity period will be checked against the current time during the validation of submissions. This will cause expired certificates to be rejected.")
@@ -173,9 +174,13 @@ func main() {
 		klog.Exitf("failed to initialize GCS backup storage for remotely fetched roots: %v", err)
 	}
 
+	if len(rootsRemoteFetchURLs) == 0 {
+		rootsRemoteFetchURLs = []string{"https://ccadb.my.salesforce-sites.com/ccadb/RootCACertificatesIncludedByRSReportCSV"}
+	}
+
 	chainValidationConfig := tesseract.ChainValidationConfig{
 		RootsPEMFile:             *rootsPemFile,
-		RootsRemoteFetchURL:      *rootsRemoteFetchURL,
+		RootsRemoteFetchURLs:     rootsRemoteFetchURLs,
 		RootsRemoteFetchInterval: *rootsRemoteFetchInterval,
 		RootsRemoteFetchBackup:   fetchedRootsBackupStorage,
 		RejectExpired:            *rejectExpired,

cmd/tesseract/posix/main.go

@@ -53,6 +53,7 @@ func init() {
 	flag.Float64Var(&dedupRL, "rate_limit_dedup", 100, "Rate limit for resolving duplicate submissions, in requests per second - i.e. duplicate requests for already integrated entries, which need to be fetched from the log storage by TesseraCT to extract their timestamp. When 0, all duplicate submissions are rejected. When negative, no rate limit is applied.")
 	// DEPRECATED: will be removed shortly
 	flag.Float64Var(&dedupRL, "pushback_max_dedupe_in_flight", 100, "DEPRECATED: use rate_limit_dedup. Maximum number of in-flight duplicate add requests - i.e. the number of requests matching entries that have already been integrated, but need to be fetched by the client to retrieve their timestamp. When 0, duplicate entries are always pushed back.")
+	flag.Var(&rootsRemoteFetchURLs, "roots_remote_fetch_url", "URL to fetch additional trusted roots from. May be specified multiple times.")
 }
 
 // Global flags that affect all log instances.
@@ -61,6 +62,7 @@ var (
 	notAfterLimit           timestampFlag
 	additionalSigners       multiStringFlag
 	rootsRejectFingerprints multiStringFlag
+	rootsRemoteFetchURLs    multiStringFlag
 	dedupRL                 float64
 
 	// Functionality flags
@@ -69,7 +71,6 @@ var (
 	origin                   = flag.String("origin", "", "Origin of the log, for checkpoints. This MUST match the log's submission prefix as per https://c2sp.org/static-ct-api.")
 	pathPrefix               = flag.String("path_prefix", "", "Prefix to use on endpoints URL paths: HOST:PATH_PREFIX/ct/v1/ENDPOINT.")
 	rootsPemFile             = flag.String("roots_pem_file", "", "Path to the file containing root certificates that are acceptable to the log.")
-	rootsRemoteFetchURL      = flag.String("roots_remote_fetch_url", "https://ccadb.my.salesforce-sites.com/ccadb/RootCACertificatesIncludedByRSReportCSV", "URL to fetch additional trusted roots from. Leave empty to disable.")
 	rootsRemoteFetchInterval = flag.Duration("roots_remote_fetch_interval", time.Duration(0), "Interval between two fetches from roots_fetch_url, e.g. \"1h\". Set to \"0s\" to disable.")
 	rejectExpired            = flag.Bool("reject_expired", false, "If true then the certificate validity period will be checked against the current time during the validation of submissions. This will cause expired certificates to be rejected.")
 	rejectUnexpired          = flag.Bool("reject_unexpired", false, "If true then TesseraCT rejects certificates that are either currently valid or not yet valid.")
@@ -114,9 +115,13 @@ func main() {
 		klog.Exitf("failed to initialize POSIX backup storage for remotely fetched roots: %v", err)
 	}
 
+	if len(rootsRemoteFetchURLs) == 0 {
+		rootsRemoteFetchURLs = []string{"https://ccadb.my.salesforce-sites.com/ccadb/RootCACertificatesIncludedByRSReportCSV"}
+	}
+
 	chainValidationConfig := tesseract.ChainValidationConfig{
 		RootsPEMFile:             *rootsPemFile,
-		RootsRemoteFetchURL:      *rootsRemoteFetchURL,
+		RootsRemoteFetchURLs:     rootsRemoteFetchURLs,
 		RootsRemoteFetchInterval: *rootsRemoteFetchInterval,
 		RootsRemoteFetchBackup:   fetchedRootsBackupStorage,
 		RejectExpired:            *rejectExpired,

ctlog.go

@@ -41,8 +41,8 @@ type ChainValidationConfig struct {
 	// are acceptable to the log. The certs are served through get-roots
 	// endpoint.
 	RootsPEMFile string
-	// RootsRemoteFetchURL configures an endpoint to fetch additional roots from.
-	RootsRemoteFetchURL string
+	// RootsRemoteFetchURLs configures an endpoint to fetch additional roots from.
+	RootsRemoteFetchURLs []string
 	// RootsRemoteFetchInterval configures the frequency at which to fetch
 	// roots from RootsRemoteEndpoint.
 	RootsRemoteFetchInterval time.Duration
@@ -151,24 +151,24 @@ func newChainValidator(ctx context.Context, cfg ChainValidationConfig) (ct.Chain
 		klog.Infof("Fetched %d roots, parsed %d, and loaded %d new ones from remote root backup storage", len(certs), parsed, added)
 	}
 
-	if cfg.RootsRemoteFetchInterval > 0 && cfg.RootsRemoteFetchURL != "" {
-		fetchAndAppendRemoteRoots := func() {
-			rr, err := ccadb.Fetch(ctx, cfg.RootsRemoteFetchURL, []string{ccadb.ColPEM})
+	if cfg.RootsRemoteFetchInterval > 0 && len(cfg.RootsRemoteFetchURLs) > 0 {
+		fetchAndAppendRemoteRoots := func(url string) {
+			rr, err := ccadb.Fetch(ctx, url, []string{ccadb.ColPEM})
 			if err != nil {
-				klog.Errorf("Couldn't fetch roots from %q: %s", cfg.RootsRemoteFetchURL, err)
+				klog.Errorf("Couldn't fetch roots from %q: %s", url, err)
 				return
 			}
 			pems := make([][]byte, 0, len(rr))
 			for _, r := range rr {
 				if len(r) < 1 {
-					klog.Errorf("Couldn't parse root from %q: empty row", cfg.RootsRemoteFetchURL)
+					klog.Errorf("Couldn't parse root from %q: empty row", url)
 					continue
 				}
 				pems = append(pems, r[0])
 				if cfg.RootsRemoteFetchBackup != nil {
 					block, _ := pem.Decode(r[0])
 					if block == nil {
-						klog.Errorf("Failed to decode PEM block in data fetched from %q", cfg.RootsRemoteFetchURL)
+						klog.Errorf("Failed to decode PEM block in data fetched from %q", url)
 						continue
 					}
 					sha := sha256.Sum256(block.Bytes)
@@ -180,10 +180,12 @@ func newChainValidator(ctx context.Context, cfg ChainValidationConfig) (ct.Chain
 				}
 			}
 			parsed, added := roots.AppendCertsFromPEMs(pems...)
-			klog.Infof("Fetched %d roots, parsed %d, and loaded %d new ones from %q", len(pems), parsed, added, cfg.RootsRemoteFetchURL)
+			klog.Infof("Fetched %d roots, parsed %d, and loaded %d new ones from %q", len(pems), parsed, added, url)
 		}
 
-		fetchAndAppendRemoteRoots()
+		for _, url := range cfg.RootsRemoteFetchURLs {
+			fetchAndAppendRemoteRoots(url)
+		}
 
 		go func() {
 			ticker := time.NewTicker(cfg.RootsRemoteFetchInterval)
@@ -193,7 +195,9 @@ func newChainValidator(ctx context.Context, cfg ChainValidationConfig) (ct.Chain
 				case <-ctx.Done():
 					return
 				case <-ticker.C:
-					fetchAndAppendRemoteRoots()
+					for _, url := range cfg.RootsRemoteFetchURLs {
+						fetchAndAppendRemoteRoots(url)
+					}
 				}
 			}
 		}()

ctlog_test.go

@@ -219,6 +219,7 @@ func TestNewChainValidatorRootsRemoteFetch(t *testing.T) {
 		desc       string
 		cvCfg      ChainValidationConfig
 		rsps       []ccadbRsp
+		rsps2      []ccadbRsp
 		wantNRoots int
 	}{
 		{
@@ -318,12 +319,44 @@ func TestNewChainValidatorRootsRemoteFetch(t *testing.T) {
 			},
 			wantNRoots: 2,
 		},
+		{
+			desc: "two-remote-endpoints",
+			cvCfg: ChainValidationConfig{
+				RootsPEMFile:             "./internal/testdata/fake-ca.cert",
+				RootsRemoteFetchInterval: fetchInterval,
+			},
+			rsps: []ccadbRsp{
+				{
+					code: 200,
+					crts: []string{
+						testdata.CACertPEM,
+					},
+				},
+			},
+			rsps2: []ccadbRsp{
+				{
+					code: 200,
+					crts: []string{
+						testdata.CACertPEM, // Same root, shouldn't duplicate
+						testdata.FakeCACertPEM, // New root.
+					},
+				},
+			},
+			wantNRoots: 3, // one from file, one from server 1, one from server 2
+		},
 	} {
 		t.Run(tc.desc, func(t *testing.T) {
 			ts := newCCADBTestServer(t, tc.rsps)
 			ts.Start()
 			defer ts.Close()
-			tc.cvCfg.RootsRemoteFetchURL = ts.URL
+			urls := []string{ts.URL}
+			if len(tc.rsps2) > 0 {
+				ts2 := newCCADBTestServer(t, tc.rsps2)
+				ts2.Start()
+				defer ts2.Close()
+				urls = append(urls, ts2.URL)
+			}
+			tc.cvCfg.RootsRemoteFetchURLs = urls
 			cv, err := newChainValidator(t.Context(), tc.cvCfg)
 			if err == nil && cv == nil {
 				t.Error("err and ValidatedLogConfig are both nil")
@@ -466,7 +499,7 @@ func TestNewChainValidatorRootsFiltering(t *testing.T) {
 			ts := newCCADBTestServer(t, tc.rsps)
 			ts.Start()
 			defer ts.Close()
-			tc.cvCfg.RootsRemoteFetchURL = ts.URL
+			tc.cvCfg.RootsRemoteFetchURLs = []string{ts.URL}
 			tc.cvCfg.RootsRemoteFetchBackup = &memoryRootsStorage{m: make(map[string][]byte)}
 			if err := tc.cvCfg.RootsRemoteFetchBackup.AddIfNotExist(t.Context(), tc.backupRoots); err != nil {
 				t.Fatalf("Can't initialize root backup storage: %v", err)

deployment/live/gcp/static-ct-ci/logs/ci/terragrunt.hcl

@@ -12,6 +12,8 @@ locals {
   log_private_key_secret_name = "projects/223810646869/secrets/${local.safe_origin}-log-secret/versions/1"
   server_docker_image         = "${include.root.locals.location}-docker.pkg.dev/${include.root.locals.project_id}/docker-${local.env}/conformance-gcp:latest"
   ephemeral                   = true
+  rootsRemoteFetchURL         = ["https://ccadb.my.salesforce-sites.com/ccadb/RootCACertificatesIncludedByRSReportCSV", "https://ccadb.my.salesforce-sites.com/ccadb/RootCACertificatesInclusionReportCSV"]
+  rootsRemoteFetchInterval    = "10m"
 }
 
 include "root" {

deployment/modules/aws/tesseract/conformance/main.tf

@@ -176,7 +176,7 @@ resource "aws_ecs_task_definition" "conformance" {
       "--antispam_db_name=${var.antispam_database_name}",
       "--inmemory_antispam_cache_size=256k",
       "--enable_publication_awaiter=true",
-      "--roots_remote_fetch_url=${var.roots_remote_fetch_url}",
+      formatlist("--roots_remote_fetch_url=%s", var.roots_remote_fetch_url),
       "--roots_remote_fetch_interval=${var.roots_remote_fetch_interval}",
       "-v=2",
     ]),

deployment/modules/aws/tesseract/conformance/variables.tf

@@ -55,9 +55,9 @@ variable "antispam_database_name" {
 }
 
 variable "roots_remote_fetch_url" {
-  description = "URL to fetch trusted roots from."
-  type        = string
-  default     = "https://ccadb.my.salesforce-sites.com/ccadb/RootCACertificatesIncludedByRSReportCSV"
+  description = "URLs to fetch trusted roots from."
+  type        = list(string)
+  default     = ["https://ccadb.my.salesforce-sites.com/ccadb/RootCACertificatesIncludedByRSReportCSV"]
 }
 
 variable "roots_remote_fetch_interval" {

deployment/modules/gcp/cloudrun/main.tf

@@ -60,7 +60,7 @@ resource "google_cloud_run_v2_service" "default" {
         "--trace_fraction=${var.trace_fraction}",
         "--batch_max_size=${var.batch_max_size}",
         "--batch_max_age=${var.batch_max_age}",
-        "--roots_remote_fetch_url=${var.roots_remote_fetch_url}",
+        formatlist("--roots_remote_fetch_url=%s", var.roots_remote_fetch_url),
         "--roots_remote_fetch_interval=${var.roots_remote_fetch_interval}",
         "--gcs_use_grpc=true",
       ])

deployment/modules/gcp/cloudrun/variables.tf

@@ -84,9 +84,9 @@ variable "batch_max_age" {
 }
 
 variable "roots_remote_fetch_url" {
-  description = "URL to fetch trusted roots from."
-  type        = string
-  default     = "https://ccadb.my.salesforce-sites.com/ccadb/RootCACertificatesIncludedByRSReportCSV"
+  description = "URLs to fetch trusted roots from."
+  type        = list(string)
+  default     = ["https://ccadb.my.salesforce-sites.com/ccadb/RootCACertificatesIncludedByRSReportCSV"]
 }
 
 variable "roots_remote_fetch_interval" {

deployment/modules/gcp/gce/tesseract/main.tf

@@ -62,7 +62,7 @@ locals {
     "-accept_sha1_signing_algorithms=true",
     "-rate_limit_old_not_before=${var.rate_limit_old_not_before}",
     "-rate_limit_dedup=${var.rate_limit_dedup}",
-    "-roots_remote_fetch_url=${var.roots_remote_fetch_url}",
+    length(var.roots_remote_fetch_url) == 0 ? "" : join(" ", formatlist("-roots_remote_fetch_url=%s", var.roots_remote_fetch_url)),
     "-roots_remote_fetch_interval=${var.roots_remote_fetch_interval}",
     length(var.roots_reject_fingerprints) == 0 ? "" : join(" ", formatlist("-roots_reject_fingerprints=%s", var.roots_reject_fingerprints)),
     var.witness_policy == "" ? "" : "-witness_policy_file=${local.witness_policy_file}",

deployment/modules/gcp/gce/tesseract/variables.tf

@@ -141,9 +141,9 @@ variable "accepted_roots" {
 }
 
 variable "roots_remote_fetch_url" {
-  description = "URL to fetch trusted roots from."
-  type        = string
-  default     = "https://ccadb.my.salesforce-sites.com/ccadb/RootCACertificatesIncludedByRSReportCSV"
+  description = "URLs to fetch trusted roots from."
+  type        = list(string)
+  default     = ["https://ccadb.my.salesforce-sites.com/ccadb/RootCACertificatesIncludedByRSReportCSV"]
 }
 
 variable "roots_remote_fetch_interval" {

deployment/modules/gcp/tesseract/cloudrun/variables.tf

@@ -86,9 +86,9 @@ variable "log_private_key_secret_name" {
 }
 
 variable "roots_remote_fetch_url" {
-  description = "URL to fetch trusted roots from."
-  type        = string
-  default     = "https://ccadb.my.salesforce-sites.com/ccadb/RootCACertificatesIncludedByRSReportCSV"
+  description = "URLs to fetch trusted roots from."
+  type        = list(string)
+  default     = ["https://ccadb.my.salesforce-sites.com/ccadb/RootCACertificatesIncludedByRSReportCSV"]
 }
 
 variable "roots_remote_fetch_interval" {

deployment/modules/gcp/tesseract/gce/variables.tf

@@ -151,9 +151,9 @@ variable "accepted_roots" {
 }
 
 variable "roots_remote_fetch_url" {
-  description = "URL to fetch trusted roots from."
-  type        = string
-  default     = "https://ccadb.my.salesforce-sites.com/ccadb/RootCACertificatesIncludedByRSReportCSV"
+  description = "URLs to fetch trusted roots from."
+  type        = list(string)
+  default     = ["https://ccadb.my.salesforce-sites.com/ccadb/RootCACertificatesIncludedByRSReportCSV"]
 }
 
 variable "roots_remote_fetch_interval" {