From 3ee83403e58bbfea733dd886166a16036e27173d Mon Sep 17 00:00:00 2001 From: Aaron Son Date: Thu, 7 May 2026 16:33:02 -0700 Subject: [PATCH 1/3] go/utils/cigotests: add unified Go test runner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A small Go program that reads an embedded config (config.yaml) and drives the test matrix for the new ci-go-tests workflow. Three subcommands: list -> JSON {os, shard} cells plan --shard X --os Y --event Z -> describe-only (no exec) run --shard X --os Y --event Z -> exec go test Each shard declares packages (with optional ./... + exclude), timeout, env, and race_on conditions over (os, event). The list command emits one matrix cell per (os, shard) the workflow should run, with shards able to narrow OS coverage via runs_on. Adding, splitting, or retuning a shard is a single-file edit — the workflow re-discovers cells on every run. This commit is the runner only; the workflow that invokes it is in the next commit. --- go/utils/cigotests/config.go | 187 +++++++++++++++++++++++++++++ go/utils/cigotests/config.yaml | 63 ++++++++++ go/utils/cigotests/config_test.go | 146 +++++++++++++++++++++++ go/utils/cigotests/main.go | 189 ++++++++++++++++++++++++++++++ 4 files changed, 585 insertions(+) create mode 100644 go/utils/cigotests/config.go create mode 100644 go/utils/cigotests/config.yaml create mode 100644 go/utils/cigotests/config_test.go create mode 100644 go/utils/cigotests/main.go diff --git a/go/utils/cigotests/config.go b/go/utils/cigotests/config.go new file mode 100644 index 00000000000..e65e6baf604 --- /dev/null +++ b/go/utils/cigotests/config.go @@ -0,0 +1,187 @@ +// Copyright 2026 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + _ "embed" + "fmt" + "os/exec" + "slices" + "strings" + + "gopkg.in/yaml.v3" +) + +//go:embed config.yaml +var configBytes []byte + +type Config struct { + DefaultOS []string `yaml:"default_os"` + Shards []Shard `yaml:"shards"` +} + +type Shard struct { + Name string `yaml:"name"` + Packages []string `yaml:"packages"` + Exclude []string `yaml:"exclude,omitempty"` + Timeout string `yaml:"timeout,omitempty"` + Env map[string]string `yaml:"env,omitempty"` + EnvWithRace map[string]string `yaml:"env_with_race,omitempty"` + RunsOn *Condition `yaml:"runs_on,omitempty"` + RaceOn *Condition `yaml:"race_on,omitempty"` +} + +// Condition is a predicate over (os, event). An empty list for either +// field means "any value." A nil Condition is true. +type Condition struct { + OS []string `yaml:"os,omitempty"` + Events []string `yaml:"events,omitempty"` +} + +// Combo is one cell of the GitHub Actions matrix. +type Combo struct { + OS string `json:"os"` + Shard string `json:"shard"` +} + +func LoadConfig() (*Config, error) { + var cfg Config + if err := yaml.Unmarshal(configBytes, &cfg); err != nil { + return nil, fmt.Errorf("parse config: %w", err) + } + if len(cfg.DefaultOS) == 0 { + return nil, fmt.Errorf("config: default_os must be non-empty") + } + seen := map[string]bool{} + for _, s := range cfg.Shards { + if s.Name == "" { + return nil, fmt.Errorf("config: shard with empty name") + } + if seen[s.Name] { + return nil, fmt.Errorf("config: duplicate shard name %q", s.Name) + } + seen[s.Name] = true + if len(s.Packages) == 0 { + return nil, fmt.Errorf("config: shard %q has no packages", s.Name) + } + } + return &cfg, nil +} + +func (c *Config) FindShard(name string) *Shard { + for i := range c.Shards { + if c.Shards[i].Name == name { + return &c.Shards[i] + } + } + return nil +} + +// ListCombos returns the {os, shard} cells the workflow should run. +// A shard's runs_on.os overrides default_os when specified. +func (c *Config) ListCombos() []Combo { + var combos []Combo + for _, s := range c.Shards { + oses := c.DefaultOS + if s.RunsOn != nil && len(s.RunsOn.OS) > 0 { + oses = s.RunsOn.OS + } + for _, o := range oses { + combos = append(combos, Combo{OS: o, Shard: s.Name}) + } + } + return combos +} + +func (cond *Condition) Matches(osName, eventName string) bool { + if cond == nil { + return true + } + if len(cond.OS) > 0 && !slices.Contains(cond.OS, osName) { + return false + } + if len(cond.Events) > 0 && !slices.Contains(cond.Events, eventName) { + return false + } + return true +} + +func (s *Shard) ResolveRace(osName, eventName string) bool { + if s.RaceOn == nil { + return false + } + return s.RaceOn.Matches(osName, eventName) +} + +// ResolvePackages expands `./...` patterns and applies excludes by +// shelling out to `go list`. Cwd should be the Go module root. +func (s *Shard) ResolvePackages() ([]string, error) { + if !needsExpansion(s.Packages) && len(s.Exclude) == 0 { + return s.Packages, nil + } + includes, err := goList(s.Packages...) + if err != nil { + return nil, fmt.Errorf("expand includes: %w", err) + } + if len(s.Exclude) == 0 { + return includes, nil + } + excludes, err := goList(s.Exclude...) + if err != nil { + return nil, fmt.Errorf("expand excludes: %w", err) + } + excludeSet := make(map[string]bool, len(excludes)) + for _, e := range excludes { + excludeSet[e] = true + } + filtered := includes[:0] + for _, p := range includes { + if !excludeSet[p] { + filtered = append(filtered, p) + } + } + return filtered, nil +} + +func needsExpansion(patterns []string) bool { + for _, p := range patterns { + if strings.Contains(p, "...") { + return true + } + } + return false +} + +// goList runs `go list ...` and returns the resolved package +// import paths, one per line. +func goList(pats ...string) ([]string, error) { + args := append([]string{"list"}, pats...) + cmd := exec.Command("go", args...) + out, err := cmd.Output() + if err != nil { + if ee, ok := err.(*exec.ExitError); ok { + return nil, fmt.Errorf("go list failed: %s", strings.TrimSpace(string(ee.Stderr))) + } + return nil, err + } + var lines []string + for line := range strings.SplitSeq(strings.TrimSpace(string(out)), "\n") { + line = strings.TrimSpace(line) + if line != "" { + lines = append(lines, line) + } + } + return lines, nil +} diff --git a/go/utils/cigotests/config.yaml b/go/utils/cigotests/config.yaml new file mode 100644 index 00000000000..621c7b7bd25 --- /dev/null +++ b/go/utils/cigotests/config.yaml @@ -0,0 +1,63 @@ +# Shard configuration for the unified Go test workflow. Each entry +# describes a set of packages, the OSes it runs on, and when -race is +# applied. The runner (./go/utils/cigotests) reads this file and emits +# {os, shard} matrix combinations for the workflow. +# +# Editing rules: +# - Adding/removing a shard, packages, or env var: just edit this file; +# the workflow re-discovers shards on every run. +# - Adding a new heavy package: split it into its own shard so the +# other matrix cells aren't blocked by it. + +# OSes a shard runs on by default. A shard can override with `runs_on.os`. +default_os: + - macos-latest + - ubuntu-22.04 + - windows-latest + +shards: + # SQL engine tests are the slowest single package; they get their own shard. + # On a push to main we also exercise them with -race; the prepared variant + # is too slow under race so we skip it via DOLT_SKIP_PREPARED_ENGINETESTS. + - name: enginetest + packages: [./libraries/doltcore/sqle/enginetest] + timeout: 60m + env: + DOLT_DEFAULT_BIN_FORMAT: __DOLT__ + env_with_race: + DOLT_SKIP_PREPARED_ENGINETESTS: "1" + race_on: + os: [ubuntu-22.04] + events: [push, workflow_dispatch] + + # binlogreplication has been Linux-only historically. Preserve that until + # someone validates it on macOS/Windows. Race coverage on ubuntu. + - name: binlogreplication + packages: [./libraries/doltcore/sqle/binlogreplication/...] + timeout: 60m + runs_on: + os: [ubuntu-22.04] + race_on: + os: [ubuntu-22.04] + + # integration_test contains tests gated by SkipByDefaultInCI() that need + # DOLT_TEST_RUN_NON_RACE_TESTS=true to run and that don't tolerate -race. + # The "rest" shard exercises this package's non-gated tests with -race; + # this shard exercises the gated ones on every OS without -race. + - name: integration_test_gated + packages: [./libraries/doltcore/sqle/integration_test] + timeout: 30m + env: + DOLT_TEST_RUN_NON_RACE_TESTS: "true" + + # Everything else. enginetest and binlogreplication are excluded since + # they have their own shards. integration_test stays in here so its + # non-gated tests run with -race on ubuntu. + - name: rest + packages: [./...] + exclude: + - ./libraries/doltcore/sqle/enginetest + - ./libraries/doltcore/sqle/binlogreplication/... + timeout: 45m + race_on: + os: [ubuntu-22.04] diff --git a/go/utils/cigotests/config_test.go b/go/utils/cigotests/config_test.go new file mode 100644 index 00000000000..f5843abf6b3 --- /dev/null +++ b/go/utils/cigotests/config_test.go @@ -0,0 +1,146 @@ +// Copyright 2026 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "testing" + + "gopkg.in/yaml.v3" +) + +func TestConditionMatches(t *testing.T) { + cases := []struct { + name string + cond *Condition + os string + event string + want bool + }{ + {"nil condition matches anything", nil, "ubuntu-22.04", "push", true}, + {"empty condition matches anything", &Condition{}, "ubuntu-22.04", "push", true}, + {"os match", &Condition{OS: []string{"ubuntu-22.04"}}, "ubuntu-22.04", "push", true}, + {"os miss", &Condition{OS: []string{"ubuntu-22.04"}}, "macos-latest", "push", false}, + {"event match", &Condition{Events: []string{"push"}}, "ubuntu-22.04", "push", true}, + {"event miss", &Condition{Events: []string{"push"}}, "ubuntu-22.04", "pull_request", false}, + { + "both match", + &Condition{OS: []string{"ubuntu-22.04"}, Events: []string{"push", "workflow_dispatch"}}, + "ubuntu-22.04", "push", true, + }, + { + "os match, event miss", + &Condition{OS: []string{"ubuntu-22.04"}, Events: []string{"push"}}, + "ubuntu-22.04", "pull_request", false, + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := c.cond.Matches(c.os, c.event); got != c.want { + t.Fatalf("Matches(%q, %q) = %v, want %v", c.os, c.event, got, c.want) + } + }) + } +} + +func TestListCombos(t *testing.T) { + cfg := &Config{ + DefaultOS: []string{"macos-latest", "ubuntu-22.04", "windows-latest"}, + Shards: []Shard{ + {Name: "all", Packages: []string{"./..."}}, + {Name: "ubuntu_only", Packages: []string{"./..."}, RunsOn: &Condition{OS: []string{"ubuntu-22.04"}}}, + }, + } + combos := cfg.ListCombos() + want := []Combo{ + {OS: "macos-latest", Shard: "all"}, + {OS: "ubuntu-22.04", Shard: "all"}, + {OS: "windows-latest", Shard: "all"}, + {OS: "ubuntu-22.04", Shard: "ubuntu_only"}, + } + if len(combos) != len(want) { + t.Fatalf("got %d combos, want %d: %+v", len(combos), len(want), combos) + } + for i, c := range combos { + if c != want[i] { + t.Errorf("combo[%d] = %+v, want %+v", i, c, want[i]) + } + } +} + +func TestResolveRace(t *testing.T) { + s := Shard{ + Name: "x", + RaceOn: &Condition{OS: []string{"ubuntu-22.04"}, Events: []string{"push"}}, + } + if !s.ResolveRace("ubuntu-22.04", "push") { + t.Error("expected race on ubuntu push") + } + if s.ResolveRace("ubuntu-22.04", "pull_request") { + t.Error("expected no race on ubuntu pull_request") + } + if s.ResolveRace("macos-latest", "push") { + t.Error("expected no race on macos push") + } + + noRace := Shard{Name: "no_race"} + if noRace.ResolveRace("ubuntu-22.04", "push") { + t.Error("expected no race when RaceOn is nil") + } +} + +func TestEmbeddedConfigParses(t *testing.T) { + cfg, err := LoadConfig() + if err != nil { + t.Fatalf("LoadConfig: %v", err) + } + if len(cfg.Shards) == 0 { + t.Fatal("embedded config has no shards") + } + if len(cfg.DefaultOS) == 0 { + t.Fatal("embedded config has no default_os") + } + // Sanity: every shard has a name and at least one package. + for _, s := range cfg.Shards { + if s.Name == "" { + t.Error("shard with empty name") + } + if len(s.Packages) == 0 { + t.Errorf("shard %q has no packages", s.Name) + } + } +} + +func TestLoadConfigRejectsDuplicateShardNames(t *testing.T) { + bad := []byte(` +default_os: [ubuntu-22.04] +shards: + - name: x + packages: [./a] + - name: x + packages: [./b] +`) + var cfg Config + if err := yaml.Unmarshal(bad, &cfg); err != nil { + t.Fatalf("yaml.Unmarshal: %v", err) + } + // Re-route through LoadConfig path by swapping configBytes. + old := configBytes + configBytes = bad + defer func() { configBytes = old }() + _, err := LoadConfig() + if err == nil { + t.Fatal("expected duplicate-name error") + } +} diff --git a/go/utils/cigotests/main.go b/go/utils/cigotests/main.go new file mode 100644 index 00000000000..23bc7ea55fe --- /dev/null +++ b/go/utils/cigotests/main.go @@ -0,0 +1,189 @@ +// Copyright 2026 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// cigotests is the test runner used by .github/workflows/ci-go-tests.yaml. +// +// It reads config.yaml (embedded at build time) and supports two +// subcommands: +// +// list +// Prints a JSON array of {"os": "...", "shard": "..."} entries to +// stdout, one per matrix cell the workflow should run. +// +// run --shard --os --event +// Resolves the shard's package list, decides whether to enable -race +// based on the (os, event) pair, sets per-shard env vars, and execs +// `go test`. Cwd must be the Go module root (./go in this repo). +// +// Editing the matrix shape is a config-only operation: edit config.yaml +// and the workflow re-discovers shards on the next run. +package main + +import ( + "encoding/json" + "flag" + "fmt" + "os" + "os/exec" + "strings" +) + +func main() { + if len(os.Args) < 2 { + usage() + os.Exit(2) + } + switch os.Args[1] { + case "list": + cmdList() + case "plan": + cmdPlan(os.Args[2:]) + case "run": + cmdRun(os.Args[2:]) + case "-h", "--help", "help": + usage() + default: + fmt.Fprintf(os.Stderr, "unknown subcommand %q\n", os.Args[1]) + usage() + os.Exit(2) + } +} + +func usage() { + fmt.Fprintln(os.Stderr, "usage:") + fmt.Fprintln(os.Stderr, " cigotests list") + fmt.Fprintln(os.Stderr, " cigotests plan --shard --os --event ") + fmt.Fprintln(os.Stderr, " cigotests run --shard --os --event ") +} + +func cmdList() { + cfg, err := LoadConfig() + if err != nil { + die("load config: %v", err) + } + combos := cfg.ListCombos() + out, err := json.Marshal(combos) + if err != nil { + die("marshal combos: %v", err) + } + fmt.Println(string(out)) +} + +// resolvedPlan is the materialized go-test invocation for a (shard, os, +// event) triple: the resolved package list, the final go-test args, and +// the env additions that should be layered on os.Environ(). +type resolvedPlan struct { + Shard *Shard + OSName string + Event string + RaceOn bool + Pkgs []string + Args []string + EnvAdds []string +} + +func resolveShardArgs(args []string) (*resolvedPlan, error) { + fs := flag.NewFlagSet("", flag.ExitOnError) + shardName := fs.String("shard", "", "shard name (matches a name in config.yaml)") + osName := fs.String("os", "", "matrix os value, e.g. ubuntu-22.04") + eventName := fs.String("event", "", "github event name, e.g. pull_request, push") + if err := fs.Parse(args); err != nil { + return nil, err + } + if *shardName == "" || *osName == "" || *eventName == "" { + fs.Usage() + return nil, fmt.Errorf("--shard, --os, and --event are required") + } + + cfg, err := LoadConfig() + if err != nil { + return nil, fmt.Errorf("load config: %w", err) + } + shard := cfg.FindShard(*shardName) + if shard == nil { + return nil, fmt.Errorf("unknown shard %q", *shardName) + } + + pkgs, err := shard.ResolvePackages() + if err != nil { + return nil, fmt.Errorf("resolve packages: %w", err) + } + if len(pkgs) == 0 { + return nil, fmt.Errorf("shard %q resolved to zero packages", shard.Name) + } + + raceOn := shard.ResolveRace(*osName, *eventName) + + testArgs := []string{"test", "-vet=off"} + if shard.Timeout != "" { + testArgs = append(testArgs, "-timeout", shard.Timeout) + } + if raceOn { + testArgs = append(testArgs, "-race") + } + testArgs = append(testArgs, pkgs...) + + var envAdds []string + for k, v := range shard.Env { + envAdds = append(envAdds, k+"="+v) + } + if raceOn { + for k, v := range shard.EnvWithRace { + envAdds = append(envAdds, k+"="+v) + } + } + + return &resolvedPlan{ + Shard: shard, OSName: *osName, Event: *eventName, + RaceOn: raceOn, Pkgs: pkgs, Args: testArgs, EnvAdds: envAdds, + }, nil +} + +func (p *resolvedPlan) describe(w *os.File) { + fmt.Fprintf(w, "shard=%s os=%s event=%s race=%v packages=%d\n", + p.Shard.Name, p.OSName, p.Event, p.RaceOn, len(p.Pkgs)) + for _, e := range p.EnvAdds { + fmt.Fprintf(w, "env: %s\n", e) + } + fmt.Fprintf(w, "+ go %s\n", strings.Join(p.Args, " ")) +} + +func cmdPlan(args []string) { + plan, err := resolveShardArgs(args) + if err != nil { + die("%v", err) + } + plan.describe(os.Stdout) +} + +func cmdRun(args []string) { + plan, err := resolveShardArgs(args) + if err != nil { + die("%v", err) + } + plan.describe(os.Stderr) + + cmd := exec.Command("go", plan.Args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Env = append(os.Environ(), plan.EnvAdds...) + if err := cmd.Run(); err != nil { + os.Exit(1) + } +} + +func die(format string, args ...any) { + fmt.Fprintf(os.Stderr, format+"\n", args...) + os.Exit(1) +} From a8b119091108ae0ec9514a1c3ec775e975d8812d Mon Sep 17 00:00:00 2001 From: Aaron Son Date: Thu, 7 May 2026 16:36:19 -0700 Subject: [PATCH 2/3] .github: replace three Go-test workflows with one matrix workflow Consolidates ci-go-tests.yaml, ci-go-tests-adaptive.yaml, ci-go-race-tests.yaml, and ci-binlog-tests.yaml into a single ci-go-tests.yaml driven by the cigotests runner. The workflow has two jobs: enumerate-shards runs cigotests list, emits a JSON array of {os, shard} cells via job outputs. test strategy.matrix.include is fromJson of the above. Each cell runs cigotests run for its (shard, os, github.event_name) triple. What changed from the old workflows: - The per-package bash loop is gone. Each shard hands its full package list to a single 'go test pkg1 pkg2 ...' invocation, which lets go's test runner schedule packages concurrently. - The OS-conditional '-race' baked into ci-go-tests.yaml is now a per-shard race_on predicate in cigotests/config.yaml. - ci-go-tests-adaptive.yaml is dropped entirely. After the adaptive-encoding default flip in #11011 it was running the same matrix as ci-go-tests.yaml. - The push-only enginetest+race coverage from ci-go-race-tests.yaml is now expressed as race_on: {os: [ubuntu-22.04], events: [push, workflow_dispatch]} on the enginetest shard. Concurrentmap, which had its own race job, is just one of many packages in the 'rest' shard which already gets -race on ubuntu. - ci-binlog-tests.yaml folds in as a 'binlogreplication' shard pinned to ubuntu (matching the workflow it replaces). Adding/splitting/retuning a shard is now a one-file edit to config.yaml; no workflow change is needed. --- .github/workflows/ci-binlog-tests.yaml | 62 ----------- .github/workflows/ci-go-race-tests.yaml | 39 ------- .github/workflows/ci-go-tests-adaptive.yaml | 74 ------------- .github/workflows/ci-go-tests.yaml | 117 +++++++------------- 4 files changed, 43 insertions(+), 249 deletions(-) delete mode 100644 .github/workflows/ci-binlog-tests.yaml delete mode 100644 .github/workflows/ci-go-race-tests.yaml delete mode 100644 .github/workflows/ci-go-tests-adaptive.yaml diff --git a/.github/workflows/ci-binlog-tests.yaml b/.github/workflows/ci-binlog-tests.yaml deleted file mode 100644 index 4afd9925b32..00000000000 --- a/.github/workflows/ci-binlog-tests.yaml +++ /dev/null @@ -1,62 +0,0 @@ -name: Test Binlog - -on: - pull_request: - branches: [ main ] - paths: - - 'go/**' - workflow_dispatch: - -concurrency: - group: ci-binlog-tests-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - binlog-test: - name: Binlog tests - defaults: - run: - shell: bash - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-22.04] - adaptive_encoding: ["false", "true"] - steps: - - uses: actions/checkout@v6 - - name: Set up Go toolchain - uses: ./.github/actions/setup-go-toolchain - - name: Test Binlog - working-directory: ./go - run: | - # Test binlog packages - go test -vet=off -timeout 60m ./libraries/doltcore/sqle/binlogreplication/... - env: - MATRIX_OS: ${{ matrix.os }} - DOLT_USE_ADAPTIVE_ENCODING: ${{ matrix.adaptive_encoding }} - binlog-race-test: - name: Binlog tests - race - defaults: - run: - shell: bash - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-22.04] - # TODO: bin log doesn't yet handle adapative encoding -# adaptive_encoding: ["false", "true"] - adaptive_encoding: ["false"] - steps: - - uses: actions/checkout@v6 - - name: Set up Go toolchain - uses: ./.github/actions/setup-go-toolchain - - name: Test Binlog with Race - working-directory: ./go - run: | - # Test binlog packages with race detector - go test -vet=off -timeout 60m -race ./libraries/doltcore/sqle/binlogreplication/... - env: - MATRIX_OS: ${{ matrix.os }} - DOLT_USE_ADAPTIVE_ENCODING: ${{ matrix.adaptive_encoding }} diff --git a/.github/workflows/ci-go-race-tests.yaml b/.github/workflows/ci-go-race-tests.yaml deleted file mode 100644 index 53fda58a9ca..00000000000 --- a/.github/workflows/ci-go-race-tests.yaml +++ /dev/null @@ -1,39 +0,0 @@ -name: Race tests - -on: - push: - branches: [ main ] - paths: - - 'go/**' - workflow_dispatch: - -jobs: - racetests: - name: Go race tests - defaults: - run: - shell: bash - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ ubuntu-22.04 ] - dolt_fmt: [ "__DOLT__" ] - adaptive_encoding: [ "false", "true" ] - steps: - - uses: actions/checkout@v6 - - name: Set up Go toolchain - uses: ./.github/actions/setup-go-toolchain - - name: Test engine - working-directory: ./go - run: | - DOLT_SKIP_PREPARED_ENGINETESTS=1 go test -vet=off -v -race -timeout 30m github.com/dolthub/dolt/go/libraries/doltcore/sqle/enginetest - env: - DOLT_DEFAULT_BIN_FORMAT: ${{ matrix.dolt_fmt }} - DOLT_USE_ADAPTIVE_ENCODING: ${{ matrix.adaptive_encoding }} - - name: Test concurrentmap - working-directory: ./go - run: | - go test -vet=off -v -race -timeout 1m github.com/dolthub/dolt/go/libraries/utils/concurrentmap - env: - DOLT_USE_ADAPTIVE_ENCODING: ${{ matrix.adaptive_encoding }} diff --git a/.github/workflows/ci-go-tests-adaptive.yaml b/.github/workflows/ci-go-tests-adaptive.yaml deleted file mode 100644 index 388fbf25f03..00000000000 --- a/.github/workflows/ci-go-tests-adaptive.yaml +++ /dev/null @@ -1,74 +0,0 @@ -name: Test Go (Adaptive Encoding) - -on: - pull_request: - branches: [ main ] - paths: - - 'go/**' - - '.github/workflows/ci-go-tests-adaptive.yaml' - workflow_dispatch: - -concurrency: - group: ci-go-tests-adaptive-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - test: - name: Go tests (adaptive encoding) - defaults: - run: - shell: bash - runs-on: ubuntu-22.04 - env: - DOLT_USE_ADAPTIVE_ENCODING: "true" - steps: - - uses: actions/checkout@v6 - - name: Set up Go toolchain - uses: ./.github/actions/setup-go-toolchain - - name: Test All - working-directory: ./go - run: | - files=$(go list ./...) - SAVEIFS=$IFS - IFS=$'\n' - file_arr=($files) - IFS=$SAVEIFS - - for (( i=0; i<${#file_arr[@]}; i++ )) - do - # Skip binlog tests as they run in a separate CI job - if [[ "${file_arr[$i]}" == *binlogreplication* ]]; then - echo "Skipping binlog package: ${file_arr[$i]} (runs in separate CI)" - continue - fi - - echo "Testing Package: ${file_arr[$i]}" - if [[ "${file_arr[$i]}" != *enginetest* ]]; then - go test -vet=off -timeout 45m -race "${file_arr[$i]}" - else - echo "skipping enginetests for -race" - fi - succeeded=$(echo "$?") - if [ "$succeeded" -ne 0 ]; then - echo "Testing failed in package ${file_arr[$i]}" - exit 1; - fi - done - noracetest: - name: Go tests - no race (adaptive encoding) - defaults: - run: - shell: bash - runs-on: ubuntu-22.04 - env: - DOLT_USE_ADAPTIVE_ENCODING: "true" - steps: - - uses: actions/checkout@v6 - - name: Set up Go toolchain - uses: ./.github/actions/setup-go-toolchain - - name: Test All - working-directory: ./go - run: | - go test -vet=off -timeout 30m ./libraries/doltcore/sqle/integration_test - env: - DOLT_TEST_RUN_NON_RACE_TESTS: "true" diff --git a/.github/workflows/ci-go-tests.yaml b/.github/workflows/ci-go-tests.yaml index a32d682e13e..6e8c9dd6098 100644 --- a/.github/workflows/ci-go-tests.yaml +++ b/.github/workflows/ci-go-tests.yaml @@ -2,10 +2,17 @@ name: Test Go on: pull_request: - branches: [ main ] + branches: [main] paths: - 'go/**' - '.github/workflows/ci-go-tests.yaml' + - '.github/actions/setup-go-toolchain/**' + push: + branches: [main] + paths: + - 'go/**' + - '.github/workflows/ci-go-tests.yaml' + - '.github/actions/setup-go-toolchain/**' workflow_dispatch: concurrency: @@ -13,84 +20,46 @@ concurrency: cancel-in-progress: true jobs: + enumerate-shards: + name: Enumerate shards + runs-on: ubuntu-22.04 + outputs: + combos: ${{ steps.list.outputs.combos }} + steps: + - uses: actions/checkout@v6 + - uses: ./.github/actions/setup-go-toolchain + - name: List shard combos + id: list + working-directory: ./go + run: | + combos=$(go run ./utils/cigotests list) + echo "combos=$combos" >> "$GITHUB_OUTPUT" + echo "$combos" | python3 -m json.tool + test: - name: Go tests - defaults: - run: - shell: bash - runs-on: ${{ matrix.os }} + name: ${{ matrix.shard }} (${{ matrix.os }}) + needs: enumerate-shards strategy: fail-fast: false matrix: - os: [macos-latest, ubuntu-22.04, windows-latest] - steps: - - uses: actions/checkout@v6 - - name: Set up Go toolchain - id: toolchain - uses: ./.github/actions/setup-go-toolchain - - name: Test All - working-directory: ./go - run: | - files=$(go list ./...) - SAVEIFS=$IFS - IFS=$'\n' - file_arr=($files) - IFS=$SAVEIFS - if [ "$MATRIX_OS" == 'windows-latest' ]; then - export PATH=$(cygpath -u "$MSYS2_LOCATION"/ucrt64/bin):"$PATH" - fi - - for (( i=0; i<${#file_arr[@]}; i++ )) - do - # Skip binlog tests as they run in a separate CI job - if [[ "${file_arr[$i]}" == *binlogreplication* ]]; then - echo "Skipping binlog package: ${file_arr[$i]} (runs in separate CI)" - continue - fi - - echo "Testing Package: ${file_arr[$i]}" - if [ "$MATRIX_OS" == 'ubuntu-22.04' ] - then - if [[ "${file_arr[$i]}" != *enginetest* ]]; then - go test -vet=off -timeout 45m -race "${file_arr[$i]}" - else - echo "skipping enginetests for -race" - fi - else - go test -vet=off -timeout 45m "${file_arr[$i]}" - fi - succeeded=$(echo "$?") - if [ "$succeeded" -ne 0 ]; then - echo "Testing failed in package ${file_arr[$i]}" - exit 1; - fi - done - env: - MATRIX_OS: ${{ matrix.os }} - MSYS2_LOCATION: ${{ steps.toolchain.outputs.msys2-location }} - noracetest: - name: Go tests - no race + include: ${{ fromJson(needs.enumerate-shards.outputs.combos) }} + runs-on: ${{ matrix.os }} defaults: run: shell: bash - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [macos-latest, ubuntu-22.04, windows-latest] steps: - - uses: actions/checkout@v6 - - name: Set up Go toolchain - id: toolchain - uses: ./.github/actions/setup-go-toolchain - - name: Test All - working-directory: ./go - run: | - if [ "$MATRIX_OS" == 'windows-latest' ]; then - export PATH=$(cygpath -u "$MSYS2_LOCATION"/ucrt64/bin):"$PATH" - fi - go test -vet=off -timeout 30m ./libraries/doltcore/sqle/integration_test - env: - MATRIX_OS: ${{ matrix.os }} - DOLT_TEST_RUN_NON_RACE_TESTS: "true" - MSYS2_LOCATION: ${{ steps.toolchain.outputs.msys2-location }} + - uses: actions/checkout@v6 + - uses: ./.github/actions/setup-go-toolchain + id: toolchain + - name: Run shard + working-directory: ./go + run: | + if [ "$RUNNER_OS" = "Windows" ]; then + export PATH=$(cygpath -u "$MSYS2_LOCATION"/ucrt64/bin):"$PATH" + fi + go run ./utils/cigotests run \ + --shard "${{ matrix.shard }}" \ + --os "${{ matrix.os }}" \ + --event "${{ github.event_name }}" + env: + MSYS2_LOCATION: ${{ steps.toolchain.outputs.msys2-location }} From c418775398301f1d88d618b5db4e75bfe128d964 Mon Sep 17 00:00:00 2001 From: reltuk Date: Fri, 8 May 2026 00:06:16 +0000 Subject: [PATCH 3/3] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- go/utils/cigotests/main.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/go/utils/cigotests/main.go b/go/utils/cigotests/main.go index 23bc7ea55fe..9729c16840a 100644 --- a/go/utils/cigotests/main.go +++ b/go/utils/cigotests/main.go @@ -17,14 +17,14 @@ // It reads config.yaml (embedded at build time) and supports two // subcommands: // -// list -// Prints a JSON array of {"os": "...", "shard": "..."} entries to -// stdout, one per matrix cell the workflow should run. +// list +// Prints a JSON array of {"os": "...", "shard": "..."} entries to +// stdout, one per matrix cell the workflow should run. // -// run --shard --os --event -// Resolves the shard's package list, decides whether to enable -race -// based on the (os, event) pair, sets per-shard env vars, and execs -// `go test`. Cwd must be the Go module root (./go in this repo). +// run --shard --os --event +// Resolves the shard's package list, decides whether to enable -race +// based on the (os, event) pair, sets per-shard env vars, and execs +// `go test`. Cwd must be the Go module root (./go in this repo). // // Editing the matrix shape is a config-only operation: edit config.yaml // and the workflow re-discovers shards on the next run.