diff --git a/config/config.go b/config/config.go index 58212bce..8a57ee2e 100644 --- a/config/config.go +++ b/config/config.go @@ -69,15 +69,13 @@ func Parse(configFilePath string) (*Config, error) { if config.Storage.Type == "" { config.Storage.Type = StorageTypeMemory } - if config.Service.WorkspacesRoot == "" { - return nil, fmt.Errorf("service.workspaces_root must be set") - } - if config.Service.WorkerRootPath == "" { - config.Service.WorkerRootPath = filepath.Join(config.Service.WorkspacesRoot, ".workers") + if config.Service.WorkspacesRootPath == "" { + return nil, fmt.Errorf("service.workspaces_root_path must be set") } if config.Service.MaxWorkerPoolSize <= 0 { return nil, fmt.Errorf("service.max_worker_pool_size must be > 0, got %d", config.Service.MaxWorkerPoolSize) } + config.Service.WorkerRootPath = filepath.Join(config.Service.WorkspacesRootPath, ".workers") if config.Service.Streaming.MaxNumTargets <= 0 { config.Service.Streaming.MaxNumTargets = _defaultMaxNumTargets } diff --git a/config/config_test.go b/config/config_test.go index 2b00edb4..25a2235c 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -25,7 +25,7 @@ import ( const _baseServiceYAML = ` service: - workspaces_root: "/tmp/x" + workspaces_root_path: "/tmp/x" max_worker_pool_size: 1 ` @@ -42,7 +42,7 @@ func TestParse_ServiceValidation(t *testing.T) { give string }{ { - name: "workspaces_root required", + name: "workspaces_root_path required", give: ` service: max_worker_pool_size: 1 @@ -54,7 +54,7 @@ repository: name: "max_worker_pool_size must be positive", give: ` service: - workspaces_root: "/tmp/x" + workspaces_root_path: "/tmp/x" max_worker_pool_size: 0 repository: - remote: "r1" @@ -80,7 +80,7 @@ func TestParse_ServiceDefaults(t *testing.T) { wantMaxNumMetadataEntries int }{ { - name: "worker_root_path and streaming default when unset", + name: "streaming defaults when unset", give: _baseServiceYAML + ` repository: - remote: "r1" @@ -90,21 +90,6 @@ repository: wantMaxNumChangedTargets: _defaultMaxNumChangedTargets, wantMaxNumMetadataEntries: _defaultMaxNumMetadataEntries, }, - { - name: "worker_root_path explicit value preserved", - give: ` -service: - workspaces_root: "/tmp/x" - worker_root_path: "/tmp/custom-workers" - max_worker_pool_size: 1 -repository: - - remote: "r1" -`, - wantWorkerRootPath: "/tmp/custom-workers", - wantMaxNumTargets: _defaultMaxNumTargets, - wantMaxNumChangedTargets: _defaultMaxNumChangedTargets, - wantMaxNumMetadataEntries: _defaultMaxNumMetadataEntries, - }, { name: "streaming explicit values preserved", give: _baseServiceYAML + ` diff --git a/config/repository_config.go b/config/repository_config.go index 5d1f56ef..f7adbb4b 100644 --- a/config/repository_config.go +++ b/config/repository_config.go @@ -21,27 +21,18 @@ type RepositoryConfig struct { // unique across all entries and match exactly what clients send in // BuildDescription.remote. Remote string `yaml:"remote"` - // TODO: FullHashRepos, ExcludedFiles, and ExcludeExternalTargets are not - // documented in config/README.md. Delete them if they turn out to be unneeded, - // otherwise document them there. - FullHashRepos []string `yaml:"full_hash_repos"` - ExcludedFiles []string `yaml:"excluded_files"` - ExcludeExternalTargets bool `yaml:"exclude_external_targets"` // BzlmodEnabled indicates whether this repository uses Bzlmod for external // dependency management. Defaults to true if unset. Set to false only for // repositories still using WORKSPACE. BzlmodEnabled *bool `yaml:"bzlmod_enabled"` - // BazelCommand overrides the Bazel binary path. When empty, Tango + // BazelCommandPath overrides the Bazel binary path. When empty, Tango // automatically downloads and caches Bazelisk from GitHub. - BazelCommand string `yaml:"bazel_command"` - // QueryTimeoutSeconds is the Bazel query timeout in seconds. Defaults to 600. - QueryTimeoutSeconds int64 `yaml:"query_timeout_seconds"` + BazelCommandPath string `yaml:"bazel_command_path"` // BazelExtraArgs are extra arguments passed to `bazel query` invocations, // inserted between the `query` subcommand and the query expression. BazelExtraArgs []string `yaml:"bazel_extra_args"` - // TODO: StreamBazelLogs is not documented in config/README.md. Delete it if - // it turns out to be unneeded, otherwise document it there. - StreamBazelLogs bool `yaml:"stream_bazel_logs"` + // QueryTimeoutSeconds is the Bazel query timeout in seconds. Defaults to 600. + QueryTimeoutSeconds int64 `yaml:"query_timeout_seconds"` } // RepositoryConfigProvider looks up per-repository configuration by remote. diff --git a/config/service_config.go b/config/service_config.go index ef150396..41fbcbde 100644 --- a/config/service_config.go +++ b/config/service_config.go @@ -20,15 +20,16 @@ type ServiceConfig struct { // Each worker is a lightweight local clone (hardlinked to the origin, not a // full copy) that handles one request at a time. Must be greater than 0. MaxWorkerPoolSize int `yaml:"max_worker_pool_size"` - // WorkspacesRoot is the root directory where Tango stores repository clones - // and worker checkouts. Required. Layout: // for - // origin clones and /.workers//worker-{1..N}/ for + // WorkspacesRootPath is the root directory where Tango stores repository clones + // and worker checkouts. Required. Layout: // for + // origin clones and /.workers//worker-{1..N}/ for // worker checkouts. - WorkspacesRoot string `yaml:"workspaces_root"` - // TODO: WorkerRootPath is not documented in config/README.md. Delete it if - // it turns out to be unneeded, otherwise document it there. - WorkerRootPath string `yaml:"worker_root_path"` // root directory for worker workspace checkouts; defaults to workspaces_root/.workers - Streaming ChunkConfig `yaml:"streaming"` // streaming chunk sizes; zero values fall back to package defaults + WorkspacesRootPath string `yaml:"workspaces_root_path"` + Streaming ChunkConfig `yaml:"streaming"` // streaming chunk sizes; zero values fall back to package defaults + + // WorkerRootPath is the root directory for worker workspace checkouts, + // set by Parse to /.workers. Not settable via YAML. + WorkerRootPath string `yaml:"-"` } // ChunkConfig controls the number of entries per gRPC stream message. diff --git a/example/main.go b/example/main.go index 906aed60..6aba8d0c 100644 --- a/example/main.go +++ b/example/main.go @@ -71,7 +71,7 @@ func run() error { logger.Infof("Using storage type: %s", cfg.Storage.Type) // Repo manager and orchestrator - repoManagerClonePath := cfg.Service.WorkspacesRoot + repoManagerClonePath := cfg.Service.WorkspacesRootPath workerRootPath := cfg.Service.WorkerRootPath if err := os.MkdirAll(repoManagerClonePath, 0o755); err != nil { return fmt.Errorf("failed to create repo manager clone path: %w", err) diff --git a/example/tango-config.yaml b/example/tango-config.yaml index cd0fd3a1..b453057b 100644 --- a/example/tango-config.yaml +++ b/example/tango-config.yaml @@ -9,11 +9,6 @@ storage: # Repository configuration repository: - remote: "https://github.com/uber/tango.git" - full_hash_repos: - - "" - excluded_files: - - "^@@?bazel_tools/" - exclude_external_targets: true bzlmod_enabled: true query_timeout_seconds: 300 @@ -21,6 +16,4 @@ repository: service: max_worker_pool_size: 5 # root for origin clones; required - workspaces_root: "/tmp/tango-repo-manager" - # root for worker checkouts; defaults to workspaces_root/.workers - worker_root_path: "/tmp/tango/workers" + workspaces_root_path: "/tmp/tango-repo-manager" diff --git a/graphrunner/native.go b/graphrunner/native.go index 0eb1c0af..5cb415be 100644 --- a/graphrunner/native.go +++ b/graphrunner/native.go @@ -58,9 +58,12 @@ func NewNativeGraphRunner(p NativeGraphRunnerParams) GraphRunner { } func (g *nativeGraphRunner) Compute(ctx context.Context, ws workspace.Workspace) (targethasher.Result, error) { - query := "//external:all-targets + deps(//...:all-targets)" - if g.config.ExcludeExternalTargets { - query = "deps(//...:all-targets)" + bzlmodEnabled := g.config.BzlmodEnabled == nil || *g.config.BzlmodEnabled + query := "deps(//...:all-targets)" + if !bzlmodEnabled { + // //external is only queryable under legacy WORKSPACE resolution; + // Bzlmod repos resolve external deps under @@ instead. + query = "//external:all-targets + " + query } additionalArgs := append( []string{"--order_output=no", "--proto:locations", "--noproto:default_values"}, @@ -74,8 +77,6 @@ func (g *nativeGraphRunner) Compute(ctx context.Context, ws workspace.Workspace) // --proto:locations: we need to get external file location to make CTC more accurate // --noproto: parameters exclude fields from the output that are not used for hashing anyways, making // proto blob smaller and serialization/deserialization faster - // TODO: pass in --enable_workspace or --enable_bzlmod based on the config - AdditionalArgs: additionalArgs, }) g.scope.Timer("bazel_query_duration").Record(time.Since(bazelStart)) @@ -92,9 +93,8 @@ func (g *nativeGraphRunner) Compute(ctx context.Context, ws workspace.Workspace) hashConfig := targethasher.HashConfig{ KnownSourceHashes: knownSourceHashes, - FullHashRepos: g.config.FullHashRepos, - ExcludedRegex: append(g.config.ExcludedFiles, g.extraExcludedFiles...), - UseBzlmod: g.config.BzlmodEnabled == nil || *g.config.BzlmodEnabled, + ExcludedRegex: g.extraExcludedFiles, + UseBzlmod: bzlmodEnabled, } hashStart := time.Now() diff --git a/integration/integration_test.go b/integration/integration_test.go index c3373596..8d4ca164 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -48,6 +48,11 @@ const ( configTemplateFile = "testdata/tango-config.yaml.tmpl" ) +// bazelToolsExcludeRegex excludes @bazel_tools's source files from disk-based hashing. +// It ships platform-specific files (e.g. tools/test/tw.exe, a Windows-only test wrapper) +// that don't exist on every OS, so stat-ing them to hash fails on machines missing them. +var bazelToolsExcludeRegex = []string{"^@@?bazel_tools/"} + func repoRemote(t *testing.T) string { t.Helper() remote := os.Getenv("TANGO_REPO_REMOTE") @@ -266,6 +271,9 @@ func getChangedTargets(t *testing.T, client pb.TangoYARPCClient, remote, firstSH Remote: remote, BaseSha: secondSHA, }, + RequestOptions: &pb.RequestOptions{ + ExtraExcludeFilesRegex: bazelToolsExcludeRegex, + }, }) require.NoError(t, err, "failed to initiate GetChangedTargets stream") @@ -361,6 +369,9 @@ func TestIntegration_GetTargetGraph(t *testing.T) { Remote: remote, BaseSha: pinnedSHA, }, + RequestOptions: &pb.RequestOptions{ + ExtraExcludeFilesRegex: bazelToolsExcludeRegex, + }, }) require.NoError(t, err, "failed to initiate GetTargetGraph stream") diff --git a/integration/testdata/tango-config.yaml.tmpl b/integration/testdata/tango-config.yaml.tmpl index 186ba9df..7edf0396 100644 --- a/integration/testdata/tango-config.yaml.tmpl +++ b/integration/testdata/tango-config.yaml.tmpl @@ -5,15 +5,11 @@ repository: - remote: {{.Remote}} default_branch: "main" {{- if .BazelCommand}} - bazel_command: {{.BazelCommand}} + bazel_command_path: {{.BazelCommand}} {{- end}} - full_hash_repos: [""] - excluded_files: ["^@@?bazel_tools/"] - exclude_external_targets: true bzlmod_enabled: true query_timeout_seconds: 600 service: max_worker_pool_size: 2 - workspaces_root: {{.ClonePath}} - worker_root_path: {{.WorkerPath}} + workspaces_root_path: {{.ClonePath}} diff --git a/orchestrator/native_orchestrator.go b/orchestrator/native_orchestrator.go index 47062c59..3f9ab40a 100644 --- a/orchestrator/native_orchestrator.go +++ b/orchestrator/native_orchestrator.go @@ -189,9 +189,8 @@ func (b *nativeOrchestrator) GetTargetGraph(ctx context.Context, req entity.GetT client, err := bazel.NewBazelClient(ctx, bazel.Params{ WorkspacePath: ws.Path(), Logger: b.logger, - BazelCommand: repoCfg.BazelCommand, + BazelCommand: repoCfg.BazelCommandPath, QueryTimeout: time.Duration(repoCfg.QueryTimeoutSeconds) * time.Second, - StreamLogs: repoCfg.StreamBazelLogs, }) if err != nil { return nil, fmt.Errorf("create bazel client: %w", err) diff --git a/orchestrator/testdata/config.yaml b/orchestrator/testdata/config.yaml index 5a9483db..93f44392 100644 --- a/orchestrator/testdata/config.yaml +++ b/orchestrator/testdata/config.yaml @@ -1,17 +1,11 @@ repository: - remote: "git@github:uber/tango" - full_hash_repos: - - "//" - - "//external" - excluded_files: - - "*.gen.go" - exclude_external_targets: true bzlmod_enabled: true query_timeout_seconds: 15 service: max_worker_pool_size: 3 - workspaces_root: "/tmp/tango-repo-manager" + workspaces_root_path: "/tmp/tango-repo-manager" streaming: max_num_targets: 250 max_num_changed_targets: 125