From 676408ba8caf0c9bdcb0a6936df54b6086b11787 Mon Sep 17 00:00:00 2001 From: Sophia Yang Date: Sat, 6 Jun 2026 15:37:16 -0400 Subject: [PATCH 001/108] fix(internal/librarian/java): refine whitespace delimiters and exclude test classes during sample extraction for README.md --- cmd/librarian/doc.go | 3 +- go.mod | 2 +- internal/librarian/generate.go | 15 ++- internal/librarian/install_test.go | 4 +- internal/librarian/java/clean.go | 21 +++- internal/librarian/java/generate.go | 13 +-- internal/librarian/java/generate_test.go | 30 ++++-- internal/librarian/java/postgenerate_test.go | 5 +- internal/librarian/java/postprocess.go | 7 ++ internal/librarian/java/postprocess_test.go | 108 ++++++++++++++++++- internal/librarian/java/repometadata_test.go | 5 +- internal/sidekick/parser/protobuf.go | 1 - 12 files changed, 181 insertions(+), 33 deletions(-) diff --git a/cmd/librarian/doc.go b/cmd/librarian/doc.go index 14db3410070..c8a86fed508 100644 --- a/cmd/librarian/doc.go +++ b/cmd/librarian/doc.go @@ -102,7 +102,8 @@ Examples: Flags: - --all generate all libraries + --all generate all libraries + --use-go-postprocessor use the new Go postprocessor for Java libraries A typical librarian workflow for regenerating every library against the latest API definitions is: diff --git a/go.mod b/go.mod index 58f30075608..5720b43c448 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( cloud.google.com/go/iam v1.5.3 cloud.google.com/go/longrunning v0.8.0 github.com/bazelbuild/buildtools v0.0.0-20260202105709-e24971d9d1a7 + github.com/bmatcuk/doublestar/v4 v4.10.0 github.com/cbroglie/mustache v1.4.0 github.com/go-git/go-git/v5 v5.19.1 github.com/google/go-cmp v0.7.0 @@ -71,7 +72,6 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bkielbasa/cyclop v1.2.3 // indirect github.com/blizzy78/varnamelen v0.8.0 // indirect - github.com/bmatcuk/doublestar/v4 v4.10.0 // indirect github.com/bombsimon/wsl/v4 v4.7.0 // indirect github.com/bombsimon/wsl/v5 v5.6.0 // indirect github.com/breml/bidichk v0.3.3 // indirect diff --git a/internal/librarian/generate.go b/internal/librarian/generate.go index 43833550b33..10a5e54f4e0 100644 --- a/internal/librarian/generate.go +++ b/internal/librarian/generate.go @@ -73,9 +73,14 @@ latest API definitions is: Name: "all", Usage: "generate all libraries", }, + &cli.BoolFlag{ + Name: "use-go-postprocessor", + Usage: "use the new Go postprocessor for Java libraries", + }, }, Action: func(ctx context.Context, cmd *cli.Command) error { all := cmd.Bool("all") + useGoPostprocessor := cmd.Bool("use-go-postprocessor") libraryName := cmd.Args().First() if !all && libraryName == "" { return errMissingLibraryOrAllFlag @@ -87,12 +92,12 @@ latest API definitions is: if err != nil { return err } - return runGenerate(ctx, cfg, all, libraryName) + return runGenerate(ctx, cfg, all, libraryName, useGoPostprocessor) }, } } -func runGenerate(ctx context.Context, cfg *config.Config, all bool, libraryName string) error { +func runGenerate(ctx context.Context, cfg *config.Config, all bool, libraryName string, useGoPostprocessor bool) error { sources, err := LoadSources(ctx, cfg.Sources) if err != nil { return err @@ -139,7 +144,7 @@ func runGenerate(ctx context.Context, cfg *config.Config, all bool, libraryName if err := cleanLibraries(cfg.Language, libraries); err != nil { return err } - return generateLibraries(ctx, cfg, libraries, sources) + return generateLibraries(ctx, cfg, libraries, sources, useGoPostprocessor) } // cleanLibraries iterates over all the given libraries sequentially, @@ -181,7 +186,7 @@ func cleanLibraries(language string, libraries []*config.Library) error { // generateLibraries generates and formats all the given libraries, // delegating to language-specific code. Each language chooses its own // concurrency strategy for these two steps. -func generateLibraries(ctx context.Context, cfg *config.Config, libraries []*config.Library, src *sources.Sources) error { +func generateLibraries(ctx context.Context, cfg *config.Config, libraries []*config.Library, src *sources.Sources, useGoPostprocessor bool) error { switch cfg.Language { case config.LanguageDart: g, gctx := errgroup.WithContext(ctx) @@ -241,7 +246,7 @@ func generateLibraries(ctx context.Context, cfg *config.Config, libraries []*con allMissingArtifacts = append(allMissingArtifacts, java.MissingArtifact{ID: id, Library: library}) } - if err := java.Generate(ctx, cfg, library, src); err != nil { + if err := java.Generate(ctx, cfg, library, src, useGoPostprocessor); err != nil { return fmt.Errorf("generate library %q (%s): %w", library.Name, cfg.Language, err) } if err := java.Format(ctx, library); err != nil { diff --git a/internal/librarian/install_test.go b/internal/librarian/install_test.go index a9f22409b3e..a9090367213 100644 --- a/internal/librarian/install_test.go +++ b/internal/librarian/install_test.go @@ -62,7 +62,7 @@ func TestGenerate(t *testing.T) { tmpDir := t.TempDir() t.Chdir(tmpDir) - if err := generateLibraries(t.Context(), cfg, []*config.Library{library}, nil); err != nil { + if err := generateLibraries(t.Context(), cfg, []*config.Library{library}, nil, false); err != nil { t.Fatal(err) } @@ -92,7 +92,7 @@ func TestCleanLibraries(t *testing.T) { tmpDir := t.TempDir() t.Chdir(tmpDir) - if err := generateLibraries(t.Context(), cfg, []*config.Library{library}, nil); err != nil { + if err := generateLibraries(t.Context(), cfg, []*config.Library{library}, nil, false); err != nil { t.Fatal(err) } diff --git a/internal/librarian/java/clean.go b/internal/librarian/java/clean.go index 7d36826493e..96bdccd933c 100644 --- a/internal/librarian/java/clean.go +++ b/internal/librarian/java/clean.go @@ -20,6 +20,7 @@ import ( "io" "io/fs" "os" + "path" "path/filepath" "regexp" "slices" @@ -54,7 +55,7 @@ func Clean(library *config.Library) error { patterns := cleanPatterns(library) keepSet := make(map[string]bool) for _, k := range library.Keep { - keepSet[filepath.ToSlash(k)] = true + keepSet[strings.TrimSuffix(filepath.ToSlash(k), "/")] = true } for pattern, useMarker := range patterns { matches, err := filepath.Glob(filepath.Join(library.Output, pattern)) @@ -168,8 +169,20 @@ func isDirNotEmpty(err error) bool { // shouldPreserve returns true if the given slash-separated path should be preserved // based on the keepSet or standard preservation patterns. -func shouldPreserve(path string, keepSet map[string]bool) bool { - return keepSet[path] || isDefaultPreserved(path) +// It also checks if any ancestor directory is in the keepSet. +func shouldPreserve(p string, keepSet map[string]bool) bool { + if keepSet[p] || isDefaultPreserved(p) { + return true + } + + dir := path.Dir(p) + for dir != "." && dir != "/" && dir != "" { + if keepSet[dir] { + return true + } + dir = path.Dir(dir) + } + return false } func isDefaultPreserved(path string) bool { @@ -204,7 +217,7 @@ func shouldCleanMarkerPath(path string, d os.DirEntry) (bool, error) { // hasMarker checks if the file at path contains the auto-generated marker. // It scans the file line by line using [bufio.Reader] and returns true as soon // as the marker is found, avoiding reading the entire file. -func hasMarker(path string) (bool, error) { +func hasMarker(path string) (has bool, err error) { f, err := os.Open(path) if err != nil { return false, err diff --git a/internal/librarian/java/generate.go b/internal/librarian/java/generate.go index 26972398e5c..6b5e52ee7ac 100644 --- a/internal/librarian/java/generate.go +++ b/internal/librarian/java/generate.go @@ -67,7 +67,7 @@ type generateAPIParams struct { } // Generate generates a Java client library. -func Generate(ctx context.Context, cfg *config.Config, library *config.Library, srcs *sources.Sources) error { +func Generate(ctx context.Context, cfg *config.Config, library *config.Library, srcs *sources.Sources, useGoPostprocessor bool) error { if library.Java.GroupID == fakeGroupID { return errUnrecognizedAPI } @@ -110,11 +110,12 @@ func Generate(ctx context.Context, cfg *config.Config, library *config.Library, } if err := postProcessLibrary(ctx, libraryPostProcessParams{ - cfg: cfg, - library: library, - outDir: outdir, - metadata: metadata, - transports: transports, + cfg: cfg, + library: library, + outDir: outdir, + metadata: metadata, + transports: transports, + UseGoPostprocessor: useGoPostprocessor, }); err != nil { return err } diff --git a/internal/librarian/java/generate_test.go b/internal/librarian/java/generate_test.go index c511a55653f..1a4cd04a097 100644 --- a/internal/librarian/java/generate_test.go +++ b/internal/librarian/java/generate_test.go @@ -302,7 +302,11 @@ func TestGenerateAPI(t *testing.T) { t.Fatal(err) } } - apiCfg, err := serviceconfig.Find(googleapisDir, "google/cloud/secretmanager/v1", config.LanguageJava) + absGoogleapisDir, err := filepath.Abs(googleapisDir) + if err != nil { + t.Fatal(err) + } + apiCfg, err := serviceconfig.Find(absGoogleapisDir, "google/cloud/secretmanager/v1", config.LanguageJava) if err != nil { t.Fatal(err) } @@ -310,7 +314,7 @@ func TestGenerateAPI(t *testing.T) { cfg: cfg, api: library.APIs[0], library: library, - srcCfg: sources.NewSourceConfig(&sources.Sources{Googleapis: googleapisDir}, nil), + srcCfg: sources.NewSourceConfig(&sources.Sources{Googleapis: absGoogleapisDir}, nil), outdir: outdir, metadata: &repoMetadata{ NamePretty: "Secret Manager", @@ -367,7 +371,11 @@ func TestGenerateAPI_ProtoOnly(t *testing.T) { t.Fatal(err) } } - apiCfg, err := serviceconfig.Find(googleapisDir, "google/cloud/gkehub/policycontroller/v1beta", config.LanguageJava) + absGoogleapisDir, err := filepath.Abs(googleapisDir) + if err != nil { + t.Fatal(err) + } + apiCfg, err := serviceconfig.Find(absGoogleapisDir, "google/cloud/gkehub/policycontroller/v1beta", config.LanguageJava) if err != nil { t.Fatal(err) } @@ -375,7 +383,7 @@ func TestGenerateAPI_ProtoOnly(t *testing.T) { cfg: cfg, api: library.APIs[0], library: library, - srcCfg: sources.NewSourceConfig(&sources.Sources{Googleapis: googleapisDir}, nil), + srcCfg: sources.NewSourceConfig(&sources.Sources{Googleapis: absGoogleapisDir}, nil), outdir: outdir, metadata: &repoMetadata{ NamePretty: "GKE Hub API", @@ -511,7 +519,11 @@ func TestGenerateAPI_WithAdditionalProtosToGenerateAndCopy(t *testing.T) { t.Fatal(err) } } - apiCfg, err := serviceconfig.Find(googleapisDir, "google/cloud/secretmanager/v1", config.LanguageJava) + absGoogleapisDir, err := filepath.Abs(googleapisDir) + if err != nil { + t.Fatal(err) + } + apiCfg, err := serviceconfig.Find(absGoogleapisDir, "google/cloud/secretmanager/v1", config.LanguageJava) if err != nil { t.Fatal(err) } @@ -519,7 +531,7 @@ func TestGenerateAPI_WithAdditionalProtosToGenerateAndCopy(t *testing.T) { cfg: cfg, api: library.APIs[0], library: library, - srcCfg: sources.NewSourceConfig(&sources.Sources{Googleapis: googleapisDir}, nil), + srcCfg: sources.NewSourceConfig(&sources.Sources{Googleapis: absGoogleapisDir}, nil), outdir: outdir, metadata: &repoMetadata{ NamePretty: "Secret Manager", @@ -640,7 +652,7 @@ func TestGenerateLibrary_Error(t *testing.T) { }, Libraries: []*config.Library{test.library}, } - err := Generate(t.Context(), cfg, test.library, &sources.Sources{Googleapis: googleapisDir}) + err := Generate(t.Context(), cfg, test.library, &sources.Sources{Googleapis: googleapisDir}, false) if !errors.Is(err, test.wantErr) { t.Errorf("generate() error = %v, wantErr %v", err, test.wantErr) } @@ -693,7 +705,7 @@ func TestGenerate_Logic(t *testing.T) { t.Fatal(err) } - err := Generate(t.Context(), cfg, library, &sources.Sources{Googleapis: googleapisDir}) + err := Generate(t.Context(), cfg, library, &sources.Sources{Googleapis: googleapisDir}, false) if err != nil { t.Fatal(err) } @@ -755,7 +767,7 @@ func TestGenerate_ProtoExclusion(t *testing.T) { {Name: rootLibrary, Version: "1.2.3"}, }, } - err := Generate(t.Context(), cfg, library, &sources.Sources{Googleapis: googleapisDir}) + err := Generate(t.Context(), cfg, library, &sources.Sources{Googleapis: googleapisDir}, false) if err != nil { t.Fatal(err) } diff --git a/internal/librarian/java/postgenerate_test.go b/internal/librarian/java/postgenerate_test.go index e097fe25d5b..8b943804b95 100644 --- a/internal/librarian/java/postgenerate_test.go +++ b/internal/librarian/java/postgenerate_test.go @@ -33,7 +33,10 @@ func TestPostGenerate(t *testing.T) { t.Parallel() tmpDir := t.TempDir() // Copy testdata to tmpDir - testdataDir := filepath.Join("testdata", "postgenerate") + testdataDir, err := filepath.Abs(filepath.Join("testdata", "postgenerate")) + if err != nil { + t.Fatal(err) + } if err := copyDir(testdataDir, tmpDir); err != nil { t.Fatal(err) } diff --git a/internal/librarian/java/postprocess.go b/internal/librarian/java/postprocess.go index 1c396a6d37a..54b3e8636e8 100644 --- a/internal/librarian/java/postprocess.go +++ b/internal/librarian/java/postprocess.go @@ -65,9 +65,16 @@ type libraryPostProcessParams struct { outDir string metadata *repoMetadata transports map[string]serviceconfig.Transport + UseGoPostprocessor bool } func postProcessLibrary(ctx context.Context, params libraryPostProcessParams) error { + if params.UseGoPostprocessor { + yamlPath := filepath.Join(params.outDir, "postprocess.yaml") + if _, err := os.Stat(yamlPath); err == nil { + return postProcessLibraryNew(params) + } + } if err := createOrVerifyOwlbotPy(params.outDir); err != nil { return err } diff --git a/internal/librarian/java/postprocess_test.go b/internal/librarian/java/postprocess_test.go index ca9de2f147c..7f7224730f8 100644 --- a/internal/librarian/java/postprocess_test.go +++ b/internal/librarian/java/postprocess_test.go @@ -196,9 +196,13 @@ func TestRestructureModules(t *testing.T) { if err := os.WriteFile(reflectConfigPath, []byte("{}"), 0644); err != nil { t.Fatal(err) } - protoPath := filepath.Join(googleapisDir, "google", "cloud", "secretmanager", "v1", "service.proto") + absGoogleapisDir, err := filepath.Abs(googleapisDir) + if err != nil { + t.Fatal(err) + } + protoPath := filepath.Join(absGoogleapisDir, "google", "cloud", "secretmanager", "v1", "service.proto") - additionalProtoPath := filepath.Join(googleapisDir, "google", "cloud", "oslogin", "common", "common.proto") + additionalProtoPath := filepath.Join(absGoogleapisDir, "google", "cloud", "oslogin", "common", "common.proto") params := postProcessParams{ outDir: tmpDir, library: &config.Library{ @@ -1137,3 +1141,103 @@ func TestCreateOrVerifyOwlbotPy_Error(t *testing.T) { t.Errorf("error = %v, wantErr %v", err, fs.ErrPermission) } } + +func TestPostProcessLibrary_Branching(t *testing.T) { + + t.Run("UseGoPostprocessor false", func(t *testing.T) { + outDir := t.TempDir() + // Create a dummy owlbot.py to avoid failure in createOrVerifyOwlbotPy + if err := os.WriteFile(filepath.Join(outDir, "owlbot.py"), []byte(""), 0755); err != nil { + t.Fatal(err) + } + p := libraryPostProcessParams{ + outDir: outDir, + cfg: &config.Config{ + Libraries: []*config.Library{ + {Name: "google-cloud-java", Version: "1.2.3"}, + }, + }, + library: &config.Library{ + Name: "test-library", + }, + } + err := postProcessLibrary(t.Context(), p) + if err == nil { + t.Fatal("expected error, got nil") + } + if strings.Contains(err.Error(), "postProcessLibraryNew not implemented") { + t.Errorf("expected legacy flow, but got new flow error: %v", err) + } + }) + + t.Run("UseGoPostprocessor true, no yaml", func(t *testing.T) { + outDir := t.TempDir() + if err := os.WriteFile(filepath.Join(outDir, "owlbot.py"), []byte(""), 0755); err != nil { + t.Fatal(err) + } + p := libraryPostProcessParams{ + outDir: outDir, + UseGoPostprocessor: true, + cfg: &config.Config{ + Libraries: []*config.Library{ + {Name: "google-cloud-java", Version: "1.2.3"}, + }, + }, + library: &config.Library{ + Name: "test-library", + }, + } + err := postProcessLibrary(t.Context(), p) + if err == nil { + t.Fatal("expected error, got nil") + } + if strings.Contains(err.Error(), "postProcessLibraryNew not implemented") { + t.Errorf("expected legacy flow, but got new flow error: %v", err) + } + }) + + t.Run("UseGoPostprocessor true, with yaml", func(t *testing.T) { + outDir := t.TempDir() + t.Chdir(outDir) + + if err := os.WriteFile(filepath.Join(outDir, "postprocess.yaml"), []byte(""), 0644); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(outDir, "owl-bot-staging"), 0755); err != nil { + t.Fatal(err) + } + metadata := `{"repo": {"name_pretty": "My API"}}` + if err := os.WriteFile(filepath.Join(outDir, ".repo-metadata.json"), []byte(metadata), 0644); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(outDir, "template"), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(outDir, "template", "README.md.go.tmpl"), []byte("dummy"), 0644); err != nil { + t.Fatal(err) + } + + p := libraryPostProcessParams{ + outDir: outDir, + UseGoPostprocessor: true, + cfg: &config.Config{ + Default: &config.Default{ + Java: &config.JavaModule{ + LibrariesBOMVersion: "1.0.0", + }, + }, + Libraries: []*config.Library{ + {Name: "google-cloud-java", Version: "1.2.3"}, + }, + }, + library: &config.Library{ + Name: "test-library", + Version: "1.2.3", + }, + } + err := postProcessLibrary(t.Context(), p) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + }) +} diff --git a/internal/librarian/java/repometadata_test.go b/internal/librarian/java/repometadata_test.go index c1f42a8ac91..ef94400cf07 100644 --- a/internal/librarian/java/repometadata_test.go +++ b/internal/librarian/java/repometadata_test.go @@ -76,7 +76,10 @@ func TestRepoMetadata_write(t *testing.T) { func TestDeriveRepoMetadata_Overrides(t *testing.T) { t.Parallel() apiPath := "google/cloud/secretmanager/v1" - googleapis := "../../testdata/googleapis" + googleapis, err := filepath.Abs("../../testdata/googleapis") + if err != nil { + t.Fatal(err) + } cfg := sample.Config() cfg.Language = config.LanguageJava diff --git a/internal/sidekick/parser/protobuf.go b/internal/sidekick/parser/protobuf.go index 197a8ac0895..30df14d56ec 100644 --- a/internal/sidekick/parser/protobuf.go +++ b/internal/sidekick/parser/protobuf.go @@ -169,7 +169,6 @@ func runProtoc(files []string, sourceCfg *sources.SourceConfig) ([]byte, error) args := []string{ "--include_imports", "--include_source_info", - "--retain_options", "--descriptor_set_out", tempFile.Name(), } if sourceCfg != nil { From 680b8f6835977a37a6eb301f070db312b2ed329a Mon Sep 17 00:00:00 2001 From: Sophia Yang Date: Sat, 6 Jun 2026 15:54:59 -0400 Subject: [PATCH 002/108] fix(java): ensure zero diff for README whitespace --- .../librarian/java/template/README.md.go.tmpl | 252 ++++++++++++++++++ 1 file changed, 252 insertions(+) create mode 100644 internal/librarian/java/template/README.md.go.tmpl diff --git a/internal/librarian/java/template/README.md.go.tmpl b/internal/librarian/java/template/README.md.go.tmpl new file mode 100644 index 00000000000..3a4a7cc0a88 --- /dev/null +++ b/internal/librarian/java/template/README.md.go.tmpl @@ -0,0 +1,252 @@ +# Google {{ .Metadata.Repo.NamePretty }} Client for Java + +Java idiomatic client for [{{ .Metadata.Repo.NamePretty }}][product-docs]. + +[![Maven][maven-version-image]][maven-version-link] +![Stability][stability-image] + +- [Product Documentation][product-docs] +- [Client Library Documentation][javadocs] + +{{ if and .Metadata.Partials .Metadata.Partials.DeprecationWarning }}{{ .Metadata.Partials.DeprecationWarning }} +{{ end }}{{ if eq .Metadata.Repo.ReleaseLevel "preview" }}> Note: This client is a work-in-progress, and may occasionally +> make backwards-incompatible changes. +{{ end }}{{ if .MigratedSplitRepo }}:bus: In October 2022, this library has moved to +[google-cloud-java/{{ .Metadata.Repo.RepoShort }}]( +https://github.com/googleapis/google-cloud-java/tree/main/{{ .Metadata.Repo.RepoShort }}). +This repository will be archived in the future. +Future releases will appear in the new repository (https://github.com/googleapis/google-cloud-java/releases). +The Maven artifact coordinates (`{{ .GroupID }}:{{ .ArtifactID }}`) remain the same. +{{ end }} +## Quickstart + +{{ if and .Metadata.Snippets (index .Metadata.Snippets (print .Metadata.Repo.APIShortname "_install_with_bom")) -}} +If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file: + +```xml +{{ index .Metadata.Snippets (print .Metadata.Repo.APIShortname "_install_with_bom") }} +``` + +If you are using Maven without the BOM, add this to your dependencies: +{{- else if and .Monorepo (or (eq .GroupID "com.google.cloud") (eq .GroupID "com.google.analytics") (eq .GroupID "com.google.area120")) }} +If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file: + +```xml + + + + com.google.cloud + libraries-bom + {{ .Metadata.LibrariesBOMVersion }} + pom + import + + + + + + + {{ .GroupID }} + {{ .ArtifactID }} + + +``` + +If you are using Maven without the BOM, add this to your dependencies: +{{ else }}If you are using Maven, add this to your pom.xml file: +{{ end }} + + +```xml +{{ if and .Metadata.Snippets (index .Metadata.Snippets (print .Metadata.Repo.APIShortname "_install_without_bom")) -}} +{{ index .Metadata.Snippets (print .Metadata.Repo.APIShortname "_install_without_bom") }} +{{- else -}} + + {{ .GroupID }} + {{ .ArtifactID }} + {{ .Version }} + +{{- end }} +``` + +{{ if and .Metadata.Snippets (index .Metadata.Snippets (print .Metadata.Repo.APIShortname "_install_with_bom")) -}} +If you are using Gradle 5.x or later, add this to your dependencies: + +```Groovy +implementation platform('com.google.cloud:libraries-bom:{{ .Metadata.LibrariesBOMVersion }}') + +implementation '{{ .GroupID }}:{{ .ArtifactID }}' +``` +{{ end }}If you are using Gradle without BOM, add this to your dependencies: + +```Groovy +implementation '{{ .GroupID }}:{{ .ArtifactID }}:{{ .Version }}' +``` + +If you are using SBT, add this to your dependencies: + +```Scala +libraryDependencies += "{{ .GroupID }}" % "{{ .ArtifactID }}" % "{{ .Version }}" +``` + +## Authentication + +See the [Authentication][authentication] section in the base directory's README. + +## Authorization + +The client application making API calls must be granted [authorization scopes][auth-scopes] required for the desired {{ .Metadata.Repo.NamePretty }} APIs, and the authenticated principal must have the [IAM role(s)][predefined-iam-roles] required to access GCP resources using the {{ .Metadata.Repo.NamePretty }} API calls. + +## Getting Started + +### Prerequisites + +You will need a [Google Cloud Platform Console][developer-console] project with the {{ .Metadata.Repo.NamePretty }} [API enabled][enable-api]. +{{ if .Metadata.Repo.RequiresBilling }}You will need to [enable billing][enable-billing] to use Google {{ .Metadata.Repo.NamePretty }}.{{ end }} +[Follow these instructions][create-project] to get your project set up. You will also need to set up the local development environment by +[installing the Google Cloud Command Line Interface][cloud-cli] and running the following commands in command line: +`gcloud auth login` and `gcloud config set project [YOUR PROJECT ID]`. + +### Installation and setup + +You'll need to obtain the `{{ .ArtifactID }}` library. See the [Quickstart](#quickstart) section +to add `{{ .ArtifactID }}` as a dependency in your code. + +## About {{ .Metadata.Repo.NamePretty }} + +{{ if and .Metadata.Partials .Metadata.Partials.About }} +{{ .Metadata.Partials.About }} +{{ else }} +[{{ .Metadata.Repo.NamePretty }}][product-docs] {{ .Metadata.Repo.APIDescription }} + +See the [{{ .Metadata.Repo.NamePretty }} client library docs][javadocs] to learn how to +use this {{ .Metadata.Repo.NamePretty }} Client Library. +{{ end -}} +{{ if and .Metadata.Partials .Metadata.Partials.CustomContent }} + +{{ .Metadata.Partials.CustomContent }} +{{ end }} + +{{ if .Metadata.Samples }} +## Samples + +Samples are in the [`samples/`](https://github.com/{{ .Metadata.Repo.Repo }}/tree/main/samples) directory. + +| Sample | Source Code | Try it | +| --------------------------- | --------------------------------- | ------ | +{{ range .Metadata.Samples }}| {{ .Title }} | [source code](https://github.com/{{ $.Metadata.Repo.Repo }}/blob/main/{{ .File }}) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/{{ $.Metadata.Repo.Repo }}&page=editor&open_in_editor={{ .File }}) | +{{ end }} +{{ end }} + +## Troubleshooting + +To get help, follow the instructions in the [shared Troubleshooting document][troubleshooting]. + +{{ if .Metadata.Repo.Transport }}## Transport + +{{ if eq .Metadata.Repo.Transport "grpc" }}{{ .Metadata.Repo.NamePretty }} uses gRPC for the transport layer. +{{ else if eq .Metadata.Repo.Transport "http" }}{{ .Metadata.Repo.NamePretty }} uses HTTP/JSON for the transport layer. +{{ else if eq .Metadata.Repo.Transport "both" }}{{ .Metadata.Repo.NamePretty }} uses both gRPC and HTTP/JSON for the transport layer. +{{ end }} +{{ end }}## Supported Java Versions + +Java {{ .Metadata.MinJavaVersion }} or above is required for using this client. + +Google's Java client libraries, +[Google Cloud Client Libraries][cloudlibs] +and +[Google Cloud API Libraries][apilibs], +follow the +[Oracle Java SE support roadmap][oracle] +(see the Oracle Java SE Product Releases section). + +### For new development + +In general, new feature development occurs with support for the lowest Java +LTS version covered by Oracle's Premier Support (which typically lasts 5 years +from initial General Availability). If the minimum required JVM for a given +library is changed, it is accompanied by a [semver][semver] major release. + +Java 11 and (in September 2021) Java 17 are the best choices for new +development. + +### Keeping production systems current + +Google tests its client libraries with all current LTS versions covered by +Oracle's Extended Support (which typically lasts 8 years from initial +General Availability). + +#### Legacy support + +Google's client libraries support legacy versions of Java runtimes with long +term stable libraries that don't receive feature updates on a best efforts basis +as it may not be possible to backport all patches. + +Google provides updates on a best efforts basis to apps that continue to use +Java 7, though apps might need to upgrade to current versions of the library +that supports their JVM. + +#### Where to find specific information + +The latest versions and the supported Java versions are identified on +the individual GitHub repository `github.com/GoogleAPIs/java-SERVICENAME` +and on [google-cloud-java][g-c-j]. + +## Versioning + +{{ if and .Metadata.Partials .Metadata.Partials.Versioning }} +{{ .Metadata.Partials.Versioning }} +{{ else }} +This library follows [Semantic Versioning](http://semver.org/). + +{{ if eq .Metadata.Repo.ReleaseLevel "preview" }} +It is currently in major version zero (``0.y.z``), which means that anything may change at any time +and the public API should not be considered stable. +{{ end }}{{ end }} + +## Contributing + +{{ if and .Metadata.Partials .Metadata.Partials.Contributing }} +{{ .Metadata.Partials.Contributing }} +{{ else }} +Contributions to this library are always welcome and highly encouraged. + +See [CONTRIBUTING][contributing] for more information how to get started. + +Please note that this project is released with a Contributor Code of Conduct. By participating in +this project you agree to abide by its terms. See [Code of Conduct][code-of-conduct] for more +information. +{{ end }} + +## License + +Apache 2.0 - See [LICENSE][license] for more information. + +Java is a registered trademark of Oracle and/or its affiliates. + +[product-docs]: {{ .Metadata.Repo.ProductDocumentation }} +[javadocs]: {{ .Metadata.Repo.ClientDocumentation }} +[stability-image]: https://img.shields.io/badge/stability-{{ if eq .Metadata.Repo.ReleaseLevel "stable" }}stable-green{{ else if eq .Metadata.Repo.ReleaseLevel "preview" }}preview-yellow{{ else }}unknown-red{{ end }} +[maven-version-image]: https://img.shields.io/maven-central/v/{{ .GroupID }}/{{ .ArtifactID }}.svg +[maven-version-link]: https://central.sonatype.com/artifact/{{ .GroupID }}/{{ .ArtifactID }}/{{ .Version }} +[authentication]: https://github.com/googleapis/google-cloud-java#authentication +[auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes +[predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles +[iam-policy]: https://cloud.google.com/iam/docs/overview#cloud-iam-policy +[developer-console]: https://console.developers.google.com/ +[create-project]: https://cloud.google.com/resource-manager/docs/creating-managing-projects +[cloud-cli]: https://cloud.google.com/cli +[troubleshooting]: https://github.com/googleapis/google-cloud-java/blob/main/TROUBLESHOOTING.md +[contributing]: https://github.com/{{ .Metadata.Repo.Repo }}/blob/main/CONTRIBUTING.md +[code-of-conduct]: https://github.com/{{ .Metadata.Repo.Repo }}/blob/main/CODE_OF_CONDUCT.md#contributor-code-of-conduct +[license]: https://github.com/{{ .Metadata.Repo.Repo }}/blob/main/LICENSE +{{ if .Metadata.Repo.RequiresBilling }}[enable-billing]: https://cloud.google.com/apis/docs/getting-started#enabling_billing{{ end }} +{{ if .Metadata.Repo.APIID }}[enable-api]: https://console.cloud.google.com/flows/enableapi?apiid={{ .Metadata.Repo.APIID }}{{ end }} +[libraries-bom]: https://github.com/GoogleCloudPlatform/cloud-opensource-java/wiki/The-Google-Cloud-Platform-Libraries-BOM +[shell_img]: https://gstatic.com/cloudssh/images/open-btn.png + +[semver]: https://semver.org/ +[cloudlibs]: https://cloud.google.com/apis/docs/client-libraries-explained +[apilibs]: https://cloud.google.com/apis/docs/client-libraries-explained#google_api_client_libraries +[oracle]: https://www.oracle.com/java/technologies/java-se-support-roadmap.html +[g-c-j]: http://github.com/googleapis/google-cloud-java From 68eaa857acf69a2d424fcc48f45f73fd46c0fca9 Mon Sep 17 00:00:00 2001 From: Sophia Yang Date: Sat, 6 Jun 2026 16:22:14 -0400 Subject: [PATCH 003/108] fix(java): remove left hyphen from else if in README template to prevent newline stripping --- internal/librarian/java/postprocess.go | 10 +++++----- internal/librarian/java/template/README.md.go.tmpl | 3 +-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/internal/librarian/java/postprocess.go b/internal/librarian/java/postprocess.go index 54b3e8636e8..7468809a229 100644 --- a/internal/librarian/java/postprocess.go +++ b/internal/librarian/java/postprocess.go @@ -60,11 +60,11 @@ type postProcessParams struct { } type libraryPostProcessParams struct { - cfg *config.Config - library *config.Library - outDir string - metadata *repoMetadata - transports map[string]serviceconfig.Transport + cfg *config.Config + library *config.Library + outDir string + metadata *repoMetadata + transports map[string]serviceconfig.Transport UseGoPostprocessor bool } diff --git a/internal/librarian/java/template/README.md.go.tmpl b/internal/librarian/java/template/README.md.go.tmpl index 3a4a7cc0a88..1f50d9473b0 100644 --- a/internal/librarian/java/template/README.md.go.tmpl +++ b/internal/librarian/java/template/README.md.go.tmpl @@ -28,7 +28,7 @@ If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file: ``` If you are using Maven without the BOM, add this to your dependencies: -{{- else if and .Monorepo (or (eq .GroupID "com.google.cloud") (eq .GroupID "com.google.analytics") (eq .GroupID "com.google.area120")) }} +{{ else if and .Monorepo (or (eq .GroupID "com.google.cloud") (eq .GroupID "com.google.analytics") (eq .GroupID "com.google.area120")) }} If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file: ```xml @@ -56,7 +56,6 @@ If you are using Maven without the BOM, add this to your dependencies: {{ else }}If you are using Maven, add this to your pom.xml file: {{ end }} - ```xml {{ if and .Metadata.Snippets (index .Metadata.Snippets (print .Metadata.Repo.APIShortname "_install_without_bom")) -}} {{ index .Metadata.Snippets (print .Metadata.Repo.APIShortname "_install_without_bom") }} From eb2265548375c1c3174715d014c7da47626ec1e8 Mon Sep 17 00:00:00 2001 From: Sophia Yang Date: Sat, 6 Jun 2026 16:27:18 -0400 Subject: [PATCH 004/108] fix(java): match legacy Troubleshooting spacing exactly for missing samples --- internal/librarian/java/template/README.md.go.tmpl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/librarian/java/template/README.md.go.tmpl b/internal/librarian/java/template/README.md.go.tmpl index 1f50d9473b0..d9177ec037e 100644 --- a/internal/librarian/java/template/README.md.go.tmpl +++ b/internal/librarian/java/template/README.md.go.tmpl @@ -137,6 +137,9 @@ Samples are in the [`samples/`](https://github.com/{{ .Metadata.Repo.Repo }}/tre {{ end }} {{ end }} +{{ if not .Metadata.Samples }} + +{{ end -}} ## Troubleshooting To get help, follow the instructions in the [shared Troubleshooting document][troubleshooting]. From f8e57c6a522306832a671600740ae460a26ea636 Mon Sep 17 00:00:00 2001 From: Sophia Yang Date: Sun, 7 Jun 2026 10:49:08 -0400 Subject: [PATCH 005/108] fix: fallback to .yml for readme partials and save postprocess modernization work --- internal/filesystem/filesystem.go | 57 ++++ internal/librarian/java/generate.go | 45 +-- internal/librarian/java/postprocess.go | 95 +++++-- internal/librarian/java/postprocess_new.go | 148 ++++++++++ .../librarian/java/postprocess_new_test.go | 146 ++++++++++ internal/librarian/java/postprocess_test.go | 10 +- internal/librarian/java/samples.go | 260 +++++++++++++++++ internal/librarian/java/samples_test.go | 164 +++++++++++ .../librarian/java/template/README.md.tmpl | 266 ++++++++++++++++++ internal/librarian/java/template_render.go | 232 +++++++++++++++ .../librarian/java/template_render_test.go | 132 +++++++++ internal/postprocessing/config.go | 73 +++++ internal/postprocessing/config_test.go | 132 +++++++++ internal/postprocessing/fileops.go | 77 +++++ internal/postprocessing/fileops_test.go | 69 +++-- 15 files changed, 1845 insertions(+), 61 deletions(-) create mode 100644 internal/librarian/java/postprocess_new.go create mode 100644 internal/librarian/java/postprocess_new_test.go create mode 100644 internal/librarian/java/samples.go create mode 100644 internal/librarian/java/samples_test.go create mode 100644 internal/librarian/java/template/README.md.tmpl create mode 100644 internal/librarian/java/template_render.go create mode 100644 internal/librarian/java/template_render_test.go create mode 100644 internal/postprocessing/config.go create mode 100644 internal/postprocessing/config_test.go diff --git a/internal/filesystem/filesystem.go b/internal/filesystem/filesystem.go index 079671c54b0..7d95dc8a90e 100644 --- a/internal/filesystem/filesystem.go +++ b/internal/filesystem/filesystem.go @@ -60,6 +60,63 @@ func MoveAndMerge(sourceDir, targetDir string) error { return nil } +// MoveAndMergeWithKeep moves entries from sourceDir to targetDir. +// It merges directories recursively if they exist in both source and target. +// If an entry in sourceDir is a file that already exists in targetDir: +// - If keepFunc is not nil and returns true for the path relative to libraryRoot, it is preserved (source is deleted). +// - Otherwise, the target is overwritten. +func MoveAndMergeWithKeep(sourceDir, targetDir, libraryRoot string, keepFunc func(string) bool) error { + entries, err := os.ReadDir(sourceDir) + if err != nil { + return err + } + for _, entry := range entries { + oldPath := filepath.Join(sourceDir, entry.Name()) + newPath := filepath.Join(targetDir, entry.Name()) + if entry.IsDir() { + if _, err := os.Stat(newPath); err == nil { + // Destination exists, merge contents. + if err := MoveAndMergeWithKeep(oldPath, newPath, libraryRoot, keepFunc); err != nil { + return err + } + // Remove the now-empty source directory after successful merge. + if err := os.Remove(oldPath); err != nil { + return err + } + continue + } + } else { + if _, err := os.Stat(newPath); err == nil { + rel, err := filepath.Rel(libraryRoot, newPath) + if err != nil { + return err + } + if keepFunc != nil && keepFunc(filepath.ToSlash(rel)) { + // Preserve the target file, remove the source file. + if err := os.Remove(oldPath); err != nil { + return err + } + continue + } + // Overwrite existing file + if err := os.Remove(newPath); err != nil { + return err + } + } + } + if err := os.Rename(oldPath, newPath); err != nil { + // Fallback for cross-device links + if err := CopyFile(oldPath, newPath); err != nil { + return err + } + if err := os.Remove(oldPath); err != nil { + return err + } + } + } + return nil +} + // CopyFile copies a file from src to dest. func CopyFile(src, dest string) error { in, err := os.Open(src) diff --git a/internal/librarian/java/generate.go b/internal/librarian/java/generate.go index 6b5e52ee7ac..dbf0ef07c1b 100644 --- a/internal/librarian/java/generate.go +++ b/internal/librarian/java/generate.go @@ -57,13 +57,14 @@ var ( ) type generateAPIParams struct { - cfg *config.Config - api *config.API - library *config.Library - srcCfg *sources.SourceConfig - outdir string - metadata *repoMetadata - apiCfg *serviceconfig.API + cfg *config.Config + api *config.API + library *config.Library + srcCfg *sources.SourceConfig + outdir string + metadata *repoMetadata + apiCfg *serviceconfig.API + useGoPostprocessor bool } // Generate generates a Java client library. @@ -97,13 +98,14 @@ func Generate(ctx context.Context, cfg *config.Config, library *config.Library, transports[api.Path] = apiCfg.Transport(config.LanguageJava) // metadata is needed for pom.xml generation in post process if err := generateAPI(ctx, generateAPIParams{ - cfg: cfg, - api: api, - library: library, - srcCfg: srcCfg, - outdir: outdir, - metadata: metadata, - apiCfg: apiCfg, + cfg: cfg, + api: api, + library: library, + srcCfg: srcCfg, + outdir: outdir, + metadata: metadata, + apiCfg: apiCfg, + useGoPostprocessor: useGoPostprocessor, }); err != nil { return fmt.Errorf("failed to generate api %q: %w", api.Path, err) } @@ -139,13 +141,14 @@ func generateAPI(ctx context.Context, params generateAPIParams) error { additionalProtosToCopyRel := processAdditionalProtos(javaAPI, googleapisDir) postParams := postProcessParams{ - cfg: params.cfg, - library: params.library, - javaAPI: javaAPI, - metadata: params.metadata, - outDir: params.outdir, - apiBase: deriveAPIBase(params.library, params.api.Path), - includeSamples: *javaAPI.Samples, + cfg: params.cfg, + library: params.library, + javaAPI: javaAPI, + metadata: params.metadata, + outDir: params.outdir, + apiBase: deriveAPIBase(params.library, params.api.Path), + includeSamples: *javaAPI.Samples, + UseGoPostprocessor: params.useGoPostprocessor, } gapicDir := postParams.gapicDir() gRPCDir := postParams.gRPCDir() diff --git a/internal/librarian/java/postprocess.go b/internal/librarian/java/postprocess.go index 7468809a229..768e06aee25 100644 --- a/internal/librarian/java/postprocess.go +++ b/internal/librarian/java/postprocess.go @@ -49,14 +49,15 @@ type protoFileToCopy struct { } type postProcessParams struct { - cfg *config.Config - library *config.Library - javaAPI *config.JavaAPI - metadata *repoMetadata - outDir string - apiBase string - protosToCopy []protoFileToCopy - includeSamples bool + cfg *config.Config + library *config.Library + javaAPI *config.JavaAPI + metadata *repoMetadata + outDir string + apiBase string + protosToCopy []protoFileToCopy + includeSamples bool + UseGoPostprocessor bool } type libraryPostProcessParams struct { @@ -72,7 +73,18 @@ func postProcessLibrary(ctx context.Context, params libraryPostProcessParams) er if params.UseGoPostprocessor { yamlPath := filepath.Join(params.outDir, "postprocess.yaml") if _, err := os.Stat(yamlPath); err == nil { - return postProcessLibraryNew(params) + if err := postProcessLibraryNew(params); err != nil { + return err + } + + monorepoVersion, err := findMonorepoVersion(params.cfg) + if err != nil { + return err + } + if err := syncPOMs(params.library, params.outDir, monorepoVersion, params.metadata, params.transports); err != nil { + return fmt.Errorf("%w: %w", errSyncPOMs, err) + } + return nil } } if err := createOrVerifyOwlbotPy(params.outDir); err != nil { @@ -132,6 +144,38 @@ func postProcessAPI(ctx context.Context, params postProcessParams) error { if err := copyFiles(params); err != nil { return fmt.Errorf("failed to copy files: %w", err) } + coords := params.coords() + + if params.UseGoPostprocessor { + yamlPath := filepath.Join(params.outDir, "postprocess.yaml") + if _, err := os.Stat(yamlPath); err == nil { + keepSet := make(map[string]bool) + for _, k := range params.library.Keep { + keepSet[strings.TrimSuffix(filepath.ToSlash(k), "/")] = true + } + if err := restructureModules(params, params.outDir, keepSet, params.outDir); err != nil { + return fmt.Errorf("failed to restructure direct to outDir: %w", err) + } + + protoModuleRepoRoot := filepath.Join(params.outDir, coords.Proto.ArtifactID) + shouldGenerate, err := clirrIgnoreShouldGenerate(coords.Proto.ArtifactID, protoModuleRepoRoot, params.javaAPI.Monolithic) + if err != nil { + return fmt.Errorf("failed to check for clirr ignore file: %w", err) + } + if shouldGenerate { + if err := generateClirrIgnore(protoModuleRepoRoot); err != nil { + return fmt.Errorf("failed to generate clirr ignore file: %w", err) + } + } + + // Cleanup intermediate protoc output directory + if err := os.RemoveAll(filepath.Join(params.outDir, params.apiBase)); err != nil { + return fmt.Errorf("failed to cleanup intermediate files: %w", err) + } + return nil + } + } + if err := restructureToStaging(params); err != nil { return fmt.Errorf("failed to restructure to staging: %w", err) } @@ -139,7 +183,6 @@ func postProcessAPI(ctx context.Context, params postProcessParams) error { // Generate clirr-ignored-differences.xml for the proto module. // We target the staging directory because runOwlBot hasn't moved the files // to their final destination yet. - coords := params.coords() protoModuleRepoRoot := filepath.Join(params.outDir, coords.Proto.ArtifactID) shouldGenerate, err := clirrIgnoreShouldGenerate(coords.Proto.ArtifactID, protoModuleRepoRoot, params.javaAPI.Monolithic) if err != nil { @@ -274,7 +317,7 @@ func restructureToStaging(params postProcessParams) error { if err := os.MkdirAll(destRoot, 0755); err != nil { return fmt.Errorf("failed to create staging directory: %w", err) } - return restructureModules(params, destRoot) + return restructureModules(params, destRoot, nil, "") } type moveAction struct { @@ -282,14 +325,34 @@ type moveAction struct { description string } -func restructure(actions []moveAction) error { +func restructure(actions []moveAction, keepSet map[string]bool, libraryRoot string) error { for _, action := range actions { if _, err := os.Stat(action.src); err == nil { if err := os.MkdirAll(action.dest, 0755); err != nil { return fmt.Errorf("failed to create directory %s: %w", action.dest, err) } - if err := filesystem.MoveAndMerge(action.src, action.dest); err != nil { - return fmt.Errorf("failed to move %s: %w", action.description, err) + if keepSet != nil { + keepFunc := func(p string) bool { + if shouldPreserve(p, keepSet) { + return true + } + // Also preserve existing files that lack the auto-generated marker. + destPath := filepath.Join(libraryRoot, p) + if _, err := os.Stat(destPath); err == nil { + isGen, err := hasMarker(destPath) + if err == nil && !isGen { + return true + } + } + return false + } + if err := filesystem.MoveAndMergeWithKeep(action.src, action.dest, libraryRoot, keepFunc); err != nil { + return fmt.Errorf("failed to move with keep %s: %w", action.description, err) + } + } else { + if err := filesystem.MoveAndMerge(action.src, action.dest); err != nil { + return fmt.Errorf("failed to move %s: %w", action.description, err) + } } } } @@ -299,7 +362,7 @@ func restructure(actions []moveAction) error { // restructureModules moves the generated code from the temporary versioned directory // tree into the destination root directory for GAPIC, Proto, gRPC, and samples. // It also copies the relevant proto files into the proto module. -func restructureModules(params postProcessParams, destRoot string) error { +func restructureModules(params postProcessParams, destRoot string, keepSet map[string]bool, libraryRoot string) error { coords := params.coords() tempProtoSrcDir := params.protoDir() if params.library.Name != commonProtosLibrary { @@ -365,7 +428,7 @@ func restructureModules(params postProcessParams, destRoot string) error { description: "samples", }) } - if err := restructure(actions); err != nil { + if err := restructure(actions, keepSet, libraryRoot); err != nil { return err } // Copy proto files to proto-*/src/main/proto diff --git a/internal/librarian/java/postprocess_new.go b/internal/librarian/java/postprocess_new.go new file mode 100644 index 00000000000..231c608ca60 --- /dev/null +++ b/internal/librarian/java/postprocess_new.go @@ -0,0 +1,148 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package java + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/bmatcuk/doublestar/v4" + "github.com/googleapis/librarian/internal/filesystem" + "github.com/googleapis/librarian/internal/postprocessing" +) + +// postProcessLibraryNew implements the new postprocessing flow, bypassing owlbot.py. +// It applies operations from postprocess.yaml, and renders the README.md directly +// on the generated files in their final destinations. +func postProcessLibraryNew(p libraryPostProcessParams) error { + // 1. Load postprocess.yaml and apply operations + postprocessYamlPath := filepath.Join(p.outDir, "postprocess.yaml") + if _, err := os.Stat(postprocessYamlPath); err == nil { + cfg, err := postprocessing.ParseConfig(postprocessYamlPath) + if err != nil { + return fmt.Errorf("failed to parse postprocess.yaml: %w", err) + } + + // 1. Apply Copies + for _, c := range cfg.CopyFile { + srcAbs := filepath.Join(p.outDir, c.Src) + dstAbs := filepath.Join(p.outDir, c.Dst) + if err := filesystem.CopyFile(srcAbs, dstAbs); err != nil { + return fmt.Errorf("failed to copy file from %s to %s: %w", c.Src, c.Dst, err) + } + } + + // 2. Apply Removes + for _, rem := range cfg.RemoveFile { + absPath := filepath.Join(p.outDir, rem) + if err := postprocessing.RemoveFile(absPath); err != nil { + return fmt.Errorf("failed to remove file %s: %w", rem, err) + } + } + + // 3. Apply Replacements + for _, r := range cfg.Replace { + absPath := filepath.Join(p.outDir, r.Path) + if err := postprocessing.Replace(absPath, r.Original, r.Replacement); err != nil { + return fmt.Errorf("failed to apply replacement in %s: %w", r.Path, err) + } + } + + // 4. Apply Regex Replacements + for _, r := range cfg.ReplaceRegex { + absPath := filepath.Join(p.outDir, r.Path) + if err := postprocessing.ReplaceRegex(absPath, r.Pattern, r.Replacement); err != nil { + return fmt.Errorf("failed to apply regex replacement in %s: %w", r.Path, err) + } + } + + // 5. Apply Method Operations + for _, mo := range cfg.MethodOperations { + files, err := resolveGlobs(p.outDir, mo.Path) + if err != nil { + return fmt.Errorf("failed to resolve glob for %s: %w", mo.Path, err) + } + for _, file := range files { + switch mo.Action { + case "delete": + if err := postprocessing.DeleteFunc(file, mo.FuncName, "java"); err != nil { + if strings.Contains(err.Error(), "not found") { + continue + } + return fmt.Errorf("failed to delete method %q in %s: %w", mo.FuncName, file, err) + } + case "duplicate": + if err := postprocessing.DuplicateMethod(file, mo.FuncName, mo.NewName, "java"); err != nil { + if strings.Contains(err.Error(), "not found") { + continue + } + return fmt.Errorf("failed to duplicate method %q in %s: %w", mo.FuncName, file, err) + } + case "deprecate": + if err := postprocessing.DeprecateMethod(file, mo.FuncName, mo.DeprecationMessage, "java"); err != nil { + if strings.Contains(err.Error(), "not found") { + continue + } + return fmt.Errorf("failed to deprecate method %q in %s: %w", mo.FuncName, file, err) + } + default: + return fmt.Errorf("unsupported method operation action %q", mo.Action) + } + } + } + } + + // 4. Render README.md + templatePath := filepath.Join("template", "README.md.go.tmpl") + if _, err := os.Stat(templatePath); err != nil { + // Fallback to absolute path for this workspace + templatePath = "/usr/local/google/home/sophieeee/workspace/owlbot-modernization/librarian/internal/librarian/java/template/README.md.go.tmpl" + } + + libraryVersion, err := deriveLastReleasedVersion(p.library.Version) + if err != nil { + return fmt.Errorf("failed to derive library version: %w", err) + } + + if p.cfg == nil { + return fmt.Errorf("cfg is nil") + } + if p.cfg.Default == nil { + return fmt.Errorf("cfg.Default is nil") + } + if p.cfg.Default.Java == nil { + return fmt.Errorf("cfg.Default.Java is nil") + } + + bomVersion, err := findBOMVersion(p.cfg) + if err != nil { + return fmt.Errorf("failed to find BOM version: %w", err) + } + + if err := RenderREADME(p.outDir, templatePath, bomVersion, libraryVersion); err != nil { + return fmt.Errorf("failed to render README: %w", err) + } + + return nil +} + +func resolveGlobs(outDir, pathPattern string) ([]string, error) { + if strings.ContainsAny(pathPattern, "*?[]{}") { + return doublestar.FilepathGlob(filepath.Join(outDir, pathPattern)) + } + return []string{filepath.Join(outDir, pathPattern)}, nil +} diff --git a/internal/librarian/java/postprocess_new_test.go b/internal/librarian/java/postprocess_new_test.go new file mode 100644 index 00000000000..5bfe8f265a1 --- /dev/null +++ b/internal/librarian/java/postprocess_new_test.go @@ -0,0 +1,146 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package java + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/googleapis/librarian/internal/config" +) + +func TestPostProcessLibraryNew(t *testing.T) { + tmpDir := t.TempDir() + t.Chdir(tmpDir) + + // Setup structure directly in outDir + destDir := filepath.Join(tmpDir, "my-module", "src", "main", "java") + if err := os.MkdirAll(destDir, 0755); err != nil { + t.Fatal(err) + } + + fileContent := `package com.example; +public class File { + public void oldFunc() {} + public void toDelete() { + System.out.println("delete me"); + } +}` + filePath := filepath.Join(destDir, "File.java") + if err := os.WriteFile(filePath, []byte(fileContent), 0644); err != nil { + t.Fatal(err) + } + + // Setup postprocess.yaml + postprocessYaml := ` +replace: + - path: "**/File.java" + original: "oldFunc" + replacement: "newFunc" +method_operations: + - path: "**/File.java" + action: delete + func_name: "public void toDelete()" + - path: "**/File.java" + action: duplicate + func_name: "public void newFunc()" + new_name: "newFuncCopy" + - path: "**/File.java" + action: deprecate + func_name: "public void newFuncCopy()" + deprecation_message: "Use newFunc instead." +` + if err := os.WriteFile(filepath.Join(tmpDir, "postprocess.yaml"), []byte(postprocessYaml), 0644); err != nil { + t.Fatal(err) + } + + // Setup .repo-metadata.json + metadata := `{ + "repo": { + "name_pretty": "My API", + "distribution_name": "com.google.cloud:google-cloud-myapi", + "repo": "googleapis/google-cloud-java" + }, + "library_version": "1.2.3" +}` + if err := os.WriteFile(filepath.Join(tmpDir, ".repo-metadata.json"), []byte(metadata), 0644); err != nil { + t.Fatal(err) + } + + // Setup template + if err := os.MkdirAll(filepath.Join(tmpDir, "template"), 0755); err != nil { + t.Fatal(err) + } + templateContent := `# {{ .Metadata.Repo.NamePretty }}` + if err := os.WriteFile(filepath.Join(tmpDir, "template", "README.md.go.tmpl"), []byte(templateContent), 0644); err != nil { + t.Fatal(err) + } + + p := libraryPostProcessParams{ + outDir: tmpDir, + cfg: &config.Config{ + Default: &config.Default{ + Java: &config.JavaModule{ + LibrariesBOMVersion: "1.0.0", + }, + }, + }, + library: &config.Library{ + Version: "1.2.3", + }, + } + + err := postProcessLibraryNew(p) + if err != nil { + t.Fatal(err) + } + + // Verify File was modified + content, err := os.ReadFile(filePath) + if err != nil { + t.Fatal(err) + } + sContent := string(content) + if !strings.Contains(sContent, "newFunc") { + t.Errorf("Replacement was not applied. Content: %s", sContent) + } + if strings.Contains(sContent, "toDelete") { + t.Errorf("Delete function was not applied. Content: %s", sContent) + } + if !strings.Contains(sContent, "newFuncCopy") { + t.Errorf("Duplicate method operation was not applied. Content: %s", sContent) + } + if !strings.Contains(sContent, "@Deprecated\n\tpublic void newFuncCopy()") { + t.Errorf("@Deprecated annotation was not applied correctly. Content: %s", sContent) + } + if !strings.Contains(sContent, "* @deprecated Use newFunc instead.") { + t.Errorf("Javadoc deprecation tag was not applied correctly. Content: %s", sContent) + } + + // Verify README was rendered + readmePath := filepath.Join(tmpDir, "README.md") + if _, err := os.Stat(readmePath); err != nil { + t.Errorf("README.md was not rendered: %v", err) + } + readmeContent, err := os.ReadFile(readmePath) + if err != nil { + t.Fatal(err) + } + if string(readmeContent) != "# My API" { + t.Errorf("README content mismatch. Got: %s, expected: # My API", string(readmeContent)) + } +} diff --git a/internal/librarian/java/postprocess_test.go b/internal/librarian/java/postprocess_test.go index 7f7224730f8..fd262832099 100644 --- a/internal/librarian/java/postprocess_test.go +++ b/internal/librarian/java/postprocess_test.go @@ -227,7 +227,7 @@ func TestRestructureModules(t *testing.T) { javaAPI: &config.JavaAPI{}, } destRoot := filepath.Join(tmpDir, "dest") - if err := restructureModules(params, destRoot); err != nil { + if err := restructureModules(params, destRoot, nil, ""); err != nil { t.Fatal(err) } @@ -274,7 +274,7 @@ func TestRestructureModules_CommonProtos(t *testing.T) { }, } destRoot := filepath.Join(tmpDir, "dest") - if err := restructureModules(params, destRoot); err != nil { + if err := restructureModules(params, destRoot, nil, ""); err != nil { t.Fatal(err) } wantPath := filepath.Join(destRoot, "proto-google-common-protos", "src", "main", "java", "com", "google", "cloud", "location", "LocationsProto.java") @@ -302,7 +302,7 @@ func TestRestructureModules_ShouldRemoveClasses(t *testing.T) { javaAPI: &config.JavaAPI{}, } destRoot := filepath.Join(tmpDir, "dest") - if err := restructureModules(params, destRoot); err != nil { + if err := restructureModules(params, destRoot, nil, ""); err != nil { t.Fatal(err) } wantPath := filepath.Join(destRoot, "proto-google-cloud-secretmanager-v1", "src", "main", "java", "com", "google", "cloud", "location", "LocationsProto.java") @@ -359,7 +359,7 @@ func TestRestructureModules_SamplesDisabled(t *testing.T) { javaAPI: &config.JavaAPI{}, } destRoot := filepath.Join(tmpDir, "dest") - if err := restructureModules(params, destRoot); err != nil { + if err := restructureModules(params, destRoot, nil, ""); err != nil { t.Fatal(err) } // Verify sample file location DOES NOT exist @@ -415,7 +415,7 @@ func TestRestructureModules_Monolithic(t *testing.T) { }, } destRoot := filepath.Join(tmpDir, "dest", "src") - if err := restructureModules(params, destRoot); err != nil { + if err := restructureModules(params, destRoot, nil, ""); err != nil { t.Fatal(err) } diff --git a/internal/librarian/java/samples.go b/internal/librarian/java/samples.go new file mode 100644 index 00000000000..81b004b0ccc --- /dev/null +++ b/internal/librarian/java/samples.go @@ -0,0 +1,260 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package java + +import ( + "bufio" + "errors" + "io/fs" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "unicode" + + "gopkg.in/yaml.v3" +) + +var ( + openSnippetRegex = regexp.MustCompile(`\[START ([a-zA-Z0-9_-]+)\]`) + closeSnippetRegex = regexp.MustCompile(`\[END ([a-zA-Z0-9_-]+)\]`) + openExcludeRegex = regexp.MustCompile(`\[START_EXCLUDE\]`) + closeExcludeRegex = regexp.MustCompile(`\[END_EXCLUDE\]`) + + reMetadataBlock = regexp.MustCompile(`(?m)^[ \t]*//[ \t]*sample-metadata:([^\n]+|\n[ \t]*//)+`) + reCommentPrefix = regexp.MustCompile(`(?m)^[ \t]*(?:#|//)[ \t]?`) + + reDecamelize1 = regexp.MustCompile(`([A-Z]+)([A-Z])([a-z0-9])`) + reDecamelize2 = regexp.MustCompile(`([a-z0-9])([A-Z])`) +) + +// decamelize converts CamelCase or PascalCase titles into space-separated words, +// exactly reproducing Python synthtool's _decamelize(value: str). +func decamelize(value string) string { + if value == "" { + return "" + } + r := []rune(value) + r[0] = unicode.ToUpper(r[0]) + s := string(r) + + s = reDecamelize1.ReplaceAllString(s, "${1} ${2}${3}") + return reDecamelize2.ReplaceAllString(s, "${1} ${2}") +} + +// ExtractSamples walks the "samples" directory locating all .java source files. +// It parses embedded multiline "// sample-metadata:" YAML blocks to derive title and path metadata. +func ExtractSamples(dir string) ([]map[string]interface{}, error) { + samplesDir := filepath.Join(dir, "samples") + var files []string + + err := filepath.WalkDir(samplesDir, func(path string, d os.DirEntry, err error) error { + if err != nil { + if errors.Is(err, fs.ErrNotExist) || os.IsNotExist(err) { + return nil + } + return err + } + if d.IsDir() && d.Name() == "test" { + return filepath.SkipDir + } + if !d.IsDir() && d.Type().IsRegular() && filepath.Ext(path) == ".java" && !strings.Contains(filepath.ToSlash(path), "/src/test/") { + files = append(files, path) + } + return nil + }) + if err != nil { + return nil, err + } + + if len(files) == 0 { + return nil, nil + } + + sort.Strings(files) + var samples []map[string]interface{} + + for _, file := range files { + rel, err := filepath.Rel(dir, file) + if err != nil { + continue + } + base := strings.TrimSuffix(filepath.Base(file), ".java") + title := decamelize(base) + + slashPath := filepath.ToSlash(rel) + item := map[string]interface{}{ + "Title": title, + "File": slashPath, + "title": title, + "file": slashPath, + } + + contentBytes, err := os.ReadFile(file) + if err == nil { + if match := reMetadataBlock.FindString(string(contentBytes)); match != "" { + cleaned := reCommentPrefix.ReplaceAllString(match, "") + var meta map[string]map[string]interface{} + if err := yaml.Unmarshal([]byte(cleaned), &meta); err == nil { + if sm, ok := meta["sample-metadata"]; ok { + for k, v := range sm { + item[k] = v + if len(k) > 0 { + upperKey := strings.ToUpper(k[:1]) + k[1:] + item[upperKey] = v + } + } + if t, ok := sm["title"].(string); ok && strings.TrimSpace(t) != "" { + item["Title"] = t + item["title"] = t + } + } + } + } + } + + samples = append(samples, item) + } + return samples, nil +} + +// ExtractSnippets walks the "samples" directory locating *.java and *.xml files. +// It line-scans for [START name] and [END name] tags while supporting [START_EXCLUDE] blocks, +// returning trimmed minimum plain-space indentation blocks. +func ExtractSnippets(dir string) (map[string]string, error) { + samplesDir := filepath.Join(dir, "samples") + var files []string + + err := filepath.WalkDir(samplesDir, func(path string, d os.DirEntry, err error) error { + if err != nil { + if errors.Is(err, fs.ErrNotExist) || os.IsNotExist(err) { + return nil + } + return err + } + if d.IsDir() { + if d.Name() == "test" { + return filepath.SkipDir + } + return nil + } + if !d.Type().IsRegular() { + return nil + } + ext := filepath.Ext(path) + if ext == ".java" || ext == ".xml" { + files = append(files, path) + } + return nil + }) + if err != nil { + return nil, err + } + + if len(files) == 0 { + return nil, nil + } + + sort.Strings(files) + snippetLines := make(map[string][]string) + + for _, file := range files { + f, err := os.Open(file) + if err != nil { + continue + } + + openSnippets := make(map[string]bool) + excluding := false + scanner := bufio.NewScanner(f) + scanner.Buffer(make([]byte, 64*1024), 10*1024*1024) + + for scanner.Scan() { + line := scanner.Text() + openMatch := openSnippetRegex.FindStringSubmatch(line) + closeMatch := closeSnippetRegex.FindStringSubmatch(line) + + if len(openMatch) > 1 && !excluding { + name := openMatch[1] + openSnippets[name] = true + if _, exists := snippetLines[name]; !exists { + snippetLines[name] = []string{} + } + } else if len(closeMatch) > 1 && !excluding { + delete(openSnippets, closeMatch[1]) + } else if openExcludeRegex.MatchString(line) { + excluding = true + } else if closeExcludeRegex.MatchString(line) { + excluding = false + } else if !excluding { + for s := range openSnippets { + snippetLines[s] = append(snippetLines[s], line) + } + } + } + if err := scanner.Err(); err != nil { + f.Close() + return nil, err + } + f.Close() + } + + if len(snippetLines) == 0 { + return nil, nil + } + + result := make(map[string]string) + for snippet, lines := range snippetLines { + result[snippet] = trimLeadingWhitespace(lines) + } + return result, nil +} + +// trimLeadingWhitespace computes the minimum plain-space indentation across non-empty lines, +// trimming that common whitespace while preserving newlines. +func trimLeadingWhitespace(lines []string) string { + if len(lines) == 0 { + return "" + } + + minSpaces := -1 + for _, line := range lines { + if strings.TrimSpace(line) != "" { + spaces := len(line) - len(strings.TrimLeft(line, " ")) + if minSpaces == -1 || spaces < minSpaces { + minSpaces = spaces + } + } + } + if minSpaces == -1 { + minSpaces = 0 + } + + var sb strings.Builder + for _, line := range lines { + if strings.TrimSpace(line) == "" { + sb.WriteString("\n") + } else { + if len(line) >= minSpaces { + sb.WriteString(line[minSpaces:]) + } else { + sb.WriteString(strings.TrimLeft(line, " ")) + } + sb.WriteString("\n") + } + } + return sb.String() +} diff --git a/internal/librarian/java/samples_test.go b/internal/librarian/java/samples_test.go new file mode 100644 index 00000000000..af0a1b4d215 --- /dev/null +++ b/internal/librarian/java/samples_test.go @@ -0,0 +1,164 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package java + +import ( + "os" + "path/filepath" + "testing" +) + +func TestDecamelize(t *testing.T) { + tests := []struct { + input string + expected string + }{ + {"requesterPays", "Requester Pays"}, + {"ACLBatman", "ACL Batman"}, + {"NativeImageLoggingSample", "Native Image Logging Sample"}, + {"simpleTest", "Simple Test"}, + {"", ""}, + } + + for _, tc := range tests { + actual := decamelize(tc.input) + if actual != tc.expected { + t.Errorf("decamelize(%q) = %q; expected %q", tc.input, actual, tc.expected) + } + } +} + +func TestExtractSamples_MissingDir(t *testing.T) { + tempDir := t.TempDir() + samples, err := ExtractSamples(tempDir) + if err != nil { + t.Fatalf("ExtractSamples returned error for missing dir: %v", err) + } + if samples != nil { + t.Errorf("Expected nil samples for missing dir, got %v", samples) + } +} + +func TestExtractSamples_Success(t *testing.T) { + tempDir := t.TempDir() + samplesDir := filepath.Join(tempDir, "samples", "src", "main", "java") + if err := os.MkdirAll(samplesDir, 0755); err != nil { + t.Fatal(err) + } + + file1 := filepath.Join(samplesDir, "RequesterPays.java") + content1 := `// sample-metadata: +// title: Custom Title Override +// description: A custom demo sample +public class RequesterPays {}` + if err := os.WriteFile(file1, []byte(content1), 0644); err != nil { + t.Fatal(err) + } + + file2 := filepath.Join(samplesDir, "demoSample.java") + content2 := `public class demoSample {}` + if err := os.WriteFile(file2, []byte(content2), 0644); err != nil { + t.Fatal(err) + } + + samples, err := ExtractSamples(tempDir) + if err != nil { + t.Fatal(err) + } + + if len(samples) != 2 { + t.Fatalf("Expected 2 samples, got %d", len(samples)) + } + + // First should be RequesterPays.java with custom YAML override (uppercase R comes before lowercase d) + s0 := samples[0] + if s0["Title"] != "Custom Title Override" || s0["title"] != "Custom Title Override" { + t.Errorf("Sample 0 title = %v; expected 'Custom Title Override'", s0["Title"]) + } + if s0["File"] != "samples/src/main/java/RequesterPays.java" { + t.Errorf("Sample 0 file = %v", s0["File"]) + } + + // Second should be demoSample.java + s1 := samples[1] + if s1["Title"] != "Demo Sample" || s1["title"] != "Demo Sample" { + t.Errorf("Sample 1 title = %v; expected 'Demo Sample'", s1["Title"]) + } + if s1["File"] != "samples/src/main/java/demoSample.java" { + t.Errorf("Sample 1 file = %v", s1["File"]) + } +} + +func TestExtractSnippets(t *testing.T) { + tempDir := t.TempDir() + samplesDir := filepath.Join(tempDir, "samples") + if err := os.MkdirAll(samplesDir, 0755); err != nil { + t.Fatal(err) + } + + pomPath := filepath.Join(samplesDir, "pom.xml") + pomContent := ` + + + com.google.cloud + + +` + if err := os.WriteFile(pomPath, []byte(pomContent), 0644); err != nil { + t.Fatal(err) + } + + javaPath := filepath.Join(samplesDir, "Demo.java") + javaContent := `public class Demo { + // [START quickstart] + public void run() { + // [START_EXCLUDE] + System.out.println("hidden"); + // [END_EXCLUDE] + System.out.println("visible"); + } + // [END quickstart] +}` + if err := os.WriteFile(javaPath, []byte(javaContent), 0644); err != nil { + t.Fatal(err) + } + + snippets, err := ExtractSnippets(tempDir) + if err != nil { + t.Fatal(err) + } + + if len(snippets) != 2 { + t.Fatalf("Expected 2 snippets, got %d", len(snippets)) + } + + depSnippet := snippets["dependency_snippet"] + expectedDep := ` + com.google.cloud + +` + if depSnippet != expectedDep { + t.Errorf("dependency_snippet = %q; expected %q", depSnippet, expectedDep) + } + + quickSnippet := snippets["quickstart"] + expectedQuick := `public void run() { + System.out.println("visible"); +} +` + if quickSnippet != expectedQuick { + t.Errorf("quickstart = %q; expected %q", quickSnippet, expectedQuick) + } +} diff --git a/internal/librarian/java/template/README.md.tmpl b/internal/librarian/java/template/README.md.tmpl new file mode 100644 index 00000000000..a2d5a55b477 --- /dev/null +++ b/internal/librarian/java/template/README.md.tmpl @@ -0,0 +1,266 @@ +# Google {{ .Metadata.Repo.NamePretty }} Client for Java + +Java idiomatic client for [{{ .Metadata.Repo.NamePretty }}][product-docs]. + +[![Maven][maven-version-image]][maven-version-link] +![Stability][stability-image] + +- [Product Documentation][product-docs] +- [Client Library Documentation][javadocs] + +{{ if and .Metadata.Partials .Metadata.Partials.DeprecationWarning }} +{{ .Metadata.Partials.DeprecationWarning }} +{{ else if eq .Metadata.Repo.ReleaseLevel "preview" }} +> Note: This client is a work-in-progress, and may occasionally +> make backwards-incompatible changes. +{{ end }} + +{{ if .MigratedSplitRepo }} +:bus: In October 2022, this library has moved to +[google-cloud-java/{{ .Metadata.Repo.RepoShort }}]( +https://github.com/googleapis/google-cloud-java/tree/main/{{ .Metadata.Repo.RepoShort }}). +This repository will be archived in the future. +Future releases will appear in the new repository (https://github.com/googleapis/google-cloud-java/releases). +The Maven artifact coordinates (`{{ .GroupID }}:{{ .ArtifactID }}`) remain the same. +{{ end }} + +## Quickstart + +{{ if and .Metadata.Snippets (index .Metadata.Snippets (print .Metadata.Repo.APIShortname "_install_with_bom")) }} +If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file: + +```xml +{{ index .Metadata.Snippets (print .Metadata.Repo.APIShortname "_install_with_bom") }} +``` + +If you are using Maven without the BOM, add this to your dependencies: +{{ else if and .Monorepo (or (eq .GroupID "com.google.cloud") (eq .GroupID "com.google.analytics") (eq .GroupID "com.google.area120")) }} +If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file: + +```xml + + + + com.google.cloud + libraries-bom + {{ .Metadata.LibrariesBOMVersion }} + pom + import + + + + + + + {{ .GroupID }} + {{ .ArtifactID }} + + +``` + +If you are using Maven without the BOM, add this to your dependencies: +{{ else }} +If you are using Maven, add this to your pom.xml file: +{{ end }} + +```xml +{{ if and .Metadata.Snippets (index .Metadata.Snippets (print .Metadata.Repo.APIShortname "_install_without_bom")) }} +{{ index .Metadata.Snippets (print .Metadata.Repo.APIShortname "_install_without_bom") }} +{{ else }} + + {{ .GroupID }} + {{ .ArtifactID }} + {{ .Version }} + +{{ end }} +``` + +{{ if and .Metadata.Snippets (index .Metadata.Snippets (print .Metadata.Repo.APIShortname "_install_with_bom")) }} +If you are using Gradle 5.x or later, add this to your dependencies: + +```Groovy +implementation platform('com.google.cloud:libraries-bom:{{ .Metadata.LibrariesBOMVersion }}') + +implementation '{{ .GroupID }}:{{ .ArtifactID }}' +``` +{{ end }} + +If you are using Gradle without BOM, add this to your dependencies: + +```Groovy +implementation '{{ .GroupID }}:{{ .ArtifactID }}:{{ .Version }}' +``` + +If you are using SBT, add this to your dependencies: + +```Scala +libraryDependencies += "{{ .GroupID }}" % "{{ .ArtifactID }}" % "{{ .Version }}" +``` + +## Authentication + +See the [Authentication][authentication] section in the base directory's README. + +## Authorization + +The client application making API calls must be granted [authorization scopes][auth-scopes] required for the desired {{ .Metadata.Repo.NamePretty }} APIs, and the authenticated principal must have the [IAM role(s)][predefined-iam-roles] required to access GCP resources using the {{ .Metadata.Repo.NamePretty }} API calls. + +## Getting Started + +### Prerequisites + +You will need a [Google Cloud Platform Console][developer-console] project with the {{ .Metadata.Repo.NamePretty }} [API enabled][enable-api]. +{{ if .Metadata.Repo.RequiresBilling }}You will need to [enable billing][enable-billing] to use Google {{ .Metadata.Repo.NamePretty }}.{{ end }} +[Follow these instructions][create-project] to get your project set up. You will also need to set up the local development environment by +[installing the Google Cloud Command Line Interface][cloud-cli] and running the following commands in command line: +`gcloud auth login` and `gcloud config set project [YOUR PROJECT ID]`. + +### Installation and setup + +You'll need to obtain the `{{ .ArtifactID }}` library. See the [Quickstart](#quickstart) section +to add `{{ .ArtifactID }}` as a dependency in your code. + +## About {{ .Metadata.Repo.NamePretty }} + +{{ if and .Metadata.Partials .Metadata.Partials.About }} +{{ .Metadata.Partials.About }} +{{ else }} +[{{ .Metadata.Repo.NamePretty }}][product-docs] {{ .Metadata.Repo.APIDescription }} + +See the [{{ .Metadata.Repo.NamePretty }} client library docs][javadocs] to learn how to +use this {{ .Metadata.Repo.NamePretty }} Client Library. +{{ end }} + +{{ if and .Metadata.Partials .Metadata.Partials.CustomContent }} +{{ .Metadata.Partials.CustomContent }} +{{ end }} + +{{ if .Metadata.Samples }} +## Samples + +Samples are in the [`samples/`](https://github.com/{{ .Metadata.Repo.Repo }}/tree/main/samples) directory. + +| Sample | Source Code | Try it | +| --------------------------- | --------------------------------- | ------ | +{{ range .Metadata.Samples }}| {{ .Title }} | [source code](https://github.com/{{ $.Metadata.Repo.Repo }}/blob/main/{{ .File }}) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/{{ $.Metadata.Repo.Repo }}&page=editor&open_in_editor={{ .File }}) | +{{ end }} +{{ end }} + +## Troubleshooting + +To get help, follow the instructions in the [shared Troubleshooting document][troubleshooting]. + +{{ if .Metadata.Repo.Transport }} +## Transport + +{{ if eq .Metadata.Repo.Transport "grpc" }} +{{ .Metadata.Repo.NamePretty }} uses gRPC for the transport layer. +{{ else if eq .Metadata.Repo.Transport "http" }} +{{ .Metadata.Repo.NamePretty }} uses HTTP/JSON for the transport layer. +{{ else if eq .Metadata.Repo.Transport "both" }} +{{ .Metadata.Repo.NamePretty }} uses both gRPC and HTTP/JSON for the transport layer. +{{ end }} +{{ end }} + +## Supported Java Versions + +Java {{ .Metadata.MinJavaVersion }} or above is required for using this client. + +Google's Java client libraries, +[Google Cloud Client Libraries][cloudlibs] +and +[Google Cloud API Libraries][apilibs], +follow the +[Oracle Java SE support roadmap][oracle] +(see the Oracle Java SE Product Releases section). + +### For new development + +In general, new feature development occurs with support for the lowest Java +LTS version covered by Oracle's Premier Support (which typically lasts 5 years +from initial General Availability). If the minimum required JVM for a given +library is changed, it is accompanied by a [semver][semver] major release. + +Java 11 and (in September 2021) Java 17 are the best choices for new +development. + +### Keeping production systems current + +Google tests its client libraries with all current LTS versions covered by +Oracle's Extended Support (which typically lasts 8 years from initial +General Availability). + +#### Legacy support + +Google's client libraries support legacy versions of Java runtimes with long +term stable libraries that don't receive feature updates on a best efforts basis +as it may not be possible to backport all patches. + +Google provides updates on a best efforts basis to apps that continue to use +Java 7, though apps might need to upgrade to current versions of the library +that supports their JVM. + +#### Where to find specific information + +The latest versions and the supported Java versions are identified on +the individual GitHub repository `github.com/GoogleAPIs/java-SERVICENAME` +and on [google-cloud-java][g-c-j]. + +## Versioning + +{{ if and .Metadata.Partials .Metadata.Partials.Versioning }} +{{ .Metadata.Partials.Versioning }} +{{ else }} +This library follows [Semantic Versioning](http://semver.org/). + +{{ if eq .Metadata.Repo.ReleaseLevel "preview" }} +It is currently in major version zero (``0.y.z``), which means that anything may change at any time +and the public API should not be considered stable. +{{ end }}{{ end }} + +## Contributing + +{{ if and .Metadata.Partials .Metadata.Partials.Contributing }} +{{ .Metadata.Partials.Contributing }} +{{ else }} +Contributions to this library are always welcome and highly encouraged. + +See [CONTRIBUTING][contributing] for more information how to get started. + +Please note that this project is released with a Contributor Code of Conduct. By participating in +this project you agree to abide by its terms. See [Code of Conduct][code-of-conduct] for more +information. +{{ end }} + +## License + +Apache 2.0 - See [LICENSE][license] for more information. + +Java is a registered trademark of Oracle and/or its affiliates. + +[product-docs]: {{ .Metadata.Repo.ProductDocumentation }} +[javadocs]: {{ .Metadata.Repo.ClientDocumentation }} +[stability-image]: https://img.shields.io/badge/stability-{{ if eq .Metadata.Repo.ReleaseLevel "stable" }}stable-green{{ else if eq .Metadata.Repo.ReleaseLevel "preview" }}preview-yellow{{ else }}unknown-red{{ end }} +[maven-version-image]: https://img.shields.io/maven-central/v/{{ .GroupID }}/{{ .ArtifactID }}.svg +[maven-version-link]: https://central.sonatype.com/artifact/{{ .GroupID }}/{{ .ArtifactID }}/{{ .Version }} +[authentication]: https://github.com/googleapis/google-cloud-java#authentication +[auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes +[predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles +[iam-policy]: https://cloud.google.com/iam/docs/overview#cloud-iam-policy +[developer-console]: https://console.developers.google.com/ +[create-project]: https://cloud.google.com/resource-manager/docs/creating-managing-projects +[cloud-cli]: https://cloud.google.com/cli +[troubleshooting]: https://github.com/googleapis/google-cloud-java/blob/main/TROUBLESHOOTING.md +[contributing]: https://github.com/{{ .Metadata.Repo.Repo }}/blob/main/CONTRIBUTING.md +[code-of-conduct]: https://github.com/{{ .Metadata.Repo.Repo }}/blob/main/CODE_OF_CONDUCT.md#contributor-code-of-conduct +[license]: https://github.com/{{ .Metadata.Repo.Repo }}/blob/main/LICENSE +{{ if .Metadata.Repo.RequiresBilling }}[enable-billing]: https://cloud.google.com/apis/docs/getting-started#enabling_billing{{ end }} +{{ if .Metadata.Repo.APIID }}[enable-api]: https://console.cloud.google.com/flows/enableapi?apiid={{ .Metadata.Repo.APIID }}{{ end }} +[libraries-bom]: https://github.com/GoogleCloudPlatform/cloud-opensource-java/wiki/The-Google-Cloud-Platform-Libraries-BOM +[shell_img]: https://gstatic.com/cloudssh/images/open-btn.png + +[semver]: https://semver.org/ +[cloudlibs]: https://cloud.google.com/apis/docs/client-libraries-explained +[apilibs]: https://cloud.google.com/apis/docs/client-libraries-explained#google_api_client_libraries +[oracle]: https://www.oracle.com/java/technologies/java-se-support-roadmap.html +[g-c-j]: http://github.com/googleapis/google-cloud-java diff --git a/internal/librarian/java/template_render.go b/internal/librarian/java/template_render.go new file mode 100644 index 00000000000..eccc4b9c635 --- /dev/null +++ b/internal/librarian/java/template_render.go @@ -0,0 +1,232 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package java + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "reflect" + "strings" + "text/template" + + "github.com/iancoleman/strcase" + "gopkg.in/yaml.v3" +) + +func isNilOrEmpty(v interface{}) bool { + if v == nil { + return true + } + rv := reflect.ValueOf(v) + switch rv.Kind() { + case reflect.Slice, reflect.Map, reflect.Array, reflect.String: + return rv.Len() == 0 + } + return false +} + +// RenderREADME renders the README.md file using the template and metadata. +// dir is the directory containing .repo-metadata.json and where README.md will be written. +// templatePath is the path to the README.md.go.tmpl file. +func RenderREADME(dir, templatePath, bomVersion, libraryVersion string) error { + metadataPath := filepath.Join(dir, ".repo-metadata.json") + partialsPath := filepath.Join(dir, ".readme-partials.yaml") + if _, err := os.Stat(partialsPath); os.IsNotExist(err) { + partialsPath = filepath.Join(dir, ".readme-partials.yml") + } + outputPath := filepath.Join(dir, "README.md") + + // Read metadata + metadataBytes, err := os.ReadFile(metadataPath) + if err != nil { + return fmt.Errorf("failed to read metadata: %w", err) + } + + var metadata map[string]interface{} + if err := json.Unmarshal(metadataBytes, &metadata); err != nil { + return fmt.Errorf("failed to unmarshal metadata: %w", err) + } + + // Read partials if exist + partials := make(map[string]interface{}) + if _, err := os.Stat(partialsPath); err == nil { + partialsBytes, err := os.ReadFile(partialsPath) + if err != nil { + return fmt.Errorf("failed to read partials: %w", err) + } + if err := yaml.Unmarshal(partialsBytes, &partials); err != nil { + return fmt.Errorf("failed to unmarshal partials: %w", err) + } + } + + // Capitalize keys of partials for template + capitalizedPartials := make(map[string]interface{}) + for k, v := range partials { + capitalizedPartials[strcase.ToCamel(k)] = v + } + + // Prepare data for template + distName, _ := metadata["distribution_name"].(string) + if distName == "" { + if repo, ok := metadata["repo"].(map[string]interface{}); ok { + distName, _ = repo["distribution_name"].(string) + } + } + distParts := strings.Split(distName, ":") + groupID := "" + artifactID := "" + if len(distParts) > 0 { + groupID = distParts[0] + } + if len(distParts) > 1 { + artifactID = distParts[1] + } + + repoName, _ := metadata["repo"].(string) + if repoName == "" { + if repo, ok := metadata["repo"].(map[string]interface{}); ok { + repoName, _ = repo["repo"].(string) + } + } + repoParts := strings.Split(repoName, "/") + repoShort := "" + if len(repoParts) > 0 { + repoShort = repoParts[len(repoParts)-1] + } + + version, _ := metadata["library_version"].(string) + if version == "" { + version = libraryVersion + } + + // Construct the nested Metadata object for the template + templateRepo := map[string]interface{}{ + "NamePretty": metadata["name_pretty"], + "DistributionName": metadata["distribution_name"], + "Repo": metadata["repo"], + "APIShortname": metadata["api_shortname"], + "APIDescription": metadata["api_description"], + "ClientDocumentation": metadata["client_documentation"], + "ProductDocumentation": metadata["product_documentation"], + "ReleaseLevel": metadata["release_level"], + "Transport": metadata["transport"], + "RequiresBilling": metadata["requires_billing"], + "APIID": metadata["api_id"], + "RepoShort": metadata["repo_short"], + } + + // If flat structure failed, try to get from nested repo + if templateRepo["NamePretty"] == nil { + if repo, ok := metadata["repo"].(map[string]interface{}); ok { + templateRepo["NamePretty"] = repo["name_pretty"] + templateRepo["DistributionName"] = repo["distribution_name"] + templateRepo["Repo"] = repo["repo"] + templateRepo["APIShortname"] = repo["api_shortname"] + templateRepo["APIDescription"] = repo["api_description"] + templateRepo["ClientDocumentation"] = repo["client_documentation"] + templateRepo["ProductDocumentation"] = repo["product_documentation"] + templateRepo["ReleaseLevel"] = repo["release_level"] + templateRepo["Transport"] = repo["transport"] + templateRepo["RequiresBilling"] = repo["requires_billing"] + templateRepo["APIID"] = repo["api_id"] + templateRepo["RepoShort"] = repo["repo_short"] + } + } + + minJavaVersion, _ := metadata["min_java_version"].(string) + if minJavaVersion == "" { + minJavaVersion = "8" // Default to Java 8 + } + fmt.Println("DEBUG minJavaVersion:", minJavaVersion) + + samples := metadata["samples"] + if isNilOrEmpty(samples) { + extSamples, err := ExtractSamples(dir) + if err != nil { + return fmt.Errorf("failed to extract samples: %w", err) + } + if len(extSamples) > 0 { + samples = extSamples + } + } + + snippets := metadata["snippets"] + if isNilOrEmpty(snippets) { + extSnippets, err := ExtractSnippets(dir) + if err != nil { + return fmt.Errorf("failed to extract snippets: %w", err) + } + if len(extSnippets) > 0 { + snippets = extSnippets + } + } + + templateMetadata := map[string]interface{}{ + "Repo": templateRepo, + "LibraryVersion": version, + "LibrariesBOMVersion": bomVersion, + "Samples": samples, + "Snippets": snippets, + "MinJavaVersion": minJavaVersion, + } + + if len(capitalizedPartials) > 0 { + templateMetadata["Partials"] = capitalizedPartials + } + + data := struct { + Metadata map[string]interface{} + GroupID string + ArtifactID string + Version string + RepoShort string + MigratedSplitRepo bool + Monorepo bool + BOMVersion string + LibraryVersion string + }{ + Metadata: templateMetadata, + GroupID: groupID, + ArtifactID: artifactID, + Version: version, + RepoShort: repoShort, + MigratedSplitRepo: false, + Monorepo: true, + BOMVersion: bomVersion, + LibraryVersion: libraryVersion, + } + + // Read and parse template + tmplBytes, err := os.ReadFile(templatePath) + if err != nil { + return fmt.Errorf("failed to read template: %w", err) + } + + tmpl, err := template.New("README").Parse(string(tmplBytes)) + if err != nil { + return fmt.Errorf("failed to parse template: %w", err) + } + + // Execute template + var buf strings.Builder + if err := tmpl.Execute(&buf, data); err != nil { + return fmt.Errorf("failed to execute template: %w", err) + } + + // Write output + return os.WriteFile(outputPath, []byte(buf.String()), 0644) +} diff --git a/internal/librarian/java/template_render_test.go b/internal/librarian/java/template_render_test.go new file mode 100644 index 00000000000..7c51d2160d6 --- /dev/null +++ b/internal/librarian/java/template_render_test.go @@ -0,0 +1,132 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package java + +import ( + "os" + "path/filepath" + "strings" + "testing" + "text/template" +) + +func TestRenderREADME(t *testing.T) { + tmpDir := t.TempDir() + + // Create dummy template + templatePath := filepath.Join(tmpDir, "README.md.go.tmpl") + templateContent := `# Google {{ .Metadata.Repo.NamePretty }} Client for Java +Artifact: {{ .GroupID }}:{{ .ArtifactID }} +Version: {{ .Version }} +BOMVersion: {{ .BOMVersion }} +LibraryVersion: {{ .LibraryVersion }} +{{ if and .Metadata.Partials .Metadata.Partials.About }} +About: {{ .Metadata.Partials.About }} +{{ end }} +` + err := os.WriteFile(templatePath, []byte(templateContent), 0644) + if err != nil { + t.Fatal(err) + } + + // Create dummy metadata + metadataPath := filepath.Join(tmpDir, ".repo-metadata.json") + metadataContent := `{ + "repo": { + "name_pretty": "My API", + "distribution_name": "com.google.cloud:google-cloud-myapi", + "repo": "googleapis/google-cloud-java" + }, + "library_version": "1.2.3" +}` + err = os.WriteFile(metadataPath, []byte(metadataContent), 0644) + if err != nil { + t.Fatal(err) + } + + // Test case 1: Without partials + err = RenderREADME(tmpDir, templatePath, "1.0.0-BOM", "1.2.3-LIB") + if err != nil { + t.Fatal(err) + } + + outputPath := filepath.Join(tmpDir, "README.md") + outputContent, err := os.ReadFile(outputPath) + if err != nil { + t.Fatal(err) + } + + expected := `# Google My API Client for Java +Artifact: com.google.cloud:google-cloud-myapi +Version: 1.2.3 +BOMVersion: 1.0.0-BOM +LibraryVersion: 1.2.3-LIB +` + if strings.TrimSpace(string(outputContent)) != strings.TrimSpace(expected) { + t.Errorf("expected:\n%s\ngot:\n%s", expected, string(outputContent)) + } + + // Test case 2: With partials + partialsPath := filepath.Join(tmpDir, ".readme-partials.yaml") + partialsContent := `about: "This is a great API."` + err = os.WriteFile(partialsPath, []byte(partialsContent), 0644) + if err != nil { + t.Fatal(err) + } + + err = RenderREADME(tmpDir, templatePath, "1.0.0-BOM", "1.2.3-LIB") + if err != nil { + t.Fatal(err) + } + + outputContent, err = os.ReadFile(outputPath) + if err != nil { + t.Fatal(err) + } + + expectedWithPartials := `# Google My API Client for Java +Artifact: com.google.cloud:google-cloud-myapi +Version: 1.2.3 +BOMVersion: 1.0.0-BOM +LibraryVersion: 1.2.3-LIB + +About: This is a great API. +` + if strings.TrimSpace(string(outputContent)) != strings.TrimSpace(expectedWithPartials) { + t.Errorf("expected:\n%s\ngot:\n%s", expectedWithPartials, string(outputContent)) + } +} + +func TestRealTemplateParses(t *testing.T) { + // Find template path + // Current dir is internal/librarian/java when running tests in this package. + templatePath := filepath.Join("template", "README.md.tmpl") + + tmplBytes, err := os.ReadFile(templatePath) + if err != nil { + // If running from repo root, path might be different. + // Let's try to find it relative to repo root if needed. + templatePath = filepath.Join("internal", "librarian", "java", "template", "README.md.go.tmpl") + tmplBytes, err = os.ReadFile(templatePath) + if err != nil { + t.Fatal(err) + } + } + + _, err = template.New("README").Parse(string(tmplBytes)) + if err != nil { + t.Fatalf("failed to parse real template: %v", err) + } +} diff --git a/internal/postprocessing/config.go b/internal/postprocessing/config.go new file mode 100644 index 00000000000..ec0078d5a93 --- /dev/null +++ b/internal/postprocessing/config.go @@ -0,0 +1,73 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package postprocessing provides tools for postprocessing generated code. +package postprocessing + +import ( + "os" + + "gopkg.in/yaml.v3" +) + +// Config represents the postprocess.yaml configuration. +type Config struct { + Replace []ReplaceConfig `yaml:"replace,omitempty"` + ReplaceRegex []ReplaceRegexConfig `yaml:"replace_regex,omitempty"` + CopyFile []CopyConfig `yaml:"copy_file,omitempty"` + RemoveFile []string `yaml:"remove_file,omitempty"` + MethodOperations []MethodOperation `yaml:"method_operations,omitempty"` +} + +// MethodOperation represents a method-level operation like delete, duplicate, or deprecate. +type MethodOperation struct { + Path string `yaml:"path"` + Action string `yaml:"action"` + FuncName string `yaml:"func_name"` + NewName string `yaml:"new_name,omitempty"` // Used for duplicate + DeprecationMessage string `yaml:"deprecation_message,omitempty"` // Used for deprecate +} + +// ReplaceConfig represents a replacement rule. +type ReplaceConfig struct { + Path string `yaml:"path"` + Original string `yaml:"original"` + Replacement string `yaml:"replacement"` +} + +// ReplaceRegexConfig represents a regex replacement rule. +type ReplaceRegexConfig struct { + Path string `yaml:"path"` + Pattern string `yaml:"pattern"` + Replacement string `yaml:"replacement"` +} + +// CopyConfig represents a file copy rule. +type CopyConfig struct { + Src string `yaml:"src"` + Dst string `yaml:"dst"` +} + +// ParseConfig parses the postprocess.yaml file. +func ParseConfig(path string) (*Config, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var config Config + if err := yaml.Unmarshal(data, &config); err != nil { + return nil, err + } + return &config, nil +} diff --git a/internal/postprocessing/config_test.go b/internal/postprocessing/config_test.go new file mode 100644 index 00000000000..59d78ef1871 --- /dev/null +++ b/internal/postprocessing/config_test.go @@ -0,0 +1,132 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postprocessing + +import ( + "os" + "path/filepath" + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestParseConfig(t *testing.T) { + yamlContent := ` +replace: + - path: path/to/file.java + original: "old string" + replacement: "new string" +replace_regex: + - path: path/to/file.java + pattern: "pattern" + replacement: "replacement" +copy_file: + - src: path/to/src.java + dst: path/to/dst.java +remove_file: + - path/to/file_to_remove.java + +method_operations: + - path: path/to/file.java + action: delete + func_name: "public void toDelete()" + - path: path/to/file.java + action: duplicate + func_name: "public void toDuplicate()" + new_name: "duplicated" + - path: path/to/file.java + action: deprecate + func_name: "public void toDeprecate()" + deprecation_message: "Use alternative instead." +` + dir := t.TempDir() + configPath := filepath.Join(dir, "postprocess.yaml") + if err := os.WriteFile(configPath, []byte(yamlContent), 0644); err != nil { + t.Fatal(err) + } + + got, err := ParseConfig(configPath) + if err != nil { + t.Fatal(err) + } + + want := &Config{ + Replace: []ReplaceConfig{ + { + Path: "path/to/file.java", + Original: "old string", + Replacement: "new string", + }, + }, + ReplaceRegex: []ReplaceRegexConfig{ + { + Path: "path/to/file.java", + Pattern: "pattern", + Replacement: "replacement", + }, + }, + CopyFile: []CopyConfig{ + { + Src: "path/to/src.java", + Dst: "path/to/dst.java", + }, + }, + RemoveFile: []string{"path/to/file_to_remove.java"}, + + MethodOperations: []MethodOperation{ + { + Path: "path/to/file.java", + Action: "delete", + FuncName: "public void toDelete()", + }, + { + Path: "path/to/file.java", + Action: "duplicate", + FuncName: "public void toDuplicate()", + NewName: "duplicated", + }, + { + Path: "path/to/file.java", + Action: "deprecate", + FuncName: "public void toDeprecate()", + DeprecationMessage: "Use alternative instead.", + }, + }, + } + + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("ParseConfig() mismatch (-want +got):\n%s", diff) + } +} + +func TestParseConfig_FileNotFound(t *testing.T) { + _, err := ParseConfig("non-existent-file.yaml") + if err == nil { + t.Error("ParseConfig() expected error for non-existent file, got nil") + } +} + +func TestParseConfig_InvalidYAML(t *testing.T) { + dir := t.TempDir() + configPath := filepath.Join(dir, "postprocess.yaml") + if err := os.WriteFile(configPath, []byte("invalid yaml content"), 0644); err != nil { + t.Fatal(err) + } + + _, err := ParseConfig(configPath) + if err == nil { + t.Error("ParseConfig() expected error for invalid YAML, got nil") + } +} diff --git a/internal/postprocessing/fileops.go b/internal/postprocessing/fileops.go index 3ca493be699..ae51114effd 100644 --- a/internal/postprocessing/fileops.go +++ b/internal/postprocessing/fileops.go @@ -23,6 +23,7 @@ import ( "regexp" "strings" + "github.com/bmatcuk/doublestar/v4" "github.com/googleapis/librarian/internal/filesystem" ) @@ -36,6 +37,37 @@ func CopyFile(src, dst string) error { return filesystem.CopyFile(src, dst) } +// Replace replaces all instances of original with replacement in the file at path. +// It supports glob patterns if path contains glob characters. +func Replace(path, original, replacement string) error { + var files []string + var err error + + if strings.ContainsAny(path, "*?[]{}") { + files, err = doublestar.FilepathGlob(path) + if err != nil { + return err + } + } else { + files = []string{path} + } + + for _, f := range files { + content, err := os.ReadFile(f) + if err != nil { + if os.IsNotExist(err) { + continue // Silently ignore missing files, like synthtool + } + return err + } + newContent := strings.ReplaceAll(string(content), original, replacement) + if err := os.WriteFile(f, []byte(newContent), 0644); err != nil { + return err + } + } + return nil +} + // RemoveFile removes the file at the specified path. func RemoveFile(path string) error { return os.Remove(path) @@ -77,3 +109,48 @@ func ReplaceRegex(path, pattern, replacement string) error { newContent := re.ReplaceAll(content, []byte(replacement)) return os.WriteFile(path, newContent, 0644) } + +// ReplaceRegex replaces all instances of pattern with replacement in the file at path. +// It converts Python-style capture groups like \g<1> to Go-style $1. +// It supports glob patterns if path contains glob characters. +func ReplaceRegex(path, pattern, replacement string) error { + var files []string + var err error + + if strings.ContainsAny(path, "*?[]{}") { + files, err = doublestar.FilepathGlob(path) + if err != nil { + return err + } + } else { + files = []string{path} + } + + // Convert Python-style capture groups \g<1> or \g to Go-style $1 or $name + rePythonGroup := regexp.MustCompile(`\\g<(\w+)>`) + replacement = rePythonGroup.ReplaceAllString(replacement, `$$$1`) + + // Convert Python-style numeric capture groups \1, \2 to Go-style $1, $2 + rePythonNumGroup := regexp.MustCompile(`(^|[^\\])\\(\d+)`) + replacement = rePythonNumGroup.ReplaceAllString(replacement, `${1}$$${2}`) + + re, err := regexp.Compile(pattern) + if err != nil { + return err + } + + for _, f := range files { + content, err := os.ReadFile(f) + if err != nil { + if os.IsNotExist(err) { + continue // Silently ignore missing files, like synthtool + } + return err + } + newContent := re.ReplaceAllString(string(content), replacement) + if err := os.WriteFile(f, []byte(newContent), 0644); err != nil { + return err + } + } + return nil +} diff --git a/internal/postprocessing/fileops_test.go b/internal/postprocessing/fileops_test.go index 10aa4df27cb..9e0f8d3832e 100644 --- a/internal/postprocessing/fileops_test.go +++ b/internal/postprocessing/fileops_test.go @@ -95,25 +95,35 @@ func TestRemoveFile_Error(t *testing.T) { } func TestReplace(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "test.txt") - content := "Hello World" - original := "World" - replacement := "Go" - want := "Hello Go" - if err := os.WriteFile(path, []byte(content), 0644); err != nil { - t.Fatal(err) - } - if err := Replace(path, original, replacement); err != nil { - t.Fatal(err) - } - gotBytes, err := os.ReadFile(path) - if err != nil { - t.Fatal(err) - } - got := string(gotBytes) - if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("mismatch (-want +got):\n%s", diff) + t.Parallel() + for _, test := range []struct { + name string + content string + original string + replacement string + want string + }{ + {"success", "Hello World", "World", "Go", "Hello Go"}, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := filepath.Join(dir, "test.txt") + if err := os.WriteFile(path, []byte(test.content), 0644); err != nil { + t.Fatal(err) + } + if err := Replace(path, test.original, test.replacement); err != nil { + t.Fatal(err) + } + gotBytes, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + got := string(gotBytes) + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + }) } } @@ -140,6 +150,27 @@ func TestReplaceRegex(t *testing.T) { replacement: "Number", want: "Hello Number World", }, + { + name: "python style capture group", + content: "Hello World", + pattern: `(Hello) (World)`, + replacement: `\g<2> \g<1>`, + want: "World Hello", + }, + { + name: "python style capture group with named group", + content: "Hello World", + pattern: `(?PHello) (?PWorld)`, + replacement: `\g \g`, + want: "World Hello", + }, + { + name: "python style numeric capture group", + content: "Hello World", + pattern: `(Hello) (World)`, + replacement: `\2 \1`, + want: "World Hello", + }, } { t.Run(test.name, func(t *testing.T) { t.Parallel() From cecd7d24de7d21aa21c03142375885bfeafaba24 Mon Sep 17 00:00:00 2001 From: Sophia Yang Date: Wed, 10 Jun 2026 12:30:45 -0400 Subject: [PATCH 006/108] WIP: save all postprocessing modernization work --- internal/librarian/java/clean.go | 8 +- internal/librarian/java/postprocess.go | 19 +++-- internal/librarian/java/postprocess_new.go | 79 +++++++++++++------ .../librarian/java/postprocess_new_test.go | 13 ++- internal/librarian/java/postprocess_test.go | 15 +++- internal/librarian/java/samples.go | 15 +++- .../librarian/java/template/README.md.go.tmpl | 3 +- internal/librarian/java/template_render.go | 16 ++-- .../librarian/java/template_render_test.go | 34 +++----- internal/postprocessing/fileops.go | 4 + 10 files changed, 123 insertions(+), 83 deletions(-) diff --git a/internal/librarian/java/clean.go b/internal/librarian/java/clean.go index 96bdccd933c..044e4599221 100644 --- a/internal/librarian/java/clean.go +++ b/internal/librarian/java/clean.go @@ -17,6 +17,7 @@ package java import ( "bufio" "errors" + "fmt" "io" "io/fs" "os" @@ -33,6 +34,8 @@ import ( const ( autoGeneratedMarker = "// AUTO-GENERATED DOCUMENTATION AND CLASS." autoGeneratedAnnotation = "@Generated(\"by gapic-generator-java\")" + protobufMarker = "// Generated by the protocol buffer compiler. DO NOT EDIT!" + grpcMarker = "@io.grpc.stub.annotations.GrpcGenerated" ) var ( @@ -53,6 +56,7 @@ var ( // It targets patterns like proto-*, grpc-*, and the main GAPIC module. func Clean(library *config.Library) error { patterns := cleanPatterns(library) + fmt.Printf("Clean patterns for %s: %v\n", library.Output, patterns) keepSet := make(map[string]bool) for _, k := range library.Keep { keepSet[strings.TrimSuffix(filepath.ToSlash(k), "/")] = true @@ -141,6 +145,8 @@ func cleanPath(targetPath, root string, keepSet map[string]bool, useMarker bool) return nil } } + fmt.Println("DELETING:", path) + fmt.Println("DELETING:", path) return os.Remove(path) }) if err != nil && !errors.Is(err, fs.ErrNotExist) { @@ -231,7 +237,7 @@ func hasMarker(path string) (has bool, err error) { reader := bufio.NewReader(f) for { line, rerr := reader.ReadString('\n') - if strings.Contains(line, autoGeneratedMarker) || strings.Contains(line, autoGeneratedAnnotation) { + if strings.Contains(line, autoGeneratedMarker) || strings.Contains(line, autoGeneratedAnnotation) || strings.Contains(line, protobufMarker) || strings.Contains(line, grpcMarker) { return true, nil } if rerr != nil { diff --git a/internal/librarian/java/postprocess.go b/internal/librarian/java/postprocess.go index 768e06aee25..ac6c4297d8e 100644 --- a/internal/librarian/java/postprocess.go +++ b/internal/librarian/java/postprocess.go @@ -339,9 +339,11 @@ func restructure(actions []moveAction, keepSet map[string]bool, libraryRoot stri // Also preserve existing files that lack the auto-generated marker. destPath := filepath.Join(libraryRoot, p) if _, err := os.Stat(destPath); err == nil { - isGen, err := hasMarker(destPath) - if err == nil && !isGen { - return true + if filepath.Ext(destPath) == ".java" { + isGen, err := hasMarker(destPath) + if err == nil && !isGen { + return true + } } } return false @@ -378,11 +380,12 @@ func restructureModules(params postProcessParams, destRoot string, keepSet map[s protoFilesDestDir := filepath.Join(destRoot, coords.Proto.ArtifactID, "src", "main", "proto") if params.javaAPI.Monolithic { - protoDest = filepath.Join(destRoot, "main", "java") - grpcDest = filepath.Join(destRoot, "main", "java") - gapicMainDest = filepath.Join(destRoot, "main") - gapicTestDest = filepath.Join(destRoot, "test") - protoFilesDestDir = filepath.Join(destRoot, "main", "proto") + protoDest = filepath.Join(destRoot, "src", "main", "java") + grpcDest = filepath.Join(destRoot, "src", "main", "java") + gapicMainDest = filepath.Join(destRoot, "src", "main") + gapicTestDest = filepath.Join(destRoot, "src", "test") + protoFilesDestDir = filepath.Join(destRoot, "src", "main", "proto") + } } var actions []moveAction diff --git a/internal/librarian/java/postprocess_new.go b/internal/librarian/java/postprocess_new.go index 231c608ca60..53b7e92e00b 100644 --- a/internal/librarian/java/postprocess_new.go +++ b/internal/librarian/java/postprocess_new.go @@ -37,8 +37,16 @@ func postProcessLibraryNew(p libraryPostProcessParams) error { return fmt.Errorf("failed to parse postprocess.yaml: %w", err) } + keepSet := make(map[string]bool) + for _, k := range p.library.Keep { + keepSet[strings.TrimSuffix(filepath.ToSlash(k), "/")] = true + } + // 1. Apply Copies for _, c := range cfg.CopyFile { + if shouldPreserve(filepath.ToSlash(c.Dst), keepSet) { + continue + } srcAbs := filepath.Join(p.outDir, c.Src) dstAbs := filepath.Join(p.outDir, c.Dst) if err := filesystem.CopyFile(srcAbs, dstAbs); err != nil { @@ -48,70 +56,76 @@ func postProcessLibraryNew(p libraryPostProcessParams) error { // 2. Apply Removes for _, rem := range cfg.RemoveFile { - absPath := filepath.Join(p.outDir, rem) - if err := postprocessing.RemoveFile(absPath); err != nil { - return fmt.Errorf("failed to remove file %s: %w", rem, err) + if err := applyToFiles(p.outDir, rem, keepSet, func(file string) error { + if err := postprocessing.RemoveFile(file); err != nil { + return fmt.Errorf("failed to remove file %s: %w", file, err) + } + return nil + }); err != nil { + return err } } // 3. Apply Replacements for _, r := range cfg.Replace { - absPath := filepath.Join(p.outDir, r.Path) - if err := postprocessing.Replace(absPath, r.Original, r.Replacement); err != nil { - return fmt.Errorf("failed to apply replacement in %s: %w", r.Path, err) + if err := applyToFiles(p.outDir, r.Path, keepSet, func(file string) error { + if err := postprocessing.Replace(file, r.Original, r.Replacement); err != nil { + return fmt.Errorf("failed to apply replacement in %s: %w", file, err) + } + return nil + }); err != nil { + return err } } // 4. Apply Regex Replacements for _, r := range cfg.ReplaceRegex { - absPath := filepath.Join(p.outDir, r.Path) - if err := postprocessing.ReplaceRegex(absPath, r.Pattern, r.Replacement); err != nil { - return fmt.Errorf("failed to apply regex replacement in %s: %w", r.Path, err) + if err := applyToFiles(p.outDir, r.Path, keepSet, func(file string) error { + if err := postprocessing.ReplaceRegex(file, r.Pattern, r.Replacement); err != nil { + return fmt.Errorf("failed to apply regex replacement in %s: %w", file, err) + } + return nil + }); err != nil { + return err } } // 5. Apply Method Operations for _, mo := range cfg.MethodOperations { - files, err := resolveGlobs(p.outDir, mo.Path) - if err != nil { - return fmt.Errorf("failed to resolve glob for %s: %w", mo.Path, err) - } - for _, file := range files { + if err := applyToFiles(p.outDir, mo.Path, keepSet, func(file string) error { switch mo.Action { case "delete": if err := postprocessing.DeleteFunc(file, mo.FuncName, "java"); err != nil { if strings.Contains(err.Error(), "not found") { - continue + return nil } return fmt.Errorf("failed to delete method %q in %s: %w", mo.FuncName, file, err) } case "duplicate": if err := postprocessing.DuplicateMethod(file, mo.FuncName, mo.NewName, "java"); err != nil { if strings.Contains(err.Error(), "not found") { - continue + return nil } return fmt.Errorf("failed to duplicate method %q in %s: %w", mo.FuncName, file, err) } case "deprecate": if err := postprocessing.DeprecateMethod(file, mo.FuncName, mo.DeprecationMessage, "java"); err != nil { if strings.Contains(err.Error(), "not found") { - continue + return nil } return fmt.Errorf("failed to deprecate method %q in %s: %w", mo.FuncName, file, err) } default: return fmt.Errorf("unsupported method operation action %q", mo.Action) } + return nil + }); err != nil { + return err } } } - // 4. Render README.md - templatePath := filepath.Join("template", "README.md.go.tmpl") - if _, err := os.Stat(templatePath); err != nil { - // Fallback to absolute path for this workspace - templatePath = "/usr/local/google/home/sophieeee/workspace/owlbot-modernization/librarian/internal/librarian/java/template/README.md.go.tmpl" - } + // 6. Render README.md libraryVersion, err := deriveLastReleasedVersion(p.library.Version) if err != nil { @@ -133,7 +147,7 @@ func postProcessLibraryNew(p libraryPostProcessParams) error { return fmt.Errorf("failed to find BOM version: %w", err) } - if err := RenderREADME(p.outDir, templatePath, bomVersion, libraryVersion); err != nil { + if err := RenderREADME(p.outDir, bomVersion, libraryVersion); err != nil { return fmt.Errorf("failed to render README: %w", err) } @@ -146,3 +160,20 @@ func resolveGlobs(outDir, pathPattern string) ([]string, error) { } return []string{filepath.Join(outDir, pathPattern)}, nil } + +func applyToFiles(outDir string, pathPattern string, keepSet map[string]bool, action func(string) error) error { + files, err := resolveGlobs(outDir, pathPattern) + if err != nil { + return fmt.Errorf("failed to resolve glob for %s: %w", pathPattern, err) + } + for _, file := range files { + relPath, _ := filepath.Rel(outDir, file) + if shouldPreserve(filepath.ToSlash(relPath), keepSet) { + continue + } + if err := action(file); err != nil { + return err + } + } + return nil +} diff --git a/internal/librarian/java/postprocess_new_test.go b/internal/librarian/java/postprocess_new_test.go index 5bfe8f265a1..a03c8f74f70 100644 --- a/internal/librarian/java/postprocess_new_test.go +++ b/internal/librarian/java/postprocess_new_test.go @@ -81,14 +81,11 @@ method_operations: t.Fatal(err) } - // Setup template - if err := os.MkdirAll(filepath.Join(tmpDir, "template"), 0755); err != nil { - t.Fatal(err) - } - templateContent := `# {{ .Metadata.Repo.NamePretty }}` - if err := os.WriteFile(filepath.Join(tmpDir, "template", "README.md.go.tmpl"), []byte(templateContent), 0644); err != nil { - t.Fatal(err) - } + oldTemplate := readmeTemplate + readmeTemplate = `# {{ .Metadata.Repo.NamePretty }}` + defer func() { + readmeTemplate = oldTemplate + }() p := libraryPostProcessParams{ outDir: tmpDir, diff --git a/internal/librarian/java/postprocess_test.go b/internal/librarian/java/postprocess_test.go index fd262832099..f75315b7577 100644 --- a/internal/librarian/java/postprocess_test.go +++ b/internal/librarian/java/postprocess_test.go @@ -414,16 +414,16 @@ func TestRestructureModules_Monolithic(t *testing.T) { Monolithic: true, }, } - destRoot := filepath.Join(tmpDir, "dest", "src") + destRoot := filepath.Join(tmpDir, "dest") if err := restructureModules(params, destRoot, nil, ""); err != nil { t.Fatal(err) } // Verify all files are in the same src directory files := []string{ - filepath.Join(destRoot, "main", "java", "Gapic.java"), - filepath.Join(destRoot, "main", "java", "Grpc.java"), - filepath.Join(destRoot, "main", "java", "Proto.java"), + filepath.Join(destRoot, "src", "main", "java", "Gapic.java"), + filepath.Join(destRoot, "src", "main", "java", "Grpc.java"), + filepath.Join(destRoot, "src", "main", "java", "Proto.java"), } for _, f := range files { if _, err := os.Stat(f); err != nil { @@ -1220,6 +1220,9 @@ func TestPostProcessLibrary_Branching(t *testing.T) { p := libraryPostProcessParams{ outDir: outDir, UseGoPostprocessor: true, + metadata: &repoMetadata{ + NamePretty: "test-library", + }, cfg: &config.Config{ Default: &config.Default{ Java: &config.JavaModule{ @@ -1233,6 +1236,10 @@ func TestPostProcessLibrary_Branching(t *testing.T) { library: &config.Library{ Name: "test-library", Version: "1.2.3", + Java: &config.JavaModule{ + GroupID: "com.google.cloud", + ArtifactID: "test-library", + }, }, } err := postProcessLibrary(t.Context(), p) diff --git a/internal/librarian/java/samples.go b/internal/librarian/java/samples.go index 81b004b0ccc..1047d2985e5 100644 --- a/internal/librarian/java/samples.go +++ b/internal/librarian/java/samples.go @@ -68,10 +68,16 @@ func ExtractSamples(dir string) ([]map[string]interface{}, error) { } return err } - if d.IsDir() && d.Name() == "test" { - return filepath.SkipDir + if d.IsDir() { + if d.Name() == "test" { + return filepath.SkipDir + } + if d.Name() == "generated" && filepath.Base(filepath.Dir(path)) == "snippets" { + return filepath.SkipDir + } + return nil } - if !d.IsDir() && d.Type().IsRegular() && filepath.Ext(path) == ".java" && !strings.Contains(filepath.ToSlash(path), "/src/test/") { + if !d.IsDir() && d.Type().IsRegular() && filepath.Ext(path) == ".java" && strings.Contains(filepath.ToSlash(path), "/src/main/java/") { files = append(files, path) } return nil @@ -149,6 +155,9 @@ func ExtractSnippets(dir string) (map[string]string, error) { if d.Name() == "test" { return filepath.SkipDir } + if d.Name() == "generated" && filepath.Base(filepath.Dir(path)) == "snippets" { + return filepath.SkipDir + } return nil } if !d.Type().IsRegular() { diff --git a/internal/librarian/java/template/README.md.go.tmpl b/internal/librarian/java/template/README.md.go.tmpl index d9177ec037e..12e4433fb77 100644 --- a/internal/librarian/java/template/README.md.go.tmpl +++ b/internal/librarian/java/template/README.md.go.tmpl @@ -11,6 +11,7 @@ Java idiomatic client for [{{ .Metadata.Repo.NamePretty }}][product-docs]. {{ if and .Metadata.Partials .Metadata.Partials.DeprecationWarning }}{{ .Metadata.Partials.DeprecationWarning }} {{ end }}{{ if eq .Metadata.Repo.ReleaseLevel "preview" }}> Note: This client is a work-in-progress, and may occasionally > make backwards-incompatible changes. + {{ end }}{{ if .MigratedSplitRepo }}:bus: In October 2022, this library has moved to [google-cloud-java/{{ .Metadata.Repo.RepoShort }}]( https://github.com/googleapis/google-cloud-java/tree/main/{{ .Metadata.Repo.RepoShort }}). @@ -196,7 +197,7 @@ and on [google-cloud-java][g-c-j]. ## Versioning -{{ if and .Metadata.Partials .Metadata.Partials.Versioning }} +{{- if and .Metadata.Partials .Metadata.Partials.Versioning }} {{ .Metadata.Partials.Versioning }} {{ else }} This library follows [Semantic Versioning](http://semver.org/). diff --git a/internal/librarian/java/template_render.go b/internal/librarian/java/template_render.go index eccc4b9c635..4b84bb518e3 100644 --- a/internal/librarian/java/template_render.go +++ b/internal/librarian/java/template_render.go @@ -15,6 +15,7 @@ package java import ( + _ "embed" "encoding/json" "fmt" "os" @@ -27,6 +28,9 @@ import ( "gopkg.in/yaml.v3" ) +//go:embed template/README.md.go.tmpl +var readmeTemplate string + func isNilOrEmpty(v interface{}) bool { if v == nil { return true @@ -41,8 +45,7 @@ func isNilOrEmpty(v interface{}) bool { // RenderREADME renders the README.md file using the template and metadata. // dir is the directory containing .repo-metadata.json and where README.md will be written. -// templatePath is the path to the README.md.go.tmpl file. -func RenderREADME(dir, templatePath, bomVersion, libraryVersion string) error { +func RenderREADME(dir, bomVersion, libraryVersion string) error { metadataPath := filepath.Join(dir, ".repo-metadata.json") partialsPath := filepath.Join(dir, ".readme-partials.yaml") if _, err := os.Stat(partialsPath); os.IsNotExist(err) { @@ -210,13 +213,8 @@ func RenderREADME(dir, templatePath, bomVersion, libraryVersion string) error { LibraryVersion: libraryVersion, } - // Read and parse template - tmplBytes, err := os.ReadFile(templatePath) - if err != nil { - return fmt.Errorf("failed to read template: %w", err) - } - - tmpl, err := template.New("README").Parse(string(tmplBytes)) + // Parse template + tmpl, err := template.New("README").Parse(readmeTemplate) if err != nil { return fmt.Errorf("failed to parse template: %w", err) } diff --git a/internal/librarian/java/template_render_test.go b/internal/librarian/java/template_render_test.go index 7c51d2160d6..4f4e7010775 100644 --- a/internal/librarian/java/template_render_test.go +++ b/internal/librarian/java/template_render_test.go @@ -25,8 +25,6 @@ import ( func TestRenderREADME(t *testing.T) { tmpDir := t.TempDir() - // Create dummy template - templatePath := filepath.Join(tmpDir, "README.md.go.tmpl") templateContent := `# Google {{ .Metadata.Repo.NamePretty }} Client for Java Artifact: {{ .GroupID }}:{{ .ArtifactID }} Version: {{ .Version }} @@ -36,10 +34,11 @@ LibraryVersion: {{ .LibraryVersion }} About: {{ .Metadata.Partials.About }} {{ end }} ` - err := os.WriteFile(templatePath, []byte(templateContent), 0644) - if err != nil { - t.Fatal(err) - } + oldTemplate := readmeTemplate + readmeTemplate = templateContent + defer func() { + readmeTemplate = oldTemplate + }() // Create dummy metadata metadataPath := filepath.Join(tmpDir, ".repo-metadata.json") @@ -51,13 +50,13 @@ About: {{ .Metadata.Partials.About }} }, "library_version": "1.2.3" }` - err = os.WriteFile(metadataPath, []byte(metadataContent), 0644) + err := os.WriteFile(metadataPath, []byte(metadataContent), 0644) if err != nil { t.Fatal(err) } // Test case 1: Without partials - err = RenderREADME(tmpDir, templatePath, "1.0.0-BOM", "1.2.3-LIB") + err = RenderREADME(tmpDir, "1.0.0-BOM", "1.2.3-LIB") if err != nil { t.Fatal(err) } @@ -86,7 +85,7 @@ LibraryVersion: 1.2.3-LIB t.Fatal(err) } - err = RenderREADME(tmpDir, templatePath, "1.0.0-BOM", "1.2.3-LIB") + err = RenderREADME(tmpDir, "1.0.0-BOM", "1.2.3-LIB") if err != nil { t.Fatal(err) } @@ -110,22 +109,7 @@ About: This is a great API. } func TestRealTemplateParses(t *testing.T) { - // Find template path - // Current dir is internal/librarian/java when running tests in this package. - templatePath := filepath.Join("template", "README.md.tmpl") - - tmplBytes, err := os.ReadFile(templatePath) - if err != nil { - // If running from repo root, path might be different. - // Let's try to find it relative to repo root if needed. - templatePath = filepath.Join("internal", "librarian", "java", "template", "README.md.go.tmpl") - tmplBytes, err = os.ReadFile(templatePath) - if err != nil { - t.Fatal(err) - } - } - - _, err = template.New("README").Parse(string(tmplBytes)) + _, err := template.New("README").Parse(readmeTemplate) if err != nil { t.Fatalf("failed to parse real template: %v", err) } diff --git a/internal/postprocessing/fileops.go b/internal/postprocessing/fileops.go index ae51114effd..96d036af901 100644 --- a/internal/postprocessing/fileops.go +++ b/internal/postprocessing/fileops.go @@ -134,6 +134,10 @@ func ReplaceRegex(path, pattern, replacement string) error { rePythonNumGroup := regexp.MustCompile(`(^|[^\\])\\(\d+)`) replacement = rePythonNumGroup.ReplaceAllString(replacement, `${1}$$${2}`) + if !strings.HasPrefix(pattern, "(?") { + pattern = "(?m)" + pattern + } + re, err := regexp.Compile(pattern) if err != nil { return err From 8ca2d237f7591a386078b03fd21048578c090ebd Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Mon, 15 Jun 2026 14:38:13 -0400 Subject: [PATCH 007/108] chore: temporary commit --- internal/librarian/java/postprocess.go | 2 +- internal/librarian/java/postprocess_new.go | 11 ++++++----- internal/librarian/java/postprocess_new_test.go | 2 +- internal/postprocessing/config.go | 11 ++++++----- internal/postprocessing/config_test.go | 7 ++++--- 5 files changed, 18 insertions(+), 15 deletions(-) diff --git a/internal/librarian/java/postprocess.go b/internal/librarian/java/postprocess.go index ac6c4297d8e..ce334122c11 100644 --- a/internal/librarian/java/postprocess.go +++ b/internal/librarian/java/postprocess.go @@ -73,7 +73,7 @@ func postProcessLibrary(ctx context.Context, params libraryPostProcessParams) er if params.UseGoPostprocessor { yamlPath := filepath.Join(params.outDir, "postprocess.yaml") if _, err := os.Stat(yamlPath); err == nil { - if err := postProcessLibraryNew(params); err != nil { + if err := postProcessLibraryNew(ctx, params); err != nil { return err } diff --git a/internal/librarian/java/postprocess_new.go b/internal/librarian/java/postprocess_new.go index 53b7e92e00b..3cbf1de1ee3 100644 --- a/internal/librarian/java/postprocess_new.go +++ b/internal/librarian/java/postprocess_new.go @@ -15,6 +15,7 @@ package java import ( + "context" "fmt" "os" "path/filepath" @@ -28,11 +29,11 @@ import ( // postProcessLibraryNew implements the new postprocessing flow, bypassing owlbot.py. // It applies operations from postprocess.yaml, and renders the README.md directly // on the generated files in their final destinations. -func postProcessLibraryNew(p libraryPostProcessParams) error { +func postProcessLibraryNew(ctx context.Context, p libraryPostProcessParams) error { // 1. Load postprocess.yaml and apply operations postprocessYamlPath := filepath.Join(p.outDir, "postprocess.yaml") if _, err := os.Stat(postprocessYamlPath); err == nil { - cfg, err := postprocessing.ParseConfig(postprocessYamlPath) + cfg, err := postprocessing.ParseConfig(ctx, postprocessYamlPath) if err != nil { return fmt.Errorf("failed to parse postprocess.yaml: %w", err) } @@ -95,21 +96,21 @@ func postProcessLibraryNew(p libraryPostProcessParams) error { if err := applyToFiles(p.outDir, mo.Path, keepSet, func(file string) error { switch mo.Action { case "delete": - if err := postprocessing.DeleteFunc(file, mo.FuncName, "java"); err != nil { + if err := postprocessing.DeleteFunc(ctx, file, mo.FuncName, "java"); err != nil { if strings.Contains(err.Error(), "not found") { return nil } return fmt.Errorf("failed to delete method %q in %s: %w", mo.FuncName, file, err) } case "duplicate": - if err := postprocessing.DuplicateMethod(file, mo.FuncName, mo.NewName, "java"); err != nil { + if err := postprocessing.DuplicateMethod(ctx, file, mo.FuncName, mo.NewName, "java"); err != nil { if strings.Contains(err.Error(), "not found") { return nil } return fmt.Errorf("failed to duplicate method %q in %s: %w", mo.FuncName, file, err) } case "deprecate": - if err := postprocessing.DeprecateMethod(file, mo.FuncName, mo.DeprecationMessage, "java"); err != nil { + if err := postprocessing.DeprecateMethod(ctx, file, mo.FuncName, mo.DeprecationMessage, "java"); err != nil { if strings.Contains(err.Error(), "not found") { return nil } diff --git a/internal/librarian/java/postprocess_new_test.go b/internal/librarian/java/postprocess_new_test.go index a03c8f74f70..875705c97cd 100644 --- a/internal/librarian/java/postprocess_new_test.go +++ b/internal/librarian/java/postprocess_new_test.go @@ -101,7 +101,7 @@ method_operations: }, } - err := postProcessLibraryNew(p) + err := postProcessLibraryNew(t.Context(), p) if err != nil { t.Fatal(err) } diff --git a/internal/postprocessing/config.go b/internal/postprocessing/config.go index ec0078d5a93..c35d22d6804 100644 --- a/internal/postprocessing/config.go +++ b/internal/postprocessing/config.go @@ -16,9 +16,10 @@ package postprocessing import ( + "context" "os" - "gopkg.in/yaml.v3" + "github.com/googleapis/librarian/internal/yaml" ) // Config represents the postprocess.yaml configuration. @@ -60,14 +61,14 @@ type CopyConfig struct { } // ParseConfig parses the postprocess.yaml file. -func ParseConfig(path string) (*Config, error) { +func ParseConfig(ctx context.Context, path string) (*Config, error) { data, err := os.ReadFile(path) if err != nil { return nil, err } - var config Config - if err := yaml.Unmarshal(data, &config); err != nil { + configPtr, err := yaml.Unmarshal[Config](data) + if err != nil { return nil, err } - return &config, nil + return configPtr, nil } diff --git a/internal/postprocessing/config_test.go b/internal/postprocessing/config_test.go index 59d78ef1871..154800cebb6 100644 --- a/internal/postprocessing/config_test.go +++ b/internal/postprocessing/config_test.go @@ -15,6 +15,7 @@ package postprocessing import ( + "context" "os" "path/filepath" "testing" @@ -57,7 +58,7 @@ method_operations: t.Fatal(err) } - got, err := ParseConfig(configPath) + got, err := ParseConfig(context.Background(), configPath) if err != nil { t.Fatal(err) } @@ -112,7 +113,7 @@ method_operations: } func TestParseConfig_FileNotFound(t *testing.T) { - _, err := ParseConfig("non-existent-file.yaml") + _, err := ParseConfig(context.Background(), "non-existent-file.yaml") if err == nil { t.Error("ParseConfig() expected error for non-existent file, got nil") } @@ -125,7 +126,7 @@ func TestParseConfig_InvalidYAML(t *testing.T) { t.Fatal(err) } - _, err := ParseConfig(configPath) + _, err := ParseConfig(context.Background(), configPath) if err == nil { t.Error("ParseConfig() expected error for invalid YAML, got nil") } From 5bdf8bc67cb3a39e3d4ea3334806cb4b7e2ae10b Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Mon, 15 Jun 2026 18:36:33 -0400 Subject: [PATCH 008/108] feat(postprocessing): validate method signatures during config parsing --- internal/librarian/java/postprocess_new.go | 2 +- internal/postprocessing/config.go | 17 ++++++++++++++++ internal/postprocessing/config_test.go | 23 ++++++++++++++++++++++ 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/internal/librarian/java/postprocess_new.go b/internal/librarian/java/postprocess_new.go index 3cbf1de1ee3..7caf3261c4f 100644 --- a/internal/librarian/java/postprocess_new.go +++ b/internal/librarian/java/postprocess_new.go @@ -96,7 +96,7 @@ func postProcessLibraryNew(ctx context.Context, p libraryPostProcessParams) erro if err := applyToFiles(p.outDir, mo.Path, keepSet, func(file string) error { switch mo.Action { case "delete": - if err := postprocessing.DeleteFunc(ctx, file, mo.FuncName, "java"); err != nil { + if err := postprocessing.DeleteMethod(file, mo.FuncName, "java"); err != nil { if strings.Contains(err.Error(), "not found") { return nil } diff --git a/internal/postprocessing/config.go b/internal/postprocessing/config.go index c35d22d6804..682f8bd5eff 100644 --- a/internal/postprocessing/config.go +++ b/internal/postprocessing/config.go @@ -17,7 +17,9 @@ package postprocessing import ( "context" + "fmt" "os" + "strings" "github.com/googleapis/librarian/internal/yaml" ) @@ -60,6 +62,18 @@ type CopyConfig struct { Dst string `yaml:"dst"` } +// Validate validates the postprocess configuration. +func (c *Config) Validate() error { + for _, mo := range c.MethodOperations { + if mo.Action == "delete" { + if !strings.Contains(mo.FuncName, "(") || !strings.Contains(mo.FuncName, ")") { + return fmt.Errorf("%w: %q (must contain parameter list in parentheses)", errInvalidSignature, mo.FuncName) + } + } + } + return nil +} + // ParseConfig parses the postprocess.yaml file. func ParseConfig(ctx context.Context, path string) (*Config, error) { data, err := os.ReadFile(path) @@ -70,5 +84,8 @@ func ParseConfig(ctx context.Context, path string) (*Config, error) { if err != nil { return nil, err } + if err := configPtr.Validate(); err != nil { + return nil, fmt.Errorf("invalid config: %w", err) + } return configPtr, nil } diff --git a/internal/postprocessing/config_test.go b/internal/postprocessing/config_test.go index 154800cebb6..cd9ef27cf4c 100644 --- a/internal/postprocessing/config_test.go +++ b/internal/postprocessing/config_test.go @@ -16,6 +16,7 @@ package postprocessing import ( "context" + "errors" "os" "path/filepath" "testing" @@ -131,3 +132,25 @@ func TestParseConfig_InvalidYAML(t *testing.T) { t.Error("ParseConfig() expected error for invalid YAML, got nil") } } + +func TestParseConfig_InvalidSignature(t *testing.T) { + yamlContent := ` +method_operations: + - path: path/to/file.java + action: delete + func_name: "invalidSignatureNoParentheses" +` + dir := t.TempDir() + configPath := filepath.Join(dir, "postprocess.yaml") + if err := os.WriteFile(configPath, []byte(yamlContent), 0644); err != nil { + t.Fatal(err) + } + + _, err := ParseConfig(context.Background(), configPath) + if err == nil { + t.Fatal("ParseConfig() expected error for invalid signature, got nil") + } + if !errors.Is(err, errInvalidSignature) { + t.Errorf("expected error to wrap errInvalidSignature, got %v", err) + } +} From cf6d4556b63956bc1a15dd54053e373f9b2be257 Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Mon, 15 Jun 2026 20:55:45 -0400 Subject: [PATCH 009/108] fix: revert fileops to upstream and fix postprocess syntax --- internal/librarian/java/postprocess.go | 1 - internal/postprocessing/fileops.go | 81 ------------------------- internal/postprocessing/fileops_test.go | 69 ++++++--------------- 3 files changed, 19 insertions(+), 132 deletions(-) diff --git a/internal/librarian/java/postprocess.go b/internal/librarian/java/postprocess.go index ce334122c11..6783ea114bb 100644 --- a/internal/librarian/java/postprocess.go +++ b/internal/librarian/java/postprocess.go @@ -386,7 +386,6 @@ func restructureModules(params postProcessParams, destRoot string, keepSet map[s gapicTestDest = filepath.Join(destRoot, "src", "test") protoFilesDestDir = filepath.Join(destRoot, "src", "main", "proto") } - } var actions []moveAction if shouldGenerateProto(params.javaAPI) { diff --git a/internal/postprocessing/fileops.go b/internal/postprocessing/fileops.go index 96d036af901..3ca493be699 100644 --- a/internal/postprocessing/fileops.go +++ b/internal/postprocessing/fileops.go @@ -23,7 +23,6 @@ import ( "regexp" "strings" - "github.com/bmatcuk/doublestar/v4" "github.com/googleapis/librarian/internal/filesystem" ) @@ -37,37 +36,6 @@ func CopyFile(src, dst string) error { return filesystem.CopyFile(src, dst) } -// Replace replaces all instances of original with replacement in the file at path. -// It supports glob patterns if path contains glob characters. -func Replace(path, original, replacement string) error { - var files []string - var err error - - if strings.ContainsAny(path, "*?[]{}") { - files, err = doublestar.FilepathGlob(path) - if err != nil { - return err - } - } else { - files = []string{path} - } - - for _, f := range files { - content, err := os.ReadFile(f) - if err != nil { - if os.IsNotExist(err) { - continue // Silently ignore missing files, like synthtool - } - return err - } - newContent := strings.ReplaceAll(string(content), original, replacement) - if err := os.WriteFile(f, []byte(newContent), 0644); err != nil { - return err - } - } - return nil -} - // RemoveFile removes the file at the specified path. func RemoveFile(path string) error { return os.Remove(path) @@ -109,52 +77,3 @@ func ReplaceRegex(path, pattern, replacement string) error { newContent := re.ReplaceAll(content, []byte(replacement)) return os.WriteFile(path, newContent, 0644) } - -// ReplaceRegex replaces all instances of pattern with replacement in the file at path. -// It converts Python-style capture groups like \g<1> to Go-style $1. -// It supports glob patterns if path contains glob characters. -func ReplaceRegex(path, pattern, replacement string) error { - var files []string - var err error - - if strings.ContainsAny(path, "*?[]{}") { - files, err = doublestar.FilepathGlob(path) - if err != nil { - return err - } - } else { - files = []string{path} - } - - // Convert Python-style capture groups \g<1> or \g to Go-style $1 or $name - rePythonGroup := regexp.MustCompile(`\\g<(\w+)>`) - replacement = rePythonGroup.ReplaceAllString(replacement, `$$$1`) - - // Convert Python-style numeric capture groups \1, \2 to Go-style $1, $2 - rePythonNumGroup := regexp.MustCompile(`(^|[^\\])\\(\d+)`) - replacement = rePythonNumGroup.ReplaceAllString(replacement, `${1}$$${2}`) - - if !strings.HasPrefix(pattern, "(?") { - pattern = "(?m)" + pattern - } - - re, err := regexp.Compile(pattern) - if err != nil { - return err - } - - for _, f := range files { - content, err := os.ReadFile(f) - if err != nil { - if os.IsNotExist(err) { - continue // Silently ignore missing files, like synthtool - } - return err - } - newContent := re.ReplaceAllString(string(content), replacement) - if err := os.WriteFile(f, []byte(newContent), 0644); err != nil { - return err - } - } - return nil -} diff --git a/internal/postprocessing/fileops_test.go b/internal/postprocessing/fileops_test.go index 9e0f8d3832e..10aa4df27cb 100644 --- a/internal/postprocessing/fileops_test.go +++ b/internal/postprocessing/fileops_test.go @@ -95,35 +95,25 @@ func TestRemoveFile_Error(t *testing.T) { } func TestReplace(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - content string - original string - replacement string - want string - }{ - {"success", "Hello World", "World", "Go", "Hello Go"}, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - dir := t.TempDir() - path := filepath.Join(dir, "test.txt") - if err := os.WriteFile(path, []byte(test.content), 0644); err != nil { - t.Fatal(err) - } - if err := Replace(path, test.original, test.replacement); err != nil { - t.Fatal(err) - } - gotBytes, err := os.ReadFile(path) - if err != nil { - t.Fatal(err) - } - got := string(gotBytes) - if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("mismatch (-want +got):\n%s", diff) - } - }) + dir := t.TempDir() + path := filepath.Join(dir, "test.txt") + content := "Hello World" + original := "World" + replacement := "Go" + want := "Hello Go" + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatal(err) + } + if err := Replace(path, original, replacement); err != nil { + t.Fatal(err) + } + gotBytes, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + got := string(gotBytes) + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -150,27 +140,6 @@ func TestReplaceRegex(t *testing.T) { replacement: "Number", want: "Hello Number World", }, - { - name: "python style capture group", - content: "Hello World", - pattern: `(Hello) (World)`, - replacement: `\g<2> \g<1>`, - want: "World Hello", - }, - { - name: "python style capture group with named group", - content: "Hello World", - pattern: `(?PHello) (?PWorld)`, - replacement: `\g \g`, - want: "World Hello", - }, - { - name: "python style numeric capture group", - content: "Hello World", - pattern: `(Hello) (World)`, - replacement: `\2 \1`, - want: "World Hello", - }, } { t.Run(test.name, func(t *testing.T) { t.Parallel() From 4641cf25da4bebcb3348e6844073910f763d75cc Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Thu, 18 Jun 2026 15:04:41 -0400 Subject: [PATCH 010/108] feat(internal/postprocessing): integrate robust Java deprecation logic and fix linter/compilation issues --- internal/librarian/java/postprocess_new.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/librarian/java/postprocess_new.go b/internal/librarian/java/postprocess_new.go index 7caf3261c4f..4e09e619628 100644 --- a/internal/librarian/java/postprocess_new.go +++ b/internal/librarian/java/postprocess_new.go @@ -110,7 +110,7 @@ func postProcessLibraryNew(ctx context.Context, p libraryPostProcessParams) erro return fmt.Errorf("failed to duplicate method %q in %s: %w", mo.FuncName, file, err) } case "deprecate": - if err := postprocessing.DeprecateMethod(ctx, file, mo.FuncName, mo.DeprecationMessage, "java"); err != nil { + if err := postprocessing.DeprecateMethod(file, mo.FuncName, mo.DeprecationMessage, "java"); err != nil { if strings.Contains(err.Error(), "not found") { return nil } From a3f80de3bad8038966c455bfb76df911e1b0d628 Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Thu, 18 Jun 2026 16:38:22 -0400 Subject: [PATCH 011/108] feat(internal/postprocessing): validate method operations and replace configs strictly --- internal/librarian/java/postprocess_new.go | 9 ------- internal/postprocessing/config.go | 30 +++++++++++++++++++++- internal/postprocessing/fileops.go | 1 - 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/internal/librarian/java/postprocess_new.go b/internal/librarian/java/postprocess_new.go index 4e09e619628..c96f864382b 100644 --- a/internal/librarian/java/postprocess_new.go +++ b/internal/librarian/java/postprocess_new.go @@ -97,23 +97,14 @@ func postProcessLibraryNew(ctx context.Context, p libraryPostProcessParams) erro switch mo.Action { case "delete": if err := postprocessing.DeleteMethod(file, mo.FuncName, "java"); err != nil { - if strings.Contains(err.Error(), "not found") { - return nil - } return fmt.Errorf("failed to delete method %q in %s: %w", mo.FuncName, file, err) } case "duplicate": if err := postprocessing.DuplicateMethod(ctx, file, mo.FuncName, mo.NewName, "java"); err != nil { - if strings.Contains(err.Error(), "not found") { - return nil - } return fmt.Errorf("failed to duplicate method %q in %s: %w", mo.FuncName, file, err) } case "deprecate": if err := postprocessing.DeprecateMethod(file, mo.FuncName, mo.DeprecationMessage, "java"); err != nil { - if strings.Contains(err.Error(), "not found") { - return nil - } return fmt.Errorf("failed to deprecate method %q in %s: %w", mo.FuncName, file, err) } default: diff --git a/internal/postprocessing/config.go b/internal/postprocessing/config.go index 682f8bd5eff..45a5b88bf6e 100644 --- a/internal/postprocessing/config.go +++ b/internal/postprocessing/config.go @@ -17,6 +17,7 @@ package postprocessing import ( "context" + "errors" "fmt" "os" "strings" @@ -24,6 +25,8 @@ import ( "github.com/googleapis/librarian/internal/yaml" ) +var errEmptyNewName = errors.New("new method name is required and cannot be empty") + // Config represents the postprocess.yaml configuration. type Config struct { Replace []ReplaceConfig `yaml:"replace,omitempty"` @@ -64,11 +67,36 @@ type CopyConfig struct { // Validate validates the postprocess configuration. func (c *Config) Validate() error { + for _, r := range c.Replace { + if strings.TrimSpace(r.Original) == "" { + return fmt.Errorf("replace rule original text to replace cannot be empty") + } + } + for _, r := range c.ReplaceRegex { + if strings.TrimSpace(r.Pattern) == "" { + return fmt.Errorf("replace_regex rule pattern cannot be empty") + } + } for _, mo := range c.MethodOperations { - if mo.Action == "delete" { + switch mo.Action { + case "delete": if !strings.Contains(mo.FuncName, "(") || !strings.Contains(mo.FuncName, ")") { return fmt.Errorf("%w: %q (must contain parameter list in parentheses)", errInvalidSignature, mo.FuncName) } + case "duplicate": + if !strings.Contains(mo.FuncName, "(") || !strings.Contains(mo.FuncName, ")") { + return fmt.Errorf("%w: %q (must contain parameter list in parentheses)", errInvalidSignature, mo.FuncName) + } + if strings.TrimSpace(mo.NewName) == "" { + return fmt.Errorf("%w for duplicate method signature %q", errEmptyNewName, mo.FuncName) + } + case "deprecate": + if !strings.Contains(mo.FuncName, "(") || !strings.Contains(mo.FuncName, ")") { + return fmt.Errorf("%w: %q (must contain parameter list in parentheses)", errInvalidSignature, mo.FuncName) + } + if strings.TrimSpace(mo.DeprecationMessage) == "" { + return fmt.Errorf("%w for deprecating method %q", errEmptyDeprecationMessage, mo.FuncName) + } } } return nil diff --git a/internal/postprocessing/fileops.go b/internal/postprocessing/fileops.go index 3ca493be699..14b038cff6e 100644 --- a/internal/postprocessing/fileops.go +++ b/internal/postprocessing/fileops.go @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package postprocessing provides tools for the YAML-based postprocessing workflow. package postprocessing import ( From 520fe9d501df0f31227289cfa821da67f1006ceb Mon Sep 17 00:00:00 2001 From: Joe Wang <106995533+JoeWang1127@users.noreply.github.com> Date: Mon, 22 Jun 2026 13:08:28 +0000 Subject: [PATCH 012/108] chore(internal/librarian/nodejs): remove symlink workarounds for issue 6313 (#6521) The temporary build-step workarounds and associated TODO comments for the symlink extraction issue are removed from the Node.js installer tool configuration and its unit tests. Since native symlink extraction is now supported by the fetch package, the manual symlink creation steps in the generator build configuration are no longer necessary, and the test mock setup is simplified. Fixes https://github.com/googleapis/librarian/issues/6313 --- internal/librarian/nodejs/install_test.go | 6 +----- internal/librarian/nodejs/librarian.yaml | 6 ------ 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/internal/librarian/nodejs/install_test.go b/internal/librarian/nodejs/install_test.go index b85c9003c68..8017ab77dff 100644 --- a/internal/librarian/nodejs/install_test.go +++ b/internal/librarian/nodejs/install_test.go @@ -41,12 +41,8 @@ func TestInstall(t *testing.T) { genDir := filepath.Join(cache, repo+"@"+tool.Version, gapicGeneratorSubdir) - // TODO(https://github.com/googleapis/librarian/issues/6313): - // this can be just "templates", "protos" once the workaround in - // librarian.yaml is removed. for _, sub := range []string{ - "templates/cjs/typescript_gapic", - "templates/esm/typescript_gapic", + "templates", "protos", } { if err := os.MkdirAll(filepath.Join(genDir, sub), 0o755); err != nil { diff --git a/internal/librarian/nodejs/librarian.yaml b/internal/librarian/nodejs/librarian.yaml index 9fbe3ddadce..2cac2d4ace6 100644 --- a/internal/librarian/nodejs/librarian.yaml +++ b/internal/librarian/nodejs/librarian.yaml @@ -27,12 +27,6 @@ tools: # tsc only compiles TypeScript; the generator also needs templates/ # and protos/ in build/ (resolved via __dirname at runtime). - "./node_modules/.bin/tsc" - # Recreate the symlinks that are in the tarball, as fetch doesn't handle them - # at the moment. - # TODO(https://github.com/googleapis/librarian/issues/6313): - # remove this workaround when the issue is fixed. - - "ln -sf package.json templates/cjs/typescript_gapic/package.json.njk" - - "ln -sf package.json templates/esm/typescript_gapic/package.json.njk" - "cp -a templates protos build/" # Pass the full directory name into "pnpm add" so that the generator # is registered with the name "gapic-generator-typescript" rather than From 9cd8ee95b5be29f702dad03801c006e438448aa4 Mon Sep 17 00:00:00 2001 From: Joe Wang <106995533+JoeWang1127@users.noreply.github.com> Date: Mon, 22 Jun 2026 16:00:41 +0000 Subject: [PATCH 013/108] fix(internal/librarian/nodejs): correct product doc link in readme template (#6519) The product documentation link in the README template is corrected by removing the redundant trailing "/overview" path suffix. For #6442 --- internal/librarian/nodejs/template/_README.md.txt | 2 +- .../with_partials/google-cloud-secretmanager/README.md | 2 +- .../without_partials/google-cloud-secretmanager/README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/librarian/nodejs/template/_README.md.txt b/internal/librarian/nodejs/template/_README.md.txt index 821ebfb62a4..a054d764110 100644 --- a/internal/librarian/nodejs/template/_README.md.txt +++ b/internal/librarian/nodejs/template/_README.md.txt @@ -18,7 +18,7 @@ A comprehensive list of changes in each version may be found in [the CHANGELOG][homepage_changelog]. * [{{.Name}} API Nodejs Client API Reference]({{.ClientDoc}}) -* [{{.Name}} API Documentation]({{.ProductDoc}}/overview) +* [{{.Name}} API Documentation]({{.ProductDoc}}) Read more about the client libraries for Cloud APIs, including the older Google APIs Client Libraries, in [Client Libraries Explained][explained]. diff --git a/internal/librarian/nodejs/testdata/generate_readme/with_partials/google-cloud-secretmanager/README.md b/internal/librarian/nodejs/testdata/generate_readme/with_partials/google-cloud-secretmanager/README.md index e860d1a0a4e..442ca2a04eb 100644 --- a/internal/librarian/nodejs/testdata/generate_readme/with_partials/google-cloud-secretmanager/README.md +++ b/internal/librarian/nodejs/testdata/generate_readme/with_partials/google-cloud-secretmanager/README.md @@ -21,7 +21,7 @@ A comprehensive list of changes in each version may be found in [the CHANGELOG][homepage_changelog]. * [Secret Manager API Nodejs Client API Reference](https://cloud.google.com/nodejs/docs/reference/secret-manager/latest) -* [Secret Manager API Documentation](https://cloud.google.com/secret-manager/docs/overview) +* [Secret Manager API Documentation](https://cloud.google.com/secret-manager/docs) Read more about the client libraries for Cloud APIs, including the older Google APIs Client Libraries, in [Client Libraries Explained][explained]. diff --git a/internal/librarian/nodejs/testdata/generate_readme/without_partials/google-cloud-secretmanager/README.md b/internal/librarian/nodejs/testdata/generate_readme/without_partials/google-cloud-secretmanager/README.md index 3bde36bd62f..a836d590ef2 100644 --- a/internal/librarian/nodejs/testdata/generate_readme/without_partials/google-cloud-secretmanager/README.md +++ b/internal/librarian/nodejs/testdata/generate_readme/without_partials/google-cloud-secretmanager/README.md @@ -17,7 +17,7 @@ A comprehensive list of changes in each version may be found in [the CHANGELOG][homepage_changelog]. * [Secret Manager API Nodejs Client API Reference](https://cloud.google.com/nodejs/docs/reference/secret-manager/latest) -* [Secret Manager API Documentation](https://cloud.google.com/secret-manager/docs/overview) +* [Secret Manager API Documentation](https://cloud.google.com/secret-manager/docs) Read more about the client libraries for Cloud APIs, including the older Google APIs Client Libraries, in [Client Libraries Explained][explained]. From 68c0a2089d53f9f8604f23d86762afe26b66c884 Mon Sep 17 00:00:00 2001 From: Joe Wang <106995533+JoeWang1127@users.noreply.github.com> Date: Mon, 22 Jun 2026 16:00:57 +0000 Subject: [PATCH 014/108] feat(internal/librarian/nodejs): generate README in Node library (#6520) The new pure Go-based README generator is fully integrated into the Node.js post processing generation pipeline, completely replacing the old external script-based generator. The generator now creates the package `README.md` from the embedded Go text template during the post-processing phase, unless the `README.md` is explicitly listed in the library's keep configuration to preserve hand-written versions. This feature will not bring changes to `README.md` in google-cloud-node since `README.md` is in the default keep list. We will gradually rollout this feature after the partial migration is done. For https://github.com/googleapis/librarian/issues/6442 --- internal/librarian/nodejs/generate.go | 41 ++++------------------ internal/librarian/nodejs/generate_test.go | 13 ++----- internal/librarian/nodejs/readme.go | 2 +- internal/librarian/nodejs/readme_test.go | 4 +-- 4 files changed, 11 insertions(+), 49 deletions(-) diff --git a/internal/librarian/nodejs/generate.go b/internal/librarian/nodejs/generate.go index 2431ca85a13..8245b32f0d6 100644 --- a/internal/librarian/nodejs/generate.go +++ b/internal/librarian/nodejs/generate.go @@ -34,7 +34,6 @@ import ( "github.com/googleapis/librarian/internal/filesystem" "github.com/googleapis/librarian/internal/serviceconfig" "github.com/googleapis/librarian/internal/sources" - "github.com/googleapis/librarian/internal/yaml" ) const ( @@ -309,8 +308,12 @@ func runPostProcessor(ctx context.Context, cfg *config.Config, library *config.L return fmt.Errorf("librarian.js failed: %w", err) } } - if err := generateReadme(ctx, outDir); err != nil { - return err + // TODO(https://github.com/googleapis/librarian/issues/6442): remove this if block + // once all readme are generated in google-cloud-node. + if !slices.Contains(library.Keep, "README.md") { + if err := generateReadme(cfg, library, googleapisDir, outDir); err != nil { + return fmt.Errorf("failed to generate README.md: %w", err) + } } if err := removeRedundantLinterFiles(library, outDir); err != nil { return fmt.Errorf("failed to remove redundant linter files: %w", err) @@ -370,38 +373,6 @@ func movePackageFromStaging(ctx context.Context, library *config.Library, repoRo return nil } -// generateReadme generates the README.md file in the package root, replacing -// any introduction and body specified in .readme-partials.yaml. -func generateReadme(ctx context.Context, outDir string) error { - readmePartials := filepath.Join(outDir, ".readme-partials.yaml") - if _, err := os.Stat(readmePartials); err == nil { - type partials struct { - Introduction string `yaml:"introduction"` - Body string `yaml:"body"` - } - p, err := yaml.Read[partials](readmePartials) - if err != nil { - return fmt.Errorf("failed to parse %s: %w", readmePartials, err) - } - for name, replacement := range map[string]string{ - "introduction": p.Introduction, - "body": p.Body, - } { - if replacement == "" { - continue - } - if err := command.RunInDir(ctx, outDir, "npx", "gapic-node-processing", "generate-readme", - fmt.Sprintf("--source-path=%s", outDir), - fmt.Sprintf("--string-to-replace=[//]: # \"partials.%s\"", name), - fmt.Sprintf("--replacement-string=%s", replacement), - ); err != nil { - return fmt.Errorf("generate-readme (%s) failed: %w", name, err) - } - } - } - return nil -} - // TODO(https://github.com/googleapis/google-cloud-node/issues/8286): gapic-generator-typescript // unconditionally generates redundant linter configuration files (.eslintignore, .eslintrc.json, etc.). // This post-processing cleanup function removes them unless explicitly kept in librarian.yaml. diff --git a/internal/librarian/nodejs/generate_test.go b/internal/librarian/nodejs/generate_test.go index 670bef9044a..9c31969dd0f 100644 --- a/internal/librarian/nodejs/generate_test.go +++ b/internal/librarian/nodejs/generate_test.go @@ -626,7 +626,7 @@ func TestRunPostProcessor_CustomScripts(t *testing.T) { library := &config.Library{ Name: "google-cloud-secretmanager", APIs: []*config.API{{Path: "google/cloud/secretmanager/v1"}}, - Keep: []string{"librarian.js", ".readme-partials.yaml", "README.md"}, + Keep: []string{"librarian.js", ".readme-partials.yaml"}, } outDir := filepath.Join(repoRoot, "packages", library.Name) if err := os.MkdirAll(outDir, 0755); err != nil { @@ -653,22 +653,14 @@ func TestRunPostProcessor_CustomScripts(t *testing.T) { if err := os.WriteFile(filepath.Join(protoDir, "service.proto"), []byte(protoContent), 0644); err != nil { t.Fatal(err) } - librarianJS := filepath.Join(outDir, "librarian.js") if err := os.WriteFile(librarianJS, []byte("const fs = require('fs');\nfs.writeFileSync('librarian-ran.txt', 'yes');\n"), 0644); err != nil { t.Fatal(err) } - - readmePath := filepath.Join(outDir, "README.md") - if err := os.WriteFile(readmePath, []byte("Some Title\n[//]: # \"partials.introduction\"\n[//]: # \"partials.body\"\nFooter"), 0644); err != nil { - t.Fatal(err) - } - readmePartials := filepath.Join(outDir, ".readme-partials.yaml") if err := os.WriteFile(readmePartials, []byte("introduction: 'intro text'\nbody: 'body text'"), 0644); err != nil { t.Fatal(err) } - cfg := &config.Config{ Language: config.LanguageNodejs, Repo: "googleapis/google-cloud-node", @@ -684,11 +676,10 @@ func TestRunPostProcessor_CustomScripts(t *testing.T) { if _, err := os.Stat(filepath.Join(repoRoot, "owl-bot-staging")); err != nil { t.Error("expected top-level owl-bot-staging directory to remain intact") } - if _, err := os.Stat(filepath.Join(repoRoot, "librarian-ran.txt")); err != nil { t.Errorf("expected librarian.js to run and create librarian-ran.txt in repoRoot: %v", err) } - + readmePath := filepath.Join(outDir, "README.md") content, err := os.ReadFile(readmePath) if err != nil { t.Fatal(err) diff --git a/internal/librarian/nodejs/readme.go b/internal/librarian/nodejs/readme.go index 8f6396bea45..bab997d25a2 100644 --- a/internal/librarian/nodejs/readme.go +++ b/internal/librarian/nodejs/readme.go @@ -55,7 +55,7 @@ type sampleMetadata struct { FilePath string } -func generateReadmeNew(cfg *config.Config, library *config.Library, googleapisDir, output string) (err error) { +func generateReadme(cfg *config.Config, library *config.Library, googleapisDir, output string) (err error) { metadata, err := generateRepoMetadata(cfg, library, googleapisDir) if err != nil { return err diff --git a/internal/librarian/nodejs/readme_test.go b/internal/librarian/nodejs/readme_test.go index a83058cb515..4ab554936b8 100644 --- a/internal/librarian/nodejs/readme_test.go +++ b/internal/librarian/nodejs/readme_test.go @@ -80,7 +80,7 @@ func TestGenerateReadme(t *testing.T) { if test.setup != nil { test.setup(output) } - if err := generateReadmeNew(cfg, library, absGoogleapisDir, output); err != nil { + if err := generateReadme(cfg, library, absGoogleapisDir, output); err != nil { t.Fatal(err) } got, err := os.ReadFile(filepath.Join(output, "README.md")) @@ -160,7 +160,7 @@ func TestGenerateReadme_Error(t *testing.T) { } { t.Run(test.name, func(t *testing.T) { outputDir := test.output(t) - err := generateReadmeNew(cfg, test.library, test.googleapisDir, outputDir) + err := generateReadme(cfg, test.library, test.googleapisDir, outputDir) if !errors.Is(err, test.wantErr) { t.Errorf("expected error %v, got %v", test.wantErr, err) } From 7a891726ea1062afa3641a40732948ca3800d199 Mon Sep 17 00:00:00 2001 From: haphungw Date: Mon, 22 Jun 2026 10:37:53 -0700 Subject: [PATCH 015/108] feat(sidekick/rust): add condition to include `google-cloud-lro` as dependency (#6503) Include the `google-cloud-lro` dependency in a generated crate's `Cargo.toml` if any of its methods are configured as LRO pollers (`IsLroPoller`). --- internal/sidekick/rust/codec.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/sidekick/rust/codec.go b/internal/sidekick/rust/codec.go index 0f556733200..928298ead46 100644 --- a/internal/sidekick/rust/codec.go +++ b/internal/sidekick/rust/codec.go @@ -385,7 +385,7 @@ func resolveUsedPackages(model *api.API, extraPackages []*packagez) { // not save us any computations. for _, m := range s.Methods { - if m.OperationInfo != nil || m.DiscoveryLro != nil { + if m.OperationInfo != nil || m.IsLroPoller { hasLROs = true } if len(m.AutoPopulated) != 0 { From 09c74f696003106ccdfc104e8436b297a657b7bb Mon Sep 17 00:00:00 2001 From: Carlos O'Ryan Date: Mon, 22 Jun 2026 13:59:43 -0400 Subject: [PATCH 016/108] fix(sidekick/swift): UrlSafe requires custom serialization (#6522) Sidekick needs to generate a custom `encode()` and a custom initializer for messages when any field is url-safe encoded. --- internal/sidekick/swift/annotate_message.go | 16 +-- .../sidekick/swift/annotate_message_test.go | 127 ++++++++++++++++++ 2 files changed, 135 insertions(+), 8 deletions(-) diff --git a/internal/sidekick/swift/annotate_message.go b/internal/sidekick/swift/annotate_message.go index 06812c89fa8..74d5f3c6a06 100644 --- a/internal/sidekick/swift/annotate_message.go +++ b/internal/sidekick/swift/annotate_message.go @@ -133,16 +133,9 @@ func (c *codec) annotateMessage(message *api.Message, model *modelAnnotations) e if err != nil { return err } - if fieldCodec.Name != field.JSONName { + if fieldCodec.Name != field.JSONName || fieldCodec.UrlSafeValue { annotations.CustomSerialization = true } - if fieldCodec.PackageName != "" && fieldCodec.PackageName != c.Model.PackageName { - dep, err := c.addApiPackageDependency(fieldCodec.PackageName) - if err != nil { - return err - } - annotations.DependsOn[dep.Name] = dep - } if field.Map && !fieldCodec.IsStringKeyed() { // In ProtoJSON map fields with non-string keys need to be // serialized as JSON objects with key fields. In the generated @@ -150,6 +143,13 @@ func (c *codec) annotateMessage(message *api.Message, model *modelAnnotations) e // `Decodable` and `Encodable` protocol. annotations.CustomSerialization = true } + if fieldCodec.PackageName != "" && fieldCodec.PackageName != c.Model.PackageName { + dep, err := c.addApiPackageDependency(fieldCodec.PackageName) + if err != nil { + return err + } + annotations.DependsOn[dep.Name] = dep + } } if message.Pagination != nil { diff --git a/internal/sidekick/swift/annotate_message_test.go b/internal/sidekick/swift/annotate_message_test.go index 128cfeb7b96..dc2678eb56f 100644 --- a/internal/sidekick/swift/annotate_message_test.go +++ b/internal/sidekick/swift/annotate_message_test.go @@ -145,6 +145,133 @@ func TestAnnotateMessage(t *testing.T) { } } +func TestAnnotateMessage_Discovery(t *testing.T) { + mapMessage := &api.Message{ + Name: "map", + ID: "$map", + IsMap: true, + Fields: []*api.Field{ + {Name: "key", JSONName: "key", Typez: api.TypezString}, + {Name: "value", JSONName: "value", Typez: api.TypezBytes}, + }, + } + + for _, test := range []struct { + name string + message *api.Message + want *messageAnnotations + }{ + { + name: "simple", + message: &api.Message{ + Name: "Secret", + ID: ".test.Secret", + Package: "test", + Fields: []*api.Field{ + {Name: "field", JSONName: "field", ID: ".test.Secret.field", Typez: api.TypezString}, + }, + }, + want: &messageAnnotations{ + Name: "Secret", + TypeURL: "type.googleapis.com/test.Secret", + CustomSerialization: false, + SampleField: "field", + }, + }, + { + name: "required", + message: &api.Message{ + Name: "Secret", + ID: ".test.Secret", + Package: "test", + Fields: []*api.Field{ + {Name: "field", JSONName: "field", ID: ".test.Secret.field", Typez: api.TypezBytes}, + }, + }, + want: &messageAnnotations{ + Name: "Secret", + TypeURL: "type.googleapis.com/test.Secret", + CustomSerialization: true, + SampleField: "field", + }, + }, + { + name: "optional", + message: &api.Message{ + Name: "Secret", + ID: ".test.Secret", + Package: "test", + Fields: []*api.Field{ + {Name: "field", JSONName: "field", ID: ".test.Secret.field", Typez: api.TypezBytes, Optional: true}, + }, + }, + want: &messageAnnotations{ + Name: "Secret", + TypeURL: "type.googleapis.com/test.Secret", + CustomSerialization: true, + SampleField: "field", + }, + }, + { + name: "repeated", + message: &api.Message{ + Name: "Secret", + ID: ".test.Secret", + Package: "test", + Fields: []*api.Field{ + {Name: "field", JSONName: "field", ID: ".test.Secret.field", Typez: api.TypezBytes, Repeated: true}, + }, + }, + want: &messageAnnotations{ + Name: "Secret", + TypeURL: "type.googleapis.com/test.Secret", + CustomSerialization: true, + SampleField: "field", + }, + }, + { + name: "map", + message: &api.Message{ + Name: "Secret", + ID: ".test.Secret", + Package: "test", + Fields: []*api.Field{ + { + Name: "field", + JSONName: "field", + ID: ".test.Secret.field", + Typez: api.TypezMessage, + TypezID: mapMessage.ID, + Map: true, + }, + }, + }, + want: &messageAnnotations{ + Name: "Secret", + TypeURL: "type.googleapis.com/test.Secret", + CustomSerialization: true, + SampleField: "field", + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + for _, f := range test.message.Fields { + f.Parent = test.message + } + model := api.NewTestAPI([]*api.Message{test.message}, []*api.Enum{}, []*api.Service{}) + model.AddMessage(mapMessage) + codec := newTestCodec(t, model, map[string]string{}) + codec.UrlSafeForBytes = true + if err := codec.annotateModel(); err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(test.want, test.message.Codec, cmpopts.IgnoreFields(messageAnnotations{}, "Model", "DependsOn")); diff != "" { + t.Errorf("mismatch (-want, +got):\n%s", diff) + } + }) + } +} + func TestAnnotateMessage_Pagination(t *testing.T) { pageSizeField := &api.Field{Name: "page_size", JSONName: "pageSize", Typez: api.TypezInt32} pageTokenField := &api.Field{Name: "page_token", JSONName: "pageToken", Typez: api.TypezString} From 6ee3e9554a129098bfd2befcaa6c0e243a055250 Mon Sep 17 00:00:00 2001 From: Noah Dietz Date: Mon, 22 Jun 2026 11:42:12 -0700 Subject: [PATCH 017/108] chore(librarian): mention release-please in add (#6526) --- cmd/librarian/doc.go | 4 ++++ internal/librarian/add.go | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/cmd/librarian/doc.go b/cmd/librarian/doc.go index 14db3410070..face1f6cdef 100644 --- a/cmd/librarian/doc.go +++ b/cmd/librarian/doc.go @@ -66,6 +66,10 @@ If the API path should naturally be included in an existing library, and if the language supports doing so, that library is modified. Otherwise, a new library is created. +While release-please is responsible for library releases, the relevant +release-please configuration will be updated as necessary to onboard any new +library. + To add a preview client of an existing library, prefix the API path with "preview/". diff --git a/internal/librarian/add.go b/internal/librarian/add.go index 1629f47ee7b..4c2e5d995d9 100644 --- a/internal/librarian/add.go +++ b/internal/librarian/add.go @@ -61,6 +61,10 @@ If the API path should naturally be included in an existing library, and if the language supports doing so, that library is modified. Otherwise, a new library is created. +While release-please is responsible for library releases, the relevant +release-please configuration will be updated as necessary to onboard any new +library. + To add a preview client of an existing library, prefix the API path with "preview/". From 9489715246601f6f39268afbd4e4ad68240997b4 Mon Sep 17 00:00:00 2001 From: Carlos O'Ryan Date: Mon, 22 Jun 2026 16:36:50 -0400 Subject: [PATCH 018/108] feat(sidekick/swift): qualified names for requests (#6506) In discovery-based services the requests will be nested messages. We need to generate code that fully qualifies the request type, like we do for fields. This requires new annotations for the messages. --- internal/sidekick/swift/annotate_message.go | 15 ++++++++++ .../sidekick/swift/annotate_message_test.go | 25 +++++++++++++---- .../sidekick/swift/annotate_method_test.go | 28 +++++++++++-------- .../swift/generate_service_swift_test.go | 5 ++-- internal/sidekick/swift/generate_stub_test.go | 2 +- .../templates/common/logging.swift.mustache | 2 +- .../common/method_signature/lro.mustache | 2 +- .../method_signature/lro_options.mustache | 2 +- .../method_signature/pagination.mustache | 2 +- .../pagination_options.mustache | 2 +- .../common/method_signature/request.mustache | 2 +- .../method_signature/request_options.mustache | 2 +- .../templates/common/retry.swift.mustache | 2 +- .../templates/common/service.swift.mustache | 6 ++-- .../snippet/method_request_body.mustache | 2 +- 15 files changed, 66 insertions(+), 33 deletions(-) diff --git a/internal/sidekick/swift/annotate_message.go b/internal/sidekick/swift/annotate_message.go index 74d5f3c6a06..36719fbce32 100644 --- a/internal/sidekick/swift/annotate_message.go +++ b/internal/sidekick/swift/annotate_message.go @@ -51,6 +51,16 @@ type messageAnnotations struct { // service that needs them is enabled. Messages that do not map to any // service use " && ". GatedOp string + + // The message type name when it appears as a method parameter name. + // + // Most of the time the request types are in the package namespace, or are + // imported with the mixin, e.g. `import GoogleCloudIamV1` imports the IAM + // mixin request types. + // + // For discovery-based APIs, the request are synthetic and generated within + // a scope. They need to be fully qualified. + ParameterTypeName string } // MessageImports returns the list of dependencies for this message. @@ -97,6 +107,10 @@ func (c *codec) annotateMessage(message *api.Message, model *modelAnnotations) e if len(message.Fields) != 0 { sampleField = camelCase(message.Fields[0].Name) } + parameterTypeName, err := c.messageTypeName(message) + if err != nil { + return err + } annotations := &messageAnnotations{ Name: pascalCase(message.Name), DocLines: docLines, @@ -105,6 +119,7 @@ func (c *codec) annotateMessage(message *api.Message, model *modelAnnotations) e CustomSerialization: len(message.OneOfs) > 0, DependsOn: map[string]*Dependency{}, SampleField: sampleField, + ParameterTypeName: parameterTypeName, } // Ensure the entire package depends on the package this message belongs to. diff --git a/internal/sidekick/swift/annotate_message_test.go b/internal/sidekick/swift/annotate_message_test.go index dc2678eb56f..8c9814ee196 100644 --- a/internal/sidekick/swift/annotate_message_test.go +++ b/internal/sidekick/swift/annotate_message_test.go @@ -46,6 +46,7 @@ func TestAnnotateMessage(t *testing.T) { TypeURL: "type.googleapis.com/test.Secret", CustomSerialization: false, SampleField: "secretKey", + ParameterTypeName: "Secret", }, wantImports: []string{"GoogleCloudWkt"}, }, @@ -63,6 +64,7 @@ func TestAnnotateMessage(t *testing.T) { TypeURL: "type.googleapis.com/test.Protocol", CustomSerialization: false, SampleField: "", + ParameterTypeName: "Protocol_", }, wantImports: []string{"GoogleCloudWkt"}, }, @@ -79,6 +81,7 @@ func TestAnnotateMessage(t *testing.T) { TypeURL: "type.googleapis.com/test.WithOneof", CustomSerialization: true, SampleField: "", + ParameterTypeName: "WithOneof", }, wantImports: []string{"GoogleCloudWkt"}, }, @@ -97,6 +100,7 @@ func TestAnnotateMessage(t *testing.T) { TypeURL: "type.googleapis.com/test.WithCustomJSON", CustomSerialization: true, SampleField: "secretKey", + ParameterTypeName: "WithCustomJSON", }, wantImports: []string{"GoogleCloudWkt"}, }, @@ -122,6 +126,7 @@ func TestAnnotateMessage(t *testing.T) { PageableItemField: "secretKey", PageableItemType: "SecretKey", SampleField: "secretKey", + ParameterTypeName: "WithPagination", }, wantImports: []string{"GoogleCloudGax", "GoogleCloudWkt"}, }, @@ -176,6 +181,7 @@ func TestAnnotateMessage_Discovery(t *testing.T) { TypeURL: "type.googleapis.com/test.Secret", CustomSerialization: false, SampleField: "field", + ParameterTypeName: "Secret", }, }, { @@ -193,6 +199,7 @@ func TestAnnotateMessage_Discovery(t *testing.T) { TypeURL: "type.googleapis.com/test.Secret", CustomSerialization: true, SampleField: "field", + ParameterTypeName: "Secret", }, }, { @@ -210,6 +217,7 @@ func TestAnnotateMessage_Discovery(t *testing.T) { TypeURL: "type.googleapis.com/test.Secret", CustomSerialization: true, SampleField: "field", + ParameterTypeName: "Secret", }, }, { @@ -227,6 +235,7 @@ func TestAnnotateMessage_Discovery(t *testing.T) { TypeURL: "type.googleapis.com/test.Secret", CustomSerialization: true, SampleField: "field", + ParameterTypeName: "Secret", }, }, { @@ -251,6 +260,7 @@ func TestAnnotateMessage_Discovery(t *testing.T) { TypeURL: "type.googleapis.com/test.Secret", CustomSerialization: true, SampleField: "field", + ParameterTypeName: "Secret", }, }, } { @@ -338,9 +348,10 @@ func TestAnnotateMessage_Pagination(t *testing.T) { // Verify annotations on request message gotRequest := inputType.Codec.(*messageAnnotations) wantRequest := &messageAnnotations{ - Name: "ListSecretsRequest", - TypeURL: "type.googleapis.com/google.cloud.secretmanager.v1.ListSecretsRequest", - SampleField: "pageSize", + Name: "ListSecretsRequest", + TypeURL: "type.googleapis.com/google.cloud.secretmanager.v1.ListSecretsRequest", + SampleField: "pageSize", + ParameterTypeName: "ListSecretsRequest", } if diff := cmp.Diff(wantRequest, gotRequest, cmpopts.IgnoreFields(messageAnnotations{}, "Model", "DependsOn")); diff != "" { t.Errorf("mismatch (-want, +got):\n%s", diff) @@ -359,6 +370,7 @@ func TestAnnotateMessage_Pagination(t *testing.T) { PageableItemField: "secrets", PageableItemType: "Secret", SampleField: "secrets", + ParameterTypeName: "ListSecretsResponse", } if diff := cmp.Diff(wantResponse, gotResponse, cmpopts.IgnoreFields(messageAnnotations{}, "Model", "DependsOn")); diff != "" { t.Errorf("mismatch (-want, +got):\n%s", diff) @@ -408,9 +420,10 @@ func TestAnnotateMessage_RecursiveNested(t *testing.T) { gotOuter := outerMessage.Codec.(*messageAnnotations) wantOuter := &messageAnnotations{ - Name: "OuterMessage", - TypeURL: "type.googleapis.com/google.cloud.secretmanager.v1.OuterMessage", - SampleField: "", + Name: "OuterMessage", + TypeURL: "type.googleapis.com/google.cloud.secretmanager.v1.OuterMessage", + SampleField: "", + ParameterTypeName: "OuterMessage", } if diff := cmp.Diff(wantOuter, gotOuter, cmpopts.IgnoreFields(messageAnnotations{}, "Model", "DependsOn")); diff != "" { t.Errorf("mismatch (-want, +got):\n%s", diff) diff --git a/internal/sidekick/swift/annotate_method_test.go b/internal/sidekick/swift/annotate_method_test.go index ed816f747d9..fc4b6c8de6e 100644 --- a/internal/sidekick/swift/annotate_method_test.go +++ b/internal/sidekick/swift/annotate_method_test.go @@ -26,14 +26,16 @@ import ( func TestAnnotateMethod(t *testing.T) { keyField := &api.Field{Name: "key", ID: ".test.Request.key", Typez: api.TypezString} inputType := &api.Message{ - Name: "Request", - ID: ".test.Request", - Fields: []*api.Field{keyField}, + Name: "Request", + ID: ".test.Request", + Package: "test", + Fields: []*api.Field{keyField}, } keyField.Parent = inputType outputType := &api.Message{ - Name: "Response", - ID: ".test.Response", + Name: "Response", + ID: ".test.Response", + Package: "test", Fields: []*api.Field{ {Name: "value", ID: ".test.Request.value", Typez: api.TypezString}, }, @@ -64,7 +66,7 @@ func TestAnnotateMethod(t *testing.T) { DocLines: []string{"Gets a thing.", "", "Test multiple comment lines.", ""}, HTTPMethod: "GET", HasBody: false, - ReturnType: "Response", + ReturnType: "GoogleTest.Response", }, }, { @@ -88,7 +90,7 @@ func TestAnnotateMethod(t *testing.T) { HasBody: true, IsBodyWildcard: false, BodyField: "key", - ReturnType: "Response", + ReturnType: "GoogleTest.Response", }, }, { @@ -111,7 +113,7 @@ func TestAnnotateMethod(t *testing.T) { HTTPMethod: "POST", HasBody: true, IsBodyWildcard: true, - ReturnType: "Response", + ReturnType: "GoogleTest.Response", }, }, { @@ -136,7 +138,7 @@ func TestAnnotateMethod(t *testing.T) { HTTPMethod: "GET", HasBody: false, QueryParams: []*api.Field{keyField}, - ReturnType: "Response", + ReturnType: "GoogleTest.Response", }, }, } { @@ -371,9 +373,10 @@ func TestAnnotateMethod_Pagination(t *testing.T) { // Verify request message annotations gotRequest := inputType.Codec.(*messageAnnotations) wantRequest := &messageAnnotations{ - Name: "ListRequest", - TypeURL: "type.googleapis.com/test.ListRequest", - SampleField: "pageSize", + Name: "ListRequest", + TypeURL: "type.googleapis.com/test.ListRequest", + SampleField: "pageSize", + ParameterTypeName: "ListRequest", } if diff := cmp.Diff(wantRequest, gotRequest, cmpopts.IgnoreFields(messageAnnotations{}, "Model", "DependsOn")); diff != "" { t.Errorf("mismatch (-want, +got):\n%s", diff) @@ -392,6 +395,7 @@ func TestAnnotateMethod_Pagination(t *testing.T) { PageableItemField: "items", PageableItemType: "Item", SampleField: "items", + ParameterTypeName: "ListResponse", } if diff := cmp.Diff(wantResponse, gotResponse, cmpopts.IgnoreFields(messageAnnotations{}, "Model", "DependsOn")); diff != "" { t.Errorf("mismatch (-want, +got):\n%s", diff) diff --git a/internal/sidekick/swift/generate_service_swift_test.go b/internal/sidekick/swift/generate_service_swift_test.go index 3f2e2200e76..f0ec98e9b5f 100644 --- a/internal/sidekick/swift/generate_service_swift_test.go +++ b/internal/sidekick/swift/generate_service_swift_test.go @@ -175,7 +175,8 @@ func TestGenerateService_Delegation(t *testing.T) { func TestGenerateService_SnippetFiles(t *testing.T) { outDir := t.TempDir() - dummyMessage := &api.Message{Name: "DummyMessage"} + packageName := "google.cloud.test.v1" + dummyMessage := &api.Message{Name: "DummyMessage", Package: packageName} iam := &api.Service{ Name: "IAM", Methods: []*api.Method{ @@ -202,7 +203,7 @@ func TestGenerateService_SnippetFiles(t *testing.T) { } model := api.NewTestAPI([]*api.Message{dummyMessage}, nil, []*api.Service{iam, secretManager}) - model.PackageName = "google.cloud.test.v1" + model.PackageName = packageName cfg := &parser.ModelConfig{ Codec: map[string]string{ diff --git a/internal/sidekick/swift/generate_stub_test.go b/internal/sidekick/swift/generate_stub_test.go index 8208b430355..f59a19ef66e 100644 --- a/internal/sidekick/swift/generate_stub_test.go +++ b/internal/sidekick/swift/generate_stub_test.go @@ -86,7 +86,7 @@ func TestGenerateService_StubStructure(t *testing.T) { got := extractBlock(t, contentStr, ` protocol ProtocolStub {`, "\n"+` }`) want := ` protocol ProtocolStub { func getThing( - request: Request, options: GoogleCloudGax.RequestOptions + request: SomeTestPackage.Request, options: GoogleCloudGax.RequestOptions ) async throws -> SomeTestPackage.Response }` diff --git a/internal/sidekick/swift/templates/common/logging.swift.mustache b/internal/sidekick/swift/templates/common/logging.swift.mustache index 7d7ff02b463..0f67c57f7a9 100644 --- a/internal/sidekick/swift/templates/common/logging.swift.mustache +++ b/internal/sidekick/swift/templates/common/logging.swift.mustache @@ -73,7 +73,7 @@ extension Clients { request: request, options: options, name: "{{Codec.Name}}", - action: { (r: {{InputType.Codec.Name}}, o: GoogleCloudGax.RequestOptions) async throws -> + action: { (r: {{InputType.Codec.ParameterTypeName}}, o: GoogleCloudGax.RequestOptions) async throws -> {{^ReturnsEmpty}}{{Codec.ReturnType}}{{/ReturnsEmpty}} {{#ReturnsEmpty}}Void{{/ReturnsEmpty}} in return try await self.inner.{{Codec.Name}}(request: r, options: o) diff --git a/internal/sidekick/swift/templates/common/method_signature/lro.mustache b/internal/sidekick/swift/templates/common/method_signature/lro.mustache index 1b5ae37656c..bb23580c7e5 100644 --- a/internal/sidekick/swift/templates/common/method_signature/lro.mustache +++ b/internal/sidekick/swift/templates/common/method_signature/lro.mustache @@ -13,4 +13,4 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. }} -{{Codec.Name}}(withPolling: {{InputType.Codec.Name}}) async throws -> any GoogleCloudGax.PollableOperation<{{Codec.LRO.ReturnType}}> +{{Codec.Name}}(withPolling: {{InputType.Codec.ParameterTypeName}}) async throws -> any GoogleCloudGax.PollableOperation<{{Codec.LRO.ReturnType}}> diff --git a/internal/sidekick/swift/templates/common/method_signature/lro_options.mustache b/internal/sidekick/swift/templates/common/method_signature/lro_options.mustache index 40129993df4..ecde5dcd33c 100644 --- a/internal/sidekick/swift/templates/common/method_signature/lro_options.mustache +++ b/internal/sidekick/swift/templates/common/method_signature/lro_options.mustache @@ -14,5 +14,5 @@ See the License for the specific language governing permissions and limitations under the License. }} {{Codec.Name}}( - withPolling: {{InputType.Codec.Name}}, options: GoogleCloudGax.RequestOptions + withPolling: {{InputType.Codec.ParameterTypeName}}, options: GoogleCloudGax.RequestOptions ) async throws -> any GoogleCloudGax.PollableOperation<{{Codec.LRO.ReturnType}}> diff --git a/internal/sidekick/swift/templates/common/method_signature/pagination.mustache b/internal/sidekick/swift/templates/common/method_signature/pagination.mustache index 24f8b1e063d..70d60f5670e 100644 --- a/internal/sidekick/swift/templates/common/method_signature/pagination.mustache +++ b/internal/sidekick/swift/templates/common/method_signature/pagination.mustache @@ -14,5 +14,5 @@ See the License for the specific language governing permissions and limitations under the License. }} {{Codec.Name}}( - byItem: {{InputType.Codec.Name}} + byItem: {{InputType.Codec.ParameterTypeName}} ) throws -> any AsyncSequence<{{OutputType.Codec.PageableItemType}}, Swift.Error> diff --git a/internal/sidekick/swift/templates/common/method_signature/pagination_options.mustache b/internal/sidekick/swift/templates/common/method_signature/pagination_options.mustache index a440db8c554..14be131a4bd 100644 --- a/internal/sidekick/swift/templates/common/method_signature/pagination_options.mustache +++ b/internal/sidekick/swift/templates/common/method_signature/pagination_options.mustache @@ -14,5 +14,5 @@ See the License for the specific language governing permissions and limitations under the License. }} {{Codec.Name}}( - byItem: {{InputType.Codec.Name}}, options: GoogleCloudGax.RequestOptions + byItem: {{InputType.Codec.ParameterTypeName}}, options: GoogleCloudGax.RequestOptions ) throws -> any AsyncSequence<{{OutputType.Codec.PageableItemType}}, Swift.Error> diff --git a/internal/sidekick/swift/templates/common/method_signature/request.mustache b/internal/sidekick/swift/templates/common/method_signature/request.mustache index 88c1a6c6fbc..f5e6eeaeefb 100644 --- a/internal/sidekick/swift/templates/common/method_signature/request.mustache +++ b/internal/sidekick/swift/templates/common/method_signature/request.mustache @@ -13,4 +13,4 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. }} -{{Codec.Name}}(request: {{InputType.Codec.Name}}) async throws{{^ReturnsEmpty}} -> {{Codec.ReturnType}}{{/ReturnsEmpty}} +{{Codec.Name}}(request: {{InputType.Codec.ParameterTypeName}}) async throws{{^ReturnsEmpty}} -> {{Codec.ReturnType}}{{/ReturnsEmpty}} diff --git a/internal/sidekick/swift/templates/common/method_signature/request_options.mustache b/internal/sidekick/swift/templates/common/method_signature/request_options.mustache index fbfcdf62dff..495a019bcfc 100644 --- a/internal/sidekick/swift/templates/common/method_signature/request_options.mustache +++ b/internal/sidekick/swift/templates/common/method_signature/request_options.mustache @@ -14,5 +14,5 @@ See the License for the specific language governing permissions and limitations under the License. }} {{Codec.Name}}( - request: {{InputType.Codec.Name}}, options: GoogleCloudGax.RequestOptions + request: {{InputType.Codec.ParameterTypeName}}, options: GoogleCloudGax.RequestOptions ) async throws{{^ReturnsEmpty}} -> {{Codec.ReturnType}}{{/ReturnsEmpty}} diff --git a/internal/sidekick/swift/templates/common/retry.swift.mustache b/internal/sidekick/swift/templates/common/retry.swift.mustache index 6bb0cd0376e..026a8ef364e 100644 --- a/internal/sidekick/swift/templates/common/retry.swift.mustache +++ b/internal/sidekick/swift/templates/common/retry.swift.mustache @@ -66,7 +66,7 @@ extension Clients { request: request, options: options, idempotent: {{Codec.Idempotent}}, - action: { (r: {{InputType.Codec.Name}}, o: GoogleCloudGax.RequestOptions) async throws -> + action: { (r: {{InputType.Codec.ParameterTypeName}}, o: GoogleCloudGax.RequestOptions) async throws -> {{^ReturnsEmpty}}{{Codec.ReturnType}}{{/ReturnsEmpty}} {{#ReturnsEmpty}}Void{{/ReturnsEmpty}} in return try await self.inner.{{Codec.Name}}(request: r, options: o) diff --git a/internal/sidekick/swift/templates/common/service.swift.mustache b/internal/sidekick/swift/templates/common/service.swift.mustache index 12e6ecc8408..356ab1d990f 100644 --- a/internal/sidekick/swift/templates/common/service.swift.mustache +++ b/internal/sidekick/swift/templates/common/service.swift.mustache @@ -202,7 +202,7 @@ extension {{Codec.Name}} { {{#Signatures}} public func {{> /templates/common/method_signature/overload}} { - let request = {{Method.InputType.Codec.Name}}().with { + let request = {{Method.InputType.Codec.ParameterTypeName}}().with { {{#Fields}} $0.{{Codec.Name}} = {{Codec.Name}} {{/Fields}} @@ -231,7 +231,7 @@ extension {{Codec.Name}} { {{#Signatures}} public func {{> /templates/common/method_signature/pagination_overload}} { - let request = {{Method.InputType.Codec.Name}}().with { + let request = {{Method.InputType.Codec.ParameterTypeName}}().with { {{#Fields}} $0.{{Codec.Name}} = {{Codec.Name}} {{/Fields}} @@ -255,7 +255,7 @@ extension {{Codec.Name}} { {{#Signatures}} public func {{> /templates/common/method_signature/lro_overload}} { - let request = {{Method.InputType.Codec.Name}}().with { + let request = {{Method.InputType.Codec.ParameterTypeName}}().with { {{#Fields}} $0.{{Codec.Name}} = {{Codec.Name}} {{/Fields}} diff --git a/internal/sidekick/swift/templates/snippet/method_request_body.mustache b/internal/sidekick/swift/templates/snippet/method_request_body.mustache index 4ee84caad26..557467cef5f 100644 --- a/internal/sidekick/swift/templates/snippet/method_request_body.mustache +++ b/internal/sidekick/swift/templates/snippet/method_request_body.mustache @@ -22,7 +22,7 @@ limitations under the License. Conceptually there is a `list response = try await client.<>( ... )` around this partial. }} -{{InputType.Codec.Name}}() +{{InputType.Codec.ParameterTypeName}}() {{#SampleInfo}} .with { {{#IsRequestResourceName}} From aa8abbf0e2ef1a59cf9ba33562f800cf20223309 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Tue, 23 Jun 2026 19:56:11 +0000 Subject: [PATCH 019/108] chore(main): release 0.22.0 (#6448) :robot: I have created a release *beep* *boop* --- ## [0.22.0](https://github.com/googleapis/librarian/compare/v0.21.0...v0.22.0) (2026-06-22) ### Features * **internal/librarian/java:** remove redundant keep items in librarian.yaml ([#6291](https://github.com/googleapis/librarian/issues/6291)) ([2965478](https://github.com/googleapis/librarian/commit/2965478f6348bdcdb53dc8ee9a142a1a0dfceac9)) * **internal/librarian/java:** support alternate_headers for monolithc libraries ([#6481](https://github.com/googleapis/librarian/issues/6481)) ([4165a09](https://github.com/googleapis/librarian/commit/4165a0982afee3c8ee28170b9c476eeeff2f2d17)) * **internal/librarian/nodejs:** add release level markdown generation ([#6476](https://github.com/googleapis/librarian/issues/6476)) ([1d1281f](https://github.com/googleapis/librarian/commit/1d1281f939b2b97df259c605fa70a7395626345e)) * **internal/librarian/nodejs:** add support for readme partials ([#6505](https://github.com/googleapis/librarian/issues/6505)) ([eca8e3d](https://github.com/googleapis/librarian/commit/eca8e3d26641893deea785e2d5850ace165ffee3)), closes [#6442](https://github.com/googleapis/librarian/issues/6442) * **internal/librarian/nodejs:** extract sample metadata for node readme ([#6454](https://github.com/googleapis/librarian/issues/6454)) ([00e5e0d](https://github.com/googleapis/librarian/commit/00e5e0d8d0ff8307a861a8f5e08f8b896a1b8c50)), closes [#6442](https://github.com/googleapis/librarian/issues/6442) * **internal/librarian/nodejs:** generate README in Node library ([#6520](https://github.com/googleapis/librarian/issues/6520)) ([68c0a20](https://github.com/googleapis/librarian/commit/68c0a2089d53f9f8604f23d86762afe26b66c884)) * **internal/librarian/nodejs:** implement README generation without partials ([#6488](https://github.com/googleapis/librarian/issues/6488)) ([44e6954](https://github.com/googleapis/librarian/commit/44e69540b74ad4f8b8dd212e0d97952d1af6814d)) * **internal/postprocessing:** implement Java method deprecation ([#6497](https://github.com/googleapis/librarian/issues/6497)) ([289a385](https://github.com/googleapis/librarian/commit/289a385ffb83092022fa8c98ec36bd90f4883dd7)), closes [#6298](https://github.com/googleapis/librarian/issues/6298) * **internal/postprocessing:** implement Java method duplication ([#6484](https://github.com/googleapis/librarian/issues/6484)) ([0c5959c](https://github.com/googleapis/librarian/commit/0c5959c38e5c50805a7bd463c29a0351eb2e134c)), closes [#6298](https://github.com/googleapis/librarian/issues/6298) * **nodejs:** support per-API version mixin configuration([#6462](https://github.com/googleapis/librarian/issues/6462)) ([71cd24e](https://github.com/googleapis/librarian/commit/71cd24eed5cc7bd8ac35493c597a493b3081e36d)) * **postprocessing:** implement Java method deletion ([#6436](https://github.com/googleapis/librarian/issues/6436)) ([820646f](https://github.com/googleapis/librarian/commit/820646f3db936c15e4efb8a7998fdc20201d70a2)), closes [#6298](https://github.com/googleapis/librarian/issues/6298) * **sidekick/rust:** add condition to include `google-cloud-lro` as dependency ([#6503](https://github.com/googleapis/librarian/issues/6503)) ([7a89172](https://github.com/googleapis/librarian/commit/7a891726ea1062afa3641a40732948ca3800d199)) * **sidekick/rust:** bigquery query metadata ([#6407](https://github.com/googleapis/librarian/issues/6407)) ([6989ebc](https://github.com/googleapis/librarian/commit/6989ebcebf4cd2a04c36db7fcd544e8c464105b1)) * **sidekick/swift:** `bytes` for discovery docs ([#6433](https://github.com/googleapis/librarian/issues/6433)) ([d20f64c](https://github.com/googleapis/librarian/commit/d20f64c48ee852854344361eccaa60e5c76e9c58)) * **sidekick/swift:** generate method signature overloads ([#6473](https://github.com/googleapis/librarian/issues/6473)) ([27a72be](https://github.com/googleapis/librarian/commit/27a72beb520feb5d7c04701e967a3454f1367b39)) * **sidekick/swift:** qualified names for requests ([#6506](https://github.com/googleapis/librarian/issues/6506)) ([9489715](https://github.com/googleapis/librarian/commit/9489715246601f6f39268afbd4e4ad68240997b4)) * **sidekick:** parse method signatures ([#6451](https://github.com/googleapis/librarian/issues/6451)) ([7a433e7](https://github.com/googleapis/librarian/commit/7a433e71b701feda02bcf63e78624dcd08c5eb27)) * **sidekick:** parse method signatures ([#6461](https://github.com/googleapis/librarian/issues/6461)) ([16aa2e6](https://github.com/googleapis/librarian/commit/16aa2e6b1d00b63374d8571a0ba3e613b3768b28)) ### Bug Fixes * **internal/librarian/nodejs:** correct product doc link in readme template ([#6519](https://github.com/googleapis/librarian/issues/6519)) ([9cd8ee9](https://github.com/googleapis/librarian/commit/9cd8ee95b5be29f702dad03801c006e438448aa4)), closes [#6442](https://github.com/googleapis/librarian/issues/6442) * **internal/librarian/nodejs:** path leak during generte_readme ([#6470](https://github.com/googleapis/librarian/issues/6470)) ([d3e7c16](https://github.com/googleapis/librarian/commit/d3e7c169c3e028720cd8d3c1369972bc376d2ede)) * **internal/postprocessing:** support deleting multiple methods and extract boundary finder ([#6471](https://github.com/googleapis/librarian/issues/6471)) ([20442d8](https://github.com/googleapis/librarian/commit/20442d805274eec9e1ab3362ad4586f3afe0957c)), closes [#6298](https://github.com/googleapis/librarian/issues/6298) * **librarian:** print errors on failure ([#6458](https://github.com/googleapis/librarian/issues/6458)) ([37e4f91](https://github.com/googleapis/librarian/commit/37e4f915221045cba9e26f78c4e036d8d08076ed)) * **sidekick/rust:** disable docs/clippy warning for BQ generated files ([#6498](https://github.com/googleapis/librarian/issues/6498)) ([0a6a4d8](https://github.com/googleapis/librarian/commit/0a6a4d8f95b552a52d2d637d6db5f95499e5a9d8)) * **sidekick/rust:** use struct initializer for QueryMetadata ([#6504](https://github.com/googleapis/librarian/issues/6504)) ([2bdb3b5](https://github.com/googleapis/librarian/commit/2bdb3b5f262bad05ce2a569828f06b9445ab78bd)) * **sidekick/swift:** UrlSafe requires custom serialization ([#6522](https://github.com/googleapis/librarian/issues/6522)) ([09c74f6](https://github.com/googleapis/librarian/commit/09c74f696003106ccdfc104e8436b297a657b7bb)) * **surfer:** print errors on failure ([#6465](https://github.com/googleapis/librarian/issues/6465)) ([d91bf4c](https://github.com/googleapis/librarian/commit/d91bf4c4c6895fe7401cdea02fe0f2c64fb286d8)) --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). --------- Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> Co-authored-by: Joe Wang <106995533+JoeWang1127@users.noreply.github.com> --- .release-please-manifest.json | 2 +- CHANGELOG.md | 36 +++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index d3535156fcb..788b0fa7632 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "0.21.0" + ".": "0.22.0" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 3206216703d..25fad1c6fa0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,41 @@ # Changelog +## [0.22.0](https://github.com/googleapis/librarian/compare/v0.21.0...v0.22.0) (2026-06-22) + + +### Features + +* **internal/librarian/java:** remove redundant keep items in librarian.yaml ([#6291](https://github.com/googleapis/librarian/issues/6291)) ([2965478](https://github.com/googleapis/librarian/commit/2965478f6348bdcdb53dc8ee9a142a1a0dfceac9)) +* **internal/librarian/java:** support alternate_headers for monolithc libraries ([#6481](https://github.com/googleapis/librarian/issues/6481)) ([4165a09](https://github.com/googleapis/librarian/commit/4165a0982afee3c8ee28170b9c476eeeff2f2d17)) +* **internal/librarian/nodejs:** add release level markdown generation ([#6476](https://github.com/googleapis/librarian/issues/6476)) ([1d1281f](https://github.com/googleapis/librarian/commit/1d1281f939b2b97df259c605fa70a7395626345e)) +* **internal/librarian/nodejs:** add support for readme partials ([#6505](https://github.com/googleapis/librarian/issues/6505)) ([eca8e3d](https://github.com/googleapis/librarian/commit/eca8e3d26641893deea785e2d5850ace165ffee3)), closes [#6442](https://github.com/googleapis/librarian/issues/6442) +* **internal/librarian/nodejs:** extract sample metadata for node readme ([#6454](https://github.com/googleapis/librarian/issues/6454)) ([00e5e0d](https://github.com/googleapis/librarian/commit/00e5e0d8d0ff8307a861a8f5e08f8b896a1b8c50)), closes [#6442](https://github.com/googleapis/librarian/issues/6442) +* **internal/librarian/nodejs:** generate README in Node library ([#6520](https://github.com/googleapis/librarian/issues/6520)) ([68c0a20](https://github.com/googleapis/librarian/commit/68c0a2089d53f9f8604f23d86762afe26b66c884)) +* **internal/librarian/nodejs:** implement README generation without partials ([#6488](https://github.com/googleapis/librarian/issues/6488)) ([44e6954](https://github.com/googleapis/librarian/commit/44e69540b74ad4f8b8dd212e0d97952d1af6814d)) +* **internal/postprocessing:** implement Java method deprecation ([#6497](https://github.com/googleapis/librarian/issues/6497)) ([289a385](https://github.com/googleapis/librarian/commit/289a385ffb83092022fa8c98ec36bd90f4883dd7)), closes [#6298](https://github.com/googleapis/librarian/issues/6298) +* **internal/postprocessing:** implement Java method duplication ([#6484](https://github.com/googleapis/librarian/issues/6484)) ([0c5959c](https://github.com/googleapis/librarian/commit/0c5959c38e5c50805a7bd463c29a0351eb2e134c)), closes [#6298](https://github.com/googleapis/librarian/issues/6298) +* **nodejs:** support per-API version mixin configuration([#6462](https://github.com/googleapis/librarian/issues/6462)) ([71cd24e](https://github.com/googleapis/librarian/commit/71cd24eed5cc7bd8ac35493c597a493b3081e36d)) +* **postprocessing:** implement Java method deletion ([#6436](https://github.com/googleapis/librarian/issues/6436)) ([820646f](https://github.com/googleapis/librarian/commit/820646f3db936c15e4efb8a7998fdc20201d70a2)), closes [#6298](https://github.com/googleapis/librarian/issues/6298) +* **sidekick/rust:** add condition to include `google-cloud-lro` as dependency ([#6503](https://github.com/googleapis/librarian/issues/6503)) ([7a89172](https://github.com/googleapis/librarian/commit/7a891726ea1062afa3641a40732948ca3800d199)) +* **sidekick/rust:** bigquery query metadata ([#6407](https://github.com/googleapis/librarian/issues/6407)) ([6989ebc](https://github.com/googleapis/librarian/commit/6989ebcebf4cd2a04c36db7fcd544e8c464105b1)) +* **sidekick/swift:** `bytes` for discovery docs ([#6433](https://github.com/googleapis/librarian/issues/6433)) ([d20f64c](https://github.com/googleapis/librarian/commit/d20f64c48ee852854344361eccaa60e5c76e9c58)) +* **sidekick/swift:** generate method signature overloads ([#6473](https://github.com/googleapis/librarian/issues/6473)) ([27a72be](https://github.com/googleapis/librarian/commit/27a72beb520feb5d7c04701e967a3454f1367b39)) +* **sidekick/swift:** qualified names for requests ([#6506](https://github.com/googleapis/librarian/issues/6506)) ([9489715](https://github.com/googleapis/librarian/commit/9489715246601f6f39268afbd4e4ad68240997b4)) +* **sidekick:** parse method signatures ([#6451](https://github.com/googleapis/librarian/issues/6451)) ([7a433e7](https://github.com/googleapis/librarian/commit/7a433e71b701feda02bcf63e78624dcd08c5eb27)) +* **sidekick:** parse method signatures ([#6461](https://github.com/googleapis/librarian/issues/6461)) ([16aa2e6](https://github.com/googleapis/librarian/commit/16aa2e6b1d00b63374d8571a0ba3e613b3768b28)) + + +### Bug Fixes + +* **internal/librarian/nodejs:** correct product doc link in readme template ([#6519](https://github.com/googleapis/librarian/issues/6519)) ([9cd8ee9](https://github.com/googleapis/librarian/commit/9cd8ee95b5be29f702dad03801c006e438448aa4)), closes [#6442](https://github.com/googleapis/librarian/issues/6442) +* **internal/librarian/nodejs:** path leak during generate_readme ([#6470](https://github.com/googleapis/librarian/issues/6470)) ([d3e7c16](https://github.com/googleapis/librarian/commit/d3e7c169c3e028720cd8d3c1369972bc376d2ede)) +* **internal/postprocessing:** support deleting multiple methods and extract boundary finder ([#6471](https://github.com/googleapis/librarian/issues/6471)) ([20442d8](https://github.com/googleapis/librarian/commit/20442d805274eec9e1ab3362ad4586f3afe0957c)), closes [#6298](https://github.com/googleapis/librarian/issues/6298) +* **librarian:** print errors on failure ([#6458](https://github.com/googleapis/librarian/issues/6458)) ([37e4f91](https://github.com/googleapis/librarian/commit/37e4f915221045cba9e26f78c4e036d8d08076ed)) +* **sidekick/rust:** disable docs/clippy warning for BQ generated files ([#6498](https://github.com/googleapis/librarian/issues/6498)) ([0a6a4d8](https://github.com/googleapis/librarian/commit/0a6a4d8f95b552a52d2d637d6db5f95499e5a9d8)) +* **sidekick/rust:** use struct initializer for QueryMetadata ([#6504](https://github.com/googleapis/librarian/issues/6504)) ([2bdb3b5](https://github.com/googleapis/librarian/commit/2bdb3b5f262bad05ce2a569828f06b9445ab78bd)) +* **sidekick/swift:** UrlSafe requires custom serialization ([#6522](https://github.com/googleapis/librarian/issues/6522)) ([09c74f6](https://github.com/googleapis/librarian/commit/09c74f696003106ccdfc104e8436b297a657b7bb)) +* **surfer:** print errors on failure ([#6465](https://github.com/googleapis/librarian/issues/6465)) ([d91bf4c](https://github.com/googleapis/librarian/commit/d91bf4c4c6895fe7401cdea02fe0f2c64fb286d8)) + ## [0.21.0](https://github.com/googleapis/librarian/compare/v0.20.0...v0.21.0) (2026-06-16) From 1373628c9c8c639a666fe17813a4f313fe0f8c40 Mon Sep 17 00:00:00 2001 From: Tomo Suzuki Date: Tue, 23 Jun 2026 16:34:36 -0400 Subject: [PATCH 020/108] feat(nodejs): bump pnpm version to 11.7.0 and support v11+ global bin layout (#6494) This PR bumps `pnpm` version to `11.7.0` and `Node.js` to `22` in the Dockerfile, and updates the Node.js tool installer to support `pnpm` v11+ global bin directory layout. We assume `pnpm` v11 is used. ### Changes * **Dockerfile (pnpm & Node bump):** Bumps `NODEJS_PNPM_VERSION` to `11.7.0` and `NODEJS_NODE_VERSION` to `22` in `cmd/librarian/Dockerfile`. Node 22 is required for `pnpm` v11. * **Dockerfile (Build Fix):** Sets `PNPM_CONFIG_DANGEROUSLY_ALLOW_ALL_BUILDS=true` during the Docker image build (`librarian install nodejs`) to bypass interactive build approval prompts for dependencies (fixing `[ERR_PNPM_IGNORED_BUILDS]`). * *Reference:* Starting with pnpm v10, build scripts are blocked by default for security. See the [pnpm v10.0.0 release notes](https://github.com/pnpm/pnpm/releases/tag/v10.0.0) for details on why this is needed. * **PNPM_HOME Adjustment:** Updates `getPNPMEnv` in `internal/librarian/nodejs/install.go` to use the parent directory of Node bin directory as `PNPM_HOME` (`filepath.Dir(globalBin)`). This ensures `pnpm` (which installs to `PNPM_HOME/bin` in v11+) places the binaries back in the Node bin directory, which is in `PATH`. ### Scope Notes * The use of `LIBRARIAN_INSTALL_DIR` is outside the scope of this pull request. This pull request does not make it harder to use `LIBRARIAN_INSTALL_DIR`. Fixes #6480 --- cmd/librarian/Dockerfile | 6 +++--- internal/librarian/nodejs/install.go | 7 ++++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/cmd/librarian/Dockerfile b/cmd/librarian/Dockerfile index b91daf79e5d..cb5f7f798d1 100644 --- a/cmd/librarian/Dockerfile +++ b/cmd/librarian/Dockerfile @@ -70,8 +70,8 @@ ARG JAVA_VERSION=17 ARG JAVA_PYTHON_VERSION=3.12 ARG NODEJS_NVM_VERSION=0.40.4 ARG NODEJS_NVM_CHECKSUM=a8e082d8d1a9b61a09e5d3e1902d2930e5b1b84a86f9777c7d2eb50ea204c0141f6a97c54a860bc3282e7b000f1c669c755f5e0db7bd6d492072744c302c0a21 -ARG NODEJS_NODE_VERSION=18 -ARG NODEJS_PNPM_VERSION=7.32.2 +ARG NODEJS_NODE_VERSION=22 +ARG NODEJS_PNPM_VERSION=11.7.0 # ============== Build Stage ============== # Use the MOSS-compliant base image. @@ -285,7 +285,7 @@ RUN curl -fsSL --retry 5 --retry-delay 15 -o /tmp/protoc.zip \ cd /usr/local && unzip -o /tmp/protoc.zip && rm /tmp/protoc.zip # Install all the tools that Librarian needs for Node generation. -RUN /app/librarian install nodejs -v +RUN PNPM_CONFIG_DANGEROUSLY_ALLOW_ALL_BUILDS=true /app/librarian install nodejs -v # Allow normal users to read /root (as the generator is installed # in /root/.cache, for example) diff --git a/internal/librarian/nodejs/install.go b/internal/librarian/nodejs/install.go index 35c0e9ce2b5..a4e50e241ad 100644 --- a/internal/librarian/nodejs/install.go +++ b/internal/librarian/nodejs/install.go @@ -88,8 +88,13 @@ func getPNPMEnv(ctx context.Context) ([]string, error) { } globalBin := strings.TrimSpace(binOut) + // In pnpm v11+, globally installed binaries are stored in PNPM_HOME/bin. + // We want them to be stored directly in globalBin (node's bin directory). + // See https://pnpm.io/blog/releases/11.0#isolated-global-virtual-store-global-installs + pnpmHome := filepath.Dir(globalBin) + env := os.Environ() - env = append(env, "PNPM_HOME="+globalBin) + env = append(env, "PNPM_HOME="+pnpmHome) env = append(env, "PNPM_CONFIG_GLOBAL_BIN_DIR="+globalBin) env = append(env, "PNPM_CONFIG_GLOBAL_DIR="+filepath.Join(globalBin, "pnpm-global")) env = append(env, "PNPM_CONFIG_STORE_DIR="+filepath.Join(globalBin, "pnpm-store")) From 730364823f86ac636ddea676bb439fce8f8e5a6c Mon Sep 17 00:00:00 2001 From: Carlos O'Ryan Date: Wed, 24 Jun 2026 15:00:18 -0400 Subject: [PATCH 021/108] feat(sidekick/swift): generate synthetic messages (#6530) For discovery-based APIs the generator produces structs corresponding to the synthetic request messages. To avoid name clashes,these structs are generated in extensions to the client struct. --- internal/sidekick/swift/annotate_message.go | 11 +++ .../sidekick/swift/annotate_message_test.go | 73 +++++++++++++++++++ internal/sidekick/swift/field_type_name.go | 3 + internal/sidekick/swift/generate.go | 10 ++- .../swift/generate_service_swift_test.go | 72 ++++++++++++++++++ .../templates/common/placeholder.mustache | 21 ++++++ .../common/placeholder_file.swift.mustache | 34 +++++++++ 7 files changed, 222 insertions(+), 2 deletions(-) create mode 100644 internal/sidekick/swift/templates/common/placeholder.mustache create mode 100644 internal/sidekick/swift/templates/common/placeholder_file.swift.mustache diff --git a/internal/sidekick/swift/annotate_message.go b/internal/sidekick/swift/annotate_message.go index 36719fbce32..7e3398d1d29 100644 --- a/internal/sidekick/swift/annotate_message.go +++ b/internal/sidekick/swift/annotate_message.go @@ -52,6 +52,14 @@ type messageAnnotations struct { // service use " && ". GatedOp string + // In discovery-based APIs, the requests messages are nested messages of a + // message that is not generated, it is just a placeholder to represent the + // service. This placeholder provides a namespace for the requests. + // + // In the generated code, the namespace is implemented by the client struct, + // this is the name of this struct. + PlaceholderName string + // The message type name when it appears as a method parameter name. // // Most of the time the request types are in the package namespace, or are @@ -121,6 +129,9 @@ func (c *codec) annotateMessage(message *api.Message, model *modelAnnotations) e SampleField: sampleField, ParameterTypeName: parameterTypeName, } + if message.ServicePlaceholder { + annotations.PlaceholderName = pascalCase(message.Name + "Client") + } // Ensure the entire package depends on the package this message belongs to. if _, err := c.addApiPackageDependency(message.Package); err != nil { diff --git a/internal/sidekick/swift/annotate_message_test.go b/internal/sidekick/swift/annotate_message_test.go index 8c9814ee196..33f021366ae 100644 --- a/internal/sidekick/swift/annotate_message_test.go +++ b/internal/sidekick/swift/annotate_message_test.go @@ -130,6 +130,24 @@ func TestAnnotateMessage(t *testing.T) { }, wantImports: []string{"GoogleCloudGax", "GoogleCloudWkt"}, }, + { + name: "service placeholder", + message: &api.Message{ + Name: "Service", + ID: ".test.Service", + Package: "test", + ServicePlaceholder: true, + }, + want: &messageAnnotations{ + Name: "Service", + TypeURL: "type.googleapis.com/test.Service", + CustomSerialization: false, + SampleField: "", + ParameterTypeName: "Clients.ServiceClient", + PlaceholderName: "ServiceClient", + }, + wantImports: []string{"GoogleCloudWkt"}, + }, } { t.Run(test.name, func(t *testing.T) { for _, f := range test.message.Fields { @@ -282,6 +300,61 @@ func TestAnnotateMessage_Discovery(t *testing.T) { } } +func TestAnnotateMessage_DiscoveryRequests(t *testing.T) { + for _, test := range []struct { + name string + service *api.Service + request *api.Message + want *messageAnnotations + }{ + { + name: "basic message", + service: &api.Service{Name: "Service", Package: "test", ID: ".test.Service"}, + request: &api.Message{Name: "getRequest", Package: "test", ID: ".test.Service.getRequest", SyntheticRequest: true}, + want: &messageAnnotations{ + Name: "GetRequest", + TypeURL: "type.googleapis.com/test.Service.getRequest", + SampleField: "", + ParameterTypeName: "Clients.ServiceClient.GetRequest", + }, + }, + { + name: "service with reserved name", + service: &api.Service{Name: "Protocol", Package: "test", ID: ".test.Protocol"}, + request: &api.Message{Name: "listRequest", Package: "test", ID: ".test.Protocol.listRequest", SyntheticRequest: true}, + want: &messageAnnotations{ + Name: "ListRequest", + TypeURL: "type.googleapis.com/test.Protocol.listRequest", + SampleField: "", + ParameterTypeName: "Clients.ProtocolClient.ListRequest", + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + // Discovery requests are synthetic. The messages are injected into the data + // model by sidekick. To avoid clashes, sidekick puts the request messages + // within a placeholder named after the service. + servicePlaceholder := &api.Message{ + Name: test.service.Name, + Package: test.service.Package, + ID: test.service.ID, + ServicePlaceholder: true, + } + test.request.Parent = servicePlaceholder + servicePlaceholder.Messages = append(servicePlaceholder.Messages, test.request) + model := api.NewTestAPI([]*api.Message{servicePlaceholder}, []*api.Enum{}, []*api.Service{test.service}) + model.AddMessage(test.request) + codec := newTestCodec(t, model, map[string]string{}) + if err := codec.annotateModel(); err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(test.want, test.request.Codec, cmpopts.IgnoreFields(messageAnnotations{}, "Model", "DependsOn")); diff != "" { + t.Errorf("mismatch (-want, +got):\n%s", diff) + } + }) + } +} + func TestAnnotateMessage_Pagination(t *testing.T) { pageSizeField := &api.Field{Name: "page_size", JSONName: "pageSize", Typez: api.TypezInt32} pageTokenField := &api.Field{Name: "page_token", JSONName: "pageToken", Typez: api.TypezString} diff --git a/internal/sidekick/swift/field_type_name.go b/internal/sidekick/swift/field_type_name.go index 81c76efeec3..c4c0a9bb478 100644 --- a/internal/sidekick/swift/field_type_name.go +++ b/internal/sidekick/swift/field_type_name.go @@ -146,6 +146,9 @@ func scalarFieldTypeName(field *api.Field) (string, error) { func (c *codec) messageTypeName(m *api.Message) (string, error) { name := pascalCase(m.Name) + if m.ServicePlaceholder { + name = "Clients." + pascalCase(m.Name+"Client") + } if m.Parent == nil { prefix, err := c.externalTypePrefix(m.Package) if err != nil { diff --git a/internal/sidekick/swift/generate.go b/internal/sidekick/swift/generate.go index ccfd416e8c9..4dc22e9b9df 100644 --- a/internal/sidekick/swift/generate.go +++ b/internal/sidekick/swift/generate.go @@ -84,9 +84,15 @@ func (c *codec) outputPath(elem ...string) string { func (c *codec) generateMessages(outdir string, model *api.API, provider language.TemplateProvider) error { for _, m := range model.Messages { + output := c.outputPath(m.Name + ".swift") + template := "templates/common/message_file.swift.mustache" + if m.ServicePlaceholder { + output = c.outputPath(m.Name + "+Requests.swift") + template = "templates/common/placeholder_file.swift.mustache" + } generated := language.GeneratedFile{ - TemplatePath: "templates/common/message_file.swift.mustache", - OutputPath: c.outputPath(m.Name + ".swift"), + TemplatePath: template, + OutputPath: output, } if err := language.GenerateMessage(outdir, m, provider, generated); err != nil { return err diff --git a/internal/sidekick/swift/generate_service_swift_test.go b/internal/sidekick/swift/generate_service_swift_test.go index f0ec98e9b5f..9ba451012d8 100644 --- a/internal/sidekick/swift/generate_service_swift_test.go +++ b/internal/sidekick/swift/generate_service_swift_test.go @@ -15,6 +15,8 @@ package swift import ( + "bytes" + "fmt" "os" "path/filepath" "strings" @@ -888,3 +890,73 @@ func TestGenerateService_LRO_Empty(t *testing.T) { } } } + +func TestGenerateDiscoveryService_Files(t *testing.T) { + testdataDir, err := filepath.Abs("../../testdata") + if err != nil { + t.Fatal(err) + } + outDir := t.TempDir() + + cfg := &parser.ModelConfig{ + SpecificationFormat: config.SpecDiscovery, + ServiceConfig: filepath.Join(testdataDir, "googleapis/google/cloud/compute/v1/small-compute_v1.yaml"), + SpecificationSource: filepath.Join(testdataDir, "discovery/small-compute.v1.json"), + Codec: map[string]string{ + "copyright-year": "2038", + }, + } + model, err := parser.CreateModel(cfg) + if err != nil { + t.Fatal(err) + } + + if err := Generate(t.Context(), model, outDir, cfg, swiftConfig(t, nil)); err != nil { + t.Fatal(err) + } + + // Verify files + expectedDir := filepath.Join(outDir, "Sources", "GoogleCloudComputeV1") + + for _, test := range []struct { + filename string + serviceName string + structName string + }{ + { + filename: "AcceleratorTypes+Requests.swift", + serviceName: "AcceleratorTypes", + structName: "ListRequest", + }, + { + filename: "Addresses+Requests.swift", + serviceName: "Addresses", + structName: "DeleteRequest", + }, + { + filename: "instances+Requests.swift", + serviceName: "Instances", + structName: "GetRequest", + }, + } { + t.Run(test.serviceName, func(t *testing.T) { + filename := filepath.Join(expectedDir, test.filename) + content, err := os.ReadFile(filename) + if err != nil { + t.Fatal(err) + } + + // Verify it contains an extension to the right Clients.$ServiceName type. + wantExtension := fmt.Appendf(nil, "extension Clients.%sClient {", test.serviceName) + if !bytes.Contains(content, wantExtension) { + t.Errorf("expected extension %q in %s, got:\n%s", wantExtension, filename, content) + } + + // Verify the request struct definition appears in that file. + wantStruct := fmt.Appendf(nil, "public struct %s: ", test.structName) + if !bytes.Contains(content, wantStruct) { + t.Errorf("expected struct %q in %s, got:\n%s", wantStruct, filename, content) + } + }) + } +} diff --git a/internal/sidekick/swift/templates/common/placeholder.mustache b/internal/sidekick/swift/templates/common/placeholder.mustache new file mode 100644 index 00000000000..a061daba4bf --- /dev/null +++ b/internal/sidekick/swift/templates/common/placeholder.mustache @@ -0,0 +1,21 @@ +{{! +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +}} +extension Clients.{{Codec.PlaceholderName}} { + {{#Messages}} + + {{> /templates/common/message}} + {{/Messages}} +} diff --git a/internal/sidekick/swift/templates/common/placeholder_file.swift.mustache b/internal/sidekick/swift/templates/common/placeholder_file.swift.mustache new file mode 100644 index 00000000000..2b074b5b330 --- /dev/null +++ b/internal/sidekick/swift/templates/common/placeholder_file.swift.mustache @@ -0,0 +1,34 @@ +{{! +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +}} +// Code generated by sidekick. DO NOT EDIT. +// +// Copyright {{Codec.Model.CopyrightYear}} Google LLC +{{#Codec.Model.BoilerPlate}} +//{{{.}}} +{{/Codec.Model.BoilerPlate}} + +{{#Codec.IsGated}} +#if {{Codec.GateExpression}} +{{/Codec.IsGated}} +import Foundation +{{#Codec.MessageImports}} +import {{.}} +{{/Codec.MessageImports}} + +{{> /templates/common/placeholder}} +{{#Codec.IsGated}} +#endif +{{/Codec.IsGated}} From a6d950a157338083086d2b3d8c03150179628b10 Mon Sep 17 00:00:00 2001 From: yangyzs <171981480+yangyzs@users.noreply.github.com> Date: Wed, 24 Jun 2026 23:00:05 -0400 Subject: [PATCH 022/108] feat(internal/librarian/java): add decamelize utility for README generation (#6538) This utility splits CamelCase Java class names into space-separated words to generate human-readable titles for the README code samples table. For #6515 --- internal/librarian/java/readme.go | 30 +++++++++++++ internal/librarian/java/readme_test.go | 62 ++++++++++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 internal/librarian/java/readme.go create mode 100644 internal/librarian/java/readme_test.go diff --git a/internal/librarian/java/readme.go b/internal/librarian/java/readme.go new file mode 100644 index 00000000000..5ddd7d9f8ce --- /dev/null +++ b/internal/librarian/java/readme.go @@ -0,0 +1,30 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package java + +import ( + "regexp" + "strings" +) + +var ( + // Matches lowercase/digit followed by uppercase (e.g., "FooBar" -> "Foo Bar"). + camelCaseRegexp = regexp.MustCompile(`([a-z0-9])([A-Z])`) +) + +// decamelize converts CamelCase string to space-separated string (e.g. "CamelCase" -> "Camel Case"). +func decamelize(value string) string { + return strings.TrimSpace(camelCaseRegexp.ReplaceAllString(value, `$1 $2`)) +} diff --git a/internal/librarian/java/readme_test.go b/internal/librarian/java/readme_test.go new file mode 100644 index 00000000000..fc022d3c45e --- /dev/null +++ b/internal/librarian/java/readme_test.go @@ -0,0 +1,62 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package java + +import ( + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestDecamelize(t *testing.T) { + for _, test := range []struct { + name string + input string + want string + }{ + { + name: "camel case", + input: "CamelCase", + want: "Camel Case", + }, + { + name: "simple word", + input: "Word", + want: "Word", + }, + { + name: "already separated", + input: "Camel Case", + want: "Camel Case", + }, + { + name: "java acronym IamPolicy", + input: "IamPolicy", + want: "Iam Policy", + }, + { + name: "java acronym GcsBucket", + input: "GcsBucket", + want: "Gcs Bucket", + }, + } { + t.Run(test.name, func(t *testing.T) { + got := decamelize(test.input) + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + }) + } +} From 56bf365d10860f00bc9059a6d78f5f554c59406c Mon Sep 17 00:00:00 2001 From: Carlos O'Ryan Date: Thu, 25 Jun 2026 16:49:35 -0400 Subject: [PATCH 023/108] feat(sidekick/swift): handle clashing names (#6543) Filenames must be unique on case-insensitive filesystems. Swift also requires the basename to be unique, so I removed the `Clients/` intermediate directory because it was solving the problem of filename clashes, but not Swift source clashes. --- internal/sidekick/swift/codec.go | 15 +++++ internal/sidekick/swift/generate.go | 60 ++++++++++++++----- .../swift/generate_enum_swift_test.go | 17 +++++- .../swift/generate_message_swift_test.go | 14 ++++- .../swift/generate_service_swift_test.go | 29 +++++---- internal/sidekick/swift/generate_stub_test.go | 4 +- 6 files changed, 108 insertions(+), 31 deletions(-) diff --git a/internal/sidekick/swift/codec.go b/internal/sidekick/swift/codec.go index 42957860d2b..8ca8929892d 100644 --- a/internal/sidekick/swift/codec.go +++ b/internal/sidekick/swift/codec.go @@ -89,6 +89,21 @@ type codec struct { // If true, bytes need to be serialized and deserialized using the URL-safe // base64 alphabet. UrlSafeForBytes bool + + // Tracks generated files, considering case-insensitive filesystems. + // + // The generated file names are based on message, enum, and service names. + // When using case-insensitive filesystems (such as APFS on macOS, or NTFS + // on Windows) this can result in filename clashes when two messages differ + // only in case, such as `HTTPCheckName` vs. `HttpCheckName`. + // + // Furthermore, Swift requires all the files in a package to have different + // names, even if they are in different subdirectories. + // + // The codec uses this map to disambiguate names, the number of clashes for + // each (lowercased) name are tracked in this hash, and if necessary, the + // output file is disambiguated by appending `+${Counter}` to the basename. + GeneratedFiles map[string]int } func newCodec(model *api.API, cfg *parser.ModelConfig, swiftCfg *config.SwiftPackage, outdir string) (*codec, error) { diff --git a/internal/sidekick/swift/generate.go b/internal/sidekick/swift/generate.go index 4dc22e9b9df..ca60461ddde 100644 --- a/internal/sidekick/swift/generate.go +++ b/internal/sidekick/swift/generate.go @@ -20,6 +20,7 @@ import ( "embed" "fmt" "path/filepath" + "strings" "github.com/googleapis/librarian/internal/config" "github.com/googleapis/librarian/internal/sidekick/api" @@ -72,22 +73,36 @@ func Generate(ctx context.Context, model *api.API, outdir string, cfg *parser.Mo return language.GenerateFromModel(outdir, model, provider, generatedFiles) } -func (c *codec) outputPath(elem ...string) string { +func (c *codec) swiftFilename(basename string) string { + name := basename + ".swift" + key := strings.ToLower(basename) + if c.GeneratedFiles == nil { + c.GeneratedFiles = map[string]int{key: 0} + } else { + count, ok := c.GeneratedFiles[key] + if !ok { + c.GeneratedFiles[key] = 0 + } else { + c.GeneratedFiles[key] = count + 1 + // This can only deal with 1000 conflicts on the same name. If we + // ever have more, we need to have a serious discussion with the + // API team that produced that many conflicts. + name = fmt.Sprintf("%s+%03d.swift", basename, count) + } + + } if c.Module { - return filepath.Join(elem...) + return name } - full := make([]string, 0, len(elem)+2) - full = append(full, "Sources", c.PackageName) - full = append(full, elem...) - return filepath.Join(full...) + return filepath.Join("Sources", c.PackageName, name) } func (c *codec) generateMessages(outdir string, model *api.API, provider language.TemplateProvider) error { for _, m := range model.Messages { - output := c.outputPath(m.Name + ".swift") + output := c.swiftFilename(m.Name) template := "templates/common/message_file.swift.mustache" if m.ServicePlaceholder { - output = c.outputPath(m.Name + "+Requests.swift") + output = c.swiftFilename(m.Name + "+Requests") template = "templates/common/placeholder_file.swift.mustache" } generated := language.GeneratedFile{ @@ -105,7 +120,7 @@ func (c *codec) generateEnums(outdir string, model *api.API, provider language.T for _, e := range model.Enums { generated := language.GeneratedFile{ TemplatePath: "templates/common/enum_file.swift.mustache", - OutputPath: c.outputPath(e.Name + ".swift"), + OutputPath: c.swiftFilename(e.Name), } if err := language.GenerateEnum(outdir, e, provider, generated); err != nil { return err @@ -118,7 +133,7 @@ func (c *codec) generateServices(outdir string, model *api.API, provider languag for _, s := range model.Services { generated := language.GeneratedFile{ TemplatePath: "templates/common/service.swift.mustache", - OutputPath: c.outputPath(s.Name + ".swift"), + OutputPath: c.swiftFilename(s.Name), } if err := language.GenerateService(outdir, s, provider, generated); err != nil { return err @@ -133,13 +148,13 @@ func (c *codec) generateStubs(outdir string, model *api.API, provider language.T suffix string template string }{ - {suffix: "Stub.swift", template: "templates/common/stub.swift.mustache"}, - {suffix: "Logging.swift", template: "templates/common/logging.swift.mustache"}, - {suffix: "Retry.swift", template: "templates/common/retry.swift.mustache"}, + {suffix: "+Stub", template: "templates/common/stub.swift.mustache"}, + {suffix: "+Logging", template: "templates/common/logging.swift.mustache"}, + {suffix: "+Retry", template: "templates/common/retry.swift.mustache"}, } { generated := language.GeneratedFile{ TemplatePath: stub.template, - OutputPath: c.outputPath("Clients", s.Name+stub.suffix), + OutputPath: c.swiftFilename(s.Name + stub.suffix), } if err := language.GenerateService(outdir, s, provider, generated); err != nil { return err @@ -151,6 +166,21 @@ func (c *codec) generateStubs(outdir string, model *api.API, provider language.T func (c *codec) generateSnippets(outdir string, model *api.API, provider language.TemplateProvider) error { for _, s := range model.Services { + // If two services differ only in case (such as `fooService` and + // `FooService`), then this might generate clashing filenames in + // filesystems that are case insensitive. + // + // This seems unlikely: the services are always in a flat namespace, and + // they always use consistent naming conventions. We have only found + // clashes between messages and services or between messages, not + // between two services. + // + // Furthermore, fixing the problem would require changing the generated + // code, as the name of the snippet file is referenced in the generated + // comments. The effort to fix that does not seem worthwhile given how + // unlikely is the problem. + // + // If I (coryan@) am wrong, we can fix the generator at that time. generated := language.GeneratedFile{ TemplatePath: "templates/common/client_snippet.swift.mustache", OutputPath: filepath.Join("Snippets", s.Name+"Quickstart.swift"), @@ -180,7 +210,7 @@ func (c *codec) generateClients(outdir string, model *api.API, provider language } generated := language.GeneratedFile{ TemplatePath: "templates/common/clients.swift.mustache", - OutputPath: c.outputPath("Clients.swift"), + OutputPath: c.swiftFilename("Clients"), } return language.GenerateFromModel(outdir, model, provider, []language.GeneratedFile{generated}) } diff --git a/internal/sidekick/swift/generate_enum_swift_test.go b/internal/sidekick/swift/generate_enum_swift_test.go index c6172c5ea45..f5b17de8eeb 100644 --- a/internal/sidekick/swift/generate_enum_swift_test.go +++ b/internal/sidekick/swift/generate_enum_swift_test.go @@ -34,7 +34,14 @@ func TestGenerateEnum_Files(t *testing.T) { kind.Values = []*api.EnumValue{{Name: "KIND_UNSPECIFIED", Number: 0, Parent: kind}} kind.UniqueNumberValues = kind.Values - model := api.NewTestAPI([]*api.Message{}, []*api.Enum{color, kind}, []*api.Service{}) + clash0 := &api.Enum{Name: "ClashName", Package: "google.cloud.test.v1", ID: ".google.cloud.test.v1.ClashName"} + clash0.Values = []*api.EnumValue{{Name: "CLASH_UNSPECIFIED", Number: 0, Parent: clash0}} + clash0.UniqueNumberValues = clash0.Values + clash1 := &api.Enum{Name: "clashName", Package: "google.cloud.test.v1", ID: ".google.cloud.test.v1.clashName"} + clash1.Values = []*api.EnumValue{{Name: "CLASH_UNSPECIFIED", Number: 0, Parent: clash1}} + clash1.UniqueNumberValues = clash1.Values + + model := api.NewTestAPI([]*api.Message{}, []*api.Enum{color, kind, clash0, clash1}, []*api.Service{}) model.PackageName = "google.cloud.test.v1" cfg := &parser.ModelConfig{ @@ -48,7 +55,13 @@ func TestGenerateEnum_Files(t *testing.T) { } expectedDir := filepath.Join(outDir, "Sources", "GoogleCloudTestV1") - for _, expected := range []string{"Color.swift", "Kind.swift"} { + want := []string{ + "Color.swift", + "Kind.swift", + "ClashName.swift", + "clashName+000.swift", + } + for _, expected := range want { filename := filepath.Join(expectedDir, expected) if _, err := os.Stat(filename); err != nil { t.Error(err) diff --git a/internal/sidekick/swift/generate_message_swift_test.go b/internal/sidekick/swift/generate_message_swift_test.go index 9c144c856de..3428757dbeb 100644 --- a/internal/sidekick/swift/generate_message_swift_test.go +++ b/internal/sidekick/swift/generate_message_swift_test.go @@ -44,8 +44,11 @@ func TestGenerateMessage_Files(t *testing.T) { secret := &api.Message{Name: "Secret", Package: "google.cloud.test.v1", ID: ".google.cloud.test.v1.Secret"} volume := &api.Message{Name: "Volume", Package: "google.cloud.test.v1", ID: ".google.cloud.test.v1.Volume"} + clash0 := &api.Message{Name: "HttpHealthCheck", Package: "google.cloud.test.v1", ID: ".google.cloud.test.v1.HttpHealthCheck"} + clash1 := &api.Message{Name: "HTTPHealthCheck", Package: "google.cloud.test.v1", ID: ".google.cloud.test.v1.HTTPHealthCheck"} + clash2 := &api.Message{Name: "httpHealthCheck", Package: "google.cloud.test.v1", ID: ".google.cloud.test.v1.httpHealthCheck"} - model := api.NewTestAPI([]*api.Message{secret, volume}, []*api.Enum{}, []*api.Service{}) + model := api.NewTestAPI([]*api.Message{secret, volume, clash0, clash1, clash2}, []*api.Enum{}, []*api.Service{}) model.PackageName = "google.cloud.test.v1" cfg := &parser.ModelConfig{ @@ -59,7 +62,14 @@ func TestGenerateMessage_Files(t *testing.T) { } expectedDir := filepath.Join(outDir, "Sources", "GoogleCloudTestV1") - for _, expected := range []string{"Secret.swift", "Volume.swift"} { + want := []string{ + "Secret.swift", + "Volume.swift", + "HttpHealthCheck.swift", + "HTTPHealthCheck+000.swift", + "httpHealthCheck+001.swift", + } + for _, expected := range want { filename := filepath.Join(expectedDir, expected) if _, err := os.Stat(filename); err != nil { t.Error(err) diff --git a/internal/sidekick/swift/generate_service_swift_test.go b/internal/sidekick/swift/generate_service_swift_test.go index 9ba451012d8..582209c564b 100644 --- a/internal/sidekick/swift/generate_service_swift_test.go +++ b/internal/sidekick/swift/generate_service_swift_test.go @@ -31,11 +31,15 @@ import ( func TestGenerateService_Files(t *testing.T) { outDir := t.TempDir() - iam := &api.Service{Name: "IAM"} - secretManager := &api.Service{Name: "SecretManagerService"} + // We need explicit Package and ID fields because we generate both messages + // and services. + iam := &api.Service{Name: "IAM", Package: "test", ID: ".test.IAM"} + secretManager := &api.Service{Name: "SecretManagerService", Package: "test", ID: ".test.SecretManagerService"} + clash0 := &api.Message{Name: "InstanceSettings", Package: "test", ID: ".test.InstanceSettings"} + clash1 := &api.Service{Name: "instanceSettings", Package: "test", ID: ".test.instanceSettings"} - model := api.NewTestAPI(nil, nil, []*api.Service{iam, secretManager}) - model.PackageName = "google.cloud.test.v1" + model := api.NewTestAPI([]*api.Message{clash0}, nil, []*api.Service{iam, secretManager, clash1}) + model.PackageName = "test" cfg := &parser.ModelConfig{ Codec: map[string]string{ @@ -47,14 +51,19 @@ func TestGenerateService_Files(t *testing.T) { t.Fatal(err) } - expectedDir := filepath.Join(outDir, "Sources", "GoogleCloudTestV1") + expectedDir := filepath.Join(outDir, "Sources", "GoogleTest") wantFiles := []string{ "IAM.swift", - "SecretManagerService.swift", "Clients.swift", - "Clients/SecretManagerServiceStub.swift", - "Clients/SecretManagerServiceLogging.swift", - "Clients/SecretManagerServiceRetry.swift", + "SecretManagerService.swift", + "SecretManagerService+Stub.swift", + "SecretManagerService+Logging.swift", + "SecretManagerService+Retry.swift", + "InstanceSettings.swift", + "instanceSettings+000.swift", + "instanceSettings+Stub.swift", + "instanceSettings+Logging.swift", + "instanceSettings+Retry.swift", } for _, expected := range wantFiles { filename := filepath.Join(expectedDir, expected) @@ -435,7 +444,7 @@ func TestGenerateService_PathParameters(t *testing.T) { t.Fatal(err) } - filename := filepath.Join(outDir, "Sources", "GoogleCloudSecretmanagerV1", "Clients", "SecretManagerServiceStub.swift") + filename := filepath.Join(outDir, "Sources", "GoogleCloudSecretmanagerV1", "SecretManagerService+Stub.swift") content, err := os.ReadFile(filename) if err != nil { t.Fatal(err) diff --git a/internal/sidekick/swift/generate_stub_test.go b/internal/sidekick/swift/generate_stub_test.go index f59a19ef66e..80f0285e9c6 100644 --- a/internal/sidekick/swift/generate_stub_test.go +++ b/internal/sidekick/swift/generate_stub_test.go @@ -76,7 +76,7 @@ func TestGenerateService_StubStructure(t *testing.T) { t.Fatal(err) } - filename := filepath.Join(outDir, "Sources", "GoogleCloudTestV1", "Clients", "ProtocolStub.swift") + filename := filepath.Join(outDir, "Sources", "GoogleCloudTestV1", "Protocol+Stub.swift") content, err := os.ReadFile(filename) if err != nil { t.Fatal(err) @@ -190,7 +190,7 @@ func TestGenerateService_QueryParameters(t *testing.T) { t.Fatal(err) } - filename := filepath.Join(outDir, "Sources", "GoogleTest", "Clients", "ServiceStub.swift") + filename := filepath.Join(outDir, "Sources", "GoogleTest", "Service+Stub.swift") content, err := os.ReadFile(filename) if err != nil { t.Fatal(err) From 0986d6a2cd5d3e4736b9e336a47dd85c9240e14b Mon Sep 17 00:00:00 2001 From: yangyzs <171981480+yangyzs@users.noreply.github.com> Date: Fri, 26 Jun 2026 11:25:32 -0400 Subject: [PATCH 024/108] feat(internal/librarian/java): add title override extractor (#6546) Adds 'extractTitle' to extract 'sample-metadata:' titles. It returns an error if the title is missing or empty. For #6515 --- internal/librarian/java/readme.go | 29 +++++++++ internal/librarian/java/readme_test.go | 83 ++++++++++++++++++++++++++ 2 files changed, 112 insertions(+) diff --git a/internal/librarian/java/readme.go b/internal/librarian/java/readme.go index 5ddd7d9f8ce..f9009affcec 100644 --- a/internal/librarian/java/readme.go +++ b/internal/librarian/java/readme.go @@ -15,6 +15,7 @@ package java import ( + "errors" "regexp" "strings" ) @@ -22,9 +23,37 @@ import ( var ( // Matches lowercase/digit followed by uppercase (e.g., "FooBar" -> "Foo Bar"). camelCaseRegexp = regexp.MustCompile(`([a-z0-9])([A-Z])`) + + // reTitle matches a "sample-metadata:" marker followed immediately by a "title:" line on the next comment line. + reTitle = regexp.MustCompile(`sample-metadata:\s*\n\s*(?://|#)\s*title:\s*(.*)`) + + // errMissingTitle indicates the "title:" line is missing immediately following "sample-metadata:". + errMissingTitle = errors.New("missing title line immediately following sample-metadata") + + // errEmptyTitle indicates the extracted title value is empty. + errEmptyTitle = errors.New("title value cannot be empty") ) // decamelize converts CamelCase string to space-separated string (e.g. "CamelCase" -> "Camel Case"). func decamelize(value string) string { return strings.TrimSpace(camelCaseRegexp.ReplaceAllString(value, `$1 $2`)) } + +// extractTitle extracts and validates the title override from Java comment blocks. +// It expects a "title:" line to immediately follow the "sample-metadata:" marker. +// Returns an error if the marker is present but the title line is missing, malformed, or empty. +func extractTitle(content string) (string, error) { + if !strings.Contains(content, "sample-metadata:") { + return "", nil + } + matches := reTitle.FindStringSubmatch(content) + if len(matches) < 2 { + return "", errMissingTitle + } + // Trim surrounding whitespace, quotes, and carriage returns. + title := strings.Trim(matches[1], " \t\"'\r\n") + if title == "" { + return "", errEmptyTitle + } + return title, nil +} diff --git a/internal/librarian/java/readme_test.go b/internal/librarian/java/readme_test.go index fc022d3c45e..0a3b117e49e 100644 --- a/internal/librarian/java/readme_test.go +++ b/internal/librarian/java/readme_test.go @@ -15,6 +15,7 @@ package java import ( + "errors" "testing" "github.com/google/go-cmp/cmp" @@ -60,3 +61,85 @@ func TestDecamelize(t *testing.T) { }) } } + +func TestExtractTitle(t *testing.T) { + for _, test := range []struct { + name string + content string + want string + }{ + { + name: "success with standard comment", + content: `// sample-metadata: +// title: Standard Title`, + want: "Standard Title", + }, + { + name: "success with indented comment", + content: `// sample-metadata: +// title: Indented Title`, + want: "Indented Title", + }, + { + name: "success with single quotes", + content: `// sample-metadata: +// title: 'Single Quotes Title'`, + want: "Single Quotes Title", + }, + { + name: "success with double quotes", + content: `// sample-metadata: +// title: "Double Quotes Title"`, + want: "Double Quotes Title", + }, + { + name: "success with windows carriage returns", + content: "// sample-metadata:\r\n// title: Windows Title\r\n", + want: "Windows Title", + }, + { + name: "no metadata block present", + content: `// This is a standard java file. +public class Normal {}`, + want: "", + }, + } { + t.Run(test.name, func(t *testing.T) { + got, err := extractTitle(test.content) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestExtractTitle_Error(t *testing.T) { + for _, test := range []struct { + name string + content string + wantErr error + }{ + { + name: "missing title line returns error", + content: `// sample-metadata: +// description: No title line immediately following!`, + wantErr: errMissingTitle, + }, + { + name: "empty title value returns error", + content: `// sample-metadata: +// title: ""`, + wantErr: errEmptyTitle, + }, + } { + t.Run(test.name, func(t *testing.T) { + _, gotErr := extractTitle(test.content) + if !errors.Is(gotErr, test.wantErr) { + t.Errorf("extractTitle() error = %v, wantErr %v", gotErr, test.wantErr) + } + }) + } +} From a0930f4b0f52383ee4b19f23ae666ae4017f40f6 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Fri, 26 Jun 2026 11:55:40 -0400 Subject: [PATCH 025/108] feat(internal/librarian/python): update gapic-generator to 1.36.0 (#6548) The version of gapic-generator in the embedded librarian.yaml (used by librarian install) is updated to v1.36.0. This adds Python 3.15 pre-release testing to generated libraries to prepare for the [Python 3.15 release in October](https://peps.python.org/pep-0790/#schedule). Signed-off-by: Anthonios Partheniou --- internal/librarian/python/librarian.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/librarian/python/librarian.yaml b/internal/librarian/python/librarian.yaml index d261cbb10b7..3d06913c223 100644 --- a/internal/librarian/python/librarian.yaml +++ b/internal/librarian/python/librarian.yaml @@ -19,7 +19,7 @@ language: python tools: pip: - name: gapic-generator - version: "1.35.0" + version: "1.36.0" - name: nox version: "2025.11.12" - name: ruff From eba5cb5c5b0b8135f117594e2d6c79212cc80b15 Mon Sep 17 00:00:00 2001 From: Tomo Suzuki Date: Fri, 26 Jun 2026 13:42:30 -0400 Subject: [PATCH 026/108] ci: auto-fix workflow YAML files (#6544) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit go/github-zizmor-help The 1st commit was an automated fix (`--fix`). The 2nd commit was a manual fix. ``` suztomo@suztomo:~/librarian-2026/librarian$ zizmor .github/workflows INFO zizmor: 🌈 zizmor v1.25.2 INFO audit: zizmor: 🌈 completed .github/workflows/ci.yaml INFO audit: zizmor: 🌈 completed .github/workflows/create-issue-on-failure.yaml INFO audit: zizmor: 🌈 completed .github/workflows/dart.yaml INFO audit: zizmor: 🌈 completed .github/workflows/go.yaml INFO audit: zizmor: 🌈 completed .github/workflows/java.yaml INFO audit: zizmor: 🌈 completed .github/workflows/linter.yaml INFO audit: zizmor: 🌈 completed .github/workflows/multi_approvers.yaml INFO audit: zizmor: 🌈 completed .github/workflows/nodejs.yaml INFO audit: zizmor: 🌈 completed .github/workflows/python.yaml INFO audit: zizmor: 🌈 completed .github/workflows/sidekick.yaml No findings to report. Good job! (1 ignored, 43 suppressed) ``` --- .github/workflows/ci.yaml | 15 +++++-- .../workflows/create-issue-on-failure.yaml | 4 +- .github/workflows/dart.yaml | 4 +- .github/workflows/go.yaml | 14 +++++-- .github/workflows/java.yaml | 27 +++++++----- .github/workflows/linter.yaml | 42 ++++++++++++------- .github/workflows/multi_approvers.yaml | 1 + .github/workflows/nodejs.yaml | 8 ++-- .github/workflows/python.yaml | 6 ++- .github/workflows/sidekick.yaml | 18 +++++--- 10 files changed, 94 insertions(+), 45 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 14ad9552415..552f4a08fbb 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -15,14 +15,15 @@ name: Librarian on: [push, pull_request, merge_group] permissions: contents: read - issues: write jobs: legacylibrarian: runs-on: ubuntu-24.04 # Presubmit jobs must complete within 5 minutes. See CONTRIBUTING.md. timeout-minutes: 5 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false - uses: ./.github/actions/setup-librarian - name: Run tests and check coverage run: go run ./tool/cmd/coverage ./internal/legacylibrarian/... @@ -31,7 +32,9 @@ jobs: # Presubmit jobs must complete within 5 minutes. See CONTRIBUTING.md. timeout-minutes: 5 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false - uses: ./.github/actions/setup-librarian - name: Setup librarian.yaml for test tool dependency installation run: | @@ -45,10 +48,14 @@ jobs: $(go list ./... | grep -v -E 'github.com/googleapis/librarian$|internal/librarian/(dart|golang|java|nodejs|python|rust|swift)|internal/sidekick|internal/legacylibrarian|internal/sample|internal/testhelper') create-issue-on-failure: runs-on: ubuntu-24.04 + permissions: + issues: write needs: [legacylibrarian, test] if: ${{ always() && contains(needs.*.result, 'failure') && github.event_name == 'push' && github.ref == 'refs/heads/main' }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false - uses: ./.github/actions/create-issue-on-failure with: title: "all: tests failed at HEAD" diff --git a/.github/workflows/create-issue-on-failure.yaml b/.github/workflows/create-issue-on-failure.yaml index e01d75c198b..d8e2cba7200 100644 --- a/.github/workflows/create-issue-on-failure.yaml +++ b/.github/workflows/create-issue-on-failure.yaml @@ -30,7 +30,9 @@ jobs: create-issue-on-failure: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false - uses: ./.github/actions/create-issue-on-failure with: title: "${{ inputs.repository }}: librarian generate fails when run at HEAD" diff --git a/.github/workflows/dart.yaml b/.github/workflows/dart.yaml index 82cbf4b3511..003bbf95c41 100644 --- a/.github/workflows/dart.yaml +++ b/.github/workflows/dart.yaml @@ -21,7 +21,9 @@ jobs: # Presubmit jobs must complete within 5 minutes. See CONTRIBUTING.md. timeout-minutes: 5 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false - uses: ./.github/actions/setup-librarian - name: Display Go version run: go version diff --git a/.github/workflows/go.yaml b/.github/workflows/go.yaml index 4e8b710661e..fce4306f8fe 100644 --- a/.github/workflows/go.yaml +++ b/.github/workflows/go.yaml @@ -15,14 +15,15 @@ name: Go on: [push, pull_request, merge_group] permissions: contents: read - issues: write jobs: test: runs-on: ubuntu-24.04 # Presubmit jobs must complete within 5 minutes. See CONTRIBUTING.md. timeout-minutes: 5 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false - uses: ./.github/actions/setup-librarian - name: Run tests run: go run ./tool/cmd/coverage ./internal/librarian/golang @@ -30,13 +31,16 @@ jobs: runs-on: ubuntu-24.04 if: github.event_name == 'push' && github.ref == 'refs/heads/main' steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false - uses: ./.github/actions/setup-librarian - name: Checkout google-cloud-go - uses: actions/checkout@v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: repository: googleapis/google-cloud-go path: google-cloud-go + persist-credentials: false - name: Run librarian install working-directory: google-cloud-go run: librarian install @@ -45,6 +49,8 @@ jobs: run: librarian generate --all create-issue-on-failure: needs: [integration] + permissions: + issues: write if: ${{ always() && contains(needs.*.result, 'failure') && github.event_name == 'push' && github.ref == 'refs/heads/main' }} uses: ./.github/workflows/create-issue-on-failure.yaml with: diff --git a/.github/workflows/java.yaml b/.github/workflows/java.yaml index 04a21c5d99f..c48828f755b 100644 --- a/.github/workflows/java.yaml +++ b/.github/workflows/java.yaml @@ -15,33 +15,35 @@ name: Java on: [push, pull_request, merge_group] permissions: contents: read - issues: write jobs: test: runs-on: ubuntu-24.04 # Presubmit jobs must complete within 5 minutes. See CONTRIBUTING.md. timeout-minutes: 5 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false - uses: ./.github/actions/setup-librarian - name: Checkout google-cloud-java - uses: actions/checkout@v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: repository: googleapis/google-cloud-java path: google-cloud-java + persist-credentials: false - name: Display Go version run: go version - - uses: actions/setup-java@v4 + - uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 with: java-version: "17" distribution: "temurin" cache: "maven" - - uses: actions/setup-python@v5 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.12" - name: Cache Librarian Tools & Venv id: cache-tools - uses: actions/cache@v5 + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 with: path: | ~/.cache/librarian @@ -80,14 +82,17 @@ jobs: runs-on: ubuntu-24.04 if: github.event_name == 'push' && (github.ref == 'refs/heads/main') steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false - uses: ./.github/actions/setup-librarian - name: Checkout google-cloud-java - uses: actions/checkout@v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: repository: googleapis/google-cloud-java path: google-cloud-java - - uses: actions/setup-java@v4 + persist-credentials: false + - uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 with: java-version: "17" distribution: "temurin" @@ -100,7 +105,7 @@ jobs: sudo apt-get update && sudo apt-get install -y maven fi mvn -version - - uses: actions/setup-python@v5 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.12" - name: Run librarian install @@ -115,6 +120,8 @@ jobs: run: librarian generate --all create-issue-on-failure: needs: [integration] + permissions: + issues: write if: ${{ always() && contains(needs.*.result, 'failure') && github.event_name == 'push' && github.ref == 'refs/heads/main' }} uses: ./.github/workflows/create-issue-on-failure.yaml with: diff --git a/.github/workflows/linter.yaml b/.github/workflows/linter.yaml index 9e1121d1ce2..8e861e5cd42 100644 --- a/.github/workflows/linter.yaml +++ b/.github/workflows/linter.yaml @@ -21,8 +21,10 @@ jobs: # Presubmit jobs must complete within 5 minutes. See CONTRIBUTING.md. timeout-minutes: 5 steps: - - uses: actions/checkout@v6 - - uses: actions/setup-go@v6 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false + - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 with: go-version-file: "go.mod" - name: Run TestAddLicense @@ -32,15 +34,19 @@ jobs: # Presubmit jobs must complete within 5 minutes. See CONTRIBUTING.md. timeout-minutes: 5 steps: - - uses: actions/checkout@v6 - - uses: crate-ci/typos@v1.45.0 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false + - uses: crate-ci/typos@02ea592e44b3a53c302f697cddca7641cd051c3d # v1.45.0 yamlfmt: runs-on: ubuntu-24.04 # Presubmit jobs must complete within 5 minutes. See CONTRIBUTING.md. timeout-minutes: 5 steps: - - uses: actions/checkout@v6 - - uses: actions/setup-go@v6 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false + - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 with: go-version-file: "go.mod" - name: Run TestYAMLFormat @@ -50,8 +56,10 @@ jobs: # Presubmit jobs must complete within 5 minutes. See CONTRIBUTING.md. timeout-minutes: 5 steps: - - uses: actions/checkout@v6 - - uses: actions/setup-go@v6 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false + - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 with: go-version-file: "go.mod" - name: Run TestGoImports @@ -61,8 +69,10 @@ jobs: # Presubmit jobs must complete within 5 minutes. See CONTRIBUTING.md. timeout-minutes: 5 steps: - - uses: actions/checkout@v6 - - uses: actions/setup-go@v6 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false + - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 with: go-version-file: "go.mod" - name: Run TestGolangCILint @@ -72,8 +82,10 @@ jobs: # Presubmit jobs must complete within 5 minutes. See CONTRIBUTING.md. timeout-minutes: 5 steps: - - uses: actions/checkout@v6 - - uses: actions/setup-go@v6 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false + - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 with: go-version-file: "go.mod" - name: Run TestGoModTidy @@ -83,8 +95,10 @@ jobs: # Presubmit jobs must complete within 5 minutes. See CONTRIBUTING.md. timeout-minutes: 5 steps: - - uses: actions/checkout@v6 - - uses: actions/setup-go@v6 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false + - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 with: go-version-file: "go.mod" - name: Run TestGoGenerate diff --git a/.github/workflows/multi_approvers.yaml b/.github/workflows/multi_approvers.yaml index 5f79857503d..fdfa39e4624 100644 --- a/.github/workflows/multi_approvers.yaml +++ b/.github/workflows/multi_approvers.yaml @@ -27,6 +27,7 @@ on: # branch and does not check out or run untrusted code from the PR branch. # For more details, see the action's documentation: # https://github.com/abcxyz/actions/blob/main/.github/actions/multi-approvers/README.md + # zizmor: ignore[dangerous-triggers] - {Multi-approver check needs this} pull_request_target: types: - 'opened' diff --git a/.github/workflows/nodejs.yaml b/.github/workflows/nodejs.yaml index 87821da25ff..f05ab83a32a 100644 --- a/.github/workflows/nodejs.yaml +++ b/.github/workflows/nodejs.yaml @@ -21,9 +21,11 @@ jobs: # Presubmit jobs must complete within 5 minutes. See CONTRIBUTING.md. timeout-minutes: 5 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false - uses: ./.github/actions/setup-librarian - - uses: actions/setup-node@v6 + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 with: node-version: "18" - name: "Set up pnpm via corepack" @@ -35,7 +37,7 @@ jobs: run: | echo "npm=$(npm config get prefix)" >> "$GITHUB_OUTPUT" - name: "Install Node.js dependencies (restore cache)" - uses: actions/cache@v5 + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 id: tools-cache with: path: | diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml index 9df36375d87..dab3a451bf7 100644 --- a/.github/workflows/python.yaml +++ b/.github/workflows/python.yaml @@ -21,9 +21,11 @@ jobs: # Presubmit jobs must complete within 5 minutes. See CONTRIBUTING.md. timeout-minutes: 5 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false - uses: ./.github/actions/setup-librarian - - uses: actions/setup-python@v6 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: "3.14" - name: Install pandoc diff --git a/.github/workflows/sidekick.yaml b/.github/workflows/sidekick.yaml index 33c1ee212e0..908ec433eaf 100644 --- a/.github/workflows/sidekick.yaml +++ b/.github/workflows/sidekick.yaml @@ -15,19 +15,20 @@ name: Sidekick (Rust, Swift) on: [push, pull_request, merge_group] permissions: contents: read - issues: write jobs: test: runs-on: ubuntu-24.04 # Presubmit jobs must complete within 5 minutes. See CONTRIBUTING.md. timeout-minutes: 5 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false - uses: ./.github/actions/setup-librarian - name: Display Go version run: go version - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # v1 - name: Display Cargo version run: cargo version - name: Display rustc version @@ -43,16 +44,19 @@ jobs: runs-on: ubuntu-24.04 if: github.event_name == 'push' && github.ref == 'refs/heads/main' steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false - uses: ./.github/actions/setup-librarian - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # v1 - uses: ./.github/actions/install-taplo - name: Checkout google-cloud-rust - uses: actions/checkout@v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: repository: googleapis/google-cloud-rust path: google-cloud-rust + persist-credentials: false - name: Run librarian generate working-directory: google-cloud-rust run: librarian generate --all @@ -61,6 +65,8 @@ jobs: run: cargo check -p google-cloud-showcase-v1beta1 create-issue-on-failure: needs: [test, integration] + permissions: + issues: write if: ${{ always() && contains(needs.*.result, 'failure') && github.event_name == 'push' && github.ref == 'refs/heads/main' }} uses: ./.github/workflows/create-issue-on-failure.yaml with: From a698dddfcfe1f16e97739ff7bd3529d13760b276 Mon Sep 17 00:00:00 2001 From: Joe Wang <106995533+JoeWang1127@users.noreply.github.com> Date: Sat, 27 Jun 2026 02:53:21 +0000 Subject: [PATCH 027/108] chore(internal/librarianops): update generate commit type to `feat` (#6552) Update generate commit type to `feat`. Fixes https://github.com/googleapis/librarian/issues/6529 --- internal/librarianops/generate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/librarianops/generate.go b/internal/librarianops/generate.go index ba5c3984a0e..20fecdcd11c 100644 --- a/internal/librarianops/generate.go +++ b/internal/librarianops/generate.go @@ -196,7 +196,7 @@ func createPR(ctx context.Context, repoName string) error { if repoName == repoRust { sources = "googleapis and discovery-artifact-manager" } - title := fmt.Sprintf("chore: update %s and regenerate", sources) + title := fmt.Sprintf("feat: update %s and regenerate", sources) body := fmt.Sprintf("Update %s to the latest commit and regenerate all client libraries.", sources) return command.Run(ctx, "gh", "pr", "create", "--title", title, "--body", body) } From 91af191bc6aaec04c2fd0873b58ffbee4f09f48b Mon Sep 17 00:00:00 2001 From: Joe Wang <106995533+JoeWang1127@users.noreply.github.com> Date: Sat, 27 Jun 2026 02:53:30 +0000 Subject: [PATCH 028/108] chore(internal/serviceconfig): rename api-allowlist-schema to sdk-yaml-schema (#6551) Rename `doc/api-allowlist-schema.md` to `doc/sdk-yaml-schema.md`. Fixes https://github.com/googleapis/librarian/issues/6540 --- doc/{api-allowlist-schema.md => sdk-yaml-schema.md} | 4 ++-- internal/serviceconfig/api.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) rename doc/{api-allowlist-schema.md => sdk-yaml-schema.md} (97%) diff --git a/doc/api-allowlist-schema.md b/doc/sdk-yaml-schema.md similarity index 97% rename from doc/api-allowlist-schema.md rename to doc/sdk-yaml-schema.md index 13530ff70b3..389b5073020 100644 --- a/doc/api-allowlist-schema.md +++ b/doc/sdk-yaml-schema.md @@ -1,6 +1,6 @@ -# API Allowlist Schema +# SDK YAML Schema -This document describes the schema for the API Allowlist. +This document describes the schema for the SDK YAML. ## API Configuration diff --git a/internal/serviceconfig/api.go b/internal/serviceconfig/api.go index 4d3ba0223d6..16878f488a7 100644 --- a/internal/serviceconfig/api.go +++ b/internal/serviceconfig/api.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:generate go run -tags configdocgen ../../cmd/config_doc_generate.go -input . -output ../../doc/api-allowlist-schema.md -root API -root-title API -title "API Allowlist" +//go:generate go run -tags configdocgen ../../cmd/config_doc_generate.go -input . -output ../../doc/sdk-yaml-schema.md -root API -root-title API -title "SDK YAML" package serviceconfig From 2fb20a667398c99154d9510ad865585b955f7ecd Mon Sep 17 00:00:00 2001 From: Tomo Suzuki Date: Sat, 27 Jun 2026 09:42:12 -0400 Subject: [PATCH 029/108] ci: add contents: read permission to create-issue-on-failure jobs (#6556) Fixes https://github.com/googleapis/librarian/issues/6555 --- .github/workflows/ci.yaml | 1 + .github/workflows/go.yaml | 1 + .github/workflows/java.yaml | 1 + .github/workflows/sidekick.yaml | 5 +++-- 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 552f4a08fbb..1d145649899 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -49,6 +49,7 @@ jobs: create-issue-on-failure: runs-on: ubuntu-24.04 permissions: + contents: read issues: write needs: [legacylibrarian, test] if: ${{ always() && contains(needs.*.result, 'failure') && github.event_name == 'push' && github.ref == 'refs/heads/main' }} diff --git a/.github/workflows/go.yaml b/.github/workflows/go.yaml index fce4306f8fe..47478637ad7 100644 --- a/.github/workflows/go.yaml +++ b/.github/workflows/go.yaml @@ -50,6 +50,7 @@ jobs: create-issue-on-failure: needs: [integration] permissions: + contents: read issues: write if: ${{ always() && contains(needs.*.result, 'failure') && github.event_name == 'push' && github.ref == 'refs/heads/main' }} uses: ./.github/workflows/create-issue-on-failure.yaml diff --git a/.github/workflows/java.yaml b/.github/workflows/java.yaml index c48828f755b..1c506506978 100644 --- a/.github/workflows/java.yaml +++ b/.github/workflows/java.yaml @@ -121,6 +121,7 @@ jobs: create-issue-on-failure: needs: [integration] permissions: + contents: read issues: write if: ${{ always() && contains(needs.*.result, 'failure') && github.event_name == 'push' && github.ref == 'refs/heads/main' }} uses: ./.github/workflows/create-issue-on-failure.yaml diff --git a/.github/workflows/sidekick.yaml b/.github/workflows/sidekick.yaml index 908ec433eaf..67e0618b922 100644 --- a/.github/workflows/sidekick.yaml +++ b/.github/workflows/sidekick.yaml @@ -28,7 +28,7 @@ jobs: - name: Display Go version run: go version - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # v1 + uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable as of 2026-06-26 - name: Display Cargo version run: cargo version - name: Display rustc version @@ -49,7 +49,7 @@ jobs: persist-credentials: false - uses: ./.github/actions/setup-librarian - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # v1 + uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable as of 2026-06-26 - uses: ./.github/actions/install-taplo - name: Checkout google-cloud-rust uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 @@ -66,6 +66,7 @@ jobs: create-issue-on-failure: needs: [test, integration] permissions: + contents: read issues: write if: ${{ always() && contains(needs.*.result, 'failure') && github.event_name == 'push' && github.ref == 'refs/heads/main' }} uses: ./.github/workflows/create-issue-on-failure.yaml From df213ecb97834adb13c62b902fe090955845ba19 Mon Sep 17 00:00:00 2001 From: yangyzs <171981480+yangyzs@users.noreply.github.com> Date: Sat, 27 Jun 2026 14:47:44 -0400 Subject: [PATCH 030/108] feat(internal/librarian/java): add production sample filter (#6545) Adds 'isProductionSample' helper to identify production Java source files under '/src/main/java/' with unit tests. For #6515 --- internal/librarian/java/readme.go | 9 +++++++ internal/librarian/java/readme_test.go | 36 ++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/internal/librarian/java/readme.go b/internal/librarian/java/readme.go index f9009affcec..9a7db30a34b 100644 --- a/internal/librarian/java/readme.go +++ b/internal/librarian/java/readme.go @@ -16,6 +16,7 @@ package java import ( "errors" + "path/filepath" "regexp" "strings" ) @@ -39,6 +40,14 @@ func decamelize(value string) string { return strings.TrimSpace(camelCaseRegexp.ReplaceAllString(value, `$1 $2`)) } +// isProductionSample reports whether the given path represents a production Java source file +// located under a standard "src/main/java" path. +func isProductionSample(path string) bool { + slashed := filepath.ToSlash(path) + return strings.HasSuffix(slashed, ".java") && + (strings.HasPrefix(slashed, "src/main/java/") || strings.Contains(slashed, "/src/main/java/")) +} + // extractTitle extracts and validates the title override from Java comment blocks. // It expects a "title:" line to immediately follow the "sample-metadata:" marker. // Returns an error if the marker is present but the title line is missing, malformed, or empty. diff --git a/internal/librarian/java/readme_test.go b/internal/librarian/java/readme_test.go index 0a3b117e49e..595c87cb645 100644 --- a/internal/librarian/java/readme_test.go +++ b/internal/librarian/java/readme_test.go @@ -62,6 +62,42 @@ func TestDecamelize(t *testing.T) { } } +func TestIsProductionSample(t *testing.T) { + for _, test := range []struct { + name string + path string + want bool + }{ + { + name: "valid production sample", + path: "samples/src/main/java/com/example/Sample.java", + want: true, + }, + { + name: "valid production sample at root", + path: "src/main/java/com/example/Sample.java", + want: true, + }, + { + name: "non-java file", + path: "samples/src/main/java/README.md", + want: false, + }, + { + name: "not in src/main/java", + path: "samples/src/test/java/com/example/Sample.java", + want: false, + }, + } { + t.Run(test.name, func(t *testing.T) { + got := isProductionSample(test.path) + if got != test.want { + t.Errorf("isProductionSample() = %t, want %t", got, test.want) + } + }) + } +} + func TestExtractTitle(t *testing.T) { for _, test := range []struct { name string From 867f637a8f4ebc274d334e3a298ff580e90a5c42 Mon Sep 17 00:00:00 2001 From: Sophia Yang Date: Sat, 6 Jun 2026 15:37:16 -0400 Subject: [PATCH 031/108] fix(internal/librarian/java): refine whitespace delimiters and exclude test classes during sample extraction for README.md --- cmd/librarian/doc.go | 3 +- go.mod | 2 +- internal/librarian/generate.go | 15 ++- internal/librarian/install_test.go | 4 +- internal/librarian/java/clean.go | 21 +++- internal/librarian/java/generate.go | 13 +-- internal/librarian/java/generate_test.go | 30 ++++-- internal/librarian/java/postgenerate_test.go | 5 +- internal/librarian/java/postprocess.go | 7 ++ internal/librarian/java/postprocess_test.go | 108 ++++++++++++++++++- internal/librarian/java/repometadata_test.go | 5 +- internal/sidekick/parser/protobuf.go | 1 - 12 files changed, 181 insertions(+), 33 deletions(-) diff --git a/cmd/librarian/doc.go b/cmd/librarian/doc.go index face1f6cdef..3b3293559c5 100644 --- a/cmd/librarian/doc.go +++ b/cmd/librarian/doc.go @@ -106,7 +106,8 @@ Examples: Flags: - --all generate all libraries + --all generate all libraries + --use-go-postprocessor use the new Go postprocessor for Java libraries A typical librarian workflow for regenerating every library against the latest API definitions is: diff --git a/go.mod b/go.mod index 58f30075608..5720b43c448 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( cloud.google.com/go/iam v1.5.3 cloud.google.com/go/longrunning v0.8.0 github.com/bazelbuild/buildtools v0.0.0-20260202105709-e24971d9d1a7 + github.com/bmatcuk/doublestar/v4 v4.10.0 github.com/cbroglie/mustache v1.4.0 github.com/go-git/go-git/v5 v5.19.1 github.com/google/go-cmp v0.7.0 @@ -71,7 +72,6 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bkielbasa/cyclop v1.2.3 // indirect github.com/blizzy78/varnamelen v0.8.0 // indirect - github.com/bmatcuk/doublestar/v4 v4.10.0 // indirect github.com/bombsimon/wsl/v4 v4.7.0 // indirect github.com/bombsimon/wsl/v5 v5.6.0 // indirect github.com/breml/bidichk v0.3.3 // indirect diff --git a/internal/librarian/generate.go b/internal/librarian/generate.go index 43833550b33..10a5e54f4e0 100644 --- a/internal/librarian/generate.go +++ b/internal/librarian/generate.go @@ -73,9 +73,14 @@ latest API definitions is: Name: "all", Usage: "generate all libraries", }, + &cli.BoolFlag{ + Name: "use-go-postprocessor", + Usage: "use the new Go postprocessor for Java libraries", + }, }, Action: func(ctx context.Context, cmd *cli.Command) error { all := cmd.Bool("all") + useGoPostprocessor := cmd.Bool("use-go-postprocessor") libraryName := cmd.Args().First() if !all && libraryName == "" { return errMissingLibraryOrAllFlag @@ -87,12 +92,12 @@ latest API definitions is: if err != nil { return err } - return runGenerate(ctx, cfg, all, libraryName) + return runGenerate(ctx, cfg, all, libraryName, useGoPostprocessor) }, } } -func runGenerate(ctx context.Context, cfg *config.Config, all bool, libraryName string) error { +func runGenerate(ctx context.Context, cfg *config.Config, all bool, libraryName string, useGoPostprocessor bool) error { sources, err := LoadSources(ctx, cfg.Sources) if err != nil { return err @@ -139,7 +144,7 @@ func runGenerate(ctx context.Context, cfg *config.Config, all bool, libraryName if err := cleanLibraries(cfg.Language, libraries); err != nil { return err } - return generateLibraries(ctx, cfg, libraries, sources) + return generateLibraries(ctx, cfg, libraries, sources, useGoPostprocessor) } // cleanLibraries iterates over all the given libraries sequentially, @@ -181,7 +186,7 @@ func cleanLibraries(language string, libraries []*config.Library) error { // generateLibraries generates and formats all the given libraries, // delegating to language-specific code. Each language chooses its own // concurrency strategy for these two steps. -func generateLibraries(ctx context.Context, cfg *config.Config, libraries []*config.Library, src *sources.Sources) error { +func generateLibraries(ctx context.Context, cfg *config.Config, libraries []*config.Library, src *sources.Sources, useGoPostprocessor bool) error { switch cfg.Language { case config.LanguageDart: g, gctx := errgroup.WithContext(ctx) @@ -241,7 +246,7 @@ func generateLibraries(ctx context.Context, cfg *config.Config, libraries []*con allMissingArtifacts = append(allMissingArtifacts, java.MissingArtifact{ID: id, Library: library}) } - if err := java.Generate(ctx, cfg, library, src); err != nil { + if err := java.Generate(ctx, cfg, library, src, useGoPostprocessor); err != nil { return fmt.Errorf("generate library %q (%s): %w", library.Name, cfg.Language, err) } if err := java.Format(ctx, library); err != nil { diff --git a/internal/librarian/install_test.go b/internal/librarian/install_test.go index a9f22409b3e..a9090367213 100644 --- a/internal/librarian/install_test.go +++ b/internal/librarian/install_test.go @@ -62,7 +62,7 @@ func TestGenerate(t *testing.T) { tmpDir := t.TempDir() t.Chdir(tmpDir) - if err := generateLibraries(t.Context(), cfg, []*config.Library{library}, nil); err != nil { + if err := generateLibraries(t.Context(), cfg, []*config.Library{library}, nil, false); err != nil { t.Fatal(err) } @@ -92,7 +92,7 @@ func TestCleanLibraries(t *testing.T) { tmpDir := t.TempDir() t.Chdir(tmpDir) - if err := generateLibraries(t.Context(), cfg, []*config.Library{library}, nil); err != nil { + if err := generateLibraries(t.Context(), cfg, []*config.Library{library}, nil, false); err != nil { t.Fatal(err) } diff --git a/internal/librarian/java/clean.go b/internal/librarian/java/clean.go index 7d36826493e..96bdccd933c 100644 --- a/internal/librarian/java/clean.go +++ b/internal/librarian/java/clean.go @@ -20,6 +20,7 @@ import ( "io" "io/fs" "os" + "path" "path/filepath" "regexp" "slices" @@ -54,7 +55,7 @@ func Clean(library *config.Library) error { patterns := cleanPatterns(library) keepSet := make(map[string]bool) for _, k := range library.Keep { - keepSet[filepath.ToSlash(k)] = true + keepSet[strings.TrimSuffix(filepath.ToSlash(k), "/")] = true } for pattern, useMarker := range patterns { matches, err := filepath.Glob(filepath.Join(library.Output, pattern)) @@ -168,8 +169,20 @@ func isDirNotEmpty(err error) bool { // shouldPreserve returns true if the given slash-separated path should be preserved // based on the keepSet or standard preservation patterns. -func shouldPreserve(path string, keepSet map[string]bool) bool { - return keepSet[path] || isDefaultPreserved(path) +// It also checks if any ancestor directory is in the keepSet. +func shouldPreserve(p string, keepSet map[string]bool) bool { + if keepSet[p] || isDefaultPreserved(p) { + return true + } + + dir := path.Dir(p) + for dir != "." && dir != "/" && dir != "" { + if keepSet[dir] { + return true + } + dir = path.Dir(dir) + } + return false } func isDefaultPreserved(path string) bool { @@ -204,7 +217,7 @@ func shouldCleanMarkerPath(path string, d os.DirEntry) (bool, error) { // hasMarker checks if the file at path contains the auto-generated marker. // It scans the file line by line using [bufio.Reader] and returns true as soon // as the marker is found, avoiding reading the entire file. -func hasMarker(path string) (bool, error) { +func hasMarker(path string) (has bool, err error) { f, err := os.Open(path) if err != nil { return false, err diff --git a/internal/librarian/java/generate.go b/internal/librarian/java/generate.go index 26972398e5c..6b5e52ee7ac 100644 --- a/internal/librarian/java/generate.go +++ b/internal/librarian/java/generate.go @@ -67,7 +67,7 @@ type generateAPIParams struct { } // Generate generates a Java client library. -func Generate(ctx context.Context, cfg *config.Config, library *config.Library, srcs *sources.Sources) error { +func Generate(ctx context.Context, cfg *config.Config, library *config.Library, srcs *sources.Sources, useGoPostprocessor bool) error { if library.Java.GroupID == fakeGroupID { return errUnrecognizedAPI } @@ -110,11 +110,12 @@ func Generate(ctx context.Context, cfg *config.Config, library *config.Library, } if err := postProcessLibrary(ctx, libraryPostProcessParams{ - cfg: cfg, - library: library, - outDir: outdir, - metadata: metadata, - transports: transports, + cfg: cfg, + library: library, + outDir: outdir, + metadata: metadata, + transports: transports, + UseGoPostprocessor: useGoPostprocessor, }); err != nil { return err } diff --git a/internal/librarian/java/generate_test.go b/internal/librarian/java/generate_test.go index c511a55653f..1a4cd04a097 100644 --- a/internal/librarian/java/generate_test.go +++ b/internal/librarian/java/generate_test.go @@ -302,7 +302,11 @@ func TestGenerateAPI(t *testing.T) { t.Fatal(err) } } - apiCfg, err := serviceconfig.Find(googleapisDir, "google/cloud/secretmanager/v1", config.LanguageJava) + absGoogleapisDir, err := filepath.Abs(googleapisDir) + if err != nil { + t.Fatal(err) + } + apiCfg, err := serviceconfig.Find(absGoogleapisDir, "google/cloud/secretmanager/v1", config.LanguageJava) if err != nil { t.Fatal(err) } @@ -310,7 +314,7 @@ func TestGenerateAPI(t *testing.T) { cfg: cfg, api: library.APIs[0], library: library, - srcCfg: sources.NewSourceConfig(&sources.Sources{Googleapis: googleapisDir}, nil), + srcCfg: sources.NewSourceConfig(&sources.Sources{Googleapis: absGoogleapisDir}, nil), outdir: outdir, metadata: &repoMetadata{ NamePretty: "Secret Manager", @@ -367,7 +371,11 @@ func TestGenerateAPI_ProtoOnly(t *testing.T) { t.Fatal(err) } } - apiCfg, err := serviceconfig.Find(googleapisDir, "google/cloud/gkehub/policycontroller/v1beta", config.LanguageJava) + absGoogleapisDir, err := filepath.Abs(googleapisDir) + if err != nil { + t.Fatal(err) + } + apiCfg, err := serviceconfig.Find(absGoogleapisDir, "google/cloud/gkehub/policycontroller/v1beta", config.LanguageJava) if err != nil { t.Fatal(err) } @@ -375,7 +383,7 @@ func TestGenerateAPI_ProtoOnly(t *testing.T) { cfg: cfg, api: library.APIs[0], library: library, - srcCfg: sources.NewSourceConfig(&sources.Sources{Googleapis: googleapisDir}, nil), + srcCfg: sources.NewSourceConfig(&sources.Sources{Googleapis: absGoogleapisDir}, nil), outdir: outdir, metadata: &repoMetadata{ NamePretty: "GKE Hub API", @@ -511,7 +519,11 @@ func TestGenerateAPI_WithAdditionalProtosToGenerateAndCopy(t *testing.T) { t.Fatal(err) } } - apiCfg, err := serviceconfig.Find(googleapisDir, "google/cloud/secretmanager/v1", config.LanguageJava) + absGoogleapisDir, err := filepath.Abs(googleapisDir) + if err != nil { + t.Fatal(err) + } + apiCfg, err := serviceconfig.Find(absGoogleapisDir, "google/cloud/secretmanager/v1", config.LanguageJava) if err != nil { t.Fatal(err) } @@ -519,7 +531,7 @@ func TestGenerateAPI_WithAdditionalProtosToGenerateAndCopy(t *testing.T) { cfg: cfg, api: library.APIs[0], library: library, - srcCfg: sources.NewSourceConfig(&sources.Sources{Googleapis: googleapisDir}, nil), + srcCfg: sources.NewSourceConfig(&sources.Sources{Googleapis: absGoogleapisDir}, nil), outdir: outdir, metadata: &repoMetadata{ NamePretty: "Secret Manager", @@ -640,7 +652,7 @@ func TestGenerateLibrary_Error(t *testing.T) { }, Libraries: []*config.Library{test.library}, } - err := Generate(t.Context(), cfg, test.library, &sources.Sources{Googleapis: googleapisDir}) + err := Generate(t.Context(), cfg, test.library, &sources.Sources{Googleapis: googleapisDir}, false) if !errors.Is(err, test.wantErr) { t.Errorf("generate() error = %v, wantErr %v", err, test.wantErr) } @@ -693,7 +705,7 @@ func TestGenerate_Logic(t *testing.T) { t.Fatal(err) } - err := Generate(t.Context(), cfg, library, &sources.Sources{Googleapis: googleapisDir}) + err := Generate(t.Context(), cfg, library, &sources.Sources{Googleapis: googleapisDir}, false) if err != nil { t.Fatal(err) } @@ -755,7 +767,7 @@ func TestGenerate_ProtoExclusion(t *testing.T) { {Name: rootLibrary, Version: "1.2.3"}, }, } - err := Generate(t.Context(), cfg, library, &sources.Sources{Googleapis: googleapisDir}) + err := Generate(t.Context(), cfg, library, &sources.Sources{Googleapis: googleapisDir}, false) if err != nil { t.Fatal(err) } diff --git a/internal/librarian/java/postgenerate_test.go b/internal/librarian/java/postgenerate_test.go index e097fe25d5b..8b943804b95 100644 --- a/internal/librarian/java/postgenerate_test.go +++ b/internal/librarian/java/postgenerate_test.go @@ -33,7 +33,10 @@ func TestPostGenerate(t *testing.T) { t.Parallel() tmpDir := t.TempDir() // Copy testdata to tmpDir - testdataDir := filepath.Join("testdata", "postgenerate") + testdataDir, err := filepath.Abs(filepath.Join("testdata", "postgenerate")) + if err != nil { + t.Fatal(err) + } if err := copyDir(testdataDir, tmpDir); err != nil { t.Fatal(err) } diff --git a/internal/librarian/java/postprocess.go b/internal/librarian/java/postprocess.go index 1c396a6d37a..54b3e8636e8 100644 --- a/internal/librarian/java/postprocess.go +++ b/internal/librarian/java/postprocess.go @@ -65,9 +65,16 @@ type libraryPostProcessParams struct { outDir string metadata *repoMetadata transports map[string]serviceconfig.Transport + UseGoPostprocessor bool } func postProcessLibrary(ctx context.Context, params libraryPostProcessParams) error { + if params.UseGoPostprocessor { + yamlPath := filepath.Join(params.outDir, "postprocess.yaml") + if _, err := os.Stat(yamlPath); err == nil { + return postProcessLibraryNew(params) + } + } if err := createOrVerifyOwlbotPy(params.outDir); err != nil { return err } diff --git a/internal/librarian/java/postprocess_test.go b/internal/librarian/java/postprocess_test.go index ca9de2f147c..7f7224730f8 100644 --- a/internal/librarian/java/postprocess_test.go +++ b/internal/librarian/java/postprocess_test.go @@ -196,9 +196,13 @@ func TestRestructureModules(t *testing.T) { if err := os.WriteFile(reflectConfigPath, []byte("{}"), 0644); err != nil { t.Fatal(err) } - protoPath := filepath.Join(googleapisDir, "google", "cloud", "secretmanager", "v1", "service.proto") + absGoogleapisDir, err := filepath.Abs(googleapisDir) + if err != nil { + t.Fatal(err) + } + protoPath := filepath.Join(absGoogleapisDir, "google", "cloud", "secretmanager", "v1", "service.proto") - additionalProtoPath := filepath.Join(googleapisDir, "google", "cloud", "oslogin", "common", "common.proto") + additionalProtoPath := filepath.Join(absGoogleapisDir, "google", "cloud", "oslogin", "common", "common.proto") params := postProcessParams{ outDir: tmpDir, library: &config.Library{ @@ -1137,3 +1141,103 @@ func TestCreateOrVerifyOwlbotPy_Error(t *testing.T) { t.Errorf("error = %v, wantErr %v", err, fs.ErrPermission) } } + +func TestPostProcessLibrary_Branching(t *testing.T) { + + t.Run("UseGoPostprocessor false", func(t *testing.T) { + outDir := t.TempDir() + // Create a dummy owlbot.py to avoid failure in createOrVerifyOwlbotPy + if err := os.WriteFile(filepath.Join(outDir, "owlbot.py"), []byte(""), 0755); err != nil { + t.Fatal(err) + } + p := libraryPostProcessParams{ + outDir: outDir, + cfg: &config.Config{ + Libraries: []*config.Library{ + {Name: "google-cloud-java", Version: "1.2.3"}, + }, + }, + library: &config.Library{ + Name: "test-library", + }, + } + err := postProcessLibrary(t.Context(), p) + if err == nil { + t.Fatal("expected error, got nil") + } + if strings.Contains(err.Error(), "postProcessLibraryNew not implemented") { + t.Errorf("expected legacy flow, but got new flow error: %v", err) + } + }) + + t.Run("UseGoPostprocessor true, no yaml", func(t *testing.T) { + outDir := t.TempDir() + if err := os.WriteFile(filepath.Join(outDir, "owlbot.py"), []byte(""), 0755); err != nil { + t.Fatal(err) + } + p := libraryPostProcessParams{ + outDir: outDir, + UseGoPostprocessor: true, + cfg: &config.Config{ + Libraries: []*config.Library{ + {Name: "google-cloud-java", Version: "1.2.3"}, + }, + }, + library: &config.Library{ + Name: "test-library", + }, + } + err := postProcessLibrary(t.Context(), p) + if err == nil { + t.Fatal("expected error, got nil") + } + if strings.Contains(err.Error(), "postProcessLibraryNew not implemented") { + t.Errorf("expected legacy flow, but got new flow error: %v", err) + } + }) + + t.Run("UseGoPostprocessor true, with yaml", func(t *testing.T) { + outDir := t.TempDir() + t.Chdir(outDir) + + if err := os.WriteFile(filepath.Join(outDir, "postprocess.yaml"), []byte(""), 0644); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(outDir, "owl-bot-staging"), 0755); err != nil { + t.Fatal(err) + } + metadata := `{"repo": {"name_pretty": "My API"}}` + if err := os.WriteFile(filepath.Join(outDir, ".repo-metadata.json"), []byte(metadata), 0644); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(outDir, "template"), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(outDir, "template", "README.md.go.tmpl"), []byte("dummy"), 0644); err != nil { + t.Fatal(err) + } + + p := libraryPostProcessParams{ + outDir: outDir, + UseGoPostprocessor: true, + cfg: &config.Config{ + Default: &config.Default{ + Java: &config.JavaModule{ + LibrariesBOMVersion: "1.0.0", + }, + }, + Libraries: []*config.Library{ + {Name: "google-cloud-java", Version: "1.2.3"}, + }, + }, + library: &config.Library{ + Name: "test-library", + Version: "1.2.3", + }, + } + err := postProcessLibrary(t.Context(), p) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + }) +} diff --git a/internal/librarian/java/repometadata_test.go b/internal/librarian/java/repometadata_test.go index c1f42a8ac91..ef94400cf07 100644 --- a/internal/librarian/java/repometadata_test.go +++ b/internal/librarian/java/repometadata_test.go @@ -76,7 +76,10 @@ func TestRepoMetadata_write(t *testing.T) { func TestDeriveRepoMetadata_Overrides(t *testing.T) { t.Parallel() apiPath := "google/cloud/secretmanager/v1" - googleapis := "../../testdata/googleapis" + googleapis, err := filepath.Abs("../../testdata/googleapis") + if err != nil { + t.Fatal(err) + } cfg := sample.Config() cfg.Language = config.LanguageJava diff --git a/internal/sidekick/parser/protobuf.go b/internal/sidekick/parser/protobuf.go index 197a8ac0895..30df14d56ec 100644 --- a/internal/sidekick/parser/protobuf.go +++ b/internal/sidekick/parser/protobuf.go @@ -169,7 +169,6 @@ func runProtoc(files []string, sourceCfg *sources.SourceConfig) ([]byte, error) args := []string{ "--include_imports", "--include_source_info", - "--retain_options", "--descriptor_set_out", tempFile.Name(), } if sourceCfg != nil { From a59e405104592d6a2d1487171aaf215498242cd5 Mon Sep 17 00:00:00 2001 From: Sophia Yang Date: Sat, 6 Jun 2026 15:54:59 -0400 Subject: [PATCH 032/108] fix(java): ensure zero diff for README whitespace --- .../librarian/java/template/README.md.go.tmpl | 252 ++++++++++++++++++ 1 file changed, 252 insertions(+) create mode 100644 internal/librarian/java/template/README.md.go.tmpl diff --git a/internal/librarian/java/template/README.md.go.tmpl b/internal/librarian/java/template/README.md.go.tmpl new file mode 100644 index 00000000000..3a4a7cc0a88 --- /dev/null +++ b/internal/librarian/java/template/README.md.go.tmpl @@ -0,0 +1,252 @@ +# Google {{ .Metadata.Repo.NamePretty }} Client for Java + +Java idiomatic client for [{{ .Metadata.Repo.NamePretty }}][product-docs]. + +[![Maven][maven-version-image]][maven-version-link] +![Stability][stability-image] + +- [Product Documentation][product-docs] +- [Client Library Documentation][javadocs] + +{{ if and .Metadata.Partials .Metadata.Partials.DeprecationWarning }}{{ .Metadata.Partials.DeprecationWarning }} +{{ end }}{{ if eq .Metadata.Repo.ReleaseLevel "preview" }}> Note: This client is a work-in-progress, and may occasionally +> make backwards-incompatible changes. +{{ end }}{{ if .MigratedSplitRepo }}:bus: In October 2022, this library has moved to +[google-cloud-java/{{ .Metadata.Repo.RepoShort }}]( +https://github.com/googleapis/google-cloud-java/tree/main/{{ .Metadata.Repo.RepoShort }}). +This repository will be archived in the future. +Future releases will appear in the new repository (https://github.com/googleapis/google-cloud-java/releases). +The Maven artifact coordinates (`{{ .GroupID }}:{{ .ArtifactID }}`) remain the same. +{{ end }} +## Quickstart + +{{ if and .Metadata.Snippets (index .Metadata.Snippets (print .Metadata.Repo.APIShortname "_install_with_bom")) -}} +If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file: + +```xml +{{ index .Metadata.Snippets (print .Metadata.Repo.APIShortname "_install_with_bom") }} +``` + +If you are using Maven without the BOM, add this to your dependencies: +{{- else if and .Monorepo (or (eq .GroupID "com.google.cloud") (eq .GroupID "com.google.analytics") (eq .GroupID "com.google.area120")) }} +If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file: + +```xml + + + + com.google.cloud + libraries-bom + {{ .Metadata.LibrariesBOMVersion }} + pom + import + + + + + + + {{ .GroupID }} + {{ .ArtifactID }} + + +``` + +If you are using Maven without the BOM, add this to your dependencies: +{{ else }}If you are using Maven, add this to your pom.xml file: +{{ end }} + + +```xml +{{ if and .Metadata.Snippets (index .Metadata.Snippets (print .Metadata.Repo.APIShortname "_install_without_bom")) -}} +{{ index .Metadata.Snippets (print .Metadata.Repo.APIShortname "_install_without_bom") }} +{{- else -}} + + {{ .GroupID }} + {{ .ArtifactID }} + {{ .Version }} + +{{- end }} +``` + +{{ if and .Metadata.Snippets (index .Metadata.Snippets (print .Metadata.Repo.APIShortname "_install_with_bom")) -}} +If you are using Gradle 5.x or later, add this to your dependencies: + +```Groovy +implementation platform('com.google.cloud:libraries-bom:{{ .Metadata.LibrariesBOMVersion }}') + +implementation '{{ .GroupID }}:{{ .ArtifactID }}' +``` +{{ end }}If you are using Gradle without BOM, add this to your dependencies: + +```Groovy +implementation '{{ .GroupID }}:{{ .ArtifactID }}:{{ .Version }}' +``` + +If you are using SBT, add this to your dependencies: + +```Scala +libraryDependencies += "{{ .GroupID }}" % "{{ .ArtifactID }}" % "{{ .Version }}" +``` + +## Authentication + +See the [Authentication][authentication] section in the base directory's README. + +## Authorization + +The client application making API calls must be granted [authorization scopes][auth-scopes] required for the desired {{ .Metadata.Repo.NamePretty }} APIs, and the authenticated principal must have the [IAM role(s)][predefined-iam-roles] required to access GCP resources using the {{ .Metadata.Repo.NamePretty }} API calls. + +## Getting Started + +### Prerequisites + +You will need a [Google Cloud Platform Console][developer-console] project with the {{ .Metadata.Repo.NamePretty }} [API enabled][enable-api]. +{{ if .Metadata.Repo.RequiresBilling }}You will need to [enable billing][enable-billing] to use Google {{ .Metadata.Repo.NamePretty }}.{{ end }} +[Follow these instructions][create-project] to get your project set up. You will also need to set up the local development environment by +[installing the Google Cloud Command Line Interface][cloud-cli] and running the following commands in command line: +`gcloud auth login` and `gcloud config set project [YOUR PROJECT ID]`. + +### Installation and setup + +You'll need to obtain the `{{ .ArtifactID }}` library. See the [Quickstart](#quickstart) section +to add `{{ .ArtifactID }}` as a dependency in your code. + +## About {{ .Metadata.Repo.NamePretty }} + +{{ if and .Metadata.Partials .Metadata.Partials.About }} +{{ .Metadata.Partials.About }} +{{ else }} +[{{ .Metadata.Repo.NamePretty }}][product-docs] {{ .Metadata.Repo.APIDescription }} + +See the [{{ .Metadata.Repo.NamePretty }} client library docs][javadocs] to learn how to +use this {{ .Metadata.Repo.NamePretty }} Client Library. +{{ end -}} +{{ if and .Metadata.Partials .Metadata.Partials.CustomContent }} + +{{ .Metadata.Partials.CustomContent }} +{{ end }} + +{{ if .Metadata.Samples }} +## Samples + +Samples are in the [`samples/`](https://github.com/{{ .Metadata.Repo.Repo }}/tree/main/samples) directory. + +| Sample | Source Code | Try it | +| --------------------------- | --------------------------------- | ------ | +{{ range .Metadata.Samples }}| {{ .Title }} | [source code](https://github.com/{{ $.Metadata.Repo.Repo }}/blob/main/{{ .File }}) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/{{ $.Metadata.Repo.Repo }}&page=editor&open_in_editor={{ .File }}) | +{{ end }} +{{ end }} + +## Troubleshooting + +To get help, follow the instructions in the [shared Troubleshooting document][troubleshooting]. + +{{ if .Metadata.Repo.Transport }}## Transport + +{{ if eq .Metadata.Repo.Transport "grpc" }}{{ .Metadata.Repo.NamePretty }} uses gRPC for the transport layer. +{{ else if eq .Metadata.Repo.Transport "http" }}{{ .Metadata.Repo.NamePretty }} uses HTTP/JSON for the transport layer. +{{ else if eq .Metadata.Repo.Transport "both" }}{{ .Metadata.Repo.NamePretty }} uses both gRPC and HTTP/JSON for the transport layer. +{{ end }} +{{ end }}## Supported Java Versions + +Java {{ .Metadata.MinJavaVersion }} or above is required for using this client. + +Google's Java client libraries, +[Google Cloud Client Libraries][cloudlibs] +and +[Google Cloud API Libraries][apilibs], +follow the +[Oracle Java SE support roadmap][oracle] +(see the Oracle Java SE Product Releases section). + +### For new development + +In general, new feature development occurs with support for the lowest Java +LTS version covered by Oracle's Premier Support (which typically lasts 5 years +from initial General Availability). If the minimum required JVM for a given +library is changed, it is accompanied by a [semver][semver] major release. + +Java 11 and (in September 2021) Java 17 are the best choices for new +development. + +### Keeping production systems current + +Google tests its client libraries with all current LTS versions covered by +Oracle's Extended Support (which typically lasts 8 years from initial +General Availability). + +#### Legacy support + +Google's client libraries support legacy versions of Java runtimes with long +term stable libraries that don't receive feature updates on a best efforts basis +as it may not be possible to backport all patches. + +Google provides updates on a best efforts basis to apps that continue to use +Java 7, though apps might need to upgrade to current versions of the library +that supports their JVM. + +#### Where to find specific information + +The latest versions and the supported Java versions are identified on +the individual GitHub repository `github.com/GoogleAPIs/java-SERVICENAME` +and on [google-cloud-java][g-c-j]. + +## Versioning + +{{ if and .Metadata.Partials .Metadata.Partials.Versioning }} +{{ .Metadata.Partials.Versioning }} +{{ else }} +This library follows [Semantic Versioning](http://semver.org/). + +{{ if eq .Metadata.Repo.ReleaseLevel "preview" }} +It is currently in major version zero (``0.y.z``), which means that anything may change at any time +and the public API should not be considered stable. +{{ end }}{{ end }} + +## Contributing + +{{ if and .Metadata.Partials .Metadata.Partials.Contributing }} +{{ .Metadata.Partials.Contributing }} +{{ else }} +Contributions to this library are always welcome and highly encouraged. + +See [CONTRIBUTING][contributing] for more information how to get started. + +Please note that this project is released with a Contributor Code of Conduct. By participating in +this project you agree to abide by its terms. See [Code of Conduct][code-of-conduct] for more +information. +{{ end }} + +## License + +Apache 2.0 - See [LICENSE][license] for more information. + +Java is a registered trademark of Oracle and/or its affiliates. + +[product-docs]: {{ .Metadata.Repo.ProductDocumentation }} +[javadocs]: {{ .Metadata.Repo.ClientDocumentation }} +[stability-image]: https://img.shields.io/badge/stability-{{ if eq .Metadata.Repo.ReleaseLevel "stable" }}stable-green{{ else if eq .Metadata.Repo.ReleaseLevel "preview" }}preview-yellow{{ else }}unknown-red{{ end }} +[maven-version-image]: https://img.shields.io/maven-central/v/{{ .GroupID }}/{{ .ArtifactID }}.svg +[maven-version-link]: https://central.sonatype.com/artifact/{{ .GroupID }}/{{ .ArtifactID }}/{{ .Version }} +[authentication]: https://github.com/googleapis/google-cloud-java#authentication +[auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes +[predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles +[iam-policy]: https://cloud.google.com/iam/docs/overview#cloud-iam-policy +[developer-console]: https://console.developers.google.com/ +[create-project]: https://cloud.google.com/resource-manager/docs/creating-managing-projects +[cloud-cli]: https://cloud.google.com/cli +[troubleshooting]: https://github.com/googleapis/google-cloud-java/blob/main/TROUBLESHOOTING.md +[contributing]: https://github.com/{{ .Metadata.Repo.Repo }}/blob/main/CONTRIBUTING.md +[code-of-conduct]: https://github.com/{{ .Metadata.Repo.Repo }}/blob/main/CODE_OF_CONDUCT.md#contributor-code-of-conduct +[license]: https://github.com/{{ .Metadata.Repo.Repo }}/blob/main/LICENSE +{{ if .Metadata.Repo.RequiresBilling }}[enable-billing]: https://cloud.google.com/apis/docs/getting-started#enabling_billing{{ end }} +{{ if .Metadata.Repo.APIID }}[enable-api]: https://console.cloud.google.com/flows/enableapi?apiid={{ .Metadata.Repo.APIID }}{{ end }} +[libraries-bom]: https://github.com/GoogleCloudPlatform/cloud-opensource-java/wiki/The-Google-Cloud-Platform-Libraries-BOM +[shell_img]: https://gstatic.com/cloudssh/images/open-btn.png + +[semver]: https://semver.org/ +[cloudlibs]: https://cloud.google.com/apis/docs/client-libraries-explained +[apilibs]: https://cloud.google.com/apis/docs/client-libraries-explained#google_api_client_libraries +[oracle]: https://www.oracle.com/java/technologies/java-se-support-roadmap.html +[g-c-j]: http://github.com/googleapis/google-cloud-java From 54039c2b3d32bc63d0001bfbe1d451f2cb04acd7 Mon Sep 17 00:00:00 2001 From: Sophia Yang Date: Sat, 6 Jun 2026 16:22:14 -0400 Subject: [PATCH 033/108] fix(java): remove left hyphen from else if in README template to prevent newline stripping --- internal/librarian/java/postprocess.go | 10 +++++----- internal/librarian/java/template/README.md.go.tmpl | 3 +-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/internal/librarian/java/postprocess.go b/internal/librarian/java/postprocess.go index 54b3e8636e8..7468809a229 100644 --- a/internal/librarian/java/postprocess.go +++ b/internal/librarian/java/postprocess.go @@ -60,11 +60,11 @@ type postProcessParams struct { } type libraryPostProcessParams struct { - cfg *config.Config - library *config.Library - outDir string - metadata *repoMetadata - transports map[string]serviceconfig.Transport + cfg *config.Config + library *config.Library + outDir string + metadata *repoMetadata + transports map[string]serviceconfig.Transport UseGoPostprocessor bool } diff --git a/internal/librarian/java/template/README.md.go.tmpl b/internal/librarian/java/template/README.md.go.tmpl index 3a4a7cc0a88..1f50d9473b0 100644 --- a/internal/librarian/java/template/README.md.go.tmpl +++ b/internal/librarian/java/template/README.md.go.tmpl @@ -28,7 +28,7 @@ If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file: ``` If you are using Maven without the BOM, add this to your dependencies: -{{- else if and .Monorepo (or (eq .GroupID "com.google.cloud") (eq .GroupID "com.google.analytics") (eq .GroupID "com.google.area120")) }} +{{ else if and .Monorepo (or (eq .GroupID "com.google.cloud") (eq .GroupID "com.google.analytics") (eq .GroupID "com.google.area120")) }} If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file: ```xml @@ -56,7 +56,6 @@ If you are using Maven without the BOM, add this to your dependencies: {{ else }}If you are using Maven, add this to your pom.xml file: {{ end }} - ```xml {{ if and .Metadata.Snippets (index .Metadata.Snippets (print .Metadata.Repo.APIShortname "_install_without_bom")) -}} {{ index .Metadata.Snippets (print .Metadata.Repo.APIShortname "_install_without_bom") }} From f09bc19da72180b2557472b9cdfa382572f5317e Mon Sep 17 00:00:00 2001 From: Sophia Yang Date: Sat, 6 Jun 2026 16:27:18 -0400 Subject: [PATCH 034/108] fix(java): match legacy Troubleshooting spacing exactly for missing samples --- internal/librarian/java/template/README.md.go.tmpl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/librarian/java/template/README.md.go.tmpl b/internal/librarian/java/template/README.md.go.tmpl index 1f50d9473b0..d9177ec037e 100644 --- a/internal/librarian/java/template/README.md.go.tmpl +++ b/internal/librarian/java/template/README.md.go.tmpl @@ -137,6 +137,9 @@ Samples are in the [`samples/`](https://github.com/{{ .Metadata.Repo.Repo }}/tre {{ end }} {{ end }} +{{ if not .Metadata.Samples }} + +{{ end -}} ## Troubleshooting To get help, follow the instructions in the [shared Troubleshooting document][troubleshooting]. From 6eb0889fdf94b01df0eb78119c564499d8156933 Mon Sep 17 00:00:00 2001 From: Sophia Yang Date: Sun, 7 Jun 2026 10:49:08 -0400 Subject: [PATCH 035/108] fix: fallback to .yml for readme partials and save postprocess modernization work --- internal/filesystem/filesystem.go | 57 ++++ internal/librarian/java/generate.go | 45 +-- internal/librarian/java/postprocess.go | 95 +++++-- internal/librarian/java/postprocess_new.go | 148 ++++++++++ .../librarian/java/postprocess_new_test.go | 146 ++++++++++ internal/librarian/java/postprocess_test.go | 10 +- internal/librarian/java/samples.go | 260 +++++++++++++++++ internal/librarian/java/samples_test.go | 164 +++++++++++ .../librarian/java/template/README.md.tmpl | 266 ++++++++++++++++++ internal/librarian/java/template_render.go | 232 +++++++++++++++ .../librarian/java/template_render_test.go | 132 +++++++++ internal/postprocessing/config.go | 73 +++++ internal/postprocessing/config_test.go | 132 +++++++++ internal/postprocessing/fileops.go | 77 +++++ internal/postprocessing/fileops_test.go | 69 +++-- 15 files changed, 1845 insertions(+), 61 deletions(-) create mode 100644 internal/librarian/java/postprocess_new.go create mode 100644 internal/librarian/java/postprocess_new_test.go create mode 100644 internal/librarian/java/samples.go create mode 100644 internal/librarian/java/samples_test.go create mode 100644 internal/librarian/java/template/README.md.tmpl create mode 100644 internal/librarian/java/template_render.go create mode 100644 internal/librarian/java/template_render_test.go create mode 100644 internal/postprocessing/config.go create mode 100644 internal/postprocessing/config_test.go diff --git a/internal/filesystem/filesystem.go b/internal/filesystem/filesystem.go index 079671c54b0..7d95dc8a90e 100644 --- a/internal/filesystem/filesystem.go +++ b/internal/filesystem/filesystem.go @@ -60,6 +60,63 @@ func MoveAndMerge(sourceDir, targetDir string) error { return nil } +// MoveAndMergeWithKeep moves entries from sourceDir to targetDir. +// It merges directories recursively if they exist in both source and target. +// If an entry in sourceDir is a file that already exists in targetDir: +// - If keepFunc is not nil and returns true for the path relative to libraryRoot, it is preserved (source is deleted). +// - Otherwise, the target is overwritten. +func MoveAndMergeWithKeep(sourceDir, targetDir, libraryRoot string, keepFunc func(string) bool) error { + entries, err := os.ReadDir(sourceDir) + if err != nil { + return err + } + for _, entry := range entries { + oldPath := filepath.Join(sourceDir, entry.Name()) + newPath := filepath.Join(targetDir, entry.Name()) + if entry.IsDir() { + if _, err := os.Stat(newPath); err == nil { + // Destination exists, merge contents. + if err := MoveAndMergeWithKeep(oldPath, newPath, libraryRoot, keepFunc); err != nil { + return err + } + // Remove the now-empty source directory after successful merge. + if err := os.Remove(oldPath); err != nil { + return err + } + continue + } + } else { + if _, err := os.Stat(newPath); err == nil { + rel, err := filepath.Rel(libraryRoot, newPath) + if err != nil { + return err + } + if keepFunc != nil && keepFunc(filepath.ToSlash(rel)) { + // Preserve the target file, remove the source file. + if err := os.Remove(oldPath); err != nil { + return err + } + continue + } + // Overwrite existing file + if err := os.Remove(newPath); err != nil { + return err + } + } + } + if err := os.Rename(oldPath, newPath); err != nil { + // Fallback for cross-device links + if err := CopyFile(oldPath, newPath); err != nil { + return err + } + if err := os.Remove(oldPath); err != nil { + return err + } + } + } + return nil +} + // CopyFile copies a file from src to dest. func CopyFile(src, dest string) error { in, err := os.Open(src) diff --git a/internal/librarian/java/generate.go b/internal/librarian/java/generate.go index 6b5e52ee7ac..dbf0ef07c1b 100644 --- a/internal/librarian/java/generate.go +++ b/internal/librarian/java/generate.go @@ -57,13 +57,14 @@ var ( ) type generateAPIParams struct { - cfg *config.Config - api *config.API - library *config.Library - srcCfg *sources.SourceConfig - outdir string - metadata *repoMetadata - apiCfg *serviceconfig.API + cfg *config.Config + api *config.API + library *config.Library + srcCfg *sources.SourceConfig + outdir string + metadata *repoMetadata + apiCfg *serviceconfig.API + useGoPostprocessor bool } // Generate generates a Java client library. @@ -97,13 +98,14 @@ func Generate(ctx context.Context, cfg *config.Config, library *config.Library, transports[api.Path] = apiCfg.Transport(config.LanguageJava) // metadata is needed for pom.xml generation in post process if err := generateAPI(ctx, generateAPIParams{ - cfg: cfg, - api: api, - library: library, - srcCfg: srcCfg, - outdir: outdir, - metadata: metadata, - apiCfg: apiCfg, + cfg: cfg, + api: api, + library: library, + srcCfg: srcCfg, + outdir: outdir, + metadata: metadata, + apiCfg: apiCfg, + useGoPostprocessor: useGoPostprocessor, }); err != nil { return fmt.Errorf("failed to generate api %q: %w", api.Path, err) } @@ -139,13 +141,14 @@ func generateAPI(ctx context.Context, params generateAPIParams) error { additionalProtosToCopyRel := processAdditionalProtos(javaAPI, googleapisDir) postParams := postProcessParams{ - cfg: params.cfg, - library: params.library, - javaAPI: javaAPI, - metadata: params.metadata, - outDir: params.outdir, - apiBase: deriveAPIBase(params.library, params.api.Path), - includeSamples: *javaAPI.Samples, + cfg: params.cfg, + library: params.library, + javaAPI: javaAPI, + metadata: params.metadata, + outDir: params.outdir, + apiBase: deriveAPIBase(params.library, params.api.Path), + includeSamples: *javaAPI.Samples, + UseGoPostprocessor: params.useGoPostprocessor, } gapicDir := postParams.gapicDir() gRPCDir := postParams.gRPCDir() diff --git a/internal/librarian/java/postprocess.go b/internal/librarian/java/postprocess.go index 7468809a229..768e06aee25 100644 --- a/internal/librarian/java/postprocess.go +++ b/internal/librarian/java/postprocess.go @@ -49,14 +49,15 @@ type protoFileToCopy struct { } type postProcessParams struct { - cfg *config.Config - library *config.Library - javaAPI *config.JavaAPI - metadata *repoMetadata - outDir string - apiBase string - protosToCopy []protoFileToCopy - includeSamples bool + cfg *config.Config + library *config.Library + javaAPI *config.JavaAPI + metadata *repoMetadata + outDir string + apiBase string + protosToCopy []protoFileToCopy + includeSamples bool + UseGoPostprocessor bool } type libraryPostProcessParams struct { @@ -72,7 +73,18 @@ func postProcessLibrary(ctx context.Context, params libraryPostProcessParams) er if params.UseGoPostprocessor { yamlPath := filepath.Join(params.outDir, "postprocess.yaml") if _, err := os.Stat(yamlPath); err == nil { - return postProcessLibraryNew(params) + if err := postProcessLibraryNew(params); err != nil { + return err + } + + monorepoVersion, err := findMonorepoVersion(params.cfg) + if err != nil { + return err + } + if err := syncPOMs(params.library, params.outDir, monorepoVersion, params.metadata, params.transports); err != nil { + return fmt.Errorf("%w: %w", errSyncPOMs, err) + } + return nil } } if err := createOrVerifyOwlbotPy(params.outDir); err != nil { @@ -132,6 +144,38 @@ func postProcessAPI(ctx context.Context, params postProcessParams) error { if err := copyFiles(params); err != nil { return fmt.Errorf("failed to copy files: %w", err) } + coords := params.coords() + + if params.UseGoPostprocessor { + yamlPath := filepath.Join(params.outDir, "postprocess.yaml") + if _, err := os.Stat(yamlPath); err == nil { + keepSet := make(map[string]bool) + for _, k := range params.library.Keep { + keepSet[strings.TrimSuffix(filepath.ToSlash(k), "/")] = true + } + if err := restructureModules(params, params.outDir, keepSet, params.outDir); err != nil { + return fmt.Errorf("failed to restructure direct to outDir: %w", err) + } + + protoModuleRepoRoot := filepath.Join(params.outDir, coords.Proto.ArtifactID) + shouldGenerate, err := clirrIgnoreShouldGenerate(coords.Proto.ArtifactID, protoModuleRepoRoot, params.javaAPI.Monolithic) + if err != nil { + return fmt.Errorf("failed to check for clirr ignore file: %w", err) + } + if shouldGenerate { + if err := generateClirrIgnore(protoModuleRepoRoot); err != nil { + return fmt.Errorf("failed to generate clirr ignore file: %w", err) + } + } + + // Cleanup intermediate protoc output directory + if err := os.RemoveAll(filepath.Join(params.outDir, params.apiBase)); err != nil { + return fmt.Errorf("failed to cleanup intermediate files: %w", err) + } + return nil + } + } + if err := restructureToStaging(params); err != nil { return fmt.Errorf("failed to restructure to staging: %w", err) } @@ -139,7 +183,6 @@ func postProcessAPI(ctx context.Context, params postProcessParams) error { // Generate clirr-ignored-differences.xml for the proto module. // We target the staging directory because runOwlBot hasn't moved the files // to their final destination yet. - coords := params.coords() protoModuleRepoRoot := filepath.Join(params.outDir, coords.Proto.ArtifactID) shouldGenerate, err := clirrIgnoreShouldGenerate(coords.Proto.ArtifactID, protoModuleRepoRoot, params.javaAPI.Monolithic) if err != nil { @@ -274,7 +317,7 @@ func restructureToStaging(params postProcessParams) error { if err := os.MkdirAll(destRoot, 0755); err != nil { return fmt.Errorf("failed to create staging directory: %w", err) } - return restructureModules(params, destRoot) + return restructureModules(params, destRoot, nil, "") } type moveAction struct { @@ -282,14 +325,34 @@ type moveAction struct { description string } -func restructure(actions []moveAction) error { +func restructure(actions []moveAction, keepSet map[string]bool, libraryRoot string) error { for _, action := range actions { if _, err := os.Stat(action.src); err == nil { if err := os.MkdirAll(action.dest, 0755); err != nil { return fmt.Errorf("failed to create directory %s: %w", action.dest, err) } - if err := filesystem.MoveAndMerge(action.src, action.dest); err != nil { - return fmt.Errorf("failed to move %s: %w", action.description, err) + if keepSet != nil { + keepFunc := func(p string) bool { + if shouldPreserve(p, keepSet) { + return true + } + // Also preserve existing files that lack the auto-generated marker. + destPath := filepath.Join(libraryRoot, p) + if _, err := os.Stat(destPath); err == nil { + isGen, err := hasMarker(destPath) + if err == nil && !isGen { + return true + } + } + return false + } + if err := filesystem.MoveAndMergeWithKeep(action.src, action.dest, libraryRoot, keepFunc); err != nil { + return fmt.Errorf("failed to move with keep %s: %w", action.description, err) + } + } else { + if err := filesystem.MoveAndMerge(action.src, action.dest); err != nil { + return fmt.Errorf("failed to move %s: %w", action.description, err) + } } } } @@ -299,7 +362,7 @@ func restructure(actions []moveAction) error { // restructureModules moves the generated code from the temporary versioned directory // tree into the destination root directory for GAPIC, Proto, gRPC, and samples. // It also copies the relevant proto files into the proto module. -func restructureModules(params postProcessParams, destRoot string) error { +func restructureModules(params postProcessParams, destRoot string, keepSet map[string]bool, libraryRoot string) error { coords := params.coords() tempProtoSrcDir := params.protoDir() if params.library.Name != commonProtosLibrary { @@ -365,7 +428,7 @@ func restructureModules(params postProcessParams, destRoot string) error { description: "samples", }) } - if err := restructure(actions); err != nil { + if err := restructure(actions, keepSet, libraryRoot); err != nil { return err } // Copy proto files to proto-*/src/main/proto diff --git a/internal/librarian/java/postprocess_new.go b/internal/librarian/java/postprocess_new.go new file mode 100644 index 00000000000..231c608ca60 --- /dev/null +++ b/internal/librarian/java/postprocess_new.go @@ -0,0 +1,148 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package java + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/bmatcuk/doublestar/v4" + "github.com/googleapis/librarian/internal/filesystem" + "github.com/googleapis/librarian/internal/postprocessing" +) + +// postProcessLibraryNew implements the new postprocessing flow, bypassing owlbot.py. +// It applies operations from postprocess.yaml, and renders the README.md directly +// on the generated files in their final destinations. +func postProcessLibraryNew(p libraryPostProcessParams) error { + // 1. Load postprocess.yaml and apply operations + postprocessYamlPath := filepath.Join(p.outDir, "postprocess.yaml") + if _, err := os.Stat(postprocessYamlPath); err == nil { + cfg, err := postprocessing.ParseConfig(postprocessYamlPath) + if err != nil { + return fmt.Errorf("failed to parse postprocess.yaml: %w", err) + } + + // 1. Apply Copies + for _, c := range cfg.CopyFile { + srcAbs := filepath.Join(p.outDir, c.Src) + dstAbs := filepath.Join(p.outDir, c.Dst) + if err := filesystem.CopyFile(srcAbs, dstAbs); err != nil { + return fmt.Errorf("failed to copy file from %s to %s: %w", c.Src, c.Dst, err) + } + } + + // 2. Apply Removes + for _, rem := range cfg.RemoveFile { + absPath := filepath.Join(p.outDir, rem) + if err := postprocessing.RemoveFile(absPath); err != nil { + return fmt.Errorf("failed to remove file %s: %w", rem, err) + } + } + + // 3. Apply Replacements + for _, r := range cfg.Replace { + absPath := filepath.Join(p.outDir, r.Path) + if err := postprocessing.Replace(absPath, r.Original, r.Replacement); err != nil { + return fmt.Errorf("failed to apply replacement in %s: %w", r.Path, err) + } + } + + // 4. Apply Regex Replacements + for _, r := range cfg.ReplaceRegex { + absPath := filepath.Join(p.outDir, r.Path) + if err := postprocessing.ReplaceRegex(absPath, r.Pattern, r.Replacement); err != nil { + return fmt.Errorf("failed to apply regex replacement in %s: %w", r.Path, err) + } + } + + // 5. Apply Method Operations + for _, mo := range cfg.MethodOperations { + files, err := resolveGlobs(p.outDir, mo.Path) + if err != nil { + return fmt.Errorf("failed to resolve glob for %s: %w", mo.Path, err) + } + for _, file := range files { + switch mo.Action { + case "delete": + if err := postprocessing.DeleteFunc(file, mo.FuncName, "java"); err != nil { + if strings.Contains(err.Error(), "not found") { + continue + } + return fmt.Errorf("failed to delete method %q in %s: %w", mo.FuncName, file, err) + } + case "duplicate": + if err := postprocessing.DuplicateMethod(file, mo.FuncName, mo.NewName, "java"); err != nil { + if strings.Contains(err.Error(), "not found") { + continue + } + return fmt.Errorf("failed to duplicate method %q in %s: %w", mo.FuncName, file, err) + } + case "deprecate": + if err := postprocessing.DeprecateMethod(file, mo.FuncName, mo.DeprecationMessage, "java"); err != nil { + if strings.Contains(err.Error(), "not found") { + continue + } + return fmt.Errorf("failed to deprecate method %q in %s: %w", mo.FuncName, file, err) + } + default: + return fmt.Errorf("unsupported method operation action %q", mo.Action) + } + } + } + } + + // 4. Render README.md + templatePath := filepath.Join("template", "README.md.go.tmpl") + if _, err := os.Stat(templatePath); err != nil { + // Fallback to absolute path for this workspace + templatePath = "/usr/local/google/home/sophieeee/workspace/owlbot-modernization/librarian/internal/librarian/java/template/README.md.go.tmpl" + } + + libraryVersion, err := deriveLastReleasedVersion(p.library.Version) + if err != nil { + return fmt.Errorf("failed to derive library version: %w", err) + } + + if p.cfg == nil { + return fmt.Errorf("cfg is nil") + } + if p.cfg.Default == nil { + return fmt.Errorf("cfg.Default is nil") + } + if p.cfg.Default.Java == nil { + return fmt.Errorf("cfg.Default.Java is nil") + } + + bomVersion, err := findBOMVersion(p.cfg) + if err != nil { + return fmt.Errorf("failed to find BOM version: %w", err) + } + + if err := RenderREADME(p.outDir, templatePath, bomVersion, libraryVersion); err != nil { + return fmt.Errorf("failed to render README: %w", err) + } + + return nil +} + +func resolveGlobs(outDir, pathPattern string) ([]string, error) { + if strings.ContainsAny(pathPattern, "*?[]{}") { + return doublestar.FilepathGlob(filepath.Join(outDir, pathPattern)) + } + return []string{filepath.Join(outDir, pathPattern)}, nil +} diff --git a/internal/librarian/java/postprocess_new_test.go b/internal/librarian/java/postprocess_new_test.go new file mode 100644 index 00000000000..5bfe8f265a1 --- /dev/null +++ b/internal/librarian/java/postprocess_new_test.go @@ -0,0 +1,146 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package java + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/googleapis/librarian/internal/config" +) + +func TestPostProcessLibraryNew(t *testing.T) { + tmpDir := t.TempDir() + t.Chdir(tmpDir) + + // Setup structure directly in outDir + destDir := filepath.Join(tmpDir, "my-module", "src", "main", "java") + if err := os.MkdirAll(destDir, 0755); err != nil { + t.Fatal(err) + } + + fileContent := `package com.example; +public class File { + public void oldFunc() {} + public void toDelete() { + System.out.println("delete me"); + } +}` + filePath := filepath.Join(destDir, "File.java") + if err := os.WriteFile(filePath, []byte(fileContent), 0644); err != nil { + t.Fatal(err) + } + + // Setup postprocess.yaml + postprocessYaml := ` +replace: + - path: "**/File.java" + original: "oldFunc" + replacement: "newFunc" +method_operations: + - path: "**/File.java" + action: delete + func_name: "public void toDelete()" + - path: "**/File.java" + action: duplicate + func_name: "public void newFunc()" + new_name: "newFuncCopy" + - path: "**/File.java" + action: deprecate + func_name: "public void newFuncCopy()" + deprecation_message: "Use newFunc instead." +` + if err := os.WriteFile(filepath.Join(tmpDir, "postprocess.yaml"), []byte(postprocessYaml), 0644); err != nil { + t.Fatal(err) + } + + // Setup .repo-metadata.json + metadata := `{ + "repo": { + "name_pretty": "My API", + "distribution_name": "com.google.cloud:google-cloud-myapi", + "repo": "googleapis/google-cloud-java" + }, + "library_version": "1.2.3" +}` + if err := os.WriteFile(filepath.Join(tmpDir, ".repo-metadata.json"), []byte(metadata), 0644); err != nil { + t.Fatal(err) + } + + // Setup template + if err := os.MkdirAll(filepath.Join(tmpDir, "template"), 0755); err != nil { + t.Fatal(err) + } + templateContent := `# {{ .Metadata.Repo.NamePretty }}` + if err := os.WriteFile(filepath.Join(tmpDir, "template", "README.md.go.tmpl"), []byte(templateContent), 0644); err != nil { + t.Fatal(err) + } + + p := libraryPostProcessParams{ + outDir: tmpDir, + cfg: &config.Config{ + Default: &config.Default{ + Java: &config.JavaModule{ + LibrariesBOMVersion: "1.0.0", + }, + }, + }, + library: &config.Library{ + Version: "1.2.3", + }, + } + + err := postProcessLibraryNew(p) + if err != nil { + t.Fatal(err) + } + + // Verify File was modified + content, err := os.ReadFile(filePath) + if err != nil { + t.Fatal(err) + } + sContent := string(content) + if !strings.Contains(sContent, "newFunc") { + t.Errorf("Replacement was not applied. Content: %s", sContent) + } + if strings.Contains(sContent, "toDelete") { + t.Errorf("Delete function was not applied. Content: %s", sContent) + } + if !strings.Contains(sContent, "newFuncCopy") { + t.Errorf("Duplicate method operation was not applied. Content: %s", sContent) + } + if !strings.Contains(sContent, "@Deprecated\n\tpublic void newFuncCopy()") { + t.Errorf("@Deprecated annotation was not applied correctly. Content: %s", sContent) + } + if !strings.Contains(sContent, "* @deprecated Use newFunc instead.") { + t.Errorf("Javadoc deprecation tag was not applied correctly. Content: %s", sContent) + } + + // Verify README was rendered + readmePath := filepath.Join(tmpDir, "README.md") + if _, err := os.Stat(readmePath); err != nil { + t.Errorf("README.md was not rendered: %v", err) + } + readmeContent, err := os.ReadFile(readmePath) + if err != nil { + t.Fatal(err) + } + if string(readmeContent) != "# My API" { + t.Errorf("README content mismatch. Got: %s, expected: # My API", string(readmeContent)) + } +} diff --git a/internal/librarian/java/postprocess_test.go b/internal/librarian/java/postprocess_test.go index 7f7224730f8..fd262832099 100644 --- a/internal/librarian/java/postprocess_test.go +++ b/internal/librarian/java/postprocess_test.go @@ -227,7 +227,7 @@ func TestRestructureModules(t *testing.T) { javaAPI: &config.JavaAPI{}, } destRoot := filepath.Join(tmpDir, "dest") - if err := restructureModules(params, destRoot); err != nil { + if err := restructureModules(params, destRoot, nil, ""); err != nil { t.Fatal(err) } @@ -274,7 +274,7 @@ func TestRestructureModules_CommonProtos(t *testing.T) { }, } destRoot := filepath.Join(tmpDir, "dest") - if err := restructureModules(params, destRoot); err != nil { + if err := restructureModules(params, destRoot, nil, ""); err != nil { t.Fatal(err) } wantPath := filepath.Join(destRoot, "proto-google-common-protos", "src", "main", "java", "com", "google", "cloud", "location", "LocationsProto.java") @@ -302,7 +302,7 @@ func TestRestructureModules_ShouldRemoveClasses(t *testing.T) { javaAPI: &config.JavaAPI{}, } destRoot := filepath.Join(tmpDir, "dest") - if err := restructureModules(params, destRoot); err != nil { + if err := restructureModules(params, destRoot, nil, ""); err != nil { t.Fatal(err) } wantPath := filepath.Join(destRoot, "proto-google-cloud-secretmanager-v1", "src", "main", "java", "com", "google", "cloud", "location", "LocationsProto.java") @@ -359,7 +359,7 @@ func TestRestructureModules_SamplesDisabled(t *testing.T) { javaAPI: &config.JavaAPI{}, } destRoot := filepath.Join(tmpDir, "dest") - if err := restructureModules(params, destRoot); err != nil { + if err := restructureModules(params, destRoot, nil, ""); err != nil { t.Fatal(err) } // Verify sample file location DOES NOT exist @@ -415,7 +415,7 @@ func TestRestructureModules_Monolithic(t *testing.T) { }, } destRoot := filepath.Join(tmpDir, "dest", "src") - if err := restructureModules(params, destRoot); err != nil { + if err := restructureModules(params, destRoot, nil, ""); err != nil { t.Fatal(err) } diff --git a/internal/librarian/java/samples.go b/internal/librarian/java/samples.go new file mode 100644 index 00000000000..81b004b0ccc --- /dev/null +++ b/internal/librarian/java/samples.go @@ -0,0 +1,260 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package java + +import ( + "bufio" + "errors" + "io/fs" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "unicode" + + "gopkg.in/yaml.v3" +) + +var ( + openSnippetRegex = regexp.MustCompile(`\[START ([a-zA-Z0-9_-]+)\]`) + closeSnippetRegex = regexp.MustCompile(`\[END ([a-zA-Z0-9_-]+)\]`) + openExcludeRegex = regexp.MustCompile(`\[START_EXCLUDE\]`) + closeExcludeRegex = regexp.MustCompile(`\[END_EXCLUDE\]`) + + reMetadataBlock = regexp.MustCompile(`(?m)^[ \t]*//[ \t]*sample-metadata:([^\n]+|\n[ \t]*//)+`) + reCommentPrefix = regexp.MustCompile(`(?m)^[ \t]*(?:#|//)[ \t]?`) + + reDecamelize1 = regexp.MustCompile(`([A-Z]+)([A-Z])([a-z0-9])`) + reDecamelize2 = regexp.MustCompile(`([a-z0-9])([A-Z])`) +) + +// decamelize converts CamelCase or PascalCase titles into space-separated words, +// exactly reproducing Python synthtool's _decamelize(value: str). +func decamelize(value string) string { + if value == "" { + return "" + } + r := []rune(value) + r[0] = unicode.ToUpper(r[0]) + s := string(r) + + s = reDecamelize1.ReplaceAllString(s, "${1} ${2}${3}") + return reDecamelize2.ReplaceAllString(s, "${1} ${2}") +} + +// ExtractSamples walks the "samples" directory locating all .java source files. +// It parses embedded multiline "// sample-metadata:" YAML blocks to derive title and path metadata. +func ExtractSamples(dir string) ([]map[string]interface{}, error) { + samplesDir := filepath.Join(dir, "samples") + var files []string + + err := filepath.WalkDir(samplesDir, func(path string, d os.DirEntry, err error) error { + if err != nil { + if errors.Is(err, fs.ErrNotExist) || os.IsNotExist(err) { + return nil + } + return err + } + if d.IsDir() && d.Name() == "test" { + return filepath.SkipDir + } + if !d.IsDir() && d.Type().IsRegular() && filepath.Ext(path) == ".java" && !strings.Contains(filepath.ToSlash(path), "/src/test/") { + files = append(files, path) + } + return nil + }) + if err != nil { + return nil, err + } + + if len(files) == 0 { + return nil, nil + } + + sort.Strings(files) + var samples []map[string]interface{} + + for _, file := range files { + rel, err := filepath.Rel(dir, file) + if err != nil { + continue + } + base := strings.TrimSuffix(filepath.Base(file), ".java") + title := decamelize(base) + + slashPath := filepath.ToSlash(rel) + item := map[string]interface{}{ + "Title": title, + "File": slashPath, + "title": title, + "file": slashPath, + } + + contentBytes, err := os.ReadFile(file) + if err == nil { + if match := reMetadataBlock.FindString(string(contentBytes)); match != "" { + cleaned := reCommentPrefix.ReplaceAllString(match, "") + var meta map[string]map[string]interface{} + if err := yaml.Unmarshal([]byte(cleaned), &meta); err == nil { + if sm, ok := meta["sample-metadata"]; ok { + for k, v := range sm { + item[k] = v + if len(k) > 0 { + upperKey := strings.ToUpper(k[:1]) + k[1:] + item[upperKey] = v + } + } + if t, ok := sm["title"].(string); ok && strings.TrimSpace(t) != "" { + item["Title"] = t + item["title"] = t + } + } + } + } + } + + samples = append(samples, item) + } + return samples, nil +} + +// ExtractSnippets walks the "samples" directory locating *.java and *.xml files. +// It line-scans for [START name] and [END name] tags while supporting [START_EXCLUDE] blocks, +// returning trimmed minimum plain-space indentation blocks. +func ExtractSnippets(dir string) (map[string]string, error) { + samplesDir := filepath.Join(dir, "samples") + var files []string + + err := filepath.WalkDir(samplesDir, func(path string, d os.DirEntry, err error) error { + if err != nil { + if errors.Is(err, fs.ErrNotExist) || os.IsNotExist(err) { + return nil + } + return err + } + if d.IsDir() { + if d.Name() == "test" { + return filepath.SkipDir + } + return nil + } + if !d.Type().IsRegular() { + return nil + } + ext := filepath.Ext(path) + if ext == ".java" || ext == ".xml" { + files = append(files, path) + } + return nil + }) + if err != nil { + return nil, err + } + + if len(files) == 0 { + return nil, nil + } + + sort.Strings(files) + snippetLines := make(map[string][]string) + + for _, file := range files { + f, err := os.Open(file) + if err != nil { + continue + } + + openSnippets := make(map[string]bool) + excluding := false + scanner := bufio.NewScanner(f) + scanner.Buffer(make([]byte, 64*1024), 10*1024*1024) + + for scanner.Scan() { + line := scanner.Text() + openMatch := openSnippetRegex.FindStringSubmatch(line) + closeMatch := closeSnippetRegex.FindStringSubmatch(line) + + if len(openMatch) > 1 && !excluding { + name := openMatch[1] + openSnippets[name] = true + if _, exists := snippetLines[name]; !exists { + snippetLines[name] = []string{} + } + } else if len(closeMatch) > 1 && !excluding { + delete(openSnippets, closeMatch[1]) + } else if openExcludeRegex.MatchString(line) { + excluding = true + } else if closeExcludeRegex.MatchString(line) { + excluding = false + } else if !excluding { + for s := range openSnippets { + snippetLines[s] = append(snippetLines[s], line) + } + } + } + if err := scanner.Err(); err != nil { + f.Close() + return nil, err + } + f.Close() + } + + if len(snippetLines) == 0 { + return nil, nil + } + + result := make(map[string]string) + for snippet, lines := range snippetLines { + result[snippet] = trimLeadingWhitespace(lines) + } + return result, nil +} + +// trimLeadingWhitespace computes the minimum plain-space indentation across non-empty lines, +// trimming that common whitespace while preserving newlines. +func trimLeadingWhitespace(lines []string) string { + if len(lines) == 0 { + return "" + } + + minSpaces := -1 + for _, line := range lines { + if strings.TrimSpace(line) != "" { + spaces := len(line) - len(strings.TrimLeft(line, " ")) + if minSpaces == -1 || spaces < minSpaces { + minSpaces = spaces + } + } + } + if minSpaces == -1 { + minSpaces = 0 + } + + var sb strings.Builder + for _, line := range lines { + if strings.TrimSpace(line) == "" { + sb.WriteString("\n") + } else { + if len(line) >= minSpaces { + sb.WriteString(line[minSpaces:]) + } else { + sb.WriteString(strings.TrimLeft(line, " ")) + } + sb.WriteString("\n") + } + } + return sb.String() +} diff --git a/internal/librarian/java/samples_test.go b/internal/librarian/java/samples_test.go new file mode 100644 index 00000000000..af0a1b4d215 --- /dev/null +++ b/internal/librarian/java/samples_test.go @@ -0,0 +1,164 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package java + +import ( + "os" + "path/filepath" + "testing" +) + +func TestDecamelize(t *testing.T) { + tests := []struct { + input string + expected string + }{ + {"requesterPays", "Requester Pays"}, + {"ACLBatman", "ACL Batman"}, + {"NativeImageLoggingSample", "Native Image Logging Sample"}, + {"simpleTest", "Simple Test"}, + {"", ""}, + } + + for _, tc := range tests { + actual := decamelize(tc.input) + if actual != tc.expected { + t.Errorf("decamelize(%q) = %q; expected %q", tc.input, actual, tc.expected) + } + } +} + +func TestExtractSamples_MissingDir(t *testing.T) { + tempDir := t.TempDir() + samples, err := ExtractSamples(tempDir) + if err != nil { + t.Fatalf("ExtractSamples returned error for missing dir: %v", err) + } + if samples != nil { + t.Errorf("Expected nil samples for missing dir, got %v", samples) + } +} + +func TestExtractSamples_Success(t *testing.T) { + tempDir := t.TempDir() + samplesDir := filepath.Join(tempDir, "samples", "src", "main", "java") + if err := os.MkdirAll(samplesDir, 0755); err != nil { + t.Fatal(err) + } + + file1 := filepath.Join(samplesDir, "RequesterPays.java") + content1 := `// sample-metadata: +// title: Custom Title Override +// description: A custom demo sample +public class RequesterPays {}` + if err := os.WriteFile(file1, []byte(content1), 0644); err != nil { + t.Fatal(err) + } + + file2 := filepath.Join(samplesDir, "demoSample.java") + content2 := `public class demoSample {}` + if err := os.WriteFile(file2, []byte(content2), 0644); err != nil { + t.Fatal(err) + } + + samples, err := ExtractSamples(tempDir) + if err != nil { + t.Fatal(err) + } + + if len(samples) != 2 { + t.Fatalf("Expected 2 samples, got %d", len(samples)) + } + + // First should be RequesterPays.java with custom YAML override (uppercase R comes before lowercase d) + s0 := samples[0] + if s0["Title"] != "Custom Title Override" || s0["title"] != "Custom Title Override" { + t.Errorf("Sample 0 title = %v; expected 'Custom Title Override'", s0["Title"]) + } + if s0["File"] != "samples/src/main/java/RequesterPays.java" { + t.Errorf("Sample 0 file = %v", s0["File"]) + } + + // Second should be demoSample.java + s1 := samples[1] + if s1["Title"] != "Demo Sample" || s1["title"] != "Demo Sample" { + t.Errorf("Sample 1 title = %v; expected 'Demo Sample'", s1["Title"]) + } + if s1["File"] != "samples/src/main/java/demoSample.java" { + t.Errorf("Sample 1 file = %v", s1["File"]) + } +} + +func TestExtractSnippets(t *testing.T) { + tempDir := t.TempDir() + samplesDir := filepath.Join(tempDir, "samples") + if err := os.MkdirAll(samplesDir, 0755); err != nil { + t.Fatal(err) + } + + pomPath := filepath.Join(samplesDir, "pom.xml") + pomContent := ` + + + com.google.cloud + + +` + if err := os.WriteFile(pomPath, []byte(pomContent), 0644); err != nil { + t.Fatal(err) + } + + javaPath := filepath.Join(samplesDir, "Demo.java") + javaContent := `public class Demo { + // [START quickstart] + public void run() { + // [START_EXCLUDE] + System.out.println("hidden"); + // [END_EXCLUDE] + System.out.println("visible"); + } + // [END quickstart] +}` + if err := os.WriteFile(javaPath, []byte(javaContent), 0644); err != nil { + t.Fatal(err) + } + + snippets, err := ExtractSnippets(tempDir) + if err != nil { + t.Fatal(err) + } + + if len(snippets) != 2 { + t.Fatalf("Expected 2 snippets, got %d", len(snippets)) + } + + depSnippet := snippets["dependency_snippet"] + expectedDep := ` + com.google.cloud + +` + if depSnippet != expectedDep { + t.Errorf("dependency_snippet = %q; expected %q", depSnippet, expectedDep) + } + + quickSnippet := snippets["quickstart"] + expectedQuick := `public void run() { + System.out.println("visible"); +} +` + if quickSnippet != expectedQuick { + t.Errorf("quickstart = %q; expected %q", quickSnippet, expectedQuick) + } +} diff --git a/internal/librarian/java/template/README.md.tmpl b/internal/librarian/java/template/README.md.tmpl new file mode 100644 index 00000000000..a2d5a55b477 --- /dev/null +++ b/internal/librarian/java/template/README.md.tmpl @@ -0,0 +1,266 @@ +# Google {{ .Metadata.Repo.NamePretty }} Client for Java + +Java idiomatic client for [{{ .Metadata.Repo.NamePretty }}][product-docs]. + +[![Maven][maven-version-image]][maven-version-link] +![Stability][stability-image] + +- [Product Documentation][product-docs] +- [Client Library Documentation][javadocs] + +{{ if and .Metadata.Partials .Metadata.Partials.DeprecationWarning }} +{{ .Metadata.Partials.DeprecationWarning }} +{{ else if eq .Metadata.Repo.ReleaseLevel "preview" }} +> Note: This client is a work-in-progress, and may occasionally +> make backwards-incompatible changes. +{{ end }} + +{{ if .MigratedSplitRepo }} +:bus: In October 2022, this library has moved to +[google-cloud-java/{{ .Metadata.Repo.RepoShort }}]( +https://github.com/googleapis/google-cloud-java/tree/main/{{ .Metadata.Repo.RepoShort }}). +This repository will be archived in the future. +Future releases will appear in the new repository (https://github.com/googleapis/google-cloud-java/releases). +The Maven artifact coordinates (`{{ .GroupID }}:{{ .ArtifactID }}`) remain the same. +{{ end }} + +## Quickstart + +{{ if and .Metadata.Snippets (index .Metadata.Snippets (print .Metadata.Repo.APIShortname "_install_with_bom")) }} +If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file: + +```xml +{{ index .Metadata.Snippets (print .Metadata.Repo.APIShortname "_install_with_bom") }} +``` + +If you are using Maven without the BOM, add this to your dependencies: +{{ else if and .Monorepo (or (eq .GroupID "com.google.cloud") (eq .GroupID "com.google.analytics") (eq .GroupID "com.google.area120")) }} +If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file: + +```xml + + + + com.google.cloud + libraries-bom + {{ .Metadata.LibrariesBOMVersion }} + pom + import + + + + + + + {{ .GroupID }} + {{ .ArtifactID }} + + +``` + +If you are using Maven without the BOM, add this to your dependencies: +{{ else }} +If you are using Maven, add this to your pom.xml file: +{{ end }} + +```xml +{{ if and .Metadata.Snippets (index .Metadata.Snippets (print .Metadata.Repo.APIShortname "_install_without_bom")) }} +{{ index .Metadata.Snippets (print .Metadata.Repo.APIShortname "_install_without_bom") }} +{{ else }} + + {{ .GroupID }} + {{ .ArtifactID }} + {{ .Version }} + +{{ end }} +``` + +{{ if and .Metadata.Snippets (index .Metadata.Snippets (print .Metadata.Repo.APIShortname "_install_with_bom")) }} +If you are using Gradle 5.x or later, add this to your dependencies: + +```Groovy +implementation platform('com.google.cloud:libraries-bom:{{ .Metadata.LibrariesBOMVersion }}') + +implementation '{{ .GroupID }}:{{ .ArtifactID }}' +``` +{{ end }} + +If you are using Gradle without BOM, add this to your dependencies: + +```Groovy +implementation '{{ .GroupID }}:{{ .ArtifactID }}:{{ .Version }}' +``` + +If you are using SBT, add this to your dependencies: + +```Scala +libraryDependencies += "{{ .GroupID }}" % "{{ .ArtifactID }}" % "{{ .Version }}" +``` + +## Authentication + +See the [Authentication][authentication] section in the base directory's README. + +## Authorization + +The client application making API calls must be granted [authorization scopes][auth-scopes] required for the desired {{ .Metadata.Repo.NamePretty }} APIs, and the authenticated principal must have the [IAM role(s)][predefined-iam-roles] required to access GCP resources using the {{ .Metadata.Repo.NamePretty }} API calls. + +## Getting Started + +### Prerequisites + +You will need a [Google Cloud Platform Console][developer-console] project with the {{ .Metadata.Repo.NamePretty }} [API enabled][enable-api]. +{{ if .Metadata.Repo.RequiresBilling }}You will need to [enable billing][enable-billing] to use Google {{ .Metadata.Repo.NamePretty }}.{{ end }} +[Follow these instructions][create-project] to get your project set up. You will also need to set up the local development environment by +[installing the Google Cloud Command Line Interface][cloud-cli] and running the following commands in command line: +`gcloud auth login` and `gcloud config set project [YOUR PROJECT ID]`. + +### Installation and setup + +You'll need to obtain the `{{ .ArtifactID }}` library. See the [Quickstart](#quickstart) section +to add `{{ .ArtifactID }}` as a dependency in your code. + +## About {{ .Metadata.Repo.NamePretty }} + +{{ if and .Metadata.Partials .Metadata.Partials.About }} +{{ .Metadata.Partials.About }} +{{ else }} +[{{ .Metadata.Repo.NamePretty }}][product-docs] {{ .Metadata.Repo.APIDescription }} + +See the [{{ .Metadata.Repo.NamePretty }} client library docs][javadocs] to learn how to +use this {{ .Metadata.Repo.NamePretty }} Client Library. +{{ end }} + +{{ if and .Metadata.Partials .Metadata.Partials.CustomContent }} +{{ .Metadata.Partials.CustomContent }} +{{ end }} + +{{ if .Metadata.Samples }} +## Samples + +Samples are in the [`samples/`](https://github.com/{{ .Metadata.Repo.Repo }}/tree/main/samples) directory. + +| Sample | Source Code | Try it | +| --------------------------- | --------------------------------- | ------ | +{{ range .Metadata.Samples }}| {{ .Title }} | [source code](https://github.com/{{ $.Metadata.Repo.Repo }}/blob/main/{{ .File }}) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/{{ $.Metadata.Repo.Repo }}&page=editor&open_in_editor={{ .File }}) | +{{ end }} +{{ end }} + +## Troubleshooting + +To get help, follow the instructions in the [shared Troubleshooting document][troubleshooting]. + +{{ if .Metadata.Repo.Transport }} +## Transport + +{{ if eq .Metadata.Repo.Transport "grpc" }} +{{ .Metadata.Repo.NamePretty }} uses gRPC for the transport layer. +{{ else if eq .Metadata.Repo.Transport "http" }} +{{ .Metadata.Repo.NamePretty }} uses HTTP/JSON for the transport layer. +{{ else if eq .Metadata.Repo.Transport "both" }} +{{ .Metadata.Repo.NamePretty }} uses both gRPC and HTTP/JSON for the transport layer. +{{ end }} +{{ end }} + +## Supported Java Versions + +Java {{ .Metadata.MinJavaVersion }} or above is required for using this client. + +Google's Java client libraries, +[Google Cloud Client Libraries][cloudlibs] +and +[Google Cloud API Libraries][apilibs], +follow the +[Oracle Java SE support roadmap][oracle] +(see the Oracle Java SE Product Releases section). + +### For new development + +In general, new feature development occurs with support for the lowest Java +LTS version covered by Oracle's Premier Support (which typically lasts 5 years +from initial General Availability). If the minimum required JVM for a given +library is changed, it is accompanied by a [semver][semver] major release. + +Java 11 and (in September 2021) Java 17 are the best choices for new +development. + +### Keeping production systems current + +Google tests its client libraries with all current LTS versions covered by +Oracle's Extended Support (which typically lasts 8 years from initial +General Availability). + +#### Legacy support + +Google's client libraries support legacy versions of Java runtimes with long +term stable libraries that don't receive feature updates on a best efforts basis +as it may not be possible to backport all patches. + +Google provides updates on a best efforts basis to apps that continue to use +Java 7, though apps might need to upgrade to current versions of the library +that supports their JVM. + +#### Where to find specific information + +The latest versions and the supported Java versions are identified on +the individual GitHub repository `github.com/GoogleAPIs/java-SERVICENAME` +and on [google-cloud-java][g-c-j]. + +## Versioning + +{{ if and .Metadata.Partials .Metadata.Partials.Versioning }} +{{ .Metadata.Partials.Versioning }} +{{ else }} +This library follows [Semantic Versioning](http://semver.org/). + +{{ if eq .Metadata.Repo.ReleaseLevel "preview" }} +It is currently in major version zero (``0.y.z``), which means that anything may change at any time +and the public API should not be considered stable. +{{ end }}{{ end }} + +## Contributing + +{{ if and .Metadata.Partials .Metadata.Partials.Contributing }} +{{ .Metadata.Partials.Contributing }} +{{ else }} +Contributions to this library are always welcome and highly encouraged. + +See [CONTRIBUTING][contributing] for more information how to get started. + +Please note that this project is released with a Contributor Code of Conduct. By participating in +this project you agree to abide by its terms. See [Code of Conduct][code-of-conduct] for more +information. +{{ end }} + +## License + +Apache 2.0 - See [LICENSE][license] for more information. + +Java is a registered trademark of Oracle and/or its affiliates. + +[product-docs]: {{ .Metadata.Repo.ProductDocumentation }} +[javadocs]: {{ .Metadata.Repo.ClientDocumentation }} +[stability-image]: https://img.shields.io/badge/stability-{{ if eq .Metadata.Repo.ReleaseLevel "stable" }}stable-green{{ else if eq .Metadata.Repo.ReleaseLevel "preview" }}preview-yellow{{ else }}unknown-red{{ end }} +[maven-version-image]: https://img.shields.io/maven-central/v/{{ .GroupID }}/{{ .ArtifactID }}.svg +[maven-version-link]: https://central.sonatype.com/artifact/{{ .GroupID }}/{{ .ArtifactID }}/{{ .Version }} +[authentication]: https://github.com/googleapis/google-cloud-java#authentication +[auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes +[predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles +[iam-policy]: https://cloud.google.com/iam/docs/overview#cloud-iam-policy +[developer-console]: https://console.developers.google.com/ +[create-project]: https://cloud.google.com/resource-manager/docs/creating-managing-projects +[cloud-cli]: https://cloud.google.com/cli +[troubleshooting]: https://github.com/googleapis/google-cloud-java/blob/main/TROUBLESHOOTING.md +[contributing]: https://github.com/{{ .Metadata.Repo.Repo }}/blob/main/CONTRIBUTING.md +[code-of-conduct]: https://github.com/{{ .Metadata.Repo.Repo }}/blob/main/CODE_OF_CONDUCT.md#contributor-code-of-conduct +[license]: https://github.com/{{ .Metadata.Repo.Repo }}/blob/main/LICENSE +{{ if .Metadata.Repo.RequiresBilling }}[enable-billing]: https://cloud.google.com/apis/docs/getting-started#enabling_billing{{ end }} +{{ if .Metadata.Repo.APIID }}[enable-api]: https://console.cloud.google.com/flows/enableapi?apiid={{ .Metadata.Repo.APIID }}{{ end }} +[libraries-bom]: https://github.com/GoogleCloudPlatform/cloud-opensource-java/wiki/The-Google-Cloud-Platform-Libraries-BOM +[shell_img]: https://gstatic.com/cloudssh/images/open-btn.png + +[semver]: https://semver.org/ +[cloudlibs]: https://cloud.google.com/apis/docs/client-libraries-explained +[apilibs]: https://cloud.google.com/apis/docs/client-libraries-explained#google_api_client_libraries +[oracle]: https://www.oracle.com/java/technologies/java-se-support-roadmap.html +[g-c-j]: http://github.com/googleapis/google-cloud-java diff --git a/internal/librarian/java/template_render.go b/internal/librarian/java/template_render.go new file mode 100644 index 00000000000..eccc4b9c635 --- /dev/null +++ b/internal/librarian/java/template_render.go @@ -0,0 +1,232 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package java + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "reflect" + "strings" + "text/template" + + "github.com/iancoleman/strcase" + "gopkg.in/yaml.v3" +) + +func isNilOrEmpty(v interface{}) bool { + if v == nil { + return true + } + rv := reflect.ValueOf(v) + switch rv.Kind() { + case reflect.Slice, reflect.Map, reflect.Array, reflect.String: + return rv.Len() == 0 + } + return false +} + +// RenderREADME renders the README.md file using the template and metadata. +// dir is the directory containing .repo-metadata.json and where README.md will be written. +// templatePath is the path to the README.md.go.tmpl file. +func RenderREADME(dir, templatePath, bomVersion, libraryVersion string) error { + metadataPath := filepath.Join(dir, ".repo-metadata.json") + partialsPath := filepath.Join(dir, ".readme-partials.yaml") + if _, err := os.Stat(partialsPath); os.IsNotExist(err) { + partialsPath = filepath.Join(dir, ".readme-partials.yml") + } + outputPath := filepath.Join(dir, "README.md") + + // Read metadata + metadataBytes, err := os.ReadFile(metadataPath) + if err != nil { + return fmt.Errorf("failed to read metadata: %w", err) + } + + var metadata map[string]interface{} + if err := json.Unmarshal(metadataBytes, &metadata); err != nil { + return fmt.Errorf("failed to unmarshal metadata: %w", err) + } + + // Read partials if exist + partials := make(map[string]interface{}) + if _, err := os.Stat(partialsPath); err == nil { + partialsBytes, err := os.ReadFile(partialsPath) + if err != nil { + return fmt.Errorf("failed to read partials: %w", err) + } + if err := yaml.Unmarshal(partialsBytes, &partials); err != nil { + return fmt.Errorf("failed to unmarshal partials: %w", err) + } + } + + // Capitalize keys of partials for template + capitalizedPartials := make(map[string]interface{}) + for k, v := range partials { + capitalizedPartials[strcase.ToCamel(k)] = v + } + + // Prepare data for template + distName, _ := metadata["distribution_name"].(string) + if distName == "" { + if repo, ok := metadata["repo"].(map[string]interface{}); ok { + distName, _ = repo["distribution_name"].(string) + } + } + distParts := strings.Split(distName, ":") + groupID := "" + artifactID := "" + if len(distParts) > 0 { + groupID = distParts[0] + } + if len(distParts) > 1 { + artifactID = distParts[1] + } + + repoName, _ := metadata["repo"].(string) + if repoName == "" { + if repo, ok := metadata["repo"].(map[string]interface{}); ok { + repoName, _ = repo["repo"].(string) + } + } + repoParts := strings.Split(repoName, "/") + repoShort := "" + if len(repoParts) > 0 { + repoShort = repoParts[len(repoParts)-1] + } + + version, _ := metadata["library_version"].(string) + if version == "" { + version = libraryVersion + } + + // Construct the nested Metadata object for the template + templateRepo := map[string]interface{}{ + "NamePretty": metadata["name_pretty"], + "DistributionName": metadata["distribution_name"], + "Repo": metadata["repo"], + "APIShortname": metadata["api_shortname"], + "APIDescription": metadata["api_description"], + "ClientDocumentation": metadata["client_documentation"], + "ProductDocumentation": metadata["product_documentation"], + "ReleaseLevel": metadata["release_level"], + "Transport": metadata["transport"], + "RequiresBilling": metadata["requires_billing"], + "APIID": metadata["api_id"], + "RepoShort": metadata["repo_short"], + } + + // If flat structure failed, try to get from nested repo + if templateRepo["NamePretty"] == nil { + if repo, ok := metadata["repo"].(map[string]interface{}); ok { + templateRepo["NamePretty"] = repo["name_pretty"] + templateRepo["DistributionName"] = repo["distribution_name"] + templateRepo["Repo"] = repo["repo"] + templateRepo["APIShortname"] = repo["api_shortname"] + templateRepo["APIDescription"] = repo["api_description"] + templateRepo["ClientDocumentation"] = repo["client_documentation"] + templateRepo["ProductDocumentation"] = repo["product_documentation"] + templateRepo["ReleaseLevel"] = repo["release_level"] + templateRepo["Transport"] = repo["transport"] + templateRepo["RequiresBilling"] = repo["requires_billing"] + templateRepo["APIID"] = repo["api_id"] + templateRepo["RepoShort"] = repo["repo_short"] + } + } + + minJavaVersion, _ := metadata["min_java_version"].(string) + if minJavaVersion == "" { + minJavaVersion = "8" // Default to Java 8 + } + fmt.Println("DEBUG minJavaVersion:", minJavaVersion) + + samples := metadata["samples"] + if isNilOrEmpty(samples) { + extSamples, err := ExtractSamples(dir) + if err != nil { + return fmt.Errorf("failed to extract samples: %w", err) + } + if len(extSamples) > 0 { + samples = extSamples + } + } + + snippets := metadata["snippets"] + if isNilOrEmpty(snippets) { + extSnippets, err := ExtractSnippets(dir) + if err != nil { + return fmt.Errorf("failed to extract snippets: %w", err) + } + if len(extSnippets) > 0 { + snippets = extSnippets + } + } + + templateMetadata := map[string]interface{}{ + "Repo": templateRepo, + "LibraryVersion": version, + "LibrariesBOMVersion": bomVersion, + "Samples": samples, + "Snippets": snippets, + "MinJavaVersion": minJavaVersion, + } + + if len(capitalizedPartials) > 0 { + templateMetadata["Partials"] = capitalizedPartials + } + + data := struct { + Metadata map[string]interface{} + GroupID string + ArtifactID string + Version string + RepoShort string + MigratedSplitRepo bool + Monorepo bool + BOMVersion string + LibraryVersion string + }{ + Metadata: templateMetadata, + GroupID: groupID, + ArtifactID: artifactID, + Version: version, + RepoShort: repoShort, + MigratedSplitRepo: false, + Monorepo: true, + BOMVersion: bomVersion, + LibraryVersion: libraryVersion, + } + + // Read and parse template + tmplBytes, err := os.ReadFile(templatePath) + if err != nil { + return fmt.Errorf("failed to read template: %w", err) + } + + tmpl, err := template.New("README").Parse(string(tmplBytes)) + if err != nil { + return fmt.Errorf("failed to parse template: %w", err) + } + + // Execute template + var buf strings.Builder + if err := tmpl.Execute(&buf, data); err != nil { + return fmt.Errorf("failed to execute template: %w", err) + } + + // Write output + return os.WriteFile(outputPath, []byte(buf.String()), 0644) +} diff --git a/internal/librarian/java/template_render_test.go b/internal/librarian/java/template_render_test.go new file mode 100644 index 00000000000..7c51d2160d6 --- /dev/null +++ b/internal/librarian/java/template_render_test.go @@ -0,0 +1,132 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package java + +import ( + "os" + "path/filepath" + "strings" + "testing" + "text/template" +) + +func TestRenderREADME(t *testing.T) { + tmpDir := t.TempDir() + + // Create dummy template + templatePath := filepath.Join(tmpDir, "README.md.go.tmpl") + templateContent := `# Google {{ .Metadata.Repo.NamePretty }} Client for Java +Artifact: {{ .GroupID }}:{{ .ArtifactID }} +Version: {{ .Version }} +BOMVersion: {{ .BOMVersion }} +LibraryVersion: {{ .LibraryVersion }} +{{ if and .Metadata.Partials .Metadata.Partials.About }} +About: {{ .Metadata.Partials.About }} +{{ end }} +` + err := os.WriteFile(templatePath, []byte(templateContent), 0644) + if err != nil { + t.Fatal(err) + } + + // Create dummy metadata + metadataPath := filepath.Join(tmpDir, ".repo-metadata.json") + metadataContent := `{ + "repo": { + "name_pretty": "My API", + "distribution_name": "com.google.cloud:google-cloud-myapi", + "repo": "googleapis/google-cloud-java" + }, + "library_version": "1.2.3" +}` + err = os.WriteFile(metadataPath, []byte(metadataContent), 0644) + if err != nil { + t.Fatal(err) + } + + // Test case 1: Without partials + err = RenderREADME(tmpDir, templatePath, "1.0.0-BOM", "1.2.3-LIB") + if err != nil { + t.Fatal(err) + } + + outputPath := filepath.Join(tmpDir, "README.md") + outputContent, err := os.ReadFile(outputPath) + if err != nil { + t.Fatal(err) + } + + expected := `# Google My API Client for Java +Artifact: com.google.cloud:google-cloud-myapi +Version: 1.2.3 +BOMVersion: 1.0.0-BOM +LibraryVersion: 1.2.3-LIB +` + if strings.TrimSpace(string(outputContent)) != strings.TrimSpace(expected) { + t.Errorf("expected:\n%s\ngot:\n%s", expected, string(outputContent)) + } + + // Test case 2: With partials + partialsPath := filepath.Join(tmpDir, ".readme-partials.yaml") + partialsContent := `about: "This is a great API."` + err = os.WriteFile(partialsPath, []byte(partialsContent), 0644) + if err != nil { + t.Fatal(err) + } + + err = RenderREADME(tmpDir, templatePath, "1.0.0-BOM", "1.2.3-LIB") + if err != nil { + t.Fatal(err) + } + + outputContent, err = os.ReadFile(outputPath) + if err != nil { + t.Fatal(err) + } + + expectedWithPartials := `# Google My API Client for Java +Artifact: com.google.cloud:google-cloud-myapi +Version: 1.2.3 +BOMVersion: 1.0.0-BOM +LibraryVersion: 1.2.3-LIB + +About: This is a great API. +` + if strings.TrimSpace(string(outputContent)) != strings.TrimSpace(expectedWithPartials) { + t.Errorf("expected:\n%s\ngot:\n%s", expectedWithPartials, string(outputContent)) + } +} + +func TestRealTemplateParses(t *testing.T) { + // Find template path + // Current dir is internal/librarian/java when running tests in this package. + templatePath := filepath.Join("template", "README.md.tmpl") + + tmplBytes, err := os.ReadFile(templatePath) + if err != nil { + // If running from repo root, path might be different. + // Let's try to find it relative to repo root if needed. + templatePath = filepath.Join("internal", "librarian", "java", "template", "README.md.go.tmpl") + tmplBytes, err = os.ReadFile(templatePath) + if err != nil { + t.Fatal(err) + } + } + + _, err = template.New("README").Parse(string(tmplBytes)) + if err != nil { + t.Fatalf("failed to parse real template: %v", err) + } +} diff --git a/internal/postprocessing/config.go b/internal/postprocessing/config.go new file mode 100644 index 00000000000..ec0078d5a93 --- /dev/null +++ b/internal/postprocessing/config.go @@ -0,0 +1,73 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package postprocessing provides tools for postprocessing generated code. +package postprocessing + +import ( + "os" + + "gopkg.in/yaml.v3" +) + +// Config represents the postprocess.yaml configuration. +type Config struct { + Replace []ReplaceConfig `yaml:"replace,omitempty"` + ReplaceRegex []ReplaceRegexConfig `yaml:"replace_regex,omitempty"` + CopyFile []CopyConfig `yaml:"copy_file,omitempty"` + RemoveFile []string `yaml:"remove_file,omitempty"` + MethodOperations []MethodOperation `yaml:"method_operations,omitempty"` +} + +// MethodOperation represents a method-level operation like delete, duplicate, or deprecate. +type MethodOperation struct { + Path string `yaml:"path"` + Action string `yaml:"action"` + FuncName string `yaml:"func_name"` + NewName string `yaml:"new_name,omitempty"` // Used for duplicate + DeprecationMessage string `yaml:"deprecation_message,omitempty"` // Used for deprecate +} + +// ReplaceConfig represents a replacement rule. +type ReplaceConfig struct { + Path string `yaml:"path"` + Original string `yaml:"original"` + Replacement string `yaml:"replacement"` +} + +// ReplaceRegexConfig represents a regex replacement rule. +type ReplaceRegexConfig struct { + Path string `yaml:"path"` + Pattern string `yaml:"pattern"` + Replacement string `yaml:"replacement"` +} + +// CopyConfig represents a file copy rule. +type CopyConfig struct { + Src string `yaml:"src"` + Dst string `yaml:"dst"` +} + +// ParseConfig parses the postprocess.yaml file. +func ParseConfig(path string) (*Config, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var config Config + if err := yaml.Unmarshal(data, &config); err != nil { + return nil, err + } + return &config, nil +} diff --git a/internal/postprocessing/config_test.go b/internal/postprocessing/config_test.go new file mode 100644 index 00000000000..59d78ef1871 --- /dev/null +++ b/internal/postprocessing/config_test.go @@ -0,0 +1,132 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postprocessing + +import ( + "os" + "path/filepath" + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestParseConfig(t *testing.T) { + yamlContent := ` +replace: + - path: path/to/file.java + original: "old string" + replacement: "new string" +replace_regex: + - path: path/to/file.java + pattern: "pattern" + replacement: "replacement" +copy_file: + - src: path/to/src.java + dst: path/to/dst.java +remove_file: + - path/to/file_to_remove.java + +method_operations: + - path: path/to/file.java + action: delete + func_name: "public void toDelete()" + - path: path/to/file.java + action: duplicate + func_name: "public void toDuplicate()" + new_name: "duplicated" + - path: path/to/file.java + action: deprecate + func_name: "public void toDeprecate()" + deprecation_message: "Use alternative instead." +` + dir := t.TempDir() + configPath := filepath.Join(dir, "postprocess.yaml") + if err := os.WriteFile(configPath, []byte(yamlContent), 0644); err != nil { + t.Fatal(err) + } + + got, err := ParseConfig(configPath) + if err != nil { + t.Fatal(err) + } + + want := &Config{ + Replace: []ReplaceConfig{ + { + Path: "path/to/file.java", + Original: "old string", + Replacement: "new string", + }, + }, + ReplaceRegex: []ReplaceRegexConfig{ + { + Path: "path/to/file.java", + Pattern: "pattern", + Replacement: "replacement", + }, + }, + CopyFile: []CopyConfig{ + { + Src: "path/to/src.java", + Dst: "path/to/dst.java", + }, + }, + RemoveFile: []string{"path/to/file_to_remove.java"}, + + MethodOperations: []MethodOperation{ + { + Path: "path/to/file.java", + Action: "delete", + FuncName: "public void toDelete()", + }, + { + Path: "path/to/file.java", + Action: "duplicate", + FuncName: "public void toDuplicate()", + NewName: "duplicated", + }, + { + Path: "path/to/file.java", + Action: "deprecate", + FuncName: "public void toDeprecate()", + DeprecationMessage: "Use alternative instead.", + }, + }, + } + + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("ParseConfig() mismatch (-want +got):\n%s", diff) + } +} + +func TestParseConfig_FileNotFound(t *testing.T) { + _, err := ParseConfig("non-existent-file.yaml") + if err == nil { + t.Error("ParseConfig() expected error for non-existent file, got nil") + } +} + +func TestParseConfig_InvalidYAML(t *testing.T) { + dir := t.TempDir() + configPath := filepath.Join(dir, "postprocess.yaml") + if err := os.WriteFile(configPath, []byte("invalid yaml content"), 0644); err != nil { + t.Fatal(err) + } + + _, err := ParseConfig(configPath) + if err == nil { + t.Error("ParseConfig() expected error for invalid YAML, got nil") + } +} diff --git a/internal/postprocessing/fileops.go b/internal/postprocessing/fileops.go index 3ca493be699..ae51114effd 100644 --- a/internal/postprocessing/fileops.go +++ b/internal/postprocessing/fileops.go @@ -23,6 +23,7 @@ import ( "regexp" "strings" + "github.com/bmatcuk/doublestar/v4" "github.com/googleapis/librarian/internal/filesystem" ) @@ -36,6 +37,37 @@ func CopyFile(src, dst string) error { return filesystem.CopyFile(src, dst) } +// Replace replaces all instances of original with replacement in the file at path. +// It supports glob patterns if path contains glob characters. +func Replace(path, original, replacement string) error { + var files []string + var err error + + if strings.ContainsAny(path, "*?[]{}") { + files, err = doublestar.FilepathGlob(path) + if err != nil { + return err + } + } else { + files = []string{path} + } + + for _, f := range files { + content, err := os.ReadFile(f) + if err != nil { + if os.IsNotExist(err) { + continue // Silently ignore missing files, like synthtool + } + return err + } + newContent := strings.ReplaceAll(string(content), original, replacement) + if err := os.WriteFile(f, []byte(newContent), 0644); err != nil { + return err + } + } + return nil +} + // RemoveFile removes the file at the specified path. func RemoveFile(path string) error { return os.Remove(path) @@ -77,3 +109,48 @@ func ReplaceRegex(path, pattern, replacement string) error { newContent := re.ReplaceAll(content, []byte(replacement)) return os.WriteFile(path, newContent, 0644) } + +// ReplaceRegex replaces all instances of pattern with replacement in the file at path. +// It converts Python-style capture groups like \g<1> to Go-style $1. +// It supports glob patterns if path contains glob characters. +func ReplaceRegex(path, pattern, replacement string) error { + var files []string + var err error + + if strings.ContainsAny(path, "*?[]{}") { + files, err = doublestar.FilepathGlob(path) + if err != nil { + return err + } + } else { + files = []string{path} + } + + // Convert Python-style capture groups \g<1> or \g to Go-style $1 or $name + rePythonGroup := regexp.MustCompile(`\\g<(\w+)>`) + replacement = rePythonGroup.ReplaceAllString(replacement, `$$$1`) + + // Convert Python-style numeric capture groups \1, \2 to Go-style $1, $2 + rePythonNumGroup := regexp.MustCompile(`(^|[^\\])\\(\d+)`) + replacement = rePythonNumGroup.ReplaceAllString(replacement, `${1}$$${2}`) + + re, err := regexp.Compile(pattern) + if err != nil { + return err + } + + for _, f := range files { + content, err := os.ReadFile(f) + if err != nil { + if os.IsNotExist(err) { + continue // Silently ignore missing files, like synthtool + } + return err + } + newContent := re.ReplaceAllString(string(content), replacement) + if err := os.WriteFile(f, []byte(newContent), 0644); err != nil { + return err + } + } + return nil +} diff --git a/internal/postprocessing/fileops_test.go b/internal/postprocessing/fileops_test.go index 10aa4df27cb..9e0f8d3832e 100644 --- a/internal/postprocessing/fileops_test.go +++ b/internal/postprocessing/fileops_test.go @@ -95,25 +95,35 @@ func TestRemoveFile_Error(t *testing.T) { } func TestReplace(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "test.txt") - content := "Hello World" - original := "World" - replacement := "Go" - want := "Hello Go" - if err := os.WriteFile(path, []byte(content), 0644); err != nil { - t.Fatal(err) - } - if err := Replace(path, original, replacement); err != nil { - t.Fatal(err) - } - gotBytes, err := os.ReadFile(path) - if err != nil { - t.Fatal(err) - } - got := string(gotBytes) - if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("mismatch (-want +got):\n%s", diff) + t.Parallel() + for _, test := range []struct { + name string + content string + original string + replacement string + want string + }{ + {"success", "Hello World", "World", "Go", "Hello Go"}, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := filepath.Join(dir, "test.txt") + if err := os.WriteFile(path, []byte(test.content), 0644); err != nil { + t.Fatal(err) + } + if err := Replace(path, test.original, test.replacement); err != nil { + t.Fatal(err) + } + gotBytes, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + got := string(gotBytes) + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + }) } } @@ -140,6 +150,27 @@ func TestReplaceRegex(t *testing.T) { replacement: "Number", want: "Hello Number World", }, + { + name: "python style capture group", + content: "Hello World", + pattern: `(Hello) (World)`, + replacement: `\g<2> \g<1>`, + want: "World Hello", + }, + { + name: "python style capture group with named group", + content: "Hello World", + pattern: `(?PHello) (?PWorld)`, + replacement: `\g \g`, + want: "World Hello", + }, + { + name: "python style numeric capture group", + content: "Hello World", + pattern: `(Hello) (World)`, + replacement: `\2 \1`, + want: "World Hello", + }, } { t.Run(test.name, func(t *testing.T) { t.Parallel() From 8c57cfb74dc323bd737cc3fde1082d88df25f888 Mon Sep 17 00:00:00 2001 From: Sophia Yang Date: Wed, 10 Jun 2026 12:30:45 -0400 Subject: [PATCH 036/108] WIP: save all postprocessing modernization work --- internal/librarian/java/clean.go | 8 +- internal/librarian/java/postprocess.go | 19 +++-- internal/librarian/java/postprocess_new.go | 79 +++++++++++++------ .../librarian/java/postprocess_new_test.go | 13 ++- internal/librarian/java/postprocess_test.go | 15 +++- internal/librarian/java/samples.go | 15 +++- .../librarian/java/template/README.md.go.tmpl | 3 +- internal/librarian/java/template_render.go | 16 ++-- .../librarian/java/template_render_test.go | 34 +++----- internal/postprocessing/fileops.go | 4 + 10 files changed, 123 insertions(+), 83 deletions(-) diff --git a/internal/librarian/java/clean.go b/internal/librarian/java/clean.go index 96bdccd933c..044e4599221 100644 --- a/internal/librarian/java/clean.go +++ b/internal/librarian/java/clean.go @@ -17,6 +17,7 @@ package java import ( "bufio" "errors" + "fmt" "io" "io/fs" "os" @@ -33,6 +34,8 @@ import ( const ( autoGeneratedMarker = "// AUTO-GENERATED DOCUMENTATION AND CLASS." autoGeneratedAnnotation = "@Generated(\"by gapic-generator-java\")" + protobufMarker = "// Generated by the protocol buffer compiler. DO NOT EDIT!" + grpcMarker = "@io.grpc.stub.annotations.GrpcGenerated" ) var ( @@ -53,6 +56,7 @@ var ( // It targets patterns like proto-*, grpc-*, and the main GAPIC module. func Clean(library *config.Library) error { patterns := cleanPatterns(library) + fmt.Printf("Clean patterns for %s: %v\n", library.Output, patterns) keepSet := make(map[string]bool) for _, k := range library.Keep { keepSet[strings.TrimSuffix(filepath.ToSlash(k), "/")] = true @@ -141,6 +145,8 @@ func cleanPath(targetPath, root string, keepSet map[string]bool, useMarker bool) return nil } } + fmt.Println("DELETING:", path) + fmt.Println("DELETING:", path) return os.Remove(path) }) if err != nil && !errors.Is(err, fs.ErrNotExist) { @@ -231,7 +237,7 @@ func hasMarker(path string) (has bool, err error) { reader := bufio.NewReader(f) for { line, rerr := reader.ReadString('\n') - if strings.Contains(line, autoGeneratedMarker) || strings.Contains(line, autoGeneratedAnnotation) { + if strings.Contains(line, autoGeneratedMarker) || strings.Contains(line, autoGeneratedAnnotation) || strings.Contains(line, protobufMarker) || strings.Contains(line, grpcMarker) { return true, nil } if rerr != nil { diff --git a/internal/librarian/java/postprocess.go b/internal/librarian/java/postprocess.go index 768e06aee25..ac6c4297d8e 100644 --- a/internal/librarian/java/postprocess.go +++ b/internal/librarian/java/postprocess.go @@ -339,9 +339,11 @@ func restructure(actions []moveAction, keepSet map[string]bool, libraryRoot stri // Also preserve existing files that lack the auto-generated marker. destPath := filepath.Join(libraryRoot, p) if _, err := os.Stat(destPath); err == nil { - isGen, err := hasMarker(destPath) - if err == nil && !isGen { - return true + if filepath.Ext(destPath) == ".java" { + isGen, err := hasMarker(destPath) + if err == nil && !isGen { + return true + } } } return false @@ -378,11 +380,12 @@ func restructureModules(params postProcessParams, destRoot string, keepSet map[s protoFilesDestDir := filepath.Join(destRoot, coords.Proto.ArtifactID, "src", "main", "proto") if params.javaAPI.Monolithic { - protoDest = filepath.Join(destRoot, "main", "java") - grpcDest = filepath.Join(destRoot, "main", "java") - gapicMainDest = filepath.Join(destRoot, "main") - gapicTestDest = filepath.Join(destRoot, "test") - protoFilesDestDir = filepath.Join(destRoot, "main", "proto") + protoDest = filepath.Join(destRoot, "src", "main", "java") + grpcDest = filepath.Join(destRoot, "src", "main", "java") + gapicMainDest = filepath.Join(destRoot, "src", "main") + gapicTestDest = filepath.Join(destRoot, "src", "test") + protoFilesDestDir = filepath.Join(destRoot, "src", "main", "proto") + } } var actions []moveAction diff --git a/internal/librarian/java/postprocess_new.go b/internal/librarian/java/postprocess_new.go index 231c608ca60..53b7e92e00b 100644 --- a/internal/librarian/java/postprocess_new.go +++ b/internal/librarian/java/postprocess_new.go @@ -37,8 +37,16 @@ func postProcessLibraryNew(p libraryPostProcessParams) error { return fmt.Errorf("failed to parse postprocess.yaml: %w", err) } + keepSet := make(map[string]bool) + for _, k := range p.library.Keep { + keepSet[strings.TrimSuffix(filepath.ToSlash(k), "/")] = true + } + // 1. Apply Copies for _, c := range cfg.CopyFile { + if shouldPreserve(filepath.ToSlash(c.Dst), keepSet) { + continue + } srcAbs := filepath.Join(p.outDir, c.Src) dstAbs := filepath.Join(p.outDir, c.Dst) if err := filesystem.CopyFile(srcAbs, dstAbs); err != nil { @@ -48,70 +56,76 @@ func postProcessLibraryNew(p libraryPostProcessParams) error { // 2. Apply Removes for _, rem := range cfg.RemoveFile { - absPath := filepath.Join(p.outDir, rem) - if err := postprocessing.RemoveFile(absPath); err != nil { - return fmt.Errorf("failed to remove file %s: %w", rem, err) + if err := applyToFiles(p.outDir, rem, keepSet, func(file string) error { + if err := postprocessing.RemoveFile(file); err != nil { + return fmt.Errorf("failed to remove file %s: %w", file, err) + } + return nil + }); err != nil { + return err } } // 3. Apply Replacements for _, r := range cfg.Replace { - absPath := filepath.Join(p.outDir, r.Path) - if err := postprocessing.Replace(absPath, r.Original, r.Replacement); err != nil { - return fmt.Errorf("failed to apply replacement in %s: %w", r.Path, err) + if err := applyToFiles(p.outDir, r.Path, keepSet, func(file string) error { + if err := postprocessing.Replace(file, r.Original, r.Replacement); err != nil { + return fmt.Errorf("failed to apply replacement in %s: %w", file, err) + } + return nil + }); err != nil { + return err } } // 4. Apply Regex Replacements for _, r := range cfg.ReplaceRegex { - absPath := filepath.Join(p.outDir, r.Path) - if err := postprocessing.ReplaceRegex(absPath, r.Pattern, r.Replacement); err != nil { - return fmt.Errorf("failed to apply regex replacement in %s: %w", r.Path, err) + if err := applyToFiles(p.outDir, r.Path, keepSet, func(file string) error { + if err := postprocessing.ReplaceRegex(file, r.Pattern, r.Replacement); err != nil { + return fmt.Errorf("failed to apply regex replacement in %s: %w", file, err) + } + return nil + }); err != nil { + return err } } // 5. Apply Method Operations for _, mo := range cfg.MethodOperations { - files, err := resolveGlobs(p.outDir, mo.Path) - if err != nil { - return fmt.Errorf("failed to resolve glob for %s: %w", mo.Path, err) - } - for _, file := range files { + if err := applyToFiles(p.outDir, mo.Path, keepSet, func(file string) error { switch mo.Action { case "delete": if err := postprocessing.DeleteFunc(file, mo.FuncName, "java"); err != nil { if strings.Contains(err.Error(), "not found") { - continue + return nil } return fmt.Errorf("failed to delete method %q in %s: %w", mo.FuncName, file, err) } case "duplicate": if err := postprocessing.DuplicateMethod(file, mo.FuncName, mo.NewName, "java"); err != nil { if strings.Contains(err.Error(), "not found") { - continue + return nil } return fmt.Errorf("failed to duplicate method %q in %s: %w", mo.FuncName, file, err) } case "deprecate": if err := postprocessing.DeprecateMethod(file, mo.FuncName, mo.DeprecationMessage, "java"); err != nil { if strings.Contains(err.Error(), "not found") { - continue + return nil } return fmt.Errorf("failed to deprecate method %q in %s: %w", mo.FuncName, file, err) } default: return fmt.Errorf("unsupported method operation action %q", mo.Action) } + return nil + }); err != nil { + return err } } } - // 4. Render README.md - templatePath := filepath.Join("template", "README.md.go.tmpl") - if _, err := os.Stat(templatePath); err != nil { - // Fallback to absolute path for this workspace - templatePath = "/usr/local/google/home/sophieeee/workspace/owlbot-modernization/librarian/internal/librarian/java/template/README.md.go.tmpl" - } + // 6. Render README.md libraryVersion, err := deriveLastReleasedVersion(p.library.Version) if err != nil { @@ -133,7 +147,7 @@ func postProcessLibraryNew(p libraryPostProcessParams) error { return fmt.Errorf("failed to find BOM version: %w", err) } - if err := RenderREADME(p.outDir, templatePath, bomVersion, libraryVersion); err != nil { + if err := RenderREADME(p.outDir, bomVersion, libraryVersion); err != nil { return fmt.Errorf("failed to render README: %w", err) } @@ -146,3 +160,20 @@ func resolveGlobs(outDir, pathPattern string) ([]string, error) { } return []string{filepath.Join(outDir, pathPattern)}, nil } + +func applyToFiles(outDir string, pathPattern string, keepSet map[string]bool, action func(string) error) error { + files, err := resolveGlobs(outDir, pathPattern) + if err != nil { + return fmt.Errorf("failed to resolve glob for %s: %w", pathPattern, err) + } + for _, file := range files { + relPath, _ := filepath.Rel(outDir, file) + if shouldPreserve(filepath.ToSlash(relPath), keepSet) { + continue + } + if err := action(file); err != nil { + return err + } + } + return nil +} diff --git a/internal/librarian/java/postprocess_new_test.go b/internal/librarian/java/postprocess_new_test.go index 5bfe8f265a1..a03c8f74f70 100644 --- a/internal/librarian/java/postprocess_new_test.go +++ b/internal/librarian/java/postprocess_new_test.go @@ -81,14 +81,11 @@ method_operations: t.Fatal(err) } - // Setup template - if err := os.MkdirAll(filepath.Join(tmpDir, "template"), 0755); err != nil { - t.Fatal(err) - } - templateContent := `# {{ .Metadata.Repo.NamePretty }}` - if err := os.WriteFile(filepath.Join(tmpDir, "template", "README.md.go.tmpl"), []byte(templateContent), 0644); err != nil { - t.Fatal(err) - } + oldTemplate := readmeTemplate + readmeTemplate = `# {{ .Metadata.Repo.NamePretty }}` + defer func() { + readmeTemplate = oldTemplate + }() p := libraryPostProcessParams{ outDir: tmpDir, diff --git a/internal/librarian/java/postprocess_test.go b/internal/librarian/java/postprocess_test.go index fd262832099..f75315b7577 100644 --- a/internal/librarian/java/postprocess_test.go +++ b/internal/librarian/java/postprocess_test.go @@ -414,16 +414,16 @@ func TestRestructureModules_Monolithic(t *testing.T) { Monolithic: true, }, } - destRoot := filepath.Join(tmpDir, "dest", "src") + destRoot := filepath.Join(tmpDir, "dest") if err := restructureModules(params, destRoot, nil, ""); err != nil { t.Fatal(err) } // Verify all files are in the same src directory files := []string{ - filepath.Join(destRoot, "main", "java", "Gapic.java"), - filepath.Join(destRoot, "main", "java", "Grpc.java"), - filepath.Join(destRoot, "main", "java", "Proto.java"), + filepath.Join(destRoot, "src", "main", "java", "Gapic.java"), + filepath.Join(destRoot, "src", "main", "java", "Grpc.java"), + filepath.Join(destRoot, "src", "main", "java", "Proto.java"), } for _, f := range files { if _, err := os.Stat(f); err != nil { @@ -1220,6 +1220,9 @@ func TestPostProcessLibrary_Branching(t *testing.T) { p := libraryPostProcessParams{ outDir: outDir, UseGoPostprocessor: true, + metadata: &repoMetadata{ + NamePretty: "test-library", + }, cfg: &config.Config{ Default: &config.Default{ Java: &config.JavaModule{ @@ -1233,6 +1236,10 @@ func TestPostProcessLibrary_Branching(t *testing.T) { library: &config.Library{ Name: "test-library", Version: "1.2.3", + Java: &config.JavaModule{ + GroupID: "com.google.cloud", + ArtifactID: "test-library", + }, }, } err := postProcessLibrary(t.Context(), p) diff --git a/internal/librarian/java/samples.go b/internal/librarian/java/samples.go index 81b004b0ccc..1047d2985e5 100644 --- a/internal/librarian/java/samples.go +++ b/internal/librarian/java/samples.go @@ -68,10 +68,16 @@ func ExtractSamples(dir string) ([]map[string]interface{}, error) { } return err } - if d.IsDir() && d.Name() == "test" { - return filepath.SkipDir + if d.IsDir() { + if d.Name() == "test" { + return filepath.SkipDir + } + if d.Name() == "generated" && filepath.Base(filepath.Dir(path)) == "snippets" { + return filepath.SkipDir + } + return nil } - if !d.IsDir() && d.Type().IsRegular() && filepath.Ext(path) == ".java" && !strings.Contains(filepath.ToSlash(path), "/src/test/") { + if !d.IsDir() && d.Type().IsRegular() && filepath.Ext(path) == ".java" && strings.Contains(filepath.ToSlash(path), "/src/main/java/") { files = append(files, path) } return nil @@ -149,6 +155,9 @@ func ExtractSnippets(dir string) (map[string]string, error) { if d.Name() == "test" { return filepath.SkipDir } + if d.Name() == "generated" && filepath.Base(filepath.Dir(path)) == "snippets" { + return filepath.SkipDir + } return nil } if !d.Type().IsRegular() { diff --git a/internal/librarian/java/template/README.md.go.tmpl b/internal/librarian/java/template/README.md.go.tmpl index d9177ec037e..12e4433fb77 100644 --- a/internal/librarian/java/template/README.md.go.tmpl +++ b/internal/librarian/java/template/README.md.go.tmpl @@ -11,6 +11,7 @@ Java idiomatic client for [{{ .Metadata.Repo.NamePretty }}][product-docs]. {{ if and .Metadata.Partials .Metadata.Partials.DeprecationWarning }}{{ .Metadata.Partials.DeprecationWarning }} {{ end }}{{ if eq .Metadata.Repo.ReleaseLevel "preview" }}> Note: This client is a work-in-progress, and may occasionally > make backwards-incompatible changes. + {{ end }}{{ if .MigratedSplitRepo }}:bus: In October 2022, this library has moved to [google-cloud-java/{{ .Metadata.Repo.RepoShort }}]( https://github.com/googleapis/google-cloud-java/tree/main/{{ .Metadata.Repo.RepoShort }}). @@ -196,7 +197,7 @@ and on [google-cloud-java][g-c-j]. ## Versioning -{{ if and .Metadata.Partials .Metadata.Partials.Versioning }} +{{- if and .Metadata.Partials .Metadata.Partials.Versioning }} {{ .Metadata.Partials.Versioning }} {{ else }} This library follows [Semantic Versioning](http://semver.org/). diff --git a/internal/librarian/java/template_render.go b/internal/librarian/java/template_render.go index eccc4b9c635..4b84bb518e3 100644 --- a/internal/librarian/java/template_render.go +++ b/internal/librarian/java/template_render.go @@ -15,6 +15,7 @@ package java import ( + _ "embed" "encoding/json" "fmt" "os" @@ -27,6 +28,9 @@ import ( "gopkg.in/yaml.v3" ) +//go:embed template/README.md.go.tmpl +var readmeTemplate string + func isNilOrEmpty(v interface{}) bool { if v == nil { return true @@ -41,8 +45,7 @@ func isNilOrEmpty(v interface{}) bool { // RenderREADME renders the README.md file using the template and metadata. // dir is the directory containing .repo-metadata.json and where README.md will be written. -// templatePath is the path to the README.md.go.tmpl file. -func RenderREADME(dir, templatePath, bomVersion, libraryVersion string) error { +func RenderREADME(dir, bomVersion, libraryVersion string) error { metadataPath := filepath.Join(dir, ".repo-metadata.json") partialsPath := filepath.Join(dir, ".readme-partials.yaml") if _, err := os.Stat(partialsPath); os.IsNotExist(err) { @@ -210,13 +213,8 @@ func RenderREADME(dir, templatePath, bomVersion, libraryVersion string) error { LibraryVersion: libraryVersion, } - // Read and parse template - tmplBytes, err := os.ReadFile(templatePath) - if err != nil { - return fmt.Errorf("failed to read template: %w", err) - } - - tmpl, err := template.New("README").Parse(string(tmplBytes)) + // Parse template + tmpl, err := template.New("README").Parse(readmeTemplate) if err != nil { return fmt.Errorf("failed to parse template: %w", err) } diff --git a/internal/librarian/java/template_render_test.go b/internal/librarian/java/template_render_test.go index 7c51d2160d6..4f4e7010775 100644 --- a/internal/librarian/java/template_render_test.go +++ b/internal/librarian/java/template_render_test.go @@ -25,8 +25,6 @@ import ( func TestRenderREADME(t *testing.T) { tmpDir := t.TempDir() - // Create dummy template - templatePath := filepath.Join(tmpDir, "README.md.go.tmpl") templateContent := `# Google {{ .Metadata.Repo.NamePretty }} Client for Java Artifact: {{ .GroupID }}:{{ .ArtifactID }} Version: {{ .Version }} @@ -36,10 +34,11 @@ LibraryVersion: {{ .LibraryVersion }} About: {{ .Metadata.Partials.About }} {{ end }} ` - err := os.WriteFile(templatePath, []byte(templateContent), 0644) - if err != nil { - t.Fatal(err) - } + oldTemplate := readmeTemplate + readmeTemplate = templateContent + defer func() { + readmeTemplate = oldTemplate + }() // Create dummy metadata metadataPath := filepath.Join(tmpDir, ".repo-metadata.json") @@ -51,13 +50,13 @@ About: {{ .Metadata.Partials.About }} }, "library_version": "1.2.3" }` - err = os.WriteFile(metadataPath, []byte(metadataContent), 0644) + err := os.WriteFile(metadataPath, []byte(metadataContent), 0644) if err != nil { t.Fatal(err) } // Test case 1: Without partials - err = RenderREADME(tmpDir, templatePath, "1.0.0-BOM", "1.2.3-LIB") + err = RenderREADME(tmpDir, "1.0.0-BOM", "1.2.3-LIB") if err != nil { t.Fatal(err) } @@ -86,7 +85,7 @@ LibraryVersion: 1.2.3-LIB t.Fatal(err) } - err = RenderREADME(tmpDir, templatePath, "1.0.0-BOM", "1.2.3-LIB") + err = RenderREADME(tmpDir, "1.0.0-BOM", "1.2.3-LIB") if err != nil { t.Fatal(err) } @@ -110,22 +109,7 @@ About: This is a great API. } func TestRealTemplateParses(t *testing.T) { - // Find template path - // Current dir is internal/librarian/java when running tests in this package. - templatePath := filepath.Join("template", "README.md.tmpl") - - tmplBytes, err := os.ReadFile(templatePath) - if err != nil { - // If running from repo root, path might be different. - // Let's try to find it relative to repo root if needed. - templatePath = filepath.Join("internal", "librarian", "java", "template", "README.md.go.tmpl") - tmplBytes, err = os.ReadFile(templatePath) - if err != nil { - t.Fatal(err) - } - } - - _, err = template.New("README").Parse(string(tmplBytes)) + _, err := template.New("README").Parse(readmeTemplate) if err != nil { t.Fatalf("failed to parse real template: %v", err) } diff --git a/internal/postprocessing/fileops.go b/internal/postprocessing/fileops.go index ae51114effd..96d036af901 100644 --- a/internal/postprocessing/fileops.go +++ b/internal/postprocessing/fileops.go @@ -134,6 +134,10 @@ func ReplaceRegex(path, pattern, replacement string) error { rePythonNumGroup := regexp.MustCompile(`(^|[^\\])\\(\d+)`) replacement = rePythonNumGroup.ReplaceAllString(replacement, `${1}$$${2}`) + if !strings.HasPrefix(pattern, "(?") { + pattern = "(?m)" + pattern + } + re, err := regexp.Compile(pattern) if err != nil { return err From fea33cb7b66fb8765b2a6675c45fb256ed586023 Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Mon, 15 Jun 2026 14:38:13 -0400 Subject: [PATCH 037/108] chore: temporary commit --- internal/librarian/java/postprocess.go | 2 +- internal/librarian/java/postprocess_new.go | 11 ++++++----- internal/librarian/java/postprocess_new_test.go | 2 +- internal/postprocessing/config.go | 11 ++++++----- internal/postprocessing/config_test.go | 7 ++++--- 5 files changed, 18 insertions(+), 15 deletions(-) diff --git a/internal/librarian/java/postprocess.go b/internal/librarian/java/postprocess.go index ac6c4297d8e..ce334122c11 100644 --- a/internal/librarian/java/postprocess.go +++ b/internal/librarian/java/postprocess.go @@ -73,7 +73,7 @@ func postProcessLibrary(ctx context.Context, params libraryPostProcessParams) er if params.UseGoPostprocessor { yamlPath := filepath.Join(params.outDir, "postprocess.yaml") if _, err := os.Stat(yamlPath); err == nil { - if err := postProcessLibraryNew(params); err != nil { + if err := postProcessLibraryNew(ctx, params); err != nil { return err } diff --git a/internal/librarian/java/postprocess_new.go b/internal/librarian/java/postprocess_new.go index 53b7e92e00b..3cbf1de1ee3 100644 --- a/internal/librarian/java/postprocess_new.go +++ b/internal/librarian/java/postprocess_new.go @@ -15,6 +15,7 @@ package java import ( + "context" "fmt" "os" "path/filepath" @@ -28,11 +29,11 @@ import ( // postProcessLibraryNew implements the new postprocessing flow, bypassing owlbot.py. // It applies operations from postprocess.yaml, and renders the README.md directly // on the generated files in their final destinations. -func postProcessLibraryNew(p libraryPostProcessParams) error { +func postProcessLibraryNew(ctx context.Context, p libraryPostProcessParams) error { // 1. Load postprocess.yaml and apply operations postprocessYamlPath := filepath.Join(p.outDir, "postprocess.yaml") if _, err := os.Stat(postprocessYamlPath); err == nil { - cfg, err := postprocessing.ParseConfig(postprocessYamlPath) + cfg, err := postprocessing.ParseConfig(ctx, postprocessYamlPath) if err != nil { return fmt.Errorf("failed to parse postprocess.yaml: %w", err) } @@ -95,21 +96,21 @@ func postProcessLibraryNew(p libraryPostProcessParams) error { if err := applyToFiles(p.outDir, mo.Path, keepSet, func(file string) error { switch mo.Action { case "delete": - if err := postprocessing.DeleteFunc(file, mo.FuncName, "java"); err != nil { + if err := postprocessing.DeleteFunc(ctx, file, mo.FuncName, "java"); err != nil { if strings.Contains(err.Error(), "not found") { return nil } return fmt.Errorf("failed to delete method %q in %s: %w", mo.FuncName, file, err) } case "duplicate": - if err := postprocessing.DuplicateMethod(file, mo.FuncName, mo.NewName, "java"); err != nil { + if err := postprocessing.DuplicateMethod(ctx, file, mo.FuncName, mo.NewName, "java"); err != nil { if strings.Contains(err.Error(), "not found") { return nil } return fmt.Errorf("failed to duplicate method %q in %s: %w", mo.FuncName, file, err) } case "deprecate": - if err := postprocessing.DeprecateMethod(file, mo.FuncName, mo.DeprecationMessage, "java"); err != nil { + if err := postprocessing.DeprecateMethod(ctx, file, mo.FuncName, mo.DeprecationMessage, "java"); err != nil { if strings.Contains(err.Error(), "not found") { return nil } diff --git a/internal/librarian/java/postprocess_new_test.go b/internal/librarian/java/postprocess_new_test.go index a03c8f74f70..875705c97cd 100644 --- a/internal/librarian/java/postprocess_new_test.go +++ b/internal/librarian/java/postprocess_new_test.go @@ -101,7 +101,7 @@ method_operations: }, } - err := postProcessLibraryNew(p) + err := postProcessLibraryNew(t.Context(), p) if err != nil { t.Fatal(err) } diff --git a/internal/postprocessing/config.go b/internal/postprocessing/config.go index ec0078d5a93..c35d22d6804 100644 --- a/internal/postprocessing/config.go +++ b/internal/postprocessing/config.go @@ -16,9 +16,10 @@ package postprocessing import ( + "context" "os" - "gopkg.in/yaml.v3" + "github.com/googleapis/librarian/internal/yaml" ) // Config represents the postprocess.yaml configuration. @@ -60,14 +61,14 @@ type CopyConfig struct { } // ParseConfig parses the postprocess.yaml file. -func ParseConfig(path string) (*Config, error) { +func ParseConfig(ctx context.Context, path string) (*Config, error) { data, err := os.ReadFile(path) if err != nil { return nil, err } - var config Config - if err := yaml.Unmarshal(data, &config); err != nil { + configPtr, err := yaml.Unmarshal[Config](data) + if err != nil { return nil, err } - return &config, nil + return configPtr, nil } diff --git a/internal/postprocessing/config_test.go b/internal/postprocessing/config_test.go index 59d78ef1871..154800cebb6 100644 --- a/internal/postprocessing/config_test.go +++ b/internal/postprocessing/config_test.go @@ -15,6 +15,7 @@ package postprocessing import ( + "context" "os" "path/filepath" "testing" @@ -57,7 +58,7 @@ method_operations: t.Fatal(err) } - got, err := ParseConfig(configPath) + got, err := ParseConfig(context.Background(), configPath) if err != nil { t.Fatal(err) } @@ -112,7 +113,7 @@ method_operations: } func TestParseConfig_FileNotFound(t *testing.T) { - _, err := ParseConfig("non-existent-file.yaml") + _, err := ParseConfig(context.Background(), "non-existent-file.yaml") if err == nil { t.Error("ParseConfig() expected error for non-existent file, got nil") } @@ -125,7 +126,7 @@ func TestParseConfig_InvalidYAML(t *testing.T) { t.Fatal(err) } - _, err := ParseConfig(configPath) + _, err := ParseConfig(context.Background(), configPath) if err == nil { t.Error("ParseConfig() expected error for invalid YAML, got nil") } From f2e3676cb558e6f4a6118e38bbcfb26075c53888 Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Mon, 15 Jun 2026 18:36:33 -0400 Subject: [PATCH 038/108] feat(postprocessing): validate method signatures during config parsing --- internal/librarian/java/postprocess_new.go | 2 +- internal/postprocessing/config.go | 17 ++++++++++++++++ internal/postprocessing/config_test.go | 23 ++++++++++++++++++++++ 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/internal/librarian/java/postprocess_new.go b/internal/librarian/java/postprocess_new.go index 3cbf1de1ee3..7caf3261c4f 100644 --- a/internal/librarian/java/postprocess_new.go +++ b/internal/librarian/java/postprocess_new.go @@ -96,7 +96,7 @@ func postProcessLibraryNew(ctx context.Context, p libraryPostProcessParams) erro if err := applyToFiles(p.outDir, mo.Path, keepSet, func(file string) error { switch mo.Action { case "delete": - if err := postprocessing.DeleteFunc(ctx, file, mo.FuncName, "java"); err != nil { + if err := postprocessing.DeleteMethod(file, mo.FuncName, "java"); err != nil { if strings.Contains(err.Error(), "not found") { return nil } diff --git a/internal/postprocessing/config.go b/internal/postprocessing/config.go index c35d22d6804..682f8bd5eff 100644 --- a/internal/postprocessing/config.go +++ b/internal/postprocessing/config.go @@ -17,7 +17,9 @@ package postprocessing import ( "context" + "fmt" "os" + "strings" "github.com/googleapis/librarian/internal/yaml" ) @@ -60,6 +62,18 @@ type CopyConfig struct { Dst string `yaml:"dst"` } +// Validate validates the postprocess configuration. +func (c *Config) Validate() error { + for _, mo := range c.MethodOperations { + if mo.Action == "delete" { + if !strings.Contains(mo.FuncName, "(") || !strings.Contains(mo.FuncName, ")") { + return fmt.Errorf("%w: %q (must contain parameter list in parentheses)", errInvalidSignature, mo.FuncName) + } + } + } + return nil +} + // ParseConfig parses the postprocess.yaml file. func ParseConfig(ctx context.Context, path string) (*Config, error) { data, err := os.ReadFile(path) @@ -70,5 +84,8 @@ func ParseConfig(ctx context.Context, path string) (*Config, error) { if err != nil { return nil, err } + if err := configPtr.Validate(); err != nil { + return nil, fmt.Errorf("invalid config: %w", err) + } return configPtr, nil } diff --git a/internal/postprocessing/config_test.go b/internal/postprocessing/config_test.go index 154800cebb6..cd9ef27cf4c 100644 --- a/internal/postprocessing/config_test.go +++ b/internal/postprocessing/config_test.go @@ -16,6 +16,7 @@ package postprocessing import ( "context" + "errors" "os" "path/filepath" "testing" @@ -131,3 +132,25 @@ func TestParseConfig_InvalidYAML(t *testing.T) { t.Error("ParseConfig() expected error for invalid YAML, got nil") } } + +func TestParseConfig_InvalidSignature(t *testing.T) { + yamlContent := ` +method_operations: + - path: path/to/file.java + action: delete + func_name: "invalidSignatureNoParentheses" +` + dir := t.TempDir() + configPath := filepath.Join(dir, "postprocess.yaml") + if err := os.WriteFile(configPath, []byte(yamlContent), 0644); err != nil { + t.Fatal(err) + } + + _, err := ParseConfig(context.Background(), configPath) + if err == nil { + t.Fatal("ParseConfig() expected error for invalid signature, got nil") + } + if !errors.Is(err, errInvalidSignature) { + t.Errorf("expected error to wrap errInvalidSignature, got %v", err) + } +} From 845c73ea2b1c818493169ceb42f563759fc3db30 Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Mon, 15 Jun 2026 20:55:45 -0400 Subject: [PATCH 039/108] fix: revert fileops to upstream and fix postprocess syntax --- internal/librarian/java/postprocess.go | 1 - internal/postprocessing/fileops.go | 81 ------------------------- internal/postprocessing/fileops_test.go | 69 ++++++--------------- 3 files changed, 19 insertions(+), 132 deletions(-) diff --git a/internal/librarian/java/postprocess.go b/internal/librarian/java/postprocess.go index ce334122c11..6783ea114bb 100644 --- a/internal/librarian/java/postprocess.go +++ b/internal/librarian/java/postprocess.go @@ -386,7 +386,6 @@ func restructureModules(params postProcessParams, destRoot string, keepSet map[s gapicTestDest = filepath.Join(destRoot, "src", "test") protoFilesDestDir = filepath.Join(destRoot, "src", "main", "proto") } - } var actions []moveAction if shouldGenerateProto(params.javaAPI) { diff --git a/internal/postprocessing/fileops.go b/internal/postprocessing/fileops.go index 96d036af901..3ca493be699 100644 --- a/internal/postprocessing/fileops.go +++ b/internal/postprocessing/fileops.go @@ -23,7 +23,6 @@ import ( "regexp" "strings" - "github.com/bmatcuk/doublestar/v4" "github.com/googleapis/librarian/internal/filesystem" ) @@ -37,37 +36,6 @@ func CopyFile(src, dst string) error { return filesystem.CopyFile(src, dst) } -// Replace replaces all instances of original with replacement in the file at path. -// It supports glob patterns if path contains glob characters. -func Replace(path, original, replacement string) error { - var files []string - var err error - - if strings.ContainsAny(path, "*?[]{}") { - files, err = doublestar.FilepathGlob(path) - if err != nil { - return err - } - } else { - files = []string{path} - } - - for _, f := range files { - content, err := os.ReadFile(f) - if err != nil { - if os.IsNotExist(err) { - continue // Silently ignore missing files, like synthtool - } - return err - } - newContent := strings.ReplaceAll(string(content), original, replacement) - if err := os.WriteFile(f, []byte(newContent), 0644); err != nil { - return err - } - } - return nil -} - // RemoveFile removes the file at the specified path. func RemoveFile(path string) error { return os.Remove(path) @@ -109,52 +77,3 @@ func ReplaceRegex(path, pattern, replacement string) error { newContent := re.ReplaceAll(content, []byte(replacement)) return os.WriteFile(path, newContent, 0644) } - -// ReplaceRegex replaces all instances of pattern with replacement in the file at path. -// It converts Python-style capture groups like \g<1> to Go-style $1. -// It supports glob patterns if path contains glob characters. -func ReplaceRegex(path, pattern, replacement string) error { - var files []string - var err error - - if strings.ContainsAny(path, "*?[]{}") { - files, err = doublestar.FilepathGlob(path) - if err != nil { - return err - } - } else { - files = []string{path} - } - - // Convert Python-style capture groups \g<1> or \g to Go-style $1 or $name - rePythonGroup := regexp.MustCompile(`\\g<(\w+)>`) - replacement = rePythonGroup.ReplaceAllString(replacement, `$$$1`) - - // Convert Python-style numeric capture groups \1, \2 to Go-style $1, $2 - rePythonNumGroup := regexp.MustCompile(`(^|[^\\])\\(\d+)`) - replacement = rePythonNumGroup.ReplaceAllString(replacement, `${1}$$${2}`) - - if !strings.HasPrefix(pattern, "(?") { - pattern = "(?m)" + pattern - } - - re, err := regexp.Compile(pattern) - if err != nil { - return err - } - - for _, f := range files { - content, err := os.ReadFile(f) - if err != nil { - if os.IsNotExist(err) { - continue // Silently ignore missing files, like synthtool - } - return err - } - newContent := re.ReplaceAllString(string(content), replacement) - if err := os.WriteFile(f, []byte(newContent), 0644); err != nil { - return err - } - } - return nil -} diff --git a/internal/postprocessing/fileops_test.go b/internal/postprocessing/fileops_test.go index 9e0f8d3832e..10aa4df27cb 100644 --- a/internal/postprocessing/fileops_test.go +++ b/internal/postprocessing/fileops_test.go @@ -95,35 +95,25 @@ func TestRemoveFile_Error(t *testing.T) { } func TestReplace(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - content string - original string - replacement string - want string - }{ - {"success", "Hello World", "World", "Go", "Hello Go"}, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - dir := t.TempDir() - path := filepath.Join(dir, "test.txt") - if err := os.WriteFile(path, []byte(test.content), 0644); err != nil { - t.Fatal(err) - } - if err := Replace(path, test.original, test.replacement); err != nil { - t.Fatal(err) - } - gotBytes, err := os.ReadFile(path) - if err != nil { - t.Fatal(err) - } - got := string(gotBytes) - if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("mismatch (-want +got):\n%s", diff) - } - }) + dir := t.TempDir() + path := filepath.Join(dir, "test.txt") + content := "Hello World" + original := "World" + replacement := "Go" + want := "Hello Go" + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatal(err) + } + if err := Replace(path, original, replacement); err != nil { + t.Fatal(err) + } + gotBytes, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + got := string(gotBytes) + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -150,27 +140,6 @@ func TestReplaceRegex(t *testing.T) { replacement: "Number", want: "Hello Number World", }, - { - name: "python style capture group", - content: "Hello World", - pattern: `(Hello) (World)`, - replacement: `\g<2> \g<1>`, - want: "World Hello", - }, - { - name: "python style capture group with named group", - content: "Hello World", - pattern: `(?PHello) (?PWorld)`, - replacement: `\g \g`, - want: "World Hello", - }, - { - name: "python style numeric capture group", - content: "Hello World", - pattern: `(Hello) (World)`, - replacement: `\2 \1`, - want: "World Hello", - }, } { t.Run(test.name, func(t *testing.T) { t.Parallel() From 0ad8357ea85433ea89a1c8012e20598334981194 Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Thu, 18 Jun 2026 15:04:41 -0400 Subject: [PATCH 040/108] feat(internal/postprocessing): integrate robust Java deprecation logic and fix linter/compilation issues --- internal/librarian/java/postprocess_new.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/librarian/java/postprocess_new.go b/internal/librarian/java/postprocess_new.go index 7caf3261c4f..4e09e619628 100644 --- a/internal/librarian/java/postprocess_new.go +++ b/internal/librarian/java/postprocess_new.go @@ -110,7 +110,7 @@ func postProcessLibraryNew(ctx context.Context, p libraryPostProcessParams) erro return fmt.Errorf("failed to duplicate method %q in %s: %w", mo.FuncName, file, err) } case "deprecate": - if err := postprocessing.DeprecateMethod(ctx, file, mo.FuncName, mo.DeprecationMessage, "java"); err != nil { + if err := postprocessing.DeprecateMethod(file, mo.FuncName, mo.DeprecationMessage, "java"); err != nil { if strings.Contains(err.Error(), "not found") { return nil } From 4049c9c3d31f4ff436e66a8783e0b75ccd120570 Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Thu, 18 Jun 2026 16:38:22 -0400 Subject: [PATCH 041/108] feat(internal/postprocessing): validate method operations and replace configs strictly --- internal/librarian/java/postprocess_new.go | 9 ------- internal/postprocessing/config.go | 30 +++++++++++++++++++++- internal/postprocessing/fileops.go | 1 - 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/internal/librarian/java/postprocess_new.go b/internal/librarian/java/postprocess_new.go index 4e09e619628..c96f864382b 100644 --- a/internal/librarian/java/postprocess_new.go +++ b/internal/librarian/java/postprocess_new.go @@ -97,23 +97,14 @@ func postProcessLibraryNew(ctx context.Context, p libraryPostProcessParams) erro switch mo.Action { case "delete": if err := postprocessing.DeleteMethod(file, mo.FuncName, "java"); err != nil { - if strings.Contains(err.Error(), "not found") { - return nil - } return fmt.Errorf("failed to delete method %q in %s: %w", mo.FuncName, file, err) } case "duplicate": if err := postprocessing.DuplicateMethod(ctx, file, mo.FuncName, mo.NewName, "java"); err != nil { - if strings.Contains(err.Error(), "not found") { - return nil - } return fmt.Errorf("failed to duplicate method %q in %s: %w", mo.FuncName, file, err) } case "deprecate": if err := postprocessing.DeprecateMethod(file, mo.FuncName, mo.DeprecationMessage, "java"); err != nil { - if strings.Contains(err.Error(), "not found") { - return nil - } return fmt.Errorf("failed to deprecate method %q in %s: %w", mo.FuncName, file, err) } default: diff --git a/internal/postprocessing/config.go b/internal/postprocessing/config.go index 682f8bd5eff..45a5b88bf6e 100644 --- a/internal/postprocessing/config.go +++ b/internal/postprocessing/config.go @@ -17,6 +17,7 @@ package postprocessing import ( "context" + "errors" "fmt" "os" "strings" @@ -24,6 +25,8 @@ import ( "github.com/googleapis/librarian/internal/yaml" ) +var errEmptyNewName = errors.New("new method name is required and cannot be empty") + // Config represents the postprocess.yaml configuration. type Config struct { Replace []ReplaceConfig `yaml:"replace,omitempty"` @@ -64,11 +67,36 @@ type CopyConfig struct { // Validate validates the postprocess configuration. func (c *Config) Validate() error { + for _, r := range c.Replace { + if strings.TrimSpace(r.Original) == "" { + return fmt.Errorf("replace rule original text to replace cannot be empty") + } + } + for _, r := range c.ReplaceRegex { + if strings.TrimSpace(r.Pattern) == "" { + return fmt.Errorf("replace_regex rule pattern cannot be empty") + } + } for _, mo := range c.MethodOperations { - if mo.Action == "delete" { + switch mo.Action { + case "delete": if !strings.Contains(mo.FuncName, "(") || !strings.Contains(mo.FuncName, ")") { return fmt.Errorf("%w: %q (must contain parameter list in parentheses)", errInvalidSignature, mo.FuncName) } + case "duplicate": + if !strings.Contains(mo.FuncName, "(") || !strings.Contains(mo.FuncName, ")") { + return fmt.Errorf("%w: %q (must contain parameter list in parentheses)", errInvalidSignature, mo.FuncName) + } + if strings.TrimSpace(mo.NewName) == "" { + return fmt.Errorf("%w for duplicate method signature %q", errEmptyNewName, mo.FuncName) + } + case "deprecate": + if !strings.Contains(mo.FuncName, "(") || !strings.Contains(mo.FuncName, ")") { + return fmt.Errorf("%w: %q (must contain parameter list in parentheses)", errInvalidSignature, mo.FuncName) + } + if strings.TrimSpace(mo.DeprecationMessage) == "" { + return fmt.Errorf("%w for deprecating method %q", errEmptyDeprecationMessage, mo.FuncName) + } } } return nil diff --git a/internal/postprocessing/fileops.go b/internal/postprocessing/fileops.go index 3ca493be699..14b038cff6e 100644 --- a/internal/postprocessing/fileops.go +++ b/internal/postprocessing/fileops.go @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package postprocessing provides tools for the YAML-based postprocessing workflow. package postprocessing import ( From 60f00a23e9551b7d1b1d48ad3540accd313c56d1 Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Sun, 21 Jun 2026 22:47:02 -0400 Subject: [PATCH 042/108] Merge upstream/main into modernize-postprocess and fix compilation/test issues --- internal/librarian/java/postprocess_new.go | 2 +- internal/librarian/java/postprocess_new_test.go | 2 +- internal/librarian/java/postprocess_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/librarian/java/postprocess_new.go b/internal/librarian/java/postprocess_new.go index c96f864382b..2a97020e7b6 100644 --- a/internal/librarian/java/postprocess_new.go +++ b/internal/librarian/java/postprocess_new.go @@ -134,7 +134,7 @@ func postProcessLibraryNew(ctx context.Context, p libraryPostProcessParams) erro return fmt.Errorf("cfg.Default.Java is nil") } - bomVersion, err := findBOMVersion(p.cfg) + bomVersion, err := findBOMVersion(p.cfg, p.library) if err != nil { return fmt.Errorf("failed to find BOM version: %w", err) } diff --git a/internal/librarian/java/postprocess_new_test.go b/internal/librarian/java/postprocess_new_test.go index 875705c97cd..014014539c3 100644 --- a/internal/librarian/java/postprocess_new_test.go +++ b/internal/librarian/java/postprocess_new_test.go @@ -91,7 +91,7 @@ method_operations: outDir: tmpDir, cfg: &config.Config{ Default: &config.Default{ - Java: &config.JavaModule{ + Java: &config.JavaDefault{ LibrariesBOMVersion: "1.0.0", }, }, diff --git a/internal/librarian/java/postprocess_test.go b/internal/librarian/java/postprocess_test.go index f75315b7577..1618a1cb40e 100644 --- a/internal/librarian/java/postprocess_test.go +++ b/internal/librarian/java/postprocess_test.go @@ -1225,7 +1225,7 @@ func TestPostProcessLibrary_Branching(t *testing.T) { }, cfg: &config.Config{ Default: &config.Default{ - Java: &config.JavaModule{ + Java: &config.JavaDefault{ LibrariesBOMVersion: "1.0.0", }, }, From 027784dd49f87c67221be99b068a3d50d434395c Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Mon, 22 Jun 2026 10:26:47 -0400 Subject: [PATCH 043/108] refactor(java): apply Go best practices and style optimizations to README rendering and postprocessing --- internal/librarian/java/generate.go | 4 +- internal/librarian/java/postprocess.go | 8 +-- internal/librarian/java/postprocess_test.go | 4 +- internal/librarian/java/samples.go | 51 ++++++++------ internal/librarian/java/template_render.go | 77 ++++++++++----------- 5 files changed, 75 insertions(+), 69 deletions(-) diff --git a/internal/librarian/java/generate.go b/internal/librarian/java/generate.go index dbf0ef07c1b..2a3696daa47 100644 --- a/internal/librarian/java/generate.go +++ b/internal/librarian/java/generate.go @@ -117,7 +117,7 @@ func Generate(ctx context.Context, cfg *config.Config, library *config.Library, outDir: outdir, metadata: metadata, transports: transports, - UseGoPostprocessor: useGoPostprocessor, + useGoPostprocessor: useGoPostprocessor, }); err != nil { return err } @@ -148,7 +148,7 @@ func generateAPI(ctx context.Context, params generateAPIParams) error { outDir: params.outdir, apiBase: deriveAPIBase(params.library, params.api.Path), includeSamples: *javaAPI.Samples, - UseGoPostprocessor: params.useGoPostprocessor, + useGoPostprocessor: params.useGoPostprocessor, } gapicDir := postParams.gapicDir() gRPCDir := postParams.gRPCDir() diff --git a/internal/librarian/java/postprocess.go b/internal/librarian/java/postprocess.go index 6783ea114bb..19a97a09b9e 100644 --- a/internal/librarian/java/postprocess.go +++ b/internal/librarian/java/postprocess.go @@ -57,7 +57,7 @@ type postProcessParams struct { apiBase string protosToCopy []protoFileToCopy includeSamples bool - UseGoPostprocessor bool + useGoPostprocessor bool } type libraryPostProcessParams struct { @@ -66,11 +66,11 @@ type libraryPostProcessParams struct { outDir string metadata *repoMetadata transports map[string]serviceconfig.Transport - UseGoPostprocessor bool + useGoPostprocessor bool } func postProcessLibrary(ctx context.Context, params libraryPostProcessParams) error { - if params.UseGoPostprocessor { + if params.useGoPostprocessor { yamlPath := filepath.Join(params.outDir, "postprocess.yaml") if _, err := os.Stat(yamlPath); err == nil { if err := postProcessLibraryNew(ctx, params); err != nil { @@ -146,7 +146,7 @@ func postProcessAPI(ctx context.Context, params postProcessParams) error { } coords := params.coords() - if params.UseGoPostprocessor { + if params.useGoPostprocessor { yamlPath := filepath.Join(params.outDir, "postprocess.yaml") if _, err := os.Stat(yamlPath); err == nil { keepSet := make(map[string]bool) diff --git a/internal/librarian/java/postprocess_test.go b/internal/librarian/java/postprocess_test.go index 1618a1cb40e..157147a6c4d 100644 --- a/internal/librarian/java/postprocess_test.go +++ b/internal/librarian/java/postprocess_test.go @@ -1177,7 +1177,7 @@ func TestPostProcessLibrary_Branching(t *testing.T) { } p := libraryPostProcessParams{ outDir: outDir, - UseGoPostprocessor: true, + useGoPostprocessor: true, cfg: &config.Config{ Libraries: []*config.Library{ {Name: "google-cloud-java", Version: "1.2.3"}, @@ -1219,7 +1219,7 @@ func TestPostProcessLibrary_Branching(t *testing.T) { p := libraryPostProcessParams{ outDir: outDir, - UseGoPostprocessor: true, + useGoPostprocessor: true, metadata: &repoMetadata{ NamePretty: "test-library", }, diff --git a/internal/librarian/java/samples.go b/internal/librarian/java/samples.go index 1047d2985e5..61abe27eaf2 100644 --- a/internal/librarian/java/samples.go +++ b/internal/librarian/java/samples.go @@ -25,7 +25,7 @@ import ( "strings" "unicode" - "gopkg.in/yaml.v3" + "github.com/googleapis/librarian/internal/yaml" ) var ( @@ -110,28 +110,37 @@ func ExtractSamples(dir string) ([]map[string]interface{}, error) { } contentBytes, err := os.ReadFile(file) - if err == nil { - if match := reMetadataBlock.FindString(string(contentBytes)); match != "" { - cleaned := reCommentPrefix.ReplaceAllString(match, "") - var meta map[string]map[string]interface{} - if err := yaml.Unmarshal([]byte(cleaned), &meta); err == nil { - if sm, ok := meta["sample-metadata"]; ok { - for k, v := range sm { - item[k] = v - if len(k) > 0 { - upperKey := strings.ToUpper(k[:1]) + k[1:] - item[upperKey] = v - } - } - if t, ok := sm["title"].(string); ok && strings.TrimSpace(t) != "" { - item["Title"] = t - item["title"] = t - } - } - } + if err != nil { + samples = append(samples, item) + continue + } + match := reMetadataBlock.FindString(string(contentBytes)) + if match == "" { + samples = append(samples, item) + continue + } + cleaned := reCommentPrefix.ReplaceAllString(match, "") + meta, err := yaml.Unmarshal[map[string]map[string]interface{}]([]byte(cleaned)) + if err != nil { + samples = append(samples, item) + continue + } + sm, ok := (*meta)["sample-metadata"] + if !ok { + samples = append(samples, item) + continue + } + for k, v := range sm { + item[k] = v + if len(k) > 0 { + upperKey := strings.ToUpper(k[:1]) + k[1:] + item[upperKey] = v } } - + if t, ok := sm["title"].(string); ok && strings.TrimSpace(t) != "" { + item["Title"] = t + item["title"] = t + } samples = append(samples, item) } return samples, nil diff --git a/internal/librarian/java/template_render.go b/internal/librarian/java/template_render.go index 4b84bb518e3..24f19a80df3 100644 --- a/internal/librarian/java/template_render.go +++ b/internal/librarian/java/template_render.go @@ -20,12 +20,11 @@ import ( "fmt" "os" "path/filepath" - "reflect" "strings" "text/template" + "github.com/googleapis/librarian/internal/yaml" "github.com/iancoleman/strcase" - "gopkg.in/yaml.v3" ) //go:embed template/README.md.go.tmpl @@ -35,14 +34,33 @@ func isNilOrEmpty(v interface{}) bool { if v == nil { return true } - rv := reflect.ValueOf(v) - switch rv.Kind() { - case reflect.Slice, reflect.Map, reflect.Array, reflect.String: - return rv.Len() == 0 + switch val := v.(type) { + case string: + return val == "" + case []interface{}: + return len(val) == 0 + case []map[string]interface{}: + return len(val) == 0 + case map[string]interface{}: + return len(val) == 0 + case map[string]string: + return len(val) == 0 } return false } +// getMetadataField attempts to retrieve a field from the top level of metadata, +// falling back to the nested "repo" map if it exists. +func getMetadataField(metadata map[string]interface{}, key string) interface{} { + if val, ok := metadata[key]; ok && val != nil { + return val + } + if repo, ok := metadata["repo"].(map[string]interface{}); ok { + return repo[key] + } + return nil +} + // RenderREADME renders the README.md file using the template and metadata. // dir is the directory containing .repo-metadata.json and where README.md will be written. func RenderREADME(dir, bomVersion, libraryVersion string) error { @@ -65,15 +83,17 @@ func RenderREADME(dir, bomVersion, libraryVersion string) error { } // Read partials if exist - partials := make(map[string]interface{}) + var partials map[string]interface{} if _, err := os.Stat(partialsPath); err == nil { partialsBytes, err := os.ReadFile(partialsPath) if err != nil { return fmt.Errorf("failed to read partials: %w", err) } - if err := yaml.Unmarshal(partialsBytes, &partials); err != nil { + p, err := yaml.Unmarshal[map[string]interface{}](partialsBytes) + if err != nil { return fmt.Errorf("failed to unmarshal partials: %w", err) } + partials = *p } // Capitalize keys of partials for template @@ -116,38 +136,15 @@ func RenderREADME(dir, bomVersion, libraryVersion string) error { version = libraryVersion } - // Construct the nested Metadata object for the template - templateRepo := map[string]interface{}{ - "NamePretty": metadata["name_pretty"], - "DistributionName": metadata["distribution_name"], - "Repo": metadata["repo"], - "APIShortname": metadata["api_shortname"], - "APIDescription": metadata["api_description"], - "ClientDocumentation": metadata["client_documentation"], - "ProductDocumentation": metadata["product_documentation"], - "ReleaseLevel": metadata["release_level"], - "Transport": metadata["transport"], - "RequiresBilling": metadata["requires_billing"], - "APIID": metadata["api_id"], - "RepoShort": metadata["repo_short"], - } - - // If flat structure failed, try to get from nested repo - if templateRepo["NamePretty"] == nil { - if repo, ok := metadata["repo"].(map[string]interface{}); ok { - templateRepo["NamePretty"] = repo["name_pretty"] - templateRepo["DistributionName"] = repo["distribution_name"] - templateRepo["Repo"] = repo["repo"] - templateRepo["APIShortname"] = repo["api_shortname"] - templateRepo["APIDescription"] = repo["api_description"] - templateRepo["ClientDocumentation"] = repo["client_documentation"] - templateRepo["ProductDocumentation"] = repo["product_documentation"] - templateRepo["ReleaseLevel"] = repo["release_level"] - templateRepo["Transport"] = repo["transport"] - templateRepo["RequiresBilling"] = repo["requires_billing"] - templateRepo["APIID"] = repo["api_id"] - templateRepo["RepoShort"] = repo["repo_short"] - } + // Construct the nested Metadata object for the template using our helper + fields := []string{ + "name_pretty", "distribution_name", "repo", "api_shortname", + "api_description", "client_documentation", "product_documentation", + "release_level", "transport", "requires_billing", "api_id", "repo_short", + } + templateRepo := make(map[string]interface{}) + for _, f := range fields { + templateRepo[strcase.ToCamel(f)] = getMetadataField(metadata, f) } minJavaVersion, _ := metadata["min_java_version"].(string) From a8b2d7f950a8954602ac6234648fdc7561f8c821 Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Mon, 22 Jun 2026 10:59:50 -0400 Subject: [PATCH 044/108] feat(java): load README template from disk dynamically instead of embedding --- .../librarian/java/postprocess_new_test.go | 13 ++++++++----- internal/librarian/java/template_render.go | 14 ++++++++------ .../librarian/java/template_render_test.go | 19 +++++++++++++------ 3 files changed, 29 insertions(+), 17 deletions(-) diff --git a/internal/librarian/java/postprocess_new_test.go b/internal/librarian/java/postprocess_new_test.go index 014014539c3..3279f7c8ad3 100644 --- a/internal/librarian/java/postprocess_new_test.go +++ b/internal/librarian/java/postprocess_new_test.go @@ -81,11 +81,14 @@ method_operations: t.Fatal(err) } - oldTemplate := readmeTemplate - readmeTemplate = `# {{ .Metadata.Repo.NamePretty }}` - defer func() { - readmeTemplate = oldTemplate - }() + // Write mock template to disk + tmplDir := filepath.Join(tmpDir, "template") + if err := os.MkdirAll(tmplDir, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(tmplDir, "README.md.go.tmpl"), []byte(`# {{ .Metadata.Repo.NamePretty }}`), 0644); err != nil { + t.Fatal(err) + } p := libraryPostProcessParams{ outDir: tmpDir, diff --git a/internal/librarian/java/template_render.go b/internal/librarian/java/template_render.go index 24f19a80df3..af74925f433 100644 --- a/internal/librarian/java/template_render.go +++ b/internal/librarian/java/template_render.go @@ -15,7 +15,6 @@ package java import ( - _ "embed" "encoding/json" "fmt" "os" @@ -27,9 +26,6 @@ import ( "github.com/iancoleman/strcase" ) -//go:embed template/README.md.go.tmpl -var readmeTemplate string - func isNilOrEmpty(v interface{}) bool { if v == nil { return true @@ -210,8 +206,14 @@ func RenderREADME(dir, bomVersion, libraryVersion string) error { LibraryVersion: libraryVersion, } - // Parse template - tmpl, err := template.New("README").Parse(readmeTemplate) + // Read and parse template from disk + templatePath := filepath.Join(dir, "template", "README.md.go.tmpl") + tmplBytes, err := os.ReadFile(templatePath) + if err != nil { + return fmt.Errorf("failed to read template from %s: %w", templatePath, err) + } + + tmpl, err := template.New("README").Parse(string(tmplBytes)) if err != nil { return fmt.Errorf("failed to parse template: %w", err) } diff --git a/internal/librarian/java/template_render_test.go b/internal/librarian/java/template_render_test.go index 4f4e7010775..233229a2ced 100644 --- a/internal/librarian/java/template_render_test.go +++ b/internal/librarian/java/template_render_test.go @@ -34,11 +34,14 @@ LibraryVersion: {{ .LibraryVersion }} About: {{ .Metadata.Partials.About }} {{ end }} ` - oldTemplate := readmeTemplate - readmeTemplate = templateContent - defer func() { - readmeTemplate = oldTemplate - }() + // Write mock template to disk + tmplDir := filepath.Join(tmpDir, "template") + if err := os.MkdirAll(tmplDir, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(tmplDir, "README.md.go.tmpl"), []byte(templateContent), 0644); err != nil { + t.Fatal(err) + } // Create dummy metadata metadataPath := filepath.Join(tmpDir, ".repo-metadata.json") @@ -109,7 +112,11 @@ About: This is a great API. } func TestRealTemplateParses(t *testing.T) { - _, err := template.New("README").Parse(readmeTemplate) + tmplBytes, err := os.ReadFile("template/README.md.go.tmpl") + if err != nil { + t.Fatalf("failed to read real template: %v", err) + } + _, err = template.New("README").Parse(string(tmplBytes)) if err != nil { t.Fatalf("failed to parse real template: %v", err) } From be77556acb63836a727548f18a14e028d286848e Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Tue, 23 Jun 2026 10:21:31 -0400 Subject: [PATCH 045/108] refactor(java): render README using typed repoMetadata struct directly --- internal/librarian/java/postprocess_new.go | 2 +- .../librarian/java/postprocess_new_test.go | 18 +-- internal/librarian/java/template_render.go | 110 +++--------------- .../librarian/java/template_render_test.go | 25 ++-- 4 files changed, 29 insertions(+), 126 deletions(-) diff --git a/internal/librarian/java/postprocess_new.go b/internal/librarian/java/postprocess_new.go index 2a97020e7b6..c255984d657 100644 --- a/internal/librarian/java/postprocess_new.go +++ b/internal/librarian/java/postprocess_new.go @@ -139,7 +139,7 @@ func postProcessLibraryNew(ctx context.Context, p libraryPostProcessParams) erro return fmt.Errorf("failed to find BOM version: %w", err) } - if err := RenderREADME(p.outDir, bomVersion, libraryVersion); err != nil { + if err := RenderREADME(p.outDir, p.metadata, bomVersion, libraryVersion); err != nil { return fmt.Errorf("failed to render README: %w", err) } diff --git a/internal/librarian/java/postprocess_new_test.go b/internal/librarian/java/postprocess_new_test.go index 3279f7c8ad3..a363bf440a4 100644 --- a/internal/librarian/java/postprocess_new_test.go +++ b/internal/librarian/java/postprocess_new_test.go @@ -68,19 +68,6 @@ method_operations: t.Fatal(err) } - // Setup .repo-metadata.json - metadata := `{ - "repo": { - "name_pretty": "My API", - "distribution_name": "com.google.cloud:google-cloud-myapi", - "repo": "googleapis/google-cloud-java" - }, - "library_version": "1.2.3" -}` - if err := os.WriteFile(filepath.Join(tmpDir, ".repo-metadata.json"), []byte(metadata), 0644); err != nil { - t.Fatal(err) - } - // Write mock template to disk tmplDir := filepath.Join(tmpDir, "template") if err := os.MkdirAll(tmplDir, 0755); err != nil { @@ -102,6 +89,11 @@ method_operations: library: &config.Library{ Version: "1.2.3", }, + metadata: &repoMetadata{ + NamePretty: "My API", + DistributionName: "com.google.cloud:google-cloud-myapi", + Repo: "googleapis/google-cloud-java", + }, } err := postProcessLibraryNew(t.Context(), p) diff --git a/internal/librarian/java/template_render.go b/internal/librarian/java/template_render.go index af74925f433..68ba569b581 100644 --- a/internal/librarian/java/template_render.go +++ b/internal/librarian/java/template_render.go @@ -15,7 +15,6 @@ package java import ( - "encoding/json" "fmt" "os" "path/filepath" @@ -26,58 +25,15 @@ import ( "github.com/iancoleman/strcase" ) -func isNilOrEmpty(v interface{}) bool { - if v == nil { - return true - } - switch val := v.(type) { - case string: - return val == "" - case []interface{}: - return len(val) == 0 - case []map[string]interface{}: - return len(val) == 0 - case map[string]interface{}: - return len(val) == 0 - case map[string]string: - return len(val) == 0 - } - return false -} - -// getMetadataField attempts to retrieve a field from the top level of metadata, -// falling back to the nested "repo" map if it exists. -func getMetadataField(metadata map[string]interface{}, key string) interface{} { - if val, ok := metadata[key]; ok && val != nil { - return val - } - if repo, ok := metadata["repo"].(map[string]interface{}); ok { - return repo[key] - } - return nil -} - // RenderREADME renders the README.md file using the template and metadata. -// dir is the directory containing .repo-metadata.json and where README.md will be written. -func RenderREADME(dir, bomVersion, libraryVersion string) error { - metadataPath := filepath.Join(dir, ".repo-metadata.json") +// dir is the directory containing where README.md will be written. +func RenderREADME(dir string, metadata *repoMetadata, bomVersion, libraryVersion string) error { partialsPath := filepath.Join(dir, ".readme-partials.yaml") if _, err := os.Stat(partialsPath); os.IsNotExist(err) { partialsPath = filepath.Join(dir, ".readme-partials.yml") } outputPath := filepath.Join(dir, "README.md") - // Read metadata - metadataBytes, err := os.ReadFile(metadataPath) - if err != nil { - return fmt.Errorf("failed to read metadata: %w", err) - } - - var metadata map[string]interface{} - if err := json.Unmarshal(metadataBytes, &metadata); err != nil { - return fmt.Errorf("failed to unmarshal metadata: %w", err) - } - // Read partials if exist var partials map[string]interface{} if _, err := os.Stat(partialsPath); err == nil { @@ -99,12 +55,7 @@ func RenderREADME(dir, bomVersion, libraryVersion string) error { } // Prepare data for template - distName, _ := metadata["distribution_name"].(string) - if distName == "" { - if repo, ok := metadata["repo"].(map[string]interface{}); ok { - distName, _ = repo["distribution_name"].(string) - } - } + distName := metadata.DistributionName distParts := strings.Split(distName, ":") groupID := "" artifactID := "" @@ -115,64 +66,33 @@ func RenderREADME(dir, bomVersion, libraryVersion string) error { artifactID = distParts[1] } - repoName, _ := metadata["repo"].(string) - if repoName == "" { - if repo, ok := metadata["repo"].(map[string]interface{}); ok { - repoName, _ = repo["repo"].(string) - } - } + repoName := metadata.Repo repoParts := strings.Split(repoName, "/") repoShort := "" if len(repoParts) > 0 { repoShort = repoParts[len(repoParts)-1] } - version, _ := metadata["library_version"].(string) - if version == "" { - version = libraryVersion - } + version := libraryVersion - // Construct the nested Metadata object for the template using our helper - fields := []string{ - "name_pretty", "distribution_name", "repo", "api_shortname", - "api_description", "client_documentation", "product_documentation", - "release_level", "transport", "requires_billing", "api_id", "repo_short", - } - templateRepo := make(map[string]interface{}) - for _, f := range fields { - templateRepo[strcase.ToCamel(f)] = getMetadataField(metadata, f) - } - - minJavaVersion, _ := metadata["min_java_version"].(string) - if minJavaVersion == "" { - minJavaVersion = "8" // Default to Java 8 + minJavaVersion := metadata.MinJavaVersion + if minJavaVersion == 0 { + minJavaVersion = 8 // Default to Java 8 } fmt.Println("DEBUG minJavaVersion:", minJavaVersion) - samples := metadata["samples"] - if isNilOrEmpty(samples) { - extSamples, err := ExtractSamples(dir) - if err != nil { - return fmt.Errorf("failed to extract samples: %w", err) - } - if len(extSamples) > 0 { - samples = extSamples - } + samples, err := ExtractSamples(dir) + if err != nil { + return fmt.Errorf("failed to extract samples: %w", err) } - snippets := metadata["snippets"] - if isNilOrEmpty(snippets) { - extSnippets, err := ExtractSnippets(dir) - if err != nil { - return fmt.Errorf("failed to extract snippets: %w", err) - } - if len(extSnippets) > 0 { - snippets = extSnippets - } + snippets, err := ExtractSnippets(dir) + if err != nil { + return fmt.Errorf("failed to extract snippets: %w", err) } templateMetadata := map[string]interface{}{ - "Repo": templateRepo, + "Repo": metadata, "LibraryVersion": version, "LibrariesBOMVersion": bomVersion, "Samples": samples, diff --git a/internal/librarian/java/template_render_test.go b/internal/librarian/java/template_render_test.go index 233229a2ced..fc57b79993b 100644 --- a/internal/librarian/java/template_render_test.go +++ b/internal/librarian/java/template_render_test.go @@ -43,23 +43,14 @@ About: {{ .Metadata.Partials.About }} t.Fatal(err) } - // Create dummy metadata - metadataPath := filepath.Join(tmpDir, ".repo-metadata.json") - metadataContent := `{ - "repo": { - "name_pretty": "My API", - "distribution_name": "com.google.cloud:google-cloud-myapi", - "repo": "googleapis/google-cloud-java" - }, - "library_version": "1.2.3" -}` - err := os.WriteFile(metadataPath, []byte(metadataContent), 0644) - if err != nil { - t.Fatal(err) + metadata := &repoMetadata{ + NamePretty: "My API", + DistributionName: "com.google.cloud:google-cloud-myapi", + Repo: "googleapis/google-cloud-java", } // Test case 1: Without partials - err = RenderREADME(tmpDir, "1.0.0-BOM", "1.2.3-LIB") + err := RenderREADME(tmpDir, metadata, "1.0.0-BOM", "1.2.3-LIB") if err != nil { t.Fatal(err) } @@ -72,7 +63,7 @@ About: {{ .Metadata.Partials.About }} expected := `# Google My API Client for Java Artifact: com.google.cloud:google-cloud-myapi -Version: 1.2.3 +Version: 1.2.3-LIB BOMVersion: 1.0.0-BOM LibraryVersion: 1.2.3-LIB ` @@ -88,7 +79,7 @@ LibraryVersion: 1.2.3-LIB t.Fatal(err) } - err = RenderREADME(tmpDir, "1.0.0-BOM", "1.2.3-LIB") + err = RenderREADME(tmpDir, metadata, "1.0.0-BOM", "1.2.3-LIB") if err != nil { t.Fatal(err) } @@ -100,7 +91,7 @@ LibraryVersion: 1.2.3-LIB expectedWithPartials := `# Google My API Client for Java Artifact: com.google.cloud:google-cloud-myapi -Version: 1.2.3 +Version: 1.2.3-LIB BOMVersion: 1.0.0-BOM LibraryVersion: 1.2.3-LIB From e347a8898546b379617abe824e3219442a53b767 Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Sat, 27 Jun 2026 16:01:59 -0400 Subject: [PATCH 046/108] feat(internal/librarian/java): integrate yaml metadata extraction and snippets Integrates full sample and snippet extraction logic into readme.go, removing redundant samples.go. Updates tests to use cmp.Diff. --- cmd/librarian/doc.go | 157 ------------- internal/librarian/java/readme.go | 252 ++++++++++++++++++++- internal/librarian/java/readme_test.go | 241 ++++++++++++++------ internal/librarian/java/samples.go | 278 ------------------------ internal/librarian/java/samples_test.go | 164 -------------- 5 files changed, 421 insertions(+), 671 deletions(-) delete mode 100644 internal/librarian/java/samples.go delete mode 100644 internal/librarian/java/samples_test.go diff --git a/cmd/librarian/doc.go b/cmd/librarian/doc.go index 3b3293559c5..8d8a24b2d3f 100644 --- a/cmd/librarian/doc.go +++ b/cmd/librarian/doc.go @@ -31,162 +31,5 @@ Usage: Global flags: --verbose, -v enable verbose logging - -# Read and write librarian.yaml configuration - -Usage: - - librarian config [get|set] [path] [value] - -# Get a configuration value - -Usage: - - librarian config get [path] - -# Set a configuration value - -Usage: - - librarian config set [path] [value] - -# Add a new client library - -Usage: - - librarian add - -add registers a single API in librarian.yaml. - -The is a path within the configured googleapis source, such as -"google/cloud/secretmanager/v1". The library name and other defaults are -derived from the first API path using language-specific rules. - -If the API path should naturally be included in an existing library, and if the -language supports doing so, that library is modified. Otherwise, a new library -is created. - -While release-please is responsible for library releases, the relevant -release-please configuration will be updated as necessary to onboard any new -library. - -To add a preview client of an existing library, prefix the API path with -"preview/". - -Examples: - - librarian add google/cloud/secretmanager/v1 - librarian add preview/google/cloud/secretmanager/v1beta - -A typical librarian workflow for adding a new client library is: - - librarian add # onboard a new API into librarian.yaml - librarian generate # generate the client library - -# Generate a client library - -Usage: - - librarian generate - -generate produces client library code from the APIs configured in -librarian.yaml. - -The library name argument selects a single library to regenerate. Use the ---all flag to regenerate every library in the workspace instead. Exactly -one of or --all must be provided. - -Generation is delegated to the language-specific tooling configured in -librarian.yaml. Libraries marked with skip_generate are skipped. - -Examples: - - librarian generate # regenerate one library - librarian generate --all # regenerate every library - -Flags: - - --all generate all libraries - --use-go-postprocessor use the new Go postprocessor for Java libraries - -A typical librarian workflow for regenerating every library against the -latest API definitions is: - - librarian update googleapis - librarian generate --all - -# Install tool dependencies for a language - -Usage: - - librarian install [language] - -install installs the language-specific tools that librarian uses to -generate and build client libraries (for example, language SDKs and code -generators). - -If [language] is omitted, the language is read from librarian.yaml in the -current directory. - -Examples: - - librarian install # use language from librarian.yaml - librarian install go # install Go-specific tools - -# Tidy and validate librarian.yaml - -Usage: - - librarian tidy - -tidy reads librarian.yaml, validates its contents, applies any -language-specific defaults and normalization, and writes the file back -with a canonical formatting. - -Run tidy after editing librarian.yaml by hand, or as a quick check that -the configuration is well-formed. - -# Update sources or version to the latest version - -Usage: - - librarian update ... - -update refreshes the upstream source repositories declared in -librarian.yaml to their latest commits and updates the recorded commit -SHAs in librarian.yaml accordingly. It also supports updating the librarian version. - -Supported targets: - - - sources.conformance: protocolbuffers/protobuf conformance tests - - sources.discovery: googleapis/discovery-artifact-manager - - sources.googleapis: googleapis/googleapis (the API definitions) - - sources.protobuf: protocolbuffers/protobuf - - sources.showcase: googleapis/gapic-showcase - - version: the librarian tool version - -At least one target must be specified. - -Examples: - - librarian update sources.googleapis - librarian update sources.googleapis sources.protobuf - librarian update version - -A typical librarian workflow for regenerating every library against the -latest API definitions is: - - librarian update sources.googleapis - librarian generate --all - -# Print the binary version - -Usage: - - librarian version - -version prints the librarian binary version and exits. The version is -embedded at build time and follows the conventions described at -https://go.dev/ref/mod#versions. */ package main diff --git a/internal/librarian/java/readme.go b/internal/librarian/java/readme.go index 9a7db30a34b..9cc7a4c8f28 100644 --- a/internal/librarian/java/readme.go +++ b/internal/librarian/java/readme.go @@ -15,15 +15,31 @@ package java import ( + "bufio" "errors" + "fmt" + "io/fs" + "os" "path/filepath" "regexp" + "sort" "strings" + "unicode" + + "github.com/googleapis/librarian/internal/yaml" ) var ( - // Matches lowercase/digit followed by uppercase (e.g., "FooBar" -> "Foo Bar"). - camelCaseRegexp = regexp.MustCompile(`([a-z0-9])([A-Z])`) + openSnippetRegex = regexp.MustCompile(`\[START ([a-zA-Z0-9_-]+)\]`) + closeSnippetRegex = regexp.MustCompile(`\[END ([a-zA-Z0-9_-]+)\]`) + openExcludeRegex = regexp.MustCompile(`\[START_EXCLUDE\]`) + closeExcludeRegex = regexp.MustCompile(`\[END_EXCLUDE\]`) + + reMetadataBlock = regexp.MustCompile(`(?m)^[ \t]*//[ \t]*sample-metadata:([^\n]+|\n[ \t]*//)+`) + reCommentPrefix = regexp.MustCompile(`(?m)^[ \t]*(?:#|//)[ \t]?`) + + reDecamelize1 = regexp.MustCompile(`([A-Z]+)([A-Z])([a-z0-9])`) + reDecamelize2 = regexp.MustCompile(`([a-z0-9])([A-Z])`) // reTitle matches a "sample-metadata:" marker followed immediately by a "title:" line on the next comment line. reTitle = regexp.MustCompile(`sample-metadata:\s*\n\s*(?://|#)\s*title:\s*(.*)`) @@ -35,9 +51,18 @@ var ( errEmptyTitle = errors.New("title value cannot be empty") ) -// decamelize converts CamelCase string to space-separated string (e.g. "CamelCase" -> "Camel Case"). +// decamelize converts CamelCase or PascalCase titles into space-separated words, +// exactly reproducing Python synthtool's _decamelize(value: str). func decamelize(value string) string { - return strings.TrimSpace(camelCaseRegexp.ReplaceAllString(value, `$1 $2`)) + if value == "" { + return "" + } + r := []rune(value) + r[0] = unicode.ToUpper(r[0]) + s := string(r) + + s = reDecamelize1.ReplaceAllString(s, "${1} ${2}${3}") + return reDecamelize2.ReplaceAllString(s, "${1} ${2}") } // isProductionSample reports whether the given path represents a production Java source file @@ -66,3 +91,222 @@ func extractTitle(content string) (string, error) { } return title, nil } + +// ExtractSamples walks the "samples" directory locating all .java source files. +// It parses embedded multiline "// sample-metadata:" YAML blocks to derive title and path metadata. +func ExtractSamples(dir string) ([]map[string]interface{}, error) { + samplesDir := filepath.Join(dir, "samples") + var files []string + + err := filepath.WalkDir(samplesDir, func(path string, d os.DirEntry, err error) error { + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil + } + return err + } + if d.IsDir() { + if d.Name() == "test" { + return filepath.SkipDir + } + if d.Name() == "generated" && filepath.Base(filepath.Dir(path)) == "snippets" { + return filepath.SkipDir + } + return nil + } + if d.Type().IsRegular() && isProductionSample(path) { + files = append(files, path) + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("failed to walk samples directory: %w", err) + } + + if len(files) == 0 { + return nil, nil + } + + sort.Strings(files) + var samples []map[string]interface{} + + for _, file := range files { + rel, err := filepath.Rel(dir, file) + if err != nil { + continue + } + base := strings.TrimSuffix(filepath.Base(file), ".java") + title := decamelize(base) + + slashPath := filepath.ToSlash(rel) + item := map[string]interface{}{ + "Title": title, + "File": slashPath, + "title": title, + "file": slashPath, + } + + contentBytes, err := os.ReadFile(file) + if err != nil { + return nil, fmt.Errorf("failed to read sample file %s: %w", file, err) + } + matchBytes := reMetadataBlock.Find(contentBytes) + if matchBytes == nil { + samples = append(samples, item) + continue + } + cleanedBytes := reCommentPrefix.ReplaceAll(matchBytes, nil) + meta, err := yaml.Unmarshal[map[string]map[string]interface{}](cleanedBytes) + if err != nil { + return nil, fmt.Errorf("failed to parse YAML metadata in %s: %w", file, err) + } + sm, ok := (*meta)["sample-metadata"] + if !ok { + return nil, fmt.Errorf("missing 'sample-metadata' key in YAML of %s", file) + } + for k, v := range sm { + item[k] = v + if len(k) > 0 { + upperKey := strings.ToUpper(k[:1]) + k[1:] + item[upperKey] = v + } + } + if t, ok := sm["title"].(string); ok && strings.TrimSpace(t) != "" { + item["Title"] = t + item["title"] = t + } + samples = append(samples, item) + } + return samples, nil +} + +// ExtractSnippets walks the "samples" directory locating *.java and *.xml files. +// It line-scans for [START name] and [END name] tags while supporting [START_EXCLUDE] blocks, +// returning trimmed minimum plain-space indentation blocks. +func ExtractSnippets(dir string) (map[string]string, error) { + samplesDir := filepath.Join(dir, "samples") + var files []string + + err := filepath.WalkDir(samplesDir, func(path string, d os.DirEntry, err error) error { + if err != nil { + if errors.Is(err, fs.ErrNotExist) || os.IsNotExist(err) { + return nil + } + return err + } + if d.IsDir() { + if d.Name() == "test" { + return filepath.SkipDir + } + if d.Name() == "generated" && filepath.Base(filepath.Dir(path)) == "snippets" { + return filepath.SkipDir + } + return nil + } + if !d.Type().IsRegular() { + return nil + } + ext := filepath.Ext(path) + if ext == ".java" || ext == ".xml" { + files = append(files, path) + } + return nil + }) + if err != nil { + return nil, err + } + + if len(files) == 0 { + return nil, nil + } + + sort.Strings(files) + snippetLines := make(map[string][]string) + + for _, file := range files { + f, err := os.Open(file) + if err != nil { + continue + } + + openSnippets := make(map[string]bool) + excluding := false + scanner := bufio.NewScanner(f) + scanner.Buffer(make([]byte, 64*1024), 10*1024*1024) + + for scanner.Scan() { + line := scanner.Text() + openMatch := openSnippetRegex.FindStringSubmatch(line) + closeMatch := closeSnippetRegex.FindStringSubmatch(line) + + if len(openMatch) > 1 && !excluding { + name := openMatch[1] + openSnippets[name] = true + if _, exists := snippetLines[name]; !exists { + snippetLines[name] = []string{} + } + } else if len(closeMatch) > 1 && !excluding { + delete(openSnippets, closeMatch[1]) + } else if openExcludeRegex.MatchString(line) { + excluding = true + } else if closeExcludeRegex.MatchString(line) { + excluding = false + } else if !excluding { + for s := range openSnippets { + snippetLines[s] = append(snippetLines[s], line) + } + } + } + if err := scanner.Err(); err != nil { + f.Close() + return nil, err + } + f.Close() + } + + if len(snippetLines) == 0 { + return nil, nil + } + + result := make(map[string]string) + for snippet, lines := range snippetLines { + result[snippet] = trimLeadingWhitespace(lines) + } + return result, nil +} + +// trimLeadingWhitespace computes the minimum plain-space indentation across non-empty lines, +// trimming that common whitespace while preserving newlines. +func trimLeadingWhitespace(lines []string) string { + if len(lines) == 0 { + return "" + } + + minSpaces := -1 + for _, line := range lines { + if strings.TrimSpace(line) != "" { + spaces := len(line) - len(strings.TrimLeft(line, " ")) + if minSpaces == -1 || spaces < minSpaces { + minSpaces = spaces + } + } + } + if minSpaces == -1 { + minSpaces = 0 + } + + var sb strings.Builder + for _, line := range lines { + if strings.TrimSpace(line) == "" { + sb.WriteString("\n") + } else { + if len(line) >= minSpaces { + sb.WriteString(line[minSpaces:]) + } else { + sb.WriteString(strings.TrimLeft(line, " ")) + } + sb.WriteString("\n") + } + } + return sb.String() +} diff --git a/internal/librarian/java/readme_test.go b/internal/librarian/java/readme_test.go index 595c87cb645..447f6087185 100644 --- a/internal/librarian/java/readme_test.go +++ b/internal/librarian/java/readme_test.go @@ -16,6 +16,8 @@ package java import ( "errors" + "os" + "path/filepath" "testing" "github.com/google/go-cmp/cmp" @@ -23,76 +25,21 @@ import ( func TestDecamelize(t *testing.T) { for _, test := range []struct { - name string - input string - want string + input string + expected string }{ - { - name: "camel case", - input: "CamelCase", - want: "Camel Case", - }, - { - name: "simple word", - input: "Word", - want: "Word", - }, - { - name: "already separated", - input: "Camel Case", - want: "Camel Case", - }, - { - name: "java acronym IamPolicy", - input: "IamPolicy", - want: "Iam Policy", - }, - { - name: "java acronym GcsBucket", - input: "GcsBucket", - want: "Gcs Bucket", - }, + {"requesterPays", "Requester Pays"}, + {"ACLBatman", "ACL Batman"}, + {"NativeImageLoggingSample", "Native Image Logging Sample"}, + {"simpleTest", "Simple Test"}, + {"", ""}, + {"IamPolicy", "Iam Policy"}, + {"GcsBucket", "Gcs Bucket"}, } { - t.Run(test.name, func(t *testing.T) { - got := decamelize(test.input) - if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestIsProductionSample(t *testing.T) { - for _, test := range []struct { - name string - path string - want bool - }{ - { - name: "valid production sample", - path: "samples/src/main/java/com/example/Sample.java", - want: true, - }, - { - name: "valid production sample at root", - path: "src/main/java/com/example/Sample.java", - want: true, - }, - { - name: "non-java file", - path: "samples/src/main/java/README.md", - want: false, - }, - { - name: "not in src/main/java", - path: "samples/src/test/java/com/example/Sample.java", - want: false, - }, - } { - t.Run(test.name, func(t *testing.T) { - got := isProductionSample(test.path) - if got != test.want { - t.Errorf("isProductionSample() = %t, want %t", got, test.want) + t.Run(test.input, func(t *testing.T) { + actual := decamelize(test.input) + if actual != test.expected { + t.Errorf("decamelize(%q) = %q; expected %q", test.input, actual, test.expected) } }) } @@ -179,3 +126,161 @@ func TestExtractTitle_Error(t *testing.T) { }) } } + +func TestIsProductionSample(t *testing.T) { + for _, test := range []struct { + name string + path string + want bool + }{ + { + name: "valid production sample", + path: "samples/src/main/java/com/example/Sample.java", + want: true, + }, + { + name: "valid production sample at root", + path: "src/main/java/com/example/Sample.java", + want: true, + }, + { + name: "non-java file", + path: "samples/src/main/java/README.md", + want: false, + }, + { + name: "not in src/main/java", + path: "samples/src/test/java/com/example/Sample.java", + want: false, + }, + } { + t.Run(test.name, func(t *testing.T) { + got := isProductionSample(test.path) + if got != test.want { + t.Errorf("isProductionSample() = %t, want %t", got, test.want) + } + }) + } +} + +func TestExtractSamples_MissingDir(t *testing.T) { + tempDir := t.TempDir() + samples, err := ExtractSamples(tempDir) + if err != nil { + t.Fatalf("ExtractSamples returned error for missing dir: %v", err) + } + if samples != nil { + t.Errorf("Expected nil samples for missing dir, got %v", samples) + } +} + +func TestExtractSamples_Success(t *testing.T) { + tempDir := t.TempDir() + samplesDir := filepath.Join(tempDir, "samples", "src", "main", "java") + if err := os.MkdirAll(samplesDir, 0755); err != nil { + t.Fatal(err) + } + + file1 := filepath.Join(samplesDir, "RequesterPays.java") + content1 := `// sample-metadata: +// title: Custom Title Override +// description: A custom demo sample +public class RequesterPays {}` + if err := os.WriteFile(file1, []byte(content1), 0644); err != nil { + t.Fatal(err) + } + + file2 := filepath.Join(samplesDir, "demoSample.java") + content2 := `public class demoSample {}` + if err := os.WriteFile(file2, []byte(content2), 0644); err != nil { + t.Fatal(err) + } + + samples, err := ExtractSamples(tempDir) + if err != nil { + t.Fatal(err) + } + + want := []map[string]interface{}{ + { + "Title": "Custom Title Override", + "title": "Custom Title Override", + "description": "A custom demo sample", + "Description": "A custom demo sample", + "File": "samples/src/main/java/RequesterPays.java", + "file": "samples/src/main/java/RequesterPays.java", + }, + { + "Title": "Demo Sample", + "title": "Demo Sample", + "File": "samples/src/main/java/demoSample.java", + "file": "samples/src/main/java/demoSample.java", + }, + } + + if diff := cmp.Diff(want, samples); diff != "" { + t.Errorf("ExtractSamples() mismatch (-want +got):\n%s", diff) + } +} + +func TestExtractSnippets(t *testing.T) { + tempDir := t.TempDir() + samplesDir := filepath.Join(tempDir, "samples") + if err := os.MkdirAll(samplesDir, 0755); err != nil { + t.Fatal(err) + } + + pomPath := filepath.Join(samplesDir, "pom.xml") + pomContent := ` + + + com.google.cloud + + +` + if err := os.WriteFile(pomPath, []byte(pomContent), 0644); err != nil { + t.Fatal(err) + } + + javaPath := filepath.Join(samplesDir, "Demo.java") + javaContent := `public class Demo { + // [START quickstart] + public void run() { + // [START_EXCLUDE] + System.out.println("hidden"); + // [END_EXCLUDE] + System.out.println("visible"); + } + // [END quickstart] +}` + if err := os.WriteFile(javaPath, []byte(javaContent), 0644); err != nil { + t.Fatal(err) + } + + snippets, err := ExtractSnippets(tempDir) + if err != nil { + t.Fatal(err) + } + + if len(snippets) != 2 { + t.Fatalf("Expected 2 snippets, got %d", len(snippets)) + } + + depSnippet := snippets["dependency_snippet"] + expectedDep := ` + com.google.cloud + +` + if depSnippet != expectedDep { + t.Errorf("dependency_snippet = %q; expected %q", depSnippet, expectedDep) + } + + quickSnippet := snippets["quickstart"] + expectedQuick := `public void run() { + System.out.println("visible"); +} +` + if quickSnippet != expectedQuick { + t.Errorf("quickstart = %q; expected %q", quickSnippet, expectedQuick) + } +} diff --git a/internal/librarian/java/samples.go b/internal/librarian/java/samples.go deleted file mode 100644 index 61abe27eaf2..00000000000 --- a/internal/librarian/java/samples.go +++ /dev/null @@ -1,278 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package java - -import ( - "bufio" - "errors" - "io/fs" - "os" - "path/filepath" - "regexp" - "sort" - "strings" - "unicode" - - "github.com/googleapis/librarian/internal/yaml" -) - -var ( - openSnippetRegex = regexp.MustCompile(`\[START ([a-zA-Z0-9_-]+)\]`) - closeSnippetRegex = regexp.MustCompile(`\[END ([a-zA-Z0-9_-]+)\]`) - openExcludeRegex = regexp.MustCompile(`\[START_EXCLUDE\]`) - closeExcludeRegex = regexp.MustCompile(`\[END_EXCLUDE\]`) - - reMetadataBlock = regexp.MustCompile(`(?m)^[ \t]*//[ \t]*sample-metadata:([^\n]+|\n[ \t]*//)+`) - reCommentPrefix = regexp.MustCompile(`(?m)^[ \t]*(?:#|//)[ \t]?`) - - reDecamelize1 = regexp.MustCompile(`([A-Z]+)([A-Z])([a-z0-9])`) - reDecamelize2 = regexp.MustCompile(`([a-z0-9])([A-Z])`) -) - -// decamelize converts CamelCase or PascalCase titles into space-separated words, -// exactly reproducing Python synthtool's _decamelize(value: str). -func decamelize(value string) string { - if value == "" { - return "" - } - r := []rune(value) - r[0] = unicode.ToUpper(r[0]) - s := string(r) - - s = reDecamelize1.ReplaceAllString(s, "${1} ${2}${3}") - return reDecamelize2.ReplaceAllString(s, "${1} ${2}") -} - -// ExtractSamples walks the "samples" directory locating all .java source files. -// It parses embedded multiline "// sample-metadata:" YAML blocks to derive title and path metadata. -func ExtractSamples(dir string) ([]map[string]interface{}, error) { - samplesDir := filepath.Join(dir, "samples") - var files []string - - err := filepath.WalkDir(samplesDir, func(path string, d os.DirEntry, err error) error { - if err != nil { - if errors.Is(err, fs.ErrNotExist) || os.IsNotExist(err) { - return nil - } - return err - } - if d.IsDir() { - if d.Name() == "test" { - return filepath.SkipDir - } - if d.Name() == "generated" && filepath.Base(filepath.Dir(path)) == "snippets" { - return filepath.SkipDir - } - return nil - } - if !d.IsDir() && d.Type().IsRegular() && filepath.Ext(path) == ".java" && strings.Contains(filepath.ToSlash(path), "/src/main/java/") { - files = append(files, path) - } - return nil - }) - if err != nil { - return nil, err - } - - if len(files) == 0 { - return nil, nil - } - - sort.Strings(files) - var samples []map[string]interface{} - - for _, file := range files { - rel, err := filepath.Rel(dir, file) - if err != nil { - continue - } - base := strings.TrimSuffix(filepath.Base(file), ".java") - title := decamelize(base) - - slashPath := filepath.ToSlash(rel) - item := map[string]interface{}{ - "Title": title, - "File": slashPath, - "title": title, - "file": slashPath, - } - - contentBytes, err := os.ReadFile(file) - if err != nil { - samples = append(samples, item) - continue - } - match := reMetadataBlock.FindString(string(contentBytes)) - if match == "" { - samples = append(samples, item) - continue - } - cleaned := reCommentPrefix.ReplaceAllString(match, "") - meta, err := yaml.Unmarshal[map[string]map[string]interface{}]([]byte(cleaned)) - if err != nil { - samples = append(samples, item) - continue - } - sm, ok := (*meta)["sample-metadata"] - if !ok { - samples = append(samples, item) - continue - } - for k, v := range sm { - item[k] = v - if len(k) > 0 { - upperKey := strings.ToUpper(k[:1]) + k[1:] - item[upperKey] = v - } - } - if t, ok := sm["title"].(string); ok && strings.TrimSpace(t) != "" { - item["Title"] = t - item["title"] = t - } - samples = append(samples, item) - } - return samples, nil -} - -// ExtractSnippets walks the "samples" directory locating *.java and *.xml files. -// It line-scans for [START name] and [END name] tags while supporting [START_EXCLUDE] blocks, -// returning trimmed minimum plain-space indentation blocks. -func ExtractSnippets(dir string) (map[string]string, error) { - samplesDir := filepath.Join(dir, "samples") - var files []string - - err := filepath.WalkDir(samplesDir, func(path string, d os.DirEntry, err error) error { - if err != nil { - if errors.Is(err, fs.ErrNotExist) || os.IsNotExist(err) { - return nil - } - return err - } - if d.IsDir() { - if d.Name() == "test" { - return filepath.SkipDir - } - if d.Name() == "generated" && filepath.Base(filepath.Dir(path)) == "snippets" { - return filepath.SkipDir - } - return nil - } - if !d.Type().IsRegular() { - return nil - } - ext := filepath.Ext(path) - if ext == ".java" || ext == ".xml" { - files = append(files, path) - } - return nil - }) - if err != nil { - return nil, err - } - - if len(files) == 0 { - return nil, nil - } - - sort.Strings(files) - snippetLines := make(map[string][]string) - - for _, file := range files { - f, err := os.Open(file) - if err != nil { - continue - } - - openSnippets := make(map[string]bool) - excluding := false - scanner := bufio.NewScanner(f) - scanner.Buffer(make([]byte, 64*1024), 10*1024*1024) - - for scanner.Scan() { - line := scanner.Text() - openMatch := openSnippetRegex.FindStringSubmatch(line) - closeMatch := closeSnippetRegex.FindStringSubmatch(line) - - if len(openMatch) > 1 && !excluding { - name := openMatch[1] - openSnippets[name] = true - if _, exists := snippetLines[name]; !exists { - snippetLines[name] = []string{} - } - } else if len(closeMatch) > 1 && !excluding { - delete(openSnippets, closeMatch[1]) - } else if openExcludeRegex.MatchString(line) { - excluding = true - } else if closeExcludeRegex.MatchString(line) { - excluding = false - } else if !excluding { - for s := range openSnippets { - snippetLines[s] = append(snippetLines[s], line) - } - } - } - if err := scanner.Err(); err != nil { - f.Close() - return nil, err - } - f.Close() - } - - if len(snippetLines) == 0 { - return nil, nil - } - - result := make(map[string]string) - for snippet, lines := range snippetLines { - result[snippet] = trimLeadingWhitespace(lines) - } - return result, nil -} - -// trimLeadingWhitespace computes the minimum plain-space indentation across non-empty lines, -// trimming that common whitespace while preserving newlines. -func trimLeadingWhitespace(lines []string) string { - if len(lines) == 0 { - return "" - } - - minSpaces := -1 - for _, line := range lines { - if strings.TrimSpace(line) != "" { - spaces := len(line) - len(strings.TrimLeft(line, " ")) - if minSpaces == -1 || spaces < minSpaces { - minSpaces = spaces - } - } - } - if minSpaces == -1 { - minSpaces = 0 - } - - var sb strings.Builder - for _, line := range lines { - if strings.TrimSpace(line) == "" { - sb.WriteString("\n") - } else { - if len(line) >= minSpaces { - sb.WriteString(line[minSpaces:]) - } else { - sb.WriteString(strings.TrimLeft(line, " ")) - } - sb.WriteString("\n") - } - } - return sb.String() -} diff --git a/internal/librarian/java/samples_test.go b/internal/librarian/java/samples_test.go deleted file mode 100644 index af0a1b4d215..00000000000 --- a/internal/librarian/java/samples_test.go +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package java - -import ( - "os" - "path/filepath" - "testing" -) - -func TestDecamelize(t *testing.T) { - tests := []struct { - input string - expected string - }{ - {"requesterPays", "Requester Pays"}, - {"ACLBatman", "ACL Batman"}, - {"NativeImageLoggingSample", "Native Image Logging Sample"}, - {"simpleTest", "Simple Test"}, - {"", ""}, - } - - for _, tc := range tests { - actual := decamelize(tc.input) - if actual != tc.expected { - t.Errorf("decamelize(%q) = %q; expected %q", tc.input, actual, tc.expected) - } - } -} - -func TestExtractSamples_MissingDir(t *testing.T) { - tempDir := t.TempDir() - samples, err := ExtractSamples(tempDir) - if err != nil { - t.Fatalf("ExtractSamples returned error for missing dir: %v", err) - } - if samples != nil { - t.Errorf("Expected nil samples for missing dir, got %v", samples) - } -} - -func TestExtractSamples_Success(t *testing.T) { - tempDir := t.TempDir() - samplesDir := filepath.Join(tempDir, "samples", "src", "main", "java") - if err := os.MkdirAll(samplesDir, 0755); err != nil { - t.Fatal(err) - } - - file1 := filepath.Join(samplesDir, "RequesterPays.java") - content1 := `// sample-metadata: -// title: Custom Title Override -// description: A custom demo sample -public class RequesterPays {}` - if err := os.WriteFile(file1, []byte(content1), 0644); err != nil { - t.Fatal(err) - } - - file2 := filepath.Join(samplesDir, "demoSample.java") - content2 := `public class demoSample {}` - if err := os.WriteFile(file2, []byte(content2), 0644); err != nil { - t.Fatal(err) - } - - samples, err := ExtractSamples(tempDir) - if err != nil { - t.Fatal(err) - } - - if len(samples) != 2 { - t.Fatalf("Expected 2 samples, got %d", len(samples)) - } - - // First should be RequesterPays.java with custom YAML override (uppercase R comes before lowercase d) - s0 := samples[0] - if s0["Title"] != "Custom Title Override" || s0["title"] != "Custom Title Override" { - t.Errorf("Sample 0 title = %v; expected 'Custom Title Override'", s0["Title"]) - } - if s0["File"] != "samples/src/main/java/RequesterPays.java" { - t.Errorf("Sample 0 file = %v", s0["File"]) - } - - // Second should be demoSample.java - s1 := samples[1] - if s1["Title"] != "Demo Sample" || s1["title"] != "Demo Sample" { - t.Errorf("Sample 1 title = %v; expected 'Demo Sample'", s1["Title"]) - } - if s1["File"] != "samples/src/main/java/demoSample.java" { - t.Errorf("Sample 1 file = %v", s1["File"]) - } -} - -func TestExtractSnippets(t *testing.T) { - tempDir := t.TempDir() - samplesDir := filepath.Join(tempDir, "samples") - if err := os.MkdirAll(samplesDir, 0755); err != nil { - t.Fatal(err) - } - - pomPath := filepath.Join(samplesDir, "pom.xml") - pomContent := ` - - - com.google.cloud - - -` - if err := os.WriteFile(pomPath, []byte(pomContent), 0644); err != nil { - t.Fatal(err) - } - - javaPath := filepath.Join(samplesDir, "Demo.java") - javaContent := `public class Demo { - // [START quickstart] - public void run() { - // [START_EXCLUDE] - System.out.println("hidden"); - // [END_EXCLUDE] - System.out.println("visible"); - } - // [END quickstart] -}` - if err := os.WriteFile(javaPath, []byte(javaContent), 0644); err != nil { - t.Fatal(err) - } - - snippets, err := ExtractSnippets(tempDir) - if err != nil { - t.Fatal(err) - } - - if len(snippets) != 2 { - t.Fatalf("Expected 2 snippets, got %d", len(snippets)) - } - - depSnippet := snippets["dependency_snippet"] - expectedDep := ` - com.google.cloud - -` - if depSnippet != expectedDep { - t.Errorf("dependency_snippet = %q; expected %q", depSnippet, expectedDep) - } - - quickSnippet := snippets["quickstart"] - expectedQuick := `public void run() { - System.out.println("visible"); -} -` - if quickSnippet != expectedQuick { - t.Errorf("quickstart = %q; expected %q", quickSnippet, expectedQuick) - } -} From c61084a009151ebd4c83eba6dc3b9a23dac136a7 Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Sat, 27 Jun 2026 17:27:33 -0400 Subject: [PATCH 047/108] feat(internal/librarian/java): migrate ExtractSamples to return Sample slice and simplify metadata extraction --- internal/librarian/java/readme.go | 87 +++++---------- internal/librarian/java/readme_test.go | 142 ++++++++++++++++--------- 2 files changed, 121 insertions(+), 108 deletions(-) diff --git a/internal/librarian/java/readme.go b/internal/librarian/java/readme.go index 9cc7a4c8f28..78147e322e7 100644 --- a/internal/librarian/java/readme.go +++ b/internal/librarian/java/readme.go @@ -25,8 +25,6 @@ import ( "sort" "strings" "unicode" - - "github.com/googleapis/librarian/internal/yaml" ) var ( @@ -35,9 +33,6 @@ var ( openExcludeRegex = regexp.MustCompile(`\[START_EXCLUDE\]`) closeExcludeRegex = regexp.MustCompile(`\[END_EXCLUDE\]`) - reMetadataBlock = regexp.MustCompile(`(?m)^[ \t]*//[ \t]*sample-metadata:([^\n]+|\n[ \t]*//)+`) - reCommentPrefix = regexp.MustCompile(`(?m)^[ \t]*(?:#|//)[ \t]?`) - reDecamelize1 = regexp.MustCompile(`([A-Z]+)([A-Z])([a-z0-9])`) reDecamelize2 = regexp.MustCompile(`([a-z0-9])([A-Z])`) @@ -92,12 +87,17 @@ func extractTitle(content string) (string, error) { return title, nil } +// Sample represents a Java code sample and its metadata for README generation. +type Sample struct { + Title string + File string +} + // ExtractSamples walks the "samples" directory locating all .java source files. -// It parses embedded multiline "// sample-metadata:" YAML blocks to derive title and path metadata. -func ExtractSamples(dir string) ([]map[string]interface{}, error) { +// It extracts title overrides from source file comments using extractTitle. +func ExtractSamples(dir string) ([]Sample, error) { samplesDir := filepath.Join(dir, "samples") var files []string - err := filepath.WalkDir(samplesDir, func(path string, d os.DirEntry, err error) error { if err != nil { if errors.Is(err, fs.ErrNotExist) { @@ -105,17 +105,15 @@ func ExtractSamples(dir string) ([]map[string]interface{}, error) { } return err } - if d.IsDir() { - if d.Name() == "test" { - return filepath.SkipDir - } - if d.Name() == "generated" && filepath.Base(filepath.Dir(path)) == "snippets" { - return filepath.SkipDir - } + if !d.Type().IsRegular() { return nil } - if d.Type().IsRegular() && isProductionSample(path) { - files = append(files, path) + rel, err := filepath.Rel(dir, path) + if err != nil { + return err + } + if isProductionSample(rel) { + files = append(files, rel) } return nil }) @@ -123,59 +121,28 @@ func ExtractSamples(dir string) ([]map[string]interface{}, error) { return nil, fmt.Errorf("failed to walk samples directory: %w", err) } - if len(files) == 0 { - return nil, nil - } - - sort.Strings(files) - var samples []map[string]interface{} - + var samples []Sample for _, file := range files { - rel, err := filepath.Rel(dir, file) - if err != nil { - continue - } base := strings.TrimSuffix(filepath.Base(file), ".java") title := decamelize(base) + slashPath := filepath.ToSlash(file) - slashPath := filepath.ToSlash(rel) - item := map[string]interface{}{ - "Title": title, - "File": slashPath, - "title": title, - "file": slashPath, - } - - contentBytes, err := os.ReadFile(file) + absPath := filepath.Join(dir, file) + contentBytes, err := os.ReadFile(absPath) if err != nil { return nil, fmt.Errorf("failed to read sample file %s: %w", file, err) } - matchBytes := reMetadataBlock.Find(contentBytes) - if matchBytes == nil { - samples = append(samples, item) - continue - } - cleanedBytes := reCommentPrefix.ReplaceAll(matchBytes, nil) - meta, err := yaml.Unmarshal[map[string]map[string]interface{}](cleanedBytes) + titleOverride, err := extractTitle(string(contentBytes)) if err != nil { - return nil, fmt.Errorf("failed to parse YAML metadata in %s: %w", file, err) - } - sm, ok := (*meta)["sample-metadata"] - if !ok { - return nil, fmt.Errorf("missing 'sample-metadata' key in YAML of %s", file) - } - for k, v := range sm { - item[k] = v - if len(k) > 0 { - upperKey := strings.ToUpper(k[:1]) + k[1:] - item[upperKey] = v - } + return nil, fmt.Errorf("failed to extract title from %s: %w", file, err) } - if t, ok := sm["title"].(string); ok && strings.TrimSpace(t) != "" { - item["Title"] = t - item["title"] = t + if titleOverride != "" { + title = titleOverride } - samples = append(samples, item) + samples = append(samples, Sample{ + Title: title, + File: slashPath, + }) } return samples, nil } diff --git a/internal/librarian/java/readme_test.go b/internal/librarian/java/readme_test.go index 447f6087185..990cf495161 100644 --- a/internal/librarian/java/readme_test.go +++ b/internal/librarian/java/readme_test.go @@ -163,63 +163,109 @@ func TestIsProductionSample(t *testing.T) { } } -func TestExtractSamples_MissingDir(t *testing.T) { - tempDir := t.TempDir() - samples, err := ExtractSamples(tempDir) - if err != nil { - t.Fatalf("ExtractSamples returned error for missing dir: %v", err) - } - if samples != nil { - t.Errorf("Expected nil samples for missing dir, got %v", samples) - } -} - -func TestExtractSamples_Success(t *testing.T) { - tempDir := t.TempDir() - samplesDir := filepath.Join(tempDir, "samples", "src", "main", "java") - if err := os.MkdirAll(samplesDir, 0755); err != nil { - t.Fatal(err) - } - - file1 := filepath.Join(samplesDir, "RequesterPays.java") - content1 := `// sample-metadata: +func TestExtractSamples(t *testing.T) { + for _, test := range []struct { + name string + setupFiles func(t *testing.T, dir string) + want []Sample + }{ + { + name: "missing samples directory", + setupFiles: func(t *testing.T, dir string) { + // Do nothing, tempDir is empty. + }, + want: nil, + }, + { + name: "extract successfully", + setupFiles: func(t *testing.T, dir string) { + samplesDir := filepath.Join(dir, "samples", "src", "main", "java") + if err := os.MkdirAll(samplesDir, 0755); err != nil { + t.Fatal(err) + } + file1 := filepath.Join(samplesDir, "RequesterPays.java") + content1 := `// sample-metadata: // title: Custom Title Override -// description: A custom demo sample public class RequesterPays {}` - if err := os.WriteFile(file1, []byte(content1), 0644); err != nil { - t.Fatal(err) - } - - file2 := filepath.Join(samplesDir, "demoSample.java") - content2 := `public class demoSample {}` - if err := os.WriteFile(file2, []byte(content2), 0644); err != nil { - t.Fatal(err) - } + if err := os.WriteFile(file1, []byte(content1), 0644); err != nil { + t.Fatal(err) + } + file2 := filepath.Join(samplesDir, "demoSample.java") + content2 := `public class demoSample {}` + if err := os.WriteFile(file2, []byte(content2), 0644); err != nil { + t.Fatal(err) + } + }, + want: []Sample{ + { + Title: "Custom Title Override", + File: "samples/src/main/java/RequesterPays.java", + }, + { + Title: "Demo Sample", + File: "samples/src/main/java/demoSample.java", + }, + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + tempDir := t.TempDir() + test.setupFiles(t, tempDir) - samples, err := ExtractSamples(tempDir) - if err != nil { - t.Fatal(err) + samples, err := ExtractSamples(tempDir) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(test.want, samples); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + }) } +} - want := []map[string]interface{}{ +func TestExtractSamples_Error(t *testing.T) { + for _, test := range []struct { + name string + content string + wantErr error + }{ + { + name: "error on empty title override", + content: `// sample-metadata: +// title: "" +public class Invalid {}`, + wantErr: errEmptyTitle, + }, { - "Title": "Custom Title Override", - "title": "Custom Title Override", - "description": "A custom demo sample", - "Description": "A custom demo sample", - "File": "samples/src/main/java/RequesterPays.java", - "file": "samples/src/main/java/RequesterPays.java", + name: "error on capitalized Title", + content: `// sample-metadata: +// Title: Capitalized Title +public class Invalid {}`, + wantErr: errMissingTitle, }, { - "Title": "Demo Sample", - "title": "Demo Sample", - "File": "samples/src/main/java/demoSample.java", - "file": "samples/src/main/java/demoSample.java", + name: "error on missing title line immediately following sample-metadata", + content: `// sample-metadata: +// description: missing title line +public class Invalid {}`, + wantErr: errMissingTitle, }, - } - - if diff := cmp.Diff(want, samples); diff != "" { - t.Errorf("ExtractSamples() mismatch (-want +got):\n%s", diff) + } { + t.Run(test.name, func(t *testing.T) { + tempDir := t.TempDir() + samplesDir := filepath.Join(tempDir, "samples", "src", "main", "java") + if err := os.MkdirAll(samplesDir, 0755); err != nil { + t.Fatal(err) + } + file := filepath.Join(samplesDir, "Sample.java") + if err := os.WriteFile(file, []byte(test.content), 0644); err != nil { + t.Fatal(err) + } + _, err := ExtractSamples(tempDir) + if !errors.Is(err, test.wantErr) { + t.Errorf("ExtractSamples() err = %v, want %v", err, test.wantErr) + } + }) } } From 11f23c4e00f5eac80fd2afe57e8efeaed95c65da Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Sat, 27 Jun 2026 17:41:06 -0400 Subject: [PATCH 048/108] feat(internal/librarian/java): rename CLI flag to go-postprocessor and decouple config existence from execution --- cmd/librarian/doc.go | 157 ++++++++++++++++++++ internal/librarian/generate.go | 7 +- internal/librarian/java/postprocess.go | 23 ++- internal/librarian/java/postprocess_test.go | 39 ++++- 4 files changed, 202 insertions(+), 24 deletions(-) diff --git a/cmd/librarian/doc.go b/cmd/librarian/doc.go index 8d8a24b2d3f..3b806378821 100644 --- a/cmd/librarian/doc.go +++ b/cmd/librarian/doc.go @@ -31,5 +31,162 @@ Usage: Global flags: --verbose, -v enable verbose logging + +# Read and write librarian.yaml configuration + +Usage: + + librarian config [get|set] [path] [value] + +# Get a configuration value + +Usage: + + librarian config get [path] + +# Set a configuration value + +Usage: + + librarian config set [path] [value] + +# Add a new client library + +Usage: + + librarian add + +add registers a single API in librarian.yaml. + +The is a path within the configured googleapis source, such as +"google/cloud/secretmanager/v1". The library name and other defaults are +derived from the first API path using language-specific rules. + +If the API path should naturally be included in an existing library, and if the +language supports doing so, that library is modified. Otherwise, a new library +is created. + +While release-please is responsible for library releases, the relevant +release-please configuration will be updated as necessary to onboard any new +library. + +To add a preview client of an existing library, prefix the API path with +"preview/". + +Examples: + + librarian add google/cloud/secretmanager/v1 + librarian add preview/google/cloud/secretmanager/v1beta + +A typical librarian workflow for adding a new client library is: + + librarian add # onboard a new API into librarian.yaml + librarian generate # generate the client library + +# Generate a client library + +Usage: + + librarian generate + +generate produces client library code from the APIs configured in +librarian.yaml. + +The library name argument selects a single library to regenerate. Use the +--all flag to regenerate every library in the workspace instead. Exactly +one of or --all must be provided. + +Generation is delegated to the language-specific tooling configured in +librarian.yaml. Libraries marked with skip_generate are skipped. + +Examples: + + librarian generate # regenerate one library + librarian generate --all # regenerate every library + +Flags: + + --all generate all libraries + --go-postprocessor, --go-post use the new Go postprocessor for Java libraries + +A typical librarian workflow for regenerating every library against the +latest API definitions is: + + librarian update googleapis + librarian generate --all + +# Install tool dependencies for a language + +Usage: + + librarian install [language] + +install installs the language-specific tools that librarian uses to +generate and build client libraries (for example, language SDKs and code +generators). + +If [language] is omitted, the language is read from librarian.yaml in the +current directory. + +Examples: + + librarian install # use language from librarian.yaml + librarian install go # install Go-specific tools + +# Tidy and validate librarian.yaml + +Usage: + + librarian tidy + +tidy reads librarian.yaml, validates its contents, applies any +language-specific defaults and normalization, and writes the file back +with a canonical formatting. + +Run tidy after editing librarian.yaml by hand, or as a quick check that +the configuration is well-formed. + +# Update sources or version to the latest version + +Usage: + + librarian update ... + +update refreshes the upstream source repositories declared in +librarian.yaml to their latest commits and updates the recorded commit +SHAs in librarian.yaml accordingly. It also supports updating the librarian version. + +Supported targets: + + - sources.conformance: protocolbuffers/protobuf conformance tests + - sources.discovery: googleapis/discovery-artifact-manager + - sources.googleapis: googleapis/googleapis (the API definitions) + - sources.protobuf: protocolbuffers/protobuf + - sources.showcase: googleapis/gapic-showcase + - version: the librarian tool version + +At least one target must be specified. + +Examples: + + librarian update sources.googleapis + librarian update sources.googleapis sources.protobuf + librarian update version + +A typical librarian workflow for regenerating every library against the +latest API definitions is: + + librarian update sources.googleapis + librarian generate --all + +# Print the binary version + +Usage: + + librarian version + +version prints the librarian binary version and exits. The version is +embedded at build time and follows the conventions described at +https://go.dev/ref/mod#versions. */ package main diff --git a/internal/librarian/generate.go b/internal/librarian/generate.go index 10a5e54f4e0..3814cdff243 100644 --- a/internal/librarian/generate.go +++ b/internal/librarian/generate.go @@ -74,13 +74,14 @@ latest API definitions is: Usage: "generate all libraries", }, &cli.BoolFlag{ - Name: "use-go-postprocessor", - Usage: "use the new Go postprocessor for Java libraries", + Name: "go-postprocessor", + Aliases: []string{"go-post"}, + Usage: "use the new Go postprocessor for Java libraries", }, }, Action: func(ctx context.Context, cmd *cli.Command) error { all := cmd.Bool("all") - useGoPostprocessor := cmd.Bool("use-go-postprocessor") + useGoPostprocessor := cmd.Bool("go-postprocessor") libraryName := cmd.Args().First() if !all && libraryName == "" { return errMissingLibraryOrAllFlag diff --git a/internal/librarian/java/postprocess.go b/internal/librarian/java/postprocess.go index 19a97a09b9e..a202e180689 100644 --- a/internal/librarian/java/postprocess.go +++ b/internal/librarian/java/postprocess.go @@ -71,21 +71,18 @@ type libraryPostProcessParams struct { func postProcessLibrary(ctx context.Context, params libraryPostProcessParams) error { if params.useGoPostprocessor { - yamlPath := filepath.Join(params.outDir, "postprocess.yaml") - if _, err := os.Stat(yamlPath); err == nil { - if err := postProcessLibraryNew(ctx, params); err != nil { - return err - } + if err := postProcessLibraryNew(ctx, params); err != nil { + return err + } - monorepoVersion, err := findMonorepoVersion(params.cfg) - if err != nil { - return err - } - if err := syncPOMs(params.library, params.outDir, monorepoVersion, params.metadata, params.transports); err != nil { - return fmt.Errorf("%w: %w", errSyncPOMs, err) - } - return nil + monorepoVersion, err := findMonorepoVersion(params.cfg) + if err != nil { + return err + } + if err := syncPOMs(params.library, params.outDir, monorepoVersion, params.metadata, params.transports); err != nil { + return fmt.Errorf("%w: %w", errSyncPOMs, err) } + return nil } if err := createOrVerifyOwlbotPy(params.outDir); err != nil { return err diff --git a/internal/librarian/java/postprocess_test.go b/internal/librarian/java/postprocess_test.go index 157147a6c4d..56955da84ea 100644 --- a/internal/librarian/java/postprocess_test.go +++ b/internal/librarian/java/postprocess_test.go @@ -1170,29 +1170,52 @@ func TestPostProcessLibrary_Branching(t *testing.T) { } }) - t.Run("UseGoPostprocessor true, no yaml", func(t *testing.T) { + t.Run("UseGoPostprocessor true, no yaml, success", func(t *testing.T) { outDir := t.TempDir() - if err := os.WriteFile(filepath.Join(outDir, "owlbot.py"), []byte(""), 0755); err != nil { + t.Chdir(outDir) + + if err := os.MkdirAll(filepath.Join(outDir, "owl-bot-staging"), 0755); err != nil { + t.Fatal(err) + } + metadata := `{"repo": {"name_pretty": "My API"}}` + if err := os.WriteFile(filepath.Join(outDir, ".repo-metadata.json"), []byte(metadata), 0644); err != nil { t.Fatal(err) } + if err := os.MkdirAll(filepath.Join(outDir, "template"), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(outDir, "template", "README.md.go.tmpl"), []byte("dummy"), 0644); err != nil { + t.Fatal(err) + } + p := libraryPostProcessParams{ outDir: outDir, useGoPostprocessor: true, + metadata: &repoMetadata{ + NamePretty: "test-library", + }, cfg: &config.Config{ + Default: &config.Default{ + Java: &config.JavaDefault{ + LibrariesBOMVersion: "1.0.0", + }, + }, Libraries: []*config.Library{ {Name: "google-cloud-java", Version: "1.2.3"}, }, }, library: &config.Library{ - Name: "test-library", + Name: "test-library", + Version: "1.2.3", + Java: &config.JavaModule{ + GroupID: "com.google.cloud", + ArtifactID: "test-library", + }, }, } err := postProcessLibrary(t.Context(), p) - if err == nil { - t.Fatal("expected error, got nil") - } - if strings.Contains(err.Error(), "postProcessLibraryNew not implemented") { - t.Errorf("expected legacy flow, but got new flow error: %v", err) + if err != nil { + t.Fatalf("expected no error, got: %v", err) } }) From e76e25d059fca274a7426c393978421ef0b59fb2 Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Sat, 27 Jun 2026 17:45:31 -0400 Subject: [PATCH 049/108] feat(internal/postprocessing): migrate postprocessing config into librarian.yaml and retire postprocess.yaml --- doc/config-schema.md | 44 +++++ internal/config/config.go | 42 ++++ internal/librarian/java/postprocess.go | 43 ++--- internal/librarian/java/postprocess_new.go | 14 +- .../librarian/java/postprocess_new_test.go | 51 ++--- internal/librarian/java/postprocess_test.go | 28 ++- internal/postprocessing/config.go | 63 +----- internal/postprocessing/config_test.go | 182 +++++++++--------- 8 files changed, 260 insertions(+), 207 deletions(-) diff --git a/doc/config-schema.md b/doc/config-schema.md index 351c8d34076..2503039ad3d 100644 --- a/doc/config-schema.md +++ b/doc/config-schema.md @@ -129,6 +129,50 @@ This document describes the schema for the librarian.yaml. | `python` | [PythonPackage](#pythonpackage-configuration) (optional) | Contains Python-specific library configuration. | | `rust` | [RustCrate](#rustcrate-configuration) (optional) | Contains Rust-specific library configuration. | | `swift` | [SwiftPackage](#swiftpackage-configuration) (optional) | Contains Swift-specific library configuration. | +| `postprocess` | [Postprocess](#postprocess-configuration) (optional) | Contains post-processing operations (copies, removes, replacements) that are executed after code generation. | + +## Postprocess Configuration + +| Field | Type | Description | +| :--- | :--- | :--- | +| `replace` | list of [ReplaceConfig](#replaceconfig-configuration) | | +| `replace_regex` | list of [ReplaceRegexConfig](#replaceregexconfig-configuration) | | +| `copy_file` | list of [CopyConfig](#copyconfig-configuration) | | +| `remove_file` | list of string | | +| `method_operations` | list of [MethodOperation](#methodoperation-configuration) | | + +## MethodOperation Configuration + +| Field | Type | Description | +| :--- | :--- | :--- | +| `path` | string | | +| `action` | string | | +| `func_name` | string | | +| `new_name` | string | | +| `deprecation_message` | string | | + +## ReplaceConfig Configuration + +| Field | Type | Description | +| :--- | :--- | :--- | +| `path` | string | | +| `original` | string | | +| `replacement` | string | | + +## ReplaceRegexConfig Configuration + +| Field | Type | Description | +| :--- | :--- | :--- | +| `path` | string | | +| `pattern` | string | | +| `replacement` | string | | + +## CopyConfig Configuration + +| Field | Type | Description | +| :--- | :--- | :--- | +| `src` | string | | +| `dst` | string | | ## API Configuration diff --git a/internal/config/config.go b/internal/config/config.go index 81b80f5476f..7c76fda9a2e 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -323,6 +323,48 @@ type Library struct { // Swift contains Swift-specific library configuration. Swift *SwiftPackage `yaml:"swift,omitempty"` + + // Postprocess contains post-processing operations (copies, removes, replacements) + // that are executed after code generation. + Postprocess *Postprocess `yaml:"postprocess,omitempty"` +} + +// Postprocess represents post-processing configuration options integrated into librarian.yaml. +type Postprocess struct { + Replace []ReplaceConfig `yaml:"replace,omitempty"` + ReplaceRegex []ReplaceRegexConfig `yaml:"replace_regex,omitempty"` + CopyFile []CopyConfig `yaml:"copy_file,omitempty"` + RemoveFile []string `yaml:"remove_file,omitempty"` + MethodOperations []MethodOperation `yaml:"method_operations,omitempty"` +} + +// MethodOperation represents a method-level operation like delete, duplicate, or deprecate. +type MethodOperation struct { + Path string `yaml:"path"` + Action string `yaml:"action"` + FuncName string `yaml:"func_name"` + NewName string `yaml:"new_name,omitempty"` // Used for duplicate + DeprecationMessage string `yaml:"deprecation_message,omitempty"` // Used for deprecate +} + +// ReplaceConfig represents a replacement rule. +type ReplaceConfig struct { + Path string `yaml:"path"` + Original string `yaml:"original"` + Replacement string `yaml:"replacement"` +} + +// ReplaceRegexConfig represents a regex replacement rule. +type ReplaceRegexConfig struct { + Path string `yaml:"path"` + Pattern string `yaml:"pattern"` + Replacement string `yaml:"replacement"` +} + +// CopyConfig represents a file copy rule. +type CopyConfig struct { + Src string `yaml:"src"` + Dst string `yaml:"dst"` } // API describes an API to include in a library. diff --git a/internal/librarian/java/postprocess.go b/internal/librarian/java/postprocess.go index a202e180689..6fce9595e10 100644 --- a/internal/librarian/java/postprocess.go +++ b/internal/librarian/java/postprocess.go @@ -144,33 +144,30 @@ func postProcessAPI(ctx context.Context, params postProcessParams) error { coords := params.coords() if params.useGoPostprocessor { - yamlPath := filepath.Join(params.outDir, "postprocess.yaml") - if _, err := os.Stat(yamlPath); err == nil { - keepSet := make(map[string]bool) - for _, k := range params.library.Keep { - keepSet[strings.TrimSuffix(filepath.ToSlash(k), "/")] = true - } - if err := restructureModules(params, params.outDir, keepSet, params.outDir); err != nil { - return fmt.Errorf("failed to restructure direct to outDir: %w", err) - } + keepSet := make(map[string]bool) + for _, k := range params.library.Keep { + keepSet[strings.TrimSuffix(filepath.ToSlash(k), "/")] = true + } + if err := restructureModules(params, params.outDir, keepSet, params.outDir); err != nil { + return fmt.Errorf("failed to restructure direct to outDir: %w", err) + } - protoModuleRepoRoot := filepath.Join(params.outDir, coords.Proto.ArtifactID) - shouldGenerate, err := clirrIgnoreShouldGenerate(coords.Proto.ArtifactID, protoModuleRepoRoot, params.javaAPI.Monolithic) - if err != nil { - return fmt.Errorf("failed to check for clirr ignore file: %w", err) - } - if shouldGenerate { - if err := generateClirrIgnore(protoModuleRepoRoot); err != nil { - return fmt.Errorf("failed to generate clirr ignore file: %w", err) - } + protoModuleRepoRoot := filepath.Join(params.outDir, coords.Proto.ArtifactID) + shouldGenerate, err := clirrIgnoreShouldGenerate(coords.Proto.ArtifactID, protoModuleRepoRoot, params.javaAPI.Monolithic) + if err != nil { + return fmt.Errorf("failed to check for clirr ignore file: %w", err) + } + if shouldGenerate { + if err := generateClirrIgnore(protoModuleRepoRoot); err != nil { + return fmt.Errorf("failed to generate clirr ignore file: %w", err) } + } - // Cleanup intermediate protoc output directory - if err := os.RemoveAll(filepath.Join(params.outDir, params.apiBase)); err != nil { - return fmt.Errorf("failed to cleanup intermediate files: %w", err) - } - return nil + // Cleanup intermediate protoc output directory + if err := os.RemoveAll(filepath.Join(params.outDir, params.apiBase)); err != nil { + return fmt.Errorf("failed to cleanup intermediate files: %w", err) } + return nil } if err := restructureToStaging(params); err != nil { diff --git a/internal/librarian/java/postprocess_new.go b/internal/librarian/java/postprocess_new.go index c255984d657..bf63bee2835 100644 --- a/internal/librarian/java/postprocess_new.go +++ b/internal/librarian/java/postprocess_new.go @@ -17,7 +17,6 @@ package java import ( "context" "fmt" - "os" "path/filepath" "strings" @@ -27,16 +26,15 @@ import ( ) // postProcessLibraryNew implements the new postprocessing flow, bypassing owlbot.py. -// It applies operations from postprocess.yaml, and renders the README.md directly +// It applies post-processing operations configured in librarian.yaml, and renders the README.md directly // on the generated files in their final destinations. func postProcessLibraryNew(ctx context.Context, p libraryPostProcessParams) error { - // 1. Load postprocess.yaml and apply operations - postprocessYamlPath := filepath.Join(p.outDir, "postprocess.yaml") - if _, err := os.Stat(postprocessYamlPath); err == nil { - cfg, err := postprocessing.ParseConfig(ctx, postprocessYamlPath) - if err != nil { - return fmt.Errorf("failed to parse postprocess.yaml: %w", err) + // 1. Load postprocess configuration and apply operations + if p.library.Postprocess != nil { + if err := postprocessing.Validate(p.library.Postprocess); err != nil { + return fmt.Errorf("invalid postprocess config: %w", err) } + cfg := p.library.Postprocess keepSet := make(map[string]bool) for _, k := range p.library.Keep { diff --git a/internal/librarian/java/postprocess_new_test.go b/internal/librarian/java/postprocess_new_test.go index a363bf440a4..ff0b3055884 100644 --- a/internal/librarian/java/postprocess_new_test.go +++ b/internal/librarian/java/postprocess_new_test.go @@ -45,29 +45,6 @@ public class File { t.Fatal(err) } - // Setup postprocess.yaml - postprocessYaml := ` -replace: - - path: "**/File.java" - original: "oldFunc" - replacement: "newFunc" -method_operations: - - path: "**/File.java" - action: delete - func_name: "public void toDelete()" - - path: "**/File.java" - action: duplicate - func_name: "public void newFunc()" - new_name: "newFuncCopy" - - path: "**/File.java" - action: deprecate - func_name: "public void newFuncCopy()" - deprecation_message: "Use newFunc instead." -` - if err := os.WriteFile(filepath.Join(tmpDir, "postprocess.yaml"), []byte(postprocessYaml), 0644); err != nil { - t.Fatal(err) - } - // Write mock template to disk tmplDir := filepath.Join(tmpDir, "template") if err := os.MkdirAll(tmplDir, 0755); err != nil { @@ -88,6 +65,34 @@ method_operations: }, library: &config.Library{ Version: "1.2.3", + Postprocess: &config.Postprocess{ + Replace: []config.ReplaceConfig{ + { + Path: "**/File.java", + Original: "oldFunc", + Replacement: "newFunc", + }, + }, + MethodOperations: []config.MethodOperation{ + { + Path: "**/File.java", + Action: "delete", + FuncName: "public void toDelete()", + }, + { + Path: "**/File.java", + Action: "duplicate", + FuncName: "public void newFunc()", + NewName: "newFuncCopy", + }, + { + Path: "**/File.java", + Action: "deprecate", + FuncName: "public void newFuncCopy()", + DeprecationMessage: "Use newFunc instead.", + }, + }, + }, }, metadata: &repoMetadata{ NamePretty: "My API", diff --git a/internal/librarian/java/postprocess_test.go b/internal/librarian/java/postprocess_test.go index 56955da84ea..cdcb4291f31 100644 --- a/internal/librarian/java/postprocess_test.go +++ b/internal/librarian/java/postprocess_test.go @@ -1219,13 +1219,10 @@ func TestPostProcessLibrary_Branching(t *testing.T) { } }) - t.Run("UseGoPostprocessor true, with yaml", func(t *testing.T) { + t.Run("UseGoPostprocessor true, with postprocess config in library", func(t *testing.T) { outDir := t.TempDir() t.Chdir(outDir) - if err := os.WriteFile(filepath.Join(outDir, "postprocess.yaml"), []byte(""), 0644); err != nil { - t.Fatal(err) - } if err := os.MkdirAll(filepath.Join(outDir, "owl-bot-staging"), 0755); err != nil { t.Fatal(err) } @@ -1236,6 +1233,11 @@ func TestPostProcessLibrary_Branching(t *testing.T) { if err := os.MkdirAll(filepath.Join(outDir, "template"), 0755); err != nil { t.Fatal(err) } + // Write a file to apply replacements on + testFile := filepath.Join(outDir, "TestFile.java") + if err := os.WriteFile(testFile, []byte("Hello World"), 0644); err != nil { + t.Fatal(err) + } if err := os.WriteFile(filepath.Join(outDir, "template", "README.md.go.tmpl"), []byte("dummy"), 0644); err != nil { t.Fatal(err) } @@ -1259,6 +1261,15 @@ func TestPostProcessLibrary_Branching(t *testing.T) { library: &config.Library{ Name: "test-library", Version: "1.2.3", + Postprocess: &config.Postprocess{ + Replace: []config.ReplaceConfig{ + { + Path: "TestFile.java", + Original: "World", + Replacement: "Go", + }, + }, + }, Java: &config.JavaModule{ GroupID: "com.google.cloud", ArtifactID: "test-library", @@ -1269,5 +1280,14 @@ func TestPostProcessLibrary_Branching(t *testing.T) { if err != nil { t.Fatalf("expected no error, got: %v", err) } + + // Verify replacement was applied + content, err := os.ReadFile(testFile) + if err != nil { + t.Fatal(err) + } + if string(content) != "Hello Go" { + t.Errorf("expected Hello Go, got %q", string(content)) + } }) } diff --git a/internal/postprocessing/config.go b/internal/postprocessing/config.go index 45a5b88bf6e..0a5fb3d56c6 100644 --- a/internal/postprocessing/config.go +++ b/internal/postprocessing/config.go @@ -16,57 +16,20 @@ package postprocessing import ( - "context" "errors" "fmt" - "os" "strings" - "github.com/googleapis/librarian/internal/yaml" + "github.com/googleapis/librarian/internal/config" ) var errEmptyNewName = errors.New("new method name is required and cannot be empty") -// Config represents the postprocess.yaml configuration. -type Config struct { - Replace []ReplaceConfig `yaml:"replace,omitempty"` - ReplaceRegex []ReplaceRegexConfig `yaml:"replace_regex,omitempty"` - CopyFile []CopyConfig `yaml:"copy_file,omitempty"` - RemoveFile []string `yaml:"remove_file,omitempty"` - MethodOperations []MethodOperation `yaml:"method_operations,omitempty"` -} - -// MethodOperation represents a method-level operation like delete, duplicate, or deprecate. -type MethodOperation struct { - Path string `yaml:"path"` - Action string `yaml:"action"` - FuncName string `yaml:"func_name"` - NewName string `yaml:"new_name,omitempty"` // Used for duplicate - DeprecationMessage string `yaml:"deprecation_message,omitempty"` // Used for deprecate -} - -// ReplaceConfig represents a replacement rule. -type ReplaceConfig struct { - Path string `yaml:"path"` - Original string `yaml:"original"` - Replacement string `yaml:"replacement"` -} - -// ReplaceRegexConfig represents a regex replacement rule. -type ReplaceRegexConfig struct { - Path string `yaml:"path"` - Pattern string `yaml:"pattern"` - Replacement string `yaml:"replacement"` -} - -// CopyConfig represents a file copy rule. -type CopyConfig struct { - Src string `yaml:"src"` - Dst string `yaml:"dst"` -} - // Validate validates the postprocess configuration. -func (c *Config) Validate() error { +func Validate(c *config.Postprocess) error { + if c == nil { + return nil + } for _, r := range c.Replace { if strings.TrimSpace(r.Original) == "" { return fmt.Errorf("replace rule original text to replace cannot be empty") @@ -101,19 +64,3 @@ func (c *Config) Validate() error { } return nil } - -// ParseConfig parses the postprocess.yaml file. -func ParseConfig(ctx context.Context, path string) (*Config, error) { - data, err := os.ReadFile(path) - if err != nil { - return nil, err - } - configPtr, err := yaml.Unmarshal[Config](data) - if err != nil { - return nil, err - } - if err := configPtr.Validate(); err != nil { - return nil, fmt.Errorf("invalid config: %w", err) - } - return configPtr, nil -} diff --git a/internal/postprocessing/config_test.go b/internal/postprocessing/config_test.go index cd9ef27cf4c..d71cb374e15 100644 --- a/internal/postprocessing/config_test.go +++ b/internal/postprocessing/config_test.go @@ -15,79 +15,36 @@ package postprocessing import ( - "context" "errors" - "os" - "path/filepath" "testing" - "github.com/google/go-cmp/cmp" + "github.com/googleapis/librarian/internal/config" ) -func TestParseConfig(t *testing.T) { - yamlContent := ` -replace: - - path: path/to/file.java - original: "old string" - replacement: "new string" -replace_regex: - - path: path/to/file.java - pattern: "pattern" - replacement: "replacement" -copy_file: - - src: path/to/src.java - dst: path/to/dst.java -remove_file: - - path/to/file_to_remove.java - -method_operations: - - path: path/to/file.java - action: delete - func_name: "public void toDelete()" - - path: path/to/file.java - action: duplicate - func_name: "public void toDuplicate()" - new_name: "duplicated" - - path: path/to/file.java - action: deprecate - func_name: "public void toDeprecate()" - deprecation_message: "Use alternative instead." -` - dir := t.TempDir() - configPath := filepath.Join(dir, "postprocess.yaml") - if err := os.WriteFile(configPath, []byte(yamlContent), 0644); err != nil { - t.Fatal(err) - } - - got, err := ParseConfig(context.Background(), configPath) - if err != nil { - t.Fatal(err) - } - - want := &Config{ - Replace: []ReplaceConfig{ +func TestValidate_Success(t *testing.T) { + c := &config.Postprocess{ + Replace: []config.ReplaceConfig{ { Path: "path/to/file.java", Original: "old string", Replacement: "new string", }, }, - ReplaceRegex: []ReplaceRegexConfig{ + ReplaceRegex: []config.ReplaceRegexConfig{ { Path: "path/to/file.java", Pattern: "pattern", Replacement: "replacement", }, }, - CopyFile: []CopyConfig{ + CopyFile: []config.CopyConfig{ { Src: "path/to/src.java", Dst: "path/to/dst.java", }, }, RemoveFile: []string{"path/to/file_to_remove.java"}, - - MethodOperations: []MethodOperation{ + MethodOperations: []config.MethodOperation{ { Path: "path/to/file.java", Action: "delete", @@ -108,49 +65,92 @@ method_operations: }, } - if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("ParseConfig() mismatch (-want +got):\n%s", diff) - } -} - -func TestParseConfig_FileNotFound(t *testing.T) { - _, err := ParseConfig(context.Background(), "non-existent-file.yaml") - if err == nil { - t.Error("ParseConfig() expected error for non-existent file, got nil") - } -} - -func TestParseConfig_InvalidYAML(t *testing.T) { - dir := t.TempDir() - configPath := filepath.Join(dir, "postprocess.yaml") - if err := os.WriteFile(configPath, []byte("invalid yaml content"), 0644); err != nil { - t.Fatal(err) - } - - _, err := ParseConfig(context.Background(), configPath) - if err == nil { - t.Error("ParseConfig() expected error for invalid YAML, got nil") + if err := Validate(c); err != nil { + t.Fatalf("Validate() expected no error, got: %v", err) } } -func TestParseConfig_InvalidSignature(t *testing.T) { - yamlContent := ` -method_operations: - - path: path/to/file.java - action: delete - func_name: "invalidSignatureNoParentheses" -` - dir := t.TempDir() - configPath := filepath.Join(dir, "postprocess.yaml") - if err := os.WriteFile(configPath, []byte(yamlContent), 0644); err != nil { - t.Fatal(err) - } - - _, err := ParseConfig(context.Background(), configPath) - if err == nil { - t.Fatal("ParseConfig() expected error for invalid signature, got nil") - } - if !errors.Is(err, errInvalidSignature) { - t.Errorf("expected error to wrap errInvalidSignature, got %v", err) +func TestValidate_Errors(t *testing.T) { + for _, test := range []struct { + name string + config *config.Postprocess + wantErr error + }{ + { + name: "invalid signature for delete", + config: &config.Postprocess{ + MethodOperations: []config.MethodOperation{ + { + Path: "path/to/file.java", + Action: "delete", + FuncName: "invalidSignature", + }, + }, + }, + wantErr: errInvalidSignature, + }, + { + name: "invalid signature for duplicate", + config: &config.Postprocess{ + MethodOperations: []config.MethodOperation{ + { + Path: "path/to/file.java", + Action: "duplicate", + FuncName: "invalidSignature", + NewName: "foo", + }, + }, + }, + wantErr: errInvalidSignature, + }, + { + name: "missing new name for duplicate", + config: &config.Postprocess{ + MethodOperations: []config.MethodOperation{ + { + Path: "path/to/file.java", + Action: "duplicate", + FuncName: "void foo()", + NewName: "", + }, + }, + }, + wantErr: errEmptyNewName, + }, + { + name: "invalid signature for deprecate", + config: &config.Postprocess{ + MethodOperations: []config.MethodOperation{ + { + Path: "path/to/file.java", + Action: "deprecate", + FuncName: "invalidSignature", + DeprecationMessage: "foo", + }, + }, + }, + wantErr: errInvalidSignature, + }, + { + name: "missing message for deprecate", + config: &config.Postprocess{ + MethodOperations: []config.MethodOperation{ + { + Path: "path/to/file.java", + Action: "deprecate", + FuncName: "void foo()", + DeprecationMessage: "", + }, + }, + }, + wantErr: errEmptyDeprecationMessage, + }, + } { + t.Run(test.name, func(t *testing.T) { + err := Validate(test.config) + if !errors.Is(err, test.wantErr) { + t.Errorf("Validate() error = %v, wantErr %v", err, test.wantErr) + } + }) } } From f0c1611ca3e95b9293597853681a9e27811d34bb Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Sat, 27 Jun 2026 18:22:30 -0400 Subject: [PATCH 050/108] feat(internal/librarian/java): fallback to embedded template in RenderREADME Uses go:embed to load default template and uses it as fallback if template file is missing on disk. --- internal/librarian/java/template_render.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/internal/librarian/java/template_render.go b/internal/librarian/java/template_render.go index 68ba569b581..1db99d082ff 100644 --- a/internal/librarian/java/template_render.go +++ b/internal/librarian/java/template_render.go @@ -15,6 +15,8 @@ package java import ( + "embed" + "errors" "fmt" "os" "path/filepath" @@ -25,6 +27,9 @@ import ( "github.com/iancoleman/strcase" ) +//go:embed template/README.md.go.tmpl +var defaultTemplateFS embed.FS + // RenderREADME renders the README.md file using the template and metadata. // dir is the directory containing where README.md will be written. func RenderREADME(dir string, metadata *repoMetadata, bomVersion, libraryVersion string) error { @@ -130,7 +135,13 @@ func RenderREADME(dir string, metadata *repoMetadata, bomVersion, libraryVersion templatePath := filepath.Join(dir, "template", "README.md.go.tmpl") tmplBytes, err := os.ReadFile(templatePath) if err != nil { - return fmt.Errorf("failed to read template from %s: %w", templatePath, err) + if errors.Is(err, os.ErrNotExist) { + // Fallback to embedded default template + tmplBytes, err = defaultTemplateFS.ReadFile("template/README.md.go.tmpl") + } + if err != nil { + return fmt.Errorf("failed to read template: %w", err) + } } tmpl, err := template.New("README").Parse(string(tmplBytes)) From 095ebecb3ac07069a4a71b60611849803065bf2b Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Sat, 27 Jun 2026 18:44:06 -0400 Subject: [PATCH 051/108] refactor(internal/librarian/java): adjust function orders and enhance text replacement in glob patterns Reorders decamelize, isProductionSample, and extractTitle helpers. Updates applyToFiles to tolerate ErrTextNotFound in glob matches if at least one file is replaced. --- internal/librarian/java/postprocess_new.go | 12 ++++ internal/librarian/java/readme.go | 77 ++++++++++------------ 2 files changed, 46 insertions(+), 43 deletions(-) diff --git a/internal/librarian/java/postprocess_new.go b/internal/librarian/java/postprocess_new.go index bf63bee2835..71f4f19b52b 100644 --- a/internal/librarian/java/postprocess_new.go +++ b/internal/librarian/java/postprocess_new.go @@ -16,6 +16,7 @@ package java import ( "context" + "errors" "fmt" "path/filepath" "strings" @@ -156,14 +157,25 @@ func applyToFiles(outDir string, pathPattern string, keepSet map[string]bool, ac if err != nil { return fmt.Errorf("failed to resolve glob for %s: %w", pathPattern, err) } + isGlob := strings.ContainsAny(pathPattern, "*?[]{}") + var replacedAny bool + var lastTextNotFoundErr error for _, file := range files { relPath, _ := filepath.Rel(outDir, file) if shouldPreserve(filepath.ToSlash(relPath), keepSet) { continue } if err := action(file); err != nil { + if isGlob && errors.Is(err, postprocessing.ErrTextNotFound) { + lastTextNotFoundErr = err + continue + } return err } + replacedAny = true + } + if isGlob && len(files) > 0 && !replacedAny { + return lastTextNotFoundErr } return nil } diff --git a/internal/librarian/java/readme.go b/internal/librarian/java/readme.go index 78147e322e7..adbee5d875f 100644 --- a/internal/librarian/java/readme.go +++ b/internal/librarian/java/readme.go @@ -33,8 +33,8 @@ var ( openExcludeRegex = regexp.MustCompile(`\[START_EXCLUDE\]`) closeExcludeRegex = regexp.MustCompile(`\[END_EXCLUDE\]`) - reDecamelize1 = regexp.MustCompile(`([A-Z]+)([A-Z])([a-z0-9])`) - reDecamelize2 = regexp.MustCompile(`([a-z0-9])([A-Z])`) + // Matches lowercase/digit followed by uppercase (e.g., "FooBar" -> "Foo Bar"). + camelCaseRegexp = regexp.MustCompile(`([a-z0-9])([A-Z])`) // reTitle matches a "sample-metadata:" marker followed immediately by a "title:" line on the next comment line. reTitle = regexp.MustCompile(`sample-metadata:\s*\n\s*(?://|#)\s*title:\s*(.*)`) @@ -46,47 +46,6 @@ var ( errEmptyTitle = errors.New("title value cannot be empty") ) -// decamelize converts CamelCase or PascalCase titles into space-separated words, -// exactly reproducing Python synthtool's _decamelize(value: str). -func decamelize(value string) string { - if value == "" { - return "" - } - r := []rune(value) - r[0] = unicode.ToUpper(r[0]) - s := string(r) - - s = reDecamelize1.ReplaceAllString(s, "${1} ${2}${3}") - return reDecamelize2.ReplaceAllString(s, "${1} ${2}") -} - -// isProductionSample reports whether the given path represents a production Java source file -// located under a standard "src/main/java" path. -func isProductionSample(path string) bool { - slashed := filepath.ToSlash(path) - return strings.HasSuffix(slashed, ".java") && - (strings.HasPrefix(slashed, "src/main/java/") || strings.Contains(slashed, "/src/main/java/")) -} - -// extractTitle extracts and validates the title override from Java comment blocks. -// It expects a "title:" line to immediately follow the "sample-metadata:" marker. -// Returns an error if the marker is present but the title line is missing, malformed, or empty. -func extractTitle(content string) (string, error) { - if !strings.Contains(content, "sample-metadata:") { - return "", nil - } - matches := reTitle.FindStringSubmatch(content) - if len(matches) < 2 { - return "", errMissingTitle - } - // Trim surrounding whitespace, quotes, and carriage returns. - title := strings.Trim(matches[1], " \t\"'\r\n") - if title == "" { - return "", errEmptyTitle - } - return title, nil -} - // Sample represents a Java code sample and its metadata for README generation. type Sample struct { Title string @@ -147,6 +106,38 @@ func ExtractSamples(dir string) ([]Sample, error) { return samples, nil } +// decamelize converts CamelCase string to space-separated string (e.g. "CamelCase" -> "Camel Case"). +func decamelize(value string) string { + return strings.TrimSpace(camelCaseRegexp.ReplaceAllString(value, `$1 $2`)) +} + +// isProductionSample reports whether the given path represents a production Java source file +// located under a standard "src/main/java" path. +func isProductionSample(path string) bool { + slashed := filepath.ToSlash(path) + return strings.HasSuffix(slashed, ".java") && + (strings.HasPrefix(slashed, "src/main/java/") || strings.Contains(slashed, "/src/main/java/")) +} + +// extractTitle extracts and validates the title override from Java comment blocks. +// It expects a "title:" line to immediately follow the "sample-metadata:" marker. +// Returns an error if the marker is present but the title line is missing, malformed, or empty. +func extractTitle(content string) (string, error) { + if !strings.Contains(content, "sample-metadata:") { + return "", nil + } + matches := reTitle.FindStringSubmatch(content) + if len(matches) < 2 { + return "", errMissingTitle + } + // Trim surrounding whitespace, quotes, and carriage returns. + title := strings.Trim(matches[1], " \t\"'\r\n") + if title == "" { + return "", errEmptyTitle + } + return title, nil +} + // ExtractSnippets walks the "samples" directory locating *.java and *.xml files. // It line-scans for [START name] and [END name] tags while supporting [START_EXCLUDE] blocks, // returning trimmed minimum plain-space indentation blocks. From 1f046cd6d331a666d360f35f2a0b512c266ba705 Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Sat, 27 Jun 2026 18:53:47 -0400 Subject: [PATCH 052/108] build: fix compile errors by exporting ErrTextNotFound and aligning readme tests with main --- internal/librarian/java/readme.go | 1 - internal/librarian/java/readme_test.go | 20 +++++++++----------- internal/postprocessing/fileops.go | 8 ++++---- internal/postprocessing/fileops_test.go | 4 ++-- 4 files changed, 15 insertions(+), 18 deletions(-) diff --git a/internal/librarian/java/readme.go b/internal/librarian/java/readme.go index adbee5d875f..1b5dda9168a 100644 --- a/internal/librarian/java/readme.go +++ b/internal/librarian/java/readme.go @@ -24,7 +24,6 @@ import ( "regexp" "sort" "strings" - "unicode" ) var ( diff --git a/internal/librarian/java/readme_test.go b/internal/librarian/java/readme_test.go index 990cf495161..e4ef754d824 100644 --- a/internal/librarian/java/readme_test.go +++ b/internal/librarian/java/readme_test.go @@ -28,11 +28,9 @@ func TestDecamelize(t *testing.T) { input string expected string }{ - {"requesterPays", "Requester Pays"}, - {"ACLBatman", "ACL Batman"}, - {"NativeImageLoggingSample", "Native Image Logging Sample"}, - {"simpleTest", "Simple Test"}, - {"", ""}, + {"CamelCase", "Camel Case"}, + {"Word", "Word"}, + {"Camel Case", "Camel Case"}, {"IamPolicy", "Iam Policy"}, {"GcsBucket", "Gcs Bucket"}, } { @@ -190,20 +188,20 @@ public class RequesterPays {}` if err := os.WriteFile(file1, []byte(content1), 0644); err != nil { t.Fatal(err) } - file2 := filepath.Join(samplesDir, "demoSample.java") - content2 := `public class demoSample {}` + file2 := filepath.Join(samplesDir, "DemoSample.java") + content2 := `public class DemoSample {}` if err := os.WriteFile(file2, []byte(content2), 0644); err != nil { t.Fatal(err) } }, want: []Sample{ { - Title: "Custom Title Override", - File: "samples/src/main/java/RequesterPays.java", + Title: "Demo Sample", + File: "samples/src/main/java/DemoSample.java", }, { - Title: "Demo Sample", - File: "samples/src/main/java/demoSample.java", + Title: "Custom Title Override", + File: "samples/src/main/java/RequesterPays.java", }, }, }, diff --git a/internal/postprocessing/fileops.go b/internal/postprocessing/fileops.go index 14b038cff6e..15d07c6e351 100644 --- a/internal/postprocessing/fileops.go +++ b/internal/postprocessing/fileops.go @@ -25,8 +25,8 @@ import ( "github.com/googleapis/librarian/internal/filesystem" ) -// errTextNotFound is returned when the target text or pattern is not found in the file. -var errTextNotFound = errors.New("text not found") +// ErrTextNotFound is returned when the target text or pattern is not found in the file. +var ErrTextNotFound = errors.New("text not found") // CopyFile copies a single file from the src path to the dst path. // It acts as a wrapper around filesystem.CopyFile to provide a unified @@ -49,7 +49,7 @@ func Replace(path, original, replacement string) error { } oldBytes := []byte(original) if !bytes.Contains(content, oldBytes) { - return fmt.Errorf("%w: %q in file %s", errTextNotFound, original, path) + return fmt.Errorf("%w: %q in file %s", ErrTextNotFound, original, path) } newContent := bytes.ReplaceAll(content, oldBytes, []byte(replacement)) return os.WriteFile(path, newContent, 0644) @@ -71,7 +71,7 @@ func ReplaceRegex(path, pattern, replacement string) error { return err } if !re.Match(content) { - return fmt.Errorf("%w: pattern %q in file %s", errTextNotFound, pattern, path) + return fmt.Errorf("%w: pattern %q in file %s", ErrTextNotFound, pattern, path) } newContent := re.ReplaceAll(content, []byte(replacement)) return os.WriteFile(path, newContent, 0644) diff --git a/internal/postprocessing/fileops_test.go b/internal/postprocessing/fileops_test.go index 10aa4df27cb..0756ec76450 100644 --- a/internal/postprocessing/fileops_test.go +++ b/internal/postprocessing/fileops_test.go @@ -182,7 +182,7 @@ func TestReplace_Error(t *testing.T) { content: "Hello World", original: "Apple", replacement: "Go", - wantErr: errTextNotFound, + wantErr: ErrTextNotFound, }, } { t.Run(test.name, func(t *testing.T) { @@ -221,7 +221,7 @@ func TestReplaceRegex_Error(t *testing.T) { content: "Hello World", pattern: `\d+`, replacement: "Number", - wantErr: errTextNotFound, + wantErr: ErrTextNotFound, }, } { t.Run(test.name, func(t *testing.T) { From c6a0faaf84983d62ac1a6beac796000e11e5401f Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Sat, 27 Jun 2026 19:01:26 -0400 Subject: [PATCH 053/108] test(internal/librarian/java): align TestDecamelize and TestIsProductionSample structure with main --- internal/librarian/java/readme_test.go | 115 +++++++++++++++---------- 1 file changed, 68 insertions(+), 47 deletions(-) diff --git a/internal/librarian/java/readme_test.go b/internal/librarian/java/readme_test.go index e4ef754d824..a8472bb6b48 100644 --- a/internal/librarian/java/readme_test.go +++ b/internal/librarian/java/readme_test.go @@ -25,19 +25,76 @@ import ( func TestDecamelize(t *testing.T) { for _, test := range []struct { - input string - expected string + name string + input string + want string }{ - {"CamelCase", "Camel Case"}, - {"Word", "Word"}, - {"Camel Case", "Camel Case"}, - {"IamPolicy", "Iam Policy"}, - {"GcsBucket", "Gcs Bucket"}, + { + name: "camel case", + input: "CamelCase", + want: "Camel Case", + }, + { + name: "simple word", + input: "Word", + want: "Word", + }, + { + name: "already separated", + input: "Camel Case", + want: "Camel Case", + }, + { + name: "java acronym IamPolicy", + input: "IamPolicy", + want: "Iam Policy", + }, + { + name: "java acronym GcsBucket", + input: "GcsBucket", + want: "Gcs Bucket", + }, } { - t.Run(test.input, func(t *testing.T) { - actual := decamelize(test.input) - if actual != test.expected { - t.Errorf("decamelize(%q) = %q; expected %q", test.input, actual, test.expected) + t.Run(test.name, func(t *testing.T) { + got := decamelize(test.input) + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestIsProductionSample(t *testing.T) { + for _, test := range []struct { + name string + path string + want bool + }{ + { + name: "valid production sample", + path: "samples/src/main/java/com/example/Sample.java", + want: true, + }, + { + name: "valid production sample at root", + path: "src/main/java/com/example/Sample.java", + want: true, + }, + { + name: "non-java file", + path: "samples/src/main/java/README.md", + want: false, + }, + { + name: "not in src/main/java", + path: "samples/src/test/java/com/example/Sample.java", + want: false, + }, + } { + t.Run(test.name, func(t *testing.T) { + got := isProductionSample(test.path) + if got != test.want { + t.Errorf("isProductionSample() = %t, want %t", got, test.want) } }) } @@ -125,42 +182,6 @@ func TestExtractTitle_Error(t *testing.T) { } } -func TestIsProductionSample(t *testing.T) { - for _, test := range []struct { - name string - path string - want bool - }{ - { - name: "valid production sample", - path: "samples/src/main/java/com/example/Sample.java", - want: true, - }, - { - name: "valid production sample at root", - path: "src/main/java/com/example/Sample.java", - want: true, - }, - { - name: "non-java file", - path: "samples/src/main/java/README.md", - want: false, - }, - { - name: "not in src/main/java", - path: "samples/src/test/java/com/example/Sample.java", - want: false, - }, - } { - t.Run(test.name, func(t *testing.T) { - got := isProductionSample(test.path) - if got != test.want { - t.Errorf("isProductionSample() = %t, want %t", got, test.want) - } - }) - } -} - func TestExtractSamples(t *testing.T) { for _, test := range []struct { name string From 3ecc3318f828a552782dc970c3d78362693958c9 Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Sat, 27 Jun 2026 20:20:51 -0400 Subject: [PATCH 054/108] fix(java): remove duplicate 'src' directory prefix in restructureToStaging for monolithic libraries --- internal/librarian/java/postprocess.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/internal/librarian/java/postprocess.go b/internal/librarian/java/postprocess.go index 6fce9595e10..160f2df93dd 100644 --- a/internal/librarian/java/postprocess.go +++ b/internal/librarian/java/postprocess.go @@ -305,9 +305,6 @@ func removeConflictingFiles(protoSrcDir string) error { func restructureToStaging(params postProcessParams) error { stagingDir := stagingDir(params.outDir) destRoot := filepath.Join(stagingDir, params.apiBase) - if params.javaAPI.Monolithic { - destRoot = filepath.Join(destRoot, "src") - } if err := os.MkdirAll(destRoot, 0755); err != nil { return fmt.Errorf("failed to create staging directory: %w", err) } From 9228f24bf16bd763cc047ae40846cffbbde7eb6e Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Sun, 28 Jun 2026 11:50:41 -0400 Subject: [PATCH 055/108] refactor(java): enhance postprocess keep logic, resource handling and style compliance - Extract newKeepSet and path.Clean normalization for robust file preservation - Refactor ExtractSnippets with extractSnippetsFromFile helper to ensure defer f.Close() - Migrate os.IsNotExist to errors.Is(err, fs.ErrNotExist) across java package - Standardize log/slog logging and correct unexported acronym casing - Replace manual test assertions with cmp.Diff and remove non-thread-safe t.Chdir --- internal/librarian/java/clean.go | 44 ++++++---- internal/librarian/java/generate_test.go | 5 +- internal/librarian/java/postprocess.go | 13 +-- internal/librarian/java/postprocess_new.go | 16 ++-- .../librarian/java/postprocess_new_test.go | 1 - internal/librarian/java/postprocess_test.go | 2 - internal/librarian/java/readme.go | 81 ++++++++++--------- internal/librarian/java/template_render.go | 30 ++++--- .../librarian/java/template_render_test.go | 35 ++++++-- 9 files changed, 134 insertions(+), 93 deletions(-) diff --git a/internal/librarian/java/clean.go b/internal/librarian/java/clean.go index 044e4599221..f559676332d 100644 --- a/internal/librarian/java/clean.go +++ b/internal/librarian/java/clean.go @@ -17,8 +17,8 @@ package java import ( "bufio" "errors" - "fmt" "io" + "log/slog" "io/fs" "os" "path" @@ -56,11 +56,8 @@ var ( // It targets patterns like proto-*, grpc-*, and the main GAPIC module. func Clean(library *config.Library) error { patterns := cleanPatterns(library) - fmt.Printf("Clean patterns for %s: %v\n", library.Output, patterns) - keepSet := make(map[string]bool) - for _, k := range library.Keep { - keepSet[strings.TrimSuffix(filepath.ToSlash(k), "/")] = true - } + slog.Debug("Clean patterns", "output", library.Output, "patterns", patterns) + keepSet := newKeepSet(library.Keep) for pattern, useMarker := range patterns { matches, err := filepath.Glob(filepath.Join(library.Output, pattern)) if err != nil { @@ -145,8 +142,7 @@ func cleanPath(targetPath, root string, keepSet map[string]bool, useMarker bool) return nil } } - fmt.Println("DELETING:", path) - fmt.Println("DELETING:", path) + slog.Info("Deleting path", "path", path) return os.Remove(path) }) if err != nil && !errors.Is(err, fs.ErrNotExist) { @@ -173,15 +169,28 @@ func isDirNotEmpty(err error) bool { return errors.Is(err, syscall.ENOTEMPTY) || errors.Is(err, syscall.EEXIST) } -// shouldPreserve returns true if the given slash-separated path should be preserved -// based on the keepSet or standard preservation patterns. -// It also checks if any ancestor directory is in the keepSet. -func shouldPreserve(p string, keepSet map[string]bool) bool { - if keepSet[p] || isDefaultPreserved(p) { +// newKeepSet normalizes a list of keep paths using [path.Clean] and returns +// a map for fast lookup. +func newKeepSet(keep []string) map[string]bool { + keepSet := make(map[string]bool) + for _, k := range keep { + normalized := path.Clean(filepath.ToSlash(k)) + if normalized == "." { + continue + } + keepSet[strings.TrimSuffix(normalized, "/")] = true + } + return keepSet +} + +// isKept returns true if the path is explicitly kept by the user. +func isKept(p string, keepSet map[string]bool) bool { + cleanP := path.Clean(p) + if keepSet[cleanP] { return true } - dir := path.Dir(p) + dir := path.Dir(cleanP) for dir != "." && dir != "/" && dir != "" { if keepSet[dir] { return true @@ -191,6 +200,13 @@ func shouldPreserve(p string, keepSet map[string]bool) bool { return false } +// shouldPreserve returns true if the given slash-separated path should be preserved +// based on the keepSet or standard preservation patterns. +// It also checks if any ancestor directory is in the keepSet. +func shouldPreserve(p string, keepSet map[string]bool) bool { + return isKept(p, keepSet) || isDefaultPreserved(p) +} + func isDefaultPreserved(path string) bool { return itTestRegexp.MatchString(path) || versionRegexp.MatchString(path) } diff --git a/internal/librarian/java/generate_test.go b/internal/librarian/java/generate_test.go index 1a4cd04a097..0eeb6a4e713 100644 --- a/internal/librarian/java/generate_test.go +++ b/internal/librarian/java/generate_test.go @@ -17,6 +17,7 @@ package java import ( "context" "errors" + "io/fs" "os" "path/filepath" "strings" @@ -1113,7 +1114,7 @@ func TestGenerateAPI_Gating(t *testing.T) { if gotProtoDir { resNameFile := filepath.Join(stagingProtoPath, "com", "google", "cloud", "secretmanager", "v1", "SecretName.java") _, errRes := os.Stat(resNameFile) - gotResNameFiles := !os.IsNotExist(errRes) + gotResNameFiles := !errors.Is(errRes, fs.ErrNotExist) if gotResNameFiles != test.wantResNameFiles { t.Errorf("gotResNameFiles = %v, want %v (file: %s)", gotResNameFiles, test.wantResNameFiles, resNameFile) } @@ -1125,7 +1126,7 @@ func TestGenerateAPI_Gating(t *testing.T) { func assertDirExists(t *testing.T, path string, want bool, desc string) bool { t.Helper() _, err := os.Stat(path) - got := !os.IsNotExist(err) + got := !errors.Is(err, fs.ErrNotExist) if got != want { t.Errorf("expected %s existence to be %v, got %v (path: %s)", desc, want, got, path) } diff --git a/internal/librarian/java/postprocess.go b/internal/librarian/java/postprocess.go index 160f2df93dd..37bd2872734 100644 --- a/internal/librarian/java/postprocess.go +++ b/internal/librarian/java/postprocess.go @@ -144,10 +144,7 @@ func postProcessAPI(ctx context.Context, params postProcessParams) error { coords := params.coords() if params.useGoPostprocessor { - keepSet := make(map[string]bool) - for _, k := range params.library.Keep { - keepSet[strings.TrimSuffix(filepath.ToSlash(k), "/")] = true - } + keepSet := newKeepSet(params.library.Keep) if err := restructureModules(params, params.outDir, keepSet, params.outDir); err != nil { return fmt.Errorf("failed to restructure direct to outDir: %w", err) } @@ -495,14 +492,10 @@ func copyProtos(protos []protoFileToCopy, destDir string) error { // then matched against the library's Keep configuration. func removeKeptFilesFromStaging(library *config.Library, outDir string) error { stagingDir := stagingDir(outDir) - if _, err := os.Stat(stagingDir); os.IsNotExist(err) { + if _, err := os.Stat(stagingDir); errors.Is(err, fs.ErrNotExist) { return nil } - keepSet := make(map[string]bool) - for _, keep := range library.Keep { - normalized := strings.TrimSuffix(filepath.ToSlash(keep), "/") - keepSet[normalized] = true - } + keepSet := newKeepSet(library.Keep) return filepath.WalkDir(stagingDir, func(path string, d os.DirEntry, err error) error { if err != nil { return err diff --git a/internal/librarian/java/postprocess_new.go b/internal/librarian/java/postprocess_new.go index 71f4f19b52b..076106df5e7 100644 --- a/internal/librarian/java/postprocess_new.go +++ b/internal/librarian/java/postprocess_new.go @@ -30,6 +30,8 @@ import ( // It applies post-processing operations configured in librarian.yaml, and renders the README.md directly // on the generated files in their final destinations. func postProcessLibraryNew(ctx context.Context, p libraryPostProcessParams) error { + keepSet := newKeepSet(p.library.Keep) + // 1. Load postprocess configuration and apply operations if p.library.Postprocess != nil { if err := postprocessing.Validate(p.library.Postprocess); err != nil { @@ -37,11 +39,6 @@ func postProcessLibraryNew(ctx context.Context, p libraryPostProcessParams) erro } cfg := p.library.Postprocess - keepSet := make(map[string]bool) - for _, k := range p.library.Keep { - keepSet[strings.TrimSuffix(filepath.ToSlash(k), "/")] = true - } - // 1. Apply Copies for _, c := range cfg.CopyFile { if shouldPreserve(filepath.ToSlash(c.Dst), keepSet) { @@ -138,7 +135,7 @@ func postProcessLibraryNew(ctx context.Context, p libraryPostProcessParams) erro return fmt.Errorf("failed to find BOM version: %w", err) } - if err := RenderREADME(p.outDir, p.metadata, bomVersion, libraryVersion); err != nil { + if err := RenderREADME(p.outDir, p.metadata, bomVersion, libraryVersion, keepSet); err != nil { return fmt.Errorf("failed to render README: %w", err) } @@ -161,8 +158,11 @@ func applyToFiles(outDir string, pathPattern string, keepSet map[string]bool, ac var replacedAny bool var lastTextNotFoundErr error for _, file := range files { - relPath, _ := filepath.Rel(outDir, file) - if shouldPreserve(filepath.ToSlash(relPath), keepSet) { + relPath, err := filepath.Rel(outDir, file) + if err != nil { + return fmt.Errorf("failed to get relative path for %s: %w", file, err) + } + if isKept(filepath.ToSlash(relPath), keepSet) { continue } if err := action(file); err != nil { diff --git a/internal/librarian/java/postprocess_new_test.go b/internal/librarian/java/postprocess_new_test.go index ff0b3055884..4f667806dbc 100644 --- a/internal/librarian/java/postprocess_new_test.go +++ b/internal/librarian/java/postprocess_new_test.go @@ -25,7 +25,6 @@ import ( func TestPostProcessLibraryNew(t *testing.T) { tmpDir := t.TempDir() - t.Chdir(tmpDir) // Setup structure directly in outDir destDir := filepath.Join(tmpDir, "my-module", "src", "main", "java") diff --git a/internal/librarian/java/postprocess_test.go b/internal/librarian/java/postprocess_test.go index cdcb4291f31..20410e6bb5e 100644 --- a/internal/librarian/java/postprocess_test.go +++ b/internal/librarian/java/postprocess_test.go @@ -1172,7 +1172,6 @@ func TestPostProcessLibrary_Branching(t *testing.T) { t.Run("UseGoPostprocessor true, no yaml, success", func(t *testing.T) { outDir := t.TempDir() - t.Chdir(outDir) if err := os.MkdirAll(filepath.Join(outDir, "owl-bot-staging"), 0755); err != nil { t.Fatal(err) @@ -1221,7 +1220,6 @@ func TestPostProcessLibrary_Branching(t *testing.T) { t.Run("UseGoPostprocessor true, with postprocess config in library", func(t *testing.T) { outDir := t.TempDir() - t.Chdir(outDir) if err := os.MkdirAll(filepath.Join(outDir, "owl-bot-staging"), 0755); err != nil { t.Fatal(err) diff --git a/internal/librarian/java/readme.go b/internal/librarian/java/readme.go index 1b5dda9168a..51962c7cca7 100644 --- a/internal/librarian/java/readme.go +++ b/internal/librarian/java/readme.go @@ -146,7 +146,7 @@ func ExtractSnippets(dir string) (map[string]string, error) { err := filepath.WalkDir(samplesDir, func(path string, d os.DirEntry, err error) error { if err != nil { - if errors.Is(err, fs.ErrNotExist) || os.IsNotExist(err) { + if errors.Is(err, fs.ErrNotExist) { return nil } return err @@ -181,44 +181,9 @@ func ExtractSnippets(dir string) (map[string]string, error) { snippetLines := make(map[string][]string) for _, file := range files { - f, err := os.Open(file) - if err != nil { - continue - } - - openSnippets := make(map[string]bool) - excluding := false - scanner := bufio.NewScanner(f) - scanner.Buffer(make([]byte, 64*1024), 10*1024*1024) - - for scanner.Scan() { - line := scanner.Text() - openMatch := openSnippetRegex.FindStringSubmatch(line) - closeMatch := closeSnippetRegex.FindStringSubmatch(line) - - if len(openMatch) > 1 && !excluding { - name := openMatch[1] - openSnippets[name] = true - if _, exists := snippetLines[name]; !exists { - snippetLines[name] = []string{} - } - } else if len(closeMatch) > 1 && !excluding { - delete(openSnippets, closeMatch[1]) - } else if openExcludeRegex.MatchString(line) { - excluding = true - } else if closeExcludeRegex.MatchString(line) { - excluding = false - } else if !excluding { - for s := range openSnippets { - snippetLines[s] = append(snippetLines[s], line) - } - } - } - if err := scanner.Err(); err != nil { - f.Close() + if err := extractSnippetsFromFile(file, snippetLines); err != nil { return nil, err } - f.Close() } if len(snippetLines) == 0 { @@ -267,3 +232,45 @@ func trimLeadingWhitespace(lines []string) string { } return sb.String() } + +// extractSnippetsFromFile parses a single file to extract tagged code snippets. +func extractSnippetsFromFile(file string, snippetLines map[string][]string) error { + f, err := os.Open(file) + if err != nil { + return fmt.Errorf("failed to open file %s: %w", file, err) + } + defer f.Close() + + openSnippets := make(map[string]bool) + excluding := false + scanner := bufio.NewScanner(f) + scanner.Buffer(make([]byte, 64*1024), 10*1024*1024) + + for scanner.Scan() { + line := scanner.Text() + openMatch := openSnippetRegex.FindStringSubmatch(line) + closeMatch := closeSnippetRegex.FindStringSubmatch(line) + + if len(openMatch) > 1 && !excluding { + name := openMatch[1] + openSnippets[name] = true + if _, exists := snippetLines[name]; !exists { + snippetLines[name] = []string{} + } + } else if len(closeMatch) > 1 && !excluding { + delete(openSnippets, closeMatch[1]) + } else if openExcludeRegex.MatchString(line) { + excluding = true + } else if closeExcludeRegex.MatchString(line) { + excluding = false + } else if !excluding { + for s := range openSnippets { + snippetLines[s] = append(snippetLines[s], line) + } + } + } + if err := scanner.Err(); err != nil { + return fmt.Errorf("failed scanning file %s: %w", file, err) + } + return nil +} diff --git a/internal/librarian/java/template_render.go b/internal/librarian/java/template_render.go index 1db99d082ff..cb57c39ddd8 100644 --- a/internal/librarian/java/template_render.go +++ b/internal/librarian/java/template_render.go @@ -18,6 +18,7 @@ import ( "embed" "errors" "fmt" + "io/fs" "os" "path/filepath" "strings" @@ -28,16 +29,20 @@ import ( ) //go:embed template/README.md.go.tmpl -var defaultTemplateFS embed.FS +var defaultTemplateFs embed.FS // RenderREADME renders the README.md file using the template and metadata. // dir is the directory containing where README.md will be written. -func RenderREADME(dir string, metadata *repoMetadata, bomVersion, libraryVersion string) error { +func RenderREADME(dir string, metadata *repoMetadata, bomVersion, libraryVersion string, keepSet map[string]bool) error { + outputPath := filepath.Join(dir, "README.md") + if isKept("README.md", keepSet) { + return nil + } + partialsPath := filepath.Join(dir, ".readme-partials.yaml") - if _, err := os.Stat(partialsPath); os.IsNotExist(err) { + if _, err := os.Stat(partialsPath); errors.Is(err, fs.ErrNotExist) { partialsPath = filepath.Join(dir, ".readme-partials.yml") } - outputPath := filepath.Join(dir, "README.md") // Read partials if exist var partials map[string]interface{} @@ -62,13 +67,13 @@ func RenderREADME(dir string, metadata *repoMetadata, bomVersion, libraryVersion // Prepare data for template distName := metadata.DistributionName distParts := strings.Split(distName, ":") - groupID := "" - artifactID := "" + groupId := "" + artifactId := "" if len(distParts) > 0 { - groupID = distParts[0] + groupId = distParts[0] } if len(distParts) > 1 { - artifactID = distParts[1] + artifactId = distParts[1] } repoName := metadata.Repo @@ -84,7 +89,6 @@ func RenderREADME(dir string, metadata *repoMetadata, bomVersion, libraryVersion if minJavaVersion == 0 { minJavaVersion = 8 // Default to Java 8 } - fmt.Println("DEBUG minJavaVersion:", minJavaVersion) samples, err := ExtractSamples(dir) if err != nil { @@ -121,8 +125,8 @@ func RenderREADME(dir string, metadata *repoMetadata, bomVersion, libraryVersion LibraryVersion string }{ Metadata: templateMetadata, - GroupID: groupID, - ArtifactID: artifactID, + GroupID: groupId, + ArtifactID: artifactId, Version: version, RepoShort: repoShort, MigratedSplitRepo: false, @@ -135,9 +139,9 @@ func RenderREADME(dir string, metadata *repoMetadata, bomVersion, libraryVersion templatePath := filepath.Join(dir, "template", "README.md.go.tmpl") tmplBytes, err := os.ReadFile(templatePath) if err != nil { - if errors.Is(err, os.ErrNotExist) { + if errors.Is(err, fs.ErrNotExist) { // Fallback to embedded default template - tmplBytes, err = defaultTemplateFS.ReadFile("template/README.md.go.tmpl") + tmplBytes, err = defaultTemplateFs.ReadFile("template/README.md.go.tmpl") } if err != nil { return fmt.Errorf("failed to read template: %w", err) diff --git a/internal/librarian/java/template_render_test.go b/internal/librarian/java/template_render_test.go index fc57b79993b..a784d031f28 100644 --- a/internal/librarian/java/template_render_test.go +++ b/internal/librarian/java/template_render_test.go @@ -20,6 +20,8 @@ import ( "strings" "testing" "text/template" + + "github.com/google/go-cmp/cmp" ) func TestRenderREADME(t *testing.T) { @@ -50,7 +52,7 @@ About: {{ .Metadata.Partials.About }} } // Test case 1: Without partials - err := RenderREADME(tmpDir, metadata, "1.0.0-BOM", "1.2.3-LIB") + err := RenderREADME(tmpDir, metadata, "1.0.0-BOM", "1.2.3-LIB", nil) if err != nil { t.Fatal(err) } @@ -67,8 +69,8 @@ Version: 1.2.3-LIB BOMVersion: 1.0.0-BOM LibraryVersion: 1.2.3-LIB ` - if strings.TrimSpace(string(outputContent)) != strings.TrimSpace(expected) { - t.Errorf("expected:\n%s\ngot:\n%s", expected, string(outputContent)) + if diff := cmp.Diff(strings.TrimSpace(expected), strings.TrimSpace(string(outputContent))); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) } // Test case 2: With partials @@ -79,7 +81,7 @@ LibraryVersion: 1.2.3-LIB t.Fatal(err) } - err = RenderREADME(tmpDir, metadata, "1.0.0-BOM", "1.2.3-LIB") + err = RenderREADME(tmpDir, metadata, "1.0.0-BOM", "1.2.3-LIB", nil) if err != nil { t.Fatal(err) } @@ -97,8 +99,29 @@ LibraryVersion: 1.2.3-LIB About: This is a great API. ` - if strings.TrimSpace(string(outputContent)) != strings.TrimSpace(expectedWithPartials) { - t.Errorf("expected:\n%s\ngot:\n%s", expectedWithPartials, string(outputContent)) + if diff := cmp.Diff(strings.TrimSpace(expectedWithPartials), strings.TrimSpace(string(outputContent))); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + + // Test case 3: With README.md in keep list + keepSet := map[string]bool{"README.md": true} + customContent := "Custom README content" + err = os.WriteFile(outputPath, []byte(customContent), 0644) + if err != nil { + t.Fatal(err) + } + + err = RenderREADME(tmpDir, metadata, "1.0.0-BOM", "1.2.3-LIB", keepSet) + if err != nil { + t.Fatal(err) + } + + outputContent, err = os.ReadFile(outputPath) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(customContent, string(outputContent)); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) } } From b28577166a6dcac776cd40cd678b3a70128b2e37 Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Sun, 28 Jun 2026 15:47:27 -0400 Subject: [PATCH 056/108] refactor(java): decouple keepSet from postprocessing and revert clean.go to upstream - Revert clean.go completely to upstream/main to maintain clean separation of pre-gen cleaning - Remove keepSet check from applyToFiles in declarative postprocessing to align with legacy behavior - Simplify README preservation check in RenderREADME to direct keepSet map lookup --- internal/librarian/java/clean.go | 51 ++++------------------ internal/librarian/java/postprocess.go | 13 +++++- internal/librarian/java/postprocess_new.go | 25 +++++------ internal/librarian/java/template_render.go | 2 +- 4 files changed, 31 insertions(+), 60 deletions(-) diff --git a/internal/librarian/java/clean.go b/internal/librarian/java/clean.go index f559676332d..7d36826493e 100644 --- a/internal/librarian/java/clean.go +++ b/internal/librarian/java/clean.go @@ -18,10 +18,8 @@ import ( "bufio" "errors" "io" - "log/slog" "io/fs" "os" - "path" "path/filepath" "regexp" "slices" @@ -34,8 +32,6 @@ import ( const ( autoGeneratedMarker = "// AUTO-GENERATED DOCUMENTATION AND CLASS." autoGeneratedAnnotation = "@Generated(\"by gapic-generator-java\")" - protobufMarker = "// Generated by the protocol buffer compiler. DO NOT EDIT!" - grpcMarker = "@io.grpc.stub.annotations.GrpcGenerated" ) var ( @@ -56,8 +52,10 @@ var ( // It targets patterns like proto-*, grpc-*, and the main GAPIC module. func Clean(library *config.Library) error { patterns := cleanPatterns(library) - slog.Debug("Clean patterns", "output", library.Output, "patterns", patterns) - keepSet := newKeepSet(library.Keep) + keepSet := make(map[string]bool) + for _, k := range library.Keep { + keepSet[filepath.ToSlash(k)] = true + } for pattern, useMarker := range patterns { matches, err := filepath.Glob(filepath.Join(library.Output, pattern)) if err != nil { @@ -142,7 +140,6 @@ func cleanPath(targetPath, root string, keepSet map[string]bool, useMarker bool) return nil } } - slog.Info("Deleting path", "path", path) return os.Remove(path) }) if err != nil && !errors.Is(err, fs.ErrNotExist) { @@ -169,42 +166,10 @@ func isDirNotEmpty(err error) bool { return errors.Is(err, syscall.ENOTEMPTY) || errors.Is(err, syscall.EEXIST) } -// newKeepSet normalizes a list of keep paths using [path.Clean] and returns -// a map for fast lookup. -func newKeepSet(keep []string) map[string]bool { - keepSet := make(map[string]bool) - for _, k := range keep { - normalized := path.Clean(filepath.ToSlash(k)) - if normalized == "." { - continue - } - keepSet[strings.TrimSuffix(normalized, "/")] = true - } - return keepSet -} - -// isKept returns true if the path is explicitly kept by the user. -func isKept(p string, keepSet map[string]bool) bool { - cleanP := path.Clean(p) - if keepSet[cleanP] { - return true - } - - dir := path.Dir(cleanP) - for dir != "." && dir != "/" && dir != "" { - if keepSet[dir] { - return true - } - dir = path.Dir(dir) - } - return false -} - // shouldPreserve returns true if the given slash-separated path should be preserved // based on the keepSet or standard preservation patterns. -// It also checks if any ancestor directory is in the keepSet. -func shouldPreserve(p string, keepSet map[string]bool) bool { - return isKept(p, keepSet) || isDefaultPreserved(p) +func shouldPreserve(path string, keepSet map[string]bool) bool { + return keepSet[path] || isDefaultPreserved(path) } func isDefaultPreserved(path string) bool { @@ -239,7 +204,7 @@ func shouldCleanMarkerPath(path string, d os.DirEntry) (bool, error) { // hasMarker checks if the file at path contains the auto-generated marker. // It scans the file line by line using [bufio.Reader] and returns true as soon // as the marker is found, avoiding reading the entire file. -func hasMarker(path string) (has bool, err error) { +func hasMarker(path string) (bool, error) { f, err := os.Open(path) if err != nil { return false, err @@ -253,7 +218,7 @@ func hasMarker(path string) (has bool, err error) { reader := bufio.NewReader(f) for { line, rerr := reader.ReadString('\n') - if strings.Contains(line, autoGeneratedMarker) || strings.Contains(line, autoGeneratedAnnotation) || strings.Contains(line, protobufMarker) || strings.Contains(line, grpcMarker) { + if strings.Contains(line, autoGeneratedMarker) || strings.Contains(line, autoGeneratedAnnotation) { return true, nil } if rerr != nil { diff --git a/internal/librarian/java/postprocess.go b/internal/librarian/java/postprocess.go index 37bd2872734..327bccf6340 100644 --- a/internal/librarian/java/postprocess.go +++ b/internal/librarian/java/postprocess.go @@ -144,7 +144,7 @@ func postProcessAPI(ctx context.Context, params postProcessParams) error { coords := params.coords() if params.useGoPostprocessor { - keepSet := newKeepSet(params.library.Keep) + keepSet := toKeepSet(params.library.Keep) if err := restructureModules(params, params.outDir, keepSet, params.outDir); err != nil { return fmt.Errorf("failed to restructure direct to outDir: %w", err) } @@ -490,12 +490,21 @@ func copyProtos(protos []protoFileToCopy, destDir string) error { // It strips this first component (the API base like "v1") from the relative // path to reconstruct the expected path relative to the library root, which is // then matched against the library's Keep configuration. +func toKeepSet(keep []string) map[string]bool { + keepSet := make(map[string]bool, len(keep)) + for _, k := range keep { + normalized := strings.TrimSuffix(filepath.ToSlash(k), "/") + keepSet[normalized] = true + } + return keepSet +} + func removeKeptFilesFromStaging(library *config.Library, outDir string) error { stagingDir := stagingDir(outDir) if _, err := os.Stat(stagingDir); errors.Is(err, fs.ErrNotExist) { return nil } - keepSet := newKeepSet(library.Keep) + keepSet := toKeepSet(library.Keep) return filepath.WalkDir(stagingDir, func(path string, d os.DirEntry, err error) error { if err != nil { return err diff --git a/internal/librarian/java/postprocess_new.go b/internal/librarian/java/postprocess_new.go index 076106df5e7..05033ccc2b4 100644 --- a/internal/librarian/java/postprocess_new.go +++ b/internal/librarian/java/postprocess_new.go @@ -30,7 +30,11 @@ import ( // It applies post-processing operations configured in librarian.yaml, and renders the README.md directly // on the generated files in their final destinations. func postProcessLibraryNew(ctx context.Context, p libraryPostProcessParams) error { - keepSet := newKeepSet(p.library.Keep) + keepSet := make(map[string]bool, len(p.library.Keep)) + for _, k := range p.library.Keep { + normalized := strings.TrimSuffix(filepath.ToSlash(k), "/") + keepSet[normalized] = true + } // 1. Load postprocess configuration and apply operations if p.library.Postprocess != nil { @@ -41,7 +45,7 @@ func postProcessLibraryNew(ctx context.Context, p libraryPostProcessParams) erro // 1. Apply Copies for _, c := range cfg.CopyFile { - if shouldPreserve(filepath.ToSlash(c.Dst), keepSet) { + if keepSet[filepath.ToSlash(c.Dst)] { continue } srcAbs := filepath.Join(p.outDir, c.Src) @@ -53,7 +57,7 @@ func postProcessLibraryNew(ctx context.Context, p libraryPostProcessParams) erro // 2. Apply Removes for _, rem := range cfg.RemoveFile { - if err := applyToFiles(p.outDir, rem, keepSet, func(file string) error { + if err := applyToFiles(p.outDir, rem, func(file string) error { if err := postprocessing.RemoveFile(file); err != nil { return fmt.Errorf("failed to remove file %s: %w", file, err) } @@ -65,7 +69,7 @@ func postProcessLibraryNew(ctx context.Context, p libraryPostProcessParams) erro // 3. Apply Replacements for _, r := range cfg.Replace { - if err := applyToFiles(p.outDir, r.Path, keepSet, func(file string) error { + if err := applyToFiles(p.outDir, r.Path, func(file string) error { if err := postprocessing.Replace(file, r.Original, r.Replacement); err != nil { return fmt.Errorf("failed to apply replacement in %s: %w", file, err) } @@ -77,7 +81,7 @@ func postProcessLibraryNew(ctx context.Context, p libraryPostProcessParams) erro // 4. Apply Regex Replacements for _, r := range cfg.ReplaceRegex { - if err := applyToFiles(p.outDir, r.Path, keepSet, func(file string) error { + if err := applyToFiles(p.outDir, r.Path, func(file string) error { if err := postprocessing.ReplaceRegex(file, r.Pattern, r.Replacement); err != nil { return fmt.Errorf("failed to apply regex replacement in %s: %w", file, err) } @@ -89,7 +93,7 @@ func postProcessLibraryNew(ctx context.Context, p libraryPostProcessParams) erro // 5. Apply Method Operations for _, mo := range cfg.MethodOperations { - if err := applyToFiles(p.outDir, mo.Path, keepSet, func(file string) error { + if err := applyToFiles(p.outDir, mo.Path, func(file string) error { switch mo.Action { case "delete": if err := postprocessing.DeleteMethod(file, mo.FuncName, "java"); err != nil { @@ -149,7 +153,7 @@ func resolveGlobs(outDir, pathPattern string) ([]string, error) { return []string{filepath.Join(outDir, pathPattern)}, nil } -func applyToFiles(outDir string, pathPattern string, keepSet map[string]bool, action func(string) error) error { +func applyToFiles(outDir string, pathPattern string, action func(string) error) error { files, err := resolveGlobs(outDir, pathPattern) if err != nil { return fmt.Errorf("failed to resolve glob for %s: %w", pathPattern, err) @@ -158,13 +162,6 @@ func applyToFiles(outDir string, pathPattern string, keepSet map[string]bool, ac var replacedAny bool var lastTextNotFoundErr error for _, file := range files { - relPath, err := filepath.Rel(outDir, file) - if err != nil { - return fmt.Errorf("failed to get relative path for %s: %w", file, err) - } - if isKept(filepath.ToSlash(relPath), keepSet) { - continue - } if err := action(file); err != nil { if isGlob && errors.Is(err, postprocessing.ErrTextNotFound) { lastTextNotFoundErr = err diff --git a/internal/librarian/java/template_render.go b/internal/librarian/java/template_render.go index cb57c39ddd8..93f21b4d0ea 100644 --- a/internal/librarian/java/template_render.go +++ b/internal/librarian/java/template_render.go @@ -35,7 +35,7 @@ var defaultTemplateFs embed.FS // dir is the directory containing where README.md will be written. func RenderREADME(dir string, metadata *repoMetadata, bomVersion, libraryVersion string, keepSet map[string]bool) error { outputPath := filepath.Join(dir, "README.md") - if isKept("README.md", keepSet) { + if keepSet["README.md"] { return nil } From 47595390a5d26f9bbc35294f0e0daeef200d7465 Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Sun, 28 Jun 2026 15:56:11 -0400 Subject: [PATCH 057/108] chore(java): remove unused legacy README.md.tmpl --- .../librarian/java/template/README.md.tmpl | 266 ------------------ 1 file changed, 266 deletions(-) delete mode 100644 internal/librarian/java/template/README.md.tmpl diff --git a/internal/librarian/java/template/README.md.tmpl b/internal/librarian/java/template/README.md.tmpl deleted file mode 100644 index a2d5a55b477..00000000000 --- a/internal/librarian/java/template/README.md.tmpl +++ /dev/null @@ -1,266 +0,0 @@ -# Google {{ .Metadata.Repo.NamePretty }} Client for Java - -Java idiomatic client for [{{ .Metadata.Repo.NamePretty }}][product-docs]. - -[![Maven][maven-version-image]][maven-version-link] -![Stability][stability-image] - -- [Product Documentation][product-docs] -- [Client Library Documentation][javadocs] - -{{ if and .Metadata.Partials .Metadata.Partials.DeprecationWarning }} -{{ .Metadata.Partials.DeprecationWarning }} -{{ else if eq .Metadata.Repo.ReleaseLevel "preview" }} -> Note: This client is a work-in-progress, and may occasionally -> make backwards-incompatible changes. -{{ end }} - -{{ if .MigratedSplitRepo }} -:bus: In October 2022, this library has moved to -[google-cloud-java/{{ .Metadata.Repo.RepoShort }}]( -https://github.com/googleapis/google-cloud-java/tree/main/{{ .Metadata.Repo.RepoShort }}). -This repository will be archived in the future. -Future releases will appear in the new repository (https://github.com/googleapis/google-cloud-java/releases). -The Maven artifact coordinates (`{{ .GroupID }}:{{ .ArtifactID }}`) remain the same. -{{ end }} - -## Quickstart - -{{ if and .Metadata.Snippets (index .Metadata.Snippets (print .Metadata.Repo.APIShortname "_install_with_bom")) }} -If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file: - -```xml -{{ index .Metadata.Snippets (print .Metadata.Repo.APIShortname "_install_with_bom") }} -``` - -If you are using Maven without the BOM, add this to your dependencies: -{{ else if and .Monorepo (or (eq .GroupID "com.google.cloud") (eq .GroupID "com.google.analytics") (eq .GroupID "com.google.area120")) }} -If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file: - -```xml - - - - com.google.cloud - libraries-bom - {{ .Metadata.LibrariesBOMVersion }} - pom - import - - - - - - - {{ .GroupID }} - {{ .ArtifactID }} - - -``` - -If you are using Maven without the BOM, add this to your dependencies: -{{ else }} -If you are using Maven, add this to your pom.xml file: -{{ end }} - -```xml -{{ if and .Metadata.Snippets (index .Metadata.Snippets (print .Metadata.Repo.APIShortname "_install_without_bom")) }} -{{ index .Metadata.Snippets (print .Metadata.Repo.APIShortname "_install_without_bom") }} -{{ else }} - - {{ .GroupID }} - {{ .ArtifactID }} - {{ .Version }} - -{{ end }} -``` - -{{ if and .Metadata.Snippets (index .Metadata.Snippets (print .Metadata.Repo.APIShortname "_install_with_bom")) }} -If you are using Gradle 5.x or later, add this to your dependencies: - -```Groovy -implementation platform('com.google.cloud:libraries-bom:{{ .Metadata.LibrariesBOMVersion }}') - -implementation '{{ .GroupID }}:{{ .ArtifactID }}' -``` -{{ end }} - -If you are using Gradle without BOM, add this to your dependencies: - -```Groovy -implementation '{{ .GroupID }}:{{ .ArtifactID }}:{{ .Version }}' -``` - -If you are using SBT, add this to your dependencies: - -```Scala -libraryDependencies += "{{ .GroupID }}" % "{{ .ArtifactID }}" % "{{ .Version }}" -``` - -## Authentication - -See the [Authentication][authentication] section in the base directory's README. - -## Authorization - -The client application making API calls must be granted [authorization scopes][auth-scopes] required for the desired {{ .Metadata.Repo.NamePretty }} APIs, and the authenticated principal must have the [IAM role(s)][predefined-iam-roles] required to access GCP resources using the {{ .Metadata.Repo.NamePretty }} API calls. - -## Getting Started - -### Prerequisites - -You will need a [Google Cloud Platform Console][developer-console] project with the {{ .Metadata.Repo.NamePretty }} [API enabled][enable-api]. -{{ if .Metadata.Repo.RequiresBilling }}You will need to [enable billing][enable-billing] to use Google {{ .Metadata.Repo.NamePretty }}.{{ end }} -[Follow these instructions][create-project] to get your project set up. You will also need to set up the local development environment by -[installing the Google Cloud Command Line Interface][cloud-cli] and running the following commands in command line: -`gcloud auth login` and `gcloud config set project [YOUR PROJECT ID]`. - -### Installation and setup - -You'll need to obtain the `{{ .ArtifactID }}` library. See the [Quickstart](#quickstart) section -to add `{{ .ArtifactID }}` as a dependency in your code. - -## About {{ .Metadata.Repo.NamePretty }} - -{{ if and .Metadata.Partials .Metadata.Partials.About }} -{{ .Metadata.Partials.About }} -{{ else }} -[{{ .Metadata.Repo.NamePretty }}][product-docs] {{ .Metadata.Repo.APIDescription }} - -See the [{{ .Metadata.Repo.NamePretty }} client library docs][javadocs] to learn how to -use this {{ .Metadata.Repo.NamePretty }} Client Library. -{{ end }} - -{{ if and .Metadata.Partials .Metadata.Partials.CustomContent }} -{{ .Metadata.Partials.CustomContent }} -{{ end }} - -{{ if .Metadata.Samples }} -## Samples - -Samples are in the [`samples/`](https://github.com/{{ .Metadata.Repo.Repo }}/tree/main/samples) directory. - -| Sample | Source Code | Try it | -| --------------------------- | --------------------------------- | ------ | -{{ range .Metadata.Samples }}| {{ .Title }} | [source code](https://github.com/{{ $.Metadata.Repo.Repo }}/blob/main/{{ .File }}) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/{{ $.Metadata.Repo.Repo }}&page=editor&open_in_editor={{ .File }}) | -{{ end }} -{{ end }} - -## Troubleshooting - -To get help, follow the instructions in the [shared Troubleshooting document][troubleshooting]. - -{{ if .Metadata.Repo.Transport }} -## Transport - -{{ if eq .Metadata.Repo.Transport "grpc" }} -{{ .Metadata.Repo.NamePretty }} uses gRPC for the transport layer. -{{ else if eq .Metadata.Repo.Transport "http" }} -{{ .Metadata.Repo.NamePretty }} uses HTTP/JSON for the transport layer. -{{ else if eq .Metadata.Repo.Transport "both" }} -{{ .Metadata.Repo.NamePretty }} uses both gRPC and HTTP/JSON for the transport layer. -{{ end }} -{{ end }} - -## Supported Java Versions - -Java {{ .Metadata.MinJavaVersion }} or above is required for using this client. - -Google's Java client libraries, -[Google Cloud Client Libraries][cloudlibs] -and -[Google Cloud API Libraries][apilibs], -follow the -[Oracle Java SE support roadmap][oracle] -(see the Oracle Java SE Product Releases section). - -### For new development - -In general, new feature development occurs with support for the lowest Java -LTS version covered by Oracle's Premier Support (which typically lasts 5 years -from initial General Availability). If the minimum required JVM for a given -library is changed, it is accompanied by a [semver][semver] major release. - -Java 11 and (in September 2021) Java 17 are the best choices for new -development. - -### Keeping production systems current - -Google tests its client libraries with all current LTS versions covered by -Oracle's Extended Support (which typically lasts 8 years from initial -General Availability). - -#### Legacy support - -Google's client libraries support legacy versions of Java runtimes with long -term stable libraries that don't receive feature updates on a best efforts basis -as it may not be possible to backport all patches. - -Google provides updates on a best efforts basis to apps that continue to use -Java 7, though apps might need to upgrade to current versions of the library -that supports their JVM. - -#### Where to find specific information - -The latest versions and the supported Java versions are identified on -the individual GitHub repository `github.com/GoogleAPIs/java-SERVICENAME` -and on [google-cloud-java][g-c-j]. - -## Versioning - -{{ if and .Metadata.Partials .Metadata.Partials.Versioning }} -{{ .Metadata.Partials.Versioning }} -{{ else }} -This library follows [Semantic Versioning](http://semver.org/). - -{{ if eq .Metadata.Repo.ReleaseLevel "preview" }} -It is currently in major version zero (``0.y.z``), which means that anything may change at any time -and the public API should not be considered stable. -{{ end }}{{ end }} - -## Contributing - -{{ if and .Metadata.Partials .Metadata.Partials.Contributing }} -{{ .Metadata.Partials.Contributing }} -{{ else }} -Contributions to this library are always welcome and highly encouraged. - -See [CONTRIBUTING][contributing] for more information how to get started. - -Please note that this project is released with a Contributor Code of Conduct. By participating in -this project you agree to abide by its terms. See [Code of Conduct][code-of-conduct] for more -information. -{{ end }} - -## License - -Apache 2.0 - See [LICENSE][license] for more information. - -Java is a registered trademark of Oracle and/or its affiliates. - -[product-docs]: {{ .Metadata.Repo.ProductDocumentation }} -[javadocs]: {{ .Metadata.Repo.ClientDocumentation }} -[stability-image]: https://img.shields.io/badge/stability-{{ if eq .Metadata.Repo.ReleaseLevel "stable" }}stable-green{{ else if eq .Metadata.Repo.ReleaseLevel "preview" }}preview-yellow{{ else }}unknown-red{{ end }} -[maven-version-image]: https://img.shields.io/maven-central/v/{{ .GroupID }}/{{ .ArtifactID }}.svg -[maven-version-link]: https://central.sonatype.com/artifact/{{ .GroupID }}/{{ .ArtifactID }}/{{ .Version }} -[authentication]: https://github.com/googleapis/google-cloud-java#authentication -[auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes -[predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles -[iam-policy]: https://cloud.google.com/iam/docs/overview#cloud-iam-policy -[developer-console]: https://console.developers.google.com/ -[create-project]: https://cloud.google.com/resource-manager/docs/creating-managing-projects -[cloud-cli]: https://cloud.google.com/cli -[troubleshooting]: https://github.com/googleapis/google-cloud-java/blob/main/TROUBLESHOOTING.md -[contributing]: https://github.com/{{ .Metadata.Repo.Repo }}/blob/main/CONTRIBUTING.md -[code-of-conduct]: https://github.com/{{ .Metadata.Repo.Repo }}/blob/main/CODE_OF_CONDUCT.md#contributor-code-of-conduct -[license]: https://github.com/{{ .Metadata.Repo.Repo }}/blob/main/LICENSE -{{ if .Metadata.Repo.RequiresBilling }}[enable-billing]: https://cloud.google.com/apis/docs/getting-started#enabling_billing{{ end }} -{{ if .Metadata.Repo.APIID }}[enable-api]: https://console.cloud.google.com/flows/enableapi?apiid={{ .Metadata.Repo.APIID }}{{ end }} -[libraries-bom]: https://github.com/GoogleCloudPlatform/cloud-opensource-java/wiki/The-Google-Cloud-Platform-Libraries-BOM -[shell_img]: https://gstatic.com/cloudssh/images/open-btn.png - -[semver]: https://semver.org/ -[cloudlibs]: https://cloud.google.com/apis/docs/client-libraries-explained -[apilibs]: https://cloud.google.com/apis/docs/client-libraries-explained#google_api_client_libraries -[oracle]: https://www.oracle.com/java/technologies/java-se-support-roadmap.html -[g-c-j]: http://github.com/googleapis/google-cloud-java From d21710809bc724c876fb0d6d9a4dea4d665b15d9 Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Sun, 28 Jun 2026 16:22:48 -0400 Subject: [PATCH 058/108] chore(internal/librarian): revert unrelated changes and minimize diff against upstream --- internal/librarian/java/postgenerate_test.go | 5 +---- internal/librarian/java/repometadata_test.go | 5 +---- internal/postprocessing/fileops.go | 1 + internal/sidekick/parser/protobuf.go | 1 + 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/internal/librarian/java/postgenerate_test.go b/internal/librarian/java/postgenerate_test.go index 8b943804b95..e097fe25d5b 100644 --- a/internal/librarian/java/postgenerate_test.go +++ b/internal/librarian/java/postgenerate_test.go @@ -33,10 +33,7 @@ func TestPostGenerate(t *testing.T) { t.Parallel() tmpDir := t.TempDir() // Copy testdata to tmpDir - testdataDir, err := filepath.Abs(filepath.Join("testdata", "postgenerate")) - if err != nil { - t.Fatal(err) - } + testdataDir := filepath.Join("testdata", "postgenerate") if err := copyDir(testdataDir, tmpDir); err != nil { t.Fatal(err) } diff --git a/internal/librarian/java/repometadata_test.go b/internal/librarian/java/repometadata_test.go index ef94400cf07..c1f42a8ac91 100644 --- a/internal/librarian/java/repometadata_test.go +++ b/internal/librarian/java/repometadata_test.go @@ -76,10 +76,7 @@ func TestRepoMetadata_write(t *testing.T) { func TestDeriveRepoMetadata_Overrides(t *testing.T) { t.Parallel() apiPath := "google/cloud/secretmanager/v1" - googleapis, err := filepath.Abs("../../testdata/googleapis") - if err != nil { - t.Fatal(err) - } + googleapis := "../../testdata/googleapis" cfg := sample.Config() cfg.Language = config.LanguageJava diff --git a/internal/postprocessing/fileops.go b/internal/postprocessing/fileops.go index 15d07c6e351..f6ad2c4c11b 100644 --- a/internal/postprocessing/fileops.go +++ b/internal/postprocessing/fileops.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package postprocessing provides tools for the YAML-based postprocessing workflow. package postprocessing import ( diff --git a/internal/sidekick/parser/protobuf.go b/internal/sidekick/parser/protobuf.go index 30df14d56ec..197a8ac0895 100644 --- a/internal/sidekick/parser/protobuf.go +++ b/internal/sidekick/parser/protobuf.go @@ -169,6 +169,7 @@ func runProtoc(files []string, sourceCfg *sources.SourceConfig) ([]byte, error) args := []string{ "--include_imports", "--include_source_info", + "--retain_options", "--descriptor_set_out", tempFile.Name(), } if sourceCfg != nil { From d1252dde414346d77fb96025ebe078a0bfec4725 Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Sun, 28 Jun 2026 16:26:48 -0400 Subject: [PATCH 059/108] chore(internal/librarian/java): revert clean.go to upstream and remove redundant sample files --- internal/librarian/java/clean.go | 29 +-- internal/librarian/java/samples.go | 269 ------------------------ internal/librarian/java/samples_test.go | 164 --------------- 3 files changed, 5 insertions(+), 457 deletions(-) delete mode 100644 internal/librarian/java/samples.go delete mode 100644 internal/librarian/java/samples_test.go diff --git a/internal/librarian/java/clean.go b/internal/librarian/java/clean.go index 044e4599221..7d36826493e 100644 --- a/internal/librarian/java/clean.go +++ b/internal/librarian/java/clean.go @@ -17,11 +17,9 @@ package java import ( "bufio" "errors" - "fmt" "io" "io/fs" "os" - "path" "path/filepath" "regexp" "slices" @@ -34,8 +32,6 @@ import ( const ( autoGeneratedMarker = "// AUTO-GENERATED DOCUMENTATION AND CLASS." autoGeneratedAnnotation = "@Generated(\"by gapic-generator-java\")" - protobufMarker = "// Generated by the protocol buffer compiler. DO NOT EDIT!" - grpcMarker = "@io.grpc.stub.annotations.GrpcGenerated" ) var ( @@ -56,10 +52,9 @@ var ( // It targets patterns like proto-*, grpc-*, and the main GAPIC module. func Clean(library *config.Library) error { patterns := cleanPatterns(library) - fmt.Printf("Clean patterns for %s: %v\n", library.Output, patterns) keepSet := make(map[string]bool) for _, k := range library.Keep { - keepSet[strings.TrimSuffix(filepath.ToSlash(k), "/")] = true + keepSet[filepath.ToSlash(k)] = true } for pattern, useMarker := range patterns { matches, err := filepath.Glob(filepath.Join(library.Output, pattern)) @@ -145,8 +140,6 @@ func cleanPath(targetPath, root string, keepSet map[string]bool, useMarker bool) return nil } } - fmt.Println("DELETING:", path) - fmt.Println("DELETING:", path) return os.Remove(path) }) if err != nil && !errors.Is(err, fs.ErrNotExist) { @@ -175,20 +168,8 @@ func isDirNotEmpty(err error) bool { // shouldPreserve returns true if the given slash-separated path should be preserved // based on the keepSet or standard preservation patterns. -// It also checks if any ancestor directory is in the keepSet. -func shouldPreserve(p string, keepSet map[string]bool) bool { - if keepSet[p] || isDefaultPreserved(p) { - return true - } - - dir := path.Dir(p) - for dir != "." && dir != "/" && dir != "" { - if keepSet[dir] { - return true - } - dir = path.Dir(dir) - } - return false +func shouldPreserve(path string, keepSet map[string]bool) bool { + return keepSet[path] || isDefaultPreserved(path) } func isDefaultPreserved(path string) bool { @@ -223,7 +204,7 @@ func shouldCleanMarkerPath(path string, d os.DirEntry) (bool, error) { // hasMarker checks if the file at path contains the auto-generated marker. // It scans the file line by line using [bufio.Reader] and returns true as soon // as the marker is found, avoiding reading the entire file. -func hasMarker(path string) (has bool, err error) { +func hasMarker(path string) (bool, error) { f, err := os.Open(path) if err != nil { return false, err @@ -237,7 +218,7 @@ func hasMarker(path string) (has bool, err error) { reader := bufio.NewReader(f) for { line, rerr := reader.ReadString('\n') - if strings.Contains(line, autoGeneratedMarker) || strings.Contains(line, autoGeneratedAnnotation) || strings.Contains(line, protobufMarker) || strings.Contains(line, grpcMarker) { + if strings.Contains(line, autoGeneratedMarker) || strings.Contains(line, autoGeneratedAnnotation) { return true, nil } if rerr != nil { diff --git a/internal/librarian/java/samples.go b/internal/librarian/java/samples.go deleted file mode 100644 index 1047d2985e5..00000000000 --- a/internal/librarian/java/samples.go +++ /dev/null @@ -1,269 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package java - -import ( - "bufio" - "errors" - "io/fs" - "os" - "path/filepath" - "regexp" - "sort" - "strings" - "unicode" - - "gopkg.in/yaml.v3" -) - -var ( - openSnippetRegex = regexp.MustCompile(`\[START ([a-zA-Z0-9_-]+)\]`) - closeSnippetRegex = regexp.MustCompile(`\[END ([a-zA-Z0-9_-]+)\]`) - openExcludeRegex = regexp.MustCompile(`\[START_EXCLUDE\]`) - closeExcludeRegex = regexp.MustCompile(`\[END_EXCLUDE\]`) - - reMetadataBlock = regexp.MustCompile(`(?m)^[ \t]*//[ \t]*sample-metadata:([^\n]+|\n[ \t]*//)+`) - reCommentPrefix = regexp.MustCompile(`(?m)^[ \t]*(?:#|//)[ \t]?`) - - reDecamelize1 = regexp.MustCompile(`([A-Z]+)([A-Z])([a-z0-9])`) - reDecamelize2 = regexp.MustCompile(`([a-z0-9])([A-Z])`) -) - -// decamelize converts CamelCase or PascalCase titles into space-separated words, -// exactly reproducing Python synthtool's _decamelize(value: str). -func decamelize(value string) string { - if value == "" { - return "" - } - r := []rune(value) - r[0] = unicode.ToUpper(r[0]) - s := string(r) - - s = reDecamelize1.ReplaceAllString(s, "${1} ${2}${3}") - return reDecamelize2.ReplaceAllString(s, "${1} ${2}") -} - -// ExtractSamples walks the "samples" directory locating all .java source files. -// It parses embedded multiline "// sample-metadata:" YAML blocks to derive title and path metadata. -func ExtractSamples(dir string) ([]map[string]interface{}, error) { - samplesDir := filepath.Join(dir, "samples") - var files []string - - err := filepath.WalkDir(samplesDir, func(path string, d os.DirEntry, err error) error { - if err != nil { - if errors.Is(err, fs.ErrNotExist) || os.IsNotExist(err) { - return nil - } - return err - } - if d.IsDir() { - if d.Name() == "test" { - return filepath.SkipDir - } - if d.Name() == "generated" && filepath.Base(filepath.Dir(path)) == "snippets" { - return filepath.SkipDir - } - return nil - } - if !d.IsDir() && d.Type().IsRegular() && filepath.Ext(path) == ".java" && strings.Contains(filepath.ToSlash(path), "/src/main/java/") { - files = append(files, path) - } - return nil - }) - if err != nil { - return nil, err - } - - if len(files) == 0 { - return nil, nil - } - - sort.Strings(files) - var samples []map[string]interface{} - - for _, file := range files { - rel, err := filepath.Rel(dir, file) - if err != nil { - continue - } - base := strings.TrimSuffix(filepath.Base(file), ".java") - title := decamelize(base) - - slashPath := filepath.ToSlash(rel) - item := map[string]interface{}{ - "Title": title, - "File": slashPath, - "title": title, - "file": slashPath, - } - - contentBytes, err := os.ReadFile(file) - if err == nil { - if match := reMetadataBlock.FindString(string(contentBytes)); match != "" { - cleaned := reCommentPrefix.ReplaceAllString(match, "") - var meta map[string]map[string]interface{} - if err := yaml.Unmarshal([]byte(cleaned), &meta); err == nil { - if sm, ok := meta["sample-metadata"]; ok { - for k, v := range sm { - item[k] = v - if len(k) > 0 { - upperKey := strings.ToUpper(k[:1]) + k[1:] - item[upperKey] = v - } - } - if t, ok := sm["title"].(string); ok && strings.TrimSpace(t) != "" { - item["Title"] = t - item["title"] = t - } - } - } - } - } - - samples = append(samples, item) - } - return samples, nil -} - -// ExtractSnippets walks the "samples" directory locating *.java and *.xml files. -// It line-scans for [START name] and [END name] tags while supporting [START_EXCLUDE] blocks, -// returning trimmed minimum plain-space indentation blocks. -func ExtractSnippets(dir string) (map[string]string, error) { - samplesDir := filepath.Join(dir, "samples") - var files []string - - err := filepath.WalkDir(samplesDir, func(path string, d os.DirEntry, err error) error { - if err != nil { - if errors.Is(err, fs.ErrNotExist) || os.IsNotExist(err) { - return nil - } - return err - } - if d.IsDir() { - if d.Name() == "test" { - return filepath.SkipDir - } - if d.Name() == "generated" && filepath.Base(filepath.Dir(path)) == "snippets" { - return filepath.SkipDir - } - return nil - } - if !d.Type().IsRegular() { - return nil - } - ext := filepath.Ext(path) - if ext == ".java" || ext == ".xml" { - files = append(files, path) - } - return nil - }) - if err != nil { - return nil, err - } - - if len(files) == 0 { - return nil, nil - } - - sort.Strings(files) - snippetLines := make(map[string][]string) - - for _, file := range files { - f, err := os.Open(file) - if err != nil { - continue - } - - openSnippets := make(map[string]bool) - excluding := false - scanner := bufio.NewScanner(f) - scanner.Buffer(make([]byte, 64*1024), 10*1024*1024) - - for scanner.Scan() { - line := scanner.Text() - openMatch := openSnippetRegex.FindStringSubmatch(line) - closeMatch := closeSnippetRegex.FindStringSubmatch(line) - - if len(openMatch) > 1 && !excluding { - name := openMatch[1] - openSnippets[name] = true - if _, exists := snippetLines[name]; !exists { - snippetLines[name] = []string{} - } - } else if len(closeMatch) > 1 && !excluding { - delete(openSnippets, closeMatch[1]) - } else if openExcludeRegex.MatchString(line) { - excluding = true - } else if closeExcludeRegex.MatchString(line) { - excluding = false - } else if !excluding { - for s := range openSnippets { - snippetLines[s] = append(snippetLines[s], line) - } - } - } - if err := scanner.Err(); err != nil { - f.Close() - return nil, err - } - f.Close() - } - - if len(snippetLines) == 0 { - return nil, nil - } - - result := make(map[string]string) - for snippet, lines := range snippetLines { - result[snippet] = trimLeadingWhitespace(lines) - } - return result, nil -} - -// trimLeadingWhitespace computes the minimum plain-space indentation across non-empty lines, -// trimming that common whitespace while preserving newlines. -func trimLeadingWhitespace(lines []string) string { - if len(lines) == 0 { - return "" - } - - minSpaces := -1 - for _, line := range lines { - if strings.TrimSpace(line) != "" { - spaces := len(line) - len(strings.TrimLeft(line, " ")) - if minSpaces == -1 || spaces < minSpaces { - minSpaces = spaces - } - } - } - if minSpaces == -1 { - minSpaces = 0 - } - - var sb strings.Builder - for _, line := range lines { - if strings.TrimSpace(line) == "" { - sb.WriteString("\n") - } else { - if len(line) >= minSpaces { - sb.WriteString(line[minSpaces:]) - } else { - sb.WriteString(strings.TrimLeft(line, " ")) - } - sb.WriteString("\n") - } - } - return sb.String() -} diff --git a/internal/librarian/java/samples_test.go b/internal/librarian/java/samples_test.go deleted file mode 100644 index af0a1b4d215..00000000000 --- a/internal/librarian/java/samples_test.go +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package java - -import ( - "os" - "path/filepath" - "testing" -) - -func TestDecamelize(t *testing.T) { - tests := []struct { - input string - expected string - }{ - {"requesterPays", "Requester Pays"}, - {"ACLBatman", "ACL Batman"}, - {"NativeImageLoggingSample", "Native Image Logging Sample"}, - {"simpleTest", "Simple Test"}, - {"", ""}, - } - - for _, tc := range tests { - actual := decamelize(tc.input) - if actual != tc.expected { - t.Errorf("decamelize(%q) = %q; expected %q", tc.input, actual, tc.expected) - } - } -} - -func TestExtractSamples_MissingDir(t *testing.T) { - tempDir := t.TempDir() - samples, err := ExtractSamples(tempDir) - if err != nil { - t.Fatalf("ExtractSamples returned error for missing dir: %v", err) - } - if samples != nil { - t.Errorf("Expected nil samples for missing dir, got %v", samples) - } -} - -func TestExtractSamples_Success(t *testing.T) { - tempDir := t.TempDir() - samplesDir := filepath.Join(tempDir, "samples", "src", "main", "java") - if err := os.MkdirAll(samplesDir, 0755); err != nil { - t.Fatal(err) - } - - file1 := filepath.Join(samplesDir, "RequesterPays.java") - content1 := `// sample-metadata: -// title: Custom Title Override -// description: A custom demo sample -public class RequesterPays {}` - if err := os.WriteFile(file1, []byte(content1), 0644); err != nil { - t.Fatal(err) - } - - file2 := filepath.Join(samplesDir, "demoSample.java") - content2 := `public class demoSample {}` - if err := os.WriteFile(file2, []byte(content2), 0644); err != nil { - t.Fatal(err) - } - - samples, err := ExtractSamples(tempDir) - if err != nil { - t.Fatal(err) - } - - if len(samples) != 2 { - t.Fatalf("Expected 2 samples, got %d", len(samples)) - } - - // First should be RequesterPays.java with custom YAML override (uppercase R comes before lowercase d) - s0 := samples[0] - if s0["Title"] != "Custom Title Override" || s0["title"] != "Custom Title Override" { - t.Errorf("Sample 0 title = %v; expected 'Custom Title Override'", s0["Title"]) - } - if s0["File"] != "samples/src/main/java/RequesterPays.java" { - t.Errorf("Sample 0 file = %v", s0["File"]) - } - - // Second should be demoSample.java - s1 := samples[1] - if s1["Title"] != "Demo Sample" || s1["title"] != "Demo Sample" { - t.Errorf("Sample 1 title = %v; expected 'Demo Sample'", s1["Title"]) - } - if s1["File"] != "samples/src/main/java/demoSample.java" { - t.Errorf("Sample 1 file = %v", s1["File"]) - } -} - -func TestExtractSnippets(t *testing.T) { - tempDir := t.TempDir() - samplesDir := filepath.Join(tempDir, "samples") - if err := os.MkdirAll(samplesDir, 0755); err != nil { - t.Fatal(err) - } - - pomPath := filepath.Join(samplesDir, "pom.xml") - pomContent := ` - - - com.google.cloud - - -` - if err := os.WriteFile(pomPath, []byte(pomContent), 0644); err != nil { - t.Fatal(err) - } - - javaPath := filepath.Join(samplesDir, "Demo.java") - javaContent := `public class Demo { - // [START quickstart] - public void run() { - // [START_EXCLUDE] - System.out.println("hidden"); - // [END_EXCLUDE] - System.out.println("visible"); - } - // [END quickstart] -}` - if err := os.WriteFile(javaPath, []byte(javaContent), 0644); err != nil { - t.Fatal(err) - } - - snippets, err := ExtractSnippets(tempDir) - if err != nil { - t.Fatal(err) - } - - if len(snippets) != 2 { - t.Fatalf("Expected 2 snippets, got %d", len(snippets)) - } - - depSnippet := snippets["dependency_snippet"] - expectedDep := ` - com.google.cloud - -` - if depSnippet != expectedDep { - t.Errorf("dependency_snippet = %q; expected %q", depSnippet, expectedDep) - } - - quickSnippet := snippets["quickstart"] - expectedQuick := `public void run() { - System.out.println("visible"); -} -` - if quickSnippet != expectedQuick { - t.Errorf("quickstart = %q; expected %q", quickSnippet, expectedQuick) - } -} From 01ea0acba624af20cbbc5269c89644cbf985ff6b Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Sun, 28 Jun 2026 17:05:19 -0400 Subject: [PATCH 060/108] fix(internal/librarian/java): respect ReleasedVersion and restore mutual exclusion in README template Prioritizes library.Java.ReleasedVersion when computing libraryVersion for README rendering. Restores if/else if mutual exclusion between DeprecationWarning and preview work-in-progress warning banner in README.md.go.tmpl. Adds unit tests. --- internal/librarian/java/postprocess_new.go | 12 +++-- .../librarian/java/postprocess_new_test.go | 44 +++++++++++++++++++ .../librarian/java/template/README.md.go.tmpl | 2 +- 3 files changed, 54 insertions(+), 4 deletions(-) diff --git a/internal/librarian/java/postprocess_new.go b/internal/librarian/java/postprocess_new.go index 05033ccc2b4..0341c6827a5 100644 --- a/internal/librarian/java/postprocess_new.go +++ b/internal/librarian/java/postprocess_new.go @@ -119,9 +119,15 @@ func postProcessLibraryNew(ctx context.Context, p libraryPostProcessParams) erro // 6. Render README.md - libraryVersion, err := deriveLastReleasedVersion(p.library.Version) - if err != nil { - return fmt.Errorf("failed to derive library version: %w", err) + var libraryVersion string + if p.library.Java != nil && p.library.Java.ReleasedVersion != "" { + libraryVersion = p.library.Java.ReleasedVersion + } else { + var err error + libraryVersion, err = deriveLastReleasedVersion(p.library.Version) + if err != nil { + return fmt.Errorf("failed to derive library version: %w", err) + } } if p.cfg == nil { diff --git a/internal/librarian/java/postprocess_new_test.go b/internal/librarian/java/postprocess_new_test.go index 4f667806dbc..6e058a7cf13 100644 --- a/internal/librarian/java/postprocess_new_test.go +++ b/internal/librarian/java/postprocess_new_test.go @@ -140,3 +140,47 @@ public class File { t.Errorf("README content mismatch. Got: %s, expected: # My API", string(readmeContent)) } } + +func TestPostProcessLibraryNew_ReleasedVersion(t *testing.T) { + tmpDir := t.TempDir() + tmplDir := filepath.Join(tmpDir, "template") + if err := os.MkdirAll(tmplDir, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(tmplDir, "README.md.go.tmpl"), []byte(`Version: {{ .Version }}`), 0644); err != nil { + t.Fatal(err) + } + + p := libraryPostProcessParams{ + outDir: tmpDir, + cfg: &config.Config{ + Default: &config.Default{ + Java: &config.JavaDefault{ + LibrariesBOMVersion: "1.0.0", + }, + }, + }, + library: &config.Library{ + Version: "3.44.0-SNAPSHOT", + Java: &config.JavaModule{ + ReleasedVersion: "3.43.1", + }, + }, + metadata: &repoMetadata{ + NamePretty: "My API", + }, + } + + if err := postProcessLibraryNew(t.Context(), p); err != nil { + t.Fatal(err) + } + + readmePath := filepath.Join(tmpDir, "README.md") + readmeContent, err := os.ReadFile(readmePath) + if err != nil { + t.Fatal(err) + } + if string(readmeContent) != "Version: 3.43.1" { + t.Errorf("README content mismatch. Got: %s, expected: Version: 3.43.1", string(readmeContent)) + } +} diff --git a/internal/librarian/java/template/README.md.go.tmpl b/internal/librarian/java/template/README.md.go.tmpl index 12e4433fb77..69ad944b362 100644 --- a/internal/librarian/java/template/README.md.go.tmpl +++ b/internal/librarian/java/template/README.md.go.tmpl @@ -9,7 +9,7 @@ Java idiomatic client for [{{ .Metadata.Repo.NamePretty }}][product-docs]. - [Client Library Documentation][javadocs] {{ if and .Metadata.Partials .Metadata.Partials.DeprecationWarning }}{{ .Metadata.Partials.DeprecationWarning }} -{{ end }}{{ if eq .Metadata.Repo.ReleaseLevel "preview" }}> Note: This client is a work-in-progress, and may occasionally +{{ else if eq .Metadata.Repo.ReleaseLevel "preview" }}> Note: This client is a work-in-progress, and may occasionally > make backwards-incompatible changes. {{ end }}{{ if .MigratedSplitRepo }}:bus: In October 2022, this library has moved to From c3c8bbe5c83b6ee73dbd070cd8a14fab8cd0437c Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Sun, 28 Jun 2026 17:22:52 -0400 Subject: [PATCH 061/108] refactor(internal/librarian/java): consolidate postprocessing and tests with fallback routing --- cmd/librarian/doc.go | 3 +- internal/librarian/generate.go | 16 +- internal/librarian/install_test.go | 4 +- internal/librarian/java/generate.go | 58 +++-- internal/librarian/java/generate_test.go | 6 +- internal/librarian/java/postprocess.go | 224 +++++++++++++++--- internal/librarian/java/postprocess_new.go | 184 -------------- .../librarian/java/postprocess_new_test.go | 186 --------------- internal/librarian/java/postprocess_test.go | 175 +++++++++++++- 9 files changed, 402 insertions(+), 454 deletions(-) delete mode 100644 internal/librarian/java/postprocess_new.go delete mode 100644 internal/librarian/java/postprocess_new_test.go diff --git a/cmd/librarian/doc.go b/cmd/librarian/doc.go index 3b806378821..face1f6cdef 100644 --- a/cmd/librarian/doc.go +++ b/cmd/librarian/doc.go @@ -106,8 +106,7 @@ Examples: Flags: - --all generate all libraries - --go-postprocessor, --go-post use the new Go postprocessor for Java libraries + --all generate all libraries A typical librarian workflow for regenerating every library against the latest API definitions is: diff --git a/internal/librarian/generate.go b/internal/librarian/generate.go index 3814cdff243..43833550b33 100644 --- a/internal/librarian/generate.go +++ b/internal/librarian/generate.go @@ -73,15 +73,9 @@ latest API definitions is: Name: "all", Usage: "generate all libraries", }, - &cli.BoolFlag{ - Name: "go-postprocessor", - Aliases: []string{"go-post"}, - Usage: "use the new Go postprocessor for Java libraries", - }, }, Action: func(ctx context.Context, cmd *cli.Command) error { all := cmd.Bool("all") - useGoPostprocessor := cmd.Bool("go-postprocessor") libraryName := cmd.Args().First() if !all && libraryName == "" { return errMissingLibraryOrAllFlag @@ -93,12 +87,12 @@ latest API definitions is: if err != nil { return err } - return runGenerate(ctx, cfg, all, libraryName, useGoPostprocessor) + return runGenerate(ctx, cfg, all, libraryName) }, } } -func runGenerate(ctx context.Context, cfg *config.Config, all bool, libraryName string, useGoPostprocessor bool) error { +func runGenerate(ctx context.Context, cfg *config.Config, all bool, libraryName string) error { sources, err := LoadSources(ctx, cfg.Sources) if err != nil { return err @@ -145,7 +139,7 @@ func runGenerate(ctx context.Context, cfg *config.Config, all bool, libraryName if err := cleanLibraries(cfg.Language, libraries); err != nil { return err } - return generateLibraries(ctx, cfg, libraries, sources, useGoPostprocessor) + return generateLibraries(ctx, cfg, libraries, sources) } // cleanLibraries iterates over all the given libraries sequentially, @@ -187,7 +181,7 @@ func cleanLibraries(language string, libraries []*config.Library) error { // generateLibraries generates and formats all the given libraries, // delegating to language-specific code. Each language chooses its own // concurrency strategy for these two steps. -func generateLibraries(ctx context.Context, cfg *config.Config, libraries []*config.Library, src *sources.Sources, useGoPostprocessor bool) error { +func generateLibraries(ctx context.Context, cfg *config.Config, libraries []*config.Library, src *sources.Sources) error { switch cfg.Language { case config.LanguageDart: g, gctx := errgroup.WithContext(ctx) @@ -247,7 +241,7 @@ func generateLibraries(ctx context.Context, cfg *config.Config, libraries []*con allMissingArtifacts = append(allMissingArtifacts, java.MissingArtifact{ID: id, Library: library}) } - if err := java.Generate(ctx, cfg, library, src, useGoPostprocessor); err != nil { + if err := java.Generate(ctx, cfg, library, src); err != nil { return fmt.Errorf("generate library %q (%s): %w", library.Name, cfg.Language, err) } if err := java.Format(ctx, library); err != nil { diff --git a/internal/librarian/install_test.go b/internal/librarian/install_test.go index a9090367213..a9f22409b3e 100644 --- a/internal/librarian/install_test.go +++ b/internal/librarian/install_test.go @@ -62,7 +62,7 @@ func TestGenerate(t *testing.T) { tmpDir := t.TempDir() t.Chdir(tmpDir) - if err := generateLibraries(t.Context(), cfg, []*config.Library{library}, nil, false); err != nil { + if err := generateLibraries(t.Context(), cfg, []*config.Library{library}, nil); err != nil { t.Fatal(err) } @@ -92,7 +92,7 @@ func TestCleanLibraries(t *testing.T) { tmpDir := t.TempDir() t.Chdir(tmpDir) - if err := generateLibraries(t.Context(), cfg, []*config.Library{library}, nil, false); err != nil { + if err := generateLibraries(t.Context(), cfg, []*config.Library{library}, nil); err != nil { t.Fatal(err) } diff --git a/internal/librarian/java/generate.go b/internal/librarian/java/generate.go index 2a3696daa47..adb211cae33 100644 --- a/internal/librarian/java/generate.go +++ b/internal/librarian/java/generate.go @@ -57,18 +57,17 @@ var ( ) type generateAPIParams struct { - cfg *config.Config - api *config.API - library *config.Library - srcCfg *sources.SourceConfig - outdir string - metadata *repoMetadata - apiCfg *serviceconfig.API - useGoPostprocessor bool + cfg *config.Config + api *config.API + library *config.Library + srcCfg *sources.SourceConfig + outdir string + metadata *repoMetadata + apiCfg *serviceconfig.API } // Generate generates a Java client library. -func Generate(ctx context.Context, cfg *config.Config, library *config.Library, srcs *sources.Sources, useGoPostprocessor bool) error { +func Generate(ctx context.Context, cfg *config.Config, library *config.Library, srcs *sources.Sources) error { if library.Java.GroupID == fakeGroupID { return errUnrecognizedAPI } @@ -98,26 +97,24 @@ func Generate(ctx context.Context, cfg *config.Config, library *config.Library, transports[api.Path] = apiCfg.Transport(config.LanguageJava) // metadata is needed for pom.xml generation in post process if err := generateAPI(ctx, generateAPIParams{ - cfg: cfg, - api: api, - library: library, - srcCfg: srcCfg, - outdir: outdir, - metadata: metadata, - apiCfg: apiCfg, - useGoPostprocessor: useGoPostprocessor, + cfg: cfg, + api: api, + library: library, + srcCfg: srcCfg, + outdir: outdir, + metadata: metadata, + apiCfg: apiCfg, }); err != nil { return fmt.Errorf("failed to generate api %q: %w", api.Path, err) } } if err := postProcessLibrary(ctx, libraryPostProcessParams{ - cfg: cfg, - library: library, - outDir: outdir, - metadata: metadata, - transports: transports, - useGoPostprocessor: useGoPostprocessor, + cfg: cfg, + library: library, + outDir: outdir, + metadata: metadata, + transports: transports, }); err != nil { return err } @@ -141,14 +138,13 @@ func generateAPI(ctx context.Context, params generateAPIParams) error { additionalProtosToCopyRel := processAdditionalProtos(javaAPI, googleapisDir) postParams := postProcessParams{ - cfg: params.cfg, - library: params.library, - javaAPI: javaAPI, - metadata: params.metadata, - outDir: params.outdir, - apiBase: deriveAPIBase(params.library, params.api.Path), - includeSamples: *javaAPI.Samples, - useGoPostprocessor: params.useGoPostprocessor, + cfg: params.cfg, + library: params.library, + javaAPI: javaAPI, + metadata: params.metadata, + outDir: params.outdir, + apiBase: deriveAPIBase(params.library, params.api.Path), + includeSamples: *javaAPI.Samples, } gapicDir := postParams.gapicDir() gRPCDir := postParams.gRPCDir() diff --git a/internal/librarian/java/generate_test.go b/internal/librarian/java/generate_test.go index 0eeb6a4e713..6fb4b7f5f28 100644 --- a/internal/librarian/java/generate_test.go +++ b/internal/librarian/java/generate_test.go @@ -653,7 +653,7 @@ func TestGenerateLibrary_Error(t *testing.T) { }, Libraries: []*config.Library{test.library}, } - err := Generate(t.Context(), cfg, test.library, &sources.Sources{Googleapis: googleapisDir}, false) + err := Generate(t.Context(), cfg, test.library, &sources.Sources{Googleapis: googleapisDir}) if !errors.Is(err, test.wantErr) { t.Errorf("generate() error = %v, wantErr %v", err, test.wantErr) } @@ -706,7 +706,7 @@ func TestGenerate_Logic(t *testing.T) { t.Fatal(err) } - err := Generate(t.Context(), cfg, library, &sources.Sources{Googleapis: googleapisDir}, false) + err := Generate(t.Context(), cfg, library, &sources.Sources{Googleapis: googleapisDir}) if err != nil { t.Fatal(err) } @@ -768,7 +768,7 @@ func TestGenerate_ProtoExclusion(t *testing.T) { {Name: rootLibrary, Version: "1.2.3"}, }, } - err := Generate(t.Context(), cfg, library, &sources.Sources{Googleapis: googleapisDir}, false) + err := Generate(t.Context(), cfg, library, &sources.Sources{Googleapis: googleapisDir}) if err != nil { t.Fatal(err) } diff --git a/internal/librarian/java/postprocess.go b/internal/librarian/java/postprocess.go index 327bccf6340..08aafbdb555 100644 --- a/internal/librarian/java/postprocess.go +++ b/internal/librarian/java/postprocess.go @@ -25,10 +25,12 @@ import ( "strings" "time" + "github.com/bmatcuk/doublestar/v4" "github.com/googleapis/librarian/internal/command" "github.com/googleapis/librarian/internal/config" "github.com/googleapis/librarian/internal/filesystem" "github.com/googleapis/librarian/internal/license" + "github.com/googleapis/librarian/internal/postprocessing" "github.com/googleapis/librarian/internal/serviceconfig" ) @@ -49,41 +51,48 @@ type protoFileToCopy struct { } type postProcessParams struct { - cfg *config.Config - library *config.Library - javaAPI *config.JavaAPI - metadata *repoMetadata - outDir string - apiBase string - protosToCopy []protoFileToCopy - includeSamples bool - useGoPostprocessor bool + cfg *config.Config + library *config.Library + javaAPI *config.JavaAPI + metadata *repoMetadata + outDir string + apiBase string + protosToCopy []protoFileToCopy + includeSamples bool } type libraryPostProcessParams struct { - cfg *config.Config - library *config.Library - outDir string - metadata *repoMetadata - transports map[string]serviceconfig.Transport - useGoPostprocessor bool + cfg *config.Config + library *config.Library + outDir string + metadata *repoMetadata + transports map[string]serviceconfig.Transport } -func postProcessLibrary(ctx context.Context, params libraryPostProcessParams) error { - if params.useGoPostprocessor { - if err := postProcessLibraryNew(ctx, params); err != nil { - return err - } +func isGoPostprocessor(outDir string) (bool, error) { + owlbotPath := filepath.Join(outDir, "owlbot.py") + _, err := os.Stat(owlbotPath) + if errors.Is(err, fs.ErrNotExist) { + return true, nil + } + if err == nil { + return false, nil + } + return false, fmt.Errorf("failed to stat owlbot script: %w", err) +} - monorepoVersion, err := findMonorepoVersion(params.cfg) - if err != nil { - return err - } - if err := syncPOMs(params.library, params.outDir, monorepoVersion, params.metadata, params.transports); err != nil { - return fmt.Errorf("%w: %w", errSyncPOMs, err) - } - return nil +func postProcessLibrary(ctx context.Context, params libraryPostProcessParams) error { + useGo, err := isGoPostprocessor(params.outDir) + if err != nil { + return err } + if useGo { + return runGoPostprocessor(ctx, params) + } + return runLegacyPythonPostprocessor(ctx, params) +} + +func runLegacyPythonPostprocessor(ctx context.Context, params libraryPostProcessParams) error { if err := createOrVerifyOwlbotPy(params.outDir); err != nil { return err } @@ -143,7 +152,11 @@ func postProcessAPI(ctx context.Context, params postProcessParams) error { } coords := params.coords() - if params.useGoPostprocessor { + useGo, err := isGoPostprocessor(params.outDir) + if err != nil { + return err + } + if useGo { keepSet := toKeepSet(params.library.Keep) if err := restructureModules(params, params.outDir, keepSet, params.outDir); err != nil { return fmt.Errorf("failed to restructure direct to outDir: %w", err) @@ -569,3 +582,156 @@ func createOrVerifyOwlbotPy(outDir string) (err error) { } return nil } + +func runGoPostprocessor(ctx context.Context, p libraryPostProcessParams) error { + keepSet := make(map[string]bool, len(p.library.Keep)) + for _, k := range p.library.Keep { + normalized := strings.TrimSuffix(filepath.ToSlash(k), "/") + keepSet[normalized] = true + } + + // 1. Load postprocess configuration and apply operations + if p.library.Postprocess != nil { + if err := postprocessing.Validate(p.library.Postprocess); err != nil { + return fmt.Errorf("invalid postprocess config: %w", err) + } + cfg := p.library.Postprocess + + // 1. Apply Copies + for _, c := range cfg.CopyFile { + if keepSet[filepath.ToSlash(c.Dst)] { + continue + } + srcAbs := filepath.Join(p.outDir, c.Src) + dstAbs := filepath.Join(p.outDir, c.Dst) + if err := filesystem.CopyFile(srcAbs, dstAbs); err != nil { + return fmt.Errorf("failed to copy file from %s to %s: %w", c.Src, c.Dst, err) + } + } + + // 2. Apply Removes + for _, rem := range cfg.RemoveFile { + if err := applyToFiles(p.outDir, rem, func(file string) error { + if err := postprocessing.RemoveFile(file); err != nil { + return fmt.Errorf("failed to remove file %s: %w", file, err) + } + return nil + }); err != nil { + return err + } + } + + // 3. Apply Replacements + for _, r := range cfg.Replace { + if err := applyToFiles(p.outDir, r.Path, func(file string) error { + if err := postprocessing.Replace(file, r.Original, r.Replacement); err != nil { + return fmt.Errorf("failed to apply replacement in %s: %w", file, err) + } + return nil + }); err != nil { + return err + } + } + + // 4. Apply Regex Replacements + for _, r := range cfg.ReplaceRegex { + if err := applyToFiles(p.outDir, r.Path, func(file string) error { + if err := postprocessing.ReplaceRegex(file, r.Pattern, r.Replacement); err != nil { + return fmt.Errorf("failed to apply regex replacement in %s: %w", file, err) + } + return nil + }); err != nil { + return err + } + } + + // 5. Apply Method Operations + for _, mo := range cfg.MethodOperations { + if err := applyToFiles(p.outDir, mo.Path, func(file string) error { + switch mo.Action { + case "delete": + if err := postprocessing.DeleteMethod(file, mo.FuncName, "java"); err != nil { + return fmt.Errorf("failed to delete method %q in %s: %w", mo.FuncName, file, err) + } + case "duplicate": + if err := postprocessing.DuplicateMethod(ctx, file, mo.FuncName, mo.NewName, "java"); err != nil { + return fmt.Errorf("failed to duplicate method %q in %s: %w", mo.FuncName, file, err) + } + case "deprecate": + if err := postprocessing.DeprecateMethod(file, mo.FuncName, mo.DeprecationMessage, "java"); err != nil { + return fmt.Errorf("failed to deprecate method %q in %s: %w", mo.FuncName, file, err) + } + default: + return fmt.Errorf("unsupported method operation action %q", mo.Action) + } + return nil + }); err != nil { + return err + } + } + } + + // 6. Render README.md + var libraryVersion string + if p.library.Java != nil && p.library.Java.ReleasedVersion != "" { + libraryVersion = p.library.Java.ReleasedVersion + } else { + var err error + libraryVersion, err = deriveLastReleasedVersion(p.library.Version) + if err != nil { + return fmt.Errorf("failed to derive library version: %w", err) + } + } + + if p.cfg == nil { + return fmt.Errorf("cfg is nil") + } + if p.cfg.Default == nil { + return fmt.Errorf("cfg.Default is nil") + } + if p.cfg.Default.Java == nil { + return fmt.Errorf("cfg.Default.Java is nil") + } + + bomVersion, err := findBOMVersion(p.cfg, p.library) + if err != nil { + return fmt.Errorf("failed to find BOM version: %w", err) + } + + if err := RenderREADME(p.outDir, p.metadata, bomVersion, libraryVersion, keepSet); err != nil { + return fmt.Errorf("failed to render README: %w", err) + } + + return nil +} + +func resolveGlobs(outDir, pathPattern string) ([]string, error) { + if strings.ContainsAny(pathPattern, "*?[]{}") { + return doublestar.FilepathGlob(filepath.Join(outDir, pathPattern)) + } + return []string{filepath.Join(outDir, pathPattern)}, nil +} + +func applyToFiles(outDir string, pathPattern string, action func(string) error) error { + files, err := resolveGlobs(outDir, pathPattern) + if err != nil { + return fmt.Errorf("failed to resolve glob for %s: %w", pathPattern, err) + } + isGlob := strings.ContainsAny(pathPattern, "*?[]{}") + var replacedAny bool + var lastTextNotFoundErr error + for _, file := range files { + if err := action(file); err != nil { + if isGlob && errors.Is(err, postprocessing.ErrTextNotFound) { + lastTextNotFoundErr = err + continue + } + return err + } + replacedAny = true + } + if isGlob && len(files) > 0 && !replacedAny { + return lastTextNotFoundErr + } + return nil +} diff --git a/internal/librarian/java/postprocess_new.go b/internal/librarian/java/postprocess_new.go deleted file mode 100644 index 0341c6827a5..00000000000 --- a/internal/librarian/java/postprocess_new.go +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package java - -import ( - "context" - "errors" - "fmt" - "path/filepath" - "strings" - - "github.com/bmatcuk/doublestar/v4" - "github.com/googleapis/librarian/internal/filesystem" - "github.com/googleapis/librarian/internal/postprocessing" -) - -// postProcessLibraryNew implements the new postprocessing flow, bypassing owlbot.py. -// It applies post-processing operations configured in librarian.yaml, and renders the README.md directly -// on the generated files in their final destinations. -func postProcessLibraryNew(ctx context.Context, p libraryPostProcessParams) error { - keepSet := make(map[string]bool, len(p.library.Keep)) - for _, k := range p.library.Keep { - normalized := strings.TrimSuffix(filepath.ToSlash(k), "/") - keepSet[normalized] = true - } - - // 1. Load postprocess configuration and apply operations - if p.library.Postprocess != nil { - if err := postprocessing.Validate(p.library.Postprocess); err != nil { - return fmt.Errorf("invalid postprocess config: %w", err) - } - cfg := p.library.Postprocess - - // 1. Apply Copies - for _, c := range cfg.CopyFile { - if keepSet[filepath.ToSlash(c.Dst)] { - continue - } - srcAbs := filepath.Join(p.outDir, c.Src) - dstAbs := filepath.Join(p.outDir, c.Dst) - if err := filesystem.CopyFile(srcAbs, dstAbs); err != nil { - return fmt.Errorf("failed to copy file from %s to %s: %w", c.Src, c.Dst, err) - } - } - - // 2. Apply Removes - for _, rem := range cfg.RemoveFile { - if err := applyToFiles(p.outDir, rem, func(file string) error { - if err := postprocessing.RemoveFile(file); err != nil { - return fmt.Errorf("failed to remove file %s: %w", file, err) - } - return nil - }); err != nil { - return err - } - } - - // 3. Apply Replacements - for _, r := range cfg.Replace { - if err := applyToFiles(p.outDir, r.Path, func(file string) error { - if err := postprocessing.Replace(file, r.Original, r.Replacement); err != nil { - return fmt.Errorf("failed to apply replacement in %s: %w", file, err) - } - return nil - }); err != nil { - return err - } - } - - // 4. Apply Regex Replacements - for _, r := range cfg.ReplaceRegex { - if err := applyToFiles(p.outDir, r.Path, func(file string) error { - if err := postprocessing.ReplaceRegex(file, r.Pattern, r.Replacement); err != nil { - return fmt.Errorf("failed to apply regex replacement in %s: %w", file, err) - } - return nil - }); err != nil { - return err - } - } - - // 5. Apply Method Operations - for _, mo := range cfg.MethodOperations { - if err := applyToFiles(p.outDir, mo.Path, func(file string) error { - switch mo.Action { - case "delete": - if err := postprocessing.DeleteMethod(file, mo.FuncName, "java"); err != nil { - return fmt.Errorf("failed to delete method %q in %s: %w", mo.FuncName, file, err) - } - case "duplicate": - if err := postprocessing.DuplicateMethod(ctx, file, mo.FuncName, mo.NewName, "java"); err != nil { - return fmt.Errorf("failed to duplicate method %q in %s: %w", mo.FuncName, file, err) - } - case "deprecate": - if err := postprocessing.DeprecateMethod(file, mo.FuncName, mo.DeprecationMessage, "java"); err != nil { - return fmt.Errorf("failed to deprecate method %q in %s: %w", mo.FuncName, file, err) - } - default: - return fmt.Errorf("unsupported method operation action %q", mo.Action) - } - return nil - }); err != nil { - return err - } - } - } - - // 6. Render README.md - - var libraryVersion string - if p.library.Java != nil && p.library.Java.ReleasedVersion != "" { - libraryVersion = p.library.Java.ReleasedVersion - } else { - var err error - libraryVersion, err = deriveLastReleasedVersion(p.library.Version) - if err != nil { - return fmt.Errorf("failed to derive library version: %w", err) - } - } - - if p.cfg == nil { - return fmt.Errorf("cfg is nil") - } - if p.cfg.Default == nil { - return fmt.Errorf("cfg.Default is nil") - } - if p.cfg.Default.Java == nil { - return fmt.Errorf("cfg.Default.Java is nil") - } - - bomVersion, err := findBOMVersion(p.cfg, p.library) - if err != nil { - return fmt.Errorf("failed to find BOM version: %w", err) - } - - if err := RenderREADME(p.outDir, p.metadata, bomVersion, libraryVersion, keepSet); err != nil { - return fmt.Errorf("failed to render README: %w", err) - } - - return nil -} - -func resolveGlobs(outDir, pathPattern string) ([]string, error) { - if strings.ContainsAny(pathPattern, "*?[]{}") { - return doublestar.FilepathGlob(filepath.Join(outDir, pathPattern)) - } - return []string{filepath.Join(outDir, pathPattern)}, nil -} - -func applyToFiles(outDir string, pathPattern string, action func(string) error) error { - files, err := resolveGlobs(outDir, pathPattern) - if err != nil { - return fmt.Errorf("failed to resolve glob for %s: %w", pathPattern, err) - } - isGlob := strings.ContainsAny(pathPattern, "*?[]{}") - var replacedAny bool - var lastTextNotFoundErr error - for _, file := range files { - if err := action(file); err != nil { - if isGlob && errors.Is(err, postprocessing.ErrTextNotFound) { - lastTextNotFoundErr = err - continue - } - return err - } - replacedAny = true - } - if isGlob && len(files) > 0 && !replacedAny { - return lastTextNotFoundErr - } - return nil -} diff --git a/internal/librarian/java/postprocess_new_test.go b/internal/librarian/java/postprocess_new_test.go deleted file mode 100644 index 6e058a7cf13..00000000000 --- a/internal/librarian/java/postprocess_new_test.go +++ /dev/null @@ -1,186 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package java - -import ( - "os" - "path/filepath" - "strings" - "testing" - - "github.com/googleapis/librarian/internal/config" -) - -func TestPostProcessLibraryNew(t *testing.T) { - tmpDir := t.TempDir() - - // Setup structure directly in outDir - destDir := filepath.Join(tmpDir, "my-module", "src", "main", "java") - if err := os.MkdirAll(destDir, 0755); err != nil { - t.Fatal(err) - } - - fileContent := `package com.example; -public class File { - public void oldFunc() {} - public void toDelete() { - System.out.println("delete me"); - } -}` - filePath := filepath.Join(destDir, "File.java") - if err := os.WriteFile(filePath, []byte(fileContent), 0644); err != nil { - t.Fatal(err) - } - - // Write mock template to disk - tmplDir := filepath.Join(tmpDir, "template") - if err := os.MkdirAll(tmplDir, 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(tmplDir, "README.md.go.tmpl"), []byte(`# {{ .Metadata.Repo.NamePretty }}`), 0644); err != nil { - t.Fatal(err) - } - - p := libraryPostProcessParams{ - outDir: tmpDir, - cfg: &config.Config{ - Default: &config.Default{ - Java: &config.JavaDefault{ - LibrariesBOMVersion: "1.0.0", - }, - }, - }, - library: &config.Library{ - Version: "1.2.3", - Postprocess: &config.Postprocess{ - Replace: []config.ReplaceConfig{ - { - Path: "**/File.java", - Original: "oldFunc", - Replacement: "newFunc", - }, - }, - MethodOperations: []config.MethodOperation{ - { - Path: "**/File.java", - Action: "delete", - FuncName: "public void toDelete()", - }, - { - Path: "**/File.java", - Action: "duplicate", - FuncName: "public void newFunc()", - NewName: "newFuncCopy", - }, - { - Path: "**/File.java", - Action: "deprecate", - FuncName: "public void newFuncCopy()", - DeprecationMessage: "Use newFunc instead.", - }, - }, - }, - }, - metadata: &repoMetadata{ - NamePretty: "My API", - DistributionName: "com.google.cloud:google-cloud-myapi", - Repo: "googleapis/google-cloud-java", - }, - } - - err := postProcessLibraryNew(t.Context(), p) - if err != nil { - t.Fatal(err) - } - - // Verify File was modified - content, err := os.ReadFile(filePath) - if err != nil { - t.Fatal(err) - } - sContent := string(content) - if !strings.Contains(sContent, "newFunc") { - t.Errorf("Replacement was not applied. Content: %s", sContent) - } - if strings.Contains(sContent, "toDelete") { - t.Errorf("Delete function was not applied. Content: %s", sContent) - } - if !strings.Contains(sContent, "newFuncCopy") { - t.Errorf("Duplicate method operation was not applied. Content: %s", sContent) - } - if !strings.Contains(sContent, "@Deprecated\n\tpublic void newFuncCopy()") { - t.Errorf("@Deprecated annotation was not applied correctly. Content: %s", sContent) - } - if !strings.Contains(sContent, "* @deprecated Use newFunc instead.") { - t.Errorf("Javadoc deprecation tag was not applied correctly. Content: %s", sContent) - } - - // Verify README was rendered - readmePath := filepath.Join(tmpDir, "README.md") - if _, err := os.Stat(readmePath); err != nil { - t.Errorf("README.md was not rendered: %v", err) - } - readmeContent, err := os.ReadFile(readmePath) - if err != nil { - t.Fatal(err) - } - if string(readmeContent) != "# My API" { - t.Errorf("README content mismatch. Got: %s, expected: # My API", string(readmeContent)) - } -} - -func TestPostProcessLibraryNew_ReleasedVersion(t *testing.T) { - tmpDir := t.TempDir() - tmplDir := filepath.Join(tmpDir, "template") - if err := os.MkdirAll(tmplDir, 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(tmplDir, "README.md.go.tmpl"), []byte(`Version: {{ .Version }}`), 0644); err != nil { - t.Fatal(err) - } - - p := libraryPostProcessParams{ - outDir: tmpDir, - cfg: &config.Config{ - Default: &config.Default{ - Java: &config.JavaDefault{ - LibrariesBOMVersion: "1.0.0", - }, - }, - }, - library: &config.Library{ - Version: "3.44.0-SNAPSHOT", - Java: &config.JavaModule{ - ReleasedVersion: "3.43.1", - }, - }, - metadata: &repoMetadata{ - NamePretty: "My API", - }, - } - - if err := postProcessLibraryNew(t.Context(), p); err != nil { - t.Fatal(err) - } - - readmePath := filepath.Join(tmpDir, "README.md") - readmeContent, err := os.ReadFile(readmePath) - if err != nil { - t.Fatal(err) - } - if string(readmeContent) != "Version: 3.43.1" { - t.Errorf("README content mismatch. Got: %s, expected: Version: 3.43.1", string(readmeContent)) - } -} diff --git a/internal/librarian/java/postprocess_test.go b/internal/librarian/java/postprocess_test.go index 20410e6bb5e..a427375af60 100644 --- a/internal/librarian/java/postprocess_test.go +++ b/internal/librarian/java/postprocess_test.go @@ -51,6 +51,9 @@ func TestPostProcessAPI(t *testing.T) { t.Fatal(err) } } + if err := os.WriteFile(filepath.Join(outdir, "owlbot.py"), []byte("#!/usr/bin/env python3\npass"), 0755); err != nil { + t.Fatal(err) + } content := "package com.google.cloud.secretmanager.v1;" grpcFile := filepath.Join(gRPCDir, "GRPCFile.java") if err := os.WriteFile(grpcFile, []byte(content), 0644); err != nil { @@ -1170,7 +1173,7 @@ func TestPostProcessLibrary_Branching(t *testing.T) { } }) - t.Run("UseGoPostprocessor true, no yaml, success", func(t *testing.T) { + t.Run("No owlbot script present, no yaml, success", func(t *testing.T) { outDir := t.TempDir() if err := os.MkdirAll(filepath.Join(outDir, "owl-bot-staging"), 0755); err != nil { @@ -1188,8 +1191,7 @@ func TestPostProcessLibrary_Branching(t *testing.T) { } p := libraryPostProcessParams{ - outDir: outDir, - useGoPostprocessor: true, + outDir: outDir, metadata: &repoMetadata{ NamePretty: "test-library", }, @@ -1218,7 +1220,7 @@ func TestPostProcessLibrary_Branching(t *testing.T) { } }) - t.Run("UseGoPostprocessor true, with postprocess config in library", func(t *testing.T) { + t.Run("No owlbot script present, with postprocess config in library", func(t *testing.T) { outDir := t.TempDir() if err := os.MkdirAll(filepath.Join(outDir, "owl-bot-staging"), 0755); err != nil { @@ -1241,8 +1243,7 @@ func TestPostProcessLibrary_Branching(t *testing.T) { } p := libraryPostProcessParams{ - outDir: outDir, - useGoPostprocessor: true, + outDir: outDir, metadata: &repoMetadata{ NamePretty: "test-library", }, @@ -1289,3 +1290,165 @@ func TestPostProcessLibrary_Branching(t *testing.T) { } }) } + +func TestRunGoPostprocessor(t *testing.T) { + tmpDir := t.TempDir() + + // Setup structure directly in outDir + destDir := filepath.Join(tmpDir, "my-module", "src", "main", "java") + if err := os.MkdirAll(destDir, 0755); err != nil { + t.Fatal(err) + } + + fileContent := `package com.example; +public class File { + public void oldFunc() {} + public void toDelete() { + System.out.println("delete me"); + } +}` + filePath := filepath.Join(destDir, "File.java") + if err := os.WriteFile(filePath, []byte(fileContent), 0644); err != nil { + t.Fatal(err) + } + + // Write mock template to disk + tmplDir := filepath.Join(tmpDir, "template") + if err := os.MkdirAll(tmplDir, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(tmplDir, "README.md.go.tmpl"), []byte(`# {{ .Metadata.Repo.NamePretty }}`), 0644); err != nil { + t.Fatal(err) + } + + p := libraryPostProcessParams{ + outDir: tmpDir, + cfg: &config.Config{ + Default: &config.Default{ + Java: &config.JavaDefault{ + LibrariesBOMVersion: "1.0.0", + }, + }, + }, + library: &config.Library{ + Version: "1.2.3", + Postprocess: &config.Postprocess{ + Replace: []config.ReplaceConfig{ + { + Path: "**/File.java", + Original: "oldFunc", + Replacement: "newFunc", + }, + }, + MethodOperations: []config.MethodOperation{ + { + Path: "**/File.java", + Action: "delete", + FuncName: "public void toDelete()", + }, + { + Path: "**/File.java", + Action: "duplicate", + FuncName: "public void newFunc()", + NewName: "newFuncCopy", + }, + { + Path: "**/File.java", + Action: "deprecate", + FuncName: "public void newFuncCopy()", + DeprecationMessage: "Use newFunc instead.", + }, + }, + }, + }, + metadata: &repoMetadata{ + NamePretty: "My API", + DistributionName: "com.google.cloud:google-cloud-myapi", + Repo: "googleapis/google-cloud-java", + }, + } + + err := runGoPostprocessor(t.Context(), p) + if err != nil { + t.Fatal(err) + } + + // Verify File was modified + content, err := os.ReadFile(filePath) + if err != nil { + t.Fatal(err) + } + sContent := string(content) + if !strings.Contains(sContent, "newFunc") { + t.Errorf("Replacement was not applied. Content: %s", sContent) + } + if strings.Contains(sContent, "toDelete") { + t.Errorf("Delete function was not applied. Content: %s", sContent) + } + if !strings.Contains(sContent, "newFuncCopy") { + t.Errorf("Duplicate method operation was not applied. Content: %s", sContent) + } + if !strings.Contains(sContent, "@Deprecated\n\tpublic void newFuncCopy()") { + t.Errorf("@Deprecated annotation was not applied correctly. Content: %s", sContent) + } + if !strings.Contains(sContent, "* @deprecated Use newFunc instead.") { + t.Errorf("Javadoc deprecation tag was not applied correctly. Content: %s", sContent) + } + + // Verify README was rendered + readmePath := filepath.Join(tmpDir, "README.md") + if _, err := os.Stat(readmePath); err != nil { + t.Errorf("README.md was not rendered: %v", err) + } + readmeContent, err := os.ReadFile(readmePath) + if err != nil { + t.Fatal(err) + } + if string(readmeContent) != "# My API" { + t.Errorf("README content mismatch. Got: %s, expected: # My API", string(readmeContent)) + } +} + +func TestRunGoPostprocessor_ReleasedVersion(t *testing.T) { + tmpDir := t.TempDir() + tmplDir := filepath.Join(tmpDir, "template") + if err := os.MkdirAll(tmplDir, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(tmplDir, "README.md.go.tmpl"), []byte(`Version: {{ .Version }}`), 0644); err != nil { + t.Fatal(err) + } + + p := libraryPostProcessParams{ + outDir: tmpDir, + cfg: &config.Config{ + Default: &config.Default{ + Java: &config.JavaDefault{ + LibrariesBOMVersion: "1.0.0", + }, + }, + }, + library: &config.Library{ + Version: "3.44.0-SNAPSHOT", + Java: &config.JavaModule{ + ReleasedVersion: "3.43.1", + }, + }, + metadata: &repoMetadata{ + NamePretty: "My API", + }, + } + + if err := runGoPostprocessor(t.Context(), p); err != nil { + t.Fatal(err) + } + + readmePath := filepath.Join(tmpDir, "README.md") + readmeContent, err := os.ReadFile(readmePath) + if err != nil { + t.Fatal(err) + } + if string(readmeContent) != "Version: 3.43.1" { + t.Errorf("README content mismatch. Got: %s, expected: Version: 3.43.1", string(readmeContent)) + } +} From 1b04ffabd291237c70f4247e408a72addc2a9c21 Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Sun, 28 Jun 2026 17:25:45 -0400 Subject: [PATCH 062/108] chore(internal/librarian/java): revert unnecessary filepath.Abs in test files to upstream --- internal/librarian/java/postgenerate_test.go | 5 +---- internal/librarian/java/repometadata_test.go | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/internal/librarian/java/postgenerate_test.go b/internal/librarian/java/postgenerate_test.go index 8b943804b95..e097fe25d5b 100644 --- a/internal/librarian/java/postgenerate_test.go +++ b/internal/librarian/java/postgenerate_test.go @@ -33,10 +33,7 @@ func TestPostGenerate(t *testing.T) { t.Parallel() tmpDir := t.TempDir() // Copy testdata to tmpDir - testdataDir, err := filepath.Abs(filepath.Join("testdata", "postgenerate")) - if err != nil { - t.Fatal(err) - } + testdataDir := filepath.Join("testdata", "postgenerate") if err := copyDir(testdataDir, tmpDir); err != nil { t.Fatal(err) } diff --git a/internal/librarian/java/repometadata_test.go b/internal/librarian/java/repometadata_test.go index ef94400cf07..c1f42a8ac91 100644 --- a/internal/librarian/java/repometadata_test.go +++ b/internal/librarian/java/repometadata_test.go @@ -76,10 +76,7 @@ func TestRepoMetadata_write(t *testing.T) { func TestDeriveRepoMetadata_Overrides(t *testing.T) { t.Parallel() apiPath := "google/cloud/secretmanager/v1" - googleapis, err := filepath.Abs("../../testdata/googleapis") - if err != nil { - t.Fatal(err) - } + googleapis := "../../testdata/googleapis" cfg := sample.Config() cfg.Language = config.LanguageJava From be45eb84d8dd6164d08be829088e5ab268853fd5 Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Sun, 28 Jun 2026 17:29:40 -0400 Subject: [PATCH 063/108] style(internal/librarian/java): align generateAPIParams struct fields and revert unnecessary sidekick diff --- internal/librarian/java/generate.go | 2 +- internal/sidekick/parser/protobuf.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/librarian/java/generate.go b/internal/librarian/java/generate.go index adb211cae33..26972398e5c 100644 --- a/internal/librarian/java/generate.go +++ b/internal/librarian/java/generate.go @@ -57,7 +57,7 @@ var ( ) type generateAPIParams struct { - cfg *config.Config + cfg *config.Config api *config.API library *config.Library srcCfg *sources.SourceConfig diff --git a/internal/sidekick/parser/protobuf.go b/internal/sidekick/parser/protobuf.go index 30df14d56ec..197a8ac0895 100644 --- a/internal/sidekick/parser/protobuf.go +++ b/internal/sidekick/parser/protobuf.go @@ -169,6 +169,7 @@ func runProtoc(files []string, sourceCfg *sources.SourceConfig) ([]byte, error) args := []string{ "--include_imports", "--include_source_info", + "--retain_options", "--descriptor_set_out", tempFile.Name(), } if sourceCfg != nil { From f718a667250acebbf590e1ff55ff9085e34f600b Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Sun, 28 Jun 2026 17:32:41 -0400 Subject: [PATCH 064/108] chore(internal/librarian/java): remove unused legacy README.md.tmpl --- .../librarian/java/template/README.md.tmpl | 266 ------------------ 1 file changed, 266 deletions(-) delete mode 100644 internal/librarian/java/template/README.md.tmpl diff --git a/internal/librarian/java/template/README.md.tmpl b/internal/librarian/java/template/README.md.tmpl deleted file mode 100644 index a2d5a55b477..00000000000 --- a/internal/librarian/java/template/README.md.tmpl +++ /dev/null @@ -1,266 +0,0 @@ -# Google {{ .Metadata.Repo.NamePretty }} Client for Java - -Java idiomatic client for [{{ .Metadata.Repo.NamePretty }}][product-docs]. - -[![Maven][maven-version-image]][maven-version-link] -![Stability][stability-image] - -- [Product Documentation][product-docs] -- [Client Library Documentation][javadocs] - -{{ if and .Metadata.Partials .Metadata.Partials.DeprecationWarning }} -{{ .Metadata.Partials.DeprecationWarning }} -{{ else if eq .Metadata.Repo.ReleaseLevel "preview" }} -> Note: This client is a work-in-progress, and may occasionally -> make backwards-incompatible changes. -{{ end }} - -{{ if .MigratedSplitRepo }} -:bus: In October 2022, this library has moved to -[google-cloud-java/{{ .Metadata.Repo.RepoShort }}]( -https://github.com/googleapis/google-cloud-java/tree/main/{{ .Metadata.Repo.RepoShort }}). -This repository will be archived in the future. -Future releases will appear in the new repository (https://github.com/googleapis/google-cloud-java/releases). -The Maven artifact coordinates (`{{ .GroupID }}:{{ .ArtifactID }}`) remain the same. -{{ end }} - -## Quickstart - -{{ if and .Metadata.Snippets (index .Metadata.Snippets (print .Metadata.Repo.APIShortname "_install_with_bom")) }} -If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file: - -```xml -{{ index .Metadata.Snippets (print .Metadata.Repo.APIShortname "_install_with_bom") }} -``` - -If you are using Maven without the BOM, add this to your dependencies: -{{ else if and .Monorepo (or (eq .GroupID "com.google.cloud") (eq .GroupID "com.google.analytics") (eq .GroupID "com.google.area120")) }} -If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file: - -```xml - - - - com.google.cloud - libraries-bom - {{ .Metadata.LibrariesBOMVersion }} - pom - import - - - - - - - {{ .GroupID }} - {{ .ArtifactID }} - - -``` - -If you are using Maven without the BOM, add this to your dependencies: -{{ else }} -If you are using Maven, add this to your pom.xml file: -{{ end }} - -```xml -{{ if and .Metadata.Snippets (index .Metadata.Snippets (print .Metadata.Repo.APIShortname "_install_without_bom")) }} -{{ index .Metadata.Snippets (print .Metadata.Repo.APIShortname "_install_without_bom") }} -{{ else }} - - {{ .GroupID }} - {{ .ArtifactID }} - {{ .Version }} - -{{ end }} -``` - -{{ if and .Metadata.Snippets (index .Metadata.Snippets (print .Metadata.Repo.APIShortname "_install_with_bom")) }} -If you are using Gradle 5.x or later, add this to your dependencies: - -```Groovy -implementation platform('com.google.cloud:libraries-bom:{{ .Metadata.LibrariesBOMVersion }}') - -implementation '{{ .GroupID }}:{{ .ArtifactID }}' -``` -{{ end }} - -If you are using Gradle without BOM, add this to your dependencies: - -```Groovy -implementation '{{ .GroupID }}:{{ .ArtifactID }}:{{ .Version }}' -``` - -If you are using SBT, add this to your dependencies: - -```Scala -libraryDependencies += "{{ .GroupID }}" % "{{ .ArtifactID }}" % "{{ .Version }}" -``` - -## Authentication - -See the [Authentication][authentication] section in the base directory's README. - -## Authorization - -The client application making API calls must be granted [authorization scopes][auth-scopes] required for the desired {{ .Metadata.Repo.NamePretty }} APIs, and the authenticated principal must have the [IAM role(s)][predefined-iam-roles] required to access GCP resources using the {{ .Metadata.Repo.NamePretty }} API calls. - -## Getting Started - -### Prerequisites - -You will need a [Google Cloud Platform Console][developer-console] project with the {{ .Metadata.Repo.NamePretty }} [API enabled][enable-api]. -{{ if .Metadata.Repo.RequiresBilling }}You will need to [enable billing][enable-billing] to use Google {{ .Metadata.Repo.NamePretty }}.{{ end }} -[Follow these instructions][create-project] to get your project set up. You will also need to set up the local development environment by -[installing the Google Cloud Command Line Interface][cloud-cli] and running the following commands in command line: -`gcloud auth login` and `gcloud config set project [YOUR PROJECT ID]`. - -### Installation and setup - -You'll need to obtain the `{{ .ArtifactID }}` library. See the [Quickstart](#quickstart) section -to add `{{ .ArtifactID }}` as a dependency in your code. - -## About {{ .Metadata.Repo.NamePretty }} - -{{ if and .Metadata.Partials .Metadata.Partials.About }} -{{ .Metadata.Partials.About }} -{{ else }} -[{{ .Metadata.Repo.NamePretty }}][product-docs] {{ .Metadata.Repo.APIDescription }} - -See the [{{ .Metadata.Repo.NamePretty }} client library docs][javadocs] to learn how to -use this {{ .Metadata.Repo.NamePretty }} Client Library. -{{ end }} - -{{ if and .Metadata.Partials .Metadata.Partials.CustomContent }} -{{ .Metadata.Partials.CustomContent }} -{{ end }} - -{{ if .Metadata.Samples }} -## Samples - -Samples are in the [`samples/`](https://github.com/{{ .Metadata.Repo.Repo }}/tree/main/samples) directory. - -| Sample | Source Code | Try it | -| --------------------------- | --------------------------------- | ------ | -{{ range .Metadata.Samples }}| {{ .Title }} | [source code](https://github.com/{{ $.Metadata.Repo.Repo }}/blob/main/{{ .File }}) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/{{ $.Metadata.Repo.Repo }}&page=editor&open_in_editor={{ .File }}) | -{{ end }} -{{ end }} - -## Troubleshooting - -To get help, follow the instructions in the [shared Troubleshooting document][troubleshooting]. - -{{ if .Metadata.Repo.Transport }} -## Transport - -{{ if eq .Metadata.Repo.Transport "grpc" }} -{{ .Metadata.Repo.NamePretty }} uses gRPC for the transport layer. -{{ else if eq .Metadata.Repo.Transport "http" }} -{{ .Metadata.Repo.NamePretty }} uses HTTP/JSON for the transport layer. -{{ else if eq .Metadata.Repo.Transport "both" }} -{{ .Metadata.Repo.NamePretty }} uses both gRPC and HTTP/JSON for the transport layer. -{{ end }} -{{ end }} - -## Supported Java Versions - -Java {{ .Metadata.MinJavaVersion }} or above is required for using this client. - -Google's Java client libraries, -[Google Cloud Client Libraries][cloudlibs] -and -[Google Cloud API Libraries][apilibs], -follow the -[Oracle Java SE support roadmap][oracle] -(see the Oracle Java SE Product Releases section). - -### For new development - -In general, new feature development occurs with support for the lowest Java -LTS version covered by Oracle's Premier Support (which typically lasts 5 years -from initial General Availability). If the minimum required JVM for a given -library is changed, it is accompanied by a [semver][semver] major release. - -Java 11 and (in September 2021) Java 17 are the best choices for new -development. - -### Keeping production systems current - -Google tests its client libraries with all current LTS versions covered by -Oracle's Extended Support (which typically lasts 8 years from initial -General Availability). - -#### Legacy support - -Google's client libraries support legacy versions of Java runtimes with long -term stable libraries that don't receive feature updates on a best efforts basis -as it may not be possible to backport all patches. - -Google provides updates on a best efforts basis to apps that continue to use -Java 7, though apps might need to upgrade to current versions of the library -that supports their JVM. - -#### Where to find specific information - -The latest versions and the supported Java versions are identified on -the individual GitHub repository `github.com/GoogleAPIs/java-SERVICENAME` -and on [google-cloud-java][g-c-j]. - -## Versioning - -{{ if and .Metadata.Partials .Metadata.Partials.Versioning }} -{{ .Metadata.Partials.Versioning }} -{{ else }} -This library follows [Semantic Versioning](http://semver.org/). - -{{ if eq .Metadata.Repo.ReleaseLevel "preview" }} -It is currently in major version zero (``0.y.z``), which means that anything may change at any time -and the public API should not be considered stable. -{{ end }}{{ end }} - -## Contributing - -{{ if and .Metadata.Partials .Metadata.Partials.Contributing }} -{{ .Metadata.Partials.Contributing }} -{{ else }} -Contributions to this library are always welcome and highly encouraged. - -See [CONTRIBUTING][contributing] for more information how to get started. - -Please note that this project is released with a Contributor Code of Conduct. By participating in -this project you agree to abide by its terms. See [Code of Conduct][code-of-conduct] for more -information. -{{ end }} - -## License - -Apache 2.0 - See [LICENSE][license] for more information. - -Java is a registered trademark of Oracle and/or its affiliates. - -[product-docs]: {{ .Metadata.Repo.ProductDocumentation }} -[javadocs]: {{ .Metadata.Repo.ClientDocumentation }} -[stability-image]: https://img.shields.io/badge/stability-{{ if eq .Metadata.Repo.ReleaseLevel "stable" }}stable-green{{ else if eq .Metadata.Repo.ReleaseLevel "preview" }}preview-yellow{{ else }}unknown-red{{ end }} -[maven-version-image]: https://img.shields.io/maven-central/v/{{ .GroupID }}/{{ .ArtifactID }}.svg -[maven-version-link]: https://central.sonatype.com/artifact/{{ .GroupID }}/{{ .ArtifactID }}/{{ .Version }} -[authentication]: https://github.com/googleapis/google-cloud-java#authentication -[auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes -[predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles -[iam-policy]: https://cloud.google.com/iam/docs/overview#cloud-iam-policy -[developer-console]: https://console.developers.google.com/ -[create-project]: https://cloud.google.com/resource-manager/docs/creating-managing-projects -[cloud-cli]: https://cloud.google.com/cli -[troubleshooting]: https://github.com/googleapis/google-cloud-java/blob/main/TROUBLESHOOTING.md -[contributing]: https://github.com/{{ .Metadata.Repo.Repo }}/blob/main/CONTRIBUTING.md -[code-of-conduct]: https://github.com/{{ .Metadata.Repo.Repo }}/blob/main/CODE_OF_CONDUCT.md#contributor-code-of-conduct -[license]: https://github.com/{{ .Metadata.Repo.Repo }}/blob/main/LICENSE -{{ if .Metadata.Repo.RequiresBilling }}[enable-billing]: https://cloud.google.com/apis/docs/getting-started#enabling_billing{{ end }} -{{ if .Metadata.Repo.APIID }}[enable-api]: https://console.cloud.google.com/flows/enableapi?apiid={{ .Metadata.Repo.APIID }}{{ end }} -[libraries-bom]: https://github.com/GoogleCloudPlatform/cloud-opensource-java/wiki/The-Google-Cloud-Platform-Libraries-BOM -[shell_img]: https://gstatic.com/cloudssh/images/open-btn.png - -[semver]: https://semver.org/ -[cloudlibs]: https://cloud.google.com/apis/docs/client-libraries-explained -[apilibs]: https://cloud.google.com/apis/docs/client-libraries-explained#google_api_client_libraries -[oracle]: https://www.oracle.com/java/technologies/java-se-support-roadmap.html -[g-c-j]: http://github.com/googleapis/google-cloud-java From a08b989e621a2358c5b217882c3db9883028ed0d Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Sun, 28 Jun 2026 17:42:33 -0400 Subject: [PATCH 065/108] chore(internal/librarian/java): revert unnecessary filepath.Abs in generate_test.go to upstream --- internal/librarian/java/generate_test.go | 29 +++++++----------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/internal/librarian/java/generate_test.go b/internal/librarian/java/generate_test.go index 6fb4b7f5f28..c511a55653f 100644 --- a/internal/librarian/java/generate_test.go +++ b/internal/librarian/java/generate_test.go @@ -17,7 +17,6 @@ package java import ( "context" "errors" - "io/fs" "os" "path/filepath" "strings" @@ -303,11 +302,7 @@ func TestGenerateAPI(t *testing.T) { t.Fatal(err) } } - absGoogleapisDir, err := filepath.Abs(googleapisDir) - if err != nil { - t.Fatal(err) - } - apiCfg, err := serviceconfig.Find(absGoogleapisDir, "google/cloud/secretmanager/v1", config.LanguageJava) + apiCfg, err := serviceconfig.Find(googleapisDir, "google/cloud/secretmanager/v1", config.LanguageJava) if err != nil { t.Fatal(err) } @@ -315,7 +310,7 @@ func TestGenerateAPI(t *testing.T) { cfg: cfg, api: library.APIs[0], library: library, - srcCfg: sources.NewSourceConfig(&sources.Sources{Googleapis: absGoogleapisDir}, nil), + srcCfg: sources.NewSourceConfig(&sources.Sources{Googleapis: googleapisDir}, nil), outdir: outdir, metadata: &repoMetadata{ NamePretty: "Secret Manager", @@ -372,11 +367,7 @@ func TestGenerateAPI_ProtoOnly(t *testing.T) { t.Fatal(err) } } - absGoogleapisDir, err := filepath.Abs(googleapisDir) - if err != nil { - t.Fatal(err) - } - apiCfg, err := serviceconfig.Find(absGoogleapisDir, "google/cloud/gkehub/policycontroller/v1beta", config.LanguageJava) + apiCfg, err := serviceconfig.Find(googleapisDir, "google/cloud/gkehub/policycontroller/v1beta", config.LanguageJava) if err != nil { t.Fatal(err) } @@ -384,7 +375,7 @@ func TestGenerateAPI_ProtoOnly(t *testing.T) { cfg: cfg, api: library.APIs[0], library: library, - srcCfg: sources.NewSourceConfig(&sources.Sources{Googleapis: absGoogleapisDir}, nil), + srcCfg: sources.NewSourceConfig(&sources.Sources{Googleapis: googleapisDir}, nil), outdir: outdir, metadata: &repoMetadata{ NamePretty: "GKE Hub API", @@ -520,11 +511,7 @@ func TestGenerateAPI_WithAdditionalProtosToGenerateAndCopy(t *testing.T) { t.Fatal(err) } } - absGoogleapisDir, err := filepath.Abs(googleapisDir) - if err != nil { - t.Fatal(err) - } - apiCfg, err := serviceconfig.Find(absGoogleapisDir, "google/cloud/secretmanager/v1", config.LanguageJava) + apiCfg, err := serviceconfig.Find(googleapisDir, "google/cloud/secretmanager/v1", config.LanguageJava) if err != nil { t.Fatal(err) } @@ -532,7 +519,7 @@ func TestGenerateAPI_WithAdditionalProtosToGenerateAndCopy(t *testing.T) { cfg: cfg, api: library.APIs[0], library: library, - srcCfg: sources.NewSourceConfig(&sources.Sources{Googleapis: absGoogleapisDir}, nil), + srcCfg: sources.NewSourceConfig(&sources.Sources{Googleapis: googleapisDir}, nil), outdir: outdir, metadata: &repoMetadata{ NamePretty: "Secret Manager", @@ -1114,7 +1101,7 @@ func TestGenerateAPI_Gating(t *testing.T) { if gotProtoDir { resNameFile := filepath.Join(stagingProtoPath, "com", "google", "cloud", "secretmanager", "v1", "SecretName.java") _, errRes := os.Stat(resNameFile) - gotResNameFiles := !errors.Is(errRes, fs.ErrNotExist) + gotResNameFiles := !os.IsNotExist(errRes) if gotResNameFiles != test.wantResNameFiles { t.Errorf("gotResNameFiles = %v, want %v (file: %s)", gotResNameFiles, test.wantResNameFiles, resNameFile) } @@ -1126,7 +1113,7 @@ func TestGenerateAPI_Gating(t *testing.T) { func assertDirExists(t *testing.T, path string, want bool, desc string) bool { t.Helper() _, err := os.Stat(path) - got := !errors.Is(err, fs.ErrNotExist) + got := !os.IsNotExist(err) if got != want { t.Errorf("expected %s existence to be %v, got %v (path: %s)", desc, want, got, path) } From 48461260c54af2b35ce043571afb917c754d0966 Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Sun, 28 Jun 2026 18:10:28 -0400 Subject: [PATCH 066/108] refactor(internal/librarian/java): decouple legacy staging restructure from modern direct restructure --- internal/librarian/java/postprocess.go | 147 ++++++++++++++++---- internal/librarian/java/postprocess_test.go | 72 +++++++++- 2 files changed, 183 insertions(+), 36 deletions(-) diff --git a/internal/librarian/java/postprocess.go b/internal/librarian/java/postprocess.go index 08aafbdb555..af5161b5825 100644 --- a/internal/librarian/java/postprocess.go +++ b/internal/librarian/java/postprocess.go @@ -158,7 +158,7 @@ func postProcessAPI(ctx context.Context, params postProcessParams) error { } if useGo { keepSet := toKeepSet(params.library.Keep) - if err := restructureModules(params, params.outDir, keepSet, params.outDir); err != nil { + if err := restructureModulesDirect(params, params.outDir, keepSet); err != nil { return fmt.Errorf("failed to restructure direct to outDir: %w", err) } @@ -315,10 +315,13 @@ func removeConflictingFiles(protoSrcDir string) error { func restructureToStaging(params postProcessParams) error { stagingDir := stagingDir(params.outDir) destRoot := filepath.Join(stagingDir, params.apiBase) + if params.javaAPI.Monolithic { + destRoot = filepath.Join(destRoot, "src") + } if err := os.MkdirAll(destRoot, 0755); err != nil { return fmt.Errorf("failed to create staging directory: %w", err) } - return restructureModules(params, destRoot, nil, "") + return restructureModules(params, destRoot) } type moveAction struct { @@ -326,36 +329,14 @@ type moveAction struct { description string } -func restructure(actions []moveAction, keepSet map[string]bool, libraryRoot string) error { +func restructure(actions []moveAction) error { for _, action := range actions { if _, err := os.Stat(action.src); err == nil { if err := os.MkdirAll(action.dest, 0755); err != nil { return fmt.Errorf("failed to create directory %s: %w", action.dest, err) } - if keepSet != nil { - keepFunc := func(p string) bool { - if shouldPreserve(p, keepSet) { - return true - } - // Also preserve existing files that lack the auto-generated marker. - destPath := filepath.Join(libraryRoot, p) - if _, err := os.Stat(destPath); err == nil { - if filepath.Ext(destPath) == ".java" { - isGen, err := hasMarker(destPath) - if err == nil && !isGen { - return true - } - } - } - return false - } - if err := filesystem.MoveAndMergeWithKeep(action.src, action.dest, libraryRoot, keepFunc); err != nil { - return fmt.Errorf("failed to move with keep %s: %w", action.description, err) - } - } else { - if err := filesystem.MoveAndMerge(action.src, action.dest); err != nil { - return fmt.Errorf("failed to move %s: %w", action.description, err) - } + if err := filesystem.MoveAndMerge(action.src, action.dest); err != nil { + return fmt.Errorf("failed to move %s: %w", action.description, err) } } } @@ -365,7 +346,85 @@ func restructure(actions []moveAction, keepSet map[string]bool, libraryRoot stri // restructureModules moves the generated code from the temporary versioned directory // tree into the destination root directory for GAPIC, Proto, gRPC, and samples. // It also copies the relevant proto files into the proto module. -func restructureModules(params postProcessParams, destRoot string, keepSet map[string]bool, libraryRoot string) error { +func restructureModules(params postProcessParams, destRoot string) error { + coords := params.coords() + tempProtoSrcDir := params.protoDir() + if params.library.Name != commonProtosLibrary { + if err := removeConflictingFiles(tempProtoSrcDir); err != nil { + return err + } + } + + protoDest := filepath.Join(destRoot, coords.Proto.ArtifactID, "src", "main", "java") + grpcDest := filepath.Join(destRoot, coords.GRPC.ArtifactID, "src", "main", "java") + gapicMainDest := filepath.Join(destRoot, coords.GAPIC.ArtifactID, "src", "main") + gapicTestDest := filepath.Join(destRoot, coords.GAPIC.ArtifactID, "src", "test") + protoFilesDestDir := filepath.Join(destRoot, coords.Proto.ArtifactID, "src", "main", "proto") + + if params.javaAPI.Monolithic { + protoDest = filepath.Join(destRoot, "main", "java") + grpcDest = filepath.Join(destRoot, "main", "java") + gapicMainDest = filepath.Join(destRoot, "main") + gapicTestDest = filepath.Join(destRoot, "test") + protoFilesDestDir = filepath.Join(destRoot, "main", "proto") + } + + var actions []moveAction + if shouldGenerateProto(params.javaAPI) { + actions = append(actions, moveAction{ + src: tempProtoSrcDir, + dest: protoDest, + description: "proto source", + }) + } + if shouldGenerateGRPC(params.javaAPI) { + actions = append(actions, moveAction{ + src: params.gRPCDir(), + dest: grpcDest, + description: "grpc source", + }) + } + if shouldGenerateGAPIC(params.javaAPI) { + actions = append(actions, []moveAction{ + { + src: filepath.Join(params.gapicDir(), "src", "main"), + dest: gapicMainDest, + description: "gapic source", + }, + { + src: filepath.Join(params.gapicDir(), "src", "test"), + dest: gapicTestDest, + description: "gapic test", + }, + }...) + } + if shouldGenerateResourceNames(params.javaAPI) { + actions = append(actions, moveAction{ + src: filepath.Join(params.gapicDir(), "proto", "src", "main", "java"), + dest: protoDest, + description: "resource name source", + }) + } + if params.includeSamples && shouldGenerateGAPIC(params.javaAPI) { + actions = append(actions, moveAction{ + src: filepath.Join(params.gapicDir(), "samples", "snippets", "generated", "src", "main", "java"), + dest: filepath.Join(destRoot, "samples", "snippets", "generated"), + description: "samples", + }) + } + if err := restructure(actions); err != nil { + return err + } + // Copy proto files to proto-*/src/main/proto + if shouldGenerateProto(params.javaAPI) { + if err := copyProtos(params.protosToCopy, protoFilesDestDir); err != nil { + return fmt.Errorf("failed to copy proto files: %w", err) + } + } + return nil +} + +func restructureModulesDirect(params postProcessParams, destRoot string, keepSet map[string]bool) error { coords := params.coords() tempProtoSrcDir := params.protoDir() if params.library.Name != commonProtosLibrary { @@ -431,10 +490,9 @@ func restructureModules(params postProcessParams, destRoot string, keepSet map[s description: "samples", }) } - if err := restructure(actions, keepSet, libraryRoot); err != nil { + if err := restructureDirect(actions, keepSet, destRoot); err != nil { return err } - // Copy proto files to proto-*/src/main/proto if shouldGenerateProto(params.javaAPI) { if err := copyProtos(params.protosToCopy, protoFilesDestDir); err != nil { return fmt.Errorf("failed to copy proto files: %w", err) @@ -443,6 +501,35 @@ func restructureModules(params postProcessParams, destRoot string, keepSet map[s return nil } +func restructureDirect(actions []moveAction, keepSet map[string]bool, libraryRoot string) error { + for _, action := range actions { + if _, err := os.Stat(action.src); err == nil { + if err := os.MkdirAll(action.dest, 0755); err != nil { + return fmt.Errorf("failed to create directory %s: %w", action.dest, err) + } + keepFunc := func(p string) bool { + if shouldPreserve(p, keepSet) { + return true + } + destPath := filepath.Join(libraryRoot, p) + if _, err := os.Stat(destPath); err == nil { + if filepath.Ext(destPath) == ".java" { + isGen, err := hasMarker(destPath) + if err == nil && !isGen { + return true + } + } + } + return false + } + if err := filesystem.MoveAndMergeWithKeep(action.src, action.dest, libraryRoot, keepFunc); err != nil { + return fmt.Errorf("failed to move with keep %s: %w", action.description, err) + } + } + } + return nil +} + // runOwlBot executes the owlbot.py script located in outDir to restructure the // generated code and apply templates (e.g., for README.md). // diff --git a/internal/librarian/java/postprocess_test.go b/internal/librarian/java/postprocess_test.go index a427375af60..1c2ce5a6996 100644 --- a/internal/librarian/java/postprocess_test.go +++ b/internal/librarian/java/postprocess_test.go @@ -230,7 +230,7 @@ func TestRestructureModules(t *testing.T) { javaAPI: &config.JavaAPI{}, } destRoot := filepath.Join(tmpDir, "dest") - if err := restructureModules(params, destRoot, nil, ""); err != nil { + if err := restructureModules(params, destRoot); err != nil { t.Fatal(err) } @@ -277,7 +277,7 @@ func TestRestructureModules_CommonProtos(t *testing.T) { }, } destRoot := filepath.Join(tmpDir, "dest") - if err := restructureModules(params, destRoot, nil, ""); err != nil { + if err := restructureModules(params, destRoot); err != nil { t.Fatal(err) } wantPath := filepath.Join(destRoot, "proto-google-common-protos", "src", "main", "java", "com", "google", "cloud", "location", "LocationsProto.java") @@ -305,7 +305,7 @@ func TestRestructureModules_ShouldRemoveClasses(t *testing.T) { javaAPI: &config.JavaAPI{}, } destRoot := filepath.Join(tmpDir, "dest") - if err := restructureModules(params, destRoot, nil, ""); err != nil { + if err := restructureModules(params, destRoot); err != nil { t.Fatal(err) } wantPath := filepath.Join(destRoot, "proto-google-cloud-secretmanager-v1", "src", "main", "java", "com", "google", "cloud", "location", "LocationsProto.java") @@ -362,7 +362,7 @@ func TestRestructureModules_SamplesDisabled(t *testing.T) { javaAPI: &config.JavaAPI{}, } destRoot := filepath.Join(tmpDir, "dest") - if err := restructureModules(params, destRoot, nil, ""); err != nil { + if err := restructureModules(params, destRoot); err != nil { t.Fatal(err) } // Verify sample file location DOES NOT exist @@ -418,11 +418,71 @@ func TestRestructureModules_Monolithic(t *testing.T) { }, } destRoot := filepath.Join(tmpDir, "dest") - if err := restructureModules(params, destRoot, nil, ""); err != nil { + if err := restructureModules(params, destRoot); err != nil { + t.Fatal(err) + } + + // Verify all files are in the same main directory (upstream staging structure) + files := []string{ + filepath.Join(destRoot, "main", "java", "Gapic.java"), + filepath.Join(destRoot, "main", "java", "Grpc.java"), + filepath.Join(destRoot, "main", "java", "Proto.java"), + } + for _, f := range files { + if _, err := os.Stat(f); err != nil { + t.Errorf("expected file %s to exist, but it was not found: %v", f, err) + } + } +} + +func TestRestructureModulesDirect_Monolithic(t *testing.T) { + t.Parallel() + tmpDir := t.TempDir() + apiBase := "v1" + libraryID := "grafeas" + + dirs := []string{ + filepath.Join(tmpDir, apiBase, "gapic", "src", "main", "java"), + filepath.Join(tmpDir, apiBase, "grpc"), + filepath.Join(tmpDir, apiBase, "proto"), + } + for _, dir := range dirs { + if err := os.MkdirAll(dir, 0755); err != nil { + t.Fatal(err) + } + } + gapicFile := filepath.Join(tmpDir, apiBase, "gapic", "src", "main", "java", "Gapic.java") + if err := os.WriteFile(gapicFile, []byte("public class Gapic {}"), 0644); err != nil { + t.Fatal(err) + } + grpcFile := filepath.Join(tmpDir, apiBase, "grpc", "Grpc.java") + if err := os.WriteFile(grpcFile, []byte("public class Grpc {}"), 0644); err != nil { + t.Fatal(err) + } + protoFile := filepath.Join(tmpDir, apiBase, "proto", "Proto.java") + if err := os.WriteFile(protoFile, []byte("public class Proto {}"), 0644); err != nil { + t.Fatal(err) + } + params := postProcessParams{ + outDir: tmpDir, + library: &config.Library{ + Name: libraryID, + Java: &config.JavaModule{ + GroupID: "com.google.cloud", + }, + }, + apiBase: apiBase, + + includeSamples: false, + javaAPI: &config.JavaAPI{ + Monolithic: true, + }, + } + destRoot := filepath.Join(tmpDir, "dest") + if err := restructureModulesDirect(params, destRoot, nil); err != nil { t.Fatal(err) } - // Verify all files are in the same src directory files := []string{ filepath.Join(destRoot, "src", "main", "java", "Gapic.java"), filepath.Join(destRoot, "src", "main", "java", "Grpc.java"), From 53a55e37dd452a23fc1bb38bd645fb95949df666 Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Sun, 28 Jun 2026 18:15:40 -0400 Subject: [PATCH 067/108] refactor(internal/librarian/java): revert removeKeptFilesFromStaging to exact upstream implementation --- internal/librarian/java/postprocess.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/internal/librarian/java/postprocess.go b/internal/librarian/java/postprocess.go index af5161b5825..10034064e5a 100644 --- a/internal/librarian/java/postprocess.go +++ b/internal/librarian/java/postprocess.go @@ -601,10 +601,14 @@ func toKeepSet(keep []string) map[string]bool { func removeKeptFilesFromStaging(library *config.Library, outDir string) error { stagingDir := stagingDir(outDir) - if _, err := os.Stat(stagingDir); errors.Is(err, fs.ErrNotExist) { + if _, err := os.Stat(stagingDir); os.IsNotExist(err) { return nil } - keepSet := toKeepSet(library.Keep) + keepSet := make(map[string]bool) + for _, keep := range library.Keep { + normalized := strings.TrimSuffix(filepath.ToSlash(keep), "/") + keepSet[normalized] = true + } return filepath.WalkDir(stagingDir, func(path string, d os.DirEntry, err error) error { if err != nil { return err From 4d8ff2f5087b08d233f30f35443ef6f97f3113f1 Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Sun, 28 Jun 2026 18:44:28 -0400 Subject: [PATCH 068/108] refactor(internal/librarian/java): reuse toKeepSet helper in runGoPostprocessor --- internal/librarian/java/postprocess.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/internal/librarian/java/postprocess.go b/internal/librarian/java/postprocess.go index 10034064e5a..7d1e17fca6f 100644 --- a/internal/librarian/java/postprocess.go +++ b/internal/librarian/java/postprocess.go @@ -675,11 +675,7 @@ func createOrVerifyOwlbotPy(outDir string) (err error) { } func runGoPostprocessor(ctx context.Context, p libraryPostProcessParams) error { - keepSet := make(map[string]bool, len(p.library.Keep)) - for _, k := range p.library.Keep { - normalized := strings.TrimSuffix(filepath.ToSlash(k), "/") - keepSet[normalized] = true - } + keepSet := toKeepSet(p.library.Keep) // 1. Load postprocess configuration and apply operations if p.library.Postprocess != nil { From a563367fc377c14171a0b2b2f7f11380b0037b60 Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Sun, 28 Jun 2026 18:57:39 -0400 Subject: [PATCH 069/108] refactor(internal/librarian/java): make README template render hermetic with go:embed --- internal/librarian/java/postprocess_test.go | 37 ++----------------- internal/librarian/java/template_render.go | 31 +++++----------- .../librarian/java/template_render_test.go | 24 +++--------- 3 files changed, 20 insertions(+), 72 deletions(-) diff --git a/internal/librarian/java/postprocess_test.go b/internal/librarian/java/postprocess_test.go index 1c2ce5a6996..e41e6e40ff6 100644 --- a/internal/librarian/java/postprocess_test.go +++ b/internal/librarian/java/postprocess_test.go @@ -1243,13 +1243,6 @@ func TestPostProcessLibrary_Branching(t *testing.T) { if err := os.WriteFile(filepath.Join(outDir, ".repo-metadata.json"), []byte(metadata), 0644); err != nil { t.Fatal(err) } - if err := os.MkdirAll(filepath.Join(outDir, "template"), 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(outDir, "template", "README.md.go.tmpl"), []byte("dummy"), 0644); err != nil { - t.Fatal(err) - } - p := libraryPostProcessParams{ outDir: outDir, metadata: &repoMetadata{ @@ -1290,17 +1283,11 @@ func TestPostProcessLibrary_Branching(t *testing.T) { if err := os.WriteFile(filepath.Join(outDir, ".repo-metadata.json"), []byte(metadata), 0644); err != nil { t.Fatal(err) } - if err := os.MkdirAll(filepath.Join(outDir, "template"), 0755); err != nil { - t.Fatal(err) - } // Write a file to apply replacements on testFile := filepath.Join(outDir, "TestFile.java") if err := os.WriteFile(testFile, []byte("Hello World"), 0644); err != nil { t.Fatal(err) } - if err := os.WriteFile(filepath.Join(outDir, "template", "README.md.go.tmpl"), []byte("dummy"), 0644); err != nil { - t.Fatal(err) - } p := libraryPostProcessParams{ outDir: outDir, @@ -1372,15 +1359,6 @@ public class File { t.Fatal(err) } - // Write mock template to disk - tmplDir := filepath.Join(tmpDir, "template") - if err := os.MkdirAll(tmplDir, 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(tmplDir, "README.md.go.tmpl"), []byte(`# {{ .Metadata.Repo.NamePretty }}`), 0644); err != nil { - t.Fatal(err) - } - p := libraryPostProcessParams{ outDir: tmpDir, cfg: &config.Config{ @@ -1464,20 +1442,13 @@ public class File { if err != nil { t.Fatal(err) } - if string(readmeContent) != "# My API" { - t.Errorf("README content mismatch. Got: %s, expected: # My API", string(readmeContent)) + if !strings.Contains(string(readmeContent), "# Google My API Client for Java") { + t.Errorf("README content mismatch. Got: %s", string(readmeContent)) } } func TestRunGoPostprocessor_ReleasedVersion(t *testing.T) { tmpDir := t.TempDir() - tmplDir := filepath.Join(tmpDir, "template") - if err := os.MkdirAll(tmplDir, 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(tmplDir, "README.md.go.tmpl"), []byte(`Version: {{ .Version }}`), 0644); err != nil { - t.Fatal(err) - } p := libraryPostProcessParams{ outDir: tmpDir, @@ -1508,7 +1479,7 @@ func TestRunGoPostprocessor_ReleasedVersion(t *testing.T) { if err != nil { t.Fatal(err) } - if string(readmeContent) != "Version: 3.43.1" { - t.Errorf("README content mismatch. Got: %s, expected: Version: 3.43.1", string(readmeContent)) + if !strings.Contains(string(readmeContent), "3.43.1") { + t.Errorf("README content mismatch. Got: %s", string(readmeContent)) } } diff --git a/internal/librarian/java/template_render.go b/internal/librarian/java/template_render.go index 93f21b4d0ea..9bb22844cc9 100644 --- a/internal/librarian/java/template_render.go +++ b/internal/librarian/java/template_render.go @@ -15,7 +15,7 @@ package java import ( - "embed" + _ "embed" "errors" "fmt" "io/fs" @@ -28,12 +28,19 @@ import ( "github.com/iancoleman/strcase" ) -//go:embed template/README.md.go.tmpl -var defaultTemplateFs embed.FS +var ( + //go:embed template/README.md.go.tmpl + readmeTmpl string + readmeTmplParsed = template.Must(template.New("README").Parse(readmeTmpl)) +) // RenderREADME renders the README.md file using the template and metadata. // dir is the directory containing where README.md will be written. func RenderREADME(dir string, metadata *repoMetadata, bomVersion, libraryVersion string, keepSet map[string]bool) error { + return renderREADMEWithTemplate(dir, metadata, bomVersion, libraryVersion, keepSet, readmeTmplParsed) +} + +func renderREADMEWithTemplate(dir string, metadata *repoMetadata, bomVersion, libraryVersion string, keepSet map[string]bool, tmpl *template.Template) error { outputPath := filepath.Join(dir, "README.md") if keepSet["README.md"] { return nil @@ -135,24 +142,6 @@ func RenderREADME(dir string, metadata *repoMetadata, bomVersion, libraryVersion LibraryVersion: libraryVersion, } - // Read and parse template from disk - templatePath := filepath.Join(dir, "template", "README.md.go.tmpl") - tmplBytes, err := os.ReadFile(templatePath) - if err != nil { - if errors.Is(err, fs.ErrNotExist) { - // Fallback to embedded default template - tmplBytes, err = defaultTemplateFs.ReadFile("template/README.md.go.tmpl") - } - if err != nil { - return fmt.Errorf("failed to read template: %w", err) - } - } - - tmpl, err := template.New("README").Parse(string(tmplBytes)) - if err != nil { - return fmt.Errorf("failed to parse template: %w", err) - } - // Execute template var buf strings.Builder if err := tmpl.Execute(&buf, data); err != nil { diff --git a/internal/librarian/java/template_render_test.go b/internal/librarian/java/template_render_test.go index a784d031f28..bf23bd607e7 100644 --- a/internal/librarian/java/template_render_test.go +++ b/internal/librarian/java/template_render_test.go @@ -36,14 +36,7 @@ LibraryVersion: {{ .LibraryVersion }} About: {{ .Metadata.Partials.About }} {{ end }} ` - // Write mock template to disk - tmplDir := filepath.Join(tmpDir, "template") - if err := os.MkdirAll(tmplDir, 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(tmplDir, "README.md.go.tmpl"), []byte(templateContent), 0644); err != nil { - t.Fatal(err) - } + mockTmpl := template.Must(template.New("mock").Parse(templateContent)) metadata := &repoMetadata{ NamePretty: "My API", @@ -52,7 +45,7 @@ About: {{ .Metadata.Partials.About }} } // Test case 1: Without partials - err := RenderREADME(tmpDir, metadata, "1.0.0-BOM", "1.2.3-LIB", nil) + err := renderREADMEWithTemplate(tmpDir, metadata, "1.0.0-BOM", "1.2.3-LIB", nil, mockTmpl) if err != nil { t.Fatal(err) } @@ -81,7 +74,7 @@ LibraryVersion: 1.2.3-LIB t.Fatal(err) } - err = RenderREADME(tmpDir, metadata, "1.0.0-BOM", "1.2.3-LIB", nil) + err = renderREADMEWithTemplate(tmpDir, metadata, "1.0.0-BOM", "1.2.3-LIB", nil, mockTmpl) if err != nil { t.Fatal(err) } @@ -111,7 +104,7 @@ About: This is a great API. t.Fatal(err) } - err = RenderREADME(tmpDir, metadata, "1.0.0-BOM", "1.2.3-LIB", keepSet) + err = renderREADMEWithTemplate(tmpDir, metadata, "1.0.0-BOM", "1.2.3-LIB", keepSet, mockTmpl) if err != nil { t.Fatal(err) } @@ -126,12 +119,7 @@ About: This is a great API. } func TestRealTemplateParses(t *testing.T) { - tmplBytes, err := os.ReadFile("template/README.md.go.tmpl") - if err != nil { - t.Fatalf("failed to read real template: %v", err) - } - _, err = template.New("README").Parse(string(tmplBytes)) - if err != nil { - t.Fatalf("failed to parse real template: %v", err) + if readmeTmplParsed == nil { + t.Fatal("readmeTmplParsed is nil") } } From 11774e4e512c2c9fd3f4388dbb4de114b3ff4416 Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Sun, 28 Jun 2026 21:44:50 -0400 Subject: [PATCH 070/108] refactor(internal/librarian/java): consolidate README rendering and extraction into readme.go --- internal/librarian/java/readme.go | 127 +++++++++++++++ internal/librarian/java/readme_test.go | 102 ++++++++++++ internal/librarian/java/template_render.go | 153 ------------------ .../librarian/java/template_render_test.go | 125 -------------- 4 files changed, 229 insertions(+), 278 deletions(-) delete mode 100644 internal/librarian/java/template_render.go delete mode 100644 internal/librarian/java/template_render_test.go diff --git a/internal/librarian/java/readme.go b/internal/librarian/java/readme.go index 51962c7cca7..f58bafb467a 100644 --- a/internal/librarian/java/readme.go +++ b/internal/librarian/java/readme.go @@ -16,6 +16,7 @@ package java import ( "bufio" + _ "embed" "errors" "fmt" "io/fs" @@ -24,9 +25,17 @@ import ( "regexp" "sort" "strings" + "text/template" + + "github.com/googleapis/librarian/internal/yaml" + "github.com/iancoleman/strcase" ) var ( + //go:embed template/README.md.go.tmpl + readmeTmpl string + readmeTmplParsed = template.Must(template.New("README").Parse(readmeTmpl)) + openSnippetRegex = regexp.MustCompile(`\[START ([a-zA-Z0-9_-]+)\]`) closeSnippetRegex = regexp.MustCompile(`\[END ([a-zA-Z0-9_-]+)\]`) openExcludeRegex = regexp.MustCompile(`\[START_EXCLUDE\]`) @@ -274,3 +283,121 @@ func extractSnippetsFromFile(file string, snippetLines map[string][]string) erro } return nil } + +// RenderREADME renders the README.md file using the template and metadata. +// dir is the directory containing where README.md will be written. +func RenderREADME(dir string, metadata *repoMetadata, bomVersion, libraryVersion string, keepSet map[string]bool) error { + return renderREADMEWithTemplate(dir, metadata, bomVersion, libraryVersion, keepSet, readmeTmplParsed) +} + +func renderREADMEWithTemplate(dir string, metadata *repoMetadata, bomVersion, libraryVersion string, keepSet map[string]bool, tmpl *template.Template) error { + outputPath := filepath.Join(dir, "README.md") + if keepSet["README.md"] { + return nil + } + + partialsPath := filepath.Join(dir, ".readme-partials.yaml") + if _, err := os.Stat(partialsPath); errors.Is(err, fs.ErrNotExist) { + partialsPath = filepath.Join(dir, ".readme-partials.yml") + } + + // Read partials if exist + var partials map[string]interface{} + if _, err := os.Stat(partialsPath); err == nil { + partialsBytes, err := os.ReadFile(partialsPath) + if err != nil { + return fmt.Errorf("failed to read partials: %w", err) + } + p, err := yaml.Unmarshal[map[string]interface{}](partialsBytes) + if err != nil { + return fmt.Errorf("failed to unmarshal partials: %w", err) + } + partials = *p + } + + // Capitalize keys of partials for template + capitalizedPartials := make(map[string]interface{}) + for k, v := range partials { + capitalizedPartials[strcase.ToCamel(k)] = v + } + + // Prepare data for template + distName := metadata.DistributionName + distParts := strings.Split(distName, ":") + groupId := "" + artifactId := "" + if len(distParts) > 0 { + groupId = distParts[0] + } + if len(distParts) > 1 { + artifactId = distParts[1] + } + + repoName := metadata.Repo + repoParts := strings.Split(repoName, "/") + repoShort := "" + if len(repoParts) > 0 { + repoShort = repoParts[len(repoParts)-1] + } + + version := libraryVersion + + minJavaVersion := metadata.MinJavaVersion + if minJavaVersion == 0 { + minJavaVersion = 8 // Default to Java 8 + } + + samples, err := ExtractSamples(dir) + if err != nil { + return fmt.Errorf("failed to extract samples: %w", err) + } + + snippets, err := ExtractSnippets(dir) + if err != nil { + return fmt.Errorf("failed to extract snippets: %w", err) + } + + templateMetadata := map[string]interface{}{ + "Repo": metadata, + "LibraryVersion": version, + "LibrariesBOMVersion": bomVersion, + "Samples": samples, + "Snippets": snippets, + "MinJavaVersion": minJavaVersion, + } + + if len(capitalizedPartials) > 0 { + templateMetadata["Partials"] = capitalizedPartials + } + + data := struct { + Metadata map[string]interface{} + GroupID string + ArtifactID string + Version string + RepoShort string + MigratedSplitRepo bool + Monorepo bool + BOMVersion string + LibraryVersion string + }{ + Metadata: templateMetadata, + GroupID: groupId, + ArtifactID: artifactId, + Version: version, + RepoShort: repoShort, + MigratedSplitRepo: false, + Monorepo: true, + BOMVersion: bomVersion, + LibraryVersion: libraryVersion, + } + + // Execute template + var buf strings.Builder + if err := tmpl.Execute(&buf, data); err != nil { + return fmt.Errorf("failed to execute template: %w", err) + } + + // Write output + return os.WriteFile(outputPath, []byte(buf.String()), 0644) +} diff --git a/internal/librarian/java/readme_test.go b/internal/librarian/java/readme_test.go index a8472bb6b48..2bdef007d0b 100644 --- a/internal/librarian/java/readme_test.go +++ b/internal/librarian/java/readme_test.go @@ -18,7 +18,9 @@ import ( "errors" "os" "path/filepath" + "strings" "testing" + "text/template" "github.com/google/go-cmp/cmp" ) @@ -349,3 +351,103 @@ func TestExtractSnippets(t *testing.T) { t.Errorf("quickstart = %q; expected %q", quickSnippet, expectedQuick) } } + +func TestRenderREADME(t *testing.T) { + tmpDir := t.TempDir() + + templateContent := `# Google {{ .Metadata.Repo.NamePretty }} Client for Java +Artifact: {{ .GroupID }}:{{ .ArtifactID }} +Version: {{ .Version }} +BOMVersion: {{ .BOMVersion }} +LibraryVersion: {{ .LibraryVersion }} +{{ if and .Metadata.Partials .Metadata.Partials.About }} +About: {{ .Metadata.Partials.About }} +{{ end }} +` + mockTmpl := template.Must(template.New("mock").Parse(templateContent)) + + metadata := &repoMetadata{ + NamePretty: "My API", + DistributionName: "com.google.cloud:google-cloud-myapi", + Repo: "googleapis/google-cloud-java", + } + + // Test case 1: Without partials + err := renderREADMEWithTemplate(tmpDir, metadata, "1.0.0-BOM", "1.2.3-LIB", nil, mockTmpl) + if err != nil { + t.Fatal(err) + } + + outputPath := filepath.Join(tmpDir, "README.md") + outputContent, err := os.ReadFile(outputPath) + if err != nil { + t.Fatal(err) + } + + expected := `# Google My API Client for Java +Artifact: com.google.cloud:google-cloud-myapi +Version: 1.2.3-LIB +BOMVersion: 1.0.0-BOM +LibraryVersion: 1.2.3-LIB +` + if diff := cmp.Diff(strings.TrimSpace(expected), strings.TrimSpace(string(outputContent))); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + + // Test case 2: With partials + partialsPath := filepath.Join(tmpDir, ".readme-partials.yaml") + partialsContent := `about: "This is a great API."` + err = os.WriteFile(partialsPath, []byte(partialsContent), 0644) + if err != nil { + t.Fatal(err) + } + + err = renderREADMEWithTemplate(tmpDir, metadata, "1.0.0-BOM", "1.2.3-LIB", nil, mockTmpl) + if err != nil { + t.Fatal(err) + } + + outputContent, err = os.ReadFile(outputPath) + if err != nil { + t.Fatal(err) + } + + expectedWithPartials := `# Google My API Client for Java +Artifact: com.google.cloud:google-cloud-myapi +Version: 1.2.3-LIB +BOMVersion: 1.0.0-BOM +LibraryVersion: 1.2.3-LIB + +About: This is a great API. +` + if diff := cmp.Diff(strings.TrimSpace(expectedWithPartials), strings.TrimSpace(string(outputContent))); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + + // Test case 3: With README.md in keep list + keepSet := map[string]bool{"README.md": true} + customContent := "Custom README content" + err = os.WriteFile(outputPath, []byte(customContent), 0644) + if err != nil { + t.Fatal(err) + } + + err = renderREADMEWithTemplate(tmpDir, metadata, "1.0.0-BOM", "1.2.3-LIB", keepSet, mockTmpl) + if err != nil { + t.Fatal(err) + } + + outputContent, err = os.ReadFile(outputPath) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(customContent, string(outputContent)); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } +} + +func TestRealTemplateParses(t *testing.T) { + if readmeTmplParsed == nil { + t.Fatal("readmeTmplParsed is nil") + } +} diff --git a/internal/librarian/java/template_render.go b/internal/librarian/java/template_render.go deleted file mode 100644 index 9bb22844cc9..00000000000 --- a/internal/librarian/java/template_render.go +++ /dev/null @@ -1,153 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package java - -import ( - _ "embed" - "errors" - "fmt" - "io/fs" - "os" - "path/filepath" - "strings" - "text/template" - - "github.com/googleapis/librarian/internal/yaml" - "github.com/iancoleman/strcase" -) - -var ( - //go:embed template/README.md.go.tmpl - readmeTmpl string - readmeTmplParsed = template.Must(template.New("README").Parse(readmeTmpl)) -) - -// RenderREADME renders the README.md file using the template and metadata. -// dir is the directory containing where README.md will be written. -func RenderREADME(dir string, metadata *repoMetadata, bomVersion, libraryVersion string, keepSet map[string]bool) error { - return renderREADMEWithTemplate(dir, metadata, bomVersion, libraryVersion, keepSet, readmeTmplParsed) -} - -func renderREADMEWithTemplate(dir string, metadata *repoMetadata, bomVersion, libraryVersion string, keepSet map[string]bool, tmpl *template.Template) error { - outputPath := filepath.Join(dir, "README.md") - if keepSet["README.md"] { - return nil - } - - partialsPath := filepath.Join(dir, ".readme-partials.yaml") - if _, err := os.Stat(partialsPath); errors.Is(err, fs.ErrNotExist) { - partialsPath = filepath.Join(dir, ".readme-partials.yml") - } - - // Read partials if exist - var partials map[string]interface{} - if _, err := os.Stat(partialsPath); err == nil { - partialsBytes, err := os.ReadFile(partialsPath) - if err != nil { - return fmt.Errorf("failed to read partials: %w", err) - } - p, err := yaml.Unmarshal[map[string]interface{}](partialsBytes) - if err != nil { - return fmt.Errorf("failed to unmarshal partials: %w", err) - } - partials = *p - } - - // Capitalize keys of partials for template - capitalizedPartials := make(map[string]interface{}) - for k, v := range partials { - capitalizedPartials[strcase.ToCamel(k)] = v - } - - // Prepare data for template - distName := metadata.DistributionName - distParts := strings.Split(distName, ":") - groupId := "" - artifactId := "" - if len(distParts) > 0 { - groupId = distParts[0] - } - if len(distParts) > 1 { - artifactId = distParts[1] - } - - repoName := metadata.Repo - repoParts := strings.Split(repoName, "/") - repoShort := "" - if len(repoParts) > 0 { - repoShort = repoParts[len(repoParts)-1] - } - - version := libraryVersion - - minJavaVersion := metadata.MinJavaVersion - if minJavaVersion == 0 { - minJavaVersion = 8 // Default to Java 8 - } - - samples, err := ExtractSamples(dir) - if err != nil { - return fmt.Errorf("failed to extract samples: %w", err) - } - - snippets, err := ExtractSnippets(dir) - if err != nil { - return fmt.Errorf("failed to extract snippets: %w", err) - } - - templateMetadata := map[string]interface{}{ - "Repo": metadata, - "LibraryVersion": version, - "LibrariesBOMVersion": bomVersion, - "Samples": samples, - "Snippets": snippets, - "MinJavaVersion": minJavaVersion, - } - - if len(capitalizedPartials) > 0 { - templateMetadata["Partials"] = capitalizedPartials - } - - data := struct { - Metadata map[string]interface{} - GroupID string - ArtifactID string - Version string - RepoShort string - MigratedSplitRepo bool - Monorepo bool - BOMVersion string - LibraryVersion string - }{ - Metadata: templateMetadata, - GroupID: groupId, - ArtifactID: artifactId, - Version: version, - RepoShort: repoShort, - MigratedSplitRepo: false, - Monorepo: true, - BOMVersion: bomVersion, - LibraryVersion: libraryVersion, - } - - // Execute template - var buf strings.Builder - if err := tmpl.Execute(&buf, data); err != nil { - return fmt.Errorf("failed to execute template: %w", err) - } - - // Write output - return os.WriteFile(outputPath, []byte(buf.String()), 0644) -} diff --git a/internal/librarian/java/template_render_test.go b/internal/librarian/java/template_render_test.go deleted file mode 100644 index bf23bd607e7..00000000000 --- a/internal/librarian/java/template_render_test.go +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package java - -import ( - "os" - "path/filepath" - "strings" - "testing" - "text/template" - - "github.com/google/go-cmp/cmp" -) - -func TestRenderREADME(t *testing.T) { - tmpDir := t.TempDir() - - templateContent := `# Google {{ .Metadata.Repo.NamePretty }} Client for Java -Artifact: {{ .GroupID }}:{{ .ArtifactID }} -Version: {{ .Version }} -BOMVersion: {{ .BOMVersion }} -LibraryVersion: {{ .LibraryVersion }} -{{ if and .Metadata.Partials .Metadata.Partials.About }} -About: {{ .Metadata.Partials.About }} -{{ end }} -` - mockTmpl := template.Must(template.New("mock").Parse(templateContent)) - - metadata := &repoMetadata{ - NamePretty: "My API", - DistributionName: "com.google.cloud:google-cloud-myapi", - Repo: "googleapis/google-cloud-java", - } - - // Test case 1: Without partials - err := renderREADMEWithTemplate(tmpDir, metadata, "1.0.0-BOM", "1.2.3-LIB", nil, mockTmpl) - if err != nil { - t.Fatal(err) - } - - outputPath := filepath.Join(tmpDir, "README.md") - outputContent, err := os.ReadFile(outputPath) - if err != nil { - t.Fatal(err) - } - - expected := `# Google My API Client for Java -Artifact: com.google.cloud:google-cloud-myapi -Version: 1.2.3-LIB -BOMVersion: 1.0.0-BOM -LibraryVersion: 1.2.3-LIB -` - if diff := cmp.Diff(strings.TrimSpace(expected), strings.TrimSpace(string(outputContent))); diff != "" { - t.Errorf("mismatch (-want +got):\n%s", diff) - } - - // Test case 2: With partials - partialsPath := filepath.Join(tmpDir, ".readme-partials.yaml") - partialsContent := `about: "This is a great API."` - err = os.WriteFile(partialsPath, []byte(partialsContent), 0644) - if err != nil { - t.Fatal(err) - } - - err = renderREADMEWithTemplate(tmpDir, metadata, "1.0.0-BOM", "1.2.3-LIB", nil, mockTmpl) - if err != nil { - t.Fatal(err) - } - - outputContent, err = os.ReadFile(outputPath) - if err != nil { - t.Fatal(err) - } - - expectedWithPartials := `# Google My API Client for Java -Artifact: com.google.cloud:google-cloud-myapi -Version: 1.2.3-LIB -BOMVersion: 1.0.0-BOM -LibraryVersion: 1.2.3-LIB - -About: This is a great API. -` - if diff := cmp.Diff(strings.TrimSpace(expectedWithPartials), strings.TrimSpace(string(outputContent))); diff != "" { - t.Errorf("mismatch (-want +got):\n%s", diff) - } - - // Test case 3: With README.md in keep list - keepSet := map[string]bool{"README.md": true} - customContent := "Custom README content" - err = os.WriteFile(outputPath, []byte(customContent), 0644) - if err != nil { - t.Fatal(err) - } - - err = renderREADMEWithTemplate(tmpDir, metadata, "1.0.0-BOM", "1.2.3-LIB", keepSet, mockTmpl) - if err != nil { - t.Fatal(err) - } - - outputContent, err = os.ReadFile(outputPath) - if err != nil { - t.Fatal(err) - } - if diff := cmp.Diff(customContent, string(outputContent)); diff != "" { - t.Errorf("mismatch (-want +got):\n%s", diff) - } -} - -func TestRealTemplateParses(t *testing.T) { - if readmeTmplParsed == nil { - t.Fatal("readmeTmplParsed is nil") - } -} From b483eed1f80586ebbb4f0810da34f0515464839d Mon Sep 17 00:00:00 2001 From: Joe Wang <106995533+JoeWang1127@users.noreply.github.com> Date: Mon, 29 Jun 2026 13:36:03 +0000 Subject: [PATCH 071/108] chore(tool/cmd): remove configcheck tool (#6561) Remove configcheck tool since Go and Python are already migrated from legacylibrarian. --- tool/cmd/configcheck/main.go | 155 -------- tool/cmd/configcheck/main_test.go | 360 ------------------ .../config-check/.librarian/state.yaml | 71 ---- .../testdata/config-check/librarian.yaml | 39 -- .../no-librarian/.librarian/config.yaml | 14 - .../no-librarian/.librarian/state.yaml | 90 ----- 6 files changed, 729 deletions(-) delete mode 100644 tool/cmd/configcheck/main.go delete mode 100644 tool/cmd/configcheck/main_test.go delete mode 100644 tool/cmd/configcheck/testdata/config-check/.librarian/state.yaml delete mode 100644 tool/cmd/configcheck/testdata/config-check/librarian.yaml delete mode 100644 tool/cmd/configcheck/testdata/no-librarian/.librarian/config.yaml delete mode 100644 tool/cmd/configcheck/testdata/no-librarian/.librarian/state.yaml diff --git a/tool/cmd/configcheck/main.go b/tool/cmd/configcheck/main.go deleted file mode 100644 index 29c85369376..00000000000 --- a/tool/cmd/configcheck/main.go +++ /dev/null @@ -1,155 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Command configcheck is a tool to verify the consistency of library versions -// between librarian.yaml and .librarian/state.yaml. -package main - -import ( - "errors" - "flag" - "fmt" - "io/fs" - "log" - "os" - "path/filepath" - "slices" - - "github.com/googleapis/librarian/internal/config" - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" - "github.com/googleapis/librarian/internal/yaml" -) - -var ( - errRepoNotFound = errors.New("repo argument is required") - errLibNotFoundInLibrarianYAML = errors.New("library not found in librarian.yaml") - errLibNotFoundInStateYAML = errors.New("library not found in state.yaml") - errLibraryVersionNotSame = errors.New("library version not same") - errLibraryAPINotSame = errors.New("library API not same") - errLibraryReleaseBlockedNotSame = errors.New("library release blocked not same") -) - -type library struct { - version string - apis []string -} - -func main() { - if err := run(os.Args[1:]); err != nil { - log.Fatal(err) - } -} - -func run(args []string) error { - flagSet := flag.NewFlagSet("configcheck", flag.ContinueOnError) - if err := flagSet.Parse(args); err != nil { - return err - } - if flagSet.NArg() < 1 { - return errRepoNotFound - } - - repoPath := flagSet.Arg(0) - abs, err := filepath.Abs(repoPath) - if err != nil { - return err - } - stateFile := filepath.Join(abs, legacyconfig.LibrarianDir, legacyconfig.LibrarianStateFile) - state, err := yaml.Read[legacyconfig.LibrarianState](stateFile) - if err != nil { - return err - } - cfg, err := yaml.Read[config.Config](filepath.Join(abs, config.LibrarianYAML)) - if err != nil { - return err - } - var lcfg *legacyconfig.LibrarianConfig - configFile := filepath.Join(abs, legacyconfig.LibrarianDir, legacyconfig.LibrarianConfigFile) - lcfg, err = yaml.Read[legacyconfig.LibrarianConfig](configFile) - if err != nil { - if !errors.Is(err, fs.ErrNotExist) { - return err - } - lcfg = nil - } - return configCheck(state, cfg, lcfg) -} - -// configCheck verifies that the libraries and their versions defined in the -// state.yaml match those in librarian.yaml. -func configCheck(state *legacyconfig.LibrarianState, cfg *config.Config, lcfg *legacyconfig.LibrarianConfig) error { - legacyLibs := convertLegacyLibs(state.Libraries) - libs := convertLibs(cfg.Libraries) - for id, legacyLib := range legacyLibs { - lib, ok := libs[id] - if !ok { - return fmt.Errorf("library %s: %w", id, errLibNotFoundInLibrarianYAML) - } - if lib.version != legacyLib.version { - return fmt.Errorf("library %s: %w", id, errLibraryVersionNotSame) - } - if !slices.Equal(lib.apis, legacyLib.apis) { - return fmt.Errorf("library %s: %w", id, errLibraryAPINotSame) - } - } - for name := range libs { - if _, ok := legacyLibs[name]; !ok { - return fmt.Errorf("library %s: %w", name, errLibNotFoundInStateYAML) - } - } - releaseBlocked := make(map[string]bool) - if lcfg != nil { - for _, l := range lcfg.Libraries { - releaseBlocked[l.LibraryID] = l.ReleaseBlocked - } - } - for _, lib := range cfg.Libraries { - if lib.SkipRelease != releaseBlocked[lib.Name] { - return fmt.Errorf("%w: library %s: skip_release=%v != release_blocked=%v", errLibraryReleaseBlockedNotSame, lib.Name, lib.SkipRelease, releaseBlocked[lib.Name]) - } - } - return nil -} - -func convertLegacyLibs(libs []*legacyconfig.LibraryState) map[string]*library { - res := make(map[string]*library) - for _, lib := range libs { - apis := make([]string, 0, len(lib.APIs)) - for _, api := range lib.APIs { - apis = append(apis, api.Path) - } - slices.Sort(apis) - res[lib.ID] = &library{ - version: lib.Version, - apis: apis, - } - } - return res -} - -func convertLibs(libs []*config.Library) map[string]*library { - res := make(map[string]*library) - for _, lib := range libs { - apis := make([]string, 0, len(lib.APIs)) - for _, api := range lib.APIs { - apis = append(apis, api.Path) - } - slices.Sort(apis) - res[lib.Name] = &library{ - version: lib.Version, - apis: apis, - } - } - return res -} diff --git a/tool/cmd/configcheck/main_test.go b/tool/cmd/configcheck/main_test.go deleted file mode 100644 index b2d27e9d06e..00000000000 --- a/tool/cmd/configcheck/main_test.go +++ /dev/null @@ -1,360 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "errors" - "io/fs" - "testing" - - "github.com/googleapis/librarian/internal/config" - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" -) - -func TestRun(t *testing.T) { - for _, test := range []struct { - name string - repoPath string - }{ - { - name: "success", - repoPath: "testdata/config-check", - }, - } { - t.Run(test.name, func(t *testing.T) { - if err := run([]string{test.repoPath}); err != nil { - t.Fatal(err) - } - }) - } -} - -func TestRun_Error(t *testing.T) { - for _, test := range []struct { - name string - repoPath string - wantErr error - }{ - { - name: "no state.yaml", - repoPath: "testdata/no-state", - wantErr: fs.ErrNotExist, - }, - { - name: "no librarian.yaml", - repoPath: "testdata/no-librarian", - wantErr: fs.ErrNotExist, - }, - } { - t.Run(test.name, func(t *testing.T) { - err := run([]string{test.repoPath}) - if !errors.Is(err, test.wantErr) { - t.Errorf("run(%q) error = %v, want %v", test.repoPath, err, test.wantErr) - } - }) - } -} - -func TestConfigCheck(t *testing.T) { - for _, test := range []struct { - name string - state *legacyconfig.LibrarianState - cfg *config.Config - lcfg *legacyconfig.LibrarianConfig - }{ - { - name: "library match in state.yaml and librarian.yaml", - state: &legacyconfig.LibrarianState{ - Image: "test-image", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "lib-1", - Version: "1.0.0", - APIs: []*legacyconfig.API{ - {Path: "google/lib1"}, - {Path: "google/lib1/sub1"}, - }, - SourceRoots: []string{"existing"}, - TagFormat: "{id}/v{version}", - }, - { - ID: "lib-2", - Version: "1.2.0", - APIs: []*legacyconfig.API{ - {Path: "google/lib2"}, - {Path: "google/lib2/sub1"}, - }, - SourceRoots: []string{"another"}, - TagFormat: "{id}/v{version}", - }, - }, - }, - cfg: &config.Config{ - Libraries: []*config.Library{ - { - Name: "lib-1", - Version: "1.0.0", - APIs: []*config.API{ - {Path: "google/lib1"}, - {Path: "google/lib1/sub1"}, - }, - }, - { - Name: "lib-2", - Version: "1.2.0", - APIs: []*config.API{ - {Path: "google/lib2"}, - {Path: "google/lib2/sub1"}, - }, - }, - }, - }, - }, - { - name: "skip_release and release_blocked are both true", - state: &legacyconfig.LibrarianState{ - Image: "test-image", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "lib-1", - Version: "1.0.0", - SourceRoots: []string{"existing"}, - TagFormat: "{id}/v{version}", - }, - }, - }, - cfg: &config.Config{ - Libraries: []*config.Library{ - { - Name: "lib-1", - Version: "1.0.0", - SkipRelease: true, - }, - }, - }, - lcfg: &legacyconfig.LibrarianConfig{ - Libraries: []*legacyconfig.LibraryConfig{ - { - LibraryID: "lib-1", - ReleaseBlocked: true, - }, - }, - }, - }, - { - name: "skip_release and release_blocked are both false (lcfg exists but empty)", - state: &legacyconfig.LibrarianState{ - Image: "test-image", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "lib-1", - Version: "1.0.0", - SourceRoots: []string{"existing"}, - TagFormat: "{id}/v{version}", - }, - }, - }, - cfg: &config.Config{ - Libraries: []*config.Library{ - { - Name: "lib-1", - Version: "1.0.0", - SkipRelease: false, - }, - }, - }, - lcfg: &legacyconfig.LibrarianConfig{ - Libraries: []*legacyconfig.LibraryConfig{}, - }, - }, - } { - t.Run(test.name, func(t *testing.T) { - if err := configCheck(test.state, test.cfg, test.lcfg); err != nil { - t.Fatal(err) - } - }) - } -} - -func TestConfigCheck_Error(t *testing.T) { - for _, test := range []struct { - name string - state *legacyconfig.LibrarianState - cfg *config.Config - lcfg *legacyconfig.LibrarianConfig - wantErr error - }{ - { - name: "a library exists in state.yaml but not librarian.yaml", - state: &legacyconfig.LibrarianState{ - Image: "test-image", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "lib-1", - Version: "1.0.0", - SourceRoots: []string{"existing"}, - TagFormat: "{id}/v{version}", - }, - { - ID: "lib-2", - Version: "1.2.0", - SourceRoots: []string{"another"}, - TagFormat: "{id}/v{version}", - }, - }, - }, - cfg: &config.Config{ - Libraries: []*config.Library{ - {Name: "lib-1", Version: "1.0.0"}, - }, - }, - wantErr: errLibNotFoundInLibrarianYAML, - }, - { - name: "a library exists in librarian.yaml but not state.yaml", - state: &legacyconfig.LibrarianState{ - Image: "test-image", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "lib-1", - Version: "1.0.0", - SourceRoots: []string{"existing"}, - TagFormat: "{id}/v{version}", - }, - }, - }, - cfg: &config.Config{ - Libraries: []*config.Library{ - {Name: "lib-1", Version: "1.0.0"}, - {Name: "lib-2", Version: "1.2.0"}, - }, - }, - wantErr: errLibNotFoundInStateYAML, - }, - { - name: "a library exists in two configs but version is different", - state: &legacyconfig.LibrarianState{ - Image: "test-image", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "lib-1", - Version: "1.0.0", - SourceRoots: []string{"existing"}, - TagFormat: "{id}/v{version}", - }, - { - ID: "lib-2", - Version: "1.1.0", - SourceRoots: []string{"another"}, - TagFormat: "{id}/v{version}", - }, - }, - }, - cfg: &config.Config{ - Libraries: []*config.Library{ - {Name: "lib-1", Version: "1.0.0"}, - {Name: "lib-2", Version: "1.2.0"}, - }, - }, - wantErr: errLibraryVersionNotSame, - }, - { - name: "a library exists in two configs but api is different", - state: &legacyconfig.LibrarianState{ - Image: "test-image", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "lib-1", - Version: "1.0.0", - SourceRoots: []string{"existing"}, - TagFormat: "{id}/v{version}", - }, - { - ID: "lib-2", - Version: "1.2.0", - APIs: []*legacyconfig.API{ - {Path: "google/lib2"}, - {Path: "google/lib2/sub1"}, - }, - SourceRoots: []string{"another"}, - TagFormat: "{id}/v{version}", - }, - }, - }, - cfg: &config.Config{ - Libraries: []*config.Library{ - {Name: "lib-1", Version: "1.0.0"}, - {Name: "lib-2", Version: "1.2.0", APIs: []*config.API{{Path: "google/lib2"}}}, - }, - }, - wantErr: errLibraryAPINotSame, - }, - { - name: "skip_release is true but release_blocked is false (missing in lcfg)", - state: &legacyconfig.LibrarianState{ - Image: "test-image", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "lib-1", - Version: "1.0.0", - SourceRoots: []string{"existing"}, - TagFormat: "{id}/v{version}", - }, - }, - }, - cfg: &config.Config{ - Libraries: []*config.Library{ - {Name: "lib-1", Version: "1.0.0", SkipRelease: true}, - }, - }, - wantErr: errLibraryReleaseBlockedNotSame, - }, - { - name: "skip_release is false but release_blocked is true", - state: &legacyconfig.LibrarianState{ - Image: "test-image", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "lib-1", - Version: "1.0.0", - SourceRoots: []string{"existing"}, - TagFormat: "{id}/v{version}", - }, - }, - }, - cfg: &config.Config{ - Libraries: []*config.Library{ - {Name: "lib-1", Version: "1.0.0", SkipRelease: false}, - }, - }, - lcfg: &legacyconfig.LibrarianConfig{ - Libraries: []*legacyconfig.LibraryConfig{ - { - LibraryID: "lib-1", - ReleaseBlocked: true, - }, - }, - }, - wantErr: errLibraryReleaseBlockedNotSame, - }, - } { - t.Run(test.name, func(t *testing.T) { - err := configCheck(test.state, test.cfg, test.lcfg) - if !errors.Is(err, test.wantErr) { - t.Errorf("configCheck() error = %v, want %v", err, test.wantErr) - } - }) - } -} diff --git a/tool/cmd/configcheck/testdata/config-check/.librarian/state.yaml b/tool/cmd/configcheck/testdata/config-check/.librarian/state.yaml deleted file mode 100644 index 93307c6a159..00000000000 --- a/tool/cmd/configcheck/testdata/config-check/.librarian/state.yaml +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -image: test-image -libraries: - - id: accessapproval - version: 1.8.8 - last_generated_commit: "" - apis: - - path: google/cloud/accessapproval/v1 - service_config: accessapproval_v1.yaml - source_roots: - - accessapproval - - internal/generated/snippets/accessapproval - preserve_regex: [] - remove_regex: [] - release_exclude_paths: - - internal/generated/snippets/accessapproval/ - tag_format: '{id}/v{version}' - - id: accesscontextmanager - version: 1.9.7 - last_generated_commit: "" - apis: - - path: google/identity/accesscontextmanager/v1 - service_config: accesscontextmanager_v1.yaml - source_roots: - - accesscontextmanager - - internal/generated/snippets/accesscontextmanager - preserve_regex: [] - remove_regex: [] - release_exclude_paths: - - internal/generated/snippets/accesscontextmanager/ - tag_format: '{id}/v{version}' - - id: advisorynotifications - version: 1.5.6 - last_generated_commit: "" - apis: - - path: google/cloud/advisorynotifications/v1 - service_config: advisorynotifications_v1.yaml - source_roots: - - advisorynotifications - - internal/generated/snippets/advisorynotifications - preserve_regex: [] - remove_regex: [] - release_exclude_paths: - - internal/generated/snippets/advisorynotifications/ - tag_format: '{id}/v{version}' - - id: apigeeconnect - version: 1.7.7 - last_generated_commit: "" - apis: - - path: google/cloud/apigeeconnect/v1 - service_config: apigeeconnect_v1.yaml - source_roots: - - apigeeconnect - - internal/generated/snippets/apigeeconnect - preserve_regex: [] - remove_regex: [] - release_exclude_paths: - - internal/generated/snippets/apigeeconnect/ - tag_format: '{id}/v{version}' diff --git a/tool/cmd/configcheck/testdata/config-check/librarian.yaml b/tool/cmd/configcheck/testdata/config-check/librarian.yaml deleted file mode 100644 index ccbcfd6c389..00000000000 --- a/tool/cmd/configcheck/testdata/config-check/librarian.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -language: "" -sources: - googleapis: - commit: 6c3dce4a02401667e9dd6d28304fee3f98e20ff8 - sha256: c119da87c44afc55cd7afc0829c7e321a0fb03ff5c8bdbe0c00d27e7f30a8c63 -libraries: - - name: accessapproval - version: 1.8.8 - apis: - - path: google/cloud/accessapproval/v1 - - name: accesscontextmanager - version: 1.9.7 - apis: - - path: google/identity/accesscontextmanager/v1 - go: - go_apis: - - import_path: accesscontextmanager/apiv1 - path: google/identity/accesscontextmanager/v1 - - name: advisorynotifications - version: 1.5.6 - apis: - - path: google/cloud/advisorynotifications/v1 - - name: apigeeconnect - version: 1.7.7 - apis: - - path: google/cloud/apigeeconnect/v1 diff --git a/tool/cmd/configcheck/testdata/no-librarian/.librarian/config.yaml b/tool/cmd/configcheck/testdata/no-librarian/.librarian/config.yaml deleted file mode 100644 index 9f71a2dc3c5..00000000000 --- a/tool/cmd/configcheck/testdata/no-librarian/.librarian/config.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - diff --git a/tool/cmd/configcheck/testdata/no-librarian/.librarian/state.yaml b/tool/cmd/configcheck/testdata/no-librarian/.librarian/state.yaml deleted file mode 100644 index 27c64f8a09c..00000000000 --- a/tool/cmd/configcheck/testdata/no-librarian/.librarian/state.yaml +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -image: us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/librarian-go@sha256:19bb93e8f1f916c61b597db2bad65dc432f79baaabb210499d7d0e4ad1dffe29 -libraries: - - id: accessapproval - version: 1.9.0 - last_generated_commit: 6c3dce4a02401667e9dd6d28304fee3f98e20ff8 - apis: - - path: google/cloud/accessapproval/v1 - service_config: accessapproval_v1.yaml - source_roots: - - accessapproval - - internal/generated/snippets/accessapproval - preserve_regex: [] - remove_regex: - - ^internal/generated/snippets/accessapproval/ - - ^accessapproval/apiv1/[^/]*_client\.go$ - - ^accessapproval/apiv1/[^/]*_client_example_go123_test\.go$ - - ^accessapproval/apiv1/[^/]*_client_example_test\.go$ - - ^accessapproval/apiv1/auxiliary\.go$ - - ^accessapproval/apiv1/auxiliary_go123\.go$ - - ^accessapproval/apiv1/doc\.go$ - - ^accessapproval/apiv1/gapic_metadata\.json$ - - ^accessapproval/apiv1/helpers\.go$ - - ^accessapproval/apiv1/accessapprovalpb/.*$ - - ^accessapproval/apiv1/\.repo-metadata\.json$ - release_exclude_paths: - - internal/generated/snippets/accessapproval/ - tag_format: '{id}/v{version}' - - id: accesscontextmanager - version: 1.10.0 - last_generated_commit: 6c3dce4a02401667e9dd6d28304fee3f98e20ff8 - apis: - - path: google/identity/accesscontextmanager/v1 - service_config: accesscontextmanager_v1.yaml - source_roots: - - accesscontextmanager - - internal/generated/snippets/accesscontextmanager - preserve_regex: [] - remove_regex: - - ^internal/generated/snippets/accesscontextmanager/ - - ^accesscontextmanager/apiv1/[^/]*_client\.go$ - - ^accesscontextmanager/apiv1/[^/]*_client_example_go123_test\.go$ - - ^accesscontextmanager/apiv1/[^/]*_client_example_test\.go$ - - ^accesscontextmanager/apiv1/auxiliary\.go$ - - ^accesscontextmanager/apiv1/auxiliary_go123\.go$ - - ^accesscontextmanager/apiv1/doc\.go$ - - ^accesscontextmanager/apiv1/gapic_metadata\.json$ - - ^accesscontextmanager/apiv1/helpers\.go$ - - ^accesscontextmanager/apiv1/accesscontextmanagerpb/.*$ - - ^accesscontextmanager/apiv1/\.repo-metadata\.json$ - release_exclude_paths: - - internal/generated/snippets/accesscontextmanager/ - tag_format: '{id}/v{version}' - - id: advisorynotifications - version: 1.5.6 - last_generated_commit: 6c3dce4a02401667e9dd6d28304fee3f98e20ff8 - apis: - - path: google/cloud/advisorynotifications/v1 - service_config: advisorynotifications_v1.yaml - source_roots: - - advisorynotifications - - internal/generated/snippets/advisorynotifications - preserve_regex: [] - remove_regex: - - ^internal/generated/snippets/advisorynotifications/ - - ^advisorynotifications/apiv1/[^/]*_client\.go$ - - ^advisorynotifications/apiv1/[^/]*_client_example_go123_test\.go$ - - ^advisorynotifications/apiv1/[^/]*_client_example_test\.go$ - - ^advisorynotifications/apiv1/auxiliary\.go$ - - ^advisorynotifications/apiv1/auxiliary_go123\.go$ - - ^advisorynotifications/apiv1/doc\.go$ - - ^advisorynotifications/apiv1/gapic_metadata\.json$ - - ^advisorynotifications/apiv1/helpers\.go$ - - ^advisorynotifications/apiv1/advisorynotificationspb/.*$ - - ^advisorynotifications/apiv1/\.repo-metadata\.json$ - release_exclude_paths: - - internal/generated/snippets/advisorynotifications/ - tag_format: '{id}/v{version}' From 27a2e8930f0110ae92884012dff58db6ef8787ed Mon Sep 17 00:00:00 2001 From: Joe Wang <106995533+JoeWang1127@users.noreply.github.com> Date: Mon, 29 Jun 2026 13:37:05 +0000 Subject: [PATCH 072/108] chore(internal/librarian/python): restore function visibility (#6560) Restore function visibility after migration. Fixes #4175 --- internal/librarian/python/generate.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/internal/librarian/python/generate.go b/internal/librarian/python/generate.go index 87a8cdf99ca..fd42abf37af 100644 --- a/internal/librarian/python/generate.go +++ b/internal/librarian/python/generate.go @@ -215,7 +215,7 @@ func createRepoMetadata(cfg *config.Config, library *config.Library, googleapisD repoMetadata.Name = library.Name } repoMetadata.LibraryType = packageOptions.LibraryType - repoMetadata.ClientDocumentation = BuildClientDocumentationURI(library.Name, repoMetadata.Name) + repoMetadata.ClientDocumentation = buildClientDocumentationURI(library.Name, repoMetadata.Name) // Even after migration oddities, just a few libraries don't fit into the // normal pattern for client documentation URI (e.g. the documentation is // in cloud.google.com when it would be expected to be in googleapis.dev). @@ -229,12 +229,9 @@ func createRepoMetadata(cfg *config.Config, library *config.Library, googleapisD return repoMetadata, nil } -// BuildClientDocumentationURI builds the URI for the client documentation +// buildClientDocumentationURI builds the URI for the client documentation // for the library. -// TODO(https://github.com/googleapis/librarian/issues/4175): make this function -// package-private (or inline it) after migration, when we won't need to -// determine whether or not to specify an override. -func BuildClientDocumentationURI(libraryName, repoMetadataName string) string { +func buildClientDocumentationURI(libraryName, repoMetadataName string) string { // Work out the right documentation URI based on whether this is a Cloud // or non-Cloud API. docTemplate := cloudGoogleComDocumentationTemplate @@ -252,7 +249,6 @@ func generateAPI(ctx context.Context, api *config.API, library *config.Library, // the correct final position in the repository. // TODO(https://github.com/googleapis/librarian/issues/3210): generate // directly in place. - protoOnly := isProtoOnly(api, library) stagingChildDirectory := getStagingChildDirectory(api.Path, protoOnly) stagingDir := filepath.Join(generationRoot, "owl-bot-staging", library.Name, stagingChildDirectory) From 660ab2bcbb654ad546945b4a107fd77923ee57a9 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Mon, 29 Jun 2026 14:33:04 +0000 Subject: [PATCH 073/108] feat(internal/librarian/java): Add JSpecify dep to proto module template (#6564) This changes adds the JSpecify dep to Java proto module template. --- internal/librarian/java/template/module_proto_pom.xml.tmpl | 4 ++++ .../proto-google-cloud-secretmanager-v1/pom.xml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/internal/librarian/java/template/module_proto_pom.xml.tmpl b/internal/librarian/java/template/module_proto_pom.xml.tmpl index 4860f10445c..0493e3fbc39 100644 --- a/internal/librarian/java/template/module_proto_pom.xml.tmpl +++ b/internal/librarian/java/template/module_proto_pom.xml.tmpl @@ -33,5 +33,9 @@ com.google.guava guava + + org.jspecify + jspecify + diff --git a/internal/librarian/java/testdata/syncpoms/secretmanager-v1/proto-google-cloud-secretmanager-v1/pom.xml b/internal/librarian/java/testdata/syncpoms/secretmanager-v1/proto-google-cloud-secretmanager-v1/pom.xml index b4d8e0fc0cd..df2b3493c21 100644 --- a/internal/librarian/java/testdata/syncpoms/secretmanager-v1/proto-google-cloud-secretmanager-v1/pom.xml +++ b/internal/librarian/java/testdata/syncpoms/secretmanager-v1/proto-google-cloud-secretmanager-v1/pom.xml @@ -33,5 +33,9 @@ com.google.guava guava + + org.jspecify + jspecify + From d6d50547a8c8200ac402e1cdf760b748a268a3c9 Mon Sep 17 00:00:00 2001 From: Min Zhu Date: Mon, 29 Jun 2026 11:42:32 -0400 Subject: [PATCH 074/108] docs(internal/librarian/java): add java package developer guide (#6479) Quick guide to uniform our dev workflow when making breaking changes to google-cloud-java. I expect this will be pretty often with post-migrate tasks. --- internal/librarian/java/README.md | 99 +++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 internal/librarian/java/README.md diff --git a/internal/librarian/java/README.md b/internal/librarian/java/README.md new file mode 100644 index 00000000000..56e3fd30516 --- /dev/null +++ b/internal/librarian/java/README.md @@ -0,0 +1,99 @@ +# Java Package Developer Guide + +This guide describes how to handle changes in the `librarian` repository +that affect client library generation in `google-cloud-java`. It covers +two scenarios: + +1. **Breaking Changes:** Changes that cause code generation failures, + compilation errors, or integration test failures in + `google-cloud-java` (see + [Handling Breaking Changes](#handling-breaking-changes-in-google-cloud-java)). +2. **Non-Breaking Diffs:** Changes that introduce diffs in the + generated code but do not break the build or tests (see + [Handling Changes That Cause Generation Diffs](#handling-changes-that-cause-generation-diffs)). + +## Handling Breaking Changes in `google-cloud-java` + +If you are making changes in `librarian` that are expected to cause code +generation failure or other breakages in the `google-cloud-java` repository +(such as in the integration tests; see +[Example](#example-of-a-breaking-change)): + +1. **Disable the Java Workflow:** + Temporarily disable the Java integration workflow by modifying + [java.yaml](/.github/workflows/java.yaml). + You can do this by prepending `false && ` to the `if` condition of the + `integration` job: + + ```yaml + integration: + runs-on: ubuntu-24.04 + if: false && github.event_name == 'push' && (github.ref == 'refs/heads/main') + ``` +2. **Add a TODO:** + Add a `TODO` comment in [java.yaml](/.github/workflows/java.yaml) linking to + the GitHub issue or pull request you are working on to track the reinstate + task: + + ```yaml + integration: + runs-on: ubuntu-24.04 + # TODO(https://github.com/googleapis/librarian/issues/XXXX): Reinstate this job + if: false && github.event_name == 'push' && (github.ref == 'refs/heads/main') + ``` +3. **Merge Librarian Changes:** + Merge your changes into the `librarian` repository. +4. **Update `google-cloud-java`:** + After the `librarian` changes are merged, update the `google-cloud-java` + repository to use the pseudo-version containing your changes. + + You can update the version in `librarian.yaml` by running the following + commands in the `google-cloud-java` repository: + + ```bash + # Get the latest pseudo-version of librarian from main + PSEUDO=$(GOPROXY=direct go list -m -f '{{.Version}}' github.com/googleapis/librarian@main) + + # Get the current librarian version used in the repo + V=$(go run github.com/googleapis/librarian/cmd/librarian@latest config get version) + + # Update the version in librarian.yaml using the current tool version + go run github.com/googleapis/librarian/cmd/librarian@${V} config set version $PSEUDO + ``` + + After updating the version, run `generate -all` to apply the changes. +5. **Reinstate the Java Workflow:** + Once `google-cloud-java` is updated and working with the new changes, remove + the `TODO` and reinstate the + [java.yaml](/.github/workflows/java.yaml) + workflow. + +### Example of a Breaking Change + +[PR #6432](https://github.com/googleapis/librarian/pull/6432) updated +`pom.xml` templates. It passed local tests but broke +`librarian generate --all` in `google-cloud-java` +([Issue #6446](https://github.com/googleapis/librarian/issues/6446)). +Because the integration test only runs in postsubmit, the failure +wasn't caught before merge, requiring a revert +([PR #6449](https://github.com/googleapis/librarian/pull/6449)). If +anticipated, the author should have disabled the workflow +beforehand. + +## Handling Changes That Cause Generation Diffs + +If you are making changes in `librarian` that do not cause generation failure in +`google-cloud-java` but will introduce a diff in the generated code: + +1. **Librarian CI Stays Green:** + The [java.yaml](/.github/workflows/java.yaml) integration check in the + `librarian` repository will not fail on such changes. +2. **Submit `google-cloud-java` PR:** + It is good practice to immediately open a pull request in the + `google-cloud-java` repository. This PR should update the `librarian` + dependency to **the new pseudo-version** containing your changes (using the + commands described in [Handling Breaking Changes](#handling-breaking-changes-in-google-cloud-java)) + and run `generate -all` to apply the generated diff. +3. **Prevent Weekly Update Diffs:** + Proactively applying these diffs prevents them from being introduced + abruptly during the weekly automated `librarian` updates. From 1ec1f6ec6ca3681d6b8e6a2e604940d2c7f02bb4 Mon Sep 17 00:00:00 2001 From: Joe Wang <106995533+JoeWang1127@users.noreply.github.com> Date: Mon, 29 Jun 2026 15:45:37 +0000 Subject: [PATCH 075/108] chore(internal/librarian): refactor unit tests (#6557) For unit tests of `applyDefaults`, split it into success and error cases. See https://github.com/googleapis/librarian/blob/main/doc/howwewritego.md#separate-error-tests --- internal/librarian/library.go | 7 ++++- internal/librarian/library_test.go | 47 +++++++++++++++++++++--------- 2 files changed, 39 insertions(+), 15 deletions(-) diff --git a/internal/librarian/library.go b/internal/librarian/library.go index 6b25f7abfce..13a56f6651e 100644 --- a/internal/librarian/library.go +++ b/internal/librarian/library.go @@ -15,6 +15,7 @@ package librarian import ( + "errors" "fmt" "maps" "strings" @@ -28,6 +29,10 @@ import ( "github.com/googleapis/librarian/internal/librarian/swift" ) +var ( + errNoExplicitOutput = errors.New("library requires an explicit output path") +) + // fillDefaults populates empty library fields from the provided defaults. func fillDefaults(lib *config.Library, d *config.Default) *config.Library { if d == nil { @@ -288,7 +293,7 @@ func applyDefaults(language string, lib *config.Library, defaults *config.Defaul } if lib.Output == "" { if isMixedLibrary(language, lib) { - return nil, fmt.Errorf("library %q requires an explicit output path", lib.Name) + return nil, fmt.Errorf("%s: %w", lib.Name, errNoExplicitOutput) } var apiPath string if len(lib.APIs) > 0 { diff --git a/internal/librarian/library_test.go b/internal/librarian/library_test.go index 3e76cf3ef0e..d79eeb2e891 100644 --- a/internal/librarian/library_test.go +++ b/internal/librarian/library_test.go @@ -15,6 +15,7 @@ package librarian import ( + "errors" "testing" "github.com/google/go-cmp/cmp" @@ -626,7 +627,7 @@ func TestFillDefaults_Go(t *testing.T) { } } -func TestPrepareLibrary(t *testing.T) { +func TestApplyDefaults(t *testing.T) { for _, test := range []struct { name string language string @@ -634,7 +635,6 @@ func TestPrepareLibrary(t *testing.T) { rust *config.RustCrate apis []*config.API wantOutput string - wantErr bool wantAPIPath string }{ { @@ -671,12 +671,6 @@ func TestPrepareLibrary(t *testing.T) { apis: nil, wantOutput: "src/storage/test/v1", }, - { - name: "veneer without output returns error", - language: config.LanguageRust, - rust: &config.RustCrate{Modules: []*config.RustModule{{APIPath: "google/storage/v2"}}}, - wantErr: true, - }, { name: "veneer with explicit output succeeds", language: config.LanguageRust, @@ -708,12 +702,6 @@ func TestPrepareLibrary(t *testing.T) { Output: "src/generated", } got, err := applyDefaults(test.language, lib, defaults) - if test.wantErr { - if err == nil { - t.Fatal("expected error, got nil") - } - return - } if err != nil { t.Fatal(err) } @@ -730,6 +718,37 @@ func TestPrepareLibrary(t *testing.T) { } } +func TestApplyDefaults_Error(t *testing.T) { + for _, test := range []struct { + name string + language string + lib *config.Library + wantErr error + }{ + { + name: "veneer without output returns error", + language: config.LanguageRust, + lib: &config.Library{ + Name: "storage", + Rust: &config.RustCrate{ + Modules: []*config.RustModule{{APIPath: "google/storage/v2"}}, + }, + }, + wantErr: errNoExplicitOutput, + }, + } { + t.Run(test.name, func(t *testing.T) { + defaults := &config.Default{ + Output: "src/generated", + } + _, err := applyDefaults(test.language, test.lib, defaults) + if !errors.Is(err, test.wantErr) { + t.Errorf("got error %v, want %v", err, test.wantErr) + } + }) + } +} + func TestCanDeriveAPIPath(t *testing.T) { for _, test := range []struct { name string From feb2ef22fa2b45f2bf216a7da95c106eeaacbad8 Mon Sep 17 00:00:00 2001 From: haphungw Date: Mon, 29 Jun 2026 10:48:17 -0700 Subject: [PATCH 076/108] feat(sidekick/rust): track recording error info for discovery LROs (#6304) Add error telemetry for Discovery LROs so that their traces contain the same error information as standard LROs. Staged @ https://github.com/googleapis/google-cloud-rust/pull/5861 Fixes #6286 --- .../sidekick/rust/templates/crate/src/tracing.rs.mustache | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/internal/sidekick/rust/templates/crate/src/tracing.rs.mustache b/internal/sidekick/rust/templates/crate/src/tracing.rs.mustache index 7b8e10198ce..ffb4b04f508 100644 --- a/internal/sidekick/rust/templates/crate/src/tracing.rs.mustache +++ b/internal/sidekick/rust/templates/crate/src/tracing.rs.mustache @@ -109,9 +109,7 @@ where T: super::stub::{{Codec.Name}} + std::fmt::Debug + Send + Sync { } {{/Codec.IsDiscoveryLro}} {{#Codec.IsDiscoveryLro}} - // TODO(https://github.com/googleapis/librarian/issues/6286): Track recording error info for Discovery LROs - let done = google_cloud_lro::internal::DiscoveryOperation::done(op); - _span.record("gcp.longrunning.done", done); + google_cloud_lro::record_discovery_polling_result!(&_span, op); {{/Codec.IsDiscoveryLro}} } Err(e) => { From 62d6c15c3fe89e6a8413a42d2e94c4843fb2ee82 Mon Sep 17 00:00:00 2001 From: Joe Wang <106995533+JoeWang1127@users.noreply.github.com> Date: Mon, 29 Jun 2026 18:14:54 +0000 Subject: [PATCH 077/108] chore(internal/librarian): rename `cfg_value_utility.go` to `config_value.go` (#6562) Rename `cfg_value_utility.go` to `config_value.go` --- internal/librarian/{cfg_value_utility.go => config_value.go} | 0 .../librarian/{cfg_value_utility_test.go => config_value_test.go} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename internal/librarian/{cfg_value_utility.go => config_value.go} (100%) rename internal/librarian/{cfg_value_utility_test.go => config_value_test.go} (100%) diff --git a/internal/librarian/cfg_value_utility.go b/internal/librarian/config_value.go similarity index 100% rename from internal/librarian/cfg_value_utility.go rename to internal/librarian/config_value.go diff --git a/internal/librarian/cfg_value_utility_test.go b/internal/librarian/config_value_test.go similarity index 100% rename from internal/librarian/cfg_value_utility_test.go rename to internal/librarian/config_value_test.go From bb509a740d375c1ca1709e624e404b69924fdfa7 Mon Sep 17 00:00:00 2001 From: Joe Wang <106995533+JoeWang1127@users.noreply.github.com> Date: Mon, 29 Jun 2026 18:18:10 +0000 Subject: [PATCH 078/108] refactor(internal/fetch): rename helper functions in fetch.go (#6559) Export functions in `fetch` package and create `filter` function when extracting tarball. Preparations to implement `protoc` installation. For #6558 --- internal/fetch/fetch.go | 48 +++++++++++++++++++----------------- internal/fetch/fetch_test.go | 29 +++++++++++----------- 2 files changed, 41 insertions(+), 36 deletions(-) diff --git a/internal/fetch/fetch.go b/internal/fetch/fetch.go index e73674d8437..ee6ac986489 100644 --- a/internal/fetch/fetch.go +++ b/internal/fetch/fetch.go @@ -128,7 +128,7 @@ func Repo(ctx context.Context, repo, commit, expectedSHA256 string) (string, err if err := os.MkdirAll(outDir, 0755); err != nil { return "", fmt.Errorf("failed creating %q: %w", outDir, err) } - if err := extractTarball(tgz, outDir); err == nil { + if err := ExtractTarball(tgz, outDir, stripTopLevelDir); err == nil { return outDir, nil } } @@ -146,10 +146,10 @@ func Repo(ctx context.Context, repo, commit, expectedSHA256 string) (string, err if err := os.MkdirAll(outDir, 0755); err != nil { return "", fmt.Errorf("failed creating %q: %w", outDir, err) } - if err := download(ctx, tgz, sourceURL, expectedSHA256); err != nil { + if err := Download(ctx, tgz, sourceURL, expectedSHA256); err != nil { return "", err } - if err := extractTarball(tgz, outDir); err != nil { + if err := ExtractTarball(tgz, outDir, stripTopLevelDir); err != nil { return "", fmt.Errorf("failed to extract tarball: %w", err) } return outDir, nil @@ -286,14 +286,14 @@ func tarballLink(githubDownload string, repo *RepoRef, sha string) string { return fmt.Sprintf("%s/%s/%s/archive/%s.tar.gz", githubDownload, repo.Org, repo.Name, sha) } -// download downloads a file from the given url to the target path, verifying -// its SHA256 checksum matches expectedSha256. It retries up to +// Download downloads a file from the given url to the target path, verifying +// its SHA256 checksum matches expectedSHA256. It retries up to // maxDownloadRetries times with exponential backoff on failure. -func download(ctx context.Context, target, url, expectedSha256 string) error { +func Download(ctx context.Context, target, url, expectedSHA256 string) error { if fileExists(target) { return nil } - if expectedSha256 == "" { + if expectedSHA256 == "" { return errMissingSHA256 } if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { @@ -319,8 +319,8 @@ func download(ctx context.Context, target, url, expectedSha256 string) error { if err != nil { return err } - if sha != expectedSha256 { - return fmt.Errorf("%w: expected=%s, got=%s", errChecksumMismatch, expectedSha256, sha) + if sha != expectedSHA256 { + return fmt.Errorf("%w: expected=%s, got=%s", errChecksumMismatch, expectedSHA256, sha) } return os.Rename(tempPath, target) } @@ -392,9 +392,21 @@ func fileExists(name string) bool { return stat.Mode().IsRegular() } -// extractTarball extracts a gzipped tarball to the specified directory, -// stripping the top-level directory prefix that GitHub adds to tarballs. -func extractTarball(tarballPath, destDir string) error { +// stripTopLevelDir removes the top-level directory prefix (such as "{repo}-{commit}/") +// that GitHub automatically adds to repository archive entries. +func stripTopLevelDir(name string) (string, bool) { + parts := strings.SplitN(name, "/", 2) + if len(parts) == 2 { + return parts[1], true + } + return "", false +} + +// ExtractTarball extracts a gzipped tarball to the specified directory. +func ExtractTarball(tarballPath, destDir string, filter func(string) (string, bool)) error { + if filter == nil { + filter = func(name string) (string, bool) { return name, true } + } f, err := os.Open(tarballPath) if err != nil { return err @@ -417,18 +429,10 @@ func extractTarball(tarballPath, destDir string) error { return err } - // When GitHub creates a tarball archive of a repository, it wraps all - // the files in a top-level directory named in the format - // "{repo}-{commit}/". Remove the GitHub top-level "repo-/" - // prefix. - name := hdr.Name - parts := strings.SplitN(name, "/", 2) - if len(parts) == 2 { - name = parts[1] - } else { + name, ok := filter(hdr.Name) + if !ok { continue } - target := filepath.Join(destDir, name) switch hdr.Typeflag { case tar.TypeDir: diff --git a/internal/fetch/fetch_test.go b/internal/fetch/fetch_test.go index 69d15937392..f03f86f8b52 100644 --- a/internal/fetch/fetch_test.go +++ b/internal/fetch/fetch_test.go @@ -446,7 +446,7 @@ func TestDownload_TgzExists(t *testing.T) { if err := os.WriteFile(target, tarball.Contents, 0644); err != nil { t.Fatal(err) } - if err := download(t.Context(), target, "https://unused/placeholder.tar.gz", tarball.Sha256); err != nil { + if err := Download(t.Context(), target, "https://unused/placeholder.tar.gz", tarball.Sha256); err != nil { t.Fatal(err) } } @@ -464,7 +464,7 @@ func TestDownload_NeedsDownload(t *testing.T) { defer server.Close() expected := path.Join(testDir, "new-file") - if err := download(t.Context(), expected, server.URL+"/placeholder.tar.gz", tarball.Sha256); err != nil { + if err := Download(t.Context(), expected, server.URL+"/placeholder.tar.gz", tarball.Sha256); err != nil { t.Fatal(err) } got, err := os.ReadFile(expected) @@ -488,7 +488,7 @@ func TestDownload_ChecksumMismatch(t *testing.T) { target := path.Join(testDir, "target-file") wrongSha := "0000000000000000000000000000000000000000000000000000000000000000" - err := download(t.Context(), target, server.URL+"/test.tar.gz", wrongSha) + err := Download(t.Context(), target, server.URL+"/test.tar.gz", wrongSha) if !errors.Is(err, errChecksumMismatch) { t.Fatalf("expected errChecksumMismatch, got: %v", err) } @@ -516,7 +516,7 @@ func TestDownload_ContextCanceled(t *testing.T) { cancel() }() - err := download(ctx, target, server.URL+"/test.tar.gz", "any-sha") + err := Download(ctx, target, server.URL+"/test.tar.gz", "any-sha") if !errors.Is(err, context.Canceled) { t.Fatalf("expected context.Canceled, got: %v", err) } @@ -557,7 +557,7 @@ func TestExtractTarball(t *testing.T) { } destDir := t.TempDir() - if err := extractTarball(tarballPath, destDir); err != nil { + if err := ExtractTarball(tarballPath, destDir, stripTopLevelDir); err != nil { t.Fatal(err) } @@ -760,7 +760,7 @@ func TestExtractTarball_Error(t *testing.T) { }, } { t.Run(test.name, func(t *testing.T) { - err := extractTarball(test.tarballPath(t), test.dest(t)) + err := ExtractTarball(test.tarballPath(t), test.dest(t), stripTopLevelDir) if !errors.Is(err, test.wantErr) { t.Fatalf("got error %v, want %v", err, test.wantErr) } @@ -803,7 +803,7 @@ func TestExtractTarball_PathError(t *testing.T) { }, } { t.Run(test.name, func(t *testing.T) { - err := extractTarball(test.tarballPath(t), test.dest(t)) + err := ExtractTarball(test.tarballPath(t), test.dest(t), stripTopLevelDir) var pathErr *fs.PathError if !errors.As(err, &pathErr) { t.Fatalf("got error %v, want *fs.PathError", err) @@ -834,7 +834,8 @@ func TestDownload_Error(t *testing.T) { }, sha: "any-sha", wantErr: true, - }, { + }, + { name: "cannot create parent directory", target: func(t *testing.T) string { // Create a read-only directory to trigger a permission error. @@ -860,9 +861,9 @@ func TestDownload_Error(t *testing.T) { t.Cleanup(func() { defaultBackoff = 10 * time.Second }) - err := download(context.Background(), test.target(t), test.url(t), test.sha) + err := Download(context.Background(), test.target(t), test.url(t), test.sha) if (err != nil) != test.wantErr { - t.Errorf("download() error = %v, wantErr %v", err, test.wantErr) + t.Errorf("Download() error = %v, wantErr %v", err, test.wantErr) } }) } @@ -870,7 +871,7 @@ func TestDownload_Error(t *testing.T) { func TestDownload_EmptySha(t *testing.T) { target := path.Join(t.TempDir(), "target") - err := download(t.Context(), target, "https://any-url", "") + err := Download(t.Context(), target, "https://any-url", "") if !errors.Is(err, errMissingSHA256) { t.Errorf("expected errMissingSHA256, got: %v", err) } @@ -980,7 +981,7 @@ func TestDownload_RetryErrorIncludesLastFailure(t *testing.T) { defer server.Close() target := path.Join(t.TempDir(), "target-file") - err := download(t.Context(), target, server.URL+"/test.tar.gz", "any-sha") + err := Download(t.Context(), target, server.URL+"/test.tar.gz", "any-sha") if err == nil { t.Fatal("expected an error") } @@ -1011,7 +1012,7 @@ func TestDownload_RetrySucceeds(t *testing.T) { defer server.Close() target := path.Join(t.TempDir(), "target-file") - if err := download(t.Context(), target, server.URL+"/test.tar.gz", tarball.Sha256); err != nil { + if err := Download(t.Context(), target, server.URL+"/test.tar.gz", tarball.Sha256); err != nil { t.Fatal(err) } @@ -1114,7 +1115,7 @@ func TestExtractTarball_Symlink(t *testing.T) { } destDir := t.TempDir() - if err := extractTarball(tarballPath, destDir); err != nil { + if err := ExtractTarball(tarballPath, destDir, stripTopLevelDir); err != nil { t.Fatal(err) } From a71db8e3f1e34626f2da92f518402e4ca051ff6d Mon Sep 17 00:00:00 2001 From: yangyzs <171981480+yangyzs@users.noreply.github.com> Date: Mon, 29 Jun 2026 15:11:40 -0400 Subject: [PATCH 079/108] refactor(internal/librarian/java): accept file path in extractTitle (#6573) Accept file path in extractTitle to read file content from disk directly. Update unit tests to use t.TempDir(). Follow-up to #6546 --- internal/librarian/java/readme.go | 17 +++++++++++++---- internal/librarian/java/readme_test.go | 14 ++++++++++++-- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/internal/librarian/java/readme.go b/internal/librarian/java/readme.go index 9a7db30a34b..2058f7334e7 100644 --- a/internal/librarian/java/readme.go +++ b/internal/librarian/java/readme.go @@ -15,7 +15,10 @@ package java import ( + "bytes" "errors" + "fmt" + "os" "path/filepath" "regexp" "strings" @@ -48,13 +51,19 @@ func isProductionSample(path string) bool { (strings.HasPrefix(slashed, "src/main/java/") || strings.Contains(slashed, "/src/main/java/")) } -// extractTitle extracts and validates the title override from Java comment blocks. +// extractTitle reads a file from disk and extracts the title override from Java comment blocks. // It expects a "title:" line to immediately follow the "sample-metadata:" marker. -// Returns an error if the marker is present but the title line is missing, malformed, or empty. -func extractTitle(content string) (string, error) { - if !strings.Contains(content, "sample-metadata:") { +// Returns an error if the file cannot be read, or if the marker is present but the title line +// is missing, malformed, or empty. +func extractTitle(filePath string) (string, error) { + contentBytes, err := os.ReadFile(filePath) + if err != nil { + return "", fmt.Errorf("failed to read file: %w", err) + } + if !bytes.Contains(contentBytes, []byte("sample-metadata:")) { return "", nil } + content := string(contentBytes) matches := reTitle.FindStringSubmatch(content) if len(matches) < 2 { return "", errMissingTitle diff --git a/internal/librarian/java/readme_test.go b/internal/librarian/java/readme_test.go index 595c87cb645..a2261c597f2 100644 --- a/internal/librarian/java/readme_test.go +++ b/internal/librarian/java/readme_test.go @@ -16,6 +16,8 @@ package java import ( "errors" + "os" + "path/filepath" "testing" "github.com/google/go-cmp/cmp" @@ -141,7 +143,11 @@ public class Normal {}`, }, } { t.Run(test.name, func(t *testing.T) { - got, err := extractTitle(test.content) + tmpPath := filepath.Join(t.TempDir(), "Sample.java") + if err := os.WriteFile(tmpPath, []byte(test.content), 0644); err != nil { + t.Fatal(err) + } + got, err := extractTitle(tmpPath) if err != nil { t.Fatal(err) } @@ -172,7 +178,11 @@ func TestExtractTitle_Error(t *testing.T) { }, } { t.Run(test.name, func(t *testing.T) { - _, gotErr := extractTitle(test.content) + tmpPath := filepath.Join(t.TempDir(), "Sample.java") + if err := os.WriteFile(tmpPath, []byte(test.content), 0644); err != nil { + t.Fatal(err) + } + _, gotErr := extractTitle(tmpPath) if !errors.Is(gotErr, test.wantErr) { t.Errorf("extractTitle() error = %v, wantErr %v", gotErr, test.wantErr) } From faa88f0efe1efbfd2b588f4f69afda48ed26c5a1 Mon Sep 17 00:00:00 2001 From: g-husam Date: Mon, 29 Jun 2026 15:50:54 -0400 Subject: [PATCH 080/108] chore(actions): update version comments and specify setup-dart release version (#6570) This commit fixes some inconsistent version comments for commit SHAs, and specifies the latest release's commit SHA for setup-dart. Dependabot should be able to keep this up to date by updating to the commit SHA for newer releases when they are available. --- .github/workflows/dart.yaml | 2 +- .github/workflows/java.yaml | 2 +- .github/workflows/nodejs.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dart.yaml b/.github/workflows/dart.yaml index 003bbf95c41..06ef31794e2 100644 --- a/.github/workflows/dart.yaml +++ b/.github/workflows/dart.yaml @@ -28,7 +28,7 @@ jobs: - name: Display Go version run: go version - name: Install Dart SDK - uses: dart-lang/setup-dart@e51d8e571e22473a2ddebf0ef8a2123f0ab2c02c + uses: dart-lang/setup-dart@65eb853c7ba17dde3be364c3d2858773e7144260 # v1.7.2 with: sdk: 3.9.0 - name: Display Dart version diff --git a/.github/workflows/java.yaml b/.github/workflows/java.yaml index 1c506506978..7125494571a 100644 --- a/.github/workflows/java.yaml +++ b/.github/workflows/java.yaml @@ -43,7 +43,7 @@ jobs: python-version: "3.12" - name: Cache Librarian Tools & Venv id: cache-tools - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: | ~/.cache/librarian diff --git a/.github/workflows/nodejs.yaml b/.github/workflows/nodejs.yaml index f05ab83a32a..b722226456a 100644 --- a/.github/workflows/nodejs.yaml +++ b/.github/workflows/nodejs.yaml @@ -37,7 +37,7 @@ jobs: run: | echo "npm=$(npm config get prefix)" >> "$GITHUB_OUTPUT" - name: "Install Node.js dependencies (restore cache)" - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 id: tools-cache with: path: | From 14e186cb2cdc1d8c545d022aba24bbd864e13ea0 Mon Sep 17 00:00:00 2001 From: Carlos O'Ryan Date: Mon, 29 Jun 2026 15:52:07 -0400 Subject: [PATCH 081/108] feat(sidekick/swift): swap client vs. protocol (#6566) Move the client struct to the top-level namespace. At the same time, move the protocol to the nested `Clients` namespace. This makes the client libraries easier to use, as most applications only interact with the client type, which now has a shorter name. --- .../sidekick/swift/annotate_message_test.go | 6 +- internal/sidekick/swift/field_type_name.go | 2 +- .../swift/generate_pagination_swift_test.go | 16 +- .../swift/generate_service_swift_test.go | 29 +- .../templates/common/client_protocol.mustache | 143 ++++++++++ .../common/client_snippet.swift.mustache | 2 +- .../common/method_snippet.swift.mustache | 4 +- .../templates/common/placeholder.mustache | 2 +- .../templates/common/service.swift.mustache | 259 +++++------------- 9 files changed, 234 insertions(+), 229 deletions(-) create mode 100644 internal/sidekick/swift/templates/common/client_protocol.mustache diff --git a/internal/sidekick/swift/annotate_message_test.go b/internal/sidekick/swift/annotate_message_test.go index 33f021366ae..0f6e9d6862d 100644 --- a/internal/sidekick/swift/annotate_message_test.go +++ b/internal/sidekick/swift/annotate_message_test.go @@ -143,7 +143,7 @@ func TestAnnotateMessage(t *testing.T) { TypeURL: "type.googleapis.com/test.Service", CustomSerialization: false, SampleField: "", - ParameterTypeName: "Clients.ServiceClient", + ParameterTypeName: "ServiceClient", PlaceholderName: "ServiceClient", }, wantImports: []string{"GoogleCloudWkt"}, @@ -315,7 +315,7 @@ func TestAnnotateMessage_DiscoveryRequests(t *testing.T) { Name: "GetRequest", TypeURL: "type.googleapis.com/test.Service.getRequest", SampleField: "", - ParameterTypeName: "Clients.ServiceClient.GetRequest", + ParameterTypeName: "ServiceClient.GetRequest", }, }, { @@ -326,7 +326,7 @@ func TestAnnotateMessage_DiscoveryRequests(t *testing.T) { Name: "ListRequest", TypeURL: "type.googleapis.com/test.Protocol.listRequest", SampleField: "", - ParameterTypeName: "Clients.ProtocolClient.ListRequest", + ParameterTypeName: "ProtocolClient.ListRequest", }, }, } { diff --git a/internal/sidekick/swift/field_type_name.go b/internal/sidekick/swift/field_type_name.go index c4c0a9bb478..da6c54c1787 100644 --- a/internal/sidekick/swift/field_type_name.go +++ b/internal/sidekick/swift/field_type_name.go @@ -147,7 +147,7 @@ func scalarFieldTypeName(field *api.Field) (string, error) { func (c *codec) messageTypeName(m *api.Message) (string, error) { name := pascalCase(m.Name) if m.ServicePlaceholder { - name = "Clients." + pascalCase(m.Name+"Client") + name = pascalCase(m.Name + "Client") } if m.Parent == nil { prefix, err := c.externalTypePrefix(m.Package) diff --git a/internal/sidekick/swift/generate_pagination_swift_test.go b/internal/sidekick/swift/generate_pagination_swift_test.go index ab0f87bfae5..be60d43168c 100644 --- a/internal/sidekick/swift/generate_pagination_swift_test.go +++ b/internal/sidekick/swift/generate_pagination_swift_test.go @@ -155,18 +155,18 @@ func verifyGeneratedMapService(t *testing.T, outDir string) { contentStr := string(content) gotMethodOverload := extractBlock(t, contentStr, ` public func listSecrets( - byItem: `, "\n }") + byItem: `, "\n }") wantMethodOverload := ` public func listSecrets( byItem: ListSecretsRequest, options: GoogleCloudGax.RequestOptions ) throws -> any AsyncSequence<(Swift.String, Secret), Swift.Error> { - let listRpc = { (token: String) async throws -> GoogleCloudSecretmanagerV1.ListSecretsResponse in - var request = byItem - request.pageToken = token - return try await self.listSecrets(request: request, options: options) - } - return GoogleCloudGax.PaginatedResponseSequence(listRpc: listRpc) - }` + let listRpc = { (token: String) async throws -> GoogleCloudSecretmanagerV1.ListSecretsResponse in + var request = byItem + request.pageToken = token + return try await self.listSecrets(request: request, options: options) + } + return GoogleCloudGax.PaginatedResponseSequence(listRpc: listRpc) + }` if diff := cmp.Diff(wantMethodOverload, gotMethodOverload); diff != "" { t.Errorf("mismatch (-want +got):\n%s", diff) } diff --git a/internal/sidekick/swift/generate_service_swift_test.go b/internal/sidekick/swift/generate_service_swift_test.go index 582209c564b..476e88b1017 100644 --- a/internal/sidekick/swift/generate_service_swift_test.go +++ b/internal/sidekick/swift/generate_service_swift_test.go @@ -100,9 +100,8 @@ func TestGenerateServiceSwift_SnippetReference(t *testing.T) { } contentStr := string(content) - gotBlock := extractBlock(t, contentStr, "/// @Snippet", "public protocol Protocol_ {") - wantBlock := `/// @Snippet(path: "ProtocolQuickstart") -public protocol Protocol_ {` + gotBlock := extractBlock(t, contentStr, "public protocol ", "{") + wantBlock := `public protocol ProtocolProtocol {` if diff := cmp.Diff(wantBlock, gotBlock); diff != "" { t.Errorf("mismatch (-want +got):\n%s", diff) } @@ -173,8 +172,8 @@ func TestGenerateService_Delegation(t *testing.T) { contentStr := string(content) for _, want := range []string{ - "let inner: any IAMStub", - "var inner: any IAMStub = try IAMTransport(options)", + "let inner: any Clients.IAMStub", + "var inner: any Clients.IAMStub = try Clients.IAMTransport(options)", "try await self.inner.createRole(request: request, options: options)", } { if !strings.Contains(contentStr, want) { @@ -579,18 +578,18 @@ func verifyGeneratedService(t *testing.T, outDir string) { contentStr := string(content) gotMethodOverload := extractBlock(t, contentStr, ` public func listSecrets( - byItem: ListSecretsRequest, options: `, "\n }") + byItem: ListSecretsRequest, options: `, "\n }") wantMethodOverload := ` public func listSecrets( byItem: ListSecretsRequest, options: GoogleCloudGax.RequestOptions ) throws -> any AsyncSequence { - let listRpc = { (token: String) async throws -> GoogleCloudSecretmanagerV1.ListSecretsResponse in - var request = byItem - request.pageToken = token - return try await self.listSecrets(request: request, options: options) - } - return GoogleCloudGax.PaginatedResponseSequence(listRpc: listRpc) - }` + let listRpc = { (token: String) async throws -> GoogleCloudSecretmanagerV1.ListSecretsResponse in + var request = byItem + request.pageToken = token + return try await self.listSecrets(request: request, options: options) + } + return GoogleCloudGax.PaginatedResponseSequence(listRpc: listRpc) + }` if diff := cmp.Diff(wantMethodOverload, gotMethodOverload); diff != "" { t.Errorf("mismatch (-want +got):\n%s", diff) } @@ -955,8 +954,8 @@ func TestGenerateDiscoveryService_Files(t *testing.T) { t.Fatal(err) } - // Verify it contains an extension to the right Clients.$ServiceName type. - wantExtension := fmt.Appendf(nil, "extension Clients.%sClient {", test.serviceName) + // Verify it contains an extension to the right ${ServiceName}Client type. + wantExtension := fmt.Appendf(nil, "extension %sClient {", test.serviceName) if !bytes.Contains(content, wantExtension) { t.Errorf("expected extension %q in %s, got:\n%s", wantExtension, filename, content) } diff --git a/internal/sidekick/swift/templates/common/client_protocol.mustache b/internal/sidekick/swift/templates/common/client_protocol.mustache new file mode 100644 index 00000000000..cc40ca0025a --- /dev/null +++ b/internal/sidekick/swift/templates/common/client_protocol.mustache @@ -0,0 +1,143 @@ +{{! +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +}} +extension Clients { + /// A Swift protocol to mock `{{Codec.ClientName}}`. + /// + /// To mock `{{Codec.ClientName}}` change your functions to receive + /// `some {{Codec.StubPrefix}}Protocol` or `any {{Codec.StubPrefix}}Protocol` + /// and pass a mock implementation in your tests. + public protocol {{Codec.StubPrefix}}Protocol { + {{#Codec.RestMethods}} + /// See `{{Service.Codec.ClientName}}.{{Codec.Name}}`. + func {{> /templates/common/method_signature/request}} + {{#Codec.PlainRPC}} + {{#Signatures}} + /// See `{{Service.Codec.ClientName}}.{{Codec.Name}}`. + func {{> /templates/common/method_signature/overload}} + {{/Signatures}} + {{/Codec.PlainRPC}} + {{#Codec.Pagination}} + /// See `{{Service.Codec.ClientName}}.{{Codec.Name}}`. + func {{> /templates/common/method_signature/pagination}} + {{#Signatures}} + /// See `{{Service.Codec.ClientName}}.{{Codec.Name}}`. + func {{> /templates/common/method_signature/pagination_overload}} + {{/Signatures}} + {{/Codec.Pagination}} + {{#Codec.LRO}} + /// See `{{Service.Codec.ClientName}}.{{Codec.Name}}`. + func {{> /templates/common/method_signature/lro}} + {{#Signatures}} + /// See `{{Service.Codec.ClientName}}.{{Codec.Name}}`. + func {{> /templates/common/method_signature/lro_overload}} + {{/Signatures}} + {{/Codec.LRO}} + {{/Codec.RestMethods}} + + {{#Codec.RestMethods}} + /// See `{{Service.Codec.ClientName}}.{{Codec.Name}}`. + func {{> /templates/common/method_signature/request_options}} + {{#Codec.Pagination}} + /// See `{{Service.Codec.ClientName}}.{{Codec.Name}}`. + func {{> /templates/common/method_signature/pagination_options}} + {{/Codec.Pagination}} + {{#Codec.LRO}} + /// See `{{Service.Codec.ClientName}}.{{Codec.Name}}`. + func {{> /templates/common/method_signature/lro_options}} + {{/Codec.LRO}} + {{/Codec.RestMethods}} + } +} + +// Default implementations +extension Clients.{{Codec.StubPrefix}}Protocol { + {{#Codec.RestMethods}} + + public func {{> /templates/common/method_signature/request}} { + try await self.{{Codec.Name}}(request: request, options: .init()) + } + + public func {{> /templates/common/method_signature/request_options}} { + throw GoogleCloudGax.RequestError.unimplemented + } + {{#Codec.PlainRPC}} + {{#Signatures}} + + public func {{> /templates/common/method_signature/overload}} { + let request = {{Method.InputType.Codec.ParameterTypeName}}().with { + {{#Fields}} + $0.{{Codec.Name}} = {{Codec.Name}} + {{/Fields}} + } + {{#Method.ReturnsEmpty}} + try await self.{{Method.Codec.Name}}(request: request) + {{/Method.ReturnsEmpty}} + {{^Method.ReturnsEmpty}} + return try await self.{{Method.Codec.Name}}(request: request) + {{/Method.ReturnsEmpty}} + } + {{/Signatures}} + {{/Codec.PlainRPC}} + {{#Codec.Pagination}} + + public func {{> /templates/common/method_signature/pagination}} { + try self.{{Codec.Name}}(byItem: byItem, options: .init()) + } + + public func {{> /templates/common/method_signature/pagination_options}} { + let listRpc = { (token: String) async throws -> {{Codec.ReturnType}} in + throw GoogleCloudGax.RequestError.unimplemented + } + return GoogleCloudGax.PaginatedResponseSequence(listRpc: listRpc) + } + {{#Signatures}} + + public func {{> /templates/common/method_signature/pagination_overload}} { + let request = {{Method.InputType.Codec.ParameterTypeName}}().with { + {{#Fields}} + $0.{{Codec.Name}} = {{Codec.Name}} + {{/Fields}} + } + return try self.{{Codec.Name}}(byItem: request) + } + {{/Signatures}} + {{/Codec.Pagination}} + {{#Codec.LRO}} + + public func {{> /templates/common/method_signature/lro}} { + try await self.{{Codec.Name}}(withPolling: withPolling, options: .init()) + } + + public func {{> /templates/common/method_signature/lro_options}} { + let poll = { () async throws -> GoogleCloudGax._PollableOperationImpl<{{Codec.LRO.ReturnType}}>.State in + throw GoogleCloudGax.RequestError.unimplemented + } + return GoogleCloudGax._PollableOperationImpl(initialState: .init(done: false, result: nil), poll: poll) + } + {{#Signatures}} + + public func {{> /templates/common/method_signature/lro_overload}} { + let request = {{Method.InputType.Codec.ParameterTypeName}}().with { + {{#Fields}} + $0.{{Codec.Name}} = {{Codec.Name}} + {{/Fields}} + } + return try await self.{{Codec.Name}}(withPolling: request) + } + {{/Signatures}} + {{/Codec.LRO}} + {{/Codec.RestMethods}} +} diff --git a/internal/sidekick/swift/templates/common/client_snippet.swift.mustache b/internal/sidekick/swift/templates/common/client_snippet.swift.mustache index a5a470ff5b1..8b40344e0ce 100644 --- a/internal/sidekick/swift/templates/common/client_snippet.swift.mustache +++ b/internal/sidekick/swift/templates/common/client_snippet.swift.mustache @@ -32,7 +32,7 @@ import {{.}} {{/Codec.SnippetImports}} func sample({{#Codec.QuickstartMethod.SampleInfo}}{{#Codec.Parameters}}{{.}}: String, {{/Codec.Parameters}}{{/Codec.QuickstartMethod.SampleInfo}}) async throws { - let client = try {{Codec.PackageName}}.Clients.{{Codec.ClientName}}() + let client = try {{Codec.PackageName}}.{{Codec.ClientName}}() {{#Codec.QuickstartMethod}} {{> /templates/snippet/generic}} {{/Codec.QuickstartMethod}} diff --git a/internal/sidekick/swift/templates/common/method_snippet.swift.mustache b/internal/sidekick/swift/templates/common/method_snippet.swift.mustache index 720299706e9..e7017a758ea 100644 --- a/internal/sidekick/swift/templates/common/method_snippet.swift.mustache +++ b/internal/sidekick/swift/templates/common/method_snippet.swift.mustache @@ -31,7 +31,7 @@ import {{Service.Codec.PackageName}} import {{.}} {{/Service.Codec.SnippetImports}} -func sample(client: some {{Service.Codec.Name}}{{#SampleInfo}}{{#Codec.Parameters}}, {{.}}: String{{/Codec.Parameters}}{{/SampleInfo}}) async throws { +func sample(client: {{Service.Codec.ClientName}}{{#SampleInfo}}{{#Codec.Parameters}}, {{.}}: String{{/Codec.Parameters}}{{/SampleInfo}}) async throws { {{> /templates/snippet/generic}} } // snippet.hide @@ -40,7 +40,7 @@ func sample(client: some {{Service.Codec.Name}}{{#SampleInfo}}{{#Codec.Parameter struct SnippetRunner { static func main() async throws { do { - let client = try {{Service.Codec.PackageName}}.Clients.{{Service.Codec.ClientName}}() + let client = try {{Service.Codec.PackageName}}.{{Service.Codec.ClientName}}() try await sample(client: client{{#SampleInfo}}{{#Codec.Parameters}}, {{.}}: "[placeholder]"{{/Codec.Parameters}}{{/SampleInfo}}) } catch { print("Error: \(error)") diff --git a/internal/sidekick/swift/templates/common/placeholder.mustache b/internal/sidekick/swift/templates/common/placeholder.mustache index a061daba4bf..b99eff729f6 100644 --- a/internal/sidekick/swift/templates/common/placeholder.mustache +++ b/internal/sidekick/swift/templates/common/placeholder.mustache @@ -13,7 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. }} -extension Clients.{{Codec.PlaceholderName}} { +extension {{Codec.PlaceholderName}} { {{#Messages}} {{> /templates/common/message}} diff --git a/internal/sidekick/swift/templates/common/service.swift.mustache b/internal/sidekick/swift/templates/common/service.swift.mustache index 356ab1d990f..ca923d57675 100644 --- a/internal/sidekick/swift/templates/common/service.swift.mustache +++ b/internal/sidekick/swift/templates/common/service.swift.mustache @@ -38,234 +38,97 @@ import {{.}} {{/Codec.DocLines}} /// /// @Snippet(path: "{{Name}}Quickstart") -public protocol {{Codec.Name}} { +public class {{Codec.ClientName}}: Clients.{{Codec.StubPrefix}}Protocol { + let inner: any Clients.{{Codec.StubPrefix}}Stub + + /// Creates a new `{{Codec.ClientName}}` instance. + public init(_ options: GoogleCloudGax.ClientOptions = .init()) throws { + var inner: any Clients.{{Codec.StubPrefix}}Stub = try Clients.{{Codec.StubPrefix}}Transport(options) + inner = Clients.{{Codec.StubPrefix}}Retry(inner, options: options) + if let logger = options.logger { + inner = Clients.{{Codec.StubPrefix}}Logging(inner, logger: logger) + } + self.inner = inner + } {{#Codec.RestMethods}} + {{#Codec.DocLines}} /// {{{.}}} {{/Codec.DocLines}} /// /// @Snippet(path: "{{Service.Name}}_{{Name}}") - func {{> /templates/common/method_signature/request}} - {{#Codec.PlainRPC}} - {{#Signatures}} - {{#Codec.DocLines}} - /// {{{.}}} - {{/Codec.DocLines}} - func {{> /templates/common/method_signature/overload}} - {{/Signatures}} - {{/Codec.PlainRPC}} + public func {{> /templates/common/method_signature/request_options}} { + try await self.inner.{{Codec.Name}}(request: request, options: options) + } {{#Codec.Pagination}} - {{#Codec.DocLines}} - /// {{{.}}} - {{/Codec.DocLines}} - func {{> /templates/common/method_signature/pagination}} - {{#Signatures}} - {{#Codec.DocLines}} - /// {{{.}}} - {{/Codec.DocLines}} - func {{> /templates/common/method_signature/pagination_overload}} - {{/Signatures}} - {{/Codec.Pagination}} - {{#Codec.LRO}} - {{#Codec.DocLines}} - /// {{{.}}} - {{/Codec.DocLines}} - func {{> /templates/common/method_signature/lro}} - {{#Signatures}} - {{#Codec.DocLines}} - /// {{{.}}} - {{/Codec.DocLines}} - func {{> /templates/common/method_signature/lro_overload}} - {{/Signatures}} - {{/Codec.LRO}} - {{/Codec.RestMethods}} - {{#Codec.RestMethods}} {{#Codec.DocLines}} /// {{{.}}} {{/Codec.DocLines}} /// /// @Snippet(path: "{{Service.Name}}_{{Name}}") - func {{> /templates/common/method_signature/request_options}} - {{#Codec.Pagination}} - {{#Codec.DocLines}} - /// {{{.}}} - {{/Codec.DocLines}} - func {{> /templates/common/method_signature/pagination_options}} + public func {{> /templates/common/method_signature/pagination_options}} { + let listRpc = { (token: String) async throws -> {{Codec.ReturnType}} in + var request = byItem + request.pageToken = token + return try await self.{{Codec.Name}}(request: request, options: options) + } + return GoogleCloudGax.PaginatedResponseSequence(listRpc: listRpc) + } {{/Codec.Pagination}} {{#Codec.LRO}} + {{#Codec.DocLines}} /// {{{.}}} {{/Codec.DocLines}} - func {{> /templates/common/method_signature/lro_options}} - {{/Codec.LRO}} - {{/Codec.RestMethods}} -} - -extension Clients { - /// The recommended implementation for ``{{Codec.Name}}``. - public class {{Codec.ClientName}}: {{Codec.Name}} { - let inner: any {{Codec.StubPrefix}}Stub - - /// Creates a new `{{Codec.ClientName}}` instance. - public init(_ options: GoogleCloudGax.ClientOptions = .init()) throws { - var inner: any {{Codec.StubPrefix}}Stub = try {{Codec.StubPrefix}}Transport(options) - inner = {{Codec.StubPrefix}}Retry(inner, options: options) - if let logger = options.logger { - inner = {{Codec.StubPrefix}}Logging(inner, logger: logger) - } - self.inner = inner - } - {{#Codec.RestMethods}} - - /// See `{{Service.Codec.Name}}.{{Codec.Name}}` - public func {{> /templates/common/method_signature/request_options}} { - try await self.inner.{{Codec.Name}}(request: request, options: options) - } - {{#Codec.Pagination}} - - {{#Codec.DocLines}} - /// {{{.}}} - {{/Codec.DocLines}} - public func {{> /templates/common/method_signature/pagination_options}} { - let listRpc = { (token: String) async throws -> {{Codec.ReturnType}} in - var request = byItem - request.pageToken = token - return try await self.{{Codec.Name}}(request: request, options: options) + /// + /// @Snippet(path: "{{Service.Name}}_{{Name}}") + public func {{> /templates/common/method_signature/lro_options}} { + let extractStatus = { (op: {{Codec.ReturnType}}) throws -> GoogleCloudGax._PollableOperationImpl<{{Codec.LRO.ReturnType}}>.State in + guard op.done else { + return .init(done: false, result: nil) } - return GoogleCloudGax.PaginatedResponseSequence(listRpc: listRpc) - } - {{/Codec.Pagination}} - {{#Codec.LRO}} - {{#Codec.DocLines}} - /// {{{.}}} - {{/Codec.DocLines}} - public func {{> /templates/common/method_signature/lro_options}} { - let extractStatus = { (op: {{Codec.ReturnType}}) throws -> GoogleCloudGax._PollableOperationImpl<{{Codec.LRO.ReturnType}}>.State in - guard op.done else { - return .init(done: false, result: nil) + switch op.result { + {{#Codec.LRO.ResponseIsEmpty}} + case .response: + return .init(done: true, result: .success(())) + {{/Codec.LRO.ResponseIsEmpty}} + {{^Codec.LRO.ResponseIsEmpty}} + case .response(let anyValue): + guard let anyValueUnwrapped = anyValue else { + return .init(done: true, result: .failure(GoogleCloudGax.RequestError.binding("Operation completed but response value was missing"))) } - - switch op.result { - {{#Codec.LRO.ResponseIsEmpty}} - case .response: - return .init(done: true, result: .success(())) - {{/Codec.LRO.ResponseIsEmpty}} - {{^Codec.LRO.ResponseIsEmpty}} - case .response(let anyValue): - guard let anyValueUnwrapped = anyValue else { - return .init(done: true, result: .failure(GoogleCloudGax.RequestError.binding("Operation completed but response value was missing"))) - } - let response = try {{Codec.LRO.ReturnType}}(fromAny: anyValueUnwrapped) - return .init(done: true, result: .success(response)) - {{/Codec.LRO.ResponseIsEmpty}} - case .error(let status): - guard let statusUnwrapped = status else { - return .init(done: true, result: .failure(GoogleCloudGax.RequestError.binding("Operation completed but error value was missing"))) - } - let error = GoogleCloudGax.RequestError.service( - GoogleCloudGax.ServiceError(code: GoogleRpc.Code(intValue: Int(statusUnwrapped.code)), message: statusUnwrapped.message)) - return .init(done: true, result: .failure(error)) - case .none: - return .init( - done: true, - result: .failure( - GoogleCloudGax.RequestError.binding("Operation completed but result was missing"))) + let response = try {{Codec.LRO.ReturnType}}(fromAny: anyValueUnwrapped) + return .init(done: true, result: .success(response)) + {{/Codec.LRO.ResponseIsEmpty}} + case .error(let status): + guard let statusUnwrapped = status else { + return .init(done: true, result: .failure(GoogleCloudGax.RequestError.binding("Operation completed but error value was missing"))) } + let error = GoogleCloudGax.RequestError.service( + GoogleCloudGax.ServiceError(code: GoogleRpc.Code(intValue: Int(statusUnwrapped.code)), message: statusUnwrapped.message)) + return .init(done: true, result: .failure(error)) + case .none: + return .init( + done: true, + result: .failure( + GoogleCloudGax.RequestError.binding("Operation completed but result was missing"))) } - let rawOp = try await self.{{Codec.Name}}(request: withPolling, options: options) - let initialState = try extractStatus(rawOp) - let poll = { () async throws -> GoogleCloudGax._PollableOperationImpl<{{Codec.LRO.ReturnType}}>.State in - let op = try await self.getOperation(request: .init().with { $0.name = rawOp.name }, options: options) - return try extractStatus(op) - } - return GoogleCloudGax._PollableOperationImpl(initialState: initialState, poll: poll) - } - {{/Codec.LRO}} - {{/Codec.RestMethods}} - } -} - -// Default implementations -extension {{Codec.Name}} { - {{#Codec.RestMethods}} - - public func {{> /templates/common/method_signature/request}} { - try await self.{{Codec.Name}}(request: request, options: .init()) - } - - public func {{> /templates/common/method_signature/request_options}} { - throw GoogleCloudGax.RequestError.unimplemented - } - {{#Codec.PlainRPC}} - {{#Signatures}} - - public func {{> /templates/common/method_signature/overload}} { - let request = {{Method.InputType.Codec.ParameterTypeName}}().with { - {{#Fields}} - $0.{{Codec.Name}} = {{Codec.Name}} - {{/Fields}} - } - {{#Method.ReturnsEmpty}} - try await self.{{Method.Codec.Name}}(request: request) - {{/Method.ReturnsEmpty}} - {{^Method.ReturnsEmpty}} - return try await self.{{Method.Codec.Name}}(request: request) - {{/Method.ReturnsEmpty}} - } - {{/Signatures}} - {{/Codec.PlainRPC}} - {{#Codec.Pagination}} - - public func {{> /templates/common/method_signature/pagination}} { - try self.{{Codec.Name}}(byItem: byItem, options: .init()) - } - - public func {{> /templates/common/method_signature/pagination_options}} { - let listRpc = { (token: String) async throws -> {{Codec.ReturnType}} in - throw GoogleCloudGax.RequestError.unimplemented } - return GoogleCloudGax.PaginatedResponseSequence(listRpc: listRpc) - } - {{#Signatures}} - - public func {{> /templates/common/method_signature/pagination_overload}} { - let request = {{Method.InputType.Codec.ParameterTypeName}}().with { - {{#Fields}} - $0.{{Codec.Name}} = {{Codec.Name}} - {{/Fields}} - } - return try self.{{Codec.Name}}(byItem: request) - } - {{/Signatures}} - {{/Codec.Pagination}} - {{#Codec.LRO}} - - public func {{> /templates/common/method_signature/lro}} { - try await self.{{Codec.Name}}(withPolling: withPolling, options: .init()) - } - - public func {{> /templates/common/method_signature/lro_options}} { + let rawOp = try await self.{{Codec.Name}}(request: withPolling, options: options) + let initialState = try extractStatus(rawOp) let poll = { () async throws -> GoogleCloudGax._PollableOperationImpl<{{Codec.LRO.ReturnType}}>.State in - throw GoogleCloudGax.RequestError.unimplemented + let op = try await self.getOperation(request: .init().with { $0.name = rawOp.name }, options: options) + return try extractStatus(op) } - return GoogleCloudGax._PollableOperationImpl(initialState: .init(done: false, result: nil), poll: poll) + return GoogleCloudGax._PollableOperationImpl(initialState: initialState, poll: poll) } - {{#Signatures}} - - public func {{> /templates/common/method_signature/lro_overload}} { - let request = {{Method.InputType.Codec.ParameterTypeName}}().with { - {{#Fields}} - $0.{{Codec.Name}} = {{Codec.Name}} - {{/Fields}} - } - return try await self.{{Codec.Name}}(withPolling: request) - } - {{/Signatures}} {{/Codec.LRO}} {{/Codec.RestMethods}} } + +{{> /templates/common/client_protocol}} {{#Codec.IsGated}} #endif {{/Codec.IsGated}} From 8f81089ebb620e6255f64b0cf56ff87017206a51 Mon Sep 17 00:00:00 2001 From: Cody Oss <6331106+codyoss@users.noreply.github.com> Date: Mon, 29 Jun 2026 16:03:37 -0400 Subject: [PATCH 082/108] chore: remove legacylibrarian (#6572) The legacylibrarian part of this project is being turned down. This change deletes all associated code, including legacy automation, CLI commands, config checkers, and infrastructure build configurations. Internal Bug: b/518756763 --- .github/workflows/ci.yaml | 11 - .golangci.yaml | 3 - AGENTS.md | 1 - CONTRIBUTING.md | 2 +- cmd/legacyautomation/doc.go | 81 - cmd/legacyautomation/main.go | 29 - cmd/legacylibrarian/doc.go | 377 --- cmd/legacylibrarian/main.go | 31 - doc/legacylibrarian/config-schema.md | 61 - doc/legacylibrarian/language-onboarding.md | 381 --- .../library-maintainer-guide.md | 190 -- doc/legacylibrarian/onboarding.md | 133 - .../repository-library-onboarding.md | 33 - doc/legacylibrarian/state-schema.md | 57 - doc/legacylibrarian/testing.md | 49 - go.mod | 38 +- go.sum | 88 - infra/imagebuilders/cli/Dockerfile | 99 - .../cloudbuild-exitgate-dispatcher.yaml | 34 - ...cloudbuild-exitgate-release-container.yaml | 33 - infra/imagebuilders/cloudbuild-exitgate.yaml | 41 - infra/imagebuilders/cloudbuild-test.yaml | 56 - .../container-structure-test.yaml | 32 - infra/imagebuilders/container/Dockerfile | 27 - infra/imagebuilders/container/run.sh | 31 - infra/imagebuilders/dispatcher/Dockerfile | 55 - infra/prod/generate-worker.yaml | 27 - infra/prod/generate.yaml | 76 - infra/prod/publish-release-worker.yaml | 30 - infra/prod/publish-release.yaml | 46 - infra/prod/stage-release-worker.yaml | 26 - infra/prod/stage-release.yaml | 67 - infra/prod/update-image-worker.yaml | 27 - infra/prod/update-image.yaml | 72 - infra/test/librarian-image-test.yaml | 58 - infra/test/token-access-test.yaml | 57 - .../legacyautomation/automation.go | 89 - .../legacyautomation/automation_test.go | 24 - .../legacylibrarian/legacyautomation/cli.go | 70 - .../legacyautomation/cli_test.go | 137 - .../legacyautomation/cloudbuild.go | 78 - .../legacyautomation/cloudbuild_test.go | 225 -- .../legacylibrarian/legacyautomation/flags.go | 33 - .../legacyautomation/generate.go | 44 - .../legacyautomation/generate_test.go | 116 - .../legacylibrarian/legacyautomation/help.go | 26 - .../legacyautomation/prod/repositories.yaml | 38 - .../legacyautomation/publish_release.go | 39 - .../legacyautomation/publish_release_test.go | 95 - .../legacyautomation/repositories.go | 134 - .../legacyautomation/repositories_test.go | 361 --- .../legacyautomation/stage_release.go | 41 - .../legacyautomation/stage_release_test.go | 99 - .../legacyautomation/trigger.go | 144 - .../legacyautomation/trigger_test.go | 383 --- internal/legacylibrarian/legacycli/cli.go | 191 -- .../legacylibrarian/legacycli/cli_test.go | 379 --- internal/legacylibrarian/legacycli/version.go | 78 - .../legacylibrarian/legacycli/version.txt | 1 - .../legacylibrarian/legacycli/version_test.go | 100 - .../legacylibrarian/legacyconfig/config.go | 383 --- .../legacyconfig/config_test.go | 539 ---- .../legacyconfig/librarian_config.go | 105 - .../legacyconfig/librarian_config_test.go | 273 -- .../legacylibrarian/legacyconfig/state.go | 306 -- .../legacyconfig/state_test.go | 557 ---- .../legacyconfig/tag_format.go | 59 - .../legacyconfig/tag_format_test.go | 133 - .../legacycontainer/java/.gitignore | 1 - .../legacycontainer/java/Dockerfile | 65 - .../legacycontainer/java/README.md | 3 - .../legacycontainer/java/bazel/parser.go | 133 - .../legacycontainer/java/bazel/parser_test.go | 259 -- .../java/build-docker-and-test.sh | 32 - .../java/cloudbuild-exitgate.yaml | 46 - .../legacycontainer/java/execv/execv.go | 52 - .../legacycontainer/java/execv/execv_test.go | 78 - .../java/generate/generator.go | 306 -- .../java/generate/generator_test.go | 635 ----- .../languagecontainer/generate/generate.go | 87 - .../generate/generate_test.go | 113 - .../languagecontainer/languagecontainer.go | 149 - .../languagecontainer_test.go | 269 -- .../java/languagecontainer/release/release.go | 35 - .../languagecontainer/release/release_test.go | 76 - .../testdata/generate-request.json | 22 - .../testdata/release-stage-request.json | 37 - .../legacycontainer/java/main.go | 68 - .../legacycontainer/java/main_test.go | 66 - .../legacycontainer/java/message/message.go | 94 - .../java/message/message_test.go | 98 - .../legacycontainer/java/pom/pom.go | 264 -- .../legacycontainer/java/pom/pom_test.go | 147 - .../legacycontainer/java/pom/pom_update.go | 101 - .../java/pom/pom_update_test.go | 120 - .../java/pom/template/bom_pom.xml.tmpl | 34 - .../java/pom/template/cloud_pom.xml.tmpl | 134 - .../java/pom/template/grpc_pom.xml.tmpl | 46 - .../java/pom/template/parent_pom.xml.tmpl | 45 - .../java/pom/template/proto_pom.xml.tmpl | 38 - .../java/pom/testdata/happy_path_bom_pom.xml | 42 - .../pom/testdata/happy_path_cloud_pom.xml | 130 - .../java/pom/testdata/happy_path_grpc_pom.xml | 46 - .../pom/testdata/happy_path_parent_pom.xml | 53 - .../pom/testdata/happy_path_proto_pom.xml | 38 - .../java/pom/testdata/only_proto_bom_pom.xml | 37 - .../pom/testdata/only_proto_cloud_pom.xml | 125 - .../pom/testdata/only_proto_parent_pom.xml | 47 - .../pom/testdata/only_proto_proto_pom.xml | 38 - .../legacycontainer/java/protoc/protoc.go | 112 - .../java/protoc/protoc_test.go | 146 - .../legacycontainer/java/release/release.go | 45 - .../java/release/release_test.go | 118 - .../java-foo/google-cloud-foo/pom.xml | 7 - .../java/release/testdata/java-foo/pom.xml | 7 - .../java/run-generate-library.sh | 126 - .../generate/librarian/generate-request.json | 22 - .../secret-manager-generate-request.json | 28 - .../source/google/api/annotations.proto | 46 - .../cloud/secretmanager/v1beta2/BUILD.bazel | 67 - .../secretmanager/v1beta2/secretmanager.proto | 48 - .../v1beta2/secretmanager_v1beta2.yaml | 22 - .../google/cloud/workflows/v1/BUILD.bazel | 64 - .../google/cloud/workflows/v1/workflows.proto | 48 - .../cloud/workflows/v1/workflows_v1.yaml | 62 - .../legacylibrarian/legacydocker/docker.go | 478 ---- .../legacydocker/docker_test.go | 1159 -------- .../release-stage-request.json | 23 - .../empty-librarian-state.json | 4 - .../write-librarian-state-example.json | 39 - .../empty-library-state.json | 0 .../omit-empty-status.json | 19 - .../successful-marshaling-and-writing.json | 20 - .../legacylibrarian/legacygithub/github.go | 433 --- .../legacygithub/github_test.go | 1231 -------- .../legacygitrepo/conventional_commits.go | 417 --- .../conventional_commits_test.go | 903 ------ .../legacylibrarian/legacygitrepo/gitrepo.go | 780 ----- .../legacygitrepo/gitrepo_test.go | 2133 -------------- .../legacylibrarian/legacyimages/images.go | 167 -- .../legacyimages/images_test.go | 79 - .../legacylibrarian/legacyintegration/doc.go | 16 - .../legacyintegration/e2e_test.go | 909 ------ .../legacyintegration/system_test.go | 568 ---- .../testdata/e2e-test.Dockerfile | 36 - .../cloud/another-library/v3/another_v3.yaml | 14 - .../cloud/new-library-path/v2/library_v2.yaml | 14 - .../google/cloud/pubsub/v1/pubsub_v1.yaml | 14 - .../repo_init/.librarian/config.yaml | 14 - .../configure/repo_init/.librarian/state.yaml | 22 - .../google-cloud-pubsub/v1/example.txt | 0 .../testdata/e2e/configure/updated-state.yaml | 40 - .../google/cloud/future/v2/future_v2.yaml | 14 - .../google/cloud/pubsub/v1/pubsub_v1.yaml | 14 - .../.librarian/state.yaml | 27 - .../multi_repo_init/.librarian/state.yaml | 27 - .../.librarian/state.yaml | 27 - .../generate/repo_init/.librarian/config.yaml | 14 - .../generate/repo_init/.librarian/state.yaml | 31 - .../google-cloud-future/v2/example.txt | 0 .../google-cloud-pubsub/v1/example.txt | 0 .../stage/multiple_commits/CHANGELOG.md | 19 - .../stage/multiple_commits/commit_msg.txt | 38 - .../repo_init/.librarian/state.yaml | 22 - .../repo_init/google-cloud-pubsub/v1/dummy.go | 15 - .../release/stage/multiple_commits/state.yaml | 22 - .../multiple_nested_commits/CHANGELOG.md | 9 - .../multiple_nested_commits/commit_msg.txt | 111 - .../repo_init/.librarian/state.yaml | 24 - .../stage/multiple_nested_commits/state.yaml | 24 - .../release/stage/single_commit/CHANGELOG.md | 3 - .../stage/single_commit/commit_msg.txt | 5 - .../repo_init/.librarian/state.yaml | 23 - .../release/stage/single_commit/state.yaml | 23 - .../legacyintegration/testdata/e2e_func.go | 553 ---- .../legacylibrarian/action_test.go | 94 - .../legacylibrarian/legacylibrarian/build.go | 59 - .../legacylibrarian/build_test.go | 146 - .../legacylibrarian/command.go | 816 ------ .../legacylibrarian/command_test.go | 2370 --------------- .../commit_version_analyzer.go | 203 -- .../commit_version_analyzer_test.go | 806 ------ .../legacylibrarian/legacylibrarian/flags.go | 158 - .../legacylibrarian/gapic_metadata.go | 199 -- .../legacylibrarian/gapic_metadata_test.go | 451 --- .../legacylibrarian/generate.go | 92 - .../legacylibrarian/generate_command.go | 515 ---- .../legacylibrarian/generate_command_test.go | 1705 ----------- .../legacylibrarian/generate_pull_request.go | 278 -- .../generate_pull_request_test.go | 925 ------ .../legacylibrarian/generate_test.go | 122 - .../legacylibrarian/legacylibrarian/help.go | 151 - .../legacylibrarian/librarian.go | 213 -- .../legacylibrarian/librarian_test.go | 386 --- .../legacylibrarian/mocks_test.go | 591 ---- .../legacylibrarian/release_notes.go | 341 --- .../legacylibrarian/release_notes_test.go | 986 ------- .../legacylibrarian/release_stage.go | 420 --- .../legacylibrarian/release_stage_test.go | 2529 ----------------- .../legacylibrarian/repository.go | 56 - .../legacylibrarian/repository_mock.go | 48 - .../legacylibrarian/repository_test.go | 142 - .../legacylibrarian/legacylibrarian/state.go | 238 -- .../legacylibrarian/state_test.go | 553 ---- .../legacylibrarian/tag_and_release.go | 397 --- .../legacylibrarian/tag_test.go | 848 ------ .../test_container_generate.go | 447 --- .../test_container_generate_test.go | 712 ----- .../find_a_service_config/another_config.yaml | 14 - .../find_a_service_config/random_file.txt | 0 .../find_a_service_config/service_config.yaml | 15 - .../testdata/invalid_yaml/invalid_yaml.yaml | 14 - .../no_service_config/some_config.yaml | 14 - .../example/api/example_api_config.yaml | 15 - .../invalid-global-config.yaml | 16 - .../successful-parsing-config.yaml | 20 - .../empty-libraryState.json | 1 - .../successful-unmarshal-libraryState.json | 19 - ...unmarshal-libraryState-with-error-msg.json | 14 - .../legacylibrarian/update_image.go | 295 -- .../legacylibrarian/update_image_test.go | 1012 ------- 221 files changed, 2 insertions(+), 42937 deletions(-) delete mode 100644 cmd/legacyautomation/doc.go delete mode 100644 cmd/legacyautomation/main.go delete mode 100644 cmd/legacylibrarian/doc.go delete mode 100644 cmd/legacylibrarian/main.go delete mode 100644 doc/legacylibrarian/config-schema.md delete mode 100644 doc/legacylibrarian/language-onboarding.md delete mode 100644 doc/legacylibrarian/library-maintainer-guide.md delete mode 100644 doc/legacylibrarian/onboarding.md delete mode 100644 doc/legacylibrarian/repository-library-onboarding.md delete mode 100644 doc/legacylibrarian/state-schema.md delete mode 100644 doc/legacylibrarian/testing.md delete mode 100644 infra/imagebuilders/cli/Dockerfile delete mode 100644 infra/imagebuilders/cloudbuild-exitgate-dispatcher.yaml delete mode 100644 infra/imagebuilders/cloudbuild-exitgate-release-container.yaml delete mode 100644 infra/imagebuilders/cloudbuild-exitgate.yaml delete mode 100644 infra/imagebuilders/cloudbuild-test.yaml delete mode 100644 infra/imagebuilders/container-structure-test.yaml delete mode 100644 infra/imagebuilders/container/Dockerfile delete mode 100755 infra/imagebuilders/container/run.sh delete mode 100644 infra/imagebuilders/dispatcher/Dockerfile delete mode 100644 infra/prod/generate-worker.yaml delete mode 100644 infra/prod/generate.yaml delete mode 100644 infra/prod/publish-release-worker.yaml delete mode 100644 infra/prod/publish-release.yaml delete mode 100644 infra/prod/stage-release-worker.yaml delete mode 100644 infra/prod/stage-release.yaml delete mode 100644 infra/prod/update-image-worker.yaml delete mode 100644 infra/prod/update-image.yaml delete mode 100644 infra/test/librarian-image-test.yaml delete mode 100644 infra/test/token-access-test.yaml delete mode 100644 internal/legacylibrarian/legacyautomation/automation.go delete mode 100644 internal/legacylibrarian/legacyautomation/automation_test.go delete mode 100644 internal/legacylibrarian/legacyautomation/cli.go delete mode 100644 internal/legacylibrarian/legacyautomation/cli_test.go delete mode 100644 internal/legacylibrarian/legacyautomation/cloudbuild.go delete mode 100644 internal/legacylibrarian/legacyautomation/cloudbuild_test.go delete mode 100644 internal/legacylibrarian/legacyautomation/flags.go delete mode 100644 internal/legacylibrarian/legacyautomation/generate.go delete mode 100644 internal/legacylibrarian/legacyautomation/generate_test.go delete mode 100644 internal/legacylibrarian/legacyautomation/help.go delete mode 100644 internal/legacylibrarian/legacyautomation/prod/repositories.yaml delete mode 100644 internal/legacylibrarian/legacyautomation/publish_release.go delete mode 100644 internal/legacylibrarian/legacyautomation/publish_release_test.go delete mode 100644 internal/legacylibrarian/legacyautomation/repositories.go delete mode 100644 internal/legacylibrarian/legacyautomation/repositories_test.go delete mode 100644 internal/legacylibrarian/legacyautomation/stage_release.go delete mode 100644 internal/legacylibrarian/legacyautomation/stage_release_test.go delete mode 100644 internal/legacylibrarian/legacyautomation/trigger.go delete mode 100644 internal/legacylibrarian/legacyautomation/trigger_test.go delete mode 100644 internal/legacylibrarian/legacycli/cli.go delete mode 100644 internal/legacylibrarian/legacycli/cli_test.go delete mode 100644 internal/legacylibrarian/legacycli/version.go delete mode 100644 internal/legacylibrarian/legacycli/version.txt delete mode 100644 internal/legacylibrarian/legacycli/version_test.go delete mode 100644 internal/legacylibrarian/legacyconfig/config.go delete mode 100644 internal/legacylibrarian/legacyconfig/config_test.go delete mode 100644 internal/legacylibrarian/legacyconfig/librarian_config.go delete mode 100644 internal/legacylibrarian/legacyconfig/librarian_config_test.go delete mode 100644 internal/legacylibrarian/legacyconfig/state.go delete mode 100644 internal/legacylibrarian/legacyconfig/state_test.go delete mode 100644 internal/legacylibrarian/legacyconfig/tag_format.go delete mode 100644 internal/legacylibrarian/legacyconfig/tag_format_test.go delete mode 100644 internal/legacylibrarian/legacycontainer/java/.gitignore delete mode 100644 internal/legacylibrarian/legacycontainer/java/Dockerfile delete mode 100644 internal/legacylibrarian/legacycontainer/java/README.md delete mode 100644 internal/legacylibrarian/legacycontainer/java/bazel/parser.go delete mode 100644 internal/legacylibrarian/legacycontainer/java/bazel/parser_test.go delete mode 100755 internal/legacylibrarian/legacycontainer/java/build-docker-and-test.sh delete mode 100644 internal/legacylibrarian/legacycontainer/java/cloudbuild-exitgate.yaml delete mode 100644 internal/legacylibrarian/legacycontainer/java/execv/execv.go delete mode 100644 internal/legacylibrarian/legacycontainer/java/execv/execv_test.go delete mode 100644 internal/legacylibrarian/legacycontainer/java/generate/generator.go delete mode 100644 internal/legacylibrarian/legacycontainer/java/generate/generator_test.go delete mode 100644 internal/legacylibrarian/legacycontainer/java/languagecontainer/generate/generate.go delete mode 100644 internal/legacylibrarian/legacycontainer/java/languagecontainer/generate/generate_test.go delete mode 100644 internal/legacylibrarian/legacycontainer/java/languagecontainer/languagecontainer.go delete mode 100644 internal/legacylibrarian/legacycontainer/java/languagecontainer/languagecontainer_test.go delete mode 100644 internal/legacylibrarian/legacycontainer/java/languagecontainer/release/release.go delete mode 100644 internal/legacylibrarian/legacycontainer/java/languagecontainer/release/release_test.go delete mode 100644 internal/legacylibrarian/legacycontainer/java/languagecontainer/testdata/generate-request.json delete mode 100644 internal/legacylibrarian/legacycontainer/java/languagecontainer/testdata/release-stage-request.json delete mode 100644 internal/legacylibrarian/legacycontainer/java/main.go delete mode 100644 internal/legacylibrarian/legacycontainer/java/main_test.go delete mode 100644 internal/legacylibrarian/legacycontainer/java/message/message.go delete mode 100644 internal/legacylibrarian/legacycontainer/java/message/message_test.go delete mode 100644 internal/legacylibrarian/legacycontainer/java/pom/pom.go delete mode 100644 internal/legacylibrarian/legacycontainer/java/pom/pom_test.go delete mode 100644 internal/legacylibrarian/legacycontainer/java/pom/pom_update.go delete mode 100644 internal/legacylibrarian/legacycontainer/java/pom/pom_update_test.go delete mode 100644 internal/legacylibrarian/legacycontainer/java/pom/template/bom_pom.xml.tmpl delete mode 100644 internal/legacylibrarian/legacycontainer/java/pom/template/cloud_pom.xml.tmpl delete mode 100644 internal/legacylibrarian/legacycontainer/java/pom/template/grpc_pom.xml.tmpl delete mode 100644 internal/legacylibrarian/legacycontainer/java/pom/template/parent_pom.xml.tmpl delete mode 100644 internal/legacylibrarian/legacycontainer/java/pom/template/proto_pom.xml.tmpl delete mode 100644 internal/legacylibrarian/legacycontainer/java/pom/testdata/happy_path_bom_pom.xml delete mode 100644 internal/legacylibrarian/legacycontainer/java/pom/testdata/happy_path_cloud_pom.xml delete mode 100644 internal/legacylibrarian/legacycontainer/java/pom/testdata/happy_path_grpc_pom.xml delete mode 100644 internal/legacylibrarian/legacycontainer/java/pom/testdata/happy_path_parent_pom.xml delete mode 100644 internal/legacylibrarian/legacycontainer/java/pom/testdata/happy_path_proto_pom.xml delete mode 100644 internal/legacylibrarian/legacycontainer/java/pom/testdata/only_proto_bom_pom.xml delete mode 100644 internal/legacylibrarian/legacycontainer/java/pom/testdata/only_proto_cloud_pom.xml delete mode 100644 internal/legacylibrarian/legacycontainer/java/pom/testdata/only_proto_parent_pom.xml delete mode 100644 internal/legacylibrarian/legacycontainer/java/pom/testdata/only_proto_proto_pom.xml delete mode 100644 internal/legacylibrarian/legacycontainer/java/protoc/protoc.go delete mode 100644 internal/legacylibrarian/legacycontainer/java/protoc/protoc_test.go delete mode 100644 internal/legacylibrarian/legacycontainer/java/release/release.go delete mode 100644 internal/legacylibrarian/legacycontainer/java/release/release_test.go delete mode 100644 internal/legacylibrarian/legacycontainer/java/release/testdata/java-foo/google-cloud-foo/pom.xml delete mode 100644 internal/legacylibrarian/legacycontainer/java/release/testdata/java-foo/pom.xml delete mode 100755 internal/legacylibrarian/legacycontainer/java/run-generate-library.sh delete mode 100644 internal/legacylibrarian/legacycontainer/java/testdata/generate/librarian/generate-request.json delete mode 100644 internal/legacylibrarian/legacycontainer/java/testdata/generate/librarian/secret-manager-generate-request.json delete mode 100644 internal/legacylibrarian/legacycontainer/java/testdata/generate/source/google/api/annotations.proto delete mode 100644 internal/legacylibrarian/legacycontainer/java/testdata/generate/source/google/cloud/secretmanager/v1beta2/BUILD.bazel delete mode 100644 internal/legacylibrarian/legacycontainer/java/testdata/generate/source/google/cloud/secretmanager/v1beta2/secretmanager.proto delete mode 100644 internal/legacylibrarian/legacycontainer/java/testdata/generate/source/google/cloud/secretmanager/v1beta2/secretmanager_v1beta2.yaml delete mode 100644 internal/legacylibrarian/legacycontainer/java/testdata/generate/source/google/cloud/workflows/v1/BUILD.bazel delete mode 100644 internal/legacylibrarian/legacycontainer/java/testdata/generate/source/google/cloud/workflows/v1/workflows.proto delete mode 100644 internal/legacylibrarian/legacycontainer/java/testdata/generate/source/google/cloud/workflows/v1/workflows_v1.yaml delete mode 100644 internal/legacylibrarian/legacydocker/docker.go delete mode 100644 internal/legacylibrarian/legacydocker/docker_test.go delete mode 100644 internal/legacylibrarian/legacydocker/testdata/release-stage-request/release-stage-request.json delete mode 100644 internal/legacylibrarian/legacydocker/testdata/test-write-librarian-state/empty-librarian-state.json delete mode 100644 internal/legacylibrarian/legacydocker/testdata/test-write-librarian-state/write-librarian-state-example.json delete mode 100755 internal/legacylibrarian/legacydocker/testdata/test-write-library-state/empty-library-state.json delete mode 100644 internal/legacylibrarian/legacydocker/testdata/test-write-library-state/omit-empty-status.json delete mode 100644 internal/legacylibrarian/legacydocker/testdata/test-write-library-state/successful-marshaling-and-writing.json delete mode 100644 internal/legacylibrarian/legacygithub/github.go delete mode 100644 internal/legacylibrarian/legacygithub/github_test.go delete mode 100644 internal/legacylibrarian/legacygitrepo/conventional_commits.go delete mode 100644 internal/legacylibrarian/legacygitrepo/conventional_commits_test.go delete mode 100644 internal/legacylibrarian/legacygitrepo/gitrepo.go delete mode 100644 internal/legacylibrarian/legacygitrepo/gitrepo_test.go delete mode 100644 internal/legacylibrarian/legacyimages/images.go delete mode 100644 internal/legacylibrarian/legacyimages/images_test.go delete mode 100644 internal/legacylibrarian/legacyintegration/doc.go delete mode 100644 internal/legacylibrarian/legacyintegration/e2e_test.go delete mode 100644 internal/legacylibrarian/legacyintegration/system_test.go delete mode 100644 internal/legacylibrarian/legacyintegration/testdata/e2e-test.Dockerfile delete mode 100644 internal/legacylibrarian/legacyintegration/testdata/e2e/configure/api_root/google/cloud/another-library/v3/another_v3.yaml delete mode 100644 internal/legacylibrarian/legacyintegration/testdata/e2e/configure/api_root/google/cloud/new-library-path/v2/library_v2.yaml delete mode 100644 internal/legacylibrarian/legacyintegration/testdata/e2e/configure/api_root/google/cloud/pubsub/v1/pubsub_v1.yaml delete mode 100644 internal/legacylibrarian/legacyintegration/testdata/e2e/configure/repo_init/.librarian/config.yaml delete mode 100644 internal/legacylibrarian/legacyintegration/testdata/e2e/configure/repo_init/.librarian/state.yaml delete mode 100644 internal/legacylibrarian/legacyintegration/testdata/e2e/configure/repo_init/google-cloud-pubsub/v1/example.txt delete mode 100644 internal/legacylibrarian/legacyintegration/testdata/e2e/configure/updated-state.yaml delete mode 100644 internal/legacylibrarian/legacyintegration/testdata/e2e/generate/api_root/google/cloud/future/v2/future_v2.yaml delete mode 100644 internal/legacylibrarian/legacyintegration/testdata/e2e/generate/api_root/google/cloud/pubsub/v1/pubsub_v1.yaml delete mode 100644 internal/legacylibrarian/legacyintegration/testdata/e2e/generate/multi_repo_all_fail_init/.librarian/state.yaml delete mode 100644 internal/legacylibrarian/legacyintegration/testdata/e2e/generate/multi_repo_init/.librarian/state.yaml delete mode 100644 internal/legacylibrarian/legacyintegration/testdata/e2e/generate/multi_repo_one_fails_init/.librarian/state.yaml delete mode 100644 internal/legacylibrarian/legacyintegration/testdata/e2e/generate/repo_init/.librarian/config.yaml delete mode 100644 internal/legacylibrarian/legacyintegration/testdata/e2e/generate/repo_init/.librarian/state.yaml delete mode 100644 internal/legacylibrarian/legacyintegration/testdata/e2e/generate/repo_init/google-cloud-future/v2/example.txt delete mode 100644 internal/legacylibrarian/legacyintegration/testdata/e2e/generate/repo_init/google-cloud-pubsub/v1/example.txt delete mode 100644 internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/multiple_commits/CHANGELOG.md delete mode 100644 internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/multiple_commits/commit_msg.txt delete mode 100644 internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/multiple_commits/repo_init/.librarian/state.yaml delete mode 100644 internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/multiple_commits/repo_init/google-cloud-pubsub/v1/dummy.go delete mode 100644 internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/multiple_commits/state.yaml delete mode 100644 internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/multiple_nested_commits/CHANGELOG.md delete mode 100644 internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/multiple_nested_commits/commit_msg.txt delete mode 100644 internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/multiple_nested_commits/repo_init/.librarian/state.yaml delete mode 100644 internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/multiple_nested_commits/state.yaml delete mode 100644 internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/single_commit/CHANGELOG.md delete mode 100644 internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/single_commit/commit_msg.txt delete mode 100644 internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/single_commit/repo_init/.librarian/state.yaml delete mode 100644 internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/single_commit/state.yaml delete mode 100644 internal/legacylibrarian/legacyintegration/testdata/e2e_func.go delete mode 100644 internal/legacylibrarian/legacylibrarian/action_test.go delete mode 100644 internal/legacylibrarian/legacylibrarian/build.go delete mode 100644 internal/legacylibrarian/legacylibrarian/build_test.go delete mode 100644 internal/legacylibrarian/legacylibrarian/command.go delete mode 100644 internal/legacylibrarian/legacylibrarian/command_test.go delete mode 100644 internal/legacylibrarian/legacylibrarian/commit_version_analyzer.go delete mode 100644 internal/legacylibrarian/legacylibrarian/commit_version_analyzer_test.go delete mode 100644 internal/legacylibrarian/legacylibrarian/flags.go delete mode 100644 internal/legacylibrarian/legacylibrarian/gapic_metadata.go delete mode 100644 internal/legacylibrarian/legacylibrarian/gapic_metadata_test.go delete mode 100644 internal/legacylibrarian/legacylibrarian/generate.go delete mode 100644 internal/legacylibrarian/legacylibrarian/generate_command.go delete mode 100644 internal/legacylibrarian/legacylibrarian/generate_command_test.go delete mode 100644 internal/legacylibrarian/legacylibrarian/generate_pull_request.go delete mode 100644 internal/legacylibrarian/legacylibrarian/generate_pull_request_test.go delete mode 100644 internal/legacylibrarian/legacylibrarian/generate_test.go delete mode 100644 internal/legacylibrarian/legacylibrarian/help.go delete mode 100644 internal/legacylibrarian/legacylibrarian/librarian.go delete mode 100644 internal/legacylibrarian/legacylibrarian/librarian_test.go delete mode 100644 internal/legacylibrarian/legacylibrarian/mocks_test.go delete mode 100644 internal/legacylibrarian/legacylibrarian/release_notes.go delete mode 100644 internal/legacylibrarian/legacylibrarian/release_notes_test.go delete mode 100644 internal/legacylibrarian/legacylibrarian/release_stage.go delete mode 100644 internal/legacylibrarian/legacylibrarian/release_stage_test.go delete mode 100644 internal/legacylibrarian/legacylibrarian/repository.go delete mode 100644 internal/legacylibrarian/legacylibrarian/repository_mock.go delete mode 100644 internal/legacylibrarian/legacylibrarian/repository_test.go delete mode 100644 internal/legacylibrarian/legacylibrarian/state.go delete mode 100644 internal/legacylibrarian/legacylibrarian/state_test.go delete mode 100644 internal/legacylibrarian/legacylibrarian/tag_and_release.go delete mode 100644 internal/legacylibrarian/legacylibrarian/tag_test.go delete mode 100644 internal/legacylibrarian/legacylibrarian/test_container_generate.go delete mode 100644 internal/legacylibrarian/legacylibrarian/test_container_generate_test.go delete mode 100644 internal/legacylibrarian/legacylibrarian/testdata/find_a_service_config/another_config.yaml delete mode 100644 internal/legacylibrarian/legacylibrarian/testdata/find_a_service_config/random_file.txt delete mode 100644 internal/legacylibrarian/legacylibrarian/testdata/find_a_service_config/service_config.yaml delete mode 100644 internal/legacylibrarian/legacylibrarian/testdata/invalid_yaml/invalid_yaml.yaml delete mode 100644 internal/legacylibrarian/legacylibrarian/testdata/no_service_config/some_config.yaml delete mode 100644 internal/legacylibrarian/legacylibrarian/testdata/populate_service_config/example/api/example_api_config.yaml delete mode 100644 internal/legacylibrarian/legacylibrarian/testdata/test-parse-global-config/invalid-global-config.yaml delete mode 100644 internal/legacylibrarian/legacylibrarian/testdata/test-parse-global-config/successful-parsing-config.yaml delete mode 100644 internal/legacylibrarian/legacylibrarian/testdata/test-read-library-state/empty-libraryState.json delete mode 100644 internal/legacylibrarian/legacylibrarian/testdata/test-read-library-state/successful-unmarshal-libraryState.json delete mode 100644 internal/legacylibrarian/legacylibrarian/testdata/test-read-library-state/unmarshal-libraryState-with-error-msg.json delete mode 100644 internal/legacylibrarian/legacylibrarian/update_image.go delete mode 100644 internal/legacylibrarian/legacylibrarian/update_image_test.go diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1d145649899..433672ac8df 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -16,17 +16,6 @@ on: [push, pull_request, merge_group] permissions: contents: read jobs: - legacylibrarian: - runs-on: ubuntu-24.04 - # Presubmit jobs must complete within 5 minutes. See CONTRIBUTING.md. - timeout-minutes: 5 - steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - with: - persist-credentials: false - - uses: ./.github/actions/setup-librarian - - name: Run tests and check coverage - run: go run ./tool/cmd/coverage ./internal/legacylibrarian/... test: runs-on: ubuntu-24.04 # Presubmit jobs must complete within 5 minutes. See CONTRIBUTING.md. diff --git a/.golangci.yaml b/.golangci.yaml index 51bc186f46a..c6a14c7818e 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -38,9 +38,6 @@ linters: length: 120 exclusions: rules: - - path: internal/legacylibrarian/ - linters: - - godoclint - path: _test\.go$ linters: - godoclint diff --git a/AGENTS.md b/AGENTS.md index f8c28b30ac2..9abead90165 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -27,7 +27,6 @@ Before submitting changes, run the full test suite: ## Codebase Map -- `**/legacylibrarian/`: **STRICT IGNORE.** Never read or edit this legacy code, unless explicitly requested to do so. - `go.mod`: **NO NEW DEPENDENCIES.** Use only what is already available. - `cmd/`: Main entrypoint to CLI commands. - `internal/command`: Use `command.Run` for execution. `os/exec` is permitted for other tasks. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fe417d45903..c2eab2d949b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -60,7 +60,7 @@ we operate with a limited contribution model. ## Becoming a contributor Review the guide for -[Onboarding to Librarian](/doc/legacylibrarian/onboarding.md). +[How to Generate Client Libraries with Librarian](/doc/developer-guide.md). To contribute to this repository, ask a member of [cloud-sdk-librarian-admin](https://github.com/orgs/googleapis/teams/cloud-sdk-librarian-admin) diff --git a/cmd/legacyautomation/doc.go b/cmd/legacyautomation/doc.go deleted file mode 100644 index 15d989f4fe1..00000000000 --- a/cmd/legacyautomation/doc.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* -Automation provides logic to trigger Cloud Build jobs that run Librarian commands for -any repository listed in internal/legacylibrarian/legacyautomation/prod/repositories.yaml. - -Usage: - - automation [arguments] - -The commands are: - -# generate - -The generate command triggers a Cloud Build job that runs librarian generate command for every -repository onboarded to Librarian generate automation. - -Usage: - - automation generate [flags] - -Flags: - - -build - The _BUILD flag (true/false) to Librarian CLI's -build option - -project string - Google Cloud Platform project ID (default "cloud-sdk-librarian-prod") - -push - The _PUSH flag (true/false) to Librarian CLI's -push option - -# publish-release - -The publish-release command triggers a Cloud Build job that runs librarian release tag command -for every repository onboarded to Librarian publish-release automation. - -Usage: - - automation publish-release [flags] - -Flags: - - -project string - Google Cloud Platform project ID (default "cloud-sdk-librarian-prod") - -# stage-release - -The stage-release command triggers a Cloud Build job that runs librarian release stage command for -every repository onboarded to Librarian stage-release automation. - -Usage: - - automation stage-release [flags] - -Flags: - - -project string - Google Cloud Platform project ID (default "cloud-sdk-librarian-prod") - -push - The _PUSH flag (true/false) to Librarian CLI's -push option - -# version - -Version prints version information for the automation binary. - -Usage: - - automation version -*/ -package main diff --git a/cmd/legacyautomation/main.go b/cmd/legacyautomation/main.go deleted file mode 100644 index 8e12f22e767..00000000000 --- a/cmd/legacyautomation/main.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "context" - "log" - "os" - - "github.com/googleapis/librarian/internal/legacylibrarian/legacyautomation" -) - -func main() { - if err := legacyautomation.Run(context.Background(), os.Args[1:]); err != nil { - log.Fatal(err) - } -} diff --git a/cmd/legacylibrarian/doc.go b/cmd/legacylibrarian/doc.go deleted file mode 100644 index 8ca33d70ef7..00000000000 --- a/cmd/legacylibrarian/doc.go +++ /dev/null @@ -1,377 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* -Librarian manages Google API client libraries by automating onboarding, -regeneration, and release. It runs language-agnostic workflows while -delegating language-specific tasks—such as code generation, building, and -testing—to Docker images. - -Usage: - - librarian [arguments] - -The commands are: - -# generate - -The generate command is the primary tool for all code generation -tasks. It handles both the initial setup of a new library (onboarding) and the -regeneration of existing ones. Librarian works by delegating language-specific -tasks to a container, which is configured in the .librarian/state.yaml file. -Librarian is environment aware and will check if the current directory is the -root of a librarian repository. If you are not executing in such a directory the -'-repo' flag must be provided. - -# Onboarding a new library - -To configure and generate a new library for the first time, you must specify the -API to be generated and the library it will belong to. Librarian will invoke the -'configure' command in the language container to set up the repository, add the -new library's configuration to the '.librarian/state.yaml' file, and then -proceed with generation. - -Example: - - legacylibrarian generate -library=secretmanager -api=google/cloud/secretmanager/v1 - -# Regenerating existing libraries - -You can regenerate a single, existing library by specifying either the library -ID or the API path. If no specific library or API is provided, Librarian will -regenerate all libraries listed in '.librarian/state.yaml'. If '-library' or -'-api' is specified the whole library will be regenerated. - -Examples: - - # Regenerate a single library by its ID - legacylibrarian generate -library=secretmanager - - # Regenerate a single library by its API path - legacylibrarian generate -api=google/cloud/secretmanager/v1 - - # Regenerate all libraries in the repository - legacylibrarian generate - -# Workflow and Options: - -The generation process involves delegating to the language container's -'generate' command. After the code is generated, the tool cleans the destination -directories and copies the new files into place, according to the configuration -in '.librarian/state.yaml'. - - - If the '-build' flag is specified, the 'build' command is also executed in - the container to compile and validate the generated code. - - If the '-push' flag is provided, the changes are committed to a new branch, - and a pull request is created on GitHub. Otherwise, the changes are left in - your local working directory for inspection. When pushing to a remote branch, - you have the option of using HTTPS or SSH. Librarian will automatically determine - whether to use HTTPS or SSH based on the remote URI. - -Example with build and push: - - LIBRARIAN_GITHUB_TOKEN=xxx legacylibrarian generate -push -build - -Usage: - - legacylibrarian generate [flags] - -Flags: - - -api string - Relative path to the API to be configured/generated (e.g., google/cloud/functions/v2). - Must be specified when generating a new library. - -api-source string - The location of an API specification repository. - Can be a remote URL or a local file path. (default "https://github.com/googleapis/googleapis") - -api-source-branch string - The target branch of the API specification repository to checkout. - Can only be used with a remote -api-source. (default "master") - -branch string - The branch to use with remote code repositories. It is ignored if - you are using a local repository. This is used to specify which branch to clone - and which branch to use as the base for a pull request. (default "main") - -build - If true, Librarian will build each generated library by invoking the - language-specific container. - -generate-unchanged - If true, librarian generates libraries even if none of their associated APIs - have changed. This does not override generation being blocked by configuration. - -host-mount string - For use when librarian is running in a container. A mapping of a - directory from the host to the container, in the format - :. - -image string - Language specific image used to invoke code generation and releasing. - If not specified, the image configured in the state.yaml is used. - -library string - The library ID to generate or release (e.g. secretmanager). - This corresponds to a releasable language unit. - -output string - Working directory root. When this is not specified, a working directory - will be created in /tmp. - -push - If true, Librarian will create a commit, - push and create a pull request for the changes. - A GitHub token with push access must be provided via the - LIBRARIAN_GITHUB_TOKEN environment variable. - -repo string - Code repository where the generated code will reside. Can be a remote - in the format of a remote URL such as https://github.com/{owner}/{repo} or a - local file path like /path/to/repo. Both absolute and relative paths are - supported. If not specified, will try to detect if the current working directory - is configured as a language repository. - Note: When using a local repository (either by providing a path or by defaulting - to the current directory), Librarian creates a new branch from the currently checked-out - branch and commits changes. If the --push flag is also specified, a pull request is - created against the main branch. The --branch flag is ignored for local repositories. - -v enables verbose logging - -# release - -Manages releases of libraries. - -Usage: - - legacylibrarian release [arguments] - -Commands: - - stage stages a release by creating a release pull request. - tag tags and creates a GitHub release for a merged pull request. - -# release stage - -The 'release stage' command is the primary entry point for staging -a new release. It automates the creation of a release pull request by parsing -conventional commits, determining the next semantic version for each library, -and generating a changelog. Librarian is environment aware and will check if the -current directory is the root of a librarian repository. If you are not -executing in such a directory the '-repo' flag must be provided. - -This command scans the git history since the last release, identifies changes -(feat, fix, BREAKING CHANGE), and calculates the appropriate version bump -according to semver rules. It then delegates all language-specific file -modifications, such as updating a CHANGELOG.md or bumping the version in a pom.xml, -to the configured language-specific container. - -If a specific library is configured for release via the '-library' flag, a single -releasable change is needed to automatically calculate a version bump. If there are -no releasable changes since the last release, the '-version' flag should be included -to set a new version for the library. The new version must be "SemVer" greater than the -current version. - -By default, 'release stage' leaves the changes in your local working directory -for inspection. Use the '-push' flag to automatically commit the changes to -a new branch and create a pull request on GitHub. The '-commit' flag may be -used to create a local commit without creating a pull request; this flag is -ignored if '-push' is also specified. When pushing to a remote branch, -you have the option of using HTTPS or SSH. Librarian will automatically determine -whether to use HTTPS or SSH based on the remote URI. - -Examples: - - # Create a release PR for all libraries with pending changes. - legacylibrarian release stage -push - - # Create a release PR for a single library. - legacylibrarian release stage -library=secretmanager -push - - # Manually specify a version for a single library, overriding the calculation. - legacylibrarian release stage -library=secretmanager -library-version=2.0.0 -push - -Usage: - - legacylibrarian release stage [flags] - -Flags: - - -branch string - The branch to use with remote code repositories. It is ignored if - you are using a local repository. This is used to specify which branch to clone - and which branch to use as the base for a pull request. (default "main") - -commit - If true, librarian will create a commit for the change but not create - a pull request. This flag is ignored if push is set to true. - -image string - Language specific image used to invoke code generation and releasing. - If not specified, the image configured in the state.yaml is used. - -library string - The library ID to generate or release (e.g. secretmanager). - This corresponds to a releasable language unit. - -library-version string - Overrides the automatic semantic version calculation and forces a specific - version for a library. Requires the --library flag to be specified. - -output string - Working directory root. When this is not specified, a working directory - will be created in /tmp. - -push - If true, Librarian will create a commit, - push and create a pull request for the changes. - A GitHub token with push access must be provided via the - LIBRARIAN_GITHUB_TOKEN environment variable. - -repo string - Code repository where the generated code will reside. Can be a remote - in the format of a remote URL such as https://github.com/{owner}/{repo} or a - local file path like /path/to/repo. Both absolute and relative paths are - supported. If not specified, will try to detect if the current working directory - is configured as a language repository. - Note: When using a local repository (either by providing a path or by defaulting - to the current directory), Librarian creates a new branch from the currently checked-out - branch and commits changes. If the --push flag is also specified, a pull request is - created against the main branch. The --branch flag is ignored for local repositories. - -v enables verbose logging - -# release tag - -The 'tag' command is the final step in the release -process. It is designed to be run after a release pull request, created by -'release stage', has been merged. - -This command's primary responsibilities are to: - - - Create a Git tag for each library version included in the merged pull request. - - Create a corresponding GitHub Release for each tag, using the release notes - from the pull request body. - - Update the pull request's label from 'release:pending' to 'release:done' to - mark the process as complete. - -You can target a specific merged pull request using the '-pr' flag. If no pull -request is specified, the command will automatically search for and process all -merged pull requests with the 'release:pending' label from the last 30 days. - -Examples: - - # Tag and create a GitHub release for a specific merged PR. - legacylibrarian release tag -repo=https://github.com/googleapis/google-cloud-go -pr=https://github.com/googleapis/google-cloud-go/pull/123 - - # Find and process all pending merged release PRs in a repository. - legacylibrarian release tag -repo=https://github.com/googleapis/google-cloud-go - -Usage: - - legacylibrarian release tag [arguments] - -Flags: - - -github-api-endpoint string - The GitHub API endpoint to use for all GitHub API operations. - This is intended for testing and should not be used in production. - -pr string - The URL of a pull request to operate on. - It should be in the format of https://github.com/{owner}/{repo}/pull/{number}. - If not specified, will search for all merged pull requests with the label - "release:pending" in the last 30 days. - -repo string - Code repository where the generated code will reside. Can be a remote - in the format of a remote URL such as https://github.com/{owner}/{repo} or a - local file path like /path/to/repo. Both absolute and relative paths are - supported. If not specified, will try to detect if the current working directory - is configured as a language repository. - Note: When using a local repository (either by providing a path or by defaulting - to the current directory), Librarian creates a new branch from the currently checked-out - branch and commits changes. If the --push flag is also specified, a pull request is - created against the main branch. The --branch flag is ignored for local repositories. - -v enables verbose logging - -# update-image - -The 'update-image' command is used to update the 'image' SHA -of the language container for a language repository. - -This command's primary responsibilities are to: - - - Update the 'image' field in '.librarian/state.yaml' - - Regenerate each library with the new language container using googleapis' - proto definitions at the 'last_generated_commit' - -Examples: - - # Create a PR that updates the language container to latest image. - legacylibrarian update-image -commit -push - - # Create a PR that updates the language container to the specified image. - legacylibrarian update-image -commit -push -image= - -Usage: - - legacylibrarian update-image [flags] - -Flags: - - -api-source string - The location of an API specification repository. - Can be a remote URL or a local file path. (default "https://github.com/googleapis/googleapis") - -api-source-branch string - The target branch of the API specification repository to checkout. - Can only be used with a remote -api-source. (default "master") - -branch string - The branch to use with remote code repositories. It is ignored if - you are using a local repository. This is used to specify which branch to clone - and which branch to use as the base for a pull request. (default "main") - -build - If true, Librarian will build each generated library by invoking the - language-specific container. - -check-unexpected-changes - Defaults to false. When used with --test, this flag verifies that no - unexpected files are added, deleted, or modified outside of the changes caused - by proto updates. You may want to skip this check when testing a container image - change that is expected to add or delete files. - -commit - If true, librarian will create a commit for the change but not create - a pull request. This flag is ignored if push is set to true. - -host-mount string - For use when librarian is running in a container. A mapping of a - directory from the host to the container, in the format - :. - -image string - Language specific image used to invoke code generation and releasing. - If not specified, the image configured in the state.yaml is used. - -library-to-test string - When used with --test, this flag specifies the library ID to test - (e.g. secretmanager). Will test on all configured libraries if omitted. - -output string - Working directory root. When this is not specified, a working directory - will be created in /tmp. - -push - If true, Librarian will create a commit, - push and create a pull request for the changes. - A GitHub token with push access must be provided via the - LIBRARIAN_GITHUB_TOKEN environment variable. - -repo string - Code repository where the generated code will reside. Can be a remote - in the format of a remote URL such as https://github.com/{owner}/{repo} or a - local file path like /path/to/repo. Both absolute and relative paths are - supported. If not specified, will try to detect if the current working directory - is configured as a language repository. - Note: When using a local repository (either by providing a path or by defaulting - to the current directory), Librarian creates a new branch from the currently checked-out - branch and commits changes. If the --push flag is also specified, a pull request is - created against the main branch. The --branch flag is ignored for local repositories. - -test - If true, run container tests after generation but before committing and pushing. - These tests verify the interaction between language containers and the Librarian CLI's - 'generate' command. If a test fails, temporary branches and files will be preserved for - debugging. This flag can be used with 'library-to-test' and 'check-unexpected-changes'. - -v enables verbose logging - -# version - -Version prints version information for the legacylibrarian binary. - -Usage: - - legacylibrarian version -*/ -package main diff --git a/cmd/legacylibrarian/main.go b/cmd/legacylibrarian/main.go deleted file mode 100644 index 22fec7c9391..00000000000 --- a/cmd/legacylibrarian/main.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "context" - "log/slog" - "os" - - "github.com/googleapis/librarian/internal/legacylibrarian/legacylibrarian" -) - -func main() { - ctx := context.Background() - if err := legacylibrarian.Run(ctx, os.Args[1:]...); err != nil { - slog.Error("librarian command failed", "err", err) - os.Exit(1) - } -} diff --git a/doc/legacylibrarian/config-schema.md b/doc/legacylibrarian/config-schema.md deleted file mode 100644 index b01bac28876..00000000000 --- a/doc/legacylibrarian/config-schema.md +++ /dev/null @@ -1,61 +0,0 @@ -# config.yaml Schema - -This document describes the schema for the `config.yaml` file, which is used by Librarian to specify repository-level -and library-level configuration. This file is maintained by the repository owner. - -For more details, see the Go implementation in [librarian_config.go](../internal/config/librarian_config.go). - -## Top-Level Fields - -| Field | Type | Description | Required | Validation Constraints | -|--------------------------|------|--------------------------------------------------------|----------|------------------------| -| `global_files_allowlist` | list | A list of [global files](#global-files-object). | No | See details below. | -| `libraries` | list | A list of [library configurations](#libraries-object). | No | See details below. | - -## `global-files` Object - -Each object in the `global_files_allowlist` list represents a global file that Librarian is able to modify. - -| Field | Type | Description | Required | Validation Constraints | -|---------------|--------|----------------------------------|----------|-------------------------------------------------------------------------------------| -| `path` | string | A path from the repository root. | Yes. | Cannot be empty. May include relative paths, but cannot escape the repository root. | -| `permissions` | string | Permissions of the mounted file. | Yes | One of `read-only`, `write-only`, `read-write`. | - -## `libraries` Object - -Each object in the `libraries` list represents a single library and has the following fields: - -| Field | Type | Description | Required | Validation Constraints | -|-------------------------|--------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|-----------------------------------------------------------| -| `id` | string | A unique identifier for the library, in a language-specific format. It should not be empty and only contains alphanumeric characters, slashes, periods, underscores, and hyphens. | Yes | Must be a valid library ID. | -| `next_version` | string | The next released version of the library. Ignored unless it would increase the release version. | No | Must be a valid semantic version, "v" prefix is optional. | -| `generate_blocked` | bool | (When this library is not explicitly specified in the `-library` argument) Set this to `true` to skip the generation of this library. It's `false` by default. | No | | -| `release_blocked` | bool | (When this library is not explicitly specified in the `-library` argument) Set this to `true` to skip the release of this library. It's `false` by default. | No | | - -## Example - -```yaml -# .librarian/config.yaml - -# A list of files that will be provided to the 'configure' and 'release-stage' -# container invocations. -global_files_allowlist: - # Allow the container to read and write the root go.work file during the - # 'configure' step to add new modules. - - path: "go.work" - permissions: "read-write" - - # Allow the container to read a template. - - path: "internal/README.md.template" - permissions: "read-only" - - # Allow publishing the updated root README.md. - - path: "README.md" - permissions: "write-only" -# A list of library overrides -libraries: - - id: "secretmanager" - next_version: "2.3.4" - generate_blocked: false - release_blocked: false -``` diff --git a/doc/legacylibrarian/language-onboarding.md b/doc/legacylibrarian/language-onboarding.md deleted file mode 100644 index 5a3dd1e2862..00000000000 --- a/doc/legacylibrarian/language-onboarding.md +++ /dev/null @@ -1,381 +0,0 @@ -# Language Onboarding Guide - -This document provides a comprehensive guide for language teams to onboard their projects to the Librarian platform. It -details the necessary steps and configurations required to develop a language-specific container that Librarian can -delegate tasks to. - -## Core Concepts - -Before diving into the specifics, it's important to understand the key components of the Librarian ecosystem: - -* **Librarian:** The core orchestration tool that automates the generation, release, and maintenance of client - libraries. -* **Language-Specific Container:** A Docker container, built by each language team, that encapsulates the logic for - generating, building, and releasing libraries for that specific language. Librarian interacts with this container by - invoking different commands. -* **`state.yaml`:** A manifest file within each language repository that defines the libraries managed by Librarian, - their versions, and other essential metadata. -* **`config.yaml`:** A configuration file that allows for repository-level customization of Librarian's behavior, such - as specifying which files the container can access. - -## Configure repository to work with Librarian CLI - -Librarian relies on two key configuration files to manage its operations: `state.yaml` and `config.yaml`. These files -must be present in the `.librarian` directory at the root of the language repository. - -### `state.yaml` - -The `state.yaml` file is the primary manifest that informs Librarian about the libraries it is responsible for managing. -It contains a comprehensive list of all libraries within the repository, along with their current state and -configuration. Repository maintainers **SHOULD NOT** modify this file manually. - -For a detailed breakdown of all the fields in the `state.yaml` file, please refer to [state-schema.md]. - -### `config.yaml` - -The `config.yaml` file is a handwritten configuration file that allows you to customize Librarian's behavior at the -repository level. Its primary use is to define which files the language-specific container is allowed to access. -Repository maintainers are expected to maintain this file. Librarian will not modify this file. - -For a detailed breakdown of all the fields in the `config.yaml` file, please refer to [config-schema.md]. - -## Implement a Language Container - -Librarian orchestrates its workflows by making a series of invocations to a language-specific container. Each invocation -corresponds to a specific command and is designed to perform a distinct task. For the container to function correctly, -it must have a binary entrypoint that can accept the arguments passed by Librarian. - -A successful container invocation is expected to exit with a code of `0`. Any non-zero exit code will be treated as an -error and will halt the current workflow. If a container would like to send an error message back to librarian it can do -so by including a field in the various response files outlined below. Additionally, any logs sent to stderr/stdout will -be surfaced to the CLI. - -Additionally, Librarian specifies a user and group ID when executing the language-specific container. This means that -the container **MUST** be able to run as an arbitrary user (the caller of Librarian's user). Any commands used will -need to be executable by any user ID within the container. - -* Create a docker file for your container [example](https://github.com/googleapis/google-cloud-go/blob/main/internal/librariangen/Dockerfile) -* Create a cloudbuild file [example](https://github.com/googleapis/google-cloud-go/blob/main/internal/librariangen/cloudbuild-exitgate.yaml) that uploads your image to us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-dev - -### Guidelines on Language Container Runtimes - -You should be able to run the `generate` or `release-stage` commands for an API such as Google Cloud Secret Manager in less than a -minute. We understand that some libraries may take longer to process, however, long runtimes can adversely affect your -ability to roll out emergency changes. While the CLI typically calls the container only for libraries with changes, a -generator update could trigger a run for all your libraries. - -### Implement Container Contracts - -The following sections detail the contracts for each container command. - -### `configure` - -The `configure` command is invoked only during the onboarding of a new API. Its primary responsibility is to process -the new API information and generate the necessary configuration for the library. - -The container is expected to produce up to two artifacts: - -* A `configure-response.json` file, which is derived from the `configure-request.json` and contains language-specific - details. This response will be committed back to the `state.yaml` file by Librarian. -* Any "side-configuration" files that the language may need for its libraries. These should be written to the `/input` - mount, which corresponds to the `.librarian/generator-input` directory in the language repository. - -**Contract:** - -| Context | Type | Description | -| :----------- | :------------------ |:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `/librarian` | Mount (Read/Write) | Contains `configure-request.json`. The container must process this and write back `configure-response.json`. | -| `/input` | Mount (Read/Write) | The contents of the `.librarian/generator-input` directory. The container can add new language-specific configuration here. | -| `/repo` | Mount (Read) | Contains all of the files are specified in the libraries source_roots , if any already exist, as well as the files specified in the global_files_allowlist from `config.yaml`. | -| `/source` | Mount (Read). | Contains the complete contents of the API definition repository (e.g., [googleapis/googleapis](https://github.com/googleapis/googleapis)). | -| `/output` | Mount (Read/Write) | An output directory for writing any global file edits allowed by `global_files_allowlist`.
Additionally, the container can write arbitrary files as long as they are contained within the library’s source_roots specified in the container's response message.| -| `command` | Positional Argument | The value will always be `configure`. | -| flags | Flags | Flags indicating the locations of the mounts: `--librarian`, `--input`, `--source`, `--repo`, `--output` | - -**Example `configure-request.json`:** - -*Note: There will be only one API with a `status` of `new`.* - -```json -{ - "libraries": [ - { - "id": "pubsub", - "apis": [ - { - "path": "google/cloud/pubsub/v1", - "service_config": "pubsub_v1.yaml", - "status": "existing" - } - ], - "source_roots": [ "pubsub" ] - }, - { - "id": "secretmanager", - "apis": [ - { - "path": "google/cloud/secretmanager/v1", - "service_config": "secretmanager_v1.yaml", - "status": "new" - } - ] - } - ] -} -``` - -**Example `configure-response.json`:** - -*Note: Only the library with a `status` of `new` should be returned.* - -```json -{ - "id": "secretmanager", - "apis": [ - { - "path": "google/cloud/secretmanager/v1" - } - ], - "source_roots": [ "secretmanager" ], - "preserve_regex": [ - "secretmanager/subdir/handwritten-file.go" - ], - "remove_regex": [ - "secretmanager/generated-dir" - ], - "version": "0.0.0", - "tag_format": "{id}/v{version}", - "error": "An optional field to share error context back to Librarian." -} -``` - -### `generate` - -The `generate` command is where the core work of code generation happens. The container is expected to generate the library code and write it to the `/output` mount, preserving the directory structure of the language repository. - -**Contract:** - -| Context | Type | Description | -| :----------- | :------------------ | :------------------------------------------------------------------------------ | -| `/librarian` | Mount (Read/Write) | Contains `generate-request.json`. Container can optionally write back a `generate-response.json`. | -| `/input` | Mount (Read/Write) | The contents of the `.librarian/generator-input` directory. | -| `/output` | Mount (Write) | The destination for the generated code. The output structure should match the target repository. | -| `/source` | Mount (Read) | The complete contents of the API definition repository. (e.g. googlapis/googleapis) | -| `command` | Positional Argument | The value will always be `generate`. | -| flags | Flags | Flags indicating the locations of the mounts: `--librarian`, `--input`, `--output`, `--source` | - -**Example `generate-request.json`:** - -```json -{ - "id": "secretmanager", - "apis": [ - { - "path": "google/cloud/secretmanager/v1", - "service_config": "secretmanager_v1.yaml" - } - ], - "source_paths": [ - "secretmanager" - ], - "preserve_regex": [ - "secretmanager/subdir/handwritten-file.go" - ], - "remove_regex": [ - "secretmanager/generated-dir" - ], - "version": "0.0.0", - "tag_format": "{id}/v{version}" -} -``` - -**Example `generate-response.json`:** - -```json -{ - "error": "An optional field to share error context back to Librarian." -} -``` - -After the `generate` container finishes, Librarian is responsible for copying the generated code to the language -repository and handling any merging or deleting actions as defined in the library's state. - -### `build` - -The `build` command is responsible for building and testing the newly generated library to ensure its integrity. - -**Contract:** - -| Context | Type | Description | -| :----------- | :------------------ | :------------------------------------------------------------------------------ | -| `/librarian` | Mount (Read/Write) | Contains `build-request.json`. Container can optionally write back a `build-response.json`. | -| `/repo` | Mount (Read/Write) | The entire language repository. This is a deep copy, so any changes made here will not affect the final generated code. | -| `command` | Positional Argument | The value will always be `build`. | -| flags. | Flags | Flags indicating the locations of the mounts: `--librarian`, `--repo` | - -**Example `build-request.json`:** - -```json -{ - "id": "secretmanager", - "apis": [ - { - "path": "google/cloud/secretmanager/v1", - "service_config": "secretmanager_v1.yaml" - } - ], - "source_paths": [ - "secretmanager" - ], - "preserve_regex": [ - "secretmanager/subdir/handwritten-file.go" - ], - "remove_regex": [ - "secretmanager/generated-dir" - ], - "version": "0.0.0", - "tag_format": "{id}/v{version}" -} -``` - -**Example `build-response.json`:** - -```json -{ - "error": "An optional field to share error context back to Librarian." -} -``` - -### `release-stage` - -The `release-stage` command is the core of the release workflow. After Librarian determines the new version and collates -the commits for a release, it invokes this container command to apply the necessary changes to the repository. - -The container command's primary responsibility is to update all required files with the new version and commit -information for libraries that have the `release_triggered` set to true. This includes, but is not limited to, updating -`CHANGELOG.md` files, bumping version numbers in metadata files (e.g., `pom.xml`, `package.json`), and updating any -global files that reference the libraries being released. - -**Contract:** - -| Context | Type | Description | -| :----------- | :------------------ | :------------------------------------------------------------------------------ | -| `/librarian` | Mount (Read/Write) | Contains `release-stage-request.json`. Container writes back a `release-stage-response.json`. | -| `/repo` | Mount (Read) | Read-only contents of the language repo including any global files declared in the `config.yaml`. | -| `/output` | Mount (Write) | Any files updated during the release phase should be moved to this directory, preserving their original paths. | -| `command` | Positional Argument | The value will always be `release-stage`. | -| flags. | Flags | Flags indicating the locations of the mounts: `--librarian`, `--repo`, `--output` | - -**Example `release-stage-request.json`:** - -The request will have entries for all libraries configured in the state.yaml -- this information may be needed for any -global file edits. The libraries that are being released will be marked by the `release_triggered` field being set to -`true`. - -```json -{ - "libraries": [ - { - "id": "secretmanager", - "version": "1.3.0", - "changes": [ - { - "type": "feat", - "subject": "add new UpdateRepository API", - "body": "This adds the ability to update a repository's properties.", - "piper_cl_number": "786353207", - "commit_hash": "9461532e7d19c8d71709ec3b502e5d81340fb661" - }, - { - "type": "docs", - "subject": "fix typo in BranchRule comment", - "body": "", - "piper_cl_number": "786353207", - "commit_hash": "9461532e7d19c8d71709ec3b502e5d81340fb661" - } - ], - "apis": [ - { - "path": "google/cloud/secretmanager/v1" - }, - { - "path": "google/cloud/secretmanager/v1beta" - } - ], - "source_roots": [ - "secretmanager", - "other/location/secretmanager" - ], - "release_triggered": true - } - ] -} -``` - -**Example `release-stage-response.json`:** - -```json -{ - "error": "An optional field to share error context back to Librarian." -} -``` - -[config-schema.md]:config-schema.md -[state-schema.md]: state-schema.md - -## Pin the Language Container version in `state.yaml` - -You should pin the container version so that changes that appear as a result of updates to the language specific container -can be tracked and properly documented in client library release notes. - -The `update-image` command is used to update and pin the language specific container in `state.yaml` and re-generate all libraries. -You can optionally specify an image using the `-image` flag. - -*Note: If the `-image` flag is not specified, the latest container image will be used. -This requires application default credentials which have access to the corresponding artifact registry. -Use `gcloud auth application-default login` to configure ADC.* - -When the job completes, a PR will be opened by librarian with the changes related to the container update. You can edit the pull -request title to set a global commit message which will be applied to all libraries. - -*Note: If the container SHA in `state.yaml` was updated without using `update-image`, there could be unrelated diffs.* - -As a quick check to verify that changes are solely due to the language-specific container, execute the `update-image` command -for the current SHA to ensure a clean state before updating to a newer version of a language specific container. - -librarian update-image -image=us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/-librarian-generator@sha256: -push - -Merge the PR which includes changes for the current container SHA prior to updating to a new container SHA. Moving forward, always use `update-image` -to update the language specific SHA in `state.yaml` to ensure that all changes are correctly tracked and documented. - -## Validate Commands are working - -For each command you should be able to run the CLI on your remote desktop and have it create the expected PR. - -**Configure Command:** - -``` -export LIBRARIAN_GITHUB_TOKEN=$(gh auth token) -go run ./cmd/librarian/ generate -repo= -library= -push -``` - -**Generate Command:** - -``` -export LIBRARIAN_GITHUB_TOKEN=$(gh auth token) -go run ./cmd/librarian/ generate -repo= -push -``` - -**Release Command:** - -``` -export LIBRARIAN_GITHUB_TOKEN=$(gh auth token) -go run ./cmd/librarian/ release stage -repo= -push -``` - -**Update Image Command:** - -``` -export LIBRARIAN_GITHUB_TOKEN=$(gh auth token) -go run ./cmd/librarian/ update-image -repo= -push -``` diff --git a/doc/legacylibrarian/library-maintainer-guide.md b/doc/legacylibrarian/library-maintainer-guide.md deleted file mode 100644 index 768d7dc5fe4..00000000000 --- a/doc/legacylibrarian/library-maintainer-guide.md +++ /dev/null @@ -1,190 +0,0 @@ -# Library Maintainer Guide - -This guide is task-oriented, specifically for core/handwritten/hybrid library -maintainers. See the -[generated CLI documentation](https://pkg.go.dev/github.com/googleapis/librarian/cmd/legacylibrarian) -for a more comprehensive list of commands and flags. - -For libraries onboarded to automation, please see [automation section below](#using-automated-releases). - -This guide uses the term Librarian (capital L, regular font) for the overall -Librarian system, and `legacylibrarian` (lower case L, code font) for the CLI. - -## Internal support (Googlers) - -If anything in this guide is unclear, please see go/g3doc-cloud-sdk-librarian-support -for appropriate ways of obtaining more support. - -## Configuring development environment - -See [Setup Environment to Run Librarian](onboarding.md#step-1-setup-environment-to-run-librarian). - -## Running `legacylibrarian` - -See [Running Librarian](onboarding.md#step-6-running-librarian). - -## Initiating a release - -The release process consists of four steps (at a high level; there are more -details which aren't relevant to this guide): - -1. Creating a release PR using `legacylibrarian` -2. Reviewing/merging the release PR -3. Creating a tag for the commit created by the merged release PR -4. Running a release job (to build, test, publish) on the tagged commit - -Step 1 is described in this section. - -Step 2 is simply the normal process of reviewing and merging a PR. - -Steps 3 and 4 are automated; step 3 should occur within about 5 minutes of the -release PR being merged, and this will trigger step 4. The instructions for the -remainder of this section are about step 1. - -### Hands-off release PR creation - -If you are reasonably confident that the release notes won't need editing, the -simplest way to initiate a release is to ask `legacylibrarian` to create the release -PR for you: - -```sh -$ LIBRARIAN_GITHUB_TOKEN=$(gh auth token) legacylibrarian release stage -push \ - -repo=https://github.com/googleapis/google-cloud-go -library=bigtable -``` - -This will use the conventional commits since the last release to populate -any release notes in both the PR and the relevant changelog file in the repo. - -If you want to release a version other than the one inferred by the conventional -commits (e.g. for a prerelease or a patch), you can use the `-library-version` -flag: - -```sh -$ LIBRARIAN_GITHUB_TOKEN=$(gh auth token) legacylibrarian release stage -push \ - -repo=https://github.com/googleapis/google-cloud-go -library=bigtable \ - -library-version=1.2.3 -``` - -Note that if `librarian` doesn't detect any conventional commits that would -trigger a release, you *must* specify the `-library-version` flag. - -### Manual release PR creation - -If you expect to need to edit the release notes, it's simplest to run -`librarian` *without* the `-push` flag (and using a local repo), -then create the pull request yourself: - -1. Make sure your local clone is up-to-date -2. Create a new branch for the release (e.g. `git checkout -b release-bigtable-1.2.3`) -3. Run `librarian`, specifying `-library-version` if you want/need to, as above: - ```sh - $ legacylibrarian release stage -library=bigtable -library-version=1.2.3 - ``` -4. Note the line of the `librarian` output near the end, which tells you where - it has written a `pr-body.txt` file (split by key below, but all on one line - in the output): - ```text - time=2025-09-26T15:35:23.124Z - level=INFO - msg="Wrote body of pull request that might have been created" - file=/tmp/librarian-837968205/pr-body.txt - ``` -5. Perform any edits to the release notes, and commit the change using a "chore" - conventional commit, e.g. `git commit -a -m "chore: create release"` -6. Push your change to GitHub *in the main fork* (rather than any personal fork you may use), - and create a PR from the branch. Use the content of the `pr-body.txt` file as the body - of the PR, editing it to be consistent with any changes you've made in the release notes. -7. Add the "release:pending" label to the PR. -8. Ask a colleague to review and merge the PR. - -## Updating generated code - -Librarian automation updates GAPIC/proto-generated code on a weekly basis on a -Wednesday, but generation can be run manually if an API change urgently needs -to be included in client libraries. - -### Create a PR with updated code - -In the common case where you just need to expedite generation, `librarian` can -create the generation PR for you: - -```sh -$ LIBRARIAN_GITHUB_TOKEN=$(gh auth token) legacylibrarian generate -push \ - -repo=https://github.com/googleapis/google-cloud-go -library=bigtable -``` - -Ask a colleague to review and merge the PR. - -### Test a local API change - -If you need to test the potential impact of an API change which isn't yet in the -`googleapis` repository, you can use the `-api-source` flag to specify a local -clone of `googleapis`. You should *not* push this to GitHub other than for -sharing purposes. - -Typical flow, assuming a directory structure of `~/github/googleapis` containing -clones of `googleapis` and `google-cloud-go` (and assuming they're up-to-date): - -```sh -$ cd ~/github/googleapis/googleapis -$ git checkout -b test-api-changes -# Make changes here -$ git commit -a -m "Test API changes" -$ cd ../google-cloud-go -$ git checkout -b test-generated-api-changes -$ legacylibrarian generate -api-source=../googleapis -library=bigtable -``` - -## Using automated releases - -Maintainers *may* configure Librarian for automated releases, but should do so -with a significant amount of care. Using automated releases is convenient, but -cedes control - and once a release has been published, it can't generally be -rolled back in anything like a "clean" way. - -### Impact of automated releases - -When automated releases are enabled, they will be initiated on a regular -[cadence](https://goto.google.com/g3doc-librarian-automation), in release PRs that contain other libraries needing releasing from -the same repository. These pull requests will be approved and merged by the -Cloud SDK Platform team. - -The version number and release notes will be automatically determined by -Librarian from conventional commits. These will *not* be vetted by the -Cloud SDK Platform team before merging. - -Using automated releases doesn't *prevent* manual releases - a maintainer team -can always use the process above to create and merge release PRs themselves, -customizing the version number and release notes as they see fit. Creating -a single manual release does not interrupt automated releases - any subsequent -"release-worthy" changes will still cause an automated release to be created. - -### Enabling automated releases - -If the repository containing the library is not already using automated releases -for other libraries (i.e. if it's a split repo instead of a monorepo), -first [open a ticket](https://buganizer.corp.google.com/issues/new?component=1198207&template=2190445) with the Cloud Platform SDK team to enable automated releases. - -Next, edit the `.librarian/config.yaml` file in your repository. You will -see YAML describing additional configuration for the libraries in the -repository, particularly those which have release automation blocked. Find -the library for which you wish to enable release automation, and remove or -comment out the `release_blocked` key. We recommend commenting out rather -than deleting, ideally adding an explanation and potentially caveats, for -future readers. - -Create a PR with the configuration change, get it reviewed and merged, and -the next time release automation runs against the repository, it will consider -the library eligible for automatic releases. - -### Ownership of PRs once onboarded to automation - -Once a library is onboarded to Librarian automation, the Librarian team is -responsible for approving and merging PRs generated by Librarian. Maintainers are -not expected to be involved in this process, unless PR checks fail. In that -case a ticket will be opened up in the repository and needs to be addressed by -Maintainers. This can potentially block generation/release until issue has been -resolved. - -## Support -If you need support please reach out to cloud-sdk-librarian-oncall@google.com. diff --git a/doc/legacylibrarian/onboarding.md b/doc/legacylibrarian/onboarding.md deleted file mode 100644 index 8b9d8ea6244..00000000000 --- a/doc/legacylibrarian/onboarding.md +++ /dev/null @@ -1,133 +0,0 @@ -# Onboarding to Legacy Librarian - -Welcome! This guide is intended to help you get started with the Librarian -project and begin contributing effectively. - -## Step 1: Setup Environment to Run Legacy Librarian - -`librarian` requires: - -- Linux -- [Go](https://go.dev/doc/install) -- [sudoless Docker](http://go/installdocker) -- git (if you wish to build it locally) -- [gcloud](https://g3doc.corp.google.com/company/teams/cloud-sdk/cli/index.md?cl=head#installing-and-using-the-cloud-sdk) (to set up Docker access to container images) -- [gh](https://github.com/cli/cli) for GitHub access tokens - -While in theory `librarian` should be run from your local remote desktop. - -> Note that installing Docker will cause gLinux to warn you that Docker is -> unsupported and discouraged. Within Cloud, support for Docker is a core -> expectation (e.g. for Cloud Run and Cloud Build). - -Docker needs to be configured to use gcloud for authentication. The following -command line needs to be run, just once: - -```sh -gcloud auth configure-docker us-central1-docker.pkg.dev -``` - -## Step 2: Set Up Your Editor -Install the Go extension following the -[instructions for your preferred editor](https://github.com/golang/tools/tree/master/gopls#editors) - -These extensions provide support for essential tools like -[gofmt](https://pkg.go.dev/golang.org/x/tools/cmd/goimports) (automatic code -formatting) and -[goimports](https://pkg.go.dev/golang.org/x/tools/cmd/goimports) (automatic -import management). - -## Step 3: Understand How We Work - -Read the -[CONTRIBUTING.md](https://github.com/googleapis/librarian/blob/main/CONTRIBUTING.md) -for information on how we work, how to submit code, and what to expect. - -## Step 4: Learn Go - -If you are new to Go, complete these tutorials: - -- [Tutorial: Get started with Go](https://go.dev/doc/tutorial/getting-started) -- [Tutorial: Create a Go module](https://go.dev/doc/tutorial/create-module) -- [A Tour of Go](https://go.dev/tour/welcome) - -These will teach you the foundations for how to write, run, and test Go code. - -## Step 5: Understand How We Write Go - -Read our guide on -[How We Write Go](https://github.com/googleapis/librarian/blob/main/doc/howwewritego.md), for -[project-specific guidance on writing idiomatic, consistent Go code. - -## Step 6: Running Legacy Librarian - -Currently running legacy librarian from main is unstable, please use the `latest` tag when running -locally. - -### Using `go run` - -```sh -$ go run github.com/googleapis/librarian/cmd/legacylibrarian@latest -``` - -### Using `go install` - -To install a binary locally, and then run it (assuming the `$GOBIN` directory -is in your path): - -```sh -$ go install github.com/googleapis/librarian/cmd/legacylibrarian@latest -``` - - -### Obtaining a GitHub access token - -`librarian` commands which perform write operations on GitHub require -a GitHub access token to be specified via the `LIBRARIAN_GITHUB_TOKEN` -environment variable. While access tokens can be generated manually -and then stored in environment variables in other ways, it's simplest -to use the [`gh` tool](https://github.com/cli/cli). - -Once installed, use `gh auth login` to log into GitHub. After that, -when running `librarian` you can use `gh auth token` to obtain an access -token and set it in the environment variable just for that invocation: - -```sh -$ LIBRARIAN_GITHUB_TOKEN=$(gh auth token) legacylibrarian ... -``` - -The examples below assume include this for convenience; if you have -set the environment variable in a different way, just remove the -`LIBRARIAN_GITHUB_TOKEN=$(gh auth token)` part from the command. - -### Repository and library options - -`legacylibrarian` can either operate on a local clone of the library repo, -or it can clone the repo itself. Unless you have a particular need to use a -local clone (e.g. to the impact of a local change) we recommend you let -`legacylibrarian` clone the library repo itself, using the `-repo` flag - just specify -the GitHub repository, e.g. -`-repo=https://github.com/googleapis/google-cloud-go`. This avoids any -risk of unwanted local changes accidentally becoming part of a generation/release -pull request. - -If you wish to use a local clone, you can specify the directory in the `-repo` -flag, or just run `legacylibrarian` from the root directory of the clone and omit the -`-repo` flag entirely. - -The commands in this guide are specifically for generating/releasing a single -library, specified with the `library` flag. This is typically the name of the -package or module, e.g. `bigtable` or `google-cloud-bigtable`. Consult the -state file (`.librarian/state.yaml`) in the library repository to find the -library IDs in use (and ideally record this in a team-specific playbook). - -## Helpful Links - -Use these links to deepen your understanding as you go: - -- **Play with Go** (https://go.dev/play): Playground to run and share Go snippets in your browser. - -- **Browse Go Packages** (https://pkg.go.dev): Go's official site for discovering and reading documentation for any Go - package. - -- **Explore the Standard Library** (https://pkg.go.dev/std): Documentation for the Go standard library. diff --git a/doc/legacylibrarian/repository-library-onboarding.md b/doc/legacylibrarian/repository-library-onboarding.md deleted file mode 100644 index a400605b59c..00000000000 --- a/doc/legacylibrarian/repository-library-onboarding.md +++ /dev/null @@ -1,33 +0,0 @@ -# Repository/Library Onboarding Guide - -This guide should be followed when onboarding new repositories/libraries. - -## Repository Setup: -1) [Create a ticket](http://go/onboard-repository-to-librarian) to onboard a repository to Librarian automation (automation is per repository not library). At a minimum, you should onboard to [tag-and-release](https://pkg.go.dev/github.com/googleapis/librarian/cmd/librarian#hdr-release_tag_and_release) automation. -2) Add `.librarian` directory to your repository with appropriate configuration files. See details [here](https://github.com/googleapis/librarian/blob/main/doc/language-onboarding.md#configure-repository-to-work-with-librarian-cli) -3) You should only start with 1 library to validate the flow (follow instructions below) -4) If your repository contains multiple libraries, start ramping up slowly until all libraries are in your `state.yaml` file and have migrated to librarian. -5) To complete onboarding you should run the librarian test-container generate command to validate that all libraries are getting generated correctly. Note this standalone command is WIP, currently you can run [update-image](https://pkg.go.dev/github.com/googleapis/librarian/cmd/librarian#hdr-update_image) command with `-test` flag to trigger tests after generation. -6) To correctly parse the commit message of a merge commit, only allow squash merging -and set the default commit message to **Pull request title and description**. -![Pull request settings](assets/setting-pull-requests.webp) - -## Library Setup: - -### If you currently are using owlbot/release-please for generation and release: -1) Ensure all OwlBot PRs for that library have been merged and then release the library using a release-please PR -2) Remove the library from your OwlBot config - - If you have an .Owlbot.yaml config file with multiple libraries, remove the library from .Owlbot.yaml file. - - If your .Owlbot.yaml config file contains only the single library, remove the .Owlbot.yaml file itself. -3) Remove the library from your release-please config - - For a monolithic repo remove the path entry for the library in your release-please-config.json and .release-please-manifest.json files - - For a single library repository, remove all the release-please config (.github/release-please.yml, release-please-config.json if it exists, .release-please-manifest.json if it exists) -4) There is no requirement to stop using library-specific post-processing files as part of this migration. However, any post processing should be included in a file named "librarian.", where corresponds to the script's file extension (e.g., "sh", "py"). While migrating, please also consider opening an issue in your generator repository for any improvements that could reduce your library's post-processing logic. - -### General Library Setup Steps -1) Add your library to your [libraries object](https://github.com/googleapis/librarian/blob/main/doc/state-schema.md#libraries-object) in your [state.yaml](https://github.com/googleapis/librarian/blob/main/doc/state-schema.md#stateyaml-schema) file. -2) Run [librarian generate command](https://pkg.go.dev/github.com/googleapis/librarian/cmd/librarian#hdr-generate). The output should be 0 diff, check with your language lead/generator owner if this is not the case. -3) Be aware of the `generate_blocked` and `release_blocked` fields in the [config.yaml file](https://github.com/googleapis/librarian/blob/main/doc/config-schema.md#libraries-object). If these are not set and automation is enabled for the repository ([check here](https://github.com/googleapis/librarian/blob/main/internal/legacylibrarian/legacyautomation/prod/repositories.yaml)), then generate and release PRs will be created and merged automatically. If these actions are blocked, or your repository is not set up for automation, you will have to perform these actions manually. See this [guide](https://github.com/googleapis/librarian/blob/main/doc/library-maintainer-guide.md) for details. - -## Support -If you need support please reach out to cloud-sdk-librarian-oncall@google.com. diff --git a/doc/legacylibrarian/state-schema.md b/doc/legacylibrarian/state-schema.md deleted file mode 100644 index e85530d7e9d..00000000000 --- a/doc/legacylibrarian/state-schema.md +++ /dev/null @@ -1,57 +0,0 @@ -# state.yaml Schema - -This document describes the schema for the `state.yaml` file, which is used by Librarian to track the status of managed files. This file should not be edited manually. - -For more details, see the Go implementation in [state.go](../internal/config/state.go). - -## Top-Level Fields - -| Field | Type | Description | Required | Validation Constraints | -|-------------|--------|-----------------------------------------------------|----------|------------------------------------------------------------------------------------| -| `image` | string | The name and tag of the generator image to use. | Yes | Must be a container image reference that includes a tag and contains no whitespace. | -| `libraries` | list | A list of [library configurations](#libraries-object). | Yes | Must not be empty. | - -## `libraries` Object - -Each object in the `libraries` list represents a single library and has the following fields: - -| Field | Type | Description | Required | Validation Constraints | -|-------------------------|--------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|------------------------| -| `id` | string | A unique identifier for the library, in a language-specific format. It should not be empty and only contains alphanumeric characters, slashes, periods, underscores, and hyphens. | Yes | Must be a valid library ID. | -| `version` | string | The last released version of the library. | No | Must be a valid semantic version, "v" prefix is optional. | -| `last_generated_commit` | string | The commit hash from the API definition repository at which the library was last generated. | No | Must be a 40-character hexadecimal string. | -| `apis` | list | A list of [APIs](#apis-object) that are part of this library. | Yes | Must not be empty. | -| `source_roots` | list | A list of directories in the language repository where Librarian contributes code. | Yes | Must not be empty, and each path must be a valid directory path. | -| `preserve_regex` | list | A list of regular expressions for files and directories to preserve during the copy and remove process. | No | Each entry must be a valid regular expression. | -| `remove_regex` | list | A list of regular expressions for files and directories to remove before copying generated code. If not set, this defaults to the `source_roots`. A more specific `preserve_regex` takes precedence. | No | Each entry must be a valid regular expression. | -| `release_exclude_paths` | list | A list of paths to exclude from the release. Files matching these paths will not be considered part of a commit for this library. | No | Each entry must be a valid directory file/path. | -| `tag_format` | string | A format string for the release tag. The supported placeholders are `{id}` and `{version}`. | No | Must contain `{version}` and may optionally contain `{id}`. No other placeholders are allowed. | - -## `apis` Object - -Each object in the `apis` list represents a single API and has the following fields: - -| Field | Type | Description | Required | Validation Constraints | -|------------------|--------|---------------------------------------------------------------------------------------------------------|----------|------------------------| -| `path` | string | The path to the API, relative to the root of the API definition repository (e.g., `google/storage/v1`). | Yes | Must be a valid directory path. | -| `service_config` | string | The name of the service config file, relative to the API `path`. | No | None. | - -## Example - -```yaml -image: "gcr.io/my-special-project/language-generator:v1.2.5" -libraries: - - id: "secretmanager" - version: "1.15.0" - last_generated_commit: "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2" - apis: - - path: "google/cloud/secretmanager/v1" - service_config: "secretmanager_v1.yaml" - source_roots: - - "src/google/cloud/secretmanager" - - "test/google/cloud/secretmanager" - preserve_regex: - - "src/google/cloud/secretmanager/generated-dir/HandWrittenFile.java" - remove_regex: - - "src/google/cloud/secretmanager/generated-dir" -``` \ No newline at end of file diff --git a/doc/legacylibrarian/testing.md b/doc/legacylibrarian/testing.md deleted file mode 100644 index de46f00c854..00000000000 --- a/doc/legacylibrarian/testing.md +++ /dev/null @@ -1,49 +0,0 @@ -# Testing - -## Unit Tests - -These tests are designed to test the internal librarian golang logic. - -Usage: - -```bash -go test ./... -``` - -## End-to-End (e2e) Tests - -These tests are designed to test the CLI interface for each supported Librarian command -on a local system. It tests the interface with docker using a fake repo and test docker -image. These tests are run as presubmits and postsubmits via GitHub actions. - -Setup: - -```bash -DOCKER_BUILDKIT=1 docker build \ - -f ./internal/legacylibrarian/legacyintegration/testdata/e2e-test.Dockerfile \ - -t test-image:latest \ - . -``` - -Usage: - -```bash -go test -tags e2e ./internal/legacylibrarian/legacyintegration/... -``` - -## Integration Tests - -These tests are designed to test interactions with remote systems (e.g. GitHub). These -tests are **NOT** run automatically as they create pull requests and branches. - -Usage: - -```bash -LIBRARIAN_TEST_GITHUB_TOKEN= \ - LIBRARIAN_TEST_GITHUB_REPO= \ - go test -tags integration ./... -``` - -Note: `LIBRARIAN_TEST_GITHUB_TOKEN` must have write access to `LIBRARIAN_TEST_GITHUB_REPO`. - -Note: These tests are skipped unless these environment variables are set. diff --git a/go.mod b/go.mod index 58f30075608..16b88c68197 100644 --- a/go.mod +++ b/go.mod @@ -3,24 +3,18 @@ module github.com/googleapis/librarian go 1.26.1 require ( - cloud.google.com/go/artifactregistry v1.19.0 - cloud.google.com/go/cloudbuild v1.25.0 cloud.google.com/go/iam v1.5.3 cloud.google.com/go/longrunning v0.8.0 github.com/bazelbuild/buildtools v0.0.0-20260202105709-e24971d9d1a7 github.com/cbroglie/mustache v1.4.0 github.com/go-git/go-git/v5 v5.19.1 github.com/google/go-cmp v0.7.0 - github.com/google/go-github/v69 v69.2.0 - github.com/google/uuid v1.6.0 github.com/google/yamlfmt v0.21.0 - github.com/googleapis/gax-go/v2 v2.17.0 github.com/iancoleman/strcase v0.3.0 github.com/pb33f/libopenapi v0.25.9 github.com/pelletier/go-toml/v2 v2.2.4 github.com/urfave/cli/v3 v3.6.2 github.com/yuin/goldmark v1.7.16 - golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f golang.org/x/mod v0.35.0 golang.org/x/sync v0.20.0 golang.org/x/tools v0.44.0 @@ -34,13 +28,8 @@ require ( require ( 4d63.com/gocheckcompilerdirectives v1.3.0 // indirect 4d63.com/gochecknoglobals v0.2.2 // indirect - cloud.google.com/go v0.123.0 // indirect - cloud.google.com/go/auth v0.18.1 // indirect - cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect - cloud.google.com/go/compute/metadata v0.9.0 // indirect codeberg.org/chavacava/garif v0.2.0 // indirect codeberg.org/polyfloyd/go-errorlint v1.9.0 // indirect - dario.cat/mergo v1.0.2 // indirect dev.gaijin.team/go/exhaustruct/v4 v4.0.0 // indirect dev.gaijin.team/go/golib v0.6.0 // indirect github.com/4meepo/tagalign v1.4.3 // indirect @@ -53,10 +42,8 @@ require ( github.com/BurntSushi/toml v1.6.0 // indirect github.com/Djarvur/go-err113 v0.1.1 // indirect github.com/Masterminds/semver/v3 v3.4.0 // indirect - github.com/Microsoft/go-winio v0.6.2 // indirect github.com/MirrexOne/unqueryvet v1.5.4 // indirect github.com/OpenPeeDeeP/depguard/v2 v2.2.1 // indirect - github.com/ProtonMail/go-crypto v1.3.0 // indirect github.com/alecthomas/chroma/v2 v2.23.1 // indirect github.com/alecthomas/go-check-sumtype v0.3.1 // indirect github.com/alexkohler/nakedret/v2 v2.0.6 // indirect @@ -89,19 +76,15 @@ require ( github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd // indirect github.com/charmbracelet/x/term v0.2.1 // indirect github.com/ckaznocha/intrange v0.3.1 // indirect - github.com/cloudflare/circl v1.6.3 // indirect github.com/curioswitch/go-reassign v0.3.0 // indirect - github.com/cyphar/filepath-securejoin v0.6.1 // indirect github.com/daixiang0/gci v0.13.7 // indirect github.com/dave/dst v0.27.3 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/denis-tingaikin/go-header v0.5.0 // indirect github.com/dlclark/regexp2 v1.11.5 // indirect - github.com/emirpasic/gods v1.18.1 // indirect github.com/ettle/strcase v0.2.0 // indirect github.com/fatih/color v1.19.0 // indirect github.com/fatih/structtag v1.2.0 // indirect - github.com/felixge/httpsnoop v1.0.4 // indirect github.com/firefart/nonamedreturns v1.0.6 // indirect github.com/fsnotify/fsnotify v1.5.4 // indirect github.com/fzipp/gocyclo v0.6.0 // indirect @@ -109,8 +92,6 @@ require ( github.com/go-critic/go-critic v0.14.3 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.9.0 // indirect - github.com/go-logr/logr v1.4.3 // indirect - github.com/go-logr/stdr v1.2.2 // indirect github.com/go-toolsmith/astcast v1.1.0 // indirect github.com/go-toolsmith/astcopy v1.1.0 // indirect github.com/go-toolsmith/astequal v1.2.0 // indirect @@ -123,7 +104,6 @@ require ( github.com/gobwas/glob v0.2.3 // indirect github.com/godoc-lint/godoc-lint v0.11.2 // indirect github.com/gofrs/flock v0.13.0 // indirect - github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golangci/asciicheck v0.5.0 // indirect github.com/golangci/dupl v0.0.0-20250308024227-f665c8d69b32 // indirect @@ -137,9 +117,6 @@ require ( github.com/golangci/swaggoswag v0.0.0-20250504205917-77f2aca3143e // indirect github.com/golangci/unconvert v0.0.0-20250410112200-a129a6e6413e // indirect github.com/google/addlicense v1.2.0 // indirect - github.com/google/go-querystring v1.1.0 // indirect - github.com/google/s2a-go v0.1.9 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.12 // indirect github.com/gordonklaus/ineffassign v0.2.0 // indirect github.com/gostaticanalysis/analysisutil v0.7.1 // indirect github.com/gostaticanalysis/comment v1.5.0 // indirect @@ -157,10 +134,8 @@ require ( github.com/jjti/go-spancheck v0.6.5 // indirect github.com/julz/importas v0.2.0 // indirect github.com/karamaru-alpha/copyloopvar v1.2.2 // indirect - github.com/kevinburke/ssh_config v1.4.0 // indirect github.com/kisielk/errcheck v1.10.0 // indirect github.com/kkHAIKE/contextcheck v1.1.6 // indirect - github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/kulti/thelper v0.7.1 // indirect github.com/kunwardeep/paralleltest v1.0.15 // indirect github.com/lasiar/canonicalheader v1.1.2 // indirect @@ -194,7 +169,6 @@ require ( github.com/nunnatsa/ginkgolinter v0.23.0 // indirect github.com/pb33f/ordered-map/v2 v2.3.0 // indirect github.com/pelletier/go-toml v1.9.5 // indirect - github.com/pjbgf/sha1cd v0.6.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_golang v1.12.1 // indirect github.com/prometheus/client_model v0.2.0 // indirect @@ -219,7 +193,6 @@ require ( github.com/sergi/go-diff v1.4.0 // indirect github.com/sirupsen/logrus v1.9.4 // indirect github.com/sivchari/containedctx v1.0.3 // indirect - github.com/skeema/knownhosts v1.3.2 // indirect github.com/sonatard/noctx v0.5.1 // indirect github.com/sourcegraph/go-diff v0.7.0 // indirect github.com/speakeasy-api/jsonpath v0.6.2 // indirect @@ -243,7 +216,6 @@ require ( github.com/ultraware/whitespace v0.2.0 // indirect github.com/uudashr/gocognit v1.2.1 // indirect github.com/uudashr/iface v1.4.1 // indirect - github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xen0n/gosmopolitan v1.3.0 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect github.com/yagipy/maintidx v1.0.0 // indirect @@ -254,25 +226,17 @@ require ( go-simpler.org/sloglint v0.11.1 // indirect go.augendre.info/arangolint v0.4.0 // indirect go.augendre.info/fatcontext v0.9.0 // indirect - go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.65.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 // indirect go.opentelemetry.io/otel v1.41.0 // indirect - go.opentelemetry.io/otel/metric v1.41.0 // indirect - go.opentelemetry.io/otel/trace v1.41.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.40.0 // indirect go.uber.org/multierr v1.10.0 // indirect go.uber.org/zap v1.27.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect go.yaml.in/yaml/v4 v4.0.0-rc.4 // indirect - golang.org/x/crypto v0.50.0 // indirect golang.org/x/exp/typeparams v0.0.0-20260209203927-2842357ff358 // indirect golang.org/x/net v0.53.0 // indirect - golang.org/x/oauth2 v0.35.0 // indirect golang.org/x/sys v0.43.0 // indirect golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa // indirect golang.org/x/text v0.36.0 // indirect - golang.org/x/time v0.14.0 // indirect - google.golang.org/api v0.266.0 // indirect google.golang.org/grpc v1.79.3 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect diff --git a/go.sum b/go.sum index ad05382789c..86fe1cfc0e8 100644 --- a/go.sum +++ b/go.sum @@ -17,24 +17,12 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= -cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= -cloud.google.com/go/artifactregistry v1.19.0 h1:DaOHWeURq93K27/6Sa2fy3rJoftrVXKeT3tonM4fxtI= -cloud.google.com/go/artifactregistry v1.19.0/go.mod h1:UEAPCgHDFC1q+A8nnVxXHPEy9KCVOeavFBF1fEChQvU= -cloud.google.com/go/auth v0.18.1 h1:IwTEx92GFUo2pJ6Qea0EU3zYvKnTAeRCODxfA/G5UWs= -cloud.google.com/go/auth v0.18.1/go.mod h1:GfTYoS9G3CWpRA3Va9doKN9mjPGRS+v41jmZAhBzbrA= -cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= -cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/cloudbuild v1.25.0 h1:Fkg+iJdN7bfICZJzLr/XV+k9aVxXS/hakIlhjDIRIDw= -cloud.google.com/go/cloudbuild v1.25.0/go.mod h1:lCu+T6IPkobPo2Nw+vCE7wuaAl9HbXLzdPx/tcF+oWo= -cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= -cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/iam v1.5.3 h1:+vMINPiDF2ognBJ97ABAYYwRgsaqxPbQDlMnbHMjolc= @@ -54,8 +42,6 @@ codeberg.org/chavacava/garif v0.2.0 h1:F0tVjhYbuOCnvNcU3YSpO6b3Waw6Bimy4K0mM8y6M codeberg.org/chavacava/garif v0.2.0/go.mod h1:P2BPbVbT4QcvLZrORc2T29szK3xEOlnl0GiPTJmEqBQ= codeberg.org/polyfloyd/go-errorlint v1.9.0 h1:VkdEEmA1VBpH6ecQoMR4LdphVI3fA4RrCh2an7YmodI= codeberg.org/polyfloyd/go-errorlint v1.9.0/go.mod h1:GPRRu2LzVijNn4YkrZYJfatQIdS+TrcK8rL5Xs24qw8= -dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= -dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= dev.gaijin.team/go/exhaustruct/v4 v4.0.0 h1:873r7aNneqoBB3IaFIzhvt2RFYTuHgmMjoKfwODoI1Y= dev.gaijin.team/go/exhaustruct/v4 v4.0.0/go.mod h1:aZ/k2o4Y05aMJtiux15x8iXaumE88YdiB0Ai4fXOzPI= dev.gaijin.team/go/golib v0.6.0 h1:v6nnznFTs4bppib/NyU1PQxobwDHwCXXl15P7DV5Zgo= @@ -83,15 +69,10 @@ github.com/Djarvur/go-err113 v0.1.1 h1:eHfopDqXRwAi+YmCUas75ZE0+hoBHJ2GQNLYRSxao github.com/Djarvur/go-err113 v0.1.1/go.mod h1:IaWJdYFLg76t2ihfflPZnM1LIQszWOsFDh2hhhAVF6k= github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= -github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= -github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= -github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/MirrexOne/unqueryvet v1.5.4 h1:38QOxShO7JmMWT+eCdDMbcUgGCOeJphVkzzRgyLJgsQ= github.com/MirrexOne/unqueryvet v1.5.4/go.mod h1:fs9Zq6eh1LRIhsDIsxf9PONVUjYdFHdtkHIgZdJnyPU= github.com/OpenPeeDeeP/depguard/v2 v2.2.1 h1:vckeWVESWp6Qog7UZSARNqfu/cZqvki8zsuj3piCMx4= github.com/OpenPeeDeeP/depguard/v2 v2.2.1/go.mod h1:q4DKzC4UcVaAvcfd41CZh0PWpGgzrVxUYBlgKNGquUo= -github.com/ProtonMail/go-crypto v1.3.0 h1:ILq8+Sf5If5DCpHQp4PbZdS1J7HDFRXz/+xKBiRGFrw= -github.com/ProtonMail/go-crypto v1.3.0/go.mod h1:9whxjD8Rbs29b4XWbB8irEcE8KHMqaR2e7GWU1R+/PE= github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= github.com/alecthomas/chroma/v2 v2.23.1 h1:nv2AVZdTyClGbVQkIzlDm/rnhk1E9bU9nXwmZ/Vk/iY= @@ -115,10 +96,6 @@ github.com/alingse/asasalint v0.0.11 h1:SFwnQXJ49Kx/1GghOFz1XGqHYKp21Kq1nHad/0WQ github.com/alingse/asasalint v0.0.11/go.mod h1:nCaoMhw7a9kSJObvQyVzNTPBDbNpdocqrSP7t/cW5+I= github.com/alingse/nilnesserr v0.2.0 h1:raLem5KG7EFVb4UIDAXgrv3N2JIaffeKNtcEXkEWd/w= github.com/alingse/nilnesserr v0.2.0/go.mod h1:1xJPrXonEtX7wyTq8Dytns5P2hNzoWymVUIaKm4HNFg= -github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= -github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/ashanbrown/forbidigo/v2 v2.3.0 h1:OZZDOchCgsX5gvToVtEBoV2UWbFfI6RKQTir2UZzSxo= github.com/ashanbrown/forbidigo/v2 v2.3.0/go.mod h1:5p6VmsG5/1xx3E785W9fouMxIOkvY2rRV9nMdWadd6c= github.com/ashanbrown/makezero/v2 v2.1.0 h1:snuKYMbqosNokUKm+R6/+vOPs8yVAi46La7Ck6QYSaE= @@ -183,16 +160,10 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/ckaznocha/intrange v0.3.1 h1:j1onQyXvHUsPWujDH6WIjhyH26gkRt/txNlV7LspvJs= github.com/ckaznocha/intrange v0.3.1/go.mod h1:QVepyz1AkUoFQkpEqksSYpNpUo3c5W7nWh/s6SHIJJk= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8= -github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 h1:6xNmx7iTtyBRev0+D/Tv1FZd4SCg8axKApyNyRsAt/w= -github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5/go.mod h1:KdCmV+x/BuvyMxRnYBlmVaq4OLiKW6iRQfvC62cvdkI= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/curioswitch/go-reassign v0.3.0 h1:dh3kpQHuADL3cobV/sSGETA8DOv457dwl+fbBAhrQPs= github.com/curioswitch/go-reassign v0.3.0/go.mod h1:nApPCCTtqLJN/s8HfItCcKV0jIPwluBOvZP+dsJGA88= -github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE= -github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc= github.com/daixiang0/gci v0.13.7 h1:+0bG5eK9vlI08J+J/NWGbWPTNiXPG4WhNLJOkSxWITQ= github.com/daixiang0/gci v0.13.7/go.mod h1:812WVN6JLFY9S6Tv76twqmNqevN0pa3SX3nih0brVzQ= github.com/dave/dst v0.27.3 h1:P1HPoMza3cMEquVf9kKy8yXsFirry4zEnWOdYPOoIzY= @@ -207,27 +178,16 @@ github.com/denis-tingaikin/go-header v0.5.0 h1:SRdnP5ZKvcO9KKRP1KJrhFR3RrlGuD+42 github.com/denis-tingaikin/go-header v0.5.0/go.mod h1:mMenU5bWrok6Wl2UsZjy+1okegmwQ3UgWl4V1D8gjlY= github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= -github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= -github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= -github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= -github.com/envoyproxy/go-control-plane/envoy v1.36.0 h1:yg/JjO5E7ubRyKX3m07GF3reDNEnfOboJ0QySbH736g= -github.com/envoyproxy/go-control-plane/envoy v1.36.0/go.mod h1:ty89S1YCCVruQAm9OtKeEkQLTb+Lkz0k8v9W0Oxsv98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/envoyproxy/protoc-gen-validate v1.3.0 h1:TvGH1wof4H33rezVKWSpqKz5NXWg5VPuZ0uONDT6eb4= -github.com/envoyproxy/protoc-gen-validate v1.3.0/go.mod h1:HvYl7zwPa5mffgyeTUHA9zHIH36nmrm7oCbo4YKoSWA= github.com/ettle/strcase v0.2.0 h1:fGNiVF21fHXpX1niBgk0aROov1LagYsOwV/xqKDKR/Q= github.com/ettle/strcase v0.2.0/go.mod h1:DajmHElDSaX76ITe3/VHVyMin4LWSJN5Z909Wp+ED1A= github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w= github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE= github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/firefart/nonamedreturns v1.0.6 h1:vmiBcKV/3EqKY3ZiPxCINmpS431OcE1S47AQUwhrg8E= github.com/firefart/nonamedreturns v1.0.6/go.mod h1:R8NisJnSIpvPWheCq0mNRXJok6D8h7fagJTF8EMEwCo= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= @@ -238,16 +198,12 @@ github.com/fzipp/gocyclo v0.6.0 h1:lsblElZG7d3ALtGMx9fmxeTKZaLLpU8mET09yN4BBLo= github.com/fzipp/gocyclo v0.6.0/go.mod h1:rXPyn8fnlpa0R2csP/31uerbiVBugk5whMdlyaLkLoA= github.com/ghostiam/protogetter v0.3.20 h1:oW7OPFit2FxZOpmMRPP9FffU4uUpfeE/rEdE1f+MzD0= github.com/ghostiam/protogetter v0.3.20/go.mod h1:FjIu5Yfs6FT391m+Fjp3fbAYJ6rkL/J6ySpZBfnODuI= -github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= -github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= github.com/go-critic/go-critic v0.14.3 h1:5R1qH2iFeo4I/RJU8vTezdqs08Egi4u5p6vOESA0pog= github.com/go-critic/go-critic v0.14.3/go.mod h1:xwntfW6SYAd7h1OqDzmN6hBX/JxsEKl5up/Y2bsxgVQ= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= github.com/go-git/go-billy/v5 v5.9.0 h1:jItGXszUDRtR/AlferWPTMN4j38BQ88XnXKbilmmBPA= github.com/go-git/go-billy/v5 v5.9.0/go.mod h1:jCnQMLj9eUgGU7+ludSTYoZL/GGmii14RxKFj7ROgHw= -github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= -github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= github.com/go-git/go-git/v5 v5.19.1 h1:nX27AnaU43/K5bKktKwgBmR9lawoYVe1Ckg0rgzzN00= github.com/go-git/go-git/v5 v5.19.1/go.mod h1:Pb1v0c7/g8aGQJwx9Us09W85yGoyvSwuhEGMH7zjDKQ= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -259,7 +215,6 @@ github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vb github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -303,8 +258,6 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= -github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -369,10 +322,6 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/go-github/v69 v69.2.0 h1:wR+Wi/fN2zdUx9YxSmYE0ktiX9IAR/BeePzeaUUbEHE= -github.com/google/go-github/v69 v69.2.0/go.mod h1:xne4jymxLR6Uj9b7J7PyTpkMYstEMMwGZa0Aehh1azM= -github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= -github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -386,18 +335,12 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 h1:z2ogiKUYzX5Is6zr/vP9vJGqPwcdqsWjOt+V8J7+bTc= github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= -github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/yamlfmt v0.21.0 h1:9FKApQkDpMKgBjwLFytBHUCgqnQgxaQnci0uiESfbzs= github.com/google/yamlfmt v0.21.0/go.mod h1:q6FYExB+Ueu7jZDjKECJk+EaeDXJzJ6Ne0dxx69GWfI= -github.com/googleapis/enterprise-certificate-proxy v0.3.12 h1:Fg+zsqzYEs1ZnvmcztTYxhgCBsx3eEhEwQ1W/lHq/sQ= -github.com/googleapis/enterprise-certificate-proxy v0.3.12/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.17.0 h1:RksgfBpxqff0EZkDWYuz9q/uWsTVz+kf43LsZ1J6SMc= -github.com/googleapis/gax-go/v2 v2.17.0/go.mod h1:mzaqghpQp4JDh3HvADwrat+6M3MOIDp5YKHhb9PAgDY= github.com/gordonklaus/ineffassign v0.2.0 h1:Uths4KnmwxNJNzq87fwQQDDnbNb7De00VOk9Nu0TySs= github.com/gordonklaus/ineffassign v0.2.0/go.mod h1:TIpymnagPSexySzs7F9FnO1XFTy8IT3a59vmZp5Y9Lw= github.com/gostaticanalysis/analysisutil v0.7.1 h1:ZMCjoue3DtDWQ5WyU16YbjbQEQ3VuzwxALrpYd+HeKk= @@ -453,15 +396,11 @@ github.com/julz/importas v0.2.0 h1:y+MJN/UdL63QbFJHws9BVC5RpA2iq0kpjrFajTGivjQ= github.com/julz/importas v0.2.0/go.mod h1:pThlt589EnCYtMnmhmRYY/qn9lCf/frPOK+WMx3xiJY= github.com/karamaru-alpha/copyloopvar v1.2.2 h1:yfNQvP9YaGQR7VaWLYcfZUlRP2eo2vhExWKxD/fP6q0= github.com/karamaru-alpha/copyloopvar v1.2.2/go.mod h1:oY4rGZqZ879JkJMtX3RRkcXRkmUvH0x35ykgaKgsgJY= -github.com/kevinburke/ssh_config v1.4.0 h1:6xxtP5bZ2E4NF5tuQulISpTO2z8XbtH8cg1PWkxoFkQ= -github.com/kevinburke/ssh_config v1.4.0/go.mod h1:q2RIzfka+BXARoNexmF9gkxEX7DmvbW9P4hIVx2Kg4M= github.com/kisielk/errcheck v1.10.0 h1:Lvs/YAHP24YKg08LA8oDw2z9fJVme090RAXd90S+rrw= github.com/kisielk/errcheck v1.10.0/go.mod h1:kQxWMMVZgIkDq7U8xtG/n2juOjbLgZtedi0D+/VL/i8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkHAIKE/contextcheck v1.1.6 h1:7HIyRcnyzxL9Lz06NGhiKvenXq7Zw6Q0UQu/ttjfJCE= github.com/kkHAIKE/contextcheck v1.1.6/go.mod h1:3dDbMRNBFaq8HFXWC1JyvDSPm43CmE6IuHam8Wr0rkg= -github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= -github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -562,14 +501,10 @@ github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3v github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= -github.com/pjbgf/sha1cd v0.6.0 h1:3WJ8Wz8gvDz29quX1OcEmkAlUg9diU4GxJHqs0/XiwU= -github.com/pjbgf/sha1cd v0.6.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H0iCM= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= -github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -637,13 +572,10 @@ github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOms github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/sivchari/containedctx v1.0.3 h1:x+etemjbsh2fB5ewm5FeLNi5bUjK0V8n0RB+Wwfd0XE= github.com/sivchari/containedctx v1.0.3/go.mod h1:c1RDvCbnJLtH4lLcYD/GqwiBSSf4F5Qk0xld2rBqzJ4= -github.com/skeema/knownhosts v1.3.2 h1:EDL9mgf4NzwMXCTfaxSD/o/a5fxDw/xL9nkU28JjdBg= -github.com/skeema/knownhosts v1.3.2/go.mod h1:bEg3iQAuw+jyiw+484wwFJoKSLwcfd7fqRy+N0QTiow= github.com/sonatard/noctx v0.5.1 h1:wklWg9c9ZYugOAk7qG4yP4PBrlQsmSLPTvW1K4PRQMs= github.com/sonatard/noctx v0.5.1/go.mod h1:64XdbzFb18XL4LporKXp8poqZtPKbCrqQ402CV+kJas= github.com/sourcegraph/go-diff v0.7.0 h1:9uLlrd5T46OXs5qpp8L/MTltk0zikUGi0sNNyCpA8G0= @@ -704,8 +636,6 @@ github.com/uudashr/gocognit v1.2.1 h1:CSJynt5txTnORn/DkhiB4mZjwPuifyASC8/6Q0I/QS github.com/uudashr/gocognit v1.2.1/go.mod h1:acaubQc6xYlXFEMb9nWX2dYBzJ/bIjEkc1zzvyIZg5Q= github.com/uudashr/iface v1.4.1 h1:J16Xl1wyNX9ofhpHmQ9h9gk5rnv2A6lX/2+APLTo0zU= github.com/uudashr/iface v1.4.1/go.mod h1:pbeBPlbuU2qkNDn0mmfrxP2X+wjPMIQAy+r1MBXSXtg= -github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= -github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xen0n/gosmopolitan v1.3.0 h1:zAZI1zefvo7gcpbCOrPSHJZJYA9ZgLfJqtKzZ5pHqQM= github.com/xen0n/gosmopolitan v1.3.0/go.mod h1:rckfr5T6o4lBtM1ga7mLGKZmLxswUoH1zxHgNXOsEt4= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= @@ -744,10 +674,6 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.65.0 h1:XmiuHzgJt067+a6kwyAzkhXooYVv3/TOw9cM2VfJgUM= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.65.0/go.mod h1:KDgtbWKTQs4bM+VPUr6WlL9m/WXcmkCcBlIzqxPGzmI= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 h1:7iP2uCb7sGddAr30RRS6xjKy7AZ2JtTOPA3oolgVSw8= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0/go.mod h1:c7hN3ddxs/z6q9xwvfLPk+UHlWRQyaeR1LdgfL/66l0= go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ= @@ -777,11 +703,8 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= -golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -860,7 +783,6 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= @@ -874,8 +796,6 @@ golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ= -golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -906,7 +826,6 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -938,7 +857,6 @@ golang.org/x/sys v0.0.0-20211105183446-c75c47738b0c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -955,8 +873,6 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= -golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY= -golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -972,8 +888,6 @@ golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= -golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -1052,8 +966,6 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.266.0 h1:hco+oNCf9y7DmLeAtHJi/uBAY7n/7XC9mZPxu1ROiyk= -google.golang.org/api v0.266.0/go.mod h1:Jzc0+ZfLnyvXma3UtaTl023TdhZu6OMBP9tJ+0EmFD0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= diff --git a/infra/imagebuilders/cli/Dockerfile b/infra/imagebuilders/cli/Dockerfile deleted file mode 100644 index 49d7659bb86..00000000000 --- a/infra/imagebuilders/cli/Dockerfile +++ /dev/null @@ -1,99 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This stage builds the librariangen binary using the MOSS-compliant base image. -FROM marketplace.gcr.io/google/debian12@sha256:326ccf35aa72f7cc8cbd25b69e819a81c5fd9ed675c6c9ffb51f3214a64f23cf AS builder - -# Set environment variables for tool versions for easy updates. -ENV GO_VERSION=1.25.3 - -# Install build dependencies. -RUN apt-get update && \ - apt-get install -y \ - build-essential \ - ca-certificates \ - curl \ - git \ - wget && \ - rm -rf /var/lib/apt/lists/* - -# Install the specific Go version required for compatibility. -RUN wget https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz -O go.tar.gz && \ - tar -C /usr/local -xzf go.tar.gz && \ - rm go.tar.gz -ENV PATH=/usr/local/go/bin:$PATH -RUN go version - -WORKDIR /src - -COPY go.mod . -COPY go.sum . -RUN go mod download - -COPY . . -# Renaming the binary to librarian to match the old binary name and avoid -# breaking existing Cloud Build steps that reference the binary by name. -RUN CGO_ENABLED=0 GOOS=linux go build -o librarian ./cmd/legacylibrarian - -# Using docker:dind so we can run docker from the CLI, -# while in Docker. Note that for this to work, *this* -# docker image should be run with -# -v /var/run/docker.sock:/var/run/docker.sock -FROM marketplace.gcr.io/google/debian12@sha256:326ccf35aa72f7cc8cbd25b69e819a81c5fd9ed675c6c9ffb51f3214a64f23cf -WORKDIR /app - -# From https://docs.docker.com/engine/install/debian/ - -RUN apt update -RUN apt-get install -y unzip \ - gnupg \ - apt-transport-https \ - ca-certificates \ - curl \ - && rm -rf /var/lib/apt/lists/* - -# Add Docker's official GPG key -RUN install -m 0755 -d /etc/apt/keyrings -RUN curl -fsSL --retry 5 --retry-delay 15 https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc -RUN chmod a+r /etc/apt/keyrings/docker.asc - -# Add the repository to Apt sources: -RUN echo \ - "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian \ - $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ - tee /etc/apt/sources.list.d/docker.list > /dev/null - -# Install Docker and pin version -# Cloud Build runs Docker Engine v20.10.24, which supports a maximum API version of v1.41. -# https://docs.cloud.google.com/build/docs/overview#docker -# Docker client v29.0.0 and newer require API version v1.44 or later. -# https://docs.docker.com/engine/release-notes/29/#2900 -RUN apt update && \ - apt-get -y install \ - docker-ce=5:27.1.1-1~debian.12~bookworm \ - docker-ce-cli=5:27.1.1-1~debian.12~bookworm \ - docker-ce-rootless-extras=5:27.1.1-1~debian.12~bookworm && \ - rm -rf /var/lib/apt/lists/* - -# Add the Google Cloud SDK distribution URI as a package source -RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" \ - > /etc/apt/sources.list.d/google-cloud-sdk.list && \ - curl -fsSL --retry 5 --retry-delay 15 https://packages.cloud.google.com/apt/doc/apt-key.gpg | \ - gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg - -# Install the gcloud CLI -RUN apt-get update && apt-get install -y google-cloud-sdk - -COPY --from=builder /src/librarian . -ENTRYPOINT ["/app/librarian"] diff --git a/infra/imagebuilders/cloudbuild-exitgate-dispatcher.yaml b/infra/imagebuilders/cloudbuild-exitgate-dispatcher.yaml deleted file mode 100644 index 2b8c2acaac7..00000000000 --- a/infra/imagebuilders/cloudbuild-exitgate-dispatcher.yaml +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# This Cloud Build configuration is used by a Louhi flow for the Artifact -# Registry (AR) Exit Gate process (go/cloud-sdk-ar-exit-gate-onboarding). -# -# This build step creates the librarian container image and publishes it to the -# 'images-dev' repository, which serves as the entry point for the AR Exit Gate. -# After passing the gate's security checks, the image is promoted and -# published to the 'images-prod' repository. -steps: - - id: build-dispatcher - name: 'gcr.io/cloud-builders/docker' - waitFor: ['-'] - args: - - 'build' - - '-t' - - 'us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-dev/librarian-dispatcher' - - '.' - - '-f' - - 'infra/imagebuilders/dispatcher/Dockerfile' -images: - - 'us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-dev/librarian-dispatcher' diff --git a/infra/imagebuilders/cloudbuild-exitgate-release-container.yaml b/infra/imagebuilders/cloudbuild-exitgate-release-container.yaml deleted file mode 100644 index a770ba98fb4..00000000000 --- a/infra/imagebuilders/cloudbuild-exitgate-release-container.yaml +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# This Cloud Build configuration is used by a Louhi flow for the Artifact -# Registry (AR) Exit Gate process (go/cloud-sdk-ar-exit-gate-onboarding). -# -# This build step creates the librarian container image and publishes it to the -# 'images-dev' repository, which serves as the entry point for the AR Exit Gate. -# After passing the gate's security checks, the image is promoted and -# published to the 'images-prod' repository. -steps: - - name: 'gcr.io/cloud-builders/docker' - waitFor: ['-'] - args: - - 'build' - - '-f' - - 'infra/imagebuilders/container/Dockerfile' - - '-t' - - 'us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-dev/librarian-release-container' - - '.' -images: - - 'us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-dev/librarian-release-container' diff --git a/infra/imagebuilders/cloudbuild-exitgate.yaml b/infra/imagebuilders/cloudbuild-exitgate.yaml deleted file mode 100644 index 179b36e6c6a..00000000000 --- a/infra/imagebuilders/cloudbuild-exitgate.yaml +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# This Cloud Build configuration is used by a Louhi flow for the Artifact -# Registry (AR) Exit Gate process (go/cloud-sdk-ar-exit-gate-onboarding). -# -# This build step creates the librarian container image and publishes it to the -# 'images-dev' repository, which serves as the entry point for the AR Exit Gate. -# After passing the gate's security checks, the image is promoted and -# published to the 'images-prod' repository. -steps: - - name: 'gcr.io/cloud-builders/docker' - waitFor: ['-'] - args: - - 'build' - - '-t' - - 'us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-dev/librarian' - - '.' - - '-f' - - 'infra/imagebuilders/cli/Dockerfile' - - id: structure-test - name: gcr.io/gcp-runtimes/structure_test - args: - - '-i' - - 'us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-dev/librarian' - - '--config' - - '/workspace/infra/imagebuilders/container-structure-test.yaml' - - '-v' -images: - - 'us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-dev/librarian' diff --git a/infra/imagebuilders/cloudbuild-test.yaml b/infra/imagebuilders/cloudbuild-test.yaml deleted file mode 100644 index 9b332ad509c..00000000000 --- a/infra/imagebuilders/cloudbuild-test.yaml +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# This Cloud Build configuration is used by a Louhi flow for the Artifact -# Registry (AR) Exit Gate process (go/cloud-sdk-ar-exit-gate-onboarding). -# -# This build step creates the librarian container image and publishes it to the -# 'images-dev' repository, which serves as the entry point for the AR Exit Gate. -# After passing the gate's security checks, the image is promoted and -# published to the 'images-prod' repository. -# -# Local test: -# gcloud --project= builds submit . --config=cloudbuild-test.yaml -steps: - - id: 'build' - name: 'gcr.io/cloud-builders/docker' - waitFor: ['-'] - args: - - 'build' - - '-t' - - 'us-central1-docker.pkg.dev/$PROJECT_ID/librarian' - - '.' - - '-f' - - 'infra/imagebuilders/cli/Dockerfile' - - id: structure-test - name: gcr.io/gcp-runtimes/structure_test - waitFor: ['build'] - args: - - '-i' - - 'us-central1-docker.pkg.dev/$PROJECT_ID/librarian' - - '--config' - - '/workspace/infra/imagebuilders/container-structure-test.yaml' - - '-v' - - id: 'build-dispatcher' - name: 'gcr.io/cloud-builders/docker' - waitFor: ['-'] - args: - - 'build' - - '-t' - - 'us-central1-docker.pkg.dev/$PROJECT_ID/librarian-dispatcher' - - '.' - - '-f' - - 'infra/imagebuilders/dispatcher/Dockerfile' -options: - logging: CLOUD_LOGGING_ONLY diff --git a/infra/imagebuilders/container-structure-test.yaml b/infra/imagebuilders/container-structure-test.yaml deleted file mode 100644 index 4cb155a8972..00000000000 --- a/infra/imagebuilders/container-structure-test.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -schemaVersion: 1.0.0 -commandTests: - - name: "librarian version" - command: ["/app/librarian", "version"] - expectedOutput: ["\\d+\\.\\d+\\.\\d+"] - - name: "docker" - command: ["docker", "version"] - expectedOutput: ["Docker Engine", "Version:"] - envVars: - # Pin the Docker API version to match the one supported by the - # Docker server in the Cloud Build environment. - - key: "DOCKER_API_VERSION" - value: "1.41" - - name: "gcloud" - command: ["gcloud", "--version"] - expectedOutput: ["Google Cloud SDK"] - - name: "git CLI" - command: ["git", "version"] - expectedOutput: ["git version 2.*"] diff --git a/infra/imagebuilders/container/Dockerfile b/infra/imagebuilders/container/Dockerfile deleted file mode 100644 index 5a0ccaadcb1..00000000000 --- a/infra/imagebuilders/container/Dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This stage builds the librariangen binary using the MOSS-compliant base image. -FROM marketplace.gcr.io/google/debian12@sha256:326ccf35aa72f7cc8cbd25b69e819a81c5fd9ed675c6c9ffb51f3214a64f23cf AS builder - -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - jq \ - && apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -WORKDIR /app -COPY infra/imagebuilders/container/run.sh /app/run.sh - -ENTRYPOINT [ "/app/run.sh" ] diff --git a/infra/imagebuilders/container/run.sh b/infra/imagebuilders/container/run.sh deleted file mode 100755 index 658431f4ff2..00000000000 --- a/infra/imagebuilders/container/run.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -echo "in release container" -cat /librarian/release-stage-request.json - -ls -al /librarian -ls -al /repo -ls -al /output - -new_version=$(jq -r '.libraries[0].version' /librarian/release-stage-request.json) -echo "release version: ${new_version}" -mkdir -p /output/internal/cli/ -echo "${new_version}" > /output/internal/cli/version.txt - -ls -al /output - -echo "writing empty response" -echo "{}" > /librarian/release-stage-response.json diff --git a/infra/imagebuilders/dispatcher/Dockerfile b/infra/imagebuilders/dispatcher/Dockerfile deleted file mode 100644 index f4f7f32f14a..00000000000 --- a/infra/imagebuilders/dispatcher/Dockerfile +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This stage builds the librariangen binary using the MOSS-compliant base image. -FROM marketplace.gcr.io/google/debian12@sha256:326ccf35aa72f7cc8cbd25b69e819a81c5fd9ed675c6c9ffb51f3214a64f23cf AS builder - -# Set environment variables for tool versions for easy updates. -ENV GO_VERSION=1.25.3 - -# Install build dependencies. -RUN apt-get update && \ - apt-get install -y \ - build-essential \ - ca-certificates \ - curl \ - wget && \ - rm -rf /var/lib/apt/lists/* - -# Install the specific Go version required for compatibility. -RUN wget https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz -O go.tar.gz && \ - tar -C /usr/local -xzf go.tar.gz && \ - rm go.tar.gz -ENV PATH=/usr/local/go/bin:$PATH -RUN go version - -WORKDIR /src - -COPY go.mod . -COPY go.sum . -RUN go mod download - -COPY . . -# Renaming the binary to automation to match the old binary name and avoid -# breaking existing Cloud Build steps that reference the binary by name. -RUN CGO_ENABLED=0 GOOS=linux go build -o automation ./cmd/legacyautomation - -FROM marketplace.gcr.io/google/debian12@sha256:326ccf35aa72f7cc8cbd25b69e819a81c5fd9ed675c6c9ffb51f3214a64f23cf -RUN apt-get update && \ - apt-get install -y ca-certificates && \ - rm -rf /var/lib/apt/lists/* -WORKDIR /app - -COPY --from=builder /src/automation . -ENTRYPOINT [ "/app/automation" ] diff --git a/infra/prod/generate-worker.yaml b/infra/prod/generate-worker.yaml deleted file mode 100644 index 663e760505d..00000000000 --- a/infra/prod/generate-worker.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# This runs the `librarian generate` command with a provided repository, -# secret name, and optional library ID -steps: - - name: us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/librarian-dispatcher - id: generate-dispatcher - args: - - '--project=$PROJECT_ID' - - '--command=generate' - - '--push=$_PUSH' - - '--build=$_BUILD' -tags: ['generate-dispatcher'] -options: - logging: CLOUD_LOGGING_ONLY diff --git a/infra/prod/generate.yaml b/infra/prod/generate.yaml deleted file mode 100644 index 0e98337655d..00000000000 --- a/infra/prod/generate.yaml +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# This runs the `librarian generate` command with a provided repository, -# secret name, and optional library ID -steps: - - name: gcr.io/cloud-builders/git - id: clone-language-repo - waitFor: ['-'] - args: - - 'clone' - - '--depth=1' - - '--branch=$_BRANCH' - - $_FULL_REPOSITORY - - name: gcr.io/cloud-builders/git - id: configure-language-repo-name - waitFor: ['clone-language-repo'] - dir: /workspace/$_REPOSITORY - args: - - 'config' - - 'user.name' - - 'Cloud SDK Librarian' - - name: gcr.io/cloud-builders/git - id: configure-language-repo-email - waitFor: ['configure-language-repo-name'] - dir: /workspace/$_REPOSITORY - args: - - 'config' - - 'user.email' - - 'cloud-sdk-librarian-robot@google.com' - - name: gcr.io/cloud-builders/git - id: clone-googleapis - waitFor: ['-'] - args: - - 'clone' - - '--single-branch' - - '--branch=master' - - https://github.com/googleapis/googleapis - - name: 'us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/librarian@$_IMAGE_SHA' - id: generate - waitFor: ['configure-language-repo-email', 'clone-googleapis'] - dir: tmp - args: - - 'generate' - - '-repo' - - '/workspace/$_REPOSITORY' - - '-branch=$_BRANCH' - - '-library' - - $_LIBRARY_ID - - '-api-source' - - '/workspace/googleapis' - - '-output' - - /workspace/tmp - - '-push=$_PUSH' - - '-build=$_BUILD' - secretEnv: ['LIBRARIAN_GITHUB_TOKEN'] -tags: ['generate-$_REPOSITORY'] -availableSecrets: - secretManager: - - versionName: projects/$PROJECT_ID/secrets/$_GITHUB_TOKEN_SECRET_NAME/versions/latest - env: 'LIBRARIAN_GITHUB_TOKEN' -options: - logging: CLOUD_LOGGING_ONLY - machineType: 'E2_HIGHCPU_8' -timeout: 10h diff --git a/infra/prod/publish-release-worker.yaml b/infra/prod/publish-release-worker.yaml deleted file mode 100644 index b8063de89d2..00000000000 --- a/infra/prod/publish-release-worker.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# This runs the `librarian release tag` command with a provided repository, -# secret name, and optional PR -steps: - - name: us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/librarian-dispatcher - id: publish-release-dispatcher - args: - - '--project=$PROJECT_ID' - - '--command=publish-release' - secretEnv: ['LIBRARIAN_GITHUB_TOKEN'] -tags: ['publish-release-dispatcher'] -availableSecrets: - secretManager: - - versionName: projects/$PROJECT_ID/secrets/$_GITHUB_TOKEN_SECRET_NAME/versions/latest - env: 'LIBRARIAN_GITHUB_TOKEN' -options: - logging: CLOUD_LOGGING_ONLY diff --git a/infra/prod/publish-release.yaml b/infra/prod/publish-release.yaml deleted file mode 100644 index a474702228e..00000000000 --- a/infra/prod/publish-release.yaml +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# This Cloud Build configuration is used by a Louhi flow for the Artifact -# Registry (AR) Exit Gate process (go/cloud-sdk-ar-exit-gate-onboarding). -# -# This runs the `librarian release tag` command with a provided repository, -# secret name, and optional PR -steps: - - name: gcr.io/cloud-builders/git - id: clone-language-repo - waitFor: ['-'] - args: - - 'clone' - - '--depth=1' - - '--single-branch' - - '--branch=$_BRANCH' - - $_FULL_REPOSITORY - - name: 'us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/librarian@$_IMAGE_SHA' - id: publish-release - waitFor: ['clone-language-repo'] - args: - - 'release' - - 'tag' - - '-repo' - - '/workspace/$_REPOSITORY' - - '-pr' - - $_PR - secretEnv: ['LIBRARIAN_GITHUB_TOKEN'] -tags: ['publish-release-$_REPOSITORY'] -availableSecrets: - secretManager: - - versionName: projects/$PROJECT_ID/secrets/$_GITHUB_TOKEN_SECRET_NAME/versions/latest - env: 'LIBRARIAN_GITHUB_TOKEN' -options: - logging: CLOUD_LOGGING_ONLY diff --git a/infra/prod/stage-release-worker.yaml b/infra/prod/stage-release-worker.yaml deleted file mode 100644 index 08c9a5113d9..00000000000 --- a/infra/prod/stage-release-worker.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# This runs the `librarian release stage` command with a provided repository, -# secret name, and optional library ID -steps: - - name: us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/librarian-dispatcher - id: stage-release-dispatcher - args: - - '--project=$PROJECT_ID' - - '--command=stage-release' - - '--push=$_PUSH' -tags: ['stage-release-dispatcher'] -options: - logging: CLOUD_LOGGING_ONLY diff --git a/infra/prod/stage-release.yaml b/infra/prod/stage-release.yaml deleted file mode 100644 index 30ecf41d666..00000000000 --- a/infra/prod/stage-release.yaml +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# This Cloud Build configuration is used by a Louhi flow for the Artifact -# Registry (AR) Exit Gate process (go/cloud-sdk-ar-exit-gate-onboarding). -# -# This runs the `librarian release stage` command with a provided repository, -# secret name, and optional library ID -steps: - - name: gcr.io/cloud-builders/git - id: clone-language-repo - waitFor: ['-'] - args: - - 'clone' - - '--single-branch' - - '--branch=$_BRANCH' - - $_FULL_REPOSITORY - - name: gcr.io/cloud-builders/git - id: configure-language-repo-name - waitFor: ['clone-language-repo'] - dir: /workspace/$_REPOSITORY - args: - - 'config' - - 'user.name' - - 'Cloud SDK Librarian' - - name: gcr.io/cloud-builders/git - id: configure-language-repo-email - waitFor: ['configure-language-repo-name'] - dir: /workspace/$_REPOSITORY - args: - - 'config' - - 'user.email' - - 'cloud-sdk-librarian-robot@google.com' - - name: 'us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/librarian@$_IMAGE_SHA' - id: stage-release - waitFor: ['configure-language-repo-email'] - dir: tmp - args: - - 'release' - - 'stage' - - '-repo' - - '/workspace/$_REPOSITORY' - - '-branch=$_BRANCH' - - '-library' - - $_LIBRARY_ID - - '-output' - - /workspace/tmp - - '-push=$_PUSH' - secretEnv: ['LIBRARIAN_GITHUB_TOKEN'] -tags: ['stage-release-$_REPOSITORY'] -availableSecrets: - secretManager: - - versionName: projects/$PROJECT_ID/secrets/$_GITHUB_TOKEN_SECRET_NAME/versions/latest - env: 'LIBRARIAN_GITHUB_TOKEN' -options: - logging: CLOUD_LOGGING_ONLY -timeout: 2h diff --git a/infra/prod/update-image-worker.yaml b/infra/prod/update-image-worker.yaml deleted file mode 100644 index 5c7ce287e2b..00000000000 --- a/infra/prod/update-image-worker.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# This triggers the `update-image` workflow for all repositories registered -# to automate the `librarian update-image` command -steps: - - name: us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/librarian-dispatcher - id: update-image-dispatcher - args: - - '--project=$PROJECT_ID' - - '--command=update-image' - - '--push=$_PUSH' - - '--build=$_BUILD' -tags: ['update-image-dispatcher'] -options: - logging: CLOUD_LOGGING_ONLY diff --git a/infra/prod/update-image.yaml b/infra/prod/update-image.yaml deleted file mode 100644 index 08c73e2a650..00000000000 --- a/infra/prod/update-image.yaml +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# This runs the `librarian update-image` command with a provided repository, -# secret name, and optional library ID -steps: - - name: gcr.io/cloud-builders/git - id: clone-language-repo - waitFor: ['-'] - args: - - 'clone' - - '--depth=1' - - '--branch=$_BRANCH' - - $_FULL_REPOSITORY - - name: gcr.io/cloud-builders/git - id: configure-language-repo-name - waitFor: ['clone-language-repo'] - dir: /workspace/$_REPOSITORY - args: - - 'config' - - 'user.name' - - 'Cloud SDK Librarian' - - name: gcr.io/cloud-builders/git - id: configure-language-repo-email - waitFor: ['configure-language-repo-name'] - dir: /workspace/$_REPOSITORY - args: - - 'config' - - 'user.email' - - 'cloud-sdk-librarian-robot@google.com' - - name: gcr.io/cloud-builders/git - id: clone-googleapis - waitFor: ['-'] - args: - - 'clone' - - '--single-branch' - - '--branch=master' - - https://github.com/googleapis/googleapis - - name: 'us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/librarian@$_IMAGE_SHA' - id: update-image - waitFor: ['configure-language-repo-email', 'clone-googleapis'] - dir: tmp - args: - - 'update-image' - - '-repo' - - '/workspace/$_REPOSITORY' - - '-api-source' - - '/workspace/googleapis' - - '-branch=$_BRANCH' - - '-commit=true' - - '-push=$_PUSH' - - '-build=$_BUILD' - secretEnv: ['LIBRARIAN_GITHUB_TOKEN'] -tags: ['update-image-$_REPOSITORY'] -availableSecrets: - secretManager: - - versionName: projects/$PROJECT_ID/secrets/$_GITHUB_TOKEN_SECRET_NAME/versions/latest - env: 'LIBRARIAN_GITHUB_TOKEN' -options: - logging: CLOUD_LOGGING_ONLY -timeout: 10h diff --git a/infra/test/librarian-image-test.yaml b/infra/test/librarian-image-test.yaml deleted file mode 100644 index 82574579496..00000000000 --- a/infra/test/librarian-image-test.yaml +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -steps: - - id: get-librarian-image-sha - name: bash - args: - - '-c' - - | - wget https://github.com/mikefarah/yq/releases/download/v4.48.2/yq_linux_amd64 -O /usr/local/bin/yq - chmod +x /usr/local/bin/yq - yq -r '.librarian-image-sha' internal/legacylibrarian/legacyautomation/prod/repositories.yaml > /workspace/librarian_image_sha.txt - cat /workspace/librarian_image_sha.txt - - id: test-generate - name: gcr.io/cloud-builders/docker - entrypoint: bash - waitFor: ['get-librarian-image-sha'] - args: - - -c - - | - docker run \ - --rm \ - us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/librarian@$(cat /workspace/librarian_image_sha.txt) \ - generate --help - - id: test-release-stage - name: gcr.io/cloud-builders/docker - entrypoint: bash - waitFor: ['get-librarian-image-sha'] - args: - - -c - - | - docker run \ - --rm \ - us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/librarian@$(cat /workspace/librarian_image_sha.txt) \ - release stage --help - - id: test-release-tag - name: gcr.io/cloud-builders/docker - entrypoint: bash - waitFor: ['get-librarian-image-sha'] - args: - - -c - - | - docker run \ - --rm \ - us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/librarian@$(cat /workspace/librarian_image_sha.txt) \ - release tag --help -options: - logging: CLOUD_LOGGING_ONLY diff --git a/infra/test/token-access-test.yaml b/infra/test/token-access-test.yaml deleted file mode 100644 index 771505d97f5..00000000000 --- a/infra/test/token-access-test.yaml +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# This Cloud Build configuration is used by a Louhi flow for the Artifact -# Registry (AR) Exit Gate process (go/cloud-sdk-ar-exit-gate-onboarding). -# -# This runs the `librarian generate` command with a provided repository, -# secret name, and optional library ID -steps: - - name: 'gcr.io/cloud-builders/gcloud-slim' - id: validate-credentials - waitFor: ['-'] - script: | - #!/usr/bin/env bash - - # "-o pipefail" stops execution of the piped command fails, especially - # the body of the while loop in this case. - set -eo pipefail - echo "gcloud config get-value project:" - gcloud config get-value project - echo "gcloud config get-value core/account:" - gcloud config get-value core/account - echo "pwd is $(pwd)" - echo "ls -la ." - ls -la . - ROBOT_ACCOUNT=cloud-sdk-librarian-robot - if [[ $- == *x* ]]; then - echo "xtrace is ON. Exiting to avoid credentials showing up in logs." - exit 1 - fi - sed -n 's/.* - name: \"\(.*\)".*/\1/p' internal/legacylibrarian/legacyautomation/prod/repositories.yaml | while read -r repo_name; do - echo "Validating credentials for repository: ${repo_name}" - GITHUB_TOKEN=$(gcloud secrets versions access latest --secret="${repo_name}-github-token") - if [[ -z "${GITHUB_TOKEN}" ]]; then - echo "GitHub token for repository ${repo_name} is not set." - exit 1 - fi - permission_url="https://api.github.com/repos/googleapis/${repo_name}/collaborators/${ROBOT_ACCOUNT}/permission" - curl --retry 5 --retry-delay 15 --fail -H "Authorization: token ${GITHUB_TOKEN}" "${permission_url}" - if [[ $? -ne 0 ]]; then - echo "Failed to validate credentials for repository: ${repo_name} via ${permission_url}" - exit 1 - fi - done - echo "Finished validating credentials." -options: - logging: CLOUD_LOGGING_ONLY diff --git a/internal/legacylibrarian/legacyautomation/automation.go b/internal/legacylibrarian/legacyautomation/automation.go deleted file mode 100644 index 10ab92ccf55..00000000000 --- a/internal/legacylibrarian/legacyautomation/automation.go +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacyautomation - -import ( - "context" - - "github.com/googleapis/librarian/internal/legacylibrarian/legacycli" -) - -func newAutomationCommand() *legacycli.Command { - commands := []*legacycli.Command{ - newCmdGenerate(), - newCmdPublishRelease(), - newCmdStageRelease(), - } - - return legacycli.NewCommandSet( - commands, - "automation manages Cloud Build resources to run Librarian CLI.", - "automation [arguments]", - automationLongHelp) -} - -func newCmdGenerate() *legacycli.Command { - cmdGenerate := &legacycli.Command{ - Short: "generate", - UsageLine: "automation generate [flags]", - Long: generateLongHelp, - Action: func(ctx context.Context, cmd *legacycli.Command) error { - runner := newGenerateRunner(cmd.Config) - return runner.run(ctx) - }, - } - - cmdGenerate.Init() - addFlagBuild(cmdGenerate.Flags, cmdGenerate.Config) - addFlagProject(cmdGenerate.Flags, cmdGenerate.Config) - addFlagPush(cmdGenerate.Flags, cmdGenerate.Config) - - return cmdGenerate -} - -func newCmdPublishRelease() *legacycli.Command { - cmdPublishRelease := &legacycli.Command{ - Short: "publish-release", - UsageLine: "automation publish-release [flags]", - Long: publishLongHelp, - Action: func(ctx context.Context, cmd *legacycli.Command) error { - runner := newPublishRunner(cmd.Config) - return runner.run(ctx) - }, - } - - cmdPublishRelease.Init() - addFlagProject(cmdPublishRelease.Flags, cmdPublishRelease.Config) - - return cmdPublishRelease -} - -func newCmdStageRelease() *legacycli.Command { - cmdStageRelease := &legacycli.Command{ - Short: "stage-release", - UsageLine: "automation stage-release [flags]", - Long: stageLongHelp, - Action: func(ctx context.Context, cmd *legacycli.Command) error { - runner := newStageRunner(cmd.Config) - return runner.run(ctx) - }, - } - - cmdStageRelease.Init() - addFlagProject(cmdStageRelease.Flags, cmdStageRelease.Config) - addFlagPush(cmdStageRelease.Flags, cmdStageRelease.Config) - - return cmdStageRelease -} diff --git a/internal/legacylibrarian/legacyautomation/automation_test.go b/internal/legacylibrarian/legacyautomation/automation_test.go deleted file mode 100644 index ba40fbf7054..00000000000 --- a/internal/legacylibrarian/legacyautomation/automation_test.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacyautomation - -import "testing" - -func TestAutomationCmdRun(t *testing.T) { - cmd := newAutomationCommand() - if err := cmd.Run(t.Context(), []string{"version"}); err != nil { - t.Fatal(err) - } -} diff --git a/internal/legacylibrarian/legacyautomation/cli.go b/internal/legacylibrarian/legacyautomation/cli.go deleted file mode 100644 index f5a55fbb285..00000000000 --- a/internal/legacylibrarian/legacyautomation/cli.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package legacyautomation implements the command-line interface and core logic -// for Librarian's automated workflows. -package legacyautomation - -import ( - "context" - "flag" -) - -// runCommandFn is a function type that matches RunCommand, for mocking in tests. -var runCommandFn = RunCommand - -// Run parses the command line arguments and triggers the specified command. -func Run(ctx context.Context, args []string) error { - // TODO(https://github.com/googleapis/librarian/issues/2889) refactor this function after all commands are migrated. - if len(args) == 0 || args[0] == "version" || args[0] == generateCmdName || args[0] == publishCmdName || args[0] == stageCmdName { - cmd := newAutomationCommand() - return cmd.Run(ctx, args) - } - - options, err := parseFlags(args) - if err != nil { - return err - } - - err = runCommandFn(ctx, options.Command, options.ProjectId, options.Push, options.Build) - if err != nil { - return err - } - return nil -} - -type runOptions struct { - Command string - ProjectId string - Push bool - Build bool -} - -func parseFlags(args []string) (*runOptions, error) { - flagSet := flag.NewFlagSet("dispatcher", flag.ContinueOnError) - projectId := flagSet.String("project", "cloud-sdk-librarian-prod", "GCP project ID") - command := flagSet.String("command", "generate", "The librarian command to run") - push := flagSet.Bool("push", true, "The _PUSH flag (true/false) to Librarian CLI's -push option") - build := flagSet.Bool("build", true, "The _BUILD flag (true/false) to Librarian CLI's -build option") - err := flagSet.Parse(args) - if err != nil { - return nil, err - } - return &runOptions{ - ProjectId: *projectId, - Command: *command, - Push: *push, - Build: *build, - }, nil -} diff --git a/internal/legacylibrarian/legacyautomation/cli_test.go b/internal/legacylibrarian/legacyautomation/cli_test.go deleted file mode 100644 index 2d09af0e215..00000000000 --- a/internal/legacylibrarian/legacyautomation/cli_test.go +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacyautomation - -import ( - "context" - "errors" - "testing" - - "github.com/google/go-cmp/cmp" -) - -func TestRun(t *testing.T) { - - tests := []struct { - name string - args []string - runCommandErr error - wantErr bool - }{ - { - name: "success", - args: []string{"--command=generate"}, - wantErr: false, - }, - { - name: "error parsing flags", - args: []string{"--unknown-flag"}, - wantErr: true, - }, - { - name: "error from RunCommand", - args: []string{"--command=generate"}, - runCommandErr: errors.New("run command failed"), - wantErr: true, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - runCommandFn = func(ctx context.Context, command string, projectId string, push bool, build bool) error { - return test.runCommandErr - } - if err := Run(context.Background(), test.args); (err != nil) != test.wantErr { - t.Errorf("Run() error = %v, wantErr %v", err, test.wantErr) - } - }) - } -} - -func TestParseArgs(t *testing.T) { - for _, test := range []struct { - name string - args []string - want *runOptions - wantErr bool - }{ - { - name: "parses defaults", - args: []string{}, - wantErr: false, - want: &runOptions{ - Command: "generate", - ProjectId: "cloud-sdk-librarian-prod", - Push: true, - Build: true, - }, - }, - { - name: "sets project", - args: []string{"--project=some-project-id"}, - wantErr: false, - want: &runOptions{ - Command: "generate", - ProjectId: "some-project-id", - Push: true, - Build: true, - }, - }, - { - name: "sets command", - args: []string{"--command=stage-release"}, - wantErr: false, - want: &runOptions{ - Command: "stage-release", - ProjectId: "cloud-sdk-librarian-prod", - Push: true, - Build: true, - }, - }, - { - name: "sets command", - args: []string{"--command=stage-release", "--push=false"}, - wantErr: false, - want: &runOptions{ - Command: "stage-release", - ProjectId: "cloud-sdk-librarian-prod", - Push: false, - Build: true, - }, - }, - { - name: "sets build", - args: []string{"--command=generate", "--build=false"}, - wantErr: false, - want: &runOptions{ - Command: "generate", - ProjectId: "cloud-sdk-librarian-prod", - Push: true, - Build: false, - }, - }, - } { - t.Run(test.name, func(t *testing.T) { - got, err := parseFlags(test.args) - if test.wantErr && err == nil { - t.Fatal("expected error, but did not return one") - } else if !test.wantErr && err != nil { - t.Errorf("did not expect error, but received one: %s", err) - } - if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("mismatch (-want +got):\n%s", diff) - } - }) - } -} diff --git a/internal/legacylibrarian/legacyautomation/cloudbuild.go b/internal/legacylibrarian/legacyautomation/cloudbuild.go deleted file mode 100644 index f09f6a41f6e..00000000000 --- a/internal/legacylibrarian/legacyautomation/cloudbuild.go +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacyautomation - -import ( - "context" - "fmt" - "iter" - - "cloud.google.com/go/cloudbuild/apiv1/v2/cloudbuildpb" - "github.com/googleapis/gax-go/v2" - "golang.org/x/exp/slog" -) - -// CloudBuildClient is an interface for mocking calls to Cloud Build. -type CloudBuildClient interface { - RunBuildTrigger(ctx context.Context, req *cloudbuildpb.RunBuildTriggerRequest, opts ...gax.CallOption) error - ListBuildTriggers(ctx context.Context, req *cloudbuildpb.ListBuildTriggersRequest, opts ...gax.CallOption) iter.Seq2[*cloudbuildpb.BuildTrigger, error] -} - -func runCloudBuildTriggerByName(ctx context.Context, c CloudBuildClient, projectId string, location string, triggerName string, substitutions map[string]string) error { - triggerId, err := findTriggerIdByName(ctx, c, projectId, location, triggerName) - if err != nil { - return fmt.Errorf("error finding triggerid: %w", err) - } - slog.Info("found triggerId", slog.String("triggerId", triggerId)) - return runCloudBuildTrigger(ctx, c, projectId, location, triggerId, substitutions) -} - -func findTriggerIdByName(ctx context.Context, c CloudBuildClient, projectId string, location string, triggerName string) (string, error) { - slog.Info("looking for triggerId by name", - slog.String("projectId", projectId), - slog.String("location", location), - slog.String("triggerName", triggerName), - ) - req := &cloudbuildpb.ListBuildTriggersRequest{ - Parent: fmt.Sprintf("projects/%s/locations/%s", projectId, location), - } - for resp, err := range c.ListBuildTriggers(ctx, req) { - if err != nil { - return "", fmt.Errorf("error running trigger %w", err) - } - if resp.Name == triggerName { - return resp.Id, nil - } - } - return "", fmt.Errorf("could not find trigger id") -} - -func runCloudBuildTrigger(ctx context.Context, c CloudBuildClient, projectId string, location string, triggerId string, substitutions map[string]string) error { - triggerName := fmt.Sprintf("projects/%s/locations/%s/triggers/%s", projectId, location, triggerId) - req := &cloudbuildpb.RunBuildTriggerRequest{ - Name: triggerName, - ProjectId: projectId, - TriggerId: triggerId, - Source: &cloudbuildpb.RepoSource{ - Substitutions: substitutions, - }, - } - slog.Info("triggering", slog.String("triggerName", triggerName), slog.String("triggerId", triggerId)) - err := c.RunBuildTrigger(ctx, req) - if err != nil { - return fmt.Errorf("error running trigger %w", err) - } - return nil -} diff --git a/internal/legacylibrarian/legacyautomation/cloudbuild_test.go b/internal/legacylibrarian/legacyautomation/cloudbuild_test.go deleted file mode 100644 index f1d1895f5db..00000000000 --- a/internal/legacylibrarian/legacyautomation/cloudbuild_test.go +++ /dev/null @@ -1,225 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacyautomation - -import ( - "context" - "fmt" - "iter" - "testing" - - "cloud.google.com/go/cloudbuild/apiv1/v2/cloudbuildpb" - "github.com/google/go-cmp/cmp" - "github.com/googleapis/gax-go/v2" - "golang.org/x/exp/slog" -) - -type mockCloudBuildClient struct { - runError error - buildTriggers []*cloudbuildpb.BuildTrigger - triggersRun []string - substitutions []map[string]string -} - -func (c *mockCloudBuildClient) RunBuildTrigger(ctx context.Context, req *cloudbuildpb.RunBuildTriggerRequest, opts ...gax.CallOption) error { - slog.Info("running fake RunBuildTrigger") - if c.runError != nil { - return c.runError - } - for _, t := range c.triggersRun { - if t == req.TriggerId { - return nil - } - } - c.triggersRun = append(c.triggersRun, req.TriggerId) - c.substitutions = append(c.substitutions, req.GetSource().GetSubstitutions()) - return nil -} - -func (c *mockCloudBuildClient) ListBuildTriggers(ctx context.Context, req *cloudbuildpb.ListBuildTriggersRequest, opts ...gax.CallOption) iter.Seq2[*cloudbuildpb.BuildTrigger, error] { - return func(yield func(*cloudbuildpb.BuildTrigger, error) bool) { - for _, v := range c.buildTriggers { - var err error - if c.runError != nil { - v = nil - err = c.runError - } - if !yield(v, err) { - return // Stop iteration if yield returns false - } - } - } -} - -func TestRunCloudBuildTrigger(t *testing.T) { - for _, test := range []struct { - name string - runError error - wantErr bool - }{ - { - name: "pass", - wantErr: false, - }, - { - name: "error", - runError: fmt.Errorf("some-error"), - wantErr: true, - }, - } { - t.Run(test.name, func(t *testing.T) { - ctx := t.Context() - client := &mockCloudBuildClient{ - runError: test.runError, - buildTriggers: make([]*cloudbuildpb.BuildTrigger, 0), - } - substitutions := make(map[string]string) - err := runCloudBuildTrigger(ctx, client, "some-project", "some-location", "some-trigger-id", substitutions) - if diff := cmp.Diff(test.wantErr, err != nil); diff != "" { - t.Errorf("mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestFindTriggerIdByName(t *testing.T) { - for _, test := range []struct { - name string - want string - runError error - wantErr bool - buildTriggers []*cloudbuildpb.BuildTrigger - }{ - { - name: "finds trigger", - want: "some-trigger-id", - wantErr: false, - buildTriggers: []*cloudbuildpb.BuildTrigger{ - { - Name: "different-trigger", - Id: "different-trigger-id", - }, - { - Name: "some-trigger-name", - Id: "some-trigger-id", - }, - }, - }, - { - name: "not found", - want: "", - wantErr: true, - buildTriggers: []*cloudbuildpb.BuildTrigger{ - { - Name: "different-trigger", - Id: "different-trigger-id", - }, - }, - }, - { - name: "runtime error", - want: "", - runError: fmt.Errorf("some-error"), - wantErr: true, - buildTriggers: []*cloudbuildpb.BuildTrigger{ - { - Name: "different-trigger", - Id: "different-trigger-id", - }, - { - Name: "some-trigger-name", - Id: "some-trigger-id", - }, - }, - }, - } { - t.Run(test.name, func(t *testing.T) { - ctx := t.Context() - client := &mockCloudBuildClient{ - runError: test.runError, - buildTriggers: test.buildTriggers, - } - triggerId, err := findTriggerIdByName(ctx, client, "some-project", "some-location", "some-trigger-name") - if diff := cmp.Diff(test.wantErr, err != nil); diff != "" { - t.Errorf("mismatch (-want +got):\n%s", diff) - } - if diff := cmp.Diff(test.want, triggerId); diff != "" { - t.Errorf("mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestRunCloudBuildTriggerByName(t *testing.T) { - for _, test := range []struct { - name string - want string - runError error - wantErr bool - buildTriggers []*cloudbuildpb.BuildTrigger - }{ - { - name: "finds trigger", - wantErr: false, - buildTriggers: []*cloudbuildpb.BuildTrigger{ - { - Name: "different-trigger", - Id: "different-trigger-id", - }, - { - Name: "some-trigger-name", - Id: "some-trigger-id", - }, - }, - }, - { - name: "not found", - wantErr: true, - buildTriggers: []*cloudbuildpb.BuildTrigger{ - { - Name: "different-trigger", - Id: "different-trigger-id", - }, - }, - }, - { - name: "runtime error", - runError: fmt.Errorf("some-error"), - wantErr: true, - buildTriggers: []*cloudbuildpb.BuildTrigger{ - { - Name: "different-trigger", - Id: "different-trigger-id", - }, - { - Name: "some-trigger-name", - Id: "some-trigger-id", - }, - }, - }, - } { - t.Run(test.name, func(t *testing.T) { - ctx := t.Context() - client := &mockCloudBuildClient{ - runError: test.runError, - buildTriggers: test.buildTriggers, - } - err := runCloudBuildTriggerByName(ctx, client, "some-project", "some-location", "some-trigger-name", make(map[string]string)) - if diff := cmp.Diff(test.wantErr, err != nil); diff != "" { - t.Errorf("mismatch (-want +got):\n%s", diff) - } - }) - } -} diff --git a/internal/legacylibrarian/legacyautomation/flags.go b/internal/legacylibrarian/legacyautomation/flags.go deleted file mode 100644 index 263e9840079..00000000000 --- a/internal/legacylibrarian/legacyautomation/flags.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacyautomation - -import ( - "flag" - - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" -) - -func addFlagBuild(fs *flag.FlagSet, cfg *legacyconfig.Config) { - fs.BoolVar(&cfg.Build, "build", false, "The _BUILD flag (true/false) to Librarian CLI's -build option") -} - -func addFlagProject(fs *flag.FlagSet, cfg *legacyconfig.Config) { - fs.StringVar(&cfg.Project, "project", "cloud-sdk-librarian-prod", "Google Cloud Platform project ID") -} - -func addFlagPush(fs *flag.FlagSet, cfg *legacyconfig.Config) { - fs.BoolVar(&cfg.Push, "push", false, "The _PUSH flag (true/false) to Librarian CLI's -push option") -} diff --git a/internal/legacylibrarian/legacyautomation/generate.go b/internal/legacylibrarian/legacyautomation/generate.go deleted file mode 100644 index 51cc9fe4f27..00000000000 --- a/internal/legacylibrarian/legacyautomation/generate.go +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacyautomation - -import ( - "context" - - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" -) - -const ( - generateCmdName = "generate" -) - -type generateRunner struct { - build bool - projectID string - push bool -} - -func newGenerateRunner(cfg *legacyconfig.Config) *generateRunner { - return &generateRunner{ - build: cfg.Build, - projectID: cfg.Project, - push: cfg.Push, - } -} - -func (r *generateRunner) run(ctx context.Context) error { - // TODO(https://github.com/googleapis/librarian/issues/2890): refactor this function after all commands are migrated. - return runCommandFn(ctx, generateCmdName, r.projectID, r.push, r.build) -} diff --git a/internal/legacylibrarian/legacyautomation/generate_test.go b/internal/legacylibrarian/legacyautomation/generate_test.go deleted file mode 100644 index 2565a9d5b73..00000000000 --- a/internal/legacylibrarian/legacyautomation/generate_test.go +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacyautomation - -import ( - "context" - "errors" - "testing" - - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" -) - -func TestNewGenerateRunner(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - cfg *legacyconfig.Config - }{ - { - name: "create_a_runner", - cfg: &legacyconfig.Config{ - Build: true, - Project: "example-project", - Push: true, - }, - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - runner := newGenerateRunner(test.cfg) - if runner.build != test.cfg.Build { - t.Errorf("newGenerateRunner() build is not set") - } - if runner.projectID != test.cfg.Project { - t.Errorf("newGenerateRunner() projectID is not set") - } - if runner.push != test.cfg.Push { - t.Errorf("newGenerateRunner() push is not set") - } - }) - } -} - -func TestGenerateRunnerRun(t *testing.T) { - originalRunCommandFn := runCommandFn - defer func() { runCommandFn = originalRunCommandFn }() - - tests := []struct { - name string - runner *generateRunner - runCommandErr error - wantErr bool - wantCmd string - wantProjectID string - wantPush bool - wantBuild bool - }{ - { - name: "success", - runner: &generateRunner{ - build: true, - projectID: "test-project", - push: true, - }, - wantCmd: generateCmdName, - wantProjectID: "test-project", - wantPush: true, - wantBuild: true, - }, - { - name: "error from RunCommand", - runner: &generateRunner{}, - runCommandErr: errors.New("run command failed"), - wantErr: true, - wantCmd: generateCmdName, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - runCommandFn = func(ctx context.Context, command string, projectId string, push bool, build bool) error { - if command != test.wantCmd { - t.Errorf("runCommandFn() command = %v, want %v", command, test.wantCmd) - } - // Only check other args on success case to avoid nil pointer with empty runner - if test.runCommandErr == nil { - if projectId != test.wantProjectID { - t.Errorf("runCommandFn() projectId = %v, want %v", projectId, test.wantProjectID) - } - if push != test.wantPush { - t.Errorf("runCommandFn() push = %v, want %v", push, test.wantPush) - } - if build != test.wantBuild { - t.Errorf("runCommandFn() build = %v, want %v", build, test.wantBuild) - } - } - return test.runCommandErr - } - - if err := test.runner.run(t.Context()); (err != nil) != test.wantErr { - t.Errorf("run() error = %v, wantErr %v", err, test.wantErr) - } - }) - } -} diff --git a/internal/legacylibrarian/legacyautomation/help.go b/internal/legacylibrarian/legacyautomation/help.go deleted file mode 100644 index ce43ff050aa..00000000000 --- a/internal/legacylibrarian/legacyautomation/help.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacyautomation - -const ( - automationLongHelp = `Automation provides logic to trigger Cloud Build jobs that run Librarian commands for -any repository listed in internal/legacylibrarian/legacyautomation/prod/repositories.yaml.` - generateLongHelp = `The generate command triggers a Cloud Build job that runs librarian generate command for every -repository onboarded to Librarian generate automation.` - publishLongHelp = `The publish-release command triggers a Cloud Build job that runs librarian release tag command -for every repository onboarded to Librarian publish-release automation.` - stageLongHelp = `The stage-release command triggers a Cloud Build job that runs librarian release stage command for -every repository onboarded to Librarian stage-release automation.` -) diff --git a/internal/legacylibrarian/legacyautomation/prod/repositories.yaml b/internal/legacylibrarian/legacyautomation/prod/repositories.yaml deleted file mode 100644 index aade9a63427..00000000000 --- a/internal/legacylibrarian/legacyautomation/prod/repositories.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# SHA corresponds https://github.com/googleapis/librarian/commit/2e86731a438a346f789999998944bc4085c1cab6. -librarian-image-sha: sha256:7860dfbb8432b6292c493d7c06589a713e52bcaef9e08e2564d3b5f9e66051c7 -repositories: - - name: "gapic-generator-go" - full-name: https://github.com/googleapis/gapic-generator-go - github-token-secret-name: "gapic-generator-go-github-token" - supported-commands: - - publish-release - - name: "gax-go" - full-name: https://github.com/googleapis/gax-go - github-token-secret-name: "gax-go-github-token" - supported-commands: - - publish-release - - name: "google-cloud-go" - full-name: https://github.com/googleapis/google-cloud-go - github-token-secret-name: "google-cloud-go-github-token" - supported-commands: - - stage-release - - publish-release - - name: "google-cloud-python" - full-name: https://github.com/googleapis/google-cloud-python - github-token-secret-name: "google-cloud-python-github-token" - supported-commands: - - stage-release - - publish-release diff --git a/internal/legacylibrarian/legacyautomation/publish_release.go b/internal/legacylibrarian/legacyautomation/publish_release.go deleted file mode 100644 index 100c5630df0..00000000000 --- a/internal/legacylibrarian/legacyautomation/publish_release.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacyautomation - -import ( - "context" - - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" -) - -const ( - publishCmdName = "publish-release" -) - -type publishRunner struct { - projectID string -} - -func newPublishRunner(cfg *legacyconfig.Config) *publishRunner { - return &publishRunner{ - projectID: cfg.Project, - } -} - -func (r *publishRunner) run(ctx context.Context) error { - return runCommandFn(ctx, publishCmdName, r.projectID, false, false) -} diff --git a/internal/legacylibrarian/legacyautomation/publish_release_test.go b/internal/legacylibrarian/legacyautomation/publish_release_test.go deleted file mode 100644 index 7b3b4fb2b5d..00000000000 --- a/internal/legacylibrarian/legacyautomation/publish_release_test.go +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacyautomation - -import ( - "context" - "errors" - "testing" - - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" -) - -func TestNewPublishRunner(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - cfg *legacyconfig.Config - }{ - { - name: "create_a_runner", - cfg: &legacyconfig.Config{ - Project: "example-project", - }, - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - runner := newPublishRunner(test.cfg) - if runner.projectID != test.cfg.Project { - t.Errorf("newPublishRunner() projectID is not set") - } - }) - } -} - -func TestPublishRunnerRun(t *testing.T) { - tests := []struct { - name string - runner *publishRunner - runCommandErr error - wantErr bool - wantCmd string - wantProjectID string - wantPush bool - wantBuild bool - }{ - { - name: "success", - runner: &publishRunner{ - projectID: "test-project", - }, - wantCmd: publishCmdName, - wantProjectID: "test-project", - }, - { - name: "error from RunCommand", - runner: &publishRunner{}, - runCommandErr: errors.New("run command failed"), - wantErr: true, - wantCmd: publishCmdName, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - runCommandFn = func(ctx context.Context, command string, projectId string, push bool, build bool) error { - if command != test.wantCmd { - t.Errorf("runCommandFn() command = %v, want %v", command, test.wantCmd) - } - // Only check other args on success case to avoid nil pointer with empty runner - if test.runCommandErr == nil { - if projectId != test.wantProjectID { - t.Errorf("runCommandFn() projectId = %v, want %v", projectId, test.wantProjectID) - } - } - return test.runCommandErr - } - - if err := test.runner.run(t.Context()); (err != nil) != test.wantErr { - t.Errorf("run() error = %v, wantErr %v", err, test.wantErr) - } - }) - } -} diff --git a/internal/legacylibrarian/legacyautomation/repositories.go b/internal/legacylibrarian/legacyautomation/repositories.go deleted file mode 100644 index 5ea37bf49ee..00000000000 --- a/internal/legacylibrarian/legacyautomation/repositories.go +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacyautomation - -import ( - "errors" - "fmt" - "slices" - - "gopkg.in/yaml.v3" - - _ "embed" -) - -//go:embed prod/repositories.yaml -var prodRepositoriesYaml []byte - -var errImageSHANotFound = errors.New("image SHA not found") - -var availableCommands = map[string]bool{ - "generate": true, - "stage-release": true, - "publish-release": true, - "update-image": true, -} - -// RepositoryConfig represents a single registered librarian GitHub repository. -type RepositoryConfig struct { - Name string `yaml:"name"` - FullName string `yaml:"full-name"` - SecretName string `yaml:"github-token-secret-name"` - SupportedCommands []string `yaml:"supported-commands"` - - // Branch configures the repository branch to checkout in Cloud Build prior - // to command execution. Furthermore, it dictates the value set on the - // [github.com/googleapis/librarian/internal/legacyconfig.Config] Branch property - // via the --branch flag. - // - // This property is optional. Downstream usage defaults to "main". - Branch string `yaml:"branch"` -} - -// RepositoriesConfig represents all the registered librarian GitHub repositories. -type RepositoriesConfig struct { - ImageSHA string `yaml:"librarian-image-sha"` - Repositories []*RepositoryConfig `yaml:"repositories"` -} - -// GitURL returns the full git url to clone. If full name is not available, -// it defaults to "https://github.com/googleapis/". -func (c *RepositoryConfig) GitURL() (string, error) { - if c.FullName == "" { - if c.Name == "" { - return "", fmt.Errorf("name or full name is required") - } - return fmt.Sprintf("https://github.com/googleapis/%s", c.Name), nil - } - return c.FullName, nil -} - -// Validate checks the RepositoryConfig is valid. -func (c *RepositoryConfig) Validate() error { - if c.FullName == "" && c.Name == "" { - return fmt.Errorf("name or full name is required") - } - if c.SecretName == "" { - return fmt.Errorf("secret name is required") - } - if len(c.SupportedCommands) == 0 { - return fmt.Errorf("supported commands cannot be empty") - } - for _, command := range c.SupportedCommands { - if !availableCommands[command] { - return fmt.Errorf("unsupported command: %s", command) - } - } - return nil -} - -// Validate checks the RepositoriesConfig is valid. -func (c *RepositoriesConfig) Validate() error { - if c.ImageSHA == "" { - return errImageSHANotFound - } - for i, r := range c.Repositories { - err := r.Validate() - if err != nil { - return fmt.Errorf("invalid repository config at index %d: %w", i, err) - } - } - return nil -} - -// RepositoriesForCommand return a subset of repositories that support the provided command. -func (c *RepositoriesConfig) RepositoriesForCommand(command string) []*RepositoryConfig { - var repositories []*RepositoryConfig - for _, r := range c.Repositories { - if slices.Contains(r.SupportedCommands, command) { - repositories = append(repositories, r) - } - } - return repositories -} - -func parseRepositoriesConfig(contentLoader func(file string) ([]byte, error), path string) (*RepositoriesConfig, error) { - bytes, err := contentLoader(path) - if err != nil { - return nil, err - } - var c RepositoriesConfig - if err := yaml.Unmarshal(bytes, &c); err != nil { - return nil, fmt.Errorf("unmarshaling repositories config state: %w", err) - } - if err := c.Validate(); err != nil { - return nil, fmt.Errorf("validating repositories config state: %w", err) - } - return &c, nil -} - -func loadRepositoriesConfig() (*RepositoriesConfig, error) { - return parseRepositoriesConfig(func(file string) ([]byte, error) { return prodRepositoriesYaml, nil }, "unused") -} diff --git a/internal/legacylibrarian/legacyautomation/repositories_test.go b/internal/legacylibrarian/legacyautomation/repositories_test.go deleted file mode 100644 index b4b79d7ddb4..00000000000 --- a/internal/legacylibrarian/legacyautomation/repositories_test.go +++ /dev/null @@ -1,361 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacyautomation - -import ( - "testing" - - "github.com/google/go-cmp/cmp" -) - -func TestRepositoriesConfig_Validate(t *testing.T) { - for _, test := range []struct { - name string - config *RepositoriesConfig - wantErr bool - }{ - { - name: "valid state", - config: &RepositoriesConfig{ - ImageSHA: "example-sha", - Repositories: []*RepositoryConfig{ - { - Name: "google-cloud-foo", - SecretName: "google-cloud-foo-github-token", - SupportedCommands: []string{"generate", "stage-release", "publish-release"}, - }, - }, - }, - }, - { - name: "valid full name", - config: &RepositoriesConfig{ - ImageSHA: "example-sha", - Repositories: []*RepositoryConfig{ - { - FullName: "https://github.com/googleapis/google-cloud-foo", - SecretName: "google-cloud-foo-github-token", - SupportedCommands: []string{"generate", "stage-release", "publish-release"}, - }, - }, - }, - }, - { - name: "missing name", - config: &RepositoriesConfig{ - ImageSHA: "example-sha", - Repositories: []*RepositoryConfig{ - { - SecretName: "google-cloud-foo-github-token", - SupportedCommands: []string{"generate", "stage-release", "publish-release"}, - }, - }, - }, - wantErr: true, - }, - { - name: "missing secret name", - config: &RepositoriesConfig{ - ImageSHA: "example-sha", - Repositories: []*RepositoryConfig{ - { - Name: "google-cloud-foo", - SupportedCommands: []string{"generate", "stage-release", "publish-release"}, - }, - }, - }, - wantErr: true, - }, - { - name: "missing commands", - config: &RepositoriesConfig{ - ImageSHA: "example-sha", - Repositories: []*RepositoryConfig{ - { - Name: "google-cloud-foo", - SecretName: "google-cloud-foo-github-token", - }, - }, - }, - wantErr: true, - }, - { - name: "empty commands", - config: &RepositoriesConfig{ - ImageSHA: "example-sha", - Repositories: []*RepositoryConfig{ - { - Name: "google-cloud-foo", - SecretName: "google-cloud-foo-github-token", - SupportedCommands: []string{}, - }, - }, - }, - wantErr: true, - }, - { - name: "invalid command", - config: &RepositoriesConfig{ - ImageSHA: "example-sha", - Repositories: []*RepositoryConfig{ - { - Name: "google-cloud-foo", - SecretName: "google-cloud-foo-github-token", - SupportedCommands: []string{"generate", "invalid", "publish-release"}, - }, - }, - }, - wantErr: true, - }, - { - name: "empty image sha", - config: &RepositoriesConfig{}, - wantErr: true, - }, - } { - t.Run(test.name, func(t *testing.T) { - if err := test.config.Validate(); (err != nil) != test.wantErr { - t.Errorf("LibrarianState.Validate() error = %v, wantErr %v", err, test.wantErr) - } - }) - } -} - -func TestParseRepositoriesConfig(t *testing.T) { - for _, test := range []struct { - name string - content string - want *RepositoriesConfig - wantErr bool - }{ - { - name: "valid state", - content: `librarian-image-sha: example-sha -repositories: - - name: google-cloud-python - github-token-secret-name: google-cloud-python-github-token - supported-commands: - - generate - - stage-release -`, - want: &RepositoriesConfig{ - ImageSHA: "example-sha", - Repositories: []*RepositoryConfig{ - { - Name: "google-cloud-python", - SecretName: "google-cloud-python-github-token", - SupportedCommands: []string{"generate", "stage-release"}, - }, - }, - }, - }, - { - name: "valid state with full name", - content: `librarian-image-sha: example-sha -repositories: - - name: google-cloud-python - full-name: https://github.com/some-org/google-cloud-python - github-token-secret-name: google-cloud-python-github-token - supported-commands: - - generate - - stage-release -`, - want: &RepositoriesConfig{ - ImageSHA: "example-sha", - Repositories: []*RepositoryConfig{ - { - Name: "google-cloud-python", - FullName: "https://github.com/some-org/google-cloud-python", - SecretName: "google-cloud-python-github-token", - SupportedCommands: []string{"generate", "stage-release"}, - }, - }, - }, - }, - { - name: "valid state with branch", - content: `librarian-image-sha: example-sha -repositories: - - name: google-cloud-python - branch: preview - github-token-secret-name: google-cloud-python-github-token - supported-commands: - - generate - - stage-release -`, - want: &RepositoriesConfig{ - ImageSHA: "example-sha", - Repositories: []*RepositoryConfig{ - { - Name: "google-cloud-python", - Branch: "preview", - SecretName: "google-cloud-python-github-token", - SupportedCommands: []string{"generate", "stage-release"}, - }, - }, - }, - }, - { - name: "invalid yaml", - content: `librarian-image-sha: example-sha -repositories: - - name: google-cloud-python - github-token-secret-name: google-cloud-python-github-token # bad indent - supported-commands: - - generate - - stage-release -`, - want: nil, - wantErr: true, - }, - { - name: "validation error", - content: `librarian-image-sha: example-sha -repositories: - - name: google-cloud-python - github-token-secret-name: google-cloud-python-github-token - # missing supported-commands -`, - want: nil, - wantErr: true, - }, - } { - t.Run(test.name, func(t *testing.T) { - contentLoader := func(path string) ([]byte, error) { - return []byte(path), nil - } - got, err := parseRepositoriesConfig(contentLoader, test.content) - if (err != nil) != test.wantErr { - t.Errorf("parseRepositoriesConfig() error = %v, wantErr %v", err, test.wantErr) - return - } - if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("parseRepositoriesConfig() mismatch (-want +got): %s", diff) - } - }) - } -} - -func TestRepositoriesForCommand(t *testing.T) { - testConfig := &RepositoriesConfig{ - Repositories: []*RepositoryConfig{ - { - Name: "google-cloud-python", - SupportedCommands: []string{"generate", "release"}, - }, - { - Name: "google-cloud-ruby", - SupportedCommands: []string{"release"}, - }, - { - Name: "google-cloud-dotnet", - SupportedCommands: []string{"generate", "release"}, - }, - }, - } - for _, test := range []struct { - name string - command string - want []string - }{ - { - name: "finds subset", - command: "generate", - want: []string{"google-cloud-python", "google-cloud-dotnet"}, - }, - { - name: "finds all", - command: "release", - want: []string{"google-cloud-python", "google-cloud-ruby", "google-cloud-dotnet"}, - }, - { - name: "finds none", - command: "non-existent", - want: []string{}, - }, - } { - t.Run(test.name, func(t *testing.T) { - got := testConfig.RepositoriesForCommand(test.command) - var names = make([]string, 0) - for _, r := range got { - names = append(names, r.Name) - } - if diff := cmp.Diff(test.want, names); diff != "" { - t.Errorf("parseRepositoriesConfig() mismatch (-want +got): %s", diff) - } - }) - } -} - -func TestRepositoryGitUrl(t *testing.T) { - for _, test := range []struct { - name string - repository *RepositoryConfig - want string - wantError bool - }{ - { - name: "sets default organization", - repository: &RepositoryConfig{ - Name: "google-cloud-python", - }, - want: "https://github.com/googleapis/google-cloud-python", - }, - { - name: "reads full name", - repository: &RepositoryConfig{ - FullName: "https://github.com/some-org/google-cloud-python", - }, - want: "https://github.com/some-org/google-cloud-python", - }, - { - name: "prefers full name", - repository: &RepositoryConfig{ - Name: "google-cloud-python", - FullName: "https://github.com/some-org/google-cloud-python", - }, - want: "https://github.com/some-org/google-cloud-python", - }, - { - name: "missing name", - repository: &RepositoryConfig{}, - wantError: true, - }, - } { - t.Run(test.name, func(t *testing.T) { - got, err := test.repository.GitURL() - if (err != nil) != test.wantError { - t.Errorf("repository.GitURL() expected to return error") - return - } - if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("repository.GitURL() mismatch (-want +got): %s", diff) - } - }) - } -} - -func TestProdRepositoriesAreSorted(t *testing.T) { - config, err := loadRepositoriesConfig() - if err != nil { - t.Fatal(err) - } - for i := 1; i < len(config.Repositories); i++ { - if config.Repositories[i-1].Name > config.Repositories[i].Name { - t.Errorf("repositories are not sorted: %s should be before %s", - config.Repositories[i].Name, config.Repositories[i-1].Name) - } - } -} diff --git a/internal/legacylibrarian/legacyautomation/stage_release.go b/internal/legacylibrarian/legacyautomation/stage_release.go deleted file mode 100644 index 908c3c653f6..00000000000 --- a/internal/legacylibrarian/legacyautomation/stage_release.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacyautomation - -import ( - "context" - - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" -) - -const ( - stageCmdName = "stage-release" -) - -type stageRunner struct { - projectID string - push bool -} - -func newStageRunner(cfg *legacyconfig.Config) *stageRunner { - return &stageRunner{ - projectID: cfg.Project, - push: cfg.Push, - } -} - -func (r *stageRunner) run(ctx context.Context) error { - return runCommandFn(ctx, stageCmdName, r.projectID, r.push, false) -} diff --git a/internal/legacylibrarian/legacyautomation/stage_release_test.go b/internal/legacylibrarian/legacyautomation/stage_release_test.go deleted file mode 100644 index e2064fbc68b..00000000000 --- a/internal/legacylibrarian/legacyautomation/stage_release_test.go +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacyautomation - -import ( - "context" - "errors" - "testing" - - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" -) - -func TestNewStageRunner(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - cfg *legacyconfig.Config - }{ - { - name: "create_a_runner", - cfg: &legacyconfig.Config{ - Project: "example-project", - }, - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - runner := newStageRunner(test.cfg) - if runner.projectID != test.cfg.Project { - t.Errorf("newStageRunner() projectID is not set") - } - }) - } -} - -func TestStageRunnerRun(t *testing.T) { - tests := []struct { - name string - runner *stageRunner - runCommandErr error - wantErr bool - wantCmd string - wantProjectID string - wantPush bool - }{ - { - name: "success", - runner: &stageRunner{ - projectID: "test-project", - push: true, - }, - wantCmd: stageCmdName, - wantProjectID: "test-project", - wantPush: true, - }, - { - name: "error from RunCommand", - runner: &stageRunner{}, - runCommandErr: errors.New("run command failed"), - wantErr: true, - wantCmd: stageCmdName, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - runCommandFn = func(ctx context.Context, command string, projectId string, push bool, build bool) error { - if command != test.wantCmd { - t.Errorf("runCommandFn() command = %v, want %v", command, test.wantCmd) - } - // Only check other args on success case to avoid nil pointer with empty runner - if test.runCommandErr == nil { - if projectId != test.wantProjectID { - t.Errorf("runCommandFn() projectId = %v, want %v", projectId, test.wantProjectID) - } - if push != test.wantPush { - t.Errorf("runCommandFn() push = %v, want %v", push, test.wantPush) - } - } - return test.runCommandErr - } - - if err := test.runner.run(t.Context()); (err != nil) != test.wantErr { - t.Errorf("run() error = %v, wantErr %v", err, test.wantErr) - } - }) - } -} diff --git a/internal/legacylibrarian/legacyautomation/trigger.go b/internal/legacylibrarian/legacyautomation/trigger.go deleted file mode 100644 index 71996309b9e..00000000000 --- a/internal/legacylibrarian/legacyautomation/trigger.go +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacyautomation - -import ( - "context" - "errors" - "fmt" - "iter" - "log/slog" - "os" - "strings" - - cloudbuild "cloud.google.com/go/cloudbuild/apiv1/v2" - "cloud.google.com/go/cloudbuild/apiv1/v2/cloudbuildpb" - "github.com/googleapis/gax-go/v2" - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" - "github.com/googleapis/librarian/internal/legacylibrarian/legacygithub" -) - -var triggerNameByCommandName = map[string]string{ - "generate": "generate", - "stage-release": "stage-release", - "publish-release": "publish-release", - "update-image": "update-image", -} - -const region = "global" - -// GitHubClient handles communication with the GitHub API. -type GitHubClient interface { - FindMergedPullRequestsWithPendingReleaseLabel(ctx context.Context, owner, repo string) ([]*legacygithub.PullRequest, error) -} - -type wrappedCloudBuildClient struct { - client *cloudbuild.Client -} - -// RunBuildTrigger executes the RPC to trigger a Cloud Build trigger. -func (c *wrappedCloudBuildClient) RunBuildTrigger(ctx context.Context, req *cloudbuildpb.RunBuildTriggerRequest, opts ...gax.CallOption) error { - resp, err := c.client.RunBuildTrigger(ctx, req, opts...) - if err != nil { - return err - } - - slog.Debug("triggered", slog.String("LRO Name", resp.Name())) - return err -} - -// ListBuildTriggers executes the RPC to list Cloud Build triggers. -func (c *wrappedCloudBuildClient) ListBuildTriggers(ctx context.Context, req *cloudbuildpb.ListBuildTriggersRequest, opts ...gax.CallOption) iter.Seq2[*cloudbuildpb.BuildTrigger, error] { - return c.client.ListBuildTriggers(ctx, req, opts...).All() -} - -// RunCommand triggers a command for each registered repository that supports it. -func RunCommand(ctx context.Context, command string, projectId string, push bool, build bool) error { - c, err := cloudbuild.NewClient(ctx) - if err != nil { - return fmt.Errorf("error creating cloudbuild client: %w", err) - } - defer c.Close() - wrappedClient := &wrappedCloudBuildClient{ - client: c, - } - ghClient := legacygithub.NewClient(os.Getenv(legacyconfig.LibrarianGithubToken), nil) - return runCommandWithClient(ctx, wrappedClient, ghClient, command, projectId, push, build) -} - -func runCommandWithClient(ctx context.Context, client CloudBuildClient, ghClient GitHubClient, command string, projectId string, push bool, build bool) error { - repositoriesConfig, err := loadRepositoriesConfig() - if err != nil { - return fmt.Errorf("error loading repositories config: %w", err) - } - return runCommandWithConfig(ctx, client, ghClient, command, projectId, push, build, repositoriesConfig) -} - -func runCommandWithConfig(ctx context.Context, client CloudBuildClient, ghClient GitHubClient, command string, projectId string, push bool, build bool, config *RepositoriesConfig) error { - // validate command is allowed - triggerName := triggerNameByCommandName[command] - if triggerName == "" { - return fmt.Errorf("unsupported command: %s", command) - } - - errs := make([]error, 0) - - repositories := config.RepositoriesForCommand(command) - for _, repository := range repositories { - slog.Debug("running command", "command", command, "repository", repository.Name) - - gitUrl, err := repository.GitURL() - if err != nil { - slog.Error("repository has no configured git url", slog.Any("repository", repository)) - return err - } - - substitutions := map[string]string{ - "_IMAGE_SHA": config.ImageSHA, - "_REPOSITORY": repository.Name, - "_FULL_REPOSITORY": gitUrl, - "_GITHUB_TOKEN_SECRET_NAME": repository.SecretName, - "_PUSH": fmt.Sprintf("%v", push), - } - if repository.Branch != "" { - substitutions["_BRANCH"] = repository.Branch - } - - if command == "publish-release" { - parts := strings.Split(gitUrl, "/") - repositoryOwner := parts[len(parts)-2] - prs, err := ghClient.FindMergedPullRequestsWithPendingReleaseLabel(ctx, repositoryOwner, repository.Name) - if err != nil { - slog.Error("error finding merged pull requests for publish-release", slog.Any("err", err), slog.String("repository", repository.Name)) - errs = append(errs, err) - continue - } - if len(prs) == 0 { - slog.Info("no pull requests with label 'release:pending' found. Skipping 'publish-release' trigger.", slog.String("repository", repository.Name)) - continue - } else { - substitutions["_PR"] = fmt.Sprintf("%v", prs[0].GetHTMLURL()) - } - } else if command == "generate" || command == "update-image" { - substitutions["_BUILD"] = fmt.Sprintf("%v", build) - } - err = runCloudBuildTriggerByName(ctx, client, projectId, region, triggerName, substitutions) - if err != nil { - slog.Error("error triggering cloudbuild", slog.Any("err", err)) - errs = append(errs, err) - } - } - return errors.Join(errs...) -} diff --git a/internal/legacylibrarian/legacyautomation/trigger_test.go b/internal/legacylibrarian/legacyautomation/trigger_test.go deleted file mode 100644 index 840beefd056..00000000000 --- a/internal/legacylibrarian/legacyautomation/trigger_test.go +++ /dev/null @@ -1,383 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacyautomation - -import ( - "context" - "fmt" - "testing" - - "cloud.google.com/go/cloudbuild/apiv1/v2/cloudbuildpb" - "github.com/google/go-cmp/cmp" - "github.com/google/go-github/v69/github" - "github.com/googleapis/librarian/internal/legacylibrarian/legacygithub" -) - -type mockGitHubClient struct { - prs []*legacygithub.PullRequest - err error -} - -func (m *mockGitHubClient) FindMergedPullRequestsWithPendingReleaseLabel(ctx context.Context, owner, repo string) ([]*legacygithub.PullRequest, error) { - return m.prs, m.err -} - -func TestRunCommandWithClient(t *testing.T) { - for _, test := range []struct { - name string - command string - push bool - build bool - want string - runError error - wantErr bool - buildTriggers []*cloudbuildpb.BuildTrigger - ghPRs []*legacygithub.PullRequest - ghError error - wantTriggersRun []string - }{ - { - name: "runs prepare-release trigger", - command: "stage-release", - push: true, - buildTriggers: []*cloudbuildpb.BuildTrigger{ - { - Name: "generate", - Id: "generate-trigger-id", - }, - { - Name: "stage-release", - Id: "stage-release-trigger-id", - }, - }, - wantTriggersRun: []string{"stage-release-trigger-id"}, - }, - { - name: "invalid command", - command: "invalid-command", - push: true, - wantErr: true, - buildTriggers: []*cloudbuildpb.BuildTrigger{ - { - Name: "generate", - Id: "generate-trigger-id", - }, - { - Name: "stage-release", - Id: "stage-release-trigger-id", - }, - }, - wantTriggersRun: nil, - }, - { - name: "error triggering", - command: "stage-release", - push: true, - runError: fmt.Errorf("some-error"), - wantErr: true, - buildTriggers: []*cloudbuildpb.BuildTrigger{ - { - Name: "generate", - Id: "generate-trigger-id", - }, - { - Name: "stage-release", - Id: "stage-release-trigger-id", - }, - }, - wantTriggersRun: nil, - }, - { - name: "runs publish-release trigger", - command: "publish-release", - push: true, - buildTriggers: []*cloudbuildpb.BuildTrigger{ - { - Name: "publish-release", - Id: "publish-release-trigger-id", - }, - }, - ghPRs: []*legacygithub.PullRequest{{HTMLURL: github.Ptr("https://github.com/googleapis/librarian/pull/1")}}, - wantTriggersRun: []string{"publish-release-trigger-id"}, - }, - { - name: "skips publish-release with no PRs", - command: "publish-release", - push: true, - buildTriggers: []*cloudbuildpb.BuildTrigger{ - { - Name: "publish-release", - Id: "publish-release-trigger-id", - }, - }, - ghPRs: []*legacygithub.PullRequest{}, - wantTriggersRun: nil, - }, - { - name: "error finding PRs for publish-release", - command: "publish-release", - push: true, - wantErr: true, - buildTriggers: []*cloudbuildpb.BuildTrigger{ - { - Name: "publish-release", - Id: "publish-release-trigger-id", - }, - }, - ghError: fmt.Errorf("github error"), - wantTriggersRun: nil, - }, - } { - t.Run(test.name, func(t *testing.T) { - ctx := t.Context() - client := &mockCloudBuildClient{ - runError: test.runError, - buildTriggers: test.buildTriggers, - } - ghClient := &mockGitHubClient{ - prs: test.ghPRs, - err: test.ghError, - } - err := runCommandWithClient(ctx, client, ghClient, test.command, "some-project", test.push, test.build) - if test.wantErr && err == nil { - t.Fatal("expected error, but did not return one") - } else if !test.wantErr && err != nil { - t.Errorf("did not expect error, but received one: %s", err) - } - if diff := cmp.Diff(test.wantTriggersRun, client.triggersRun); diff != "" { - t.Errorf("runCommandWithClient() triggersRun diff (-want, +got):\n%s", diff) - } - }) - } -} - -func TestRunCommandWithConfig(t *testing.T) { - var buildTriggers = []*cloudbuildpb.BuildTrigger{ - { - Name: "generate", - Id: "generate-trigger-id", - }, - { - Name: "stage-release", - Id: "stage-release-trigger-id", - }, - { - Name: "publish-release", - Id: "publish-release-trigger-id", - }, - { - Name: "update-image", - Id: "update-image-trigger-id", - }, - { - Name: "prepare-release", - Id: "prepare-release-trigger-id", - }, - } - for _, test := range []struct { - name string - command string - config *RepositoriesConfig - want string - runError error - wantErr bool - ghPRs []*legacygithub.PullRequest - ghError error - wantTriggersRun []string - wantSubstitutions []map[string]string - }{ - { - name: "runs generate trigger with name", - command: "generate", - config: &RepositoriesConfig{ - ImageSHA: "test-sha", - Repositories: []*RepositoryConfig{ - { - Name: "google-cloud-python", - SupportedCommands: []string{"generate"}, - SecretName: "foo", - }, - }, - }, - wantErr: false, - wantTriggersRun: []string{"generate-trigger-id"}, - wantSubstitutions: []map[string]string{{ - "_REPOSITORY": "google-cloud-python", - "_FULL_REPOSITORY": "https://github.com/googleapis/google-cloud-python", - "_GITHUB_TOKEN_SECRET_NAME": "foo", - "_IMAGE_SHA": "test-sha", - "_PUSH": "true", - "_BUILD": "true", - }}, - }, - { - name: "runs generate trigger with full name", - command: "generate", - config: &RepositoriesConfig{ - ImageSHA: "test-sha", - - Repositories: []*RepositoryConfig{ - { - Name: "google-cloud-python", - FullName: "https://github.com/googleapis/google-cloud-python", - SupportedCommands: []string{"generate"}, - SecretName: "bar", - }, - }, - }, - wantErr: false, - wantTriggersRun: []string{"generate-trigger-id"}, - wantSubstitutions: []map[string]string{{ - "_REPOSITORY": "google-cloud-python", - "_FULL_REPOSITORY": "https://github.com/googleapis/google-cloud-python", - "_GITHUB_TOKEN_SECRET_NAME": "bar", - "_IMAGE_SHA": "test-sha", - "_PUSH": "true", - "_BUILD": "true", - }}, - }, - { - name: "runs generate trigger without name", - command: "generate", - config: &RepositoriesConfig{ - Repositories: []*RepositoryConfig{ - { - SupportedCommands: []string{"generate"}, - }, - }, - }, - wantErr: true, - wantTriggersRun: nil, - }, - { - name: "runs stage-release trigger", - command: "stage-release", - config: &RepositoriesConfig{ - ImageSHA: "test-sha", - Repositories: []*RepositoryConfig{ - { - Name: "google-cloud-python", - SupportedCommands: []string{"stage-release"}, - SecretName: "baz", - }, - }, - }, - wantTriggersRun: []string{"stage-release-trigger-id"}, - wantSubstitutions: []map[string]string{{ - "_REPOSITORY": "google-cloud-python", - "_FULL_REPOSITORY": "https://github.com/googleapis/google-cloud-python", - "_GITHUB_TOKEN_SECRET_NAME": "baz", - "_IMAGE_SHA": "test-sha", - "_PUSH": "true", - }}, - }, - { - name: "runs publish-release trigger", - command: "publish-release", - config: &RepositoriesConfig{ - ImageSHA: "test-sha", - Repositories: []*RepositoryConfig{ - { - Name: "google-cloud-python", - SupportedCommands: []string{"publish-release"}, - SecretName: "qux", - }, - }, - }, - ghPRs: []*legacygithub.PullRequest{{HTMLURL: github.Ptr("https://github.com/googleapis/google-cloud-python/pull/42")}}, - wantTriggersRun: []string{"publish-release-trigger-id"}, - wantSubstitutions: []map[string]string{{ - "_REPOSITORY": "google-cloud-python", - "_FULL_REPOSITORY": "https://github.com/googleapis/google-cloud-python", - "_GITHUB_TOKEN_SECRET_NAME": "qux", - "_IMAGE_SHA": "test-sha", - "_PUSH": "true", - "_PR": "https://github.com/googleapis/google-cloud-python/pull/42", - }}, - }, - { - name: "runs update-image trigger", - command: "update-image", - config: &RepositoriesConfig{ - ImageSHA: "test-sha", - Repositories: []*RepositoryConfig{ - { - Name: "google-cloud-python", - SupportedCommands: []string{"update-image"}, - SecretName: "quux", - }, - }, - }, - wantTriggersRun: []string{"update-image-trigger-id"}, - wantSubstitutions: []map[string]string{{ - "_REPOSITORY": "google-cloud-python", - "_FULL_REPOSITORY": "https://github.com/googleapis/google-cloud-python", - "_GITHUB_TOKEN_SECRET_NAME": "quux", - "_IMAGE_SHA": "test-sha", - "_PUSH": "true", - "_BUILD": "true", - }}, - }, - { - name: "runs generate trigger on branch", - command: "generate", - config: &RepositoriesConfig{ - ImageSHA: "test-sha", - Repositories: []*RepositoryConfig{ - { - Name: "google-cloud-python", - SupportedCommands: []string{"generate"}, - Branch: "preview", - SecretName: "foo", - }, - }, - }, - wantTriggersRun: []string{"generate-trigger-id"}, - wantSubstitutions: []map[string]string{{ - "_REPOSITORY": "google-cloud-python", - "_FULL_REPOSITORY": "https://github.com/googleapis/google-cloud-python", - "_GITHUB_TOKEN_SECRET_NAME": "foo", - "_PUSH": "true", - "_IMAGE_SHA": "test-sha", - "_BRANCH": "preview", - "_BUILD": "true", - }}, - }, - } { - t.Run(test.name, func(t *testing.T) { - ctx := t.Context() - client := &mockCloudBuildClient{ - runError: test.runError, - buildTriggers: buildTriggers, - } - ghClient := &mockGitHubClient{ - prs: test.ghPRs, - err: test.ghError, - } - err := runCommandWithConfig(ctx, client, ghClient, test.command, "some-project", true, true, test.config) - if test.wantErr && err == nil { - t.Fatal("expected error, but did not return one") - } else if !test.wantErr && err != nil { - t.Errorf("did not expect error, but received one: %s", err) - } - if diff := cmp.Diff(test.wantTriggersRun, client.triggersRun); diff != "" { - t.Errorf("runCommandWithConfig() triggersRun diff (-want, +got):\n%s", diff) - } - if diff := cmp.Diff(test.wantSubstitutions, client.substitutions); diff != "" { - t.Errorf("runCommandWithConfig() substitutions diff (-want, +got):\n%s", diff) - } - }) - } -} diff --git a/internal/legacylibrarian/legacycli/cli.go b/internal/legacylibrarian/legacycli/cli.go deleted file mode 100644 index c88fae8258b..00000000000 --- a/internal/legacylibrarian/legacycli/cli.go +++ /dev/null @@ -1,191 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package legacycli defines a lightweight framework for building CLI commands. -// It's designed to be generic and self-contained, with no embedded business logic -// or dependencies on the surrounding application's configuration or behavior. -package legacycli - -import ( - "context" - "errors" - "flag" - "fmt" - "io" - "strings" - - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" -) - -// Command represents a single command that can be executed by the application. -type Command struct { - // Short is a concise one-line description of the command. - Short string - - // UsageLine is the one line usage. - UsageLine string - - // Long is the full description of the command. - Long string - - // Action executes the command. - Action func(context.Context, *Command) error - - // Commands are the sub commands. - Commands []*Command - - // Flags is the command's flag set for parsing arguments and generating - // usage messages. This is populated for each command in init(). - Flags *flag.FlagSet - - // Config contains the configs for the command. - Config *legacyconfig.Config -} - -// NewCommandSet creates and initializes a root *Command object. It automatically appends a "version" subcommand to -// the list. -func NewCommandSet(commands []*Command, short, usageLine, long string) *Command { - cmd := &Command{ - Short: short, - UsageLine: usageLine, - Long: long, - } - verifyCommandDocs(cmd) - pkg := strings.Split(cmd.Short, " ")[0] - cmd.Commands = append(make([]*Command, 0, len(commands)+1), commands...) - cmd.Commands = append(cmd.Commands, newCmdVersion(pkg)) - - cmd.Init() - return cmd -} - -func newCmdVersion(pkg string) *Command { - cmdVersion := &Command{ - Short: "version prints the version information", - UsageLine: fmt.Sprintf("%s version", pkg), - Long: fmt.Sprintf("Version prints version information for the %s binary.", pkg), - Action: func(ctx context.Context, cmd *Command) error { - fmt.Println(Version()) - return nil - }, - } - cmdVersion.Init() - return cmdVersion -} - -// Run executes the command with the provided arguments. -func (c *Command) Run(ctx context.Context, args []string) error { - cmd, remaining, err := lookupCommand(c, args) - if err != nil { - return err - } - if err := cmd.Flags.Parse(remaining); err != nil { - if errors.Is(err, flag.ErrHelp) { - return nil - } - return err - } - - if cmd.Action == nil { - cmd.Flags.Usage() - if len(cmd.Commands) > 0 { - return nil - } - return fmt.Errorf("no action defined for command %q", cmd.Name()) - } - return cmd.Action(ctx, cmd) -} - -// Name is the command name. Command.Short is always expected to begin with -// this name. -func (c *Command) Name() string { - if c.Short == "" { - panic("command is missing documentation") - } - parts := strings.Fields(c.Short) - return parts[0] -} - -func (c *Command) usage(w io.Writer) { - verifyCommandDocs(c) - fmt.Fprintf(w, "%s\n\n", c.Long) - fmt.Fprintf(w, "Usage:\n\n %s", c.UsageLine) - if len(c.Commands) > 0 { - fmt.Fprint(w, "\n\nCommands:\n") - for _, c := range c.Commands { - parts := strings.Fields(c.Short) - short := strings.Join(parts[1:], " ") - fmt.Fprintf(w, "\n %-25s %s", c.Name(), short) - } - } - if hasFlags(c.Flags) { - fmt.Fprint(w, "\n\nFlags:\n\n") - } - c.Flags.SetOutput(w) - c.Flags.PrintDefaults() - fmt.Fprintf(w, "\n\n") -} - -// Init creates a new set of flags for the command and initializes -// them such that any parsing failures result in the command usage being -// displayed. -func (c *Command) Init() *Command { - c.Flags = flag.NewFlagSet(c.Name(), flag.ContinueOnError) - c.Flags.Usage = func() { - c.usage(c.Flags.Output()) - } - c.Config = legacyconfig.New(c.Name()) - return c -} - -func hasFlags(fs *flag.FlagSet) bool { - visited := false - fs.VisitAll(func(f *flag.Flag) { - visited = true - }) - return visited -} - -func verifyCommandDocs(c *Command) { - if c.Short == "" || c.UsageLine == "" || c.Long == "" { - panic(fmt.Sprintf("command %q is missing documentation", c.Name())) - } -} - -// lookupCommand looks up the command specified by the given arguments. -// It returns the command, the remaining arguments, and an error if the command -// is not found. -func lookupCommand(cmd *Command, args []string) (*Command, []string, error) { - // If the first character of the argument is '-' assume it is a flag. - for len(args) > 0 && args[0][0] != '-' { - name := args[0] - var found *Command - for _, sub := range cmd.Commands { - if sub.Name() == name { - found = sub - break - } - } - if found == nil { - if len(cmd.Commands) == 0 { - break - } - cmd.Flags.Usage() - return nil, nil, fmt.Errorf("invalid command: %q", name) - } - cmd = found - args = args[1:] - } - return cmd, args, nil -} diff --git a/internal/legacylibrarian/legacycli/cli_test.go b/internal/legacylibrarian/legacycli/cli_test.go deleted file mode 100644 index 6c9254b3e20..00000000000 --- a/internal/legacylibrarian/legacycli/cli_test.go +++ /dev/null @@ -1,379 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacycli - -import ( - "bytes" - "context" - "flag" - "fmt" - "testing" - - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" -) - -func TestParseAndSetFlags(t *testing.T) { - var ( - strFlag string - intFlag int - ) - - cmd := &Command{ - Short: "test is used for testing", - Long: "This is the long documentation for command test.", - UsageLine: "foobar test [arguments]", - } - cmd.Init() - cmd.Flags.StringVar(&strFlag, "name", "default", "name flag") - cmd.Flags.IntVar(&intFlag, "count", 0, "count flag") - - args := []string{"-name=foo", "-count=5"} - if err := cmd.Flags.Parse(args); err != nil { - t.Fatalf("Parse() failed: %v", err) - } - - if strFlag != "foo" { - t.Errorf("expected name=foo, got %q", strFlag) - } - if intFlag != 5 { - t.Errorf("expected count=5, got %d", intFlag) - } -} - -func TestAction(t *testing.T) { - executed := false - cmd := &Command{ - Short: "run runs the command", - Action: func(ctx context.Context, cmd *Command) error { - executed = true - return nil - }, - } - - if err := cmd.Action(t.Context(), cmd); err != nil { - t.Fatal(err) - } - if !executed { - t.Errorf("cmd.Action was not executed") - } -} - -func TestNamePanicsOnEmptyShort(t *testing.T) { - defer func() { - if r := recover(); r == nil { - t.Error("The code did not panic") - } - }() - c := &Command{Short: ""} - c.Name() -} - -func TestUsagePanicsOnMissingDoc(t *testing.T) { - for _, test := range []struct { - name string - cmd *Command - }{ - {"missing short", &Command{UsageLine: "l", Long: "l"}}, - {"missing usage", &Command{Short: "s", Long: "l"}}, - {"missing long", &Command{Short: "s", UsageLine: "l"}}, - } { - t.Run(test.name, func(t *testing.T) { - defer func() { - if r := recover(); r == nil { - t.Error("The code did not panic") - } - }() - test.cmd.Init() - var buf bytes.Buffer - test.cmd.usage(&buf) - }) - } -} - -func TestUsage(t *testing.T) { - preamble := `Test prints test information. - -Usage: - - test [flags] - -` - - for _, test := range []struct { - name string - flags []func(fs *flag.FlagSet) - subcommands []*Command - want string - }{ - { - name: "no flags", - flags: nil, - want: preamble, - }, - { - name: "with string flag", - flags: []func(fs *flag.FlagSet){ - func(fs *flag.FlagSet) { - fs.String("name", "default", "name flag") - }, - }, - want: fmt.Sprintf(`%sFlags: - - -name string - name flag (default "default") - - -`, preamble), - }, - { - name: "with subcommand", - subcommands: []*Command{ - {Short: "sub runs a subcommand"}, - }, - want: fmt.Sprintf(`%sCommands: - - sub runs a subcommand - -`, preamble), - }, - } { - t.Run(test.name, func(t *testing.T) { - c := &Command{ - Short: "test prints test information", - UsageLine: "test [flags]", - Long: "Test prints test information.", - Commands: test.subcommands, - } - c.Init() - for _, fn := range test.flags { - fn(c.Flags) - } - - var buf bytes.Buffer - c.usage(&buf) - got := buf.String() - if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("mismatch(-want + got):\n%s", diff) - } - }) - } -} - -func TestLookupCommand(t *testing.T) { - sub1sub1 := &Command{ - Short: "sub1sub1 does something", - UsageLine: "sub1sub1", - Long: "sub1sub1 does something", - } - sub1 := &Command{ - Short: "sub1 does something", - UsageLine: "sub1", - Long: "sub1 does something", - Commands: []*Command{sub1sub1}, - } - sub2 := &Command{ - Short: "sub2 does something", - UsageLine: "sub2", - Long: "sub2 does something", - } - root := &Command{ - Short: "root does something", - UsageLine: "root", - Long: "root does something", - Commands: []*Command{ - sub1, - sub2, - }, - } - root.Init() - sub1.Init() - sub2.Init() - sub1sub1.Init() - - for _, test := range []struct { - name string - cmd *Command - args []string - wantCmd *Command - wantArgs []string - wantErr bool - }{ - { - name: "no args", - cmd: root, - wantCmd: root, - }, - { - name: "find sub1", - cmd: root, - args: []string{"sub1"}, - wantCmd: sub1, - }, - { - name: "find sub2", - cmd: root, - args: []string{"sub2"}, - wantCmd: sub2, - }, - { - name: "find sub1sub1", - cmd: root, - args: []string{"sub1", "sub1sub1"}, - wantCmd: sub1sub1, - }, - { - name: "find sub1sub1 with args", - cmd: root, - args: []string{"sub1", "sub1sub1", "arg1"}, - wantCmd: sub1sub1, - wantArgs: []string{"arg1"}, - }, - { - name: "unknown command", - cmd: root, - args: []string{"unknown"}, - wantErr: true, - }, - { - name: "unknown subcommand", - cmd: root, - args: []string{"sub1", "unknown"}, - wantErr: true, - }, - { - name: "find sub1 with flag arguments", - cmd: root, - args: []string{"sub1", "-h"}, - wantCmd: sub1, - wantArgs: []string{"-h"}, - }, - { - name: "find sub1sub1 with flag arguments", - cmd: root, - args: []string{"sub1", "sub1sub1", "-h"}, - wantCmd: sub1sub1, - wantArgs: []string{"-h"}, - }, - { - name: "find sub1 with a flag argument in between subcommands", - cmd: root, - args: []string{"sub1", "-h", "sub1sub1"}, - wantCmd: sub1, - wantArgs: []string{"-h", "sub1sub1"}, - }, - } { - t.Run(test.name, func(t *testing.T) { - gotCmd, gotArgs, err := lookupCommand(test.cmd, test.args) - if (err != nil) != test.wantErr { - t.Fatalf("error = %v, wantErr %v", err, test.wantErr) - } - if gotCmd != test.wantCmd { - var gotName, wantName string - if gotCmd != nil { - gotName = gotCmd.Name() - } - if test.wantCmd != nil { - wantName = test.wantCmd.Name() - } - t.Errorf("gotCmd.Name() = %q, want %q", gotName, wantName) - } - if diff := cmp.Diff(test.wantArgs, gotArgs, cmpopts.EquateEmpty()); diff != "" { - t.Errorf("mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestRun(t *testing.T) { - actionExecuted := false - subcmd := &Command{ - Short: "bar is a subcommand", - Long: "bar is a subcommand.", - UsageLine: "bar", - Action: func(ctx context.Context, cmd *Command) error { - actionExecuted = true - return nil - }, - } - subcmd.Init() - - root := &Command{ - Short: "foo is the root command", - Long: "foo is the root command.", - UsageLine: "foo", - Commands: []*Command{subcmd}, - } - root.Init() - - noaction := &Command{ - Short: "noaction has no action", - Long: "noaction has no action.", - UsageLine: "noaction", - } - noaction.Init() - - for _, test := range []struct { - name string - cmd *Command - args []string - wantErr bool - actionExecuted bool - }{ - { - name: "execute foo with subcommand bar", - cmd: root, - args: []string{"bar"}, - actionExecuted: true, - }, - { - name: "unknown subcommand", - cmd: root, - args: []string{"unknown"}, - wantErr: true, - }, - { - name: "flag parse error", - cmd: subcmd, - args: []string{"-unknown"}, - wantErr: true, - }, - { - name: "no action defined on command with no subcommands", - cmd: noaction, - wantErr: true, - }, - } { - t.Run(test.name, func(t *testing.T) { - actionExecuted = false - err := test.cmd.Run(t.Context(), test.args) - if err != nil { - if !test.wantErr { - t.Errorf("error = %v, wantErr %v", err, test.wantErr) - } - } - if actionExecuted != test.actionExecuted { - t.Errorf("actionExecuted = %v, want %v", actionExecuted, test.actionExecuted) - } - }) - } -} - -func TestNewCommandSet(t *testing.T) { - cmd := NewCommandSet(nil, "short", "usage", "long usageLine") - if len(cmd.Commands) != 1 || cmd.Commands[0].Name() != "version" { - t.Errorf("NewCommandSet(nil) did not produce a single 'version' command") - } -} diff --git a/internal/legacylibrarian/legacycli/version.go b/internal/legacylibrarian/legacycli/version.go deleted file mode 100644 index 036e06caea9..00000000000 --- a/internal/legacylibrarian/legacycli/version.go +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacycli - -import ( - _ "embed" - "runtime/debug" - "strings" - "time" -) - -//go:embed version.txt -var versionString string - -// Version return the version information for the binary, which is constructed -// following https://go.dev/ref/mod#versions. -func Version() string { - info, ok := debug.ReadBuildInfo() - if !ok { - return "" - } - return version(info) -} - -func version(info *debug.BuildInfo) string { - if info.Main.Version != "" && info.Main.Version != "(devel)" { - return info.Main.Version - } - - var revision, at string - for _, s := range info.Settings { - if s.Key == "vcs.revision" { - revision = s.Value - } - if s.Key == "vcs.time" { - at = s.Value - } - } - - if revision == "" && at == "" { - return "not available" - } - - // Construct the pseudo-version string per - // https://go.dev/ref/mod#pseudo-versions. - var buf strings.Builder - buf.WriteString(strings.TrimSpace(versionString)) - if revision != "" { - buf.WriteString("-") - // Per https://go.dev/ref/mod#pseudo-versions, only use the first 12 - // letters of the commit hash. - if len(revision) > 12 { - revision = revision[:12] - } - buf.WriteString(revision) - } - if at != "" { - // commit time is of the form 2023-01-25T19:57:54Z - p, err := time.Parse(time.RFC3339, at) - if err == nil { - buf.WriteString("-") - buf.WriteString(p.Format("20060102150405")) - } - } - return buf.String() -} diff --git a/internal/legacylibrarian/legacycli/version.txt b/internal/legacylibrarian/legacycli/version.txt deleted file mode 100644 index 0ea3a944b39..00000000000 --- a/internal/legacylibrarian/legacycli/version.txt +++ /dev/null @@ -1 +0,0 @@ -0.2.0 diff --git a/internal/legacylibrarian/legacycli/version_test.go b/internal/legacylibrarian/legacycli/version_test.go deleted file mode 100644 index ea47eb0ea79..00000000000 --- a/internal/legacylibrarian/legacycli/version_test.go +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacycli - -import ( - "fmt" - "runtime/debug" - "strings" - "testing" -) - -func TestVersion(t *testing.T) { - baseVersion := strings.TrimSpace(versionString) - for _, test := range []struct { - name string - want string - buildinfo *debug.BuildInfo - }{ - { - name: "tagged version", - want: "1.2.3", - buildinfo: &debug.BuildInfo{ - Main: debug.Module{ - Version: "1.2.3", - }, - }, - }, - { - name: "pseudoversion", - want: fmt.Sprintf("%s-123456789000-20230125195754", baseVersion), - buildinfo: &debug.BuildInfo{ - Settings: []debug.BuildSetting{ - {Key: "vcs.revision", Value: "1234567890001234"}, - {Key: "vcs.time", Value: "2023-01-25T19:57:54Z"}, - }, - }, - }, - { - name: "pseudoversion only revision", - want: fmt.Sprintf("%s-123456789000", baseVersion), - buildinfo: &debug.BuildInfo{ - Settings: []debug.BuildSetting{ - {Key: "vcs.revision", Value: "1234567890001234"}, - }, - }, - }, - { - name: "pseudoversion only time", - want: fmt.Sprintf("%s-20230102150405", baseVersion), - buildinfo: &debug.BuildInfo{ - Settings: []debug.BuildSetting{ - {Key: "vcs.time", Value: "2023-01-02T15:04:05Z"}, - }, - }, - }, - { - name: "pseudoversion invalid time", - want: fmt.Sprintf("%s-123456789000", baseVersion), - buildinfo: &debug.BuildInfo{ - Settings: []debug.BuildSetting{ - {Key: "vcs.revision", Value: "123456789000"}, - {Key: "vcs.time", Value: "invalid-time"}, - }, - }, - }, - { - name: "revision less than 12 chars", - want: fmt.Sprintf("%s-shortrev-20230125195754", baseVersion), - buildinfo: &debug.BuildInfo{ - Settings: []debug.BuildSetting{ - {Key: "vcs.revision", Value: "shortrev"}, - {Key: "vcs.time", Value: "2023-01-25T19:57:54Z"}, - }, - }, - }, - { - name: "local development", - want: "not available", - buildinfo: &debug.BuildInfo{}, - }, - } { - t.Run(test.name, func(t *testing.T) { - if got := version(test.buildinfo); got != test.want { - t.Errorf("got %s; want %s", got, test.want) - } - }) - } -} diff --git a/internal/legacylibrarian/legacyconfig/config.go b/internal/legacylibrarian/legacyconfig/config.go deleted file mode 100644 index b3ac569e944..00000000000 --- a/internal/legacylibrarian/legacyconfig/config.go +++ /dev/null @@ -1,383 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package legacyconfig defines configuration used by the CLI. -package legacyconfig - -import ( - "errors" - "fmt" - "log/slog" - "os" - "os/user" - "path/filepath" - "regexp" - "strings" -) - -const ( - // BuildRequest is a JSON file that describes which library to build/test. - BuildRequest = "build-request.json" - // BuildResponse is a JSON file that describes which library to change after - // built/test. - BuildResponse = "build-response.json" - // ConfigureRequest is a JSON file that describes which library to configure. - ConfigureRequest = "configure-request.json" - // ConfigureResponse is a JSON file that describes which library to change - // after initial configuration. - ConfigureResponse = "configure-response.json" - // GeneratorInputDir is the default directory to store files that generator - // needs to regenerate libraries from an empty directory. - GeneratorInputDir = ".librarian/generator-input" - // GenerateRequest is a JSON file that describes which library to generate. - GenerateRequest = "generate-request.json" - // GenerateResponse is a JSON file that describes which library to change - // after re-generation. - GenerateResponse = "generate-response.json" - // LibrarianDir is the default directory to store librarian state/config files, - // along with any additional configuration. - LibrarianDir = ".librarian" - // ReleaseStageRequest is a JSON file that describes which library to release. - ReleaseStageRequest = "release-stage-request.json" - // ReleaseStageResponse is a JSON file that describes which library to change - // after release. - ReleaseStageResponse = "release-stage-response.json" - // LibrarianStateFile is the name of the pipeline state file. - LibrarianStateFile = "state.yaml" - // LibrarianConfigFile is the name of the language-repository config file. - LibrarianConfigFile = "config.yaml" - // LibrarianGithubToken is the name of the env var used to store the GitHub token. - LibrarianGithubToken = "LIBRARIAN_GITHUB_TOKEN" -) - -// are variables so it can be replaced during testing. -var ( - tempDir = os.TempDir - currentUser = user.Current -) - -var ( - // pullRequestRegexp is regular expression that describes an uri of a pull request. - pullRequestRegexp = regexp.MustCompile(`^https://github\.com/([a-zA-Z0-9-._]+)/([a-zA-Z0-9-._]+)/pull/([0-9]+)$`) -) - -// Config holds all configuration values parsed from flags or environment -// variables. When adding members to this struct, please keep them in -// alphabetical order. -type Config struct { - // API is the path to the API to be configured or generated, - // relative to the root of the googleapis repository. It is a directory - // name as far as (and including) the version (v1, v2, v1alpha etc.). It - // is expected to contain a service config YAML file. - // Example: "google/cloud/functions/v2" - // - // API is used by generate and configure commands. - // - // API Path is specified with the -api flag. - API string - - // APISource is the path to the root of the googleapis repository. - // When this is not specified, the googleapis repository is cloned - // automatically. - // - // APISource is used by generate and update-image commands. - // - // APISource is specified with the -api-source flag. - APISource string - - // APISourceBranch is the branch of the API source repository to checkout - // after cloning. When this is not specified, the default branch 'master' - // 'master' is used. This is ignored when -api-source is a local repository. - // - // APISourceBranch is specified with the -api-source-branch flag. - APISourceBranch string - - // APISourceDepth controls the depth of the repository closing **IF** - // APISource is a GitHub repository, and it is cloned. - APISourceDepth int - - // Branch is the remote branch of the language repository to use. - // This is the branch which is cloned when Repo is a URL, and also used - // as the base reference for any pull requests created by the command. - // By default, the branch "main" is used. - Branch string - - // Build determines whether to build the generated library, and is only - // used in the generate command. - // - // Build is specified with the -build flag. - Build bool - - // CheckUnexpectedChanges determines whether to do additional checks for - // unexpected changes during test-container generate. - CheckUnexpectedChanges bool - - // CI is the type of Continuous Integration (CI) environment in which - // the tool is executing. - CI string - - // CommandName is the name of the command being executed. - // - // commandName is populated automatically after flag parsing. No user setup is - // expected. - CommandName string - - // Commit determines whether to create a commit for the release but not create - // a pull request. - // - // This flag is ignored if Push is set to true. - Commit bool - - // GenerateUnchanged determines whether to generate libraries where none of - // the associated APIs have changed since the commit at which they were last - // generated. Note that this does not override any configuration indicating - // that the library should not be automatically generated. - GenerateUnchanged bool - - // GitHubAPIEndpoint is the GitHub API endpoint to use for all GitHub API - // operations. - // - // This is intended for testing and should not be used in production. - GitHubAPIEndpoint string - - // GitHubToken is the access token to use for all operations involving - // GitHub. - // - // GitHubToken is used by the generate, update-image, and release - // init commands when Push is true. - // - // GitHubToken is not specified by a flag, as flags are logged and the - // access token is sensitive information. Instead, it is fetched from the - // LIBRARIAN_GITHUB_TOKEN environment variable. - GitHubToken string - - // HostMount is used to remap Docker mount paths when running in environments - // where Docker containers are siblings (e.g., Kokoro). - // It specifies a mount point from the Docker host into the Docker container. - // The format is "{host-dir}:{local-dir}". - // - // HostMount is specified with the -host-mount flag. - HostMount string - - // Image is the language-specific container image to use for language-specific - // operations. It is primarily used for testing Librarian and/or new images. - // - // Image is used by all commands which perform language-specific operations. - // If this is set via the -image flag, it is expected to be used directly - // (potentially including a repository and/or tag). If the -image flag is not - // set, use an image configured in the `config.yaml`. - // - // Image is specified with the -image flag. - Image string - - // Library is the library ID to generate (e.g. secretmanager). - // This usually corresponds to a releasable language unit -- for Go this would - // be a Go module or for dotnet the name of a NuGet package. If neither this nor - // api is specified all currently managed libraries will be regenerated. - Library string - - // LibraryToTest is the library ID to test (e.g. secretmanager). - LibraryToTest string - - // LibraryVersion is the library version to release. - // - // Overrides the automatic semantic version calculation and forces a specific - // version for a library. - // This is intended for exceptional cases, such as applying a backport patch - // or forcing a major version bump. - // - // Requires the --library flag to be specified. - LibraryVersion string - - // Project is the ID of the Google Cloud project to use. - Project string - - // PullRequest to target and operate one in the context of a release. - // - // The pull request should be in the format `https://github.com/{owner}/{repo}/pull/{number}`. - // Setting this field for `tag` means librarian will only attempt - // to process this exact pull request and not search for other pull requests - // that may be ready for tagging and releasing. - PullRequest string - - // Push determines whether to push changes to GitHub. It is used in - // all commands that create commits in a language repository: - // generate, release init, update-image. - // These commands all create pull requests if they - // - // By default (when Push isn't explicitly specified), commits are created in - // the language repo (whether a fresh clone or one specified with RepoRoot) - // but no pull request is created. In this situation, the description of the - // pull request that would have been created is displayed in the output of - // the command. - // - // When Push is true, GitHubToken must also be specified. - // - // Push is specified with the -push flag. No value is required. - Push bool - - // Repo specifies the language repository to use, as either a local root directory - // or a URL to clone from. If a local directory is specified, it can - // be relative to the current working directory. The repository must - // be in a clean state (i.e. git status should report no changes) to avoid mixing - // Librarian-created changes with other changes. - // - // Repo is used by all commands which operate on a language repository: - // generate, release init, update-image. - // - // When a local directory is specified for the generate command, the repo is checked to - // determine whether the specified API path is configured as a library. See the generate - // command documentation for more details. - // For all commands other than generate, omitting Repo is equivalent to - // specifying Repo as https://github.com/googleapis/google-cloud-{Language}. - // - // Repo is specified with the -repo flag. - Repo string - - // Test determines whether to run a test after generation. - Test bool - - // UserGID is the group ID of the current user. It is used to run Docker - // containers with the same user, so that created files have the correct - // ownership. - // - // This is populated automatically after flag parsing. No user setup is - // expected. - UserGID string - - // UserUID is the user ID of the current user. It is used to run Docker - // containers with the same user, so that created files have the correct - // ownership. - // - // This is populated automatically after flag parsing. No user setup is - // expected. - UserUID string - - // WorkRoot is the root directory used for temporary working files, including - // any repositories that are cloned. By default, this is created in /tmp with - // a timestamped directory name (e.g. /tmp/librarian-20250617T083548Z) but - // can be specified with the -output flag. - // - // WorkRoot is used by all librarian commands. - WorkRoot string -} - -// New returns a new Config populated with environment variables. -func New(cmdName string) *Config { - return &Config{ - CommandName: cmdName, - GitHubToken: os.Getenv(LibrarianGithubToken), - } -} - -// setupUser performs late initialization of user-specific configuration, -// determining the current user. This is in a separate method as it -// can fail, and is called after flag parsing. -func (c *Config) setupUser() error { - user, err := currentUser() - if err != nil { - return fmt.Errorf("failed to get current user: %w", err) - } - c.UserUID = user.Uid - c.UserGID = user.Gid - return nil -} - -func (c *Config) createWorkRoot() error { - if c.WorkRoot != "" { - slog.Info("using specified working directory", "dir", c.WorkRoot) - return nil - } - path, err := os.MkdirTemp(tempDir(), "librarian-*") - if err != nil { - return err - } - - slog.Info("temporary working directory", "dir", path) - c.WorkRoot = path - return nil -} - -func (c *Config) deriveRepo() error { - if c.Repo != "" { - slog.Debug("repo value provided by user", "repo", c.Repo) - return nil - } - wd, err := os.Getwd() - if err != nil { - return fmt.Errorf("getting working directory: %w", err) - } - stateFile := filepath.Join(wd, LibrarianDir, LibrarianStateFile) - if _, err := os.Stat(stateFile); err != nil { - return fmt.Errorf("repo flag not specified and no state file found in current working directory: %w", err) - } - slog.Info("repo not specified, using current working directory as repo root", "path", wd) - c.Repo = wd - return nil -} - -// IsValid ensures the values contained in a Config are valid. -func (c *Config) IsValid() (bool, error) { - if c.Push && c.GitHubToken == "" { - return false, errors.New("no GitHub token supplied for push") - } - - if c.Library == "" && c.LibraryVersion != "" { - return false, errors.New("specified library version without library id") - } - - if c.PullRequest != "" { - matched := pullRequestRegexp.MatchString(c.PullRequest) - if !matched { - return false, errors.New("pull request URL is not valid") - } - } - - if _, err := validateHostMount(c.HostMount, ""); err != nil { - return false, err - } - - if c.Repo == "" { - return false, errors.New("language repository not specified or detected") - } - - return true, nil -} - -// SetDefaults initializes values not set directly by the user. -func (c *Config) SetDefaults() error { - if err := c.setupUser(); err != nil { - return err - } - if err := c.createWorkRoot(); err != nil { - return err - } - if err := c.deriveRepo(); err != nil { - return err - } - return nil -} - -func validateHostMount(hostMount, defaultValue string) (bool, error) { - if hostMount == defaultValue { - return true, nil - } - - parts := strings.Split(hostMount, ":") - if len(parts) != 2 || parts[0] == "" || parts[1] == "" { - return false, errors.New("unable to parse host mount") - } - - return true, nil -} diff --git a/internal/legacylibrarian/legacyconfig/config_test.go b/internal/legacylibrarian/legacyconfig/config_test.go deleted file mode 100644 index 997b3a072bc..00000000000 --- a/internal/legacylibrarian/legacyconfig/config_test.go +++ /dev/null @@ -1,539 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacyconfig - -import ( - "errors" - "os" - "os/user" - "path/filepath" - "strings" - "testing" - - "github.com/google/go-cmp/cmp" -) - -func TestNew(t *testing.T) { - t.Setenv(LibrarianGithubToken, "") - for _, test := range []struct { - name string - envVars map[string]string - want Config - }{ - { - name: "All environment variables set", - envVars: map[string]string{ - LibrarianGithubToken: "gh_token", - }, - want: Config{ - GitHubToken: "gh_token", - CommandName: "test", - }, - }, - { - name: "No environment variables set", - envVars: map[string]string{}, - want: Config{ - CommandName: "test", - }, - }, - { - name: "Some environment variables set", - envVars: map[string]string{ - LibrarianGithubToken: "gh_token", - }, - want: Config{ - GitHubToken: "gh_token", - CommandName: "test", - }, - }, - } { - t.Run(test.name, func(t *testing.T) { - for k, v := range test.envVars { - t.Setenv(k, v) - } - - got := New("test") - - if diff := cmp.Diff(&test.want, got); diff != "" { - t.Errorf("New() mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestSetupUser(t *testing.T) { - originalCurrentUser := currentUser - t.Cleanup(func() { - currentUser = originalCurrentUser - }) - - for _, test := range []struct { - name string - mockUser *user.User - mockErr error - wantUID string - wantGID string - wantErr bool - }{ - { - name: "Success", - mockUser: &user.User{ - Uid: "1001", - Gid: "1002", - }, - mockErr: nil, - wantUID: "1001", - wantGID: "1002", - wantErr: false, - }, - { - name: "Error getting user", - mockUser: nil, - mockErr: errors.New("user lookup failed"), - wantUID: "", - wantGID: "", - wantErr: true, - }, - } { - t.Run(test.name, func(t *testing.T) { - currentUser = func() (*user.User, error) { - return test.mockUser, test.mockErr - } - - cfg := &Config{} - err := cfg.setupUser() - - if (err != nil) != test.wantErr { - t.Errorf("SetupUser() error = %v, wantErr %v", err, test.wantErr) - return - } - - if cfg.UserUID != test.wantUID { - t.Errorf("SetupUser() got UID = %q, want %q", cfg.UserUID, test.wantUID) - } - if cfg.UserGID != test.wantGID { - t.Errorf("SetupUser() got GID = %q, want %q", cfg.UserGID, test.wantGID) - } - }) - } -} - -func TestIsValid(t *testing.T) { - for _, test := range []struct { - name string - cfg Config - wantErr bool - wantErrMsg string - }{ - { - name: "Valid config - Push false", - cfg: Config{ - Push: false, - Repo: "/tmp/some/repo", - }, - }, - { - name: "Valid config - Push true, token present", - cfg: Config{ - Push: true, - GitHubToken: "some_token", - Repo: "/tmp/some/repo", - }, - }, - { - name: "Valid config - missing library version", - cfg: Config{ - Push: true, - GitHubToken: "some_token", - Library: "library-id", - Repo: "/tmp/some/repo", - }, - }, - { - name: "Valid config - valid pull request", - cfg: Config{ - PullRequest: "https://github.com/owner/repo/pull/123", - Repo: "/tmp/some/repo", - }, - }, - { - name: "Invalid config - Push true, token missing", - cfg: Config{ - Push: true, - GitHubToken: "", - Repo: "/tmp/some/repo", - }, - wantErr: true, - wantErrMsg: "no GitHub token supplied for push", - }, - { - name: "Invalid config - library version presents, missing library id", - cfg: Config{ - Push: true, - GitHubToken: "some_token", - LibraryVersion: "1.2.3", - Repo: "/tmp/some/repo", - }, - wantErr: true, - wantErrMsg: "specified library version without library id", - }, - { - name: "Invalid config - host mount invalid, missing local-dir", - cfg: Config{ - HostMount: "host-dir:", - Repo: "/tmp/some/repo", - }, - wantErr: true, - wantErrMsg: "unable to parse host mount", - }, - { - name: "Invalid config - host mount invalid, missing host-dir", - cfg: Config{ - HostMount: ":local-dir", - Repo: "/tmp/some/repo", - }, - wantErr: true, - wantErrMsg: "unable to parse host mount", - }, - { - name: "Invalid config - host mount invalid, missing separator", - cfg: Config{ - HostMount: "host-dir/local-dir", - Repo: "/tmp/some/repo", - }, - wantErr: true, - wantErrMsg: "unable to parse host mount", - }, - { - name: "Invalid config - missing Repo", - cfg: Config{ - Repo: "", - }, - wantErr: true, - wantErrMsg: "language repository not specified or detected", - }, - { - name: "Invalid config - invalid pull request url", - cfg: Config{ - PullRequest: "https://github.com/owner/repo/issues/123", - }, - wantErr: true, - wantErrMsg: "pull request URL is not valid", - }, - } { - t.Run(test.name, func(t *testing.T) { - gotValid, err := test.cfg.IsValid() - - if gotValid != !test.wantErr { - t.Errorf("IsValid() got valid = %t, want %t", gotValid, !test.wantErr) - } - - if test.wantErr && !strings.Contains(err.Error(), test.wantErrMsg) { - t.Errorf("IsValid() got unexpected error message: %q", err.Error()) - } - }) - } -} - -func TestCreateWorkRoot(t *testing.T) { - localTempDir := t.TempDir() - tempDir = func() string { - return localTempDir - } - t.Cleanup(func() { - tempDir = os.TempDir - }) - for _, test := range []struct { - name string - config *Config - setup func(t *testing.T) (string, func()) - }{ - { - name: "configured root", - config: &Config{ - WorkRoot: "/some/path", - }, - setup: func(t *testing.T) (string, func()) { - return "/some/path", func() {} - }, - }, - { - name: "version command", - config: &Config{ - CommandName: "version", - WorkRoot: "/some/path", - }, - setup: func(t *testing.T) (string, func()) { - return "/some/path", func() {} - }, - }, - { - name: "without override, new dir", - config: &Config{}, - setup: func(t *testing.T) (string, func()) { - expectedPath := filepath.Join(localTempDir, "librarian-") - return expectedPath, func() { - if err := os.RemoveAll(expectedPath); err != nil { - t.Errorf("os.RemoveAll(%q) = %v; want nil", expectedPath, err) - } - } - }, - }, - } { - t.Run(test.name, func(t *testing.T) { - want, cleanup := test.setup(t) - defer cleanup() - - if err := test.config.createWorkRoot(); err != nil { - t.Errorf("createWorkRoot() got unexpected error: %v", err) - return - } - - if !strings.HasPrefix(test.config.WorkRoot, want) { - t.Errorf("createWorkRoot() = %v, want %v", test.config.WorkRoot, want) - } - }) - } -} - -func TestCreateWorkRootError(t *testing.T) { - tempDir = func() string { - return filepath.Join("--invalid--", "--not-a-directory--") - } - t.Cleanup(func() { - tempDir = os.TempDir - }) - config := &Config{} - if err := config.createWorkRoot(); err == nil { - t.Errorf("createWorkRoot() expected an error got: %v", config.WorkRoot) - } -} - -func TestDeriveRepo(t *testing.T) { - for _, test := range []struct { - name string - config *Config - setup func(t *testing.T, dir string) - wantErr bool - wantRepoPath string - }{ - { - name: "configured repo path", - config: &Config{ - Repo: "/some/path", - }, - wantRepoPath: "/some/path", - }, - { - name: "empty repo path, state file exists", - config: &Config{}, - setup: func(t *testing.T, dir string) { - stateDir := filepath.Join(dir, LibrarianDir) - if err := os.MkdirAll(stateDir, 0755); err != nil { - t.Fatal(err) - } - stateFile := filepath.Join(stateDir, LibrarianStateFile) - if err := os.WriteFile(stateFile, []byte("test"), 0644); err != nil { - t.Fatal(err) - } - }, - }, - { - name: "empty repo path, no state file", - config: &Config{}, - wantErr: true, - }, - { - name: "version command", - config: &Config{ - Repo: "/some/path", - CommandName: "version", - }, - wantRepoPath: "/some/path", - }, - } { - t.Run(test.name, func(t *testing.T) { - tmpDir := t.TempDir() - if test.setup != nil { - test.setup(t, tmpDir) - } - t.Chdir(tmpDir) - - err := test.config.deriveRepo() - if (err != nil) != test.wantErr { - t.Errorf("deriveRepoPath() error = %v, wantErr %v", err, test.wantErr) - return - } - - wantPath := test.wantRepoPath - if wantPath == "" && !test.wantErr { - wantPath = tmpDir - } - - if diff := cmp.Diff(wantPath, test.config.Repo); diff != "" { - t.Errorf("deriveRepoPath() mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestSetDefaults(t *testing.T) { - currentUser = func() (*user.User, error) { - return &user.User{ - Uid: "1001", - Gid: "1002", - }, nil - } - - t.Cleanup(func() { - currentUser = user.Current - }) - for _, test := range []struct { - name string - setup func(t *testing.T, dir string) - workRoot string - repoRoot string - wantErr bool - }{ - { - name: "all defaults", - setup: func(t *testing.T, dir string) { - stateDir := filepath.Join(dir, LibrarianDir) - if err := os.MkdirAll(stateDir, 0755); err != nil { - t.Fatal(err) - } - stateFile := filepath.Join(stateDir, LibrarianStateFile) - if err := os.WriteFile(stateFile, []byte("test"), 0644); err != nil { - t.Fatal(err) - } - }, - workRoot: "", - repoRoot: "", - }, - { - name: "work root specified", - setup: func(t *testing.T, dir string) { - stateDir := filepath.Join(dir, LibrarianDir) - if err := os.MkdirAll(stateDir, 0755); err != nil { - t.Fatal(err) - } - stateFile := filepath.Join(stateDir, LibrarianStateFile) - if err := os.WriteFile(stateFile, []byte("test"), 0644); err != nil { - t.Fatal(err) - } - }, - workRoot: "/tmp/some/path", - repoRoot: "", - }, - { - name: "repo root specified", - workRoot: "", - repoRoot: "/tmp/my-repo-root", - }, - { - name: "all specified", - workRoot: "/tmp/my-work-root", - repoRoot: "/tmp/my-repo-root", - }, - } { - t.Run(test.name, func(t *testing.T) { - localTempDir := t.TempDir() - tempDir = func() string { - return localTempDir - } - t.Cleanup(func() { - tempDir = os.TempDir - }) - if test.setup != nil { - test.setup(t, localTempDir) - t.Chdir(localTempDir) - } - cfg := &Config{ - WorkRoot: test.workRoot, - Repo: test.repoRoot, - } - - err := cfg.SetDefaults() - if (err != nil) != test.wantErr { - t.Errorf("SetDefaults() error = %v, wantErr %v", err, test.wantErr) - return - } - - if test.wantErr { - return - } - - if cfg.UserUID == "" || cfg.UserGID == "" { - t.Errorf("User UID/GID not set") - } - - if test.workRoot == "" && cfg.WorkRoot == "" { - t.Errorf("WorkRoot not set") - } - - if test.repoRoot == "" && cfg.Repo == "" { - t.Errorf("Repo not set") - } - }) - } -} - -func TestValidateHostMount(t *testing.T) { - for _, test := range []struct { - name string - hostMount string - defaultMount string - wantErr bool - wantErrMsg string - }{ - { - name: "default host mount", - hostMount: "example/path:/path", - defaultMount: "example/path:/path", - }, - { - name: "valid host mount", - hostMount: "example/path:/mounted/path", - defaultMount: "another/path:/path", - }, - { - name: "invalid host mount", - hostMount: "example/path", - defaultMount: "example/path:/path", - wantErr: true, - wantErrMsg: "unable to parse host mount", - }, - } { - t.Run(test.name, func(t *testing.T) { - ok, err := validateHostMount(test.hostMount, test.defaultMount) - if test.wantErr { - if err == nil { - t.Fatal("validateHostMount() should return error") - } - - if !strings.Contains(err.Error(), test.wantErrMsg) { - t.Errorf("want error message: %q, got %q", test.wantErrMsg, err.Error()) - } - - return - } - - if !ok || err != nil { - t.Error("validateHostMount() should not return error") - } - }) - } -} diff --git a/internal/legacylibrarian/legacyconfig/librarian_config.go b/internal/legacylibrarian/legacyconfig/librarian_config.go deleted file mode 100644 index 0529bb7bf82..00000000000 --- a/internal/legacylibrarian/legacyconfig/librarian_config.go +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacyconfig - -import ( - "fmt" -) - -const ( - PermissionReadOnly = "read-only" - PermissionWriteOnly = "write-only" - PermissionReadWrite = "read-write" -) - -// LibrarianConfig defines the contract for the config.yaml file. -type LibrarianConfig struct { - // ReleaseOnlyMode describes whether a repository is in release only mode. - // If true, generate will fail. - // We set this value to true if we want to migrate a repository from legacylibrarian - // generate to librarian generate, but still want to use legacylibrarian to - // release the repository. - ReleaseOnlyMode bool `yaml:"release_only_mode,omitempty" json:"-"` - GlobalFilesAllowlist []*GlobalFile `yaml:"global_files_allowlist,omitempty"` - Libraries []*LibraryConfig `yaml:"libraries,omitempty"` - TagFormat string `yaml:"tag_format,omitempty"` -} - -// LibraryConfig defines configuration for a single library, identified by its ID. -type LibraryConfig struct { - GenerateBlocked bool `yaml:"generate_blocked,omitempty"` - LibraryID string `yaml:"id,omitempty"` - NextVersion string `yaml:"next_version,omitempty"` - ReleaseBlocked bool `yaml:"release_blocked,omitempty"` - TagFormat string `yaml:"tag_format,omitempty"` - // Whether to create a GitHub release for this library. - SkipGitHubReleaseCreation bool `yaml:"skip_github_release_creation,omitempty"` -} - -// GlobalFile defines the global files in language repositories. -type GlobalFile struct { - Path string `yaml:"path"` - Permissions string `yaml:"permissions"` -} - -var validPermissions = map[string]bool{ - PermissionReadOnly: true, - PermissionWriteOnly: true, - PermissionReadWrite: true, -} - -// Validate checks that the LibrarianConfig is valid. -func (g *LibrarianConfig) Validate() error { - for i, globalFile := range g.GlobalFilesAllowlist { - path, permissions := globalFile.Path, globalFile.Permissions - if !isValidRelativePath(path) { - return fmt.Errorf("invalid global file path at index %d: %q", i, path) - } - if _, ok := validPermissions[permissions]; !ok { - return fmt.Errorf("invalid global file permissions at index %d: %q", i, permissions) - } - } - - return nil -} - -// LibraryConfigFor finds the LibraryConfig entry for a given LibraryID. -func (g *LibrarianConfig) LibraryConfigFor(LibraryID string) *LibraryConfig { - for _, lib := range g.Libraries { - if lib.LibraryID == LibraryID { - return lib - } - } - return nil -} - -// IsGenerationBlocked returns true if the library is configured to block generation. -func (g *LibrarianConfig) IsGenerationBlocked(libraryID string) bool { - if g == nil { - return false - } - libConfig := g.LibraryConfigFor(libraryID) - return libConfig != nil && libConfig.GenerateBlocked -} - -// GetGlobalFiles returns the global files defined in the librarian config. -func (g *LibrarianConfig) GetGlobalFiles() []string { - var globalFiles []string - for _, globalFile := range g.GlobalFilesAllowlist { - globalFiles = append(globalFiles, globalFile.Path) - } - - return globalFiles -} diff --git a/internal/legacylibrarian/legacyconfig/librarian_config_test.go b/internal/legacylibrarian/legacyconfig/librarian_config_test.go deleted file mode 100644 index d125397f411..00000000000 --- a/internal/legacylibrarian/legacyconfig/librarian_config_test.go +++ /dev/null @@ -1,273 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacyconfig - -import ( - "strings" - "testing" - - "github.com/google/go-cmp/cmp" -) - -func TestGlobalConfig_Validate(t *testing.T) { - for _, test := range []struct { - name string - config *LibrarianConfig - wantErr bool - wantErrMsg string - }{ - { - name: "valid config", - config: &LibrarianConfig{ - GlobalFilesAllowlist: []*GlobalFile{ - { - Path: "a/path", - Permissions: "read-only", - }, - { - Path: "another/path", - Permissions: "write-only", - }, - { - Path: "other/paths", - Permissions: "read-write", - }, - }, - }, - }, - { - name: "invalid path in config", - config: &LibrarianConfig{ - GlobalFilesAllowlist: []*GlobalFile{ - { - Path: "a/path", - Permissions: "read-only", - }, - { - Path: "/another/absolute/path", - Permissions: "write-only", - }, - }, - }, - wantErr: true, - wantErrMsg: "invalid global file path", - }, - { - name: "invalid permission in config", - config: &LibrarianConfig{ - GlobalFilesAllowlist: []*GlobalFile{ - { - Path: "a/path", - Permissions: "write-only", - }, - { - Path: "another/path", - Permissions: "unknown", - }, - }, - }, - wantErr: true, - wantErrMsg: "invalid global file permissions", - }, - } { - t.Run(test.name, func(t *testing.T) { - err := test.config.Validate() - if test.wantErr { - if err == nil { - t.Fatal("GlobalConfig.Validate() should return error") - } - - if !strings.Contains(err.Error(), test.wantErrMsg) { - t.Errorf("GlobalConfig.Validate() err = %v, want error containing %q", err, test.wantErrMsg) - } - - return - } - - if err != nil { - t.Errorf("GlobalConfig.Validate() error = %v, wantErr %v", err, test.wantErr) - } - }) - } -} - -func TestLibraryConfigFor(t *testing.T) { - cases := []struct { - name string - config *LibrarianConfig - LibraryID string - wantLibrary *LibraryConfig - wantErr bool - wantErrSubstr string - }{ - { - name: "library found", - config: &LibrarianConfig{ - Libraries: []*LibraryConfig{ - {LibraryID: "lib1", NextVersion: "1.0.0"}, - {LibraryID: "lib2", NextVersion: "2.0.0"}, - }, - }, - LibraryID: "lib1", - wantLibrary: &LibraryConfig{LibraryID: "lib1", NextVersion: "1.0.0"}, - }, - { - name: "library not found", - config: &LibrarianConfig{ - Libraries: []*LibraryConfig{ - {LibraryID: "lib1", NextVersion: "1.0.0"}, - {LibraryID: "lib2", NextVersion: "2.0.0"}, - }, - }, - LibraryID: "lib3", - wantLibrary: nil, - }, - { - name: "empty libraries", - config: &LibrarianConfig{ - Libraries: []*LibraryConfig{}, - }, - LibraryID: "lib1", - wantLibrary: nil, - }, - { - name: "multiple libraries with target in middle", - config: &LibrarianConfig{ - Libraries: []*LibraryConfig{ - {LibraryID: "lib1", NextVersion: "1.0.0"}, - {LibraryID: "lib2", NextVersion: "2.0.0"}, - {LibraryID: "lib3", NextVersion: "3.0.0"}, - }, - }, - LibraryID: "lib2", - wantLibrary: &LibraryConfig{LibraryID: "lib2", NextVersion: "2.0.0"}, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - gotLibrary := tc.config.LibraryConfigFor(tc.LibraryID) - - if diff := cmp.Diff(tc.wantLibrary, gotLibrary); diff != "" { - t.Errorf("LibraryConfigFor() mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestGetGlobalFiles(t *testing.T) { - for _, test := range []struct { - name string - config *LibrarianConfig - want []string - }{ - { - name: "get_global_files", - config: &LibrarianConfig{ - GlobalFilesAllowlist: []*GlobalFile{ - { - Path: "a/path", - Permissions: "read-only", - }, - { - Path: "another/path", - Permissions: "write-only", - }, - { - Path: "other/paths", - Permissions: "read-write", - }, - }, - }, - want: []string{ - "a/path", - "another/path", - "other/paths", - }, - }, - { - name: "empty_global_files", - config: &LibrarianConfig{ - GlobalFilesAllowlist: []*GlobalFile{}, - }, - want: nil, - }, - { - name: "nil_global_files", - config: &LibrarianConfig{}, - want: nil, - }, - } { - t.Run(test.name, func(t *testing.T) { - got := test.config.GetGlobalFiles() - if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("GetGlobalFiles() mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestIsGenerationBlocked(t *testing.T) { - for _, test := range []struct { - name string - config *LibrarianConfig - libraryID string - want bool - }{ - { - name: "nil config", - config: nil, - libraryID: "lib1", - want: false, - }, - { - name: "library not in config", - config: &LibrarianConfig{ - Libraries: []*LibraryConfig{ - {LibraryID: "lib2", GenerateBlocked: true}, - }, - }, - libraryID: "lib1", - want: false, - }, - { - name: "library in config, generate_blocked is false", - config: &LibrarianConfig{ - Libraries: []*LibraryConfig{ - {LibraryID: "lib1", GenerateBlocked: false}, - }, - }, - libraryID: "lib1", - want: false, - }, - { - name: "library in config, generate_blocked is true", - config: &LibrarianConfig{ - Libraries: []*LibraryConfig{ - {LibraryID: "lib1", GenerateBlocked: true}, - }, - }, - libraryID: "lib1", - want: true, - }, - } { - t.Run(test.name, func(t *testing.T) { - got := test.config.IsGenerationBlocked(test.libraryID) - if got != test.want { - t.Errorf("IsGenerationBlocked() = %v, want %v", got, test.want) - } - }) - } -} diff --git a/internal/legacylibrarian/legacyconfig/state.go b/internal/legacylibrarian/legacyconfig/state.go deleted file mode 100644 index bbd8a4e9b68..00000000000 --- a/internal/legacylibrarian/legacyconfig/state.go +++ /dev/null @@ -1,306 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacyconfig - -import ( - "fmt" - "path/filepath" - "regexp" - "strings" - - "github.com/googleapis/librarian/internal/legacylibrarian/legacygitrepo" -) - -const ( - StatusNew = "new" - StatusExisting = "existing" - // BulkChangeThreshold is a threshold to determine whether a commit is a bulk change. - BulkChangeThreshold = 10 -) - -// LibrarianState defines the contract for the state.yaml file. -type LibrarianState struct { - // The name and tag of the generator image to use. tag is required. - Image string `yaml:"image" json:"image"` - // A list of library configurations. - Libraries []*LibraryState `yaml:"libraries" json:"libraries"` -} - -// Validate checks that the LibrarianState is valid. -func (s *LibrarianState) Validate() error { - if s.Image == "" { - return fmt.Errorf("image is required") - } - if !isValidImage(s.Image) { - return fmt.Errorf("invalid image: %q", s.Image) - } - if len(s.Libraries) == 0 { - return fmt.Errorf("libraries cannot be empty") - } - seenLibraryIDs := make(map[string]bool) - for i, l := range s.Libraries { - if l == nil { - return fmt.Errorf("library at index %d cannot be nil", i) - } - if _, exists := seenLibraryIDs[l.ID]; exists { - return fmt.Errorf("duplicate library ID %s", l.ID) - } - seenLibraryIDs[l.ID] = true - if err := l.Validate(); err != nil { - return fmt.Errorf("invalid library at index %d: %w", i, err) - } - } - return nil -} - -// ImageRefAndTag extracts the image reference and tag from the full image string. -// For example, for "gcr.io/my-image:v1.2.3", it returns a reference to -// "gcr.io/my-image" and the tag "v1.2.3". -// If no tag is present, the returned tag is an empty string. -func (s *LibrarianState) ImageRefAndTag() (ref string, tag string) { - if s == nil { - return "", "" - } - return parseImage(s.Image) -} - -// LibraryByID returns the library with the given ID, or nil if not found. -func (s *LibrarianState) LibraryByID(id string) *LibraryState { - for _, lib := range s.Libraries { - if lib.ID == id { - return lib - } - } - return nil -} - -// parseImage splits an image string into its reference and tag. -// It correctly handles port numbers in the reference. -// If no tag is found, the tag part is an empty string. -func parseImage(image string) (ref string, tag string) { - if image == "" { - return "", "" - } - lastColon := strings.LastIndex(image, ":") - if lastColon < 0 { - return image, "" - } - // if there is a slash after the last colon, it's a port number, not a tag. - if strings.Contains(image[lastColon:], "/") { - return image, "" - } - return image[:lastColon], image[lastColon+1:] -} - -// LibraryState represents the state of a single library within state.yaml. -type LibraryState struct { - // A unique identifier for the library, in a language-specific format. - // A valid ID should not be empty and only contains alphanumeric characters, slashes, periods, underscores, and hyphens. - ID string `yaml:"id" json:"id"` - // The last released version of the library, following SemVer. - Version string `yaml:"version" json:"version"` - // The commit hash from the API definition repository at which the library was last generated. - LastGeneratedCommit string `yaml:"last_generated_commit" json:"-"` - // The changes from the language repository since the library was last released. - // This field is ignored when writing to state.yaml. - Changes []*Commit `yaml:"-" json:"changes,omitempty"` - // A list of APIs that are part of this library. - APIs []*API `yaml:"apis" json:"apis"` - // A list of directories in the language repository where Librarian contributes code. - SourceRoots []string `yaml:"source_roots" json:"source_roots"` - // The previous release version, this field is only for bookkeeping. - PreviousVersion string `yaml:"-" json:"-"` - // A list of regular expressions for files and directories to preserve during the copy and remove process. - PreserveRegex []string `yaml:"preserve_regex" json:"preserve_regex"` - // A list of regular expressions for files and directories to remove before copying generated code. - // If not set, this defaults to the `source_roots`. - // A more specific `preserve_regex` takes precedence. - RemoveRegex []string `yaml:"remove_regex" json:"remove_regex"` - // A list of paths to exclude from the release. - // Files matching these paths will not be considered part of a commit for this library. - ReleaseExcludePaths []string `yaml:"release_exclude_paths,omitempty" json:"release_exclude_paths,omitempty"` - // Specifying a tag format allows librarian to honor this format when creating - // a tag for the release of the library. The replacement values of {id} and {version} - // permitted to reference the values configured in the library. If not specified - // the assumed format is {id}-{version}. e.g., {id}/v{version}. - TagFormat string `yaml:"tag_format,omitempty" json:"tag_format,omitempty"` - // Whether including this library in a release. - // This field is ignored when writing to state.yaml. - ReleaseTriggered bool `yaml:"-" json:"release_triggered,omitempty"` - // An error message from the docker response. - // This field is ignored when writing to state.yaml. - ErrorMessage string `yaml:"-" json:"error,omitempty"` -} - -// Commit represents a single commit in the release notes. -type Commit struct { - // Type is the type of change (e.g., "feat", "fix", "docs"). - Type string `json:"type"` - // Subject is the short summary of the change. - Subject string `json:"subject"` - // Body is the long-form description of the change. - Body string `json:"body"` - // CommitHash is the full commit hash. - CommitHash string `json:"commit_hash,omitempty"` - // PiperCLNumber is the Piper CL number associated with the commit. - PiperCLNumber string `json:"piper_cl_number,omitempty"` - // A list of library IDs associated with the commit. - LibraryIDs string `json:"-"` -} - -// IsBulkCommit returns true if the commit is associated with 10 or more -// libraries. -func (c *Commit) IsBulkCommit() bool { - return len(strings.Split(c.LibraryIDs, ",")) >= BulkChangeThreshold -} - -var ( - libraryIDRegex = regexp.MustCompile(`^[a-zA-Z0-9/._-]+$`) - semverRegex = regexp.MustCompile(`^v?\d+\.\d+\.\d+(?:-([a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*))?$`) - hexRegex = regexp.MustCompile("^[a-fA-F0-9]+$") - tagFormatRegex = regexp.MustCompile(`{[^{}]*}`) -) - -// Validate checks that the Library is valid. -func (l *LibraryState) Validate() error { - if l.ID == "" { - return fmt.Errorf("id is required") - } - if l.ID == "." || l.ID == ".." { - return fmt.Errorf(`id cannot be "." or ".." only`) - } - if !libraryIDRegex.MatchString(l.ID) { - return fmt.Errorf("invalid id: %q", l.ID) - } - if l.Version != "" && !semverRegex.MatchString(l.Version) { - return fmt.Errorf("invalid version: %q", l.Version) - } - if l.LastGeneratedCommit != "" { - if !hexRegex.MatchString(l.LastGeneratedCommit) { - return fmt.Errorf("last_generated_commit must be a hex string") - } - if len(l.LastGeneratedCommit) != 40 { - return fmt.Errorf("last_generated_commit must be 40 characters") - } - } - for i, a := range l.APIs { - if err := a.Validate(); err != nil { - return fmt.Errorf("invalid api at index %d: %w", i, err) - } - } - if len(l.SourceRoots) == 0 { - return fmt.Errorf("source_roots cannot be empty") - } - for i, p := range l.SourceRoots { - if !isValidRelativePath(p) { - return fmt.Errorf("invalid source_path at index %d: %q", i, p) - } - } - for i, p := range l.ReleaseExcludePaths { - if !isValidRelativePath(p) { - return fmt.Errorf("invalid release_exclude_path at index %d: %q", i, p) - } - } - if l.TagFormat != "" { - if !strings.Contains(l.TagFormat, "{version}") { - return fmt.Errorf("invalid tag_format: must contain {version}") - } - matches := tagFormatRegex.FindAllString(l.TagFormat, -1) - for _, match := range matches { - if match != "{id}" && match != "{version}" { - return fmt.Errorf("invalid tag_format: placeholder %s not recognized", match) - } - } - } - for i, r := range l.PreserveRegex { - if _, err := regexp.Compile(r); err != nil { - return fmt.Errorf("invalid preserve_regex at index %d: %w", i, err) - } - } - for i, r := range l.RemoveRegex { - if _, err := regexp.Compile(r); err != nil { - return fmt.Errorf("invalid remove_regex at index %d: %w", i, err) - } - } - return nil -} - -// API represents an API that is part of a library. -type API struct { - // The path to the API, relative to the root of the API definition repository (e.g., "google/storage/v1"). - Path string `yaml:"path" json:"path"` - // The name of the service config file, relative to the API `path`. - ServiceConfig string `yaml:"service_config,omitempty" json:"service_config,omitempty"` - // The status of the API, one of "new" or "existing". - // This field is ignored when writing to state.yaml. - Status string `yaml:"-" json:"status,omitempty"` -} - -// Validate checks that the API is valid. -func (a *API) Validate() error { - if !isValidRelativePath(a.Path) { - return fmt.Errorf("invalid path: %q", a.Path) - } - return nil -} - -// invalidPathChars contains characters that are invalid in path components, -// plus path separators and the null byte. -const invalidPathChars = "<>:\"|?*/\\\x00" - -func isValidRelativePath(pathString string) bool { - if pathString == "" { - return false - } - - if pathString == legacygitrepo.RootPath { - return true - } - - // The paths are expected to be relative and use the OS-specific path separator. - // We clean the path to resolve ".." and check that it doesn't try to - // escape the root. - cleaned := filepath.Clean(pathString) - if filepath.IsAbs(pathString) || cleaned == ".." || strings.HasPrefix(cleaned, ".."+string(filepath.Separator)) { - return false - } - - // A single dot is a valid relative path, but likely not the intended input. - if cleaned == "." { - return false - } - - // Each path component must not contain invalid characters. - for _, component := range strings.Split(cleaned, string(filepath.Separator)) { - if strings.ContainsAny(component, invalidPathChars) { - return false - } - } - return true -} - -// isValidImage checks if a string is a valid container image name with a required tag. -// It validates that the image string contains a tag, separated by a colon, and has no whitespace. -// It correctly distinguishes between a tag and a port number in the registry host. -func isValidImage(image string) bool { - // Basic validation: no whitespace. - if strings.ContainsAny(image, " \t\n\r") { - return false - } - - ref, tag := parseImage(image) - - return ref != "" && tag != "" -} diff --git a/internal/legacylibrarian/legacyconfig/state_test.go b/internal/legacylibrarian/legacyconfig/state_test.go deleted file mode 100644 index b4cf6f6efe3..00000000000 --- a/internal/legacylibrarian/legacyconfig/state_test.go +++ /dev/null @@ -1,557 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacyconfig - -import ( - "strings" - "testing" - - "github.com/google/go-cmp/cmp" -) - -func TestLibrarianState_Validate(t *testing.T) { - for _, test := range []struct { - name string - state *LibrarianState - wantErr bool - wantErrMsg string - }{ - { - name: "valid state", - state: &LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*LibraryState{ - { - ID: "a/b", - SourceRoots: []string{"src/a", "src/b"}, - APIs: []*API{ - { - Path: "a/b/v1", - }, - }, - }, - }, - }, - }, - { - name: "missing image", - state: &LibrarianState{ - Libraries: []*LibraryState{ - { - ID: "a/b", - SourceRoots: []string{"src/a", "src/b"}, - APIs: []*API{ - { - Path: "a/b/v1", - }, - }, - }, - }, - }, - wantErr: true, - wantErrMsg: "image is required", - }, - { - name: "missing libraries", - state: &LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - }, - wantErr: true, - wantErrMsg: "libraries cannot be empty", - }, - { - name: "duplicate library IDs", - state: &LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*LibraryState{ - { - ID: "x", - SourceRoots: []string{"src/x"}, - }, - { - ID: "x", - SourceRoots: []string{"src/x"}, - }, - }, - }, - wantErr: true, - wantErrMsg: "duplicate library ID", - }, - } { - t.Run(test.name, func(t *testing.T) { - err := test.state.Validate() - if test.wantErr { - if err == nil { - t.Fatal("Librarian.Validate() should fail") - } - if !strings.Contains(err.Error(), test.wantErrMsg) { - t.Errorf("want error message %q, got %q", test.wantErrMsg, err.Error()) - } - - return - } - - if err != nil { - t.Errorf("Librarian.Validate() error = %v, wantErr %v", err, test.wantErr) - } - }) - } -} - -func TestLibrary_Validate(t *testing.T) { - for _, test := range []struct { - name string - library *LibraryState - wantErr bool - wantErrMsg string - }{ - { - name: "valid library", - library: &LibraryState{ - ID: "a/b", - SourceRoots: []string{"src/a", "src/b"}, - APIs: []*API{ - { - Path: "a/b/v1", - }, - }, - }, - }, - { - name: "valid library with no APIs", - library: &LibraryState{ - ID: "a/b", - SourceRoots: []string{"src/a", "src/b"}, - }, - wantErr: false, - }, - { - name: "missing id", - library: &LibraryState{}, - wantErr: true, - wantErrMsg: "id is required", - }, - { - name: "id is dot", - library: &LibraryState{ - ID: ".", - }, - wantErr: true, - wantErrMsg: "id cannot be", - }, - { - name: "id is double dot", - library: &LibraryState{ - ID: "..", - }, - wantErr: true, - wantErrMsg: "id cannot be", - }, - { - name: "missing source paths", - library: &LibraryState{ - ID: "a/b", - APIs: []*API{ - { - Path: "a/b/v1", - }, - }, - }, - wantErr: true, - wantErrMsg: "source_roots cannot be empty", - }, - { - name: "valid version without v prefix", - library: &LibraryState{ - ID: "a/b", - Version: "1.2.3", - SourceRoots: []string{"src/a", "src/b"}, - APIs: []*API{ - { - Path: "a/b/v1", - }, - }, - }, - }, - { - name: "valid semver v1 prerelease version", - library: &LibraryState{ - ID: "a/b", - Version: "1.2.3-dev1", - SourceRoots: []string{"src/a", "src/b"}, - APIs: []*API{ - { - Path: "a/b/v1", - }, - }, - }, - }, - { - name: "valid semver v2 prerelease version", - library: &LibraryState{ - ID: "a/b", - Version: "1.2.3-dev.1", - SourceRoots: []string{"src/a", "src/b"}, - APIs: []*API{ - { - Path: "a/b/v1", - }, - }, - }, - }, - { - name: "invalid version no prerelease separator", - library: &LibraryState{ - ID: "a/b", - Version: "1.2.3dev0", - SourceRoots: []string{"src/a", "src/b"}, - APIs: []*API{ - { - Path: "a/b/v1", - }, - }, - }, - wantErr: true, - wantErrMsg: "invalid version", - }, - { - name: "invalid version bad prerelease separator", - library: &LibraryState{ - ID: "a/b", - Version: "1.2.3.dev0", - SourceRoots: []string{"src/a", "src/b"}, - APIs: []*API{ - { - Path: "a/b/v1", - }, - }, - }, - wantErr: true, - wantErrMsg: "invalid version", - }, - { - name: "invalid id characters", - library: &LibraryState{ - ID: "a/b!", - }, - wantErr: true, - wantErrMsg: "invalid id", - }, - { - name: "invalid last generated commit non-hex", - library: &LibraryState{ - ID: "a/b", - LastGeneratedCommit: "not-a-hex-string", - }, - wantErr: true, - wantErrMsg: "last_generated_commit must be a hex string", - }, - { - name: "invalid last generated commit wrong length", - library: &LibraryState{ - ID: "a/b", - LastGeneratedCommit: "deadbeef", - }, - wantErr: true, - wantErrMsg: "last_generated_commit must be 40 characters", - }, - { - name: "valid preserve_regex", - library: &LibraryState{ - ID: "a/b", - SourceRoots: []string{"src/a"}, - APIs: []*API{{Path: "a/b/v1"}}, - PreserveRegex: []string{`. * \\.txt`}, - }, - }, - { - name: "invalid preserve_regex", - library: &LibraryState{ - ID: "a/b", - SourceRoots: []string{"src/a"}, - APIs: []*API{{Path: "a/b/v1"}}, - PreserveRegex: []string{"["}, - }, - wantErr: true, - wantErrMsg: "invalid preserve_regex at index", - }, - { - name: "valid remove_regex", - library: &LibraryState{ - ID: "a/b", - SourceRoots: []string{"src/a"}, - APIs: []*API{{Path: "a/b/v1"}}, - RemoveRegex: []string{`. * \\.log`}, - }, - }, - { - name: "invalid remove_regex", - library: &LibraryState{ - ID: "a/b", - SourceRoots: []string{"src/a"}, - APIs: []*API{{Path: "a/b/v1"}}, - RemoveRegex: []string{"("}, - }, - wantErr: true, - wantErrMsg: "invalid remove_regex at index", - }, - { - name: "valid release_exclude_path", - library: &LibraryState{ - ID: "a/b", - SourceRoots: []string{"src/a"}, - APIs: []*API{{Path: "a/b/v1"}}, - ReleaseExcludePaths: []string{"a/b", "c"}, - }, - }, - { - name: "invalid release_exclude_path", - library: &LibraryState{ - ID: "a/b", - SourceRoots: []string{"src/a"}, - APIs: []*API{{Path: "a/b/v1"}}, - ReleaseExcludePaths: []string{"/a/b"}, - }, - wantErr: true, - wantErrMsg: "invalid release_exclude_path at index", - }, - { - name: "valid tag_format", - library: &LibraryState{ - ID: "a/b", - SourceRoots: []string{"src/a"}, - APIs: []*API{{Path: "a/b/v1"}}, - TagFormat: "v{id}-{version}", - }, - }, - { - name: "invalid tag_format placeholder", - library: &LibraryState{ - ID: "a/b", - SourceRoots: []string{"src/a"}, - APIs: []*API{{Path: "a/b/v1"}}, - TagFormat: "{version}-{foo}", - }, - wantErr: true, - wantErrMsg: "not recognized", - }, - { - name: "invalid tag_format with id only", - library: &LibraryState{ - ID: "a/b", - SourceRoots: []string{"src/a"}, - APIs: []*API{{Path: "a/b/v1"}}, - TagFormat: "{id}v1.2.3", - }, - wantErr: true, - wantErrMsg: "must contain", - }, - { - name: "valid tag_format with version only", - library: &LibraryState{ - ID: "a/b", - SourceRoots: []string{"src/a"}, - APIs: []*API{{Path: "a/b/v1"}}, - TagFormat: "v{version}", - }, - wantErr: false, - }, - } { - t.Run(test.name, func(t *testing.T) { - err := test.library.Validate() - if test.wantErr { - if err == nil { - t.Fatal("Library.Validate() should fail") - } - if !strings.Contains(err.Error(), test.wantErrMsg) { - t.Errorf("want error message %q, got %q", test.wantErrMsg, err.Error()) - } - - return - } - - if err != nil { - t.Errorf("Library.Validate() error = %v, wantErr %v", err, test.wantErr) - } - }) - } -} - -func TestAPI_Validate(t *testing.T) { - for _, test := range []struct { - name string - api *API - wantErr bool - wantErrMsg string - }{ - { - name: "new api", - api: &API{ - Path: "a/b/v1", - }, - }, - { - name: "existing api", - api: &API{ - Path: "a/b/v1", - }, - }, - { - name: "missing path", - api: &API{}, - wantErr: true, - wantErrMsg: "invalid path", - }, - } { - t.Run(test.name, func(t *testing.T) { - err := test.api.Validate() - if test.wantErr { - if err == nil { - t.Fatal("API.Validate() should fail") - } - if !strings.Contains(err.Error(), test.wantErrMsg) { - t.Errorf("want error message %q, got %q", test.wantErrMsg, err.Error()) - } - - return - } - - if err != nil { - t.Errorf("API.Validate() error = %v, wantErr %v", err, test.wantErr) - } - }) - } -} - -func TestIsValidDirPath(t *testing.T) { - for _, test := range []struct { - name string - path string - want bool - }{ - {"valid", "a/b/c", true}, - {"valid with dots", "a/./b/../c", true}, - {"valid path area120", "google/area120/tables/v1alpha1", true}, - {"empty", "", false}, - {"absolute", "/a/b", false}, - {"up traversal", "../a", false}, - {"double dot", "..", false}, - {"single dot", ".", true}, - {"invalid chars", "a/b 2 { - if match[1] != "" { - return match[1] // Double-quoted - } - return match[2] // Single-quoted - } - slog.Debug("librariangen: failed to find string attr in BUILD.bazel", "name", name) - return "" -} - -func findBool(content, name string) (bool, error) { - re := getRegexp("findBool_"+name, fmt.Sprintf(`%s\s*=\s*(\w+)`, name)) - if match := re.FindStringSubmatch(content); len(match) > 1 { - if b, err := strconv.ParseBool(match[1]); err == nil { - return b, nil - } - return false, fmt.Errorf("librariangen: failed to parse bool attr in BUILD.bazel: %q, got: %q", name, match[1]) - } - slog.Debug("librariangen: failed to find bool attr in BUILD.bazel", "name", name) - return false, nil -} diff --git a/internal/legacylibrarian/legacycontainer/java/bazel/parser_test.go b/internal/legacylibrarian/legacycontainer/java/bazel/parser_test.go deleted file mode 100644 index 71cfbcb8247..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/bazel/parser_test.go +++ /dev/null @@ -1,259 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bazel - -import ( - "os" - "path/filepath" - "testing" -) - -func TestParse(t *testing.T) { - content := ` -java_grpc_library( - name = "asset_java_grpc", - srcs = [":asset_proto"], - deps = [":asset_java_proto"], -) - -java_gapic_library( - name = "asset_java_gapic", - srcs = [":asset_proto_with_info"], - gapic_yaml = "cloudasset_gapic.yaml", - grpc_service_config = "cloudasset_grpc_service_config.json", - rest_numeric_enums = True, - service_yaml = "cloudasset_v1.yaml", - test_deps = [ - ":asset_java_grpc", - "//google/iam/v1:iam_java_grpc", - ], - transport = 'grpc+rest', - deps = [ - ":asset_java_proto", - "//google/api:api_java_proto", - "//google/iam/v1:iam_java_proto", - ], -) -` - tmpDir := t.TempDir() - buildPath := filepath.Join(tmpDir, "BUILD.bazel") - if err := os.WriteFile(buildPath, []byte(content), 0644); err != nil { - t.Fatalf("failed to write test file: %v", err) - } - - got, err := Parse(tmpDir) - if err != nil { - t.Fatalf("Parse() failed: %v", err) - } - - t.Run("HasGAPIC", func(t *testing.T) { - if !got.HasGAPIC() { - t.Error("HasGAPIC() = false; want true") - } - }) - t.Run("ServiceYAML", func(t *testing.T) { - if want := "cloudasset_v1.yaml"; got.ServiceYAML() != want { - t.Errorf("ServiceYAML() = %q; want %q", got.ServiceYAML(), want) - } - }) - t.Run("GapicYAML", func(t *testing.T) { - if want := "cloudasset_gapic.yaml"; got.GapicYAML() != want { - t.Errorf("GapicYAML() = %q; want %q", got.GapicYAML(), want) - } - }) - t.Run("GRPCServiceConfig", func(t *testing.T) { - if want := "cloudasset_grpc_service_config.json"; got.GRPCServiceConfig() != want { - t.Errorf("GRPCServiceConfig() = %q; want %q", got.GRPCServiceConfig(), want) - } - }) - t.Run("Transport", func(t *testing.T) { - if want := "grpc+rest"; got.Transport() != want { - t.Errorf("Transport() = %q; want %q", got.Transport(), want) - } - }) - t.Run("HasRESTNumericEnums", func(t *testing.T) { - if !got.HasRESTNumericEnums() { - t.Error("HasRESTNumericEnums() = false; want true") - } - }) -} - -func TestParse_configIsTarget(t *testing.T) { - content := ` -java_grpc_library( - name = "asset_java_grpc", - srcs = [":asset_proto"], - deps = [":asset_java_proto"], -) - -java_gapic_library( - name = "asset_java_gapic", - srcs = [":asset_proto_with_info"], - gapic_yaml = ":cloudasset_gapic.yaml", - grpc_service_config = "cloudasset_grpc_service_config.json", - rest_numeric_enums = True, - service_yaml = ":cloudasset_v1.yaml", - test_deps = [ - ":asset_java_grpc", - "//google/iam/v1:iam_java_grpc", - ], - transport = "grpc+rest", - deps = [ - ":asset_java_proto", - "//google/api:api_java_proto", - "//google/iam/v1:iam_java_proto", - ], -) -` - tmpDir := t.TempDir() - buildPath := filepath.Join(tmpDir, "BUILD.bazel") - if err := os.WriteFile(buildPath, []byte(content), 0644); err != nil { - t.Fatalf("failed to write test file: %v", err) - } - - got, err := Parse(tmpDir) - if err != nil { - t.Fatalf("Parse() failed: %v", err) - } - - if want := "cloudasset_v1.yaml"; got.ServiceYAML() != want { - t.Errorf("ServiceYAML() = %q; want %q", got.ServiceYAML(), want) - } - if want := "cloudasset_gapic.yaml"; got.GapicYAML() != want { - t.Errorf("GapicYAML() = %q; want %q", got.GapicYAML(), want) - } -} - -func TestConfig_Validate(t *testing.T) { - tests := []struct { - name string - cfg *Config - wantErr bool - }{ - { - name: "valid GAPIC", - cfg: &Config{ - hasGAPIC: true, - gapicYAML: "a", - serviceYAML: "b", - grpcServiceConfig: "c", - transport: "d", - }, - wantErr: false, - }, - { - name: "valid non-GAPIC", - cfg: &Config{}, - wantErr: false, - }, - { - name: "gRPC service config and transport are optional", - cfg: &Config{hasGAPIC: true, serviceYAML: "b", gapicYAML: "a"}, - wantErr: false, - }, - { - name: "missing serviceYAML", - cfg: &Config{hasGAPIC: true, grpcServiceConfig: "c", transport: "d"}, - wantErr: true, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - if err := test.cfg.Validate(); (err != nil) != test.wantErr { - t.Errorf("Config.Validate() error = %v, wantErr %v", err, test.wantErr) - } - }) - } -} - -func TestParse_noGapic(t *testing.T) { - content := ` -java_grpc_library( - name = "asset_java_grpc", - srcs = [":asset_proto"], - deps = [":asset_java_proto"], -) -` - tmpDir := t.TempDir() - buildPath := filepath.Join(tmpDir, "BUILD.bazel") - if err := os.WriteFile(buildPath, []byte(content), 0644); err != nil { - t.Fatalf("failed to write test file: %v", err) - } - - got, err := Parse(tmpDir) - if err != nil { - t.Fatalf("Parse() failed: %v", err) - } - - if got.HasGAPIC() { - t.Error("HasGAPIC() = true; want false") - } -} - -func TestParse_missingSomeAttrs(t *testing.T) { - content := ` -java_gapic_library( - name = "asset_java_gapic", - service_yaml = "cloudasset_v1.yaml", -) -` - tmpDir := t.TempDir() - buildPath := filepath.Join(tmpDir, "BUILD.bazel") - if err := os.WriteFile(buildPath, []byte(content), 0644); err != nil { - t.Fatalf("failed to write test file: %v", err) - } - - got, err := Parse(tmpDir) - if err != nil { - t.Fatalf("Parse() failed: %v", err) - } - - if got.GRPCServiceConfig() != "" { - t.Errorf("GRPCServiceConfig() = %q; want \"\"", got.GRPCServiceConfig()) - } - if got.Transport() != "" { - t.Errorf("Transport() = %q; want \"\"", got.Transport()) - } - if got.HasRESTNumericEnums() { - t.Error("HasRESTNumericEnums() = true; want false") - } -} - -func TestParse_invalidBoolAttr(t *testing.T) { - content := ` -java_gapic_library( - name = "asset_java_gapic", - rest_numeric_enums = "not-a-bool", -) -` - tmpDir := t.TempDir() - buildPath := filepath.Join(tmpDir, "BUILD.bazel") - if err := os.WriteFile(buildPath, []byte(content), 0644); err != nil { - t.Fatalf("failed to write test file: %v", err) - } - - _, err := Parse(tmpDir) - if err == nil { - t.Error("Parse() succeeded; want error") - } -} - -func TestParse_noBuildFile(t *testing.T) { - tmpDir := t.TempDir() - _, err := Parse(tmpDir) - if err == nil { - t.Error("Parse() succeeded; want error") - } -} diff --git a/internal/legacylibrarian/legacycontainer/java/build-docker-and-test.sh b/internal/legacylibrarian/legacycontainer/java/build-docker-and-test.sh deleted file mode 100755 index e050ee5b205..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/build-docker-and-test.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -set -e -SCRIPT_DIR=$(dirname "$0") -IMAGE_NAME="librariangen-test" - -echo "Building Docker image..." -docker build -t "${IMAGE_NAME}" "${SCRIPT_DIR}" - -echo "Running version check..." -output=$(docker run --rm -e LIBRARIAN_GOOGLE_SDK_JAVA_LOGGING_LEVEL=quiet "${IMAGE_NAME}" --version) - -if [[ ! "$output" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo "Version format is incorrect. Got: $output" - exit 1 -fi - -echo "Version check passed. Version is $output" \ No newline at end of file diff --git a/internal/legacylibrarian/legacycontainer/java/cloudbuild-exitgate.yaml b/internal/legacylibrarian/legacycontainer/java/cloudbuild-exitgate.yaml deleted file mode 100644 index af6dd78c111..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/cloudbuild-exitgate.yaml +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# note: /workspace is a special directory in the docker image where all the files in this folder -# get placed on your behalf -timeout: 7200s # 2 hours -steps: - # Attempt to pull the latest image to use as a cache source. - - name: gcr.io/cloud-builders/docker - entrypoint: 'bash' - args: - - -c - - | - docker pull us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-dev/librarian-java:latest || exit 0 - - name: gcr.io/cloud-builders/docker - args: - - "build" - - "--cache-from" - - "us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-dev/librarian-java:latest" - - "-t" - - "us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-dev/librarian-java:latest" - - "-f" - - "internal/container/java/Dockerfile" - - "." -options: - machineType: 'E2_HIGHCPU_8' - requestedVerifyOption: VERIFIED # For provenance attestation generation - logging: CLOUD_LOGGING_ONLY -images: - - us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-dev/librarian-java:latest -dependencies: - - gitSource: - repository: - url: "${_LOUHI_REPOSITORY_URL}" - revision: "${_LOUHI_REF_SHA}" - destPath: "." diff --git a/internal/legacylibrarian/legacycontainer/java/execv/execv.go b/internal/legacylibrarian/legacycontainer/java/execv/execv.go deleted file mode 100644 index 8a459ae8f5d..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/execv/execv.go +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package execv provides a helper function for executing external commands. -package execv - -import ( - "context" - "errors" - "fmt" - "log/slog" - "os" - "os/exec" - "strings" -) - -// Run executes a command in a specified working directory and logs its output. -func Run(ctx context.Context, args []string, workingDir string) error { - cmd := exec.CommandContext(ctx, args[0], args[1:]...) - cmd.Env = os.Environ() - cmd.Dir = workingDir - slog.Debug("librariangen: running command", "command", strings.Join(cmd.Args, " "), "dir", cmd.Dir) - - output, err := cmd.Output() - if len(output) > 0 { - slog.Debug("librariangen: command stdout", "output", string(output)) - } - if err != nil { - var exitErr *exec.ExitError - if errors.As(err, &exitErr) { - // The command ran and exited with a non-zero exit code. - if len(exitErr.Stderr) > 0 { - slog.Debug("librariangen: command stderr", "output", string(exitErr.Stderr)) - } - return fmt.Errorf("librariangen: command failed with exit error: %s: %w", exitErr.Stderr, err) - } - // Another error occurred (e.g., command not found). - return fmt.Errorf("librariangen: command failed: %w", err) - } - return nil -} diff --git a/internal/legacylibrarian/legacycontainer/java/execv/execv_test.go b/internal/legacylibrarian/legacycontainer/java/execv/execv_test.go deleted file mode 100644 index 15884cae87a..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/execv/execv_test.go +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package execv - -import ( - "errors" - "os/exec" - "strings" - "testing" -) - -func TestRun(t *testing.T) { - tests := []struct { - name string - args []string - wantErr bool - wantExit int - wantInStderr string - }{ - { - name: "valid command", - args: []string{"echo", "hello"}, - wantErr: false, - }, - { - name: "invalid command", - args: []string{"command-that-does-not-exist"}, - wantErr: true, - }, - { - name: "command with non-zero exit", - args: []string{"sh", "-c", "exit 1"}, - wantErr: true, - wantExit: 1, - }, - { - name: "command with stderr output", - args: []string{"sh", "-c", "echo 'test error' >&2; exit 1"}, - wantErr: true, - wantExit: 1, - wantInStderr: "test error", - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - err := Run(t.Context(), test.args, ".") - if (err != nil) != test.wantErr { - t.Fatalf("Run() error = %v, wantErr %v", err, test.wantErr) - } - - if !test.wantErr { - return - } - - var exitErr *exec.ExitError - if errors.As(err, &exitErr) { - if test.wantExit != 0 && exitErr.ExitCode() != test.wantExit { - t.Errorf("Run() exit code = %d, want %d", exitErr.ExitCode(), test.wantExit) - } - if test.wantInStderr != "" && !strings.Contains(string(exitErr.Stderr), test.wantInStderr) { - t.Errorf("Run() stderr = %q, want contains %q", string(exitErr.Stderr), test.wantInStderr) - } - } - }) - } -} diff --git a/internal/legacylibrarian/legacycontainer/java/generate/generator.go b/internal/legacylibrarian/legacycontainer/java/generate/generator.go deleted file mode 100644 index 53c6333abdb..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/generate/generator.go +++ /dev/null @@ -1,306 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package generate - -import ( - "archive/zip" - "context" - "errors" - "fmt" - "io" - "io/fs" - "log/slog" - "os" - "path/filepath" - "strings" - - "github.com/googleapis/librarian/internal/legacylibrarian/legacycontainer/java/bazel" - "github.com/googleapis/librarian/internal/legacylibrarian/legacycontainer/java/execv" - "github.com/googleapis/librarian/internal/legacylibrarian/legacycontainer/java/languagecontainer/generate" - "github.com/googleapis/librarian/internal/legacylibrarian/legacycontainer/java/message" - "github.com/googleapis/librarian/internal/legacylibrarian/legacycontainer/java/pom" - "github.com/googleapis/librarian/internal/legacylibrarian/legacycontainer/java/protoc" -) - -// Test substitution vars. -var ( - bazelParse = bazel.Parse - execvRun = execv.Run - protocBuild = protoc.Build -) - -// Generate is the main entrypoint for the `generate` command. It orchestrates -// the entire generation process. -func Generate(ctx context.Context, cfg *generate.Config) error { - slog.Debug("librariangen: generate command started") - libraryID := cfg.Request.ID - for _, api := range cfg.Request.APIs { - if err := processAPI(ctx, cfg, libraryID, api); err != nil { - return err - } - } - - // Generate pom.xml files - if err := pom.Generate(cfg.Context.OutputDir, libraryID); err != nil { - return fmt.Errorf("librariangen: failed to generate poms for API %s: %w", libraryID, err) - } - - slog.Debug("librariangen: generate command finished") - return nil -} - -func processAPI(ctx context.Context, cfg *generate.Config, libraryID string, api message.API) error { - version := extractVersion(api.Path) - if version == "" { - slog.Warn("skipping api with no version", "api", api.Path) - return nil - } - slog.Info("processing api", "path", api.Path, "version", version) - outputConfig := &protoc.OutputConfig{ - GAPICDir: filepath.Join(cfg.Context.OutputDir, version, "gapic"), - GRPCDir: filepath.Join(cfg.Context.OutputDir, version, "grpc"), - ProtoDir: filepath.Join(cfg.Context.OutputDir, version, "proto"), - } - defer func() { - if err := cleanupIntermediateFiles(outputConfig); err != nil { - slog.Error("failed to cleanup", "err", err) - } - }() - - if err := invokeProtoc(ctx, cfg.Context, &api, outputConfig); err != nil { - return fmt.Errorf("librariangen: gapic generation failed: %w", err) - } - // Unzip the temp-codegen.srcjar. - srcjarPath := filepath.Join(outputConfig.GAPICDir, "temp-codegen.srcjar") - srcjarDest := outputConfig.GAPICDir - if err := unzip(srcjarPath, srcjarDest); err != nil { - return fmt.Errorf("librariangen: failed to unzip %s: %w", srcjarPath, err) - } - - if err := restructureOutput(cfg.Context.OutputDir, libraryID, version); err != nil { - return fmt.Errorf("librariangen: failed to restructure output: %w", err) - } - - return nil -} - -func extractVersion(path string) string { - parts := strings.Split(path, "/") - for i := len(parts) - 1; i >= 0; i-- { - if strings.HasPrefix(parts[i], "v") { - return parts[i] - } - } - return "" -} - -// invokeProtoc handles the protoc GAPIC generation logic for the 'generate' CLI command. -// It reads a request file, and for each API specified, it invokes protoc -// to generate the client library. It returns the module path and the path to the service YAML. -func invokeProtoc(ctx context.Context, genCtx *generate.Context, api *message.API, outputConfig *protoc.OutputConfig) error { - apiServiceDir := filepath.Join(genCtx.SourceDir, api.Path) - slog.Info("processing api", "service_dir", apiServiceDir) - bazelConfig, err := bazelParse(apiServiceDir) - if err != nil { - return fmt.Errorf("librariangen: failed to parse BUILD.bazel for %s: %w", apiServiceDir, err) - } - args, err := protocBuild(apiServiceDir, bazelConfig, genCtx.SourceDir, outputConfig) - if err != nil { - return fmt.Errorf("librariangen: failed to build protoc command for api %q: %w", api.Path, err) - } - - // Create protoc output directories. - for _, dir := range []string{outputConfig.ProtoDir, outputConfig.GRPCDir, outputConfig.GAPICDir} { - if err := os.MkdirAll(dir, 0755); err != nil { - return err - } - } - - if err := execvRun(ctx, args, genCtx.OutputDir); err != nil { - return fmt.Errorf("librariangen: protoc failed for api %q: %w, execvRun error: %v", api.Path, err, err) - } - return nil -} - -// moveFiles moves all files (and directories) from sourceDir to targetDir. -func moveFiles(sourceDir, targetDir string) error { - files, err := os.ReadDir(sourceDir) - if err != nil { - return fmt.Errorf("librariangen: failed to read dir %s: %w", sourceDir, err) - } - for _, f := range files { - oldPath := filepath.Join(sourceDir, f.Name()) - newPath := filepath.Join(targetDir, f.Name()) - slog.Debug("librariangen: moving file", "from", oldPath, "to", newPath) - if err := os.Rename(oldPath, newPath); err != nil { - return fmt.Errorf("librariangen: failed to move %s to %s: %w, os.Rename error: %v", oldPath, newPath, err, err) - } - } - return nil -} - -func restructureOutput(outputDir, libraryID, version string) error { - slog.Debug("librariangen: restructuring output directory", "dir", outputDir) - - // Define source and destination directories. - gapicSrcDir := filepath.Join(outputDir, version, "gapic", "src", "main", "java") - gapicTestDir := filepath.Join(outputDir, version, "gapic", "src", "test", "java") - protoSrcDir := filepath.Join(outputDir, version, "proto") - resourceNameSrcDir := filepath.Join(outputDir, version, "gapic", "proto", "src", "main", "java") - samplesDir := filepath.Join(outputDir, version, "gapic", "samples", "snippets") - - gapicDestDir := filepath.Join(outputDir, fmt.Sprintf("google-cloud-%s", libraryID), "src", "main", "java") - gapicTestDestDir := filepath.Join(outputDir, fmt.Sprintf("google-cloud-%s", libraryID), "src", "test", "java") - protoDestDir := filepath.Join(outputDir, fmt.Sprintf("proto-google-cloud-%s-%s", libraryID, version), "src", "main", "java") - resourceNameDestDir := filepath.Join(outputDir, fmt.Sprintf("proto-google-cloud-%s-%s", libraryID, version), "src", "main", "java") - grpcDestDir := filepath.Join(outputDir, fmt.Sprintf("grpc-google-cloud-%s-%s", libraryID, version), "src", "main", "java") - samplesDestDir := filepath.Join(outputDir, "samples", "snippets") - - // Create destination directories. - destDirs := []string{gapicDestDir, gapicTestDestDir, protoDestDir, samplesDestDir, grpcDestDir} - for _, dir := range destDirs { - if err := os.MkdirAll(dir, 0755); err != nil { - return err - } - } - - // The resource name directory is not created if there are no resource names - // to generate. We create it here to avoid errors later. - if _, err := os.Stat(resourceNameSrcDir); errors.Is(err, fs.ErrNotExist) { - if err := os.MkdirAll(resourceNameSrcDir, 0755); err != nil { - return err - } - } - - // Remove the location classes from the proto output to avoid conflicts with - // proto-google-common-protos. - if err := os.RemoveAll(filepath.Join(protoSrcDir, "com", "google", "cloud", "location")); err != nil { - return err - } - if err := os.Remove(filepath.Join(protoSrcDir, "google", "cloud", "CommonResources.java")); err != nil { - return err - } - - // Move files that won't have conflicts. - moves := map[string]string{ - filepath.Join(outputDir, version, "proto"): protoDestDir, - filepath.Join(outputDir, version, "grpc"): grpcDestDir, - } - for src, dest := range moves { - if err := moveFiles(src, dest); err != nil { - return err - } - } - - // Merge the gapic source and test files. - if err := copyAndMerge(gapicSrcDir, gapicDestDir); err != nil { - return err - } - if err := copyAndMerge(gapicTestDir, gapicTestDestDir); err != nil { - return err - } - if err := copyAndMerge(samplesDir, samplesDestDir); err != nil { - return err - } - - // Merge the resource name files into the proto destination. - if err := copyAndMerge(resourceNameSrcDir, resourceNameDestDir); err != nil { - return err - } - - return nil -} - -// copyAndMerge recursively copies the contents of src to dest, merging directories. -func copyAndMerge(src, dest string) error { - entries, err := os.ReadDir(src) - if errors.Is(err, fs.ErrNotExist) { - return nil - } - if err != nil { - return err - } - - for _, entry := range entries { - srcPath := filepath.Join(src, entry.Name()) - destPath := filepath.Join(dest, entry.Name()) - if entry.IsDir() { - if err := os.MkdirAll(destPath, 0755); err != nil { - return err - } - if err := copyAndMerge(srcPath, destPath); err != nil { - return err - } - } else { - if err := os.Rename(srcPath, destPath); err != nil { - return fmt.Errorf("librariangen: failed to move %s to %s: %w, os.Rename error: %v", srcPath, destPath, err, err) - } - } - } - return nil -} - -func cleanupIntermediateFiles(outputConfig *protoc.OutputConfig) error { - slog.Debug("librariangen: cleaning up intermediate files") - return os.RemoveAll(filepath.Dir(outputConfig.GAPICDir)) -} - -func unzip(src, dest string) error { - r, err := zip.OpenReader(src) - if err != nil { - return err - } - defer r.Close() - - for _, f := range r.File { - fpath := filepath.Join(dest, f.Name) - - if !strings.HasPrefix(fpath, filepath.Clean(dest)+string(os.PathSeparator)) { - return fmt.Errorf("librariangen: illegal file path: %s", fpath) - } - - if f.FileInfo().IsDir() { - os.MkdirAll(fpath, os.ModePerm) - continue - } - - if err := os.MkdirAll(filepath.Dir(fpath), os.ModePerm); err != nil { - return err - } - - outFile, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) - if err != nil { - return err - } - - rc, err := f.Open() - if err != nil { - outFile.Close() - return err - } - - _, copyErr := io.Copy(outFile, rc) - rc.Close() // Error on read-only file close is less critical - closeErr := outFile.Close() - - if copyErr != nil { - return copyErr - } - if closeErr != nil { - return closeErr - } - } - return nil -} diff --git a/internal/legacylibrarian/legacycontainer/java/generate/generator_test.go b/internal/legacylibrarian/legacycontainer/java/generate/generator_test.go deleted file mode 100644 index 3b6ba9b6f41..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/generate/generator_test.go +++ /dev/null @@ -1,635 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package generate - -import ( - "archive/zip" - "context" - "errors" - "io/fs" - "os" - "path/filepath" - "testing" - - "github.com/googleapis/librarian/internal/legacylibrarian/legacycontainer/java/languagecontainer/generate" - "github.com/googleapis/librarian/internal/legacylibrarian/legacycontainer/java/protoc" -) - -// testEnv encapsulates a temporary test environment. -type testEnv struct { - tmpDir string - librarianDir string - sourceDir string - outputDir string -} - -// newTestEnv creates a new test environment. -func newTestEnv(t *testing.T) *testEnv { - t.Helper() - tmpDir := t.TempDir() - e := &testEnv{tmpDir: tmpDir} - e.librarianDir = filepath.Join(tmpDir, "librarian") - e.sourceDir = filepath.Join(tmpDir, "source") - e.outputDir = filepath.Join(tmpDir, "output") - for _, dir := range []string{e.librarianDir, e.sourceDir, e.outputDir} { - if err := os.Mkdir(dir, 0755); err != nil { - t.Fatalf("failed to create dir %s: %v", dir, err) - } - } - return e -} - -// cleanup removes the temporary directory. -func (e *testEnv) cleanup(t *testing.T) { - t.Helper() - if err := os.RemoveAll(e.tmpDir); err != nil { - t.Fatalf("failed to remove temp dir: %v", err) - } -} - -// writeRequestFile writes a generate-request.json file. -func (e *testEnv) writeRequestFile(t *testing.T, content string) { - t.Helper() - p := filepath.Join(e.librarianDir, "generate-request.json") - if err := os.WriteFile(p, []byte(content), 0644); err != nil { - t.Fatalf("failed to write request file: %v", err) - } -} - -// writeBazelFile writes a BUILD.bazel file. -func (e *testEnv) writeBazelFile(t *testing.T, apiPath, content string) { - t.Helper() - apiDir := filepath.Join(e.sourceDir, apiPath) - if err := os.MkdirAll(apiDir, 0755); err != nil { - t.Fatalf("failed to create api dir: %v", err) - } - // Create a fake .proto file, which is required by the protoc command builder. - if err := os.WriteFile(filepath.Join(apiDir, "fake.proto"), nil, 0644); err != nil { - t.Fatalf("failed to write fake proto file: %v", err) - } - p := filepath.Join(apiDir, "BUILD.bazel") - if err := os.WriteFile(p, []byte(content), 0644); err != nil { - t.Fatalf("failed to write bazel file: %v", err) - } -} - -// writeServiceYAML writes a service.yaml file. -func (e *testEnv) writeServiceYAML(t *testing.T, apiPath, title string) { - t.Helper() - apiDir := filepath.Join(e.sourceDir, apiPath) - content := "title: " + title - p := filepath.Join(apiDir, "service.yaml") - if err := os.WriteFile(p, []byte(content), 0644); err != nil { - t.Fatalf("failed to write service yaml: %v", err) - } -} - -func createFakeZip(t *testing.T, path string) { - t.Helper() - // Create a new zip archive. - newZipFile, err := os.Create(path) - if err != nil { - t.Fatalf("failed to create zip file: %v", err) - } - defer newZipFile.Close() - - zipWriter := zip.NewWriter(newZipFile) - defer zipWriter.Close() - - // Add the src/main/java directory to the zip file. - _, err = zipWriter.Create("src/main/java/") - if err != nil { - t.Fatalf("failed to create directory in zip: %v", err) - } - _, err = zipWriter.Create("src/test/java/") - if err != nil { - t.Fatalf("failed to create directory in zip: %v", err) - } -} - -func setupFakeProtocOutput(t *testing.T, e *testEnv, versions []string) { - for _, v := range versions { - // Simulate protoc creating the zip file. - zipPath := filepath.Join(e.outputDir, v, "gapic", "temp-codegen.srcjar") - if err := os.MkdirAll(filepath.Dir(zipPath), 0755); err != nil { - t.Fatalf("failed to create directory: %v", err) - } - createFakeZip(t, zipPath) - // Create the directory that is expected by restructureOutput. - if err := os.MkdirAll(filepath.Join(e.outputDir, v, "gapic", "src", "main", "java"), 0755); err != nil { - t.Fatalf("failed to create directory: %v", err) - } - if err := os.MkdirAll(filepath.Join(e.outputDir, v, "gapic", "src", "test", "java"), 0755); err != nil { - t.Fatalf("failed to create directory: %v", err) - } - if err := os.MkdirAll(filepath.Join(e.outputDir, v, "gapic", "samples", "snippets"), 0755); err != nil { - t.Fatalf("failed to create directory: %v", err) - } - // Create a fake CommonResources.java file. - p := filepath.Join(e.outputDir, v, "proto", "google", "cloud", "CommonResources.java") - if err := os.MkdirAll(filepath.Dir(p), 0755); err != nil { - t.Fatalf("failed to create directory for CommonResources.java: %v", err) - } - if err := os.WriteFile(p, nil, 0644); err != nil { - t.Fatalf("failed to write CommonResources.java: %v", err) - } - } -} - -func TestGenerate(t *testing.T) { - singleAPIRequest := `{"id": "foo", "apis": [{"path": "api/v1"}]}` - validBazel := ` -java_gapic_library( - name = "v1_gapic", - grpc_service_config = "service_config.json", - service_yaml = "service.yaml", - transport = "grpc", -) -` - invalidBazel := ` -java_gapic_library( - name = "v1_gapic", -) -` - tests := []struct { - name string - setup func(e *testEnv, t *testing.T) - protocErr error - wantErr bool - wantProtocRunCount int - }{ - { - name: "happy path", - setup: func(e *testEnv, t *testing.T) { - e.writeRequestFile(t, singleAPIRequest) - e.writeBazelFile(t, "api/v1", validBazel) - e.writeServiceYAML(t, "api/v1", "My API") - }, - wantErr: false, - wantProtocRunCount: 1, - }, - { - name: "missing request file", - setup: func(e *testEnv, t *testing.T) { - e.writeBazelFile(t, "api/v1", validBazel) - }, - wantErr: true, - }, - { - name: "missing bazel file", - setup: func(e *testEnv, t *testing.T) { - e.writeRequestFile(t, singleAPIRequest) - }, - wantErr: true, - }, - { - name: "invalid bazel config", - setup: func(e *testEnv, t *testing.T) { - e.writeRequestFile(t, singleAPIRequest) - e.writeBazelFile(t, "api/v1", invalidBazel) - }, - wantErr: true, - }, - { - name: "protoc fails", - setup: func(e *testEnv, t *testing.T) { - e.writeRequestFile(t, singleAPIRequest) - e.writeBazelFile(t, "api/v1", validBazel) - e.writeServiceYAML(t, "api/v1", "My API") - }, - protocErr: errors.New("protoc failed"), - wantErr: true, - wantProtocRunCount: 1, - }, - { - name: "unzip fails", - setup: func(e *testEnv, t *testing.T) { - e.writeRequestFile(t, singleAPIRequest) - e.writeBazelFile(t, "api/v1", validBazel) - e.writeServiceYAML(t, "api/v1", "My API") - // Create a corrupt zip file. - zipPath := filepath.Join(e.outputDir, "java_gapic.zip") - if err := os.WriteFile(zipPath, []byte("not a zip"), 0644); err != nil { - t.Fatalf("failed to write corrupt zip file: %v", err) - } - }, - wantErr: true, - wantProtocRunCount: 1, - }, - { - name: "restructureOutput fails", - setup: func(e *testEnv, t *testing.T) { - e.writeRequestFile(t, `{"id": "foo", "apis": [{"path": "api/v2"}]}`) - e.writeBazelFile(t, "api/v2", validBazel) - e.writeServiceYAML(t, "api/v2", "My API 2") - // Make a directory that restructureOutput needs to write to read-only. - readOnlyDir := filepath.Join(e.outputDir, "google-cloud-foo") - if err := os.Mkdir(readOnlyDir, 0400); err != nil { - t.Fatalf("failed to create read-only dir: %v", err) - } - }, - wantErr: true, - wantProtocRunCount: 1, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - e := newTestEnv(t) - defer e.cleanup(t) - test.setup(e, t) - var protocRunCount int - execvRun = func(ctx context.Context, args []string, dir string) error { - want := "protoc" - if args[0] != want { - t.Errorf("protocRun called with %s; want %s", args[0], want) - } - if test.protocErr == nil && test.name != "unzip fails" { - setupFakeProtocOutput(t, e, []string{"v1"}) - } - protocRunCount++ - return test.protocErr - } - genCtx := &generate.Context{ - LibrarianDir: e.librarianDir, - InputDir: "fake-input", - OutputDir: e.outputDir, - SourceDir: e.sourceDir, - } - cfg, err := generate.NewConfig(genCtx) - if err != nil && test.wantErr { - // If we expect an error, and NewConfig fails, that's ok. - return - } - if err != nil { - t.Fatalf("failed to create generate config: %v", err) - } - if err := Generate(t.Context(), cfg); (err != nil) != test.wantErr { - t.Errorf("Generate() error = %v, wantErr %v", err, test.wantErr) - } - if protocRunCount != test.wantProtocRunCount { - t.Errorf("protocRun called = %v; want %v", protocRunCount, test.wantProtocRunCount) - } - }) - } -} - -func TestRestructureOutput(t *testing.T) { - tests := []struct { - name string - libraryName string - versions []string - sourceFiles map[string]string - expectedFiles []string - wantErr bool - }{ - { - name: "single version happy path", - libraryName: "my-library", - versions: []string{"v1"}, - sourceFiles: map[string]string{ - "v1/gapic/src/main/java/com/google/foo.java": "", - "v1/gapic/src/test/java/com/google/foo_test.java": "", - "v1/proto/com/google/bar.proto": "", - "v1/grpc/com/google/bar_grpc.java": "", - "v1/gapic/samples/snippets/com/google/baz.java": "", - "v1/gapic/proto/src/main/java/com/google/resname.java": "", - "v1/proto/google/cloud/CommonResources.java": "", - }, - expectedFiles: []string{ - "google-cloud-my-library/src/main/java/com/google/foo.java", - "google-cloud-my-library/src/test/java/com/google/foo_test.java", - "proto-google-cloud-my-library-v1/src/main/java/com/google/bar.proto", - "grpc-google-cloud-my-library-v1/src/main/java/com/google/bar_grpc.java", - "samples/snippets/com/google/baz.java", - "proto-google-cloud-my-library-v1/src/main/java/com/google/resname.java", - }, - wantErr: false, - }, - { - name: "multiple versions in one library", - libraryName: "my-multi-version-library", - versions: []string{"v1", "v2"}, - sourceFiles: map[string]string{ - "v1/gapic/src/main/java/com/google/v1/foo.java": "", - "v1/gapic/src/test/java/com/google/v1/foo_test.java": "", - "v1/proto/com/google/v1/bar.proto": "", - "v1/grpc/com/google/v1/bar_grpc.java": "", - "v1/gapic/samples/snippets/com/google/v1/baz.java": "", - "v1/gapic/proto/src/main/java/com/google/v1/resname.java": "", - "v1/proto/google/cloud/CommonResources.java": "", - "v2/proto/google/cloud/CommonResources.java": "", - "v2/gapic/src/main/java/com/google/v2/foo.java": "", - "v2/gapic/src/test/java/com/google/v2/foo_test.java": "", - "v2/proto/com/google/v2/bar.proto": "", - "v2/grpc/com/google/v2/bar_grpc.java": "", - "v2/gapic/samples/snippets/com/google/v2/baz.java": "", - "v2/gapic/proto/src/main/java/com/google/v2/resname.java": "", - }, - expectedFiles: []string{ - "google-cloud-my-multi-version-library/src/main/java/com/google/v1/foo.java", - "google-cloud-my-multi-version-library/src/test/java/com/google/v1/foo_test.java", - "proto-google-cloud-my-multi-version-library-v1/src/main/java/com/google/v1/bar.proto", - "grpc-google-cloud-my-multi-version-library-v1/src/main/java/com/google/v1/bar_grpc.java", - "samples/snippets/com/google/v1/baz.java", - "proto-google-cloud-my-multi-version-library-v1/src/main/java/com/google/v1/resname.java", - "google-cloud-my-multi-version-library/src/main/java/com/google/v2/foo.java", - "google-cloud-my-multi-version-library/src/test/java/com/google/v2/foo_test.java", - "proto-google-cloud-my-multi-version-library-v2/src/main/java/com/google/v2/bar.proto", - "grpc-google-cloud-my-multi-version-library-v2/src/main/java/com/google/v2/bar_grpc.java", - "samples/snippets/com/google/v2/baz.java", - "proto-google-cloud-my-multi-version-library-v2/src/main/java/com/google/v2/resname.java", - }, - wantErr: false, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - e := newTestEnv(t) - defer e.cleanup(t) - - for path, content := range test.sourceFiles { - fullPath := filepath.Join(e.outputDir, path) - if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil { - t.Fatalf("failed to create source directory for %s: %v", path, err) - } - if err := os.WriteFile(fullPath, []byte(content), 0644); err != nil { - t.Fatalf("failed to write source file for %s: %v", path, err) - } - } - - for _, version := range test.versions { - if err := restructureOutput(e.outputDir, test.libraryName, version); (err != nil) != test.wantErr { - t.Errorf("restructureOutput() for version %s error = %v, wantErr %v", version, err, test.wantErr) - } - } - - for _, path := range test.expectedFiles { - fullPath := filepath.Join(e.outputDir, path) - if _, err := os.Stat(fullPath); err != nil { - t.Errorf("expected file not found at %s: %v", fullPath, err) - } - } - }) - } -} - -func TestCopyAndMerge(t *testing.T) { - e := newTestEnv(t) - defer e.cleanup(t) - - // 1. Setup: Create source and destination directories with nested structures. - srcDir := filepath.Join(e.tmpDir, "src") - destDir := filepath.Join(e.tmpDir, "dest") - sourceFiles := map[string]string{ - "com/google/foo.java": "", - "com/google/bar/baz.java": "", - "com/google/bar/qux/quux.java": "", - } - destFiles := map[string]string{ - "com/google/existing.java": "", - "com/google/bar/another.java": "", - } - for path, content := range sourceFiles { - fullPath := filepath.Join(srcDir, path) - if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil { - t.Fatalf("failed to create source directory for %s: %v", path, err) - } - if err := os.WriteFile(fullPath, []byte(content), 0644); err != nil { - t.Fatalf("failed to write source file for %s: %v", path, err) - } - } - for path, content := range destFiles { - fullPath := filepath.Join(destDir, path) - if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil { - t.Fatalf("failed to create dest directory for %s: %v", path, err) - } - if err := os.WriteFile(fullPath, []byte(content), 0644); err != nil { - t.Fatalf("failed to write dest file for %s: %v", path, err) - } - } - - // 2. Execute: Call the function under test. - if err := copyAndMerge(srcDir, destDir); err != nil { - t.Fatalf("copyAndMerge() failed: %v", err) - } - - // 3. Verify: Check that all files were merged correctly. - for path := range sourceFiles { - fullPath := filepath.Join(destDir, path) - if _, err := os.Stat(fullPath); err != nil { - t.Errorf("source file not merged: %v", err) - } - } - for path := range destFiles { - fullPath := filepath.Join(destDir, path) - if _, err := os.Stat(fullPath); err != nil { - t.Errorf("destination file was deleted: %v", err) - } - } -} - -func TestUnzip(t *testing.T) { - t.Run("happy path", func(t *testing.T) { - e := newTestEnv(t) - defer e.cleanup(t) - // Create a valid zip file. - zipPath := filepath.Join(e.outputDir, "valid.zip") - f, err := os.Create(zipPath) - if err != nil { - t.Fatalf("failed to create zip file: %v", err) - } - defer f.Close() - zipWriter := zip.NewWriter(f) - if _, err := zipWriter.Create("file.txt"); err != nil { - t.Fatalf("failed to create file in zip: %v", err) - } - zipWriter.Close() - - // Unzip the file. - destDir := filepath.Join(e.outputDir, "unzip-dest") - if err := unzip(zipPath, destDir); err != nil { - t.Fatalf("unzip() failed: %v", err) - } - - // Check that the file was unzipped. - if _, err := os.Stat(filepath.Join(destDir, "file.txt")); err != nil { - t.Errorf("file not unzipped: %v", err) - } - }) - - t.Run("invalid zip file", func(t *testing.T) { - e := newTestEnv(t) - defer e.cleanup(t) - invalidZipPath := filepath.Join(e.outputDir, "invalid.zip") - if err := os.WriteFile(invalidZipPath, []byte("not a zip file"), 0644); err != nil { - t.Fatalf("failed to write invalid zip file: %v", err) - } - if err := unzip(invalidZipPath, e.outputDir); err == nil { - t.Error("unzip() with invalid zip file should return an error") - } - }) - - t.Run("permission denied", func(t *testing.T) { - e := newTestEnv(t) - defer e.cleanup(t) - // Create a valid zip file. - validZipPath := filepath.Join(e.outputDir, "valid.zip") - if err := os.WriteFile(validZipPath, []byte{}, 0644); err != nil { - t.Fatalf("failed to write valid zip file: %v", err) - } - // Create a zip writer to add a file to the zip. - f, err := os.OpenFile(validZipPath, os.O_RDWR, 0) - if err != nil { - t.Fatalf("failed to open zip file: %v", err) - } - defer f.Close() // Ensure file is closed - zipWriter := zip.NewWriter(f) - if _, err := zipWriter.Create("file.txt"); err != nil { - t.Fatalf("failed to create file in zip: %v", err) - } - if err := zipWriter.Close(); err != nil { // Check for errors on close - t.Fatalf("failed to close zip writer: %v", err) - } - - // Make the output directory read-only. - readOnlyDir := filepath.Join(e.tmpDir, "readonly") - if err := os.Mkdir(readOnlyDir, 0400); err != nil { - t.Fatalf("failed to create read-only dir: %v", err) - } - if err := os.Chmod(readOnlyDir, 0400); err != nil { - t.Fatalf("failed to chmod read-only dir: %v", err) - } - - if err := unzip(validZipPath, readOnlyDir); err == nil { - t.Error("unzip() with read-only destination should return an error") - } - }) - - t.Run("zip slip vulnerability", func(t *testing.T) { - e := newTestEnv(t) - defer e.cleanup(t) - // Create a zip file with a malicious file path. - maliciousZipPath := filepath.Join(e.outputDir, "malicious.zip") - f, err := os.Create(maliciousZipPath) - if err != nil { - t.Fatalf("failed to create malicious zip file: %v", err) - } - defer f.Close() - zipWriter := zip.NewWriter(f) - if _, err := zipWriter.Create("../../pwned.txt"); err != nil { - t.Fatalf("failed to create malicious file in zip: %v", err) - } - zipWriter.Close() - - destDir := filepath.Join(e.outputDir, "unzip-dest") - if err := os.Mkdir(destDir, 0755); err != nil { - t.Fatalf("failed to create unzip dest dir: %v", err) - } - - if err := unzip(maliciousZipPath, destDir); err == nil { - t.Error("unzip() with malicious zip file should return an error") - } - - // Check that the malicious file was not created. - pwnedFile := filepath.Join(e.tmpDir, "pwned.txt") - if _, err := os.Stat(pwnedFile); !errors.Is(err, fs.ErrNotExist) { - t.Errorf("malicious file was created at %s", pwnedFile) - } - }) -} - -func TestMoveFiles(t *testing.T) { - e := newTestEnv(t) - defer e.cleanup(t) - - sourceDir := t.TempDir() - destDir := filepath.Join(e.tmpDir, "dest-move-test") - if err := os.Mkdir(destDir, 0755); err != nil { - t.Fatalf("failed to create dest dir: %v", err) - } - // Make source dir unreadable. - if err := os.Chmod(sourceDir, 0000); err != nil { - t.Fatalf("failed to chmod source dir: %v", err) - } - - if err := moveFiles(sourceDir, destDir); err == nil { - t.Error("moveFiles() with unreadable source should return an error") - } -} - -func TestCleanupIntermediateFiles(t *testing.T) { - e := newTestEnv(t) - defer e.cleanup(t) - - // Create a file that cannot be deleted. - protectedDir := filepath.Join(e.outputDir, "proto") - if err := os.Mkdir(protectedDir, 0755); err != nil { - t.Fatalf("failed to create protected dir: %v", err) - } - if _, err := os.Create(filepath.Join(protectedDir, "file.txt")); err != nil { - t.Fatalf("failed to create file in protected dir: %v", err) - } - // Make the directory read-only after creating the file. - if err := os.Chmod(protectedDir, 0500); err != nil { - t.Fatalf("failed to chmod protected dir: %v", err) - } - defer os.Chmod(protectedDir, 0755) // Restore permissions for cleanup. - - outputConfig := &protoc.OutputConfig{ - GAPICDir: filepath.Join(e.outputDir, "gapic"), - GRPCDir: filepath.Join(e.outputDir, "grpc"), - ProtoDir: protectedDir, - } - if err := cleanupIntermediateFiles(outputConfig); err == nil { - t.Error("cleanupIntermediateFiles() should return an error on failure, but did not") - } -} - -func TestExtractVersion(t *testing.T) { - tests := []struct { - name string - path string - want string - }{ - { - name: "simple path", - path: "google/cloud/secretmanager/v1", - want: "v1", - }, - { - name: "no version", - path: "google/cloud/secretmanager", - want: "", - }, - { - name: "version in the middle", - path: "google/cloud/v1/secretmanager", - want: "v1", - }, - { - name: "empty path", - path: "", - want: "", - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - if got := extractVersion(test.path); got != test.want { - t.Errorf("extractVersion() = %v, want %v", got, test.want) - } - }) - } -} diff --git a/internal/legacylibrarian/legacycontainer/java/languagecontainer/generate/generate.go b/internal/legacylibrarian/legacycontainer/java/languagecontainer/generate/generate.go deleted file mode 100644 index 8461df5a3d2..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/languagecontainer/generate/generate.go +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package generate contains types for language container's generate command. -package generate - -import ( - "errors" - "fmt" - "log/slog" - "path/filepath" - - "github.com/googleapis/librarian/internal/legacylibrarian/legacycontainer/java/message" -) - -// Context holds the directory paths for the generate command. -// https://github.com/googleapis/librarian/blob/main/doc/language-onboarding.md#generate -type Context struct { - // LibrarianDir is the path to the librarian-tool input directory. - // It is expected to contain the generate-request.json file. - LibrarianDir string - // InputDir is the path to the .librarian/generator-input directory from the - // language repository. - InputDir string - // OutputDir is the path to the empty directory where a language container writes - // its output. - OutputDir string - // SourceDir is the path to a complete checkout of the googleapis repository. - SourceDir string -} - -// Validate ensures that the context is valid. -func (c *Context) Validate() error { - if c.LibrarianDir == "" { - return errors.New("languagecontainer: librarian directory must be set") - } - if c.InputDir == "" { - return errors.New("languagecontainer: input directory must be set") - } - if c.OutputDir == "" { - return errors.New("languagecontainer: output directory must be set") - } - if c.SourceDir == "" { - return errors.New("languagecontainer: source directory must be set") - } - return nil -} - -// Config for the generate command. This holds the context (the directory paths) -// and the request parsed from the generate-request.json file. -type Config struct { - Context *Context - // This request is parsed from the generate-request.json file in - // the LibrarianDir of the context. - Request *message.Library -} - -// NewConfig creates a new Config, parsing the generate-request.json file -// from the LibrarianDir in the given Context. -func NewConfig(ctx *Context) (*Config, error) { - if err := ctx.Validate(); err != nil { - return nil, fmt.Errorf("invalid context: %w", err) - } - reqPath := filepath.Join(ctx.LibrarianDir, "generate-request.json") - slog.Debug("languagecontainer: reading generate request", "path", reqPath) - - generateReq, err := message.ParseLibrary(reqPath) - if err != nil { - return nil, err - } - slog.Debug("languagecontainer: successfully unmarshalled request", "library_id", generateReq.ID) - return &Config{ - Context: ctx, - Request: generateReq, - }, nil -} diff --git a/internal/legacylibrarian/legacycontainer/java/languagecontainer/generate/generate_test.go b/internal/legacylibrarian/legacycontainer/java/languagecontainer/generate/generate_test.go deleted file mode 100644 index 8e9de67cd6a..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/languagecontainer/generate/generate_test.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package generate - -import ( - "path/filepath" - "testing" - - "github.com/google/go-cmp/cmp" - "github.com/googleapis/librarian/internal/legacylibrarian/legacycontainer/java/message" -) - -func TestNewConfig(t *testing.T) { - // This reads generate-request.json from testdata. - librarianDir := filepath.Join("..", "testdata") - want := &Config{ - Context: &Context{ - LibrarianDir: librarianDir, - InputDir: "in", - OutputDir: "out", - SourceDir: "source", - }, - Request: &message.Library{ - ID: "chronicle", - Version: "0.1.1", - APIs: []message.API{ - { - Path: "google/cloud/chronicle/v1", - ServiceConfig: "chronicle_v1.yaml", - }, - }, - SourcePaths: []string{ - "chronicle", - "internal/generated/snippets/chronicle", - }, - PreserveRegex: []string{ - "chronicle/aliasshim/aliasshim.go", - "chronicle/CHANGES.md", - }, - RemoveRegex: []string{ - "chronicle", - "internal/generated/snippets/chronicle", - }, - }, - } - - ctx := &Context{ - LibrarianDir: librarianDir, - InputDir: "in", - OutputDir: "out", - SourceDir: "source", - } - got, err := NewConfig(ctx) - if err != nil { - t.Fatal(err) - } - - if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("NewConfig() mismatch (-want +got):\n%s", diff) - } -} - -func TestNewConfig_validate(t *testing.T) { - tests := []struct { - name string - context *Context - }{ - { - name: "empty librarian dir", - context: &Context{}, - }, - { - name: "empty input dir", - context: &Context{ - LibrarianDir: "librarian", - }, - }, - { - name: "empty output dir", - context: &Context{ - LibrarianDir: "librarian", - InputDir: "in", - }, - }, - { - name: "empty source dir", - context: &Context{ - LibrarianDir: "librarian", - InputDir: "in", - OutputDir: "out", - }, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - if _, err := NewConfig(test.context); err == nil { - t.Error("NewConfig() error = nil, want not nil") - } - }) - } -} diff --git a/internal/legacylibrarian/legacycontainer/java/languagecontainer/languagecontainer.go b/internal/legacylibrarian/legacycontainer/java/languagecontainer/languagecontainer.go deleted file mode 100644 index d8ed901dacf..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/languagecontainer/languagecontainer.go +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package languagecontainer defines LanguageContainer interface and -// the Run function to execute commands within the container. -// This package should not have any language-specific implementation or -// Librarian CLI's implementation. -// TODO(b/447404382): Move this package to the https://github.com/googleapis/librarian -// GitHub repository once the interface is finalized. -package languagecontainer - -import ( - "context" - "encoding/json" - "flag" - "fmt" - "log/slog" - "os" - "path/filepath" - - "github.com/googleapis/librarian/internal/legacylibrarian/legacycontainer/java/languagecontainer/generate" - "github.com/googleapis/librarian/internal/legacylibrarian/legacycontainer/java/languagecontainer/release" - "github.com/googleapis/librarian/internal/legacylibrarian/legacycontainer/java/message" -) - -// LanguageContainer defines the functions for language-specific container operations. -type LanguageContainer struct { - Generate func(context.Context, *generate.Config) error - ReleaseStage func(context.Context, *release.Config) (*message.ReleaseStageResponse, error) - // Other container functions like Generate and Build will also be part of the struct. -} - -// Run accepts an implementation of the LanguageContainer. -// The args parameter contains the command-line arguments passed to the container, -// without including the program name. Usually it's os.Args[1:]. -func Run(ctx context.Context, args []string, container *LanguageContainer) int { - // Logic to parse args and call the appropriate method on the container. - // For example, if args[1] is "generate": - // request := ... // unmarshal the request from the expected location - // err := container.Generate(context.Background(), request) - // ... - if len(args) < 1 { - panic("args must not be empty") - } - cmd := args[0] - flags := args[1:] - switch cmd { - case "generate": - if container.Generate == nil { - slog.Error("languagecontainer: generate command is not implemented") - return 1 - } - return handleGenerate(ctx, flags, container) - case "configure": - slog.Warn("languagecontainer: configure command is missing") - return 1 - case "release-stage": - if container.ReleaseStage == nil { - slog.Error("languagecontainer: generate command is missing") - return 1 - } - return handleReleaseStage(ctx, flags, container) - case "build": - slog.Warn("languagecontainer: build command is not yet implemented") - return 1 - default: - slog.Error(fmt.Sprintf("languagecontainer: unknown command: %s (with flags %v)", cmd, flags)) - return 1 - } -} - -func handleGenerate(ctx context.Context, flags []string, container *LanguageContainer) int { - genCtx := &generate.Context{} - generateFlags := flag.NewFlagSet("generate", flag.ContinueOnError) - generateFlags.StringVar(&genCtx.LibrarianDir, "librarian", "/librarian", "Path to the librarian-tool input directory. Contains generate-request.json.") - generateFlags.StringVar(&genCtx.InputDir, "input", "/input", "Path to the .librarian/generator-input directory from the language repository.") - generateFlags.StringVar(&genCtx.OutputDir, "output", "/output", "Path to the empty directory where a language container writes its output.") - generateFlags.StringVar(&genCtx.SourceDir, "source", "/source", "Path to a complete checkout of the googleapis repository.") - if err := generateFlags.Parse(flags); err != nil { - slog.Error("failed to parse flags", "error", err) - return 1 - } - cfg, err := generate.NewConfig(genCtx) - if err != nil { - slog.Error("failed to create generate config", "error", err) - return 1 - } - if err := container.Generate(ctx, cfg); err != nil { - slog.Error("generate failed", "error", err) - return 1 - } - slog.Info("languagecontainer: generate command executed successfully") - return 0 -} - -func handleReleaseStage(ctx context.Context, flags []string, container *LanguageContainer) int { - cfg := &release.Context{} - releaseInitFlags := flag.NewFlagSet("release-stage", flag.ContinueOnError) - releaseInitFlags.StringVar(&cfg.LibrarianDir, "librarian", "/librarian", "Path to the librarian-tool input directory. Contains release-stage-request.json.") - releaseInitFlags.StringVar(&cfg.RepoDir, "repo", "/repo", "Path to the language repo.") - releaseInitFlags.StringVar(&cfg.OutputDir, "output", "/output", "Path to the output directory.") - if err := releaseInitFlags.Parse(flags); err != nil { - slog.Error("failed to parse flags", "error", err) - return 1 - } - requestPath := filepath.Join(cfg.LibrarianDir, "release-stage-request.json") - bytes, err := os.ReadFile(requestPath) - if err != nil { - slog.Error("failed to read request file", "path", requestPath, "error", err) - return 1 - } - request := &message.ReleaseStageRequest{} - if err := json.Unmarshal(bytes, request); err != nil { - slog.Error("failed to parse request JSON", "error", err) - return 1 - } - config := &release.Config{ - Context: cfg, - Request: request, - } - response, err := container.ReleaseStage(ctx, config) - if err != nil { - slog.Error("release-stage failed", "error", err) - return 1 - } - bytes, err = json.MarshalIndent(response, "", " ") - if err != nil { - slog.Error("failed to marshal response JSON", "error", err) - return 1 - } - responsePath := filepath.Join(cfg.LibrarianDir, "release-stage-response.json") - if err := os.WriteFile(responsePath, bytes, 0644); err != nil { - slog.Error("failed to write response file", "path", responsePath, "error", err) - return 1 - } - slog.Info("languagecontainer: release-stage command executed successfully") - return 0 -} diff --git a/internal/legacylibrarian/legacycontainer/java/languagecontainer/languagecontainer_test.go b/internal/legacylibrarian/legacycontainer/java/languagecontainer/languagecontainer_test.go deleted file mode 100644 index 2b420c14b10..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/languagecontainer/languagecontainer_test.go +++ /dev/null @@ -1,269 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package languagecontainer - -import ( - "context" - "encoding/json" - "io/fs" - "os" - "path/filepath" - "testing" - - "github.com/google/go-cmp/cmp" - "github.com/googleapis/librarian/internal/legacylibrarian/legacycontainer/java/languagecontainer/generate" - "github.com/googleapis/librarian/internal/legacylibrarian/legacycontainer/java/languagecontainer/release" - "github.com/googleapis/librarian/internal/legacylibrarian/legacycontainer/java/message" -) - -func TestRun(t *testing.T) { - tmpDir := t.TempDir() - if err := os.WriteFile(filepath.Join(tmpDir, "release-stage-request.json"), []byte("{}"), 0644); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(tmpDir, "generate-request.json"), []byte("{}"), 0644); err != nil { - t.Fatal(err) - } - tests := []struct { - name string - args []string - wantCode int - wantErr bool - }{ - { - name: "unknown command", - args: []string{"foo"}, - wantCode: 1, - }, - { - name: "build command", - args: []string{"build"}, - wantCode: 1, // Not implemented yet - }, - { - name: "configure command", - args: []string{"configure"}, - wantCode: 1, // Not implemented yet - }, - { - name: "generate command with default flags", - args: []string{"generate"}, - wantCode: 1, // Fails because default /librarian does not exist. - }, - { - name: "generate command success", - args: []string{"generate", "-librarian", tmpDir}, - wantCode: 0, - }, - { - name: "generate command failure", - args: []string{"generate", "-librarian", tmpDir}, - wantCode: 1, - wantErr: true, - }, - { - name: "release-stage command success", - args: []string{"release-stage", "-librarian", tmpDir}, - wantCode: 0, - }, - { - name: "release-stage command failure", - args: []string{"release-stage", "-librarian", tmpDir}, - wantCode: 1, - wantErr: true, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - container := LanguageContainer{ - Generate: func(ctx context.Context, c *generate.Config) error { - if test.wantErr { - return fs.ErrNotExist - } - return nil - }, - ReleaseStage: func(ctx context.Context, c *release.Config) (*message.ReleaseStageResponse, error) { - if test.wantErr { - return nil, fs.ErrNotExist - } - return &message.ReleaseStageResponse{}, nil - }, - } - if gotCode := Run(context.Background(), test.args, &container); gotCode != test.wantCode { - t.Errorf("Run() = %v, want %v", gotCode, test.wantCode) - } - }) - } -} - -func TestRun_noArgs(t *testing.T) { - defer func() { - if r := recover(); r == nil { - t.Errorf("The code did not panic") - } - }() - Run(context.Background(), []string{}, &LanguageContainer{}) -} - -func TestRun_ReleaseStageWritesResponse(t *testing.T) { - tmpDir := t.TempDir() - if err := os.WriteFile(filepath.Join(tmpDir, "release-stage-request.json"), []byte("{}"), 0644); err != nil { - t.Fatal(err) - } - args := []string{"release-stage", "-librarian", tmpDir} - want := &message.ReleaseStageResponse{Error: "test error"} - container := LanguageContainer{ - ReleaseStage: func(ctx context.Context, c *release.Config) (*message.ReleaseStageResponse, error) { - return want, nil - }, - } - - if code := Run(context.Background(), args, &container); code != 0 { - t.Errorf("Run() = %v, want 0", code) - } - - responsePath := filepath.Join(tmpDir, "release-stage-response.json") - bytes, err := os.ReadFile(responsePath) - if err != nil { - t.Fatal(err) - } - got := &message.ReleaseStageResponse{} - if err := json.Unmarshal(bytes, got); err != nil { - t.Fatal(err) - } - if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("response mismatch (-want +got):\n%s", diff) - } -} - -func TestRun_ReleaseStageReadsContextArgs(t *testing.T) { - tmpDir := t.TempDir() - librarianDir := filepath.Join(tmpDir, "librarian") - if err := os.Mkdir(librarianDir, 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(librarianDir, "release-stage-request.json"), []byte("{}"), 0644); err != nil { - t.Fatal(err) - } - repoDir := filepath.Join(tmpDir, "repo") - if err := os.Mkdir(repoDir, 0755); err != nil { - t.Fatal(err) - } - outputDir := filepath.Join(tmpDir, "output") - if err := os.Mkdir(outputDir, 0755); err != nil { - t.Fatal(err) - } - args := []string{"release-stage", "-librarian", librarianDir, "-repo", repoDir, "-output", outputDir} - var gotConfig *release.Config - container := LanguageContainer{ - ReleaseStage: func(ctx context.Context, c *release.Config) (*message.ReleaseStageResponse, error) { - gotConfig = c - return &message.ReleaseStageResponse{}, nil - }, - } - if code := Run(context.Background(), args, &container); code != 0 { - t.Errorf("Run() = %v, want 0", code) - } - if got, want := gotConfig.Context.LibrarianDir, librarianDir; got != want { - t.Errorf("gotConfig.Context.LibrarianDir = %q, want %q", got, want) - } - if got, want := gotConfig.Context.RepoDir, repoDir; got != want { - t.Errorf("gotConfig.Context.RepoDir = %q, want %q", got, want) - } - if got, want := gotConfig.Context.OutputDir, outputDir; got != want { - t.Errorf("gotConfig.Context.OutputDir = %q, want %q", got, want) - } -} - -func TestRun_GenerateReadsContextArgs(t *testing.T) { - tmpDir := t.TempDir() - librarianDir := filepath.Join(tmpDir, "librarian") - if err := os.Mkdir(librarianDir, 0755); err != nil { - t.Fatal(err) - } - // generate.NewConfig reads generate-request.json. - if err := os.WriteFile(filepath.Join(librarianDir, "generate-request.json"), []byte("{}"), 0644); err != nil { - t.Fatal(err) - } - inputDir := filepath.Join(tmpDir, "input") - if err := os.Mkdir(inputDir, 0755); err != nil { - t.Fatal(err) - } - outputDir := filepath.Join(tmpDir, "output") - if err := os.Mkdir(outputDir, 0755); err != nil { - t.Fatal(err) - } - sourceDir := filepath.Join(tmpDir, "source") - if err := os.Mkdir(sourceDir, 0755); err != nil { - t.Fatal(err) - } - args := []string{"generate", "-librarian", librarianDir, "-input", inputDir, "-output", outputDir, "-source", sourceDir} - var gotConfig *generate.Config - container := LanguageContainer{ - Generate: func(ctx context.Context, c *generate.Config) error { - gotConfig = c - return nil - }, - } - if code := Run(context.Background(), args, &container); code != 0 { - t.Errorf("Run() = %v, want 0", code) - } - if got, want := gotConfig.Context.LibrarianDir, librarianDir; got != want { - t.Errorf("gotConfig.Context.LibrarianDir = %q, want %q", got, want) - } - if got, want := gotConfig.Context.InputDir, inputDir; got != want { - t.Errorf("gotConfig.Context.InputDir = %q, want %q", got, want) - } - if got, want := gotConfig.Context.OutputDir, outputDir; got != want { - t.Errorf("gotConfig.Context.OutputDir = %q, want %q", got, want) - } - if got, want := gotConfig.Context.SourceDir, sourceDir; got != want { - t.Errorf("gotConfig.Context.SourceDir = %q, want %q", got, want) - } -} - -func TestRun_unimplementedCommands(t *testing.T) { - tests := []struct { - name string - args []string - container *LanguageContainer - }{ - { - name: "generate is nil", - args: []string{"generate"}, - container: &LanguageContainer{ - ReleaseStage: func(context.Context, *release.Config) (*message.ReleaseStageResponse, error) { - return nil, nil - }, - }, - }, - { - name: "release-stage is nil", - args: []string{"release-stage"}, - container: &LanguageContainer{ - Generate: func(context.Context, *generate.Config) error { - return nil - }, - }, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - if gotCode := Run(context.Background(), test.args, test.container); gotCode != 1 { - t.Errorf("Run() = %v, want 1", gotCode) - } - }) - } -} diff --git a/internal/legacylibrarian/legacycontainer/java/languagecontainer/release/release.go b/internal/legacylibrarian/legacycontainer/java/languagecontainer/release/release.go deleted file mode 100644 index f67e51159dc..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/languagecontainer/release/release.go +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package release contains types for language container's release command. -package release - -import "github.com/googleapis/librarian/internal/legacylibrarian/legacycontainer/java/message" - -// Context has the directory paths for the release-stage command. -// https://github.com/googleapis/librarian/blob/main/doc/language-onboarding.md#release-stage -type Context struct { - LibrarianDir string - RepoDir string - OutputDir string -} - -// The Config for the release-stage command. This holds the context (the directory paths) -// and the request parsed from the release-stage-request.json file. -type Config struct { - Context *Context - // This request is parsed from the release-stage-request.json file in - // the LibrarianDir of the context. - Request *message.ReleaseStageRequest -} diff --git a/internal/legacylibrarian/legacycontainer/java/languagecontainer/release/release_test.go b/internal/legacylibrarian/legacycontainer/java/languagecontainer/release/release_test.go deleted file mode 100644 index c52aa41a715..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/languagecontainer/release/release_test.go +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package release - -import ( - "encoding/json" - "os" - "path/filepath" - "testing" - - "github.com/google/go-cmp/cmp" - "github.com/googleapis/librarian/internal/legacylibrarian/legacycontainer/java/message" -) - -func TestReadReleaseStageRequest(t *testing.T) { - want := &message.ReleaseStageRequest{ - Libraries: []*message.Library{ - { - ID: "google-cloud-secretmanager-v1", - Version: "1.3.0", - Changes: []*message.Change{ - { - Type: "feat", - Subject: "add new UpdateRepository API", - Body: "This adds the ability to update a repository's properties.", - PiperCLNumber: "786353207", - CommitHash: "9461532e7d19c8d71709ec3b502e5d81340fb661", - }, - { - Type: "docs", - Subject: "fix typo in BranchRule comment", - Body: "", - PiperCLNumber: "786353207", - CommitHash: "9461532e7d19c8d71709ec3b502e5d81340fb661", - }, - }, - APIs: []message.API{ - { - Path: "google/cloud/secretmanager/v1", - }, - { - Path: "google/cloud/secretmanager/v1beta", - }, - }, - SourcePaths: []string{ - "secretmanager", - "other/location/secretmanager", - }, - ReleaseTriggered: true, - }, - }, - } - bytes, err := os.ReadFile(filepath.Join("..", "testdata", "release-stage-request.json")) - if err != nil { - t.Fatal(err) - } - got := &message.ReleaseStageRequest{} - if err := json.Unmarshal(bytes, got); err != nil { - t.Fatal(err) - } - if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("Unmarshal() mismatch (-want +got):\n%s", diff) - } -} diff --git a/internal/legacylibrarian/legacycontainer/java/languagecontainer/testdata/generate-request.json b/internal/legacylibrarian/legacycontainer/java/languagecontainer/testdata/generate-request.json deleted file mode 100644 index 894b5639fc2..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/languagecontainer/testdata/generate-request.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "id": "chronicle", - "version": "0.1.1", - "apis": [ - { - "path": "google/cloud/chronicle/v1", - "service_config": "chronicle_v1.yaml" - } - ], - "source_roots": [ - "chronicle", - "internal/generated/snippets/chronicle" - ], - "preserve_regex": [ - "chronicle/aliasshim/aliasshim.go", - "chronicle/CHANGES.md" - ], - "remove_regex": [ - "chronicle", - "internal/generated/snippets/chronicle" - ] -} \ No newline at end of file diff --git a/internal/legacylibrarian/legacycontainer/java/languagecontainer/testdata/release-stage-request.json b/internal/legacylibrarian/legacycontainer/java/languagecontainer/testdata/release-stage-request.json deleted file mode 100644 index de1cd291681..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/languagecontainer/testdata/release-stage-request.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "libraries": [ - { - "id": "google-cloud-secretmanager-v1", - "version": "1.3.0", - "changes": [ - { - "type": "feat", - "subject": "add new UpdateRepository API", - "body": "This adds the ability to update a repository's properties.", - "piper_cl_number": "786353207", - "commit_hash": "9461532e7d19c8d71709ec3b502e5d81340fb661" - }, - { - "type": "docs", - "subject": "fix typo in BranchRule comment", - "body": "", - "piper_cl_number": "786353207", - "commit_hash": "9461532e7d19c8d71709ec3b502e5d81340fb661" - } - ], - "apis": [ - { - "path": "google/cloud/secretmanager/v1" - }, - { - "path": "google/cloud/secretmanager/v1beta" - } - ], - "source_roots": [ - "secretmanager", - "other/location/secretmanager" - ], - "release_triggered": true - } - ] -} diff --git a/internal/legacylibrarian/legacycontainer/java/main.go b/internal/legacylibrarian/legacycontainer/java/main.go deleted file mode 100644 index b761fb7749a..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/main.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "context" - "fmt" - "log/slog" - "os" - - "github.com/googleapis/librarian/internal/legacylibrarian/legacycontainer/java/generate" - "github.com/googleapis/librarian/internal/legacylibrarian/legacycontainer/java/languagecontainer" - "github.com/googleapis/librarian/internal/legacylibrarian/legacycontainer/java/release" -) - -const version = "0.1.0" - -// main is the entrypoint for the librariangen CLI. -func main() { - os.Exit(runCLI(os.Args)) -} - -func runCLI(args []string) int { - logLevel := parseLogLevel(os.Getenv("LIBRARIAN_GOOGLE_SDK_JAVA_LOGGING_LEVEL")) - slog.SetDefault(slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{ - Level: logLevel, - }))) - slog.Info("librariangen: invoked", "args", args) - if len(args) < 2 { - slog.Error("librariangen: expected a command") - return 1 - } - - // The --version flag is a special case and not a command. - if args[1] == "--version" { - fmt.Println(version) - return 0 - } - - container := languagecontainer.LanguageContainer{ - Generate: generate.Generate, - ReleaseStage: release.Stage, - } - return languagecontainer.Run(context.Background(), args[1:], &container) -} - -func parseLogLevel(logLevelEnv string) slog.Level { - switch logLevelEnv { - case "debug": - return slog.LevelDebug - case "quiet": - return slog.LevelError + 1 - default: - return slog.LevelInfo - } -} diff --git a/internal/legacylibrarian/legacycontainer/java/main_test.go b/internal/legacylibrarian/legacycontainer/java/main_test.go deleted file mode 100644 index 63b26100f92..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/main_test.go +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "log/slog" - "testing" -) - -func TestRunCLI(t *testing.T) { - tests := []struct { - name string - args []string - wantCode int - }{ - { - name: "success", - args: []string{"librariangen", "--version"}, - wantCode: 0, - }, - { - name: "failure", - args: []string{"librariangen", "foo"}, - wantCode: 1, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - if gotCode := runCLI(test.args); gotCode != test.wantCode { - t.Errorf("runCLI() = %v, want %v", gotCode, test.wantCode) - } - }) - } -} - -func TestParseLogLevel(t *testing.T) { - tests := []struct { - name string - level string - want slog.Level - }{ - {"default", "", slog.LevelInfo}, - {"debug", "debug", slog.LevelDebug}, - {"quiet", "quiet", slog.LevelError + 1}, - {"invalid", "foo", slog.LevelInfo}, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - if got := parseLogLevel(test.level); got != test.want { - t.Errorf("parseLogLevel() = %v, want %v", got, test.want) - } - }) - } -} diff --git a/internal/legacylibrarian/legacycontainer/java/message/message.go b/internal/legacylibrarian/legacycontainer/java/message/message.go deleted file mode 100644 index 60aa90c69e8..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/message/message.go +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package message defines data types which the Librarian CLI and language -// containers exchange. -// There shouldn't be CLI-specific data type or language container-specific -// data types in this package. -// TODO(b/447404382): Move this package to the https://github.com/googleapis/librarian -// GitHub repository once the interface is finalized. -package message - -import ( - "encoding/json" - "fmt" - "os" -) - -// ReleaseStageRequest is the structure of the release-stage-request.json file. -type ReleaseStageRequest struct { - Libraries []*Library `json:"libraries"` -} - -// ReleaseStageResponse is the structure of the release-stage-response.json file. -type ReleaseStageResponse struct { - Error string `json:"error,omitempty"` -} - -// Library is the combination of all the fields used by CLI requests and responses. -// Each CLI command has its own request/response type, but they all use Library. -type Library struct { - ID string `json:"id,omitempty"` - Version string `json:"version,omitempty"` - APIs []API `json:"apis,omitempty"` - // SourcePaths are the directories to which librarian contributes code. - // For Go, this is typically the Go module directory. - SourcePaths []string `json:"source_roots,omitempty"` - // PreserveRegex are files/directories to leave untouched during generation. - // This is useful for preserving handwritten helper files or customizations. - PreserveRegex []string `json:"preserve_regex,omitempty"` - // RemoveRegex are files/directories to remove during generation. - RemoveRegex []string `json:"remove_regex,omitempty"` - // Changes are the changes being released (in a release request) - Changes []*Change `json:"changes,omitempty"` - // Specifying a tag format allows librarian to honor this format when creating - // a tag for the release of the library. The replacement values of {id} and {version} - // permitted to reference the values configured in the library. If not specified - // the assumed format is {id}-{version}. e.g., {id}/v{version}. - TagFormat string `yaml:"tag_format,omitempty" json:"tag_format,omitempty"` - // ReleaseTriggered indicates whether this library is being released (in a release request) - ReleaseTriggered bool `json:"release_triggered,omitempty"` -} - -// API corresponds to a single API definition within a librarian request/response. -type API struct { - Path string `json:"path,omitempty"` - ServiceConfig string `json:"service_config,omitempty"` -} - -// Change represents a single commit change for a library. -type Change struct { - Type string `json:"type"` - Subject string `json:"subject"` - Body string `json:"body"` - PiperCLNumber string `json:"piper_cl_number"` - CommitHash string `json:"commit_hash"` -} - -// ParseLibrary reads a file from the given path and unmarshals -// it into a Library struct. This is used for build and generate, where the requests -// are simply the library, with no wrapping. -func ParseLibrary(path string) (*Library, error) { - data, err := os.ReadFile(path) - if err != nil { - return nil, fmt.Errorf("languagecontainer: failed to read request file from %s: %w", path, err) - } - - var req Library - if err := json.Unmarshal(data, &req); err != nil { - return nil, fmt.Errorf("languagecontainer: failed to unmarshal request file %s: %w", path, err) - } - - return &req, nil -} diff --git a/internal/legacylibrarian/legacycontainer/java/message/message_test.go b/internal/legacylibrarian/legacycontainer/java/message/message_test.go deleted file mode 100644 index 671dddaa883..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/message/message_test.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package message - -import ( - "os" - "path/filepath" - "testing" - - "github.com/google/go-cmp/cmp" -) - -func TestParseLibrary(t *testing.T) { - testCases := []struct { - name string - content string - want *Library - wantErr bool - }{ - { - name: "valid request", - content: `{ - "id": "asset", - "version": "1.15.0", - "apis": [ - { - "path": "google/cloud/asset/v1", - "service_config": "cloudasset_v1.yaml" - } - ], - "source_roots": ["asset/apiv1"], - "preserve_regex": ["asset/apiv1/foo.go"], - "remove_regex": ["asset/apiv1/bar.go"] - }`, - want: &Library{ - ID: "asset", - Version: "1.15.0", - APIs: []API{ - { - Path: "google/cloud/asset/v1", - ServiceConfig: "cloudasset_v1.yaml", - }, - }, - SourcePaths: []string{"asset/apiv1"}, - PreserveRegex: []string{"asset/apiv1/foo.go"}, - RemoveRegex: []string{"asset/apiv1/bar.go"}, - }, - wantErr: false, - }, - { - name: "malformed json", - content: `{"id": "foo",`, - wantErr: true, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - tmpDir := t.TempDir() - reqPath := filepath.Join(tmpDir, "generate-request.json") - if err := os.WriteFile(reqPath, []byte(tc.content), 0644); err != nil { - t.Fatalf("failed to write test file: %v", err) - } - - got, err := ParseLibrary(reqPath) - - if (err != nil) != tc.wantErr { - t.Errorf("Parse() error = %v, wantErr %v", err, tc.wantErr) - return - } - - if !tc.wantErr { - if diff := cmp.Diff(tc.want, got); diff != "" { - t.Errorf("Parse() mismatch (-want +got):\n%s", diff) - } - } - }) - } -} - -func TestParseLibrary_FileNotFound(t *testing.T) { - _, err := ParseLibrary("non-existent-file.json") - if err == nil { - t.Error("Parse() expected error for non-existent file, got nil") - } -} diff --git a/internal/legacylibrarian/legacycontainer/java/pom/pom.go b/internal/legacylibrarian/legacycontainer/java/pom/pom.go deleted file mode 100644 index e5e34b6338a..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/pom/pom.go +++ /dev/null @@ -1,264 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package pom handles the generation of Maven pom.xml files for a Java library. -package pom - -import ( - "embed" - "errors" - "fmt" - "io/fs" - "os" - "path/filepath" - "sort" - "strings" - "text/template" -) - -//go:embed template/*.tmpl -var templatesFS embed.FS - -var templates *template.Template - -func init() { - templates = template.Must(template.New("").ParseFS(templatesFS, "template/*.tmpl")) -} - -// Module represents a Maven module. -type Module struct { - GroupId string - ArtifactId string - Version string -} - -// Generate generates the pom.xml files for a library. -// Precondition: module directories exist except for for the *-bom. -func Generate(libraryPath, libraryID string) error { - // 1. Create main module from libraryID. - mainModule := &Module{ - GroupId: "com.google.cloud", - ArtifactId: fmt.Sprintf("google-cloud-%s", libraryID), - Version: "0.0.1-SNAPSHOT", // Default version - } - - // 2. Find other modules (proto, grpc). - modules, protoModules, grpcModules, err := findModules(libraryPath, mainModule) - if err != nil { - return fmt.Errorf("could not find modules: %w", err) - } - - // 3. Render templates - if err := renderTemplates(libraryPath, mainModule, modules, protoModules, grpcModules, libraryID); err != nil { - return fmt.Errorf("could not render templates: %w", err) - } - - return nil -} - -func findModules(libraryPath string, mainModule *Module) (map[string]*Module, []*Module, []*Module, error) { - modules := make(map[string]*Module) - protoModules := []*Module{} - grpcModules := []*Module{} - - modules[mainModule.ArtifactId] = mainModule - - files, err := os.ReadDir(libraryPath) - if err != nil { - return nil, nil, nil, err - } - - for _, f := range files { - if f.IsDir() { - if strings.HasPrefix(f.Name(), "proto-") { - module := &Module{ - GroupId: "com.google.api.grpc", - ArtifactId: f.Name(), - Version: mainModule.Version, - } - modules[f.Name()] = module - protoModules = append(protoModules, module) - } else if strings.HasPrefix(f.Name(), "grpc-") { - module := &Module{ - GroupId: "com.google.api.grpc", - ArtifactId: f.Name(), - Version: mainModule.Version, - } - modules[f.Name()] = module - grpcModules = append(grpcModules, module) - } - } - } - return modules, protoModules, grpcModules, nil -} - -func renderTemplates(libraryPath string, mainModule *Module, modules map[string]*Module, protoModules, grpcModules []*Module, libraryID string) error { - // Render the parent pom.xml - if err := renderParentPom(libraryPath, mainModule, modules, libraryID); err != nil { - return err - } - - for path, module := range modules { - if strings.HasPrefix(path, "proto-") { - if err := renderProtoPom(filepath.Join(libraryPath, path), mainModule, module); err != nil { - return err - } - } - if strings.HasPrefix(path, "grpc-") { - protoArtifactId := strings.Replace(module.ArtifactId, "grpc-", "proto-", 1) - protoModule, ok := modules[protoArtifactId] - if !ok { - return fmt.Errorf("grpc module %s exists without a corresponding proto module", module.ArtifactId) - } - if err := renderGrpcPom(filepath.Join(libraryPath, path), mainModule, module, protoModule); err != nil { - return err - } - } - } - mainArtifactDir := filepath.Join(libraryPath, mainModule.ArtifactId) - if err := renderCloudPom(mainArtifactDir, mainModule, protoModules, grpcModules, libraryID); err != nil { - return err - } - bomDir := filepath.Join(libraryPath, mainModule.ArtifactId+"-bom") - if err := renderBomPom(bomDir, mainModule, modules, libraryID); err != nil { - return err - } - return nil -} - -func renderParentPom(libraryPath string, mainModule *Module, modules map[string]*Module, libraryID string) error { - var moduleList []*Module - for _, m := range modules { - moduleList = append(moduleList, m) - } - sort.Slice(moduleList, func(i, j int) bool { - return moduleList[i].ArtifactId < moduleList[j].ArtifactId - }) - - data := struct { - MainModule *Module - Name string - Modules []*Module - }{ - MainModule: mainModule, - Name: fmt.Sprintf("Google Cloud %s", libraryID), - Modules: moduleList, - } - return renderPom(filepath.Join(libraryPath, "pom.xml"), "parent_pom.xml.tmpl", data) -} - -// renderPom executes a template with the given data and writes the output to the outputPath. -func renderPom(outputPath, templateName string, data interface{}) error { - pomFile, err := os.Create(outputPath) - if err != nil { - return err - } - defer pomFile.Close() - - return templates.ExecuteTemplate(pomFile, templateName, data) -} - -func renderProtoPom(modulePath string, mainModule, module *Module) error { - parentModule := &Module{ - GroupId: mainModule.GroupId, - ArtifactId: mainModule.ArtifactId + "-parent", - Version: mainModule.Version, - } - - data := struct { - MainModule *Module - Module *Module - ParentModule *Module - }{ - MainModule: mainModule, - Module: module, - ParentModule: parentModule, - } - return renderPom(filepath.Join(modulePath, "pom.xml"), "proto_pom.xml.tmpl", data) -} - -func renderGrpcPom(modulePath string, mainModule, module, protoModule *Module) error { - parentModule := &Module{ - GroupId: mainModule.GroupId, - ArtifactId: mainModule.ArtifactId + "-parent", - Version: mainModule.Version, - } - - data := struct { - MainModule *Module - Module *Module - ParentModule *Module - ProtoModule *Module - }{ - MainModule: mainModule, - Module: module, - ParentModule: parentModule, - ProtoModule: protoModule, - } - return renderPom(filepath.Join(modulePath, "pom.xml"), "grpc_pom.xml.tmpl", data) -} - -func renderCloudPom(modulePath string, mainModule *Module, protoModules, grpcModules []*Module, libraryID string) error { - parentModule := &Module{ - GroupId: mainModule.GroupId, - ArtifactId: mainModule.ArtifactId + "-parent", - Version: mainModule.Version, - } - - data := struct { - Module *Module - Name string - Description string - ParentModule *Module - ProtoModules []*Module - GrpcModules []*Module - Repo string - }{ - Module: mainModule, - Name: fmt.Sprintf("Google Cloud %s", libraryID), - Description: fmt.Sprintf("Google Cloud %s client", libraryID), - ParentModule: parentModule, - ProtoModules: protoModules, - GrpcModules: grpcModules, - Repo: "googleapis/google-cloud-java", - } - - return renderPom(filepath.Join(modulePath, "pom.xml"), "cloud_pom.xml.tmpl", data) -} - -func renderBomPom(modulePath string, mainModule *Module, modules map[string]*Module, libraryID string) error { - if _, err := os.Stat(modulePath); errors.Is(err, fs.ErrNotExist) { - if err := os.MkdirAll(modulePath, 0755); err != nil { - return err - } - } - var moduleList []*Module - for _, m := range modules { - moduleList = append(moduleList, m) - } - sort.Slice(moduleList, func(i, j int) bool { - return moduleList[i].ArtifactId < moduleList[j].ArtifactId - }) - - data := struct { - MainModule *Module - Name string - Modules []*Module - }{ - MainModule: mainModule, - Name: fmt.Sprintf("Google Cloud %s", libraryID), - Modules: moduleList, - } - return renderPom(filepath.Join(modulePath, "pom.xml"), "bom_pom.xml.tmpl", data) -} diff --git a/internal/legacylibrarian/legacycontainer/java/pom/pom_test.go b/internal/legacylibrarian/legacycontainer/java/pom/pom_test.go deleted file mode 100644 index 4c183c67d79..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/pom/pom_test.go +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pom - -import ( - "errors" - "fmt" - "io/fs" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/google/go-cmp/cmp" -) - -func TestGenerate(t *testing.T) { - testCases := []struct { - name string - libraryID string - modules []string - goldenFiles map[string]string - wantErr bool - errorContains string - }{ - { - name: "happy path with proto and grpc", - libraryID: "test", - modules: []string{"proto-test", "grpc-test"}, - goldenFiles: map[string]string{ - "pom.xml": "testdata/happy_path_parent_pom.xml", - "proto-test/pom.xml": "testdata/happy_path_proto_pom.xml", - "grpc-test/pom.xml": "testdata/happy_path_grpc_pom.xml", - "google-cloud-test/pom.xml": "testdata/happy_path_cloud_pom.xml", - "google-cloud-test-bom/pom.xml": "testdata/happy_path_bom_pom.xml", - }, - wantErr: false, - }, - { - name: "only proto module", - libraryID: "test", - modules: []string{"proto-test"}, - goldenFiles: map[string]string{ - "pom.xml": "testdata/only_proto_parent_pom.xml", - "proto-test/pom.xml": "testdata/only_proto_proto_pom.xml", - "google-cloud-test/pom.xml": "testdata/only_proto_cloud_pom.xml", - "google-cloud-test-bom/pom.xml": "testdata/only_proto_bom_pom.xml", - }, - wantErr: false, - }, - { - name: "only grpc module", - libraryID: "test", - modules: []string{"grpc-test"}, - wantErr: true, - errorContains: "grpc module grpc-test exists without a corresponding proto module", - }, { - name: "non-existent libraryPath", - libraryID: "test", - modules: []string{}, - wantErr: true, - errorContains: "could not find modules", - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - var libraryPath string - if tc.name == "non-existent libraryPath" { - libraryPath = filepath.Join(t.TempDir(), "non-existent") - } else { - libraryPath = t.TempDir() - for _, module := range tc.modules { - err := os.Mkdir(filepath.Join(libraryPath, module), 0755) - if err != nil { - t.Fatalf("failed to create module directory %s: %v", module, err) - } - } - // Create main artifact directory - mainArtifactDir := filepath.Join(libraryPath, fmt.Sprintf("google-cloud-%s", tc.libraryID)) - if err := os.Mkdir(mainArtifactDir, 0755); err != nil { - t.Fatalf("failed to create main artifact directory %s: %v", mainArtifactDir, err) - } - } - - err := Generate(libraryPath, tc.libraryID) - if (err != nil) != tc.wantErr { - t.Errorf("Generate() error = %v, wantErr %v", err, tc.wantErr) - return - } - - if tc.wantErr { - if !strings.Contains(err.Error(), tc.errorContains) { - t.Errorf("Generate() error = %v, want error containing %q", err, tc.errorContains) - } - return - } - - for generatedFile, goldenFile := range tc.goldenFiles { - generatedContent, err := os.ReadFile(filepath.Join(libraryPath, generatedFile)) - if err != nil { - t.Fatalf("failed to read generated file %s: %v", generatedFile, err) - } - - goldenContent, err := os.ReadFile(goldenFile) - if err != nil { - // If golden files don't exist, create them. - if errors.Is(err, fs.ErrNotExist) { - goldenFileDir := filepath.Dir(goldenFile) - if _, err := os.Stat(goldenFileDir); errors.Is(err, fs.ErrNotExist) { - if err := os.MkdirAll(goldenFileDir, 0755); err != nil { - t.Fatalf("failed to create golden file directory %s: %v", goldenFileDir, err) - } - } - if err := os.WriteFile(goldenFile, generatedContent, 0644); err != nil { - t.Fatalf("failed to write golden file %s: %v", goldenFile, err) - } - t.Logf("created golden file %s", goldenFile) - // Reread the golden file to continue the test - goldenContent, err = os.ReadFile(goldenFile) - if err != nil { - t.Fatalf("failed to read newly created golden file %s: %v", goldenFile, err) - } - } else { - t.Fatalf("failed to read golden file %s: %v", goldenFile, err) - } - } - - if diff := cmp.Diff(string(goldenContent), string(generatedContent)); diff != "" { - t.Errorf("generated file %s content mismatch (-want +got):\n%s", generatedFile, diff) - } - } - }) - } -} diff --git a/internal/legacylibrarian/legacycontainer/java/pom/pom_update.go b/internal/legacylibrarian/legacycontainer/java/pom/pom_update.go deleted file mode 100644 index c8eda022aa7..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/pom/pom_update.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pom - -import ( - "errors" - "fmt" - "io/fs" - "os" - "path/filepath" - "regexp" -) - -var ( - versionRegex = regexp.MustCompile(`()([^<]+)(\s*)`) -) - -// UpdateVersions updates the versions of all pom.xml files in a given directory. -// It appends the "-SNAPSHOT" suffix to the version given the version parameter. -// If the directory is not present, this function creates it. -func UpdateVersions(repoDir, sourcePath, outputDir, libraryID, version string) error { - pomFiles, err := findPomFiles(sourcePath) - if err != nil { - return fmt.Errorf("failed to find pom files: %w", err) - } - for _, pomFile := range pomFiles { - relPath, err := filepath.Rel(repoDir, pomFile) - if err != nil { - return fmt.Errorf("failed to get relative path for %s: %w", pomFile, err) - } - outputPomFile := filepath.Join(outputDir, relPath) - if err := os.MkdirAll(filepath.Dir(outputPomFile), 0755); err != nil { - return fmt.Errorf("failed to create output directory for %s: %w", outputPomFile, err) - } - if err := updateVersion(pomFile, outputPomFile, libraryID, version); err != nil { - return fmt.Errorf("failed to update version in %s: %w", pomFile, err) - } - } - return nil -} - -// updateVersion updates the version in a single pom.xml file. -// It appends the "-SNAPSHOT" suffix to the the version parameter. -// The directory for outputPath must already exist. -func updateVersion(inputPath, outputPath, libraryID, version string) error { - content, err := os.ReadFile(inputPath) - if err != nil { - return fmt.Errorf("failed to read file: %w", err) - } - - newContent := versionRegex.ReplaceAllStringFunc(string(content), func(s string) string { - matches := versionRegex.FindStringSubmatch(s) - if len(matches) > 4 && matches[4] == libraryID { - // matches[1] is "" - // matches[2] is the old version - // matches[3] is " " - // matches[4] is libraryID - return fmt.Sprintf("%s%s-SNAPSHOT%s", matches[1], version, matches[3]) - } - return s - }) - - if err := os.WriteFile(outputPath, []byte(newContent), 0644); err != nil { - return fmt.Errorf("failed to write file: %w", err) - } - return nil -} - -func findPomFiles(path string) ([]string, error) { - var pomFiles []string - // Return empty if there's no matching directory. - if _, err := os.Stat(path); errors.Is(err, fs.ErrNotExist) { - return []string{}, nil - } - - err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if !info.IsDir() && info.Name() == "pom.xml" { - pomFiles = append(pomFiles, path) - } - return nil - }) - if err != nil { - return nil, fmt.Errorf("failed to walk path: %w", err) - } - return pomFiles, nil -} diff --git a/internal/legacylibrarian/legacycontainer/java/pom/pom_update_test.go b/internal/legacylibrarian/legacycontainer/java/pom/pom_update_test.go deleted file mode 100644 index e0d31395c5c..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/pom/pom_update_test.go +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pom - -import ( - "os" - "path/filepath" - "testing" -) - -func TestUpdateVersion(t *testing.T) { - tests := []struct { - name string - initial string - libraryID string - version string - expected string - expectError bool - }{ - { - name: "happy path", - initial: ` - 1.0.0-SNAPSHOT -`, - libraryID: "google-cloud-java", - version: "2.0.0", - expected: ` - 2.0.0-SNAPSHOT -`, - }, - { - name: "no match", - initial: ` - 1.0.0-SNAPSHOT -`, - libraryID: "wrong-library-id", - version: "2.0.0", - expected: ` - 1.0.0-SNAPSHOT -`, - }, - { - name: "multiple versions", - initial: ` - 1.0.0-SNAPSHOT - - com.google.cloud - google-cloud-secretmanager - 1.2.3-SNAPSHOT - -`, - libraryID: "google-cloud-secretmanager", - version: "2.0.0", - expected: ` - 1.0.0-SNAPSHOT - - com.google.cloud - google-cloud-secretmanager - 2.0.0-SNAPSHOT - -`, - }, - { - name: "no comment", - initial: ` - 1.0.0-SNAPSHOT -`, - libraryID: "google-cloud-java", - version: "2.0.0", - expected: ` - 1.0.0-SNAPSHOT -`, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - tmpDir := t.TempDir() - pomPath := filepath.Join(tmpDir, "pom.xml") - outPath := filepath.Join(tmpDir, "out", "pom.xml") - if err := os.WriteFile(pomPath, []byte(test.initial), 0644); err != nil { - t.Fatalf("failed to write initial pom.xml: %v", err) - } - - if err := os.MkdirAll(filepath.Dir(outPath), 0755); err != nil { - t.Fatalf("failed to create output directory: %v", err) - } - err := updateVersion(pomPath, outPath, test.libraryID, test.version) - - if test.expectError { - if err == nil { - t.Errorf("expected error, got nil") - } - } else { - if err != nil { - t.Errorf("unexpected error: %v", err) - } - content, readErr := os.ReadFile(outPath) - if readErr != nil { - t.Fatalf("failed to read pom.xml: %v", readErr) - } - if string(content) != test.expected { - t.Errorf("expected:\n%s\ngot:\n%s", test.expected, string(content)) - } - } - }) - } -} diff --git a/internal/legacylibrarian/legacycontainer/java/pom/template/bom_pom.xml.tmpl b/internal/legacylibrarian/legacycontainer/java/pom/template/bom_pom.xml.tmpl deleted file mode 100644 index 0f2f5148dca..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/pom/template/bom_pom.xml.tmpl +++ /dev/null @@ -1,34 +0,0 @@ - - - 4.0.0 - {{.MainModule.GroupId}} - {{.MainModule.ArtifactId}}-bom - {{.MainModule.Version}} - pom - - com.google.cloud - google-cloud-pom-parent - 1.72.0 - - - {{.Name}} BOM - - BOM for {{.Name}} - - - - true - - - - - {{- range .Modules }} - - {{.GroupId}} - {{.ArtifactId}} - {{.Version}} - - {{- end }} - - - \ No newline at end of file diff --git a/internal/legacylibrarian/legacycontainer/java/pom/template/cloud_pom.xml.tmpl b/internal/legacylibrarian/legacycontainer/java/pom/template/cloud_pom.xml.tmpl deleted file mode 100644 index 3a84ebfd1d3..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/pom/template/cloud_pom.xml.tmpl +++ /dev/null @@ -1,134 +0,0 @@ - - - 4.0.0 - {{.Module.GroupId}} - {{.Module.ArtifactId}} - {{.Module.Version}} - jar - {{.Name}} - https://github.com/{{.Repo}} - {{.Description}} - - {{.ParentModule.GroupId}} - {{.ParentModule.ArtifactId}} - {{.ParentModule.Version}} - - - {{.Module.ArtifactId}} - - - - io.grpc - grpc-api - - - io.grpc - grpc-stub - - - io.grpc - grpc-protobuf - - - com.google.api - api-common - - - com.google.protobuf - protobuf-java - - - com.google.api.grpc - proto-google-common-protos - -{{- range .ProtoModules }} - - {{.GroupId}} - {{.ArtifactId}} - -{{- end }} - - com.google.guava - guava - - - com.google.api - gax - - - com.google.api - gax-grpc - - - com.google.api - gax-httpjson - - - com.google.api.grpc - proto-google-iam-v1 - - - org.threeten - threetenbp - - - - - com.google.api.grpc - grpc-google-common-protos - test - - - com.google.api.grpc - grpc-google-iam-v1 - test - - - junit - junit - test - -{{- range .GrpcModules }} - - {{.GroupId}} - {{.ArtifactId}} - test - -{{- end }} - - - com.google.api - gax - testlib - test - - - com.google.api - gax-grpc - testlib - test - - - com.google.api - gax-httpjson - testlib - test - - - - - - java9 - - [9,) - - - - javax.annotation - javax.annotation-api - - - - - - diff --git a/internal/legacylibrarian/legacycontainer/java/pom/template/grpc_pom.xml.tmpl b/internal/legacylibrarian/legacycontainer/java/pom/template/grpc_pom.xml.tmpl deleted file mode 100644 index 8ed26fd9841..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/pom/template/grpc_pom.xml.tmpl +++ /dev/null @@ -1,46 +0,0 @@ - - 4.0.0 - {{.Module.GroupId}} - {{.Module.ArtifactId}} - {{.Module.Version}} - {{.Module.ArtifactId}} - GRPC library for {{.MainModule.ArtifactId}} - - {{.ParentModule.GroupId}} - {{.ParentModule.ArtifactId}} - {{.ParentModule.Version}} - - - - io.grpc - grpc-api - - - io.grpc - grpc-stub - - - io.grpc - grpc-protobuf - - - com.google.protobuf - protobuf-java - - - com.google.api.grpc - proto-google-common-protos - - - {{.ProtoModule.GroupId}} - {{.ProtoModule.ArtifactId}} - - - com.google.guava - guava - - - - \ No newline at end of file diff --git a/internal/legacylibrarian/legacycontainer/java/pom/template/parent_pom.xml.tmpl b/internal/legacylibrarian/legacycontainer/java/pom/template/parent_pom.xml.tmpl deleted file mode 100644 index f6c124dee25..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/pom/template/parent_pom.xml.tmpl +++ /dev/null @@ -1,45 +0,0 @@ - - - 4.0.0 - {{.MainModule.GroupId}} - {{.MainModule.ArtifactId}}-parent - pom - {{.MainModule.Version}} - {{.Name}} Parent - - Java idiomatic client for Google Cloud Platform services. - - - - com.google.cloud - google-cloud-jar-parent - 1.72.0 - - - - UTF-8 - UTF-8 - github - {{.MainModule.ArtifactId}}-parent - - - - -{{- range .Modules }} - - {{.GroupId}} - {{.ArtifactId}} - {{.Version}} - -{{- end }} - - - - -{{- range .Modules }} - {{.ArtifactId}} -{{- end }} - {{.MainModule.ArtifactId}}-bom - - - \ No newline at end of file diff --git a/internal/legacylibrarian/legacycontainer/java/pom/template/proto_pom.xml.tmpl b/internal/legacylibrarian/legacycontainer/java/pom/template/proto_pom.xml.tmpl deleted file mode 100644 index 127f763ac0c..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/pom/template/proto_pom.xml.tmpl +++ /dev/null @@ -1,38 +0,0 @@ - - 4.0.0 - {{.Module.GroupId}} - {{.Module.ArtifactId}} - {{.Module.Version}} - {{.Module.ArtifactId}} - Proto library for {{.MainModule.ArtifactId}} - - {{.ParentModule.GroupId}} - {{.ParentModule.ArtifactId}} - {{.ParentModule.Version}} - - - - com.google.protobuf - protobuf-java - - - com.google.api.grpc - proto-google-common-protos - - - com.google.api.grpc - proto-google-iam-v1 - - - com.google.api - api-common - - - com.google.guava - guava - - - - \ No newline at end of file diff --git a/internal/legacylibrarian/legacycontainer/java/pom/testdata/happy_path_bom_pom.xml b/internal/legacylibrarian/legacycontainer/java/pom/testdata/happy_path_bom_pom.xml deleted file mode 100644 index aa2368103db..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/pom/testdata/happy_path_bom_pom.xml +++ /dev/null @@ -1,42 +0,0 @@ - - - 4.0.0 - com.google.cloud - google-cloud-test-bom - 0.0.1-SNAPSHOT - pom - - com.google.cloud - google-cloud-pom-parent - 1.72.0 - - - Google Cloud test BOM - - BOM for Google Cloud test - - - - true - - - - - - com.google.cloud - google-cloud-test - 0.0.1-SNAPSHOT - - - com.google.api.grpc - grpc-test - 0.0.1-SNAPSHOT - - - com.google.api.grpc - proto-test - 0.0.1-SNAPSHOT - - - - \ No newline at end of file diff --git a/internal/legacylibrarian/legacycontainer/java/pom/testdata/happy_path_cloud_pom.xml b/internal/legacylibrarian/legacycontainer/java/pom/testdata/happy_path_cloud_pom.xml deleted file mode 100644 index 28a7b21bf99..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/pom/testdata/happy_path_cloud_pom.xml +++ /dev/null @@ -1,130 +0,0 @@ - - - 4.0.0 - com.google.cloud - google-cloud-test - 0.0.1-SNAPSHOT - jar - Google Cloud test - https://github.com/googleapis/google-cloud-java - Google Cloud test client - - com.google.cloud - google-cloud-test-parent - 0.0.1-SNAPSHOT - - - google-cloud-test - - - - io.grpc - grpc-api - - - io.grpc - grpc-stub - - - io.grpc - grpc-protobuf - - - com.google.api - api-common - - - com.google.protobuf - protobuf-java - - - com.google.api.grpc - proto-google-common-protos - - - com.google.api.grpc - proto-test - - - com.google.guava - guava - - - com.google.api - gax - - - com.google.api - gax-grpc - - - com.google.api - gax-httpjson - - - com.google.api.grpc - proto-google-iam-v1 - - - org.threeten - threetenbp - - - - - com.google.api.grpc - grpc-google-common-protos - test - - - com.google.api.grpc - grpc-google-iam-v1 - test - - - junit - junit - test - - - com.google.api.grpc - grpc-test - test - - - - com.google.api - gax - testlib - test - - - com.google.api - gax-grpc - testlib - test - - - com.google.api - gax-httpjson - testlib - test - - - - - - java9 - - [9,) - - - - javax.annotation - javax.annotation-api - - - - - - diff --git a/internal/legacylibrarian/legacycontainer/java/pom/testdata/happy_path_grpc_pom.xml b/internal/legacylibrarian/legacycontainer/java/pom/testdata/happy_path_grpc_pom.xml deleted file mode 100644 index 2f18d7b0436..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/pom/testdata/happy_path_grpc_pom.xml +++ /dev/null @@ -1,46 +0,0 @@ - - 4.0.0 - com.google.api.grpc - grpc-test - 0.0.1-SNAPSHOT - grpc-test - GRPC library for google-cloud-test - - com.google.cloud - google-cloud-test-parent - 0.0.1-SNAPSHOT - - - - io.grpc - grpc-api - - - io.grpc - grpc-stub - - - io.grpc - grpc-protobuf - - - com.google.protobuf - protobuf-java - - - com.google.api.grpc - proto-google-common-protos - - - com.google.api.grpc - proto-test - - - com.google.guava - guava - - - - \ No newline at end of file diff --git a/internal/legacylibrarian/legacycontainer/java/pom/testdata/happy_path_parent_pom.xml b/internal/legacylibrarian/legacycontainer/java/pom/testdata/happy_path_parent_pom.xml deleted file mode 100644 index aea1ac1a0de..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/pom/testdata/happy_path_parent_pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - com.google.cloud - google-cloud-test-parent - pom - 0.0.1-SNAPSHOT - Google Cloud test Parent - - Java idiomatic client for Google Cloud Platform services. - - - - com.google.cloud - google-cloud-jar-parent - 1.72.0 - - - - UTF-8 - UTF-8 - github - google-cloud-test-parent - - - - - - com.google.cloud - google-cloud-test - 0.0.1-SNAPSHOT - - - com.google.api.grpc - grpc-test - 0.0.1-SNAPSHOT - - - com.google.api.grpc - proto-test - 0.0.1-SNAPSHOT - - - - - - google-cloud-test - grpc-test - proto-test - google-cloud-test-bom - - - \ No newline at end of file diff --git a/internal/legacylibrarian/legacycontainer/java/pom/testdata/happy_path_proto_pom.xml b/internal/legacylibrarian/legacycontainer/java/pom/testdata/happy_path_proto_pom.xml deleted file mode 100644 index 2a487693a66..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/pom/testdata/happy_path_proto_pom.xml +++ /dev/null @@ -1,38 +0,0 @@ - - 4.0.0 - com.google.api.grpc - proto-test - 0.0.1-SNAPSHOT - proto-test - Proto library for google-cloud-test - - com.google.cloud - google-cloud-test-parent - 0.0.1-SNAPSHOT - - - - com.google.protobuf - protobuf-java - - - com.google.api.grpc - proto-google-common-protos - - - com.google.api.grpc - proto-google-iam-v1 - - - com.google.api - api-common - - - com.google.guava - guava - - - - \ No newline at end of file diff --git a/internal/legacylibrarian/legacycontainer/java/pom/testdata/only_proto_bom_pom.xml b/internal/legacylibrarian/legacycontainer/java/pom/testdata/only_proto_bom_pom.xml deleted file mode 100644 index 81267a0ac6a..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/pom/testdata/only_proto_bom_pom.xml +++ /dev/null @@ -1,37 +0,0 @@ - - - 4.0.0 - com.google.cloud - google-cloud-test-bom - 0.0.1-SNAPSHOT - pom - - com.google.cloud - google-cloud-pom-parent - 1.72.0 - - - Google Cloud test BOM - - BOM for Google Cloud test - - - - true - - - - - - com.google.cloud - google-cloud-test - 0.0.1-SNAPSHOT - - - com.google.api.grpc - proto-test - 0.0.1-SNAPSHOT - - - - \ No newline at end of file diff --git a/internal/legacylibrarian/legacycontainer/java/pom/testdata/only_proto_cloud_pom.xml b/internal/legacylibrarian/legacycontainer/java/pom/testdata/only_proto_cloud_pom.xml deleted file mode 100644 index d88913e374f..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/pom/testdata/only_proto_cloud_pom.xml +++ /dev/null @@ -1,125 +0,0 @@ - - - 4.0.0 - com.google.cloud - google-cloud-test - 0.0.1-SNAPSHOT - jar - Google Cloud test - https://github.com/googleapis/google-cloud-java - Google Cloud test client - - com.google.cloud - google-cloud-test-parent - 0.0.1-SNAPSHOT - - - google-cloud-test - - - - io.grpc - grpc-api - - - io.grpc - grpc-stub - - - io.grpc - grpc-protobuf - - - com.google.api - api-common - - - com.google.protobuf - protobuf-java - - - com.google.api.grpc - proto-google-common-protos - - - com.google.api.grpc - proto-test - - - com.google.guava - guava - - - com.google.api - gax - - - com.google.api - gax-grpc - - - com.google.api - gax-httpjson - - - com.google.api.grpc - proto-google-iam-v1 - - - org.threeten - threetenbp - - - - - com.google.api.grpc - grpc-google-common-protos - test - - - com.google.api.grpc - grpc-google-iam-v1 - test - - - junit - junit - test - - - - com.google.api - gax - testlib - test - - - com.google.api - gax-grpc - testlib - test - - - com.google.api - gax-httpjson - testlib - test - - - - - - java9 - - [9,) - - - - javax.annotation - javax.annotation-api - - - - - - diff --git a/internal/legacylibrarian/legacycontainer/java/pom/testdata/only_proto_parent_pom.xml b/internal/legacylibrarian/legacycontainer/java/pom/testdata/only_proto_parent_pom.xml deleted file mode 100644 index 42cf8e0eac9..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/pom/testdata/only_proto_parent_pom.xml +++ /dev/null @@ -1,47 +0,0 @@ - - - 4.0.0 - com.google.cloud - google-cloud-test-parent - pom - 0.0.1-SNAPSHOT - Google Cloud test Parent - - Java idiomatic client for Google Cloud Platform services. - - - - com.google.cloud - google-cloud-jar-parent - 1.72.0 - - - - UTF-8 - UTF-8 - github - google-cloud-test-parent - - - - - - com.google.cloud - google-cloud-test - 0.0.1-SNAPSHOT - - - com.google.api.grpc - proto-test - 0.0.1-SNAPSHOT - - - - - - google-cloud-test - proto-test - google-cloud-test-bom - - - \ No newline at end of file diff --git a/internal/legacylibrarian/legacycontainer/java/pom/testdata/only_proto_proto_pom.xml b/internal/legacylibrarian/legacycontainer/java/pom/testdata/only_proto_proto_pom.xml deleted file mode 100644 index 2a487693a66..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/pom/testdata/only_proto_proto_pom.xml +++ /dev/null @@ -1,38 +0,0 @@ - - 4.0.0 - com.google.api.grpc - proto-test - 0.0.1-SNAPSHOT - proto-test - Proto library for google-cloud-test - - com.google.cloud - google-cloud-test-parent - 0.0.1-SNAPSHOT - - - - com.google.protobuf - protobuf-java - - - com.google.api.grpc - proto-google-common-protos - - - com.google.api.grpc - proto-google-iam-v1 - - - com.google.api - api-common - - - com.google.guava - guava - - - - \ No newline at end of file diff --git a/internal/legacylibrarian/legacycontainer/java/protoc/protoc.go b/internal/legacylibrarian/legacycontainer/java/protoc/protoc.go deleted file mode 100644 index 6669c9b4094..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/protoc/protoc.go +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package protoc provides utilities for constructing protoc command arguments. -package protoc - -import ( - "fmt" - "os" - "path/filepath" - "strings" -) - -// ConfigProvider is an interface that describes the configuration needed -// by the Build function. This allows the protoc package to be decoupled -// from the source of the configuration (e.g., Bazel files, JSON, etc.). -type ConfigProvider interface { - ServiceYAML() string - GapicYAML() string - GRPCServiceConfig() string - Transport() string - HasRESTNumericEnums() bool - HasGAPIC() bool -} - -// OutputConfig provides paths to directories to be used for protoc output. -type OutputConfig struct { - GAPICDir string - GRPCDir string - ProtoDir string -} - -// Build constructs the full protoc command arguments for a given API. -func Build(apiServiceDir string, config ConfigProvider, sourceDir string, outputConfig *OutputConfig) ([]string, error) { - // Gather all .proto files in the API's source directory. - entries, err := os.ReadDir(apiServiceDir) - if err != nil { - return nil, fmt.Errorf("librariangen: failed to read API source directory %s: %w", apiServiceDir, err) - } - - var protoFiles []string - for _, entry := range entries { - if !entry.IsDir() && filepath.Ext(entry.Name()) == ".proto" { - protoFiles = append(protoFiles, filepath.Join(apiServiceDir, entry.Name())) - } - } - // Add common protos to the list of proto files to be compiled. - protoFiles = append(protoFiles, filepath.Join(sourceDir, "google", "cloud", "common_resources.proto")) - - if len(protoFiles) == 0 { - return nil, fmt.Errorf("librariangen: no .proto files found in %s", apiServiceDir) - } - - // Construct the protoc command arguments. - var gapicOpts []string - if config.HasGAPIC() { - if config.ServiceYAML() != "" { - gapicOpts = append(gapicOpts, fmt.Sprintf("api-service-config=%s", filepath.Join(apiServiceDir, config.ServiceYAML()))) - } - if config.GapicYAML() != "" { - gapicOpts = append(gapicOpts, fmt.Sprintf("gapic-config=%s", filepath.Join(apiServiceDir, config.GapicYAML()))) - } - if config.GRPCServiceConfig() != "" { - gapicOpts = append(gapicOpts, fmt.Sprintf("grpc-service-config=%s", filepath.Join(apiServiceDir, config.GRPCServiceConfig()))) - } - if config.Transport() != "" { - gapicOpts = append(gapicOpts, fmt.Sprintf("transport=%s", config.Transport())) - } - if config.HasRESTNumericEnums() { - gapicOpts = append(gapicOpts, "rest-numeric-enums") - } - } - - args := []string{ - "protoc", - "--experimental_allow_proto3_optional", - } - - args = append(args, fmt.Sprintf("--java_out=%s", outputConfig.ProtoDir)) - if config.Transport() != "" && config.Transport() != "rest" { - args = append(args, fmt.Sprintf("--java_grpc_out=%s", outputConfig.GRPCDir)) - } - if config.HasGAPIC() { - args = append(args, fmt.Sprintf("--java_gapic_out=metadata:%s", outputConfig.GAPICDir)) - - if len(gapicOpts) > 0 { - args = append(args, "--java_gapic_opt="+strings.Join(gapicOpts, ",")) - } - } - - args = append(args, - // The -I flag specifies the import path for protoc. All protos - // and their dependencies must be findable from this path. - // The /source mount contains the complete googleapis repository. - "-I="+sourceDir, - ) - - args = append(args, protoFiles...) - - return args, nil -} diff --git a/internal/legacylibrarian/legacycontainer/java/protoc/protoc_test.go b/internal/legacylibrarian/legacycontainer/java/protoc/protoc_test.go deleted file mode 100644 index 278092b8bca..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/protoc/protoc_test.go +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package protoc - -import ( - "path/filepath" - "strings" - "testing" - - "github.com/google/go-cmp/cmp" -) - -// mockConfigProvider is a mock implementation of the ConfigProvider interface for testing. -type mockConfigProvider struct { - serviceYAML string - gapicYAML string - grpcServiceConfig string - transport string - restNumericEnums bool - hasGAPIC bool -} - -func (m *mockConfigProvider) ServiceYAML() string { return m.serviceYAML } -func (m *mockConfigProvider) GapicYAML() string { return m.gapicYAML } -func (m *mockConfigProvider) GRPCServiceConfig() string { return m.grpcServiceConfig } -func (m *mockConfigProvider) Transport() string { return m.transport } -func (m *mockConfigProvider) HasRESTNumericEnums() bool { return m.restNumericEnums } -func (m *mockConfigProvider) HasGAPIC() bool { return m.hasGAPIC } - -func TestBuild(t *testing.T) { - // The testdata directory is a curated version of a valid protoc - // import path that contains all the necessary proto definitions. - sourceDir, err := filepath.Abs("../testdata/generate/source") - if err != nil { - t.Fatalf("failed to get absolute path for sourceDir: %v", err) - } - tests := []struct { - name string - apiPath string - config mockConfigProvider - want []string - }{ - { - name: "java_grpc_library rule", - apiPath: "google/cloud/workflows/v1", - config: mockConfigProvider{ - transport: "grpc", - grpcServiceConfig: "workflows_grpc_service_config.json", - gapicYAML: "workflows_gapic.yaml", - serviceYAML: "workflows_v1.yaml", - restNumericEnums: true, - hasGAPIC: true, - }, - want: []string{ - "protoc", - "--experimental_allow_proto3_optional", - "--java_out=/output/proto", - "--java_grpc_out=/output/grpc", - "--java_gapic_out=metadata:/output/gapic", - "--java_gapic_opt=" + strings.Join([]string{ - "api-service-config=" + filepath.Join(sourceDir, "google/cloud/workflows/v1/workflows_v1.yaml"), - "gapic-config=" + filepath.Join(sourceDir, "google/cloud/workflows/v1/workflows_gapic.yaml"), - "grpc-service-config=" + filepath.Join(sourceDir, "google/cloud/workflows/v1/workflows_grpc_service_config.json"), - "transport=grpc", - "rest-numeric-enums", - }, ","), - "-I=" + sourceDir, - filepath.Join(sourceDir, "google/cloud/workflows/v1/workflows.proto"), - filepath.Join(sourceDir, "google/cloud/common_resources.proto"), - }, - }, - { - name: "java_proto_library rule with legacy gRPC", - apiPath: "google/cloud/secretmanager/v1beta2", - config: mockConfigProvider{ - transport: "grpc", - grpcServiceConfig: "secretmanager_grpc_service_config.json", - serviceYAML: "secretmanager_v1beta2.yaml", - restNumericEnums: true, - hasGAPIC: true, - }, - want: []string{ - "protoc", - "--experimental_allow_proto3_optional", - "--java_out=/output/proto", - "--java_grpc_out=/output/grpc", - "--java_gapic_out=metadata:/output/gapic", - "--java_gapic_opt=" + strings.Join([]string{ - "api-service-config=" + filepath.Join(sourceDir, "google/cloud/secretmanager/v1beta2/secretmanager_v1beta2.yaml"), - "grpc-service-config=" + filepath.Join(sourceDir, "google/cloud/secretmanager/v1beta2/secretmanager_grpc_service_config.json"), - "transport=grpc", - "rest-numeric-enums", - }, ","), - "-I=" + sourceDir, - filepath.Join(sourceDir, "google/cloud/secretmanager/v1beta2/secretmanager.proto"), - filepath.Join(sourceDir, "google/cloud/common_resources.proto"), - }, - }, { - // Note: we don't have a separate test directory with a proto-only library; - // the config is used to say "don't generate GAPIC". - name: "proto-only", - apiPath: "google/cloud/secretmanager/v1beta2", - config: mockConfigProvider{ - hasGAPIC: false, - }, - want: []string{ - "protoc", - "--experimental_allow_proto3_optional", - "--java_out=/output/proto", - "-I=" + sourceDir, - filepath.Join(sourceDir, "google/cloud/secretmanager/v1beta2/secretmanager.proto"), - filepath.Join(sourceDir, "google/cloud/common_resources.proto"), - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - outputConfig := &OutputConfig{ - GAPICDir: "/output/gapic", - GRPCDir: "/output/grpc", - ProtoDir: "/output/proto", - } - got, err := Build(filepath.Join(sourceDir, test.apiPath), &test.config, sourceDir, outputConfig) - if err != nil { - t.Fatalf("Build() failed: %v", err) - } - - if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("Build() mismatch (-want +got):\n%s", diff) - } - }) - } -} diff --git a/internal/legacylibrarian/legacycontainer/java/release/release.go b/internal/legacylibrarian/legacycontainer/java/release/release.go deleted file mode 100644 index e4e1a0db7af..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/release/release.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package release contains the implementation of the release-stage command. -package release - -import ( - "context" - "log/slog" - "path/filepath" - - "github.com/googleapis/librarian/internal/legacylibrarian/legacycontainer/java/languagecontainer/release" - "github.com/googleapis/librarian/internal/legacylibrarian/legacycontainer/java/message" - "github.com/googleapis/librarian/internal/legacylibrarian/legacycontainer/java/pom" -) - -// Stage executes the release stage command. -func Stage(ctx context.Context, cfg *release.Config) (*message.ReleaseStageResponse, error) { - slog.Info("release-stage: invoked", "config", cfg) - response := &message.ReleaseStageResponse{} - for _, lib := range cfg.Request.Libraries { - for _, path := range lib.SourcePaths { - slog.Info("release-stage: processing library", "libraryID", lib.ID, "version", lib.Version, "sourcePath", path) - if err := pom.UpdateVersions( - cfg.Context.RepoDir, - filepath.Join(cfg.Context.RepoDir, path), - cfg.Context.OutputDir, lib.ID, lib.Version); err != nil { - response.Error = err.Error() - return response, err - } - } - } - return response, nil -} diff --git a/internal/legacylibrarian/legacycontainer/java/release/release_test.go b/internal/legacylibrarian/legacycontainer/java/release/release_test.go deleted file mode 100644 index db657dcbd49..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/release/release_test.go +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package release - -import ( - "context" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/googleapis/librarian/internal/legacylibrarian/legacycontainer/java/languagecontainer/release" - "github.com/googleapis/librarian/internal/legacylibrarian/legacycontainer/java/message" -) - -func TestStage(t *testing.T) { - tests := []struct { - name string - libraryID string - SourcePaths []string - version string - expected string - }{ - { - name: "happy path", - libraryID: "google-cloud-foo", - SourcePaths: []string{ - "java-foo", - }, - version: "2.0.0", - expected: "2.0.0-SNAPSHOT", - }, - { - name: "Source Paths not matching the folder", - libraryID: "google-cloud-java", - SourcePaths: []string{ - "java-nonexistent", - }, - version: "2.0.0", - // Do not expect the files updated since the source path does not exist. - expected: "", - }, - } - - for _, test := range tests { - test := test - t.Run(test.name, func(t *testing.T) { - // This testdata is the dummy repository root. - inputPath := filepath.Join("testdata") - - tmpDir := t.TempDir() - outputDir := filepath.Join(tmpDir, "output") - if err := os.MkdirAll(outputDir, 0755); err != nil { - t.Fatalf("failed to create output directory: %v", err) - } - cfg := &release.Config{ - Context: &release.Context{ - RepoDir: inputPath, - OutputDir: outputDir, - }, - Request: &message.ReleaseStageRequest{ - Libraries: []*message.Library{ - { - ID: test.libraryID, - Version: test.version, - SourcePaths: test.SourcePaths, - }, - }, - }, - } - - response, err := Stage(context.Background(), cfg) - if err != nil { - t.Fatalf("Stage() got unexpected error: %v", err) - } - - if response.Error != "" { - t.Errorf("expected success, got error: %s", response.Error) - } - if test.expected != "" { - // The file paths are relative to the repoDir. - for _, file := range []string{"java-foo/pom.xml", "java-foo/google-cloud-foo/pom.xml"} { - content, err := os.ReadFile(filepath.Join(outputDir, file)) - if err != nil { - t.Fatalf("failed to read output file: %v", err) - } - - hasExpectedVersionLineWithAnnotation := strings.Contains(string(content), test.expected) - if !hasExpectedVersionLineWithAnnotation { - t.Errorf("expected file to contain annotation %q and comment, got %q", test.expected, string(content)) - } - } - } else { - // Expect no files in the output directory because this operation - // does not change any files in the repodir. - entries, err := os.ReadDir(outputDir) - if err != nil { - t.Fatalf("failed to read output directory: %v", err) - } - if len(entries) != 0 { - t.Errorf("expected no files in output directory, got %d files", len(entries)) - } - } - }) - } -} diff --git a/internal/legacylibrarian/legacycontainer/java/release/testdata/java-foo/google-cloud-foo/pom.xml b/internal/legacylibrarian/legacycontainer/java/release/testdata/java-foo/google-cloud-foo/pom.xml deleted file mode 100644 index fed71948872..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/release/testdata/java-foo/google-cloud-foo/pom.xml +++ /dev/null @@ -1,7 +0,0 @@ - - 4.0.0 - - google-cloud-foo - 1.0.0-SNAPSHOT - jar - \ No newline at end of file diff --git a/internal/legacylibrarian/legacycontainer/java/release/testdata/java-foo/pom.xml b/internal/legacylibrarian/legacycontainer/java/release/testdata/java-foo/pom.xml deleted file mode 100644 index 3739fc0ec98..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/release/testdata/java-foo/pom.xml +++ /dev/null @@ -1,7 +0,0 @@ - - 4.0.0 - - google-cloud-foo-parent - 1.0.0-SNAPSHOT - pom - \ No newline at end of file diff --git a/internal/legacylibrarian/legacycontainer/java/run-generate-library.sh b/internal/legacylibrarian/legacycontainer/java/run-generate-library.sh deleted file mode 100755 index 63667274c67..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/run-generate-library.sh +++ /dev/null @@ -1,126 +0,0 @@ -#!/bin/bash -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script is a development tool for the librariangen command. It -# orchestrates the end-to-end process of generating a Java client library from a -# test configuration. -# -# Key steps include: -# 1. Setting up an isolated workspace directory. -# 2. Downloading the required version of the gapic-generator-java artifact. -# 3. Compiling the librariangen Go binary. -# 4. Executing the `librariangen generate` command with pre-defined test -# inputs and capturing its output and logs. -# -# This allows for quick, reproducible test runs of the entire generation -# process. - -set -e # Exit immediately if a command exits with a non-zero status. - -cd "$(dirname "$0")" # Change to the script's directory to make relative paths work. - -# --- Configuration --- -WORKSPACE="$(pwd)/workspace" -LIBRARIANGEN_GOOGLEAPIS_DIR=../../../../googleapis -LIBRARIANGEN_GOTOOLCHAIN=local -GAPIC_GENERATOR_VERSION="2.62.3" -GRPC_PLUGIN_VERSION="1.65.0" -LIBRARIANGEN_LOG="$WORKSPACE/librariangen.log" - -# --- Cleanup and Setup --- -echo "Cleaning up from last time..." -rm -rf "$WORKSPACE" -mkdir -p "$WORKSPACE" -echo "Working directory: $WORKSPACE" - -# --- Dependency Checks & Version Info --- -( - echo "--- Tool Versions ---" - echo "Go: $(GOWORK=off go version)" - echo "protoc: $(protoc --version 2>&1)" - echo "---------------------\n" -) >> "$LIBRARIANGEN_LOG" 2>&1 - -# Ensure that all required protoc dependencies are available in PATH. -if ! command -v "protoc" &> /dev/null; then - echo "Error: protoc not found in PATH. Please install it." - exit 1 -fi - -# Ensure that mvn is available in PATH. -if ! command -v "mvn" &> /dev/null; then - echo "Error: mvn not found in PATH. Please install it." - exit 1 -fi - -# --- Download and Prepare Tools --- -echo "Downloading gapic-generator-java version $GAPIC_GENERATOR_VERSION..." -wget -q "https://repo1.maven.org/maven2/com/google/api/gapic-generator-java/$GAPIC_GENERATOR_VERSION/gapic-generator-java-$GAPIC_GENERATOR_VERSION.jar" -O "$WORKSPACE/gapic-generator-java.jar" - -echo "Downloading protoc-gen-grpc-java..." -wget -q "https://repo1.maven.org/maven2/io/grpc/protoc-gen-grpc-java/$GRPC_PLUGIN_VERSION/protoc-gen-grpc-java-$GRPC_PLUGIN_VERSION-linux-x86_64.exe" -O "$WORKSPACE/protoc-gen-java_grpc" -chmod +x "$WORKSPACE/protoc-gen-java_grpc" - -# Create wrapper script for protoc-gen-java_gapic -echo "Creating protoc-gen-java_gapic wrapper..." -cat > "$WORKSPACE/protoc-gen-java_gapic" << EOL -#!/bin/bash -set -e -exec java -cp "$WORKSPACE/gapic-generator-java.jar" com.google.api.generator.Main \$@ -EOL -chmod +x "$WORKSPACE/protoc-gen-java_gapic" - -# --- Prepare Inputs --- -LIBRARIAN_DIR="$WORKSPACE/librarian" -OUTPUT_DIR="$WORKSPACE/output" -mkdir -p "$LIBRARIAN_DIR" "$OUTPUT_DIR" - -# Use an external googleapis checkout. -if [ ! -d "$LIBRARIANGEN_GOOGLEAPIS_DIR" ]; then - echo "Error: LIBRARIANGEN_GOOGLEAPIS_DIR is not set or not a directory." - echo "Please set it to the path of your local googleapis clone." - exit 1 -fi -echo "Using googleapis source from $LIBRARIANGEN_GOOGLEAPIS_DIR" -SOURCE_DIR=$(cd "$LIBRARIANGEN_GOOGLEAPIS_DIR" && pwd) - -# Copy the generate-request.json into the librarian directory. -cp "testdata/generate/librarian/secret-manager-generate-request.json" "$LIBRARIAN_DIR/generate-request.json" - -# --- Execute --- -# Compile the librariangen binary. -BINARY_PATH="$WORKSPACE/librariangen" -echo "Compiling librariangen..." -GOWORK=off go build -o "$BINARY_PATH" . - -# Run the librariangen generate command. -echo "Running librariangen..." -PATH="$WORKSPACE:$(GOWORK=off go env GOPATH)/bin:$HOME/go/bin:$PATH" \ -"$BINARY_PATH" generate \ - --source="$SOURCE_DIR" \ - --librarian="$LIBRARIAN_DIR" \ - --output="$OUTPUT_DIR" \ - >> "$LIBRARIANGEN_LOG" 2>&1 - -echo "Library generation complete." -echo "Generated files are available in: $OUTPUT_DIR" -echo "Librariangen logs are available in: $LIBRARIANGEN_LOG" - -# --- Build Generated Library --- -echo "Building generated library with Maven..." -(cd "$OUTPUT_DIR" && mvn clean install -Dcheckstyle.skip) >> "$LIBRARIANGEN_LOG" 2>&1 - -echo "Maven build complete." -echo "Build logs are available in: $LIBRARIANGEN_LOG" \ No newline at end of file diff --git a/internal/legacylibrarian/legacycontainer/java/testdata/generate/librarian/generate-request.json b/internal/legacylibrarian/legacycontainer/java/testdata/generate/librarian/generate-request.json deleted file mode 100644 index 51d386e96ee..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/testdata/generate/librarian/generate-request.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "id": "chronicle", - "version": "0.1.1", - "apis": [ - { - "path": "google/cloud/chronicle/v1", - "service_config": "chronicle_v1.yaml" - } - ], - "source_roots": [ - "chronicle", - "internal/generated/snippets/chronicle" - ], - "preserve_regex": [ - "chronicle/aliasshim/aliasshim.go", - "chronicle/CHANGES.md" - ], - "remove_regex": [ - "chronicle", - "internal/generated/snippets/chronicle" - ] -} \ No newline at end of file diff --git a/internal/legacylibrarian/legacycontainer/java/testdata/generate/librarian/secret-manager-generate-request.json b/internal/legacylibrarian/legacycontainer/java/testdata/generate/librarian/secret-manager-generate-request.json deleted file mode 100644 index 10842035312..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/testdata/generate/librarian/secret-manager-generate-request.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "id": "secretmanager", - "version": "0.1.1", - "apis": [ - { - "path": "google/cloud/secretmanager/v1", - "service_config": "secretmanager_v1.yaml" - }, - { - "path": "google/cloud/location", - "service_config": "locations.yaml" - }, - { - "path": "google/cloud/secretmanager/v1beta2", - "service_config": "secretmanager_v1beta2.yaml" - }, - { - "path": "google/cloud/secrets/v1beta1", - "service_config": "secretmanager_v1beta1.yaml" - } - ], - "source_roots": [ - "java-secretmanager" - ], - "preserve_regex": [ - "CHANGELOG.md" - ] -} \ No newline at end of file diff --git a/internal/legacylibrarian/legacycontainer/java/testdata/generate/source/google/api/annotations.proto b/internal/legacylibrarian/legacycontainer/java/testdata/generate/source/google/api/annotations.proto deleted file mode 100644 index c749d43d6ff..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/testdata/generate/source/google/api/annotations.proto +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.api; - -import "google/protobuf/descriptor.proto"; - -option java_multiple_files = true; -option java_outer_classname = "AnnotationsProto"; -option java_package = "com.google.api"; - -extend google.protobuf.MethodOptions { - HttpRule http = 72295728; -} - -message HttpRule { - string selector = 1; - oneof pattern { - string get = 2; - string put = 3; - string post = 4; - string delete = 5; - string patch = 6; - CustomHttpPattern custom = 8; - } - string body = 7; - repeated HttpRule additional_bindings = 11; -} - -message CustomHttpPattern { - string kind = 1; - string path = 2; -} diff --git a/internal/legacylibrarian/legacycontainer/java/testdata/generate/source/google/cloud/secretmanager/v1beta2/BUILD.bazel b/internal/legacylibrarian/legacycontainer/java/testdata/generate/source/google/cloud/secretmanager/v1beta2/BUILD.bazel deleted file mode 100644 index 7ec7192b1cc..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/testdata/generate/source/google/cloud/secretmanager/v1beta2/BUILD.bazel +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -java_proto_library( - name = "secretmanager_java_proto", - deps = [":secretmanager_proto"], -) - -java_grpc_library( - name = "secretmanager_java_grpc", - srcs = [":secretmanager_proto"], - deps = [":secretmanager_java_proto"], -) - -java_gapic_library( - name = "secretmanager_java_gapic", - srcs = [":secretmanager_proto_with_info"], - gapic_yaml = None, - grpc_service_config = "secretmanager_grpc_service_config.json", - rest_numeric_enums = True, - service_yaml = "secretmanager_v1beta2.yaml", - test_deps = [ - "//google/cloud/location:location_java_grpc", - "//google/iam/v1:iam_java_grpc", - ":secretmanager_java_grpc", - ], - transport = "grpc+rest", - deps = [ - ":secretmanager_java_proto", - "//google/api:api_java_proto", - "//google/cloud/location:location_java_proto", - "//google/iam/v1:iam_java_proto", - ], -) - -java_gapic_test( - name = "secretmanager_java_gapic_test_suite", - test_classes = [ - "com.google.cloud.secretmanager.v1beta2.SecretManagerServiceClientHttpJsonTest", - "com.google.cloud.secretmanager.v1beta2.SecretManagerServiceClientTest", - ], - runtime_deps = [":secretmanager_java_gapic_test"], -) - -# Open Source Packages -java_gapic_assembly_gradle_pkg( - name = "google-cloud-secretmanager-v1beta2-java", - transport = "grpc+rest", - deps = [ - ":secretmanager_java_gapic", - ":secretmanager_java_grpc", - ":secretmanager_java_proto", - ":secretmanager_proto", - ], - include_samples = True, -) \ No newline at end of file diff --git a/internal/legacylibrarian/legacycontainer/java/testdata/generate/source/google/cloud/secretmanager/v1beta2/secretmanager.proto b/internal/legacylibrarian/legacycontainer/java/testdata/generate/source/google/cloud/secretmanager/v1beta2/secretmanager.proto deleted file mode 100644 index a3d1df63be6..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/testdata/generate/source/google/cloud/secretmanager/v1beta2/secretmanager.proto +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.cloud.secretmanager.v1beta2; - -import "google/api/annotations.proto"; - -option java_multiple_files = true; -option java_outer_classname = "ServiceProto"; -option java_package = "com.google.cloud.secretmanager.v1beta2"; - -// A Secret is a secret value. -message Secret { - string name = 1; -} - -// Request for the `ListSecrets` method. -message ListSecretsRequest { - string parent = 1; -} - -// Response for the `ListSecrets` method. -message ListSecretsResponse { - repeated Secret Secrets = 1; -} - -// Service for managing secrets. -service Secrets { - // Lists Secrets in a given project and location. - rpc ListSecrets(ListSecretsRequest) returns (ListSecretsResponse) { - option (google.api.http) = { - get: "/v1/{parent=projects/*/locations/*}/Secrets" - }; - } -} diff --git a/internal/legacylibrarian/legacycontainer/java/testdata/generate/source/google/cloud/secretmanager/v1beta2/secretmanager_v1beta2.yaml b/internal/legacylibrarian/legacycontainer/java/testdata/generate/source/google/cloud/secretmanager/v1beta2/secretmanager_v1beta2.yaml deleted file mode 100644 index cf28439da01..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/testdata/generate/source/google/cloud/secretmanager/v1beta2/secretmanager_v1beta2.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -type: google.api.Service -config_version: 3 -name: secretmanager.googleapis.com -title: Secret Manager API -apis: - - name: google.cloud.secretmanager.v1beta2.Secrets -documentation: - summary: |- - Subset of real Secret Manager API. diff --git a/internal/legacylibrarian/legacycontainer/java/testdata/generate/source/google/cloud/workflows/v1/BUILD.bazel b/internal/legacylibrarian/legacycontainer/java/testdata/generate/source/google/cloud/workflows/v1/BUILD.bazel deleted file mode 100644 index 59e33ec9db2..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/testdata/generate/source/google/cloud/workflows/v1/BUILD.bazel +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -java_proto_library( - name = "workflows_java_proto", - deps = [":workflows_proto"], -) - -java_grpc_library( - name = "workflows_java_grpc", - srcs = [":workflows_proto"], - deps = [":workflows_java_proto"], -) - -java_gapic_library( - name = "workflows_java_gapic", - srcs = [":workflows_proto_with_info"], - grpc_service_config = "workflows_grpc_service_config.json", - rest_numeric_enums = True, - service_yaml = "workflows_v1.yaml", - test_deps = [ - "//google/cloud/location:location_java_grpc", - ":workflows_java_grpc", - ], - transport = "grpc+rest", - deps = [ - ":workflows_java_proto", - "//google/api:api_java_proto", - "//google/cloud/location:location_java_proto", - ], -) - -java_gapic_test( - name = "workflows_java_gapic_test_suite", - test_classes = [ - "com.google.cloud.workflows.v1.WorkflowsClientHttpJsonTest", - "com.google.cloud.workflows.v1.WorkflowsClientTest", - ], - runtime_deps = [":workflows_java_gapic_test"], -) - -# Open Source Packages -java_gapic_assembly_gradle_pkg( - name = "google-cloud-workflows-v1-java", - include_samples = True, - transport = "grpc+rest", - deps = [ - ":workflows_java_gapic", - ":workflows_java_grpc", - ":workflows_java_proto", - ":workflows_proto", - ], -) \ No newline at end of file diff --git a/internal/legacylibrarian/legacycontainer/java/testdata/generate/source/google/cloud/workflows/v1/workflows.proto b/internal/legacylibrarian/legacycontainer/java/testdata/generate/source/google/cloud/workflows/v1/workflows.proto deleted file mode 100644 index a65843a1546..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/testdata/generate/source/google/cloud/workflows/v1/workflows.proto +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.cloud.workflows.v1; - -import "google/api/annotations.proto"; - -option java_multiple_files = true; -option java_outer_classname = "WorkflowsProto"; -option java_package = "com.google.cloud.workflows.v1"; - -// A workflow is a collection of steps that are executed in a predefined order. -message Workflow { - string name = 1; -} - -// Request for the `ListWorkflows` method. -message ListWorkflowsRequest { - string parent = 1; -} - -// Response for the `ListWorkflows` method. -message ListWorkflowsResponse { - repeated Workflow workflows = 1; -} - -// Service for managing workflows. -service Workflows { - // Lists workflows in a given project and location. - rpc ListWorkflows(ListWorkflowsRequest) returns (ListWorkflowsResponse) { - option (google.api.http) = { - get: "/v1/{parent=projects/*/locations/*}/workflows" - }; - } -} diff --git a/internal/legacylibrarian/legacycontainer/java/testdata/generate/source/google/cloud/workflows/v1/workflows_v1.yaml b/internal/legacylibrarian/legacycontainer/java/testdata/generate/source/google/cloud/workflows/v1/workflows_v1.yaml deleted file mode 100644 index f1731abd706..00000000000 --- a/internal/legacylibrarian/legacycontainer/java/testdata/generate/source/google/cloud/workflows/v1/workflows_v1.yaml +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -type: google.api.Service -config_version: 3 -name: workflows.googleapis.com -title: Workflows API -apis: - - name: google.cloud.location.Locations - - name: google.cloud.workflows.v1.Workflows - - name: google.longrunning.Operations -types: - - name: google.cloud.workflows.v1.OperationMetadata -documentation: - summary: |- - Manage workflow definitions. To execute workflows and manage executions, - see the Workflows Executions API. - rules: - - selector: google.cloud.location.Locations.GetLocation - description: Gets information about a location. - - selector: google.cloud.location.Locations.ListLocations - description: Lists information about the supported locations for this service. -http: - rules: - - selector: google.cloud.location.Locations.GetLocation - get: '/v1/{name=projects/*/locations/*}' - - selector: google.cloud.location.Locations.ListLocations - get: '/v1/{name=projects/*}/locations' - - selector: google.longrunning.Operations.DeleteOperation - delete: '/v1/{name=projects/*/locations/*/operations/*}' - - selector: google.longrunning.Operations.GetOperation - get: '/v1/{name=projects/*/locations/*/operations/*}' - - selector: google.longrunning.Operations.ListOperations - get: '/v1/{name=projects/*/locations/*}/operations' -authentication: - rules: - - selector: google.cloud.location.Locations.GetLocation - oauth: - canonical_scopes: |- - https://www.googleapis.com/auth/cloud-platform - - selector: google.cloud.location.Locations.ListLocations - oauth: - canonical_scopes: |- - https://www.googleapis.com/auth/cloud-platform - - selector: 'google.cloud.workflows.v1.Workflows.*' - oauth: - canonical_scopes: |- - https://www.googleapis.com/auth/cloud-platform - - selector: 'google.longrunning.Operations.*' - oauth: - canonical_scopes: |- - https://www.googleapis.com/auth/cloud-platform diff --git a/internal/legacylibrarian/legacydocker/docker.go b/internal/legacylibrarian/legacydocker/docker.go deleted file mode 100644 index 6cb99fb05e8..00000000000 --- a/internal/legacylibrarian/legacydocker/docker.go +++ /dev/null @@ -1,478 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package legacydocker provides the interface for running language-specific -// Docker containers which conform to the Librarian container contract. -// TODO(https://github.com/googleapis/librarian/issues/330): link to -// the documentation when it's written. -package legacydocker - -import ( - "context" - "encoding/json" - "fmt" - "log/slog" - "os" - "os/exec" - "path/filepath" - "strings" - - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" -) - -// Command is the string representation of a command to be passed to the language-specific -// container's entry point as the first argument. -type Command string - -// The set of commands passed to the language container, in a single place to avoid typos. -const ( - // CommandBuild builds a library. - CommandBuild Command = "build" - // CommandConfigure configures a new API as a library. - CommandConfigure Command = "configure" - // CommandGenerate performs generation for a configured library. - CommandGenerate Command = "generate" - // CommandReleaseStage performs release for a library. - CommandReleaseStage Command = "release-stage" -) - -// Docker contains all the information required to run language-specific -// Docker containers. -type Docker struct { - // The Docker image to run. - Image string - - // The user ID to run the container as. - uid string - - // The group ID to run the container as. - gid string - - // HostMount specifies a mount point from the Docker host into the Docker - // container. The format is "{host-dir}:{local-dir}". - HostMount string - - // run runs the docker command. - run func(args ...string) error -} - -// BuildRequest contains all the information required for a language -// container to run the build command. -type BuildRequest struct { - // LibraryID specifies the ID of the library to build. - LibraryID string - - // RepoDir is the local root directory of the language repository. - RepoDir string - - // State is a pointer to the [legacyconfig.LibrarianState] struct, representing - // the overall state of the generation and release pipeline. - State *legacyconfig.LibrarianState - - // Image is the name of the docker image to use when running. If not - // specified, uses the default image configured for the client. - Image string -} - -// ConfigureRequest contains all the information required for a language -// container to run the configure command. -type ConfigureRequest struct { - // ApiRoot specifies the root directory of the API specification repo. - ApiRoot string - - // libraryID specifies the ID of the library to configure. - LibraryID string - - // Output specifies the empty output directory into which the command should - // generate code - Output string - - // RepoDir is the local root directory of the language repository. - RepoDir string - - // ExistingSourceRoots are existing source roots in the language repository. - ExistingSourceRoots []string - - // GlobalFiles are global files of the language repository. - GlobalFiles []string - - // State is a pointer to the [legacyconfig.LibrarianState] struct, representing - // the overall state of the generation and release pipeline. - State *legacyconfig.LibrarianState - - // Image is the name of the docker image to use when running. If not - // specified, uses the default image configured for the client. - Image string -} - -// GenerateRequest contains all the information required for a language -// container to run the generate command. -type GenerateRequest struct { - // ApiRoot specifies the root directory of the API specification repo. - ApiRoot string - - // LibraryID specifies the ID of the library to generate. - LibraryID string - - // Output specifies the empty output directory into which the command should - // generate code - Output string - - // RepoDir is the local root directory of the language repository. - RepoDir string - - // State is a pointer to the [legacyconfig.LibrarianState] struct, representing - // the overall state of the generation and release pipeline. - State *legacyconfig.LibrarianState - - // Image is the name of the docker image to use when running. If not - // specified, uses the default image configured for the client. - Image string -} - -// ReleaseStageRequest contains all the information required for a language -// container to run the stage command. -type ReleaseStageRequest struct { - // Branch is the remote branch of the language repository to use. - Branch string - - // Commit determines whether to create a commit for the release but not - // create a pull request. This flag is ignored if Push is set to true. - Commit bool - - // LibrarianConfig is a pointer to the [legacyconfig.LibrarianConfig] struct, holding - // global files configuration in a language repository. - LibrarianConfig *legacyconfig.LibrarianConfig - - // LibraryID specifies the ID of the library to release. - LibraryID string - - // LibraryVersion specifies the version of the library to release. - LibraryVersion string - - // Output specifies the empty output directory into which the command should - // generate code. - Output string - - // RepoDir is the local root directory of language repository contains - // files that make up libraries and global files. - // This is the directory that container can access. - RepoDir string - - // Push determines whether to push changes to GitHub. - Push bool - - // State is a pointer to the [legacyconfig.LibrarianState] struct, representing - // the overall state of the generation and release pipeline. - State *legacyconfig.LibrarianState - - // Image is the name of the docker image to use when running. If not - // specified, uses the default image configured for the client. - Image string -} - -// DockerOptions contains optional configuration parameters for invoking -// docker commands. -type DockerOptions struct { - // UserUID is the user ID of the current user. It is used to run Docker - // containers with the same user, so that created files have the correct - // ownership. - UserUID string - // UserGID is the group ID of the current user. It is used to run Docker - // containers with the same user, so that created files have the correct - // ownership. - UserGID string - // HostMount is used to remap Docker mount paths when running in environments - // where Docker containers are siblings (e.g., Kokoro). - // It specifies a mount point from the Docker host into the Docker container. - // The format is "{host-dir}:{local-dir}". - HostMount string -} - -// New constructs a Docker instance which will invoke the specified -// Docker image as required to implement language-specific commands, -// providing the container with required environment variables. -func New(workRoot, image string, options *DockerOptions) (*Docker, error) { - docker := &Docker{ - Image: image, - uid: options.UserUID, - gid: options.UserGID, - HostMount: options.HostMount, - } - docker.run = func(args ...string) error { - return docker.runCommand("docker", args...) - } - return docker, nil -} - -// Generate performs generation for an API which is configured as part of a -// library. -func (c *Docker) Generate(ctx context.Context, request *GenerateRequest) error { - reqFilePath := filepath.Join(request.RepoDir, legacyconfig.LibrarianDir, legacyconfig.GenerateRequest) - if err := writeLibraryState(request.State, request.LibraryID, reqFilePath); err != nil { - return err - } - defer func() { - if b, err := os.ReadFile(reqFilePath); err == nil { - slog.Debug("generate request", "content", string(b)) - } - err := os.Remove(reqFilePath) - if err != nil { - slog.Warn("fail to remove file", slog.String("name", reqFilePath), slog.Any("err", err)) - } - }() - - commandArgs := []string{ - "--librarian=/librarian", - "--input=/input", - "--output=/output", - "--source=/source", - } - - generatorInput := filepath.Join(request.RepoDir, legacyconfig.GeneratorInputDir) - librarianDir := filepath.Join(request.RepoDir, legacyconfig.LibrarianDir) - mounts := []string{ - fmt.Sprintf("%s:/librarian", librarianDir), - fmt.Sprintf("%s:/input", generatorInput), - fmt.Sprintf("%s:/output", request.Output), - fmt.Sprintf("%s:/source:ro", request.ApiRoot), // readonly volume - } - - image := c.resolveImage(request.Image) - return c.runDocker(ctx, image, CommandGenerate, mounts, commandArgs) -} - -// Build builds the library with an ID of libraryID, as configured in -// the Librarian state file for the repository with a root of repoRoot. -func (c *Docker) Build(ctx context.Context, request *BuildRequest) error { - reqFilePath := filepath.Join(request.RepoDir, legacyconfig.LibrarianDir, legacyconfig.BuildRequest) - if err := writeLibraryState(request.State, request.LibraryID, reqFilePath); err != nil { - return err - } - defer func() { - if b, err := os.ReadFile(reqFilePath); err == nil { - slog.Debug("build request", "content", string(b)) - } - err := os.Remove(reqFilePath) - if err != nil { - slog.Warn("fail to remove file", slog.String("name", reqFilePath), slog.Any("err", err)) - } - }() - - librarianDir := filepath.Join(request.RepoDir, legacyconfig.LibrarianDir) - mounts := []string{ - fmt.Sprintf("%s:/librarian", librarianDir), - fmt.Sprintf("%s:/repo", request.RepoDir), - } - commandArgs := []string{ - "--librarian=/librarian", - "--repo=/repo", - } - - image := c.resolveImage(request.Image) - return c.runDocker(ctx, image, CommandBuild, mounts, commandArgs) -} - -// Configure configures an API within a repository, either adding it to an -// existing library or creating a new library. -// -// Returns the configured library id if the command succeeds. -func (c *Docker) Configure(ctx context.Context, request *ConfigureRequest) (string, error) { - reqFilePath := filepath.Join(request.RepoDir, legacyconfig.LibrarianDir, legacyconfig.ConfigureRequest) - if err := writeLibrarianState(request.State, reqFilePath); err != nil { - return "", err - } - defer func() { - if b, err := os.ReadFile(reqFilePath); err == nil { - slog.Debug("configure request", "content", string(b)) - } - err := os.Remove(reqFilePath) - if err != nil { - slog.Warn("fail to remove file", slog.String("name", reqFilePath), slog.Any("err", err)) - } - }() - commandArgs := []string{ - "--librarian=/librarian", - "--input=/input", - "--output=/output", - "--repo=/repo", - "--source=/source", - } - generatorInput := filepath.Join(request.RepoDir, legacyconfig.GeneratorInputDir) - librarianDir := filepath.Join(request.RepoDir, legacyconfig.LibrarianDir) - mounts := []string{ - fmt.Sprintf("%s:/librarian", librarianDir), - fmt.Sprintf("%s:/input", generatorInput), - fmt.Sprintf("%s:/output", request.Output), - fmt.Sprintf("%s:/source:ro", request.ApiRoot), // readonly volume - } - // Mount existing source roots as a readonly volume. - for _, sourceRoot := range request.ExistingSourceRoots { - mounts = append(mounts, fmt.Sprintf("%s/%s:/repo/%s:ro", request.RepoDir, sourceRoot, sourceRoot)) - } - // Mount global files as a readonly volume. - for _, globalFile := range request.GlobalFiles { - mounts = append(mounts, fmt.Sprintf("%s/%s:/repo/%s:ro", request.RepoDir, globalFile, globalFile)) - } - - image := c.resolveImage(request.Image) - if err := c.runDocker(ctx, image, CommandConfigure, mounts, commandArgs); err != nil { - return "", err - } - - return request.LibraryID, nil -} - -// ReleaseStage stages a release for a given language repository. -func (c *Docker) ReleaseStage(ctx context.Context, request *ReleaseStageRequest) error { - reqFilePath := filepath.Join(request.RepoDir, legacyconfig.LibrarianDir, legacyconfig.ReleaseStageRequest) - if err := writeLibrarianState(request.State, reqFilePath); err != nil { - return err - } - defer func() { - if b, err := os.ReadFile(reqFilePath); err == nil { - slog.Debug("release stage request", "content", string(b)) - } - err := os.Remove(reqFilePath) - if err != nil { - slog.Warn("fail to remove file", slog.String("name", reqFilePath), slog.Any("err", err)) - } - }() - commandArgs := []string{ - "--librarian=/librarian", - "--repo=/repo", - "--output=/output", - } - - librarianDir := filepath.Join(request.RepoDir, legacyconfig.LibrarianDir) - mounts := []string{ - fmt.Sprintf("%s:/librarian", librarianDir), - fmt.Sprintf("%s:/repo:ro", request.RepoDir), // readonly volume - fmt.Sprintf("%s:/output", request.Output), - } - - image := c.resolveImage(request.Image) - if err := c.runDocker(ctx, image, CommandReleaseStage, mounts, commandArgs); err != nil { - return err - } - - return nil -} - -func (c *Docker) runDocker(_ context.Context, image string, command Command, mounts []string, commandArgs []string) (err error) { - mounts = maybeRelocateMounts(c.HostMount, mounts) - args := []string{ - "run", - "--rm", // Automatically delete the container after completion - } - for _, mount := range mounts { - args = append(args, "-v", mount) - } - - // Run as the current user in the container - primarily so that any files - // we create end up being owned by the current user (and easily deletable). - if c.uid != "" && c.gid != "" { - args = append(args, "--user", fmt.Sprintf("%s:%s", c.uid, c.gid)) - } - - args = append(args, image) - args = append(args, string(command)) - args = append(args, commandArgs...) - return c.run(args...) -} - -func maybeRelocateMounts(hostMount string, mounts []string) []string { - // When running in Kokoro, we'll be running sibling containers. - // Make sure we specify the "from" part of the mount as the host directory. - if hostMount == "" { - return mounts - } - - relocatedMounts := []string{} - hostMountParts := strings.Split(hostMount, ":") - for _, mount := range mounts { - if strings.HasPrefix(mount, hostMountParts[0]) { - mount = strings.Replace(mount, hostMountParts[0], hostMountParts[1], 1) - } - relocatedMounts = append(relocatedMounts, mount) - } - return relocatedMounts -} - -func (c *Docker) resolveImage(requestedImage string) string { - if requestedImage != "" { - return requestedImage - } - return c.Image -} - -func (c *Docker) runCommand(cmdName string, args ...string) error { - cmd := exec.Command(cmdName, args...) - cmd.Stderr = os.Stderr - cmd.Stdout = os.Stdout - slog.Info(fmt.Sprintf("=== Docker start %s", strings.Repeat("=", 63))) - slog.Info(cmd.String()) - slog.Info(strings.Repeat("-", 80)) - err := cmd.Run() - slog.Info(fmt.Sprintf("=== Docker end %s", strings.Repeat("=", 65))) - return err -} - -func writeLibraryState(state *legacyconfig.LibrarianState, libraryID, jsonFilePath string) error { - if err := os.MkdirAll(filepath.Dir(jsonFilePath), 0755); err != nil { - return fmt.Errorf("failed to make directory: %w", err) - } - jsonFile, err := os.Create(jsonFilePath) - if err != nil { - return fmt.Errorf("failed to create JSON file: %w", err) - } - defer jsonFile.Close() - - for _, library := range state.Libraries { - if library.ID != libraryID { - continue - } - - data, err := json.MarshalIndent(library, "", " ") - if err != nil { - return fmt.Errorf("failed to marshal state to JSON: %w", err) - } - _, err = jsonFile.Write(data) - if err != nil { - return fmt.Errorf("failed to write generate request JSON file: %w", err) - } - } - - return nil -} - -func writeLibrarianState(state *legacyconfig.LibrarianState, jsonFilePath string) error { - if err := os.MkdirAll(filepath.Dir(jsonFilePath), 0755); err != nil { - return fmt.Errorf("failed to make directory: %w", err) - } - jsonFile, err := os.Create(jsonFilePath) - if err != nil { - return fmt.Errorf("failed to create JSON file: %w", err) - } - defer jsonFile.Close() - - data, err := json.MarshalIndent(state, "", " ") - if err != nil { - return fmt.Errorf("failed to marshal state to JSON: %w", err) - } - - _, err = jsonFile.Write(data) - - return err -} diff --git a/internal/legacylibrarian/legacydocker/docker_test.go b/internal/legacylibrarian/legacydocker/docker_test.go deleted file mode 100644 index 1a98e8ed102..00000000000 --- a/internal/legacylibrarian/legacydocker/docker_test.go +++ /dev/null @@ -1,1159 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacydocker - -import ( - "context" - "errors" - "fmt" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/google/go-cmp/cmp" - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" -) - -func TestNew(t *testing.T) { - const ( - testWorkRoot = "testWorkRoot" - testImage = "testImage" - testUID = "1000" - testGID = "1001" - ) - d, err := New(testWorkRoot, testImage, &DockerOptions{ - UserUID: testUID, - UserGID: testGID, - }) - if err != nil { - t.Fatalf("New() error = %v", err) - } - if d.Image != testImage { - t.Errorf("d.Image = %q, want %q", d.Image, testImage) - } - if d.uid != testUID { - t.Errorf("d.uid = %q, want %q", d.uid, testUID) - } - if d.gid != testGID { - t.Errorf("d.gid = %q, want %q", d.gid, testGID) - } - if d.run == nil { - t.Error("d.run is nil") - } -} - -func TestDockerRun(t *testing.T) { - const ( - mockImage = "mockImage" - testAPIRoot = "testAPIRoot" - testImage = "testImage" - testLibraryID = "testLibraryID" - testOutput = "testOutput" - simulateDockerErrMsg = "simulate docker command failure for testing" - ) - - state := &legacyconfig.LibrarianState{} - repoDir := filepath.Join(os.TempDir()) - for _, test := range []struct { - name string - docker *Docker - runCommand func(ctx context.Context, d *Docker) error - want []string - wantErr bool - wantErrMsg string - }{ - { - name: "Generate", - docker: &Docker{ - Image: testImage, - }, - runCommand: func(ctx context.Context, d *Docker) error { - generateRequest := &GenerateRequest{ - State: state, - RepoDir: repoDir, - ApiRoot: testAPIRoot, - Output: testOutput, - LibraryID: testLibraryID, - } - - return d.Generate(ctx, generateRequest) - }, - want: []string{ - "run", "--rm", - "-v", fmt.Sprintf("%s/.librarian:/librarian", repoDir), - "-v", fmt.Sprintf("%s/.librarian/generator-input:/input", repoDir), - "-v", fmt.Sprintf("%s:/output", testOutput), - "-v", fmt.Sprintf("%s:/source:ro", testAPIRoot), - testImage, - string(CommandGenerate), - "--librarian=/librarian", - "--input=/input", - "--output=/output", - "--source=/source", - }, - }, - { - name: "Generate with invalid repo root", - docker: &Docker{ - Image: testImage, - }, - runCommand: func(ctx context.Context, d *Docker) error { - generateRequest := &GenerateRequest{ - State: state, - RepoDir: "/non-existed-dir", - ApiRoot: testAPIRoot, - Output: testOutput, - LibraryID: testLibraryID, - } - return d.Generate(ctx, generateRequest) - }, - want: []string{}, - wantErr: true, - wantErrMsg: "failed to make directory", - }, - { - name: "Generate with mock image", - docker: &Docker{ - Image: mockImage, - }, - runCommand: func(ctx context.Context, d *Docker) error { - generateRequest := &GenerateRequest{ - State: state, - RepoDir: repoDir, - ApiRoot: testAPIRoot, - Output: testOutput, - LibraryID: testLibraryID, - } - - return d.Generate(ctx, generateRequest) - }, - want: []string{}, - wantErr: true, - wantErrMsg: simulateDockerErrMsg, - }, - { - name: "Generate runs in docker with image override", - docker: &Docker{ - Image: testImage, - HostMount: "hostDir:localDir", - }, - runCommand: func(ctx context.Context, d *Docker) error { - generateRequest := &GenerateRequest{ - State: state, - RepoDir: repoDir, - ApiRoot: testAPIRoot, - Output: "hostDir", - LibraryID: testLibraryID, - Image: "custom-image:abcd123", - } - - return d.Generate(ctx, generateRequest) - }, - want: []string{ - "run", "--rm", - "-v", fmt.Sprintf("%s/.librarian:/librarian", repoDir), - "-v", fmt.Sprintf("%s/.librarian/generator-input:/input", repoDir), - "-v", "localDir:/output", - "-v", fmt.Sprintf("%s:/source:ro", testAPIRoot), - "custom-image:abcd123", - string(CommandGenerate), - "--librarian=/librarian", - "--input=/input", - "--output=/output", - "--source=/source", - }, - }, - { - name: "Generate runs in docker", - docker: &Docker{ - Image: testImage, - HostMount: "hostDir:localDir", - }, - runCommand: func(ctx context.Context, d *Docker) error { - generateRequest := &GenerateRequest{ - State: state, - RepoDir: repoDir, - ApiRoot: testAPIRoot, - Output: "hostDir", - LibraryID: testLibraryID, - } - - return d.Generate(ctx, generateRequest) - }, - want: []string{ - "run", "--rm", - "-v", fmt.Sprintf("%s/.librarian:/librarian", repoDir), - "-v", fmt.Sprintf("%s/.librarian/generator-input:/input", repoDir), - "-v", "localDir:/output", - "-v", fmt.Sprintf("%s:/source:ro", testAPIRoot), - testImage, - string(CommandGenerate), - "--librarian=/librarian", - "--input=/input", - "--output=/output", - "--source=/source", - }, - }, - { - name: "Build", - docker: &Docker{ - Image: testImage, - }, - runCommand: func(ctx context.Context, d *Docker) error { - buildRequest := &BuildRequest{ - State: state, - LibraryID: testLibraryID, - RepoDir: repoDir, - } - - return d.Build(ctx, buildRequest) - }, - want: []string{ - "run", "--rm", - "-v", fmt.Sprintf("%s/.librarian:/librarian", repoDir), - "-v", fmt.Sprintf("%s:/repo", repoDir), - testImage, - string(CommandBuild), - "--librarian=/librarian", - "--repo=/repo", - }, - }, - { - name: "Build runs in docker with image override", - docker: &Docker{ - Image: testImage, - HostMount: "hostDir:localDir", - }, - runCommand: func(ctx context.Context, d *Docker) error { - buildRequest := &BuildRequest{ - State: state, - LibraryID: testLibraryID, - RepoDir: repoDir, - Image: "custom-image:abcd123", - } - - return d.Build(ctx, buildRequest) - }, - want: []string{ - "run", "--rm", - "-v", fmt.Sprintf("%s/.librarian:/librarian", repoDir), - "-v", fmt.Sprintf("%s:/repo", repoDir), - "custom-image:abcd123", - string(CommandBuild), - "--librarian=/librarian", - "--repo=/repo", - }, - }, - { - name: "Build with invalid repo dir", - docker: &Docker{ - Image: testImage, - }, - runCommand: func(ctx context.Context, d *Docker) error { - buildRequest := &BuildRequest{ - State: state, - LibraryID: testLibraryID, - RepoDir: "/non-exist-dir", - } - return d.Build(ctx, buildRequest) - }, - want: []string{}, - wantErr: true, - wantErrMsg: "failed to make directory", - }, - { - name: "Build with mock image", - docker: &Docker{ - Image: mockImage, - }, - runCommand: func(ctx context.Context, d *Docker) error { - buildRequest := &BuildRequest{ - State: state, - LibraryID: testLibraryID, - RepoDir: repoDir, - } - - return d.Build(ctx, buildRequest) - }, - want: []string{}, - wantErr: true, - wantErrMsg: simulateDockerErrMsg, - }, - { - name: "Configure", - docker: &Docker{ - Image: testImage, - }, - runCommand: func(ctx context.Context, d *Docker) error { - configureRequest := &ConfigureRequest{ - State: state, - LibraryID: testLibraryID, - RepoDir: repoDir, - ApiRoot: testAPIRoot, - Output: testOutput, - GlobalFiles: []string{ - "a/b/go.mod", - "go.mod", - }, - } - - _, err := d.Configure(ctx, configureRequest) - - return err - }, - want: []string{ - "run", "--rm", - "-v", fmt.Sprintf("%s/.librarian:/librarian", repoDir), - "-v", fmt.Sprintf("%s/.librarian/generator-input:/input", repoDir), - "-v", fmt.Sprintf("%s:/output", testOutput), - "-v", fmt.Sprintf("%s:/source:ro", testAPIRoot), - "-v", fmt.Sprintf("%s/a/b/go.mod:/repo/a/b/go.mod:ro", repoDir), - "-v", fmt.Sprintf("%s/go.mod:/repo/go.mod:ro", repoDir), - testImage, - string(CommandConfigure), - "--librarian=/librarian", - "--input=/input", - "--output=/output", - "--repo=/repo", - "--source=/source", - }, - }, - { - name: "Configure runs in docker with image override", - docker: &Docker{ - Image: testImage, - }, - runCommand: func(ctx context.Context, d *Docker) error { - configureRequest := &ConfigureRequest{ - State: state, - LibraryID: testLibraryID, - RepoDir: repoDir, - ApiRoot: testAPIRoot, - Output: testOutput, - GlobalFiles: []string{ - "a/b/go.mod", - "go.mod", - }, - Image: "custom-image:abcd123", - } - - _, err := d.Configure(ctx, configureRequest) - - return err - }, - want: []string{ - "run", "--rm", - "-v", fmt.Sprintf("%s/.librarian:/librarian", repoDir), - "-v", fmt.Sprintf("%s/.librarian/generator-input:/input", repoDir), - "-v", fmt.Sprintf("%s:/output", testOutput), - "-v", fmt.Sprintf("%s:/source:ro", testAPIRoot), - "-v", fmt.Sprintf("%s/a/b/go.mod:/repo/a/b/go.mod:ro", repoDir), - "-v", fmt.Sprintf("%s/go.mod:/repo/go.mod:ro", repoDir), - "custom-image:abcd123", - string(CommandConfigure), - "--librarian=/librarian", - "--input=/input", - "--output=/output", - "--repo=/repo", - "--source=/source", - }, - }, - { - name: "configure_with_nil_global_files", - docker: &Docker{ - Image: testImage, - }, - runCommand: func(ctx context.Context, d *Docker) error { - configureRequest := &ConfigureRequest{ - State: state, - LibraryID: testLibraryID, - RepoDir: repoDir, - ApiRoot: testAPIRoot, - Output: testOutput, - GlobalFiles: nil, - } - - _, err := d.Configure(ctx, configureRequest) - - return err - }, - want: []string{ - "run", "--rm", - "-v", fmt.Sprintf("%s/.librarian:/librarian", repoDir), - "-v", fmt.Sprintf("%s/.librarian/generator-input:/input", repoDir), - "-v", fmt.Sprintf("%s:/output", testOutput), - "-v", fmt.Sprintf("%s:/source:ro", testAPIRoot), - testImage, - string(CommandConfigure), - "--librarian=/librarian", - "--input=/input", - "--output=/output", - "--repo=/repo", - "--source=/source", - }, - }, - { - name: "configure_with_source_roots", - docker: &Docker{ - Image: testImage, - }, - runCommand: func(ctx context.Context, d *Docker) error { - configureRequest := &ConfigureRequest{ - State: state, - LibraryID: testLibraryID, - RepoDir: repoDir, - ApiRoot: testAPIRoot, - Output: testOutput, - ExistingSourceRoots: []string{ - "a/path", - "b/path", - }, - } - - _, err := d.Configure(ctx, configureRequest) - - return err - }, - want: []string{ - "run", "--rm", - "-v", fmt.Sprintf("%s/.librarian:/librarian", repoDir), - "-v", fmt.Sprintf("%s/.librarian/generator-input:/input", repoDir), - "-v", fmt.Sprintf("%s:/output", testOutput), - "-v", fmt.Sprintf("%s:/source:ro", testAPIRoot), - "-v", fmt.Sprintf("%s/a/path:/repo/a/path:ro", repoDir), - "-v", fmt.Sprintf("%s/b/path:/repo/b/path:ro", repoDir), - testImage, - string(CommandConfigure), - "--librarian=/librarian", - "--input=/input", - "--output=/output", - "--repo=/repo", - "--source=/source", - }, - }, - { - name: "configure_with_nil_source_roots", - docker: &Docker{ - Image: testImage, - }, - runCommand: func(ctx context.Context, d *Docker) error { - configureRequest := &ConfigureRequest{ - State: state, - LibraryID: testLibraryID, - RepoDir: repoDir, - ApiRoot: testAPIRoot, - Output: testOutput, - ExistingSourceRoots: nil, - } - - _, err := d.Configure(ctx, configureRequest) - - return err - }, - want: []string{ - "run", "--rm", - "-v", fmt.Sprintf("%s/.librarian:/librarian", repoDir), - "-v", fmt.Sprintf("%s/.librarian/generator-input:/input", repoDir), - "-v", fmt.Sprintf("%s:/output", testOutput), - "-v", fmt.Sprintf("%s:/source:ro", testAPIRoot), - testImage, - string(CommandConfigure), - "--librarian=/librarian", - "--input=/input", - "--output=/output", - "--repo=/repo", - "--source=/source", - }, - }, - { - name: "configure_with_multiple_libraries_in_librarian_state", - docker: &Docker{ - Image: testImage, - }, - runCommand: func(ctx context.Context, d *Docker) error { - curState := &legacyconfig.LibrarianState{ - Image: testImage, - Libraries: []*legacyconfig.LibraryState{ - { - ID: testLibraryID, - APIs: []*legacyconfig.API{ - { - Path: "example/path/v1", - }, - }, - }, - { - ID: "another-example-library", - APIs: []*legacyconfig.API{ - { - Path: "another/example/path/v1", - ServiceConfig: "another_v1.yaml", - }, - }, - SourceRoots: []string{ - "another-example-source-path", - }, - }, - }, - } - configureRequest := &ConfigureRequest{ - State: curState, - LibraryID: testLibraryID, - RepoDir: repoDir, - ApiRoot: testAPIRoot, - Output: testOutput, - GlobalFiles: []string{ - "a/b/go.mod", - "go.mod", - }, - } - - configuredLibrary, err := d.Configure(ctx, configureRequest) - if configuredLibrary != testLibraryID { - return errors.New("configured library, " + configuredLibrary + " is wrong") - } - - return err - }, - want: []string{ - "run", "--rm", - "-v", fmt.Sprintf("%s/.librarian:/librarian", repoDir), - "-v", fmt.Sprintf("%s/.librarian/generator-input:/input", repoDir), - "-v", fmt.Sprintf("%s:/output", testOutput), - "-v", fmt.Sprintf("%s:/source:ro", testAPIRoot), - "-v", fmt.Sprintf("%s/a/b/go.mod:/repo/a/b/go.mod:ro", repoDir), - "-v", fmt.Sprintf("%s/go.mod:/repo/go.mod:ro", repoDir), - testImage, - string(CommandConfigure), - "--librarian=/librarian", - "--input=/input", - "--output=/output", - "--repo=/repo", - "--source=/source", - }, - }, - { - name: "Configure with invalid repo dir", - docker: &Docker{ - Image: testImage, - }, - runCommand: func(ctx context.Context, d *Docker) error { - configureRequest := &ConfigureRequest{ - State: state, - LibraryID: testLibraryID, - RepoDir: "/non-exist-dir", - ApiRoot: testAPIRoot, - } - _, err := d.Configure(ctx, configureRequest) - return err - }, - want: []string{}, - wantErr: true, - wantErrMsg: "failed to make directory", - }, - { - name: "Configure with mock image", - docker: &Docker{ - Image: mockImage, - }, - runCommand: func(ctx context.Context, d *Docker) error { - configureRequest := &ConfigureRequest{ - State: state, - LibraryID: testLibraryID, - RepoDir: repoDir, - ApiRoot: testAPIRoot, - } - - _, err := d.Configure(ctx, configureRequest) - - return err - }, - want: []string{}, - wantErr: true, - wantErrMsg: simulateDockerErrMsg, - }, - { - name: "Release stage for all libraries", - docker: &Docker{ - Image: testImage, - }, - runCommand: func(ctx context.Context, d *Docker) error { - partialRepoDir := filepath.Join(repoDir, "release-stage-all-libraries") - if err := os.MkdirAll(filepath.Join(repoDir, legacyconfig.LibrarianDir), 0755); err != nil { - t.Fatal(err) - } - - releaseInitRequest := &ReleaseStageRequest{ - State: state, - Output: testOutput, - LibrarianConfig: &legacyconfig.LibrarianConfig{}, - RepoDir: partialRepoDir, - } - - defer os.RemoveAll(partialRepoDir) - - return d.ReleaseStage(ctx, releaseInitRequest) - }, - want: []string{ - "run", "--rm", - "-v", fmt.Sprintf("%s/.librarian:/librarian", filepath.Join(repoDir, "release-stage-all-libraries")), - "-v", fmt.Sprintf("%s:/repo:ro", filepath.Join(repoDir, "release-stage-all-libraries")), - "-v", fmt.Sprintf("%s:/output", testOutput), - testImage, - string(CommandReleaseStage), - "--librarian=/librarian", - "--repo=/repo", - "--output=/output", - }, - }, - { - name: "Release stage runs in docker with image override", - docker: &Docker{ - Image: testImage, - }, - runCommand: func(ctx context.Context, d *Docker) error { - partialRepoDir := filepath.Join(repoDir, "release-stage-all-libraries") - if err := os.MkdirAll(filepath.Join(repoDir, legacyconfig.LibrarianDir), 0755); err != nil { - t.Fatal(err) - } - - releaseInitRequest := &ReleaseStageRequest{ - State: state, - Output: testOutput, - LibrarianConfig: &legacyconfig.LibrarianConfig{}, - RepoDir: partialRepoDir, - Image: "custom-image:abcd123", - } - - defer os.RemoveAll(partialRepoDir) - - return d.ReleaseStage(ctx, releaseInitRequest) - }, - want: []string{ - "run", "--rm", - "-v", fmt.Sprintf("%s/.librarian:/librarian", filepath.Join(repoDir, "release-stage-all-libraries")), - "-v", fmt.Sprintf("%s:/repo:ro", filepath.Join(repoDir, "release-stage-all-libraries")), - "-v", fmt.Sprintf("%s:/output", testOutput), - "custom-image:abcd123", - string(CommandReleaseStage), - "--librarian=/librarian", - "--repo=/repo", - "--output=/output", - }, - }, - { - name: "Release stage returns error", - docker: &Docker{ - Image: mockImage, - }, - runCommand: func(ctx context.Context, d *Docker) error { - partialRepoDir := filepath.Join(repoDir, "release-init-returns-error") - if err := os.MkdirAll(filepath.Join(repoDir, legacyconfig.LibrarianDir), 0755); err != nil { - t.Fatal(err) - } - - releaseInitRequest := &ReleaseStageRequest{ - State: state, - RepoDir: partialRepoDir, - Output: testOutput, - LibrarianConfig: &legacyconfig.LibrarianConfig{}, - } - defer os.RemoveAll(partialRepoDir) - - return d.ReleaseStage(ctx, releaseInitRequest) - }, - wantErr: true, - wantErrMsg: simulateDockerErrMsg, - }, - { - name: "Release stage with invalid partial repo dir", - docker: &Docker{ - Image: mockImage, - }, - runCommand: func(ctx context.Context, d *Docker) error { - releaseInitRequest := &ReleaseStageRequest{ - State: state, - RepoDir: "/non-exist-dir", - Output: testOutput, - } - - return d.ReleaseStage(ctx, releaseInitRequest) - }, - wantErr: true, - wantErrMsg: "failed to make directory", - }, - { - name: "Release stage for one library", - docker: &Docker{ - Image: testImage, - }, - runCommand: func(ctx context.Context, d *Docker) error { - partialRepoDir := filepath.Join(repoDir, "release-init-one-library") - if err := os.MkdirAll(filepath.Join(repoDir, legacyconfig.LibrarianDir), 0755); err != nil { - t.Fatal(err) - } - releaseInitRequest := &ReleaseStageRequest{ - State: state, - RepoDir: partialRepoDir, - Output: testOutput, - LibraryID: testLibraryID, - LibrarianConfig: &legacyconfig.LibrarianConfig{}, - } - defer os.RemoveAll(partialRepoDir) - - return d.ReleaseStage(ctx, releaseInitRequest) - }, - want: []string{ - "run", "--rm", - "-v", fmt.Sprintf("%s/.librarian:/librarian", filepath.Join(repoDir, "release-init-one-library")), - "-v", fmt.Sprintf("%s:/repo:ro", filepath.Join(repoDir, "release-init-one-library")), - "-v", fmt.Sprintf("%s:/output", testOutput), - testImage, - string(CommandReleaseStage), - "--librarian=/librarian", - "--repo=/repo", - "--output=/output", - }, - }, - { - name: "Release stage for one library with version", - docker: &Docker{ - Image: testImage, - }, - runCommand: func(ctx context.Context, d *Docker) error { - partialRepoDir := filepath.Join(repoDir, "release-init-one-library-with-version") - if err := os.MkdirAll(filepath.Join(repoDir, legacyconfig.LibrarianDir), 0755); err != nil { - t.Fatal(err) - } - - releaseInitRequest := &ReleaseStageRequest{ - State: state, - RepoDir: partialRepoDir, - Output: testOutput, - LibraryID: testLibraryID, - LibraryVersion: "1.2.3", - LibrarianConfig: &legacyconfig.LibrarianConfig{}, - } - defer os.RemoveAll(partialRepoDir) - - return d.ReleaseStage(ctx, releaseInitRequest) - }, - want: []string{ - "run", "--rm", - "-v", fmt.Sprintf("%s/.librarian:/librarian", filepath.Join(repoDir, "release-init-one-library-with-version")), - "-v", fmt.Sprintf("%s:/repo:ro", filepath.Join(repoDir, "release-init-one-library-with-version")), - "-v", fmt.Sprintf("%s:/output", testOutput), - testImage, - string(CommandReleaseStage), - "--librarian=/librarian", - "--repo=/repo", - "--output=/output", - }, - }, - } { - t.Run(test.name, func(t *testing.T) { - test.docker.run = func(args ...string) error { - if test.docker.Image == mockImage { - return errors.New("simulate docker command failure for testing") - } - if diff := cmp.Diff(test.want, args); diff != "" { - t.Errorf("mismatch(-want +got):\n%s", diff) - } - return nil - } - ctx := t.Context() - err := test.runCommand(ctx, test.docker) - - if test.wantErr { - if err == nil { - t.Fatalf("%s should return error", test.name) - } - if !strings.Contains(err.Error(), test.wantErrMsg) { - t.Errorf("want error message: %s, got: %s", test.wantErrMsg, err.Error()) - } - return - } - - if err != nil { - t.Fatal(err) - } - }) - } -} - -func TestWriteLibraryState(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - state *legacyconfig.LibrarianState - path string - filename string - wantFile string - wantErr bool - wantErrMsg string - }{ - { - name: "write library state to file", - state: &legacyconfig.LibrarianState{ - Image: "v1.0.0", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "google-cloud-go", - Version: "1.0.0", - LastGeneratedCommit: "abcd123", - APIs: []*legacyconfig.API{ - { - Path: "google/cloud/compute/v1", - ServiceConfig: "example_service_config.yaml", - Status: "new", - }, - }, - SourceRoots: []string{ - "src/example/path", - }, - PreserveRegex: []string{ - "example-preserve-regex", - }, - RemoveRegex: []string{ - "example-remove-regex", - }, - }, - { - ID: "google-cloud-storage", - Version: "1.2.3", - APIs: []*legacyconfig.API{ - { - Path: "google/storage/v1", - ServiceConfig: "storage_service_config.yaml", - Status: "existing", - }, - }, - }, - }, - }, - path: os.TempDir(), - filename: "a-library-example.json", - wantFile: "successful-marshaling-and-writing.json", - }, - { - name: "omit empty status", - state: &legacyconfig.LibrarianState{ - Image: "v1.0.0", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "google-cloud-go", - Version: "1.0.0", - APIs: []*legacyconfig.API{ - { - Path: "google/cloud/compute/v1", - ServiceConfig: "example_service_config.yaml", - }, - }, - SourceRoots: []string{ - "src/example/path", - }, - PreserveRegex: []string{ - "example-preserve-regex", - }, - RemoveRegex: []string{ - "example-remove-regex", - }, - }, - }, - }, - path: os.TempDir(), - filename: "omit-empty-status.json", - wantFile: "omit-empty-status.json", - }, - { - name: "empty library state", - state: &legacyconfig.LibrarianState{}, - path: os.TempDir(), - filename: "another-library-example.json", - wantFile: "empty-library-state.json", - }, - { - name: "nonexistent directory", - state: &legacyconfig.LibrarianState{}, - path: "/nonexistent_dir_for_test", - filename: "example.json", - wantErr: true, - wantErrMsg: "failed to make directory", - }, - { - name: "invalid file name", - state: &legacyconfig.LibrarianState{}, - path: os.TempDir(), - filename: "my\u0000file.json", - wantErr: true, - wantErrMsg: "failed to create JSON file", - }, - } { - t.Run(test.name, func(t *testing.T) { - filePath := filepath.Join(test.path, test.filename) - err := writeLibraryState(test.state, "google-cloud-go", filePath) - - if test.wantErr { - if err == nil { - t.Fatalf("writeLibraryState() should fail") - } - - if !strings.Contains(err.Error(), test.wantErrMsg) { - t.Errorf("want error message: %s, got: %s", test.wantErrMsg, err.Error()) - } - - return - } - - // Verify the file content - gotBytes, err := os.ReadFile(filePath) - if err != nil { - t.Fatalf("Failed to read generated file: %v", err) - } - - wantBytes, readErr := os.ReadFile(filepath.Join("testdata", "test-write-library-state", test.wantFile)) - if readErr != nil { - t.Fatalf("Failed to read expected state for comparison: %v", readErr) - } - - if diff := cmp.Diff(strings.TrimSpace(string(wantBytes)), string(gotBytes)); diff != "" { - t.Errorf("Generated JSON mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestWriteLibrarianState(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - state *legacyconfig.LibrarianState - path string - filename string - wantFile string - wantErr bool - wantErrMsg string - }{ - { - name: "write to a json file", - state: &legacyconfig.LibrarianState{ - Image: "v1.0.0", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "google-cloud-go", - Version: "1.0.0", - LastGeneratedCommit: "abcd123", - APIs: []*legacyconfig.API{ - { - Path: "google/cloud/compute/v1", - ServiceConfig: "example_service_config.yaml", - Status: "existing", - }, - }, - SourceRoots: []string{ - "src/example/path", - }, - PreserveRegex: []string{ - "example-preserve-regex", - }, - RemoveRegex: []string{ - "example-remove-regex", - }, - }, - { - ID: "google-cloud-storage", - Version: "1.2.3", - APIs: []*legacyconfig.API{ - { - Path: "google/storage/v1", - ServiceConfig: "storage_service_config.yaml", - Status: "existing", - }, - }, - }, - }, - }, - path: os.TempDir(), - filename: "a-librarian-example.json", - wantFile: "write-librarian-state-example.json", - }, - { - name: "empty librarian state", - state: &legacyconfig.LibrarianState{}, - path: os.TempDir(), - filename: "another-librarian-example.json", - wantFile: "empty-librarian-state.json", - }, - { - name: "nonexistent directory", - state: &legacyconfig.LibrarianState{}, - path: "/nonexistent_dir_for_test", - filename: "example.json", - wantErr: true, - wantErrMsg: "failed to make directory", - }, - { - name: "invalid file name", - state: &legacyconfig.LibrarianState{}, - path: os.TempDir(), - filename: "my\u0000file.json", - wantErr: true, - wantErrMsg: "failed to create JSON file", - }, - } { - t.Run(test.name, func(t *testing.T) { - filePath := filepath.Join(test.path, test.filename) - err := writeLibrarianState(test.state, filePath) - if test.wantErr { - if err == nil { - t.Fatalf("writeLibrarianState() should fail") - } - - if !strings.Contains(err.Error(), test.wantErrMsg) { - t.Errorf("want error message: %s, got: %s", test.wantErrMsg, err.Error()) - } - - return - } - - // Verify the file content - gotBytes, err := os.ReadFile(filePath) - if err != nil { - t.Fatalf("Failed to read generated file: %v", err) - } - - wantBytes, readErr := os.ReadFile(filepath.Join("testdata", "test-write-librarian-state", test.wantFile)) - if readErr != nil { - t.Fatalf("Failed to read expected state for comparison: %v", readErr) - } - - if diff := cmp.Diff(strings.TrimSpace(string(wantBytes)), string(gotBytes)); diff != "" { - t.Errorf("Generated JSON mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestDocker_runCommand(t *testing.T) { - for _, test := range []struct { - name string - cmdName string - args []string - wantErr bool - }{ - { - name: "success", - cmdName: "echo", - args: []string{"hello"}, - wantErr: false, - }, - { - name: "failure", - cmdName: "some-non-existent-command", - args: []string{}, - wantErr: true, - }, - } { - t.Run(test.name, func(t *testing.T) { - c := &Docker{} - if err := c.runCommand(test.cmdName, test.args...); (err != nil) != test.wantErr { - t.Errorf("Docker.runCommand() error = %v, wantErr %v", err, test.wantErr) - } - }) - } -} - -func TestReleaseStageRequestContent(t *testing.T) { - tmpDir := t.TempDir() - partialRepoDir := filepath.Join(tmpDir, "partial-repo") - librarianDir := filepath.Join(partialRepoDir, legacyconfig.LibrarianDir) - if err := os.MkdirAll(librarianDir, 0755); err != nil { - t.Fatalf("MkdirAll: %v", err) - } - - stateWithChanges := &legacyconfig.LibrarianState{ - Image: "test-image", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "my-library", - Version: "1.1.0", - ReleaseTriggered: true, - Changes: []*legacyconfig.Commit{ - { - Type: "feat", - Subject: "new feature", - Body: "body of feature", - PiperCLNumber: "12345", - CommitHash: "1234", - }, - }, - }, - }, - } - - d, err := New(tmpDir, "test-image", &DockerOptions{ - UserUID: "1000", - UserGID: "1000", - }) - if err != nil { - t.Fatalf("New() failed: %v", err) - } - - // Override the run command to intercept the arguments and verify the content - // of the release-stage-request.json file. - d.run = func(args ...string) error { - var librarianDir string - for i, arg := range args { - if arg == "-v" && i+1 < len(args) { - parts := strings.Split(args[i+1], ":") - if len(parts) == 2 && parts[1] == "/librarian" { - librarianDir = parts[0] - break - } - } - } - if librarianDir == "" { - return errors.New("could not find librarian directory mount") - } - - jsonPath := filepath.Join(librarianDir, "release-stage-request.json") - gotBytes, err := os.ReadFile(jsonPath) - if err != nil { - t.Fatalf("ReadFile failed: %v", err) - } - - wantFile := filepath.Join("testdata", "release-stage-request", "release-stage-request.json") - wantBytes, err := os.ReadFile(wantFile) - if err != nil { - t.Fatalf("ReadFile for want file failed: %v", err) - } - - if diff := cmp.Diff(strings.TrimSpace(string(wantBytes)), string(gotBytes)); diff != "" { - t.Errorf("Generated JSON mismatch (-want +got):\n%s", diff) - } - return nil - } - - req := &ReleaseStageRequest{ - State: stateWithChanges, - RepoDir: partialRepoDir, - Output: filepath.Join(tmpDir, "output"), - LibrarianConfig: &legacyconfig.LibrarianConfig{}, - } - if err := d.ReleaseStage(t.Context(), req); err != nil { - t.Fatalf("d.ReleaseStage() failed: %v", err) - } -} diff --git a/internal/legacylibrarian/legacydocker/testdata/release-stage-request/release-stage-request.json b/internal/legacylibrarian/legacydocker/testdata/release-stage-request/release-stage-request.json deleted file mode 100644 index 076e5fdc0ca..00000000000 --- a/internal/legacylibrarian/legacydocker/testdata/release-stage-request/release-stage-request.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "image": "test-image", - "libraries": [ - { - "id": "my-library", - "version": "1.1.0", - "changes": [ - { - "type": "feat", - "subject": "new feature", - "body": "body of feature", - "commit_hash": "1234", - "piper_cl_number": "12345" - } - ], - "apis": null, - "source_roots": null, - "preserve_regex": null, - "remove_regex": null, - "release_triggered": true - } - ] -} \ No newline at end of file diff --git a/internal/legacylibrarian/legacydocker/testdata/test-write-librarian-state/empty-librarian-state.json b/internal/legacylibrarian/legacydocker/testdata/test-write-librarian-state/empty-librarian-state.json deleted file mode 100644 index 10aa5849d67..00000000000 --- a/internal/legacylibrarian/legacydocker/testdata/test-write-librarian-state/empty-librarian-state.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "image": "", - "libraries": null -} diff --git a/internal/legacylibrarian/legacydocker/testdata/test-write-librarian-state/write-librarian-state-example.json b/internal/legacylibrarian/legacydocker/testdata/test-write-librarian-state/write-librarian-state-example.json deleted file mode 100644 index bddc337fee3..00000000000 --- a/internal/legacylibrarian/legacydocker/testdata/test-write-librarian-state/write-librarian-state-example.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "image": "v1.0.0", - "libraries": [ - { - "id": "google-cloud-go", - "version": "1.0.0", - "apis": [ - { - "path": "google/cloud/compute/v1", - "service_config": "example_service_config.yaml", - "status": "existing" - } - ], - "source_roots": [ - "src/example/path" - ], - "preserve_regex": [ - "example-preserve-regex" - ], - "remove_regex": [ - "example-remove-regex" - ] - }, - { - "id": "google-cloud-storage", - "version": "1.2.3", - "apis": [ - { - "path": "google/storage/v1", - "service_config": "storage_service_config.yaml", - "status": "existing" - } - ], - "source_roots": null, - "preserve_regex": null, - "remove_regex": null - } - ] -} \ No newline at end of file diff --git a/internal/legacylibrarian/legacydocker/testdata/test-write-library-state/empty-library-state.json b/internal/legacylibrarian/legacydocker/testdata/test-write-library-state/empty-library-state.json deleted file mode 100755 index e69de29bb2d..00000000000 diff --git a/internal/legacylibrarian/legacydocker/testdata/test-write-library-state/omit-empty-status.json b/internal/legacylibrarian/legacydocker/testdata/test-write-library-state/omit-empty-status.json deleted file mode 100644 index 92c2f514641..00000000000 --- a/internal/legacylibrarian/legacydocker/testdata/test-write-library-state/omit-empty-status.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "id": "google-cloud-go", - "version": "1.0.0", - "apis": [ - { - "path": "google/cloud/compute/v1", - "service_config": "example_service_config.yaml" - } - ], - "source_roots": [ - "src/example/path" - ], - "preserve_regex": [ - "example-preserve-regex" - ], - "remove_regex": [ - "example-remove-regex" - ] -} diff --git a/internal/legacylibrarian/legacydocker/testdata/test-write-library-state/successful-marshaling-and-writing.json b/internal/legacylibrarian/legacydocker/testdata/test-write-library-state/successful-marshaling-and-writing.json deleted file mode 100644 index 04cdd2ce228..00000000000 --- a/internal/legacylibrarian/legacydocker/testdata/test-write-library-state/successful-marshaling-and-writing.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "id": "google-cloud-go", - "version": "1.0.0", - "apis": [ - { - "path": "google/cloud/compute/v1", - "service_config": "example_service_config.yaml", - "status": "new" - } - ], - "source_roots": [ - "src/example/path" - ], - "preserve_regex": [ - "example-preserve-regex" - ], - "remove_regex": [ - "example-remove-regex" - ] -} \ No newline at end of file diff --git a/internal/legacylibrarian/legacygithub/github.go b/internal/legacylibrarian/legacygithub/github.go deleted file mode 100644 index 87d0d6b6bb4..00000000000 --- a/internal/legacylibrarian/legacygithub/github.go +++ /dev/null @@ -1,433 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package legacygithub provides operations on GitHub repos, abstracting away go-github -// (at least somewhat) to only the operations Librarian needs. -package legacygithub - -import ( - "context" - "fmt" - "io" - "log/slog" - "math/rand" - "net/http" - "net/url" - "strings" - "time" - - "github.com/google/go-github/v69/github" -) - -const ( - maxRetries = 5 - retryDelay = 2 * time.Second - maxDelay = 60 * time.Second - multiplier = 2.0 -) - -type retryableTransport struct { - transport http.RoundTripper - delay time.Duration - sleep func(time.Duration) -} - -// RoundTrip implements the http.RoundTripper interface and adds retry logic -// for transient server errors. -func (t *retryableTransport) RoundTrip(req *http.Request) (*http.Response, error) { - var resp *http.Response - var err error - b := newBackoff() - for i := 0; i < maxRetries; i++ { - resp, err = t.transport.RoundTrip(req) - if err == nil && - resp.StatusCode != http.StatusServiceUnavailable && - resp.StatusCode != http.StatusInternalServerError && - resp.StatusCode != http.StatusTooManyRequests && - resp.StatusCode != http.StatusBadGateway { - return resp, nil - } - if err != nil { - slog.Warn("retrying due to error", "err", err) - } else { - slog.Warn("retrying due to status code", "status_code", resp.StatusCode) - } - - delay := t.delay - if delay == 0 { - delay = b.pause() - } - - if resp != nil { - if after := resp.Header.Get("Retry-After"); after != "" { - if d, err := time.ParseDuration(after + "s"); err == nil && d > delay { - delay = d - } - } - } - if delay > maxDelay { - delay = maxDelay - } - slog.Info("retrying shortly", "delay", delay) - t.sleep(delay) - } - return resp, err -} - -// backoff implements exponential backoff for retry delays with jitter. -type backoff struct { - initial time.Duration - max time.Duration - multiplier float64 - cur time.Duration -} - -// newBackoff creates a new backoff with default settings. -func newBackoff() *backoff { - return &backoff{ - initial: retryDelay, - max: maxDelay, - multiplier: multiplier, - } -} - -// pause returns the time to wait before the next retry. -func (b *backoff) pause() time.Duration { - if b.cur == 0 { - b.cur = b.initial - } - d := time.Duration(1 + rand.Int63n(int64(b.cur))) - b.cur = time.Duration(float64(b.cur) * b.multiplier) - if b.cur > b.max { - b.cur = b.max - } - return d -} - -// PullRequest is a type alias for the go-github type. -type PullRequest = github.PullRequest - -// NewPullRequest is a type alias for the go-github type. -type NewPullRequest = github.NewPullRequest - -// RepositoryCommit is a type alias for the go-github type. -type RepositoryCommit = github.RepositoryCommit - -// PullRequestReview is a type alias for the go-github type. -type PullRequestReview = github.PullRequestReview - -// RepositoryRelease is a type alias for the go-github type. -type RepositoryRelease = github.RepositoryRelease - -// Client represents this package's abstraction of a GitHub client, including -// an access token. -type Client struct { - *github.Client - accessToken string - repo *Repository -} - -// NewClient creates a new Client to interact with GitHub. -func NewClient(accessToken string, repo *Repository) *Client { - return newClientWithHTTP(accessToken, repo, nil) -} - -func newClientWithHTTP(accessToken string, repo *Repository, httpClient *http.Client) *Client { - if httpClient == nil { - httpClient = &http.Client{} - } - transport := httpClient.Transport - if transport == nil { - transport = http.DefaultTransport - } - httpClient.Transport = &retryableTransport{transport: transport, sleep: time.Sleep} - client := github.NewClient(httpClient) - if repo != nil && repo.BaseURL != "" { - baseURL, _ := url.Parse(repo.BaseURL) - // Ensure the endpoint URL has a trailing slash. - if !strings.HasSuffix(baseURL.Path, "/") { - baseURL.Path += "/" - } - client.BaseURL = baseURL - } - if accessToken != "" { - client = client.WithAuthToken(accessToken) - } - return &Client{ - Client: client, - accessToken: accessToken, - repo: repo, - } -} - -// Token returns the access token for Client. -func (c *Client) Token() string { - return c.accessToken -} - -// Repository represents a GitHub repository with an owner (e.g. an organization or a user) -// and a repository name. -type Repository struct { - // The owner of the repository. - Owner string - // The name of the repository. - Name string - // Base URL for API requests. - BaseURL string -} - -// PullRequestMetadata identifies a pull request within a repository. -type PullRequestMetadata struct { - // Repo is the repository containing the pull request. - Repo *Repository - // Number is the number of the pull request. - Number int -} - -// ParseRemote parses a GitHub remote (anything to do with a repository) to determine -// the GitHub repo details (owner and name). -func ParseRemote(remote string) (*Repository, error) { - if strings.HasPrefix(remote, "https://github.com/") { - return parseHTTPRemote(remote) - } - if strings.HasPrefix(remote, "git@") { - return parseSSHRemote(remote) - } - return nil, fmt.Errorf("remote '%s' is not a GitHub remote", remote) -} - -func parseHTTPRemote(remote string) (*Repository, error) { - remotePath := remote[len("https://github.com/"):] - pathParts := strings.Split(remotePath, "/") - organization := pathParts[0] - repoName := pathParts[1] - repoName = strings.TrimSuffix(repoName, ".git") - return &Repository{Owner: organization, Name: repoName}, nil -} - -func parseSSHRemote(remote string) (*Repository, error) { - pathParts := strings.Split(remote, ":") - if len(pathParts) != 2 { - return nil, fmt.Errorf("remote %q is not a GitHub remote", remote) - } - orgRepo := strings.Split(pathParts[1], "/") - if len(orgRepo) != 2 { - return nil, fmt.Errorf("remote %q is not a GitHub remote", remote) - } - organization := orgRepo[0] - repoName := strings.TrimSuffix(orgRepo[1], ".git") - return &Repository{Owner: organization, Name: repoName}, nil -} - -// GetRawContent fetches the raw content of a file within a repository repo, -// identifying the file by path, at a specific commit/tag/branch of ref. -func (c *Client) GetRawContent(ctx context.Context, path, ref string) ([]byte, error) { - options := &github.RepositoryContentGetOptions{ - Ref: ref, - } - body, _, err := c.Repositories.DownloadContents(ctx, c.repo.Owner, c.repo.Name, path, options) - if err != nil { - return nil, err - } - defer body.Close() - return io.ReadAll(body) -} - -// CreatePullRequest creates a pull request in the remote repo. -// At the moment this requires a single remote to be configured, -// which must have a GitHub HTTPS URL. We assume a base branch of "main". -func (c *Client) CreatePullRequest(ctx context.Context, repo *Repository, remoteBranch, baseBranch, title, body string, isDraft bool) (*PullRequestMetadata, error) { - if body == "" { - slog.Warn("provided PR body is empty, setting default.") - body = "Regenerated all changed APIs. See individual commits for details." - } - slog.Info("creating PR", "branch", remoteBranch, "base", baseBranch, "title", title) - // The body may be excessively long, only display in debug mode. - slog.Debug("with PR body", "body", body) - newPR := &github.NewPullRequest{ - Title: &title, - Head: &remoteBranch, - Base: &baseBranch, - Body: github.Ptr(body), - MaintainerCanModify: github.Ptr(true), - Draft: github.Ptr(isDraft), - } - pr, _, err := c.PullRequests.Create(ctx, repo.Owner, repo.Name, newPR) - if err != nil { - return nil, err - } - - slog.Info("pr created", "url", pr.GetHTMLURL()) - pullRequestMetadata := &PullRequestMetadata{Repo: repo, Number: pr.GetNumber()} - return pullRequestMetadata, nil -} - -// GetLabels fetches the labels for an issue. -func (c *Client) GetLabels(ctx context.Context, number int) ([]string, error) { - slog.Info("getting labels", "number", number) - var allLabels []string - opts := &github.ListOptions{ - PerPage: 100, - } - for { - labels, resp, err := c.Issues.ListLabelsByIssue(ctx, c.repo.Owner, c.repo.Name, number, opts) - if err != nil { - return nil, err - } - for _, label := range labels { - allLabels = append(allLabels, *label.Name) - } - if resp.NextPage == 0 { - break - } - opts.Page = resp.NextPage - } - return allLabels, nil -} - -// ReplaceLabels replaces all labels for an issue. -func (c *Client) ReplaceLabels(ctx context.Context, number int, labels []string) error { - slog.Info("replacing labels", "number", number, "labels", labels) - _, _, err := c.Issues.ReplaceLabelsForIssue(ctx, c.repo.Owner, c.repo.Name, number, labels) - return err -} - -// AddLabelsToIssue adds labels to an existing issue in a GitHub repository. -func (c *Client) AddLabelsToIssue(ctx context.Context, repo *Repository, number int, labels []string) error { - slog.Info("labels added to issue", "number", number, "labels", labels) - _, _, err := c.Issues.AddLabelsToIssue(ctx, repo.Owner, repo.Name, number, labels) - return err -} - -// SearchPullRequests searches for pull requests in the repository using the provided raw query. -func (c *Client) SearchPullRequests(ctx context.Context, query string) ([]*PullRequest, error) { - var prs []*PullRequest - opts := &github.SearchOptions{ - ListOptions: github.ListOptions{PerPage: 100}, - } - query = fmt.Sprintf("repo:%s/%s %s", c.repo.Owner, c.repo.Name, query) - for { - result, resp, err := c.Search.Issues(ctx, query, opts) - if err != nil { - return nil, err - } - for _, issue := range result.Issues { - if issue.IsPullRequest() { - pr, _, err := c.PullRequests.Get(ctx, c.repo.Owner, c.repo.Name, issue.GetNumber()) - if err != nil { - return nil, err - } - prs = append(prs, pr) - } - } - if resp.NextPage == 0 { - break - } - opts.Page = resp.NextPage - } - return prs, nil -} - -// GetPullRequest gets a pull request by its number. -func (c *Client) GetPullRequest(ctx context.Context, number int) (*PullRequest, error) { - pr, _, err := c.PullRequests.Get(ctx, c.repo.Owner, c.repo.Name, number) - return pr, err -} - -// CreateRelease creates a tag and release in the repository at the given -// commit-ish. See -// https://git-scm.com/docs/gitglossary#Documentation/gitglossary.txt-commit-ishalsocommittish -// for definition of commit-ish. -func (c *Client) CreateRelease(ctx context.Context, tagName, name, body, commitish string) (*github.RepositoryRelease, error) { - r, _, err := c.Repositories.CreateRelease(ctx, c.repo.Owner, c.repo.Name, &github.RepositoryRelease{ - TagName: &tagName, - Name: &name, - Body: &body, - TargetCommitish: &commitish, - }) - return r, err -} - -// CreateIssueComment adds a comment to the issue number provided. -func (c *Client) CreateIssueComment(ctx context.Context, number int, comment string) error { - _, _, err := c.Issues.CreateComment(ctx, c.repo.Owner, c.repo.Name, number, &github.IssueComment{ - Body: &comment, - }) - return err -} - -// hasLabel checks if a pull request has a given label. -func hasLabel(pr *PullRequest, labelName string) bool { - for _, l := range pr.Labels { - if l.GetName() == labelName { - return true - } - } - return false -} - -// FindMergedPullRequestsWithPendingReleaseLabel finds all merged pull requests with the "release:pending" label. -func (c *Client) FindMergedPullRequestsWithPendingReleaseLabel(ctx context.Context, owner, repo string) ([]*PullRequest, error) { - return c.FindMergedPullRequestsWithLabel(ctx, owner, repo, "release:pending") -} - -// FindMergedPullRequestsWithLabel finds all merged pull requests with the "release:pending" label. -func (c *Client) FindMergedPullRequestsWithLabel(ctx context.Context, owner, repo, label string) ([]*PullRequest, error) { - var allPRs []*PullRequest - opt := &github.PullRequestListOptions{ - State: "closed", - ListOptions: github.ListOptions{ - PerPage: 100, - }, - } - for { - prs, resp, err := c.PullRequests.List(ctx, owner, repo, opt) - if err != nil { - return nil, err - } - for _, pr := range prs { - if !pr.GetMergedAt().IsZero() && hasLabel(pr, label) { - allPRs = append(allPRs, pr) - } - } - if resp.NextPage == 0 || len(allPRs) >= 10 { - break - } - opt.Page = resp.NextPage - } - return allPRs, nil -} - -// CreateTag creates a lightweight tag in the repository at the given commit SHA. -// This does NOT create a release, just the tag. -func (c *Client) CreateTag(ctx context.Context, tagName, commitSHA string) error { - slog.Info("creating tag", "tag", tagName, "commit", commitSHA) - ref := "refs/tags/" + tagName - tagRef := &github.Reference{ - Ref: github.Ptr(ref), - Object: &github.GitObject{SHA: github.Ptr(commitSHA), Type: github.Ptr("commit")}, - } - _, _, err := c.Git.CreateRef(ctx, c.repo.Owner, c.repo.Name, tagRef) - return err -} - -// ClosePullRequest closes the pull request specified by pull request number. -func (c *Client) ClosePullRequest(ctx context.Context, number int) error { - slog.Info("closing pull request", slog.Int("number", number)) - state := "closed" - _, _, err := c.PullRequests.Edit(ctx, c.repo.Owner, c.repo.Name, number, &github.PullRequest{ - State: &state, - }) - return err -} diff --git a/internal/legacylibrarian/legacygithub/github_test.go b/internal/legacylibrarian/legacygithub/github_test.go deleted file mode 100644 index 8c40058b6ad..00000000000 --- a/internal/legacylibrarian/legacygithub/github_test.go +++ /dev/null @@ -1,1231 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacygithub - -import ( - "encoding/json" - "fmt" - "net/http" - "net/http/httptest" - "net/url" - "strings" - "testing" - "time" - - "github.com/google/go-cmp/cmp" - "github.com/google/go-github/v69/github" -) - -func TestToken(t *testing.T) { - t.Parallel() - want := "fake-token" - repo := &Repository{Owner: "owner", Name: "repo"} - client := NewClient(want, repo) - if got := client.Token(); got != want { - t.Errorf("Token() = %q, want %q", got, want) - } -} - -func TestGetRawContent(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - handler http.HandlerFunc - wantContent []byte - wantErr bool - wantErrSubstr string - wantHTTPMethod string - wantURLPath string - }{ - { - name: "Success", - handler: func(w http.ResponseWriter, r *http.Request) { - if strings.HasSuffix(r.URL.Path, "contents/path/to") { - fmt.Fprintf(w, `[{"content":"file content", "name":"file", "download_url": "http://%s/download"}]`, r.Host) - } else if strings.HasSuffix(r.URL.Path, "download") { - fmt.Fprint(w, "file content") - } else { - t.Error("unexpected URL path") - w.WriteHeader(http.StatusNotFound) - } - }, - wantContent: []byte("file content"), - wantErr: false, - wantHTTPMethod: http.MethodGet, - wantURLPath: "/repos/owner/repo/contents/path/to/file", - }, - { - name: "Not Found", - handler: func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusNotFound) - }, - wantErr: true, - wantErrSubstr: "404", - wantHTTPMethod: http.MethodGet, - wantURLPath: "/repos/owner/repo/contents/path/to/file", - }, - { - name: "Internal Server Error", - handler: func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusInternalServerError) - }, - wantErr: true, - wantErrSubstr: "500", - wantHTTPMethod: http.MethodGet, - wantURLPath: "/repos/owner/repo/contents/path/to/file", - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - test.handler(w, r) - })) - defer server.Close() - - repo := &Repository{Owner: "owner", Name: "repo"} - client := newClientWithHTTP("fake-token", repo, server.Client()) - - client.BaseURL, _ = url.Parse(server.URL + "/") - content, err := client.GetRawContent(t.Context(), "path/to/file", "main") - - if test.wantErr { - if err == nil { - t.Fatalf("GetRawContent() err = nil, want error containing %q", test.wantErrSubstr) - } - if !strings.Contains(err.Error(), test.wantErrSubstr) { - t.Errorf("GetRawContent() err = %v, want error containing %q", err, test.wantErrSubstr) - } - } else { - if err != nil { - t.Errorf("GetRawContent() err = %v, want nil", err) - } - if diff := cmp.Diff(test.wantContent, content); diff != "" { - t.Errorf("GetRawContent() content mismatch (-want +got):\n%s", diff) - } - } - }) - } -} - -func TestParseURL(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - remoteURL string - wantRepo *Repository - wantErr bool - wantErrSubstr string - }{ - { - name: "Valid HTTPS URL", - remoteURL: "https://github.com/owner/repo.git", - wantRepo: &Repository{Owner: "owner", Name: "repo"}, - wantErr: false, - }, - { - name: "Valid HTTPS URL without .git", - remoteURL: "https://github.com/owner/repo", - wantRepo: &Repository{Owner: "owner", Name: "repo"}, - wantErr: false, - }, - { - name: "Invalid URL scheme", - remoteURL: "http://github.com/owner/repo.git", - wantErr: true, - wantErrSubstr: "not a GitHub remote", - }, - { - name: "URL with extra path components", - remoteURL: "https://github.com/owner/repo/pulls", - wantRepo: &Repository{Owner: "owner", Name: "repo"}, - wantErr: false, - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - repo, err := ParseRemote(test.remoteURL) - - if test.wantErr { - if err == nil { - t.Fatalf("ParseURL() err = nil, want error containing %q", test.wantErrSubstr) - } - if !strings.Contains(err.Error(), test.wantErrSubstr) { - t.Errorf("ParseURL() err = %v, want error containing %q", err, test.wantErrSubstr) - } - } else { - if err != nil { - t.Errorf("ParseURL() err = %v, want nil", err) - } - if diff := cmp.Diff(test.wantRepo, repo); diff != "" { - t.Errorf("ParseURL() repo mismatch (-want +got): %s", diff) - } - } - }) - } -} - -func TestParseSSHRemote(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - remote string - wantRepo *Repository - wantErr bool - wantErrSubstr string - }{ - { - name: "Valid SSH URL with .git", - remote: "git@github.com:owner/repo.git", - wantRepo: &Repository{Owner: "owner", Name: "repo"}, - }, - { - name: "Valid SSH URL without .git", - remote: "git@github.com:owner/repo", - wantRepo: &Repository{Owner: "owner", Name: "repo"}, - }, - { - name: "Invalid remote, no git@ prefix", - remote: "https://github.com/owner/repo.git", - wantErr: true, - wantErrSubstr: "not a GitHub remote", - }, - { - name: "Invalid remote, no colon", - remote: "git@github.com-owner/repo.git", - wantErr: true, - wantErrSubstr: "not a GitHub remote", - }, - { - name: "Invalid remote, no slash", - remote: "git@github.com:owner-repo.git", - wantErr: true, - wantErrSubstr: "not a GitHub remote", - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - repo, err := parseSSHRemote(test.remote) - if test.wantErr { - if err == nil { - t.Fatalf("ParseSSHRemote() err = nil, want error containing %q", test.wantErrSubstr) - } - if !strings.Contains(err.Error(), test.wantErrSubstr) { - t.Errorf("ParseSSHRemote() err = %v, want error containing %q", err, test.wantErrSubstr) - } - } else { - if err != nil { - t.Errorf("ParseSSHRemote() err = %v, want nil", err) - } - if diff := cmp.Diff(test.wantRepo, repo); diff != "" { - t.Errorf("ParseSSHRemote() repo mismatch (-want +got): %s", diff) - } - } - }) - } -} - -func TestCreatePullRequest(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - remoteBranch string - remoteBase string - title string - body string - handler http.HandlerFunc - wantMetadata *PullRequestMetadata - wantErr bool - wantErrSubstr string - }{ - { - name: "Success with provided body", - remoteBranch: "feature-branch", - remoteBase: "base-branch", - title: "New Feature", - body: "This is a new feature.", - handler: func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPost { - t.Errorf("unexpected method: got %s, want %s", r.Method, http.MethodPost) - } - if r.URL.Path != "/repos/owner/repo/pulls" { - t.Errorf("unexpected path: got %s, want %s", r.URL.Path, "/repos/owner/repo/pulls") - } - var newPR github.NewPullRequest - if err := json.NewDecoder(r.Body).Decode(&newPR); err != nil { - t.Fatalf("failed to decode request body: %v", err) - } - if *newPR.Title != "New Feature" { - t.Errorf("unexpected title: got %q, want %q", *newPR.Title, "New Feature") - } - if *newPR.Body != "This is a new feature." { - t.Errorf("unexpected body: got %q, want %q", *newPR.Body, "This is a new feature.") - } - if *newPR.Head != "feature-branch" { - t.Errorf("unexpected head: got %q, want %q", *newPR.Head, "feature-branch") - } - if *newPR.Base != "base-branch" { - t.Errorf("unexpected base: got %q, want %q", *newPR.Base, "base-branch") - } - fmt.Fprint(w, `{"number": 1, "html_url": "https://github.com/owner/repo/pull/1"}`) - }, - wantMetadata: &PullRequestMetadata{Repo: &Repository{Owner: "owner", Name: "repo"}, Number: 1}, - }, - { - name: "Success with empty body", - remoteBranch: "another-branch", - remoteBase: "main", - title: "Another PR", - body: "", - handler: func(w http.ResponseWriter, r *http.Request) { - var newPR github.NewPullRequest - if err := json.NewDecoder(r.Body).Decode(&newPR); err != nil { - t.Fatalf("failed to decode request body: %v", err) - } - expectedBody := "Regenerated all changed APIs. See individual commits for details." - if *newPR.Body != expectedBody { - t.Errorf("unexpected body: got %q, want %q", *newPR.Body, expectedBody) - } - fmt.Fprint(w, `{"number": 1, "html_url": "https://github.com/owner/repo/pull/1"}`) - }, - wantMetadata: &PullRequestMetadata{Repo: &Repository{Owner: "owner", Name: "repo"}, Number: 1}, - }, - { - name: "GitHub API error", - remoteBranch: "error-branch", - remoteBase: "main", - title: "Error PR", - body: "This will fail.", - handler: func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusInternalServerError) }, - wantErr: true, - wantErrSubstr: "500", - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - server := httptest.NewServer(test.handler) - defer server.Close() - - repo := &Repository{Owner: "owner", Name: "repo"} - client := newClientWithHTTP("fake-token", repo, server.Client()) - client.BaseURL, _ = url.Parse(server.URL + "/") - - metadata, err := client.CreatePullRequest(t.Context(), repo, test.remoteBranch, test.remoteBase, test.title, test.body, false) - - if test.wantErr { - if err == nil { - t.Fatalf("CreatePullRequest() err = nil, want error containing %q", test.wantErrSubstr) - } - if !strings.Contains(err.Error(), test.wantErrSubstr) { - t.Errorf("CreatePullRequest() err = %v, want error containing %q", err, test.wantErrSubstr) - } - } else if err != nil { - t.Errorf("CreatePullRequest() err = %v, want nil", err) - } - - if diff := cmp.Diff(test.wantMetadata, metadata); diff != "" { - t.Errorf("CreatePullRequest() metadata mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestAddLabelsToIssue(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - handler http.HandlerFunc - issueNum int - labels []string - wantErr bool - wantErrSubstr string - }{ - { - name: "add labels to an issue", - handler: func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPost { - t.Errorf("unexpected method: got %s, want %s", r.Method, http.MethodPost) - } - wantPath := "/repos/owner/repo/issues/7/labels" - if r.URL.Path != wantPath { - t.Errorf("unexpected path: got %s, want %s", r.URL.Path, wantPath) - } - var labels []string - if err := json.NewDecoder(r.Body).Decode(&labels); err != nil { - t.Fatalf("failed to decode request body: %v", err) - } - expectedBody := []string{"new-label", "another-label"} - if strings.Join(labels, ",") != strings.Join(expectedBody, ",") { - t.Errorf("unexpected body: got %q, want %q", labels, expectedBody) - } - }, - issueNum: 7, - labels: []string{"new-label", "another-label"}, - }, - { - name: "GitHub API error", - handler: func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusInternalServerError) }, - wantErr: true, - wantErrSubstr: "500", - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - server := httptest.NewServer(test.handler) - defer server.Close() - - repo := &Repository{Owner: "owner", Name: "repo"} - client := newClientWithHTTP("fake-token", repo, server.Client()) - client.BaseURL, _ = url.Parse(server.URL + "/") - - err := client.AddLabelsToIssue(t.Context(), repo, test.issueNum, test.labels) - - if test.wantErr { - if err == nil { - t.Fatal("AddLabelsToIssue() should return an error") - } - if !strings.Contains(err.Error(), test.wantErrSubstr) { - t.Errorf("AddLabelsToIssue() err = %v, want error containing %q", err, test.wantErrSubstr) - } - - return - } - - if err != nil { - t.Errorf("AddLabelsToIssue() err = %v, want nil", err) - } - }) - } -} - -func TestGetLabels(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - handler http.HandlerFunc - issueNum int - wantLabels []string - wantErr bool - wantErrSubstr string - }{ - { - name: "get labels from an issue", - handler: func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet { - t.Errorf("unexpected method: got %s, want %s", r.Method, http.MethodGet) - } - wantPath := "/repos/owner/repo/issues/7/labels" - if r.URL.Path != wantPath { - t.Errorf("unexpected path: got %s, want %s", r.URL.Path, wantPath) - } - fmt.Fprint(w, `[{"name": "label1"}, {"name": "label2"}]`) - }, - issueNum: 7, - wantLabels: []string{"label1", "label2"}, - }, - { - name: "get labels with pagination", - handler: func(w http.ResponseWriter, r *http.Request) { - if r.URL.Query().Get("page") == "2" { - fmt.Fprint(w, `[{"name": "label3"}]`) - return - } - w.Header().Set("Link", `; rel="next"`) - fmt.Fprint(w, `[{"name": "label1"}, {"name": "label2"}]`) - }, - issueNum: 7, - wantLabels: []string{"label1", "label2", "label3"}, - }, - { - name: "GitHub API error", - handler: func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusInternalServerError) }, - issueNum: 7, - wantErr: true, - wantErrSubstr: "500", - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - server := httptest.NewServer(test.handler) - defer server.Close() - - repo := &Repository{Owner: "owner", Name: "repo"} - client := newClientWithHTTP("fake-token", repo, server.Client()) - client.BaseURL, _ = url.Parse(server.URL + "/") - - gotLabels, err := client.GetLabels(t.Context(), test.issueNum) - - if test.wantErr { - if err == nil { - t.Fatal("GetLabels() should return an error") - } - if !strings.Contains(err.Error(), test.wantErrSubstr) { - t.Errorf("GetLabels() err = %v, want error containing %q", err, test.wantErrSubstr) - } - return - } - - if err != nil { - t.Errorf("GetLabels() err = %v, want nil", err) - } - if diff := cmp.Diff(test.wantLabels, gotLabels); diff != "" { - t.Errorf("GetLabels() labels mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestReplaceLabels(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - handler http.HandlerFunc - issueNum int - labels []string - wantErr bool - wantErrSubstr string - }{ - { - name: "replace labels for an issue", - handler: func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPut { - t.Errorf("unexpected method: got %s, want %s", r.Method, http.MethodPut) - } - wantPath := "/repos/owner/repo/issues/7/labels" - if r.URL.Path != wantPath { - t.Errorf("unexpected path: got %s, want %s", r.URL.Path, wantPath) - } - var labels []string - if err := json.NewDecoder(r.Body).Decode(&labels); err != nil { - t.Fatalf("failed to decode request body: %v", err) - } - expectedBody := []string{"new-label", "another-label"} - if diff := cmp.Diff(expectedBody, labels); diff != "" { - t.Errorf("ReplaceLabels() request body mismatch (-want +got):\n%s", diff) - } - fmt.Fprint(w, `[]`) - }, - issueNum: 7, - labels: []string{"new-label", "another-label"}, - }, - { - name: "GitHub API error", - handler: func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusInternalServerError) }, - issueNum: 7, - labels: []string{"some-label"}, - wantErr: true, - wantErrSubstr: "500", - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - server := httptest.NewServer(test.handler) - defer server.Close() - - repo := &Repository{Owner: "owner", Name: "repo"} - client := newClientWithHTTP("fake-token", repo, server.Client()) - client.BaseURL, _ = url.Parse(server.URL + "/") - - err := client.ReplaceLabels(t.Context(), test.issueNum, test.labels) - - if test.wantErr { - if err == nil { - t.Fatal("ReplaceLabels() should return an error") - } - if !strings.Contains(err.Error(), test.wantErrSubstr) { - t.Errorf("ReplaceLabels() err = %v, want error containing %q", err, test.wantErrSubstr) - } - return - } - - if err != nil { - t.Errorf("ReplaceLabels() err = %v, want nil", err) - } - }) - } -} - -func TestSearchPullRequests(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - query string - handler http.HandlerFunc - wantPRs []*PullRequest - wantErr bool - wantErrSubstr string - }{ - { - name: "Success with single page", - query: "is:pr is:open author:app/dependabot", - handler: func(w http.ResponseWriter, r *http.Request) { - if strings.HasPrefix(r.URL.Path, "/search/issues") { - if r.URL.Query().Get("q") != "repo:owner/repo is:pr is:open author:app/dependabot" { - t.Errorf("unexpected query: got %q", r.URL.Query().Get("q")) - } - fmt.Fprint(w, `{"items": [{"number": 1, "pull_request": {}}]}`) - } else if strings.HasPrefix(r.URL.Path, "/repos/owner/repo/pulls/1") { - fmt.Fprint(w, `{"number": 1, "title": "PR 1"}`) - } else { - w.WriteHeader(http.StatusNotFound) - } - }, - wantPRs: []*PullRequest{ - {Number: github.Ptr(1), Title: github.Ptr("PR 1")}, - }, - }, - { - name: "Success with multiple pages", - query: "is:pr is:open", - handler: func(w http.ResponseWriter, r *http.Request) { - if strings.HasPrefix(r.URL.Path, "/search/issues") { - if r.URL.Query().Get("page") == "2" { - fmt.Fprint(w, `{"items": [{"number": 2, "pull_request": {}}]}`) - } else { - w.Header().Set("Link", `; rel="next"`) - fmt.Fprint(w, `{"items": [{"number": 1, "pull_request": {}}]}`) - } - } else if strings.HasPrefix(r.URL.Path, "/repos/owner/repo/pulls/1") { - fmt.Fprint(w, `{"number": 1, "title": "PR 1"}`) - } else if strings.HasPrefix(r.URL.Path, "/repos/owner/repo/pulls/2") { - fmt.Fprint(w, `{"number": 2, "title": "PR 2"}`) - } else { - w.WriteHeader(http.StatusNotFound) - } - }, - wantPRs: []*PullRequest{ - {Number: github.Ptr(1), Title: github.Ptr("PR 1")}, - {Number: github.Ptr(2), Title: github.Ptr("PR 2")}, - }, - }, - { - name: "Search API error", - query: "is:pr", - handler: func(w http.ResponseWriter, r *http.Request) { - if strings.HasPrefix(r.URL.Path, "/search/issues") { - w.WriteHeader(http.StatusInternalServerError) - } - }, - wantErr: true, - wantErrSubstr: "500", - }, - { - name: "Get PR API error", - query: "is:pr", - handler: func(w http.ResponseWriter, r *http.Request) { - if strings.HasPrefix(r.URL.Path, "/search/issues") { - fmt.Fprint(w, `{"items": [{"number": 1, "pull_request": {}}]}`) - } else if strings.HasPrefix(r.URL.Path, "/repos/owner/repo/pulls/1") { - w.WriteHeader(http.StatusInternalServerError) - } - }, - wantErr: true, - wantErrSubstr: "500", - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - server := httptest.NewServer(test.handler) - defer server.Close() - - repo := &Repository{Owner: "owner", Name: "repo"} - client := newClientWithHTTP("fake-token", repo, server.Client()) - client.BaseURL, _ = url.Parse(server.URL + "/") - - prs, err := client.SearchPullRequests(t.Context(), test.query) - - if test.wantErr { - if err == nil { - t.Fatalf("SearchPullRequests() err = nil, want error containing %q", test.wantErrSubstr) - } - if !strings.Contains(err.Error(), test.wantErrSubstr) { - t.Errorf("SearchPullRequests() err = %v, want error containing %q", err, test.wantErrSubstr) - } - } else if err != nil { - t.Errorf("SearchPullRequests() err = %v, want nil", err) - } - - if diff := cmp.Diff(test.wantPRs, prs); diff != "" { - t.Errorf("SearchPullRequests() prs mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestGetPullRequest(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - number int - handler http.HandlerFunc - wantPR *PullRequest - wantErr bool - wantErrSubstr string - }{ - { - name: "Success", - number: 42, - handler: func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet { - t.Errorf("unexpected method: got %s, want %s", r.Method, http.MethodGet) - } - wantPath := "/repos/owner/repo/pulls/42" - if r.URL.Path != wantPath { - t.Errorf("unexpected path: got %s, want %s", r.URL.Path, wantPath) - } - fmt.Fprint(w, `{"number": 42, "title": "The Answer"}`) - }, - wantPR: &PullRequest{Number: github.Ptr(42), Title: github.Ptr("The Answer")}, - }, - { - name: "Not Found", - number: 43, - handler: func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusNotFound) - }, - wantErr: true, - wantErrSubstr: "404", - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - server := httptest.NewServer(test.handler) - defer server.Close() - - repo := &Repository{Owner: "owner", Name: "repo"} - client := newClientWithHTTP("fake-token", repo, server.Client()) - client.BaseURL, _ = url.Parse(server.URL + "/") - - pr, err := client.GetPullRequest(t.Context(), test.number) - - if test.wantErr { - if err == nil { - t.Fatalf("GetPullRequest() err = nil, want error containing %q", test.wantErrSubstr) - } - if !strings.Contains(err.Error(), test.wantErrSubstr) { - t.Errorf("GetPullRequest() err = %v, want error containing %q", err, test.wantErrSubstr) - } - } else if err != nil { - t.Errorf("GetPullRequest() err = %v, want nil", err) - } - - if diff := cmp.Diff(test.wantPR, pr); diff != "" { - t.Errorf("GetPullRequest() pr mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestCreateRelease(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - tagName string - releaseName string - body string - commitish string - handler http.HandlerFunc - wantRelease *github.RepositoryRelease - wantErr bool - wantErrSubstr string - }{ - { - name: "Success", - tagName: "v1.0.0", - releaseName: "Version 1.0.0", - body: "Initial release", - commitish: "main", - handler: func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPost { - t.Errorf("unexpected method: got %s, want %s", r.Method, http.MethodPost) - } - wantPath := "/repos/owner/repo/releases" - if r.URL.Path != wantPath { - t.Errorf("unexpected path: got %s, want %s", r.URL.Path, wantPath) - } - var newRelease github.RepositoryRelease - if err := json.NewDecoder(r.Body).Decode(&newRelease); err != nil { - t.Fatalf("failed to decode request body: %v", err) - } - if *newRelease.TagName != "v1.0.0" { - t.Errorf("unexpected tag name: got %q, want %q", *newRelease.TagName, "v1.0.0") - } - fmt.Fprint(w, `{"tag_name": "v1.0.0", "name": "Version 1.0.0"}`) - }, - wantRelease: &github.RepositoryRelease{TagName: github.Ptr("v1.0.0"), Name: github.Ptr("Version 1.0.0")}, - }, - { - name: "API Error", - tagName: "v1.0.0", - releaseName: "Version 1.0.0", - body: "Initial release", - commitish: "main", - handler: func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusInternalServerError) }, - wantErr: true, - wantErrSubstr: "500", - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - server := httptest.NewServer(test.handler) - defer server.Close() - - repo := &Repository{Owner: "owner", Name: "repo"} - client := newClientWithHTTP("fake-token", repo, server.Client()) - client.BaseURL, _ = url.Parse(server.URL + "/") - - release, err := client.CreateRelease(t.Context(), test.tagName, test.releaseName, test.body, test.commitish) - - if test.wantErr { - if err == nil { - t.Fatalf("CreateRelease() err = nil, want error containing %q", test.wantErrSubstr) - } - if !strings.Contains(err.Error(), test.wantErrSubstr) { - t.Errorf("CreateRelease() err = %v, want error containing %q", err, test.wantErrSubstr) - } - } else if err != nil { - t.Errorf("CreateRelease() err = %v, want nil", err) - } - - if diff := cmp.Diff(test.wantRelease, release); diff != "" { - t.Errorf("CreateRelease() release mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestCreateIssueComment(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - number int - body string - handler http.HandlerFunc - wantErr bool - wantErrSubstr string - }{ - { - name: "Success", - number: 123, - body: "This is a comment.", - handler: func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPost { - t.Errorf("unexpected method: got %s, want %s", r.Method, http.MethodPost) - } - wantPath := "/repos/owner/repo/issues/123/comments" - if r.URL.Path != wantPath { - t.Errorf("unexpected path: got %s, want %s", r.URL.Path, wantPath) - } - var comment github.IssueComment - if err := json.NewDecoder(r.Body).Decode(&comment); err != nil { - t.Fatalf("failed to decode request body: %v", err) - } - if *comment.Body != "This is a comment." { - t.Errorf("unexpected body: got %q, want %q", *comment.Body, "This is a comment.") - } - w.WriteHeader(http.StatusCreated) - }, - }, - { - name: "API Error", - number: 123, - body: "This is a comment.", - handler: func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusInternalServerError) }, - wantErr: true, - wantErrSubstr: "500", - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - server := httptest.NewServer(test.handler) - defer server.Close() - - repo := &Repository{Owner: "owner", Name: "repo"} - client := newClientWithHTTP("fake-token", repo, server.Client()) - client.BaseURL, _ = url.Parse(server.URL + "/") - - err := client.CreateIssueComment(t.Context(), test.number, test.body) - - if test.wantErr { - if err == nil { - t.Fatalf("CreateComment() err = nil, want error containing %q", test.wantErrSubstr) - } - if !strings.Contains(err.Error(), test.wantErrSubstr) { - t.Errorf("CreateComment() err = %v, want error containing %q", err, test.wantErrSubstr) - } - } else if err != nil { - t.Errorf("CreateComment() err = %v, want nil", err) - } - }) - } -} - -func TestFindMergedPullRequestsWithPendingReleaseLabel(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - handler http.HandlerFunc - wantPRs []*PullRequest - wantErr bool - wantErrSubstr string - }{ - { - name: "Success with single page", - handler: func(w http.ResponseWriter, r *http.Request) { - if r.URL.Query().Get("state") != "closed" { - t.Errorf("unexpected state: got %q", r.URL.Query().Get("state")) - } - pr0 := github.PullRequest{Number: github.Ptr(0), HTMLURL: github.Ptr("https://github.com/owner/repo/pull/0"), MergedAt: &github.Timestamp{Time: time.Date(2025, time.September, 5, 20, 44, 59, 0, time.UTC)}, Labels: []*github.Label{{Name: github.Ptr("release:pending")}}} - pr1 := github.PullRequest{Number: github.Ptr(1), Labels: []*github.Label{{Name: github.Ptr("release:pending")}}} - pr2 := github.PullRequest{Number: github.Ptr(2), Labels: []*github.Label{{Name: github.Ptr("other-label")}}} - pr3 := github.PullRequest{Number: github.Ptr(3), HTMLURL: github.Ptr("https://github.com/owner/repo/pull/3"), ClosedAt: &github.Timestamp{Time: time.Date(2025, time.September, 5, 20, 44, 59, 0, time.UTC)}, Labels: []*github.Label{{Name: github.Ptr("release:pending")}}} - prs := []*github.PullRequest{&pr0, &pr1, &pr2, &pr3} - b, err := json.Marshal(prs) - if err != nil { - t.Fatalf("json.Marshal() failed: %v", err) - } - fmt.Fprint(w, string(b)) - }, - wantPRs: []*PullRequest{ - {Number: github.Ptr(0), HTMLURL: github.Ptr("https://github.com/owner/repo/pull/0"), MergedAt: &github.Timestamp{Time: time.Date(2025, time.September, 5, 20, 44, 59, 0, time.UTC)}, Labels: []*github.Label{{Name: github.Ptr("release:pending")}}}, - }, - }, - { - name: "API error", - handler: func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusInternalServerError) - }, - wantErr: true, - wantErrSubstr: "500", - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - server := httptest.NewServer(test.handler) - defer server.Close() - - repo := &Repository{Owner: "owner", Name: "repo"} - client := newClientWithHTTP("fake-token", repo, server.Client()) - client.BaseURL, _ = url.Parse(server.URL + "/") - - prs, err := client.FindMergedPullRequestsWithPendingReleaseLabel(t.Context(), "owner", "repo") - - if test.wantErr { - if err == nil { - t.Fatalf("FindMergedPullRequestsWithPendingReleaseLabel() err = nil, want error containing %q", test.wantErrSubstr) - } - if !strings.Contains(err.Error(), test.wantErrSubstr) { - t.Errorf("FindMergedPullRequestsWithPendingReleaseLabel() err = %v, want error containing %q", err, test.wantErrSubstr) - } - } else if err != nil { - t.Errorf("FindMergedPullRequestsWithPendingReleaseLabel() err = %v, want nil", err) - } - - if diff := cmp.Diff(test.wantPRs, prs); diff != "" { - t.Errorf("FindMergedPullRequestsWithPendingReleaseLabel() prs mismatch (-want +got):\n%s", diff) - } - }) - } -} -func TestCreateTag(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - tagName string - commitSHA string - handler http.HandlerFunc - wantErr bool - }{ - { - name: "Success", - tagName: "v1.2.3", - commitSHA: "abcdef123456", - handler: func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPost { - t.Errorf("unexpected method: got %s, want %s", r.Method, http.MethodPost) - } - wantPath := "/repos/owner/repo/git/refs" - if r.URL.Path != wantPath { - t.Errorf("unexpected path: got %s, want %s", r.URL.Path, wantPath) - } - - var ref github.Reference - if err := json.NewDecoder(r.Body).Decode(&ref); err != nil { - t.Fatalf("failed to decode request body: %v", err) - } - if ref.Ref == nil || *ref.Ref != "refs/tags/v1.2.3" { - t.Errorf("unexpected ref: got %v, want %s", ref.Ref, "refs/tags/v1.2.3") - } - fmt.Fprint(w, `{"ref": "refs/tags/v1.2.3"}`) - }, - }, - { - name: "API Error", - tagName: "v1.2.3", - commitSHA: "abcdef123456", - handler: func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusInternalServerError) }, - wantErr: true, - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - server := httptest.NewServer(test.handler) - defer server.Close() - - repo := &Repository{Owner: "owner", Name: "repo"} - client := newClientWithHTTP("fake-token", repo, server.Client()) - client.BaseURL, _ = url.Parse(server.URL + "/") - - err := client.CreateTag(t.Context(), test.tagName, test.commitSHA) - - if test.wantErr { - if err == nil { - t.Fatal("CreateTag() err = nil, expected error") - } - } else if err != nil { - t.Errorf("CreateTag() err = %v, want nil", err) - } - }) - } -} - -func TestRetryableTransport(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - handler func(w http.ResponseWriter, r *http.Request, requestCount int) - wantStatusCode int - wantErr bool - wantRequestCount int - wantSlept time.Duration - }{ - { - name: "Success after retries (503)", - handler: func(w http.ResponseWriter, r *http.Request, requestCount int) { - if requestCount < 3 { - w.WriteHeader(http.StatusServiceUnavailable) - } else { - w.WriteHeader(http.StatusOK) - } - }, - wantStatusCode: http.StatusOK, - wantErr: false, - wantRequestCount: 3, - }, - { - name: "Success after retries (500)", - handler: func(w http.ResponseWriter, r *http.Request, requestCount int) { - if requestCount < 3 { - w.WriteHeader(http.StatusInternalServerError) - } else { - w.WriteHeader(http.StatusOK) - } - }, - wantStatusCode: http.StatusOK, - wantErr: false, - wantRequestCount: 3, - }, - { - name: "Success after retries (502)", - handler: func(w http.ResponseWriter, r *http.Request, requestCount int) { - if requestCount < 3 { - w.WriteHeader(http.StatusBadGateway) - } else { - w.WriteHeader(http.StatusOK) - } - }, - wantStatusCode: http.StatusOK, - wantErr: false, - wantRequestCount: 3, - }, - { - name: "Success after retries (429)", - handler: func(w http.ResponseWriter, r *http.Request, requestCount int) { - if requestCount < 2 { - w.Header().Set("Retry-After", "0") - w.WriteHeader(http.StatusTooManyRequests) - } else { - w.WriteHeader(http.StatusOK) - } - }, - wantStatusCode: http.StatusOK, - wantErr: false, - wantRequestCount: 2, - }, - { - name: "Success after retries with capping (Retry-After > 60s)", - handler: func(w http.ResponseWriter, r *http.Request, requestCount int) { - if requestCount < 2 { - w.Header().Set("Retry-After", "120") - w.WriteHeader(http.StatusTooManyRequests) - } else { - w.WriteHeader(http.StatusOK) - } - }, - wantStatusCode: http.StatusOK, - wantErr: false, - wantRequestCount: 2, - wantSlept: 60 * time.Second, - }, - { - name: "Failure after all retries", - handler: func(w http.ResponseWriter, r *http.Request, _ int) { - w.WriteHeader(http.StatusServiceUnavailable) - }, - wantStatusCode: http.StatusServiceUnavailable, - wantErr: true, - wantRequestCount: 5, - }, - } { - t.Run(test.name, func(t *testing.T) { - var requestCount int - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requestCount++ - test.handler(w, r, requestCount) - })) - defer server.Close() - - repo := &Repository{Owner: "owner", Name: "repo"} - httpClient := server.Client() - client := newClientWithHTTP("fake-token", repo, httpClient) - rt, ok := httpClient.Transport.(*retryableTransport) - if !ok { - t.Fatalf("expected transport to be *retryableTransport but got %T", httpClient.Transport) - } - // Mock sleep to verifying delay duration and avoid waiting - var slept time.Duration - rt.sleep = func(d time.Duration) { - slept = d - } - client.BaseURL, _ = url.Parse(server.URL + "/") - - req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, server.URL, nil) - if err != nil { - t.Fatalf("http.NewRequestWithContext() failed: %v", err) - } - resp, err := client.Do(t.Context(), req, nil) - - if test.wantErr { - if err == nil { - t.Fatal("client.Do() err = nil, want error") - } - } else { - if err != nil { - t.Fatalf("client.Do() failed: %v", err) - } - defer resp.Body.Close() - } - - if resp.StatusCode != test.wantStatusCode { - t.Errorf("client.Do() status = %d, want %d", resp.StatusCode, test.wantStatusCode) - } - if requestCount != test.wantRequestCount { - t.Errorf("requestCount = %d, want %d", requestCount, test.wantRequestCount) - } - if test.wantSlept > 0 && slept != test.wantSlept { - t.Errorf("slept = %v, want %v", slept, test.wantSlept) - } - }) - } -} - -func TestNewClient(t *testing.T) { - - t.Parallel() - for _, test := range []struct { - name string - token string - repo *Repository - }{ - { - name: "with credentials", - token: "some-token", - }, - { - name: "with custom repo", - token: "some-token", - repo: &Repository{ - Owner: "some-owner", - Name: "some-repo", - BaseURL: "https://example.com/", - }, - }, - { - name: "with repo, without base url", - token: "some-token", - repo: &Repository{ - Owner: "some-owner", - Name: "some-repo", - }, - }, - } { - t.Run(test.name, func(t *testing.T) { - client := NewClient(test.token, nil) - if client == nil { - t.Fatalf("expected to create a new client") - } - }) - } -} - -func TestBackoff(t *testing.T) { - b := &backoff{ - initial: 2 * time.Second, - max: 10 * time.Second, - multiplier: 2.0, - } - - // Attempt 1 - d1 := b.pause() - if d1 < 1 || d1 > 2*time.Second { - t.Errorf("pause() = %v, want in range [1ns, 2s]", d1) - } - - // Attempt 2 - d2 := b.pause() - if d2 < 1 || d2 > 4*time.Second { - t.Errorf("pause() = %v, want in range [1ns, 4s]", d2) - } - - // Attempt 3 - d3 := b.pause() - if d3 < 1 || d3 > 8*time.Second { - t.Errorf("pause() = %v, want in range [1ns, 8s]", d3) - } - - // Attempt 4 (should hit max) - d4 := b.pause() - if d4 < 1 || d4 > 10*time.Second { - t.Errorf("pause() = %v, want in range [1ns, 10s]", d4) - } - - // Attempt 5 (should still hit max) - d5 := b.pause() - if d5 < 1 || d5 > 10*time.Second { - t.Errorf("pause() = %v, want in range [1ns, 10s]", d5) - } -} - -func TestNewBackoff(t *testing.T) { - b := newBackoff() - if b.initial != retryDelay { - t.Errorf("newBackoff().initial = %v, want %v", b.initial, retryDelay) - } - if b.max != maxDelay { - t.Errorf("newBackoff().max = %v, want %v", b.max, maxDelay) - } - if b.multiplier != multiplier { - t.Errorf("newBackoff().multiplier = %v, want %v", b.multiplier, multiplier) - } -} diff --git a/internal/legacylibrarian/legacygitrepo/conventional_commits.go b/internal/legacylibrarian/legacygitrepo/conventional_commits.go deleted file mode 100644 index e8865949559..00000000000 --- a/internal/legacylibrarian/legacygitrepo/conventional_commits.go +++ /dev/null @@ -1,417 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacygitrepo - -import ( - "encoding/json" - "errors" - "fmt" - "log/slog" - "regexp" - "strings" - "time" -) - -const ( - // beginCommit marks the start of the commit message block. It is used in conjunction with endCommit. - beginCommit = "BEGIN_COMMIT" - // beginCommitOverride marks the start of the commit message block. It is used in conjunction with endCommitOverride. - // Deprecated: use beginCommit instead. - beginCommitOverride = "BEGIN_COMMIT_OVERRIDE" - // endCommit marks the end of the commit message block. - endCommit = "END_COMMIT" - // endCommitOverride marks the end of the commit message block. - // Deprecated: use endCommit instead. - endCommitOverride = "END_COMMIT_OVERRIDE" - beginNestedCommit = "BEGIN_NESTED_COMMIT" - endNestedCommit = "END_NESTED_COMMIT" - breakingChangeKey = "BREAKING CHANGE" - sourceLinkKey = "Source-Link" -) - -var ( - commitRegex = regexp.MustCompile(`^(?P\w+)(?:\((?P.*)\))?(?P!)?:\s?(?P.*)`) - // footerRegex defines the format for a conventional commit footer. - // A footer key consists of letters and hyphens, or is the "BREAKING CHANGE" - // literal. The key is followed by ":" and then the value. - // e.g., "Reviewed-by: G. Gemini" or "BREAKING CHANGE: an API was changed". - footerRegex = regexp.MustCompile(`^([A-Za-z-]+|` + breakingChangeKey + `):(.*)`) - sourceLinkRegex = regexp.MustCompile(`^\[googleapis/googleapis@(?P.*)]\(https://github\.com/googleapis/googleapis/commit/(?P.*)\)$`) - // libraryIDRegex extracts the libraryID from the commit message in a generation PR. - // For a generation PR, each commit is expected to have the libraryID in brackets - // ('[]'). - libraryIDRegex = regexp.MustCompile(`\[([^]]+)]`) -) - -// ErrEmptyCommitMessage returns when the commit message is empty. -var ErrEmptyCommitMessage = errors.New("empty commit message") - -// ConventionalCommit represents a parsed conventional commit message. -// See https://www.conventionalcommits.org/en/v1.0.0/ for details. -type ConventionalCommit struct { - // Type is the type of change (e.g., "feat", "fix", "docs"). - Type string `yaml:"type" json:"type"` - // Subject is the short summary of the change. - Subject string `yaml:"subject" json:"subject"` - // Body is the long-form description of the change. - Body string `yaml:"body" json:"body"` - // LibraryID is the library ID the commit associated with. - LibraryID string `yaml:"-" json:"-"` - // Footers contain metadata (e.g,"BREAKING CHANGE", "Reviewed-by"). - // Repeated footer keys not supported, only first value is kept - Footers map[string]string `yaml:"-" json:"-"` - // IsBreaking indicates if the commit introduces a breaking change. - IsBreaking bool `yaml:"-" json:"-"` - // IsNested indicates if the commit is a nested commit. - IsNested bool `yaml:"-" json:"-"` - // CommitHash is the full commit hash. - CommitHash string `yaml:"-" json:"commit_hash,omitempty"` - // When is the timestamp of the commit. - When time.Time `yaml:"-" json:"-"` -} - -// parsedHeader holds the result of parsing the header line. -type parsedHeader struct { - Type string - Scope string - Description string - IsBreaking bool -} - -// extractLibraryID pulls the text between '[]' as the commit's libraryID -// A non generation PR may not have the associated libraryID between [] and -// will return an empty libraryID. -func (header *parsedHeader) extractLibraryID() string { - matches := libraryIDRegex.FindStringSubmatch(header.Description) - if len(matches) == 0 { - return "" - } - return matches[1] -} - -// commitPart holds the raw string of a commit message and whether it's nested. -type commitPart struct { - message string - isNested bool -} - -// MarshalJSON implements a custom JSON marshaler for ConventionalCommit. -func (c *ConventionalCommit) MarshalJSON() ([]byte, error) { - type Alias ConventionalCommit - return json.Marshal(&struct { - *Alias - PiperCLNumber string `json:"piper_cl_number,omitempty"` - }{ - Alias: (*Alias)(c), - PiperCLNumber: c.Footers["PiperOrigin-RevId"], - }) -} - -// ParseCommits parses a commit message into a slice of ConventionalCommit structs. -// -// It supports a top-level commit wrapped in BEGIN_COMMIT and END_COMMIT (BEGIN_COMMIT_OVERRIDE and -// END_COMMIT_OVERRIDE are also supported for backward compatibility). -// If found, this block takes precedence, and only its content will be parsed. -// -// The message can also contain multiple nested commits, each wrapped in -// BEGIN_NESTED_COMMIT and END_NESTED_COMMIT markers. -// -// Malformed override or nested blocks (e.g., with a missing end marker) are -// ignored. Any commit part that is found but fails to parse as a valid -// conventional commit is logged and skipped. -func ParseCommits(commit *Commit, libraryID string) ([]*ConventionalCommit, error) { - message := commit.Message - if strings.TrimSpace(message) == "" { - return nil, ErrEmptyCommitMessage - } - message = extractBeginCommitMessage(message) - - var commits []*ConventionalCommit - seen := make(map[string]bool) - for _, part := range extractCommitParts(message) { - simpleCommits, err := parseSimpleCommit(part, commit, libraryID) - if err != nil { - slog.Warn("failed to parse commit part", "commit", part.message, "error", err) - continue - } - - for _, simpleCommit := range simpleCommits { - key := fmt.Sprintf("%s,%s", simpleCommit.Subject, simpleCommit.LibraryID) - if _, ok := seen[key]; ok { - continue - } - seen[key] = true - commits = append(commits, simpleCommit) - } - } - - return commits, nil -} - -func extractBeginCommitMessage(message string) string { - // Search the deprecated marker first because beginCommit is the prefix - // of beginCommitOverride. - // TODO: remove usage when we drop support for `BEGIN_COMMIT_OVERRIDE` and `END_COMMIT_OVERRIDE`. - // see https://github.com/googleapis/librarian/issues/2684 - beginMarker := beginCommitOverride - endMarker := endCommitOverride - beginIndex := strings.Index(message, beginMarker) - if beginIndex == -1 { - beginMarker = beginCommit - endMarker = endCommit - beginIndex = strings.Index(message, beginMarker) - } - if beginIndex == -1 { - return message - } - - afterBegin := message[beginIndex+len(beginMarker):] - endIndex := strings.Index(afterBegin, endMarker) - if endIndex == -1 { - return message - } - - return strings.TrimSpace(afterBegin[:endIndex]) -} - -func extractCommitParts(message string) []commitPart { - parts := strings.Split(message, beginNestedCommit) - var commitParts []commitPart - - // The first part is the primary commit. - if len(parts) > 0 && strings.TrimSpace(parts[0]) != "" { - commitParts = append(commitParts, commitPart{ - message: strings.TrimSpace(parts[0]), - isNested: false, - }) - } - - // The rest of the parts are nested commits. - for i := 1; i < len(parts); i++ { - nestedPart := parts[i] - endIndex := strings.Index(nestedPart, endNestedCommit) - if endIndex == -1 { - // Malformed, ignore. - continue - } - commitStr := strings.TrimSpace(nestedPart[:endIndex]) - if commitStr == "" { - continue - } - commitParts = append(commitParts, commitPart{ - message: commitStr, - isNested: true, - }) - } - return commitParts -} - -// parseSimpleCommit parses a simple commit message and returns a slice of ConventionalCommit. -// A simple commit message is commit that does not include override or nested commits. -func parseSimpleCommit(commitPart commitPart, commit *Commit, libraryID string) ([]*ConventionalCommit, error) { - trimmedMessage := strings.TrimSpace(commitPart.message) - if trimmedMessage == "" { - return nil, fmt.Errorf("empty commit message") - } - - lines := strings.Split(trimmedMessage, "\n") - bodyLines, footerLines := separateBodyAndFooters(lines) - footers, footerIsBreaking := parseFooters(footerLines) - processFooters(footers) - - var commits []*ConventionalCommit - // Hold the subjects of each commit. - var subjects [][]string - // Hold the body of each commit. - var body [][]string - // Whether it has seen an empty line. - var foundSeparator bool - // If the body lines have multiple headers, separate them into different conventional commit, all associated with - // the same commit sha. - for _, bodyLine := range bodyLines { - header, ok := parseHeader(bodyLine) - if !ok { - if len(commits) == 0 { - // This should not happen as we expect a conventional commit message inside a nested commit. - slog.Warn("bodyLine is not a header, not in a commit", "bodyLine", bodyLine, "hash", commit.Hash.String()) - continue - } - - bodyLine = strings.TrimSpace(bodyLine) - if bodyLine == "" { - foundSeparator = true - continue - } - - if foundSeparator { - // Since we have seen a separator, the rest of the lines are body lines of the commit. - body[len(body)-1] = append(body[len(body)-1], bodyLine) - } else { - // We haven't seen a separator, this line is the continuation of the title. - subjects[len(subjects)-1] = append(subjects[len(subjects)-1], bodyLine) - } - - continue - } - - subjects = append(subjects, []string{}) - body = append(body, []string{}) - foundSeparator = false - // If there is an association for the commit (i.e. the commit has '[LIBRARY_ID]' in the - // description), then use that libraryID. Otherwise, use the libraryID passed as the default. - headerLibraryID := header.extractLibraryID() - if headerLibraryID != "" { - libraryID = headerLibraryID - } - - commits = append(commits, &ConventionalCommit{ - Type: header.Type, - Subject: header.Description, - LibraryID: libraryID, - Footers: footers, - IsBreaking: header.IsBreaking || footerIsBreaking, - IsNested: commitPart.isNested, - CommitHash: commit.Hash.String(), - When: commit.When, - }) - } - - for i, commit := range commits { - sub := fmt.Sprintf("%s %s", commit.Subject, strings.Join(subjects[i], " ")) - commit.Subject = strings.TrimSpace(sub) - commit.Body = strings.Join(body[i], "\n") - } - - return commits, nil -} - -// parseHeader parses the header line of a commit message. -func parseHeader(headerLine string) (*parsedHeader, bool) { - match := commitRegex.FindStringSubmatch(headerLine) - if len(match) == 0 { - return nil, false - } - - capturesMap := make(map[string]string) - for i, name := range commitRegex.SubexpNames()[1:] { - if name != "" { - capturesMap[name] = match[i+1] - } - } - - return &parsedHeader{ - Type: capturesMap["type"], - Scope: capturesMap["scope"], - Description: capturesMap["description"], - IsBreaking: capturesMap["breaking"] == "!", - }, true -} - -// separateBodyAndFooters splits the lines after the header into body and footer sections. -// It parses lines from the bottom up to identify the true footer section at the end -// of the message, preventing body content (like Renovate release notes) from being -// misidentified as footers. It walks up the last blank line boundary to support -// correctly identifying footers while avoiding consuming nested commit headers. -func separateBodyAndFooters(lines []string) (bodyLines, footerLines []string) { - lastBlankLine := -1 - for i := len(lines) - 1; i >= 0; i-- { - if strings.TrimSpace(lines[i]) == "" { - lastBlankLine = i - break - } - } - - if lastBlankLine == -1 { - return lines, nil - } - - potentialFooters := lines[lastBlankLine+1:] - if len(potentialFooters) == 0 { - return lines, nil - } - - if !footerRegex.MatchString(potentialFooters[0]) { - return lines, nil - } - - // Walk up past blank lines to include preceding footers, stopping if we hit a nested commit header. - boundaryIndex := lastBlankLine - for i := lastBlankLine; i >= 0; i-- { - if strings.TrimSpace(lines[i]) == "" { - if i > 0 && footerRegex.MatchString(lines[i-1]) { - if commitRegex.MatchString(lines[i-1]) { - boundaryIndex = i - break - } - continue - } - boundaryIndex = i - break - } - } - - return lines[:boundaryIndex], lines[boundaryIndex+1:] -} - -// parseFooters parses footer lines from a conventional commit message into a map -// of key-value pairs. It supports multi-line footers and also returns a -// boolean indicating if a breaking change was detected. -func parseFooters(footerLines []string) (footers map[string]string, isBreaking bool) { - footers = make(map[string]string) - var lastKey string - for _, line := range footerLines { - footerMatches := footerRegex.FindStringSubmatch(line) - if len(footerMatches) == 0 { - // Not a new footer. If we have a previous key and the line is not - // empty, append it to the last value. - if lastKey != "" && strings.TrimSpace(line) != "" { - footers[lastKey] += "\n" + line - } - continue - } - // This is a new footer. - key := strings.TrimSpace(footerMatches[1]) - if _, ok := footers[key]; ok { - // Key already exists. Invalidate lastKey to prevent any subsequent - // continuation lines from being appended to the wrong footer. - lastKey = "" - continue - } - value := strings.TrimSpace(footerMatches[2]) - footers[key] = value - lastKey = key - if key == breakingChangeKey { - isBreaking = true - } - } - for key, value := range footers { - footers[key] = strings.TrimSpace(value) - } - return footers, isBreaking -} - -// processFooters format value of certain keys. -func processFooters(footers map[string]string) { - for key, value := range footers { - if key == sourceLinkKey { - // Exact commit sha from the googleapis commit. - matches := sourceLinkRegex.FindStringSubmatch(value) - if len(matches) == 0 { - continue - } - footers[key] = matches[2] - } - } -} diff --git a/internal/legacylibrarian/legacygitrepo/conventional_commits_test.go b/internal/legacylibrarian/legacygitrepo/conventional_commits_test.go deleted file mode 100644 index f556a7f4f2a..00000000000 --- a/internal/legacylibrarian/legacygitrepo/conventional_commits_test.go +++ /dev/null @@ -1,903 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacygitrepo - -import ( - "strings" - "testing" - "time" - - "github.com/go-git/go-git/v5/plumbing" - "github.com/google/go-cmp/cmp" -) - -func TestParseCommits(t *testing.T) { - now := time.Now() - sha := plumbing.NewHash("fake-sha") - for _, test := range []struct { - name string - message string - want []*ConventionalCommit - wantErr bool - wantErrPhrase string - }{ - { - name: "simple_commit_with_no_library_association", - message: "feat: add new feature", - want: []*ConventionalCommit{ - { - Type: "feat", - Subject: "add new feature", - LibraryID: "example-id", - IsNested: false, - Footers: make(map[string]string), - CommitHash: sha.String(), - When: now, - }, - }, - }, - { - name: "simple_commit_with_no_space_separator", - message: "feat:add new feature", - want: []*ConventionalCommit{ - { - Type: "feat", - Subject: "add new feature", - LibraryID: "example-id", - IsNested: false, - Footers: make(map[string]string), - CommitHash: sha.String(), - When: now, - }, - }, - }, - { - name: "simple_commit_with_scope", - message: "feat(scope): add new feature", - want: []*ConventionalCommit{ - { - Type: "feat", - Subject: "add new feature", - LibraryID: "example-id", - IsNested: false, - Footers: make(map[string]string), - CommitHash: sha.String(), - When: now, - }, - }, - }, - { - name: "simple_commit_with_breaking_change", - message: "feat!: add new feature", - want: []*ConventionalCommit{ - { - Type: "feat", - Subject: "add new feature", - LibraryID: "example-id", - IsBreaking: true, - IsNested: false, - Footers: make(map[string]string), - CommitHash: sha.String(), - When: now, - }, - }, - }, - { - name: "commit_with_single_footer", - message: "feat: add new feature\n\nCo-authored-by: John Doe ", - want: []*ConventionalCommit{ - { - Type: "feat", - Subject: "add new feature", - LibraryID: "example-id", - IsNested: false, - Footers: map[string]string{"Co-authored-by": "John Doe "}, - CommitHash: sha.String(), - When: now, - }, - }, - }, - { - name: "commit_with_multiple_footers", - message: "feat: add new feature\n\nCo-authored-by: John Doe \nReviewed-by: Jane Smith ", - want: []*ConventionalCommit{ - { - Type: "feat", - Subject: "add new feature", - LibraryID: "example-id", - IsNested: false, - Footers: map[string]string{ - "Co-authored-by": "John Doe ", - "Reviewed-by": "Jane Smith ", - }, - CommitHash: sha.String(), - When: now, - }, - }, - }, - { - name: "commit_with_multiple_footers_for_generated_changes", - message: "feat: [library-name] add new feature\n\nThis is the body.\n...\n\nPiperOrigin-RevId: piper_cl_number\n\nSource-Link: [googleapis/googleapis@{source_commit_hash}](https://github.com/googleapis/googleapis/commit/abcdefg1234567)", - want: []*ConventionalCommit{ - { - Type: "feat", - Subject: "[library-name] add new feature", - Body: "This is the body.\n...", - LibraryID: "library-name", - IsNested: false, - IsBreaking: false, - Footers: map[string]string{ - "PiperOrigin-RevId": "piper_cl_number", - "Source-Link": "abcdefg1234567", - }, - CommitHash: sha.String(), - When: now, - }, - }, - }, - { - name: "commit_with_breaking_change_footer", - message: "feat: add new feature\n\nBREAKING CHANGE: this is a breaking change", - want: []*ConventionalCommit{ - { - Type: "feat", - Subject: "add new feature", - Body: "", - LibraryID: "example-id", - IsNested: false, - IsBreaking: true, - Footers: map[string]string{"BREAKING CHANGE": "this is a breaking change"}, - CommitHash: sha.String(), - When: now, - }, - }, - }, - { - name: "commit_with_wrong_breaking_change_footer", - message: "feat: add new feature\n\nBreaking change: this is a breaking change", - want: []*ConventionalCommit{ - { - Type: "feat", - Subject: "add new feature", - Body: "Breaking change: this is a breaking change", - LibraryID: "example-id", - IsNested: false, - IsBreaking: false, - Footers: map[string]string{}, - CommitHash: sha.String(), - When: now, - }, - }, - }, - { - name: "commit_with_body_and_footers", - message: "feat: add new feature\n\nThis is the body of the commit message.\nIt can span multiple lines.\n\nCo-authored-by: John Doe ", - want: []*ConventionalCommit{ - { - Type: "feat", - Subject: "add new feature", - Body: "This is the body of the commit message.\nIt can span multiple lines.", - LibraryID: "example-id", - IsNested: false, - Footers: map[string]string{"Co-authored-by": "John Doe "}, - CommitHash: sha.String(), - When: now, - }, - }, - }, - { - name: "commit_with_multi-line_footer", - message: "feat: add new feature\n\nThis is the body.\n\nBREAKING CHANGE: this is a breaking change\nthat spans multiple lines.", - want: []*ConventionalCommit{ - { - Type: "feat", - Subject: "add new feature", - Body: "This is the body.", - LibraryID: "example-id", - IsNested: false, - IsBreaking: true, - Footers: map[string]string{"BREAKING CHANGE": "this is a breaking change\nthat spans multiple lines."}, - CommitHash: sha.String(), - When: now, - }, - }, - }, - { - name: "begin_commit", - message: `feat: original message - -BEGIN_COMMIT -fix(override): this is the override message - -This is the body of the override. - -Reviewed-by: Jane Doe -END_COMMIT`, - want: []*ConventionalCommit{ - { - Type: "fix", - Subject: "this is the override message", - Body: "This is the body of the override.", - LibraryID: "example-id", - IsNested: false, - Footers: map[string]string{"Reviewed-by": "Jane Doe"}, - CommitHash: sha.String(), - When: now, - }, - }, - }, - { - name: "invalid_conventional_commit", - message: "this is not a conventional commit", - wantErr: false, - want: nil, - }, - { - name: "empty_commit_message", - message: "", - wantErr: true, - wantErrPhrase: "empty commit", - }, - { - name: "commit_with_nested_commit", - message: `feat(parser): main feature - -main commit body - -BEGIN_NESTED_COMMIT -fix(sub): fix a bug - -some details for the fix -END_NESTED_COMMIT -BEGIN_NESTED_COMMIT -chore(deps): update deps -END_NESTED_COMMIT -`, - want: []*ConventionalCommit{ - { - Type: "feat", - Subject: "main feature", - Body: "main commit body", - LibraryID: "example-id", - IsNested: false, - Footers: map[string]string{}, - CommitHash: sha.String(), - When: now, - }, - { - Type: "fix", - Subject: "fix a bug", - Body: "some details for the fix", - LibraryID: "example-id", - IsNested: true, - Footers: map[string]string{}, - CommitHash: sha.String(), - When: now, - }, - { - Type: "chore", - Subject: "update deps", - Body: "", - LibraryID: "example-id", - IsNested: true, - Footers: map[string]string{}, - CommitHash: sha.String(), - When: now, - }, - }, - }, - { - name: "commit_with_empty_nested_commit", - message: `feat(parser): main feature -2nd line of title - -BEGIN_NESTED_COMMIT -END_NESTED_COMMIT -`, - want: []*ConventionalCommit{ - { - Type: "feat", - Subject: "main feature 2nd line of title", - LibraryID: "example-id", - IsNested: false, - Footers: map[string]string{}, - CommitHash: sha.String(), - When: now, - }, - }, - }, - { - name: "begin_commit_with_nested_commits", - message: `feat: API regeneration main commit - -This pull request is generated with proto changes between -... ... - -Librarian Version: {librarian_version} -Language Image: {language_image_name_and_digest} - -BEGIN_COMMIT -BEGIN_NESTED_COMMIT -feat: [abc] nested commit 1 - -body of nested commit 1 -... - -PiperOrigin-RevId: 123456 - -Source-Link: fake-link -END_NESTED_COMMIT -BEGIN_NESTED_COMMIT -feat: [abc] nested commit 2 - -body of nested commit 2 -... - -PiperOrigin-RevId: 654321 - -Source-Link: fake-link -END_NESTED_COMMIT -END_COMMIT -`, - want: []*ConventionalCommit{ - { - Type: "feat", - Subject: "[abc] nested commit 1", - Body: "body of nested commit 1\n...", - LibraryID: "abc", - IsNested: true, - Footers: map[string]string{"PiperOrigin-RevId": "123456", "Source-Link": "fake-link"}, - CommitHash: sha.String(), - When: now, - }, - { - Type: "feat", - Subject: "[abc] nested commit 2", - IsNested: true, - Body: "body of nested commit 2\n...", - LibraryID: "abc", - Footers: map[string]string{"PiperOrigin-RevId": "654321", "Source-Link": "fake-link"}, - CommitHash: sha.String(), - When: now, - }, - }, - }, - { - // This test verifies that the deprecated mark, BEGIN_COMMIT_OVERRIDE and END_COMMIT_OVERRIDE - // can be used to separate nested commits. - name: "begin_commit_override_with_nested_commits", - message: `feat: API regeneration main commit - -This pull request is generated with proto changes between -... ... - -Librarian Version: {librarian_version} -Language Image: {language_image_name_and_digest} - -BEGIN_COMMIT_OVERRIDE -BEGIN_NESTED_COMMIT -feat: [abc] nested commit 1 - -body of nested commit 1 -... - -PiperOrigin-RevId: 123456 - -Source-Link: fake-link -END_NESTED_COMMIT -BEGIN_NESTED_COMMIT -feat: [abc] nested commit 2 - -body of nested commit 2 -... - -PiperOrigin-RevId: 654321 - -Source-Link: fake-link -END_NESTED_COMMIT -END_COMMIT_OVERRIDE -`, - want: []*ConventionalCommit{ - { - Type: "feat", - Subject: "[abc] nested commit 1", - Body: "body of nested commit 1\n...", - LibraryID: "abc", - IsNested: true, - Footers: map[string]string{"PiperOrigin-RevId": "123456", "Source-Link": "fake-link"}, - CommitHash: sha.String(), - When: now, - }, - { - Type: "feat", - Subject: "[abc] nested commit 2", - IsNested: true, - Body: "body of nested commit 2\n...", - LibraryID: "abc", - Footers: map[string]string{"PiperOrigin-RevId": "654321", "Source-Link": "fake-link"}, - CommitHash: sha.String(), - When: now, - }, - }, - }, - { - name: "nest_commit_outside_of_begin_commit_ignored", - message: `feat: original message - -BEGIN_NESTED_COMMIT -ignored line -BEGIN_COMMIT -fix(override): this is the override message - -This is the body of the override. - -Reviewed-by: Jane Doe -END_COMMIT -END_NESTED_COMMIT`, - want: []*ConventionalCommit{ - { - Type: "fix", - Subject: "this is the override message", - Body: "This is the body of the override.", - LibraryID: "example-id", - IsNested: false, - Footers: map[string]string{"Reviewed-by": "Jane Doe"}, - CommitHash: sha.String(), - When: now, - }, - }, - }, - { - name: "parse_multiple_lines_message_inside_nested_commit,_one_line_header", - message: ` -chore: Update generation configuration at Tue Aug 26 02:31:23 UTC 2025 (#11734) - -This pull request is generated with proto changes between -[googleapis/googleapis@525c95a](https://github.com/googleapis/googleapis/commit/525c95a7a122ec2869ae06cd02fa5013819463f6) -(exclusive) and -[googleapis/googleapis@b738e78](https://github.com/googleapis/googleapis/commit/b738e78ed63effb7d199ed2d61c9e03291b6077f) -(inclusive). - -BEGIN_COMMIT -BEGIN_NESTED_COMMIT -feat: [texttospeech] Support promptable voices by specifying a model name and a prompt -feat: [texttospeech] Add enum value M4A to enum AudioEncoding -docs: [texttospeech] A comment for method 'StreamingSynthesize' in service 'TextToSpeech' is changed - -PiperOrigin-RevId: 799242210 - -Source-Link: [googleapis/googleapis@b738e78](https://github.com/googleapis/googleapis/commit/b738e78ed63effb7d199ed2d61c9e03291b6077f) -END_NESTED_COMMIT -END_COMMIT`, - want: []*ConventionalCommit{ - { - Type: "feat", - Subject: "[texttospeech] Support promptable voices by specifying a model name and a prompt", - LibraryID: "texttospeech", - IsNested: true, - Footers: map[string]string{ - "PiperOrigin-RevId": "799242210", - "Source-Link": "b738e78ed63effb7d199ed2d61c9e03291b6077f", - }, - CommitHash: sha.String(), - When: now, - }, - { - Type: "feat", - Subject: "[texttospeech] Add enum value M4A to enum AudioEncoding", - LibraryID: "texttospeech", - IsNested: true, - Footers: map[string]string{ - "PiperOrigin-RevId": "799242210", - "Source-Link": "b738e78ed63effb7d199ed2d61c9e03291b6077f", - }, - CommitHash: sha.String(), - When: now, - }, - { - Type: "docs", - Subject: "[texttospeech] A comment for method 'StreamingSynthesize' in service 'TextToSpeech' is changed", - LibraryID: "texttospeech", - IsNested: true, - Footers: map[string]string{ - "PiperOrigin-RevId": "799242210", - "Source-Link": "b738e78ed63effb7d199ed2d61c9e03291b6077f", - }, - CommitHash: sha.String(), - When: now, - }, - }, - }, - { - name: "parse_multiple_lines_message_inside_nested_commit_multi_line_headers", - message: ` -chore: Update generation configuration at Tue Aug 26 02:31:23 UTC 2025 (#11734) - -This pull request is generated with proto changes between -[googleapis/googleapis@525c95a](https://github.com/googleapis/googleapis/commit/525c95a7a122ec2869ae06cd02fa5013819463f6) -(exclusive) and -[googleapis/googleapis@b738e78](https://github.com/googleapis/googleapis/commit/b738e78ed63effb7d199ed2d61c9e03291b6077f) -(inclusive). - -BEGIN_COMMIT -BEGIN_NESTED_COMMIT -feat: [texttospeech] Support promptable voices by specifying a model -name and a prompt -feat: [texttospeech] Add enum value M4A to enum AudioEncoding -docs: [texttospeech] A comment for method 'StreamingSynthesize' in -service 'TextToSpeech' is changed - -END_NESTED_COMMIT -END_COMMIT`, - want: []*ConventionalCommit{ - { - Type: "feat", - Subject: "[texttospeech] Support promptable voices by specifying a model name and a prompt", - LibraryID: "texttospeech", - IsNested: true, - Footers: map[string]string{}, - CommitHash: sha.String(), - When: now, - }, - { - Type: "feat", - Subject: "[texttospeech] Add enum value M4A to enum AudioEncoding", - LibraryID: "texttospeech", - IsNested: true, - Footers: map[string]string{}, - CommitHash: sha.String(), - When: now, - }, - { - Type: "docs", - Subject: "[texttospeech] A comment for method 'StreamingSynthesize' in service 'TextToSpeech' is changed", - LibraryID: "texttospeech", - IsNested: true, - Footers: map[string]string{}, - CommitHash: sha.String(), - When: now, - }, - }, - }, - { - name: "generation_commit_has_incorrect_libraryID_passed_in", - message: ` -chore: librarian generate pull request: 20250919T072957Z (#14501) - -This pull request is generated with proto changes between - -[googleapis/googleapis@f8776fe](googleapis/googleapis@f8776fe) -(exclusive) and - -[googleapis/googleapis@36533b0](googleapis/googleapis@36533b0) -(inclusive). - -BEGIN_COMMIT -BEGIN_NESTED_COMMIT -docs: [google-cloud-video-live-stream] Update requirements of resource ID fields to be more clear - -END_NESTED_COMMIT - -BEGIN_NESTED_COMMIT -feat: [google-cloud-eventarc] add new fields to Eventarc resources - -END_NESTED_COMMIT -END_COMMIT`, - want: []*ConventionalCommit{ - { - Type: "docs", - Subject: "[google-cloud-video-live-stream] Update requirements of resource ID fields to be more clear", - LibraryID: "google-cloud-video-live-stream", - IsNested: true, - Footers: map[string]string{}, - CommitHash: sha.String(), - When: now, - }, - { - Type: "feat", - Subject: "[google-cloud-eventarc] add new fields to Eventarc resources", - LibraryID: "google-cloud-eventarc", - IsNested: true, - Footers: map[string]string{}, - CommitHash: sha.String(), - When: now, - }, - }, - }, - { - name: "nested_commits_with_identical_message", - message: ` -BEGIN_NESTED_COMMIT -fix(abc): update google.golang.org/api to 0.229.0 -END_NESTED_COMMIT -BEGIN_NESTED_COMMIT -fix(def): update google.golang.org/api to 0.229.0 -END_NESTED_COMMIT -BEGIN_NESTED_COMMIT -fix(cba): update google.golang.org/api to 0.229.0 -END_NESTED_COMMIT`, - want: []*ConventionalCommit{ - { - Type: "fix", - Subject: "update google.golang.org/api to 0.229.0", - LibraryID: "example-id", - IsNested: true, - Footers: map[string]string{}, - CommitHash: sha.String(), - When: now, - }, - }, - }, - { - name: "renovate_pr_false_positive", - message: "chore(deps): update dependency aip to v0.83.0\n\n" + - "### Release Notes for aip\n\n" + - "BREAKING CHANGE: The HAS operator on timestamp fields now only accepts...\n\n" + - "This PR was generated by Renovate.", - want: []*ConventionalCommit{ - { - Type: "chore", - Subject: "update dependency aip to v0.83.0", - Body: "### Release Notes for aip\nBREAKING CHANGE: The HAS operator on timestamp fields now only accepts...\nThis PR was generated by Renovate.", - LibraryID: "example-id", - IsNested: false, - IsBreaking: false, - Footers: map[string]string{}, - CommitHash: sha.String(), - When: now, - }, - }, - }, - { - name: "nested_commit_with_breaking_change_footer", - message: "chore: main commit\n\n" + - "BEGIN_NESTED_COMMIT\n" + - "feat: nested feature\n\n" + - "BREAKING CHANGE: nested breaking change\n" + - "END_NESTED_COMMIT", - want: []*ConventionalCommit{ - { - Type: "chore", - Subject: "main commit", - LibraryID: "example-id", - IsNested: false, - Footers: map[string]string{}, - CommitHash: sha.String(), - When: now, - }, - { - Type: "feat", - Subject: "nested feature", - LibraryID: "example-id", - IsNested: true, - IsBreaking: true, - Footers: map[string]string{"BREAKING CHANGE": "nested breaking change"}, - CommitHash: sha.String(), - When: now, - }, - }, - }, - } { - t.Run(test.name, func(t *testing.T) { - commit := &Commit{ - Message: test.message, - Hash: plumbing.NewHash("fake-sha"), - When: now, - } - got, err := ParseCommits(commit, "example-id") - if test.wantErr { - if err == nil { - t.Fatalf("%s should return error", test.name) - } - if !strings.Contains(err.Error(), test.wantErrPhrase) { - t.Errorf("ParseCommits(%q) returned error %q, want to contain %q", test.message, err.Error(), test.wantErrPhrase) - } - return - } - if err != nil { - t.Fatal(err) - } - if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestExtractCommitParts(t *testing.T) { - for _, test := range []struct { - name string - message string - want []commitPart - }{ - { - name: "empty message", - message: "", - want: nil, - }, - { - name: "no nested commits", - message: "feat: hello world", - want: []commitPart{ - {message: "feat: hello world", isNested: false}, - }, - }, - { - name: "only nested commits", - message: `BEGIN_NESTED_COMMIT -fix(sub): fix a bug -END_NESTED_COMMIT -BEGIN_NESTED_COMMIT -chore(deps): update deps -END_NESTED_COMMIT -`, - want: []commitPart{ - {message: "fix(sub): fix a bug", isNested: true}, - {message: "chore(deps): update deps", isNested: true}, - }, - }, - { - name: "primary and nested commits", - message: `feat(parser): main feature -BEGIN_NESTED_COMMIT -fix(sub): fix a bug -END_NESTED_COMMIT -`, - want: []commitPart{ - {message: "feat(parser): main feature", isNested: false}, - {message: "fix(sub): fix a bug", isNested: true}, - }, - }, - { - name: "malformed nested commit without end marker, with primary commit", - message: `feat(parser): main feature -BEGIN_NESTED_COMMIT -fix(sub): fix a bug that is never closed`, - want: []commitPart{ - {message: "feat(parser): main feature", isNested: false}, - }, - }, - { - name: "malformed nested commit without end marker, without primary commit", - message: `BEGIN_NESTED_COMMIT -fix(sub): fix a bug that is never closed`, - want: nil, - }, - } { - t.Run(test.name, func(t *testing.T) { - got := extractCommitParts(test.message) - if diff := cmp.Diff(test.want, got, cmp.AllowUnexported(commitPart{})); diff != "" { - t.Errorf("mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestConventionalCommit_MarshalJSON(t *testing.T) { - c := &ConventionalCommit{ - Type: "feat", - Subject: "new feature", - Body: "body of feature", - Footers: map[string]string{ - "PiperOrigin-RevId": "12345", - }, - CommitHash: "1234", - } - b, err := c.MarshalJSON() - if err != nil { - t.Fatalf("MarshalJSON() failed: %v", err) - } - want := `{"type":"feat","subject":"new feature","body":"body of feature","commit_hash":"1234","piper_cl_number":"12345"}` - if diff := cmp.Diff(want, string(b)); diff != "" { - t.Errorf("MarshalJSON() mismatch (-want +got):\n%s", diff) - } -} - -func TestParseFooters(t *testing.T) { - for _, test := range []struct { - name string - footerLines []string - wantFooters map[string]string - wantIsBreaking bool - }{ - { - name: "single footer", - footerLines: []string{ - "Reviewed-by: G. Gemini", - }, - wantFooters: map[string]string{ - "Reviewed-by": "G. Gemini", - }, - }, - { - name: "multiple footers", - footerLines: []string{ - "Reviewed-by: G. Gemini", - "Co-authored-by: Another Person ", - }, - wantFooters: map[string]string{ - "Reviewed-by": "G. Gemini", - "Co-authored-by": "Another Person ", - }, - }, - { - name: "repeated footer keys, keep first", - footerLines: []string{ - "PiperOrigin-RevId: 123456", - "Source-Link: first value", - "Source-Link: second value", - }, - wantFooters: map[string]string{ - "PiperOrigin-RevId": "123456", - "Source-Link": "first value", - }, - }, - { - name: "multiline footer", - footerLines: []string{ - "BREAKING CHANGE: something broke", - " and it was a big deal", - }, - wantFooters: map[string]string{ - "BREAKING CHANGE": "something broke\n and it was a big deal", - }, - wantIsBreaking: true, - }, - { - name: "empty lines", - footerLines: []string{ - "Reviewed-by: G. Gemini", - "", - "", - "Co-authored-by: Another Person ", - }, - wantFooters: map[string]string{ - "Reviewed-by": "G. Gemini", - "Co-authored-by": "Another Person ", - }, - }, - { - name: "multi-line footers with key on one line, value on the next", - footerLines: []string{ - "PiperOrigin-RevId: 123456", - "Library-IDs:", - "library-one,library-two", - "Source-Link:", - "", - "googleapis/googleapis@a12b345", - "", - "", - "Copy-Tag:", - "eyJwIjoic", - }, - wantFooters: map[string]string{ - "PiperOrigin-RevId": "123456", - "Library-IDs": "library-one,library-two", - "Source-Link": "googleapis/googleapis@a12b345", - "Copy-Tag": "eyJwIjoic", - }, - }, - } { - t.Run(test.name, func(t *testing.T) { - gotFooters, gotIsBreaking := parseFooters(test.footerLines) - if diff := cmp.Diff(test.wantFooters, gotFooters); diff != "" { - t.Errorf("parseFooters() footers mismatch (-want +got):%s", diff) - } - if gotIsBreaking != test.wantIsBreaking { - t.Errorf("parseFooters() isBreaking = %v, want %v", gotIsBreaking, test.wantIsBreaking) - } - }) - } -} diff --git a/internal/legacylibrarian/legacygitrepo/gitrepo.go b/internal/legacylibrarian/legacygitrepo/gitrepo.go deleted file mode 100644 index 938eaf59595..00000000000 --- a/internal/legacylibrarian/legacygitrepo/gitrepo.go +++ /dev/null @@ -1,780 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package legacygitrepo provides operations on git repos. -package legacygitrepo - -import ( - "errors" - "fmt" - "io/fs" - "log/slog" - "net/url" - "os" - "os/exec" - "path/filepath" - "strings" - "time" - - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/config" - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/object" - "github.com/go-git/go-git/v5/plumbing/transport" - httpAuth "github.com/go-git/go-git/v5/plumbing/transport/http" - "github.com/go-git/go-git/v5/plumbing/transport/ssh" -) - -// ErrNoModificationsToCommit is returned when a commit is attempted on a clean worktree. -var ErrNoModificationsToCommit = errors.New("no modifications to commit") - -// Repository defines the interface for git repository operations. -type Repository interface { - AddAll() error - Commit(msg string) error - IsClean() (bool, error) - Remotes() ([]*Remote, error) - GetDir() string - HeadHash() (string, error) - ChangedFilesInCommit(commitHash string) ([]string, error) - ChangedFiles() ([]string, error) - GetCommit(commitHash string) (*Commit, error) - GetLatestCommit(path string) (*Commit, error) - GetCommitsForPathsSinceTag(paths []string, tagName string) ([]*Commit, error) - GetCommitsForPathsSinceCommit(paths []string, sinceCommit string) ([]*Commit, error) - CreateBranchAndCheckout(name string) error - CheckoutCommitAndCreateBranch(name, commitHash string) error - NewAndDeletedFiles() ([]string, error) - Push(branchName string) error - Restore(paths []string) error - CleanUntracked(paths []string) error - pushRefSpec(refSpec string) error - Checkout(commitHash string) error - GetHashForPath(commitHash, path string) (string, error) - ResetHard() error - DeleteLocalBranches(names []string) error - ResetSoft(commit string) error -} - -const RootPath = "." - -// LocalRepository represents a git repository. -type LocalRepository struct { - Dir string - repo *git.Repository - gitPassword string -} - -// Commit represents a git commit. -type Commit struct { - Hash plumbing.Hash - Message string - When time.Time -} - -// Remote represent a git remote. -type Remote struct { - Name string - URLs []string -} - -// RepositoryOptions are used to configure a [LocalRepository]. -type RepositoryOptions struct { - // Dir is the directory where the repository will reside locally. Required. - Dir string - // MaybeClone will try to clone the repository if it does not exist locally. - // If set to true, RemoteURL and RemoteBranch must also be set. Optional. - MaybeClone bool - // RemoteURL is the URL of the remote repository to clone from. Required if MaybeClone is set to true. - RemoteURL string - // RemoteBranch is the remote branch to clone. Required if MaybeClone is set to true. - RemoteBranch string - // CI is the type of Continuous Integration (CI) environment in which - // the tool is executing. - CI string - // GitPassword is used for HTTP basic auth. - GitPassword string - // Depth controls the cloning depth if the repository needs to be cloned. - Depth int -} - -// NewRepository provides access to a git repository based on the provided options. -// -// If opts.Clone is CloneOptionNone, it opens an existing repository at opts.Dir. -// If opts.Clone is CloneOptionMaybe, it opens the repository if it exists, -// otherwise it clones from opts.RemoteURL. -// If opts.Clone is CloneOptionAlways, it always clones from opts.RemoteURL. -func NewRepository(opts *RepositoryOptions) (*LocalRepository, error) { - repo, err := newRepositoryWithoutUser(opts) - if err != nil { - return repo, err - } - repo.gitPassword = opts.GitPassword - return repo, nil -} - -func newRepositoryWithoutUser(opts *RepositoryOptions) (*LocalRepository, error) { - if opts.Dir == "" { - return nil, errors.New("gitrepo: dir is required") - } - - if !opts.MaybeClone { - return open(opts.Dir) - } - slog.Info("checking for repository", "dir", opts.Dir) - _, err := os.Stat(opts.Dir) - if err == nil { - return open(opts.Dir) - } - if errors.Is(err, fs.ErrNotExist) { - if opts.RemoteURL == "" { - return nil, fmt.Errorf("gitrepo: remote URL is required when cloning") - } - if opts.RemoteBranch == "" { - return nil, fmt.Errorf("gitrepo: remote branch is required when cloning") - } - slog.Info("repository not found, executing clone") - return clone(opts.Dir, opts.RemoteURL, opts.RemoteBranch, opts.CI, opts.Depth) - } - return nil, fmt.Errorf("failed to check for repository at %q: %w", opts.Dir, err) -} - -func open(dir string) (*LocalRepository, error) { - slog.Info("opening repository", "dir", dir) - repo, err := git.PlainOpen(dir) - if err != nil { - return nil, err - } - - return &LocalRepository{ - Dir: dir, - repo: repo, - }, nil -} - -func clone(dir, url, branch, ci string, depth int) (*LocalRepository, error) { - slog.Info("cloning repository", "url", url, "dir", dir) - options := &git.CloneOptions{ - URL: url, - ReferenceName: plumbing.NewBranchReferenceName(branch), - SingleBranch: true, - Tags: git.AllTags, - Depth: depth, - // .NET uses submodules for conformance tests. - // (There may be other examples too.) - RecurseSubmodules: git.DefaultSubmoduleRecursionDepth, - } - if ci == "" { - options.Progress = os.Stdout // When not a CI build, output progress. - } - - repo, err := git.PlainClone(dir, false, options) - if err != nil { - return nil, err - } - return &LocalRepository{ - Dir: dir, - repo: repo, - }, nil -} - -// AddAll adds all pending changes from the working tree to the index, -// so that the changes can later be committed. -func (r *LocalRepository) AddAll() error { - worktree, err := r.repo.Worktree() - if err != nil { - return err - } - err = worktree.AddWithOptions(&git.AddOptions{All: true}) - if err != nil { - return err - } - return nil -} - -// Commit creates a new commit with the provided message and author -// information. -func (r *LocalRepository) Commit(msg string) error { - slog.Info("committing", "message", msg) - worktree, err := r.repo.Worktree() - if err != nil { - return err - } - - status, err := worktree.Status() - if err != nil { - return err - } - if status.IsClean() { - return ErrNoModificationsToCommit - } - // The author of the commit will be read from git config. - hash, err := worktree.Commit(msg, &git.CommitOptions{}) - if err != nil { - return err - } - - slog.Info("committed", "short_hash", hash.String()[0:7], "subject", strings.Split(msg, "\n")[0]) - return nil -} - -// IsClean reports whether the working tree has no uncommitted changes. -func (r *LocalRepository) IsClean() (bool, error) { - worktree, err := r.repo.Worktree() - if err != nil { - return false, err - } - status, err := worktree.Status() - if err != nil { - return false, err - } - - return status.IsClean(), nil -} - -// ChangedFiles returns a list of files that have been modified, added, or deleted -// in the working tree, including both staged and unstaged changes. -func (r *LocalRepository) ChangedFiles() ([]string, error) { - slog.Debug("getting changed files") - worktree, err := r.repo.Worktree() - if err != nil { - return nil, err - } - status, err := worktree.Status() - if err != nil { - return nil, err - } - var changedFiles []string - for file, fileStatus := range status { - if fileStatus.Staging != git.Unmodified || fileStatus.Worktree != git.Unmodified { - changedFiles = append(changedFiles, file) - } - } - return changedFiles, nil -} - -// NewAndDeletedFiles returns a list of files that are new or deleted. -func (r *LocalRepository) NewAndDeletedFiles() ([]string, error) { - slog.Debug("getting new and deleted files") - worktree, err := r.repo.Worktree() - if err != nil { - return nil, err - } - status, err := worktree.Status() - if err != nil { - return nil, err - } - var files []string - for file, fileStatus := range status { - switch { - case fileStatus.Worktree == git.Untracked, - fileStatus.Staging == git.Added, - fileStatus.Worktree == git.Deleted, - fileStatus.Staging == git.Deleted: - files = append(files, file) - } - } - return files, nil -} - -// Remotes returns the remotes within the repository. -func (r *LocalRepository) Remotes() ([]*Remote, error) { - gitRemotes, err := r.repo.Remotes() - if err != nil { - return nil, err - } - var remotes []*Remote - for _, remote := range gitRemotes { - remotes = append(remotes, &Remote{Name: remote.Config().Name, URLs: remote.Config().URLs}) - } - - return remotes, nil -} - -// HeadHash returns hash of the commit for the repository's HEAD. -func (r *LocalRepository) HeadHash() (string, error) { - ref, err := r.repo.Head() - if err != nil { - return "", err - } - return ref.Hash().String(), nil -} - -// GetDir returns the directory of the repository. -func (r *LocalRepository) GetDir() string { - return r.Dir -} - -// GetCommit returns a commit for the given commit hash. -func (r *LocalRepository) GetCommit(commitHash string) (*Commit, error) { - commit, err := r.repo.CommitObject(plumbing.NewHash(commitHash)) - if err != nil { - return nil, err - } - - return &Commit{ - Hash: commit.Hash, - Message: commit.Message, - When: commit.Author.When, - }, nil -} - -// GetLatestCommit returns the latest commit of the given path in the repository. -func (r *LocalRepository) GetLatestCommit(path string) (*Commit, error) { - slog.Info("retrieving the latest commit", "path", path) - opt := &git.LogOptions{ - Order: git.LogOrderCommitterTime, - FileName: &path, - } - log, err := r.repo.Log(opt) - if err != nil { - return nil, err - } - - commit, err := log.Next() - if err != nil { - return nil, err - } - - return &Commit{ - Hash: commit.Hash, - Message: commit.Message, - When: commit.Author.When, - }, nil -} - -// GetCommitsForPathsSinceTag returns all commits since tagName that contains -// files in paths. -// -// If tagName empty, all commits for the given paths are returned. -func (r *LocalRepository) GetCommitsForPathsSinceTag(paths []string, tagName string) ([]*Commit, error) { - var hash string - if tagName == "" { - return r.GetCommitsForPathsSinceCommit(paths, "") - } - tagRef, err := r.repo.Tag(tagName) - if err != nil { - return nil, fmt.Errorf("failed to find tag %s: %w", tagName, err) - } - - tagCommit, err := r.repo.CommitObject(tagRef.Hash()) - if err != nil { - return nil, fmt.Errorf("failed to get commit object for tag %s: %w", tagName, err) - } - hash = tagCommit.Hash.String() - - return r.GetCommitsForPathsSinceCommit(paths, hash) -} - -// GetCommitsForPathsSinceCommit returns the commits affecting any of the given -// paths, stopping looking at the given commit (which is not included in the -// results). -// The returned commits are ordered such that the most recent commit is first. -// -// If sinceCommit is not provided, all commits are searched; otherwise, if no -// commit matching sinceCommit is found, an error is returned. -func (r *LocalRepository) GetCommitsForPathsSinceCommit(paths []string, sinceCommit string) ([]*Commit, error) { - if len(paths) == 0 { - return nil, errors.New("no paths to check for commits") - } - var commits []*Commit - finalHash := plumbing.NewHash(sinceCommit) - logOptions := git.LogOptions{Order: git.LogOrderCommitterTime} - logIterator, err := r.repo.Log(&logOptions) - if err != nil { - return nil, err - } - // Sentinel "error" - this can be replaced using LogOptions.To when that's available. - ErrStopIterating := fmt.Errorf("iteration done") - err = logIterator.ForEach(func(commit *object.Commit) error { - if commit.Hash == finalHash { - return ErrStopIterating - } - // Skips the initial commit as it has no parents. - // This is a known limitation that should be addressed in the future. - // Skip any commit with multiple parents. We shouldn't see this - // as we don't use merge commits. - if commit.NumParents() != 1 { - return nil - } - parentCommit, err := commit.Parent(0) - if err != nil { - return err - } - - // We perform filtering by finding out if the tree hash for the given - // path at the commit we're looking at is the same as the tree hash - // for the commit's parent. - // This is much, much faster than any other filtering option, it seems. - // In theory, we should be able to remember our "current" commit for each - // path, but that's likely to be significantly more complex. - for _, candidatePath := range paths { - matching, err := commitMatchesPath(candidatePath, commit, parentCommit) - if err != nil { - return err - } - // If we've found a change (including a path being added or removed), - // add it to our list of commits and proceed to the next commit. - if matching { - commits = append(commits, &Commit{ - Hash: commit.Hash, - Message: commit.Message, - When: commit.Author.When, - }) - return nil - } - } - - return nil - }) - if err != nil && !errors.Is(err, ErrStopIterating) { - return nil, err - } - if sinceCommit != "" && !errors.Is(err, ErrStopIterating) { - return nil, fmt.Errorf("did not find commit %s when iterating", sinceCommit) - } - return commits, nil -} - -func commitMatchesPath(path string, commit *object.Commit, parentCommit *object.Commit) (bool, error) { - if path == RootPath { - return true, nil - } - currentPathHash, err := getHashForPath(commit, path) - if err != nil { - return false, err - } - parentPathHash, err := getHashForPath(parentCommit, path) - if err != nil { - return false, err - } - return currentPathHash != parentPathHash, nil -} - -// getHashForPath returns the hash for a path at a given commit, or an -// empty string if the path (file or directory) did not exist. -func getHashForPath(commit *object.Commit, path string) (string, error) { - tree, err := commit.Tree() - if err != nil { - return "", err - } - treeEntry, err := tree.FindEntry(path) - if errors.Is(err, object.ErrEntryNotFound) || errors.Is(err, object.ErrDirectoryNotFound) { - return "", nil - } - if err != nil { - return "", err - } - return treeEntry.Hash.String(), nil -} - -// ChangedFilesInCommit returns the files changed in the given commit. -func (r *LocalRepository) ChangedFilesInCommit(commitHash string) ([]string, error) { - slog.Debug("getting changed files in commit", "hash", commitHash) - commit, err := r.repo.CommitObject(plumbing.NewHash(commitHash)) - if err != nil { - return nil, fmt.Errorf("failed to get commit object for hash %s: %w", commitHash, err) - } - - var fromTree *object.Tree - var toTree *object.Tree - - toTree, err = commit.Tree() - if err != nil { - return nil, fmt.Errorf("failed to get tree for commit %s: %w", commitHash, err) - } - - if commit.NumParents() == 0 { - fromTree = &object.Tree{} // Empty tree for initial commit - } else { - parent, err := commit.Parent(0) - if err != nil { - return nil, fmt.Errorf("failed to get parent for commit %s: %w", commitHash, err) - } - fromTree, err = parent.Tree() - if err != nil { - return nil, fmt.Errorf("failed to get parent tree for commit %s: %w", commitHash, err) - } - } - - changes, err := fromTree.Diff(toTree) - if err != nil { - return nil, fmt.Errorf("failed to get diff for commit %s: %w", commitHash, err) - } - var files []string - for _, change := range changes { - from := change.From.Name - to := change.To.Name - // Deletion or modification - if from != "" { - files = append(files, from) - } - // Insertion or rename - if to != "" && from != to { - files = append(files, to) - } - } - return files, nil -} - -// CreateBranchAndCheckout creates a new git branch and checks out the -// branch in the local git repository. -func (r *LocalRepository) CreateBranchAndCheckout(name string) error { - slog.Info("creating branch and checking out", "name", name) - worktree, err := r.repo.Worktree() - if err != nil { - return err - } - return worktree.Checkout(&git.CheckoutOptions{ - Branch: plumbing.NewBranchReferenceName(name), - Create: true, - Keep: true, - }) -} - -// CheckoutCommitAndCreateBranch creates a new git branch from a specific commit hash -// and checks out the branch in the local git repository. -func (r *LocalRepository) CheckoutCommitAndCreateBranch(name, commitHash string) error { - slog.Debug("creating branch from commit and checking out", "name", name, "commit", commitHash) - worktree, err := r.repo.Worktree() - if err != nil { - return err - } - return worktree.Checkout(&git.CheckoutOptions{ - Hash: plumbing.NewHash(commitHash), - Branch: plumbing.NewBranchReferenceName(name), - Create: true, - }) -} - -// Push pushes the local branch to the origin remote. -func (r *LocalRepository) Push(branchName string) error { - // https://stackoverflow.com/a/75727620 - refSpec := fmt.Sprintf("+refs/heads/%s:refs/heads/%s", branchName, branchName) - slog.Info("pushing changes", "branch name", branchName, slog.Any("refspec", refSpec)) - return r.pushRefSpec(refSpec) -} - -// DeleteBranch deletes a branch on the origin remote. -func (r *LocalRepository) DeleteBranch(branchName string) error { - refSpec := fmt.Sprintf(":refs/heads/%s", branchName) - return r.pushRefSpec(refSpec) -} - -func (r *LocalRepository) pushRefSpec(refSpec string) error { - slog.Info("pushing changes", "refSpec", refSpec) - - // Check for the configured URI for the `origin` remote. - // If there are multiple URLs, the first one is selected. - var remoteURI string - remotes, err := r.Remotes() - if err != nil { - return err - } - for _, remote := range remotes { - if remote.Name == "origin" { - if len(remote.URLs) > 0 { - remoteURI = remote.URLs[0] - } - } - } - - useSSH := canUseSSH(remoteURI) - // While cloning a public repo does not require any authCreds, pushing - // to the repo requires authentication and verification of identity - auth, err := r.authCreds(useSSH) - if err != nil { - return err - } - if err := r.repo.Push(&git.PushOptions{ - RemoteName: "origin", - RefSpecs: []config.RefSpec{config.RefSpec(refSpec)}, - Auth: auth, - }); err != nil { - return err - } - slog.Info("successfully pushed changes", "refSpec", refSpec) - return nil -} - -// canUseSSH returns if the remote URI can connect via https ssh. It attempts to -// automatically determine the type and returns false as a default if it's unable -// to make a determination. -func canUseSSH(remoteURI string) bool { - // First, try to parse it as a standard URL - // e.g. "https://github.com/golang/go.git", "ssh://git@github.com/golang/go.git" - parsedURL, err := url.Parse(remoteURI) - if err == nil && parsedURL.Scheme != "" { - // Check the scheme directly - switch parsedURL.Scheme { - case "https": - return false - case "ssh": - return true - } - } - - // If parsing fails or the scheme is not standard, check for the `user@hostname` - // SSH syntax (e.g., "git@github.com:user/repo.git"). This format doesn't - // have a "://" and contains a ":" - if !strings.Contains(remoteURI, "://") && strings.Contains(remoteURI, ":") { - return true - } - - return false -} - -// authCreds returns the configured AuthMethod to used to pushing to the -// remote repository. The useSSH determines if Basic Auth or SSH is used. -func (r *LocalRepository) authCreds(useSSH bool) (transport.AuthMethod, error) { - if useSSH { - slog.Info("authenticating with SSH") - // This is the generic `git` username when cloning via SSH. It is the value - // that exists before the URL. e.g. git@github.com:googleapis/librarian.git - auth, err := ssh.DefaultAuthBuilder("git") - if err != nil { - return nil, err - } - return auth, nil - } - slog.Info("authenticating with basic auth") - return &httpAuth.BasicAuth{ - // GitHub's authentication needs the username set to a non-empty value, but - // it does not need to match the token - Username: "cloud-sdk-librarian", - Password: r.gitPassword, - }, nil -} - -// Restore restores changes in the working tree, leaving staged area untouched. -// Note that untracked files, if any, are not touched. -// -// Wrap git operations in exec, because [git.Worktree.Restore] does not support -// this operation. -func (r *LocalRepository) Restore(paths []string) error { - args := []string{"restore"} - args = append(args, paths...) - slog.Info("restoring uncommitted changes", "paths", strings.Join(paths, ",")) - cmd := exec.Command("git", args...) - cmd.Stderr = os.Stderr - cmd.Stdout = os.Stdout - cmd.Dir = r.Dir - return cmd.Run() -} - -// CleanUntracked removes untracked files within the given paths. -func (r *LocalRepository) CleanUntracked(paths []string) error { - slog.Info("cleaning untracked files", "paths", strings.Join(paths, ",")) - worktree, err := r.repo.Worktree() - if err != nil { - return err - } - - status, err := worktree.Status() - if err != nil { - return err - } - - // Remove an untracked file if it lives within one of the given paths. - for _, path := range paths { - for file, fileStatue := range status { - if !strings.Contains(file, path) || fileStatue.Worktree != git.Untracked { - continue - } - - relPath := filepath.Join(r.Dir, file) - if err := os.Remove(relPath); err != nil { - return fmt.Errorf("failed to remove untracked file, %s: %q", relPath, err) - } - } - } - - return nil -} - -// Checkout checks out the local repository at the provided git SHA. -func (r *LocalRepository) Checkout(commitSha string) error { - worktree, err := r.repo.Worktree() - if err != nil { - return err - } - return worktree.Checkout(&git.CheckoutOptions{ - Hash: plumbing.NewHash(commitSha), - }) -} - -// GetHashForPath returns a tree hash for the specified path, -// at the given commit in this repository. If the path does not exist -// at the commit, an empty string is returned rather than an error, -// as the purpose of this function is to allow callers to determine changes -// in the tree. (A path going from missing to anything else, or vice versa, -// indicates a change. A path being missing at two different commits is not a change.) -func (r *LocalRepository) GetHashForPath(commitHash, path string) (string, error) { - // This public function just delegates to the internal function that uses a Commit - // object instead of the hash. - commit, err := r.repo.CommitObject(plumbing.NewHash(commitHash)) - if err != nil { - return "", err - } - return getHashForPath(commit, path) -} - -// ResetHard resets the repository to HEAD, discarding all local changes. -func (r *LocalRepository) ResetHard() error { - worktree, err := r.repo.Worktree() - if err != nil { - return err - } - return worktree.Reset(&git.ResetOptions{ - Mode: git.HardReset, - }) -} - -// DeleteLocalBranches deletes a list of local branches. -// It returns an error if any branch deletion fails, or nil if all succeed. -func (r *LocalRepository) DeleteLocalBranches(names []string) error { - slog.Debug("starting batch deletion of local branches", "count", len(names)) - headRef, headErr := r.repo.Head() - if headErr != nil && !errors.Is(headErr, plumbing.ErrReferenceNotFound) { - return fmt.Errorf("failed to get HEAD to protect against its deletion: %w", headErr) - } - for _, name := range names { - refName := plumbing.NewBranchReferenceName(name) - _, err := r.repo.Storer.Reference(refName) - if err != nil { - return fmt.Errorf("failed to check existence of branch %s: %w", name, err) - } - if headErr == nil && headRef.Name() == refName { - return fmt.Errorf("cannot delete branch %s: it is the currently checked out branch (HEAD)", name) - } - if err := r.repo.Storer.RemoveReference(refName); err != nil { - return fmt.Errorf("failed to delete branch %s: %w", name, err) - } - } - return nil -} - -// ResetSoft resets the current branch head to a specific commit but leaves the -// working tree and index untouched. -func (r *LocalRepository) ResetSoft(commit string) error { - worktree, err := r.repo.Worktree() - if err != nil { - return err - } - hash, err := r.repo.ResolveRevision(plumbing.Revision(commit)) - if err != nil { - return fmt.Errorf("failed to resolve revision for soft reset: %w", err) - } - return worktree.Reset(&git.ResetOptions{Commit: *hash, Mode: git.SoftReset}) -} diff --git a/internal/legacylibrarian/legacygitrepo/gitrepo_test.go b/internal/legacylibrarian/legacygitrepo/gitrepo_test.go deleted file mode 100644 index 30a58667d9b..00000000000 --- a/internal/legacylibrarian/legacygitrepo/gitrepo_test.go +++ /dev/null @@ -1,2133 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacygitrepo - -import ( - "errors" - "fmt" - "io/fs" - "os" - "path/filepath" - "slices" - "strings" - "testing" - "time" - - "github.com/go-git/go-git/v5/plumbing" - "github.com/google/go-cmp/cmp/cmpopts" - - "github.com/go-git/go-git/v5" - goGitConfig "github.com/go-git/go-git/v5/config" - "github.com/go-git/go-git/v5/plumbing/object" - "github.com/google/go-cmp/cmp" -) - -func TestNewRepository(t *testing.T) { - t.Parallel() - tmpDir := t.TempDir() - remoteDir := filepath.Join(tmpDir, "remote") - if err := os.Mkdir(remoteDir, 0755); err != nil { - t.Fatal(err) - } - remoteRepo, err := git.PlainInit(remoteDir, false) - if err != nil { - t.Fatal(err) - } - w, err := remoteRepo.Worktree() - if err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(remoteDir, "README.md"), []byte("test"), 0644); err != nil { - t.Fatal(err) - } - if _, err := w.Add("README.md"); err != nil { - t.Fatal(err) - } - if _, err := w.Commit("initial commit", &git.CommitOptions{ - Author: &object.Signature{Name: "Test", Email: "test@example.com"}, - }); err != nil { - t.Fatal(err) - } - - for _, test := range []struct { - name string - opts *RepositoryOptions - wantDir string - wantErr bool - initGit bool - setup func(t *testing.T) (cleanup func()) - }{ - { - name: "no dir", - opts: &RepositoryOptions{}, - wantErr: true, - }, - { - name: "open existing", - opts: &RepositoryOptions{ - Dir: tmpDir, - }, - wantDir: tmpDir, - initGit: true, - }, - { - name: "open existing not valid git dir", - opts: &RepositoryOptions{ - Dir: filepath.Join(tmpDir, "non-git-dir"), - }, - wantErr: true, - setup: func(t *testing.T) func() { - if err := os.Mkdir(filepath.Join(tmpDir, "non-git-dir"), 0755); err != nil { - t.Fatalf("failed to create test dir: %v", err) - } - return func() {} - }, - }, - { - name: "clone maybe", - opts: &RepositoryOptions{ - Dir: filepath.Join(tmpDir, "clone-maybe"), - MaybeClone: true, - RemoteURL: remoteDir, - RemoteBranch: "master", - }, - wantDir: filepath.Join(tmpDir, "clone-maybe"), - }, - { - name: "maybe clone with existing repo", - opts: &RepositoryOptions{ - Dir: filepath.Join(tmpDir, "existing-repo"), - MaybeClone: true, - }, - wantDir: filepath.Join(tmpDir, "existing-repo"), - initGit: true, - }, - { - name: "clone maybe no remote url", - opts: &RepositoryOptions{ - Dir: filepath.Join(tmpDir, "clone-maybe-no-remote"), - MaybeClone: true, - RemoteBranch: "main", - }, - wantErr: true, - }, - { - name: "clone maybe no remote branch", - opts: &RepositoryOptions{ - Dir: filepath.Join(tmpDir, "clone-maybe-no-remote"), - MaybeClone: true, - RemoteURL: remoteDir, - }, - wantErr: true, - }, - { - name: "stat error", - opts: &RepositoryOptions{ - Dir: filepath.Join(tmpDir, "unreadable/repo"), - MaybeClone: true, - }, - wantErr: true, - setup: func(t *testing.T) func() { - unreadableDir := filepath.Join(tmpDir, "unreadable") - if err := os.Mkdir(unreadableDir, 0000); err != nil { - t.Fatalf("os.Mkdir() failed: %v", err) - } - return func() { - if err := os.Chmod(unreadableDir, 0755); err != nil { - t.Logf("failed to restore permissions on %s: %v", unreadableDir, err) - } - } - }, - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - if test.setup != nil { - cleanup := test.setup(t) - defer cleanup() - } - if test.initGit { - if _, err := git.PlainInit(test.opts.Dir, false); err != nil { - t.Fatal(err) - } - } - got, err := NewRepository(test.opts) - if (err != nil) != test.wantErr { - t.Errorf("NewRepository() error = %v, wantErr %v", err, test.wantErr) - return - } - if err != nil { - return - } - if got.Dir != test.wantDir { - t.Errorf("NewRepository() got = %v, want %v", got.Dir, test.wantDir) - } - }) - } -} - -func TestIsClean(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - setup func(t *testing.T, dir string, w *git.Worktree) - wantClean bool - }{ - { - name: "initial state is clean", - setup: func(t *testing.T, dir string, w *git.Worktree) {}, - wantClean: true, - }, - { - name: "untracked file is not clean", - setup: func(t *testing.T, dir string, w *git.Worktree) { - filePath := filepath.Join(dir, "untracked.txt") - if err := os.WriteFile(filePath, []byte("test"), 0644); err != nil { - t.Fatalf("failed to write file: %v", err) - } - }, - wantClean: false, - }, - { - name: "added file is not clean", - setup: func(t *testing.T, dir string, w *git.Worktree) { - filePath := filepath.Join(dir, "added.txt") - if err := os.WriteFile(filePath, []byte("test"), 0644); err != nil { - t.Fatalf("failed to write file: %v", err) - } - if _, err := w.Add("added.txt"); err != nil { - t.Fatalf("failed to add file: %v", err) - } - }, - wantClean: false, - }, - { - name: "committed file is clean", - setup: func(t *testing.T, dir string, w *git.Worktree) { - filePath := filepath.Join(dir, "committed.txt") - if err := os.WriteFile(filePath, []byte("test"), 0644); err != nil { - t.Fatalf("failed to write file: %v", err) - } - if _, err := w.Add("committed.txt"); err != nil { - t.Fatalf("failed to add file: %v", err) - } - _, err := w.Commit("commit", &git.CommitOptions{ - Author: &object.Signature{Name: "Test", Email: "test@example.com"}, - }) - if err != nil { - t.Fatalf("failed to commit: %v", err) - } - }, - wantClean: true, - }, - { - name: "modified file is not clean", - setup: func(t *testing.T, dir string, w *git.Worktree) { - // First, commit a file. - filePath := filepath.Join(dir, "modified.txt") - if err := os.WriteFile(filePath, []byte("initial"), 0644); err != nil { - t.Fatalf("failed to write file: %v", err) - } - if _, err := w.Add("modified.txt"); err != nil { - t.Fatalf("failed to add file: %v", err) - } - _, err := w.Commit("commit", &git.CommitOptions{ - Author: &object.Signature{Name: "Test", Email: "test@example.com"}, - }) - if err != nil { - t.Fatalf("failed to commit: %v", err) - } - - // Now modify it. - if err := os.WriteFile(filePath, []byte("modified"), 0644); err != nil { - t.Fatalf("failed to write file: %v", err) - } - }, - wantClean: false, - }, - { - name: "deleted file is not clean", - setup: func(t *testing.T, dir string, w *git.Worktree) { - // First, commit a file. - filePath := filepath.Join(dir, "deleted.txt") - if err := os.WriteFile(filePath, []byte("initial"), 0644); err != nil { - t.Fatalf("failed to write file: %v", err) - } - if _, err := w.Add("deleted.txt"); err != nil { - t.Fatalf("failed to add file: %v", err) - } - _, err := w.Commit("commit", &git.CommitOptions{ - Author: &object.Signature{Name: "Test", Email: "test@example.com"}, - }) - if err != nil { - t.Fatalf("failed to commit: %v", err) - } - - // Now delete it. - if err := os.Remove(filePath); err != nil { - t.Fatalf("failed to remove file: %v", err) - } - }, - wantClean: false, - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - repo, dir := initTestRepo(t) - w, err := repo.Worktree() - if err != nil { - t.Fatalf("failed to get worktree: %v", err) - } - - r := &LocalRepository{ - Dir: dir, - repo: repo, - } - - test.setup(t, dir, w) - gotClean, err := r.IsClean() - if err != nil { - t.Fatalf("IsClean() returned an error: %v", err) - } - - if gotClean != test.wantClean { - t.Errorf("IsClean() = %v, want %v", gotClean, test.wantClean) - } - }) - } -} - -func TestChangedFiles(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - setup func(t *testing.T, dir string, w *git.Worktree) - wantFiles []string - wantErr bool - wantErrMsg string - }{ - { - name: "clean repository", - setup: func(t *testing.T, dir string, w *git.Worktree) { - // No changes - }, - }, - { - name: "untracked file", - setup: func(t *testing.T, dir string, w *git.Worktree) { - filePath := filepath.Join(dir, "untracked.txt") - if err := os.WriteFile(filePath, []byte("test"), 0644); err != nil { - t.Fatalf("failed to write file: %v", err) - } - }, - wantFiles: []string{"untracked.txt"}, - }, - { - name: "added file", - setup: func(t *testing.T, dir string, w *git.Worktree) { - filePath := filepath.Join(dir, "added.txt") - if err := os.WriteFile(filePath, []byte("test"), 0644); err != nil { - t.Fatalf("failed to write file: %v", err) - } - if _, err := w.Add("added.txt"); err != nil { - t.Fatalf("failed to add file: %v", err) - } - }, - wantFiles: []string{"added.txt"}, - }, - { - name: "modified file", - setup: func(t *testing.T, dir string, w *git.Worktree) { - filePath := filepath.Join(dir, "modified.txt") - if err := os.WriteFile(filePath, []byte("initial"), 0644); err != nil { - t.Fatalf("failed to write file: %v", err) - } - if _, err := w.Add("modified.txt"); err != nil { - t.Fatalf("failed to add file: %v", err) - } - _, err := w.Commit("commit", &git.CommitOptions{ - Author: &object.Signature{Name: "Test", Email: "test@example.com"}, - }) - if err != nil { - t.Fatalf("failed to commit: %v", err) - } - - if err := os.WriteFile(filePath, []byte("modified"), 0644); err != nil { - t.Fatalf("failed to write file: %v", err) - } - }, - wantFiles: []string{"modified.txt"}, - }, - { - name: "deleted file", - setup: func(t *testing.T, dir string, w *git.Worktree) { - filePath := filepath.Join(dir, "deleted.txt") - if err := os.WriteFile(filePath, []byte("initial"), 0644); err != nil { - t.Fatalf("failed to write file: %v", err) - } - if _, err := w.Add("deleted.txt"); err != nil { - t.Fatalf("failed to add file: %v", err) - } - _, err := w.Commit("commit", &git.CommitOptions{ - Author: &object.Signature{Name: "Test", Email: "test@example.com"}, - }) - if err != nil { - t.Fatalf("failed to commit: %v", err) - } - if err := os.Remove(filePath); err != nil { - t.Fatalf("failed to remove file: %v", err) - } - }, - wantFiles: []string{"deleted.txt"}, - }, - { - name: "multiple changes", - setup: func(t *testing.T, dir string, w *git.Worktree) { - // Untracked - untrackedFilePath := filepath.Join(dir, "untracked.txt") - if err := os.WriteFile(untrackedFilePath, []byte("test"), 0644); err != nil { - t.Fatalf("failed to write file: %v", err) - } - - // Modified - modifiedFilePath := filepath.Join(dir, "modified.txt") - if err := os.WriteFile(modifiedFilePath, []byte("initial"), 0644); err != nil { - t.Fatalf("failed to write file: %v", err) - } - if _, err := w.Add("modified.txt"); err != nil { - t.Fatalf("failed to add file: %v", err) - } - _, err := w.Commit("commit", &git.CommitOptions{ - Author: &object.Signature{Name: "Test", Email: "test@example.com"}, - }) - if err != nil { - t.Fatalf("failed to commit: %v", err) - } - if err := os.WriteFile(modifiedFilePath, []byte("modified"), 0644); err != nil { - t.Fatalf("failed to write file: %v", err) - } - }, - wantFiles: []string{"modified.txt", "untracked.txt"}, - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - repo, dir := initTestRepo(t) - w, err := repo.Worktree() - if err != nil { - t.Fatalf("failed to get worktree: %v", err) - } - - r := &LocalRepository{ - Dir: dir, - repo: repo, - } - - test.setup(t, dir, w) - gotFiles, err := r.ChangedFiles() - if (err != nil) != test.wantErr { - t.Errorf("ChangedFiles() error = %v, wantErr %v", err, test.wantErr) - return - } - if test.wantErr { - if !strings.Contains(err.Error(), test.wantErrMsg) { - t.Errorf("ChangedFiles() error message = %q, want to contain %q", err.Error(), test.wantErrMsg) - } - return - } - - slices.Sort(gotFiles) // Sorting makes the test deterministic. - if diff := cmp.Diff(test.wantFiles, gotFiles); diff != "" { - t.Errorf("ChangedFiles() mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestAddAll(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - setup func(t *testing.T, dir string) - wantStatusIsClean bool - wantErr bool - }{ - { - name: "add a new file", - setup: func(t *testing.T, dir string) { - filePath := filepath.Join(dir, "new_file.txt") - if err := os.WriteFile(filePath, []byte("test content"), 0644); err != nil { - t.Fatalf("failed to write file: %v", err) - } - }, - wantStatusIsClean: false, - }, - { - name: "no files to add", - setup: func(t *testing.T, dir string) { - // Do nothing, repo is clean. - }, - wantStatusIsClean: true, - }, - { - name: "add unreadable file", - setup: func(t *testing.T, dir string) { - filePath := filepath.Join(dir, "unreadable_file.txt") - if err := os.WriteFile(filePath, []byte("test content"), 0644); err != nil { - t.Fatalf("failed to write file: %v", err) - } - // Make file unreadable to cause an error during `git add`. - if err := os.Chmod(filePath, 0222); err != nil { - t.Fatalf("failed to chmod file: %v", err) - } - }, - wantErr: true, - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - gogitRepo, dir := initTestRepo(t) - r := &LocalRepository{ - Dir: dir, - repo: gogitRepo, - } - - test.setup(t, dir) - - err := r.AddAll() - if (err != nil) != test.wantErr { - t.Errorf("AddAll() error = %v, wantErr %v", err, test.wantErr) - return - } - if err != nil { - return - } - isClean, err := r.IsClean() - if err != nil { - t.Fatalf("IsClean() returned an error: %v", err) - } - if isClean != test.wantStatusIsClean { - t.Errorf("AddAll() status.IsClean() = %v, want %v", isClean, test.wantStatusIsClean) - } - }) - } - -} - -func TestCommit(t *testing.T) { - t.Parallel() - name, email := "tester", "tester@example.com" - // setupRepo is a helper to create a repository with an initial commit. - setupRepo := func(t *testing.T) *LocalRepository { - t.Helper() - goGitRepo, dir := initTestRepo(t) - - author := struct { - Name string - Email string - }{ - Name: name, - Email: email, - } - config, err := goGitRepo.Config() - if err != nil { - t.Fatalf("gitRepo.Config failed: %v", err) - } - config.User = author - if err := goGitRepo.SetConfig(config); err != nil { - t.Fatalf("gitRepo.SetConfig failed: %v", err) - } - - w, err := goGitRepo.Worktree() - if err != nil { - t.Fatalf("Worktree() failed: %v", err) - } - if _, err := w.Commit("initial commit", &git.CommitOptions{ - AllowEmptyCommits: true, - Author: &object.Signature{Name: "Test", Email: "test@example.com"}, - }); err != nil { - t.Fatalf("initial commit failed: %v", err) - } - return &LocalRepository{Dir: dir, repo: goGitRepo} - } - - for _, test := range []struct { - name string - setup func(t *testing.T) *LocalRepository - commitMsg string - wantErr bool - wantErrMsg string - check func(t *testing.T, repo *LocalRepository, commitMsg string) - }{ - { - name: "successful commit", - setup: func(t *testing.T) *LocalRepository { - repo := setupRepo(t) - // Add a file to be committed. - filePath := filepath.Join(repo.Dir, "new.txt") - if err := os.WriteFile(filePath, []byte("content"), 0644); err != nil { - t.Fatalf("os.WriteFile failed: %v", err) - } - w, err := repo.repo.Worktree() - if err != nil { - t.Fatalf("Worktree() failed: %v", err) - } - if _, err := w.Add("new.txt"); err != nil { - t.Fatalf("w.Add failed: %v", err) - } - return repo - }, - commitMsg: "feat: add new file", - check: func(t *testing.T, repo *LocalRepository, commitMsg string) { - head, err := repo.repo.Head() - if err != nil { - t.Fatalf("repo.repo.Head() failed: %v", err) - } - commit, err := repo.repo.CommitObject(head.Hash()) - if err != nil { - t.Fatalf("repo.repo.CommitObject() failed: %v", err) - } - if commit.Message != commitMsg { - t.Errorf("Commit() message = %q, want %q", commit.Message, commitMsg) - } - author := commit.Author - if author.Name != "tester" { - t.Errorf("Commit() author name = %q, want %q", author.Name, "tester") - } - if author.Email != "tester@example.com" { - t.Errorf("Commit() author email = %q, want %q", author.Email, "tester@example.com") - } - }, - }, - { - name: "clean repository", - setup: func(t *testing.T) *LocalRepository { - return setupRepo(t) - }, - commitMsg: "no-op", - wantErr: true, - wantErrMsg: "no modifications to commit", - }, - { - name: "worktree error", - setup: func(t *testing.T) *LocalRepository { - dir := t.TempDir() - // Create a bare repository which has no worktree. - goGitRepo, err := git.PlainInit(dir, true) - if err != nil { - t.Fatalf("git.PlainInit failed: %v", err) - } - return &LocalRepository{Dir: dir, repo: goGitRepo} - }, - commitMsg: "any message", - wantErr: true, - wantErrMsg: "worktree not available", - }, - { - name: "status error", - setup: func(t *testing.T) *LocalRepository { - repo := setupRepo(t) - // Add a file to make the worktree dirty. - filePath := filepath.Join(repo.Dir, "new.txt") - if err := os.WriteFile(filePath, []byte("content"), 0644); err != nil { - t.Fatalf("os.WriteFile failed: %v", err) - } - w, err := repo.repo.Worktree() - if err != nil { - t.Fatalf("Worktree() failed: %v", err) - } - if _, err := w.Add("new.txt"); err != nil { - t.Fatalf("w.Add failed: %v", err) - } - - // Make the worktree unreadable to cause worktree.Status() to fail. - if err := os.Chmod(repo.Dir, 0000); err != nil { - t.Fatalf("os.Chmod failed: %v", err) - } - t.Cleanup(func() { - if err := os.Chmod(repo.Dir, 0755); err != nil { - t.Logf("failed to restore permissions: %v", err) - } - }) - return repo - }, - commitMsg: "any message", - wantErr: true, - wantErrMsg: "permission denied", - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - repo := test.setup(t) - - err := repo.Commit(test.commitMsg) - - if test.wantErr { - if err == nil { - t.Fatalf("Commit() expected error, got nil") - } - if test.wantErrMsg != "" && !strings.Contains(err.Error(), test.wantErrMsg) { - t.Errorf("Commit() error = %q, want to contain %q", err.Error(), test.wantErrMsg) - } - return - } - - if err != nil { - t.Fatalf("Commit() unexpected error = %v", err) - } - - if test.check != nil { - test.check(t, repo, test.commitMsg) - } - }) - } -} - -func TestRemotes(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - setupRemotes map[string][]string - wantErr bool - }{ - { - name: "no remotes", - setupRemotes: map[string][]string{}, - }, - { - name: "single remote", - setupRemotes: map[string][]string{ - "origin": {"https://github.com/test/repo.git"}, - }, - }, - { - name: "multiple remotes with multiple URLs", - setupRemotes: map[string][]string{ - "origin": {"https://github.com/test/origin.git"}, - "upstream": {"https://github.com/test/upstream.git", "git@github.com:test/upstream.git"}, - }, - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - gogitRepo, dir := initTestRepo(t) - - for name, urls := range test.setupRemotes { - if _, err := gogitRepo.CreateRemote(&goGitConfig.RemoteConfig{ - Name: name, - URLs: urls, - }); err != nil { - t.Fatalf("CreateRemote failed: %v", err) - } - } - - repo := &LocalRepository{Dir: dir, repo: gogitRepo} - got, err := repo.Remotes() - if (err != nil) != test.wantErr { - t.Errorf("Remotes() error = %v, wantErr %v", err, test.wantErr) - } - - gotRemotes := make(map[string][]string) - for _, r := range got { - gotRemotes[r.Name] = r.URLs - } - if diff := cmp.Diff(test.setupRemotes, gotRemotes); diff != "" { - t.Errorf("Remotes() mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestGetCommit(t *testing.T) { - t.Parallel() - setup := func(t *testing.T, dir string) string { - gitRepo, err := git.PlainInit(dir, false) - if err != nil { - t.Fatalf("git.PlainInit failed: %v", err) - } - w, err := gitRepo.Worktree() - if err != nil { - t.Fatalf("Worktree() failed: %v", err) - } - if err := os.WriteFile(filepath.Join(dir, "README.md"), []byte("test"), 0644); err != nil { - t.Fatal(err) - } - if _, err := w.Add("README.md"); err != nil { - t.Fatal(err) - } - commitHash, err := w.Commit("initial commit", &git.CommitOptions{ - Author: &object.Signature{ - Name: "Test", - Email: "test@example.com", - When: time.Now(), - }, - }) - if err != nil { - t.Fatal(err) - } - - return commitHash.String() - } - - for _, test := range []struct { - name string - commitHash string - want *Commit - wantErr bool - wantErrMsg string - }{ - { - name: "get a commit", - want: &Commit{ - Message: "initial commit", - }, - }, - { - name: "failed to get a commit", - commitHash: "wrong-sha", - wantErr: true, - wantErrMsg: "object not found", - }, - } { - t.Run(test.name, func(t *testing.T) { - dir := t.TempDir() - commitHash := setup(t, dir) - if test.commitHash != "" { - commitHash = test.commitHash - } - - repo, err := NewRepository(&RepositoryOptions{Dir: dir}) - if err != nil { - t.Error(err) - } - - got, err := repo.GetCommit(commitHash) - if test.wantErr { - if err == nil { - t.Fatal("GetCommit() should fail") - } - if !strings.Contains(err.Error(), test.wantErrMsg) { - t.Errorf("want error message %s, got %s", test.wantErrMsg, err.Error()) - } - - return - } - if err != nil { - t.Fatalf("GetCommit() failed: %v", err) - } - - test.want.Hash = plumbing.NewHash(commitHash) - if diff := cmp.Diff(test.want, got, cmpopts.IgnoreFields(Commit{}, "When")); diff != "" { - t.Errorf("GetDir() mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestHeadHash(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - setup func(t *testing.T, dir string) - wantErr bool - }{ - { - name: "success", - setup: func(t *testing.T, dir string) { - gitRepo, err := git.PlainInit(dir, false) - if err != nil { - t.Fatalf("git.PlainInit failed: %v", err) - } - w, err := gitRepo.Worktree() - if err != nil { - t.Fatalf("Worktree() failed: %v", err) - } - if err := os.WriteFile(filepath.Join(dir, "README.md"), []byte("test"), 0644); err != nil { - t.Fatal(err) - } - if _, err := w.Add("README.md"); err != nil { - t.Fatal(err) - } - if _, err := w.Commit("initial commit", &git.CommitOptions{ - Author: &object.Signature{Name: "Test", Email: "test@example.com"}, - }); err != nil { - t.Fatal(err) - } - }, - }, - { - name: "error", - setup: func(t *testing.T, dir string) { - if _, err := git.PlainInit(dir, false); err != nil { - t.Fatalf("git.PlainInit failed: %v", err) - } - }, - wantErr: true, - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - dir := t.TempDir() - test.setup(t, dir) - repo, err := NewRepository(&RepositoryOptions{Dir: dir}) - if err != nil { - t.Fatalf("NewRepository() failed: %v", err) - } - _, err = repo.HeadHash() - if (err != nil) != test.wantErr { - t.Errorf("HeadHash() error = %v, wantErr %v", err, test.wantErr) - } - }) - } -} -func TestGetDir(t *testing.T) { - t.Parallel() - want := "/test/dir" - repo := &LocalRepository{ - Dir: want, - } - - got := repo.GetDir() - if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("GetDir() mismatch (-want +got):\n%s", diff) - } -} - -// TestGetHashForPath tests the internal getHashForPath, but -// via the public GetHashForPath function which accepts a commit hash -// instead of a Commit object, to avoid duplicate testing. -func TestGetHashForPath(t *testing.T) { - t.Parallel() - - setupInitialRepo := func(t *testing.T) (LocalRepository, *object.Commit) { - t.Helper() - repo, dir := initTestRepo(t) - commit := createAndCommit(t, repo, "initial.txt", []byte("initial content"), "initial commit") - localRepository := LocalRepository{ - Dir: dir, - repo: repo, - } - return localRepository, commit - } - - for _, test := range []struct { - name string - setup func(t *testing.T) (repo LocalRepository, commit *object.Commit, path string) - wantHash func(commit *object.Commit, path string) string - wantErr bool - }{ - { - name: "existing file", - setup: func(t *testing.T) (LocalRepository, *object.Commit, string) { - localRepository, commit := setupInitialRepo(t) - return localRepository, commit, "initial.txt" - }, - wantHash: func(commit *object.Commit, path string) string { - tree, err := commit.Tree() - if err != nil { - t.Fatalf("failed to get tree: %v", err) - } - entry, err := tree.FindEntry(path) - if err != nil { - t.Fatalf("failed to find entry for path %q: %v", path, err) - } - return entry.Hash.String() - }, - }, - { - name: "existing directory", - setup: func(t *testing.T) (LocalRepository, *object.Commit, string) { - localRepository, _ := setupInitialRepo(t) - repo := localRepository.repo - // Create a directory and a file inside it to ensure the directory gets a hash - _ = createAndCommit(t, repo, "my_dir/file_in_dir.txt", []byte("content of file in dir"), "add dir and file") - head, err := repo.Head() - if err != nil { - t.Fatalf("repo.Head failed: %v", err) - } - commit, err := repo.CommitObject(head.Hash()) - if err != nil { - t.Fatalf("repo.CommitObject failed: %v", err) - } - return localRepository, commit, "my_dir" - }, - wantHash: func(commit *object.Commit, path string) string { - tree, err := commit.Tree() - if err != nil { - t.Fatalf("failed to get tree: %v", err) - } - entry, err := tree.FindEntry(path) - if err != nil { - t.Fatalf("failed to find entry for path %q: %v", path, err) - } - return entry.Hash.String() - }, - }, - { - name: "non-existent file", - setup: func(t *testing.T) (LocalRepository, *object.Commit, string) { - localRepository, commit := setupInitialRepo(t) - return localRepository, commit, "non_existent_file.txt" - }, - wantHash: func(commit *object.Commit, path string) string { - return "" - }, - }, - { - name: "non-existent directory", - setup: func(t *testing.T) (LocalRepository, *object.Commit, string) { - localRepository, commit := setupInitialRepo(t) - return localRepository, commit, "non_existent_dir" - }, - wantHash: func(commit *object.Commit, path string) string { - return "" - }, - }, - { - name: "file in subdirectory", - setup: func(t *testing.T) (LocalRepository, *object.Commit, string) { - localRepository, _ := setupInitialRepo(t) - repo := localRepository.repo - _ = createAndCommit(t, repo, "another_dir/sub_dir/nested_file.txt", []byte("nested content"), "add nested file") - head, err := repo.Head() - if err != nil { - t.Fatalf("repo.Head failed: %v", err) - } - commit, err := repo.CommitObject(head.Hash()) - if err != nil { - t.Fatalf("repo.CommitObject failed: %v", err) - } - return localRepository, commit, "another_dir/sub_dir/nested_file.txt" - }, - wantHash: func(commit *object.Commit, path string) string { - tree, err := commit.Tree() - if err != nil { - t.Fatalf("failed to get tree: %v", err) - } - entry, err := tree.FindEntry(path) - if err != nil { - t.Fatalf("failed to find entry for path %q: %v", path, err) - } - return entry.Hash.String() - }, - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - - localRepository, commit, path := test.setup(t) - - got, err := localRepository.GetHashForPath(commit.Hash.String(), path) - if (err != nil) != test.wantErr { - t.Errorf("getHashForPath() error = %v, wantErr %v", err, test.wantErr) - return - } - - wantHash := test.wantHash(commit, path) - if diff := cmp.Diff(wantHash, got); diff != "" { - t.Errorf("getHashForPath() mismatch (-want +got):\n%s", diff) - } - }) - } -} - -// TestGetHashForPathBadCommitHash tests the one path not -// otherwise tested in TestGetHashForPath, where we can't -// get the commit for the hash. -func TestGetHashForPathBadCommitHash(t *testing.T) { - repo, dir := initTestRepo(t) - localRepository := LocalRepository{ - Dir: dir, - repo: repo, - } - for _, test := range []struct { - name string - commitHash string - }{ - { - name: "empty hash", - commitHash: "", - }, - { - name: "invalid hash", - commitHash: "bad-hash", - }, - { - name: "hash not in repo", - commitHash: "d93e160f57f0a6eccd6e230dd40f465988bede63", - }, - } { - t.Run(test.name, func(t *testing.T) { - _, err := localRepository.GetHashForPath(test.commitHash, "path/to/file") - if err == nil { - t.Error("GetHashForPath() err = nil, should fail when an invalid or absent hash is provided") - } - }) - } -} - -func TestChangedFilesInCommit(t *testing.T) { - t.Parallel() - r, commitHashes := setupRepoForChangedFilesTest(t) - - for _, test := range []struct { - name string - commitHash string - wantFiles []string - wantErr bool - }{ - { - name: "commit 1", - commitHash: commitHashes["commit 1"], - wantFiles: []string{"file1.txt"}, - }, - { - name: "commit 2", - commitHash: commitHashes["commit 2"], - wantFiles: []string{"file1.txt"}, - }, - { - name: "commit 3", - commitHash: commitHashes["commit 3"], - wantFiles: []string{"file2.txt"}, - }, - { - name: "commit 4", - commitHash: commitHashes["commit 4"], - wantFiles: []string{"file2.txt"}, - }, - { - name: "commit 5", - commitHash: commitHashes["commit 5"], - wantFiles: []string{"file1.txt", "file3.txt"}, - }, - { - name: "invalid commit hash", - commitHash: "invalid", - wantErr: true, - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - gotFiles, err := r.ChangedFilesInCommit(test.commitHash) - if (err != nil) != test.wantErr { - t.Errorf("ChangedFilesInCommit() error = %v, wantErr %v", err, test.wantErr) - return - } - if diff := cmp.Diff(test.wantFiles, gotFiles); diff != "" { - t.Errorf("ChangedFilesInCommit() mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestGetCommitsForPathsSinceCommit(t *testing.T) { - t.Parallel() - - repo, commits := setupRepoForGetCommitsTest(t) - - for _, test := range []struct { - name string - paths []string - tagName string - sinceCommit string - wantCommits []string - wantErr bool - wantErrPhrase string - }{ - { - name: "one path, one commit", - paths: []string{"file2.txt"}, - sinceCommit: commits["commit1"], - wantCommits: []string{"feat: commit 2"}, - }, - { - name: "all paths, all commits", - paths: []string{"file1.txt", "file2.txt", "file3.txt"}, - sinceCommit: "", - // The current implementation skips the initial commit. - wantCommits: []string{"feat: commit 3", "feat: commit 2"}, - }, - { - name: "no matching commits", - paths: []string{"non-existent-file.txt"}, - sinceCommit: "", - wantCommits: []string{}, - }, - { - name: "no paths specified", - paths: []string{}, - tagName: "v1.0.0", - wantCommits: []string{}, - wantErr: true, - wantErrPhrase: "no paths to check for commits", - }, - { - name: "since commit not found", - paths: []string{"file1.txt"}, - sinceCommit: "nonexistenthash", - wantCommits: []string{}, - wantErr: true, - wantErrPhrase: "did not find commit", - }, - { - name: "root path matches all commits", - paths: []string{"."}, - sinceCommit: "", - // The current implementation skips the initial commit. - wantCommits: []string{"feat: commit 3", "feat: commit 2"}, - }, - } { - - t.Run(test.name, func(t *testing.T) { - var ( - gotCommits []*Commit - err error - ) - - gotCommits, err = repo.GetCommitsForPathsSinceCommit(test.paths, test.sinceCommit) - - if (err != nil) != test.wantErr { - t.Errorf("GetCommitsForPathsSinceCommit() error = %v, wantErr %v", err, test.wantErr) - return - } - - if test.wantErr { - if err == nil { - t.Fatalf("%s should return error", test.name) - } - if !strings.Contains(err.Error(), test.wantErrPhrase) { - t.Errorf("GetCommitsForPathsSinceCommit() returned error %q, want to contain %q", err.Error(), test.wantErrPhrase) - } - return - } - if err != nil { - t.Fatal(err) - } - - gotCommitMessages := []string{} - for _, c := range gotCommits { - gotCommitMessages = append(gotCommitMessages, strings.Split(c.Message, "\n")[0]) - } - - if diff := cmp.Diff(test.wantCommits, gotCommitMessages); diff != "" { - t.Errorf("GetCommitsForPathsSinceCommit() mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestGetCommitsForPathsSinceTag(t *testing.T) { - t.Parallel() - repo, _ := setupRepoForGetCommitsTest(t) - - for _, test := range []struct { - name string - paths []string - tagName string - sinceCommit string - wantCommits []string - wantErr bool - wantErrPhrase string - }{ - { - name: "all paths, multiple commits", - paths: []string{"file2.txt", "file3.txt"}, - tagName: "v1.0.0", - wantCommits: []string{"feat: commit 3", "feat: commit 2"}, - }, - { - name: "invalid tag", - paths: []string{"file2.txt"}, - tagName: "non-existent-tag", - wantCommits: []string{}, - wantErr: true, - wantErrPhrase: "failed to find tag", - }, - } { - - t.Run(test.name, func(t *testing.T) { - var ( - gotCommits []*Commit - err error - ) - gotCommits, err = repo.GetCommitsForPathsSinceTag(test.paths, test.tagName) - - if test.wantErr { - if err == nil { - t.Fatalf("%s should return error", test.name) - } - if !strings.Contains(err.Error(), test.wantErrPhrase) { - t.Errorf("GetCommitsForPathsSinceTag() returned error %q, want to contain %q", err.Error(), test.wantErrPhrase) - } - return - } - if err != nil { - t.Fatal(err) - } - - gotCommitMessages := []string{} - for _, c := range gotCommits { - gotCommitMessages = append(gotCommitMessages, strings.Split(c.Message, "\n")[0]) - } - - if diff := cmp.Diff(test.wantCommits, gotCommitMessages); diff != "" { - t.Errorf("GetCommitsForPathsSinceTag() mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestCreateBranchAndCheckout(t *testing.T) { - for _, test := range []struct { - name string - branchName string - wantErr bool - wantErrPhrase string - }{ - { - name: "works", - branchName: "test-branch", - }, - { - name: "invalid branch name", - branchName: "invalid branch name", - wantErr: true, - wantErrPhrase: "invalid", - }, - } { - t.Run(test.name, func(t *testing.T) { - repo, _ := setupRepoForGetCommitsTest(t) - err := repo.CreateBranchAndCheckout(test.branchName) - if test.wantErr { - if err == nil { - t.Fatalf("%s should return error", test.name) - } - if !strings.Contains(err.Error(), test.wantErrPhrase) { - t.Errorf("CreateBranchAndCheckout() returned error %q, want to contain %q", err.Error(), test.wantErrPhrase) - } - return - } - if err != nil { - t.Fatal(err) - } - head, _ := repo.repo.Head() - if diff := cmp.Diff(test.branchName, head.Name().Short()); diff != "" { - t.Errorf("CreateBranchAndCheckout() mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestRestore(t *testing.T) { - for _, test := range []struct { - name string - paths []string - wantErr bool - wantErrPhrase string - }{ - { - name: "restore files in paths", - paths: []string{ - "first/path", - "second/path", - }, - }, - } { - t.Run(test.name, func(t *testing.T) { - repo, dir := initTestRepo(t) - localRepo := &LocalRepository{ - Dir: dir, - repo: repo, - } - // Create files in test.paths and commit the change. - for _, path := range test.paths { - file := filepath.Join(path, "example.txt") - createAndCommit(t, repo, file, []byte("old content"), fmt.Sprintf("commit path, %s", path)) - } - - // Change file contents. - for _, path := range test.paths { - file := filepath.Join(dir, path, "example.txt") - if err := os.WriteFile(file, []byte("new content"), 0755); err != nil { - t.Fatal(err) - } - // Create untracked files. - untrackedFile := filepath.Join(dir, path, "untracked.txt") - if err := os.WriteFile(untrackedFile, []byte("new content"), 0755); err != nil { - t.Fatal(err) - } - } - - err := localRepo.Restore(test.paths) - if test.wantErr { - if err == nil { - t.Fatalf("%s should return error", test.name) - } - if !strings.Contains(err.Error(), test.wantErrPhrase) { - t.Errorf("Restore() returned error %q, want to contain %q", err.Error(), test.wantErrPhrase) - } - return - } - - if err != nil { - t.Fatal(err) - } - - for _, path := range test.paths { - got, err := os.ReadFile(filepath.Join(dir, path, "example.txt")) - if err != nil { - t.Fatal(err) - } - // Verify file contents are restored. - if diff := cmp.Diff("old content", string(got)); diff != "" { - t.Errorf("Restore() mismatch (-want +got):\n%s", diff) - } - // Verify the untracked files are untouched. - untrackedFile := filepath.Join(dir, path, "untracked.txt") - if _, err := os.Stat(untrackedFile); err != nil { - t.Errorf("untracked file, %s should not be removed", untrackedFile) - } - } - }) - } -} - -func TestCleanUntracked(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - paths []string - }{ - { - name: "remove_untracked_files_in_paths", - paths: []string{ - "first/path", - "second/path", - }, - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - repo, dir := initTestRepo(t) - localRepo := &LocalRepository{ - Dir: dir, - repo: repo, - } - - // Create tracked files. - for _, path := range test.paths { - relPath := filepath.Join(dir, path) - if err := os.MkdirAll(relPath, 0755); err != nil { - t.Fatal(err) - } - trackedFile := filepath.Join(relPath, "tracked.txt") - if err := os.WriteFile(trackedFile, []byte("new content"), 0755); err != nil { - t.Fatal(err) - } - } - - if err := localRepo.AddAll(); err != nil { - t.Fatal(err) - } - - // Create untracked files. - for _, path := range test.paths { - relPath := filepath.Join(dir, path) - if err := os.MkdirAll(relPath, 0755); err != nil { - t.Fatal(err) - } - untrackedFile := filepath.Join(relPath, "untracked.txt") - if err := os.WriteFile(untrackedFile, []byte("new content"), 0755); err != nil { - t.Fatal(err) - } - } - - if err := localRepo.CleanUntracked(test.paths); err != nil { - t.Error(err) - - return - } - - for _, path := range test.paths { - // Verify the untracked files are removed. - untrackedFile := filepath.Join(dir, path, "untracked.txt") - if _, err := os.Stat(untrackedFile); !errors.Is(err, fs.ErrNotExist) { - t.Errorf("untracked file, %s should be removed", untrackedFile) - } - // Verify the tracked files are untouched. - trackedFile := filepath.Join(dir, path, "tracked.txt") - if _, err := os.Stat(trackedFile); err != nil { - t.Errorf("tracked file, %s should not be touched: %q", trackedFile, err) - } - } - }) - } -} - -func TestGetLatestCommit(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - path string - setup func(t *testing.T, repo *git.Repository, path string) - want *Commit - wantErr error - }{ - { - name: "get_latest_commit_of_a_path", - path: "a/path/example.txt", - setup: func(t *testing.T, repo *git.Repository, path string) { - createAndCommit(t, repo, path, []byte("1st content"), "first commit") - createAndCommit(t, repo, path, []byte("2nd content"), "second commit") - createAndCommit(t, repo, "another/path/example.txt", []byte("2nd content"), "second commit") - }, - want: &Commit{ - Message: "second commit", - }, - }, - { - name: "no_commit_of_a_path", - path: "a/path/example.txt", - setup: func(t *testing.T, repo *git.Repository, path string) { - // Do nothing. - }, - wantErr: plumbing.ErrReferenceNotFound, - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - repo, dir := initTestRepo(t) - test.setup(t, repo, test.path) - localRepo := &LocalRepository{Dir: dir, repo: repo} - got, err := localRepo.GetLatestCommit(test.path) - if test.wantErr != nil { - if !errors.Is(err, test.wantErr) { - t.Errorf("unexpected error type: got %v, want %v", err, test.wantErr) - } - - return - } - if err != nil { - t.Error(err) - return - } - if diff := cmp.Diff(test.want, got, cmpopts.IgnoreFields(Commit{}, "When", "Hash")); diff != "" { - t.Errorf("GetLatestCommit() mismatch in %s (-want +got):\n%s", test.name, diff) - } - }) - } -} - -// initTestRepo creates a new git repository in a temporary directory. -func initTestRepo(t *testing.T) (*git.Repository, string) { - t.Helper() - dir := t.TempDir() - repo, err := git.PlainInit(dir, false) - if err != nil { - t.Fatalf("git.PlainInit failed: %v", err) - } - return repo, dir -} - -// createAndCommit creates and commits a file or directory. -func createAndCommit(t *testing.T, repo *git.Repository, path string, content []byte, commitMsg string) *object.Commit { - t.Helper() - w, err := repo.Worktree() - if err != nil { - t.Fatalf("Worktree() failed: %v", err) - } - - fullPath := filepath.Join(w.Filesystem.Root(), path) - if content != nil { // It's a file - if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil { - t.Fatalf("os.MkdirAll failed: %v", err) - } - if err := os.WriteFile(fullPath, content, 0644); err != nil { - t.Fatalf("os.WriteFile failed: %v", err) - } - } else { // It's a directory - if err := os.MkdirAll(fullPath, 0755); err != nil { - t.Fatalf("os.MkdirAll failed: %v", err) - } - } - return commitChanges(t, repo, commitMsg, path) - -} - -// deleteAndCommit deletes a file in a repository and commits the change. -func deleteAndCommit(t *testing.T, repo *git.Repository, path string, commitMsg string) *object.Commit { - t.Helper() - w, err := repo.Worktree() - if err != nil { - t.Fatalf("Worktree() failed: %v", err) - } - - fullPath := filepath.Join(w.Filesystem.Root(), path) - if err := os.Remove(fullPath); err != nil { - t.Fatalf("Remove() failed: %v", err) - } - return commitChanges(t, repo, commitMsg, path) -} - -// renameAndCommit renames a file in a repository and commits the change. -func renameAndCommit(t *testing.T, repo *git.Repository, fromPath, toPath string, commitMsg string) *object.Commit { - t.Helper() - w, err := repo.Worktree() - if err != nil { - t.Fatalf("Worktree() failed: %v", err) - } - - fullFromPath := filepath.Join(w.Filesystem.Root(), fromPath) - fullToPath := filepath.Join(w.Filesystem.Root(), toPath) - if err := os.Rename(fullFromPath, fullToPath); err != nil { - t.Fatalf("Rename() failed: %v", err) - } - return commitChanges(t, repo, commitMsg, fromPath, toPath) -} - -// commitChanges adds the specified paths to the repository and commits the change. -func commitChanges(t *testing.T, repo *git.Repository, commitMsg string, paths ...string) *object.Commit { - t.Helper() - w, err := repo.Worktree() - if err != nil { - t.Fatalf("Worktree() failed: %v", err) - } - - for _, path := range paths { - if _, err := w.Add(path); err != nil { - t.Fatalf("w.Add failed: %v", err) - } - } - hash, err := w.Commit(commitMsg, &git.CommitOptions{ - Author: &object.Signature{Name: "Test", Email: "test@example.com"}, - }) - if err != nil { - t.Fatalf("w.Commit failed: %v", err) - } - commit, err := repo.CommitObject(hash) - if err != nil { - t.Fatalf("repo.CommitObject failed: %v", err) - } - return commit -} - -// setupRepoForChangedFilesTest sets up a repository with a series of commits for testing. -// It returns the repository and a map of commit names to their hashes. -func setupRepoForChangedFilesTest(t *testing.T) (*LocalRepository, map[string]string) { - t.Helper() - repo, dir := initTestRepo(t) - - commitHashes := make(map[string]string) - - // Commit 1 - commit1 := createAndCommit(t, repo, "file1.txt", []byte("content1"), "commit 1") - commitHashes["commit 1"] = commit1.Hash.String() - - // Commit 2 (modify file1.txt) - commit2 := createAndCommit(t, repo, "file1.txt", []byte("content2"), "commit 2") - commitHashes["commit 2"] = commit2.Hash.String() - - // Commit 3 (add file2.txt) - commit3 := createAndCommit(t, repo, "file2.txt", []byte("content3"), "commit 3") - commitHashes["commit 3"] = commit3.Hash.String() - - // Commit 4 (delete file2.txt) - commit4 := deleteAndCommit(t, repo, "file2.txt", "commit 4") - commitHashes["commit 4"] = commit4.Hash.String() - - // Commit 5 (rename file1.txt to file3.txt) - commit5 := renameAndCommit(t, repo, "file1.txt", "file3.txt", "commit 5") - commitHashes["commit 5"] = commit5.Hash.String() - - return &LocalRepository{Dir: dir, repo: repo}, commitHashes -} - -// setupRepoForGetCommitsTest creates a repository with a few commits and tags. -func setupRepoForGetCommitsTest(t *testing.T) (*LocalRepository, map[string]string) { - t.Helper() - repo, dir := initTestRepo(t) - commits := make(map[string]string) - - // Commit 1 - commit1 := createAndCommit(t, repo, "file1.txt", []byte("content1"), "feat: commit 1") - commits["commit1"] = commit1.Hash.String() - - // Tag for commit 1 - if _, err := repo.CreateTag("v1.0.0", commit1.Hash, nil); err != nil { - t.Fatalf("CreateTag failed: %v", err) - } - - // Commit 2 - commit2 := createAndCommit(t, repo, "file2.txt", []byte("content2"), "feat: commit 2") - commits["commit2"] = commit2.Hash.String() - - // Commit 3 - commit3 := createAndCommit(t, repo, "file3.txt", []byte("content3"), "feat: commit 3") - commits["commit3"] = commit3.Hash.String() - - return &LocalRepository{Dir: dir, repo: repo}, commits -} - -func TestCanUseSSH(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - remoteURI string - want bool - }{ - { - name: "remote_https_uri", - remoteURI: "https://github.com/googleapis/librarian.git", - want: false, - }, - { - name: "remote_ssh_uri", - remoteURI: "git@github.com:googleapis/librarian.git", - want: true, - }, - { - name: "remote_ssh_uri_with_scheme", - remoteURI: "ssh://git@github.com/googleapis/librarian.git", - want: true, - }, - { - name: "invalid_remote_uri", - remoteURI: "nonsense-uri", - want: false, - }, - } { - t.Run(test.name, func(t *testing.T) { - got := canUseSSH(test.remoteURI) - if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("canUseSSH() mismatch in %s (-want +got):\n%s", test.name, diff) - } - }) - } -} - -func TestNewAndDeletedFiles(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - setup func(t *testing.T, dir string, w *git.Worktree) - wantFiles []string - }{ - { - name: "clean repository", - setup: func(t *testing.T, dir string, w *git.Worktree) {}, - }, - { - name: "new untracked file", - setup: func(t *testing.T, dir string, w *git.Worktree) { - filePath := filepath.Join(dir, "untracked.txt") - if err := os.WriteFile(filePath, []byte("test"), 0644); err != nil { - t.Fatalf("failed to write file: %v", err) - } - }, - wantFiles: []string{"untracked.txt"}, - }, - { - name: "new staged file", - setup: func(t *testing.T, dir string, w *git.Worktree) { - filePath := filepath.Join(dir, "added.txt") - if err := os.WriteFile(filePath, []byte("test"), 0644); err != nil { - t.Fatalf("failed to write file: %v", err) - } - if _, err := w.Add("added.txt"); err != nil { - t.Fatalf("failed to add file: %v", err) - } - }, - wantFiles: []string{"added.txt"}, - }, - { - name: "deleted file", - setup: func(t *testing.T, dir string, w *git.Worktree) { - filePath := filepath.Join(dir, "deleted.txt") - if err := os.WriteFile(filePath, []byte("initial"), 0644); err != nil { - t.Fatalf("failed to write file: %v", err) - } - if _, err := w.Add("deleted.txt"); err != nil { - t.Fatalf("failed to add file: %v", err) - } - if _, err := w.Commit("commit deleted.txt", &git.CommitOptions{Author: &object.Signature{Name: "Test", Email: "test@example.com"}}); err != nil { - t.Fatalf("failed to commit: %v", err) - } - if err := os.Remove(filePath); err != nil { - t.Fatalf("failed to remove file: %v", err) - } - }, - wantFiles: []string{"deleted.txt"}, - }, - { - name: "modified file should not be included", - setup: func(t *testing.T, dir string, w *git.Worktree) { - filePath := filepath.Join(dir, "modified.txt") - if err := os.WriteFile(filePath, []byte("initial"), 0644); err != nil { - t.Fatalf("failed to write file: %v", err) - } - if _, err := w.Add("modified.txt"); err != nil { - t.Fatalf("failed to add file: %v", err) - } - if _, err := w.Commit("commit modified.txt", &git.CommitOptions{Author: &object.Signature{Name: "Test", Email: "test@example.com"}}); err != nil { - t.Fatalf("failed to commit: %v", err) - } - if err := os.WriteFile(filePath, []byte("modified"), 0644); err != nil { - t.Fatalf("failed to write file: %v", err) - } - }, - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - repo, dir := initTestRepo(t) - w, err := repo.Worktree() - if err != nil { - t.Fatalf("failed to get worktree: %v", err) - } - - r := &LocalRepository{ - Dir: dir, - repo: repo, - } - - test.setup(t, dir, w) - gotFiles, err := r.NewAndDeletedFiles() - if err != nil { - t.Fatalf("NewAndDeletedFiles() returned an error: %v", err) - } - - slices.Sort(gotFiles) - if diff := cmp.Diff(test.wantFiles, gotFiles); diff != "" { - t.Errorf("NewAndDeletedFiles() mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestResetHard(t *testing.T) { - t.Parallel() - repo, dir := initTestRepo(t) - localRepo := &LocalRepository{Dir: dir, repo: repo} - - // Commit an initial file. - trackedFilePath := filepath.Join(dir, "tracked.txt") - initialContent := "initial content" - createAndCommit(t, repo, "tracked.txt", []byte(initialContent), "initial commit") - - // 1. Modify the tracked file. - if err := os.WriteFile(trackedFilePath, []byte("modified content"), 0644); err != nil { - t.Fatalf("failed to write file: %v", err) - } - - // 2. Create a new untracked file. - untrackedFilePath := filepath.Join(dir, "untracked.txt") - if err := os.WriteFile(untrackedFilePath, []byte("untracked"), 0644); err != nil { - t.Fatalf("failed to write file: %v", err) - } - - // Call ResetHard. - if err := localRepo.ResetHard(); err != nil { - t.Fatalf("ResetHard() failed: %v", err) - } - - // Check that the repo is clean. - isClean, err := localRepo.IsClean() - if err != nil { - t.Fatalf("IsClean() failed: %v", err) - } - if !isClean { - t.Error("ResetHard() should result in a clean repository") - } - - // Check that the untracked file is gone. - if _, err := os.Stat(untrackedFilePath); !errors.Is(err, fs.ErrNotExist) { - t.Errorf("untracked file should have been deleted by ResetHard(), but it still exists") - } - - // Check that the tracked file is restored to its original content. - content, err := os.ReadFile(trackedFilePath) - if err != nil { - t.Fatalf("failed to read tracked file after reset: %v", err) - } - if string(content) != initialContent { - t.Errorf("tracked file content mismatch after reset: got %q, want %q", string(content), initialContent) - } -} - -func TestCheckoutCommitAndCreateBranch(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - branchName string - setup func(t *testing.T, repo *git.Repository) (string, plumbing.Hash) - wantErr bool - }{ - { - name: "success", - branchName: "new-branch", - setup: func(t *testing.T, repo *git.Repository) (string, plumbing.Hash) { - commit := createAndCommit(t, repo, "initial.txt", []byte("initial"), "initial commit") - return commit.Hash.String(), commit.Hash - }, - }, - { - name: "invalid commit", - branchName: "new-branch", - setup: func(t *testing.T, repo *git.Repository) (string, plumbing.Hash) { - return "invalid-hash", plumbing.ZeroHash - }, - wantErr: true, - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - repo, dir := initTestRepo(t) - localRepo := &LocalRepository{Dir: dir, repo: repo} - commitHashStr, commitHash := test.setup(t, repo) - - err := localRepo.CheckoutCommitAndCreateBranch(test.branchName, commitHashStr) - - if (err != nil) != test.wantErr { - t.Fatalf("CheckoutCommitAndCreateBranch() error = %v, wantErr %v", err, test.wantErr) - } - - if !test.wantErr { - head, err := repo.Head() - if err != nil { - t.Fatalf("repo.Head() failed: %v", err) - } - - if head.Name().Short() != test.branchName { - t.Errorf("HEAD is at %q, want %q", head.Name().Short(), test.branchName) - } - - if head.Hash() != commitHash { - t.Errorf("HEAD hash is %q, want %q", head.Hash(), commitHash) - } - } - }) - } -} - -func TestDeleteLocalBranches(t *testing.T) { - t.Parallel() - - for _, test := range []struct { - name string - branchNames []string - useEmptyRepo bool - setup func(t *testing.T, repo *LocalRepository) - wantErr bool - wantErrMsg string - }{ - { - name: "delete single existing branch", - branchNames: []string{"test-branch"}, - setup: func(t *testing.T, repo *LocalRepository) { - if err := repo.CreateBranchAndCheckout("test-branch"); err != nil { - t.Fatalf("CreateBranchAndCheckout() failed: %v", err) - } - // Checkout master so we are not on the branch to be deleted - w, err := repo.repo.Worktree() - if err != nil { - t.Fatalf("failed to get worktree: %v", err) - } - if err := w.Checkout(&git.CheckoutOptions{Branch: plumbing.NewBranchReferenceName("master")}); err != nil { - t.Fatalf("failed to checkout master: %v", err) - } - }, - }, - { - name: "delete multiple existing branches", - branchNames: []string{"branch1", "branch2"}, - setup: func(t *testing.T, repo *LocalRepository) { - if err := repo.CreateBranchAndCheckout("branch1"); err != nil { - t.Fatalf("CreateBranchAndCheckout() failed: %v", err) - } - if err := repo.CreateBranchAndCheckout("branch2"); err != nil { - t.Fatalf("CreateBranchAndCheckout() failed: %v", err) - } - // Checkout master so we are not on a branch to be deleted - w, err := repo.repo.Worktree() - if err != nil { - t.Fatalf("failed to get worktree: %v", err) - } - if err := w.Checkout(&git.CheckoutOptions{Branch: plumbing.NewBranchReferenceName("master")}); err != nil { - t.Fatalf("failed to checkout master: %v", err) - } - }, - }, - { - name: "delete non-existing branch", - branchNames: []string{"non-existing-branch"}, - setup: func(t *testing.T, repo *LocalRepository) {}, - wantErr: true, - wantErrMsg: "failed to check existence of branch non-existing-branch", - }, - { - name: "delete current branch", - branchNames: []string{"current-branch"}, - setup: func(t *testing.T, repo *LocalRepository) { - if err := repo.CreateBranchAndCheckout("current-branch"); err != nil { - t.Fatalf("CreateBranchAndCheckout() failed: %v", err) - } - }, - wantErr: true, - wantErrMsg: "cannot delete branch current-branch: it is the currently checked out branch (HEAD)", - }, - { - name: "attempt to delete multiple branches including non-existing", - branchNames: []string{"branch1", "non-existing-branch"}, - setup: func(t *testing.T, repo *LocalRepository) { - if err := repo.CreateBranchAndCheckout("branch1"); err != nil { - t.Fatalf("CreateBranchAndCheckout() failed: %v", err) - } - w, err := repo.repo.Worktree() - if err != nil { - t.Fatalf("failed to get worktree: %v", err) - } - if err := w.Checkout(&git.CheckoutOptions{Branch: plumbing.NewBranchReferenceName("master")}); err != nil { - t.Fatalf("failed to checkout master: %v", err) - } - }, - wantErr: true, - wantErrMsg: "failed to check existence of branch non-existing-branch", - }, - { - name: "attempt to delete multiple branches including current branch", - branchNames: []string{"branch1", "current-branch"}, - setup: func(t *testing.T, repo *LocalRepository) { - if err := repo.CreateBranchAndCheckout("branch1"); err != nil { - t.Fatalf("CreateBranchAndCheckout() failed: %v", err) - } - if err := repo.CreateBranchAndCheckout("current-branch"); err != nil { - t.Fatalf("CreateBranchAndCheckout() failed: %v", err) - } - }, - wantErr: true, - wantErrMsg: "cannot delete branch current-branch: it is the currently checked out branch (HEAD)", - }, - { - name: "empty repository", - branchNames: []string{}, - useEmptyRepo: true, - setup: func(t *testing.T, repo *LocalRepository) {}, - }, - } { - t.Run(test.name, func(t *testing.T) { - var localRepo *LocalRepository - if test.useEmptyRepo { - gogitRepo, dir := initTestRepo(t) - localRepo = &LocalRepository{Dir: dir, repo: gogitRepo} - } else { - localRepo, _ = setupRepoForGetCommitsTest(t) - } - test.setup(t, localRepo) - - err := localRepo.DeleteLocalBranches(test.branchNames) - - if (err != nil) != test.wantErr { - t.Fatalf("DeleteLocalBranches() error = %v, wantErr %v", err, test.wantErr) - } - - if test.wantErr { - if !strings.Contains(err.Error(), test.wantErrMsg) { - t.Errorf("DeleteLocalBranches() error message = %q, want to contain %q", err.Error(), test.wantErrMsg) - } - return - } - - for _, branchName := range test.branchNames { - _, err := localRepo.repo.Branch(branchName) - if err == nil { - t.Errorf("branch %q should have been deleted", branchName) - } - } - }) - } -} - -func TestResetSoft(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - commitRef string - wantErr bool - wantErrMsg string - }{ - { - name: "successful reset", - commitRef: "HEAD~1", - }, - { - name: "invalid commit reference", - commitRef: "invalid-ref", - wantErr: true, - wantErrMsg: "failed to resolve revision", - }, - } { - t.Run(test.name, func(t *testing.T) { - repo, dir := initTestRepo(t) - localRepo := &LocalRepository{Dir: dir, repo: repo} - - initialCommit := createAndCommit(t, repo, "file.txt", []byte("initial content"), "initial commit") - createAndCommit(t, repo, "file.txt", []byte("second content"), "second commit") - - err := localRepo.ResetSoft(test.commitRef) - - if test.wantErr { - if err == nil { - t.Fatal("ResetSoft() expected an error, but got nil") - } - if !strings.Contains(err.Error(), test.wantErrMsg) { - t.Errorf("ResetSoft() error = %q, want error containing %q", err.Error(), test.wantErrMsg) - } - return - } - - if err != nil { - t.Fatalf("ResetSoft() returned unexpected error: %v", err) - } - - // Verify HEAD is now at the initial commit. - headAfterReset, err := repo.Head() - if err != nil { - t.Fatalf("repo.Head() after reset failed: %v", err) - } - if headAfterReset.Hash() != initialCommit.Hash { - t.Errorf("HEAD after reset is at %s, want %s", headAfterReset.Hash(), initialCommit.Hash) - } - - // Verify the repo is no longer clean because the changes from the second commit are now staged. - isCleanAfterReset, err := localRepo.IsClean() - if err != nil { - t.Fatalf("IsClean() after reset failed: %v", err) - } - if isCleanAfterReset { - t.Error("repository should be dirty after soft reset") - } - - // Verify the working tree file still has the content from the second commit. - content, err := os.ReadFile(filepath.Join(dir, "file.txt")) - if err != nil { - t.Fatalf("failed to read file after reset: %v", err) - } - if string(content) != "second content" { - t.Errorf("file content after reset is %q, want %q", string(content), "second content") - } - }) - } -} diff --git a/internal/legacylibrarian/legacyimages/images.go b/internal/legacylibrarian/legacyimages/images.go deleted file mode 100644 index 3bd5fef1d81..00000000000 --- a/internal/legacylibrarian/legacyimages/images.go +++ /dev/null @@ -1,167 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package legacyimages provides operations around docker images. -package legacyimages - -import ( - "context" - "fmt" - "log/slog" - "strings" - - artifactregistry "cloud.google.com/go/artifactregistry/apiv1" - artifactregistrypb "cloud.google.com/go/artifactregistry/apiv1/artifactregistrypb" -) - -// ArtifactRegistryClient is the implementation of ImageRegistryClient -// to interact with Artifact Registry. -type ArtifactRegistryClient struct { - client *artifactregistry.Client -} - -// containerImage is a data structure for parsing Docker image parameters. -type containerImage struct { - // Name is the short name of the docker image. - Name string - // Tag is the named tag of the docker image. - Tag string - // SHA is the SHA256 (e.g. `sha256:abcd1234`). - SHA string - // Location is the Artifact Registry location (e.g. `us-central1`). - Location string - // Project is the name of the GCP project that holds the AR repository. - Project string - // Repository is the name of the AR repository. - Repository string -} - -// BaseName returns the image name without a pinned SHA or tag. -func (i *containerImage) BaseName() string { - return fmt.Sprintf("%s-docker.pkg.dev/%s/%s/%s", i.Location, i.Project, i.Repository, i.Name) -} - -// String returns the image name with pinned SHA or tag. -func (i *containerImage) String() string { - var b strings.Builder - b.WriteString(i.BaseName()) - if i.SHA != "" { - b.WriteString("@") - b.WriteString(i.SHA) - } else if i.Tag != "" { - b.WriteString(":") - b.WriteString(i.Tag) - } - return b.String() -} - -// NewArtifactRegistryClient creates a new ArtifactRegistryClient. -func NewArtifactRegistryClient(ctx context.Context) (*ArtifactRegistryClient, error) { - client, err := artifactregistry.NewClient(ctx) - if err != nil { - return nil, err - } - return &ArtifactRegistryClient{ - client: client, - }, nil -} - -// Close cleans up any open resources. -func (c *ArtifactRegistryClient) Close() error { - return c.client.Close() -} - -// FindLatest returns the latest docker image given a current image. -func (c *ArtifactRegistryClient) FindLatest(ctx context.Context, imageName string) (string, error) { - image, err := parseImage(imageName) - if err != nil { - return "", err - } - - if c.client == nil { - return "", fmt.Errorf("no client configured") - } - - it := c.client.ListVersions(ctx, &artifactregistrypb.ListVersionsRequest{ - Parent: fmt.Sprintf("projects/%s/locations/%s/repositories/%s/packages/%s", image.Project, image.Location, image.Repository, image.Name), - View: artifactregistrypb.VersionView_FULL, - OrderBy: "create_time DESC", - }) - version, err := it.Next() - if err != nil { - return "", err - } - slog.Info("found packages version", "version", version.GetName()) - - // latest SHA is found as the "subjectDigest" metadata field - latestSha := "" - for key, field := range version.GetMetadata().GetFields() { - if key == "subjectDigest" { - slog.Info("found SHA", "sha", field.GetStringValue()) - latestSha = field.GetStringValue() - break - } - } - - if latestSha == "" { - return "", fmt.Errorf("failed to find updated SHA for latest version: %s", version.GetName()) - } - - newImage := &containerImage{ - Name: image.Name, - Location: image.Location, - Repository: image.Repository, - Project: image.Project, - SHA: latestSha, - } - return newImage.String(), nil -} - -func parseImage(pinnedImage string) (*containerImage, error) { - parsedImage := &containerImage{} - baseName := "" - - atParts := strings.Split(pinnedImage, "@") - colonParts := strings.Split(pinnedImage, ":") - if len(atParts) == 2 { - baseName = atParts[0] - parsedImage.SHA = atParts[1] - } else if len(colonParts) == 2 { - baseName = colonParts[0] - parsedImage.Tag = colonParts[1] - } - - if baseName == "" { - slog.Info("image does not appear to be pinned") - baseName = pinnedImage - } - - parts := strings.Split(baseName, "/") - if len(parts) < 4 { - return nil, fmt.Errorf("unexpected image format %q, expected an AR formatted image", baseName) - } - - host := parts[0] - if strings.HasSuffix(host, "-docker.pkg.dev") { - parsedImage.Location = strings.TrimSuffix(host, "-docker.pkg.dev") - } else { - return nil, fmt.Errorf("unexpected host format %q, expected AR formatted host with -docker.pkg.dev suffix", host) - } - - parsedImage.Project = parts[1] - parsedImage.Repository = parts[2] - parsedImage.Name = parts[3] - - return parsedImage, nil -} diff --git a/internal/legacylibrarian/legacyimages/images_test.go b/internal/legacylibrarian/legacyimages/images_test.go deleted file mode 100644 index 277fbf8dbc9..00000000000 --- a/internal/legacylibrarian/legacyimages/images_test.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacyimages - -import ( - "testing" - - "github.com/google/go-cmp/cmp" -) - -func TestParseImage(t *testing.T) { - for _, test := range []struct { - name string - image string - want *containerImage - wantErr bool - }{ - { - name: "AR unpinned", - image: "us-central1-docker.pkg.dev/some-project/some-repo/some-image", - want: &containerImage{ - Name: "some-image", - Location: "us-central1", - Project: "some-project", - Repository: "some-repo", - }, - }, - { - name: "AR pinned SHA", - image: "us-central1-docker.pkg.dev/some-project/some-repo/some-image@sha256:abcdef", - want: &containerImage{ - Name: "some-image", - Location: "us-central1", - Project: "some-project", - Repository: "some-repo", - SHA: "sha256:abcdef", - }, - }, - { - name: "AR tagged", - image: "us-central1-docker.pkg.dev/some-project/some-repo/some-image:1.2.3", - want: &containerImage{ - Name: "some-image", - Location: "us-central1", - Project: "some-project", - Repository: "some-repo", - Tag: "1.2.3", - }, - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - got, err := parseImage(test.image) - if (err != nil) != test.wantErr { - t.Errorf("parseImage() error = %v, wantErr %v", err, test.wantErr) - return - } - if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("parseImage() mismatch (-want +got):\n%s", diff) - } - str := got.String() - if diff := cmp.Diff(str, test.image); diff != "" { - t.Errorf("image.String() mismatch (-want +got):\n%s", diff) - } - }) - } -} diff --git a/internal/legacylibrarian/legacyintegration/doc.go b/internal/legacylibrarian/legacyintegration/doc.go deleted file mode 100644 index a3aba3bf83a..00000000000 --- a/internal/legacylibrarian/legacyintegration/doc.go +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package integration contains integration and e2e tests for the Librarian project. -package integration diff --git a/internal/legacylibrarian/legacyintegration/e2e_test.go b/internal/legacylibrarian/legacyintegration/e2e_test.go deleted file mode 100644 index 4535b617342..00000000000 --- a/internal/legacylibrarian/legacyintegration/e2e_test.go +++ /dev/null @@ -1,909 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build e2e - -package integration_test - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "html" - "io/fs" - "net/http" - "net/http/httptest" - "os" - "os/exec" - "path/filepath" - "regexp" - "strings" - "testing" - - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - "github.com/google/go-github/v69/github" - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" - "gopkg.in/yaml.v3" -) - -// regexps for parsing commit messages -var ( - nestedCommitRegex = regexp.MustCompile(`(?s)BEGIN_NESTED_COMMIT\n(.*?)\nEND_NESTED_COMMIT`) - conventionalCommitRegex = regexp.MustCompile(`^(feat|fix|docs|chore): (.+)$`) -) - -const mockGithubTag = "mock_github" - -func TestRunGenerate(t *testing.T) { - const ( - initialRepoStateDir = "testdata/e2e/generate/repo_init" - localAPISource = "testdata/e2e/generate/api_root" - ) - t.Parallel() - for _, test := range []struct { - name string - api string - push bool - wantErr bool - wantInPrBody []string - doNotWantInPrBody []string - commits []struct { - message string - fileToAdd string - } - }{ - { - name: "testRunSuccess", - api: "google/cloud/pubsub/v1", - }, - { - name: "failed due to simulated error in generate command", - api: "google/cloud/future/v2", - wantErr: true, - }, - { - name: "testRunSuccess with push", - api: "google/cloud/pubsub/v1", - push: true, - commits: []struct { - message string - fileToAdd string - }{ - { - message: "feat: new feature pubsub", - fileToAdd: "google/cloud/pubsub/v1/pubsub.code", - }, - { - message: "feat: new feature future", - fileToAdd: "google/cloud/future/v2/future.code", - }, - }, - wantInPrBody: []string{"feat: new feature pubsub"}, - doNotWantInPrBody: []string{"feat: new feature future"}, - }, - } { - t.Run(test.name, func(t *testing.T) { - workRoot := t.TempDir() - repo := t.TempDir() - apiSourceRepo := t.TempDir() - if err := initRepo(t, repo, initialRepoStateDir, "initial commit"); err != nil { - t.Fatalf("languageRepo prepare test error = %v", err) - } - if err := initRepo(t, apiSourceRepo, localAPISource, "initial commit"); err != nil { - t.Fatalf("APISourceRepo prepare test error = %v", err) - } - if test.push { - // Create a local bare repository to act as the remote for the push. - bareRepoDir := filepath.Join(t.TempDir(), "remote.git") - if err := os.MkdirAll(bareRepoDir, 0755); err != nil { - t.Fatalf("Failed to create bare repo dir: %v", err) - } - runGit(t, bareRepoDir, "init", "--bare") - runGit(t, repo, "remote", "set-url", "origin", bareRepoDir) - // Setup state.yaml file with initial last_generated_commit, so it picks up the commits we create below. - apiSourceSHA := runGit(t, apiSourceRepo, "rev-parse", "HEAD") - setupStateFile(t, repo, apiSourceSHA) - runGit(t, repo, "commit", "-a", "-m", "populate yaml file with latest googleapis commit") - // Create commits in the api source repo to be picked in generation PR body. - for _, commit := range test.commits { - createCommit(t, apiSourceRepo, commit.fileToAdd, commit.message) - } - } - // Setup mock GitHub server. - server := newMockGitHubServer(t, "generate", test.wantInPrBody, test.doNotWantInPrBody) - defer server.Close() - cmdArgs := []string{ - "run", - "-tags", mockGithubTag, - "github.com/googleapis/librarian/cmd/legacylibrarian", - "generate", - fmt.Sprintf("--api=%s", test.api), - fmt.Sprintf("--output=%s", workRoot), - fmt.Sprintf("--repo=%s", repo), - fmt.Sprintf("--api-source=%s", apiSourceRepo), - } - if test.push { - cmdArgs = append(cmdArgs, "--push") - } - - cmd := exec.Command("go", cmdArgs...) - cmd.Env = append(os.Environ(), fmt.Sprintf("%s=fake-token", legacyconfig.LibrarianGithubToken)) - cmd.Env = append(cmd.Env, "LIBRARIAN_GITHUB_BASE_URL="+server.URL) - var stderr bytes.Buffer - cmd.Stderr = &stderr - cmd.Stdout = os.Stdout - err := cmd.Run() - if test.wantErr { - if err == nil { - t.Fatalf("%s should fail", test.name) - } - - if g, w := stderr.String(), "level=ERROR"; !strings.Contains(g, w) { - t.Errorf("got %q, wanted it to contain %q", stderr.String(), w) - } - // the exact message is not populated here, but we can check it's - // indeed an error returned from docker container. - if g, w := err.Error(), "exit status 1"; !strings.Contains(g, w) { - t.Fatalf("got %q, wanted it to contain %q", g, w) - } - - return - } - - if err != nil { - t.Fatalf("librarian generate command error = %v", err) - } - }) - } -} - -func TestCleanAndCopy(t *testing.T) { - const ( - localAPISource = "testdata/e2e/generate/api_root" - apiToGenerate = "google/cloud/pubsub/v1" - ) - // create a temp directory for writing files, so we don't have to create testdata files. - repoInitDir := t.TempDir() - - // within the source root, create a file to be removed, - // then create a sub dir with 2 files, on of them should be preserved. - pubsubDir := filepath.Join(repoInitDir, "pubsub") - if err := os.MkdirAll(filepath.Join(pubsubDir, "sub"), 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(pubsubDir, "file_to_remove.txt"), []byte("remove me"), 0644); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(pubsubDir, "sub", "file_to_preserve.txt"), []byte("preserve me"), 0644); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(pubsubDir, "sub", "another_file_to_remove.txt"), []byte("remove me"), 0644); err != nil { - t.Fatal(err) - } - // Create a file outside the source root to ensure it's not touched. - otherDir := filepath.Join(repoInitDir, "other_dir") - if err := os.MkdirAll(otherDir, 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(otherDir, "file_to_keep.txt"), []byte("keep me"), 0644); err != nil { - t.Fatal(err) - } - - setupStateFile(t, repoInitDir, "") - - workRoot := t.TempDir() - repo := t.TempDir() - apiSourceRepo := t.TempDir() - if err := initRepo(t, repo, repoInitDir, "initial commit"); err != nil { - t.Fatalf("languageRepo prepare test error = %v", err) - } - if err := initRepo(t, apiSourceRepo, localAPISource, "initial commit"); err != nil { - t.Fatalf("APISourceRepo prepare test error = %v", err) - } - - cmd := exec.Command( - "go", - "run", - "-tags", mockGithubTag, - "github.com/googleapis/librarian/cmd/legacylibrarian", - "generate", - fmt.Sprintf("--api=%s", apiToGenerate), - fmt.Sprintf("--output=%s", workRoot), - fmt.Sprintf("--repo=%s", repo), - fmt.Sprintf("--api-source=%s", apiSourceRepo), - ) - cmd.Stderr = os.Stderr - cmd.Stdout = os.Stdout - if err := cmd.Run(); err != nil { - t.Fatalf("librarian generate command error = %v", err) - } - - // Check that the file to remove is gone. - if _, err := os.Stat(filepath.Join(repo, "pubsub", "file_to_remove.txt")); !errors.Is(err, fs.ErrNotExist) { - t.Errorf("pubsub/file_to_remove.txt should have been removed") - } - // Check that the other file to remove is gone. - if _, err := os.Stat(filepath.Join(repo, "pubsub", "sub", "another_file_to_remove.txt")); !errors.Is(err, fs.ErrNotExist) { - t.Errorf("pubsub/sub/another_file_to_remove.txt should have been removed") - } - // Check that the file to preserve is still there. - if _, err := os.Stat(filepath.Join(repo, "pubsub", "sub", "file_to_preserve.txt")); errors.Is(err, fs.ErrNotExist) { - t.Errorf("pubsub/sub/file_to_preserve.txt should have been preserved") - } - // Check that the file outside the source root is still there. - if _, err := os.Stat(filepath.Join(repo, "other_dir", "file_to_keep.txt")); errors.Is(err, fs.ErrNotExist) { - t.Errorf("other_dir/file_to_keep.txt should have been preserved") - } - // check that the new files are copied. The fake generator creates a file called "example.txt". - if _, err := os.Stat(filepath.Join(repo, "pubsub", "example.txt")); errors.Is(err, fs.ErrNotExist) { - t.Errorf("pubsub/example.txt should have been copied") - } -} - -func TestRunConfigure(t *testing.T) { - const ( - localRepoDir = "testdata/e2e/configure/repo" - initialRepoStateDir = "testdata/e2e/configure/repo_init" - ) - t.Parallel() - for _, test := range []struct { - name string - api string - library string - apiSource string - updatedState string - wantErr bool - }{ - { - name: "runs successfully", - api: "google/cloud/new-library-path/v2", - library: "new-library", - apiSource: "testdata/e2e/configure/api_root", - updatedState: "testdata/e2e/configure/updated-state.yaml", - }, - { - name: "failed due to simulated error in configure command", - api: "google/cloud/another-library/v3", - library: "simulate-command-error-id", - apiSource: "testdata/e2e/configure/api_root", - updatedState: "testdata/e2e/configure/updated-state.yaml", - wantErr: true, - }, - } { - t.Run(test.name, func(t *testing.T) { - workRoot := t.TempDir() - repo := t.TempDir() - apiSourceRepo := t.TempDir() - if err := initRepo(t, repo, initialRepoStateDir, "initial commit"); err != nil { - t.Fatalf("prepare test error = %v", err) - } - if err := initRepo(t, apiSourceRepo, test.apiSource, "feat: add a new api\n\nPiperOrigin-RevId: 123456"); err != nil { - t.Fatalf("APISourceRepo prepare test error = %v", err) - } - - cmd := exec.Command( - "go", - "run", - "github.com/googleapis/librarian/cmd/legacylibrarian", - "generate", - fmt.Sprintf("--api=%s", test.api), - fmt.Sprintf("--output=%s", workRoot), - fmt.Sprintf("--repo=%s", repo), - fmt.Sprintf("--api-source=%s", apiSourceRepo), - fmt.Sprintf("--library=%s", test.library), - ) - cmd.Stderr = os.Stderr - cmd.Stdout = os.Stdout - err := cmd.Run() - if test.wantErr { - if err == nil { - t.Fatal("Configure command should fail") - } - - // the exact message is not populated here, but we can check it's - // indeed an error returned from docker container. - if g, w := err.Error(), "exit status 1"; !strings.Contains(g, w) { - t.Errorf("got %q, wanted it to contain %q", g, w) - } - return - } - if err != nil { - t.Fatalf("Failed to run configure: %v", err) - } - - // Verify the file content - gotBytes, err := os.ReadFile(filepath.Join(repo, ".librarian", "state.yaml")) - if err != nil { - t.Fatalf("Failed to read configure response file: %v", err) - } - wantBytes, readErr := os.ReadFile(test.updatedState) - if readErr != nil { - t.Fatalf("Failed to read expected state for comparison: %v", readErr) - } - var gotState *legacyconfig.LibrarianState - if err := yaml.Unmarshal(gotBytes, &gotState); err != nil { - t.Fatalf("Failed to unmarshal configure response file: %v", err) - } - var wantState *legacyconfig.LibrarianState - if err := yaml.Unmarshal(wantBytes, &wantState); err != nil { - t.Fatalf("Failed to unmarshal expected state: %v", err) - } - - if diff := cmp.Diff(wantState, gotState, cmpopts.IgnoreFields(legacyconfig.LibraryState{}, "LastGeneratedCommit")); diff != "" { - t.Fatalf("Generated yaml mismatch (-want +got):\n%s", diff) - } - for _, lib := range gotState.Libraries { - if lib.ID == test.library && lib.LastGeneratedCommit == "" { - t.Fatal("LastGeneratedCommit should not be empty") - } - } - - }) - } -} - -func TestRunGenerate_MultipleLibraries(t *testing.T) { - const localAPISource = "testdata/e2e/generate/api_root" - - for _, test := range []struct { - name string - initialRepoStateDir string - expectError bool - expectedFiles []string - unexpectedFiles []string - }{ - { - name: "Multiple libraries generated successfully", - initialRepoStateDir: "testdata/e2e/generate/multi_repo_init", - expectedFiles: []string{"pubsub/example.txt", "future/example.txt"}, - unexpectedFiles: []string{}, - }, - { - name: "One library fails to generate", - initialRepoStateDir: "testdata/e2e/generate/multi_repo_one_fails_init", - expectedFiles: []string{"pubsub/example.txt"}, - unexpectedFiles: []string{"future/example.txt"}, - }, - { - name: "All libraries fail to generate", - initialRepoStateDir: "testdata/e2e/generate/multi_repo_all_fail_init", - expectError: true, - expectedFiles: []string{}, - unexpectedFiles: []string{"future/example.txt", "another-future/example.txt"}, - }, - } { - t.Run(test.name, func(t *testing.T) { - workRoot := t.TempDir() - repo := t.TempDir() - apiSourceRepo := t.TempDir() - - if err := initRepo(t, repo, test.initialRepoStateDir, "initial commit"); err != nil { - t.Fatalf("languageRepo prepare test error = %v", err) - } - if err := initRepo(t, apiSourceRepo, localAPISource, "initial commit"); err != nil { - t.Fatalf("APISourceRepo prepare test error = %v", err) - } - - cmd := exec.Command( - "go", - "run", - "github.com/googleapis/librarian/cmd/legacylibrarian", - "generate", - fmt.Sprintf("--output=%s", workRoot), - fmt.Sprintf("--repo=%s", repo), - fmt.Sprintf("--api-source=%s", apiSourceRepo), - ) - cmd.Stderr = os.Stderr - cmd.Stdout = os.Stdout - err := cmd.Run() - - if test.expectError { - if err == nil { - t.Fatal("librarian generate command should fail") - } - return - } - - if err != nil { - t.Fatalf("librarian generate command error = %v", err) - } - - for _, f := range test.expectedFiles { - if _, err := os.Stat(filepath.Join(repo, f)); errors.Is(err, fs.ErrNotExist) { - t.Errorf("%s should have been copied", f) - } - } - - for _, f := range test.unexpectedFiles { - if _, err := os.Stat(filepath.Join(repo, f)); !errors.Is(err, fs.ErrNotExist) { - t.Errorf("%s should not have been copied", f) - } - } - }) - } -} - -func TestReleaseStage(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - testDataDir string - libraryID string - changePath string - tagID string - tagVersion string - tagFormat string - push bool - wantErr bool - }{ - { - name: "release with multiple commits", - testDataDir: "testdata/e2e/release/stage/multiple_commits", - libraryID: "go-google-cloud-pubsub-v1", - changePath: "google-cloud-pubsub/v1", - tagID: "go-google-cloud-pubsub-v1", - tagVersion: "1.0.0", - tagFormat: "%s/v%s", - }, - { - name: "release with multiple commits with push", - testDataDir: "testdata/e2e/release/stage/multiple_commits", - libraryID: "go-google-cloud-pubsub-v1", - changePath: "google-cloud-pubsub/v1", - tagID: "go-google-cloud-pubsub-v1", - tagVersion: "1.0.0", - tagFormat: "%s/v%s", - push: true, - }, - { - name: "release with multiple nested commits", - testDataDir: "testdata/e2e/release/stage/multiple_nested_commits", - libraryID: "python-google-cloud-video-live-stream-v1", - changePath: "packages/google-cloud-video-live-stream", - tagID: "python-google-cloud-video-live-stream-v1", - tagVersion: "v1.12.0", - tagFormat: "%s-%s", // Format for {id}-{version} - }, - { - name: "release with single commit", - testDataDir: "testdata/e2e/release/stage/single_commit", - libraryID: "dlp", - changePath: "dlp", - tagID: "dlp", - tagVersion: "1.24.0", - tagFormat: "%s/v%s", - }, - } { - t.Run(test.name, func(t *testing.T) { - workRoot := t.TempDir() - repo := t.TempDir() - - initialRepoStateDir := filepath.Join(test.testDataDir, "repo_init") - updatedState := filepath.Join(test.testDataDir, "state.yaml") - wantChangelog := filepath.Join(test.testDataDir, "CHANGELOG.md") - commitMsgPath := filepath.Join(test.testDataDir, "commit_msg.txt") - - if err := initRepo(t, repo, initialRepoStateDir, "initial commit"); err != nil { - t.Fatalf("prepare test error = %v", err) - } - - if test.push { - // Create a local bare repository to act as the remote for the push. - bareRepoDir := filepath.Join(t.TempDir(), "remote.git") - if err := os.MkdirAll(bareRepoDir, 0755); err != nil { - t.Fatalf("Failed to create bare repo dir: %v", err) - } - runGit(t, bareRepoDir, "init", "--bare") - runGit(t, repo, "remote", "set-url", "origin", bareRepoDir) - } - - // Dynamically create the tag based on the format string - tagName := fmt.Sprintf(test.tagFormat, test.tagID, test.tagVersion) - runGit(t, repo, "tag", tagName) - - // Add a new commit to simulate a change. - newFilePath := filepath.Join(test.changePath, "new-file.txt") - if err := os.MkdirAll(filepath.Join(repo, test.changePath), 0755); err != nil { - t.Fatalf("Failed to create directory for new file: %v", err) - } - commitMsgBytes, err := os.ReadFile(commitMsgPath) - if err != nil { - t.Fatalf("Failed to read commit message file: %v", err) - } - - createCommit(t, repo, newFilePath, string(commitMsgBytes)) - - prContentToMatch := parseCommitMessageForPRContent(string(commitMsgBytes)) - server := newMockGitHubServer(t, "release", prContentToMatch, []string{}) - defer server.Close() - - cmdArgs := []string{ - "run", - "-tags", mockGithubTag, - "github.com/googleapis/librarian/cmd/legacylibrarian", - "release", - "stage", - fmt.Sprintf("--repo=%s", repo), - fmt.Sprintf("--output=%s", workRoot), - fmt.Sprintf("--library=%s", test.libraryID), - } - if test.push { - cmdArgs = append(cmdArgs, "--push") - } - - cmd := exec.Command("go", cmdArgs...) - cmd.Env = os.Environ() - cmd.Env = append(cmd.Env, fmt.Sprintf("%s=fake-token", legacyconfig.LibrarianGithubToken)) - cmd.Env = append(cmd.Env, "LIBRARIAN_GITHUB_BASE_URL="+server.URL) - cmd.Stderr = os.Stderr - cmd.Stdout = os.Stdout - if err := cmd.Run(); err != nil { - t.Fatalf("Failed to run release stage: %v", err) - } - - // Verify the state.yaml file content - outputDir := filepath.Join(workRoot, "output") - t.Logf("Checking for output file in: %s", filepath.Join(outputDir, ".librarian", "state.yaml")) - gotBytes, err := os.ReadFile(filepath.Join(outputDir, ".librarian", "state.yaml")) - if err != nil { - t.Fatalf("Failed to read updated state.yaml from output directory: %v", err) - } - wantBytes, readErr := os.ReadFile(updatedState) - if readErr != nil { - t.Fatalf("Failed to read expected state for comparison: %v", readErr) - } - var gotState *legacyconfig.LibrarianState - if err := yaml.Unmarshal(gotBytes, &gotState); err != nil { - t.Fatalf("Failed to unmarshal configure response file: %v", err) - } - var wantState *legacyconfig.LibrarianState - if err := yaml.Unmarshal(wantBytes, &wantState); err != nil { - t.Fatalf("Failed to unmarshal expected state: %v", err) - } - - // Use cmpopts.IgnoreFields to ignore the dynamic commit hash. - if diff := cmp.Diff(wantState, gotState, cmpopts.IgnoreFields(legacyconfig.LibraryState{}, "LastGeneratedCommit")); diff != "" { - t.Fatalf("Generated yaml mismatch (-want +got): %s", diff) - } - - // Verify the CHANGELOG.md file content - gotChangelog, err := os.ReadFile(filepath.Join(outputDir, test.changePath, "CHANGELOG.md")) - if err != nil { - t.Fatalf("Failed to read CHANGELOG.md from output directory: %v", err) - } - wantChangelogBytes, err := os.ReadFile(wantChangelog) - if err != nil { - t.Fatalf("Failed to read expected changelog for comparison: %v", err) - } - if diff := cmp.Diff(string(wantChangelogBytes), string(gotChangelog)); diff != "" { - t.Fatalf("Generated changelog mismatch (-want +got): %s", diff) - } - }) - } -} - -func TestReleaseTag(t *testing.T) { - for _, test := range []struct { - name string - prBody string - repoPath string - repoURL string - push bool - wantErr bool - }{ - { - name: "runs successfully", - prBody: `
go-google-cloud-pubsub-v1: v1.0.1 - ### Features - - feat: new feature -
`, - repoURL: "https://github.com/googleapis/librarian", - }, - { - name: "runs successfully from cloned repo", - prBody: `
go-google-cloud-pubsub-v1: v1.0.1 -### Features -- feat: new feature -
`, - repoPath: "testdata/e2e/release/stage/single_commit", - }, - } { - t.Run(test.name, func(t *testing.T) { - headSHA := "abcdef123456" - - // Set up a mock GitHub API server using httptest. - // This server will intercept HTTP requests made by the librarian command - // and provide canned responses, avoiding any real calls to the GitHub API. - // The handlers below simulate the endpoints that 'release tag' interacts with. - var server *httptest.Server - server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // Verify that the GitHub token is being sent correctly. - if r.Header.Get("Authorization") != "Bearer fake-token" { - t.Errorf("missing or wrong authorization header: got %q", r.Header.Get("Authorization")) - } - - const stateYAMLContent = ` -image: gcr.io/some-project/some-image:latest -libraries: -- id: go-google-cloud-pubsub-v1 - source_roots: - - google-cloud-pubsub/v1 - tag_format: go-google-cloud-pubsub-v1-{version} -` - // The download URL can be any unique path. The mock server will handle it. - downloadURL := server.URL + "/raw/librarian/state.yaml" - - // Handler for the .librarian DIRECTORY listing request. - // The client sends this to find the state.yaml file. - if r.Method == "GET" && r.URL.Path == "/repos/googleapis/librarian/contents/.librarian" { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - // CRITICAL: The response for the directory listing must include the `download_url` for the file. - fmt.Fprintf(w, `[{"name": "state.yaml", "path": ".librarian/state.yaml", "type": "file", "download_url": %q}]`, downloadURL) - return - } - - // Handler for the raw CONTENT download request. - // The client hits this endpoint after extracting the download_url from the directory listing. - if r.Method == "GET" && r.URL.Path == "/raw/librarian/state.yaml" { - w.WriteHeader(http.StatusOK) - fmt.Fprint(w, stateYAMLContent) - return - } - - // Mock endpoint for the .librarian directory listing. - // This handles the preliminary request the GitHub client makes before fetching a file. - if r.Method == "GET" && r.URL.Path == "/repos/googleapis/librarian/contents/.librarian" { - w.WriteHeader(http.StatusOK) - // This response tells the client that the directory contains a file named state.yaml - fmt.Fprint(w, `[{"name": "state.yaml", "path": ".librarian/state.yaml", "type": "file"}]`) - return - } - - // Mock endpoint for GET /.librarian/state.yaml - if r.Method == "GET" && strings.HasSuffix(r.URL.Path, ".librarian/state.yaml") { - w.WriteHeader(http.StatusOK) - fmt.Fprint(w, stateYAMLContent) - return - } - - // Mock endpoint for GET /repos/{owner}/{repo}/pulls/{number} - if r.Method == "GET" && strings.HasSuffix(r.URL.Path, "/pulls/123") { - w.WriteHeader(http.StatusOK) - // Return a minimal PR object with the body and merge commit SHA. - fmt.Fprintf(w, `{"number": 123, "body": %q, "merge_commit_sha": %q, "base": {"ref": "main"}}`, test.prBody, headSHA) - return - } - - // Mock endpoint for POST /repos/{owner}/{repo}/git/refs (creating the release-please tag) - if r.Method == "POST" && strings.HasSuffix(r.URL.Path, "/git/refs") { - w.WriteHeader(http.StatusCreated) - fmt.Fprint(w, `{"ref": "refs/tags/release-please-123"}`) - return - } - - // Mock endpoint for POST /repos/{owner}/{repo}/releases (creating the GitHub Release) - if r.Method == "POST" && strings.HasSuffix(r.URL.Path, "/releases") { - var newRelease github.RepositoryRelease - if err := json.NewDecoder(r.Body).Decode(&newRelease); err != nil { - t.Fatalf("failed to decode request body: %v", err) - } - expectedTagName := "go-google-cloud-pubsub-v1-v1.0.1" - if *newRelease.TagName != expectedTagName { - t.Errorf("unexpected tag name: got %q, want %q", *newRelease.TagName, expectedTagName) - } - if *newRelease.TargetCommitish != headSHA { - t.Errorf("unexpected commitish: got %q, want %q", *newRelease.TargetCommitish, headSHA) - } - w.WriteHeader(http.StatusCreated) - fmt.Fprint(w, `{"name": "v1.0.1"}`) - return - } - - // Mock endpoint for PUT /repos/{owner}/{repo}/issues/{number}/labels (updating labels) - if r.Method == "PUT" && strings.HasSuffix(r.URL.Path, "/issues/123/labels") { - w.WriteHeader(http.StatusOK) - fmt.Fprint(w, `[]`) - return - } - - // If any other request is made, fail the test. - t.Fatalf("unexpected request: %s %s", r.Method, r.URL.Path) - })) - defer server.Close() - - repo := test.repoURL - if test.repoPath != "" { - repo = t.TempDir() - err := initRepo(t, repo, test.repoPath, "initial commit") - if err != nil { - t.Fatalf("error initializing fake git repo %s", err) - } - } - cmdArgs := []string{ - "run", - "github.com/googleapis/librarian/cmd/legacylibrarian", - "release", - "tag", - fmt.Sprintf("--repo=%s", repo), - fmt.Sprintf("--github-api-endpoint=%s/", server.URL), - "--pr=https://github.com/googleapis/librarian/pull/123", - } - if test.push { - cmdArgs = append(cmdArgs, "--push") - } - - cmd := exec.Command("go", cmdArgs...) - cmd.Env = append(os.Environ(), fmt.Sprintf("%s=fake-token", legacyconfig.LibrarianGithubToken)) - cmd.Stderr = os.Stderr - cmd.Stdout = os.Stdout - if err := cmd.Run(); err != nil { - if !test.wantErr { - t.Fatalf("Failed to run release tag: %v", err) - } - } - }) - } -} - -// newMockGitHubServer creates a mock GitHub API server for testing --push functionality. -func newMockGitHubServer(t *testing.T, prTitleFragment string, expectedContentInPr []string, notExpectedContentInPr []string) *httptest.Server { - t.Helper() - return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Header.Get("Authorization") != "Bearer fake-token" { - t.Errorf("missing or wrong authorization header: got %q", r.Header.Get("Authorization")) - } - - // Mock endpoint for POST /repos/{owner}/{repo}/pulls - if r.Method == "POST" && strings.HasSuffix(r.URL.Path, "/pulls") { - var newPR github.NewPullRequest - if err := json.NewDecoder(r.Body).Decode(&newPR); err != nil { - t.Fatalf("failed to decode request body: %v", err) - } - expectedTitle := fmt.Sprintf("chore: librarian %s pull request", prTitleFragment) - if !strings.Contains(*newPR.Title, expectedTitle) { - t.Errorf("unexpected PR title: got %q, want to contain %q", *newPR.Title, expectedTitle) - } - for _, expectedContent := range expectedContentInPr { - htmlEscapedContent := html.EscapeString(expectedContent) - if !strings.Contains(*newPR.Body, htmlEscapedContent) { - t.Errorf("unexpected PR description: got %q, missing %q", *newPR.Body, htmlEscapedContent) - } - } - for _, notExpectedContent := range notExpectedContentInPr { - if strings.Contains(*newPR.Body, notExpectedContent) { - t.Errorf("unexpected PR description: got %q, should not contain %q", *newPR.Body, notExpectedContent) - } - } - if *newPR.Base != "main" { - t.Errorf("unexpected PR base: got %q", *newPR.Base) - } - w.WriteHeader(http.StatusCreated) - fmt.Fprint(w, `{"number": 123, "html_url": "https://github.com/googleapis/librarian/pull/123"}`) - return - } - - // Mock endpoint for POST /repos/{owner}/{repo}/issues/{number}/labels - if r.Method == "POST" && strings.Contains(r.URL.Path, "/issues/123/labels") { - w.WriteHeader(http.StatusOK) - fmt.Fprint(w, `[]`) - return - } - - t.Fatalf("unexpected request: %s %s", r.Method, r.URL.Path) - })) -} - -// initRepo initiates a git repo in the given directory, copy -// files from source directory and create a commit. -func initRepo(t *testing.T, dir, source, message string) error { - t.Logf("initializing repo, dir %s, source %s", dir, source) - if err := os.CopyFS(dir, os.DirFS(source)); err != nil { - return err - } - runGit(t, dir, "init") - runGit(t, dir, "add", ".") - runGit(t, dir, "config", "user.email", "test@github.com") - runGit(t, dir, "config", "user.name", "Test User") - runGit(t, dir, "commit", "-m", message) - runGit(t, dir, "remote", "add", "origin", "https://github.com/googleapis/librarian.git") - return nil -} - -type genResponse struct { - ErrorMessage string `json:"error,omitempty"` -} - -// runGit runs a git command in the specified directory and returns the trimmed output as a string. -func runGit(t *testing.T, dir string, args ...string) string { - t.Helper() - cmd := exec.Command("git", args...) - cmd.Dir = dir - out, err := cmd.Output() - if err != nil { - t.Fatalf("failed to run git command: %s, %v", args, err) - } - return strings.TrimSpace(string(out)) -} - -// createCommit creates a new commit in the given repo directory with a dummy file using the specified commit message. -func createCommit(t *testing.T, dir, filePath string, commitMsg string) error { - fileToCommit := filepath.Join(dir, filePath) - - file, err := os.OpenFile(fileToCommit, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - return fmt.Errorf("failed to open file %s: %w", fileToCommit, err) - } - defer file.Close() - runGit(t, dir, "add", ".") - runGit(t, dir, "commit", "-m", commitMsg) - return nil -} - -// setupStateFile creates a .librarian/state.yaml file in the given repo directory with the provided lastGeneratedCommit. -func setupStateFile(t *testing.T, repoInitDir, lastGeneratedCommit string) { - state := &legacyconfig.LibrarianState{ - Image: "test-image:latest", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "go-google-cloud-pubsub-v1", - Version: "v1.0.0", - APIs: []*legacyconfig.API{ - { - Path: "google/cloud/pubsub/v1", - }, - }, - SourceRoots: []string{"pubsub"}, - RemoveRegex: []string{ - "pubsub/file_to_remove.txt", - "^pubsub/sub/.*.txt", - }, - PreserveRegex: []string{ - "pubsub/sub/file_to_preserve.txt", - }, - LastGeneratedCommit: lastGeneratedCommit, - }, - }, - } - stateBytes, err := yaml.Marshal(state) - if err != nil { - t.Fatal(err) - } - if err := os.MkdirAll(filepath.Join(repoInitDir, ".librarian"), 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(repoInitDir, ".librarian", "state.yaml"), stateBytes, 0644); err != nil { - t.Fatal(err) - } -} - -// parseCommitMessageForPRContent extracts lines that should be included in the PR description from commit message. -func parseCommitMessageForPRContent(content string) []string { - matches := nestedCommitRegex.FindStringSubmatch(content) - nestedContent := content - if len(matches) > 1 { - nestedContent = matches[1] - } - - var result []string - for _, line := range strings.Split(nestedContent, "\n") { - if match := conventionalCommitRegex.FindStringSubmatch(strings.TrimSpace(line)); match != nil { - result = append(result, match[2]) - } - } - - return result -} diff --git a/internal/legacylibrarian/legacyintegration/system_test.go b/internal/legacylibrarian/legacyintegration/system_test.go deleted file mode 100644 index 8fbeed3163e..00000000000 --- a/internal/legacylibrarian/legacyintegration/system_test.go +++ /dev/null @@ -1,568 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build integration - -package integration_test - -import ( - "fmt" - "log/slog" - "os" - "path" - "path/filepath" - "strings" - "testing" - "time" - - "github.com/google/go-cmp/cmp" -) - -var ( - testToken = os.Getenv("LIBRARIAN_TEST_GITHUB_TOKEN") - githubAction = os.Getenv("LIBRARIAN_GITHUB_ACTION") -) - -func TestGetRawContentSystem(t *testing.T) { - if testToken == "" { - t.Skip("LIBRARIAN_TEST_GITHUB_TOKEN not set, skipping GitHub integration test") - } - repoName := "https://github.com/googleapis/librarian" - - for _, test := range []struct { - name string - token string - path string - wantErr bool - wantErrSubstr string - }{ - { - name: "with credentials, existing file", - token: testToken, - path: ".librarian/state.yaml", - wantErr: false, - }, - { - name: "with credentials, missing file", - token: testToken, - path: "not-a-real-file.txt", - wantErr: true, - wantErrSubstr: "no file named", - }, - { - name: "without credentials, existing file", - token: "", - path: ".librarian/state.yaml", - wantErr: false, - }, - { - name: "without credentials, missing file", - token: "", - path: "not-a-real-file.txt", - wantErr: true, - wantErrSubstr: "no file named", - }, - } { - t.Run(test.name, func(t *testing.T) { - repo, err := github.ParseRemote(repoName) - if err != nil { - t.Fatalf("unexpected error in ParseRemote() %s", err) - } - - client := github.NewClient(test.token, repo) - got, err := client.GetRawContent(t.Context(), test.path, "main") - - if test.wantErr { - if err == nil { - t.Fatalf("GetRawContent() err = nil, want error containing %q", test.wantErrSubstr) - } else if !strings.Contains(err.Error(), test.wantErrSubstr) { - t.Errorf("GetRawContent() err = %v, want error containing %q", err, test.wantErrSubstr) - } - return - } - if err != nil { - t.Errorf("GetRawContent() err = %v, want nil", err) - } - if len(got) <= 0 { - t.Fatalf("GetRawContent() expected to fetch contents for %s from %s", test.path, repoName) - } - }) - } -} - -func TestPullRequestSystem(t *testing.T) { - // Clone a repo - // Create a commit and push - // Create a pull request - // Add a label to the pull request - // Fetch labels for the issue and verify - // Replace the issue labels - // Fetch labels for the issue and verify - // Add a comment - // Search for the pull request - // Fetch the pull request - // Close the pull request - if testToken == "" { - t.Skip("LIBRARIAN_TEST_GITHUB_TOKEN not set, skipping GitHub integration test") - } - testRepoName := os.Getenv("LIBRARIAN_TEST_GITHUB_REPO") - if testRepoName == "" { - t.Skip("LIBRARIAN_TEST_GITHUB_REPO not set, skipping GitHub integration test") - } - - // Clone a repo - workdir := path.Join(t.TempDir(), "test-repo") - localRepository, err := gitrepo.NewRepository(&gitrepo.RepositoryOptions{ - Dir: workdir, - MaybeClone: true, - RemoteURL: testRepoName, - RemoteBranch: "main", - GitPassword: testToken, - Depth: 1, - }) - if err != nil { - t.Fatalf("unexpected error in NewRepository() %s", err) - } - repo, err := github.ParseRemote(testRepoName) - if err != nil { - t.Fatalf("unexpected error in ParseRemote() %s", err) - } - - now := time.Now() - branchName := fmt.Sprintf("integration-test-%s", now.Format("20060102150405")) - err = localRepository.CreateBranchAndCheckout(branchName) - if err != nil { - t.Fatalf("unexpected error in CreateBranchAndCheckout() %s", err) - } - - // Create a commit and push - err = os.WriteFile(path.Join(workdir, "some-file.txt"), []byte("some-content"), 0644) - if err != nil { - t.Fatalf("unexpected error writing a file to git repo %s", err) - } - err = localRepository.AddAll() - if err != nil { - t.Fatalf("unexpected error in AddAll() %s", err) - } - err = localRepository.Commit("build: add test file") - if err != nil { - t.Fatalf("unexpected error in Commit() %s", err) - } - err = localRepository.Push(branchName) - if err != nil { - t.Fatalf("unexpected error in Push() %s", err) - } - - cleanupBranch := func() { - slog.Info("cleaning up created branch", "branch", branchName) - err := localRepository.DeleteBranch(branchName) - if err != nil { - t.Fatalf("unexpected error in DeleteBranch() %s", err) - } - } - defer cleanupBranch() - - // Create a pull request - client := github.NewClient(testToken, repo) - createdPullRequest, err := client.CreatePullRequest(t.Context(), repo, branchName, "main", "test: integration test", "do not merge", true) - if err != nil { - t.Fatalf("unexpected error in CreatePullRequest() %s", err) - } - t.Logf("created pull request: %d", createdPullRequest.Number) - - // Ensure we clean up the created PR. The pull request is closed later in the - // test, but this should make sure unless ClosePullRequest() is not working. - cleanupPR := func() { - slog.Info("cleaning up opened pull request") - client.ClosePullRequest(t.Context(), createdPullRequest.Number) - } - defer cleanupPR() - - // Add a label to the pull request - labels := []string{"do not merge", "type: cleanup"} - err = client.AddLabelsToIssue(t.Context(), repo, createdPullRequest.Number, labels) - if err != nil { - t.Fatalf("unexpected error in AddLabelsToIssue() %s", err) - } - - // Get labels and verify - foundLabels, err := client.GetLabels(t.Context(), createdPullRequest.Number) - if err != nil { - t.Fatalf("unexpected error in GetLabels() %s", err) - } - if diff := cmp.Diff(foundLabels, labels); diff != "" { - t.Fatalf("GetLabels() mismatch (-want + got):\n%s", diff) - } - - // Replace labels - labels = []string{"foo", "bar"} - err = client.ReplaceLabels(t.Context(), createdPullRequest.Number, labels) - if err != nil { - t.Fatalf("unexpected error in ReplaceLabels() %s", err) - } - - // Get labels and verify - foundLabels, err = client.GetLabels(t.Context(), createdPullRequest.Number) - if err != nil { - t.Fatalf("unexpected error in GetLabels() %s", err) - } - if diff := cmp.Diff(foundLabels, labels); diff != "" { - t.Fatalf("GetLabels() mismatch (-want + got):\n%s", diff) - } - - // Add label - err = client.AddLabelsToIssue(t.Context(), repo, createdPullRequest.Number, []string{"librarian-test", "asdf"}) - if err != nil { - t.Fatalf("unexpected error in AddLabelsToIssue() %s", err) - } - - // Get labels and verify - wantLabels := []string{"foo", "bar", "librarian-test", "asdf"} - foundLabels, err = client.GetLabels(t.Context(), createdPullRequest.Number) - if err != nil { - t.Fatalf("unexpected error in GetLabels() %s", err) - } - if diff := cmp.Diff(foundLabels, wantLabels); diff != "" { - t.Fatalf("GetLabels() mismatch (-want + got):\n%s", diff) - } - - // Add a comment - err = client.CreateIssueComment(t.Context(), createdPullRequest.Number, "some comment body") - if err != nil { - t.Fatalf("unexpected error in CreateIssueComment() %s", err) - } - - // Search for pull requests (this may take a bit of time, so try 5 times) - found := false - for i := 0; i < 5; i++ { - foundPullRequests, err := client.SearchPullRequests(t.Context(), "label:librarian-test is:open") - if err != nil { - t.Fatalf("unexpected error in SearchPullRequests() %s", err) - } - for _, pullRequest := range foundPullRequests { - // Look for the PR we created - if pullRequest.GetNumber() == createdPullRequest.Number { - found = true - - // Expect that we found a comment - if pullRequest.GetComments() == 0 { - t.Fatalf("Expected to have created a comment on the pull request.") - } - - if !pullRequest.GetDraft() { - t.Fatalf("Expected created pull request to have been a draft.") - } - break - } - } - if found { - break - } - delay := time.Duration(2 * time.Second) - t.Logf("Retrying in %v...\n", delay) - time.Sleep(delay) - } - if !found { - t.Fatalf("failed to find pull request after 5 attempts") - } - - // Close pull request - err = client.ClosePullRequest(t.Context(), createdPullRequest.Number) - if err != nil { - t.Fatalf("unexpected error in ClosePullRequest() %s", err) - } - - // Get single pull request - foundPullRequest, err := client.GetPullRequest(t.Context(), createdPullRequest.Number) - if err != nil { - t.Fatalf("unexpected error in GetPullRequest() %s", err) - } - if diff := cmp.Diff(foundPullRequest.GetNumber(), createdPullRequest.Number); diff != "" { - t.Fatalf("pull request number mismatch (-want + got):\n%s", diff) - } - if diff := cmp.Diff(foundPullRequest.GetState(), "closed"); diff != "" { - t.Fatalf("pull request state mismatch (-want + got):\n%s", diff) - } -} - -func TestFindMergedPullRequest(t *testing.T) { - if testToken == "" { - t.Skip("LIBRARIAN_TEST_GITHUB_TOKEN not set, skipping GitHub integration test") - } - repoName := "https://github.com/googleapis/librarian" - repo, err := github.ParseRemote(repoName) - if err != nil { - t.Fatalf("unexpected error in ParseRemote() %s", err) - } - - for _, test := range []struct { - name string - label string - want int - wantErr bool - wantErrSubstr string - }{ - { - name: "existing label", - label: "cla: yes", - want: 2210, - wantErr: false, - }, - { - name: "non-existing label", - label: "non-existing-label", - want: 0, - wantErr: false, - }, - } { - t.Run(test.name, func(t *testing.T) { - client := github.NewClient(testToken, repo) - prs, err := client.FindMergedPullRequestsWithLabel(t.Context(), repo.Owner, repo.Name, test.label) - if test.wantErr { - if err == nil { - t.Fatalf("FindMergedPullRequestsWithLabel() err = nil, want error containing %q", test.wantErrSubstr) - } else if !strings.Contains(err.Error(), test.wantErrSubstr) { - t.Errorf("FindMergedPullRequestsWithLabel() err = %v, want error containing %q", err, test.wantErrSubstr) - } - return - } - if err != nil { - t.Errorf("FindMergedPullRequestsWithLabel() err = %v, want nil", err) - } - if test.want == 0 { - // expect to not find any - if len(prs) > 0 { - t.Fatalf("FindMergedPullRequestWithLabel() expected to not find any PRs, found %d", len(prs)) - } - } else { - found := false - for _, pr := range prs { - pullNumber := pr.GetNumber() - t.Logf("Found PR %d", pullNumber) - if pr.Number != nil && pullNumber == test.want { - found = true - } - } - if !found { - t.Fatalf("FindMergedPullRequestsWithLabel() expected to find PR #%d", test.want) - } - } - }) - } -} - -func TestCreateTag(t *testing.T) { - if testToken == "" { - t.Skip("LIBRARIAN_TEST_GITHUB_TOKEN not set, skipping GitHub integration test") - } - testRepoName := os.Getenv("LIBRARIAN_TEST_GITHUB_REPO") - if testRepoName == "" { - t.Skip("LIBRARIAN_TEST_GITHUB_REPO not set, skipping GitHub integration test") - } - repo, err := github.ParseRemote(testRepoName) - if err != nil { - t.Fatalf("unexpected error in ParseRemote() %s", err) - } - // Clone a repo - workdir := path.Join(t.TempDir(), "test-repo") - localRepository, err := gitrepo.NewRepository(&gitrepo.RepositoryOptions{ - Dir: workdir, - MaybeClone: true, - RemoteURL: testRepoName, - RemoteBranch: "main", - GitPassword: testToken, - Depth: 1, - }) - if err != nil { - t.Fatalf("unexpected error in NewRepository() %s", err) - } - - now := time.Now() - tagName := fmt.Sprintf("create-tag-test-%s", now.Format("20060102150405")) - commitSHA, err := localRepository.HeadHash() - if err != nil { - t.Fatalf("unexpected error fetching head commit %s", err) - } - - client := github.NewClient(testToken, repo) - err = client.CreateTag(t.Context(), tagName, commitSHA) - if err != nil { - t.Fatalf("unexpected error in CreateTag() %s", err) - } -} - -func TestCreateRelease(t *testing.T) { - if testToken == "" { - t.Skip("LIBRARIAN_TEST_GITHUB_TOKEN not set, skipping GitHub integration test") - } - testRepoName := os.Getenv("LIBRARIAN_TEST_GITHUB_REPO") - if testRepoName == "" { - t.Skip("LIBRARIAN_TEST_GITHUB_REPO not set, skipping GitHub integration test") - } - repo, err := github.ParseRemote(testRepoName) - if err != nil { - t.Fatalf("unexpected error in ParseRemote() %s", err) - } - // Clone a repo - workdir := path.Join(t.TempDir(), "test-repo") - localRepository, err := gitrepo.NewRepository(&gitrepo.RepositoryOptions{ - Dir: workdir, - MaybeClone: true, - RemoteURL: testRepoName, - RemoteBranch: "main", - GitPassword: testToken, - Depth: 1, - }) - if err != nil { - t.Fatalf("unexpected error in NewRepository() %s", err) - } - - now := time.Now() - tagName := fmt.Sprintf("create-release-test-%s", now.Format("20060102150405")) - commitSHA, err := localRepository.HeadHash() - if err != nil { - t.Fatalf("unexpected error fetching head commit %s", err) - } - - client := github.NewClient(testToken, repo) - body := "some release body" - release, err := client.CreateRelease(t.Context(), tagName, tagName, body, commitSHA) - if err != nil { - t.Fatalf("unexpected error in CreateTag() %s", err) - } - if diff := cmp.Diff(release.GetBody(), body); diff != "" { - t.Fatalf("release body mismatch (-want + got):\n%s", diff) - } -} - -func TestFindLatestImage(t *testing.T) { - t.Logf("Note: This test requires authentication with the Artifact Registry in project 'cloud-sdk-librarian-prod'.") - - // If we are able to configure system tests on GitHub actions, then update this - // guard clause. - if githubAction != "" { - t.Skip("skipping on GitHub actions") - } - for _, test := range []struct { - name string - image string - wantDiff bool - wantErr bool - }{ - { - name: "AR unpinned", - image: "us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/librarian-go@sha256:dea280223eca5a0041fb5310635cec9daba2f01617dbfb1e47b90c77368b5620", - wantDiff: true, - }, - { - name: "AR pinned", - image: "us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/librarian-go@sha256:dea280223eca5a0041fb5310635cec9daba2f01617dbfb1e47b90c77368b5620", - wantDiff: true, - }, - { - name: "invalid image", - image: "gcr.io/some-project/some-name", - wantErr: true, - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - client, err := images.NewArtifactRegistryClient(t.Context()) - if err != nil { - t.Fatalf("unexpected error in NewArtifactRegistryClient() %v", err) - } - defer client.Close() - got, err := client.FindLatest(t.Context(), test.image) - if test.wantErr { - if err == nil { - t.Errorf("FindLatestImage() error = %v, wantErr %v", err, test.wantErr) - } - return - } - - if err != nil { - t.Fatalf("FindLatestImage() error = %v", err) - } - - if !strings.HasPrefix(got, "us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/librarian-go@sha256:") { - t.Fatalf("FindLatestImage() unexpected image format") - } - if test.wantDiff { - if got == test.image { - t.Fatalf("FindLatestImage() expected to change") - } - } else { - if got != test.image { - t.Fatalf("FindLatestImage() expected to stay the same") - } - } - }) - } -} - -func TestGitCheckout(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - sha string - want string - wantErr bool - }{ - { - name: "known SHA", - // v0.3.0 release - sha: "2e230f309505db42ce8becb0f3946d608a11a61c", - want: "chore: librarian release pull request: 20250925T070206Z (#2356)", - }, - { - name: "unknown SHA", - sha: "should not exist", - wantErr: true, - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - repo, err := gitrepo.NewRepository(&gitrepo.RepositoryOptions{ - Dir: filepath.Join(t.TempDir(), "librarian"), - MaybeClone: true, - RemoteURL: "https://github.com/googleapis/librarian", - RemoteBranch: "main", - }) - if err != nil { - t.Fatalf("error cloning repository, %v", err) - } - - err = repo.Checkout(test.sha) - - if test.wantErr { - if err == nil { - t.Fatal("Checkout() expected to return error") - } - return - } - - if err != nil { - t.Fatalf("Checkout() unexpected error: %v", err) - } - - headSha, err := repo.HeadHash() - if diff := cmp.Diff(test.sha, headSha); diff != "" { - t.Fatalf("Checkout() mismatch (-want +got):\n%s", diff) - } - if err != nil { - t.Fatalf("Checkout() unexpected error fetching HeadHash: %v", err) - } - }) - } -} diff --git a/internal/legacylibrarian/legacyintegration/testdata/e2e-test.Dockerfile b/internal/legacylibrarian/legacyintegration/testdata/e2e-test.Dockerfile deleted file mode 100644 index 0933e7b94d8..00000000000 --- a/internal/legacylibrarian/legacyintegration/testdata/e2e-test.Dockerfile +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Start with a Go base image -FROM golang:1.25.3 AS builder - -WORKDIR /app - -COPY go.mod . -COPY go.sum . - -RUN go mod download - -COPY ./internal/legacylibrarian/legacyintegration/testdata/e2e_func.go . - -RUN go build -o e2e_func . - -FROM alpine:latest - -WORKDIR /app - -# Copy the built executable from the builder stage -COPY --from=builder /app/e2e_func . - -ENTRYPOINT ["/app/e2e_func"] diff --git a/internal/legacylibrarian/legacyintegration/testdata/e2e/configure/api_root/google/cloud/another-library/v3/another_v3.yaml b/internal/legacylibrarian/legacyintegration/testdata/e2e/configure/api_root/google/cloud/another-library/v3/another_v3.yaml deleted file mode 100644 index d7d453cd4cf..00000000000 --- a/internal/legacylibrarian/legacyintegration/testdata/e2e/configure/api_root/google/cloud/another-library/v3/another_v3.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -type: google.api.Service diff --git a/internal/legacylibrarian/legacyintegration/testdata/e2e/configure/api_root/google/cloud/new-library-path/v2/library_v2.yaml b/internal/legacylibrarian/legacyintegration/testdata/e2e/configure/api_root/google/cloud/new-library-path/v2/library_v2.yaml deleted file mode 100644 index d7d453cd4cf..00000000000 --- a/internal/legacylibrarian/legacyintegration/testdata/e2e/configure/api_root/google/cloud/new-library-path/v2/library_v2.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -type: google.api.Service diff --git a/internal/legacylibrarian/legacyintegration/testdata/e2e/configure/api_root/google/cloud/pubsub/v1/pubsub_v1.yaml b/internal/legacylibrarian/legacyintegration/testdata/e2e/configure/api_root/google/cloud/pubsub/v1/pubsub_v1.yaml deleted file mode 100644 index d7d453cd4cf..00000000000 --- a/internal/legacylibrarian/legacyintegration/testdata/e2e/configure/api_root/google/cloud/pubsub/v1/pubsub_v1.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -type: google.api.Service diff --git a/internal/legacylibrarian/legacyintegration/testdata/e2e/configure/repo_init/.librarian/config.yaml b/internal/legacylibrarian/legacyintegration/testdata/e2e/configure/repo_init/.librarian/config.yaml deleted file mode 100644 index 9f71a2dc3c5..00000000000 --- a/internal/legacylibrarian/legacyintegration/testdata/e2e/configure/repo_init/.librarian/config.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - diff --git a/internal/legacylibrarian/legacyintegration/testdata/e2e/configure/repo_init/.librarian/state.yaml b/internal/legacylibrarian/legacyintegration/testdata/e2e/configure/repo_init/.librarian/state.yaml deleted file mode 100644 index 4233489d477..00000000000 --- a/internal/legacylibrarian/legacyintegration/testdata/e2e/configure/repo_init/.librarian/state.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -image: test-image:latest -libraries: - - id: example-library - version: - last_generated_commit: - apis: - - path: google/cloud/pubsub/v1 - source_roots: - - google-cloud-pubsub/v1 diff --git a/internal/legacylibrarian/legacyintegration/testdata/e2e/configure/repo_init/google-cloud-pubsub/v1/example.txt b/internal/legacylibrarian/legacyintegration/testdata/e2e/configure/repo_init/google-cloud-pubsub/v1/example.txt deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/internal/legacylibrarian/legacyintegration/testdata/e2e/configure/updated-state.yaml b/internal/legacylibrarian/legacyintegration/testdata/e2e/configure/updated-state.yaml deleted file mode 100644 index c00c182984e..00000000000 --- a/internal/legacylibrarian/legacyintegration/testdata/e2e/configure/updated-state.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -image: test-image:latest -libraries: - - id: example-library - version: "" - last_generated_commit: "" - apis: - - path: google/cloud/pubsub/v1 - service_config: pubsub_v1.yaml - source_roots: - - google-cloud-pubsub/v1 - preserve_regex: [] - remove_regex: [] - - id: new-library - version: 1.0.0 - last_generated_commit: "" - apis: - - path: google/cloud/new-library-path/v2 - service_config: library_v2.yaml - source_roots: - - example-source-path - - example-source-path-2 - preserve_regex: - - example-preserve-regex - - example-preserve-regex-2 - remove_regex: - - example-remove-regex - - example-remove-regex-2 diff --git a/internal/legacylibrarian/legacyintegration/testdata/e2e/generate/api_root/google/cloud/future/v2/future_v2.yaml b/internal/legacylibrarian/legacyintegration/testdata/e2e/generate/api_root/google/cloud/future/v2/future_v2.yaml deleted file mode 100644 index d7d453cd4cf..00000000000 --- a/internal/legacylibrarian/legacyintegration/testdata/e2e/generate/api_root/google/cloud/future/v2/future_v2.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -type: google.api.Service diff --git a/internal/legacylibrarian/legacyintegration/testdata/e2e/generate/api_root/google/cloud/pubsub/v1/pubsub_v1.yaml b/internal/legacylibrarian/legacyintegration/testdata/e2e/generate/api_root/google/cloud/pubsub/v1/pubsub_v1.yaml deleted file mode 100644 index d7d453cd4cf..00000000000 --- a/internal/legacylibrarian/legacyintegration/testdata/e2e/generate/api_root/google/cloud/pubsub/v1/pubsub_v1.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -type: google.api.Service diff --git a/internal/legacylibrarian/legacyintegration/testdata/e2e/generate/multi_repo_all_fail_init/.librarian/state.yaml b/internal/legacylibrarian/legacyintegration/testdata/e2e/generate/multi_repo_all_fail_init/.librarian/state.yaml deleted file mode 100644 index f5d90b9dfca..00000000000 --- a/internal/legacylibrarian/legacyintegration/testdata/e2e/generate/multi_repo_all_fail_init/.librarian/state.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -image: test-image:latest -libraries: - - id: simulate-command-error-id - version: v2.0.0 - apis: - - path: google/cloud/future/v2 - source_roots: - - future - - id: simulate-command-error-id - version: v2.0.0 - apis: - - path: google/cloud/future/v2 - source_roots: - - another-future diff --git a/internal/legacylibrarian/legacyintegration/testdata/e2e/generate/multi_repo_init/.librarian/state.yaml b/internal/legacylibrarian/legacyintegration/testdata/e2e/generate/multi_repo_init/.librarian/state.yaml deleted file mode 100644 index 46d0f8981ef..00000000000 --- a/internal/legacylibrarian/legacyintegration/testdata/e2e/generate/multi_repo_init/.librarian/state.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -image: test-image:latest -libraries: - - id: go-google-cloud-pubsub-v1 - version: v1.0.0 - apis: - - path: google/cloud/pubsub/v1 - source_roots: - - pubsub - - id: go-google-cloud-future-v2 - version: v2.0.0 - apis: - - path: google/cloud/future/v2 - source_roots: - - future diff --git a/internal/legacylibrarian/legacyintegration/testdata/e2e/generate/multi_repo_one_fails_init/.librarian/state.yaml b/internal/legacylibrarian/legacyintegration/testdata/e2e/generate/multi_repo_one_fails_init/.librarian/state.yaml deleted file mode 100644 index 598e83bf195..00000000000 --- a/internal/legacylibrarian/legacyintegration/testdata/e2e/generate/multi_repo_one_fails_init/.librarian/state.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -image: test-image:latest -libraries: - - id: go-google-cloud-pubsub-v1 - version: v1.0.0 - apis: - - path: google/cloud/pubsub/v1 - source_roots: - - pubsub - - id: simulate-command-error-id - version: v2.0.0 - apis: - - path: google/cloud/future/v2 - source_roots: - - future diff --git a/internal/legacylibrarian/legacyintegration/testdata/e2e/generate/repo_init/.librarian/config.yaml b/internal/legacylibrarian/legacyintegration/testdata/e2e/generate/repo_init/.librarian/config.yaml deleted file mode 100644 index 9f71a2dc3c5..00000000000 --- a/internal/legacylibrarian/legacyintegration/testdata/e2e/generate/repo_init/.librarian/config.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - diff --git a/internal/legacylibrarian/legacyintegration/testdata/e2e/generate/repo_init/.librarian/state.yaml b/internal/legacylibrarian/legacyintegration/testdata/e2e/generate/repo_init/.librarian/state.yaml deleted file mode 100644 index 0f494506cd4..00000000000 --- a/internal/legacylibrarian/legacyintegration/testdata/e2e/generate/repo_init/.librarian/state.yaml +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -image: test-image:latest -libraries: - - id: example-library - version: - last_generated_commit: - apis: - - path: google/cloud/pubsub/v1 - source_roots: - - google-cloud-pubsub/v1 - remove_regex: - - google-cloud-pubsub/v1 - - id: simulate-command-error-id - version: - last_generated_commit: - apis: - - path: google/cloud/future/v2 - source_roots: - - google-cloud-future/v2 diff --git a/internal/legacylibrarian/legacyintegration/testdata/e2e/generate/repo_init/google-cloud-future/v2/example.txt b/internal/legacylibrarian/legacyintegration/testdata/e2e/generate/repo_init/google-cloud-future/v2/example.txt deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/internal/legacylibrarian/legacyintegration/testdata/e2e/generate/repo_init/google-cloud-pubsub/v1/example.txt b/internal/legacylibrarian/legacyintegration/testdata/e2e/generate/repo_init/google-cloud-pubsub/v1/example.txt deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/multiple_commits/CHANGELOG.md b/internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/multiple_commits/CHANGELOG.md deleted file mode 100644 index b7cde633af8..00000000000 --- a/internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/multiple_commits/CHANGELOG.md +++ /dev/null @@ -1,19 +0,0 @@ -## 1.1.0 - -- feat: [go-google-cloud-pubsub-v1] Support promptable voices by specifying a model name and a prompt -- feat: [go-google-cloud-pubsub-v1] Add enum value M4A to enum AudioEncoding -- docs: [go-google-cloud-pubsub-v1] A comment for method 'StreamingSynthesize' in service 'TextToSpeech' is changed -- docs: [go-google-cloud-pubsub-v1] A comment for enum value 'AUDIO_ENCODING_UNSPECIFIED' in enum 'AudioEncoding' is changed -- docs: [go-google-cloud-pubsub-v1] A comment for enum value 'OGG_OPUS' in enum 'AudioEncoding' is changed -- docs: [go-google-cloud-pubsub-v1] A comment for enum value 'PCM' in enum 'AudioEncoding' is changed -- docs: [go-google-cloud-pubsub-v1] A comment for field 'low_latency_journey_synthesis' in message '.google.cloud.go-google-cloud-pubsub-v1.v1beta1.AdvancedVoiceOptions' is changed -- docs: [go-google-cloud-pubsub-v1] A comment for enum value 'PHONETIC_ENCODING_IPA' in enum 'PhoneticEncoding' is changed -- docs: [go-google-cloud-pubsub-v1] A comment for enum value 'PHONETIC_ENCODING_X_SAMPA' in enum 'PhoneticEncoding' is changed -- docs: [go-google-cloud-pubsub-v1] A comment for field 'phrase' in message '.google.cloud.go-google-cloud-pubsub-v1.v1beta1.CustomPronunciationParams' is changed -- docs: [go-google-cloud-pubsub-v1] A comment for field 'pronunciations' in message '.google.cloud.go-google-cloud-pubsub-v1.v1beta1.CustomPronunciations' is changed -- docs: [go-google-cloud-pubsub-v1] A comment for message 'MultiSpeakerMarkup' is changed -- docs: [go-google-cloud-pubsub-v1] A comment for field 'custom_pronunciations' in message '.google.cloud.go-google-cloud-pubsub-v1.v1beta1.SynthesisInput' is changed -- docs: [go-google-cloud-pubsub-v1] A comment for field 'voice_clone' in message '.google.cloud.go-google-cloud-pubsub-v1.v1beta1.VoiceSelectionParams' is changed -- docs: [go-google-cloud-pubsub-v1] A comment for field 'speaking_rate' in message '.google.cloud.go-google-cloud-pubsub-v1.v1beta1.AudioConfig' is changed -- docs: [go-google-cloud-pubsub-v1] A comment for field 'audio_encoding' in message '.google.cloud.go-google-cloud-pubsub-v1.v1beta1.StreamingAudioConfig' is changed -- docs: [go-google-cloud-pubsub-v1] A comment for field 'text' in message '.google.cloud.go-google-cloud-pubsub-v1.v1beta1.StreamingSynthesisInput' is changed diff --git a/internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/multiple_commits/commit_msg.txt b/internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/multiple_commits/commit_msg.txt deleted file mode 100644 index 1f4fb96d183..00000000000 --- a/internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/multiple_commits/commit_msg.txt +++ /dev/null @@ -1,38 +0,0 @@ -chore: Update generation configuration at Tue Aug 26 02:31:23 UTC 2025 (#11734) - -This pull request is generated with proto changes between -[googleapis/googleapis@525c95a](https://github.com/googleapis/googleapis/commit/525c95a7a122ec2869ae06cd02fa5013819463f6) -(exclusive) and -[googleapis/googleapis@b738e78](https://github.com/googleapis/googleapis/commit/b738e78ed63effb7d199ed2d61c9e03291b6077f) -(inclusive). - -Librarian Version: v0.0.0-20250918210716-c3c71af9310a -Language Image: -us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/librarian-go:latest - -BEGIN_COMMIT -BEGIN_NESTED_COMMIT -feat: [go-google-cloud-pubsub-v1] Support promptable voices by specifying a model name and a prompt -feat: [go-google-cloud-pubsub-v1] Add enum value M4A to enum AudioEncoding -docs: [go-google-cloud-pubsub-v1] A comment for method 'StreamingSynthesize' in service 'TextToSpeech' is changed -docs: [go-google-cloud-pubsub-v1] A comment for enum value 'AUDIO_ENCODING_UNSPECIFIED' in enum 'AudioEncoding' is changed -docs: [go-google-cloud-pubsub-v1] A comment for enum value 'OGG_OPUS' in enum 'AudioEncoding' is changed -docs: [go-google-cloud-pubsub-v1] A comment for enum value 'PCM' in enum 'AudioEncoding' is changed -docs: [go-google-cloud-pubsub-v1] A comment for field 'low_latency_journey_synthesis' in message '.google.cloud.go-google-cloud-pubsub-v1.v1beta1.AdvancedVoiceOptions' is changed -docs: [go-google-cloud-pubsub-v1] A comment for enum value 'PHONETIC_ENCODING_IPA' in enum 'PhoneticEncoding' is changed -docs: [go-google-cloud-pubsub-v1] A comment for enum value 'PHONETIC_ENCODING_X_SAMPA' in enum 'PhoneticEncoding' is changed -docs: [go-google-cloud-pubsub-v1] A comment for field 'phrase' in message '.google.cloud.go-google-cloud-pubsub-v1.v1beta1.CustomPronunciationParams' is changed -docs: [go-google-cloud-pubsub-v1] A comment for field 'pronunciations' in message '.google.cloud.go-google-cloud-pubsub-v1.v1beta1.CustomPronunciations' is changed -docs: [go-google-cloud-pubsub-v1] A comment for message 'MultiSpeakerMarkup' is changed -docs: [go-google-cloud-pubsub-v1] A comment for field 'custom_pronunciations' in message '.google.cloud.go-google-cloud-pubsub-v1.v1beta1.SynthesisInput' is changed -docs: [go-google-cloud-pubsub-v1] A comment for field 'voice_clone' in message '.google.cloud.go-google-cloud-pubsub-v1.v1beta1.VoiceSelectionParams' is changed -docs: [go-google-cloud-pubsub-v1] A comment for field 'speaking_rate' in message '.google.cloud.go-google-cloud-pubsub-v1.v1beta1.AudioConfig' is changed -docs: [go-google-cloud-pubsub-v1] A comment for field 'audio_encoding' in message '.google.cloud.go-google-cloud-pubsub-v1.v1beta1.StreamingAudioConfig' is changed -docs: [go-google-cloud-pubsub-v1] A comment for field 'text' in message '.google.cloud.go-google-cloud-pubsub-v1.v1beta1.StreamingSynthesisInput' is changed - -PiperOrigin-RevId: 799242210 - -Source Link: -[googleapis/googleapis@b738e78](https://github.com/googleapis/googleapis/commit/b738e78ed63effb7d199ed2d61c9e03291b6077f) -END_NESTED_COMMIT -END_COMMIT diff --git a/internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/multiple_commits/repo_init/.librarian/state.yaml b/internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/multiple_commits/repo_init/.librarian/state.yaml deleted file mode 100644 index 865a774e91f..00000000000 --- a/internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/multiple_commits/repo_init/.librarian/state.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -image: test-image:latest -libraries: - - id: go-google-cloud-pubsub-v1 - version: 1.0.0 - apis: - - path: google/cloud/pubsub/v1 - source_roots: - - google-cloud-pubsub/v1 - tag_format: "{id}/v{version}" diff --git a/internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/multiple_commits/repo_init/google-cloud-pubsub/v1/dummy.go b/internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/multiple_commits/repo_init/google-cloud-pubsub/v1/dummy.go deleted file mode 100644 index d393873c2f9..00000000000 --- a/internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/multiple_commits/repo_init/google-cloud-pubsub/v1/dummy.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pubsub diff --git a/internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/multiple_commits/state.yaml b/internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/multiple_commits/state.yaml deleted file mode 100644 index 87620c33679..00000000000 --- a/internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/multiple_commits/state.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -image: test-image:latest -libraries: - - id: go-google-cloud-pubsub-v1 - version: 1.1.0 - apis: - - path: google/cloud/pubsub/v1 - source_roots: - - google-cloud-pubsub/v1 - tag_format: "{id}/v{version}" diff --git a/internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/multiple_nested_commits/CHANGELOG.md b/internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/multiple_nested_commits/CHANGELOG.md deleted file mode 100644 index 29b82e09255..00000000000 --- a/internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/multiple_nested_commits/CHANGELOG.md +++ /dev/null @@ -1,9 +0,0 @@ -## 1.13.0 - -- feat: [python-google-cloud-video-live-stream-v1] Added H.265 (HEVC) codec support -- feat: [python-google-cloud-video-live-stream-v1] Added UHD (4k) resolution support -- feat: [python-google-cloud-video-live-stream-v1] Added Auto Transcription support -- feat: [python-google-cloud-video-live-stream-v1] Added StartDistribution/StopDistribution methods and Distribution/DistributionStream messages used for distributing live streams to external RTMP/SRT endpoints -- feat: [python-google-cloud-video-live-stream-v1] Added PreviewInput method used for the low latency input monitoring -- feat: [python-google-cloud-video-live-stream-v1] Added UpdateEncryptions event to perform key rotation without restarting a channel -- docs: [python-google-cloud-video-live-stream-v1] Update requirements of resource ID fields to be more clear diff --git a/internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/multiple_nested_commits/commit_msg.txt b/internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/multiple_nested_commits/commit_msg.txt deleted file mode 100644 index b2ac0b4678c..00000000000 --- a/internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/multiple_nested_commits/commit_msg.txt +++ /dev/null @@ -1,111 +0,0 @@ -chore: librarian generate pull request: 20250919T072957Z (#14501) - -This pull request is generated with proto changes between -[googleapis/googleapis@f8776fe](https://github.com/googleapis/googleapis/commit/f8776fec04e336527ba7279d960105533a1c4e21) -(exclusive) and -[googleapis/googleapis@36533b0](https://github.com/googleapis/googleapis/commit/36533b09a6b383f3c30d13c3c26092154ecf5388) -(inclusive). - -Librarian Version: v0.0.0-20250918210716-c3c71af9310a -Language Image: -us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/python-librarian-generator:latest - -BEGIN_COMMIT - -BEGIN_NESTED_COMMIT -fix: [python-google-cloud-eventarc-v1] upgrade gRPC service registration func - -An update to Go gRPC Protobuf generation will change service -registration function signatures to use an interface instead of a -concrete type in generated .pb.go service files. This change should -affect very few client library users. See release notes advisories in -[googleapis/google-cloud-go#11025](https://github.com/googleapis/google-cloud-go/pull/11025). - -PiperOrigin-RevId: 808289479 - -Source-link: -[googleapis/googleapis@36533b0](https://github.com/googleapis/googleapis/commit/36533b09a6b383f3c30d13c3c26092154ecf5388) -END_NESTED_COMMIT - -BEGIN_NESTED_COMMIT -feat: [python-google-cloud-video-live-stream-v1] Added H.265 (HEVC) codec support - -PiperOrigin-RevId: 808091810 - -Source-link: -[googleapis/googleapis@c4e80cd](https://github.com/googleapis/googleapis/commit/c4e80cd2d3321cc37cf1c3713bb02529857728d1) -END_NESTED_COMMIT - -BEGIN_NESTED_COMMIT -feat: [python-google-cloud-video-live-stream-v1] Added UHD (4k) resolution support - -PiperOrigin-RevId: 808091810 - -Source-link: -[googleapis/googleapis@c4e80cd](https://github.com/googleapis/googleapis/commit/c4e80cd2d3321cc37cf1c3713bb02529857728d1) -END_NESTED_COMMIT - -BEGIN_NESTED_COMMIT -feat: [python-google-cloud-video-live-stream-v1] Added Auto Transcription support - -PiperOrigin-RevId: 808091810 - -Source-link: -[googleapis/googleapis@c4e80cd](https://github.com/googleapis/googleapis/commit/c4e80cd2d3321cc37cf1c3713bb02529857728d1) -END_NESTED_COMMIT - -BEGIN_NESTED_COMMIT -feat: [python-google-cloud-video-live-stream-v1] Added StartDistribution/StopDistribution methods and Distribution/DistributionStream messages used for distributing live streams to external RTMP/SRT endpoints - -PiperOrigin-RevId: 808091810 - -Source-link: -[googleapis/googleapis@c4e80cd](https://github.com/googleapis/googleapis/commit/c4e80cd2d3321cc37cf1c3713bb02529857728d1) -END_NESTED_COMMIT - -BEGIN_NESTED_COMMIT -feat: [python-google-cloud-video-live-stream-v1] Added PreviewInput method used for the low latency input monitoring - -PiperOrigin-RevId: 808091810 - -Source-link: -[googleapis/googleapis@c4e80cd](https://github.com/googleapis/googleapis/commit/c4e80cd2d3321cc37cf1c3713bb02529857728d1) -END_NESTED_COMMIT - -BEGIN_NESTED_COMMIT -feat: [python-google-cloud-video-live-stream-v1] Added UpdateEncryptions event to perform key rotation without restarting a channel - -PiperOrigin-RevId: 808091810 - -Source-link: -[googleapis/googleapis@c4e80cd](https://github.com/googleapis/googleapis/commit/c4e80cd2d3321cc37cf1c3713bb02529857728d1) -END_NESTED_COMMIT - -BEGIN_NESTED_COMMIT -docs: [python-google-cloud-video-live-stream-v1] Update requirements of resource ID fields to be more clear - -PiperOrigin-RevId: 808091810 - -Source-link: -[googleapis/googleapis@c4e80cd](https://github.com/googleapis/googleapis/commit/c4e80cd2d3321cc37cf1c3713bb02529857728d1) -END_NESTED_COMMIT - -BEGIN_NESTED_COMMIT -feat: [python-google-cloud-eventarc-v1] add new fields to Eventarc resources - -PiperOrigin-RevId: 807339306 - -Source-link: -[googleapis/googleapis@dfce92d](https://github.com/googleapis/googleapis/commit/dfce92d6cc82c6dc133a6974b04e827e9fa4511c) -END_NESTED_COMMIT - -BEGIN_NESTED_COMMIT -docs: [python-google-cloud-eventarc-v1] correct some comments - -PiperOrigin-RevId: 807339306 - -Source-link: -[googleapis/googleapis@dfce92d](https://github.com/googleapis/googleapis/commit/dfce92d6cc82c6dc133a6974b04e827e9fa4511c) -END_NESTED_COMMIT - -END_COMMIT diff --git a/internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/multiple_nested_commits/repo_init/.librarian/state.yaml b/internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/multiple_nested_commits/repo_init/.librarian/state.yaml deleted file mode 100644 index f87cf2a85c0..00000000000 --- a/internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/multiple_nested_commits/repo_init/.librarian/state.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -image: test-image:latest -libraries: - - id: python-google-cloud-video-live-stream-v1 - version: 1.12.0 - last_generated_commit: f8776fec04e336527ba7279d960105533a1c4e21 - apis: - - path: google/cloud/video/livestream/v1 - service_config: livestream_v1.yaml - source_roots: - - packages/google-cloud-video-live-stream - tag_format: "{id}-v{version}" diff --git a/internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/multiple_nested_commits/state.yaml b/internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/multiple_nested_commits/state.yaml deleted file mode 100644 index 197f8ffabaf..00000000000 --- a/internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/multiple_nested_commits/state.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -image: test-image:latest -libraries: - - id: python-google-cloud-video-live-stream-v1 - version: 1.13.0 - last_generated_commit: some-commit-hash # This value is ignored by the test - apis: - - path: google/cloud/video/livestream/v1 - service_config: livestream_v1.yaml - source_roots: - - packages/google-cloud-video-live-stream - tag_format: "{id}-v{version}" diff --git a/internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/single_commit/CHANGELOG.md b/internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/single_commit/CHANGELOG.md deleted file mode 100644 index 94138b20d85..00000000000 --- a/internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/single_commit/CHANGELOG.md +++ /dev/null @@ -1,3 +0,0 @@ -## 1.25.0 - -- feat: [dlp] add LocationSupport,Domain,DocumentFallbackLocation diff --git a/internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/single_commit/commit_msg.txt b/internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/single_commit/commit_msg.txt deleted file mode 100644 index e164911ce93..00000000000 --- a/internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/single_commit/commit_msg.txt +++ /dev/null @@ -1,5 +0,0 @@ -feat: [dlp] add LocationSupport,Domain,DocumentFallbackLocation - -PiperOrigin-RevId: 808091810 -Source-link: https://github.com/googleapis/google-cloud-go/commit/2d9e87b206c27a5d491d324826e84cf060b2cb28 - diff --git a/internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/single_commit/repo_init/.librarian/state.yaml b/internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/single_commit/repo_init/.librarian/state.yaml deleted file mode 100644 index 51bac9c8831..00000000000 --- a/internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/single_commit/repo_init/.librarian/state.yaml +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -image: test-image:latest -libraries: - - id: dlp - version: 1.24.0 - last_generated_commit: f8776fec04e336527ba7279d960105533a1c4e21 - apis: - - path: google/privacy/dlp/v2 - source_roots: - - dlp - tag_format: '{id}/v{version}' diff --git a/internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/single_commit/state.yaml b/internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/single_commit/state.yaml deleted file mode 100644 index 3993f7f53a0..00000000000 --- a/internal/legacylibrarian/legacyintegration/testdata/e2e/release/stage/single_commit/state.yaml +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -image: test-image:latest -libraries: - - id: dlp - version: 1.25.0 - last_generated_commit: some-commit-hash # This value is ignored by the test - apis: - - path: google/privacy/dlp/v2 - source_roots: - - dlp - tag_format: '{id}/v{version}' diff --git a/internal/legacylibrarian/legacyintegration/testdata/e2e_func.go b/internal/legacylibrarian/legacyintegration/testdata/e2e_func.go deleted file mode 100644 index e014895e6ca..00000000000 --- a/internal/legacylibrarian/legacyintegration/testdata/e2e_func.go +++ /dev/null @@ -1,553 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "encoding/json" - "errors" - "fmt" - "log" - "log/slog" - "os" - "path/filepath" - "regexp" - "strings" - - "gopkg.in/yaml.v3" -) - -const ( - configureRequest = "configure-request.json" - configureResponse = "configure-response.json" - generateRequest = "generate-request.json" - generateResponse = "generate-response.json" - releaseInitRequest = "release-stage-request.json" - releaseInitResponse = "release-stage-response.json" - id = "id" - inputDir = "input" - librarian = "librarian" - outputDir = "output" - repoDir = "repo" - simulateCommandErrorID = "simulate-command-error-id" - source = "source" -) - -var ( - scopeRegex = regexp.MustCompile(`\[(.*?)\]`) -) - -func main() { - logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelInfo})) - slog.SetDefault(logger) - - if len(os.Args) <= 1 { - log.Fatal(errors.New("no command-line arguments provided")) - } - - slog.Info("received command", "args", os.Args[1:]) - switch os.Args[1] { - case "configure": - if err := doConfigure(os.Args[2:]); err != nil { - log.Fatal(err) - } - case "generate": - if err := doGenerate(os.Args[2:]); err != nil { - log.Fatal(err) - } - case "release-stage": - if err := doReleaseStage(os.Args[2:]); err != nil { - log.Fatal(err) - } - default: - log.Fatal("unrecognized command: ", os.Args[1]) - } -} - -func doConfigure(args []string) error { - request, err := parseConfigureRequest(args) - if err != nil { - return err - } - if err := validateLibrarianDir(request.librarianDir, configureRequest); err != nil { - return err - } - - state, err := readConfigureRequest(filepath.Join(request.librarianDir, configureRequest)) - if err != nil { - return err - } - - return writeConfigureResponse(request, state) -} - -func doGenerate(args []string) error { - request, err := parseGenerateOption(args) - if err != nil { - return err - } - if err := validateLibrarianDir(request.librarianDir, generateRequest); err != nil { - return err - } - - library, err := readGenerateRequest(filepath.Join(request.librarianDir, generateRequest)) - if err != nil { - return err - } - - if err := generateLibrary(library, request.outputDir); err != nil { - return fmt.Errorf("failed to generate library %s: %w", library.ID, err) - } - - return writeGenerateResponse(request) -} - -func doReleaseStage(args []string) error { - slog.Debug("doReleaseStage received args", "args", args) - request, err := parseReleaseStageRequest(args) - if err != nil { - return err - } - slog.Debug("doReleaseStage received request", "request", request) - if err := validateLibrarianDir(request.librarianDir, releaseInitRequest); err != nil { - return err - } - - state, err := readReleaseStageRequestJSON(filepath.Join(request.librarianDir, releaseInitRequest)) - if err != nil { - return err - } - - for i, library := range state.Libraries { - slog.Debug("library has SourceRoots", "index", i, "id", library.ID, "source_roots", library.SourceRoots) - } - - // Update the version of the library. - for _, library := range state.Libraries { - if !library.ReleaseTriggered { - continue - } - slog.Info("found library to update", "id", library.ID) - slog.Info("version from request", "version", library.Version) - - // Create a changelog. - var changelog strings.Builder - changelog.WriteString(fmt.Sprintf("## %s\n\n", library.Version)) - for _, change := range library.Changes { - changelog.WriteString(fmt.Sprintf("- %s: %s\n", change.Type, change.Subject)) - } - for _, sourceRoot := range library.SourceRoots { - changelogPath := filepath.Join(request.outputDir, sourceRoot, "CHANGELOG.md") - if err := os.MkdirAll(filepath.Dir(changelogPath), 0755); err != nil { - return fmt.Errorf("failed to create changelog directory: %w", err) - } - if err := os.WriteFile(changelogPath, []byte(changelog.String()), 0644); err != nil { - return fmt.Errorf("failed to write changelog: %w", err) - } - slog.Info("wrote changelog", "path", changelogPath) - } - - // After processing, clear the fields for the final state.yaml. - library.Changes = nil - library.ReleaseTriggered = false - } - - slog.Debug("state after update", "state", state) - updatedStateBytes, err := yaml.Marshal(state) - if err != nil { - return fmt.Errorf("failed to marshal updated state: %w", err) - } - slog.Debug("marshalled updated state (YAML)", "yaml", string(updatedStateBytes)) - - outputStateDir := filepath.Join(request.outputDir, ".librarian") - if err := os.MkdirAll(outputStateDir, 0755); err != nil { - return fmt.Errorf("failed to create output directory: %w", err) - } - - outputStatePath := filepath.Join(outputStateDir, "state.yaml") - if err := os.WriteFile(outputStatePath, updatedStateBytes, 0644); err != nil { - return fmt.Errorf("failed to write updated state.yaml to output: %w", err) - } - - slog.Info("wrote updated state.yaml", "path", outputStatePath) - - return writeReleaseStageResponseJSON(request) -} - -// readReleaseStageRequestJSON reads the release stage request file and creates a librarianState -// object. -func readReleaseStageRequestJSON(path string) (*librarianState, error) { - state := &librarianState{} - data, err := os.ReadFile(path) - if err != nil { - return nil, err - } - slog.Debug("readReleaseStageRequest: File content", "content", string(data)) - if err := json.Unmarshal(data, &state); err != nil { - return nil, err - } - slog.Debug("readReleaseStageRequest: Unmarshalled state", "state", state) - - // Validate the unmarshalled JSON content. - if err := validateReleaseStageRequestJSON(state); err != nil { - slog.Error("readReleaseStageRequest: Validation failed", "error", err) - return nil, err - } - - return state, nil -} - -type releaseInitOption struct { - librarianDir string - repoDir string - outputDir string -} - -func parseReleaseStageRequest(args []string) (*releaseInitOption, error) { - option := &releaseInitOption{} - for _, arg := range args { - opt, _ := strings.CutPrefix(arg, "--") - strs := strings.Split(opt, "=") - switch strs[0] { - case "librarian": - option.librarianDir = strs[1] - case "repo": - option.repoDir = strs[1] - case "output": - option.outputDir = strs[1] - default: - return nil, errors.New("unrecognized option: " + opt) - } - } - return option, nil -} - -func writeReleaseStageResponseJSON(option *releaseInitOption) error { - jsonFilePath := filepath.Join(option.librarianDir, releaseInitResponse) - jsonFile, err := os.Create(jsonFilePath) - if err != nil { - return err - } - defer jsonFile.Close() - - dataMap := map[string]string{} - data, err := json.MarshalIndent(dataMap, "", " ") - if err != nil { - return err - } - slog.Debug("about to write to file", "path", jsonFilePath, "data", string(data)) - _, err = jsonFile.Write(data) - slog.Info("wrote release stage response", "path", jsonFilePath) - - return err -} - -// validateReleaseStageRequestJSON validates the structure and content of the -// data unmarshalled from the release-stage-request.json file. -// This file is generated by the librarian tool and consumed by this mock container. -// It checks if the structure matches the expected contract and validates that -// commits are correctly associated with their libraries. -func validateReleaseStageRequestJSON(state *librarianState) error { - if state.Image == "" { - return errors.New("validation error: missing 'image' in release-stage-request.json") - } - if len(state.Libraries) == 0 { - return errors.New("validation error: no libraries found in release-stage-request.json") - } - - foundTriggered := false - for i, lib := range state.Libraries { - if lib.ID == "" { - return fmt.Errorf("validation error: library %d missing 'id'", i) - } - if !lib.ReleaseTriggered { - continue - } - foundTriggered = true - slog.Debug("validating triggered library", "id", lib.ID) - - if lib.Version == "" { - return fmt.Errorf("validation error: library %s missing 'version'", lib.ID) - } - if len(lib.SourceRoots) == 0 { - return fmt.Errorf("validation error: library %s missing 'source_roots'", lib.ID) - } - - if len(lib.Changes) == 0 { - return fmt.Errorf("validation error: no changes found for library %s", lib.ID) - } - for j, change := range lib.Changes { - // Validate fields based on the contract example in language-onboarding.md - if change.Type == "" { - return fmt.Errorf("validation error: library %s, change %d missing 'type'", lib.ID, j) - } - if change.Subject == "" { - return fmt.Errorf("validation error: library %s, change %d missing 'subject'", lib.ID, j) - } - // body is optional - - // These fields are expected based on the contract - if change.PiperCLNumber == "" { - return fmt.Errorf("validation error: library %s, change %d missing 'piper_cl_number'", lib.ID, j) - } - if change.CommitHash == "" { - return fmt.Errorf("validation error: library %s, change %d missing 'commit_hash'", lib.ID, j) - } - - // Validation logic to check for incorrect commit association. - matches := scopeRegex.FindStringSubmatch(change.Subject) - if len(matches) > 1 { - commitScope := matches[1] - isAssociated := strings.Contains(lib.ID, commitScope) - - slog.Info("running validation check", "library_id", lib.ID, "commit_scope", commitScope, "is_correctly_associated", isAssociated) - - if !isAssociated { - // If the commit scope does not appear in the library ID, it's a mismatch. - return fmt.Errorf("validation failed: commit with scope [%s] incorrectly associated with library %s", commitScope, lib.ID) - } - } - } - } - - if !foundTriggered { - slog.Warn("no library was marked with release_triggered: true in request") - } - slog.Debug("validateReleaseStageRequestJSON: Validation passed") - return nil -} - -func parseConfigureRequest(args []string) (*configureOption, error) { - configureOption := &configureOption{} - for _, arg := range args { - option, _ := strings.CutPrefix(arg, "--") - strs := strings.Split(option, "=") - switch strs[0] { - case inputDir: - configureOption.inputDir = strs[1] - case outputDir: - configureOption.outputDir = strs[1] - case librarian: - configureOption.librarianDir = strs[1] - case repoDir: - configureOption.repoDir = strs[1] - case source: - configureOption.sourceDir = strs[1] - default: - return nil, errors.New("unrecognized option: " + option) - } - } - - return configureOption, nil -} - -func parseGenerateOption(args []string) (*generateOption, error) { - generateOption := &generateOption{} - for _, arg := range args { - option, _ := strings.CutPrefix(arg, "--") - strs := strings.Split(option, "=") - switch strs[0] { - case inputDir: - generateOption.inputDir = strs[1] - case librarian: - generateOption.librarianDir = strs[1] - case outputDir: - generateOption.outputDir = strs[1] - case source: - generateOption.sourceDir = strs[1] - default: - return nil, errors.New("unrecognized option: " + option) - } - } - - return generateOption, nil -} - -func validateLibrarianDir(dir, requestFile string) error { - if _, err := os.Stat(filepath.Join(dir, requestFile)); err != nil { - return err - } - - return nil -} - -// readConfigureRequest reads the configure request file and creates a librarianState -// object. -func readConfigureRequest(path string) (*librarianState, error) { - state := &librarianState{} - data, err := os.ReadFile(path) - if err != nil { - return nil, err - } - if err := json.Unmarshal(data, state); err != nil { - return nil, err - } - - for _, library := range state.Libraries { - if library.ID == simulateCommandErrorID { - return nil, errors.New("simulate command error") - } - } - - return state, nil -} - -func writeConfigureResponse(option *configureOption, state *librarianState) error { - for _, library := range state.Libraries { - needConfigure := false - for _, oneAPI := range library.APIs { - if oneAPI.Status == "new" { - needConfigure = true - } - } - - if !needConfigure { - continue - } - - populateAdditionalFields(library) - data, err := json.MarshalIndent(library, "", " ") - if err != nil { - return err - } - - jsonFilePath := filepath.Join(option.librarianDir, configureResponse) - jsonFile, err := os.Create(jsonFilePath) - if err != nil { - return err - } - - if _, err := jsonFile.Write(data); err != nil { - return err - } - slog.Info("write configure response", "path", jsonFilePath) - } - - return nil -} - -// readGenerateRequest reads the generate request file and creates a libraryState -// object. -func readGenerateRequest(path string) (*libraryState, error) { - library := &libraryState{} - data, err := os.ReadFile(path) - if err != nil { - return nil, err - } - if err := json.Unmarshal(data, library); err != nil { - return nil, err - } - - if library.ID == simulateCommandErrorID { - // Simulate a command error - return nil, errors.New("simulate command error") - } - - return library, nil -} - -func writeGenerateResponse(option *generateOption) (err error) { - jsonFilePath := filepath.Join(option.librarianDir, generateResponse) - jsonFile, err := os.Create(jsonFilePath) - if err != nil { - return err - } - defer func() { - err = errors.Join(err, jsonFile.Close()) - }() - - dataMap := map[string]string{} - data, err := json.MarshalIndent(dataMap, "", " ") - if err != nil { - return err - } - _, err = jsonFile.Write(data) - slog.Info("write generate response", "path", jsonFilePath) - - return err -} - -func populateAdditionalFields(library *libraryState) { - library.Version = "1.0.0" - library.SourceRoots = []string{"example-source-path", "example-source-path-2"} - library.PreserveRegex = []string{"example-preserve-regex", "example-preserve-regex-2"} - library.RemoveRegex = []string{"example-remove-regex", "example-remove-regex-2"} - for _, oneAPI := range library.APIs { - oneAPI.Status = "existing" - } -} - -// generateLibrary creates files in sourceDir. -func generateLibrary(library *libraryState, outputDir string) error { - for _, src := range library.SourceRoots { - srcPath := filepath.Join(outputDir, src) - if err := os.MkdirAll(srcPath, 0755); err != nil { - return err - } - if _, err := os.Create(filepath.Join(srcPath, "example.txt")); err != nil { - return err - } - slog.Info("create file in", "path", srcPath) - } - - return nil -} - -type configureOption struct { - inputDir string - outputDir string - librarianDir string - repoDir string - sourceDir string -} - -type generateOption struct { - inputDir string - outputDir string - librarianDir string - sourceDir string -} - -type librarianState struct { - Image string `json:"image" yaml:"image"` - Libraries []*libraryState `json:"libraries" yaml:"libraries"` -} - -type libraryState struct { - ID string `json:"id" yaml:"id"` - Version string `json:"version" yaml:"version"` - APIs []*api `json:"apis,omitempty" yaml:"apis,omitempty"` - SourceRoots []string `json:"source_roots" yaml:"source_roots"` - PreserveRegex []string `json:"preserve_regex,omitempty" yaml:"preserve_regex,omitempty"` - RemoveRegex []string `json:"remove_regex,omitempty" yaml:"remove_regex,omitempty"` - ReleaseTriggered bool `json:"release_triggered,omitempty" yaml:"release_triggered,omitempty"` - Changes []*change `json:"changes,omitempty" yaml:"changes,omitempty"` - LastGeneratedCommit string `json:"last_generated_commit,omitempty" yaml:"last_generated_commit,omitempty"` - TagFormat string `json:"tag_format,omitempty" yaml:"tag_format"` - ReleaseExcludePaths []string `json:"release_exclude_paths,omitempty" yaml:"release_exclude_paths,omitempty"` -} - -type api struct { - Path string `json:"path" yaml:"path"` - ServiceConfig string `json:"service_config" yaml:"service_config"` - Status string `json:"status" yaml:"status"` -} - -type change struct { - Type string `json:"type" yaml:"type"` - Subject string `json:"subject" yaml:"subject"` - Body string `json:"body" yaml:"body"` - PiperCLNumber string `json:"piper_cl_number" yaml:"piper_cl_number"` - CommitHash string `json:"commit_hash" yaml:"commit_hash"` -} diff --git a/internal/legacylibrarian/legacylibrarian/action_test.go b/internal/legacylibrarian/legacylibrarian/action_test.go deleted file mode 100644 index e53474a46a9..00000000000 --- a/internal/legacylibrarian/legacylibrarian/action_test.go +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacylibrarian - -import ( - "strings" - "testing" - - "github.com/googleapis/librarian/internal/legacylibrarian/legacycli" - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" -) - -func TestLibrarianAction(t *testing.T) { - for _, test := range []struct { - name string - fn func() *legacycli.Command - }{ - { - name: "generate", - fn: newCmdGenerate, - }, - { - name: "init", - fn: newCmdStage, - }, - } { - t.Run(test.name, func(t *testing.T) { - testActionConfig(t, test.fn()) - }) - } -} - -// testActionConfig tests the execution flow for each Command.Action. The -// functionality of the Config methods called inside these Actions are tested -// separately in internal/legacyconfig. -func testActionConfig(t *testing.T, cmd *legacycli.Command) { - t.Helper() - for _, test := range []struct { - cfg *legacyconfig.Config - wantErr string - }{ - { - cfg: &legacyconfig.Config{ - WorkRoot: t.TempDir(), - }, - wantErr: "repo flag not specified", - }, - { - cfg: &legacyconfig.Config{ - WorkRoot: t.TempDir(), - Repo: "myrepo", - }, - wantErr: "repository does not exist", - }, - { - cfg: &legacyconfig.Config{ - WorkRoot: t.TempDir(), - Repo: "myrepo", - LibraryVersion: "1.0.0", - }, - wantErr: "specified library version without library id", - }, - { - cfg: &legacyconfig.Config{ - WorkRoot: t.TempDir(), - Repo: "https://github.com/googleapis/language-repo", - }, - wantErr: "remote branch is required when cloning", - }, - } { - t.Run(test.wantErr, func(t *testing.T) { - cmd.Config = test.cfg - err := cmd.Action(t.Context(), cmd) - if err == nil { - t.Fatal("expected an error") - } - if !strings.Contains(err.Error(), test.wantErr) { - t.Errorf("error mismatch, want: %q, got: %q", test.wantErr, err.Error()) - } - }) - } -} diff --git a/internal/legacylibrarian/legacylibrarian/build.go b/internal/legacylibrarian/legacylibrarian/build.go deleted file mode 100644 index 2c54c0873f8..00000000000 --- a/internal/legacylibrarian/legacylibrarian/build.go +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacylibrarian - -import ( - "context" - "errors" - "fmt" - "log/slog" - "path/filepath" - - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" - "github.com/googleapis/librarian/internal/legacylibrarian/legacydocker" - "github.com/googleapis/librarian/internal/legacylibrarian/legacygitrepo" -) - -func buildSingleLibrary(ctx context.Context, containerClient ContainerClient, state *legacyconfig.LibrarianState, libraryState *legacyconfig.LibraryState, repo legacygitrepo.Repository) error { - if libraryState == nil { - return fmt.Errorf("no libraryState provided") - } - buildRequest := &legacydocker.BuildRequest{ - LibraryID: libraryState.ID, - RepoDir: repo.GetDir(), - State: state, - } - slog.Info("performing build for library", "id", libraryState.ID) - if containerErr := containerClient.Build(ctx, buildRequest); containerErr != nil { - if restoreErr := restoreLibrary(libraryState, repo); restoreErr != nil { - return errors.Join(containerErr, restoreErr) - } - - return containerErr - } - - // Read the library state from the response. - if _, responseErr := readLibraryState( - filepath.Join(buildRequest.RepoDir, legacyconfig.LibrarianDir, legacyconfig.BuildResponse)); responseErr != nil { - if restoreErr := restoreLibrary(libraryState, repo); restoreErr != nil { - return errors.Join(responseErr, restoreErr) - } - - return responseErr - } - - slog.Info("build succeeds", "id", libraryState.ID) - return nil -} diff --git a/internal/legacylibrarian/legacylibrarian/build_test.go b/internal/legacylibrarian/legacylibrarian/build_test.go deleted file mode 100644 index 26c38c3b7d2..00000000000 --- a/internal/legacylibrarian/legacylibrarian/build_test.go +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacylibrarian - -import ( - "errors" - "io/fs" - "os" - "path/filepath" - "testing" - - "github.com/google/go-cmp/cmp" - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" -) - -func TestBuildSingleLibrary(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - libraryID string - container *mockContainerClient - wantBuildCalls int - wantErr bool - }{ - { - name: "build_with_library_id", - libraryID: "some-library", - container: &mockContainerClient{}, - wantBuildCalls: 1, - }, - { - name: "build_with_no_library_id", - container: &mockContainerClient{}, - wantErr: true, - wantBuildCalls: 0, - }, - { - name: "build_with_no_response", - libraryID: "some-library", - container: &mockContainerClient{ - noBuildResponse: true, - }, - wantBuildCalls: 1, - }, - { - name: "build_with_docker_command_error_files_restored", - libraryID: "some-library", - container: &mockContainerClient{ - buildErr: errors.New("simulate build error"), - }, - wantErr: true, - }, - { - name: "build_with_error_response_in_response", - libraryID: "some-library", - container: &mockContainerClient{ - wantErrorMsg: true, - }, - wantErr: true, - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - repo := newTestGitRepo(t) - state := &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "some-library", - SourceRoots: []string{ - "a/path", - "another/path", - }, - }, - }, - } - - // Create library files and commit the change. - repoDir := repo.GetDir() - for _, library := range state.Libraries { - for _, srcPath := range library.SourceRoots { - relPath := filepath.Join(repoDir, srcPath) - if err := os.MkdirAll(relPath, 0755); err != nil { - t.Fatal(err) - } - file := filepath.Join(relPath, "example.txt") - if err := os.WriteFile(file, []byte("old content"), 0755); err != nil { - t.Fatal(err) - } - } - } - if err := repo.AddAll(); err != nil { - t.Fatal(err) - } - if err := repo.Commit("test commit"); err != nil { - t.Fatal(err) - } - - libraryState := state.LibraryByID(test.libraryID) - err := buildSingleLibrary(t.Context(), test.container, state, libraryState, repo) - if test.wantErr { - if err == nil { - t.Fatal(err) - } - // Verify the library files are restore. - for _, library := range state.Libraries { - for _, srcPath := range library.SourceRoots { - file := filepath.Join(repoDir, srcPath, "example.txt") - readFile, err := os.ReadFile(file) - if err != nil { - t.Fatal(err) - } - if diff := cmp.Diff("old content", string(readFile)); diff != "" { - t.Errorf("file content mismatch (-want +got):%s", diff) - } - - newFile := filepath.Join(repoDir, srcPath, "another_example.txt") - if _, err := os.Stat(newFile); !errors.Is(err, fs.ErrNotExist) { - t.Fatal(err) - } - } - } - - return - } - - if err != nil { - t.Fatal(err) - } - if diff := cmp.Diff(test.wantBuildCalls, test.container.buildCalls); diff != "" { - t.Errorf("runBuildCommand() buildCalls mismatch (-want +got):%s", diff) - } - }) - } -} diff --git a/internal/legacylibrarian/legacylibrarian/command.go b/internal/legacylibrarian/legacylibrarian/command.go deleted file mode 100644 index a46435c6367..00000000000 --- a/internal/legacylibrarian/legacylibrarian/command.go +++ /dev/null @@ -1,816 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacylibrarian - -import ( - "cmp" - "context" - "errors" - "fmt" - "io" - "io/fs" - "log/slog" - "net/url" - "os" - "path" - "path/filepath" - "regexp" - "slices" - "strings" - "time" - - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" - "github.com/googleapis/librarian/internal/legacylibrarian/legacydocker" - "github.com/googleapis/librarian/internal/legacylibrarian/legacygithub" - "github.com/googleapis/librarian/internal/legacylibrarian/legacygitrepo" -) - -const ( - defaultAPISourceBranch = "master" - prBodyFile = "pr-body.txt" - timingFile = "timing.txt" - failedGenerationComment = `One or more libraries have failed to generate, please review PR description for a list of failed libraries. -For each failed library, open a ticket in that library’s repository and then you may resolve this comment and merge. -` -) - -type pullRequestType int - -const ( - pullRequestUnspecified pullRequestType = iota - pullRequestOnboard - pullRequestGenerate - pullRequestRelease - pullRequestUpdateImage -) - -// String returns the string representation of a pullRequestType. -// It returns unknown if the type is not a recognized constant. -func (t pullRequestType) String() string { - names := map[pullRequestType]string{ - pullRequestUnspecified: "unspecified", - pullRequestOnboard: "onboard", - pullRequestGenerate: "generate", - pullRequestRelease: "release", - pullRequestUpdateImage: "update image", - } - if name, ok := names[t]; ok { - return name - } - return "unspecified" -} - -var globalPreservePatterns = []string{ - fmt.Sprintf(`^%s(/.*)?$`, regexp.QuoteMeta(legacyconfig.GeneratorInputDir)), // Preserve the generator-input directory and its contents. -} - -// GitHubClient is an abstraction over the GitHub client. -type GitHubClient interface { - GetRawContent(ctx context.Context, path, ref string) ([]byte, error) - CreatePullRequest(ctx context.Context, repo *legacygithub.Repository, remoteBranch, remoteBase, title, body string, isDraft bool) (*legacygithub.PullRequestMetadata, error) - AddLabelsToIssue(ctx context.Context, repo *legacygithub.Repository, number int, labels []string) error - GetLabels(ctx context.Context, number int) ([]string, error) - ReplaceLabels(ctx context.Context, number int, labels []string) error - SearchPullRequests(ctx context.Context, query string) ([]*legacygithub.PullRequest, error) - GetPullRequest(ctx context.Context, number int) (*legacygithub.PullRequest, error) - CreateRelease(ctx context.Context, tagName, name, body, commitish string) (*legacygithub.RepositoryRelease, error) - CreateIssueComment(ctx context.Context, number int, comment string) error - CreateTag(ctx context.Context, tag, commitish string) error -} - -// ContainerClient is an abstraction over the Docker client. -type ContainerClient interface { - Build(ctx context.Context, request *legacydocker.BuildRequest) error - Configure(ctx context.Context, request *legacydocker.ConfigureRequest) (string, error) - Generate(ctx context.Context, request *legacydocker.GenerateRequest) error - ReleaseStage(ctx context.Context, request *legacydocker.ReleaseStageRequest) error -} - -type commitInfo struct { - // branch is the base branch of the created pull request. - branch string - // commit declares whether to create a commit. - commit bool - // commitMessage is used as the message on the actual git commit. - commitMessage string - // ghClient is used to interact with the GitHub API. - ghClient GitHubClient - // prType is an enum for which type of librarian pull request we are creating. - prType pullRequestType - // pullRequestLabels is a list of labels to add to the created pull request. - pullRequestLabels []string - // push declares whether to push the commits to GitHub. - push bool - // languageRepo is the git repository containing the language-specific libraries. - languageRepo legacygitrepo.Repository - // sourceRepo is the git repository containing the source protos. - sourceRepo legacygitrepo.Repository - // state is the librarian state.yaml contents. - state *legacyconfig.LibrarianState - // workRoot is the directory that we stage code changes in. - workRoot string - // failedGenerations is the number of generations that failed. - failedGenerations int - // api is the api path of a library, only set this value during api onboarding. - api string - // library is the ID of a library, only set this value during api onboarding. - library string - // prBodyBuilder is a callback function for building the pull request body - prBodyBuilder func() (string, error) - // isDraft declares whether to create the pull request as a draft. - isDraft bool -} - -type commandRunner struct { - repo legacygitrepo.Repository - sourceRepo legacygitrepo.Repository - state *legacyconfig.LibrarianState - librarianConfig *legacyconfig.LibrarianConfig - ghClient GitHubClient - containerClient ContainerClient - image string - workRoot string -} - -func newCommandRunner(cfg *legacyconfig.Config) (*commandRunner, error) { - languageRepo, err := cloneOrOpenRepo(cfg.WorkRoot, cfg.Repo, cfg.APISourceDepth, cfg.Branch, cfg.CI, cfg.GitHubToken) - if err != nil { - return nil, err - } - - var ( - sourceRepo legacygitrepo.Repository - sourceRepoDir string - ) - - // If APISource is set, checkout the protos repository. - if cfg.APISource != "" { - sourceRepo, err = cloneOrOpenRepo(cfg.WorkRoot, cfg.APISource, cfg.APISourceDepth, cfg.APISourceBranch, cfg.CI, cfg.GitHubToken) - if err != nil { - return nil, err - } - sourceRepoDir = sourceRepo.GetDir() - } - state, err := loadRepoState(languageRepo, sourceRepoDir) - if err != nil { - return nil, err - } - - librarianConfig, err := loadLibrarianConfig(languageRepo) - if err != nil { - return nil, err - } - - image := deriveImage(cfg.Image, state) - - gitHubRepo, err := GetGitHubRepository(cfg, languageRepo) - if err != nil { - return nil, fmt.Errorf("failed to get GitHub repository: %w", err) - } - - ghClient := legacygithub.NewClient(cfg.GitHubToken, gitHubRepo) - container, err := legacydocker.New(cfg.WorkRoot, image, &legacydocker.DockerOptions{ - UserUID: cfg.UserUID, - UserGID: cfg.UserGID, - HostMount: cfg.HostMount, - }) - if err != nil { - return nil, err - } - return &commandRunner{ - workRoot: cfg.WorkRoot, - repo: languageRepo, - sourceRepo: sourceRepo, - state: state, - librarianConfig: librarianConfig, - image: image, - ghClient: ghClient, - containerClient: container, - }, nil -} - -func cloneOrOpenRepo(workRoot, repo string, depth int, branch, ci string, gitPassword string) (*legacygitrepo.LocalRepository, error) { - if repo == "" { - return nil, fmt.Errorf("repo must be specified") - } - - if isURL(repo) { - // repo is a URL - // Take the last part of the URL as the directory name. It feels very - // unlikely that will clash with anything else (e.g. "output") - repoName := path.Base(strings.TrimSuffix(repo, "/")) - repoPath := filepath.Join(workRoot, repoName) - return legacygitrepo.NewRepository(&legacygitrepo.RepositoryOptions{ - Dir: repoPath, - MaybeClone: true, - RemoteURL: repo, - RemoteBranch: branch, - CI: ci, - GitPassword: gitPassword, - Depth: depth, - }) - } - // repo is a directory - absRepoRoot, err := filepath.Abs(repo) - if err != nil { - return nil, err - } - githubRepo, err := legacygitrepo.NewRepository(&legacygitrepo.RepositoryOptions{ - Dir: absRepoRoot, - CI: ci, - GitPassword: gitPassword, - }) - if err != nil { - return nil, err - } - cleanRepo, err := githubRepo.IsClean() - if err != nil { - return nil, err - } - if !cleanRepo { - return nil, fmt.Errorf("%s repo must be clean", repo) - } - return githubRepo, nil -} - -func deriveImage(imageOverride string, state *legacyconfig.LibrarianState) string { - if imageOverride != "" { - return imageOverride - } - if state == nil { - return "" - } - return state.Image -} - -func findLibraryIDByAPIPath(state *legacyconfig.LibrarianState, apiPath string) string { - if state == nil { - return "" - } - for _, lib := range state.Libraries { - for _, api := range lib.APIs { - if api.Path == apiPath { - return lib.ID - } - } - } - return "" -} - -func formatTimestamp(t time.Time) string { - const yyyyMMddHHmmss = "20060102T150405Z" // Expected format by time library - return t.Format(yyyyMMddHHmmss) -} - -// cleanAndCopyLibrary cleans the files of the given library in repoDir and copies -// the new files from outputDir. -func cleanAndCopyLibrary(state *legacyconfig.LibrarianState, repoDir, libraryID, outputDir string) error { - library := state.LibraryByID(libraryID) - if library == nil { - return fmt.Errorf("library %q not found during clean and copy, despite being found in earlier steps", libraryID) - } - - removePatterns := library.RemoveRegex - if len(removePatterns) == 0 { - slog.Info("remove_regex not provided, defaulting to source_roots") - removePatterns = make([]string, len(library.SourceRoots)) - // For each SourceRoot, create a regex pattern to match the source root - // directory itself, and any file or subdirectory within it. - for i, root := range library.SourceRoots { - removePatterns[i] = fmt.Sprintf("^%s(/.*)?$", regexp.QuoteMeta(root)) - } - } - - preservePatterns := append(library.PreserveRegex, globalPreservePatterns...) - - if err := clean(repoDir, library.SourceRoots, removePatterns, preservePatterns); err != nil { - return fmt.Errorf("failed to clean library, %s: %w", library.ID, err) - } - - return copyLibraryFiles(state, repoDir, libraryID, outputDir, true) -} - -// copyLibraryFiles copies the files in state.SourceRoots relative to the src folder to the dest -// folder. -// -// If `failOnExistingFile` is true, the function will check if a file already -// exists at the destination. If it does, an error is returned immediately without copying. -// If `failOnExistingFile` is false, it will overwrite any existing files. -// -// If there's no files in the library's SourceRoots under the src directory, no copy will happen. -// -// If a file is being copied to the library's SourceRoots in the dest folder but the folder does -// not exist, the copy fails. -func copyLibraryFiles(state *legacyconfig.LibrarianState, dest, libraryID, src string, failOnExistingFile bool) error { - library := state.LibraryByID(libraryID) - if library == nil { - return fmt.Errorf("library %q not found", libraryID) - } - slog.Info("copying library files", "id", library.ID, "destination", dest, "source", src) - for _, srcRoot := range library.SourceRoots { - dstPath := filepath.Join(dest, srcRoot) - srcPath := filepath.Join(src, srcRoot) - files, err := getDirectoryFilenames(srcPath) - if err != nil { - return err - } - for _, file := range files { - slog.Debug("copying file", "file", file) - srcFile := filepath.Join(srcPath, file) - dstFile := filepath.Join(dstPath, file) - if _, err := os.Stat(dstFile); failOnExistingFile && err == nil { - return fmt.Errorf("file existed in destination: %s", dstFile) - } - if err := copyFile(dstFile, srcFile); err != nil { - return fmt.Errorf("failed to copy file %q for library %s: %w", srcFile, library.ID, err) - } - } - } - return nil -} - -func getDirectoryFilenames(dir string) ([]string, error) { - if _, err := os.Stat(dir); err != nil { - // Skip dirs that don't exist - if errors.Is(err, fs.ErrNotExist) { - return nil, nil - } - return nil, err - } - - var fileNames []string - err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { - if err != nil { - return err - } - if !d.IsDir() { - relativePath, err := filepath.Rel(dir, path) - if err != nil { - return err - } - fileNames = append(fileNames, relativePath) - } - return nil - }) - if err != nil { - return nil, err - } - return fileNames, nil -} - -// commitAndPush creates a commit and push request to GitHub for the generated changes. -// It uses the GitHub client to create a PR with the specified branch, title, and -// description to the repository. -func commitAndPush(ctx context.Context, info *commitInfo) error { - repo := info.languageRepo - if !info.push && !info.commit { - slog.Info("push flag and commit flag are not specified, skipping committing") - isClean, err := repo.IsClean() - if err != nil { - return fmt.Errorf("failed to check if repo is clean: %w", err) - } - if isClean { - slog.Info("no changes to commit, skipping pull request body creation.") - return nil - } - return writePRBody(info) - } - - isClean, err := repo.IsClean() - if err != nil { - return fmt.Errorf("failed to check if repo is clean: %w", err) - } - - if isClean { - slog.Info("no changes to commit, skipping commit and push.") - return nil - } - - if err := repo.AddAll(); err != nil { - return fmt.Errorf("failed to add all files to git: %w", err) - } - - datetimeNow := formatTimestamp(time.Now()) - branch := fmt.Sprintf("librarian-%s", datetimeNow) - if err := repo.CreateBranchAndCheckout(branch); err != nil { - return fmt.Errorf("failed to create branch and checkout: %w", err) - } - - if err := repo.Commit(info.commitMessage); err != nil { - return fmt.Errorf("failed to commit: %w", err) - } - - if !info.push { - slog.Info("push flag is not specified, skipping pull request creation") - return writePRBody(info) - } - - if err := repo.Push(branch); err != nil { - return fmt.Errorf("failed to push: %w", err) - } - - gitHubRepo, err := GetGitHubRepositoryFromGitRepo(info.languageRepo) - if err != nil { - return fmt.Errorf("failed to get GitHub repository: %w", err) - } - - title := fmt.Sprintf("chore: librarian %s pull request: %s", info.prType, datetimeNow) - prBody, err := info.prBodyBuilder() - if err != nil { - return fmt.Errorf("failed to create pull request body: %w", err) - } - - pullRequestMetadata, err := info.ghClient.CreatePullRequest(ctx, gitHubRepo, branch, info.branch, title, prBody, info.isDraft) - if err != nil { - return fmt.Errorf("failed to create pull request: %w", err) - } - - if info.failedGenerations != 0 { - if err := info.ghClient.CreateIssueComment(ctx, pullRequestMetadata.Number, failedGenerationComment); err != nil { - return fmt.Errorf("failed to add pull request comment: %w", err) - } - } - - return addLabelsToPullRequest(ctx, info.ghClient, info.pullRequestLabels, pullRequestMetadata) -} - -// writePRBody attempts to log the body of a PR that would have been created if the -// -push flag had been specified. This logs any errors and returns them to the -// caller. -func writePRBody(info *commitInfo) error { - if info.prBodyBuilder == nil { - return fmt.Errorf("no prBodyBuilder provided") - } - - prBody, err := info.prBodyBuilder() - if err != nil { - slog.Warn("unable to create PR body", "error", err) - return err - } - // Note: we can't accurately predict whether a PR would have been created, - // as we're not checking whether the repo is clean or not. The intention is to be - // as light-touch as possible. - fullPath := filepath.Join(info.workRoot, prBodyFile) - // Ensure that "cat [path-to-pr-body.txt]" gives useful output. - prBody = prBody + "\n" - err = os.WriteFile(fullPath, []byte(prBody), 0644) - if err != nil { - slog.Warn("unable to save PR body", "error", err) - return err - } - slog.Info("wrote body of pull request that might have been created", "file", fullPath) - return nil -} - -// addLabelsToPullRequest adds a list of labels to a single pull request (specified by the id number). -// Should only be called on a valid Github pull request. -// Passing in `nil` for labels will no-op and an empty list for labels will clear all labels on the PR. -// TODO: Consolidate the params to a potential PullRequestInfo struct. -func addLabelsToPullRequest(ctx context.Context, ghClient GitHubClient, pullRequestLabels []string, prMetadata *legacygithub.PullRequestMetadata) error { - // Do not update if there aren't labels provided - if pullRequestLabels == nil { - return nil - } - // GitHub API treats Issues and Pull Request the same - // https://docs.github.com/en/rest/issues/labels#add-labels-to-an-issue - if err := ghClient.AddLabelsToIssue(ctx, prMetadata.Repo, prMetadata.Number, pullRequestLabels); err != nil { - return fmt.Errorf("failed to add labels to pull request: %w", err) - } - return nil -} - -// copyGlobalAllowlist copies files in the global file allowlist from src to dst. -func copyGlobalAllowlist(cfg *legacyconfig.LibrarianConfig, dst, src string, copyReadOnly bool) error { - if cfg == nil { - slog.Info("librarian config is not setup, skip copying global allowlist") - return nil - } - slog.Info("copying global allowlist files", "destination", dst, "source", src) - for _, globalFile := range cfg.GlobalFilesAllowlist { - if globalFile.Permissions == legacyconfig.PermissionReadOnly && !copyReadOnly { - slog.Debug("skipping read-only file", "path", globalFile.Path) - continue - } - - srcPath := filepath.Join(src, globalFile.Path) - if _, err := os.Lstat(srcPath); errors.Is(err, fs.ErrNotExist) { - slog.Info("skip copying a non-existent global allowlist file", "source", srcPath) - continue - } - dstPath := filepath.Join(dst, globalFile.Path) - if err := copyFile(dstPath, srcPath); err != nil { - return fmt.Errorf("failed to copy global file %s from %s: %w", dstPath, srcPath, err) - } - } - return nil -} - -func copyFile(dst, src string) (err error) { - lstat, err := os.Lstat(src) - if err != nil { - return fmt.Errorf("failed to lstat file: %q: %w", src, err) - } - - if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { - return fmt.Errorf("failed to make directory for %q: %w", dst, err) - } - - if lstat.Mode()&os.ModeSymlink == os.ModeSymlink { - linkTarget, err := os.Readlink(src) - if err != nil { - return fmt.Errorf("failed to read link: %q: %w", src, err) - } - // Remove existing file at dst if it exists. os.Symlink will fail otherwise. - if _, err := os.Lstat(dst); err == nil { - if err := os.Remove(dst); err != nil { - return fmt.Errorf("failed to remove existing file at destination: %q: %w", dst, err) - } - } - return os.Symlink(linkTarget, dst) - } - - sourceFile, err := os.Open(src) - if err != nil { - return fmt.Errorf("failed to open file: %q: %w", src, err) - } - defer sourceFile.Close() - - destinationFile, err := os.Create(dst) - if err != nil { - return fmt.Errorf("failed to create file: %s", dst) - } - defer destinationFile.Close() - - _, err = io.Copy(destinationFile, sourceFile) - - return err -} - -// clean removes files and directories from source roots based on remove and preserve patterns. -// Limit the possible files when cleaning to those in source roots (not rootDir) as regex patterns -// for preserve and remove should ONLY impact source root files. -// -// It first determines the paths to remove by applying the removePatterns and then excluding any paths -// that match the preservePatterns. It then separates the remaining paths into files and directories and -// removes them, ensuring that directories are removed last. -// -// This logic is ported from owlbot logic: https://github.com/googleapis/repo-automation-bots/blob/12dad68640960290910b660e4325630c9ace494b/packages/owl-bot/src/copy-code.ts#L1027 -func clean(rootDir string, sourceRoots, removePatterns, preservePatterns []string) error { - slog.Info("cleaning directories", "source roots", sourceRoots) - - // relPaths contains a list of files in source root's relative paths from rootDir. The - // regex patterns for preserve and remove apply to a source root's relative path - var relPaths []string - for _, sourceRoot := range sourceRoots { - sourceRootPath := filepath.Join(rootDir, sourceRoot) - if _, err := os.Lstat(sourceRootPath); err != nil { - if errors.Is(err, fs.ErrNotExist) { - // If a source root does not exist, continue searching other source roots. - slog.Debug("unable to find source root. It may be an initial generation request", "source root", sourceRoot) - continue - } - // For any other error (permissions, I/O, etc.) - slog.Error("error trying to clean source root", "source root", sourceRoot, "error", err) - return err - } - sourceRootPaths, err := findSubDirRelPaths(rootDir, sourceRootPath) - if err != nil { - // Continue processing other source roots. There may be other files that can be cleaned up. - slog.Debug("unable to search for files in a source root", "source root", sourceRoot, "error", err) - continue - } - if len(sourceRootPaths) == 0 { - slog.Info("source root does not contain any files", "source root", sourceRoot) - } - relPaths = append(relPaths, sourceRootPaths...) - } - - if len(relPaths) == 0 { - slog.Info("there are no files to be cleaned in source roots", "source roots", sourceRoots) - return nil - } - - pathsToRemove, err := filterPathsForRemoval(relPaths, removePatterns, preservePatterns) - if err != nil { - return err - } - - // prepend the rootDir to each path to ensure that os.Remove can find the file - var paths []string - for _, path := range pathsToRemove { - paths = append(paths, filepath.Join(rootDir, path)) - } - - filesToRemove, dirsToRemove, err := separateFilesAndDirs(paths) - if err != nil { - return err - } - - // Remove files first, then directories. - for _, file := range filesToRemove { - slog.Debug("removing file", "path", file) - if err := os.Remove(file); err != nil { - return err - } - } - - // Sort to remove the child directories first - slices.SortFunc(dirsToRemove, func(a, b string) int { - return strings.Count(b, string(filepath.Separator)) - strings.Count(a, string(filepath.Separator)) - }) - - for _, dir := range dirsToRemove { - slog.Debug("removing directory", "path", dir) - if err := os.Remove(dir); err != nil { - // It's possible the directory is not empty due to preserved files. - slog.Debug("failed to remove directory, it may not be empty due to preserved files", "dir", dir, "err", err) - } - } - - return nil -} - -// findSubDirRelPaths walks the subDir tree returns a slice of all file and directory paths -// relative to the dir. This is repeated for all nested directories. subDir must be under -// or the same as dir. -func findSubDirRelPaths(dir, subDir string) ([]string, error) { - dirRelPath, err := filepath.Rel(dir, subDir) - if err != nil { - return nil, fmt.Errorf("cannot establish the relationship between %s and %s: %w", dir, subDir, err) - } - // '..' signifies that the subDir exists outside of dir - if strings.HasPrefix(dirRelPath, "..") { - return nil, fmt.Errorf("subDir is not nested within the dir: %s, %s", subDir, dir) - } - - var paths []string - err = filepath.WalkDir(subDir, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - // error is ignored as we have confirmed that subDir is child or equal to rootDir - relPath, _ := filepath.Rel(dir, path) - // Special case when subDir is equal to dir. Drop the "." as it references itself - if relPath != "." { - paths = append(paths, relPath) - } - return nil - }) - return paths, err -} - -// filterPathsByRegex returns a new slice containing only the paths from the input slice -// that match at least one of the provided regular expressions. -func filterPathsByRegex(paths []string, regexps []*regexp.Regexp) []string { - var filtered []string - for _, path := range paths { - for _, re := range regexps { - if re.MatchString(path) { - filtered = append(filtered, path) - break - } - } - } - return filtered -} - -// compileRegexps takes a slice of string patterns and compiles each one into a -// regular expression. It returns a slice of compiled regexps or an error if any -// pattern is invalid. -func compileRegexps(patterns []string) ([]*regexp.Regexp, error) { - var regexps []*regexp.Regexp - for _, pattern := range patterns { - re, err := regexp.Compile(pattern) - if err != nil { - return nil, fmt.Errorf("invalid regex %q: %w", pattern, err) - } - regexps = append(regexps, re) - } - return regexps, nil -} - -// filterPathsForRemoval determines the list of paths to be removed. The logic runs as follows: -// 1. paths that match any removePatterns are marked for removal -// 2. paths that match the preservePatterns are kept (even if they match removePatterns) -// Paths that match both are kept as preserve has overrides. -func filterPathsForRemoval(paths, removePatterns, preservePatterns []string) ([]string, error) { - removeRegexps, err := compileRegexps(removePatterns) - if err != nil { - return nil, err - } - preserveRegexps, err := compileRegexps(preservePatterns) - if err != nil { - return nil, err - } - - pathsToRemove := filterPathsByRegex(paths, removeRegexps) - pathsToPreserve := filterPathsByRegex(pathsToRemove, preserveRegexps) - - // map for a quick lookup for any preserve paths - preserveMap := make(map[string]bool) - for _, p := range pathsToPreserve { - preserveMap[p] = true - } - finalPathsToRemove := slices.DeleteFunc(pathsToRemove, func(path string) bool { - return preserveMap[path] - }) - return finalPathsToRemove, nil -} - -// separateFilesAndDirs takes a list of paths and categorizes them into files -// and directories. It uses os.Lstat to avoid following symlinks, treating them -// as files. Paths that do not exist are silently ignored. -func separateFilesAndDirs(paths []string) ([]string, []string, error) { - var filePaths, dirPaths []string - for _, path := range paths { - info, err := os.Lstat(path) - if err != nil { - // The file or directory may have already been removed. - if errors.Is(err, fs.ErrNotExist) { - slog.Warn("unable to find path", "path", path) - continue - } - // For any other error (permissions, I/O, etc.) - return nil, nil, fmt.Errorf("failed to stat path %q: %w", path, err) - - } - if info.IsDir() { - dirPaths = append(dirPaths, path) - } else { - filePaths = append(filePaths, path) - } - } - return filePaths, dirPaths, nil -} - -func isURL(s string) bool { - u, err := url.ParseRequestURI(s) - if err != nil || u.Scheme == "" || u.Host == "" { - return false - } - - return true -} - -// writeTiming creates a file in the work root with diagnostic information -// about the time taken to process each library. A summary line states -// the number of individual measurements represented, as well as the total -// and the average, then the time taken for each library is recorded -// in descending order of time, to make it easier to figure out what to -// focus on. All times are rounded to the nearest millisecond. -func writeTiming(workRoot string, timeByLibrary map[string]time.Duration) error { - if len(timeByLibrary) == 0 { - slog.Info("no libraries processed; skipping timing statistics") - return nil - } - - // Work out the total and average times, and create a slice of timing - // by library, sorted in descending order of duration. - var total time.Duration - for _, duration := range timeByLibrary { - total += duration - } - average := time.Duration(int64(total) / int64(len(timeByLibrary))) - - type timing struct { - LibraryID string - Duration time.Duration - } - - var timingStructs []timing - for id, duration := range timeByLibrary { - timingStructs = append(timingStructs, timing{id, duration}) - } - - slices.SortFunc(timingStructs, func(a, b timing) int { - return -cmp.Compare(a.Duration, b.Duration) - }) - - // Create the timing log in memory: one summary line, then one line per library. - var sb strings.Builder - fmt.Fprintf(&sb, "Processed %d libraries in %s; average=%s\n", len(timeByLibrary), total.Round(time.Millisecond), average.Round(time.Millisecond)) - - for _, ts := range timingStructs { - fmt.Fprintf(&sb, "%s: %s\n", ts.LibraryID, ts.Duration.Round(time.Millisecond)) - } - - // Write it out to disk. - fullPath := filepath.Join(workRoot, timingFile) - if err := os.WriteFile(fullPath, []byte(sb.String()), 0644); err != nil { - return err - } - slog.Info("wrote timing statistics", "file", fullPath) - return nil -} diff --git a/internal/legacylibrarian/legacylibrarian/command_test.go b/internal/legacylibrarian/legacylibrarian/command_test.go deleted file mode 100644 index b68fe431363..00000000000 --- a/internal/legacylibrarian/legacylibrarian/command_test.go +++ /dev/null @@ -1,2370 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacylibrarian - -import ( - "errors" - "fmt" - "io/fs" - "os" - "os/exec" - "path/filepath" - "regexp" - "sort" - "strings" - "testing" - "time" - - "github.com/google/go-cmp/cmp" - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" - "github.com/googleapis/librarian/internal/legacylibrarian/legacygithub" - "github.com/googleapis/librarian/internal/legacylibrarian/legacygitrepo" -) - -func TestDeriveImage(t *testing.T) { - for _, test := range []struct { - name string - imageOverride string - state *legacyconfig.LibrarianState - want string - }{ - { - name: "with image override, nil state", - imageOverride: "my/custom-image:v1", - state: nil, - want: "my/custom-image:v1", - }, - { - name: "with image override, non-nil state", - imageOverride: "my/custom-image:v1", - state: &legacyconfig.LibrarianState{Image: "gcr.io/foo/bar:v1.2.3"}, - want: "my/custom-image:v1", - }, - { - name: "no override, nil state", - imageOverride: "", - state: nil, - want: "", - }, - { - name: "no override, with state", - imageOverride: "", - state: &legacyconfig.LibrarianState{Image: "gcr.io/foo/bar:v1.2.3"}, - want: "gcr.io/foo/bar:v1.2.3", - }, - } { - t.Run(test.name, func(t *testing.T) { - got := deriveImage(test.imageOverride, test.state) - - if got != test.want { - t.Errorf("deriveImage() = %q, want %q", got, test.want) - } - }) - } -} - -// newTestGitRepoWithCommit creates a new git repository with an initial commit. -// If dir is empty, a new temporary directory is created. -// It returns the path to the repository directory. -func newTestGitRepoWithCommit(t *testing.T, dir string) string { - t.Helper() - if dir == "" { - dir = t.TempDir() - } else { - if err := os.MkdirAll(dir, 0755); err != nil { - t.Fatalf("MkdirAll(%q): %v", dir, err) - } - } - for _, args := range [][]string{ - {"init"}, - {"config", "user.name", "tester"}, - {"config", "user.email", "tester@example.com"}, - } { - cmd := exec.Command("git", args...) - cmd.Dir = dir - if err := cmd.Run(); err != nil { - t.Fatalf("git %v: %v", args, err) - } - } - - filePath := filepath.Join(dir, "README.md") - if err := os.WriteFile(filePath, []byte("hello"), 0644); err != nil { - t.Fatalf("WriteFile: %v", err) - } - for _, args := range [][]string{ - {"add", "README.md"}, - {"commit", "-m", "initial commit"}, - } { - cmd := exec.Command("git", args...) - cmd.Dir = dir - if err := cmd.Run(); err != nil { - t.Fatalf("git %v: %v", args, err) - } - } - return dir -} - -func TestCloneOrOpenLanguageRepo(t *testing.T) { - workRoot := t.TempDir() - - cleanRepoPath := newTestGitRepoWithCommit(t, "") - dirtyRepoPath := newTestGitRepoWithCommit(t, "") - if err := os.WriteFile(filepath.Join(dirtyRepoPath, "untracked.txt"), []byte("dirty"), 0644); err != nil { - t.Fatalf("WriteFile: %v", err) - } - notARepoPath := t.TempDir() - - for _, test := range []struct { - name string - repo string - ci string - wantErr bool - check func(t *testing.T, repo *legacygitrepo.LocalRepository) - setup func(t *testing.T, workRoot string) func() - }{ - { - name: "with clean repoRoot", - repo: cleanRepoPath, - check: func(t *testing.T, repo *legacygitrepo.LocalRepository) { - absWantDir, _ := filepath.Abs(cleanRepoPath) - if repo.Dir != absWantDir { - t.Errorf("repo.Dir got %q, want %q", repo.Dir, absWantDir) - } - }, - }, - { - name: "with repoURL with trailing slash", - repo: "https://github.com/googleapis/google-cloud-go/", - setup: func(t *testing.T, workRoot string) func() { - // The expected directory name is `google-cloud-go`. - repoPath := filepath.Join(workRoot, "google-cloud-go") - newTestGitRepoWithCommit(t, repoPath) - return func() { - if err := os.RemoveAll(repoPath); err != nil { - t.Errorf("os.RemoveAll(%q) = %v; want nil", repoPath, err) - } - } - }, - check: func(t *testing.T, repo *legacygitrepo.LocalRepository) { - wantDir := filepath.Join(workRoot, "google-cloud-go") - if repo.Dir != wantDir { - t.Errorf("repo.Dir got %q, want %q", repo.Dir, wantDir) - } - }, - }, - { - name: "no repoRoot or repoURL", - wantErr: true, - }, - { - name: "with dirty repoRoot", - repo: dirtyRepoPath, - wantErr: true, - }, - { - name: "with repoRoot that is not a repo", - repo: notARepoPath, - wantErr: true, - }, - } { - t.Run(test.name, func(t *testing.T) { - var cleanup func() - if test.setup != nil { - cleanup = test.setup(t, workRoot) - } - defer func() { - if cleanup != nil { - cleanup() - } - }() - - repo, err := cloneOrOpenRepo(workRoot, test.repo, 1, test.ci, "main", "") - if test.wantErr { - if err == nil { - t.Fatal("cloneOrOpenLanguageRepo() expected an error but got nil") - } - return - } - if err != nil { - t.Errorf("cloneOrOpenLanguageRepo() got unexpected error: %v", err) - return - } - if test.check != nil { - if repo == nil { - t.Fatal("cloneOrOpenLanguageRepo() returned nil repo but no error") - } - test.check(t, repo) - } - }) - } -} - -func TestCleanAndCopyLibrary(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - libraryID string - state *legacyconfig.LibrarianState - repo legacygitrepo.Repository - outputDir string - setup func(t *testing.T, repoDir, outputDir string) - wantErr bool - errContains string - shouldCopy []string - shouldDelete []string - }{ - { - name: "library not found", - libraryID: "non-existent-library", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "some-library", - }, - }, - }, - repo: newTestGitRepo(t), - wantErr: true, - errContains: "not found during clean and copy", - }, - { - name: "clean fails due to invalid regex", - libraryID: "some-library", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "some-library", - RemoveRegex: []string{"["}, // Invalid regex - SourceRoots: []string{"src/a"}, - }, - }, - }, - repo: newTestGitRepo(t), - wantErr: true, - errContains: "failed to clean library", - }, - { - name: "copy should not fail on symlink", - libraryID: "some-library", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "some-library", - SourceRoots: []string{ - "symlink", - }, - }, - }, - }, - repo: newTestGitRepo(t), - setup: func(t *testing.T, repoDir, outputDir string) { - // Create a symlink in the output directory - if err := os.MkdirAll(filepath.Join(outputDir, "target"), 0755); err != nil { - t.Fatalf("os.MkdirAll() = %v", err) - } - if err := os.Symlink(filepath.Join(outputDir, "target"), filepath.Join(repoDir, "symlink")); err != nil { - t.Fatalf("os.Symlink() = %v", err) - } - if _, err := os.Create(filepath.Join(repoDir, "symlink", "example.txt")); err != nil { - t.Fatalf("os.Create() = %v", err) - } - }, - }, - { - name: "empty RemoveRegex defaults to source root", - libraryID: "some-library", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "some-library", - SourceRoots: []string{"a/path"}, - }, - }, - }, - repo: newTestGitRepo(t), - setup: func(t *testing.T, repoDir, outputDir string) { - // Create a stale file in the repo directory to test cleaning. - staleFile := filepath.Join(repoDir, "a/path/stale.txt") - if err := os.MkdirAll(filepath.Dir(staleFile), 0755); err != nil { - t.Fatal(err) - } - if _, err := os.Create(staleFile); err != nil { - t.Fatal(err) - } - - // Create generated files in the output directory. - filesToCreate := []string{ - "a/path/new_generated_file_to_copy.txt", - "skipped/path/example.txt", - } - for _, relPath := range filesToCreate { - fullPath := filepath.Join(outputDir, relPath) - if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil { - t.Fatal(err) - } - if _, err := os.Create(fullPath); err != nil { - t.Fatal(err) - } - } - }, - shouldCopy: []string{ - "a/path/new_generated_file_to_copy.txt", - }, - shouldDelete: []string{ - "skipped/path/example.txt", - "a/path/stale.txt", - }, - }, - { - name: "clean never deletes generator-input directory", - libraryID: "some-library", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "some-library", - SourceRoots: []string{"a/path"}, - RemoveRegex: []string{"a/path"}, - }, - }, - }, - repo: newTestGitRepo(t), - setup: func(t *testing.T, repoDir, outputDir string) { - // Create a file in the repo directory to test cleaning. - fileToBeDeleted := filepath.Join(repoDir, "a/path/delete.txt") - fileToNotBeDeleted := filepath.Join(repoDir, ".librarian/generator-input/a/path/do-not-delete.txt") - if err := os.MkdirAll(filepath.Dir(fileToBeDeleted), 0755); err != nil { - t.Fatal(err) - } - if _, err := os.Create(fileToBeDeleted); err != nil { - t.Fatal(err) - } - if err := os.MkdirAll(filepath.Dir(fileToNotBeDeleted), 0755); err != nil { - t.Fatal(err) - } - if _, err := os.Create(fileToNotBeDeleted); err != nil { - t.Fatal(err) - } - - // Create generated files in the output directory. - filesToCreate := []string{ - "a/path/new_generated_file_to_copy.txt", - } - for _, relPath := range filesToCreate { - fullPath := filepath.Join(outputDir, relPath) - if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil { - t.Fatal(err) - } - if _, err := os.Create(fullPath); err != nil { - t.Fatal(err) - } - } - }, - shouldCopy: []string{ - ".librarian/generator-input/a/path/do-not-delete.txt", - "a/path/new_generated_file_to_copy.txt", - }, - shouldDelete: []string{ - "a/path/delete.txt", - }, - }, - { - name: "if same value is present in preserve regex and global preserver regex list there is no conflict", - libraryID: "some-library", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "some-library", - SourceRoots: []string{"a/path"}, - RemoveRegex: []string{"a/path"}, - PreserveRegex: globalPreservePatterns, - }, - }, - }, - repo: newTestGitRepo(t), - setup: func(t *testing.T, repoDir, outputDir string) { - // Create a file in the repo directory to test cleaning. - fileToBeDeleted := filepath.Join(repoDir, "a/path/delete.txt") - fileToNotBeDeleted := filepath.Join(repoDir, ".librarian/generator-input/a/path/do-not-delete.txt") - if err := os.MkdirAll(filepath.Dir(fileToBeDeleted), 0755); err != nil { - t.Fatal(err) - } - if _, err := os.Create(fileToBeDeleted); err != nil { - t.Fatal(err) - } - if err := os.MkdirAll(filepath.Dir(fileToNotBeDeleted), 0755); err != nil { - t.Fatal(err) - } - if _, err := os.Create(fileToNotBeDeleted); err != nil { - t.Fatal(err) - } - - // Create generated files in the output directory. - filesToCreate := []string{ - "a/path/new_generated_file_to_copy.txt", - } - for _, relPath := range filesToCreate { - fullPath := filepath.Join(outputDir, relPath) - if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil { - t.Fatal(err) - } - if _, err := os.Create(fullPath); err != nil { - t.Fatal(err) - } - } - }, - shouldCopy: []string{ - ".librarian/generator-input/a/path/do-not-delete.txt", - "a/path/new_generated_file_to_copy.txt", - }, - shouldDelete: []string{ - "a/path/delete.txt", - }, - }, - } { - t.Run(test.name, func(t *testing.T) { - repoDir := test.repo.GetDir() - outputDir := t.TempDir() - if test.setup != nil { - test.setup(t, repoDir, outputDir) - } - err := cleanAndCopyLibrary(test.state, repoDir, test.libraryID, outputDir) - if test.wantErr { - if err == nil { - t.Fatalf("%s should return error", test.name) - } - if !strings.Contains(err.Error(), test.errContains) { - t.Errorf("want: %s, got %s", test.errContains, err.Error()) - } - - return - } - if err != nil { - t.Fatal(err) - } - - for _, file := range test.shouldCopy { - fullPath := filepath.Join(repoDir, file) - if _, err := os.Stat(fullPath); err != nil { - t.Errorf("file %s is not copied to %s", file, repoDir) - } - } - - for _, file := range test.shouldDelete { - fullPath := filepath.Join(repoDir, file) - if _, err := os.Stat(fullPath); !errors.Is(err, fs.ErrNotExist) { - t.Errorf("file %s should not be copied to %s", file, repoDir) - } - } - }) - } -} - -func TestClean(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - files map[string]string - setup func(t *testing.T, tmpDir string) - symlinks map[string]string - sourceRoots []string - removePatterns []string - preservePatterns []string - wantRemaining []string - wantErr bool - }{ - { - name: "remove everything", - files: map[string]string{ - "foo/file1.txt": "", - "foo/file2.txt": "", - }, - sourceRoots: []string{"foo"}, - removePatterns: []string{".*"}, - wantRemaining: []string{}, - }, - { - name: "remove all files in a folder", - files: map[string]string{ - "foo/file1.txt": "", - "foo/file2.txt": "", - }, - sourceRoots: []string{"foo"}, - removePatterns: []string{"foo/.*"}, - wantRemaining: []string{"foo"}, - }, - { - name: "remove folder", - files: map[string]string{ - "foo/file1.txt": "", - "foo/file2.txt": "", - }, - sourceRoots: []string{"foo"}, - removePatterns: []string{"foo"}, - wantRemaining: []string{}, - }, - { - name: "preserve all", - files: map[string]string{ - "foo/file1.txt": "", - "foo/file2.txt": "", - }, - sourceRoots: []string{"foo"}, - removePatterns: []string{".*"}, - preservePatterns: []string{".*"}, - wantRemaining: []string{"foo", "foo/file1.txt", "foo/file2.txt"}, - }, - { - name: "preserve all files in a folder", - files: map[string]string{ - "foo/file1.txt": "", - "foo/file2.txt": "", - }, - sourceRoots: []string{"foo"}, - removePatterns: []string{".*"}, - preservePatterns: []string{"foo/.*"}, - wantRemaining: []string{"foo", "foo/file1.txt", "foo/file2.txt"}, - }, - { - name: "preserve folder", - files: map[string]string{ - "foo/file1.txt": "", - "foo/file2.txt": "", - }, - sourceRoots: []string{"foo"}, - removePatterns: []string{".*"}, - preservePatterns: []string{"foo"}, - wantRemaining: []string{"foo", "foo/file1.txt", "foo/file2.txt"}, - }, - { - name: "remove specific folder", - files: map[string]string{ - "foo/file1.txt": "", - "foo/file2.txt": "", - "bar/file3.txt": "", - }, - sourceRoots: []string{"foo", "bar"}, - removePatterns: []string{"foo"}, - wantRemaining: []string{"bar", "bar/file3.txt"}, - }, - { - name: "no source roots configured", - files: map[string]string{ - "foo/file1.txt": "", - "foo/file2.txt": "", - "foo/file3.txt": "", - }, - sourceRoots: []string{}, - removePatterns: []string{"foo"}, - wantRemaining: []string{"foo", "foo/file1.txt", "foo/file2.txt", "foo/file3.txt"}, - }, - { - name: "invalid remove pattern", - files: map[string]string{ - "foo/file1.txt": "", - }, - sourceRoots: []string{"foo/"}, - removePatterns: []string{"["}, // Invalid regex - wantErr: true, - }, - { - name: "invalid preserve pattern", - files: map[string]string{ - "foo/file1.txt": "", - }, - sourceRoots: []string{"foo"}, - removePatterns: []string{".*"}, - preservePatterns: []string{"["}, // Invalid regex - wantErr: true, - }, - { - name: "remove symlink", - files: map[string]string{ - "foo/file1.txt": "content", - }, - symlinks: map[string]string{ - "foo/symlink_to_file1": "foo/file1.txt", - }, - sourceRoots: []string{"foo"}, - removePatterns: []string{"foo/symlink_to_file1"}, - wantRemaining: []string{"foo", "foo/file1.txt"}, - }, - { - name: "remove file symlinked to", - files: map[string]string{ - "foo/file1.txt": "content", - }, - symlinks: map[string]string{ - "foo/symlink_to_file1": "foo/file1.txt", - }, - sourceRoots: []string{"foo"}, - removePatterns: []string{".*/file1.txt"}, - // The symlink should remain, even though it's now broken, because - // it was not targeted for removal. - wantRemaining: []string{"foo", "foo/symlink_to_file1"}, - }, - { - name: "preserve file not matching remove pattern", - files: map[string]string{ - "foo/file1.txt": "", - "foo/file2.log": "", - }, - sourceRoots: []string{"foo"}, - removePatterns: []string{".*\\.txt"}, - wantRemaining: []string{"foo", "foo/file2.log"}, - }, - { - name: "remove file fails on permission error", - files: map[string]string{ - "readonlydir/file.txt": "content", - }, - setup: func(t *testing.T, tmpDir string) { - // Make the directory read-only to cause os.Remove to fail. - readOnlyDir := filepath.Join(tmpDir, "readonlydir") - if err := os.Chmod(readOnlyDir, 0555); err != nil { - t.Fatalf("os.Chmod() = %v", err) - } - // Register a cleanup function to restore permissions so TempDir can be removed. - t.Cleanup(func() { - _ = os.Chmod(readOnlyDir, 0755) - }) - }, - sourceRoots: []string{"readonlydir"}, - removePatterns: []string{"readonlydir/file.txt"}, - wantRemaining: []string{"readonlydir", "readonlydir/file.txt"}, - wantErr: true, - }, - { - name: "directories of source roots are empty", - // There are no files in any of the source roots - files: map[string]string{}, - sourceRoots: []string{"foo", "bar", "baz"}, - removePatterns: []string{".*"}, - preservePatterns: []string{".*"}, - wantRemaining: []string{}, - }, - { - name: "multiple source roots, keep all", - files: map[string]string{ - "foo/file1.txt": "", - "bar/file2.log": "", - "baz/file3.md": "", - }, - sourceRoots: []string{"foo", "bar", "baz"}, - preservePatterns: []string{".*"}, - wantRemaining: []string{"foo", "foo/file1.txt", "bar", "bar/file2.log", "baz", "baz/file3.md"}, - }, - { - name: "multiple source roots, remove all", - files: map[string]string{ - "foo/file1.txt": "", - "bar/file2.log": "", - "baz/file3.md": "", - }, - sourceRoots: []string{"foo", "bar", "baz"}, - removePatterns: []string{".*"}, - wantRemaining: []string{}, - }, - { - name: "remove regex references outside of source roots", - files: map[string]string{ - "foo/file1.txt": "", - "bar/file2.log": "", - "private/file1.txt": "", - }, - sourceRoots: []string{"foo", "bar"}, - // Regex outside of sourceRoots should effectively no-op - removePatterns: []string{"private"}, - wantRemaining: []string{"foo", "foo/file1.txt", "bar", "bar/file2.log", "private", "private/file1.txt"}, - }, - { - name: "preserve regex references outside of source roots", - files: map[string]string{ - "foo/file1.txt": "", - "foo/file2.log": "", - "private/file1.txt": "", - "other_private/file2.txt": "", - }, - sourceRoots: []string{"foo"}, - // Remove everything from source roots - removePatterns: []string{".*"}, - // Regex outside of sourceRoots should effectively no-op - preservePatterns: []string{"private"}, - wantRemaining: []string{"private", "private/file1.txt", "other_private", "other_private/file2.txt"}, - }, - { - name: "preserve and remove regex outside of source roots", - files: map[string]string{ - "foo/file1.txt": "", - "foo/file2.log": "", - "private/file1.txt": "", - "other_private/file2.txt": "", - }, - sourceRoots: []string{"foo"}, - // Regex outside of sourceRoots should effectively no-op - removePatterns: []string{"private"}, - preservePatterns: []string{"private"}, - wantRemaining: []string{"foo", "foo/file1.txt", "foo/file2.log", "private", "private/file1.txt", "other_private", "other_private/file2.txt"}, - }, - } { - t.Run(test.name, func(t *testing.T) { - tmpDir := t.TempDir() - for path, content := range test.files { - fullPath := filepath.Join(tmpDir, path) - if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil { - t.Fatalf("os.MkdirAll() = %v", err) - } - if err := os.WriteFile(fullPath, []byte(content), 0644); err != nil { - t.Fatalf("os.WriteFile() = %v", err) - } - } - for link, target := range test.symlinks { - linkPath := filepath.Join(tmpDir, link) - if err := os.Symlink(target, linkPath); err != nil { - t.Fatalf("os.Symlink() = %v", err) - } - } - if test.setup != nil { - test.setup(t, tmpDir) - } - err := clean(tmpDir, test.sourceRoots, test.removePatterns, test.preservePatterns) - if test.wantErr { - if err == nil { - t.Fatalf("%s should return error", test.name) - } - return - } - if err != nil { - t.Fatal(err) - } - - remainingPaths := []string{} - paths, _ := findSubDirRelPaths(tmpDir, tmpDir) - remainingPaths = append(remainingPaths, paths...) - if err != nil { - t.Fatalf("findSubDirRelPaths() = %v", err) - } - sort.Strings(test.wantRemaining) - sort.Strings(remainingPaths) - if diff := cmp.Diff(test.wantRemaining, remainingPaths); diff != "" { - t.Errorf("clean() remaining files mismatch (-want +got):%s", diff) - } - - }) - } -} - -func TestFindSubDirRelPaths(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - getRootDirPath func(dir string) string - getSubDirPath func(dir string) string - setup func(t *testing.T, dir string) - wantPaths []string - wantErr bool - errorString string - }{ - { - name: "success", - getRootDirPath: func(dir string) string { - return dir - }, - getSubDirPath: func(dir string) string { - return dir - }, - setup: func(t *testing.T, dir string) { - files := []string{ - "dir1/file1.txt", - "dir1/file2.txt", - "dir1/dir2/file3.txt", - } - for _, file := range files { - path := filepath.Join(dir, file) - if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { - t.Fatalf("os.MkdirAll() = %v", err) - } - if err := os.WriteFile(path, []byte("test"), 0644); err != nil { - t.Fatalf("os.WriteFile() = %v", err) - } - } - }, - wantPaths: []string{ - "dir1", - "dir1/dir2", - "dir1/dir2/file3.txt", - "dir1/file1.txt", - "dir1/file2.txt", - }, - }, - { - name: "dir and subDir are the same", - getRootDirPath: func(dir string) string { - return dir - }, - getSubDirPath: func(dir string) string { - return dir - }, - setup: func(t *testing.T, dir string) { - files := []string{ - "dir1/file1.txt", - "dir1/file2.txt", - } - for _, file := range files { - path := filepath.Join(dir, file) - if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { - t.Fatalf("os.MkdirAll() = %v", err) - } - if err := os.WriteFile(path, []byte("test"), 0644); err != nil { - t.Fatalf("os.WriteFile() = %v", err) - } - } - }, - wantPaths: []string{ - "dir1", - "dir1/file1.txt", - "dir1/file2.txt", - }, - }, - { - name: "dir and subDir's relationship cannot be established", - getRootDirPath: func(dir string) string { - return filepath.Join(dir, "dir1") - }, - getSubDirPath: func(dir string) string { - return "./doesnotexist" - }, - setup: func(t *testing.T, dir string) { - files := []string{ - "dir1/file1.txt", - "dir1/file2.txt", - } - for _, file := range files { - path := filepath.Join(dir, file) - if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { - t.Fatalf("os.MkdirAll() = %v", err) - } - if err := os.WriteFile(path, []byte("test"), 0644); err != nil { - t.Fatalf("os.WriteFile() = %v", err) - } - } - }, - wantErr: true, - errorString: "cannot establish the relationship between", - }, - { - name: "dir and subDir are not nested", - getRootDirPath: func(dir string) string { - return filepath.Join(dir, "dir/dir1") - }, - getSubDirPath: func(dir string) string { - return filepath.Join(dir, "dir/dir2") - }, - setup: func(t *testing.T, dir string) { - files := []string{ - "dir/dir1/file1.txt", - "dir/dir2/file2.txt", - } - for _, file := range files { - path := filepath.Join(dir, file) - if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { - t.Fatalf("os.MkdirAll() = %v", err) - } - if err := os.WriteFile(path, []byte("test"), 0644); err != nil { - t.Fatalf("os.WriteFile() = %v", err) - } - } - }, - wantErr: true, - errorString: "subDir is not nested within the dir", - }, - { - name: "unreadable directory", - getRootDirPath: func(dir string) string { - return dir - }, - getSubDirPath: func(dir string) string { - return dir - }, - setup: func(t *testing.T, tmpDir string) { - unreadableDir := filepath.Join(tmpDir, "unreadable") - if err := os.Mkdir(unreadableDir, 0755); err != nil { - t.Fatalf("os.Mkdir() = %v", err) - } - - // Make the directory unreadable to trigger an error in filepath.WalkDir. - if err := os.Chmod(unreadableDir, 0000); err != nil { - t.Fatalf("os.Chmod() = %v", err) - } - // Schedule cleanup to restore permissions so TempDir can be removed. - t.Cleanup(func() { - _ = os.Chmod(unreadableDir, 0755) - }) - }, - wantErr: true, - errorString: "unreadable", - }, - } { - t.Run(test.name, func(t *testing.T) { - tmpDir := t.TempDir() - if test.setup != nil { - test.setup(t, tmpDir) - } - - rootDirPath := test.getRootDirPath(tmpDir) - subDirPath := test.getSubDirPath(tmpDir) - - paths, err := findSubDirRelPaths(rootDirPath, subDirPath) - if test.wantErr { - if err == nil { - t.Fatalf("%s should return error", test.name) - } - if !strings.Contains(err.Error(), test.errorString) { - t.Errorf("runConfigureCommand() err = %v, want error containing %q", err, test.errorString) - } - return - } - if err != nil { - t.Fatal(err) - } - - // Sort both slices to ensure consistent comparison. - sort.Strings(paths) - sort.Strings(test.wantPaths) - - if diff := cmp.Diff(test.wantPaths, paths); diff != "" { - t.Errorf("findSubDirRelPaths() mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestFilterPathsByRegex(t *testing.T) { - t.Parallel() - paths := []string{ - "foo/file1.txt", - "foo/file2.log", - "bar/file3.txt", - "bar/file4.log", - } - regexps := []*regexp.Regexp{ - regexp.MustCompile(`^foo/.*\.txt$`), - regexp.MustCompile(`^bar/.*`), - } - - filtered := filterPathsByRegex(paths, regexps) - - wantFiltered := []string{ - "foo/file1.txt", - "bar/file3.txt", - "bar/file4.log", - } - - sort.Strings(filtered) - sort.Strings(wantFiltered) - - if diff := cmp.Diff(wantFiltered, filtered); diff != "" { - t.Errorf("filterPathsByRegex() mismatch (-want +got):%s", diff) - } -} - -func TestFilterPathsForRemoval(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - files map[string]string - removePatterns []string - preservePatterns []string - wantToRemove []string - wantErr bool - }{ - { - name: "remove all txt files, preserve nothing", - files: map[string]string{ - "file1.txt": "", - "dir1/file2.txt": "", - "dir2/file3.log": "", - }, - removePatterns: []string{`.*\.txt`}, - preservePatterns: []string{}, - wantToRemove: []string{"file1.txt", "dir1/file2.txt"}, - }, - { - name: "remove all files, preserve log files", - files: map[string]string{ - "file1.txt": "", - "dir1/file2.txt": "", - "dir2/file3.log": "", - }, - removePatterns: []string{".*"}, - preservePatterns: []string{`.*\.log`}, - wantToRemove: []string{"dir1", "dir2", "file1.txt", "dir1/file2.txt"}, - }, - { - name: "remove files in dir1, preserve nothing", - files: map[string]string{ - "file1.txt": "", - "dir1/file2.txt": "", - "dir1/file3.log": "", - "dir2/file4.txt": "", - }, - removePatterns: []string{`dir1/.*`}, - preservePatterns: []string{}, - wantToRemove: []string{"dir1/file2.txt", "dir1/file3.log"}, - }, - { - name: "remove all, preserve files in dir2", - files: map[string]string{ - "file1.txt": "", - "dir1/file2.txt": "", - "dir2/file3.txt": "", - }, - removePatterns: []string{".*"}, - preservePatterns: []string{`dir2/.*`}, - wantToRemove: []string{"dir1", "dir2", "file1.txt", "dir1/file2.txt"}, - }, - { - name: "no files", - files: map[string]string{}, - removePatterns: []string{".*"}, - preservePatterns: []string{}, - // Effectively an empty array, but cmp.Diff expect a nil slice instead of empty slice - wantToRemove: nil, - }, - { - name: "no matching files to remove", - files: map[string]string{ - "file1.txt": "", - "dir1/file2.txt": "", - "dir2/file3.txt": "", - }, - removePatterns: []string{"random_dir/.*"}, - preservePatterns: []string{}, - // Effectively an empty array, but cmp.Diff expect a nil slice instead of empty slice - wantToRemove: nil, - }, - { - name: "remove pattern has invalid regex", - files: map[string]string{}, - removePatterns: []string{"["}, - wantErr: true, - }, - { - name: "preserve pattern has invalid regex", - files: map[string]string{}, - preservePatterns: []string{"["}, - wantErr: true, - }, - } { - t.Run(test.name, func(t *testing.T) { - tmpDir := t.TempDir() - for path, content := range test.files { - fullPath := filepath.Join(tmpDir, path) - if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil { - t.Fatalf("os.MkdirAll() = %v", err) - } - if err := os.WriteFile(fullPath, []byte(content), 0644); err != nil { - t.Fatalf("os.WriteFile() = %v", err) - } - } - - sourceRootPaths, _ := findSubDirRelPaths(tmpDir, tmpDir) - gotToRemove, err := filterPathsForRemoval(sourceRootPaths, test.removePatterns, test.preservePatterns) - if test.wantErr { - if err == nil { - t.Fatalf("%s should return error", test.name) - } - return - } - if err != nil { - t.Fatal(err) - } - - sort.Strings(gotToRemove) - sort.Strings(test.wantToRemove) - - if diff := cmp.Diff(test.wantToRemove, gotToRemove); diff != "" { - t.Errorf("filterPathsForRemoval() toRemove mismatch in %s (-want +got):\n%s", test.name, diff) - } - }) - } -} - -func TestSeparateFilesAndDirs(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - setup func(t *testing.T, tmpDir string) - paths []string - wantFiles []string - wantDirs []string - wantErr bool - }{ - { - name: "mixed files, dirs, and non-existent path", - setup: func(t *testing.T, tmpDir string) { - files := []string{"file1.txt", "dir1/file2.txt"} - dirs := []string{"dir1", "dir2"} - for _, file := range files { - path := filepath.Join(tmpDir, file) - if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { - t.Fatalf("os.MkdirAll() = %v", err) - } - if err := os.WriteFile(path, []byte("test"), 0644); err != nil { - t.Fatalf("os.WriteFile() = %v", err) - } - } - for _, dir := range dirs { - if err := os.MkdirAll(filepath.Join(tmpDir, dir), 0755); err != nil { - t.Fatalf("os.MkdirAll() = %v", err) - } - } - }, - paths: []string{"file1.txt", "dir1/file2.txt", "dir1", "dir2", "non-existent-file"}, - wantFiles: []string{"file1.txt", "dir1/file2.txt"}, - wantDirs: []string{"dir1", "dir2"}, - }, - { - name: "stat error", - paths: []string{strings.Repeat("a", 300)}, - wantErr: true, - }, - } { - t.Run(test.name, func(t *testing.T) { - tmpDir := t.TempDir() - if test.setup != nil { - test.setup(t, tmpDir) - } - - var paths []string - for _, path := range test.paths { - paths = append(paths, filepath.Join(tmpDir, path)) - } - gotFiles, gotDirs, err := separateFilesAndDirs(paths) - if test.wantErr { - if err == nil { - t.Fatalf("%s should return error", test.name) - } - return - } - if err != nil { - t.Fatal(err) - } - - var gotFilesRelPath []string - for _, gotFile := range gotFiles { - relPath, _ := filepath.Rel(tmpDir, gotFile) - gotFilesRelPath = append(gotFilesRelPath, relPath) - } - - var gotDirsRelPath []string - for _, gotDir := range gotDirs { - relPath, _ := filepath.Rel(tmpDir, gotDir) - gotDirsRelPath = append(gotDirsRelPath, relPath) - } - - sort.Strings(gotFilesRelPath) - sort.Strings(gotDirsRelPath) - sort.Strings(test.wantFiles) - sort.Strings(test.wantDirs) - - if diff := cmp.Diff(test.wantFiles, gotFilesRelPath); diff != "" { - t.Errorf("separateFilesAndDirs() files mismatch (-want +got):\n%s", diff) - } - if diff := cmp.Diff(test.wantDirs, gotDirsRelPath); diff != "" { - t.Errorf("separateFilesAndDirs() dirs mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestCompileRegexps(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - patterns []string - wantErr bool - }{ - { - name: "valid patterns", - patterns: []string{ - `^foo.*`, - `\\.txt$`, - }, - wantErr: false, - }, - { - name: "empty patterns", - patterns: []string{}, - wantErr: false, - }, - { - name: "invalid pattern", - patterns: []string{ - `[`, - }, - wantErr: true, - }, - { - name: "mixed valid and invalid patterns", - patterns: []string{ - `^foo.*`, - `[`, - }, - wantErr: true, - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - regexps, err := compileRegexps(test.patterns) - if (err != nil) != test.wantErr { - t.Fatalf("compileRegexps() error = %v, wantErr %v", err, test.wantErr) - } - if !test.wantErr { - if len(regexps) != len(test.patterns) { - t.Errorf("compileRegexps() len = %d, want %d", len(regexps), len(test.patterns)) - } - } - }) - } -} - -func TestCommitAndPush(t *testing.T) { - for _, test := range []struct { - name string - setupMockRepo func(t *testing.T) legacygitrepo.Repository - setupMockClient func(t *testing.T) GitHubClient - state *legacyconfig.LibrarianState - prType pullRequestType - failedGenerations int - commit bool - push bool - wantErr bool - expectedErrMsg string - check func(t *testing.T, repo legacygitrepo.Repository) - wantPRBodyFile bool - prBodyBuilder func() (string, error) - }{ - { - name: "Push flag and Commit flag are not specified", - setupMockRepo: func(t *testing.T) legacygitrepo.Repository { - return &MockRepository{ - Dir: t.TempDir(), - RemotesValue: []*legacygitrepo.Remote{ - { - Name: "origin", - URLs: []string{"https://github.com/googleapis/librarian.git"}, - }, - }, - } - }, - setupMockClient: func(t *testing.T) GitHubClient { - return nil - }, - state: &legacyconfig.LibrarianState{}, - prType: pullRequestRelease, - check: func(t *testing.T, repo legacygitrepo.Repository) { - mockRepo := repo.(*MockRepository) - if mockRepo.PushCalls != 0 { - t.Errorf("Push was called %d times, expected 0", mockRepo.PushCalls) - } - }, - wantPRBodyFile: true, - }, - { - name: "No push or commit flags, and clean repo", - setupMockRepo: func(t *testing.T) legacygitrepo.Repository { - return &MockRepository{ - Dir: t.TempDir(), - RemotesValue: []*legacygitrepo.Remote{ - { - Name: "origin", - URLs: []string{"https://github.com/googleapis/librarian.git"}, - }, - }, - IsCleanValue: true, - } - }, - setupMockClient: func(t *testing.T) GitHubClient { - return nil - }, - state: &legacyconfig.LibrarianState{}, - prType: pullRequestRelease, - commit: false, - push: false, - wantErr: false, - wantPRBodyFile: false, - }, - { - name: "create a commit", - setupMockRepo: func(t *testing.T) legacygitrepo.Repository { - remote := &legacygitrepo.Remote{ - Name: "origin", - URLs: []string{"https://github.com/googleapis/librarian.git"}, - } - return &MockRepository{ - Dir: t.TempDir(), - RemotesValue: []*legacygitrepo.Remote{remote}, - } - }, - setupMockClient: func(t *testing.T) GitHubClient { - return &mockGitHubClient{ - createdPR: &legacygithub.PullRequestMetadata{Number: 123, Repo: &legacygithub.Repository{Owner: "test-owner", Name: "test-repo"}}, - } - }, - state: &legacyconfig.LibrarianState{}, - prType: pullRequestRelease, - commit: true, - check: func(t *testing.T, repo legacygitrepo.Repository) { - mockRepo := repo.(*MockRepository) - if mockRepo.PushCalls != 0 { - t.Errorf("Push was called %d times, expected 0", mockRepo.PushCalls) - } - }, - wantPRBodyFile: true, - }, - { - name: "create a generate pull request", - setupMockRepo: func(t *testing.T) legacygitrepo.Repository { - remote := &legacygitrepo.Remote{ - Name: "origin", - URLs: []string{"https://github.com/googleapis/librarian.git"}, - } - return &MockRepository{ - Dir: t.TempDir(), - RemotesValue: []*legacygitrepo.Remote{remote}, - } - }, - setupMockClient: func(t *testing.T) GitHubClient { - return &mockGitHubClient{ - createdPR: &legacygithub.PullRequestMetadata{Number: 123, Repo: &legacygithub.Repository{Owner: "test-owner", Name: "test-repo"}}, - } - }, - state: &legacyconfig.LibrarianState{}, - prType: pullRequestGenerate, - push: true, - }, - { - name: "create a release pull request", - setupMockRepo: func(t *testing.T) legacygitrepo.Repository { - remote := &legacygitrepo.Remote{ - Name: "origin", - URLs: []string{"https://github.com/googleapis/librarian.git"}, - } - return &MockRepository{ - Dir: t.TempDir(), - RemotesValue: []*legacygitrepo.Remote{remote}, - } - }, - setupMockClient: func(t *testing.T) GitHubClient { - return &mockGitHubClient{ - createdPR: &legacygithub.PullRequestMetadata{Number: 123, Repo: &legacygithub.Repository{Owner: "test-owner", Name: "test-repo"}}, - } - }, - state: &legacyconfig.LibrarianState{}, - prType: pullRequestRelease, - push: true, - }, - { - name: "No GitHub Remote", - setupMockRepo: func(t *testing.T) legacygitrepo.Repository { - return &MockRepository{ - Dir: t.TempDir(), - RemotesValue: []*legacygitrepo.Remote{}, // No remotes - } - }, - setupMockClient: func(t *testing.T) GitHubClient { - return nil - }, - prType: pullRequestGenerate, - push: true, - state: &legacyconfig.LibrarianState{}, - wantErr: true, - expectedErrMsg: "could not find an 'origin' remote", - }, - { - name: "AddAll error", - setupMockRepo: func(t *testing.T) legacygitrepo.Repository { - remote := &legacygitrepo.Remote{ - Name: "origin", - URLs: []string{"https://github.com/googleapis/librarian.git"}, - } - return &MockRepository{ - Dir: t.TempDir(), - RemotesValue: []*legacygitrepo.Remote{remote}, - AddAllError: errors.New("mock add all error"), - } - }, - setupMockClient: func(t *testing.T) GitHubClient { - return nil - }, - prType: pullRequestGenerate, - push: true, - wantErr: true, - expectedErrMsg: "mock add all error", - }, - { - name: "Create branch error", - setupMockRepo: func(t *testing.T) legacygitrepo.Repository { - remote := &legacygitrepo.Remote{ - Name: "origin", - URLs: []string{"https://github.com/googleapis/librarian.git"}, - } - return &MockRepository{ - Dir: t.TempDir(), - RemotesValue: []*legacygitrepo.Remote{remote}, - CreateBranchAndCheckoutError: errors.New("create branch error"), - } - }, - setupMockClient: func(t *testing.T) GitHubClient { - return nil - }, - prType: pullRequestGenerate, - push: true, - wantErr: true, - expectedErrMsg: "create branch error", - }, - { - name: "Commit error", - setupMockRepo: func(t *testing.T) legacygitrepo.Repository { - remote := &legacygitrepo.Remote{ - Name: "origin", - URLs: []string{"https://github.com/googleapis/librarian.git"}, - } - return &MockRepository{ - Dir: t.TempDir(), - RemotesValue: []*legacygitrepo.Remote{remote}, - CommitError: errors.New("commit error"), - } - }, - setupMockClient: func(t *testing.T) GitHubClient { - return nil - }, - prType: pullRequestGenerate, - push: true, - wantErr: true, - expectedErrMsg: "commit error", - }, - { - name: "Push error", - setupMockRepo: func(t *testing.T) legacygitrepo.Repository { - remote := &legacygitrepo.Remote{ - Name: "origin", - URLs: []string{"https://github.com/googleapis/librarian.git"}, - } - return &MockRepository{ - Dir: t.TempDir(), - RemotesValue: []*legacygitrepo.Remote{remote}, - PushError: errors.New("push error"), - } - }, - setupMockClient: func(t *testing.T) GitHubClient { - return nil - }, - prType: pullRequestGenerate, - push: true, - wantErr: true, - expectedErrMsg: "push error", - }, - { - name: "Create PR body error", - setupMockRepo: func(t *testing.T) legacygitrepo.Repository { - remote := &legacygitrepo.Remote{ - Name: "origin", - URLs: []string{"https://github.com/googleapis/librarian.git"}, - } - return &MockRepository{ - Dir: t.TempDir(), - RemotesValue: []*legacygitrepo.Remote{remote}, - } - }, - setupMockClient: func(t *testing.T) GitHubClient { - return &mockGitHubClient{} - }, - state: &legacyconfig.LibrarianState{}, - prType: 100, - push: true, - wantErr: true, - expectedErrMsg: "failed to create pull request body", - prBodyBuilder: func() (string, error) { return "", fmt.Errorf("failed to create pull request body") }, - }, - { - name: "Create pull request error", - setupMockRepo: func(t *testing.T) legacygitrepo.Repository { - remote := &legacygitrepo.Remote{ - Name: "origin", - URLs: []string{"https://github.com/googleapis/librarian.git"}, - } - return &MockRepository{ - Dir: t.TempDir(), - RemotesValue: []*legacygitrepo.Remote{remote}, - } - }, - setupMockClient: func(t *testing.T) GitHubClient { - return &mockGitHubClient{ - createPullRequestErr: errors.New("create pull request error"), - } - }, - state: &legacyconfig.LibrarianState{}, - prType: pullRequestGenerate, - push: true, - wantErr: true, - expectedErrMsg: "failed to create pull request", - }, - { - name: "No changes to commit", - setupMockRepo: func(t *testing.T) legacygitrepo.Repository { - remote := &legacygitrepo.Remote{ - Name: "origin", - URLs: []string{"https://github.com/googleapis/librarian.git"}, - } - return &MockRepository{ - Dir: t.TempDir(), - IsCleanValue: true, - RemotesValue: []*legacygitrepo.Remote{remote}, - } - }, - setupMockClient: func(t *testing.T) GitHubClient { - return nil - }, - prType: pullRequestGenerate, - push: true, - }, - { - name: "create_a_comment_on_generation_pr_error", - setupMockRepo: func(t *testing.T) legacygitrepo.Repository { - remote := &legacygitrepo.Remote{ - Name: "origin", - URLs: []string{"https://github.com/googleapis/librarian.git"}, - } - return &MockRepository{ - Dir: t.TempDir(), - RemotesValue: []*legacygitrepo.Remote{remote}, - } - }, - setupMockClient: func(t *testing.T) GitHubClient { - return &mockGitHubClient{ - createdPR: &legacygithub.PullRequestMetadata{Number: 123, Repo: &legacygithub.Repository{Owner: "test-owner", Name: "test-repo"}}, - createIssueErr: errors.New("simulate comment creation error"), - } - }, - state: &legacyconfig.LibrarianState{}, - prType: pullRequestGenerate, - failedGenerations: 1, - push: true, - wantErr: true, - expectedErrMsg: "failed to add pull request comment", - }, - } { - t.Run(test.name, func(t *testing.T) { - repo := test.setupMockRepo(t) - client := test.setupMockClient(t) - - // set default PR body builder - if test.prBodyBuilder == nil { - test.prBodyBuilder = func() (string, error) { return "some pr body", nil } - } - - commitInfo := &commitInfo{ - commit: test.commit, - commitMessage: "", - ghClient: client, - prType: test.prType, - push: test.push, - languageRepo: repo, - state: test.state, - failedGenerations: test.failedGenerations, - workRoot: t.TempDir(), - prBodyBuilder: test.prBodyBuilder, - } - - err := commitAndPush(t.Context(), commitInfo) - - if test.wantErr { - if err == nil { - t.Fatal("commitAndPush() expected error, got nil") - } - if test.expectedErrMsg != "" && !strings.Contains(err.Error(), test.expectedErrMsg) { - t.Errorf("commitAndPush() error = %v, expected to contain: %q", err, test.expectedErrMsg) - } - return - } - if err != nil { - t.Errorf("%s: commitAndPush() returned unexpected error: %v", test.name, err) - return - } - - if test.check != nil { - test.check(t, repo) - } - - gotPRBodyFile := gotPRBodyFile(t, commitInfo.workRoot) - if test.wantPRBodyFile != gotPRBodyFile { - t.Errorf("commitAndPush() wantPRBodyFile = %t, gotPRBodyFile = %t", test.wantPRBodyFile, gotPRBodyFile) - } - }) - } -} - -func TestWritePRBody(t *testing.T) { - for _, test := range []struct { - name string - info *commitInfo - wantErr bool - wantFile bool - }{ - { - name: "success", - info: &commitInfo{ - languageRepo: &MockRepository{ - Dir: t.TempDir(), - RemotesValue: []*legacygitrepo.Remote{ - { - Name: "origin", - URLs: []string{"https://github.com/googleapis/librarian.git"}, - }, - }, - }, - prType: pullRequestRelease, - state: &legacyconfig.LibrarianState{}, - workRoot: t.TempDir(), - prBodyBuilder: func() (string, error) { return "some pr body", nil }, - }, - wantFile: true, - }, - { - name: "unable to build PR body", - info: &commitInfo{ - languageRepo: &MockRepository{ - Dir: t.TempDir(), - RemotesValue: []*legacygitrepo.Remote{ - { - Name: "not-origin", - URLs: []string{"https://github.com/googleapis/librarian.git"}, - }, - }, - }, - prType: pullRequestRelease, - workRoot: t.TempDir(), - prBodyBuilder: func() (string, error) { return "", fmt.Errorf("some error") }, - }, - wantErr: true, - }, - { - name: "unable to save file", - info: &commitInfo{ - languageRepo: &MockRepository{ - Dir: t.TempDir(), - RemotesValue: []*legacygitrepo.Remote{ - { - Name: "origin", - URLs: []string{"https://github.com/googleapis/librarian.git"}, - }, - }, - }, - prType: pullRequestRelease, - state: &legacyconfig.LibrarianState{}, - workRoot: filepath.Join(t.TempDir(), "missing-directory"), - prBodyBuilder: func() (string, error) { return "some pr body", nil }, - }, - wantErr: true, - }, - { - name: "no body builder", - info: &commitInfo{ - languageRepo: &MockRepository{ - Dir: t.TempDir(), - RemotesValue: []*legacygitrepo.Remote{ - { - Name: "origin", - URLs: []string{"https://github.com/googleapis/librarian.git"}, - }, - }, - }, - prType: pullRequestRelease, - state: &legacyconfig.LibrarianState{}, - workRoot: filepath.Join(t.TempDir(), "missing-directory"), - }, - wantErr: true, - }, - } { - t.Run(test.name, func(t *testing.T) { - err := writePRBody(test.info) - - if test.wantErr { - if err == nil { - t.Fatalf("writePRBody() expected error, but no error returned") - } - return - } - if err != nil { - t.Fatalf("unexpected error %v", err) - } - - gotFile := gotPRBodyFile(t, test.info.workRoot) - if test.wantFile != gotFile { - t.Errorf("writePRBody() wantFile = %t, gotFile = %t", test.wantFile, gotFile) - } - }) - } -} - -func gotPRBodyFile(t *testing.T, workRoot string) bool { - possibleFilePath := filepath.Join(workRoot, prBodyFile) - _, err := os.Stat(possibleFilePath) - if err != nil && !errors.Is(err, fs.ErrNotExist) { - t.Fatalf("error other than IsNotExist finding status of %s", possibleFilePath) - } - return err == nil -} - -func TestAddLabelsToPullRequest(t *testing.T) { - for _, test := range []struct { - name string - setupMockRepo func(t *testing.T) legacygitrepo.Repository - mockGithubClient *mockGitHubClient - prMetadata legacygithub.PullRequestMetadata - wantPullRequestLabels []string - wantErr bool - expectedErrMsg string - }{ - { - name: "Add All Labels", - setupMockRepo: func(t *testing.T) legacygitrepo.Repository { - return &MockRepository{} - }, - mockGithubClient: &mockGitHubClient{}, - prMetadata: legacygithub.PullRequestMetadata{ - Repo: &legacygithub.Repository{Owner: "test-owner", Name: "test-repo"}, - Number: 7, - }, - wantPullRequestLabels: []string{"release:pending", "1234", "label1234"}, - }, - { - name: "Failed to add labels", - setupMockRepo: func(t *testing.T) legacygitrepo.Repository { - return &MockRepository{} - }, - mockGithubClient: &mockGitHubClient{ - addLabelsToIssuesErr: errors.New("Can't add labels"), - }, - prMetadata: legacygithub.PullRequestMetadata{ - Repo: &legacygithub.Repository{Owner: "test-owner", Name: "test-repo"}, - Number: 7, - }, - wantPullRequestLabels: []string{"release:pending", "1234", "label1234"}, - wantErr: true, - expectedErrMsg: "failed to add labels to pull request", - }, - } { - t.Run(test.name, func(t *testing.T) { - client := test.mockGithubClient - prMetadata := test.prMetadata - err := addLabelsToPullRequest(t.Context(), client, test.wantPullRequestLabels, &prMetadata) - - if test.wantErr { - if err == nil { - t.Fatal("addLabelsToPullRequest() expected error, got nil") - } - if test.expectedErrMsg != "" && !strings.Contains(err.Error(), test.expectedErrMsg) { - t.Errorf("addLabelsToPullRequest() error = %v, expected to contain: %q", err, test.expectedErrMsg) - } - return - } - if err != nil { - t.Errorf("%s: addLabelsToPullRequest() returned unexpected error: %v", test.name, err) - } - if diff := cmp.Diff(test.wantPullRequestLabels, test.mockGithubClient.labels); diff != "" { - t.Errorf("addLabelsToPullRequest() mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestCopyLibraryFiles(t *testing.T) { - t.Parallel() - setup := func(foo, contents string, files []string) { - for _, relPath := range files { - fullPath := filepath.Join(foo, relPath) - if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil { - t.Error(err) - } - - if err := os.WriteFile(fullPath, []byte(contents), 0755); err != nil { - t.Error(err) - } - } - } - for _, test := range []struct { - name string - repoDir string - outputDir string - libraryID string - state *legacyconfig.LibrarianState - existingFiles []string - filesToCreate []string - setup func(t *testing.T, outputDir string) - verify func(t *testing.T, repoDir string) - wantFiles []string - skipFiles []string - failOnExistingFile bool - wantErr bool - wantErrMsg string - }{ - { - repoDir: "/invalid-dst-path", - name: "invalid dst", - outputDir: t.TempDir(), - libraryID: "example-library", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "example-library", - SourceRoots: []string{ - "a-library/path", - }, - }, - }, - }, - filesToCreate: []string{ - "a-library/path/example.txt", - }, - wantErr: true, - wantErrMsg: "failed to make directory", - }, - { - name: "copy library files", - repoDir: filepath.Join(t.TempDir(), "dst"), - outputDir: filepath.Join(t.TempDir(), "foo"), - libraryID: "example-library", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "example-library", - SourceRoots: []string{ - "a/path", - "another/path", - }, - }, - }, - }, - filesToCreate: []string{ - "a/path/example.txt", - "another/path/example.txt", - "skipped/path/example.txt", - }, - wantFiles: []string{ - "a/path/example.txt", - "another/path/example.txt", - }, - skipFiles: []string{ - "skipped/path/example.txt", - }, - }, - { - name: "copy library files with symbolic link", - repoDir: filepath.Join(t.TempDir(), "dst"), - outputDir: filepath.Join(t.TempDir(), "src"), - libraryID: "example-library", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "example-library", - SourceRoots: []string{ - "a/path", - }, - }, - }, - }, - filesToCreate: []string{ - "a/path/target.txt", - }, - setup: func(t *testing.T, outputDir string) { - if err := os.Symlink("target.txt", filepath.Join(outputDir, "a/path", "link.txt")); err != nil { - t.Fatalf("failed to create symlink: %v", err) - } - }, - wantFiles: []string{ - "a/path/target.txt", - "a/path/link.txt", - }, - verify: func(t *testing.T, repoDir string) { - linkPath := filepath.Join(repoDir, "a/path", "link.txt") - info, err := os.Lstat(linkPath) - if err != nil { - t.Fatalf("failed to lstat symlink: %v", err) - } - if info.Mode()&os.ModeSymlink == 0 { - t.Errorf("copied file is not a symlink") - } - target, err := os.Readlink(linkPath) - if err != nil { - t.Fatalf("failed to readlink: %v", err) - } - if target != "target.txt" { - t.Errorf("symlink target is incorrect: got %q, want %q", target, "target.txt") - } - }, - }, - { - name: "library not found", - repoDir: filepath.Join(t.TempDir(), "dst"), - outputDir: filepath.Join(t.TempDir(), "foo"), - libraryID: "non-existent-library", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "example-library", - }, - }, - }, - wantErr: true, - wantErrMsg: "not found", - }, - { - repoDir: filepath.Join(t.TempDir(), "dst"), - name: "one source root empty", - outputDir: filepath.Join(t.TempDir(), "foo"), - libraryID: "example-library", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "example-library", - SourceRoots: []string{ - "a/path", - "another/path", - }, - }, - }, - }, - filesToCreate: []string{ - "a/path/example.txt", - "skipped/path/example.txt", - }, - wantFiles: []string{ - "a/path/example.txt", - }, - skipFiles: []string{ - "skipped/path/example.txt", - }, - }, - { - repoDir: filepath.Join(t.TempDir(), "dst"), - name: "only_delta_files_are_copied", - outputDir: filepath.Join(t.TempDir(), "foo"), - libraryID: "example-library", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "example-library", - SourceRoots: []string{ - "a/path", - "another/path", - }, - }, - }, - }, - existingFiles: []string{ - "a/path/example.txt", - }, - filesToCreate: []string{ - "another/path/example.txt", - }, - wantFiles: []string{ - "a/path/example.txt", - "another/path/example.txt", - }, - }, - { - name: "file_existed_in_dst", - repoDir: filepath.Join(t.TempDir(), "dst"), - outputDir: filepath.Join(t.TempDir(), "foo"), - libraryID: "example-library", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "example-library", - SourceRoots: []string{ - "a/path", - "another/path", - }, - }, - }, - }, - existingFiles: []string{ - "a/path/example.txt", - }, - filesToCreate: []string{ - "a/path/example.txt", - "another/path/example.txt", - }, - failOnExistingFile: true, - wantErr: true, - wantErrMsg: "file existed in destination", - }, - } { - t.Run(test.name, func(t *testing.T) { - if len(test.existingFiles) > 0 { - setup(test.repoDir, "old contents", test.existingFiles) - } - if len(test.filesToCreate) > 0 { - setup(test.outputDir, "new contents", test.filesToCreate) - } - if test.setup != nil { - test.setup(t, test.outputDir) - } - err := copyLibraryFiles(test.state, test.repoDir, test.libraryID, test.outputDir, test.failOnExistingFile) - if test.wantErr { - if err == nil { - t.Fatal("copyLibraryFiles() should fail") - } - if !strings.Contains(err.Error(), test.wantErrMsg) { - t.Errorf("want error message: %s, got: %s", test.wantErrMsg, err.Error()) - } - - return - } - if err != nil { - t.Errorf("failed to run copyLibraryFiles(): %s", err.Error()) - } - - for _, file := range test.wantFiles { - fullPath := filepath.Join(test.repoDir, file) - if _, err := os.Stat(fullPath); err != nil { - t.Errorf("file %s is not copied to %s", file, test.repoDir) - } - } - - for _, file := range test.existingFiles { - fullPath := filepath.Join(test.repoDir, file) - got, err := os.ReadFile(fullPath) - if err != nil { - t.Error(err) - } - if diff := cmp.Diff("old contents", string(got)); diff != "" { - t.Errorf("file contents mismatch (-want +got):\n%s", diff) - } - } - - for _, file := range test.skipFiles { - fullPath := filepath.Join(test.repoDir, file) - if _, err := os.Stat(fullPath); !errors.Is(err, fs.ErrNotExist) { - t.Errorf("file %s should not be copied to %s", file, test.repoDir) - } - } - - if test.verify != nil { - test.verify(t, test.repoDir) - } - }) - } -} - -func TestCopyFile(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - dst string - foo string - wantfooFile bool - wantErr bool - wantErrMsg string - }{ - { - name: "invalid foo", - foo: "/invalid-path/example.txt", - wantErr: true, - wantErrMsg: "failed to lstat file", - }, - { - name: "invalid dst path", - foo: filepath.Join(os.TempDir(), "example.txt"), - dst: "/invalid-path/example.txt", - wantfooFile: true, - wantErr: true, - wantErrMsg: "failed to make directory", - }, - { - name: "invalid dst file", - foo: filepath.Join(os.TempDir(), "example.txt"), - dst: filepath.Join(os.TempDir(), "example\x00.txt"), - wantfooFile: true, - wantErr: true, - wantErrMsg: "failed to create file", - }, - } { - t.Run(test.name, func(t *testing.T) { - if test.wantfooFile { - if err := os.MkdirAll(filepath.Dir(test.foo), 0755); err != nil { - t.Error(err) - } - sourceFile, err := os.Create(test.foo) - if err != nil { - t.Error(err) - } - if err := sourceFile.Close(); err != nil { - t.Error(err) - } - } - err := copyFile(test.dst, test.foo) - if test.wantErr { - if err == nil { - t.Fatal("copyFile() should fail") - } - - if !strings.Contains(err.Error(), test.wantErrMsg) { - t.Errorf("want error message: %s, got: %s", test.wantErrMsg, err.Error()) - } - - return - } - - }) - } -} - -func TestCopyGlobalAllowlist(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - cfg *legacyconfig.LibrarianConfig - files []string - copied []string - skipped []string - doNotCreateOutput bool // do not create files in output dir. - wantErr bool - wantErrMsg string - copyReadOnly bool - }{ - { - name: "copied all global allowlist", - cfg: &legacyconfig.LibrarianConfig{ - GlobalFilesAllowlist: []*legacyconfig.GlobalFile{ - { - Path: "one/path/example.txt", - Permissions: "read-write", - }, - { - Path: "another/path/example.txt", - Permissions: "write-only", - }, - }, - }, - files: []string{ - "one/path/example.txt", - "another/path/example.txt", - "ignored/path/example.txt", - }, - copied: []string{ - "one/path/example.txt", - "another/path/example.txt", - }, - skipped: []string{ - "ignored/path/example.txt", - }, - }, - { - name: "read only file is not copied", - cfg: &legacyconfig.LibrarianConfig{ - GlobalFilesAllowlist: []*legacyconfig.GlobalFile{ - { - Path: "one/path/example.txt", - Permissions: "read-write", - }, - { - Path: "another/path/example.txt", - Permissions: "read-only", - }, - }, - }, - files: []string{ - "one/path/example.txt", - "another/path/example.txt", - "ignored/path/example.txt", - }, - copied: []string{ - "one/path/example.txt", - }, - skipped: []string{ - "another/path/example.txt", - "ignored/path/example.txt", - }, - }, - { - name: "source doesn't have the global file", - cfg: &legacyconfig.LibrarianConfig{ - GlobalFilesAllowlist: []*legacyconfig.GlobalFile{ - { - Path: "one/path/example.txt", - Permissions: "read-write", - }, - }, - }, - files: []string{ - "one/path/example.txt", - }, - doNotCreateOutput: true, - }, - { - name: "copies read-only files", - copyReadOnly: true, - cfg: &legacyconfig.LibrarianConfig{ - GlobalFilesAllowlist: []*legacyconfig.GlobalFile{ - { - Path: "one/path/example.txt", - Permissions: "read-write", - }, - { - Path: "another/path/example.txt", - Permissions: "read-only", - }, - }, - }, - files: []string{ - "one/path/example.txt", - "another/path/example.txt", - "ignored/path/example.txt", - }, - copied: []string{ - "one/path/example.txt", - "another/path/example.txt", - }, - skipped: []string{ - "ignored/path/example.txt", - }, - }, - { - name: "nil_librarian_config", - copyReadOnly: true, - cfg: nil, - }, - } { - t.Run(test.name, func(t *testing.T) { - output := t.TempDir() - repo := t.TempDir() - for _, oneFile := range test.files { - // Create files in repo directory. - file := filepath.Join(repo, oneFile) - dir := filepath.Dir(file) - if err := os.MkdirAll(dir, 0755); err != nil { - t.Error(err) - } - - err := os.WriteFile(file, []byte("old content"), 0755) - if err != nil { - t.Error(err) - } - - if test.doNotCreateOutput { - continue - } - - // Create files in output directory. - file = filepath.Join(output, oneFile) - dir = filepath.Dir(file) - if err := os.MkdirAll(dir, 0755); err != nil { - t.Error(err) - } - - err = os.WriteFile(file, []byte("new content"), 0755) - if err != nil { - t.Error(err) - } - } - - err := copyGlobalAllowlist(test.cfg, repo, output, test.copyReadOnly) - - if test.wantErr { - if err == nil { - t.Fatal("cleanAndCopyGlobalAllowlist() should return error") - } - - if !strings.Contains(err.Error(), test.wantErrMsg) { - t.Errorf("want error message: %q, got %q", test.wantErrMsg, err.Error()) - } - - return - } - if err != nil { - t.Errorf("failed to run cleanAndCopyGlobalAllowlist(): %q", err.Error()) - } - - for _, wantFile := range test.copied { - got, err := os.ReadFile(filepath.Join(repo, wantFile)) - if err != nil { - return - } - if diff := cmp.Diff("new content", string(got)); diff != "" { - t.Errorf("state mismatch (-want +got):\n%s in %s", diff, wantFile) - } - } - // Make sure the skipped files are not changed. - for _, skippedFile := range test.skipped { - got, err := os.ReadFile(filepath.Join(repo, skippedFile)) - if err != nil { - return - } - if diff := cmp.Diff("old content", string(got)); diff != "" { - t.Errorf("state mismatch (-want +got):\n%s in %s", diff, skippedFile) - } - } - }) - } -} - -func TestWriteTiming_BadWorkRoot(t *testing.T) { - badWorkRoot := filepath.Join(t.TempDir(), "missing-directory") - timeByLibrary := map[string]time.Duration{ - "a": time.Second, - } - if err := writeTiming(badWorkRoot, timeByLibrary); err == nil { - t.Fatal("writeTiming() expected to return error") - } -} - -func TestWriteTiming_EmptyMap(t *testing.T) { - workRoot := filepath.Join(t.TempDir()) - timeByLibrary := map[string]time.Duration{} - if err := writeTiming(workRoot, timeByLibrary); err != nil { - t.Fatal("writeTiming() failed", err) - } - if _, err := os.Stat(filepath.Join(workRoot, timingFile)); !errors.Is(err, fs.ErrNotExist) { - t.Error("writeTiming() should not have created a file") - } -} - -func TestWriteTiming(t *testing.T) { - workRoot := t.TempDir() - timeByLibrary := map[string]time.Duration{ - "a": time.Second, - "b": 5 * time.Second, - "c": 3 * time.Second, - } - if err := writeTiming(workRoot, timeByLibrary); err != nil { - t.Fatal("writeTiming() failed", err) - } - want := `Processed 3 libraries in 9s; average=3s -b: 5s -c: 3s -a: 1s -` - bytes, err := os.ReadFile(filepath.Join(workRoot, timingFile)) - if err != nil { - t.Fatal("unable to read timing log", err) - } - got := string(bytes) - if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("writeTiming() mismatch (-want +got):%s", diff) - } -} diff --git a/internal/legacylibrarian/legacylibrarian/commit_version_analyzer.go b/internal/legacylibrarian/legacylibrarian/commit_version_analyzer.go deleted file mode 100644 index b3169806ebb..00000000000 --- a/internal/legacylibrarian/legacylibrarian/commit_version_analyzer.go +++ /dev/null @@ -1,203 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacylibrarian - -import ( - "fmt" - "log/slog" - "path/filepath" - "strings" - - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" - "github.com/googleapis/librarian/internal/legacylibrarian/legacygitrepo" - "github.com/googleapis/librarian/internal/semver" -) - -// getConventionalCommitsSinceLastRelease returns all conventional commits for the given library since the -// version specified in the state file. The repo should be the language repo. -func getConventionalCommitsSinceLastRelease(repo legacygitrepo.Repository, library *legacyconfig.LibraryState, tag string) ([]*legacygitrepo.ConventionalCommit, error) { - commits, err := repo.GetCommitsForPathsSinceTag(library.SourceRoots, tag) - - if err != nil { - return nil, fmt.Errorf("failed to get commits for library %q with source roots %q at tag %q: %w", library.ID, library.SourceRoots, tag, err) - } - - // checks that if the files in the commit are in the sources root. The release - // changes are in the language repo and NOT in the source repo. - shouldIncludeFiles := func(files []string) bool { - return shouldIncludeForRelease(files, library.SourceRoots, library.ReleaseExcludePaths) - } - - conventionalCommits, err := convertToConventionalCommits(repo, library, commits, shouldIncludeFiles) - if err != nil { - return nil, fmt.Errorf("failed to convert commits to conventional commits for library %q: %w", library.ID, err) - } - return conventionalCommits, nil -} - -// shouldIncludeForRelease determines if a commit should be included in a release. -// It returns true if there is at least one file in the commit that is under a source_root -// and not under a release_exclude_path. -func shouldIncludeForRelease(files, sourceRoots, excludePaths []string) bool { - for _, file := range files { - if isUnderAnyPath(file, sourceRoots) && !isUnderAnyPath(file, excludePaths) { - return true - } - } - return false -} - -// getConventionalCommitsSinceLastGeneration returns all conventional commits for -// all API paths in given library since the last generation. The repo input should -// be the googleapis source repo. -func getConventionalCommitsSinceLastGeneration(sourceRepo legacygitrepo.Repository, library *legacyconfig.LibraryState, lastGenCommit string) ([]*legacygitrepo.ConventionalCommit, error) { - if lastGenCommit == "" { - slog.Info("the last generation commit is empty, skip fetching conventional commits", "library", library.ID) - return make([]*legacygitrepo.ConventionalCommit, 0), nil - } - - apiPaths := make([]string, 0) - for _, oneAPI := range library.APIs { - apiPaths = append(apiPaths, oneAPI.Path) - } - - sourceCommits, err := sourceRepo.GetCommitsForPathsSinceCommit(apiPaths, lastGenCommit) - if err != nil { - return nil, fmt.Errorf("failed to get commits for library %s at commit %s: %w", library.ID, lastGenCommit, err) - } - - // checks that the files in the commit are in the api paths for the source repo. - // The generation change is for changes in the source repo and NOT the language repo. - shouldIncludeFiles := func(sourceFiles []string) bool { - return shouldIncludeForGeneration(sourceFiles, library) - } - - return convertToConventionalCommits(sourceRepo, library, sourceCommits, shouldIncludeFiles) -} - -// shouldIncludeForGeneration determines if a commit should be included in generation. -// It returns true if there is at least one file in the commit that is under the -// library's API(s) path (a library could have multiple APIs). -func shouldIncludeForGeneration(sourceFiles []string, library *legacyconfig.LibraryState) bool { - var apiPaths []string - for _, api := range library.APIs { - apiPaths = append(apiPaths, api.Path) - } - - for _, file := range sourceFiles { - if isUnderAnyPath(file, apiPaths) { - return true - } - } - return false -} - -// libraryFilter filters a list of conventional commits by library ID. -func libraryFilter(commits []*legacygitrepo.ConventionalCommit, libraryID string) []*legacygitrepo.ConventionalCommit { - var filteredCommits []*legacygitrepo.ConventionalCommit - for _, commit := range commits { - if libraryIDs, ok := commit.Footers["Library-IDs"]; ok { - ids := strings.Split(libraryIDs, ",") - for _, id := range ids { - if strings.TrimSpace(id) == libraryID { - filteredCommits = append(filteredCommits, commit) - break - } - } - } else if commit.LibraryID == libraryID { - filteredCommits = append(filteredCommits, commit) - } - } - return filteredCommits -} - -// convertToConventionalCommits converts a list of commits in a git repo into a list -// of conventional commits. The filesFilter parameter is custom filter out non-matching -// files depending on a generation or a release change. -func convertToConventionalCommits(sourceRepo legacygitrepo.Repository, library *legacyconfig.LibraryState, commits []*legacygitrepo.Commit, filesFilter func(files []string) bool) ([]*legacygitrepo.ConventionalCommit, error) { - var conventionalCommits []*legacygitrepo.ConventionalCommit - for _, commit := range commits { - files, err := sourceRepo.ChangedFilesInCommit(commit.Hash.String()) - if err != nil { - return nil, fmt.Errorf("failed to get changed files for commit %s: %w", commit.Hash.String(), err) - } - if !filesFilter(files) { - continue - } - parsedCommits, err := legacygitrepo.ParseCommits(commit, library.ID) - if err != nil { - return nil, fmt.Errorf("failed to parse commit %s: %w", commit.Hash.String(), err) - } - if parsedCommits == nil { - continue - } - - parsedCommits = libraryFilter(parsedCommits, library.ID) - - for _, pc := range parsedCommits { - pc.CommitHash = commit.Hash.String() - } - conventionalCommits = append(conventionalCommits, parsedCommits...) - } - return conventionalCommits, nil -} - -// isUnderAnyPath returns true if the file is under any of the given paths. -func isUnderAnyPath(file string, paths []string) bool { - for _, p := range paths { - if p == "." { - return true - } - rel, err := filepath.Rel(p, file) - if err == nil && !strings.HasPrefix(rel, "..") { - return true - } - } - return false -} - -// NextVersion calculates the next semantic version based on a slice of conventional commits. -func NextVersion(commits []*legacygitrepo.ConventionalCommit, currentVersion string, releaseOnlyMode bool) (string, error) { - highestChange := getHighestChange(commits) - // In release-only mode, any releasable change is treated as a minor bump. - if releaseOnlyMode && highestChange != semver.None { - highestChange = semver.Minor - } - return semver.DeriveNext(highestChange, currentVersion, semver.DeriveNextOptions{}) -} - -// getHighestChange determines the highest-ranking change type from a slice of commits. -func getHighestChange(commits []*legacygitrepo.ConventionalCommit) semver.ChangeLevel { - highestChange := semver.None - for _, commit := range commits { - var currentChange semver.ChangeLevel - switch { - case commit.IsNested: - // ignore nested commit type for version bump - // this allows for always increase minor version for generation PR - currentChange = semver.Minor - case commit.IsBreaking: - currentChange = semver.Major - case commit.Type == "feat": - currentChange = semver.Minor - case commit.Type == "fix": - currentChange = semver.Patch - } - if currentChange > highestChange { - highestChange = currentChange - } - } - return highestChange -} diff --git a/internal/legacylibrarian/legacylibrarian/commit_version_analyzer_test.go b/internal/legacylibrarian/legacylibrarian/commit_version_analyzer_test.go deleted file mode 100644 index 11def907d4f..00000000000 --- a/internal/legacylibrarian/legacylibrarian/commit_version_analyzer_test.go +++ /dev/null @@ -1,806 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacylibrarian - -import ( - "fmt" - "strings" - "testing" - - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" - "github.com/googleapis/librarian/internal/legacylibrarian/legacygitrepo" - "github.com/googleapis/librarian/internal/semver" -) - -func TestShouldIncludeForRelease(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - files []string - sourceRoots []string - excludePaths []string - want bool - }{ - { - name: "file in source root, not excluded", - files: []string{"a/b/c.go"}, - sourceRoots: []string{"a"}, - excludePaths: []string{}, - want: true, - }, - { - name: "file in source root, and excluded", - files: []string{"a/b/c.go"}, - sourceRoots: []string{"a"}, - excludePaths: []string{"a/b"}, - }, - { - name: "file not in source root", - files: []string{"x/y/z.go"}, - sourceRoots: []string{"a"}, - excludePaths: []string{}, - }, - { - name: "one file included, one file not in source root", - files: []string{"a/b/c.go", "x/y/z.go"}, - sourceRoots: []string{"a"}, - excludePaths: []string{}, - want: true, - }, - { - name: "one file included, one file excluded", - files: []string{"a/b/c.go", "a/d/e.go"}, - sourceRoots: []string{"a"}, - excludePaths: []string{"a/d"}, - want: true, - }, - { - name: "all files excluded", - files: []string{"a/b/c.go", "a/d/e.go"}, - sourceRoots: []string{"a"}, - excludePaths: []string{"a/b", "a/d"}, - }, - { - name: "all files not in source root", - files: []string{"x/y/c.go", "w/z/e.go"}, - sourceRoots: []string{"a"}, - excludePaths: []string{}, - }, - { - name: "a file not in source root and a file in exclude path", - files: []string{"a/b/c.go", "w/z/e.go"}, - sourceRoots: []string{"a"}, - excludePaths: []string{"a/b"}, - }, - { - name: "a file in source root and not in exclude path, one file in exclude path and a file outside of source", - files: []string{"a/d/c.go", "a/b/c.go", "w/z/e.go"}, - sourceRoots: []string{"a"}, - excludePaths: []string{"a/b"}, - want: true, - }, - { - name: "no source roots", - files: []string{"a/b/c.go"}, - sourceRoots: []string{}, - excludePaths: []string{}, - }, - { - name: "source root as prefix of another source root", - files: []string{"aiplatform/file.go"}, - sourceRoots: []string{"ai"}, - excludePaths: []string{}, - }, - { - name: "excluded path is a directory", - files: []string{"foo/bar/baz.go"}, - sourceRoots: []string{"foo"}, - excludePaths: []string{"foo/bar"}, - }, - { - name: "excluded path is a file, file matching it", - files: []string{"foo/bar/go.mod"}, - sourceRoots: []string{"foo"}, - excludePaths: []string{"foo/bar/go.mod"}, - }, - { - name: "excluded path is a file, file does not match it", - files: []string{"foo/go.mod"}, - sourceRoots: []string{"foo"}, - excludePaths: []string{"foo/bar/go.mod"}, - want: true, - }, - { - name: "excluded path is a file with similar name", - files: []string{"foo/bar/go.mod.bak"}, - sourceRoots: []string{"foo"}, - excludePaths: []string{"foo/bar/go.mod"}, - want: true, - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - got := shouldIncludeForRelease(test.files, test.sourceRoots, test.excludePaths) - if got != test.want { - t.Errorf("shouldIncludeForRelease(%v, %v, %v) = %v, want %v", test.files, test.sourceRoots, test.excludePaths, got, test.want) - } - }) - } -} - -func TestShouldIncludeForGeneration(t *testing.T) { - t.Parallel() - cases := []struct { - name string - sourceFiles []string - library *legacyconfig.LibraryState - want bool - }{ - { - name: "include: source files in path", - sourceFiles: []string{"google/cloud/aiplatform/v1/featurestore_service.proto"}, - library: &legacyconfig.LibraryState{ - APIs: []*legacyconfig.API{{Path: "google/cloud/aiplatform/v1"}}, - }, - want: true, - }, - { - name: "exclude: no source files in path", - sourceFiles: []string{"google/cloud/vision/v1/image_annotator.proto"}, - library: &legacyconfig.LibraryState{ - APIs: []*legacyconfig.API{{Path: "google/cloud/aiplatform/v1"}}, - }, - want: false, - }, - { - name: "include: multiple files, some matching", - sourceFiles: []string{"google/cloud/aiplatform/v1/featurestore_service.proto", "unrelated/file"}, - library: &legacyconfig.LibraryState{ - APIs: []*legacyconfig.API{{Path: "google/cloud/aiplatform/v1"}}, - }, - want: true, - }, - { - name: "include: multiple APIs", - sourceFiles: []string{"google/cloud/vision/v1/image_annotator.proto"}, - library: &legacyconfig.LibraryState{ - APIs: []*legacyconfig.API{{Path: "google/cloud/aiplatform/v1"}, {Path: "google/cloud/vision/v1"}}, - }, - want: true, - }, - { - name: "exclude: empty source files", - library: &legacyconfig.LibraryState{ - APIs: []*legacyconfig.API{{Path: "google/cloud/aiplatform/v1"}}, - }, - want: false, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - got := shouldIncludeForGeneration(tc.sourceFiles, tc.library) - if got != tc.want { - t.Errorf("shouldIncludeForGeneration() = %v, want %v", got, tc.want) - } - }) - } -} - -func TestGetConventionalCommitsSinceLastRelease(t *testing.T) { - - t.Parallel() - - pathAndMessages := []pathAndMessage{ - { - path: "foo/a.txt", - message: "feat(foo): initial commit for foo", - }, - { - path: "bar/a.txt", - message: "feat(bar): initial commit for bar", - }, - { - path: "foo/b.txt", - message: "fix(foo): a fix for foo", - }, - { - path: "foo/README.md", - message: "docs(foo): update README", - }, - { - path: "foo/c.txt", - message: "feat(foo): another feature for foo", - }, - { - path: "foo/something.txt", - message: `BEGIN_COMMIT - -BEGIN_NESTED_COMMIT -fix: a bug1 fix - -This is another body. - -PiperOrigin-RevId: 573342 -Library-IDs: foo -Source-link: [googleapis/googleapis@fedcba09](https://github.com/googleapis/googleapis/commit/fedcba09) -END_NESTED_COMMIT - -BEGIN_NESTED_COMMIT -fix: a bug2 fix - -This is another body. - -PiperOrigin-RevId: 573342 -Library-IDs: bar -Source-link: [googleapis/googleapis@fedcba09](https://github.com/googleapis/googleapis/commit/fedcba09) -END_NESTED_COMMIT - -BEGIN_NESTED_COMMIT -fix: a bug3 fix - -This is another body. - -PiperOrigin-RevId: 573342 -Library-IDs: foo, bar -Source-link: [googleapis/googleapis@fedcba09](https://github.com/googleapis/googleapis/commit/fedcba09) -END_NESTED_COMMIT - -END_COMMIT`, - }, - } - - repoWithCommits := setupRepoForGetCommits(t, pathAndMessages, []string{"foo-v1.0.0"}) - - for _, test := range []struct { - name string - repo legacygitrepo.Repository - library *legacyconfig.LibraryState - want []*legacygitrepo.ConventionalCommit - wantErr bool - wantErrPhrase string - }{ - { - name: "found_matching_commits_for_foo", - repo: repoWithCommits, - library: &legacyconfig.LibraryState{ - ID: "foo", - Version: "1.0.0", - TagFormat: "{id}-v{version}", - SourceRoots: []string{"foo"}, - ReleaseExcludePaths: []string{"foo/README.md"}, - }, - want: []*legacygitrepo.ConventionalCommit{ - { - Type: "fix", - Subject: "a bug1 fix", - LibraryID: "foo", - Footers: map[string]string{ - "PiperOrigin-RevId": "573342", - "Library-IDs": "foo", - "Source-link": "[googleapis/googleapis@fedcba09](https://github.com/googleapis/googleapis/commit/fedcba09)", - }, - IsNested: true, - }, - { - Type: "fix", - Subject: "a bug3 fix", - LibraryID: "foo", - Footers: map[string]string{ - "PiperOrigin-RevId": "573342", - "Library-IDs": "foo, bar", - "Source-link": "[googleapis/googleapis@fedcba09](https://github.com/googleapis/googleapis/commit/fedcba09)", - }, - IsNested: true, - }, - { - Type: "feat", - Subject: "another feature for foo", - LibraryID: "foo", - Footers: make(map[string]string), - }, - { - Type: "fix", - Subject: "a fix for foo", - LibraryID: "foo", - Footers: make(map[string]string), - }, - }, - }, - { - name: "no_matching_commits_for_foo", - repo: repoWithCommits, - library: &legacyconfig.LibraryState{ - ID: "foo", - Version: "1.0.0", - TagFormat: "{id}-v{version}", - SourceRoots: []string{"no_matching_dir"}, - }, - }, - { - name: "apiPaths_has_no_impact_on_release", - repo: repoWithCommits, - library: &legacyconfig.LibraryState{ - ID: "foo", - Version: "1.0.0", - TagFormat: "{id}-v{version}", - SourceRoots: []string{"no_matching_dir"}, // For release, only this is considered - APIs: []*legacyconfig.API{ - { - Path: "foo", - }, - { - Path: "bar", - }, - }, - }, - }, - { - name: "GetCommitsForPathsSinceTag error", - repo: &MockRepository{ - GetCommitsForPathsSinceTagError: fmt.Errorf("mock error from GetCommitsForPathsSinceTagError"), - }, - library: &legacyconfig.LibraryState{ID: "foo"}, - wantErr: true, - wantErrPhrase: "mock error from GetCommitsForPathsSinceTagError", - }, - { - name: "ChangedFilesInCommit error", - repo: &MockRepository{ - GetCommitsForPathsSinceTagValue: []*legacygitrepo.Commit{ - {Message: "feat(foo): a feature"}, - }, - ChangedFilesInCommitError: fmt.Errorf("mock error from ChangedFilesInCommit"), - }, - library: &legacyconfig.LibraryState{ID: "foo"}, - wantErr: true, - wantErrPhrase: "mock error from ChangedFilesInCommit", - }, - { - name: "ParseCommit error", - repo: &MockRepository{ - GetCommitsForPathsSinceTagValue: []*legacygitrepo.Commit{ - {Message: ""}, - }, - ChangedFilesInCommitValue: []string{"foo/a.txt"}, - }, - library: &legacyconfig.LibraryState{ID: "foo", SourceRoots: []string{"foo"}}, - wantErr: true, - wantErrPhrase: "failed to parse commit", - }, - } { - t.Run(test.name, func(t *testing.T) { - got, err := getConventionalCommitsSinceLastRelease(test.repo, test.library, "") - if test.wantErr { - if err == nil { - t.Fatal("getConventionalCommitsSinceLastRelease() should have failed") - } - if !strings.Contains(err.Error(), test.wantErrPhrase) { - t.Errorf("getConventionalCommitsSinceLastRelease() returned error %q, want to contain %q", err.Error(), test.wantErrPhrase) - } - return - } - if err != nil { - t.Fatalf("getConventionalCommitsSinceLastRelease() failed: %v", err) - } - if diff := cmp.Diff(test.want, got, cmpopts.IgnoreFields(legacygitrepo.ConventionalCommit{}, "CommitHash", "Body", "IsBreaking", "When")); diff != "" { - t.Errorf("getConventionalCommitsSinceLastRelease() mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestGetConventionalCommitsSinceLastGeneration(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - sourceRepo legacygitrepo.Repository - languageRepo legacygitrepo.Repository - library *legacyconfig.LibraryState - want []*legacygitrepo.ConventionalCommit - wantErr bool - wantErrMessage string - }{ - { - name: "found_matching_file_changes_for_foo", - library: &legacyconfig.LibraryState{ - ID: "foo", - SourceRoots: []string{"foo"}, - APIs: []*legacyconfig.API{ - { - Path: "foo", - }, - }, - }, - sourceRepo: &MockRepository{ - GetCommitsForPathsSinceLastGenByCommit: map[string][]*legacygitrepo.Commit{ - "1234": { - {Message: "feat(foo): a feature"}, - }, - }, - ChangedFilesInCommitValue: []string{"foo/a.proto"}, - }, - languageRepo: &MockRepository{ - IsCleanValue: true, - HeadHashValue: "5678", - ChangedFilesInCommitValue: []string{"foo/a.go"}, - }, - want: []*legacygitrepo.ConventionalCommit{ - { - Type: "feat", - Subject: "a feature", - LibraryID: "foo", - Footers: map[string]string{}, - }, - }, - }, - { - name: "found_matching_file_changes_for_foo_with_unclean_repo", - library: &legacyconfig.LibraryState{ - ID: "foo", - SourceRoots: []string{"foo"}, - APIs: []*legacyconfig.API{ - { - Path: "foo", - }, - }, - }, - sourceRepo: &MockRepository{ - GetCommitsForPathsSinceLastGenByCommit: map[string][]*legacygitrepo.Commit{ - "1234": { - {Message: "feat(foo): a feature"}, - }, - }, - ChangedFilesInCommitValue: []string{"foo/a.proto"}, - }, - languageRepo: &MockRepository{ - IsCleanValue: false, - ChangedFilesValue: []string{"foo/a.go"}, - }, - want: []*legacygitrepo.ConventionalCommit{ - { - Type: "feat", - Subject: "a feature", - LibraryID: "foo", - Footers: map[string]string{}, - }, - }, - }, - { - name: "no_matching_file_changes_for_foo", - library: &legacyconfig.LibraryState{ - ID: "foo", - SourceRoots: []string{"foo"}, - APIs: []*legacyconfig.API{ - { - Path: "foo", - }, - }, - }, - sourceRepo: &MockRepository{ - GetCommitsForPathsSinceLastGenByCommit: map[string][]*legacygitrepo.Commit{ - "1234": { - {Message: "feat(baz): a feature"}, - }, - }, - ChangedFilesInCommitValue: []string{"baz/a.proto", "baz/b.proto", "bar/a.proto"}, // file changed is not in foo/* - }, - languageRepo: &MockRepository{ - HeadHashValue: "5678", - ChangedFilesInCommitValue: []string{"foo/a.go"}, - }, - }, - { - name: "sources_root_has_no_impact", - library: &legacyconfig.LibraryState{ - ID: "foo", - APIs: []*legacyconfig.API{ - { - Path: "foo", // For generation, only this is considered - }, - }, - SourceRoots: []string{ - "baz/", - "bar/", - }, - }, - sourceRepo: &MockRepository{ - GetCommitsForPathsSinceLastGenByCommit: map[string][]*legacygitrepo.Commit{ - "1234": { - {Message: "feat(baz): a feature"}, - }, - }, - ChangedFilesInCommitValue: []string{"baz/a.proto", "baz/b.proto", "bar/a.proto"}, // file changed is not in foo/* - }, - languageRepo: &MockRepository{ - HeadHashValue: "5678", - ChangedFilesInCommitValue: []string{"foo/a.go"}, - }, - }, - } { - t.Run(test.name, func(t *testing.T) { - got, err := getConventionalCommitsSinceLastGeneration(test.sourceRepo, test.library, "1234") - if test.wantErr { - if err == nil { - t.Fatal("getConventionalCommitsSinceLastGeneration() should have failed") - } - if !strings.Contains(err.Error(), test.wantErrMessage) { - t.Errorf("getConventionalCommitsSinceLastRelease() returned error %q, want to contain %q", err.Error(), test.wantErrMessage) - } - return - } - if err != nil { - t.Fatalf("getConventionalCommitsSinceLastRelease() failed: %v", err) - } - if diff := cmp.Diff(test.want, got, cmpopts.IgnoreFields(legacygitrepo.ConventionalCommit{}, "CommitHash", "Body", "IsBreaking", "When")); diff != "" { - t.Errorf("getConventionalCommitsSinceLastRelease() mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestGetHighestChange(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - commits []*legacygitrepo.ConventionalCommit - expectedChange semver.ChangeLevel - }{ - { - name: "major change", - commits: []*legacygitrepo.ConventionalCommit{ - {Type: "feat", IsBreaking: true}, - {Type: "feat"}, - {Type: "fix"}, - }, - expectedChange: semver.Major, - }, - { - name: "minor change", - commits: []*legacygitrepo.ConventionalCommit{ - {Type: "feat"}, - {Type: "fix"}, - }, - expectedChange: semver.Minor, - }, - { - name: "patch change", - commits: []*legacygitrepo.ConventionalCommit{ - {Type: "fix"}, - }, - expectedChange: semver.Patch, - }, - { - name: "no change", - commits: []*legacygitrepo.ConventionalCommit{ - {Type: "docs"}, - {Type: "chore"}, - }, - expectedChange: semver.None, - }, - { - name: "no commits", - commits: []*legacygitrepo.ConventionalCommit{}, - expectedChange: semver.None, - }, - { - name: "nested commit forces minor bump", - commits: []*legacygitrepo.ConventionalCommit{ - {Type: "fix"}, - {Type: "feat", IsNested: true}, - }, - expectedChange: semver.Minor, - }, - { - name: "nested commit with breaking change forces minor bump", - commits: []*legacygitrepo.ConventionalCommit{ - {Type: "feat", IsBreaking: true, IsNested: true}, - {Type: "feat"}, - }, - expectedChange: semver.Minor, - }, - { - name: "major change and nested commit", - commits: []*legacygitrepo.ConventionalCommit{ - {Type: "feat", IsBreaking: true}, - {Type: "fix", IsNested: true}, - }, - expectedChange: semver.Major, - }, - { - name: "nested commit before major change", - commits: []*legacygitrepo.ConventionalCommit{ - {Type: "fix", IsNested: true}, - {Type: "feat", IsBreaking: true}, - }, - expectedChange: semver.Major, - }, - { - name: "nested commit with only fixes forces minor bump", - commits: []*legacygitrepo.ConventionalCommit{ - {Type: "fix"}, - {Type: "fix", IsNested: true}, - }, - expectedChange: semver.Minor, - }, - } { - t.Run(test.name, func(t *testing.T) { - highestChange := getHighestChange(test.commits) - if diff := cmp.Diff(test.expectedChange, highestChange); diff != "" { - t.Errorf("mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestNextVersion(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - commits []*legacygitrepo.ConventionalCommit - currentVersion string - releaseOnlyMode bool - wantVersion string - wantErr bool - }{ - { - name: "without override version", - commits: []*legacygitrepo.ConventionalCommit{ - {Type: "feat"}, - }, - currentVersion: "1.0.0", - wantVersion: "1.1.0", - }, - { - name: "derive next returns error", - commits: []*legacygitrepo.ConventionalCommit{ - {Type: "feat"}, - }, - currentVersion: "invalid-version", - wantVersion: "", - wantErr: true, - }, - { - name: "breaking change on nested commit results in minor bump", - commits: []*legacygitrepo.ConventionalCommit{ - {Type: "feat", IsBreaking: true, IsNested: true}, - }, - currentVersion: "1.2.3", - wantVersion: "1.3.0", - }, - { - name: "major change before nested commit results in major bump", - commits: []*legacygitrepo.ConventionalCommit{ - {Type: "feat", IsBreaking: true}, - {Type: "fix", IsNested: true}, - }, - currentVersion: "1.2.3", - wantVersion: "2.0.0", - }, - { - name: "a chore commit keeps current version in release only mode", - commits: []*legacygitrepo.ConventionalCommit{ - {Type: "chore"}, - }, - currentVersion: "1.2.3", - releaseOnlyMode: true, - wantVersion: "1.2.3", - }, - { - name: "a feat commit causes a minor bump in release only mode", - commits: []*legacygitrepo.ConventionalCommit{ - {Type: "feat"}, - }, - currentVersion: "1.2.3", - releaseOnlyMode: true, - wantVersion: "1.3.0", - }, - { - name: "a fix commit causes a minor bump in release only mode", - commits: []*legacygitrepo.ConventionalCommit{ - {Type: "fix"}, - }, - currentVersion: "1.2.3", - releaseOnlyMode: true, - wantVersion: "1.3.0", - }, - } { - t.Run(test.name, func(t *testing.T) { - gotVersion, err := NextVersion(test.commits, test.currentVersion, test.releaseOnlyMode) - if (err != nil) != test.wantErr { - t.Errorf("NextVersion() error = %v, wantErr %v", err, test.wantErr) - return - } - if gotVersion != test.wantVersion { - t.Errorf("NextVersion() = %v, want %v", gotVersion, test.wantVersion) - } - }) - } -} - -func TestLibraryFilter(t *testing.T) { - t.Parallel() - commits := []*legacygitrepo.ConventionalCommit{ - { - LibraryID: "foo", - Footers: map[string]string{}, - }, - { - LibraryID: "bar", - Footers: map[string]string{}, - }, - { - Footers: map[string]string{ - "Library-IDs": "foo", - }, - }, - { - Footers: map[string]string{ - "Library-IDs": "bar", - }, - }, - { - Footers: map[string]string{ - "Library-IDs": "foo, bar", - }, - }, - { - Footers: map[string]string{ - "Library-IDs": "foo,bar", - }, - }, - } - for _, test := range []struct { - name string - libraryID string - want []*legacygitrepo.ConventionalCommit - }{ - { - name: "filter by foo", - libraryID: "foo", - want: []*legacygitrepo.ConventionalCommit{ - commits[0], - commits[2], - commits[4], - commits[5], - }, - }, - { - name: "filter by bar", - libraryID: "bar", - want: []*legacygitrepo.ConventionalCommit{ - commits[1], - commits[3], - commits[4], - commits[5], - }, - }, - { - name: "filter by baz", - libraryID: "baz", - want: nil, - }, - } { - t.Run(test.name, func(t *testing.T) { - got := libraryFilter(commits, test.libraryID) - if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("libraryFilter() mismatch (-want +got):\n%s", diff) - } - }) - } -} diff --git a/internal/legacylibrarian/legacylibrarian/flags.go b/internal/legacylibrarian/legacylibrarian/flags.go deleted file mode 100644 index 61d9eb2d94f..00000000000 --- a/internal/legacylibrarian/legacylibrarian/flags.go +++ /dev/null @@ -1,158 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacylibrarian - -import ( - "flag" - "fmt" - - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" -) - -func addFlagAPI(fs *flag.FlagSet, cfg *legacyconfig.Config) { - fs.StringVar(&cfg.API, "api", "", - `Relative path to the API to be configured/generated (e.g., google/cloud/functions/v2). -Must be specified when generating a new library.`) -} - -func addFlagAPISource(fs *flag.FlagSet, cfg *legacyconfig.Config) { - fs.StringVar(&cfg.APISource, "api-source", "https://github.com/googleapis/googleapis", - `The location of an API specification repository. -Can be a remote URL or a local file path.`) -} - -func addFlagAPISourceBranch(fs *flag.FlagSet, cfg *legacyconfig.Config) { - fs.StringVar(&cfg.APISourceBranch, "api-source-branch", "master", - `The target branch of the API specification repository to checkout. -Can only be used with a remote -api-source.`) -} - -func addFlagBuild(fs *flag.FlagSet, cfg *legacyconfig.Config) { - fs.BoolVar(&cfg.Build, "build", false, - `If true, Librarian will build each generated library by invoking the -language-specific container.`) -} - -func addFlagBranch(fs *flag.FlagSet, cfg *legacyconfig.Config) { - fs.StringVar(&cfg.Branch, "branch", "main", - `The branch to use with remote code repositories. It is ignored if -you are using a local repository. This is used to specify which branch to clone -and which branch to use as the base for a pull request.`) -} - -func addFlagCheckUnexpectedChanges(fs *flag.FlagSet, cfg *legacyconfig.Config) { - fs.BoolVar(&cfg.CheckUnexpectedChanges, "check-unexpected-changes", false, - `Defaults to false. When used with --test, this flag verifies that no -unexpected files are added, deleted, or modified outside of the changes caused -by proto updates. You may want to skip this check when testing a container image -change that is expected to add or delete files.`) -} - -func addFlagCommit(fs *flag.FlagSet, cfg *legacyconfig.Config) { - fs.BoolVar(&cfg.Commit, "commit", false, - `If true, librarian will create a commit for the change but not create -a pull request. This flag is ignored if push is set to true.`) -} - -func addFlagGenerateUnchanged(fs *flag.FlagSet, cfg *legacyconfig.Config) { - fs.BoolVar(&cfg.GenerateUnchanged, "generate-unchanged", false, - `If true, librarian generates libraries even if none of their associated APIs -have changed. This does not override generation being blocked by configuration.`) -} - -func addFlagGitHubAPIEndpoint(fs *flag.FlagSet, cfg *legacyconfig.Config) { - fs.StringVar(&cfg.GitHubAPIEndpoint, "github-api-endpoint", "", - `The GitHub API endpoint to use for all GitHub API operations. -This is intended for testing and should not be used in production.`) -} - -func addFlagHostMount(fs *flag.FlagSet, cfg *legacyconfig.Config) { - defaultValue := "" - fs.StringVar(&cfg.HostMount, "host-mount", defaultValue, - `For use when librarian is running in a container. A mapping of a -directory from the host to the container, in the format -:.`) -} - -func addFlagImage(fs *flag.FlagSet, cfg *legacyconfig.Config) { - fs.StringVar(&cfg.Image, "image", "", - `Language specific image used to invoke code generation and releasing. -If not specified, the image configured in the state.yaml is used.`) -} - -func addFlagLibrary(fs *flag.FlagSet, cfg *legacyconfig.Config) { - fs.StringVar(&cfg.Library, "library", "", - `The library ID to generate or release (e.g. secretmanager). -This corresponds to a releasable language unit.`) -} - -func addFlagLibraryToTest(fs *flag.FlagSet, cfg *legacyconfig.Config) { - fs.StringVar(&cfg.LibraryToTest, "library-to-test", "", - `When used with --test, this flag specifies the library ID to test -(e.g. secretmanager). Will test on all configured libraries if omitted.`) -} - -func addFlagLibraryVersion(fs *flag.FlagSet, cfg *legacyconfig.Config) { - fs.StringVar(&cfg.LibraryVersion, "library-version", "", - `Overrides the automatic semantic version calculation and forces a specific -version for a library. Requires the --library flag to be specified.`) -} - -func addFlagPR(fs *flag.FlagSet, cfg *legacyconfig.Config) { - fs.StringVar(&cfg.PullRequest, "pr", "", - `The URL of a pull request to operate on. -It should be in the format of https://github.com/{owner}/{repo}/pull/{number}. -If not specified, will search for all merged pull requests with the label -"release:pending" in the last 30 days.`) -} - -func addFlagPush(fs *flag.FlagSet, cfg *legacyconfig.Config) { - fs.BoolVar(&cfg.Push, "push", false, - fmt.Sprintf(`If true, Librarian will create a commit, -push and create a pull request for the changes. -A GitHub token with push access must be provided via the -%s environment variable.`, legacyconfig.LibrarianGithubToken)) -} - -func addFlagRepo(fs *flag.FlagSet, cfg *legacyconfig.Config) { - fs.StringVar(&cfg.Repo, "repo", "", - `Code repository where the generated code will reside. Can be a remote -in the format of a remote URL such as https://github.com/{owner}/{repo} or a -local file path like /path/to/repo. Both absolute and relative paths are -supported. If not specified, will try to detect if the current working directory -is configured as a language repository. -Note: When using a local repository (either by providing a path or by defaulting -to the current directory), Librarian creates a new branch from the currently checked-out -branch and commits changes. If the --push flag is also specified, a pull request is -created against the main branch. The --branch flag is ignored for local repositories.`) -} - -func addFlagTest(fs *flag.FlagSet, cfg *legacyconfig.Config) { - fs.BoolVar(&cfg.Test, "test", false, - `If true, run container tests after generation but before committing and pushing. -These tests verify the interaction between language containers and the Librarian CLI's -'generate' command. If a test fails, temporary branches and files will be preserved for -debugging. This flag can be used with 'library-to-test' and 'check-unexpected-changes'.`) -} - -func addFlagWorkRoot(fs *flag.FlagSet, cfg *legacyconfig.Config) { - fs.StringVar(&cfg.WorkRoot, "output", "", - `Working directory root. When this is not specified, a working directory -will be created in /tmp.`) -} - -func addFlagVerbose(fs *flag.FlagSet, p *bool) { - fs.BoolVar(p, "v", false, "enables verbose logging") -} diff --git a/internal/legacylibrarian/legacylibrarian/gapic_metadata.go b/internal/legacylibrarian/legacylibrarian/gapic_metadata.go deleted file mode 100644 index 3d915c95ab9..00000000000 --- a/internal/legacylibrarian/legacylibrarian/gapic_metadata.go +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacylibrarian - -import ( - "bytes" - "fmt" - "html/template" - "io/fs" - "os" - "path/filepath" - "regexp" - "slices" - "strings" - - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" - gapic "google.golang.org/genproto/googleapis/gapic/metadata" - "google.golang.org/protobuf/encoding/protojson" -) - -const ( - gapicMetadataFile = "gapic_metadata.json" - serviceVersionOptimizationThreshold = 5 -) - -var ( - apiVersionReleaseNotesTemplate = template.Must(template.New("apiVersionReleaseNotes").Parse(`### API Versions -{{- range .LibraryPackageAPIVersions }} - -
{{.LibraryPackage}} -{{ range .ServiceVersions }} -* {{ .Service }}: {{ .Version }}{{ end }} - -
-{{ end }}`)) - - // Common places that may contain a gapic_metadata.json file as test data - // that should be ignored when preparing a release e.g. for client - // generators. - testdataRegex = regexp.MustCompile(`(?i)/(baselines|goldens|java-showcase|output|prototests|test|tests|testdata)/`) -) - -// serviceVersion represents a pairing of an API service interface e.g. Protobuf -// service and API service interface version. -type serviceVersion struct { - // Service is the name of the API service interface. This appears as the - // key in [gapic.GapicMetadata] Services map. For example, "LibraryService". - Service string - - // Version is the API service interface version identifier. This is sourced - // from the [gapic.GapicMetadata_ServiceForTransport] ApiVersion property. - // For example, "2025-09-14". - Version string -} - -// libraryPackageAPIVersions contains the API service interface version -// information for a generated library package. For example, the DotNet library -// package "Google.Example.Deltav.V1" has Services "SpaceService" and -// "SpaceShipService" with ApiVersion values therein of "2025-09-14" and -// "2025-04-04" respectively. -// -// There can be zero or more libraryPackageAPIVersions per -// [legacyconfig.LibraryState]. They constructed by [extractAPIVersions] following a -// [readGapicMetadata] call. -type libraryPackageAPIVersions struct { - // LibraryPackage is the language-specific, generated library package name - // as identified by the [gapic.GapicMetadata] LibraryPackage property. For - // example, a DotNet library package could be "Google.Example.Deltav.V1". - LibraryPackage string - - // ServiceVersions is the pairing of API service interface name to API - // version identifier found in the [gapic.GapicMetadata] Services mapping. - ServiceVersions []serviceVersion - - // Versions is the set of unique API version identifiers that appear in the - // [gapic.GapicMetadata] Services mapping. - Versions map[string]bool -} - -// formatAPIVersionReleaseNotes accepts the library's per-package API version -// information, and produces a change log/release note ready markdown -// "API Versions" section. It does leverage HTML for some formatting, so its -// output should be coerced to a [html/template.HTML] before being used in -// another [html/template.Template] as input data. -func formatAPIVersionReleaseNotes(lpv []*libraryPackageAPIVersions) (string, error) { - if len(lpv) == 0 { - return "", nil - } - - // Optimization for homogeneous API version used across service interfaces. - // Only triggers if there are more than 5 service interfaces in the API. - // If there are fewer, there is still value in listing them individually. - for _, v := range lpv { - if len(v.Versions) != 1 { - continue - } - if len(v.ServiceVersions) < serviceVersionOptimizationThreshold { - continue - } - - v.ServiceVersions = []serviceVersion{{Service: "All", Version: v.ServiceVersions[0].Version}} - } - - var out bytes.Buffer - if err := apiVersionReleaseNotesTemplate.Execute(&out, struct { - LibraryPackageAPIVersions []*libraryPackageAPIVersions - }{lpv}); err != nil { - return "", fmt.Errorf("error executing template: %w", err) - } - - return out.String(), nil -} - -// extractAPIVersions constructs a set of per-library package entities from the -// given [gapic.GapicMetadata] documents loaded from [readGapicMetadata]. If the -// document contains an API version identifier associated with a Services entry -// therein, it will get a [libraryPackageAPIVersions] item. -func extractAPIVersions(metadataDocuments map[string]*gapic.GapicMetadata) []*libraryPackageAPIVersions { - var result []*libraryPackageAPIVersions - for _, md := range metadataDocuments { - lpav := &libraryPackageAPIVersions{ - LibraryPackage: md.GetLibraryPackage(), - ServiceVersions: []serviceVersion{}, - Versions: make(map[string]bool), - } - for serviceName, service := range md.GetServices() { - if service.GetApiVersion() == "" { - continue - } - lpav.ServiceVersions = append(lpav.ServiceVersions, serviceVersion{Service: serviceName, Version: service.GetApiVersion()}) - lpav.Versions[service.GetApiVersion()] = true - } - if len(lpav.Versions) == 0 { - continue - } - slices.SortStableFunc(lpav.ServiceVersions, func(a, b serviceVersion) int { - return strings.Compare(a.Service, b.Service) - }) - result = append(result, lpav) - } - - slices.SortStableFunc(result, func(a, b *libraryPackageAPIVersions) int { - return strings.Compare(a.LibraryPackage, b.LibraryPackage) - }) - - return result -} - -// readGapicMetadata traverses the [legacyconfig.LibraryState] SourceRoots under the -// provided directory, parses any "gapic_metadata.json" file it finds, and -// stores it in a map keyed by the [gapic.GapicMetadata] LibraryPackage. -// There should be at most one "gapic_metadata.json" file per generated library -// package under SourceRoots, typically one per APIs entry, each with a unique -// library package name. -func readGapicMetadata(dir string, library *legacyconfig.LibraryState) (map[string]*gapic.GapicMetadata, error) { - mds := make(map[string]*gapic.GapicMetadata) - for _, root := range library.SourceRoots { - sr := filepath.Join(dir, root) - err := filepath.WalkDir(sr, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - if d.IsDir() { - return nil - } - if testdataRegex.MatchString(path) || filepath.Base(path) != gapicMetadataFile { - return nil - } - - content, err := os.ReadFile(path) - if err != nil { - return fmt.Errorf("failed to read %s: %w", path, err) - } - var metadata gapic.GapicMetadata - if err := protojson.Unmarshal(content, &metadata); err != nil { - return fmt.Errorf("failed to unmarshal %s: %w", path, err) - } - mds[metadata.LibraryPackage] = &metadata - - return nil - }) - if err != nil { - return nil, fmt.Errorf("error walking directory %s: %w", root, err) - } - } - return mds, nil -} diff --git a/internal/legacylibrarian/legacylibrarian/gapic_metadata_test.go b/internal/legacylibrarian/legacylibrarian/gapic_metadata_test.go deleted file mode 100644 index 73b38012b44..00000000000 --- a/internal/legacylibrarian/legacylibrarian/gapic_metadata_test.go +++ /dev/null @@ -1,451 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacylibrarian - -import ( - "os" - "path/filepath" - "testing" - - "github.com/google/go-cmp/cmp" - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" - gapic "google.golang.org/genproto/googleapis/gapic/metadata" - "google.golang.org/protobuf/encoding/protojson" - "google.golang.org/protobuf/testing/protocmp" -) - -func TestReadGapicMetadata(t *testing.T) { - libv1Metadata := &gapic.GapicMetadata{ - LibraryPackage: "cloud.google.com/go/library/apiv1", - Services: map[string]*gapic.GapicMetadata_ServiceForTransport{ - "Library": { - ApiVersion: "v1", - }, - }, - } - libv1JSON, err := protojson.Marshal(libv1Metadata) - if err != nil { - t.Fatalf("protojson.Marshal() failed: %v", err) - } - - libv2Metadata := &gapic.GapicMetadata{ - LibraryPackage: "cloud.google.com/go/library/apiv2", - Services: map[string]*gapic.GapicMetadata_ServiceForTransport{ - "AnotherLibraryService": { - ApiVersion: "v2", - }, - }, - } - libv2JSON, err := protojson.Marshal(libv2Metadata) - if err != nil { - t.Fatalf("protojson.Marshal() failed: %v", err) - } - - libTestMetadata := &gapic.GapicMetadata{ - LibraryPackage: "cloud.google.com/go/library/apiv2test", - Services: map[string]*gapic.GapicMetadata_ServiceForTransport{ - "AnotherLibraryService": { - ApiVersion: "v2test", - }, - }, - } - libTestJSON, err := protojson.Marshal(libTestMetadata) - if err != nil { - t.Fatalf("protojson.Marshal() failed: %v", err) - } - - for _, test := range []struct { - name string - files map[string][]byte - library *legacyconfig.LibraryState - want map[string]*gapic.GapicMetadata - }{ - { - name: "single metadata file", - files: map[string][]byte{ - "src/v1/gapic_metadata.json": libv1JSON, - }, - library: &legacyconfig.LibraryState{ - SourceRoots: []string{"src"}, - }, - want: map[string]*gapic.GapicMetadata{ - "cloud.google.com/go/library/apiv1": libv1Metadata, - }, - }, - { - name: "multiple metadata files", - files: map[string][]byte{ - "src/v1/gapic_metadata.json": libv1JSON, - "src/v2/gapic_metadata.json": libv2JSON, - }, - library: &legacyconfig.LibraryState{ - SourceRoots: []string{"src"}, - }, - want: map[string]*gapic.GapicMetadata{ - "cloud.google.com/go/library/apiv1": libv1Metadata, - "cloud.google.com/go/library/apiv2": libv2Metadata, - }, - }, - { - name: "multiple metadata files, ignore testdata", - files: map[string][]byte{ - "src/v1/gapic_metadata.json": libv1JSON, - "src/v2/gapic_metadata.json": libv2JSON, - "tests/v2/gapic_metadata.json": libTestJSON, - }, - library: &legacyconfig.LibraryState{ - SourceRoots: []string{"src"}, - }, - want: map[string]*gapic.GapicMetadata{ - "cloud.google.com/go/library/apiv1": libv1Metadata, - "cloud.google.com/go/library/apiv2": libv2Metadata, - }, - }, - { - name: "multiple source roots", - files: map[string][]byte{ - "src1/v1/gapic_metadata.json": libv1JSON, - "src2/v2/gapic_metadata.json": libv2JSON, - }, - library: &legacyconfig.LibraryState{ - SourceRoots: []string{"src1", "src2"}, - }, - want: map[string]*gapic.GapicMetadata{ - "cloud.google.com/go/library/apiv1": libv1Metadata, - "cloud.google.com/go/library/apiv2": libv2Metadata, - }, - }, - { - name: "no metadata files", - files: map[string][]byte{ - "src/v1/README.md": []byte("Hello, World!"), - }, - library: &legacyconfig.LibraryState{ - SourceRoots: []string{"src"}, - }, - want: map[string]*gapic.GapicMetadata{}, - }, - } { - t.Run(test.name, func(t *testing.T) { - tmpDir := t.TempDir() - for path, content := range test.files { - fullPath := filepath.Join(tmpDir, path) - if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil { - t.Fatalf("os.MkdirAll() failed: %v", err) - } - if err := os.WriteFile(fullPath, content, 0644); err != nil { - t.Fatalf("os.WriteFile() failed: %v", err) - } - } - - got, err := readGapicMetadata(tmpDir, test.library) - if err != nil { - t.Fatalf("readGapicMetadata() failed: %v", err) - } - if diff := cmp.Diff(test.want, got, protocmp.Transform()); diff != "" { - t.Errorf("mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestExtractAPIVersions(t *testing.T) { - for _, test := range []struct { - name string - in map[string]*gapic.GapicMetadata - want []*libraryPackageAPIVersions - }{ - { - name: "single service, single version", - in: map[string]*gapic.GapicMetadata{ - "cloud.google.com/go/library/apiv1": { - LibraryPackage: "cloud.google.com/go/library/apiv1", - Services: map[string]*gapic.GapicMetadata_ServiceForTransport{ - "Library": { - ApiVersion: "v1", - }, - }, - }, - }, - want: []*libraryPackageAPIVersions{ - { - LibraryPackage: "cloud.google.com/go/library/apiv1", - ServiceVersions: []serviceVersion{ - {Service: "Library", Version: "v1"}, - }, - Versions: map[string]bool{ - "v1": true, - }, - }, - }, - }, - { - name: "multiple services, same version", - in: map[string]*gapic.GapicMetadata{ - "cloud.google.com/go/library/apiv1": { - LibraryPackage: "cloud.google.com/go/library/apiv1", - Services: map[string]*gapic.GapicMetadata_ServiceForTransport{ - "Library": { - ApiVersion: "v1", - }, - "Management": { - ApiVersion: "v1", - }, - }, - }, - }, - want: []*libraryPackageAPIVersions{ - { - LibraryPackage: "cloud.google.com/go/library/apiv1", - ServiceVersions: []serviceVersion{ - {Service: "Library", Version: "v1"}, - {Service: "Management", Version: "v1"}, - }, - Versions: map[string]bool{ - "v1": true, - }, - }, - }, - }, - { - name: "multiple services, different versions", - in: map[string]*gapic.GapicMetadata{ - "cloud.google.com/go/library/apiv1": { - LibraryPackage: "cloud.google.com/go/library/apiv1", - Services: map[string]*gapic.GapicMetadata_ServiceForTransport{ - "ServiceA": { - ApiVersion: "v1", - }, - "ServiceB": { - ApiVersion: "v1beta1", - }, - }, - }, - }, - want: []*libraryPackageAPIVersions{ - { - LibraryPackage: "cloud.google.com/go/library/apiv1", - ServiceVersions: []serviceVersion{ - {Service: "ServiceA", Version: "v1"}, - {Service: "ServiceB", Version: "v1beta1"}, - }, - Versions: map[string]bool{ - "v1": true, - "v1beta1": true, - }, - }, - }, - }, - { - name: "multiple library packages", - in: map[string]*gapic.GapicMetadata{ - "cloud.google.com/go/library/apiv1": { - LibraryPackage: "cloud.google.com/go/library/apiv1", - Services: map[string]*gapic.GapicMetadata_ServiceForTransport{ - "Library": { - ApiVersion: "v1", - }, - }, - }, - "cloud.google.com/go/library/apiv2": { - LibraryPackage: "cloud.google.com/go/library/apiv2", - Services: map[string]*gapic.GapicMetadata_ServiceForTransport{ - "AnotherLibraryService": { - ApiVersion: "v2", - }, - }, - }, - }, - want: []*libraryPackageAPIVersions{ - { - LibraryPackage: "cloud.google.com/go/library/apiv1", - ServiceVersions: []serviceVersion{ - {Service: "Library", Version: "v1"}, - }, - Versions: map[string]bool{ - "v1": true, - }, - }, - { - LibraryPackage: "cloud.google.com/go/library/apiv2", - ServiceVersions: []serviceVersion{ - {Service: "AnotherLibraryService", Version: "v2"}, - }, - Versions: map[string]bool{ - "v2": true, - }, - }, - }, - }, - { - name: "empty input map", - in: map[string]*gapic.GapicMetadata{}, - want: nil, - }, - { - name: "nil input map", - in: nil, - want: nil, - }, - { - name: "empty services in metadata", - in: map[string]*gapic.GapicMetadata{ - "cloud.google.com/go/library/apiv1": { - LibraryPackage: "cloud.google.com/go/library/apiv1", - Services: map[string]*gapic.GapicMetadata_ServiceForTransport{}, - }, - }, - want: nil, - }, - { - name: "one service, no api_version", - in: map[string]*gapic.GapicMetadata{ - "cloud.google.com/go/library/apiv1": { - LibraryPackage: "cloud.google.com/go/library/apiv1", - Services: map[string]*gapic.GapicMetadata_ServiceForTransport{"Library": {}}, - }, - }, - want: nil, - }, - } { - t.Run(test.name, func(t *testing.T) { - got := extractAPIVersions(test.in) - if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestFormatAPIVersionReleaseNotes(t *testing.T) { - for _, test := range []struct { - name string - in []*libraryPackageAPIVersions - want string - }{ - { - name: "single library, multiple versions", - in: []*libraryPackageAPIVersions{ - { - LibraryPackage: "cloud.google.com/go/library/apiv1", - ServiceVersions: []serviceVersion{ - {Service: "BookService", Version: "2025-04-04"}, - {Service: "LibraryService", Version: "2025-09-14"}, - }, - Versions: map[string]bool{ - "2025-09-14": true, - "2025-04-04": true, - }, - }, - }, - want: `### API Versions - -
cloud.google.com/go/library/apiv1 - -* BookService: 2025-04-04 -* LibraryService: 2025-09-14 - -
-`, - }, - { - name: "multiple libraries, multiple versions", - in: []*libraryPackageAPIVersions{ - { - LibraryPackage: "cloud.google.com/go/library/apiv1", - ServiceVersions: []serviceVersion{ - {Service: "BookService", Version: "2025-04-04"}, - {Service: "LibraryService", Version: "2025-09-14"}, - }, - Versions: map[string]bool{ - "2025-09-14": true, - "2025-04-04": true, - }, - }, - { - LibraryPackage: "cloud.google.com/go/anotherlibrary/apiv1", - ServiceVersions: []serviceVersion{ - {Service: "AnotherLibraryService", Version: "2025-05-24"}, - }, - Versions: map[string]bool{ - "2025-05-24": true, - }, - }, - }, - want: `### API Versions - -
cloud.google.com/go/library/apiv1 - -* BookService: 2025-04-04 -* LibraryService: 2025-09-14 - -
- - -
cloud.google.com/go/anotherlibrary/apiv1 - -* AnotherLibraryService: 2025-05-24 - -
-`, - }, - { - name: "single library, many services, same version", - in: []*libraryPackageAPIVersions{ - { - LibraryPackage: "cloud.google.com/go/library/apiv1", - ServiceVersions: []serviceVersion{ - {Service: "BookService", Version: "2025-09-14"}, - {Service: "BookMobileService", Version: "2025-09-14"}, - {Service: "CounterService", Version: "2025-09-14"}, - {Service: "LibraryService", Version: "2025-09-14"}, - {Service: "ShelfService", Version: "2025-09-14"}, - }, - Versions: map[string]bool{ - "2025-09-14": true, - }, - }, - }, - want: `### API Versions - -
cloud.google.com/go/library/apiv1 - -* All: 2025-09-14 - -
-`, - }, - { - name: "empty input", - in: []*libraryPackageAPIVersions{}, - want: "", - }, - { - name: "nil input", - in: nil, - want: "", - }, - } { - t.Run(test.name, func(t *testing.T) { - got, err := formatAPIVersionReleaseNotes(test.in) - if err != nil { - t.Fatalf("formatAPIVersionReleaseNotes() failed: %v", err) - } - if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("mismatch (-want +got):\n%s", diff) - } - }) - } -} diff --git a/internal/legacylibrarian/legacylibrarian/generate.go b/internal/legacylibrarian/legacylibrarian/generate.go deleted file mode 100644 index 6794c2effd6..00000000000 --- a/internal/legacylibrarian/legacylibrarian/generate.go +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacylibrarian - -import ( - "context" - "fmt" - "log/slog" - "os" - "path/filepath" - "strings" - - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" - "github.com/googleapis/librarian/internal/legacylibrarian/legacydocker" - "github.com/googleapis/librarian/internal/legacylibrarian/legacygitrepo" -) - -func generateSingleLibrary(ctx context.Context, containerClient ContainerClient, state *legacyconfig.LibrarianState, libraryState *legacyconfig.LibraryState, repo legacygitrepo.Repository, sourceRepo legacygitrepo.Repository, image, outputDir string) error { - // For each library, create a separate output directory. This avoids - // libraries interfering with each other, and makes it easier to see what - // was generated for each library when debugging. - safeLibraryDirectory := getSafeDirectoryName(libraryState.ID) - libraryOutputDir := filepath.Join(outputDir, safeLibraryDirectory) - if err := os.MkdirAll(libraryOutputDir, 0755); err != nil { - return fmt.Errorf("error making output directory %w", err) - } - - apiRoot, err := filepath.Abs(sourceRepo.GetDir()) - if err != nil { - return err - } - - generateRequest := &legacydocker.GenerateRequest{ - ApiRoot: apiRoot, - LibraryID: libraryState.ID, - Output: libraryOutputDir, - RepoDir: repo.GetDir(), - State: state, - Image: image, - } - slog.Info("performing generation for library", "id", libraryState.ID, "outputDir", libraryOutputDir) - if err := containerClient.Generate(ctx, generateRequest); err != nil { - return err - } - - // Read the library state from the response. - if _, err := readLibraryState( - filepath.Join(generateRequest.RepoDir, legacyconfig.LibrarianDir, legacyconfig.GenerateResponse)); err != nil { - return err - } - - if err := cleanAndCopyLibrary(state, repo.GetDir(), libraryState.ID, libraryOutputDir); err != nil { - return err - } - - slog.Info("generation succeeds", "id", libraryState.ID) - return nil -} - -func restoreLibrary(libraryState *legacyconfig.LibraryState, repo legacygitrepo.Repository) error { - if err := repo.Restore(libraryState.SourceRoots); err != nil { - return err - } - return repo.CleanUntracked(libraryState.SourceRoots) -} - -// getSafeDirectoryName returns a directory name which doesn't contain slashes -// based on a library ID. This avoids cases where a library ID contains -// slashes, but we want generateSingleLibrary to create a directory which -// is not a subdirectory of some other directory. For example, if there -// are library IDs of "pubsub" and "pubsub/v2" we don't want to create -// "output/pubsub/v2" and then "output/pubsub" later. This function does -// not protect against malicious library IDs, e.g. ".", ".." or deliberate -// collisions (e.g. "pubsub/v2" and "pubsub-slash-v2"). -// -// The exact implementation may change over time - nothing should rely on this. -// The current implementation simply replaces any slashes with "-slash-". -func getSafeDirectoryName(libraryID string) string { - return strings.ReplaceAll(libraryID, "/", "-slash-") -} diff --git a/internal/legacylibrarian/legacylibrarian/generate_command.go b/internal/legacylibrarian/legacylibrarian/generate_command.go deleted file mode 100644 index 74d7336ee09..00000000000 --- a/internal/legacylibrarian/legacylibrarian/generate_command.go +++ /dev/null @@ -1,515 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacylibrarian - -import ( - "context" - "errors" - "fmt" - "log/slog" - "os" - "path/filepath" - - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" - "github.com/googleapis/librarian/internal/legacylibrarian/legacydocker" - "github.com/googleapis/librarian/internal/legacylibrarian/legacygitrepo" -) - -const ( - generateCmdName = "generate" -) - -var ( - errGenerateInReleaseOnlyMode = errors.New("generate in release only mode") -) - -type generateRunner struct { - api string - branch string - build bool - commit bool - generateUnchanged bool - containerClient ContainerClient - ghClient GitHubClient - hostMount string - image string - library string - push bool - repo legacygitrepo.Repository - sourceRepo legacygitrepo.Repository - state *legacyconfig.LibrarianState - librarianConfig *legacyconfig.LibrarianConfig - workRoot string -} - -// generationStatus represents the result of a single library generation. -type generationStatus struct { - // oldCommit is the SHA of the previously generated version of the library. - oldCommit string - prType pullRequestType -} - -func newGenerateRunner(cfg *legacyconfig.Config) (*generateRunner, error) { - runner, err := newCommandRunner(cfg) - if err != nil { - return nil, err - } - return &generateRunner{ - api: cfg.API, - branch: cfg.Branch, - build: cfg.Build, - commit: cfg.Commit, - containerClient: runner.containerClient, - generateUnchanged: cfg.GenerateUnchanged, - ghClient: runner.ghClient, - hostMount: cfg.HostMount, - image: runner.image, - library: cfg.Library, - push: cfg.Push, - repo: runner.repo, - sourceRepo: runner.sourceRepo, - state: runner.state, - librarianConfig: runner.librarianConfig, - workRoot: runner.workRoot, - }, nil -} - -// run executes the library generation process. -// -// It determines whether to generate a single library or all configured libraries based on the -// command-line flags. If an API or library is specified, it generates a single library. Otherwise, -// it iterates through all libraries defined in the state and generates them. -func (r *generateRunner) run(ctx context.Context) error { - if r.librarianConfig != nil && r.librarianConfig.ReleaseOnlyMode { - return errGenerateInReleaseOnlyMode - } - outputDir := filepath.Join(r.workRoot, "output") - if err := os.Mkdir(outputDir, 0755); err != nil { - return fmt.Errorf("failed to make output directory, %s: %w", outputDir, err) - } - // The last generated commit is changed after library generation, - // use this map to keep the mapping from library id to commit sha before the - // generation since we need these commits to create pull request body. - idToCommits := make(map[string]string) - var failedLibraries []string - prType := pullRequestGenerate - if r.api != "" || r.library != "" { - libraryID := r.library - if libraryID == "" { - libraryID = findLibraryIDByAPIPath(r.state, r.api) - } - status, err := r.generateSingleLibrary(ctx, libraryID, outputDir) - if err != nil { - return err - } - idToCommits[libraryID] = status.oldCommit - prType = status.prType - } else { - var succeededGenerations int - var skippedGenerations int - for _, library := range r.state.Libraries { - shouldGenerate, err := r.shouldGenerate(library) - if err != nil { - slog.Error("failed to determine whether or not to generate library", "id", library.ID, "err", err) - // While this isn't strictly a failed generation, it's a library for which - // the generate command failed, so it's close enough. - failedLibraries = append(failedLibraries, library.ID) - continue - } - if !shouldGenerate { - // We assume that the cause will have been logged in shouldGenerateLibrary. - skippedGenerations++ - continue - } - status, err := r.generateSingleLibrary(ctx, library.ID, outputDir) - if err != nil { - slog.Error("failed to generate library", "id", library.ID, "err", err) - failedLibraries = append(failedLibraries, library.ID) - } else { - // Only add the mapping if library generation is successful so that - // failed library will not appear in generation PR body. - idToCommits[library.ID] = status.oldCommit - succeededGenerations++ - } - } - - slog.Info( - "generation statistics", - "all", len(r.state.Libraries), - "successes", succeededGenerations, - "skipped", skippedGenerations, - "failures", len(failedLibraries)) - if len(failedLibraries) > 0 && len(failedLibraries)+skippedGenerations == len(r.state.Libraries) { - return fmt.Errorf("all %d libraries failed to generate (skipped: %d)", - len(failedLibraries), skippedGenerations) - } - } - - if err := saveLibrarianState(r.repo.GetDir(), r.state); err != nil { - return err - } - - var prBodyBuilder func() (string, error) - switch prType { - case pullRequestGenerate: - prBodyBuilder = func() (string, error) { - req := &generationPRRequest{ - sourceRepo: r.sourceRepo, - languageRepo: r.repo, - state: r.state, - idToCommits: idToCommits, - failedLibraries: failedLibraries, - } - return formatGenerationPRBody(req) - } - case pullRequestOnboard: - prBodyBuilder = func() (string, error) { - req := &onboardPRRequest{ - sourceRepo: r.sourceRepo, - state: r.state, - api: r.api, - library: r.library, - } - return formatOnboardPRBody(req) - } - default: - return fmt.Errorf("unexpected prType %s", prType) - } - - commitInfo := &commitInfo{ - branch: r.branch, - commit: r.commit, - commitMessage: "feat: generate libraries", - ghClient: r.ghClient, - prType: prType, - push: r.push, - languageRepo: r.repo, - sourceRepo: r.sourceRepo, - state: r.state, - workRoot: r.workRoot, - api: r.api, - library: r.library, - failedGenerations: len(failedLibraries), - prBodyBuilder: prBodyBuilder, - } - - if err := commitAndPush(ctx, commitInfo); err != nil { - return fmt.Errorf("failed to commit and push changes: %w", err) - } - return nil -} - -// generateSingleLibrary manages the generation of a single client library. -// -// The single library generation executes as follows: -// -// 1. Configure the library, if the library is not configured in the state.yaml. -// -// 2. Generate the library. -// -// 3. Build the library. -// -// 4. Update the last generated commit or initial piper id if the library needs configure. -func (r *generateRunner) generateSingleLibrary(ctx context.Context, libraryID, outputDir string) (*generationStatus, error) { - safeLibraryDirectory := getSafeDirectoryName(libraryID) - prType := pullRequestGenerate - if r.needsConfigure() { - slog.Info("library not configured, start initial configuration", "library", r.library) - configureOutputDir := filepath.Join(outputDir, safeLibraryDirectory, "configure") - if err := os.MkdirAll(configureOutputDir, 0755); err != nil { - return nil, err - } - configuredLibraryID, err := r.runConfigureCommand(ctx, configureOutputDir) - if err != nil { - return nil, err - } - - prType = pullRequestOnboard - libraryID = configuredLibraryID - } - - // At this point, we should have a library in the state. - libraryState := r.state.LibraryByID(libraryID) - if libraryState == nil { - return nil, fmt.Errorf("library %q not configured yet, generation stopped", libraryID) - } - lastGenCommit := libraryState.LastGeneratedCommit - - if len(libraryState.APIs) == 0 { - slog.Info("library has no APIs; skipping generation", "library", libraryID) - return &generationStatus{ - oldCommit: "", - prType: prType, - }, nil - } - - image := r.image - if image == "" { - image = r.state.Image - slog.Info("using image from state", "image", image) - } else { - slog.Info("using image from command line", "image", image) - } - - if err := generateSingleLibrary(ctx, r.containerClient, r.state, libraryState, r.repo, r.sourceRepo, image, outputDir); err != nil { - return nil, err - } - - if r.build { - if err := buildSingleLibrary(ctx, r.containerClient, r.state, libraryState, r.repo); err != nil { - return nil, err - } - } - - if err := r.updateLastGeneratedCommitState(libraryID); err != nil { - return nil, err - } - - return &generationStatus{ - oldCommit: lastGenCommit, - prType: prType, - }, nil -} - -func (r *generateRunner) needsConfigure() bool { - if r.api == "" || r.library == "" { - return false - } - libraryState := r.state.LibraryByID(r.library) - if libraryState == nil { - return true - } - for _, api := range libraryState.APIs { - if api.Path == r.api { - return false - } - } - return true -} - -func (r *generateRunner) updateLastGeneratedCommitState(libraryID string) error { - hash, err := r.sourceRepo.HeadHash() - if err != nil { - return err - } - for _, l := range r.state.Libraries { - if l.ID == libraryID { - l.LastGeneratedCommit = hash - break - } - } - return nil -} - -// runConfigureCommand executes the container's "configure" command for an API. -// -// This function performs the following steps: -// -// 1. Constructs a request for the language-specific container, including the API -// root, library ID, and repository directory. -// -// 2. Populates a service configuration if one is missing. -// -// 3. Delegates the configuration task to the container's `Configure` command. -// -// 4. Reads the updated library state from the `configure-response.json` file -// generated by the container. -// -// 5. Updates the in-memory librarian state with the new configuration. -// -// 6. Writes the complete, updated librarian state back to the `state.yaml` file -// in the repository. -// -// If successful, it returns the ID of the newly configured library; otherwise, -// it returns an empty string and an error. -func (r *generateRunner) runConfigureCommand(ctx context.Context, outputDir string) (string, error) { - - apiRoot, err := filepath.Abs(r.sourceRepo.GetDir()) - if err != nil { - return "", err - } - - setAllAPIStatus(r.state, legacyconfig.StatusExisting) - addAPIToLibrary(r.state, r.library, r.api) - - if err := populateServiceConfigIfEmpty( - r.state, - apiRoot); err != nil { - return "", err - } - - var globalFiles []string - if r.librarianConfig != nil { - globalFiles = r.librarianConfig.GetGlobalFiles() - } - - configureRequest := &legacydocker.ConfigureRequest{ - ApiRoot: apiRoot, - LibraryID: r.library, - Output: outputDir, - RepoDir: r.repo.GetDir(), - GlobalFiles: globalFiles, - ExistingSourceRoots: r.getExistingSrc(r.library), - State: r.state, - } - slog.Info("performing configuration for library", "id", r.library) - if _, err := r.containerClient.Configure(ctx, configureRequest); err != nil { - return "", err - } - - // Read the new library state from the response. - libraryState, err := readLibraryState( - filepath.Join(r.repo.GetDir(), legacyconfig.LibrarianDir, legacyconfig.ConfigureResponse), - ) - if err != nil { - return "", err - } - if libraryState == nil { - return "", errors.New("no response file for configure container command") - } - - if libraryState.Version == "" { - slog.Info("library doesn't receive a version, apply the default version", "id", r.library) - libraryState.Version = "0.0.0" - } - - // Update the library state in the librarian state. - for i, library := range r.state.Libraries { - if library.ID != libraryState.ID { - continue - } - r.state.Libraries[i] = libraryState - } - - if err := copyLibraryFiles(r.state, r.repo.GetDir(), libraryState.ID, outputDir, false); err != nil { - return "", err - } - - if err := copyGlobalAllowlist(r.librarianConfig, r.repo.GetDir(), outputDir, false); err != nil { - return "", err - } - - return libraryState.ID, nil -} - -// getExistingSrc returns source roots as-is of a given library ID, if the source roots exist in the language repo. -func (r *generateRunner) getExistingSrc(libraryID string) []string { - library := r.state.LibraryByID(libraryID) - if library == nil { - return nil - } - - var existingSrc []string - for _, src := range library.SourceRoots { - relPath := filepath.Join(r.repo.GetDir(), src) - if _, err := os.Stat(relPath); err == nil { - existingSrc = append(existingSrc, src) - } - } - - return existingSrc -} - -func setAllAPIStatus(state *legacyconfig.LibrarianState, status string) { - for _, library := range state.Libraries { - for _, api := range library.APIs { - api.Status = status - } - } -} - -// shouldGenerate determines whether a library should be generated by the generate -// command. It does *not* observe the -library or -api flag, as those are handled -// higher up in run. If this function returns false (with a nil error), it always -// logs why the library was skipped. -// -// The decision of whether or not a library should be generated is relatively complex, -// and should be kept centrally in this function, with a comment for each path in the flow -// for clarity. -func (r *generateRunner) shouldGenerate(library *legacyconfig.LibraryState) (bool, error) { - // If the library has a manual configuration which indicates generation is blocked, - // the library is skipped. - if r.librarianConfig.IsGenerationBlocked(library.ID) { - slog.Info("library has generate_blocked, skipping", "id", library.ID) - return false, nil - } - - // If the library has no APIs, it is skipped. - if len(library.APIs) == 0 { - slog.Info("library has no APIs, skipping", "id", library.ID) - return false, nil - } - - // If we've been asked to generate libraries even with unchanged APIs, - // we don't need to check whether any have changed: we should definitely generate. - if r.generateUnchanged { - return true, nil - } - - // If we don't know the last commit at which the library was generated, - // we can't tell whether or not it's changed, so we always generate. - if library.LastGeneratedCommit == "" { - return true, nil - } - - // Most common case: a non-generation-blocked library with APIs, and without the - // -generate-unchanged flag. Check each API to see whether anything under API.Path - // has changed between the last_generated_commit and the HEAD commit of r.sourceRepo. - // If any API has changed, the library is generated - otherwise it's skipped. - headHash, err := r.sourceRepo.HeadHash() - if err != nil { - return false, fmt.Errorf("failed to get head hash for source repo: %v", err) - } - for _, api := range library.APIs { - oldHash, err := r.sourceRepo.GetHashForPath(library.LastGeneratedCommit, api.Path) - if err != nil { - return false, fmt.Errorf("failed to get hash for path %v at commit %v: %v", api.Path, library.LastGeneratedCommit, err) - } - newHash, err := r.sourceRepo.GetHashForPath(headHash, api.Path) - if err != nil { - return false, fmt.Errorf("failed to get hash for path %v at commit %v: %v", api.Path, headHash, err) - } - if oldHash != newHash { - return true, nil - } - } - slog.Info("no APIs have changed; skipping", "library", library.ID) - return false, nil -} - -// addAPIToLibrary adds a new API to a library in the state. -// If the library does not exist, it creates a new one. -// If the API already exists in the library, do nothing. -func addAPIToLibrary(state *legacyconfig.LibrarianState, libraryID, apiPath string) { - lib := state.LibraryByID(libraryID) - if lib == nil { - // If the library is not found, create a new one. - state.Libraries = append(state.Libraries, &legacyconfig.LibraryState{ - ID: libraryID, - APIs: []*legacyconfig.API{{Path: apiPath, Status: legacyconfig.StatusNew}}, - }) - return - } - - // If the library is found, check if the API already exists. - for _, api := range lib.APIs { - if api.Path == apiPath { - return - } - } - - // For new API paths, set the status to "new". - lib.APIs = append(lib.APIs, &legacyconfig.API{Path: apiPath, Status: legacyconfig.StatusNew}) -} diff --git a/internal/legacylibrarian/legacylibrarian/generate_command_test.go b/internal/legacylibrarian/legacylibrarian/generate_command_test.go deleted file mode 100644 index b27d9456416..00000000000 --- a/internal/legacylibrarian/legacylibrarian/generate_command_test.go +++ /dev/null @@ -1,1705 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacylibrarian - -import ( - "errors" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/google/go-cmp/cmp" - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" - "github.com/googleapis/librarian/internal/legacylibrarian/legacygitrepo" -) - -func TestNewGenerateRunner(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - cfg *legacyconfig.Config - wantErr bool - wantErrMsg string - setupFunc func(*legacyconfig.Config) error - }{ - { - name: "valid config", - cfg: &legacyconfig.Config{ - API: "some/api", - APISource: newTestGitRepo(t).GetDir(), - Branch: "test-branch", - Repo: newTestGitRepo(t).GetDir(), - WorkRoot: t.TempDir(), - CommandName: generateCmdName, - }, - }, - { - name: "invalid api source", - cfg: &legacyconfig.Config{ - API: "some/api", - APISource: t.TempDir(), // Not a git repo - Repo: newTestGitRepo(t).GetDir(), - WorkRoot: t.TempDir(), - Image: "gcr.io/test/test-image", - CommandName: generateCmdName, - }, - wantErr: true, - wantErrMsg: "repository does not exist", - }, - { - name: "no state file", - cfg: &legacyconfig.Config{ - API: "some/api", - APISource: newTestGitRepo(t).GetDir(), - Branch: "test-branch", - Repo: newTestGitRepoWithState(t, nil).GetDir(), - WorkRoot: t.TempDir(), - CommandName: generateCmdName, - }, - wantErr: true, - wantErrMsg: ".librarian/state.yaml: no such file or directory", - }, - { - name: "valid config with github token", - cfg: &legacyconfig.Config{ - API: "some/api", - APISource: newTestGitRepo(t).GetDir(), - Branch: "test-branch", - Repo: newTestGitRepo(t).GetDir(), - WorkRoot: t.TempDir(), - Image: "gcr.io/test/test-image", - GitHubToken: "gh-token", - CommandName: generateCmdName, - }, - }, - { - name: "empty API source", - cfg: &legacyconfig.Config{ - API: "some/api", - APISource: "https://github.com/googleapis/googleapis", // This will trigger the clone of googleapis - APISourceBranch: "master", - APISourceDepth: 1, - Branch: "test-branch", - Repo: newTestGitRepo(t).GetDir(), - WorkRoot: t.TempDir(), - Image: "gcr.io/test/test-image", - CommandName: generateCmdName, - }, - }, - { - name: "clone googleapis fails", - cfg: &legacyconfig.Config{ - API: "some/api", - APISource: "https://github.com/googleapis/googleapis", // This will trigger the clone of googleapis - APISourceDepth: 1, - Repo: newTestGitRepo(t).GetDir(), - WorkRoot: t.TempDir(), - Image: "gcr.io/test/test-image", - CommandName: generateCmdName, - }, - wantErr: true, - wantErrMsg: "repository does not exist", - setupFunc: func(cfg *legacyconfig.Config) error { - // The function will try to clone googleapis into the current work directory. - // To make it fail, create a non-empty, non-git directory. - googleapisDir := filepath.Join(cfg.WorkRoot, "googleapis") - if err := os.MkdirAll(googleapisDir, 0755); err != nil { - return err - } - if err := os.WriteFile(filepath.Join(googleapisDir, "some-file"), []byte("foo"), 0644); err != nil { - return err - } - return nil - }, - }, - { - name: "valid config with local repo", - cfg: &legacyconfig.Config{ - API: "some/api", - APISource: newTestGitRepo(t).GetDir(), - Branch: "test-branch", - Repo: newTestGitRepo(t).GetDir(), - WorkRoot: t.TempDir(), - Image: "gcr.io/test/test-image", - CommandName: generateCmdName, - }, - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - - // custom setup - if test.setupFunc != nil { - if err := test.setupFunc(test.cfg); err != nil { - t.Fatalf("error in setup %v", err) - } - } - - r, err := newGenerateRunner(test.cfg) - if test.wantErr { - if err == nil { - t.Fatalf("newGenerateRunner() error = %v, wantErr %v", err, test.wantErr) - } - - if !strings.Contains(err.Error(), test.wantErrMsg) { - t.Fatalf("want error message: %s, got: %s", test.wantErrMsg, err.Error()) - } - - return - } - - if err != nil { - t.Fatalf("newGenerateRunner() got error: %v", err) - } - - if r.branch == "" { - t.Errorf("newGenerateRunner() branch is not set") - } - - if r.ghClient == nil { - t.Errorf("newGenerateRunner() ghClient is nil") - } - if r.containerClient == nil { - t.Errorf("newGenerateRunner() containerClient is nil") - } - if r.repo == nil { - t.Errorf("newGenerateRunner() repo is nil") - } - if r.sourceRepo == nil { - t.Errorf("newGenerateRunner() sourceRepo is nil") - } - }) - } -} - -func TestRunConfigureCommand(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - api string - repo legacygitrepo.Repository - state *legacyconfig.LibrarianState - librarianConfig *legacyconfig.LibrarianConfig - container *mockContainerClient - wantConfigureCalls int - wantErr bool - wantErrMsg string - }{ - { - name: "configures library successfully", - api: "some/api", - repo: newTestGitRepo(t), - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "some-library", - APIs: []*legacyconfig.API{{Path: "some/api"}}, - }, - }, - }, - container: &mockContainerClient{}, - wantConfigureCalls: 1, - }, - { - name: "configures library with non-existent api source", - api: "non-existent-dir/api", - repo: newTestGitRepo(t), - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "some-library", - APIs: []*legacyconfig.API{{Path: "non-existent-dir/api"}}, - }, - }, - }, - container: &mockContainerClient{}, - wantConfigureCalls: 1, - wantErr: true, - wantErrMsg: "failed to read dir", - }, - { - name: "configures library with error message in response", - api: "some/api", - repo: newTestGitRepo(t), - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "some-library", - APIs: []*legacyconfig.API{{Path: "some/api"}}, - }, - }, - }, - container: &mockContainerClient{ - wantErrorMsg: true, - }, - wantConfigureCalls: 1, - wantErr: true, - wantErrMsg: "failed with error message", - }, - { - name: "configures library with no response", - api: "some/api", - repo: newTestGitRepo(t), - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "some-library", - APIs: []*legacyconfig.API{{Path: "some/api"}}, - }, - }, - }, - container: &mockContainerClient{ - noConfigureResponse: true, - }, - wantConfigureCalls: 1, - wantErr: true, - wantErrMsg: "no response file for configure container command", - }, - { - name: "configures library without initial version", - api: "some/api", - repo: newTestGitRepo(t), - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "some-library", - APIs: []*legacyconfig.API{{Path: "some/api"}}, - }, - }, - }, - container: &mockContainerClient{ - noInitVersion: true, - }, - wantConfigureCalls: 1, - }, - { - name: "configure_library_without_global_files_in_output", - api: "some/api", - repo: newTestGitRepo(t), - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "some-library", - APIs: []*legacyconfig.API{{Path: "some/api"}}, - }, - }, - }, - librarianConfig: &legacyconfig.LibrarianConfig{ - GlobalFilesAllowlist: []*legacyconfig.GlobalFile{ - { - Path: "a/path/example.txt", - }, - }, - }, - container: &mockContainerClient{}, - wantConfigureCalls: 1, - }, - { - name: "configure command failed", - api: "some/api", - repo: newTestGitRepo(t), - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "some-library", - APIs: []*legacyconfig.API{{Path: "some/api"}}, - }, - }, - }, - container: &mockContainerClient{ - configureErr: errors.New("simulated configure command error"), - noConfigureResponse: true, - }, - wantConfigureCalls: 1, - wantErr: true, - wantErrMsg: "simulated configure command error", - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - outputDir := t.TempDir() - r := &generateRunner{ - api: test.api, - repo: test.repo, - sourceRepo: newTestGitRepo(t), - state: test.state, - librarianConfig: test.librarianConfig, - containerClient: test.container, - } - - // Create a service config - if err := os.MkdirAll(filepath.Join(r.sourceRepo.GetDir(), test.api), 0755); err != nil { - t.Fatal(err) - } - - data := []byte("type: google.api.Service") - if err := os.WriteFile(filepath.Join(r.sourceRepo.GetDir(), test.api, "example_service_v2.yaml"), data, 0755); err != nil { - t.Fatal(err) - } - - if test.name == "configures library with non-existent api source" { - // This test verifies the scenario of no service config is found - // in api path. - if err := os.RemoveAll(filepath.Join(r.sourceRepo.GetDir())); err != nil { - t.Fatal(err) - } - } - - _, err := r.runConfigureCommand(t.Context(), outputDir) - - if test.wantErr { - if err == nil { - t.Fatal("runConfigureCommand() should return error") - } - - if !strings.Contains(err.Error(), test.wantErrMsg) { - t.Errorf("runConfigureCommand() err = %v, want error containing %q", err, test.wantErrMsg) - } - - return - } - - if err != nil { - t.Errorf("runConfigureCommand() error = %v, wantErr %v", err, test.wantErr) - return - } - if diff := cmp.Diff(test.wantConfigureCalls, test.container.configureCalls); diff != "" { - t.Errorf("runConfigureCommand() configureCalls mismatch (-want +got):%s", diff) - } - }) - } -} - -func TestGenerateScenarios(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - api string - library string - state *legacyconfig.LibrarianState - librarianConfig *legacyconfig.LibrarianConfig - container *mockContainerClient - ghClient GitHubClient - build bool - forceShouldGenerateError bool - wantErr bool - wantErrMsg string - wantGenerateCalls int - wantBuildCalls int - wantConfigureCalls int - }{ - { - name: "generate_single_library_including_initial_configuration", - api: "some/api", - library: "some-library", - state: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - }, - container: &mockContainerClient{ - wantLibraryGen: true, - configureLibraryPaths: []string{ - "src/a", - }, - }, - ghClient: &mockGitHubClient{}, - build: true, - wantGenerateCalls: 1, - wantBuildCalls: 1, - wantConfigureCalls: 1, - }, - { - name: "generate_single_library_with_librarian_config", - api: "some/api", - library: "some-library", - state: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - }, - container: &mockContainerClient{ - wantLibraryGen: true, - configureLibraryPaths: []string{ - "src/a", - }, - }, - librarianConfig: &legacyconfig.LibrarianConfig{ - GlobalFilesAllowlist: []*legacyconfig.GlobalFile{ - { - Path: "a/path/example.txt", - Permissions: "read-only", - }, - }, - }, - ghClient: &mockGitHubClient{}, - build: true, - wantGenerateCalls: 1, - wantBuildCalls: 1, - wantConfigureCalls: 1, - }, - { - name: "generate single existing library by library id", - library: "some-library", - state: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "some-library", - APIs: []*legacyconfig.API{{Path: "some/api"}}, - SourceRoots: []string{ - "src/a", - }, - }, - }, - }, - container: &mockContainerClient{ - wantLibraryGen: true, - }, - ghClient: &mockGitHubClient{}, - build: true, - wantGenerateCalls: 1, - wantBuildCalls: 1, - wantConfigureCalls: 0, - }, - { - name: "generate single existing library by api", - api: "some/api", - state: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "some-library", - APIs: []*legacyconfig.API{{Path: "some/api"}}, - SourceRoots: []string{ - "src/a", - }, - }, - }, - }, - container: &mockContainerClient{ - wantLibraryGen: true, - }, - ghClient: &mockGitHubClient{}, - build: true, - wantGenerateCalls: 1, - wantBuildCalls: 1, - wantConfigureCalls: 0, - }, - { - name: "generate single existing library with library id and api", - api: "some/api", - library: "some-library", - state: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "some-library", - APIs: []*legacyconfig.API{{Path: "some/api"}}, - SourceRoots: []string{ - "src/a", - }, - }, - }, - }, - container: &mockContainerClient{ - wantLibraryGen: true, - }, - ghClient: &mockGitHubClient{}, - build: true, - wantGenerateCalls: 1, - wantBuildCalls: 1, - wantConfigureCalls: 0, - }, - { - name: "generate single existing library with invalid library id should fail", - library: "some-not-configured-library", - state: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "some-library", - APIs: []*legacyconfig.API{{Path: "some/api"}}, - }, - }, - }, - container: &mockContainerClient{}, - ghClient: &mockGitHubClient{}, - build: true, - wantErr: true, - wantErrMsg: "not configured yet, generation stopped", - }, - { - name: "generate single existing library with error message in response", - api: "some/api", - library: "some-library", - state: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "some-library", - APIs: []*legacyconfig.API{{Path: "some/api"}}, - }, - }, - }, - container: &mockContainerClient{ - wantErrorMsg: true, - }, - ghClient: &mockGitHubClient{}, - wantGenerateCalls: 1, - wantConfigureCalls: 0, - wantErr: true, - wantErrMsg: "failed with error message", - }, - { - name: "generate all libraries configured in state", - state: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "library1", - APIs: []*legacyconfig.API{{Path: "some/api1"}}, - SourceRoots: []string{ - "src/a", - }, - }, - { - ID: "library2", - APIs: []*legacyconfig.API{{Path: "some/api2"}}, - SourceRoots: []string{ - "src/b", - }, - }, - }, - }, - container: &mockContainerClient{ - wantLibraryGen: true, - }, - ghClient: &mockGitHubClient{}, - build: true, - wantGenerateCalls: 2, - wantBuildCalls: 2, - }, - { - name: "generate single library, corrupted api", - api: "corrupted/api/path", - state: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "some-library", - APIs: []*legacyconfig.API{{Path: "some/api"}}, - }, - }, - }, - container: &mockContainerClient{}, - ghClient: &mockGitHubClient{}, - build: true, - wantErr: true, - wantErrMsg: "not configured yet, generation stopped", - }, - { - name: "symlink in output", - api: "some/api", - state: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "some-library", - APIs: []*legacyconfig.API{{Path: "some/api"}}, - }, - }, - }, - container: &mockContainerClient{}, - build: true, - wantGenerateCalls: 1, - wantErr: true, - wantErrMsg: "failed to make output directory", - }, - { - name: "generate error", - api: "some/api", - state: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "some-library", - APIs: []*legacyconfig.API{{Path: "some/api"}}, - }, - }, - }, - container: &mockContainerClient{generateErr: errors.New("generate error")}, - ghClient: &mockGitHubClient{}, - build: true, - wantErr: true, - wantErrMsg: "generate error", - }, - { - name: "build error", - api: "some/api", - state: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "some-library", - APIs: []*legacyconfig.API{{Path: "some/api"}}, - SourceRoots: []string{ - "src/a", - }, - }, - }, - }, - container: &mockContainerClient{ - buildErr: errors.New("build error"), - wantLibraryGen: true, - }, - ghClient: &mockGitHubClient{}, - build: true, - wantErr: true, - wantErrMsg: "build error", - }, - { - name: "generate all, partial failure does not halt execution", - state: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "lib1", - APIs: []*legacyconfig.API{{Path: "some/api1"}}, - SourceRoots: []string{ - "src/a", - }, - }, - { - ID: "lib2", - APIs: []*legacyconfig.API{{Path: "some/api2"}}, - SourceRoots: []string{ - "src/b", - }, - }, - }, - }, - container: &mockContainerClient{ - wantLibraryGen: true, - failGenerateForID: "lib1", - generateErrForID: errors.New("generate error"), - }, - ghClient: &mockGitHubClient{}, - build: true, - wantGenerateCalls: 2, - wantBuildCalls: 1, - }, - { - name: "generate skips blocked libraries", - state: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "google.cloud.texttospeech.v1", - APIs: []*legacyconfig.API{{Path: "google/cloud/texttospeech/v1"}}, - }, - { - ID: "google.cloud.vision.v1", - APIs: []*legacyconfig.API{{Path: "google/cloud/vision/v1"}}, - }, - }, - }, - librarianConfig: &legacyconfig.LibrarianConfig{ - Libraries: []*legacyconfig.LibraryConfig{ - {LibraryID: "google.cloud.texttospeech.v1"}, - {LibraryID: "google.cloud.vision.v1", GenerateBlocked: true}, - }, - }, - container: &mockContainerClient{ - wantLibraryGen: true, - }, - ghClient: &mockGitHubClient{}, - build: true, - wantGenerateCalls: 1, - wantBuildCalls: 1, - }, - { - name: "generate runs blocked libraries if explicitly requested", - library: "google.cloud.vision.v1", - state: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "google.cloud.texttospeech.v1", - APIs: []*legacyconfig.API{{Path: "google/cloud/texttospeech/v1"}}, - }, - { - ID: "google.cloud.vision.v1", - APIs: []*legacyconfig.API{{Path: "google/cloud/vision/v1"}}, - }, - }, - }, - librarianConfig: &legacyconfig.LibrarianConfig{ - Libraries: []*legacyconfig.LibraryConfig{ - {LibraryID: "google.cloud.texttospecech.v1"}, - {LibraryID: "google.cloud.vision.v1", GenerateBlocked: true}, - }, - }, - container: &mockContainerClient{ - wantLibraryGen: true, - }, - ghClient: &mockGitHubClient{}, - build: true, - wantGenerateCalls: 1, - wantBuildCalls: 1, - }, - { - name: "generate skips a blocked library and the rest fail. should report error", - state: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "google.cloud.texttospeech.v1", - APIs: []*legacyconfig.API{{Path: "google/cloud/texttospeech/v1"}}, - }, - { - ID: "google.cloud.vision.v1", - APIs: []*legacyconfig.API{{Path: "google/cloud/vision/v1"}}, - }, - }, - }, - librarianConfig: &legacyconfig.LibrarianConfig{ - Libraries: []*legacyconfig.LibraryConfig{ - {LibraryID: "google.cloud.texttospeech.v1"}, - {LibraryID: "google.cloud.vision.v1", GenerateBlocked: true}, - }, - }, - container: &mockContainerClient{generateErr: errors.New("generate error")}, - ghClient: &mockGitHubClient{}, - build: true, - wantErr: true, - wantErrMsg: "all 1 libraries failed to generate (skipped: 1)", - }, - { - name: "generate all, all fail should report error", - state: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "lib1", - APIs: []*legacyconfig.API{{Path: "some/api1"}}, - SourceRoots: []string{ - "src/a", - }, - }, - }, - }, - container: &mockContainerClient{ - failGenerateForID: "lib1", - generateErrForID: errors.New("generate error"), - }, - ghClient: &mockGitHubClient{}, - build: true, - wantErr: true, - wantErrMsg: "all 1 libraries failed to generate", - wantGenerateCalls: 1, - wantBuildCalls: 0, - }, - { - name: "generate skips libraries with no APIs", - state: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "some-library", - }, - }, - }, - container: &mockContainerClient{}, - ghClient: &mockGitHubClient{}, - build: true, - wantGenerateCalls: 0, - wantBuildCalls: 0, - wantConfigureCalls: 0, - }, - { - name: "source_roots_have_same_global_files", - state: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "some-library", - SourceRoots: []string{ - "src/some/path", - "one/global/example.txt", - }, - APIs: []*legacyconfig.API{ - { - Path: "google/cloud/some", - }, - }, - }, - { - ID: "another-library", - SourceRoots: []string{ - "src/another/path", - "one/global/example.txt", - }, - APIs: []*legacyconfig.API{ - { - Path: "google/cloud/another", - }, - }, - }, - }, - }, - librarianConfig: &legacyconfig.LibrarianConfig{ - GlobalFilesAllowlist: []*legacyconfig.GlobalFile{ - { - Path: "one/global/example.txt", - Permissions: "read-write", - }, - }, - }, - container: &mockContainerClient{ - wantLibraryGen: true, - }, - ghClient: &mockGitHubClient{}, - build: true, - wantGenerateCalls: 2, - wantBuildCalls: 2, - wantConfigureCalls: 0, - }, - // We only have one library to generate, and we force shouldGenerate - // to fail by making the source repo's HeadHash function fail. - // As this ends up being all the libraries, the overall result is an error. - // (Forcing shouldGenerate to fail selectively would be very complicated.) - { - name: "shouldGenerate error", - state: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "some-library", - APIs: []*legacyconfig.API{{Path: "some/api"}}, - // We need the LastGeneratedCommit to force shouldGenerate - // to ask the source repo for the head hash. - LastGeneratedCommit: "LastGeneratedCommit", - }, - }, - }, - forceShouldGenerateError: true, - wantErr: true, - wantErrMsg: "all 1 libraries failed to generate", - }, - { - name: "generate in release only mode returns error", - state: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "some-library", - APIs: []*legacyconfig.API{{Path: "some/api"}}, - // We need the LastGeneratedCommit to force shouldGenerate - // to ask the source repo for the head hash. - LastGeneratedCommit: "LastGeneratedCommit", - }, - }, - }, - librarianConfig: &legacyconfig.LibrarianConfig{ - ReleaseOnlyMode: true, - }, - wantErr: true, - wantErrMsg: "generate in release only mode", - }, - } { - t.Run(test.name, func(t *testing.T) { - repo := newTestGitRepoWithState(t, test.state) - - r := &generateRunner{ - api: test.api, - library: test.library, - build: test.build, - repo: repo, - sourceRepo: newTestGitRepo(t), - state: test.state, - librarianConfig: test.librarianConfig, - containerClient: test.container, - ghClient: test.ghClient, - workRoot: t.TempDir(), - } - - // Create a service config in api path. - if err := os.MkdirAll(filepath.Join(r.sourceRepo.GetDir(), test.api), 0755); err != nil { - t.Fatal(err) - } - data := []byte("type: google.api.Service") - if err := os.WriteFile(filepath.Join(r.sourceRepo.GetDir(), test.api, "example_service_v2.yaml"), data, 0755); err != nil { - t.Fatal(err) - } - - // Commit the service config file because configure command needs - // to find the piper id associated with the commit message. - if err := r.sourceRepo.AddAll(); err != nil { - t.Fatal(err) - } - message := "feat: add an api\n\nPiperOrigin-RevId: 123456" - if err := r.sourceRepo.Commit(message); err != nil { - t.Fatal(err) - } - - if test.forceShouldGenerateError { - r.sourceRepo = &MockRepository{ - HeadHashError: errors.New("fail"), - } - } - - // Create a symlink in the output directory to trigger an error. - if test.name == "symlink in output" { - outputDir := filepath.Join(r.workRoot, "output") - if err := os.MkdirAll(outputDir, 0755); err != nil { - t.Fatalf("os.MkdirAll() = %v", err) - } - if err := os.Symlink("target", filepath.Join(outputDir, "symlink")); err != nil { - t.Fatalf("os.Symlink() = %v", err) - } - } - - err := r.run(t.Context()) - if test.wantErr { - if err == nil { - t.Fatalf("%s should return error", test.name) - } - - if !strings.Contains(err.Error(), test.wantErrMsg) { - t.Errorf("want error message %s, got %s", test.wantErrMsg, err.Error()) - } - - return - } - if err != nil { - t.Fatal(err) - } - if diff := cmp.Diff(test.wantGenerateCalls, test.container.generateCalls); diff != "" { - t.Errorf("%s: run() generateCalls mismatch (-want +got):%s", test.name, diff) - } - if diff := cmp.Diff(test.wantBuildCalls, test.container.buildCalls); diff != "" { - t.Errorf("%s: run() buildCalls mismatch (-want +got):%s", test.name, diff) - } - if diff := cmp.Diff(test.wantConfigureCalls, test.container.configureCalls); diff != "" { - t.Errorf("%s: run() configureCalls mismatch (-want +got):%s", test.name, diff) - } - }) - } -} - -func TestGenerateSingleLibraryCommand(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - api string - library string - image string - state *legacyconfig.LibrarianState - container *mockContainerClient - ghClient GitHubClient - build bool - wantErr bool - wantErrMsg string - wantPRType pullRequestType - wantImage string - }{ - { - name: "onboard library returns pullRequestOnboard", - api: "some/api", - library: "some-library", - state: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - }, - container: &mockContainerClient{ - wantLibraryGen: true, - configureLibraryPaths: []string{ - "src/a", - }, - }, - ghClient: &mockGitHubClient{}, - build: true, - wantPRType: pullRequestOnboard, - wantImage: "gcr.io/test/image:v1.2.3", - }, - { - name: "generate existing library returns pullRequestGenerate", - library: "some-library", - state: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "some-library", - APIs: []*legacyconfig.API{{Path: "some/api"}}, - SourceRoots: []string{ - "src/a", - }, - }, - }, - }, - container: &mockContainerClient{ - wantLibraryGen: true, - }, - ghClient: &mockGitHubClient{}, - build: true, - wantPRType: pullRequestGenerate, - wantImage: "gcr.io/test/image:v1.2.3", - }, - { - name: "generate existing library using image from command line", - library: "some-library", - image: "image-from-command-line", - state: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "some-library", - APIs: []*legacyconfig.API{{Path: "some/api"}}, - SourceRoots: []string{ - "src/a", - }, - }, - }, - }, - container: &mockContainerClient{ - wantLibraryGen: true, - }, - ghClient: &mockGitHubClient{}, - build: true, - wantPRType: pullRequestGenerate, - wantImage: "image-from-command-line", - }, - } { - t.Run(test.name, func(t *testing.T) { - repo := newTestGitRepoWithState(t, test.state) - sourceRepo := newTestGitRepo(t) - r := &generateRunner{ - api: test.api, - image: test.image, - library: test.library, - build: test.build, - repo: repo, - sourceRepo: sourceRepo, - state: test.state, - containerClient: test.container, - ghClient: test.ghClient, - workRoot: t.TempDir(), - } - - // Create a service config in api path. - if test.api != "" { - if err := os.MkdirAll(filepath.Join(r.sourceRepo.GetDir(), test.api), 0755); err != nil { - t.Fatal(err) - } - data := []byte("type: google.api.Service") - if err := os.WriteFile(filepath.Join(r.sourceRepo.GetDir(), test.api, "example_service_v2.yaml"), data, 0755); err != nil { - t.Fatal(err) - } - // Commit the service config file because configure command needs - // to find the piper id associated with the commit message. - if err := r.sourceRepo.AddAll(); err != nil { - t.Fatal(err) - } - message := "feat: add an api\n\nPiperOrigin-RevId: 123456" - if err := r.sourceRepo.Commit(message); err != nil { - t.Fatal(err) - } - } - - status, err := r.generateSingleLibrary(t.Context(), r.library, r.workRoot) - if test.wantErr { - if err == nil { - t.Fatalf("%s should return error", test.name) - } - if !strings.Contains(err.Error(), test.wantErrMsg) { - t.Errorf("want error message %s, got %s", test.wantErrMsg, err.Error()) - } - return - } - if err != nil { - t.Fatal(err) - } - if status.prType != test.wantPRType { - t.Errorf("generateSingleLibrary() prType = %v, want %v", status.prType, test.wantPRType) - } - if test.container.Image != test.wantImage { - t.Errorf("generateSingleLibrary() image = %v, want %v", test.container.Image, test.wantImage) - } - }) - } -} - -func TestGetExistingSrc(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - libraryID string - paths []string - want []string - }{ - { - name: "all_source_paths_existed", - libraryID: "some-library", - paths: []string{ - "a/path", - "another/path", - }, - want: []string{ - "a/path", - "another/path", - }, - }, - { - name: "one_source_paths_existed", - libraryID: "some-library", - paths: []string{ - "a/path", - }, - want: []string{ - "a/path", - }, - }, - { - name: "no_source_paths_existed", - libraryID: "some-library", - want: nil, - }, - { - name: "no_library_existed", - libraryID: "another-library", - want: nil, - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - repo := newTestGitRepo(t) - state := &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "some-library", - SourceRoots: []string{ - "a/path", - "another/path", - }, - }, - }, - } - for _, path := range test.paths { - relPath := filepath.Join(repo.GetDir(), path) - if err := os.MkdirAll(relPath, 0755); err != nil { - t.Fatal(err) - } - } - - r := &generateRunner{ - repo: repo, - state: state, - } - - got := r.getExistingSrc(test.libraryID) - if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("getExistingSrc() mismatch (-want +got):%s", diff) - } - }) - } -} - -func TestUpdateLastGeneratedCommitState(t *testing.T) { - t.Parallel() - sourceRepo := newTestGitRepo(t) - hash, err := sourceRepo.HeadHash() - if err != nil { - t.Fatal(err) - } - r := &generateRunner{ - sourceRepo: sourceRepo, - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "some-library", - }, - }, - }, - } - if err := r.updateLastGeneratedCommitState("some-library"); err != nil { - t.Fatal(err) - } - if r.state.Libraries[0].LastGeneratedCommit != hash { - t.Errorf("updateState() got = %v, want %v", r.state.Libraries[0].LastGeneratedCommit, hash) - } -} - -func TestShouldGenerate(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - config *legacyconfig.LibrarianConfig - state *legacyconfig.LibrarianState - generateUnchanged bool - sourceRepo legacygitrepo.Repository - libraryIDToTest string - want bool - wantErr bool - }{ - // Tests that don't get as far as checking for hashes. - // (The mock repo will fail if we do get that far.) - { - name: "generation blocked", - config: &legacyconfig.LibrarianConfig{ - Libraries: []*legacyconfig.LibraryConfig{ - { - LibraryID: "TestLibrary", - GenerateBlocked: true, - }, - }, - }, - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "TestLibrary", - APIs: []*legacyconfig.API{{Path: "google/cloud/test"}}, - LastGeneratedCommit: "LastGeneratedHash", - }, - }, - }, - sourceRepo: &MockRepository{ - HeadHashError: errors.New("Shouldn't get as far as checking head"), - }, - libraryIDToTest: "TestLibrary", - want: false, - }, - { - name: "library has no APIs", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "TestLibrary", - // This may be present even if it's meaningless. - LastGeneratedCommit: "LastGeneratedCommit", - }, - }, - }, - sourceRepo: &MockRepository{ - HeadHashError: errors.New("Shouldn't get as far as checking head"), - }, - libraryIDToTest: "TestLibrary", - want: false, - }, - { - name: "generateUnchanged specified", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "TestLibrary", - APIs: []*legacyconfig.API{{Path: "google/cloud/test"}}, - LastGeneratedCommit: "LastGeneratedCommit", - }, - }, - }, - generateUnchanged: true, - sourceRepo: &MockRepository{ - HeadHashError: errors.New("Shouldn't get as far as checking head"), - }, - libraryIDToTest: "TestLibrary", - want: true, - }, - { - name: "no LastGeneratedCommit", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "TestLibrary", - APIs: []*legacyconfig.API{{Path: "google/cloud/test"}}, - }, - }, - }, - sourceRepo: &MockRepository{ - HeadHashError: errors.New("Shouldn't get as far as checking head"), - }, - libraryIDToTest: "TestLibrary", - want: true, - }, - { - name: "error from HeadHash", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "TestLibrary", - APIs: []*legacyconfig.API{{Path: "google/cloud/test"}}, - LastGeneratedCommit: "LastGeneratedCommit", - }, - }, - }, - sourceRepo: &MockRepository{ - HeadHashError: errors.New("Can't get head commit"), - }, - libraryIDToTest: "TestLibrary", - wantErr: true, - }, - // Tests that do perform hash checking. - { - name: "error from GetHashForPath", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "TestLibrary", - APIs: []*legacyconfig.API{{Path: "google/cloud/test"}}, - LastGeneratedCommit: "LastGeneratedCommit", - }, - }, - }, - sourceRepo: &MockRepository{ - HeadHashValue: "HeadCommit", - GetHashForPathError: errors.New("Can't get hash for path"), - }, - libraryIDToTest: "TestLibrary", - wantErr: true, - }, - { - name: "config present but generation not blocked", - config: &legacyconfig.LibrarianConfig{ - Libraries: []*legacyconfig.LibraryConfig{ - { - LibraryID: "OtherLibrary", - GenerateBlocked: true, - }, - { - LibraryID: "TestLibrary", - // Just to have some reason to make it configured... - ReleaseBlocked: true, - }}, - }, - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "TestLibrary", - APIs: []*legacyconfig.API{{Path: "google/cloud/test"}}, - LastGeneratedCommit: "LastGeneratedCommit", - }, - }, - }, - sourceRepo: &MockRepository{ - HeadHashValue: "HeadCommit", - GetHashForPathValue: map[string]string{ - "LastGeneratedCommit:google/cloud/test": "hash1", - "HeadCommit:google/cloud/test": "hash2", - }, - }, - libraryIDToTest: "TestLibrary", - want: true, - }, - { - name: "API hasn't changed", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "TestLibrary", - APIs: []*legacyconfig.API{{Path: "google/cloud/test"}}, - LastGeneratedCommit: "LastGeneratedCommit", - }, - }, - }, - sourceRepo: &MockRepository{ - HeadHashValue: "HeadCommit", - GetHashForPathValue: map[string]string{ - "LastGeneratedCommit:google/cloud/test": "hash", - "HeadCommit:google/cloud/test": "hash", - }, - }, - libraryIDToTest: "TestLibrary", - want: false, - }, - { - name: "one API hasn't changed, one has", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "TestLibrary", - APIs: []*legacyconfig.API{ - {Path: "google/cloud/test1"}, - {Path: "google/cloud/test2"}, - }, - LastGeneratedCommit: "LastGeneratedCommit", - }, - }, - }, - sourceRepo: &MockRepository{ - HeadHashValue: "HeadCommit", - GetHashForPathValue: map[string]string{ - "LastGeneratedCommit:google/cloud/test1": "hash1", - "HeadCommit:google/cloud/test1": "hash1", - "LastGeneratedCommit:google/cloud/test2": "hash2a", - "HeadCommit:google/cloud/test2": "hash2b", - }, - }, - libraryIDToTest: "TestLibrary", - want: true, - }, - { - name: "second call to GetHashForPath fails", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "TestLibrary", - APIs: []*legacyconfig.API{{Path: "google/cloud/test"}}, - LastGeneratedCommit: "LastGeneratedCommit", - }, - }, - }, - sourceRepo: &MockRepository{ - HeadHashValue: "HeadCommit", - GetHashForPathValue: map[string]string{ - "LastGeneratedCommit:google/cloud/test": "hash", - // Entry which deliberately returns an error - "HeadCommit:google/cloud/test": "error", - }, - }, - libraryIDToTest: "TestLibrary", - wantErr: true, - }, - } { - t.Run(test.name, func(t *testing.T) { - r := &generateRunner{ - generateUnchanged: test.generateUnchanged, - librarianConfig: test.config, - state: test.state, - sourceRepo: test.sourceRepo, - } - library := test.state.LibraryByID(test.libraryIDToTest) - got, err := r.shouldGenerate(library) - if test.wantErr != (err != nil) { - t.Fatalf("shouldGenerate() error = %v, wantErr %v", err, test.wantErr) - } - if got != test.want { - t.Errorf("shouldGenerate() got = %v, want %v", got, test.want) - } - }) - } -} - -func TestAddAPIToLibrary(t *testing.T) { - t.Parallel() - testCases := []struct { - name string - initialState *legacyconfig.LibrarianState - libraryID string - apiPath string - expectedState *legacyconfig.LibrarianState - }{ - { - name: "add api to existing library", - initialState: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "lib1", - APIs: []*legacyconfig.API{ - {Path: "api1"}, - }, - }, - }, - }, - libraryID: "lib1", - apiPath: "api2", - expectedState: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "lib1", - APIs: []*legacyconfig.API{ - {Path: "api1"}, - {Path: "api2", Status: legacyconfig.StatusNew}, - }, - }, - }, - }, - }, - { - name: "add api to new library", - initialState: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "lib1", - APIs: []*legacyconfig.API{ - {Path: "api1"}, - }, - }, - }, - }, - libraryID: "lib2", - apiPath: "api2", - expectedState: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "lib1", - APIs: []*legacyconfig.API{ - {Path: "api1"}, - }, - }, - { - ID: "lib2", - APIs: []*legacyconfig.API{ - {Path: "api2", Status: legacyconfig.StatusNew}, - }, - }, - }, - }, - }, - { - name: "add existing api to existing library", - initialState: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "lib1", - APIs: []*legacyconfig.API{ - {Path: "api1"}, - }, - }, - }, - }, - libraryID: "lib1", - apiPath: "api1", - expectedState: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "lib1", - APIs: []*legacyconfig.API{ - {Path: "api1"}, - }, - }, - }, - }, - }, - { - name: "add api to empty state", - initialState: &legacyconfig.LibrarianState{}, - libraryID: "lib1", - apiPath: "api1", - expectedState: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "lib1", - APIs: []*legacyconfig.API{ - {Path: "api1", Status: legacyconfig.StatusNew}, - }, - }, - }, - }, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - addAPIToLibrary(tc.initialState, tc.libraryID, tc.apiPath) - if diff := cmp.Diff(tc.expectedState, tc.initialState); diff != "" { - t.Errorf("addAPIToLibrary() mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestNeedsConfigure(t *testing.T) { - t.Parallel() - testCases := []struct { - name string - api string - library string - state *legacyconfig.LibrarianState - want bool - }{ - { - name: "api and library set, library does not exist", - api: "some/api", - library: "some-library", - state: &legacyconfig.LibrarianState{}, - want: true, - }, - { - name: "api and library set library exists no api path in state yaml", - api: "some/api", - library: "some-library", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "some-library", - }, - }, - }, - want: true, - }, - { - name: "api and library set library exists different api path in state yaml", - api: "some/api", - library: "some-library", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "some-library", - APIs: []*legacyconfig.API{ - {Path: "another/api"}, - }, - }, - }, - }, - want: true, - }, - { - name: "api not set", - api: "", - library: "some-library", - state: &legacyconfig.LibrarianState{}, - want: false, - }, - { - name: "library not set", - api: "some/api", - library: "", - state: &legacyconfig.LibrarianState{}, - want: false, - }, - { - name: "api and library not set", - api: "", - library: "", - state: &legacyconfig.LibrarianState{}, - want: false, - }, - { - name: "api and library set, library and api exist", - api: "some/api", - library: "some-library", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "some-library", - APIs: []*legacyconfig.API{ - {Path: "some/api"}, - }, - }, - }, - }, - want: false, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - r := &generateRunner{ - api: tc.api, - library: tc.library, - state: tc.state, - } - got := r.needsConfigure() - if got != tc.want { - t.Errorf("needsConfigure() = %v, want %v", got, tc.want) - } - }) - } -} diff --git a/internal/legacylibrarian/legacylibrarian/generate_pull_request.go b/internal/legacylibrarian/legacylibrarian/generate_pull_request.go deleted file mode 100644 index ec1cf2cf6ac..00000000000 --- a/internal/legacylibrarian/legacylibrarian/generate_pull_request.go +++ /dev/null @@ -1,278 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacylibrarian - -import ( - "bytes" - "fmt" - "log/slog" - "path/filepath" - "sort" - "strings" - "time" - - "github.com/googleapis/librarian/internal/legacylibrarian/legacycli" - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" - "github.com/googleapis/librarian/internal/legacylibrarian/legacygitrepo" -) - -type generationPRRequest struct { - sourceRepo legacygitrepo.Repository - languageRepo legacygitrepo.Repository - state *legacyconfig.LibrarianState - idToCommits map[string]string - failedLibraries []string -} - -type onboardPRRequest struct { - sourceRepo legacygitrepo.Repository - state *legacyconfig.LibrarianState - api string - library string -} - -type generationPRBody struct { - StartSHA string - EndSHA string - LibrarianVersion string - ImageVersion string - Commits []*legacygitrepo.ConventionalCommit - FailedLibraries []string -} - -type onboardingPRBody struct { - ImageVersion string - LibrarianVersion string - LibraryID string - PiperID string -} - -// formatGenerationPRBody creates the body of a generation pull request. -// Only consider libraries whose ID appears in idToCommits. -func formatGenerationPRBody(request *generationPRRequest) (string, error) { - var allCommits []*legacygitrepo.ConventionalCommit - languageRepoChanges, err := languageRepoChangedFiles(request.languageRepo) - if err != nil { - return "", fmt.Errorf("failed to fetch changes in language repo: %w", err) - } - for _, library := range request.state.Libraries { - lastGenCommit, ok := request.idToCommits[library.ID] - if !ok { - continue - } - // If nothing has changed that would be significant in a release for this library, - // we don't look at the API changes either. - if !shouldIncludeForRelease(languageRepoChanges, library.SourceRoots, library.ReleaseExcludePaths) { - continue - } - - commits, err := getConventionalCommitsSinceLastGeneration(request.sourceRepo, library, lastGenCommit) - if err != nil { - return "", fmt.Errorf("failed to fetch conventional commits for library, %s: %w", library.ID, err) - } - allCommits = append(allCommits, commits...) - } - - if len(allCommits) == 0 { - return "No commit is found since last generation", nil - } - - startCommit, err := findLatestGenerationCommit(request.sourceRepo, request.state, request.idToCommits) - if err != nil { - return "", fmt.Errorf("failed to find the start commit: %w", err) - } - // Even though startCommit might be nil, it shouldn't happen in production - // because this function will return early if no conventional commit is found - // since last generation. - startSHA := startCommit.Hash.String() - groupedCommits := groupByIDAndSubject(allCommits) - // Sort the slice by commit time in reverse order, - // so that the latest commit appears first. - sort.Slice(groupedCommits, func(i, j int) bool { - return groupedCommits[i].When.After(groupedCommits[j].When) - }) - endSHA := groupedCommits[0].CommitHash - librarianVersion := legacycli.Version() - data := &generationPRBody{ - StartSHA: startSHA, - EndSHA: endSHA, - LibrarianVersion: librarianVersion, - ImageVersion: request.state.Image, - Commits: groupedCommits, - FailedLibraries: request.failedLibraries, - } - var out bytes.Buffer - if err := genBodyTemplate.Execute(&out, data); err != nil { - return "", fmt.Errorf("error executing template: %w", err) - } - - return strings.TrimSpace(out.String()), nil -} - -// languageRepoChangedFiles returns the paths of files changed in the repo as part -// of the current librarian run - either in the head commit if the repo is clean, -// or the outstanding changes otherwise. -func languageRepoChangedFiles(languageRepo legacygitrepo.Repository) ([]string, error) { - clean, err := languageRepo.IsClean() - if err != nil { - return nil, err - } - if clean { - headHash, err := languageRepo.HeadHash() - if err != nil { - return nil, err - } - return languageRepo.ChangedFilesInCommit(headHash) - } - // The commit or push flag is not set, get all locally changed files. - return languageRepo.ChangedFiles() -} - -// formatOnboardPRBody creates the body of an onboarding pull request. -func formatOnboardPRBody(request *onboardPRRequest) (string, error) { - piperID, err := getPiperID(request.state, request.sourceRepo, request.api, request.library) - if err != nil { - return "", err - } - - data := &onboardingPRBody{ - LibrarianVersion: legacycli.Version(), - ImageVersion: request.state.Image, - LibraryID: request.library, - PiperID: piperID, - } - - var out bytes.Buffer - if err := onboardingBodyTemplate.Execute(&out, data); err != nil { - return "", fmt.Errorf("error executing template: %w", err) - } - - return strings.TrimSpace(out.String()), nil -} - -// getPiperID extracts the Piper ID from the commit message that onboarded the API. -func getPiperID(state *legacyconfig.LibrarianState, sourceRepo legacygitrepo.Repository, apiPath, library string) (string, error) { - libraryState := state.LibraryByID(library) - if libraryState == nil { - return "", fmt.Errorf("library %s not found", library) - } - serviceYaml := "" - for _, api := range libraryState.APIs { - if api.Path == apiPath { - serviceYaml = api.ServiceConfig - break - } - } - - initialCommit, err := sourceRepo.GetLatestCommit(filepath.Join(apiPath, serviceYaml)) - if err != nil { - return "", err - } - - id, err := findPiperIDFrom(initialCommit, library) - if err != nil { - return "", err - } - - slog.Info("found piper id in the commit message", "piperID", id) - return id, nil -} - -func findPiperIDFrom(commit *legacygitrepo.Commit, libraryID string) (string, error) { - commits, err := legacygitrepo.ParseCommits(commit, libraryID) - if err != nil { - return "", err - } - - if len(commits) == 0 || commits[0].Footers == nil { - return "", errPiperNotFound - } - - id, ok := commits[0].Footers["PiperOrigin-RevId"] - if !ok { - return "", errPiperNotFound - } - - return id, nil -} - -// findLatestGenerationCommit returns the latest commit among the last generated -// commit of all the libraries that have been generated. -// A library is skipped if the last generated commit is empty. -// -// Note that it is possible that the returned commit is nil. -func findLatestGenerationCommit(repo legacygitrepo.Repository, state *legacyconfig.LibrarianState, idToCommits map[string]string) (*legacygitrepo.Commit, error) { - latest := time.UnixMilli(0) // the earliest timestamp. - var res *legacygitrepo.Commit - for _, library := range state.Libraries { - commitHash, ok := idToCommits[library.ID] - if !ok || commitHash == "" { - slog.Debug("skip getting last generated commit", "library", library.ID) - continue - } - commit, err := repo.GetCommit(commitHash) - if err != nil { - return nil, fmt.Errorf("can't find last generated commit for %s: %w", library.ID, err) - } - if latest.Before(commit.When) { - latest = commit.When - res = commit - } - } - - if res == nil { - slog.Warn("no library has non-empty last generated commit") - } - - return res, nil -} - -// groupByIDAndSubject aggregates conventional commits for ones have the same Piper ID and subject in the footer. -func groupByIDAndSubject(commits []*legacygitrepo.ConventionalCommit) []*legacygitrepo.ConventionalCommit { - var res []*legacygitrepo.ConventionalCommit - idToCommits := make(map[string][]*legacygitrepo.ConventionalCommit) - for _, commit := range commits { - // a commit is not considering for grouping if it doesn't have a footer or - // the footer doesn't have a Piper ID. - if commit.Footers == nil { - commit.Footers = make(map[string]string) - commit.Footers["Library-IDs"] = commit.LibraryID - res = append(res, commit) - continue - } - - id, ok := commit.Footers["PiperOrigin-RevId"] - if !ok { - commit.Footers["Library-IDs"] = commit.LibraryID - res = append(res, commit) - continue - } - - key := fmt.Sprintf("%s-%s", id, commit.Subject) - idToCommits[key] = append(idToCommits[key], commit) - } - - for _, groupCommits := range idToCommits { - var ids []string - for _, commit := range groupCommits { - ids = append(ids, commit.LibraryID) - } - firstCommit := groupCommits[0] - firstCommit.Footers["Library-IDs"] = strings.Join(ids, ",") - res = append(res, firstCommit) - } - - return res -} diff --git a/internal/legacylibrarian/legacylibrarian/generate_pull_request_test.go b/internal/legacylibrarian/legacylibrarian/generate_pull_request_test.go deleted file mode 100644 index 7cd302415e4..00000000000 --- a/internal/legacylibrarian/legacylibrarian/generate_pull_request_test.go +++ /dev/null @@ -1,925 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacylibrarian - -import ( - "errors" - "fmt" - "strings" - "testing" - "time" - - "github.com/go-git/go-git/v5/plumbing" - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - "github.com/googleapis/librarian/internal/legacylibrarian/legacycli" - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" - "github.com/googleapis/librarian/internal/legacylibrarian/legacygitrepo" -) - -func TestFormatGenerationPRBody(t *testing.T) { - t.Parallel() - - today := time.Now() - hash1 := plumbing.NewHash("1234567890abcdef") - hash2 := plumbing.NewHash("fedcba0987654321") - librarianVersion := legacycli.Version() - - for _, test := range []struct { - name string - state *legacyconfig.LibrarianState - sourceRepo legacygitrepo.Repository - languageRepo legacygitrepo.Repository - idToCommits map[string]string - failedLibraries []string - api string - library string - apiOnboarding bool - want string - wantErr bool - wantErrPhrase string - }{ - { - // This test verifies that only changed libraries appear in the pull request - // body. - name: "multiple libraries generation", - state: &legacyconfig.LibrarianState{ - Image: "go:1.21", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "one-library", - SourceRoots: []string{"path/to"}, - APIs: []*legacyconfig.API{ - { - Path: "path/to", - }, - }, - }, - { - ID: "another-library", - SourceRoots: []string{"path/to"}, - APIs: []*legacyconfig.API{ - { - Path: "path/to", - }, - }, - }, - }, - }, - sourceRepo: &MockRepository{ - RemotesValue: []*legacygitrepo.Remote{{Name: "origin", URLs: []string{"https://github.com/owner/repo.git"}}}, - GetCommitByHash: map[string]*legacygitrepo.Commit{ - "1234567890": { - Hash: plumbing.NewHash("1234567890"), - When: time.UnixMilli(200), - }, - "abcdefg": { - Hash: plumbing.NewHash("abcdefg"), - When: time.UnixMilli(300), - }, - }, - GetCommitsForPathsSinceLastGenByCommit: map[string][]*legacygitrepo.Commit{ - "1234567890": { - { - Message: "fix: a bug fix\n\nThis is another body.\n\nPiperOrigin-RevId: 573342", - Hash: hash2, - When: today.Add(time.Hour), - }, - }, - "abcdefg": {}, // no new commits since commit "abcdefg". - }, - ChangedFilesInCommitValueByHash: map[string][]string{ - hash2.String(): { - "path/to/file", - }, - }, - }, - languageRepo: &MockRepository{ - IsCleanValue: true, - HeadHashValue: "5678", - ChangedFilesInCommitValue: []string{"path/to/a.go"}, - }, - idToCommits: map[string]string{ - "one-library": "1234567890", - "another-library": "abcdefg", - }, - failedLibraries: []string{}, - want: fmt.Sprintf(`PR created by the Librarian CLI to generate Cloud Client Libraries code from protos. - -BEGIN_COMMIT - -BEGIN_NESTED_COMMIT -fix: a bug fix -This is another body. - -PiperOrigin-RevId: 573342 -Library-IDs: one-library -Source-link: [googleapis/googleapis@fedcba09](https://github.com/googleapis/googleapis/commit/fedcba09) -END_NESTED_COMMIT - -END_COMMIT - -This pull request is generated with proto changes between -[googleapis/googleapis@abcdef00](https://github.com/googleapis/googleapis/commit/abcdef0000000000000000000000000000000000) -(exclusive) and -[googleapis/googleapis@fedcba09](https://github.com/googleapis/googleapis/commit/fedcba0987654321000000000000000000000000) -(inclusive). - -Librarian Version: %s -Language Image: %s`, - librarianVersion, "go:1.21"), - }, - { - name: "group_commit_messages", - state: &legacyconfig.LibrarianState{ - Image: "go:1.21", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "one-library", - SourceRoots: []string{"path/to"}, - APIs: []*legacyconfig.API{ - { - Path: "path/to", - }, - }, - }, - { - ID: "another-library", - SourceRoots: []string{"path/to"}, - APIs: []*legacyconfig.API{ - { - Path: "path/to", - }, - }, - }, - }, - }, - sourceRepo: &MockRepository{ - RemotesValue: []*legacygitrepo.Remote{{Name: "origin", URLs: []string{"https://github.com/owner/repo.git"}}}, - GetCommitByHash: map[string]*legacygitrepo.Commit{ - "1234567890": { - Hash: plumbing.NewHash("1234567890"), - When: time.UnixMilli(200), - }, - "abcdefg": { - Hash: plumbing.NewHash("abcdefg"), - When: time.UnixMilli(300), - }, - }, - GetCommitsForPathsSinceLastGenByCommit: map[string][]*legacygitrepo.Commit{ - "1234567890": { - { - Message: "fix: a bug fix\n\nThis is another body.\n\nPiperOrigin-RevId: 573342", - Hash: hash2, - When: today.Add(time.Hour), - }, - }, - "abcdefg": { - { - Message: "fix: a bug fix\n\nThis is another body.\n\nPiperOrigin-RevId: 573342", - Hash: hash2, - When: today.Add(time.Hour), - }, - }, - }, - ChangedFilesInCommitValueByHash: map[string][]string{ - hash2.String(): { - "path/to/file", - }, - }, - }, - languageRepo: &MockRepository{ - IsCleanValue: true, - HeadHashValue: "5678", - ChangedFilesInCommitValue: []string{"path/to/a.go"}, - }, - idToCommits: map[string]string{ - "one-library": "1234567890", - "another-library": "abcdefg", - }, - failedLibraries: []string{}, - want: fmt.Sprintf(`PR created by the Librarian CLI to generate Cloud Client Libraries code from protos. - -BEGIN_COMMIT - -BEGIN_NESTED_COMMIT -fix: a bug fix -This is another body. - -PiperOrigin-RevId: 573342 -Library-IDs: one-library,another-library -Source-link: [googleapis/googleapis@fedcba09](https://github.com/googleapis/googleapis/commit/fedcba09) -END_NESTED_COMMIT - -END_COMMIT - -This pull request is generated with proto changes between -[googleapis/googleapis@abcdef00](https://github.com/googleapis/googleapis/commit/abcdef0000000000000000000000000000000000) -(exclusive) and -[googleapis/googleapis@fedcba09](https://github.com/googleapis/googleapis/commit/fedcba0987654321000000000000000000000000) -(inclusive). - -Librarian Version: %s -Language Image: %s`, - librarianVersion, "go:1.21"), - }, - { - name: "multiple libraries generation with failed libraries", - state: &legacyconfig.LibrarianState{ - Image: "go:1.21", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "one-library", - SourceRoots: []string{"path/to"}, - APIs: []*legacyconfig.API{ - { - Path: "path/to", - }, - }, - }, - { - ID: "another-library", - SourceRoots: []string{"path/to"}, - APIs: []*legacyconfig.API{ - { - Path: "path/to", - }, - }, - }, - }, - }, - sourceRepo: &MockRepository{ - RemotesValue: []*legacygitrepo.Remote{{Name: "origin", URLs: []string{"https://github.com/owner/repo.git"}}}, - GetCommitByHash: map[string]*legacygitrepo.Commit{ - "1234567890": { - Hash: plumbing.NewHash("1234567890"), - When: time.UnixMilli(200), - }, - "abcdefg": { - Hash: plumbing.NewHash("abcdefg"), - When: time.UnixMilli(300), - }, - }, - GetCommitsForPathsSinceLastGenByCommit: map[string][]*legacygitrepo.Commit{ - "1234567890": { - { - Message: "fix: a bug fix\n\nThis is another body.\n\nPiperOrigin-RevId: 573342", - Hash: hash2, - When: today.Add(time.Hour), - }, - }, - "abcdefg": {}, // no new commits since commit "abcdefg". - }, - ChangedFilesInCommitValueByHash: map[string][]string{ - hash2.String(): { - "path/to/file", - }, - }, - }, - languageRepo: &MockRepository{ - IsCleanValue: true, - HeadHashValue: "5678", - ChangedFilesInCommitValue: []string{"path/to/a.go"}, - }, - idToCommits: map[string]string{ - "one-library": "1234567890", - "another-library": "abcdefg", - }, - failedLibraries: []string{ - "failed-library-a", - "failed-library-b", - }, - want: fmt.Sprintf(`PR created by the Librarian CLI to generate Cloud Client Libraries code from protos. - -BEGIN_COMMIT - -BEGIN_NESTED_COMMIT -fix: a bug fix -This is another body. - -PiperOrigin-RevId: 573342 -Library-IDs: one-library -Source-link: [googleapis/googleapis@fedcba09](https://github.com/googleapis/googleapis/commit/fedcba09) -END_NESTED_COMMIT - -END_COMMIT - -This pull request is generated with proto changes between -[googleapis/googleapis@abcdef00](https://github.com/googleapis/googleapis/commit/abcdef0000000000000000000000000000000000) -(exclusive) and -[googleapis/googleapis@fedcba09](https://github.com/googleapis/googleapis/commit/fedcba0987654321000000000000000000000000) -(inclusive). - -Librarian Version: %s -Language Image: %s - -## Generation failed for -- failed-library-a -- failed-library-b`, - librarianVersion, "go:1.21"), - }, - { - name: "single library generation", - state: &legacyconfig.LibrarianState{ - Image: "go:1.21", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "one-library", - SourceRoots: []string{"path/to"}, - APIs: []*legacyconfig.API{ - { - Path: "path/to", - }, - }, - }, - }, - }, - sourceRepo: &MockRepository{ - RemotesValue: []*legacygitrepo.Remote{{Name: "origin", URLs: []string{"https://github.com/owner/repo.git"}}}, - GetCommitByHash: map[string]*legacygitrepo.Commit{ - "1234567890": { - Hash: plumbing.NewHash("1234567890"), - When: time.UnixMilli(200), - }, - }, - GetCommitsForPathsSinceLastGenByCommit: map[string][]*legacygitrepo.Commit{ - "1234567890": { - { - Message: "feat: new feature\n\nThis is body.\n\nPiperOrigin-RevId: 98765", - Hash: hash1, - When: today, - }, - { - Message: "fix: a bug fix\n\nThis is another body.\n\nPiperOrigin-RevId: 573342", - Hash: hash2, - When: today.Add(time.Hour), - }, - }, - }, - ChangedFilesInCommitValueByHash: map[string][]string{ - hash1.String(): { - "path/to/file", - "path/to/another/file", - }, - hash2.String(): { - "path/to/file", - }, - }, - }, - languageRepo: &MockRepository{ - IsCleanValue: true, - HeadHashValue: "5678", - ChangedFilesInCommitValue: []string{"path/to/a.go"}, - }, - idToCommits: map[string]string{ - "one-library": "1234567890", - }, - failedLibraries: []string{}, - want: fmt.Sprintf(`PR created by the Librarian CLI to generate Cloud Client Libraries code from protos. - -BEGIN_COMMIT - -BEGIN_NESTED_COMMIT -fix: a bug fix -This is another body. - -PiperOrigin-RevId: 573342 -Library-IDs: one-library -Source-link: [googleapis/googleapis@fedcba09](https://github.com/googleapis/googleapis/commit/fedcba09) -END_NESTED_COMMIT - -BEGIN_NESTED_COMMIT -feat: new feature -This is body. - -PiperOrigin-RevId: 98765 -Library-IDs: one-library -Source-link: [googleapis/googleapis@12345678](https://github.com/googleapis/googleapis/commit/12345678) -END_NESTED_COMMIT - -END_COMMIT - -This pull request is generated with proto changes between -[googleapis/googleapis@12345678](https://github.com/googleapis/googleapis/commit/1234567890000000000000000000000000000000) -(exclusive) and -[googleapis/googleapis@fedcba09](https://github.com/googleapis/googleapis/commit/fedcba0987654321000000000000000000000000) -(inclusive). - -Librarian Version: %s -Language Image: %s`, - librarianVersion, "go:1.21"), - }, - { - name: "no conventional commit is found since last generation", - state: &legacyconfig.LibrarianState{ - Image: "go:1.21", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "one-library", - SourceRoots: []string{"path/to"}, - // Intentionally set this value to verify the test can pass. - LastGeneratedCommit: "randomCommit", - APIs: []*legacyconfig.API{ - { - Path: "path/to", - }, - }, - }, - }, - }, - sourceRepo: &MockRepository{ - RemotesValue: []*legacygitrepo.Remote{{Name: "origin", URLs: []string{"https://github.com/owner/repo.git"}}}, - GetCommitError: errors.New("simulated get commit error"), - GetCommitsForPathsSinceLastGenByCommit: map[string][]*legacygitrepo.Commit{ - "1234567890": { - { - Message: "feat: new feature\n\nThis is body.\n\nPiperOrigin-RevId: 98765", - Hash: hash1, - When: today, - }, - { - Message: "fix: a bug fix\n\nThis is another body.\n\nPiperOrigin-RevId: 573342", - Hash: hash2, - When: today.Add(time.Hour), - }, - }, - }, - ChangedFilesInCommitValueByHash: map[string][]string{ - hash1.String(): { - "path/to/file", - "path/to/another/file", - }, - hash2.String(): { - "path/to/file", - }, - }, - }, - languageRepo: &MockRepository{ - IsCleanValue: true, - HeadHashValue: "5678", - ChangedFilesInCommitValue: []string{"path/to/a.go"}, - }, - idToCommits: map[string]string{ - "one-library": "1234567890", - }, - wantErr: true, - wantErrPhrase: "failed to find the start commit", - }, - { - name: "no conventional commits since last generation", - state: &legacyconfig.LibrarianState{ - Image: "go:1.21", - Libraries: []*legacyconfig.LibraryState{{ID: "one-library", SourceRoots: []string{"path/to"}}}, - }, - sourceRepo: &MockRepository{}, - languageRepo: &MockRepository{ - HeadHashValue: "5678", - ChangedFilesInCommitValue: []string{"path/to/a.go"}, - }, - idToCommits: map[string]string{ - "one-library": "", - }, - want: "No commit is found since last generation", - }, - { - name: "failed to get language repo changes commits", - state: &legacyconfig.LibrarianState{ - Image: "go:1.21", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "one-library", - SourceRoots: []string{"path/to"}, - }, - }, - }, - sourceRepo: &MockRepository{}, - languageRepo: &MockRepository{ - IsCleanError: errors.New("simulated error"), - }, - idToCommits: map[string]string{ - "one-library": "1234567890", - }, - wantErr: true, - wantErrPhrase: "failed to fetch changes in language repo", - }, - { - name: "failed to get conventional commits", - state: &legacyconfig.LibrarianState{ - Image: "go:1.21", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "one-library", - SourceRoots: []string{"path/to"}, - }, - }, - }, - sourceRepo: &MockRepository{ - GetCommitsForPathsSinceLastGenError: errors.New("simulated error"), - }, - languageRepo: &MockRepository{ - IsCleanValue: true, - HeadHashValue: "5678", - ChangedFilesInCommitValue: []string{"path/to/a.go"}, - }, - idToCommits: map[string]string{ - "one-library": "1234567890", - }, - wantErr: true, - wantErrPhrase: "failed to fetch conventional commits for library", - }, - } { - t.Run(test.name, func(t *testing.T) { - req := &generationPRRequest{ - sourceRepo: test.sourceRepo, - languageRepo: test.languageRepo, - state: test.state, - idToCommits: test.idToCommits, - failedLibraries: test.failedLibraries, - } - got, err := formatGenerationPRBody(req) - if test.wantErr { - if err == nil { - t.Fatalf("%s should return error", test.name) - } - if !strings.Contains(err.Error(), test.wantErrPhrase) { - t.Errorf("formatGenerationPRBody() returned error %q, want to contain %q", err.Error(), test.wantErrPhrase) - } - return - } - if err != nil { - t.Fatal(err) - } - if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("formatGenerationPRBody() mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestFormatOnboardPRBody(t *testing.T) { - t.Parallel() - librarianVersion := legacycli.Version() - - for _, test := range []struct { - name string - state *legacyconfig.LibrarianState - sourceRepo legacygitrepo.Repository - api string - library string - want string - wantErr bool - wantErrPhrase string - }{ - { - name: "onboarding_new_api", - state: &legacyconfig.LibrarianState{ - Image: "go:1.21", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "one-library", - SourceRoots: []string{"path/to"}, - APIs: []*legacyconfig.API{ - { - Path: "path/to", - ServiceConfig: "library_v1.yaml", - }, - }, - }, - }, - }, - sourceRepo: &MockRepository{ - GetLatestCommitByPath: map[string]*legacygitrepo.Commit{ - "path/to/library_v1.yaml": { - Message: "feat: new feature\n\nThis is body.\n\nPiperOrigin-RevId: 98765", - }, - }, - }, - api: "path/to", - library: "one-library", - want: fmt.Sprintf(`PR created by the Librarian CLI to onboard a new Cloud Client Library. - -BEGIN_COMMIT - -feat: onboard a new library - -PiperOrigin-RevId: 98765 -Library-IDs: one-library - -END_COMMIT - -Librarian Version: %s -Language Image: %s`, - librarianVersion, "go:1.21"), - }, - { - name: "no_latest_commit_during_api_onboarding", - state: &legacyconfig.LibrarianState{ - Image: "go:1.21", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "one-library", - SourceRoots: []string{"path/to"}, - APIs: []*legacyconfig.API{ - { - Path: "path/to", - ServiceConfig: "library_v1.yaml", - }, - }, - }, - }, - }, - sourceRepo: &MockRepository{ - GetLatestCommitError: errors.New("no latest commit"), - }, - api: "path/to", - library: "one-library", - wantErr: true, - wantErrPhrase: "no latest commit", - }, - { - name: "latest_commit_does_not_contain_piper_during_api_onboarding", - state: &legacyconfig.LibrarianState{ - Image: "go:1.21", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "one-library", - SourceRoots: []string{"path/to"}, - APIs: []*legacyconfig.API{ - { - Path: "path/to", - ServiceConfig: "library_v1.yaml", - }, - }, - }, - }, - }, - sourceRepo: &MockRepository{ - GetLatestCommitByPath: map[string]*legacygitrepo.Commit{ - "path/to/library_v1.yaml": { - Message: "feat: new feature\n\nThis is body.", - }, - }, - }, - api: "path/to", - library: "one-library", - wantErr: true, - wantErrPhrase: errPiperNotFound.Error(), - }, - { - name: "library_not_found_in_state", - state: &legacyconfig.LibrarianState{ - Image: "go:1.21", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "one-library", - }, - }, - }, - library: "another-library", - wantErr: true, - wantErrPhrase: "library another-library not found", - }, - } { - t.Run(test.name, func(t *testing.T) { - req := &onboardPRRequest{ - sourceRepo: test.sourceRepo, - state: test.state, - api: test.api, - library: test.library, - } - got, err := formatOnboardPRBody(req) - if test.wantErr { - if err == nil { - t.Fatalf("%s should return error", test.name) - } - if !strings.Contains(err.Error(), test.wantErrPhrase) { - t.Errorf("formatOnboardPRBody() returned error %q, want to contain %q", err.Error(), test.wantErrPhrase) - } - return - } - if err != nil { - t.Fatal(err) - } - if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("formatOnboardPRBody() mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestFindLatestCommit(t *testing.T) { - t.Parallel() - - today := time.Now() - hash1 := plumbing.NewHash("1234567890abcdef") - hash2 := plumbing.NewHash("fedcba0987654321") - hash3 := plumbing.NewHash("ghfgsfgshfsdf232") - for _, test := range []struct { - name string - state *legacyconfig.LibrarianState - repo legacygitrepo.Repository - idToCommits map[string]string - want *legacygitrepo.Commit - wantErr bool - wantErrPhrase string - }{ - { - name: "find the last generated commit", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "one-library", - }, - { - ID: "another-library", - }, - { - ID: "yet-another-library", - }, - { - ID: "skipped-library", - }, - }, - }, - repo: &MockRepository{ - GetCommitByHash: map[string]*legacygitrepo.Commit{ - hash1.String(): { - Hash: hash1, - Message: "this is a message", - When: today.Add(time.Hour), - }, - hash2.String(): { - Hash: hash2, - Message: "this is another message", - When: today.Add(2 * time.Hour).Add(time.Minute), - }, - hash3.String(): { - Hash: hash3, - Message: "yet another message", - When: today.Add(2 * time.Hour), - }, - }, - }, - idToCommits: map[string]string{ - "one-library": hash1.String(), - "another-library": hash2.String(), - "yet-another-library": hash3.String(), - }, - want: &legacygitrepo.Commit{ - Hash: hash2, - Message: "this is another message", - When: today.Add(2 * time.Hour).Add(time.Minute), - }, - }, - { - name: "failed to find last generated commit", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "one-library", - }, - }, - }, - repo: &MockRepository{ - GetCommitError: errors.New("simulated error"), - }, - idToCommits: map[string]string{ - "one-library": "1234567890", - }, - wantErr: true, - wantErrPhrase: "can't find last generated commit for", - }, - } { - t.Run(test.name, func(t *testing.T) { - got, err := findLatestGenerationCommit(test.repo, test.state, test.idToCommits) - if test.wantErr { - if err == nil { - t.Fatalf("%s should return error", test.name) - } - if !strings.Contains(err.Error(), test.wantErrPhrase) { - t.Errorf("findLatestCommit() returned error %q, want to contain %q", err.Error(), test.wantErrPhrase) - } - return - } - if err != nil { - t.Fatal(err) - } - if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("findLatestCommit() mismatch (-want +got):\n%s", diff) - } - }) - } -} -func TestGroupByPiperID(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - commits []*legacygitrepo.ConventionalCommit - want []*legacygitrepo.ConventionalCommit - }{ - { - name: "group_commits_with_same_piper_id_and_subject", - commits: []*legacygitrepo.ConventionalCommit{ - { - LibraryID: "library-1", - Subject: "one subject", - Footers: map[string]string{ - "PiperOrigin-RevId": "123456", - }, - }, - { - LibraryID: "library-2", - Subject: "a different subject", - Footers: map[string]string{ - "PiperOrigin-RevId": "123456", - }, - }, - { - LibraryID: "library-3", - Subject: "the same subject", - Footers: map[string]string{ - "PiperOrigin-RevId": "987654", - }, - }, - { - LibraryID: "library-4", - Subject: "the same subject", - Footers: map[string]string{ - "PiperOrigin-RevId": "987654", - }, - }, - { - LibraryID: "library-5", - }, - { - LibraryID: "library-6", - Footers: map[string]string{ - "random-key": "random-value", - }, - }, - }, - want: []*legacygitrepo.ConventionalCommit{ - { - LibraryID: "library-1", - Subject: "one subject", - Footers: map[string]string{ - "PiperOrigin-RevId": "123456", - "Library-IDs": "library-1", - }, - }, - { - LibraryID: "library-2", - Subject: "a different subject", - Footers: map[string]string{ - "PiperOrigin-RevId": "123456", - "Library-IDs": "library-2", - }, - }, - { - LibraryID: "library-3", - Subject: "the same subject", - Footers: map[string]string{ - "PiperOrigin-RevId": "987654", - "Library-IDs": "library-3,library-4", - }, - }, - { - LibraryID: "library-5", - Footers: map[string]string{ - "Library-IDs": "library-5", - }, - }, - { - LibraryID: "library-6", - Footers: map[string]string{ - "random-key": "random-value", - "Library-IDs": "library-6", - }, - }, - }, - }, - } { - t.Run(test.name, func(t *testing.T) { - got := groupByIDAndSubject(test.commits) - // We don't care the order in the slice but sorting makes the test deterministic. - opts := cmpopts.SortSlices(func(a, b *legacygitrepo.ConventionalCommit) bool { - return a.LibraryID < b.LibraryID - }) - if diff := cmp.Diff(test.want, got, opts); diff != "" { - t.Errorf("groupByIDAndSubject() mismatch (-want +got):\n%s", diff) - } - }) - } -} diff --git a/internal/legacylibrarian/legacylibrarian/generate_test.go b/internal/legacylibrarian/legacylibrarian/generate_test.go deleted file mode 100644 index fa69e1e4a47..00000000000 --- a/internal/legacylibrarian/legacylibrarian/generate_test.go +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacylibrarian - -import ( - "testing" - - "github.com/google/go-cmp/cmp" - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" - "github.com/googleapis/librarian/internal/legacylibrarian/legacygitrepo" -) - -func TestGenerateSingleLibrary(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - api string - repo legacygitrepo.Repository - state *legacyconfig.LibrarianState - container *mockContainerClient - ghClient GitHubClient - wantLibraryID string - wantErr bool - wantGenerateCalls int - }{ - { - name: "works", - api: "some/api", - repo: newTestGitRepo(t), - ghClient: &mockGitHubClient{}, - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "some-library", - APIs: []*legacyconfig.API{{Path: "some/api"}}, - }, - }, - }, - container: &mockContainerClient{}, - wantLibraryID: "some-library", - wantGenerateCalls: 1, - }, - { - name: "works with no response", - api: "some/api", - repo: newTestGitRepo(t), - ghClient: &mockGitHubClient{}, - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "some-library", - APIs: []*legacyconfig.API{{Path: "some/api"}}, - }, - }, - }, - container: &mockContainerClient{ - noGenerateResponse: true, - }, - wantLibraryID: "some-library", - wantGenerateCalls: 1, - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - - outputDir := t.TempDir() - libraryID := "some-library" - libraryState := test.state.LibraryByID(libraryID) - err := generateSingleLibrary(t.Context(), test.container, test.state, libraryState, newTestGitRepo(t), test.repo, "test-image", outputDir) - if (err != nil) != test.wantErr { - t.Errorf("generateSingleLibrary() error = %v, wantErr %v", err, test.wantErr) - return - } - if diff := cmp.Diff(test.wantGenerateCalls, test.container.generateCalls); diff != "" { - t.Errorf("runGenerateCommand() generateCalls mismatch (-want +got):%s", diff) - } - }) - } -} - -func TestGetSafeDirectoryName(t *testing.T) { - for _, test := range []struct { - name string - id string - want string - }{ - { - name: "simple", - id: "pubsub", - want: "pubsub", - }, - { - name: "nested", - id: "pubsub/v2", - want: "pubsub-slash-v2", - }, - { - name: "deeply nested", - id: "compute/metadata/v2", - want: "compute-slash-metadata-slash-v2", - }, - } { - t.Run(test.name, func(t *testing.T) { - got := getSafeDirectoryName(test.id) - if test.want != got { - t.Errorf("getSafeDirectoryName() = %q; want %q", got, test.want) - } - }) - } -} diff --git a/internal/legacylibrarian/legacylibrarian/help.go b/internal/legacylibrarian/legacylibrarian/help.go deleted file mode 100644 index e97e103c572..00000000000 --- a/internal/legacylibrarian/legacylibrarian/help.go +++ /dev/null @@ -1,151 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacylibrarian - -const ( - librarianLongHelp = "Librarian manages client libraries for Google APIs." - - releaseLongHelp = "Manages releases of libraries." - - generateLongHelp = `The generate command is the primary tool for all code generation -tasks. It handles both the initial setup of a new library (onboarding) and the -regeneration of existing ones. Librarian works by delegating language-specific -tasks to a container, which is configured in the .librarian/state.yaml file. -Librarian is environment aware and will check if the current directory is the -root of a librarian repository. If you are not executing in such a directory the -'-repo' flag must be provided. - -# Onboarding a new library - -To configure and generate a new library for the first time, you must specify the -API to be generated and the library it will belong to. Librarian will invoke the -'configure' command in the language container to set up the repository, add the -new library's configuration to the '.librarian/state.yaml' file, and then -proceed with generation. - -Example: - legacylibrarian generate -library=secretmanager -api=google/cloud/secretmanager/v1 - -# Regenerating existing libraries - -You can regenerate a single, existing library by specifying either the library -ID or the API path. If no specific library or API is provided, Librarian will -regenerate all libraries listed in '.librarian/state.yaml'. If '-library' or -'-api' is specified the whole library will be regenerated. - -Examples: - # Regenerate a single library by its ID - legacylibrarian generate -library=secretmanager - - # Regenerate a single library by its API path - legacylibrarian generate -api=google/cloud/secretmanager/v1 - - # Regenerate all libraries in the repository - legacylibrarian generate - -# Workflow and Options: - -The generation process involves delegating to the language container's -'generate' command. After the code is generated, the tool cleans the destination -directories and copies the new files into place, according to the configuration -in '.librarian/state.yaml'. - -- If the '-build' flag is specified, the 'build' command is also executed in - the container to compile and validate the generated code. -- If the '-push' flag is provided, the changes are committed to a new branch, - and a pull request is created on GitHub. Otherwise, the changes are left in - your local working directory for inspection. When pushing to a remote branch, - you have the option of using HTTPS or SSH. Librarian will automatically determine - whether to use HTTPS or SSH based on the remote URI. - -Example with build and push: - LIBRARIAN_GITHUB_TOKEN=xxx legacylibrarian generate -push -build` - - releaseStageLongHelp = `The 'release stage' command is the primary entry point for staging -a new release. It automates the creation of a release pull request by parsing -conventional commits, determining the next semantic version for each library, -and generating a changelog. Librarian is environment aware and will check if the -current directory is the root of a librarian repository. If you are not -executing in such a directory the '-repo' flag must be provided. - -This command scans the git history since the last release, identifies changes -(feat, fix, BREAKING CHANGE), and calculates the appropriate version bump -according to semver rules. It then delegates all language-specific file -modifications, such as updating a CHANGELOG.md or bumping the version in a pom.xml, -to the configured language-specific container. - -If a specific library is configured for release via the '-library' flag, a single -releasable change is needed to automatically calculate a version bump. If there are -no releasable changes since the last release, the '-version' flag should be included -to set a new version for the library. The new version must be "SemVer" greater than the -current version. - -By default, 'release stage' leaves the changes in your local working directory -for inspection. Use the '-push' flag to automatically commit the changes to -a new branch and create a pull request on GitHub. The '-commit' flag may be -used to create a local commit without creating a pull request; this flag is -ignored if '-push' is also specified. When pushing to a remote branch, -you have the option of using HTTPS or SSH. Librarian will automatically determine -whether to use HTTPS or SSH based on the remote URI. - -Examples: - # Create a release PR for all libraries with pending changes. - legacylibrarian release stage -push - - # Create a release PR for a single library. - legacylibrarian release stage -library=secretmanager -push - - # Manually specify a version for a single library, overriding the calculation. - legacylibrarian release stage -library=secretmanager -library-version=2.0.0 -push` - - tagLongHelp = `The 'tag' command is the final step in the release -process. It is designed to be run after a release pull request, created by -'release stage', has been merged. - -This command's primary responsibilities are to: - -- Create a Git tag for each library version included in the merged pull request. -- Create a corresponding GitHub Release for each tag, using the release notes - from the pull request body. -- Update the pull request's label from 'release:pending' to 'release:done' to - mark the process as complete. - -You can target a specific merged pull request using the '-pr' flag. If no pull -request is specified, the command will automatically search for and process all -merged pull requests with the 'release:pending' label from the last 30 days. - -Examples: - # Tag and create a GitHub release for a specific merged PR. - legacylibrarian release tag -repo=https://github.com/googleapis/google-cloud-go -pr=https://github.com/googleapis/google-cloud-go/pull/123 - - # Find and process all pending merged release PRs in a repository. - legacylibrarian release tag -repo=https://github.com/googleapis/google-cloud-go` - - updateImageLongHelp = `The 'update-image' command is used to update the 'image' SHA -of the language container for a language repository. - -This command's primary responsibilities are to: - -- Update the 'image' field in '.librarian/state.yaml' -- Regenerate each library with the new language container using googleapis' - proto definitions at the 'last_generated_commit' - -Examples: - # Create a PR that updates the language container to latest image. - legacylibrarian update-image -commit -push - - # Create a PR that updates the language container to the specified image. - legacylibrarian update-image -commit -push -image=` -) diff --git a/internal/legacylibrarian/legacylibrarian/librarian.go b/internal/legacylibrarian/legacylibrarian/librarian.go deleted file mode 100644 index b16c8ec15b8..00000000000 --- a/internal/legacylibrarian/legacylibrarian/librarian.go +++ /dev/null @@ -1,213 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package legacylibrarian provides the core implementation for the Librarian CLI tool. -package legacylibrarian - -import ( - "context" - "fmt" - "log/slog" - "os" - - "github.com/googleapis/librarian/internal/legacylibrarian/legacycli" -) - -// Run executes the Librarian CLI with the given command line arguments. -func Run(ctx context.Context, arg ...string) error { - cmd := newLibrarianCommand() - return cmd.Run(ctx, arg) -} - -func setupLogger(verbose bool) { - level := slog.LevelInfo - if verbose { - level = slog.LevelDebug - } - opts := &slog.HandlerOptions{Level: level} - handler := slog.NewTextHandler(os.Stderr, opts) - slog.SetDefault(slog.New(handler)) -} - -func newLibrarianCommand() *legacycli.Command { - commands := []*legacycli.Command{ - newCmdGenerate(), - newCmdRelease(), - newCmdUpdateImage(), - } - - return legacycli.NewCommandSet( - commands, - "legacylibrarian manages client libraries for Google APIs", - "legacylibrarian [arguments]", - librarianLongHelp) -} - -func newCmdGenerate() *legacycli.Command { - var verbose bool - cmdGenerate := &legacycli.Command{ - Short: "generate onboards and generates client library code", - UsageLine: "legacylibrarian generate [flags]", - Long: generateLongHelp, - Action: func(ctx context.Context, cmd *legacycli.Command) error { - setupLogger(verbose) - slog.Debug("generate command verbose logging") - if err := cmd.Config.SetDefaults(); err != nil { - return fmt.Errorf("failed to initialize config: %w", err) - } - if _, err := cmd.Config.IsValid(); err != nil { - return fmt.Errorf("failed to validate config: %s", err) - } - runner, err := newGenerateRunner(cmd.Config) - if err != nil { - return err - } - return runner.run(ctx) - }, - } - cmdGenerate.Init() - addFlagAPI(cmdGenerate.Flags, cmdGenerate.Config) - addFlagAPISource(cmdGenerate.Flags, cmdGenerate.Config) - addFlagAPISourceBranch(cmdGenerate.Flags, cmdGenerate.Config) - addFlagBuild(cmdGenerate.Flags, cmdGenerate.Config) - addFlagGenerateUnchanged(cmdGenerate.Flags, cmdGenerate.Config) - addFlagHostMount(cmdGenerate.Flags, cmdGenerate.Config) - addFlagImage(cmdGenerate.Flags, cmdGenerate.Config) - addFlagLibrary(cmdGenerate.Flags, cmdGenerate.Config) - addFlagRepo(cmdGenerate.Flags, cmdGenerate.Config) - addFlagBranch(cmdGenerate.Flags, cmdGenerate.Config) - addFlagWorkRoot(cmdGenerate.Flags, cmdGenerate.Config) - addFlagPush(cmdGenerate.Flags, cmdGenerate.Config) - addFlagVerbose(cmdGenerate.Flags, &verbose) - return cmdGenerate -} - -func newCmdRelease() *legacycli.Command { - cmdRelease := &legacycli.Command{ - Short: "release manages releases of libraries.", - UsageLine: "legacylibrarian release [arguments]", - Long: releaseLongHelp, - Commands: []*legacycli.Command{ - newCmdStage(), - newCmdTag(), - }, - } - cmdRelease.Init() - return cmdRelease -} - -func newCmdTag() *legacycli.Command { - var verbose bool - cmdTag := &legacycli.Command{ - Short: "tag tags and creates a GitHub release for a merged pull request.", - UsageLine: "legacylibrarian release tag [arguments]", - Long: tagLongHelp, - Action: func(ctx context.Context, cmd *legacycli.Command) error { - setupLogger(verbose) - slog.Debug("tag command verbose logging") - if err := cmd.Config.SetDefaults(); err != nil { - return fmt.Errorf("failed to initialize config: %w", err) - } - if _, err := cmd.Config.IsValid(); err != nil { - return fmt.Errorf("failed to validate config: %s", err) - } - runner, err := newTagRunner(cmd.Config) - if err != nil { - return err - } - return runner.run(ctx) - }, - } - cmdTag.Init() - addFlagRepo(cmdTag.Flags, cmdTag.Config) - addFlagPR(cmdTag.Flags, cmdTag.Config) - addFlagGitHubAPIEndpoint(cmdTag.Flags, cmdTag.Config) - addFlagVerbose(cmdTag.Flags, &verbose) - return cmdTag -} - -func newCmdStage() *legacycli.Command { - var verbose bool - cmdStage := &legacycli.Command{ - Short: "stage stages a release by creating a release pull request.", - UsageLine: "legacylibrarian release stage [flags]", - Long: releaseStageLongHelp, - Action: func(ctx context.Context, cmd *legacycli.Command) error { - setupLogger(verbose) - slog.Debug("stage command verbose logging") - if err := cmd.Config.SetDefaults(); err != nil { - return fmt.Errorf("failed to initialize config: %w", err) - } - if _, err := cmd.Config.IsValid(); err != nil { - return fmt.Errorf("failed to validate config: %s", err) - } - runner, err := newStageRunner(cmd.Config) - if err != nil { - return err - } - return runner.run(ctx) - }, - } - cmdStage.Init() - addFlagCommit(cmdStage.Flags, cmdStage.Config) - addFlagPush(cmdStage.Flags, cmdStage.Config) - addFlagImage(cmdStage.Flags, cmdStage.Config) - addFlagLibrary(cmdStage.Flags, cmdStage.Config) - addFlagLibraryVersion(cmdStage.Flags, cmdStage.Config) - addFlagRepo(cmdStage.Flags, cmdStage.Config) - addFlagBranch(cmdStage.Flags, cmdStage.Config) - addFlagWorkRoot(cmdStage.Flags, cmdStage.Config) - addFlagVerbose(cmdStage.Flags, &verbose) - return cmdStage -} - -func newCmdUpdateImage() *legacycli.Command { - var verbose bool - cmdUpdateImage := &legacycli.Command{ - Short: "update-image updates configured language image container", - UsageLine: "legacylibrarian update-image [flags]", - Long: updateImageLongHelp, - Action: func(ctx context.Context, cmd *legacycli.Command) error { - setupLogger(verbose) - slog.Debug("update image command verbose logging") - if err := cmd.Config.SetDefaults(); err != nil { - return fmt.Errorf("failed to initialize config: %w", err) - } - if _, err := cmd.Config.IsValid(); err != nil { - return fmt.Errorf("failed to validate config: %s", err) - } - runner, err := newUpdateImageRunner(cmd.Config) - if err != nil { - return err - } - return runner.run(ctx) - }, - } - cmdUpdateImage.Init() - addFlagAPISource(cmdUpdateImage.Flags, cmdUpdateImage.Config) - addFlagAPISourceBranch(cmdUpdateImage.Flags, cmdUpdateImage.Config) - addFlagBuild(cmdUpdateImage.Flags, cmdUpdateImage.Config) - addFlagCommit(cmdUpdateImage.Flags, cmdUpdateImage.Config) - addFlagHostMount(cmdUpdateImage.Flags, cmdUpdateImage.Config) - addFlagImage(cmdUpdateImage.Flags, cmdUpdateImage.Config) - addFlagRepo(cmdUpdateImage.Flags, cmdUpdateImage.Config) - addFlagBranch(cmdUpdateImage.Flags, cmdUpdateImage.Config) - addFlagWorkRoot(cmdUpdateImage.Flags, cmdUpdateImage.Config) - addFlagPush(cmdUpdateImage.Flags, cmdUpdateImage.Config) - addFlagTest(cmdUpdateImage.Flags, cmdUpdateImage.Config) - addFlagLibraryToTest(cmdUpdateImage.Flags, cmdUpdateImage.Config) - addFlagCheckUnexpectedChanges(cmdUpdateImage.Flags, cmdUpdateImage.Config) - addFlagVerbose(cmdUpdateImage.Flags, &verbose) - return cmdUpdateImage -} diff --git a/internal/legacylibrarian/legacylibrarian/librarian_test.go b/internal/legacylibrarian/legacylibrarian/librarian_test.go deleted file mode 100644 index c9ad866e3ae..00000000000 --- a/internal/legacylibrarian/legacylibrarian/librarian_test.go +++ /dev/null @@ -1,386 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacylibrarian - -import ( - "bytes" - "fmt" - "io" - "log/slog" - "math/rand" - "os" - "os/exec" - "path/filepath" - "strings" - "testing" - - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing/object" - - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" - "github.com/googleapis/librarian/internal/legacylibrarian/legacygitrepo" - "gopkg.in/yaml.v3" - - "github.com/google/go-cmp/cmp" -) - -func TestRun(t *testing.T) { - if err := Run(t.Context(), []string{"version"}...); err != nil { - t.Fatal(err) - } -} - -func TestVerboseFlag(t *testing.T) { - for _, test := range []struct { - name string - args []string - expectDebugLog bool - expectDebugSubstr string - }{ - { - name: "generate with -v flag", - args: []string{"generate", "-v"}, - expectDebugLog: true, - expectDebugSubstr: "generate command verbose logging", - }, - { - name: "generate without -v flag", - args: []string{"generate"}, - expectDebugLog: false, - }, - { - name: "release stage with -v flag", - args: []string{"release", "stage", "-v"}, - expectDebugLog: true, - expectDebugSubstr: "stage command verbose logging", - }, - { - name: "release stage without -v flag", - args: []string{"release", "stage"}, - expectDebugLog: false, - }, - { - name: "release tag with -v flag", - args: []string{"release", "tag", "-v"}, - expectDebugLog: true, - expectDebugSubstr: "tag command verbose logging", - }, - { - name: "release tag without -v flag", - args: []string{"release", "tag"}, - expectDebugLog: false, - }, - } { - t.Run(test.name, func(t *testing.T) { - // Redirect stderr to capture logs. - oldStderr := os.Stderr - r, w, _ := os.Pipe() - os.Stderr = w - // Reset logger to default for test isolation. - slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, nil))) - - _ = Run(t.Context(), test.args...) - - // Restore stderr and read the output. - w.Close() - os.Stderr = oldStderr - var buf bytes.Buffer - io.Copy(&buf, r) - output := buf.String() - - hasDebugLog := strings.Contains(output, "level=DEBUG") - if test.expectDebugLog { - if !hasDebugLog { - t.Errorf("expected debug log to be present, but it wasn't. Output:\n%s", output) - } - if !strings.Contains(output, test.expectDebugSubstr) { - t.Errorf("expected debug log to contain %q, but it didn't. Output:\n%s", test.expectDebugSubstr, output) - } - } else { - if hasDebugLog { - t.Errorf("expected debug log to be absent, but it was present. Output:\n%s", output) - } - } - }) - } -} - -func TestGenerate_DefaultBehavior(t *testing.T) { - ctx := t.Context() - - // 1. Set up a mock repository with a state file - repo := newTestGitRepo(t) - repoDir := repo.GetDir() - - // Set up a dummy API Source repo to prevent cloning googleapis/googleapis - apiSourceDir := t.TempDir() - runGit(t, apiSourceDir, "init") - runGit(t, apiSourceDir, "config", "user.email", "test@example.com") - runGit(t, apiSourceDir, "config", "user.name", "Test User") - runGit(t, apiSourceDir, "commit", "--allow-empty", "-m", "initial commit") - - t.Chdir(repoDir) - - // 2. Override dependency creation to use mocks - mockContainer := &mockContainerClient{ - wantLibraryGen: true, - } - mockGH := &mockGitHubClient{} - - // 3. Call librarian.Run - cfg := legacyconfig.New("generate") - cfg.WorkRoot = repoDir - cfg.Repo = repoDir - cfg.APISource = apiSourceDir - runner, err := newGenerateRunner(cfg) - if err != nil { - t.Fatalf("newGenerateRunner() failed: %v", err) - } - - runner.ghClient = mockGH - runner.containerClient = mockContainer - if err := runner.run(ctx); err != nil { - t.Fatalf("runner.run() failed: %v", err) - } - - // 4. Assertions - expectedGenerateCalls := 1 - if mockContainer.generateCalls != expectedGenerateCalls { - t.Errorf("Run(ctx, \"generate\"): got %d generate calls, want %d", mockContainer.generateCalls, expectedGenerateCalls) - } -} - -func TestIsURL(t *testing.T) { - for _, test := range []struct { - name string - input string - want bool - }{ - { - name: "Valid HTTPS URL", - input: "https://github.com/googleapis/librarian-go", - want: true, - }, - { - name: "Valid HTTP URL", - input: "http://example.com/path?query=value", - want: true, - }, - { - name: "Valid FTP URL", - input: "ftp://user:password@host/path", - want: true, - }, - { - name: "URL without scheme", - input: "google.com", - want: false, - }, - { - name: "URL with scheme only", - input: "https://", - want: false, - }, - { - name: "Absolute Unix file path", - input: "/home/user/file", - want: false, - }, - { - name: "Relative file path", - input: "home/user/file", - want: false, - }, - { - name: "Empty string", - input: "", - want: false, - }, - { - name: "Plain string", - input: "just-a-string", - want: false, - }, - } { - t.Run(test.name, func(t *testing.T) { - got := isURL(test.input) - if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("isURL() mismatch (-want +got):\n%s", diff) - } - }) - } -} - -// newTestGitRepo creates a new git repository in a temporary directory. -func newTestGitRepo(t *testing.T) legacygitrepo.Repository { - t.Helper() - defaultState := &legacyconfig.LibrarianState{ - Image: "some/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "some-library", - APIs: []*legacyconfig.API{ - { - Path: "some/api", - ServiceConfig: "api_config.yaml", - Status: legacyconfig.StatusExisting, - }, - }, - SourceRoots: []string{"src/a"}, - }, - }, - } - return newTestGitRepoWithState(t, defaultState) -} - -// newTestGitRepo creates a new git repository in a temporary directory. -func newTestGitRepoWithState(t *testing.T, state *legacyconfig.LibrarianState) legacygitrepo.Repository { - t.Helper() - dir := t.TempDir() - remoteURL := "https://github.com/googleapis/librarian.git" - runGit(t, dir, "init") - runGit(t, dir, "config", "user.email", "test@example.com") - runGit(t, dir, "config", "user.name", "Test User") - if err := os.WriteFile(filepath.Join(dir, "README.md"), []byte("test"), 0644); err != nil { - t.Fatalf("os.WriteFile: %v", err) - } - // If state is nil, skip creating the .librarian directory - // and the state.yaml/config.yaml files and return with a initial commit - if state == nil { - runGit(t, dir, "add", ".") - runGit(t, dir, "commit", "-m", "initial commit") - runGit(t, dir, "remote", "add", "origin", remoteURL) - repo, err := legacygitrepo.NewRepository(&legacygitrepo.RepositoryOptions{Dir: dir}) - if err != nil { - t.Fatalf("legacygitrepo.Open(%q) = %v", dir, err) - } - return repo - } - // Create a state.yaml and config.yaml file in .librarian dir. - librarianDir := filepath.Join(dir, legacyconfig.LibrarianDir) - if err := os.MkdirAll(librarianDir, 0755); err != nil { - t.Fatalf("os.MkdirAll: %v", err) - } - - // Setup each source root directory to be non-empty (one `random_file.txt`) - // that can be used to test preserve or remove regex patterns - for _, library := range state.Libraries { - for _, sourceRoot := range library.SourceRoots { - fullPath := filepath.Join(dir, sourceRoot, "random_file.txt") - if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil { - t.Fatal(err) - } - if _, err := os.Create(fullPath); err != nil { - t.Fatal(err) - } - } - } - - bytes, err := yaml.Marshal(state) - if err != nil { - t.Fatalf("yaml.Marshal() = %v", err) - } - stateFile := filepath.Join(librarianDir, "state.yaml") - if err := os.WriteFile(stateFile, bytes, 0644); err != nil { - t.Fatalf("os.WriteFile: %v", err) - } - configFile := filepath.Join(librarianDir, "config.yaml") - if err := os.WriteFile(configFile, []byte{}, 0644); err != nil { - t.Fatalf("os.WriteFile: %v", err) - } - - runGit(t, dir, "add", ".") - runGit(t, dir, "commit", "-m", "initial commit") - runGit(t, dir, "remote", "add", "origin", remoteURL) - repo, err := legacygitrepo.NewRepository(&legacygitrepo.RepositoryOptions{Dir: dir}) - if err != nil { - t.Fatalf("legacygitrepo.Open(%q) = %v", dir, err) - } - return repo -} - -func runGit(t *testing.T, dir string, args ...string) { - t.Helper() - cmd := exec.Command("git", args...) - cmd.Dir = dir - if err := cmd.Run(); err != nil { - t.Fatalf("git %v: %v", args, err) - } -} - -// setupRepoForGetCommits creates an empty gitrepo and creates some commits and -// tags. -// -// Each commit has a file path and a commit message. -// Note that pathAndMessages should at least have one element. All tags are created -// after the first commit. -func setupRepoForGetCommits(t *testing.T, pathAndMessages []pathAndMessage, tags []string) *legacygitrepo.LocalRepository { - t.Helper() - dir := t.TempDir() - gitRepo, err := git.PlainInit(dir, false) - if err != nil { - t.Fatalf("git.PlainInit failed: %v", err) - } - - createAndCommit := func(path, msg string) { - w, err := gitRepo.Worktree() - if err != nil { - t.Fatalf("Worktree() failed: %v", err) - } - fullPath := filepath.Join(dir, path) - if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil { - t.Fatalf("os.MkdirAll failed: %v", err) - } - content := fmt.Sprintf("content-%d", rand.Intn(10000)) - if err := os.WriteFile(fullPath, []byte(content), 0644); err != nil { - t.Fatalf("os.WriteFile failed: %v", err) - } - if _, err := w.Add(path); err != nil { - t.Fatalf("w.Add failed: %v", err) - } - _, err = w.Commit(msg, &git.CommitOptions{ - Author: &object.Signature{Name: "Test", Email: "test@example.com"}, - }) - if err != nil { - t.Fatalf("w.Commit failed: %v", err) - } - } - - createAndCommit(pathAndMessages[0].path, pathAndMessages[0].message) - head, err := gitRepo.Head() - if err != nil { - t.Fatalf("repo.Head() failed: %v", err) - } - for _, tag := range tags { - if _, err := gitRepo.CreateTag(tag, head.Hash(), nil); err != nil { - t.Fatalf("CreateTag failed: %v", err) - } - } - - for _, pam := range pathAndMessages[1:] { - createAndCommit(pam.path, pam.message) - } - - r, err := legacygitrepo.NewRepository(&legacygitrepo.RepositoryOptions{Dir: dir}) - if err != nil { - t.Fatalf("legacygitrepo.NewRepository failed: %v", err) - } - return r -} - -type pathAndMessage struct { - path string - message string -} diff --git a/internal/legacylibrarian/legacylibrarian/mocks_test.go b/internal/legacylibrarian/legacylibrarian/mocks_test.go deleted file mode 100644 index 482958b6b5b..00000000000 --- a/internal/legacylibrarian/legacylibrarian/mocks_test.go +++ /dev/null @@ -1,591 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacylibrarian - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "os" - "path/filepath" - - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" - "github.com/googleapis/librarian/internal/legacylibrarian/legacydocker" - "github.com/googleapis/librarian/internal/legacylibrarian/legacygithub" - "github.com/googleapis/librarian/internal/legacylibrarian/legacygitrepo" - "gopkg.in/yaml.v3" -) - -// mockGitHubClient is a mock implementation of the GitHubClient interface for testing. -type mockGitHubClient struct { - GitHubClient - rawContent []byte - rawErr error - createPullRequestCalls int - addLabelsToIssuesCalls int - getLabelsCalls int - replaceLabelsCalls int - searchPullRequestsCalls int - getPullRequestCalls int - createReleaseCalls int - createIssueCalls int - createTagCalls int - createPullRequestErr error - addLabelsToIssuesErr error - getLabelsErr error - replaceLabelsErr error - searchPullRequestsErr error - getPullRequestErr error - createReleaseErr error - createIssueErr error - createTagErr error - createdPR *legacygithub.PullRequestMetadata - labels []string - pullRequests []*legacygithub.PullRequest - pullRequest *legacygithub.PullRequest - createdRelease *legacygithub.RepositoryRelease - releaseNames []string - librarianState *legacyconfig.LibrarianState - librarianConfig *legacyconfig.LibrarianConfig - replacedLabels []string -} - -func (m *mockGitHubClient) GetRawContent(ctx context.Context, path, ref string) ([]byte, error) { - if path == ".librarian/state.yaml" && m.librarianState != nil { - return yaml.Marshal(m.librarianState) - } - - if path == ".librarian/config.yaml" && m.librarianConfig != nil { - return yaml.Marshal(m.librarianConfig) - } - return m.rawContent, m.rawErr -} - -func (m *mockGitHubClient) CreatePullRequest(ctx context.Context, repo *legacygithub.Repository, remoteBranch, remoteBase, title, body string, isDraft bool) (*legacygithub.PullRequestMetadata, error) { - m.createPullRequestCalls++ - if m.createPullRequestErr != nil { - return nil, m.createPullRequestErr - } - return m.createdPR, nil -} - -func (m *mockGitHubClient) AddLabelsToIssue(ctx context.Context, repo *legacygithub.Repository, number int, labels []string) error { - m.addLabelsToIssuesCalls++ - m.labels = append(m.labels, labels...) - return m.addLabelsToIssuesErr -} - -func (m *mockGitHubClient) GetLabels(ctx context.Context, number int) ([]string, error) { - m.getLabelsCalls++ - return m.labels, m.getLabelsErr -} - -func (m *mockGitHubClient) ReplaceLabels(ctx context.Context, number int, labels []string) error { - m.replaceLabelsCalls++ - m.replacedLabels = labels - return m.replaceLabelsErr -} - -func (m *mockGitHubClient) SearchPullRequests(ctx context.Context, query string) ([]*legacygithub.PullRequest, error) { - m.searchPullRequestsCalls++ - return m.pullRequests, m.searchPullRequestsErr -} - -func (m *mockGitHubClient) GetPullRequest(ctx context.Context, number int) (*legacygithub.PullRequest, error) { - m.getPullRequestCalls++ - return m.pullRequest, m.getPullRequestErr -} - -func (m *mockGitHubClient) CreateRelease(ctx context.Context, tagName, releaseName, body, commitish string) (*legacygithub.RepositoryRelease, error) { - m.createReleaseCalls++ - m.releaseNames = append(m.releaseNames, releaseName) - return m.createdRelease, m.createReleaseErr -} - -func (m *mockGitHubClient) CreateIssueComment(ctx context.Context, number int, comment string) error { - m.createIssueCalls++ - return m.createIssueErr -} - -func (m *mockGitHubClient) CreateTag(ctx context.Context, tagName, commitish string) error { - m.createTagCalls++ - return m.createTagErr -} - -// mockContainerClient is a mock implementation of the ContainerClient interface for testing. -type mockContainerClient struct { - ContainerClient - generateCalls int - buildCalls int - configureCalls int - stageCalls int - generateErr error - buildErr error - configureErr error - stageErr error - // Set this value if you want an error when - // generate a library with a specific id. - failGenerateForID string - // Set this value if you want an error when - // generate a library with a specific id. - generateErrForID error - // Set this value if you want an error when - // build a library with a specific id. - failBuildForID string - // Set this value if you want an error when - // build a library with a specific id. - buildErrForID error - requestLibraryID string - noBuildResponse bool - noConfigureResponse bool - noGenerateResponse bool - noReleaseResponse bool - noInitVersion bool - wantErrorMsg bool - // Set this value if you want library files - // to be generated in source roots. - wantLibraryGen bool - // Set this value if you want the configure-response - // has library source roots and remove regex. - configureLibraryPaths []string - // The last generation request - generateRequest *legacydocker.GenerateRequest - Image string -} - -func (m *mockContainerClient) Build(ctx context.Context, request *legacydocker.BuildRequest) error { - m.buildCalls++ - if m.noBuildResponse { - return m.buildErr - } - // Write a build-response.json unless we're configured not to. - if err := os.MkdirAll(filepath.Join(request.RepoDir, ".librarian"), 0755); err != nil { - return err - } - - libraryStr := "{}" - if m.wantErrorMsg { - libraryStr = "{error: simulated error message}" - } - if err := os.WriteFile(filepath.Join(request.RepoDir, ".librarian", legacyconfig.BuildResponse), []byte(libraryStr), 0755); err != nil { - return err - } - - if m.failBuildForID != "" { - if request.LibraryID == m.failBuildForID { - return m.buildErrForID - } - } - - return m.buildErr -} - -func (m *mockContainerClient) Configure(ctx context.Context, request *legacydocker.ConfigureRequest) (string, error) { - m.configureCalls++ - - if m.noConfigureResponse { - return "", m.configureErr - } - - // Write a configure-response.json unless we're configured not to. - if err := os.MkdirAll(filepath.Join(request.RepoDir, legacyconfig.LibrarianDir), 0755); err != nil { - return "", err - } - for _, library := range request.State.Libraries { - needConfigure := false - for _, oneApi := range library.APIs { - if oneApi.Status == "new" { - needConfigure = true - } - } - - if !needConfigure { - continue - } - - if !m.noInitVersion { - library.Version = "0.1.0" - } - - // Configure source root and remove regex. - if len(m.configureLibraryPaths) != 0 { - library.SourceRoots = make([]string, len(m.configureLibraryPaths)) - copy(library.SourceRoots, m.configureLibraryPaths) - - library.RemoveRegex = make([]string, len(m.configureLibraryPaths)) - copy(library.RemoveRegex, m.configureLibraryPaths) - } - - if m.wantErrorMsg { - library.ErrorMessage = "simulated error message" - } - - b, err := json.Marshal(library) - if err != nil { - return "", err - } - - if err := os.WriteFile(filepath.Join(request.RepoDir, legacyconfig.LibrarianDir, legacyconfig.ConfigureResponse), b, 0755); err != nil { - return "", err - } - } - - return "", m.configureErr -} - -func (m *mockContainerClient) Generate(ctx context.Context, request *legacydocker.GenerateRequest) error { - m.generateCalls++ - m.generateRequest = request - m.Image = request.Image - - if m.noGenerateResponse { - return m.generateErr - } - - // // Write a generate-response.json unless we're configured not to. - if err := os.MkdirAll(filepath.Join(request.RepoDir, legacyconfig.LibrarianDir), 0755); err != nil { - return err - } - - library := &legacyconfig.LibraryState{} - library.ID = request.LibraryID - if m.wantErrorMsg { - library.ErrorMessage = "simulated error message" - } - b, err := json.MarshalIndent(library, "", " ") - if err != nil { - return err - } - - if err := os.WriteFile(filepath.Join(request.RepoDir, legacyconfig.LibrarianDir, legacyconfig.GenerateResponse), b, 0755); err != nil { - return err - } - - if m.failGenerateForID != "" { - if request.LibraryID == m.failGenerateForID { - return m.generateErrForID - } - } - - m.requestLibraryID = request.LibraryID - if m.wantLibraryGen { - for _, library := range request.State.Libraries { - if request.LibraryID != library.ID { - continue - } - - for _, src := range library.SourceRoots { - srcPath := filepath.Join(request.Output, src) - if err := os.MkdirAll(srcPath, 0755); err != nil { - return err - } - if _, err := os.Create(filepath.Join(srcPath, "example.txt")); err != nil { - return err - } - } - } - } - - return m.generateErr -} - -func (m *mockContainerClient) ReleaseStage(ctx context.Context, request *legacydocker.ReleaseStageRequest) error { - m.stageCalls++ - if m.noReleaseResponse { - return m.stageErr - } - // Write a release-init-response.json unless we're configured not to. - if err := os.MkdirAll(filepath.Join(request.RepoDir, ".librarian"), 0755); err != nil { - return err - } - - library := &legacyconfig.LibraryState{} - if m.wantErrorMsg { - library.ErrorMessage = "simulated error message" - } - b, err := json.MarshalIndent(library, "", " ") - if err != nil { - return err - } - if err := os.WriteFile(filepath.Join(request.RepoDir, ".librarian", legacyconfig.ReleaseStageResponse), b, 0755); err != nil { - return err - } - return m.stageErr -} - -type MockRepository struct { - legacygitrepo.Repository - Dir string - IsCleanValue bool - IsCleanError error - AddAllError error - CommitError error - RemotesValue []*legacygitrepo.Remote - RemotesError error - CommitCalls int - ResetHardCalls int - LastCommitMessage string - GetCommitError error - GetLatestCommitError error - GetCommitByHash map[string]*legacygitrepo.Commit - GetCommitsForPathsSinceTagValue []*legacygitrepo.Commit - GetCommitsForPathsSinceTagValueByTag map[string][]*legacygitrepo.Commit - GetCommitsForPathsSinceTagError error - GetCommitsForPathsSinceTagLastTagName string - GetCommitsForPathsSinceLastGenValue []*legacygitrepo.Commit - GetCommitsForPathsSinceLastGenByCommit map[string][]*legacygitrepo.Commit - GetCommitsForPathsSinceLastGenByPath map[string][]*legacygitrepo.Commit - GetLatestCommitByPath map[string]*legacygitrepo.Commit - GetCommitsForPathsSinceLastGenError error - ChangedFilesInCommitValue []string - ChangedFilesInCommitValueByHash map[string][]string - ChangedFilesInCommitError error - ChangedFilesValue []string - ChangedFilesError error - NewAndDeletedFilesValue []string - NewAndDeletedFilesError error - CreateBranchAndCheckoutError error - CheckoutCommitAndCreateBranchError error - PushCalls int - PushError error - RestoreError error - HeadHashValue string - HeadHashError error - CheckoutCalls int - CheckoutError error - ResetHardError error - DeleteLocalBranchesCalls int - DeleteLocalBranchesError error - GetHashForPathError error - // GetHashForPathValue is a map where each key is of the form "commitHash:path", - // and the value is the hash to return. Every requested entry must be populated. - // If the value is "error", an error is returned instead. (This is useful when some - // calls must be successful, and others must fail.) - GetHashForPathValue map[string]string - ResetSoftCalls int - ResetSoftError error -} - -func (m *MockRepository) HeadHash() (string, error) { - if m.HeadHashError != nil { - return "", m.HeadHashError - } - return m.HeadHashValue, nil -} - -func (m *MockRepository) IsClean() (bool, error) { - if m.IsCleanError != nil { - return false, m.IsCleanError - } - return m.IsCleanValue, nil -} - -func (m *MockRepository) AddAll() error { - if m.AddAllError != nil { - return m.AddAllError - } - return nil -} - -func (m *MockRepository) Commit(msg string) error { - m.CommitCalls++ - m.LastCommitMessage = msg - return m.CommitError -} - -func (m *MockRepository) Remotes() ([]*legacygitrepo.Remote, error) { - if m.RemotesError != nil { - return nil, m.RemotesError - } - return m.RemotesValue, nil -} - -func (m *MockRepository) GetDir() string { - return m.Dir -} - -func (m *MockRepository) GetCommit(commitHash string) (*legacygitrepo.Commit, error) { - if m.GetCommitError != nil { - return nil, m.GetCommitError - } - - if m.GetCommitByHash != nil { - if commit, ok := m.GetCommitByHash[commitHash]; ok { - return commit, nil - } - } - - return nil, errors.New("should not reach here") -} - -func (m *MockRepository) GetLatestCommit(path string) (*legacygitrepo.Commit, error) { - if m.GetLatestCommitError != nil { - return nil, m.GetLatestCommitError - } - - if m.GetLatestCommitByPath != nil { - if commit, ok := m.GetLatestCommitByPath[path]; ok { - return commit, nil - } - } - - return nil, errors.New("should not reach here") -} - -func (m *MockRepository) GetCommitsForPathsSinceTag(paths []string, tagName string) ([]*legacygitrepo.Commit, error) { - m.GetCommitsForPathsSinceTagLastTagName = tagName - if m.GetCommitsForPathsSinceTagError != nil { - return nil, m.GetCommitsForPathsSinceTagError - } - if m.GetCommitsForPathsSinceTagValueByTag != nil { - if commits, ok := m.GetCommitsForPathsSinceTagValueByTag[tagName]; ok { - return commits, nil - } - } - return m.GetCommitsForPathsSinceTagValue, nil -} - -func (m *MockRepository) GetCommitsForPathsSinceCommit(paths []string, sinceCommit string) ([]*legacygitrepo.Commit, error) { - if m.GetCommitsForPathsSinceLastGenError != nil { - return nil, m.GetCommitsForPathsSinceLastGenError - } - - if m.GetCommitsForPathsSinceLastGenByCommit != nil { - if commits, ok := m.GetCommitsForPathsSinceLastGenByCommit[sinceCommit]; ok { - return commits, nil - } - } - - if m.GetCommitsForPathsSinceLastGenByPath != nil { - allCommits := make([]*legacygitrepo.Commit, 0) - for _, path := range paths { - if commits, ok := m.GetCommitsForPathsSinceLastGenByPath[path]; ok { - allCommits = append(allCommits, commits...) - } - } - - return allCommits, nil - } - return m.GetCommitsForPathsSinceLastGenValue, nil -} - -func (m *MockRepository) ChangedFilesInCommit(hash string) ([]string, error) { - if m.ChangedFilesInCommitError != nil { - return nil, m.ChangedFilesInCommitError - } - if m.ChangedFilesInCommitValueByHash != nil { - if files, ok := m.ChangedFilesInCommitValueByHash[hash]; ok { - return files, nil - } - } - return m.ChangedFilesInCommitValue, nil -} - -func (m *MockRepository) ChangedFiles() ([]string, error) { - if m.ChangedFilesError != nil { - return nil, m.ChangedFilesError - } - return m.ChangedFilesValue, nil -} - -func (m *MockRepository) NewAndDeletedFiles() ([]string, error) { - if m.NewAndDeletedFilesError != nil { - return nil, m.NewAndDeletedFilesError - } - return m.NewAndDeletedFilesValue, nil -} - -func (m *MockRepository) CreateBranchAndCheckout(name string) error { - if m.CreateBranchAndCheckoutError != nil { - return m.CreateBranchAndCheckoutError - } - return nil -} - -func (m *MockRepository) CheckoutCommitAndCreateBranch(name, commitHash string) error { - if m.CheckoutCommitAndCreateBranchError != nil { - return m.CheckoutCommitAndCreateBranchError - } - return nil -} - -func (m *MockRepository) Push(name string) error { - m.PushCalls++ - if m.PushError != nil { - return m.PushError - } - return nil -} - -func (m *MockRepository) Restore(paths []string) error { - return m.RestoreError -} - -func (m *MockRepository) CleanUntracked(paths []string) error { - return nil -} - -func (m *MockRepository) Checkout(commitHash string) error { - m.CheckoutCalls++ - if m.CheckoutError != nil { - return m.CheckoutError - } - return nil -} - -func (m *MockRepository) ResetHard() error { - m.ResetHardCalls++ - return m.ResetHardError -} - -func (m *MockRepository) DeleteLocalBranches(names []string) error { - m.DeleteLocalBranchesCalls++ - return m.DeleteLocalBranchesError -} - -// mockImagesClient is a mock implementation of the ImageRegistryClient interface for testing. -type mockImagesClient struct { - latestImage string - err error - findLatestCalls int -} - -func (m *mockImagesClient) FindLatest(ctx context.Context, imageName string) (string, error) { - m.findLatestCalls++ - return m.latestImage, m.err -} - -func (m *MockRepository) GetHashForPath(commitHash, path string) (string, error) { - if m.GetHashForPathError != nil { - return "", m.GetHashForPathError - } - if m.GetHashForPathValue != nil { - key := commitHash + ":" + path - if hash, ok := m.GetHashForPathValue[key]; ok { - if hash == "error" { - return "", errors.New("deliberate error from GetHashForPath") - } - return hash, nil - } - - } - return "", fmt.Errorf("should not reach here: GetHashForPath called with unhandled input (commitHash: %q, path: %q)", commitHash, path) -} - -func (m *MockRepository) ResetSoft(commit string) error { - m.ResetSoftCalls++ - return m.ResetSoftError -} diff --git a/internal/legacylibrarian/legacylibrarian/release_notes.go b/internal/legacylibrarian/legacylibrarian/release_notes.go deleted file mode 100644 index f57ae615753..00000000000 --- a/internal/legacylibrarian/legacylibrarian/release_notes.go +++ /dev/null @@ -1,341 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacylibrarian - -import ( - "bytes" - "errors" - "fmt" - "html/template" - "sort" - "strings" - "time" - - "github.com/googleapis/librarian/internal/legacylibrarian/legacycli" - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" - "github.com/googleapis/librarian/internal/legacylibrarian/legacygithub" -) - -var ( - errPiperNotFound = errors.New("piper id not found") - - commitTypeToHeading = map[string]string{ - "feat": "Features", - "fix": "Bug Fixes", - "perf": "Performance Improvements", - "revert": "Reverts", - "docs": "Documentation", - "style": "Styles", - "chore": "Miscellaneous Chores", - "refactor": "Code Refactoring", - "test": "Tests", - "build": "Build System", - "ci": "Continuous Integration", - } - - // commitTypeOrder is the order in which commit types should appear in release notes. - // Only these listed are included in release notes. - commitTypeOrder = []string{ - "feat", - "fix", - "perf", - "revert", - "docs", - } - - shortSHA = func(sha string) string { - if len(sha) < 8 { - return sha - } - return sha[:8] - } - - releaseNotesTemplate = template.Must(template.New("releaseNotes").Funcs(template.FuncMap{ - "shortSHA": shortSHA, - }).Parse(`PR created by the Librarian CLI to initialize a release. Merging this PR will auto trigger a release. - -Librarian Version: {{.LibrarianVersion}} -Language Image: {{.ImageVersion}} -{{ $prInfo := . }} -{{- range .NoteSections -}} -
{{.LibraryID}}: v{{.NewVersion}} - -## [v{{.NewVersion}}]({{"https://github.com/"}}{{$prInfo.RepoOwner}}/{{$prInfo.RepoName}}/compare/{{.PreviousTag}}...{{.NewTag}}) ({{$prInfo.Date}}) -{{ range .CommitSections }} -### {{.Heading}} -{{ range .Commits }} -{{ if not .IsBulkCommit -}} -{{ if .PiperCLNumber -}} -* {{.Subject}} (PiperOrigin-RevId: {{.PiperCLNumber}}) ([{{shortSHA .CommitHash}}]({{"https://github.com/"}}{{$prInfo.RepoOwner}}/{{$prInfo.RepoName}}/commit/{{shortSHA .CommitHash}})) -{{- else -}} -* {{.Subject}} ([{{shortSHA .CommitHash}}]({{"https://github.com/"}}{{$prInfo.RepoOwner}}/{{$prInfo.RepoName}}/commit/{{shortSHA .CommitHash}})) -{{- end }} -{{- end }} -{{ end }} - -{{- end }} -
- - -{{ end }} -{{- if .BulkChanges -}} -
Bulk Changes -{{ range .BulkChanges }} -{{ if .PiperCLNumber -}} -* {{.Type}}: {{.Subject}} (PiperOrigin-RevId: {{.PiperCLNumber}}) ([{{shortSHA .CommitHash}}]({{"https://github.com/"}}{{$prInfo.RepoOwner}}/{{$prInfo.RepoName}}/commit/{{shortSHA .CommitHash}})) - Libraries: {{.LibraryIDs}} -{{- else -}} -* {{.Type}}: {{.Subject}} ([{{shortSHA .CommitHash}}]({{"https://github.com/"}}{{$prInfo.RepoOwner}}/{{$prInfo.RepoName}}/commit/{{shortSHA .CommitHash}})) - Libraries: {{.LibraryIDs}} -{{- end }} -{{- end }} -
-{{ end }} -`)) - - genBodyTemplate = template.Must(template.New("genBody").Funcs(template.FuncMap{ - "shortSHA": shortSHA, - }).Parse(`PR created by the Librarian CLI to generate Cloud Client Libraries code from protos. - -BEGIN_COMMIT -{{ range .Commits }} -BEGIN_NESTED_COMMIT -{{.Type}}: {{.Subject}} -{{.Body}} - -PiperOrigin-RevId: {{index .Footers "PiperOrigin-RevId"}} -Library-IDs: {{index .Footers "Library-IDs"}} -Source-link: [googleapis/googleapis@{{shortSHA .CommitHash}}](https://github.com/googleapis/googleapis/commit/{{shortSHA .CommitHash}}) -END_NESTED_COMMIT -{{ end }} -END_COMMIT - -This pull request is generated with proto changes between -[googleapis/googleapis@{{shortSHA .StartSHA}}](https://github.com/googleapis/googleapis/commit/{{.StartSHA}}) -(exclusive) and -[googleapis/googleapis@{{shortSHA .EndSHA}}](https://github.com/googleapis/googleapis/commit/{{.EndSHA}}) -(inclusive). - -Librarian Version: {{.LibrarianVersion}} -Language Image: {{.ImageVersion}} - -{{- if .FailedLibraries }} - -## Generation failed for -{{- range .FailedLibraries }} -- {{ . }} -{{- end -}} -{{- end }} -`)) - - onboardingBodyTemplate = template.Must(template.New("onboardingBody").Parse(`PR created by the Librarian CLI to onboard a new Cloud Client Library. - -BEGIN_COMMIT - -feat: onboard a new library - -PiperOrigin-RevId: {{.PiperID}} -Library-IDs: {{.LibraryID}} - -END_COMMIT - -Librarian Version: {{.LibrarianVersion}} -Language Image: {{.ImageVersion}} -`)) -) - -type releasePRBody struct { - LibrarianVersion string - ImageVersion string - RepoOwner string - RepoName string - Date string - NoteSections []*releaseNoteSection - BulkChanges []*legacyconfig.Commit -} - -type releaseNoteSection struct { - LibraryID string - PreviousTag string - NewTag string - NewVersion string - CommitSections []*commitSection -} - -type commitSection struct { - Heading string - Commits []*legacyconfig.Commit -} - -// formatReleaseNotes generates the body for a release pull request. -func formatReleaseNotes(state *legacyconfig.LibrarianState, ghRepo *legacygithub.Repository) (string, error) { - librarianVersion := legacycli.Version() - // Separate commits to bulk changes (affects multiple libraries) or library-specific changes because they - // appear in different section in the release notes. - bulkChangesMap, libraryChanges := separateCommits(state) - // Process library specific changes. - var releaseSections []*releaseNoteSection - for _, library := range state.Libraries { - if !library.ReleaseTriggered { - continue - } - // No need to check the existence of the key, library.ID, because a library without library-specific changes - // may appear in the release notes, i.e., in the bulk changes section. - commits := libraryChanges[library.ID] - section := formatLibraryReleaseNotes(library, commits) - releaseSections = append(releaseSections, section) - } - // Process bulk changes - var bulkChanges []*legacyconfig.Commit - for _, commit := range bulkChangesMap { - bulkChanges = append(bulkChanges, commit) - } - sort.Slice(bulkChanges, func(i, j int) bool { - return bulkChanges[i].CommitHash < bulkChanges[j].CommitHash - }) - - data := &releasePRBody{ - LibrarianVersion: librarianVersion, - Date: time.Now().Format("2006-01-02"), - RepoOwner: ghRepo.Owner, - RepoName: ghRepo.Name, - ImageVersion: state.Image, - NoteSections: releaseSections, - BulkChanges: bulkChanges, - } - - var out bytes.Buffer - if err := releaseNotesTemplate.Execute(&out, data); err != nil { - return "", fmt.Errorf("error executing template: %w", err) - } - - return strings.TrimSpace(out.String()), nil -} - -// formatLibraryReleaseNotes generates release notes in Markdown format for a single library. -// It returns the generated release notes and the new version string. -func formatLibraryReleaseNotes(library *legacyconfig.LibraryState, commits []*legacyconfig.Commit) *releaseNoteSection { - // The version should already be updated to the next version. - newVersion := library.Version - tagFormat := legacyconfig.DetermineTagFormat(library.ID, library, nil) - newTag := legacyconfig.FormatTag(tagFormat, library.ID, newVersion) - previousTag := legacyconfig.FormatTag(tagFormat, library.ID, library.PreviousVersion) - - sort.Slice(commits, func(i, j int) bool { - return commits[i].CommitHash < commits[j].CommitHash - }) - commitsByType := make(map[string][]*legacyconfig.Commit) - for _, commit := range commits { - commitsByType[commit.Type] = append(commitsByType[commit.Type], commit) - } - - var sections []*commitSection - // Group commits by type, according to commitTypeOrder, to be used in the release notes. - for _, ct := range commitTypeOrder { - displayName, headingOK := commitTypeToHeading[ct] - typedCommits, commitsOK := commitsByType[ct] - if headingOK && commitsOK { - sections = append(sections, &commitSection{ - Heading: displayName, - Commits: typedCommits, - }) - } - } - - section := &releaseNoteSection{ - LibraryID: library.ID, - NewVersion: newVersion, - PreviousTag: previousTag, - NewTag: newTag, - CommitSections: sections, - } - - return section -} - -// separateCommits analyzes all commits associated with triggered releases in the -// given state and categorizes them into two groups: -// -// 1. Bulk Changes: Commits that affect multiple libraries. This includes: -// - Commits identified by IsBulkCommit() (e.g., librarian generation PRs). -// - Commits that appear in multiple libraries' change sets but are not -// marked as bulk commits (e.g., dependency updates, README changes). -// The Library-IDs for these are concatenated. -// -// 2. Library Changes: Commits that are unique to a single library. -// -// It returns two maps: -// - The first map contains bulk changes, keyed by a composite of commit hash and subject. -// - The second map contains library-specific changes, keyed by LibraryID. -func separateCommits(state *legacyconfig.LibrarianState) (map[string]*legacyconfig.Commit, map[string][]*legacyconfig.Commit) { - maybeBulkChanges := make(map[string][]*legacyconfig.Commit) - for _, library := range state.Libraries { - if !library.ReleaseTriggered { - continue - } - - for _, commit := range library.Changes { - key := commit.CommitHash + commit.Subject - maybeBulkChanges[key] = append(maybeBulkChanges[key], commit) - } - } - - bulkChanges := make(map[string]*legacyconfig.Commit) - libraryChanges := make(map[string][]*legacyconfig.Commit) - for key, commits := range maybeBulkChanges { - // A commit has multiple library IDs in the footer, this should come from librarian generation PR. - // All commits should be identical. - if commits[0].IsBulkCommit() { - bulkChanges[key] = commits[0] - continue - } - // More than ten commits have the same commit subject and sha, this should come from other sources, - // e.g., dependency updates, README updates, etc. - // All commits should be identical except for the library id. - // We assume this type of commits has only one library id in Footers and each id is unique among all - // commits. - if len(commits) >= legacyconfig.BulkChangeThreshold { - bulkChanges[key] = concatenateLibraryIDs(commits) - continue - } - // We assume the rest of commits are library-specific. - for _, commit := range commits { - // Non-bulk commits may have 1 - 9 library IDs. - libraryIDs := strings.Split(commit.LibraryIDs, ",") - for _, libraryID := range libraryIDs { - if libraryID == "" { - continue - } - libraryChanges[libraryID] = append(libraryChanges[libraryID], commit) - } - } - } - - return bulkChanges, libraryChanges -} - -// concatenateLibraryIDs merges the LibraryIDs from a slice of commits into the first commit. -func concatenateLibraryIDs(commits []*legacyconfig.Commit) *legacyconfig.Commit { - var libraryIDs []string - for _, commit := range commits { - libraryIDs = append(libraryIDs, commit.LibraryIDs) - } - - sort.Slice(libraryIDs, func(i, j int) bool { - return libraryIDs[i] < libraryIDs[j] - }) - commits[0].LibraryIDs = strings.Join(libraryIDs, ",") - return commits[0] -} diff --git a/internal/legacylibrarian/legacylibrarian/release_notes_test.go b/internal/legacylibrarian/legacylibrarian/release_notes_test.go deleted file mode 100644 index d289a820362..00000000000 --- a/internal/legacylibrarian/legacylibrarian/release_notes_test.go +++ /dev/null @@ -1,986 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacylibrarian - -import ( - "errors" - "fmt" - "strings" - "testing" - "time" - - "github.com/go-git/go-git/v5/plumbing" - "github.com/google/go-cmp/cmp" - "github.com/googleapis/librarian/internal/legacylibrarian/legacycli" - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" - "github.com/googleapis/librarian/internal/legacylibrarian/legacygithub" - "github.com/googleapis/librarian/internal/legacylibrarian/legacygitrepo" -) - -func TestFormatReleaseNotes(t *testing.T) { - t.Parallel() - - today := time.Now().Format("2006-01-02") - hash1 := plumbing.NewHash("1234567890abcdef") - hash2 := plumbing.NewHash("fedcba0987654321") - hash3 := plumbing.NewHash("abcdefg123456789") - hash4 := plumbing.NewHash("acdef12345678901") - hash5 := plumbing.NewHash("bcdef12345678901") - hash6 := plumbing.NewHash("cdefg12345678901") - librarianVersion := legacycli.Version() - - for _, test := range []struct { - name string - state *legacyconfig.LibrarianState - ghRepo *legacygithub.Repository - wantReleaseNote string - wantErr bool - wantErrPhrase string - }{ - { - name: "single library release", - state: &legacyconfig.LibrarianState{ - Image: "go:1.21", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "my-library", - // this is the NewVersion in the release note. - Version: "1.1.0", - PreviousVersion: "1.0.0", - Changes: []*legacyconfig.Commit{ - { - Type: "feat", - Subject: "new feature", - CommitHash: hash1.String(), - LibraryIDs: "my-library", - }, - { - Type: "fix", - Subject: "a bug fix", - CommitHash: hash2.String(), - LibraryIDs: "my-library", - }, - }, - ReleaseTriggered: true, - }, - }, - }, - ghRepo: &legacygithub.Repository{Owner: "owner", Name: "repo"}, - wantReleaseNote: fmt.Sprintf(`PR created by the Librarian CLI to initialize a release. Merging this PR will auto trigger a release. - -Librarian Version: %s -Language Image: go:1.21 -
my-library: v1.1.0 - -## [v1.1.0](https://github.com/owner/repo/compare/my-library-1.0.0...my-library-1.1.0) (%s) - -### Features - -* new feature ([12345678](https://github.com/owner/repo/commit/12345678)) - -### Bug Fixes - -* a bug fix ([fedcba09](https://github.com/owner/repo/commit/fedcba09)) - -
`, - librarianVersion, today), - }, - { - name: "single library release, with cl num", - state: &legacyconfig.LibrarianState{ - Image: "go:1.21", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "my-library", - // this is the NewVersion in the release note. - Version: "1.1.0", - PreviousVersion: "1.0.0", - Changes: []*legacyconfig.Commit{ - { - Type: "feat", - Subject: "new feature", - CommitHash: hash1.String(), - PiperCLNumber: "123456", - LibraryIDs: "my-library", - }, - { - Type: "fix", - Subject: "a bug fix", - CommitHash: hash2.String(), - PiperCLNumber: "987654", - LibraryIDs: "my-library", - }, - }, - ReleaseTriggered: true, - }, - }, - }, - ghRepo: &legacygithub.Repository{Owner: "owner", Name: "repo"}, - wantReleaseNote: fmt.Sprintf(`PR created by the Librarian CLI to initialize a release. Merging this PR will auto trigger a release. - -Librarian Version: %s -Language Image: go:1.21 -
my-library: v1.1.0 - -## [v1.1.0](https://github.com/owner/repo/compare/my-library-1.0.0...my-library-1.1.0) (%s) - -### Features - -* new feature (PiperOrigin-RevId: 123456) ([12345678](https://github.com/owner/repo/commit/12345678)) - -### Bug Fixes - -* a bug fix (PiperOrigin-RevId: 987654) ([fedcba09](https://github.com/owner/repo/commit/fedcba09)) - -
`, - librarianVersion, today), - }, - { - name: "single_library_with_multiple_features", - state: &legacyconfig.LibrarianState{ - Image: "go:1.21", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "my-library", - // this is the NewVersion in the release note. - Version: "1.1.0", - PreviousVersion: "1.0.0", - Changes: []*legacyconfig.Commit{ - { - Type: "feat", - Subject: "new feature", - CommitHash: hash1.String(), - LibraryIDs: "my-library", - }, - { - Type: "feat", - Subject: "another new feature", - CommitHash: hash2.String(), - LibraryIDs: "my-library", - }, - }, - ReleaseTriggered: true, - }, - }, - }, - ghRepo: &legacygithub.Repository{Owner: "owner", Name: "repo"}, - wantReleaseNote: fmt.Sprintf(`PR created by the Librarian CLI to initialize a release. Merging this PR will auto trigger a release. - -Librarian Version: %s -Language Image: go:1.21 -
my-library: v1.1.0 - -## [v1.1.0](https://github.com/owner/repo/compare/my-library-1.0.0...my-library-1.1.0) (%s) - -### Features - -* new feature ([12345678](https://github.com/owner/repo/commit/12345678)) - -* another new feature ([fedcba09](https://github.com/owner/repo/commit/fedcba09)) - -
`, - librarianVersion, today), - }, - { - name: "multiple library releases", - state: &legacyconfig.LibrarianState{ - Image: "go:1.21", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "lib-a", - // this is the NewVersion in the release note. - Version: "1.1.0", - PreviousVersion: "1.0.0", - ReleaseTriggered: true, - Changes: []*legacyconfig.Commit{ - { - Type: "feat", - Subject: "feature for a", - CommitHash: hash1.String(), - LibraryIDs: "lib-a", - }, - }, - }, - { - ID: "lib-b", - // this is the NewVersion in the release note. - Version: "2.0.1", - PreviousVersion: "2.0.0", - ReleaseTriggered: true, - Changes: []*legacyconfig.Commit{ - { - Type: "fix", - Subject: "fix for b", - CommitHash: hash2.String(), - LibraryIDs: "lib-b", - }, - }, - }, - }, - }, - ghRepo: &legacygithub.Repository{Owner: "owner", Name: "repo"}, - wantReleaseNote: fmt.Sprintf(`PR created by the Librarian CLI to initialize a release. Merging this PR will auto trigger a release. - -Librarian Version: %s -Language Image: go:1.21 -
lib-a: v1.1.0 - -## [v1.1.0](https://github.com/owner/repo/compare/lib-a-1.0.0...lib-a-1.1.0) (%s) - -### Features - -* feature for a ([12345678](https://github.com/owner/repo/commit/12345678)) - -
- - -
lib-b: v2.0.1 - -## [v2.0.1](https://github.com/owner/repo/compare/lib-b-2.0.0...lib-b-2.0.1) (%s) - -### Bug Fixes - -* fix for b ([fedcba09](https://github.com/owner/repo/commit/fedcba09)) - -
`, - librarianVersion, today, today), - }, - { - name: "release with ignored commit types", - state: &legacyconfig.LibrarianState{ - Image: "go:1.21", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "my-library", - // this is the newVersion in the release note. - Version: "1.1.0", - PreviousVersion: "1.0.0", - ReleaseTriggered: true, - Changes: []*legacyconfig.Commit{ - { - Type: "feat", - Subject: "new feature", - CommitHash: hash1.String(), - LibraryIDs: "my-library", - }, - { - Type: "ci", - Subject: "a ci change", - CommitHash: hash2.String(), - LibraryIDs: "my-library", - }, - }, - }, - }, - }, - ghRepo: &legacygithub.Repository{Owner: "owner", Name: "repo"}, - wantReleaseNote: fmt.Sprintf(`PR created by the Librarian CLI to initialize a release. Merging this PR will auto trigger a release. - -Librarian Version: %s -Language Image: go:1.21 -
my-library: v1.1.0 - -## [v1.1.0](https://github.com/owner/repo/compare/my-library-1.0.0...my-library-1.1.0) (%s) - -### Features - -* new feature ([12345678](https://github.com/owner/repo/commit/12345678)) - -
`, - librarianVersion, today), - }, - { - name: "release_with_commit_description_and_body", - state: &legacyconfig.LibrarianState{ - Image: "go:1.21", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "my-library", - // this is the newVersion in the release note. - Version: "1.1.0", - PreviousVersion: "1.0.0", - ReleaseTriggered: true, - Changes: []*legacyconfig.Commit{ - { - Type: "feat", - Subject: "new feature", - Body: "this is the body", - CommitHash: hash1.String(), - LibraryIDs: "my-library", - }, - }, - }, - }, - }, - ghRepo: &legacygithub.Repository{Owner: "owner", Name: "repo"}, - wantReleaseNote: fmt.Sprintf(`PR created by the Librarian CLI to initialize a release. Merging this PR will auto trigger a release. - -Librarian Version: %s -Language Image: go:1.21 -
my-library: v1.1.0 - -## [v1.1.0](https://github.com/owner/repo/compare/my-library-1.0.0...my-library-1.1.0) (%s) - -### Features - -* new feature ([12345678](https://github.com/owner/repo/commit/12345678)) - -
`, - librarianVersion, today), - }, - { - name: "no releases", - state: &legacyconfig.LibrarianState{ - Image: "go:1.21", - Libraries: []*legacyconfig.LibraryState{}, - }, - ghRepo: &legacygithub.Repository{}, - wantReleaseNote: fmt.Sprintf(`PR created by the Librarian CLI to initialize a release. Merging this PR will auto trigger a release. - -Librarian Version: %s -Language Image: go:1.21`, librarianVersion), - }, - { - name: "generate with chore", - state: &legacyconfig.LibrarianState{ - Image: "go:1.21", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "my-library", - // this is the newVersion in the release note. - Version: "1.1.0", - PreviousVersion: "1.0.0", - ReleaseTriggered: true, - Changes: []*legacyconfig.Commit{ - { - Type: "chore", - Subject: "some chore", - Body: "this is the body", - CommitHash: hash1.String(), - LibraryIDs: "my-library", - }, - }, - }, - }, - }, - ghRepo: &legacygithub.Repository{Owner: "owner", Name: "repo"}, - wantReleaseNote: fmt.Sprintf(`PR created by the Librarian CLI to initialize a release. Merging this PR will auto trigger a release. - -Librarian Version: %s -Language Image: go:1.21 -
my-library: v1.1.0 - -## [v1.1.0](https://github.com/owner/repo/compare/my-library-1.0.0...my-library-1.1.0) (%s) - -
`, - librarianVersion, today), - }, - { - name: "release_with_bulk_generation_commits", - state: &legacyconfig.LibrarianState{ - Image: "go:1.21", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "j", - Version: "1.1.0", - PreviousVersion: "1.0.0", - ReleaseTriggered: true, - Changes: []*legacyconfig.Commit{ - { - Type: "feat", - Subject: "new feature", - CommitHash: hash1.String(), - LibraryIDs: "j", - }, - { - Type: "fix", - Subject: "bulk change", - CommitHash: hash2.String(), - LibraryIDs: "a,b,c,d,e,f,g,h,i,j,k", - }, - { - Type: "chore", - Subject: "bulk change 2", - CommitHash: hash3.String(), - LibraryIDs: "j,k,l,m,n,o,p,q,r,s", - PiperCLNumber: "12345", - }, - }, - }, - { - ID: "k", - Version: "2.4.0", - PreviousVersion: "2.3.0", - ReleaseTriggered: true, - Changes: []*legacyconfig.Commit{ - { - Type: "fix", - Subject: "bulk change", - CommitHash: hash2.String(), - LibraryIDs: "a,b,c,d,e,f,g,h,i,j,k", - }, - }, - }, - }, - }, - ghRepo: &legacygithub.Repository{Owner: "owner", Name: "repo"}, - wantReleaseNote: fmt.Sprintf(`PR created by the Librarian CLI to initialize a release. Merging this PR will auto trigger a release. - -Librarian Version: %s -Language Image: go:1.21 -
j: v1.1.0 - -## [v1.1.0](https://github.com/owner/repo/compare/j-1.0.0...j-1.1.0) (%s) - -### Features - -* new feature ([12345678](https://github.com/owner/repo/commit/12345678)) - -
- - -
k: v2.4.0 - -## [v2.4.0](https://github.com/owner/repo/compare/k-2.3.0...k-2.4.0) (%s) - -
- - -
Bulk Changes - -* chore: bulk change 2 (PiperOrigin-RevId: 12345) ([abcdef00](https://github.com/owner/repo/commit/abcdef00)) - Libraries: j,k,l,m,n,o,p,q,r,s -* fix: bulk change ([fedcba09](https://github.com/owner/repo/commit/fedcba09)) - Libraries: a,b,c,d,e,f,g,h,i,j,k -
`, - librarianVersion, today, today), - }, - { - name: "release_with_other_src_bulk_commits", - state: &legacyconfig.LibrarianState{ - Image: "go:1.21", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "ignored-library", - Version: "1.4.0", - PreviousVersion: "1.3.0", - // This library will not appear in the release notes because - // release is not triggered for. - ReleaseTriggered: false, - Changes: []*legacyconfig.Commit{ - { - Type: "fix", - Subject: "this commit is ignored", - CommitHash: "123456789012345", - LibraryIDs: "ignored-library", - }, - }, - }, - { - ID: "j", - Version: "1.1.0", - PreviousVersion: "1.0.0", - ReleaseTriggered: true, - Changes: []*legacyconfig.Commit{ - { - Type: "feat", - Subject: "new feature", - CommitHash: hash1.String(), - LibraryIDs: "j", - }, - { - Type: "fix", - Subject: "bulk change", - CommitHash: hash2.String(), - LibraryIDs: "a,b,c,d,e,f,g,h,i,j,k", - }, - { - Type: "chore", - Subject: "bulk change 2", - CommitHash: hash3.String(), - LibraryIDs: "j,k,l,m,n,o,p,q,r,s", - PiperCLNumber: "12345", - }, - { - // This is a bulk commit, it appears in - // bulk changes section. - Type: "chore", - Subject: "update dependency", - CommitHash: hash4.String(), - LibraryIDs: "j", - }, - }, - }, - { - ID: "k", - Version: "2.4.0", - PreviousVersion: "2.3.0", - ReleaseTriggered: true, - Changes: []*legacyconfig.Commit{ - { - Type: "fix", - Subject: "bulk change", - CommitHash: hash2.String(), - LibraryIDs: "a,b,c,d,e,f,g,h,i,j,k", - }, - { - // This is a bulk commit, it appears in - // bulk changes section. - Type: "chore", - Subject: "update dependency", - CommitHash: hash4.String(), - LibraryIDs: "k", - }, - }, - }, - { - ID: "library-1", - Version: "2.4.0", - PreviousVersion: "2.3.0", - ReleaseTriggered: true, - Changes: []*legacyconfig.Commit{ - { - // This is a bulk commit, it appears in - // bulk changes section. - Type: "chore", - Subject: "update dependency", - CommitHash: hash4.String(), - LibraryIDs: "library-1", - }, - { - Type: "docs", - Subject: "update README", - CommitHash: hash5.String(), - LibraryIDs: "library-1", - }, - }, - }, - { - ID: "library-2", - Version: "2.4.0", - PreviousVersion: "2.3.0", - ReleaseTriggered: true, - Changes: []*legacyconfig.Commit{ - { - // This is a bulk commit, it appears in - // bulk changes section. - Type: "chore", - Subject: "update dependency", - CommitHash: hash4.String(), - LibraryIDs: "library-2", - }, - { - Type: "docs", - Subject: "update README", - CommitHash: hash5.String(), - LibraryIDs: "library-2", - }, - }, - }, - { - ID: "library-3", - Version: "2.4.0", - PreviousVersion: "2.3.0", - ReleaseTriggered: true, - Changes: []*legacyconfig.Commit{ - { - // This is a bulk commit, it appears in - // bulk changes section. - Type: "chore", - Subject: "update dependency", - CommitHash: hash4.String(), - LibraryIDs: "library-3", - }, - { - // This is a non-bulk commit because it only has two - // library ids. - // This commit will appear in library section. - Type: "fix", - Subject: "non bulk fix", - CommitHash: hash6.String(), - LibraryIDs: "library-3,library-4", - }, - }, - }, - { - ID: "library-4", - Version: "2.4.0", - PreviousVersion: "2.3.0", - ReleaseTriggered: true, - Changes: []*legacyconfig.Commit{ - { - // This is a bulk commit, it appears in - // bulk changes section. - Type: "chore", - Subject: "update dependency", - CommitHash: hash4.String(), - LibraryIDs: "library-4", - }, - }, - }, - { - ID: "library-5", - Version: "2.4.0", - PreviousVersion: "2.3.0", - ReleaseTriggered: true, - Changes: []*legacyconfig.Commit{ - { - // This is a bulk commit, it appears in - // bulk changes section. - Type: "chore", - Subject: "update dependency", - CommitHash: hash4.String(), - LibraryIDs: "library-5", - }, - }, - }, - { - ID: "library-6", - Version: "2.4.0", - PreviousVersion: "2.3.0", - ReleaseTriggered: true, - Changes: []*legacyconfig.Commit{ - { - // This is a bulk commit, it appears in - // bulk changes section. - Type: "chore", - Subject: "update dependency", - CommitHash: hash4.String(), - LibraryIDs: "library-6", - }, - }, - }, - { - ID: "library-7", - Version: "2.4.0", - PreviousVersion: "2.3.0", - ReleaseTriggered: true, - Changes: []*legacyconfig.Commit{ - { - // This is a bulk commit, it appears in - // bulk changes section. - Type: "chore", - Subject: "update dependency", - CommitHash: hash4.String(), - LibraryIDs: "library-7", - }, - }, - }, - { - ID: "library-8", - Version: "2.4.0", - PreviousVersion: "2.3.0", - ReleaseTriggered: true, - Changes: []*legacyconfig.Commit{ - { - // This is a bulk commit, it appears in - // bulk changes section. - Type: "chore", - Subject: "update dependency", - CommitHash: hash4.String(), - LibraryIDs: "library-8", - }, - }, - }, - }, - }, - ghRepo: &legacygithub.Repository{Owner: "owner", Name: "repo"}, - wantReleaseNote: fmt.Sprintf(`PR created by the Librarian CLI to initialize a release. Merging this PR will auto trigger a release. - -Librarian Version: %s -Language Image: go:1.21 -
j: v1.1.0 - -## [v1.1.0](https://github.com/owner/repo/compare/j-1.0.0...j-1.1.0) (%s) - -### Features - -* new feature ([12345678](https://github.com/owner/repo/commit/12345678)) - -
- - -
k: v2.4.0 - -## [v2.4.0](https://github.com/owner/repo/compare/k-2.3.0...k-2.4.0) (%s) - -
- - -
library-1: v2.4.0 - -## [v2.4.0](https://github.com/owner/repo/compare/library-1-2.3.0...library-1-2.4.0) (%s) - -### Documentation - -* update README ([bcdef123](https://github.com/owner/repo/commit/bcdef123)) - -
- - -
library-2: v2.4.0 - -## [v2.4.0](https://github.com/owner/repo/compare/library-2-2.3.0...library-2-2.4.0) (%s) - -### Documentation - -* update README ([bcdef123](https://github.com/owner/repo/commit/bcdef123)) - -
- - -
library-3: v2.4.0 - -## [v2.4.0](https://github.com/owner/repo/compare/library-3-2.3.0...library-3-2.4.0) (%s) - -### Bug Fixes - -* non bulk fix ([cdef0000](https://github.com/owner/repo/commit/cdef0000)) - -
- - -
library-4: v2.4.0 - -## [v2.4.0](https://github.com/owner/repo/compare/library-4-2.3.0...library-4-2.4.0) (%s) - -### Bug Fixes - -* non bulk fix ([cdef0000](https://github.com/owner/repo/commit/cdef0000)) - -
- - -
library-5: v2.4.0 - -## [v2.4.0](https://github.com/owner/repo/compare/library-5-2.3.0...library-5-2.4.0) (%s) - -
- - -
library-6: v2.4.0 - -## [v2.4.0](https://github.com/owner/repo/compare/library-6-2.3.0...library-6-2.4.0) (%s) - -
- - -
library-7: v2.4.0 - -## [v2.4.0](https://github.com/owner/repo/compare/library-7-2.3.0...library-7-2.4.0) (%s) - -
- - -
library-8: v2.4.0 - -## [v2.4.0](https://github.com/owner/repo/compare/library-8-2.3.0...library-8-2.4.0) (%s) - -
- - -
Bulk Changes - -* chore: bulk change 2 (PiperOrigin-RevId: 12345) ([abcdef00](https://github.com/owner/repo/commit/abcdef00)) - Libraries: j,k,l,m,n,o,p,q,r,s -* chore: update dependency ([acdef123](https://github.com/owner/repo/commit/acdef123)) - Libraries: j,k,library-1,library-2,library-3,library-4,library-5,library-6,library-7,library-8 -* fix: bulk change ([fedcba09](https://github.com/owner/repo/commit/fedcba09)) - Libraries: a,b,c,d,e,f,g,h,i,j,k -
`, - librarianVersion, today, today, today, today, today, today, today, today, today, today), - }, - { - name: "no library changes", - state: &legacyconfig.LibrarianState{ - Image: "go:1.21", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "one-library", - PreviousVersion: "1.1.0", - Version: "1.2.3", - ReleaseTriggered: true, - }, - { - ID: "another-library", - PreviousVersion: "2.1.0", - Version: "2.3.4", - ReleaseTriggered: true, - }, - }, - }, - ghRepo: &legacygithub.Repository{}, - wantReleaseNote: fmt.Sprintf(`PR created by the Librarian CLI to initialize a release. Merging this PR will auto trigger a release. - -Librarian Version: %s -Language Image: go:1.21 -
one-library: v1.2.3 - -## [v1.2.3](https://github.com///compare/one-library-1.1.0...one-library-1.2.3) (%s) - -
- - -
another-library: v2.3.4 - -## [v2.3.4](https://github.com///compare/another-library-2.1.0...another-library-2.3.4) (%s) - -
`, librarianVersion, today, today), - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - got, err := formatReleaseNotes(test.state, test.ghRepo) - if test.wantErr { - if err == nil { - t.Fatalf("%s should return error", test.name) - } - if !strings.Contains(err.Error(), test.wantErrPhrase) { - t.Errorf("formatReleaseNotes() returned error %q, want to contain %q", err.Error(), test.wantErrPhrase) - } - return - } - if err != nil { - t.Fatal(err) - } - if diff := cmp.Diff(test.wantReleaseNote, got); diff != "" { - t.Errorf("formatReleaseNotes() mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestFindPiperIDFrom(t *testing.T) { - for _, test := range []struct { - name string - commit *legacygitrepo.Commit - want string - wantErr error - }{ - { - name: "found_piper_id", - commit: &legacygitrepo.Commit{ - Message: "feat: add a new API\n\nPiperOrigin-RevId: 745187558", - }, - want: "745187558", - }, - { - name: "invalid_commit", - commit: &legacygitrepo.Commit{ - Message: "", - }, - wantErr: legacygitrepo.ErrEmptyCommitMessage, - }, - { - name: "unconventional_commit", - commit: &legacygitrepo.Commit{ - Message: "unconventional commit message", - }, - wantErr: errPiperNotFound, - }, - { - name: "does_not_contain_piper_id", - commit: &legacygitrepo.Commit{ - Message: "feat: add a new API", - }, - wantErr: errPiperNotFound, - }, - } { - t.Run(test.name, func(t *testing.T) { - got, err := findPiperIDFrom(test.commit, "example-id") - if test.wantErr != nil { - if !errors.Is(err, test.wantErr) { - t.Errorf("unexpected error type: got %v, want %v", err, test.wantErr) - } - - return - } - if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("findPiperIDFrom() mismatch (-want +got):%s", diff) - } - }) - } -} - -func TestLanguageRepoChangedFiles(t *testing.T) { - for _, test := range []struct { - name string - repo legacygitrepo.Repository - want []string - wantErr bool - }{ - { - name: "IsClean fails", - repo: &MockRepository{ - IsCleanError: fmt.Errorf("mock failure from IsClean"), - }, - wantErr: true, - }, - { - name: "clean, HeadHash fails", - repo: &MockRepository{ - IsCleanValue: true, - HeadHashError: fmt.Errorf("mock failure from HeadHash"), - }, - wantErr: true, - }, - { - name: "clean, ChangedFilesInCommit fails", - repo: &MockRepository{ - IsCleanValue: true, - HeadHashValue: "1234", - ChangedFilesInCommitError: fmt.Errorf("mock failure from ChangedFilesInCommit"), - }, - wantErr: true, - }, - { - name: "dirty, ChangedFiles fails", - repo: &MockRepository{ - ChangedFilesError: fmt.Errorf("mock failure from ChangedFiles"), - }, - wantErr: true, - }, - { - name: "clean success", - repo: &MockRepository{ - IsCleanValue: true, - HeadHashValue: "1234", - ChangedFilesInCommitValueByHash: map[string][]string{ - "abcd": {"a/b/c", "d/e/f"}, - "1234": {"g/h/i", "j/k/l"}, - }, - }, - want: []string{"g/h/i", "j/k/l"}, - }, - { - name: "dirty success", - repo: &MockRepository{ - ChangedFilesValue: []string{"a/b/c", "d/e/f"}, - }, - want: []string{"a/b/c", "d/e/f"}, - }, - } { - t.Run(test.name, func(t *testing.T) { - got, err := languageRepoChangedFiles(test.repo) - if (err != nil) != test.wantErr { - t.Errorf("languageRepoChangedFiles() error = %v, wantErr %v", err, test.wantErr) - return - } - if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("mismatch (-want +got):\n%s", diff) - } - }) - } -} diff --git a/internal/legacylibrarian/legacylibrarian/release_stage.go b/internal/legacylibrarian/legacylibrarian/release_stage.go deleted file mode 100644 index 9103a558740..00000000000 --- a/internal/legacylibrarian/legacylibrarian/release_stage.go +++ /dev/null @@ -1,420 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacylibrarian - -import ( - "context" - "errors" - "fmt" - "io/fs" - "log/slog" - "os" - "path/filepath" - "slices" - "strings" - - "github.com/googleapis/librarian/internal/config" - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" - "github.com/googleapis/librarian/internal/legacylibrarian/legacydocker" - "github.com/googleapis/librarian/internal/legacylibrarian/legacygitrepo" - "github.com/googleapis/librarian/internal/librarian" - "github.com/googleapis/librarian/internal/semver" - "github.com/googleapis/librarian/internal/yaml" -) - -var errVersionRegression = errors.New("version is regression") - -type stageRunner struct { - branch string - commit bool - containerClient ContainerClient - ghClient GitHubClient - image string - librarianConfig *legacyconfig.LibrarianConfig - library string - libraryVersion string - push bool - repo legacygitrepo.Repository - sourceRepo legacygitrepo.Repository - state *legacyconfig.LibrarianState - workRoot string -} - -func newStageRunner(cfg *legacyconfig.Config) (*stageRunner, error) { - runner, err := newCommandRunner(cfg) - if err != nil { - return nil, fmt.Errorf("failed to create stage runner: %w", err) - } - return &stageRunner{ - branch: cfg.Branch, - commit: cfg.Commit, - containerClient: runner.containerClient, - ghClient: runner.ghClient, - image: runner.image, - librarianConfig: runner.librarianConfig, - library: cfg.Library, - libraryVersion: cfg.LibraryVersion, - push: cfg.Push, - repo: runner.repo, - sourceRepo: runner.sourceRepo, - state: runner.state, - workRoot: runner.workRoot, - }, nil -} - -func (r *stageRunner) run(ctx context.Context) error { - outputDir := filepath.Join(r.workRoot, "output") - if err := os.MkdirAll(outputDir, 0755); err != nil { - return fmt.Errorf("failed to create output dir: %s", outputDir) - } - slog.Info("staging a release", "dir", outputDir) - if err := r.runStageCommand(ctx, outputDir); err != nil { - return err - } - - // No need to update the librarian state if there are no libraries - // that need to be released - if !hasLibrariesToRelease(r.state.Libraries) { - slog.Info("no release created; skipping the commit/PR") - return nil - } - - if err := saveLibrarianState(r.repo.GetDir(), r.state); err != nil { - return err - } - if err := r.updateLibrarianYAML(ctx); err != nil { - return err - } - - prBodyBuilder := func() (string, error) { - gitHubRepo, err := GetGitHubRepositoryFromGitRepo(r.repo) - if err != nil { - return "", fmt.Errorf("failed to get GitHub repository: %w", err) - } - return formatReleaseNotes(r.state, gitHubRepo) - } - commitInfo := &commitInfo{ - branch: r.branch, - commit: r.commit, - commitMessage: "chore: create a release", - ghClient: r.ghClient, - prType: pullRequestRelease, - // Newly created PRs from the `release stage` command should have a - // `release:pending` GitHub tab to be tracked for release. - pullRequestLabels: []string{"release:pending"}, - push: r.push, - languageRepo: r.repo, - sourceRepo: r.sourceRepo, - state: r.state, - workRoot: r.workRoot, - prBodyBuilder: prBodyBuilder, - } - if err := commitAndPush(ctx, commitInfo); err != nil { - return fmt.Errorf("failed to commit and push: %w", err) - } - - return nil -} - -// hasLibrariesToRelease searches through the state of each library and checks -// that there is a single library configured to be triggered. -func hasLibrariesToRelease(libraryStates []*legacyconfig.LibraryState) bool { - for _, library := range libraryStates { - if library.ReleaseTriggered { - return true - } - } - return false -} - -func (r *stageRunner) runStageCommand(ctx context.Context, outputDir string) error { - src := r.repo.GetDir() - librariesToRelease := r.state.Libraries - if r.library != "" { - library := r.state.LibraryByID(r.library) - if library == nil { - return fmt.Errorf("unable to find library for release: %s", r.library) - } - librariesToRelease = []*legacyconfig.LibraryState{library} - } - // Mark if there are any library that needs to be released - foundReleasableLibrary := false - for _, library := range librariesToRelease { - if r.librarianConfig != nil { - libraryConfig := r.librarianConfig.LibraryConfigFor(library.ID) - if libraryConfig != nil && libraryConfig.ReleaseBlocked && r.library != library.ID { - // Do not skip the `release_blocked` library if library ID is explicitly specified. - slog.Info("library has release_blocked, skipping", "id", library.ID) - continue - } - } - if err := r.processLibrary(ctx, library); err != nil { - return err - } - - // Copy the library files over if a release is needed - if library.ReleaseTriggered { - foundReleasableLibrary = true - } - } - - if !foundReleasableLibrary { - slog.Info("no libraries need to be released") - return nil - } - - stageRequest := &legacydocker.ReleaseStageRequest{ - Branch: r.branch, - Commit: r.commit, - LibrarianConfig: r.librarianConfig, - LibraryID: r.library, - LibraryVersion: r.libraryVersion, - Output: outputDir, - RepoDir: src, - Push: r.push, - State: r.state, - } - - if err := r.containerClient.ReleaseStage(ctx, stageRequest); err != nil { - return err - } - - // Read the response file. - if _, err := readLibraryState( - filepath.Join(stageRequest.RepoDir, legacyconfig.LibrarianDir, legacyconfig.ReleaseStageResponse)); err != nil { - return err - } - - for _, library := range librariesToRelease { - // Copy the library files back if a release is needed - if library.ReleaseTriggered { - if err := copyLibraryFiles(r.state, r.repo.GetDir(), library.ID, outputDir, false); err != nil { - return err - } - } - } - - return copyGlobalAllowlist(r.librarianConfig, r.repo.GetDir(), outputDir, false) -} - -// processLibrary wrapper to process the library for release. Helps retrieve latest commits -// since the last release and passing the changes to updateLibrary. -func (r *stageRunner) processLibrary(ctx context.Context, library *legacyconfig.LibraryState) error { - var tagName string - if library.Version != "0.0.0" { - tagFormat := legacyconfig.DetermineTagFormat(library.ID, library, r.librarianConfig) - tagName = legacyconfig.FormatTag(tagFormat, library.ID, library.Version) - } - commits, err := getConventionalCommitsSinceLastRelease(r.repo, library, tagName) - if err != nil { - return fmt.Errorf("failed to fetch conventional commits for library, %s: %w", library.ID, err) - } - // Filter specifically for commits relevant to a library - commits = filterCommitsByLibraryID(commits, library.ID) - return r.updateLibrary(ctx, library, commits) -} - -// filterCommitsByLibraryID keeps the conventional commits if the given libraryID appears in the Footer or matches -// the libraryID in the commit. -func filterCommitsByLibraryID(commits []*legacygitrepo.ConventionalCommit, libraryID string) []*legacygitrepo.ConventionalCommit { - var filteredCommits []*legacygitrepo.ConventionalCommit - for _, commit := range commits { - if commit.Footers != nil { - ids, ok := commit.Footers["Library-IDs"] - libraryIDs := strings.Split(ids, ",") - if ok && slices.Contains(libraryIDs, libraryID) { - filteredCommits = append(filteredCommits, commit) - continue - } - } - if commit.LibraryID == libraryID { - filteredCommits = append(filteredCommits, commit) - } - } - return filteredCommits -} - -// updateLibrary updates the library's state with the new release information: -// -// 1. Determines the library version's next version. -// -// 2. Updates the library's previous version and the new current version. -// -// 3. Set the library's release trigger to true. -func (r *stageRunner) updateLibrary(ctx context.Context, library *legacyconfig.LibraryState, commits []*legacygitrepo.ConventionalCommit) error { - var nextVersion string - // If library version was explicitly set, attempt to use it. Otherwise, try to determine the version from the commits. - if r.libraryVersion != "" { - slog.Info("library version override inputted", "currentVersion", library.Version, "inputVersion", r.libraryVersion) - nextVersion = semver.MaxVersion(library.Version, r.libraryVersion) - slog.Debug("determined the library's next version from version input", "library", library.ID, "nextVersion", nextVersion) - // Currently, nextVersion is the max of current version or input version. If nextVersion is equal to the current version, - // then the input version is either equal or less than current version and cannot be used for release - if nextVersion == library.Version { - return fmt.Errorf("inputted version is not SemVer greater than the current version. Set a version SemVer greater than current than: %s", library.Version) - } - } else { - var err error - nextVersion, err = r.determineNextVersion(ctx, commits, library.Version, library.ID) - if err != nil { - return err - } - slog.Debug("determined the library's next version from commits", "library", library.ID, "nextVersion", nextVersion) - // Unable to find a releasable unit from the changes - if nextVersion == library.Version { - // No library was inputted for release. Skipping this library for release - if r.library == "" { - slog.Info("library does not have any releasable units and will not be released.", "library", library.ID, "version", library.Version) - return nil - } - // Library was inputted for release, but does not contain a releasable unit - return fmt.Errorf("library does not have a releasable unit and will not be released. Use the version flag to force a release for: %s", library.ID) - } - slog.Info("updating library to the next version", "library", library.ID, "currentVersion", library.Version, "nextVersion", nextVersion) - } - - // Update the previous version, we need this value when creating release note. - library.PreviousVersion = library.Version - library.Changes = toCommit(commits, library.ID) - library.Version = nextVersion - library.ReleaseTriggered = true - return nil -} - -// determineNextVersion determines the next valid SemVer version from the commits or from -// the next_version override value in the config.yaml file. -func (r *stageRunner) determineNextVersion(ctx context.Context, commits []*legacygitrepo.ConventionalCommit, currentVersion string, libraryID string) (string, error) { - var derivedNextVersion string - var err error - if r.branch == "preview" { - var stableVersion string - stableVersion, err = r.loadStableLibraryVersion(ctx, libraryID) - if err != nil { - return "", err - } - derivedNextVersion, err = semver.DeriveNextPreview(currentVersion, stableVersion, semver.DeriveNextOptions{}) - } else { - derivedNextVersion, err = NextVersion(commits, currentVersion, r.librarianConfig != nil && r.librarianConfig.ReleaseOnlyMode) - } - if err != nil { - return "", err - } - - if r.librarianConfig == nil { - slog.Debug("no librarian config") - return derivedNextVersion, nil - } - - // Look for next_version override from config.yaml - libraryConfig := r.librarianConfig.LibraryConfigFor(libraryID) - slog.Debug("looking up library config", "library", libraryID, slog.Any("config", libraryConfig)) - if libraryConfig == nil || libraryConfig.NextVersion == "" { - return derivedNextVersion, nil - } - - // Compare versions and pick latest - return semver.MaxVersion(derivedNextVersion, libraryConfig.NextVersion), nil -} - -func (r *stageRunner) loadStableLibraryVersion(ctx context.Context, libraryID string) (string, error) { - mainState, err := loadRepoStateFromGitHub(ctx, r.ghClient, "main") - if err != nil { - return "", err - } - mainLibState := mainState.LibraryByID(libraryID) - if mainLibState == nil { - // Library not configured for generation on main branch. - // This can happen if the library is new and only exists on the preview - // branch. - // Fallback to "0.0.0" as the stable version. - return "0.0.0", nil - } - return mainLibState.Version, nil -} - -// toCommit converts a slice of legacygitrepo.ConventionalCommit to a slice of legacyconfig.Commit. -// If the ConventionalCommit has NestedCommits, they are also extracted and -// converted. -// Set LibraryIDs to the given libraryID if the conventional commit doesn't have key `Library-IDs` in the Footers; -// otherwise use the value in the Footers as LibraryIDs. -func toCommit(c []*legacygitrepo.ConventionalCommit, libraryID string) []*legacyconfig.Commit { - var commits []*legacyconfig.Commit - for _, cc := range c { - var libraryIDs string - libraryIDs, ok := cc.Footers["Library-IDs"] - if !ok { - libraryIDs = libraryID - } - - commits = append(commits, &legacyconfig.Commit{ - Type: cc.Type, - Subject: cc.Subject, - Body: cc.Body, - CommitHash: cc.CommitHash, - PiperCLNumber: cc.Footers["PiperOrigin-RevId"], - LibraryIDs: libraryIDs, - }) - } - return commits -} - -func (r *stageRunner) updateLibrarianYAML(ctx context.Context) error { - librarianYAMLPath := filepath.Join(r.repo.GetDir(), config.LibrarianYAML) - cfg, err := yaml.Read[config.Config](librarianYAMLPath) - if err != nil { - if errors.Is(err, fs.ErrNotExist) { - return nil - } - return err - } - newCfg, err := syncVersion(r.state, cfg) - if err != nil { - return err - } - return librarian.RunTidyOnConfig(ctx, r.repo.GetDir(), newCfg) -} - -func syncVersion(state *legacyconfig.LibrarianState, cfg *config.Config) (*config.Config, error) { - for _, lib := range cfg.Libraries { - var err error - lib.Version, err = syncLibraryVersion(lib.Name, state, lib.Version) - if err != nil { - return nil, err - } - - if lib.Preview != nil { - lib.Preview.Version, err = syncLibraryVersion(lib.Name+"-preview", state, lib.Preview.Version) - if err != nil { - return nil, err - } - } - } - return cfg, nil -} - -// syncLibraryVersion checks that the version in librarian.yaml is not greater than the same in -// .librarian/state.yaml, which would imply version drift. Once that check is complete, it sets the library's -// version in librarian.yaml to match that of .librarian/state.yaml, which is the current source of truth. -func syncLibraryVersion(name string, state *legacyconfig.LibrarianState, currentVersion string) (string, error) { - legacyLibrary := state.LibraryByID(name) - if legacyLibrary == nil || legacyLibrary.Version == "" { - return currentVersion, nil - } - maxVersion := semver.MaxVersion(currentVersion, legacyLibrary.Version) - if maxVersion == currentVersion && legacyLibrary.Version != currentVersion { - return "", fmt.Errorf("library %s, version in state, %s, is smaller than version in librarian.yaml, %s: %w", name, legacyLibrary.Version, currentVersion, errVersionRegression) - } - return legacyLibrary.Version, nil -} diff --git a/internal/legacylibrarian/legacylibrarian/release_stage_test.go b/internal/legacylibrarian/legacylibrarian/release_stage_test.go deleted file mode 100644 index fda53b948c2..00000000000 --- a/internal/legacylibrarian/legacylibrarian/release_stage_test.go +++ /dev/null @@ -1,2529 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacylibrarian - -import ( - "errors" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/go-git/go-git/v5/plumbing" - "github.com/google/go-cmp/cmp" - "github.com/googleapis/librarian/internal/config" - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" - "github.com/googleapis/librarian/internal/legacylibrarian/legacygitrepo" - "github.com/googleapis/librarian/internal/yaml" -) - -func TestNewStageRunner(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - cfg *legacyconfig.Config - wantErr bool - wantErrMsg string - }{ - { - name: "valid config", - cfg: &legacyconfig.Config{ - API: "some/api", - APISource: newTestGitRepo(t).GetDir(), - Repo: newTestGitRepo(t).GetDir(), - WorkRoot: t.TempDir(), - Image: "gcr.io/test/test-image", - }, - }, - { - name: "invalid config", - cfg: &legacyconfig.Config{ - APISource: newTestGitRepo(t).GetDir(), - }, - wantErr: true, - wantErrMsg: "failed to create stage runner", - }, - } { - t.Run(test.name, func(t *testing.T) { - _, err := newStageRunner(test.cfg) - if test.wantErr { - if err == nil { - t.Fatal("newStageRunner() should return error") - } - - if !strings.Contains(err.Error(), test.wantErrMsg) { - t.Errorf("want error message: %q, got %q", test.wantErrMsg, err.Error()) - } - - return - } - if err != nil { - t.Errorf("newStageRunner() = %v, want nil", err) - } - }) - } -} - -func TestStageRun(t *testing.T) { - t.Parallel() - - mockRepoWithReleasableUnit := &MockRepository{ - Dir: t.TempDir(), - RemotesValue: []*legacygitrepo.Remote{ - { - Name: "origin", - URLs: []string{"https://github.com/googleapis/librarian.git"}, - }, - }, - ChangedFilesInCommitValue: []string{"dir1/file.txt"}, - GetCommitsForPathsSinceTagValue: []*legacygitrepo.Commit{ - { - Message: "feat: a feature", - }, - }, - } - for _, test := range []struct { - name string - containerClient *mockContainerClient - dockerStageCalls int - // TODO: Pass all setup fields to the setupRunner func - setupRunner func(containerClient *mockContainerClient) *stageRunner - files map[string]string - want *legacyconfig.LibrarianState - wantErr bool - wantErrMsg string - }{ - { - name: "run release stage command for all libraries, update librarian state", - containerClient: &mockContainerClient{}, - dockerStageCalls: 1, - setupRunner: func(containerClient *mockContainerClient) *stageRunner { - return &stageRunner{ - workRoot: t.TempDir(), - containerClient: containerClient, - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "another-example-id", - Version: "1.0.0", - SourceRoots: []string{ - "dir3", - "dir4", - }, - RemoveRegex: []string{ - "dir3", - "dir4", - }, - }, - { - ID: "example-id", - Version: "2.0.0", - SourceRoots: []string{ - "dir1", - "dir2", - }, - RemoveRegex: []string{ - "dir1", - "dir2", - }, - }, - }, - }, - repo: &MockRepository{ - Dir: t.TempDir(), - RemotesValue: []*legacygitrepo.Remote{ - { - Name: "origin", - URLs: []string{"https://github.com/googleapis/librarian.git"}, - }, - }, - GetCommitsForPathsSinceTagValueByTag: map[string][]*legacygitrepo.Commit{ - "another-example-id-1.0.0": { - { - Hash: plumbing.NewHash("123456"), - Message: "feat: another new feature", - }, - }, - "example-id-2.0.0": { - { - Hash: plumbing.NewHash("abcdefg"), - Message: "feat: a new feature", - }, - }, - }, - ChangedFilesInCommitValueByHash: map[string][]string{ - plumbing.NewHash("123456").String(): { - "dir3/file3.txt", - "dir4/file4.txt", - }, - plumbing.NewHash("abcdefg").String(): { - "dir1/file1.txt", - "dir2/file2.txt", - }, - }, - }, - librarianConfig: &legacyconfig.LibrarianConfig{}, - } - }, - files: map[string]string{ - "file1.txt": "", - "dir1/file1.txt": "", - "dir2/file2.txt": "", - "dir3/file3.txt": "", - "dir4/file4.txt": "", - }, - want: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "another-example-id", - Version: "1.1.0", // version is bumped. - APIs: []*legacyconfig.API{}, - SourceRoots: []string{ - "dir3", - "dir4", - }, - PreserveRegex: []string{}, - RemoveRegex: []string{ - "dir3", - "dir4", - }, - }, - { - ID: "example-id", - Version: "2.1.0", // version is bumped. - APIs: []*legacyconfig.API{}, - SourceRoots: []string{ - "dir1", - "dir2", - }, - PreserveRegex: []string{}, - RemoveRegex: []string{ - "dir1", - "dir2", - }, - }, - }, - }, - }, - { - name: "run release stage command for one library (library id in cfg)", - containerClient: &mockContainerClient{}, - dockerStageCalls: 1, - setupRunner: func(containerClient *mockContainerClient) *stageRunner { - return &stageRunner{ - workRoot: t.TempDir(), - containerClient: containerClient, - library: "example-id", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - Version: "1.0.0", - ID: "another-example-id", - SourceRoots: []string{ - "dir3", - "dir4", - }, - }, - { - Version: "2.0.0", - ID: "example-id", - SourceRoots: []string{ - "dir1", - "dir2", - }, - RemoveRegex: []string{ - "dir1", - "dir2", - }, - }, - }, - }, - repo: mockRepoWithReleasableUnit, - librarianConfig: &legacyconfig.LibrarianConfig{}, - } - }, - files: map[string]string{ - "file1.txt": "", - "dir1/file1.txt": "", - "dir2/file2.txt": "", - }, - want: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - Version: "1.0.0", - ID: "another-example-id", - APIs: []*legacyconfig.API{}, - SourceRoots: []string{ - "dir3", - "dir4", - }, - PreserveRegex: []string{}, - RemoveRegex: []string{}, - }, - { - Version: "2.1.0", // Version is bumped only for library specified - ID: "example-id", - APIs: []*legacyconfig.API{}, - SourceRoots: []string{ - "dir1", - "dir2", - }, - PreserveRegex: []string{}, - RemoveRegex: []string{ - "dir1", - "dir2", - }, - }, - }, - }, - }, - { - name: "run release stage command for libraries have the same global files in src roots", - containerClient: &mockContainerClient{}, - dockerStageCalls: 1, - setupRunner: func(containerClient *mockContainerClient) *stageRunner { - return &stageRunner{ - workRoot: t.TempDir(), - containerClient: containerClient, - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "another-example-id", - Version: "1.0.0", - SourceRoots: []string{ - "dir3", - "one/global/example.txt", - }, - RemoveRegex: []string{ - "dir3", - }, - }, - { - ID: "example-id", - Version: "2.0.0", - SourceRoots: []string{ - "dir1", - "one/global/example.txt", - }, - RemoveRegex: []string{ - "dir1", - }, - }, - }, - }, - repo: &MockRepository{ - Dir: t.TempDir(), - RemotesValue: []*legacygitrepo.Remote{ - { - Name: "origin", - URLs: []string{"https://github.com/googleapis/librarian.git"}, - }, - }, - GetCommitsForPathsSinceTagValueByTag: map[string][]*legacygitrepo.Commit{ - "another-example-id-1.0.0": { - { - Hash: plumbing.NewHash("123456"), - Message: "feat: bump version", - }, - }, - "example-id-2.0.0": { - { - Hash: plumbing.NewHash("123456"), - Message: "feat: bump version", - }, - }, - }, - ChangedFilesInCommitValueByHash: map[string][]string{ - plumbing.NewHash("123456").String(): { - "one/global/example.txt", - }, - }, - }, - librarianConfig: &legacyconfig.LibrarianConfig{ - GlobalFilesAllowlist: []*legacyconfig.GlobalFile{ - { - Path: "one/global/example.txt", - Permissions: "read-write", - }, - }, - }, - } - }, - files: map[string]string{ - "one/global/example.txt": "", - "dir1/file1.txt": "", - "dir3/file3.txt": "", - }, - want: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "another-example-id", - Version: "1.1.0", // version is bumped. - APIs: []*legacyconfig.API{}, - SourceRoots: []string{ - "dir3", - "one/global/example.txt", - }, - PreserveRegex: []string{}, - RemoveRegex: []string{ - "dir3", - }, - }, - { - ID: "example-id", - Version: "2.1.0", // version is bumped. - APIs: []*legacyconfig.API{}, - SourceRoots: []string{ - "dir1", - "one/global/example.txt", - }, - PreserveRegex: []string{}, - RemoveRegex: []string{ - "dir1", - }, - }, - }, - }, - }, - { - name: "run release stage command, skips blocked libraries!", - containerClient: &mockContainerClient{}, - dockerStageCalls: 1, - setupRunner: func(containerClient *mockContainerClient) *stageRunner { - return &stageRunner{ - workRoot: t.TempDir(), - containerClient: containerClient, - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "blocked-example-id", - Version: "1.0.0", - SourceRoots: []string{"dir1"}, - }, - { - ID: "example-id", - Version: "2.0.0", - SourceRoots: []string{"dir1"}, - }, - }, - }, - repo: &MockRepository{ - Dir: t.TempDir(), - RemotesValue: []*legacygitrepo.Remote{ - { - Name: "origin", - URLs: []string{"https://github.com/googleapis/librarian.git"}, - }, - }, - GetCommitsForPathsSinceTagValueByTag: map[string][]*legacygitrepo.Commit{ - "blocked-example-id-1.0.0": { - { - Hash: plumbing.NewHash("123456"), - Message: "feat: another new feature", - }, - }, - "example-id-2.0.0": { - { - Hash: plumbing.NewHash("abcdefg"), - Message: "feat: a new feature", - }, - }, - }, - ChangedFilesInCommitValueByHash: map[string][]string{ - plumbing.NewHash("123456").String(): { - "dir1/file1.txt", - }, - plumbing.NewHash("abcdefg").String(): { - "dir1/file2.txt", - }, - }, - }, - librarianConfig: &legacyconfig.LibrarianConfig{ - Libraries: []*legacyconfig.LibraryConfig{ - {LibraryID: "blocked-example-id", ReleaseBlocked: true}, - {LibraryID: "example-id"}, - }, - }, - } - }, - want: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "blocked-example-id", - Version: "1.0.0", // version is NOT bumped. - APIs: []*legacyconfig.API{}, - SourceRoots: []string{"dir1"}, - PreserveRegex: []string{}, - RemoveRegex: []string{}, - }, - { - ID: "example-id", - Version: "2.1.0", // version is bumped. - APIs: []*legacyconfig.API{}, - SourceRoots: []string{"dir1"}, - PreserveRegex: []string{}, - RemoveRegex: []string{}, - }, - }, - }, - }, - { - name: "run release stage command, does not skip blocked library if explicitly specified", - containerClient: &mockContainerClient{}, - dockerStageCalls: 1, - setupRunner: func(containerClient *mockContainerClient) *stageRunner { - return &stageRunner{ - workRoot: t.TempDir(), - containerClient: containerClient, - // The library is explicitly specified. - library: "blocked-example-id", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "blocked-example-id", - Version: "1.0.0", - SourceRoots: []string{"dir1"}, - }, - }, - }, - repo: &MockRepository{ - Dir: t.TempDir(), - RemotesValue: []*legacygitrepo.Remote{ - { - Name: "origin", - URLs: []string{"https://github.com/googleapis/librarian.git"}, - }, - }, - GetCommitsForPathsSinceTagValueByTag: map[string][]*legacygitrepo.Commit{ - "blocked-example-id-1.0.0": { - { - Hash: plumbing.NewHash("123456"), - Message: "feat: another new feature", - }, - }, - }, - ChangedFilesInCommitValueByHash: map[string][]string{ - plumbing.NewHash("123456").String(): { - "dir1/file1.txt", - }, - }, - }, - librarianConfig: &legacyconfig.LibrarianConfig{ - Libraries: []*legacyconfig.LibraryConfig{ - {LibraryID: "blocked-example-id", ReleaseBlocked: true}, - }, - }, - } - }, - want: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "blocked-example-id", - Version: "1.1.0", - APIs: []*legacyconfig.API{}, - SourceRoots: []string{"dir1"}, - PreserveRegex: []string{}, - RemoveRegex: []string{}, - }, - }, - }, - }, - { - name: "run release stage command for one invalid library (invalid library id in cfg)", - containerClient: &mockContainerClient{}, - setupRunner: func(containerClient *mockContainerClient) *stageRunner { - return &stageRunner{ - workRoot: t.TempDir(), - containerClient: containerClient, - library: "does-not-exist", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "another-example-id", - }, - { - ID: "example-id", - }, - }, - }, - repo: &MockRepository{ - Dir: t.TempDir(), - }, - librarianConfig: &legacyconfig.LibrarianConfig{}, - } - }, - wantErr: true, - wantErrMsg: "unable to find library for release", - }, - { - name: "run release stage command without librarian config (no config.yaml file)", - containerClient: &mockContainerClient{}, - dockerStageCalls: 1, - setupRunner: func(containerClient *mockContainerClient) *stageRunner { - return &stageRunner{ - workRoot: t.TempDir(), - containerClient: containerClient, - library: "example-id", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - Version: "1.0.0", - ID: "another-example-id", - SourceRoots: []string{ - "dir3", - "dir4", - }, - }, - { - Version: "2.0.0", - ID: "example-id", - SourceRoots: []string{ - "dir1", - "dir2", - }, - RemoveRegex: []string{ - "dir1", - "dir2", - }, - }, - }, - }, - repo: mockRepoWithReleasableUnit, - } - }, - files: map[string]string{ - "file1.txt": "", - "dir1/file1.txt": "", - "dir2/file2.txt": "", - }, - want: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - Version: "1.0.0", - ID: "another-example-id", - APIs: []*legacyconfig.API{}, - SourceRoots: []string{ - "dir3", - "dir4", - }, - PreserveRegex: []string{}, - RemoveRegex: []string{}, - }, - { - Version: "2.1.0", - ID: "example-id", - APIs: []*legacyconfig.API{}, - SourceRoots: []string{ - "dir1", - "dir2", - }, - PreserveRegex: []string{}, - RemoveRegex: []string{ - "dir1", - "dir2", - }, - }, - }, - }, - }, - { - name: "docker command returns error", - containerClient: &mockContainerClient{ - stageErr: errors.New("simulated init error"), - }, - // error occurred inside the docker container, there was a single request made to the container - dockerStageCalls: 1, - setupRunner: func(containerClient *mockContainerClient) *stageRunner { - return &stageRunner{ - workRoot: t.TempDir(), - containerClient: containerClient, - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - Version: "1.0.0", - ID: "example-id", - SourceRoots: []string{"dir1"}, - }, - }, - }, - repo: mockRepoWithReleasableUnit, - librarianConfig: &legacyconfig.LibrarianConfig{}, - } - }, - wantErr: true, - wantErrMsg: "simulated init error", - }, - { - name: "release response from container contains error message", - containerClient: &mockContainerClient{ - wantErrorMsg: true, - }, - // error reported from the docker container, there was a single request made to the container - dockerStageCalls: 1, - setupRunner: func(containerClient *mockContainerClient) *stageRunner { - return &stageRunner{ - workRoot: t.TempDir(), - containerClient: containerClient, - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - Version: "1.0.0", - ID: "example-id", - SourceRoots: []string{"dir1"}, - }, - }, - }, - repo: mockRepoWithReleasableUnit, - librarianConfig: &legacyconfig.LibrarianConfig{}, - } - }, - wantErr: true, - wantErrMsg: "failed with error message: simulated error message", - }, - { - name: "invalid work root", - containerClient: &mockContainerClient{}, - setupRunner: func(containerClient *mockContainerClient) *stageRunner { - return &stageRunner{ - workRoot: "/invalid/path", - containerClient: containerClient, - repo: &MockRepository{ - Dir: t.TempDir(), - }, - } - }, - wantErr: true, - wantErrMsg: "failed to create output dir", - }, - { - name: "failed to get changes from repo when releasing one library", - containerClient: &mockContainerClient{}, - setupRunner: func(containerClient *mockContainerClient) *stageRunner { - return &stageRunner{ - workRoot: t.TempDir(), - containerClient: containerClient, - library: "example-id", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "example-id", - SourceRoots: []string{"dir1"}, - }, - }, - }, - repo: &MockRepository{ - Dir: t.TempDir(), - GetCommitsForPathsSinceTagError: errors.New("simulated error when getting commits"), - }, - } - }, - wantErr: true, - wantErrMsg: "failed to fetch conventional commits for library", - }, - { - name: "failed to get changes from repo when releasing multiple libraries", - containerClient: &mockContainerClient{}, - setupRunner: func(containerClient *mockContainerClient) *stageRunner { - return &stageRunner{ - workRoot: t.TempDir(), - containerClient: containerClient, - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "example-id", - SourceRoots: []string{"dir1"}, - }, - }, - }, - repo: &MockRepository{ - Dir: t.TempDir(), - GetCommitsForPathsSinceTagError: errors.New("simulated error when getting commits"), - }, - } - }, - wantErr: true, - wantErrMsg: "failed to fetch conventional commits for library", - }, - { - name: "single library has no releasable units, no state change", - containerClient: &mockContainerClient{}, - setupRunner: func(containerClient *mockContainerClient) *stageRunner { - return &stageRunner{ - workRoot: os.TempDir(), - containerClient: containerClient, - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - Version: "1.0.0", - SourceRoots: []string{"dir1"}, - }, - }, - }, - repo: &MockRepository{ - Dir: t.TempDir(), - RemotesValue: []*legacygitrepo.Remote{ - { - Name: "origin", - URLs: []string{"https://github.com/googleapis/librarian.git"}, - }, - }, - ChangedFilesInCommitValue: []string{"file.txt"}, - GetCommitsForPathsSinceTagValue: []*legacygitrepo.Commit{ - { - Message: "chore: not releasable", - }, - { - Message: "test: not releasable", - }, - { - Message: "build: not releasable", - }, - }, - }, - ghClient: &mockGitHubClient{}, - librarianConfig: &legacyconfig.LibrarianConfig{}, - } - }, - }, - { - name: "release stage has multiple libraries but only one library has a releasable unit", - containerClient: &mockContainerClient{}, - dockerStageCalls: 1, - setupRunner: func(containerClient *mockContainerClient) *stageRunner { - return &stageRunner{ - workRoot: os.TempDir(), - containerClient: containerClient, - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - Version: "1.0.0", - ID: "another-example-id", - SourceRoots: []string{"dir1"}, - }, - { - Version: "2.0.0", - ID: "example-id", - SourceRoots: []string{"dir1"}, - }, - }, - }, - repo: &MockRepository{ - Dir: t.TempDir(), - RemotesValue: []*legacygitrepo.Remote{ - { - Name: "origin", - URLs: []string{"https://github.com/googleapis/librarian.git"}, - }, - }, - ChangedFilesInCommitValue: []string{"dir1/file.txt"}, - GetCommitsForPathsSinceTagValueByTag: map[string][]*legacygitrepo.Commit{ - "another-example-id-1.0.0": { - { - Message: "chore: not releasable", - }, - }, - "example-id-2.0.0": { - { - Message: "feat: a new feature", - }, - }, - }, - }, - ghClient: &mockGitHubClient{}, - librarianConfig: &legacyconfig.LibrarianConfig{}, - } - }, - want: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "another-example-id", - Version: "1.0.0", // version is NOT bumped. - APIs: []*legacyconfig.API{}, - SourceRoots: []string{"dir1"}, - PreserveRegex: []string{}, - RemoveRegex: []string{}, - }, - { - ID: "example-id", - Version: "2.1.0", // version is bumped. - APIs: []*legacyconfig.API{}, - SourceRoots: []string{"dir1"}, - PreserveRegex: []string{}, - RemoveRegex: []string{}, - }, - }, - }, - }, - { - name: "release stage has multiple libraries bumped in release only mode", - containerClient: &mockContainerClient{}, - dockerStageCalls: 1, - setupRunner: func(containerClient *mockContainerClient) *stageRunner { - return &stageRunner{ - workRoot: os.TempDir(), - containerClient: containerClient, - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "another-example-id", - Version: "1.0.0", - SourceRoots: []string{"dir1"}, - }, - { - ID: "example-id", - Version: "2.0.0", - SourceRoots: []string{"dir2"}, - }, - { - ID: "no-bump", - Version: "1.2.3", - SourceRoots: []string{"dir3"}, - }, - }, - }, - repo: &MockRepository{ - Dir: t.TempDir(), - RemotesValue: []*legacygitrepo.Remote{ - { - Name: "origin", - URLs: []string{"https://github.com/googleapis/librarian.git"}, - }, - }, - ChangedFilesInCommitValueByHash: map[string][]string{ - plumbing.NewHash("123456").String(): {"dir1/file.txt"}, - plumbing.NewHash("654321").String(): {"dir2/file.txt"}, - }, - GetCommitsForPathsSinceTagValueByTag: map[string][]*legacygitrepo.Commit{ - "another-example-id-1.0.0": { - { - Message: "fix: releasable", - Hash: plumbing.NewHash("123456"), - }, - }, - "example-id-2.0.0": { - { - Message: "fix: any message", - Hash: plumbing.NewHash("654321"), - }, - }, - }, - }, - ghClient: &mockGitHubClient{}, - librarianConfig: &legacyconfig.LibrarianConfig{ReleaseOnlyMode: true}, - } - }, - want: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "another-example-id", - Version: "1.1.0", // version is bumped. - APIs: []*legacyconfig.API{}, - SourceRoots: []string{"dir1"}, - PreserveRegex: []string{}, - RemoveRegex: []string{}, - }, - { - ID: "example-id", - Version: "2.1.0", // version is bumped. - APIs: []*legacyconfig.API{}, - SourceRoots: []string{"dir2"}, - PreserveRegex: []string{}, - RemoveRegex: []string{}, - }, - { - ID: "no-bump", - Version: "1.2.3", // version is NOT bumped. - APIs: []*legacyconfig.API{}, - SourceRoots: []string{"dir3"}, - PreserveRegex: []string{}, - RemoveRegex: []string{}, - }, - }, - }, - }, - { - name: "inputted library does not have a releasable unit, version is inputted", - containerClient: &mockContainerClient{}, - dockerStageCalls: 1, - setupRunner: func(containerClient *mockContainerClient) *stageRunner { - return &stageRunner{ - workRoot: os.TempDir(), - containerClient: containerClient, - library: "another-example-id", // release only for this library - libraryVersion: "3.0.0", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - Version: "1.0.0", - ID: "another-example-id", - SourceRoots: []string{"dir1"}, - }, - { - Version: "2.0.0", - ID: "example-id", - SourceRoots: []string{"dir1"}, - }, - }, - }, - repo: &MockRepository{ - Dir: t.TempDir(), - RemotesValue: []*legacygitrepo.Remote{ - { - Name: "origin", - URLs: []string{"https://github.com/googleapis/librarian.git"}, - }, - }, - ChangedFilesInCommitValue: []string{"dir1/file.txt"}, - GetCommitsForPathsSinceTagValueByTag: map[string][]*legacygitrepo.Commit{ - "another-example-id-1.0.0": { - { - Message: "chore: not releasable", - }, - }, - "example-id-2.0.0": { - { - Message: "feat: a new feature", - }, - }, - }, - }, - ghClient: &mockGitHubClient{}, - librarianConfig: &legacyconfig.LibrarianConfig{}, - } - }, - want: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - Version: "3.0.0", - ID: "another-example-id", - APIs: []*legacyconfig.API{}, - SourceRoots: []string{"dir1"}, - PreserveRegex: []string{}, - RemoveRegex: []string{}, - }, - { - Version: "2.0.0", - ID: "example-id", - APIs: []*legacyconfig.API{}, - SourceRoots: []string{"dir1"}, - PreserveRegex: []string{}, - RemoveRegex: []string{}, - }, - }, - }, - }, - { - name: "inputted library does not have a releasable unit, no version inputted", - containerClient: &mockContainerClient{}, - dockerStageCalls: 0, // version was not inputted, do not trigger a release - setupRunner: func(containerClient *mockContainerClient) *stageRunner { - return &stageRunner{ - workRoot: os.TempDir(), - containerClient: containerClient, - library: "another-example-id", // release only for this library - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - Version: "1.0.0", - ID: "another-example-id", - SourceRoots: []string{"dir1"}, - }, - { - Version: "2.0.0", - ID: "example-id", - SourceRoots: []string{"dir1"}, - }, - }, - }, - repo: &MockRepository{ - Dir: t.TempDir(), - ChangedFilesInCommitValue: []string{"dir1/file.txt"}, - GetCommitsForPathsSinceTagValueByTag: map[string][]*legacygitrepo.Commit{ - "another-example-id-1.0.0": { - { - Message: "chore: not releasable", - }, - }, - "example-id-2.0.0": { - { - Message: "feat: a new feature", - }, - }, - }, - }, - ghClient: &mockGitHubClient{}, - librarianConfig: &legacyconfig.LibrarianConfig{}, - } - }, - wantErr: true, - wantErrMsg: "library does not have a releasable unit and will not be released. Use the version flag to force a release for", - }, - { - name: "failed to commit and push", - containerClient: &mockContainerClient{}, - dockerStageCalls: 1, - setupRunner: func(containerClient *mockContainerClient) *stageRunner { - return &stageRunner{ - workRoot: os.TempDir(), - containerClient: containerClient, - push: true, - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - Version: "1.0.0", - ID: "example-id", - SourceRoots: []string{"dir1"}, - }, - }, - }, - repo: &MockRepository{ - Dir: t.TempDir(), - RemotesValue: []*legacygitrepo.Remote{ - { - Name: "origin", - URLs: []string{"https://github.com/googleapis/librarian.git"}, - }, - }, - ChangedFilesInCommitValue: []string{"dir1/file.txt"}, - GetCommitsForPathsSinceTagValue: []*legacygitrepo.Commit{ - { - Message: "feat: a feature", - }, - }, - // This AddAll is used in commitAndPush(). If commitAndPush() is invoked, - // then this test should error out - AddAllError: errors.New("unable to add all files"), - }, - librarianConfig: &legacyconfig.LibrarianConfig{}, - } - }, - wantErr: true, - wantErrMsg: "failed to commit and push", - }, - { - name: "run release stage command with symbolic link", - containerClient: &mockContainerClient{}, - dockerStageCalls: 1, - setupRunner: func(containerClient *mockContainerClient) *stageRunner { - return &stageRunner{ - workRoot: t.TempDir(), - containerClient: containerClient, - library: "example-id", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - Version: "1.0.0", - ID: "example-id", - SourceRoots: []string{ - "dir1", - }, - }, - }, - }, - repo: mockRepoWithReleasableUnit, - librarianConfig: &legacyconfig.LibrarianConfig{}, - } - }, - files: map[string]string{ - "dir1/file1.txt": "hello", - }, - want: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "example-id", - Version: "1.1.0", - APIs: []*legacyconfig.API{}, - SourceRoots: []string{ - "dir1", - }, - PreserveRegex: []string{}, - RemoveRegex: []string{}, - }, - }, - }, - }, - } { - t.Run(test.name, func(t *testing.T) { - runner := test.setupRunner(test.containerClient) - - // Setup library files before running the command. - repoDir := runner.repo.GetDir() - outputDir := filepath.Join(runner.workRoot, "output") - for path, content := range test.files { - // Create files in repoDir and outputDir because the run() function - // will copy files from outputDir to repoDir. - for _, dir := range []string{repoDir, outputDir} { - fullPath := filepath.Join(dir, path) - if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil { - t.Fatalf("os.MkdirAll() = %v", err) - } - if err := os.WriteFile(fullPath, []byte(content), 0644); err != nil { - t.Fatalf("os.WriteFile() = %v", err) - } - } - } - // Create a symbolic link for the test case with symbolic links. - if test.name == "run release stage command with symbolic link" { - if err := os.Symlink(filepath.Join(repoDir, "dir1/file1.txt"), - filepath.Join(repoDir, "dir1/symlink.txt")); err != nil { - t.Fatalf("os.Symlink() = %v", err) - } - } - librarianDir := filepath.Join(repoDir, ".librarian") - if err := os.MkdirAll(librarianDir, 0755); err != nil { - t.Fatalf("os.MkdirAll() = %v", err) - } - - // Create the librarian state file. - stateFile := filepath.Join(repoDir, ".librarian/state.yaml") - if err := os.MkdirAll(filepath.Dir(stateFile), 0755); err != nil { - t.Fatalf("os.MkdirAll() = %v", err) - } - if err := os.WriteFile(stateFile, []byte{}, 0644); err != nil { - t.Fatalf("os.WriteFile() = %v", err) - } - - err := runner.run(t.Context()) - - // Check how many times the docker container has been called. If a release is to proceed - // we expect this to be 1. Otherwise, the dockerStageCalls should be 0. Run this check even - // if there is an error that is wanted to ensure that a docker request is only made when - // we want it to. - if diff := cmp.Diff(test.dockerStageCalls, test.containerClient.stageCalls); diff != "" { - t.Errorf("docker stage calls mismatch (-want +got):\n%s", diff) - } - - if test.wantErr { - if err == nil { - t.Fatal("run() should return error") - } - - if !strings.Contains(err.Error(), test.wantErrMsg) { - t.Errorf("want error message: %q, got %q", test.wantErrMsg, err.Error()) - } - - return - } - if err != nil { - t.Errorf("run() failed: %s", err.Error()) - } - - // load librarian state from state.yaml, which should contain updated - // library state. - bytes, err := os.ReadFile(filepath.Join(repoDir, ".librarian/state.yaml")) - if err != nil { - t.Fatal(err) - } - - // If there is no release triggered for any library, then the librarian state - // is not written back. The `want` value for the librarian state is nil - got, err := yaml.Unmarshal[legacyconfig.LibrarianState](bytes) - if err != nil { - t.Fatal(err) - } - if len(bytes) == 0 { - got = nil - } - - if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("state mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestRunStageCommand(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - state *legacyconfig.LibrarianState - config *legacyconfig.LibrarianConfig - repo legacygitrepo.Repository - client ContainerClient - want *legacyconfig.LibrarianState - }{ - { - name: "global_file_commits_appear_in_multiple_libraries", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "another-example-id", - Version: "1.0.0", - SourceRoots: []string{ - "dir3", - "one/global/example.txt", - }, - RemoveRegex: []string{ - "dir3", - }, - }, - { - ID: "example-id", - Version: "2.0.0", - SourceRoots: []string{ - "dir1", - "one/global/example.txt", - }, - RemoveRegex: []string{ - "dir1", - }, - }, - }, - }, - config: &legacyconfig.LibrarianConfig{ - GlobalFilesAllowlist: []*legacyconfig.GlobalFile{ - { - Path: "one/global/example.txt", - Permissions: "read-write", - }, - }, - }, - repo: &MockRepository{ - Dir: t.TempDir(), - RemotesValue: []*legacygitrepo.Remote{ - { - Name: "origin", - URLs: []string{"https://github.com/googleapis/librarian.git"}, - }, - }, - GetCommitsForPathsSinceTagValueByTag: map[string][]*legacygitrepo.Commit{ - "another-example-id-1.0.0": { - { - Hash: plumbing.NewHash("123456"), - Message: "feat: bump version", - }, - }, - "example-id-2.0.0": { - { - Hash: plumbing.NewHash("123456"), - Message: "feat: bump version", - }, - }, - }, - ChangedFilesInCommitValueByHash: map[string][]string{ - plumbing.NewHash("123456").String(): { - "one/global/example.txt", - }, - }, - }, - client: &mockContainerClient{}, - want: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "another-example-id", - Version: "1.1.0", - PreviousVersion: "1.0.0", - SourceRoots: []string{ - "dir3", - "one/global/example.txt", - }, - RemoveRegex: []string{ - "dir3", - }, - Changes: []*legacyconfig.Commit{ - { - Type: "feat", - Subject: "bump version", - CommitHash: "1234560000000000000000000000000000000000", - LibraryIDs: "another-example-id", - }, - }, - ReleaseTriggered: true, - }, - { - ID: "example-id", - Version: "2.1.0", - PreviousVersion: "2.0.0", - SourceRoots: []string{ - "dir1", - "one/global/example.txt", - }, - RemoveRegex: []string{ - "dir1", - }, - Changes: []*legacyconfig.Commit{ - { - Type: "feat", - Subject: "bump version", - CommitHash: "1234560000000000000000000000000000000000", - LibraryIDs: "example-id", - }, - }, - ReleaseTriggered: true, - }, - }, - }, - }, - } { - output := t.TempDir() - for _, globalFile := range test.config.GlobalFilesAllowlist { - file := filepath.Join(output, globalFile.Path) - if err := os.MkdirAll(filepath.Dir(file), 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(file, []byte("new content"), 0755); err != nil { - t.Fatal(err) - } - } - r := &stageRunner{ - repo: test.repo, - state: test.state, - librarianConfig: test.config, - containerClient: test.client, - } - err := r.runStageCommand(t.Context(), output) - if err != nil { - t.Errorf("failed to run runStageCommand(): %q", err.Error()) - return - } - if diff := cmp.Diff(test.want, r.state); diff != "" { - t.Errorf("commit filter mismatch (-want +got):\n%s", diff) - } - } -} - -func TestProcessLibrary_ReleaseOnlyMode(t *testing.T) { - for _, test := range []struct { - name string - releaseOnlyMode bool - libraryState *legacyconfig.LibraryState - repo legacygitrepo.Repository - want *legacyconfig.LibraryState - }{ - { - name: "libraries in repo with release only mode have changelogs", - releaseOnlyMode: true, - libraryState: &legacyconfig.LibraryState{ - ID: "one-id", - Version: "1.2.3", - SourceRoots: []string{"one-id"}, - }, - repo: &MockRepository{ - GetCommitsForPathsSinceTagValueByTag: map[string][]*legacygitrepo.Commit{ - "one-id-1.2.3": { - { - Hash: plumbing.NewHash("123456"), - Message: "chore: one feat", - }, - { - Hash: plumbing.NewHash("654321"), - Message: "fix: another feat", - }, - }, - }, - ChangedFilesInCommitValueByHash: map[string][]string{ - plumbing.NewHash("123456").String(): {"one-id/file1.txt", "one-id/file2.txt"}, - plumbing.NewHash("654321").String(): {"one-id/file3.txt", "one-id/file4.txt"}, - }, - }, - want: &legacyconfig.LibraryState{ - ID: "one-id", - PreviousVersion: "1.2.3", - Version: "1.3.0", - SourceRoots: []string{"one-id"}, - ReleaseTriggered: true, - Changes: []*legacyconfig.Commit{ - { - Type: "chore", - Subject: "one feat", - CommitHash: "1234560000000000000000000000000000000000", - LibraryIDs: "one-id", - }, - { - Type: "fix", - Subject: "another feat", - CommitHash: "6543210000000000000000000000000000000000", - LibraryIDs: "one-id", - }, - }, - }, - }, - { - name: "libraries in repo with normal mode have changelogs", - libraryState: &legacyconfig.LibraryState{ - ID: "one-id", - Version: "1.2.3", - SourceRoots: []string{"one-id"}, - }, - repo: &MockRepository{ - GetCommitsForPathsSinceTagValueByTag: map[string][]*legacygitrepo.Commit{ - "one-id-1.2.3": { - { - Hash: plumbing.NewHash("123456"), - Message: "feat: one feat", - }, - { - Hash: plumbing.NewHash("654321"), - Message: "feat: another feat", - }, - }, - }, - ChangedFilesInCommitValueByHash: map[string][]string{ - plumbing.NewHash("123456").String(): {"one-id/file1.txt", "one-id/file2.txt"}, - plumbing.NewHash("654321").String(): {"one-id/file3.txt", "one-id/file4.txt"}, - }, - }, - want: &legacyconfig.LibraryState{ - ID: "one-id", - PreviousVersion: "1.2.3", - Version: "1.3.0", - SourceRoots: []string{"one-id"}, - ReleaseTriggered: true, - Changes: []*legacyconfig.Commit{ - { - Type: "feat", - Subject: "one feat", - CommitHash: "1234560000000000000000000000000000000000", - LibraryIDs: "one-id", - }, - { - Type: "feat", - Subject: "another feat", - CommitHash: "6543210000000000000000000000000000000000", - LibraryIDs: "one-id", - }, - }, - }, - }, - } { - state := &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - test.libraryState, - }, - } - r := &stageRunner{ - repo: test.repo, - state: state, - librarianConfig: &legacyconfig.LibrarianConfig{ReleaseOnlyMode: test.releaseOnlyMode}, - } - err := r.processLibrary(t.Context(), test.libraryState) - if err != nil { - t.Error(err) - } - if diff := cmp.Diff(test.want, r.state.Libraries[0]); diff != "" { - t.Errorf("commit filter mismatch (-want +got):\n%s", diff) - } - } -} - -func TestProcessLibrary(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - libraryState *legacyconfig.LibraryState - repo legacygitrepo.Repository - wantErr bool - wantErrMsg string - }{ - { - name: "failed to get commit history of one library", - libraryState: &legacyconfig.LibraryState{ - ID: "one-id", - Version: "1.2.3", - }, - repo: &MockRepository{ - GetCommitsForPathsSinceTagError: errors.New("simulated error when getting commits"), - }, - wantErr: true, - wantErrMsg: "failed to fetch conventional commits for library", - }, - { - name: "does not search for git tag for 0.0.0 version", - libraryState: &legacyconfig.LibraryState{ - ID: "one-id", - Version: "0.0.0", - }, - repo: &MockRepository{}, - }, - } { - state := &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - test.libraryState, - }, - } - r := &stageRunner{ - repo: test.repo, - state: state, - } - err := r.processLibrary(t.Context(), test.libraryState) - if test.wantErr { - if err == nil { - t.Fatal("processLibrary() should return error") - } - if !strings.Contains(err.Error(), test.wantErrMsg) { - t.Errorf("want error message: %q, got %q", test.wantErrMsg, err.Error()) - } - return - } - if err != nil { - t.Errorf("failed to run processLibrary(): %q", err.Error()) - } - if test.libraryState.Version == "0.0.0" && test.repo.(*MockRepository).GetCommitsForPathsSinceTagLastTagName != "" { - t.Errorf("GetCommitsForPathsSinceTag should be called with an empty tag name for version 0.0.0, got %q", test.repo.(*MockRepository).GetCommitsForPathsSinceTagLastTagName) - } - } -} - -func TestFilterCommitsByLibraryID(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - commits []*legacygitrepo.ConventionalCommit - LibraryID string - want []*legacygitrepo.ConventionalCommit - }{ - { - name: "commits_all_match_libraryID", - commits: []*legacygitrepo.ConventionalCommit{ - { - LibraryID: "library-one", - Type: "feat", - }, - { - LibraryID: "library-one", - Type: "chore", - }, - { - LibraryID: "library-one", - Type: "deps", - }, - }, - LibraryID: "library-one", - want: []*legacygitrepo.ConventionalCommit{ - { - LibraryID: "library-one", - Type: "feat", - }, - { - LibraryID: "library-one", - Type: "chore", - }, - { - LibraryID: "library-one", - Type: "deps", - }, - }, - }, - { - name: "some_commits_match_libraryID", - commits: []*legacygitrepo.ConventionalCommit{ - { - LibraryID: "library-one", - Type: "feat", - }, - { - LibraryID: "library-two", - Type: "chore", - }, - { - LibraryID: "library-three", - Type: "deps", - }, - }, - LibraryID: "library-one", - want: []*legacygitrepo.ConventionalCommit{ - { - LibraryID: "library-one", - Type: "feat", - }, - }, - }, - { - name: "some_commits_have_library_id_in_footer", - commits: []*legacygitrepo.ConventionalCommit{ - { - LibraryID: "library-one", - Type: "feat", - Footers: map[string]string{ - "Library-IDs": "library-one,library-two", - }, - }, - { - LibraryID: "library-two", - Type: "chore", - Footers: map[string]string{ - "Library-IDs": "library-one,library-two", - }, - }, - { - LibraryID: "library-three", - Type: "deps", - }, - }, - LibraryID: "library-one", - want: []*legacygitrepo.ConventionalCommit{ - { - LibraryID: "library-one", - Type: "feat", - Footers: map[string]string{ - "Library-IDs": "library-one,library-two", - }, - }, - { - LibraryID: "library-two", - Type: "chore", - Footers: map[string]string{ - "Library-IDs": "library-one,library-two", - }, - }, - }, - }, - { - name: "some_commits_have_library_id_that_is_prefix_of_another", - commits: []*legacygitrepo.ConventionalCommit{ - { - LibraryID: "library-one", - Type: "feat", - Footers: map[string]string{ - "Library-IDs": "library-one,library-one_suffix", - }, - }, - { - LibraryID: "library-one-suffix", - Type: "chore", - Footers: map[string]string{ - "Library-IDs": "library-one-suffix", - }, - }, - }, - LibraryID: "library-one", - want: []*legacygitrepo.ConventionalCommit{ - { - LibraryID: "library-one", - Type: "feat", - Footers: map[string]string{ - "Library-IDs": "library-one,library-one_suffix", - }, - }, - }, - }, - { - name: "no_commits_match_libraryID", - commits: []*legacygitrepo.ConventionalCommit{ - { - LibraryID: "library-one", - Type: "feat", - }, - { - LibraryID: "library-two", - Type: "chore", - }, - { - LibraryID: "library-three", - Type: "deps", - }, - }, - LibraryID: "invalid-library", - }, - } { - t.Run(test.name, func(t *testing.T) { - got := filterCommitsByLibraryID(test.commits, test.LibraryID) - if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("commit filter mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestUpdateLibrary(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - libraryState *legacyconfig.LibraryState - library string // this is the `--library` input - libraryVersion string // this is the `--version` input - commits []*legacygitrepo.ConventionalCommit - want *legacyconfig.LibraryState - wantErr bool - wantErrMsg string - }{ - { - name: "update a library, automatic version calculation", - libraryState: &legacyconfig.LibraryState{ - ID: "one-id", - Version: "1.2.3", - }, - commits: []*legacygitrepo.ConventionalCommit{ - { - Type: "fix", - Subject: "change a typo", - }, - { - Type: "feat", - Subject: "add a config file", - Body: "This is the body.", - Footers: map[string]string{"PiperOrigin-RevId": "12345"}, - }, - }, - want: &legacyconfig.LibraryState{ - ID: "one-id", - Version: "1.3.0", - PreviousVersion: "1.2.3", - Changes: []*legacyconfig.Commit{ - { - Type: "fix", - Subject: "change a typo", - LibraryIDs: "one-id", - }, - { - Type: "feat", - Subject: "add a config file", - Body: "This is the body.", - PiperCLNumber: "12345", - LibraryIDs: "one-id", - }, - }, - ReleaseTriggered: true, - }, - }, - { - name: "update a library with releasable units, valid version inputted", - libraryState: &legacyconfig.LibraryState{ - ID: "one-id", - Version: "1.2.3", - }, - libraryVersion: "5.0.0", - commits: []*legacygitrepo.ConventionalCommit{ - { - Type: "fix", - Subject: "change a typo", - }, - { - Type: "feat", - Subject: "add a config file", - Body: "This is the body.", - Footers: map[string]string{"PiperOrigin-RevId": "12345"}, - }, - }, - want: &legacyconfig.LibraryState{ - ID: "one-id", - Version: "5.0.0", // Use the `--version` value` - PreviousVersion: "1.2.3", - Changes: []*legacyconfig.Commit{ - { - Type: "fix", - Subject: "change a typo", - LibraryIDs: "one-id", - }, - { - Type: "feat", - Subject: "add a config file", - Body: "This is the body.", - PiperCLNumber: "12345", - LibraryIDs: "one-id", - }, - }, - ReleaseTriggered: true, - }, - }, - { - name: "update a library with releasable units, invalid version inputted", - libraryState: &legacyconfig.LibraryState{ - ID: "one-id", - Version: "1.2.3", - }, - libraryVersion: "1.0.0", - commits: []*legacygitrepo.ConventionalCommit{ - { - Type: "fix", - Subject: "change a typo", - }, - { - Type: "feat", - Subject: "add a config file", - Body: "This is the body.", - Footers: map[string]string{"PiperOrigin-RevId": "12345"}, - }, - }, - wantErr: true, - wantErrMsg: "inputted version is not SemVer greater than the current version. Set a version SemVer greater than current than", - }, - { - name: "update a library with library ids in footer", - libraryState: &legacyconfig.LibraryState{ - ID: "one-id", - Version: "1.2.3", - }, - commits: []*legacygitrepo.ConventionalCommit{ - { - Type: "feat", - Subject: "add a config file", - Body: "This is the body.", - Footers: map[string]string{"Library-IDs": "a,b,c"}, - }, - }, - want: &legacyconfig.LibraryState{ - ID: "one-id", - Version: "1.3.0", - PreviousVersion: "1.2.3", - Changes: []*legacyconfig.Commit{ - { - Type: "feat", - Subject: "add a config file", - Body: "This is the body.", - LibraryIDs: "a,b,c", - }, - }, - ReleaseTriggered: true, - }, - }, - { - name: "library has breaking changes", - libraryState: &legacyconfig.LibraryState{ - ID: "one-id", - Version: "1.2.3", - }, - commits: []*legacygitrepo.ConventionalCommit{ - { - Type: "feat", - Subject: "add another config file", - Body: "This is the body", - Footers: map[string]string{ - "BREAKING CHANGE": "this is a breaking change", - }, - IsBreaking: true, - }, - { - Type: "feat", - Subject: "change a typo", - IsBreaking: true, - }, - }, - want: &legacyconfig.LibraryState{ - ID: "one-id", - Version: "2.0.0", - PreviousVersion: "1.2.3", - Changes: []*legacyconfig.Commit{ - { - Type: "feat", - Subject: "add another config file", - Body: "This is the body", - LibraryIDs: "one-id", - }, - { - Type: "feat", - Subject: "change a typo", - LibraryIDs: "one-id", - }, - }, - ReleaseTriggered: true, - }, - }, - { - name: "library has no changes", - libraryState: &legacyconfig.LibraryState{ - ID: "one-id", - Version: "1.2.3", - }, - want: &legacyconfig.LibraryState{ - ID: "one-id", - Version: "1.2.3", - }, - }, - { - name: "library has no releasable units and is inputted for release without a version", - libraryState: &legacyconfig.LibraryState{ - ID: "one-id", - Version: "1.2.3", - }, - library: "one-id", - commits: []*legacygitrepo.ConventionalCommit{ - { - Type: "chore", - Subject: "a chore", - }, - }, - wantErr: true, - wantErrMsg: "library does not have a releasable unit and will not be released. Use the version flag to force a release for", - }, - { - name: "library has no releasable units and is inputted for release with a specific version", - libraryState: &legacyconfig.LibraryState{ - ID: "one-id", - Version: "1.2.3", - }, - library: "one-id", - libraryVersion: "5.0.0", - commits: []*legacygitrepo.ConventionalCommit{ - { - Type: "chore", - Subject: "a chore", - }, - }, - want: &legacyconfig.LibraryState{ - ID: "one-id", - PreviousVersion: "1.2.3", - Version: "5.0.0", // Use the `--version` override value - ReleaseTriggered: true, - Changes: []*legacyconfig.Commit{ - { - Type: "chore", - Subject: "a chore", - LibraryIDs: "one-id", - }, - }, - }, - }, - } { - t.Run(test.name, func(t *testing.T) { - r := &stageRunner{ - state: &legacyconfig.LibrarianState{}, - library: test.library, - libraryVersion: test.libraryVersion, - } - err := r.updateLibrary(t.Context(), test.libraryState, test.commits) - - if test.wantErr { - if err == nil { - t.Fatal("updateLibrary() should return error") - } - if !strings.Contains(err.Error(), test.wantErrMsg) { - t.Errorf("want error message: %q, got %q", test.wantErrMsg, err.Error()) - } - return - } - if err != nil { - t.Fatalf("failed to run updateLibrary(): %q", err.Error()) - } - if diff := cmp.Diff(test.want, test.libraryState); diff != "" { - t.Errorf("state mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestDetermineNextVersion(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - commits []*legacygitrepo.ConventionalCommit - currentVersion string - libraryID string - config *legacyconfig.Config - librarianConfig *legacyconfig.LibrarianConfig - ghClient GitHubClient - branch string - wantVersion string - wantErr bool - wantErrMsg string - }{ - { - name: "from commits", - commits: []*legacygitrepo.ConventionalCommit{ - {Type: "feat"}, - }, - config: &legacyconfig.Config{ - Library: "some-library", - }, - libraryID: "some-library", - librarianConfig: &legacyconfig.LibrarianConfig{ - Libraries: []*legacyconfig.LibraryConfig{}, - }, - currentVersion: "1.0.0", - wantVersion: "1.1.0", - wantErr: false, - }, - { - name: "with config.yaml override version", - commits: []*legacygitrepo.ConventionalCommit{ - {Type: "feat"}, - }, - config: &legacyconfig.Config{ - Library: "some-library", - }, - libraryID: "some-library", - librarianConfig: &legacyconfig.LibrarianConfig{ - Libraries: []*legacyconfig.LibraryConfig{ - { - LibraryID: "some-library", - NextVersion: "2.3.4", - }, - }, - }, - currentVersion: "1.0.0", - wantVersion: "2.3.4", - wantErr: false, - }, - { - name: "with outdated config.yaml override version", - commits: []*legacygitrepo.ConventionalCommit{ - {Type: "feat"}, - }, - config: &legacyconfig.Config{ - Library: "some-library", - }, - libraryID: "some-library", - librarianConfig: &legacyconfig.LibrarianConfig{ - Libraries: []*legacyconfig.LibraryConfig{ - { - LibraryID: "some-library", - NextVersion: "2.3.4", - }, - }, - }, - currentVersion: "2.4.0", - wantVersion: "2.5.0", - wantErr: false, - }, - { - name: "preview ahead prerelease bump", - branch: "preview", - config: &legacyconfig.Config{ - Library: "some-library", - }, - libraryID: "some-library", - librarianConfig: &legacyconfig.LibrarianConfig{ - Libraries: []*legacyconfig.LibraryConfig{}, - }, - ghClient: &mockGitHubClient{ - librarianState: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - {ID: "some-library", Version: "1.0.0", SourceRoots: []string{"test"}}, - }, - }, - }, - currentVersion: "1.1.0-preview.1", - wantVersion: "1.1.0-preview.2", - }, - { - name: "preview-only prerelease bump", - branch: "preview", - config: &legacyconfig.Config{ - Library: "some-library", - }, - libraryID: "some-library", - librarianConfig: &legacyconfig.LibrarianConfig{ - Libraries: []*legacyconfig.LibraryConfig{}, - }, - ghClient: &mockGitHubClient{ - librarianState: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - {ID: "not-the-same-lib", Version: "1.0.0", SourceRoots: []string{"other"}}, - }, - }, - }, - currentVersion: "1.1.0-preview.1", - wantVersion: "1.1.0-preview.2", - }, - } { - t.Run(test.name, func(t *testing.T) { - runner := &stageRunner{ - state: &legacyconfig.LibrarianState{}, - libraryVersion: test.config.LibraryVersion, - librarianConfig: test.librarianConfig, - ghClient: test.ghClient, - branch: test.branch, - } - got, err := runner.determineNextVersion(t.Context(), test.commits, test.currentVersion, test.libraryID) - if test.wantErr { - if err == nil { - t.Fatal("determineNextVersion() should return error") - } - - if !strings.Contains(err.Error(), test.wantErrMsg) { - t.Errorf("want error message: %q, got %q", test.wantErrMsg, err.Error()) - } - - return - } - if err != nil { - t.Fatal(err) - } - if diff := cmp.Diff(test.wantVersion, got); diff != "" { - t.Errorf("state mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestSyncVersion(t *testing.T) { - for _, test := range []struct { - name string - legacyLibraries []*legacyconfig.LibraryState - libraries []*config.Library - want []*config.Library - }{ - { - name: "update versions for libraries in legacylibrarian state.yaml", - legacyLibraries: []*legacyconfig.LibraryState{ - {ID: "lib1", Version: "1.1.0"}, - {ID: "lib2", Version: "2.1.0"}, - }, - libraries: []*config.Library{ - {Name: "lib1", Version: "1.0.0"}, - {Name: "lib3", Version: "3.0.0"}, - }, - want: []*config.Library{ - {Name: "lib1", Version: "1.1.0"}, - {Name: "lib3", Version: "3.0.0"}, - }, - }, - { - name: "empty version is not synced", - legacyLibraries: []*legacyconfig.LibraryState{ - {ID: "lib1"}, - }, - libraries: []*config.Library{ - {Name: "lib1", Version: "1.0.0"}, - {Name: "lib2", Version: "2.2.0"}, - }, - want: []*config.Library{ - {Name: "lib1", Version: "1.0.0"}, - {Name: "lib2", Version: "2.2.0"}, - }, - }, - { - name: "same version is not changed", - legacyLibraries: []*legacyconfig.LibraryState{ - {ID: "lib1", Version: "1.0.0"}, - }, - libraries: []*config.Library{ - {Name: "lib1", Version: "1.0.0"}, - }, - want: []*config.Library{ - {Name: "lib1", Version: "1.0.0"}, - }, - }, - { - name: "sync preview version when present", - legacyLibraries: []*legacyconfig.LibraryState{ - {ID: "lib1", Version: "1.1.0"}, - {ID: "lib1-preview", Version: "1.1.0-preview.2"}, - }, - libraries: []*config.Library{ - { - Name: "lib1", - Version: "1.0.0", - Preview: &config.Library{ - Version: "1.1.0-preview.1", - }, - }, - }, - want: []*config.Library{ - { - Name: "lib1", - Version: "1.1.0", - Preview: &config.Library{ - Version: "1.1.0-preview.2", - }, - }, - }, - }, - { - name: "sync preview version when main library is missing in state", - legacyLibraries: []*legacyconfig.LibraryState{ - {ID: "lib1-preview", Version: "1.1.0-preview.2"}, - }, - libraries: []*config.Library{ - { - Name: "lib1", - Version: "1.0.0", - Preview: &config.Library{ - Version: "1.1.0-preview.1", - }, - }, - }, - want: []*config.Library{ - { - Name: "lib1", - Version: "1.0.0", - Preview: &config.Library{ - Version: "1.1.0-preview.2", - }, - }, - }, - }, - } { - t.Run(test.name, func(t *testing.T) { - state := &legacyconfig.LibrarianState{Libraries: test.legacyLibraries} - cfg := &config.Config{Libraries: test.libraries} - got, err := syncVersion(state, cfg) - if err != nil { - t.Fatal(err) - } - if diff := cmp.Diff(test.want, got.Libraries); diff != "" { - t.Errorf("mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestSyncVersion_Error(t *testing.T) { - t.Run("main library regression", func(t *testing.T) { - state := &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - {ID: "lib1", Version: "1.0.0"}, - }, - } - cfg := &config.Config{ - Libraries: []*config.Library{ - {Name: "lib1", Version: "1.1.0"}, - }, - } - _, err := syncVersion(state, cfg) - if !errors.Is(err, errVersionRegression) { - t.Errorf("got error %v, want %v", err, errVersionRegression) - } - }) - - t.Run("preview library regression", func(t *testing.T) { - state := &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - {ID: "lib1", Version: "1.1.0"}, - {ID: "lib1-preview", Version: "1.1.0-preview.1"}, - }, - } - cfg := &config.Config{ - Libraries: []*config.Library{ - { - Name: "lib1", - Version: "1.0.0", - Preview: &config.Library{ - Version: "1.1.0-preview.2", - }, - }, - }, - } - _, err := syncVersion(state, cfg) - if !errors.Is(err, errVersionRegression) { - t.Errorf("got error %v, want %v", err, errVersionRegression) - } - }) -} - -func TestUpdateLibrarianYAML(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - config *config.Config - state *legacyconfig.LibrarianState - wantConfig *config.Config - }{ - { - name: "updates versions", - config: &config.Config{ - Sources: &config.Sources{ - Googleapis: &config.Source{ - Commit: "some-commit", - }, - }, - Libraries: []*config.Library{ - { - Name: "asset", - Version: "1.0.0", - }, - { - Name: "billing", - Version: "2.0.0", - }, - }, - }, - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "asset", - Version: "1.1.0", - }, - { - ID: "billing", - Version: "2.0.0", - }, - }, - }, - wantConfig: &config.Config{ - Sources: &config.Sources{ - Googleapis: &config.Source{ - Commit: "some-commit", - }, - }, - Libraries: []*config.Library{ - { - Name: "asset", - Version: "1.1.0", - }, - { - Name: "billing", - Version: "2.0.0", - }, - }, - }, - }, - { - name: "ignores missing libraries in state", - config: &config.Config{ - Sources: &config.Sources{ - Googleapis: &config.Source{ - Commit: "some-commit", - }, - }, - Libraries: []*config.Library{ - { - Name: "asset", - Version: "1.0.0", - }, - }, - }, - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{}, - }, - wantConfig: &config.Config{ - Sources: &config.Sources{ - Googleapis: &config.Source{ - Commit: "some-commit", - }, - }, - Libraries: []*config.Library{ - { - Name: "asset", - Version: "1.0.0", - }, - }, - }, - }, - { - name: "ignores empty version in state", - config: &config.Config{ - Sources: &config.Sources{ - Googleapis: &config.Source{ - Commit: "some-commit", - }, - }, - Libraries: []*config.Library{ - { - Name: "asset", - Version: "1.0.0", - }, - }, - }, - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "asset", - }, - }, - }, - wantConfig: &config.Config{ - Sources: &config.Sources{ - Googleapis: &config.Source{ - Commit: "some-commit", - }, - }, - Libraries: []*config.Library{ - { - Name: "asset", - Version: "1.0.0", - }, - }, - }, - }, - } { - t.Run(test.name, func(t *testing.T) { - repoDir := t.TempDir() - configPath := filepath.Join(repoDir, config.LibrarianYAML) - if err := yaml.Write(configPath, test.config); err != nil { - t.Fatal(err) - } - runner := &stageRunner{ - repo: &MockRepository{Dir: repoDir}, - state: test.state, - } - err := runner.updateLibrarianYAML(t.Context()) - if err != nil { - t.Fatal(err) - } - gotConfig, err := yaml.Read[config.Config](configPath) - if err != nil { - t.Fatal(err) - } - if diff := cmp.Diff(test.wantConfig, gotConfig); diff != "" { - t.Errorf("mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestUpdateLibrarianYAML_Error(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - config *config.Config - state *legacyconfig.LibrarianState - wantError error - }{ - { - name: "updates versions", - config: &config.Config{ - Sources: &config.Sources{ - Googleapis: &config.Source{ - Commit: "some-commit", - }, - }, - Libraries: []*config.Library{ - { - Name: "asset", - Version: "1.2.0", - }, - { - Name: "billing", - Version: "2.0.0", - }, - }, - }, - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "asset", - Version: "1.1.0", - }, - { - ID: "billing", - Version: "2.0.0", - }, - }, - }, - wantError: errVersionRegression, - }, - } { - t.Run(test.name, func(t *testing.T) { - repoDir := t.TempDir() - configPath := filepath.Join(repoDir, config.LibrarianYAML) - if err := yaml.Write(configPath, test.config); err != nil { - t.Fatal(err) - } - runner := &stageRunner{ - repo: &MockRepository{Dir: repoDir}, - state: test.state, - } - err := runner.updateLibrarianYAML(t.Context()) - if !errors.Is(err, test.wantError) { - t.Fatalf("want error %q, got %q", test.wantError, err) - } - }) - } -} - -func TestUpdateLibrarianYAML_NoConfigFile(t *testing.T) { - t.Parallel() - repoDir := t.TempDir() - runner := &stageRunner{ - repo: &MockRepository{Dir: repoDir}, - } - - if err := runner.updateLibrarianYAML(t.Context()); err != nil { - t.Errorf("updateLibrarianYAML() with no config file should return nil, got %v", err) - } -} diff --git a/internal/legacylibrarian/legacylibrarian/repository.go b/internal/legacylibrarian/legacylibrarian/repository.go deleted file mode 100644 index 53905ceeda8..00000000000 --- a/internal/legacylibrarian/legacylibrarian/repository.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build !mock_github - -// This file contains the production implementations for functions that get -// GitHub repository details. - -package legacylibrarian - -import ( - "fmt" - - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" - "github.com/googleapis/librarian/internal/legacylibrarian/legacygithub" - "github.com/googleapis/librarian/internal/legacylibrarian/legacygitrepo" -) - -// GetGitHubRepository determines the GitHub repository from the configuration -// or the local git remote. -var GetGitHubRepository = func(cfg *legacyconfig.Config, languageRepo legacygitrepo.Repository) (*legacygithub.Repository, error) { - if isURL(cfg.Repo) { - return legacygithub.ParseRemote(cfg.Repo) - } - return GetGitHubRepositoryFromGitRepo(languageRepo) -} - -// GetGitHubRepositoryFromGitRepo determines the GitHub repository from the -// local git remote. -var GetGitHubRepositoryFromGitRepo = func(languageRepo legacygitrepo.Repository) (*legacygithub.Repository, error) { - remotes, err := languageRepo.Remotes() - if err != nil { - return nil, err - } - - for _, remote := range remotes { - if remote.Name == "origin" { - if len(remote.URLs) > 0 { - return legacygithub.ParseRemote(remote.URLs[0]) - } - } - } - - return nil, fmt.Errorf("could not find an 'origin' remote pointing to a GitHub https URL") -} diff --git a/internal/legacylibrarian/legacylibrarian/repository_mock.go b/internal/legacylibrarian/legacylibrarian/repository_mock.go deleted file mode 100644 index d714637f7de..00000000000 --- a/internal/legacylibrarian/legacylibrarian/repository_mock.go +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build mock_github - -// This file contains mock implementations of repository getters for use in -// end-to-end tests. It is compiled only when the 'mock_github' build tag is specified. - -package legacylibrarian - -import ( - "log/slog" - "os" - - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" - "github.com/googleapis/librarian/internal/legacylibrarian/legacygithub" - "github.com/googleapis/librarian/internal/legacylibrarian/legacygitrepo" -) - -// GetGitHubRepository returns a mock legacygithub.Repository object for e2e tests. -// It reads the LIBRARIAN_GITHUB_BASE_URL environment variable to configure -// the mock repository's BaseURL, allowing the test client to connect to a -// local httptest.Server. -var GetGitHubRepository = func(cfg *legacyconfig.Config, languageRepo legacygitrepo.Repository) (*legacygithub.Repository, error) { - slog.Info("using mock GitHub repository for e2e test") - baseURL := os.Getenv("LIBRARIAN_GITHUB_BASE_URL") - return &legacygithub.Repository{Owner: "test-owner", Name: "test-repo", BaseURL: baseURL}, nil -} - -// GetGitHubRepositoryFromGitRepo returns a mock legacygithub.Repository object for e2e tests. -// It reads the LIBRARIAN_GITHUB_BASE_URL environment variable to configure -// the mock repository's BaseURL. -var GetGitHubRepositoryFromGitRepo = func(languageRepo legacygitrepo.Repository) (*legacygithub.Repository, error) { - slog.Info("using mock GitHub repository for e2e test") - baseURL := os.Getenv("LIBRARIAN_GITHUB_BASE_URL") - return &legacygithub.Repository{Owner: "test-owner", Name: "test-repo", BaseURL: baseURL}, nil -} diff --git a/internal/legacylibrarian/legacylibrarian/repository_test.go b/internal/legacylibrarian/legacylibrarian/repository_test.go deleted file mode 100644 index 2a53afcd50b..00000000000 --- a/internal/legacylibrarian/legacylibrarian/repository_test.go +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacylibrarian - -import ( - "strings" - "testing" - - "github.com/go-git/go-git/v5" - gogitConfig "github.com/go-git/go-git/v5/config" - "github.com/google/go-cmp/cmp" - "github.com/googleapis/librarian/internal/legacylibrarian/legacygithub" - "github.com/googleapis/librarian/internal/legacylibrarian/legacygitrepo" -) - -func TestGetGitHubRepositoryFromGitRepo(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - remotes map[string][]string - wantRepo *legacygithub.Repository - wantErr bool - wantErrSubstr string - }{ - { - name: "origin is a GitHub remote", - remotes: map[string][]string{ - "origin": {"https://github.com/owner/repo.git"}, - }, - wantRepo: &legacygithub.Repository{Owner: "owner", Name: "repo"}, - }, - { - name: "No remotes", - remotes: map[string][]string{}, - wantErr: true, - wantErrSubstr: "could not find an 'origin' remote", - }, - { - name: "origin is not a GitHub remote", - remotes: map[string][]string{ - "origin": {"https://gitlab.com/owner/repo.git"}, - }, - wantErr: true, - wantErrSubstr: "is not a GitHub remote", - }, - { - name: "upstream is GitHub, but no origin", - remotes: map[string][]string{ - "gitlab": {"https://gitlab.com/owner/repo.git"}, - "upstream": {"https://github.com/gh-owner/gh-repo.git"}, - }, - wantErr: true, - wantErrSubstr: "could not find an 'origin' remote", - }, - { - name: "origin and upstream are GitHub remotes, should use origin", - remotes: map[string][]string{ - "origin": {"https://github.com/owner/repo.git"}, - "upstream": {"https://github.com/owner2/repo2.git"}, - }, - wantRepo: &legacygithub.Repository{Owner: "owner", Name: "repo"}, - }, - { - name: "origin is not GitHub, but upstream is", - remotes: map[string][]string{ - "origin": {"https://gitlab.com/owner/repo.git"}, - "upstream": {"https://github.com/gh-owner/gh-repo.git"}, - }, - wantErr: true, - wantErrSubstr: "is not a GitHub remote", - }, - { - name: "origin has multiple URLs, first is GitHub", - remotes: map[string][]string{ - "origin": {"https://github.com/owner/repo.git", "https://gitlab.com/owner/repo.git"}, - }, - wantRepo: &legacygithub.Repository{Owner: "owner", Name: "repo"}, - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - repo := newTestGitRepoWithRemotes(t, test.remotes) - - got, err := GetGitHubRepositoryFromGitRepo(repo) - - if test.wantErr { - if err == nil { - t.Fatalf("FetchGitHubRepoFromRemote() err = nil, want error containing %q", test.wantErrSubstr) - } - if !strings.Contains(err.Error(), test.wantErrSubstr) { - t.Errorf("FetchGitHubRepoFromRemote() err = %v, want error containing %q", err, test.wantErrSubstr) - } - } else { - if err != nil { - t.Errorf("FetchGitHubRepoFromRemote() err = %v, want nil", err) - } - if diff := cmp.Diff(test.wantRepo, got); diff != "" { - t.Errorf("FetchGitHubRepoFromRemote() repo mismatch (-want +got): %s", diff) - } - } - }) - } -} - -// newTestGitRepo creates a new git repository in a temporary directory with the given remotes. -func newTestGitRepoWithRemotes(t *testing.T, remotes map[string][]string) *legacygitrepo.LocalRepository { - t.Helper() - dir := t.TempDir() - - r, err := git.PlainInit(dir, false) - if err != nil { - t.Fatalf("git.PlainInit failed: %v", err) - } - - for name, urls := range remotes { - _, err := r.CreateRemote(&gogitConfig.RemoteConfig{ - Name: name, - URLs: urls, - }) - if err != nil { - t.Fatalf("CreateRemote failed: %v", err) - } - } - - repo, err := legacygitrepo.NewRepository(&legacygitrepo.RepositoryOptions{Dir: dir}) - if err != nil { - t.Fatalf("legacygitrepo.NewRepository failed: %v", err) - } - return repo -} diff --git a/internal/legacylibrarian/legacylibrarian/state.go b/internal/legacylibrarian/legacylibrarian/state.go deleted file mode 100644 index e34554719c1..00000000000 --- a/internal/legacylibrarian/legacylibrarian/state.go +++ /dev/null @@ -1,238 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacylibrarian - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "io/fs" - "log/slog" - "os" - "path" - "path/filepath" - "sort" - "strings" - - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" - "github.com/googleapis/librarian/internal/legacylibrarian/legacygitrepo" - internalyaml "github.com/googleapis/librarian/internal/yaml" - "gopkg.in/yaml.v3" -) - -const ( - librarianConfigFile = "config.yaml" - librarianStateFile = "state.yaml" - serviceConfigType = "type" - serviceConfigValue = "google.api.Service" -) - -// Utility functions for saving and loading pipeline state and config from various places. - -func loadRepoState(repo *legacygitrepo.LocalRepository, source string) (*legacyconfig.LibrarianState, error) { - if repo == nil { - slog.Info("repo is nil, skipping state loading") - return nil, nil - } - path := filepath.Join(repo.Dir, legacyconfig.LibrarianDir, librarianStateFile) - return parseLibrarianState(path, source) -} - -func loadRepoStateFromGitHub(ctx context.Context, ghClient GitHubClient, branch string) (*legacyconfig.LibrarianState, error) { - content, err := ghClient.GetRawContent(ctx, path.Join(legacyconfig.LibrarianDir, legacyconfig.LibrarianStateFile), branch) - if err != nil { - return nil, err - } - state, err := loadLibrarianStateFromBytes(content, "") - if err != nil { - return nil, err - } - return state, nil -} - -func loadLibrarianConfig(repo *legacygitrepo.LocalRepository) (*legacyconfig.LibrarianConfig, error) { - if repo == nil { - slog.Info("repo is nil, skipping state loading") - return nil, nil - } - path := filepath.Join(repo.Dir, legacyconfig.LibrarianDir, librarianConfigFile) - return parseLibrarianConfig(path) -} - -func loadLibrarianConfigFromGitHub(ctx context.Context, ghClient GitHubClient, branch string) (*legacyconfig.LibrarianConfig, error) { - content, err := ghClient.GetRawContent(ctx, path.Join(legacyconfig.LibrarianDir, legacyconfig.LibrarianConfigFile), branch) - if err != nil { - return nil, err - } - cfg, err := loadLibrarianConfigFromBytes(content) - if err != nil { - return nil, err - } - return cfg, nil -} - -func parseLibrarianState(path, source string) (*legacyconfig.LibrarianState, error) { - bytes, err := os.ReadFile(path) - if err != nil { - return nil, err - } - return loadLibrarianStateFromBytes(bytes, source) -} - -func loadLibrarianStateFromBytes(data []byte, source string) (*legacyconfig.LibrarianState, error) { - var s legacyconfig.LibrarianState - if err := yaml.Unmarshal(data, &s); err != nil { - return nil, fmt.Errorf("unmarshaling librarian state: %w", err) - } - if err := populateServiceConfigIfEmpty(&s, source); err != nil { - return nil, fmt.Errorf("populating service config: %w", err) - } - if err := s.Validate(); err != nil { - return nil, fmt.Errorf("validating librarian state: %w", err) - } - return &s, nil -} - -func parseLibrarianConfig(path string) (*legacyconfig.LibrarianConfig, error) { - bytes, err := os.ReadFile(path) - if err != nil { - if errors.Is(err, fs.ErrNotExist) { - slog.Info("config.yaml not found, proceeding") - return nil, nil - } - return nil, err - } - return loadLibrarianConfigFromBytes(bytes) -} - -func loadLibrarianConfigFromBytes(data []byte) (*legacyconfig.LibrarianConfig, error) { - var lc legacyconfig.LibrarianConfig - if err := yaml.Unmarshal(data, &lc); err != nil { - return nil, fmt.Errorf("failed to unmarshal global config: %w", err) - } - if err := lc.Validate(); err != nil { - return nil, fmt.Errorf("invalid global config: %w", err) - } - return &lc, nil -} - -func populateServiceConfigIfEmpty(state *legacyconfig.LibrarianState, source string) error { - if source == "" { - slog.Info("source not specified, skipping service config population") - return nil - } - for i, library := range state.Libraries { - for j, api := range library.APIs { - if api.ServiceConfig != "" { - // Do not change API if the service config has already been set. - continue - } - apiPath := filepath.Join(source, api.Path) - serviceConfig, err := findServiceConfigIn(apiPath) - if err != nil { - return err - } - state.Libraries[i].APIs[j].ServiceConfig = serviceConfig - } - } - - return nil -} - -// findServiceConfigIn detects the service config in a given path. -// -// Returns the file name (relative to the given path) if the following criteria -// are met: -// -// 1. the file ends with `.yaml` and it is a valid yaml file. -// -// 2. the file contains `type: google.api.Service`. -func findServiceConfigIn(path string) (string, error) { - entries, err := os.ReadDir(path) - if err != nil { - return "", fmt.Errorf("failed to read dir %q: %w", path, err) - } - - for _, entry := range entries { - if !strings.HasSuffix(entry.Name(), ".yaml") { - continue - } - bytes, err := os.ReadFile(filepath.Join(path, entry.Name())) - if err != nil { - return "", err - } - var configMap map[string]interface{} - if err := yaml.Unmarshal(bytes, &configMap); err != nil { - return "", err - } - if value, ok := configMap[serviceConfigType].(string); ok && value == serviceConfigValue { - return entry.Name(), nil - } - } - - slog.Info("no service config found; assuming proto-only package", "path", path) - return "", nil -} - -func saveLibrarianState(repoDir string, state *legacyconfig.LibrarianState) error { - sortByLibraryID(state) - stateFile := filepath.Join(repoDir, legacyconfig.LibrarianDir, librarianStateFile) - return internalyaml.Write(stateFile, state) -} - -// sortByLibraryID sorts legacyconfig.LibraryState with respect to ID. -func sortByLibraryID(state *legacyconfig.LibrarianState) { - sort.Slice(state.Libraries, func(i, j int) bool { - return state.Libraries[i].ID < state.Libraries[j].ID - }) -} - -// readLibraryState reads the library state from a container response, if it exists. -// If the response file does not exist, readLibraryState succeeds but returns a nil pointer. -// -// The response file is removed afterward. -func readLibraryState(jsonFilePath string) (*legacyconfig.LibraryState, error) { - data, err := os.ReadFile(jsonFilePath) - defer func() { - if b, err := os.ReadFile(jsonFilePath); err == nil { - slog.Debug("container response", "content", string(b)) - } - if err := os.Remove(jsonFilePath); err != nil && !errors.Is(err, fs.ErrNotExist) { - slog.Warn("fail to remove file", slog.String("name", jsonFilePath), slog.Any("err", err)) - } - }() - if err != nil { - // If we only failed to read the file because it didn't exist, just succeed - // with a nil pointer. - - if errors.Is(err, fs.ErrNotExist) { - return nil, nil - } - return nil, fmt.Errorf("failed to read response file, path: %s, error: %w", jsonFilePath, err) - } - - var libraryState *legacyconfig.LibraryState - - if err := json.Unmarshal(data, &libraryState); err != nil { - return nil, fmt.Errorf("failed to load file, %s, to state: %w", jsonFilePath, err) - } - - if libraryState.ErrorMessage != "" { - return nil, fmt.Errorf("failed with error message: %s", libraryState.ErrorMessage) - } - - return libraryState, nil -} diff --git a/internal/legacylibrarian/legacylibrarian/state_test.go b/internal/legacylibrarian/legacylibrarian/state_test.go deleted file mode 100644 index 7594e8958e7..00000000000 --- a/internal/legacylibrarian/legacylibrarian/state_test.go +++ /dev/null @@ -1,553 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacylibrarian - -import ( - "fmt" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" - "gopkg.in/yaml.v3" -) - -func TestParseLibrarianState(t *testing.T) { - for _, test := range []struct { - name string - content string - source string - want *legacyconfig.LibrarianState - wantErr bool - }{ - { - name: "valid state", - content: `image: gcr.io/test/image:v1.2.3 -libraries: - - id: a/b - source_roots: - - src/a - - src/b - apis: - - path: a/b/v1 - service_config: a/b/v1/service.yaml -`, - source: "", - want: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "a/b", - SourceRoots: []string{"src/a", "src/b"}, - APIs: []*legacyconfig.API{ - { - Path: "a/b/v1", - ServiceConfig: "a/b/v1/service.yaml", - }, - }, - }, - }, - }, - }, - { - name: "invalid source", - content: `image: gcr.io/test/image:v1.2.3 -libraries: - - id: a/b - source_roots: - - src/a - - src/b - apis: - - path: a/b/v1 - service_config: -`, - source: "/non-existed-path", - want: nil, - wantErr: true, - }, - { - name: "invalid yaml", - content: "image: gcr.io/test/image:v1.2.3\n libraries: []\n", - source: "", - wantErr: true, - }, - { - name: "validation error", - content: "image: gcr.io/test/image:v1.2.3\nlibraries: []\n", - source: "", - wantErr: true, - }, - } { - t.Run(test.name, func(t *testing.T) { - path := filepath.Join(t.TempDir(), "state.yaml") - if err := os.WriteFile(path, []byte(test.content), 0644); err != nil { - t.Fatalf("os.WriteFile() failed: %v", err) - } - got, err := parseLibrarianState(path, test.source) - if (err != nil) != test.wantErr { - t.Errorf("parseLibrarianState() error = %v, wantErr %v", err, test.wantErr) - return - } - if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("parseLibrarianState() mismatch (-want +got): %s", diff) - } - }) - } -} - -func TestFindServiceConfigIn(t *testing.T) { - for _, test := range []struct { - name string - path string - want string - wantErr bool - }{ - { - name: "find a service config", - path: filepath.Join("testdata", "find_a_service_config"), - want: "service_config.yaml", - }, - { - name: "non-existent source path", - path: filepath.Join("testdata", "non-existent-path"), - want: "", - wantErr: true, - }, - { - name: "no service config in a source path", - path: filepath.Join("testdata", "no_service_config"), - want: "", - wantErr: false, - }, - { - name: "invalid yaml", - path: filepath.Join("testdata", "invalid_yaml"), - want: "", - wantErr: true, - }, - } { - t.Run(test.name, func(t *testing.T) { - got, err := findServiceConfigIn(test.path) - if test.wantErr { - if err == nil { - t.Fatal("findServiceConfigIn() should return error") - } - - return - } - if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("fetchRemoteLibrarianState() mismatch (-want +got): %s", diff) - } - }) - } -} - -func TestParseGlobalConfig(t *testing.T) { - for _, test := range []struct { - name string - filename string - want *legacyconfig.LibrarianConfig - wantErr bool - wantErrMsg string - }{ - { - name: "valid global config", - filename: "successful-parsing-config.yaml", - want: &legacyconfig.LibrarianConfig{ - GlobalFilesAllowlist: []*legacyconfig.GlobalFile{ - { - Path: "a/path", - Permissions: "read-only", - }, - { - Path: "another/path", - Permissions: "read-write", - }, - }, - Libraries: []*legacyconfig.LibraryConfig{ - { - LibraryID: "example-library", - SkipGitHubReleaseCreation: false, - }, - }, - }, - }, - { - name: "invalid global config", - filename: "invalid-global-config.yaml", - wantErr: true, - wantErrMsg: "invalid global config", - }, - } { - t.Run(test.name, func(t *testing.T) { - path := filepath.Join("testdata", "test-parse-global-config", test.filename) - got, err := parseLibrarianConfig(path) - if test.wantErr { - if err == nil { - t.Fatal("parseGlobalConfig() should return error") - } - - if !strings.Contains(err.Error(), test.wantErrMsg) { - t.Errorf("want error message %q, got %q", test.wantErrMsg, err.Error()) - } - - return - } - if err != nil { - t.Fatalf("parseGlobalConfig() failed: %v", err) - } - if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("saveLibrarianState() mismatch (-want +got): %s", diff) - } - }) - } -} - -func TestPopulateServiceConfig(t *testing.T) { - for _, test := range []struct { - name string - state *legacyconfig.LibrarianState - path string - want *legacyconfig.LibrarianState - wantErr bool - }{ - { - name: "populate service config", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "example-id", - APIs: []*legacyconfig.API{ - { - Path: "example/api", - }, - { - Path: "another/example/api", - ServiceConfig: "another_config.yaml", - }, - }, - }, - }, - }, - path: filepath.Join("testdata", "populate_service_config"), - want: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "example-id", - APIs: []*legacyconfig.API{ - { - Path: "example/api", - ServiceConfig: "example_api_config.yaml", - }, - { - Path: "another/example/api", - ServiceConfig: "another_config.yaml", - }, - }, - }, - }, - }, - }, - { - name: "non valid api path", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "example-id", - APIs: []*legacyconfig.API{ - { - Path: "non-existed/example/api", - }, - }, - }, - }, - }, - path: "/non-existed-source-path", - want: nil, - wantErr: true, - }, - } { - t.Run(test.name, func(t *testing.T) { - err := populateServiceConfigIfEmpty(test.state, test.path) - if test.wantErr { - if err == nil { - t.Fatal("findServiceConfigIn() should return error") - } - - return - } - if diff := cmp.Diff(test.want, test.state); diff != "" { - t.Errorf("mismatch (-want +got): %s", diff) - } - }) - } -} - -func TestSaveLibrarianState(t *testing.T) { - t.Parallel() - tmpDir := t.TempDir() - if err := os.MkdirAll(filepath.Join(tmpDir, legacyconfig.LibrarianDir), 0755); err != nil { - t.Fatal(err) - } - state := &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "a/b", - SourceRoots: []string{ - "src/a", - "src/b", - }, - APIs: []*legacyconfig.API{ - { - Path: "a/b/v1", - ServiceConfig: "a/b/v1/service.yaml", - Status: "existing", - }, - }, - PreserveRegex: []string{}, - RemoveRegex: []string{}, - }, - }, - } - if err := saveLibrarianState(tmpDir, state); err != nil { - t.Fatalf("saveLibrarianState() failed: %v", err) - } - - path := filepath.Join(tmpDir, legacyconfig.LibrarianDir, "state.yaml") - gotBytes, err := os.ReadFile(path) - if err != nil { - t.Fatalf("os.ReadFile() failed: %v", err) - } - gotState := &legacyconfig.LibrarianState{} - if err := yaml.Unmarshal(gotBytes, gotState); err != nil { - t.Fatalf("yaml.Unmarshal() failed: %v", err) - } - // API status should be ignored when writing to yaml. - state.Libraries[0].APIs[0].Status = "" - if diff := cmp.Diff(state, gotState); diff != "" { - t.Errorf("saveLibrarianState() mismatch (-want +got): %s", diff) - } -} - -func TestReadLibraryState(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - filename string - want *legacyconfig.LibraryState - wantErr bool - wantErrMsg string - }{ - { - name: "successful load content", - filename: "successful-unmarshal-libraryState.json", - want: &legacyconfig.LibraryState{ - ID: "google-cloud-go", - Version: "1.0.0", - APIs: []*legacyconfig.API{ - { - Path: "google/cloud/compute/v1", - ServiceConfig: "example_service_config.yaml", - }, - }, - SourceRoots: []string{"src/example/path"}, - PreserveRegex: []string{"example-preserve-regex"}, - RemoveRegex: []string{"example-remove-regex"}, - }, - }, - { - name: "empty libraryState", - filename: "empty-libraryState.json", - want: &legacyconfig.LibraryState{}, - }, - { - name: "load content with an error message", - filename: "unmarshal-libraryState-with-error-msg.json", - wantErr: true, - wantErrMsg: "failed with error message", - }, - { - name: "missing file", - want: nil, - }, - } { - t.Run(test.name, func(t *testing.T) { - path := filepath.Join("testdata", "test-read-library-state", test.filename) - // The response file is removed by the readLibraryState() function, - // so we create a copy and read from it. - dstFilePath := filepath.Join(t.TempDir(), "copied-state", test.filename) - if err := os.CopyFS(filepath.Dir(dstFilePath), os.DirFS(filepath.Dir(path))); err != nil { - t.Error(err) - } - - got, err := readLibraryState(dstFilePath) - - if test.name == "load content with an error message" { - if err == nil { - t.Fatal("readLibraryState() expected an error but got nil") - } - - if g, w := err.Error(), "failed with error message"; !strings.Contains(g, w) { - t.Errorf("got %q, wanted it to contain %q", g, w) - } - - return - } - - if test.wantErr { - if err == nil { - t.Fatal("readLibraryState() should fail") - } - - if !strings.Contains(err.Error(), test.wantErrMsg) { - t.Errorf("want error message: %s, got %s", test.wantErrMsg, err.Error()) - } - } - - if err != nil { - t.Fatalf("readLibraryState() unexpected error: %v", err) - } - - if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("Response library state mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestLoadRepoStateFromGitHub(t *testing.T) { - t.Parallel() - - state := &legacyconfig.LibrarianState{ - Image: "gcr.io/some-project-id/some-test-image:latest", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "google-cloud-storage", - SourceRoots: []string{"some/path"}, - TagFormat: "v{version}", - }, - }, - } - for _, test := range []struct { - name string - branch string - ghClient GitHubClient - want *legacyconfig.LibrarianState - wantErr bool - wantErrMsg string - }{ - { - name: "happy path", - ghClient: &mockGitHubClient{ - librarianState: state, - }, - want: state, - }, - { - name: "missing file", - ghClient: &mockGitHubClient{ - rawErr: fmt.Errorf("file not found"), - }, - wantErr: true, - wantErrMsg: "file not found", - }, - { - name: "invalid state file", - ghClient: &mockGitHubClient{ - librarianState: &legacyconfig.LibrarianState{}, - }, - wantErr: true, - wantErrMsg: "validating librarian state", - }, - } { - t.Run(test.name, func(t *testing.T) { - got, err := loadRepoStateFromGitHub(t.Context(), test.ghClient, test.branch) - if test.wantErr { - if err == nil { - t.Fatal("loadRepoStateFromGitHub() should fail") - } - if !strings.Contains(err.Error(), test.wantErrMsg) { - t.Fatalf("want error message: %s, got %s", test.wantErrMsg, err.Error()) - } - - return - } - - if err != nil { - t.Fatalf("loadRepoStateFromGitHub() unexpected error: %v", err) - } - if diff := cmp.Diff(test.want, got, cmpopts.EquateEmpty()); diff != "" { - t.Fatalf("Response library state mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestSortByLibraryID(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - state *legacyconfig.LibrarianState - want *legacyconfig.LibrarianState - }{ - { - name: "sort_with_library_id", - state: &legacyconfig.LibrarianState{ - Image: "test-image", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "b-library", - }, - { - ID: "a-library", - }, - }, - }, - want: &legacyconfig.LibrarianState{ - Image: "test-image", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "a-library", - }, - { - ID: "b-library", - }, - }, - }, - }, - { - name: "nil_libraries", - state: &legacyconfig.LibrarianState{ - Image: "test-image", - Libraries: nil, - }, - want: &legacyconfig.LibrarianState{ - Image: "test-image", - Libraries: nil, - }, - }, - } { - t.Run(test.name, func(t *testing.T) { - sortByLibraryID(test.state) - if diff := cmp.Diff(test.want, test.state); diff != "" { - t.Errorf("sortByLibraryID mismatch (-want +got):\n%s", diff) - } - }) - } -} diff --git a/internal/legacylibrarian/legacylibrarian/tag_and_release.go b/internal/legacylibrarian/legacylibrarian/tag_and_release.go deleted file mode 100644 index 62c2d677e43..00000000000 --- a/internal/legacylibrarian/legacylibrarian/tag_and_release.go +++ /dev/null @@ -1,397 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacylibrarian - -import ( - "bytes" - "context" - "errors" - "fmt" - "html/template" - "log/slog" - "net/url" - "path/filepath" - "regexp" - "slices" - "sort" - "strconv" - "strings" - "time" - - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" - "github.com/googleapis/librarian/internal/legacylibrarian/legacygithub" - "github.com/googleapis/librarian/internal/legacylibrarian/legacygitrepo" -) - -const ( - pullRequestSegments = 7 - tagCmdName = "tag" - releasePendingLabel = "release:pending" - releaseDoneLabel = "release:done" - releaseFailedLabel = "release:failed" -) - -var ( - bulkChangeSectionRegex = regexp.MustCompile(`(feat|fix|perf|revert|docs): (.*)\nLibraries: (.*)`) - contentRegex = regexp.MustCompile(`### (Features|Bug Fixes|Performance Improvements|Reverts|Documentation)\n`) - detailsRegex = regexp.MustCompile(`(?s)
(.*?)(.*?)
`) - summaryRegex = regexp.MustCompile(`(.*?): v?(\d+\.\d+\.\d+)`) - - libraryReleaseTemplate = template.Must(template.New("libraryRelease").Parse(`### {{.Type}} -{{ range .Messages }} -{{.}} -{{ end }} - -`)) -) - -type tagRunner struct { - ghClient GitHubClient - pullRequest string -} - -// libraryRelease holds the parsed information from a pull request body. -type libraryRelease struct { - // Body contains the release notes. - Body string - // Library is the library id of the library being released - Library string - // Version is the version that is being released - Version string -} - -type libraryReleaseBuilder struct { - typeToMessages map[string][]string - title string - version string -} - -func newTagRunner(cfg *legacyconfig.Config) (*tagRunner, error) { - if cfg.GitHubToken == "" { - return nil, fmt.Errorf("`%s` must be set", legacyconfig.LibrarianGithubToken) - } - repo, err := parseRemote(cfg.Repo) - if err != nil { - return nil, err - } - ghClient := legacygithub.NewClient(cfg.GitHubToken, repo) - // If a custom GitHub API endpoint is provided (for testing), - // parse it and set it as the BaseURL on the GitHub client. - if cfg.GitHubAPIEndpoint != "" { - endpoint, err := url.Parse(cfg.GitHubAPIEndpoint) - if err != nil { - return nil, fmt.Errorf("failed to parse github-api-endpoint: %w", err) - } - ghClient.BaseURL = endpoint - } - return &tagRunner{ - ghClient: ghClient, - pullRequest: cfg.PullRequest, - }, nil -} - -func parseRemote(repo string) (*legacygithub.Repository, error) { - if isURL(repo) { - return legacygithub.ParseRemote(repo) - } - // repo is a directory - absRepoRoot, err := filepath.Abs(repo) - if err != nil { - return nil, err - } - githubRepo, err := legacygitrepo.NewRepository(&legacygitrepo.RepositoryOptions{ - Dir: absRepoRoot, - }) - if err != nil { - return nil, err - } - return GetGitHubRepositoryFromGitRepo(githubRepo) -} - -func (r *tagRunner) run(ctx context.Context) error { - slog.Info("running tag command") - prs, err := r.determinePullRequestsToProcess(ctx) - if err != nil { - return err - } - if len(prs) == 0 { - slog.Info("no pull requests to process, exiting") - return nil - } - - var hadErrors bool - for _, p := range prs { - if err := r.processPullRequest(ctx, p); err != nil { - slog.Error("failed to process pull request", "pr", p.GetNumber(), "error", err) - if err := r.replaceReleasePendingLabel(ctx, p, releaseFailedLabel); err != nil { - slog.Error("failed to replace pending label with failed label", "pr", p.GetNumber(), "error", err) - } - hadErrors = true - continue - } - slog.Info("processed pull request", "pr", p.GetNumber()) - } - slog.Info("finished processing all pull requests") - - if hadErrors { - return errors.New("failed to process some pull requests") - } - return nil -} - -func (r *tagRunner) determinePullRequestsToProcess(ctx context.Context) ([]*legacygithub.PullRequest, error) { - slog.Info("determining pull requests to process") - if r.pullRequest != "" { - slog.Info("processing a single pull request", "pr", r.pullRequest) - ss := strings.Split(r.pullRequest, "/") - if len(ss) != pullRequestSegments { - return nil, fmt.Errorf("invalid pull request format: %s", r.pullRequest) - } - prNum, err := strconv.Atoi(ss[pullRequestSegments-1]) - if err != nil { - return nil, fmt.Errorf("invalid pull request number: %s", ss[pullRequestSegments-1]) - } - pr, err := r.ghClient.GetPullRequest(ctx, prNum) - if err != nil { - return nil, fmt.Errorf("failed to get pull request %d: %w", prNum, err) - } - return []*legacygithub.PullRequest{pr}, nil - } - - slog.Info("searching for pull requests to tag and release") - thirtyDaysAgo := time.Now().Add(-30 * 24 * time.Hour).Format(time.RFC3339) - query := fmt.Sprintf("label:%s merged:>=%s", releasePendingLabel, thirtyDaysAgo) - prs, err := r.ghClient.SearchPullRequests(ctx, query) - if err != nil { - return nil, fmt.Errorf("failed to search pull requests: %w", err) - } - return prs, nil -} - -func (r *tagRunner) processPullRequest(ctx context.Context, p *legacygithub.PullRequest) error { - slog.Info("processing pull request", "pr", p.GetNumber()) - releases := parsePullRequestBody(p.GetBody()) - if len(releases) == 0 { - slog.Warn("no release details found in pull request body, skipping") - return nil - } - - // Load library state from remote repo - targetBranch := *p.Base.Ref - librarianState, err := loadRepoStateFromGitHub(ctx, r.ghClient, targetBranch) - if err != nil { - return err - } - - librarianConfig, err := loadLibrarianConfigFromGitHub(ctx, r.ghClient, targetBranch) - if err != nil { - slog.Warn("error loading .librarian/config.yaml", slog.Any("err", err)) - } - - // Add a tag to the release commit to trigger louhi flow: "release-{pr number}". - // See: go/sdk-librarian:louhi-trigger for details. - commitSha := p.GetMergeCommitSHA() - tagName := fmt.Sprintf("release-%d", p.GetNumber()) - if err := r.ghClient.CreateTag(ctx, tagName, commitSha); err != nil { - return fmt.Errorf("failed to create tag %s: %w", tagName, err) - } - for _, release := range releases { - libraryState := librarianState.LibraryByID(release.Library) - if libraryState == nil { - return fmt.Errorf("library %s not found", release.Library) - } - - var libraryConfig *legacyconfig.LibraryConfig - if librarianConfig != nil { - libraryConfig = librarianConfig.LibraryConfigFor(release.Library) - } - - if libraryConfig != nil && libraryConfig.SkipGitHubReleaseCreation { - slog.Info("skip creating release", "library", release.Library) - continue - } - - slog.Info("creating release", "library", release.Library, "version", release.Version) - tagFormat := legacyconfig.DetermineTagFormat(release.Library, libraryState, librarianConfig) - tagName := legacyconfig.FormatTag(tagFormat, release.Library, release.Version) - releaseName := fmt.Sprintf("%s: v%s", release.Library, release.Version) - if len(release.Body) > 0 { - if _, err := r.ghClient.CreateRelease(ctx, tagName, releaseName, release.Body, commitSha); err != nil { - return fmt.Errorf("failed to create release: %w", err) - } - } - } - return r.replaceReleasePendingLabel(ctx, p, releaseDoneLabel) -} - -// parsePullRequestBody parses a string containing release notes and returns a slice of ParsedPullRequestBody. -func parsePullRequestBody(body string) []libraryRelease { - slog.Info("parsing pull request body") - idToBuilder := make(map[string]*libraryReleaseBuilder) - matches := detailsRegex.FindAllStringSubmatch(body, -1) - for _, match := range matches { - summary := match[1] - content := strings.TrimSpace(match[2]) - if summary == "Bulk Changes" { - // Associated bulk changes to individual libraries. - sections := bulkChangeSectionRegex.FindAllStringSubmatch(content, -1) - for _, section := range sections { - if len(section) != 4 { - slog.Warn("bulk change does not associated with a library id", "content", section) - continue - } - - commitType, ok := commitTypeToHeading[strings.TrimSpace(section[1])] - if !ok { - slog.Warn("unrecognized commit type, skipping", "commit", section[1]) - continue - } - message := fmt.Sprintf("* %s", strings.TrimSpace(section[2])) - libraries := section[3] - for _, library := range strings.Split(libraries, ",") { - // Bulk change doesn't have title and version, put an empty string so that - // title and version are not overwritten, if exists. - updateLibraryReleaseBuilder(idToBuilder, library, commitType, "", message, "") - } - } - - continue - } - - summaryMatch := summaryRegex.FindStringSubmatch(summary) - if len(summaryMatch) != 3 { - slog.Warn("failed to parse pull request body", "match", strings.Join(match, "\n")) - continue - } - - slog.Info("parsed pull request body", "library", summaryMatch[1], "version", summaryMatch[2]) - library := strings.TrimSpace(summaryMatch[1]) - version := strings.TrimSpace(summaryMatch[2]) - // Split the content using commit types, e.g., Features, Bug Fixes, etc. - // For non-bulk changes, the first match (i = 0) is the release title, the i-th match is - // the commit messages of typeMatches[i-1]. - contentMatches := contentRegex.Split(content, -1) - title := contentMatches[0] - typeMatches := contentRegex.FindAllStringSubmatch(content, -1) - if len(typeMatches) == 0 { - // No commit message in a library. - updateLibraryReleaseBuilder(idToBuilder, library, "", title, "", version) - } - for i, typeMatch := range typeMatches { - commitType := typeMatch[1] - contentMatch := contentMatches[i+1] - messages := strings.Split(contentMatch, "\n\n") - for _, message := range messages { - message = strings.TrimSpace(message) - if message != "" { - updateLibraryReleaseBuilder(idToBuilder, library, commitType, title, message, version) - } - } - } - - } - - var parsedBodies []libraryRelease - for libraryID, builder := range idToBuilder { - parsedBodies = append(parsedBodies, libraryRelease{ - Body: buildReleaseBody(builder.typeToMessages, builder.title), - Library: libraryID, - Version: builder.version, - }) - } - - sort.Slice(parsedBodies, func(i, j int) bool { - return parsedBodies[i].Library < parsedBodies[j].Library - }) - - return parsedBodies -} - -// replaceReleasePendingLabel is a helper function that replaces the `release:pending` label with `newLabel`. -func (r *tagRunner) replaceReleasePendingLabel(ctx context.Context, p *legacygithub.PullRequest, newLabel string) error { - var currentLabels []string - for _, label := range p.Labels { - currentLabels = append(currentLabels, label.GetName()) - } - currentLabels = slices.DeleteFunc(currentLabels, func(s string) bool { - return s == releasePendingLabel - }) - currentLabels = append(currentLabels, newLabel) - if err := r.ghClient.ReplaceLabels(ctx, p.GetNumber(), currentLabels); err != nil { - return fmt.Errorf("failed to replace labels: %w", err) - } - return nil -} - -// updateLibraryReleaseBuilder finds or creates a libraryReleaseBuilder for a given library -// and updates it with new information. -func updateLibraryReleaseBuilder(idToVersionAndBody map[string]*libraryReleaseBuilder, library, commitType, title, message, version string) { - vab, ok := idToVersionAndBody[library] - if !ok { - idToVersionAndBody[library] = &libraryReleaseBuilder{ - typeToMessages: map[string][]string{ - commitType: {message}, - }, - version: version, - title: title, - } - - return - } - - vab.typeToMessages[commitType] = append(vab.typeToMessages[commitType], message) - if version == "" { - version = vab.version - } - vab.version = version - if title == "" { - title = vab.title - } - vab.title = title -} - -// buildReleaseBody formats the release notes for a single library. -// -// It takes a map of commit types (e.g., "Features", "Bug Fixes") to their corresponding messages and a title string. -// It returns a formatted string containing the title and all commit messages organized by type, following the order -// defined in commitTypeOrder. -func buildReleaseBody(body map[string][]string, title string) string { - var builder strings.Builder - builder.WriteString(title) - for _, commitType := range commitTypeOrder { - heading := commitTypeToHeading[commitType] - messages, ok := body[heading] - if !ok { - continue - } - var out bytes.Buffer - data := &struct { - Type string - Messages []string - }{ - Type: heading, - Messages: messages, - } - if err := libraryReleaseTemplate.Execute(&out, data); err != nil { - slog.Error("error executing template", "error", err) - continue - } - - builder.WriteString(strings.TrimSpace(out.String())) - builder.WriteString("\n\n") - } - - return strings.TrimSpace(builder.String()) -} diff --git a/internal/legacylibrarian/legacylibrarian/tag_test.go b/internal/legacylibrarian/legacylibrarian/tag_test.go deleted file mode 100644 index 6663e97bbf5..00000000000 --- a/internal/legacylibrarian/legacylibrarian/tag_test.go +++ /dev/null @@ -1,848 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacylibrarian - -import ( - "errors" - "slices" - "strings" - "testing" - - "github.com/google/go-cmp/cmp" - gh "github.com/google/go-github/v69/github" - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" - "github.com/googleapis/librarian/internal/legacylibrarian/legacygithub" -) - -func TestNewTagRunner(t *testing.T) { - testcases := []struct { - name string - cfg *legacyconfig.Config - wantErr bool - }{ - { - name: "valid config", - cfg: &legacyconfig.Config{ - GitHubToken: "some-token", - Repo: "https://github.com/googleapis/some-test-repo", - WorkRoot: t.TempDir(), - CommandName: tagCmdName, - }, - wantErr: false, - }, - { - name: "missing github token", - cfg: &legacyconfig.Config{ - CommandName: tagCmdName, - }, - wantErr: true, - }, - } - for _, tc := range testcases { - t.Run(tc.name, func(t *testing.T) { - r, err := newTagRunner(tc.cfg) - if (err != nil) != tc.wantErr { - t.Errorf("newTagRunner() error = %v, wantErr %v", err, tc.wantErr) - return - } - if !tc.wantErr && r == nil { - t.Errorf("newTagRunner() got nil runner, want non-nil") - } - }) - } -} - -func TestDeterminePullRequestsToProcess(t *testing.T) { - pr123 := &legacygithub.PullRequest{} - for _, test := range []struct { - name string - cfg *legacyconfig.Config - ghClient GitHubClient - want []*legacygithub.PullRequest - wantErrMsg string - }{ - { - name: "with pull request config", - cfg: &legacyconfig.Config{ - PullRequest: "https://github.com/googleapis/librarian/pulls/123", - }, - ghClient: &mockGitHubClient{ - getPullRequestCalls: 1, - pullRequest: pr123, - }, - want: []*legacygithub.PullRequest{pr123}, - }, - { - name: "invalid pull request format", - cfg: &legacyconfig.Config{ - PullRequest: "invalid", - }, - ghClient: &mockGitHubClient{}, - wantErrMsg: "invalid pull request format", - }, - { - name: "invalid pull request number", - cfg: &legacyconfig.Config{ - PullRequest: "https://github.com/googleapis/librarian/pulls/abc", - }, - ghClient: &mockGitHubClient{}, - wantErrMsg: "invalid pull request number", - }, - { - name: "get pull request error", - cfg: &legacyconfig.Config{ - PullRequest: "https://github.com/googleapis/librarian/pulls/123", - }, - ghClient: &mockGitHubClient{ - getPullRequestCalls: 1, - getPullRequestErr: errors.New("get pr error"), - }, - wantErrMsg: "failed to get pull request", - }, - { - name: "search pull requests", - cfg: &legacyconfig.Config{}, - ghClient: &mockGitHubClient{ - searchPullRequestsCalls: 1, - pullRequests: []*legacygithub.PullRequest{pr123}, - }, - want: []*legacygithub.PullRequest{pr123}, - }, - { - name: "search pull requests error", - cfg: &legacyconfig.Config{}, - ghClient: &mockGitHubClient{ - searchPullRequestsCalls: 1, - searchPullRequestsErr: errors.New("search pr error"), - }, - wantErrMsg: "failed to search pull requests", - }, - } { - t.Run(test.name, func(t *testing.T) { - r := &tagRunner{ - pullRequest: test.cfg.PullRequest, - ghClient: test.ghClient, - } - got, err := r.determinePullRequestsToProcess(t.Context()) - if err != nil { - if test.wantErrMsg == "" { - t.Fatalf("unexpected error: %v", err) - } - if !strings.Contains(err.Error(), test.wantErrMsg) { - t.Fatalf("got %q, want contains %q", err, test.wantErrMsg) - } - return - } - if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("determinePullRequestsToProcess() mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func Test_tagRunner_run(t *testing.T) { - pr123 := &legacygithub.PullRequest{} - pr456 := &legacygithub.PullRequest{} - - for _, test := range []struct { - name string - ghClient *mockGitHubClient - wantErrMsg string - wantSearchPullRequestsCalls int - wantGetPullRequestCalls int - }{ - { - name: "no pull requests to process", - ghClient: &mockGitHubClient{}, - wantSearchPullRequestsCalls: 1, - }, - { - name: "one pull request to process", - ghClient: &mockGitHubClient{ - pullRequests: []*legacygithub.PullRequest{pr123}, - }, - wantSearchPullRequestsCalls: 1, - }, - { - name: "multiple pull requests to process", - ghClient: &mockGitHubClient{ - pullRequests: []*legacygithub.PullRequest{pr123, pr456}, - }, - wantSearchPullRequestsCalls: 1, - }, - { - name: "error determining pull requests", - ghClient: &mockGitHubClient{ - searchPullRequestsErr: errors.New("search pr error"), - }, - wantSearchPullRequestsCalls: 1, - wantErrMsg: "failed to search pull requests", - }, - } { - t.Run(test.name, func(t *testing.T) { - r := &tagRunner{ - ghClient: test.ghClient, - } - err := r.run(t.Context()) - if err != nil { - if test.wantErrMsg == "" { - t.Fatalf("unexpected error: %v", err) - } - if !strings.Contains(err.Error(), test.wantErrMsg) { - t.Fatalf("got %q, want contains %q", err, test.wantErrMsg) - } - return - } - if test.ghClient.searchPullRequestsCalls != test.wantSearchPullRequestsCalls { - t.Errorf("searchPullRequestsCalls = %v, want %v", test.ghClient.searchPullRequestsCalls, test.wantSearchPullRequestsCalls) - } - if test.ghClient.getPullRequestCalls != test.wantGetPullRequestCalls { - t.Errorf("getPullRequestCalls = %v, want %v", test.ghClient.getPullRequestCalls, test.wantGetPullRequestCalls) - } - }) - } -} - -func TestParsePullRequestBody(t *testing.T) { - for _, test := range []struct { - name string - body string - want []libraryRelease - }{ - { - name: "single library", - body: ` -Librarian Version: v0.2.0 -Language Image: image - -
google-cloud-storage: v1.2.3 - -[v1.2.3](https://github.com/googleapis/google-cloud-go/compare/google-cloud-storage-v1.2.2...google-cloud-storage-v1.2.3) (2025-08-15) - -### Features - -* Add new feature ([abcdef1](https://github.com/googleapis/google-cloud-go/commit/abcdef1)) - -
`, - want: []libraryRelease{ - { - Version: "1.2.3", - Library: "google-cloud-storage", - Body: `[v1.2.3](https://github.com/googleapis/google-cloud-go/compare/google-cloud-storage-v1.2.2...google-cloud-storage-v1.2.3) (2025-08-15) - -### Features - -* Add new feature ([abcdef1](https://github.com/googleapis/google-cloud-go/commit/abcdef1))`, - }, - }, - }, - { - name: "multiple libraries", - body: ` -Librarian Version: 1.2.3 -Language Image: gcr.io/test/image:latest - -
library-one: v1.0.0 - -[v1.0.0](https://github.com/googleapis/repo/compare/library-one-v0.9.0...library-one-v1.0.0) (2025-08-15) - -### Features - -* some feature ([12345678](https://github.com/googleapis/repo/commit/12345678)) - -
- -
library-two: v2.3.4 - -[v2.3.4](https://github.com/googleapis/repo/compare/library-two-v2.3.3...library-two-v2.3.4) (2025-08-15) - -### Bug Fixes - -* some bug fix ([abcdefg](https://github.com/googleapis/repo/commit/abcdefg)) - -
`, - want: []libraryRelease{ - { - Version: "1.0.0", - Library: "library-one", - Body: `[v1.0.0](https://github.com/googleapis/repo/compare/library-one-v0.9.0...library-one-v1.0.0) (2025-08-15) - -### Features - -* some feature ([12345678](https://github.com/googleapis/repo/commit/12345678))`, - }, - { - Version: "2.3.4", - Library: "library-two", - Body: `[v2.3.4](https://github.com/googleapis/repo/compare/library-two-v2.3.3...library-two-v2.3.4) (2025-08-15) - -### Bug Fixes - -* some bug fix ([abcdefg](https://github.com/googleapis/repo/commit/abcdefg))`, - }, - }, - }, - { - name: "empty body", - body: "", - want: nil, - }, - { - name: "malformed summary", - body: ` -Librarian Version: 1.2.3 -Language Image: gcr.io/test/image:latest - -
no-version-here - -some content - -
`, - want: nil, - }, - { - name: "v prefix in version", - body: ` -
google-cloud-storage: v1.2.3 - -[v1.2.3](https://github.com/googleapis/google-cloud-go/compare/google-cloud-storage-v1.2.2...google-cloud-storage-v1.2.3) (2025-08-15) - -
`, - want: []libraryRelease{ - { - Version: "1.2.3", - Library: "google-cloud-storage", - Body: "[v1.2.3](https://github.com/googleapis/google-cloud-go/compare/google-cloud-storage-v1.2.2...google-cloud-storage-v1.2.3) (2025-08-15)", - }, - }, - }, - { - name: "with bulk changes", - body: ` -
google-cloud-storage: v1.2.3 - -[v1.2.3](https://github.com/googleapis/google-cloud-go/compare/google-cloud-storage-v1.2.2...google-cloud-storage-v1.2.3) (2025-08-15) - -
- - -
Bulk Changes - -some content - -
`, - want: []libraryRelease{ - { - Version: "1.2.3", - Library: "google-cloud-storage", - Body: "[v1.2.3](https://github.com/googleapis/google-cloud-go/compare/google-cloud-storage-v1.2.2...google-cloud-storage-v1.2.3) (2025-08-15)", - }, - }, - }, - { - name: "bulk_changes_appears_in_github_release", - body: ` -
google-cloud-storage: v1.2.3 - -[v1.2.3](https://github.com/googleapis/google-cloud-go/compare/google-cloud-storage-v1.2.2...google-cloud-storage-v1.2.3) (2025-08-15) - -### Features - -* add SaveToGcsFindingsOutput ([1234567](https://github.com/googleapis/google-cloud-go/commit/1234567)) - -* another feature ([9876543](https://github.com/googleapis/google-cloud-go/commit/9876543)) - -### Documentation - -* minor doc revision ([abcdefgh](https://github.com/googleapis/google-cloud-go/commit/abcdefgh)) - -
- - -
Bulk Changes - -* feat: this is a bulk change ([abcdefgh](https://github.com/googleapis/google-cloud-go/commit/abcdefgh)) -Libraries: a,b,google-cloud-storage - -* fix: this is another bulk change -Libraries: a,b,c - -
`, - want: []libraryRelease{ - { - Version: "", - Library: "a", - Body: `### Features - -* this is a bulk change ([abcdefgh](https://github.com/googleapis/google-cloud-go/commit/abcdefgh)) - -### Bug Fixes - -* this is another bulk change`, - }, - { - Version: "", - Library: "b", - Body: `### Features - -* this is a bulk change ([abcdefgh](https://github.com/googleapis/google-cloud-go/commit/abcdefgh)) - -### Bug Fixes - -* this is another bulk change`, - }, - { - Version: "", - Library: "c", - Body: `### Bug Fixes - -* this is another bulk change`, - }, - { - Version: "1.2.3", - Library: "google-cloud-storage", - Body: `[v1.2.3](https://github.com/googleapis/google-cloud-go/compare/google-cloud-storage-v1.2.2...google-cloud-storage-v1.2.3) (2025-08-15) - -### Features - -* add SaveToGcsFindingsOutput ([1234567](https://github.com/googleapis/google-cloud-go/commit/1234567)) - -* another feature ([9876543](https://github.com/googleapis/google-cloud-go/commit/9876543)) - -* this is a bulk change ([abcdefgh](https://github.com/googleapis/google-cloud-go/commit/abcdefgh)) - -### Documentation - -* minor doc revision ([abcdefgh](https://github.com/googleapis/google-cloud-go/commit/abcdefgh))`, - }, - }, - }, - { - name: "contentless releases", - body: ` -
google-cloud-functions: v2.3.4
-
google-cloud-storage: v1.2.3
`, - want: []libraryRelease{ - {Library: "google-cloud-functions", Version: "2.3.4"}, - {Library: "google-cloud-storage", Version: "1.2.3"}, - }, - }, - } { - t.Run(test.name, func(t *testing.T) { - got := parsePullRequestBody(test.body) - if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("ParsePullRequestBody() mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestProcessPullRequest(t *testing.T) { - prBody := `
google-cloud-storage: v1.2.3release notes
` - prNumber := 123 - mergeCommitSHA := "abcdef" - branch := "main" - prWithRelease := &legacygithub.PullRequest{ - Body: &prBody, - Number: &prNumber, - MergeCommitSHA: &mergeCommitSHA, - Labels: []*gh.Label{{Name: gh.Ptr(releasePendingLabel)}}, - Base: &gh.PullRequestBranch{ - Ref: &branch, - }, - } - bodyWithoutReleaseNotes := `
google-cloud-storage: v1.2.3
` - prWithoutReleaseNotes := &legacygithub.PullRequest{ - Body: &bodyWithoutReleaseNotes, - Number: &prNumber, - MergeCommitSHA: &mergeCommitSHA, - Labels: []*gh.Label{{Name: gh.Ptr(releasePendingLabel)}}, - Base: &gh.PullRequestBranch{ - Ref: &branch, - }, - } - body := "no release details" - prWithoutRelease := &legacygithub.PullRequest{ - Body: &body, - Number: &prNumber, - MergeCommitSHA: &mergeCommitSHA, - Labels: []*gh.Label{{Name: gh.Ptr(releasePendingLabel)}}, - Base: &gh.PullRequestBranch{ - Ref: &branch, - }, - } - state := &legacyconfig.LibrarianState{ - Image: "gcr.io/some-project-id/some-test-image:latest", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "google-cloud-storage", - SourceRoots: []string{"some/path"}, - TagFormat: "v{version}", - }, - }, - } - - for _, test := range []struct { - name string - pr *legacygithub.PullRequest - ghClient *mockGitHubClient - wantErrMsg string - wantCreateReleaseCalls int - wantReplaceLabelsCalls int - wantCreateTagCalls int - wantReleaseNames []string - }{ - { - name: "happy path", - pr: prWithRelease, - ghClient: &mockGitHubClient{ - librarianState: state, - }, - wantCreateReleaseCalls: 1, - wantReplaceLabelsCalls: 1, - wantCreateTagCalls: 1, - wantReleaseNames: []string{"google-cloud-storage: v1.2.3"}, - }, - { - name: "tag but no GitHub release", - pr: prWithoutReleaseNotes, - ghClient: &mockGitHubClient{ - librarianState: state, - }, - wantCreateReleaseCalls: 0, - wantReplaceLabelsCalls: 1, - wantCreateTagCalls: 1, - }, - { - name: "no release details", - pr: prWithoutRelease, - ghClient: &mockGitHubClient{ - librarianState: state, - }, - }, - { - name: "library not found", - pr: prWithRelease, - ghClient: &mockGitHubClient{ - librarianState: &legacyconfig.LibrarianState{ - Image: "gcr.io/some-project-id/some-test-image:latest", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "other-library", - SourceRoots: []string{"some/path"}, - TagFormat: "v{version}", - }, - }, - }, - }, - wantErrMsg: "library google-cloud-storage not found", - }, - { - name: "default tag format", - pr: prWithRelease, - ghClient: &mockGitHubClient{ - librarianState: &legacyconfig.LibrarianState{ - Image: "gcr.io/some-project-id/some-test-image:latest", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "google-cloud-storage", - SourceRoots: []string{"some/path"}, - }, - }, - }, - }, - wantCreateReleaseCalls: 1, - wantReplaceLabelsCalls: 1, - wantCreateTagCalls: 1, - wantReleaseNames: []string{"google-cloud-storage: v1.2.3"}, - }, - { - name: "skip_a_library_release", - pr: prWithRelease, - ghClient: &mockGitHubClient{ - librarianState: &legacyconfig.LibrarianState{ - Image: "gcr.io/some-project-id/some-test-image:latest", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "google-cloud-storage", - SourceRoots: []string{"some/path"}, - }, - }, - }, - librarianConfig: &legacyconfig.LibrarianConfig{ - Libraries: []*legacyconfig.LibraryConfig{ - { - LibraryID: "google-cloud-storage", - SkipGitHubReleaseCreation: true, - }, - }, - }, - }, - wantCreateReleaseCalls: 0, - wantReplaceLabelsCalls: 1, - wantCreateTagCalls: 1, - }, - { - name: "create release fails", - pr: prWithRelease, - ghClient: &mockGitHubClient{ - createReleaseErr: errors.New("create release error"), - librarianState: state, - }, - wantErrMsg: "failed to create release", - wantCreateReleaseCalls: 1, - wantCreateTagCalls: 1, - wantReleaseNames: []string{"google-cloud-storage: v1.2.3"}, - }, - { - name: "replace labels fails", - pr: prWithRelease, - ghClient: &mockGitHubClient{ - replaceLabelsErr: errors.New("replace labels error"), - librarianState: state, - }, - wantErrMsg: "failed to replace labels", - wantCreateReleaseCalls: 1, - wantReplaceLabelsCalls: 1, - wantCreateTagCalls: 1, - wantReleaseNames: []string{"google-cloud-storage: v1.2.3"}, - }, - { - name: "create tag fails", - pr: prWithRelease, - ghClient: &mockGitHubClient{ - createTagErr: errors.New("create tag error"), - librarianState: state, - }, - wantErrMsg: "failed to create tag", - wantCreateTagCalls: 1, - }, - } { - t.Run(test.name, func(t *testing.T) { - r := &tagRunner{ - ghClient: test.ghClient, - } - err := r.processPullRequest(t.Context(), test.pr) - if err != nil { - if test.wantErrMsg == "" { - t.Fatalf("unexpected error: %v", err) - } - if !strings.Contains(err.Error(), test.wantErrMsg) { - t.Fatalf("got %q, want contains %q", err, test.wantErrMsg) - } - } else if test.wantErrMsg != "" { - t.Fatalf("expected error containing %q, got nil", test.wantErrMsg) - } - - if test.ghClient.createReleaseCalls != test.wantCreateReleaseCalls { - t.Errorf("createReleaseCalls = %v, want %v", test.ghClient.createReleaseCalls, test.wantCreateReleaseCalls) - } - if test.ghClient.replaceLabelsCalls != test.wantReplaceLabelsCalls { - t.Errorf("replaceLabelsCalls = %v, want %v", test.ghClient.replaceLabelsCalls, test.wantReplaceLabelsCalls) - } - if test.wantErrMsg == "" && test.ghClient.replaceLabelsCalls > 0 { - if !slices.Contains(test.ghClient.replacedLabels, releaseDoneLabel) { - t.Errorf("expected release:done label in %v", test.ghClient.replacedLabels) - } - } - if diff := cmp.Diff(test.wantReleaseNames, test.ghClient.releaseNames); diff != "" { - t.Errorf("releaseNames mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestReplaceReleasePendingLabel(t *testing.T) { - prWithPending := &legacygithub.PullRequest{ - Number: gh.Ptr(123), - Labels: []*gh.Label{{Name: gh.Ptr(releasePendingLabel)}, {Name: gh.Ptr("label1")}}, - } - prWithoutPending := &legacygithub.PullRequest{ - Number: gh.Ptr(123), - Labels: []*gh.Label{{Name: gh.Ptr("label1")}}, - } - - for _, test := range []struct { - name string - pr *legacygithub.PullRequest - ghClient *mockGitHubClient - wantErrMsg string - newLabel string - }{ - { - name: "replace with done label", - pr: prWithPending, - ghClient: &mockGitHubClient{}, - newLabel: releaseDoneLabel, - }, - { - name: "replace with failed label", - pr: prWithPending, - ghClient: &mockGitHubClient{}, - newLabel: releaseFailedLabel, - }, - { - name: "without pending label", - pr: prWithoutPending, - ghClient: &mockGitHubClient{}, - newLabel: releaseDoneLabel, - }, - { - name: "replace labels fails", - pr: prWithPending, - ghClient: &mockGitHubClient{ - replaceLabelsErr: errors.New("replace labels error"), - }, - wantErrMsg: "failed to replace labels", - newLabel: releaseDoneLabel, - }, - } { - t.Run(test.name, func(t *testing.T) { - r := &tagRunner{ - ghClient: test.ghClient, - } - err := r.replaceReleasePendingLabel(t.Context(), test.pr, test.newLabel) - if err != nil { - if test.wantErrMsg == "" { - t.Fatalf("unexpected error: %v", err) - } - if !strings.Contains(err.Error(), test.wantErrMsg) { - t.Fatalf("got %q, want contains %q", err, test.wantErrMsg) - } - return - } else if test.wantErrMsg != "" { - t.Fatalf("expected error containing %q, got nil", test.wantErrMsg) - } - if test.ghClient.replacedLabels != nil { - if !slices.Contains(test.ghClient.replacedLabels, test.newLabel) { - t.Errorf("expected %s label in %v", test.newLabel, test.ghClient.replacedLabels) - } - } - }) - } -} - -func Test_tagRunner_run_processPullRequests(t *testing.T) { - branch := "main" - pr1 := &legacygithub.PullRequest{ - Body: gh.Ptr(`
google-cloud-storage: v1.2.3release notes
`), - Number: gh.Ptr(123), - MergeCommitSHA: gh.Ptr("abc123"), - Labels: []*gh.Label{{Name: gh.Ptr(releasePendingLabel)}}, - Base: &gh.PullRequestBranch{ - Ref: &branch, - }, - } - // This one will fail because the library details are not parsable. - pr2 := &legacygithub.PullRequest{ - Body: gh.Ptr(`
unknown-library: v1.0.0release notes
`), - Number: gh.Ptr(456), - MergeCommitSHA: gh.Ptr("xyz456"), - Labels: []*gh.Label{{Name: gh.Ptr(releasePendingLabel)}}, - Base: &gh.PullRequestBranch{ - Ref: &branch, - }, - } - ghClient := &mockGitHubClient{ - pullRequests: []*legacygithub.PullRequest{pr1, pr2}, - librarianState: &legacyconfig.LibrarianState{ - Image: "gcr.io/some-project/some-image:latest", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "google-cloud-storage", - SourceRoots: []string{"some/path"}, - TagFormat: "v{version}", - }, - }, - }, - } - - r := &tagRunner{ - ghClient: ghClient, - } - err := r.run(t.Context()) - if err == nil || !strings.Contains(err.Error(), "failed to process some pull requests") { - t.Fatalf("expected error 'failed to process some pull requests', got %v", err) - } - if ghClient.createReleaseCalls != 1 { - t.Errorf("createReleaseCalls = %v, want 1", ghClient.createReleaseCalls) - } - if ghClient.replaceLabelsCalls != 2 { - t.Errorf("replaceLabelsCalls = %v, want 2", ghClient.replaceLabelsCalls) - } - wantReleaseNames := []string{"google-cloud-storage: v1.2.3"} - if diff := cmp.Diff(wantReleaseNames, ghClient.releaseNames); diff != "" { - t.Errorf("releaseNames mismatch (-want +got):\n%s", diff) - } -} - -func TestSummaryRegex(t *testing.T) { - for _, tc := range []struct { - name string - input string - wantLibrary string - wantVersion string - wantMatch bool - }{ - { - name: "standard with v prefix", - input: "google-cloud-storage: v1.2.3", - wantLibrary: "google-cloud-storage", - wantVersion: "1.2.3", - wantMatch: true, - }, - { - name: "standard without v prefix", - input: "google-cloud-storage: 1.2.3", - wantLibrary: "google-cloud-storage", - wantVersion: "1.2.3", - wantMatch: true, - }, - { - name: "no space after colon", - input: "google-cloud-storage:v1.2.3", - wantMatch: false, - }, - { - name: "extra spaces", - input: " google-cloud-storage : v1.2.3", - wantLibrary: " google-cloud-storage ", - wantVersion: "1.2.3", - wantMatch: true, - }, - } { - t.Run(tc.name, func(t *testing.T) { - matches := summaryRegex.FindStringSubmatch(tc.input) - if !tc.wantMatch { - if matches != nil { - t.Errorf("expected no match for %q, but got %v", tc.input, matches) - } - return - } - if matches == nil { - t.Fatalf("expected match for %q, but got none", tc.input) - } - if len(matches) != 3 { - t.Fatalf("expected 3 match groups, got %d", len(matches)) - } - if got := matches[1]; got != tc.wantLibrary { - t.Errorf("library mismatch: got %q, want %q", got, tc.wantLibrary) - } - if got := matches[2]; got != tc.wantVersion { - t.Errorf("version mismatch: got %q, want %q", got, tc.wantVersion) - } - }) - } -} diff --git a/internal/legacylibrarian/legacylibrarian/test_container_generate.go b/internal/legacylibrarian/legacylibrarian/test_container_generate.go deleted file mode 100644 index 1295811e4e9..00000000000 --- a/internal/legacylibrarian/legacylibrarian/test_container_generate.go +++ /dev/null @@ -1,447 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacylibrarian - -import ( - "context" - "errors" - "fmt" - "io/fs" - "log/slog" - "os" - "path/filepath" - "strings" - - "github.com/google/uuid" - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" - "github.com/googleapis/librarian/internal/legacylibrarian/legacygitrepo" -) - -var errGenerateBlocked = errors.New("generation is blocked for library") - -type testGenerateRunner struct { - library string - repo legacygitrepo.Repository - sourceRepo legacygitrepo.Repository - state *legacyconfig.LibrarianState - librarianConfig *legacyconfig.LibrarianConfig - workRoot string - containerClient ContainerClient - checkUnexpectedChanges bool - branchesToDelete []string -} - -func (r *testGenerateRunner) run(ctx context.Context) error { - sourceRepoHead, err := r.sourceRepo.HeadHash() - if err != nil { - return fmt.Errorf("failed to get source repo head: %w", err) - } - - if err := os.MkdirAll(filepath.Join(r.workRoot, "output"), 0755); err != nil { - return fmt.Errorf("failed to create output directory under %s: %w", r.workRoot, err) - } - - return r.runTests(ctx, sourceRepoHead) -} - -// runTests executes the generation test for one or all libraries. -// -// If a single library is specified, it runs the test for that library. If the -// test is skipped (due to errGenerateBlocked), it logs and exits successfully. -// On failure, it returns an error, preserving the generated files for -// debugging. On success, it cleans up the temporary work directory. -// -// If no specific library is given, it runs tests for all libraries. It keeps -// track of failed and skipped tests. If any tests fail, it returns an error -// listing the failed libraries, preserving the generated files. If all tests -// pass or are skipped, it cleans up the work directory. -func (r *testGenerateRunner) runTests(ctx context.Context, sourceRepoHead string) error { - outputDir := filepath.Join(r.workRoot, "test-output") - if err := os.Mkdir(outputDir, 0755); err != nil { - return fmt.Errorf("failed to make output directory, %s: %w", outputDir, err) - } - if r.library != "" { - err := r.testSingleLibrary(ctx, r.library, sourceRepoHead, outputDir) - if errors.Is(err, errGenerateBlocked) { - slog.Info("test skipped for library due to generate_blocked", "library", r.library) - return nil - } - if err != nil { - return fmt.Errorf("test failed for library %s, keeping changes for debugging: %w", r.library, err) - } - slog.Info("test succeeded for library", "library", r.library) - if err := r.cleanup(); err != nil { - return err - } - return nil - } - - slog.Info("running tests for all libraries") - var failed []string - var skippedCount int - for _, library := range r.state.Libraries { - err := r.testSingleLibrary(ctx, library.ID, sourceRepoHead, outputDir) - if errors.Is(err, errGenerateBlocked) { - slog.Info("test skipped for library due to generate_blocked", "library", library.ID) - skippedCount++ - continue - } - if err != nil { - slog.Error("test failed for library", "library", library.ID, "error", err) - failed = append(failed, library.ID) - } else { - slog.Debug("test succeeded for library", "library", library.ID) - } - } - if len(failed) > 0 { - slog.Info("some tests failed, keeping changes for debugging", "branches", r.branchesToDelete) - return fmt.Errorf("generation tests failed for %d libraries: %s", len(failed), strings.Join(failed, ", ")) - } - - if skippedCount > 0 { - slog.Info("generation tests completed", "skipped_libraries", skippedCount) - } else { - slog.Info("generation tests succeeded for all libraries") - } - if err := r.cleanup(); err != nil { - return err - } - return nil -} - -func (r *testGenerateRunner) cleanup() error { - // Delete branches created during test in source repo - if err := r.sourceRepo.DeleteLocalBranches(r.branchesToDelete); err != nil { - return fmt.Errorf("failed to delete branch during cleanup: %w", err) - } - // Reset the code repo worktree to discard temp test changes at success - if err := r.repo.ResetHard(); err != nil { - return fmt.Errorf("failed to reset repo during cleanup: %w", err) - } - return nil -} - -// testSingleLibrary runs a generation test for a single library. -// The test performs the following steps: -// -// 1. Prepares the source repository: -// - Checks out the `last_generated_commit` from the source repository. -// - Injects unique GUIDs as comments into the first `message`/`enum` definition -// and the first `service` definition found in each proto file to simulate a change. -// - Commits these temporary changes to a new branch. -// -// 2. Runs the `generate` command for the specified library. -// 3. Validates the output: -// - Ensures that the generation command did not fail. -// - Verifies that every injected GUID appears in the generated output, -// confirming that the simulated changes triggered a corresponding update. -// - Optionally, checks for any unexpected file additions, deletions, or modifications. -// -// 4. Cleans up the source repository by checking out the original commit. -// -// Note: This function does not delete the temporary branch created in the source -// repository or reset the worktree in the code repository; these cleanup actions -// are handled by the caller. -func (r *testGenerateRunner) testSingleLibrary(ctx context.Context, libraryID, sourceRepoHead, outputDir string) error { - defer func() { - slog.Debug("resetting source repo to original commit", "library", libraryID) - if err := r.sourceRepo.Checkout(sourceRepoHead); err != nil { - slog.Error("failed to checkout source repo head during cleanup", "error", err) - } - }() - slog.Info("running generation test", "library", libraryID) - libraryState := r.state.LibraryByID(libraryID) - if libraryState == nil { - return fmt.Errorf("library %q not found in state", libraryID) - } - - if r.librarianConfig.IsGenerationBlocked(libraryID) { - return errGenerateBlocked - } - - protoFileToGUIDs, err := r.prepareForGenerateTest(libraryState, libraryID) - if err != nil { - return fmt.Errorf("failed in test preparing steps: %w", err) - } - - // We capture the error here and pass it to the validation step. - generateErr := generateSingleLibrary(ctx, r.containerClient, r.state, libraryState, r.repo, r.sourceRepo, r.state.Image, outputDir) - - if err := r.validateGenerateTest(generateErr, protoFileToGUIDs, libraryState); err != nil { - return fmt.Errorf("failed in test validation steps: %w", err) - } - - return nil -} - -// prepareForGenerateTest sets up the source repository for a generation test. It -// checks out a new branch from the library's last generated commit, injects unique -// GUIDs as comments into the relevant proto files, and commits these temporary -// changes. It returns a map of the modified proto file paths to the slice of -// GUIDs that were injected. -func (r *testGenerateRunner) prepareForGenerateTest(libraryState *legacyconfig.LibraryState, libraryID string) (map[string][]string, error) { - if libraryState.LastGeneratedCommit == "" { - return nil, fmt.Errorf("last_generated_commit is not set for library %q", libraryID) - } - - branchName := "test-generate-" + uuid.New().String() - if err := r.sourceRepo.CheckoutCommitAndCreateBranch(branchName, libraryState.LastGeneratedCommit); err != nil { - return nil, err - } - r.branchesToDelete = append(r.branchesToDelete, branchName) - - protoFiles, err := findProtoFiles(libraryState, r.sourceRepo) - if err != nil { - return nil, fmt.Errorf("failed finding proto files: %w", err) - } - - protoFileToGUIDs, err := injectTestGUIDsIntoProtoFiles(protoFiles, r.sourceRepo.GetDir()) - if err != nil { - return nil, fmt.Errorf("failed to inject test GUIDs into proto files: %w", err) - } - - if len(protoFileToGUIDs) == 0 { - return nil, fmt.Errorf("library %q configured to generate, but nothing to generate", libraryID) - } - - if err := r.sourceRepo.AddAll(); err != nil { - return nil, err - } - if err := r.sourceRepo.Commit("test(changes): temporary changes for generate test"); err != nil { - return nil, err - } - - return protoFileToGUIDs, nil -} - -// findProtoFiles recursively finds all .proto files within the API paths specified in -// the library's state. If no .proto files are found, it returns an empty slice -// and a nil error. An error is returned if any of the file system walks fail. -func findProtoFiles(libraryState *legacyconfig.LibraryState, repo legacygitrepo.Repository) ([]string, error) { - var protoFiles []string - repoPath := repo.GetDir() - for _, api := range libraryState.APIs { - root := filepath.Join(repoPath, api.Path) - err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() || !strings.HasSuffix(info.Name(), ".proto") { - return nil - } - relPath, err := filepath.Rel(repoPath, path) - if err != nil { - return err - } - protoFiles = append(protoFiles, relPath) - return nil - }) - if err != nil { - return nil, err - } - } - return protoFiles, nil -} - -// injectTestGUIDsIntoProtoFiles injects unique GUIDs into each proto file -// provided. It returns a map of file paths to the GUIDs that were successfully -// injected. -func injectTestGUIDsIntoProtoFiles(protoFiles []string, repoPath string) (map[string][]string, error) { - protoFileToGUIDs := make(map[string][]string) - for _, protoFile := range protoFiles { - guids, err := injectGUIDsIntoProto(filepath.Join(repoPath, protoFile)) - if err != nil { - return nil, fmt.Errorf("failed to inject GUID into %s: %w", protoFile, err) - } - if len(guids) > 0 { - protoFileToGUIDs[protoFile] = guids - } - } - return protoFileToGUIDs, nil -} - -// injectGUIDsIntoProto adds unique GUID comments to a single proto file to -// simulate a change. It finds suitable insertion points (before a message, enum, -// or service definition) and writes the modified content back to the file. It -// returns the GUIDs that were injected. -func injectGUIDsIntoProto(absPath string) ([]string, error) { - content, err := os.ReadFile(absPath) - if err != nil { - return nil, err - } - lines := strings.Split(string(content), "\n") - if len(content) == 0 { - return nil, nil - } - - commentsByLine := make(map[int][]string) - var injectedGUIDs []string - // find the first occurrence of message/enum, and the first occurrence of service separately - // because they usually correspond to separate generated files. - if guid := prepareGUIDInjection(lines, []string{"message ", "enum "}, commentsByLine); guid != "" { - injectedGUIDs = append(injectedGUIDs, guid) - } - if guid := prepareGUIDInjection(lines, []string{"service "}, commentsByLine); guid != "" { - injectedGUIDs = append(injectedGUIDs, guid) - } - - if len(injectedGUIDs) == 0 { - return nil, nil - } - - var newLines []string - for i, line := range lines { - if comments, ok := commentsByLine[i]; ok { - newLines = append(newLines, comments...) - } - newLines = append(newLines, line) - } - - output := strings.Join(newLines, "\n") - if err := os.WriteFile(absPath, []byte(output), 0644); err != nil { - return nil, err - } - return injectedGUIDs, nil -} - -// prepareGUIDInjection finds the first occurrence of any of the provided search -// terms and, if found, injects a new GUID comment into the commentsByLine map and -// returns the generated GUID. -func prepareGUIDInjection(lines []string, searchTerms []string, commentsByLine map[int][]string) string { - insertionLine := findProtoInsertionLine(lines, searchTerms) - if insertionLine != -1 { - guid := uuid.New().String() - comment := "// test-change-" + guid - commentsByLine[insertionLine] = append(commentsByLine[insertionLine], comment) - return guid - } - return "" -} - -// findProtoInsertionLine determines the best line number to inject a test comment -// in a proto file. It searches for the first occurrence of a top-level definition -// matching one of the search terms. -func findProtoInsertionLine(lines []string, searchTerms []string) int { - for i, line := range lines { - for _, term := range searchTerms { - if strings.HasPrefix(strings.TrimSpace(line), term) { - return i - } - } - } - return -1 -} - -// validateGenerateTest checks the results of the generation process. It ensures -// that the generation command did not fail, that every injected proto change -// resulted in a corresponding change in the generated code, and optionally -// verifies that no other unexpected files were added, deleted, or modified. -func (r *testGenerateRunner) validateGenerateTest(generateErr error, protoFileToGUIDs map[string][]string, libraryState *legacyconfig.LibraryState) error { - slog.Debug("validating generation results") - if generateErr != nil { - return fmt.Errorf("the generation command failed: %w", generateErr) - } - - // Get the list of uncommitted changed files from the worktree. - changedFiles, err := r.repo.ChangedFiles() - if err != nil { - return fmt.Errorf("failed to get changed files from working tree: %w", err) - } - - changedFiles = filterFilesBySourceRoots(changedFiles, libraryState.SourceRoots) - - if r.checkUnexpectedChanges { - newAndDeleted, err := r.repo.NewAndDeletedFiles() - if err != nil { - return fmt.Errorf("failed to get new and deleted files: %w", err) - } - newAndDeleted = filterFilesBySourceRoots(newAndDeleted, libraryState.SourceRoots) - if len(newAndDeleted) > 0 { - return fmt.Errorf("expected no new or deleted files, but found: %s", strings.Join(newAndDeleted, ", ")) - } - slog.Debug("validation succeeded: no new or deleted files") - } - - guidsToFind := make(map[string]bool) - for _, guids := range protoFileToGUIDs { - for _, guid := range guids { - guidsToFind[guid] = false - } - } - filesWithGUIDs := make(map[string]bool) - repoDir := r.repo.GetDir() - - for _, filePath := range changedFiles { - fullPath := filepath.Join(repoDir, filePath) - content, err := os.ReadFile(fullPath) - if err != nil { - if errors.Is(err, fs.ErrNotExist) { // The file was deleted, ignoring if not checkUnexpectedChanges - continue - } - return fmt.Errorf("failed to read changed file %s: %w", filePath, err) - } - - contentStr := string(content) - wasModifiedByTest := false - for guid := range guidsToFind { - if strings.Contains(contentStr, guid) { - guidsToFind[guid] = true - wasModifiedByTest = true - } - } - if wasModifiedByTest { - filesWithGUIDs[filePath] = true - } - } - - for protoFile, guids := range protoFileToGUIDs { - for _, guid := range guids { - if !guidsToFind[guid] { - return fmt.Errorf("change in proto file %s (GUID %s) produced no corresponding generated file changes", protoFile, guid) - } - } - } - slog.Debug("validation succeeded: all proto changes resulted in generated file changes") - - if r.checkUnexpectedChanges { - var unrelatedChanges []string - for _, filePath := range changedFiles { - if !filesWithGUIDs[filePath] { - unrelatedChanges = append(unrelatedChanges, filePath) - } - } - if len(unrelatedChanges) > 0 { - return fmt.Errorf("found unrelated file changes: %s", strings.Join(unrelatedChanges, ", ")) - } - slog.Debug("validation succeeded: no unrelated file changes found") - } - - slog.Debug("all generation validation passed") - return nil -} - -func filterFilesBySourceRoots(files []string, sourceRoots []string) []string { - var filteredFiles []string - for _, file := range files { - for _, sourceRoot := range sourceRoots { - relPath, err := filepath.Rel(sourceRoot, file) - if err == nil && !strings.HasPrefix(relPath, "..") { - filteredFiles = append(filteredFiles, file) - break - } - } - } - return filteredFiles -} diff --git a/internal/legacylibrarian/legacylibrarian/test_container_generate_test.go b/internal/legacylibrarian/legacylibrarian/test_container_generate_test.go deleted file mode 100644 index b6e5ed08509..00000000000 --- a/internal/legacylibrarian/legacylibrarian/test_container_generate_test.go +++ /dev/null @@ -1,712 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacylibrarian - -import ( - "context" - "errors" - "fmt" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" -) - -func TestValidateGenerateTest(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - filesToWrite map[string]string - changedFiles []string - newAndDeletedFiles []string - libraryState *legacyconfig.LibraryState - setup func(dir string) error - protoFileToGUIDs map[string][]string - checkUnexpectedChanges bool - wantErrMsg string - }{ - { - name: "unrelated changes", - filesToWrite: map[string]string{ - "related.go": "// some generated code\n// test-change-guid-123", - "unrelated.txt": "some other content", - }, - protoFileToGUIDs: map[string][]string{"some.proto": {"guid-123"}}, - checkUnexpectedChanges: true, - wantErrMsg: "found unrelated file changes: unrelated.txt", - }, - { - name: "unrelated changes outside source root", - filesToWrite: map[string]string{ - "src/related.go": "// some generated code\n// test-change-guid-123", - "unrelated.txt": "some other content", - }, - protoFileToGUIDs: map[string][]string{"some.proto": {"guid-123"}}, - libraryState: &legacyconfig.LibraryState{SourceRoots: []string{"src"}}, - checkUnexpectedChanges: true, - wantErrMsg: "", // No error, because unrelated.txt is ignored. - }, - { - name: "missing change", - filesToWrite: map[string]string{ - "somefile.go": "some content", - }, - protoFileToGUIDs: map[string][]string{"some.proto": {"guid-not-found"}}, - checkUnexpectedChanges: true, - wantErrMsg: "produced no corresponding generated file changes", - }, - { - name: "success", - filesToWrite: map[string]string{ - "some.go": "// some generated code\n// test-change-guid-123", - }, - protoFileToGUIDs: map[string][]string{"some.proto": {"guid-123"}}, - checkUnexpectedChanges: true, - wantErrMsg: "", - }, - { - name: "expected no file changes, but found changes", - filesToWrite: map[string]string{ - "somefile.go": "some content", - }, - newAndDeletedFiles: []string{"somefile.go"}, - protoFileToGUIDs: map[string][]string{}, - checkUnexpectedChanges: true, - wantErrMsg: "expected no new or deleted files, but found", - }, - { - name: "deleted file is a valid change when not checking for unexpected changes", - filesToWrite: map[string]string{ - // "deleted.go" is not written to the filesystem - }, - changedFiles: []string{"deleted.go"}, - newAndDeletedFiles: []string{"deleted.go"}, - protoFileToGUIDs: map[string][]string{}, - checkUnexpectedChanges: false, // This is the key - wantErrMsg: "", // No error expected - }, - { - name: "unreadable file causes an error", - filesToWrite: map[string]string{ - "unreadable.go": "some content", - }, - changedFiles: []string{"unreadable.go"}, - setup: func(dir string) error { - // Make the file unreadable - return os.Chmod(filepath.Join(dir, "unreadable.go"), 0000) - }, - protoFileToGUIDs: map[string][]string{}, - checkUnexpectedChanges: true, - wantErrMsg: "failed to read changed file", - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - tmpDir := t.TempDir() - var filesConsideredChanged []string - for filename, content := range test.filesToWrite { - path := filepath.Join(tmpDir, filename) - if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { - t.Fatalf("failed to create directory for %s: %v", filename, err) - } - if err := os.WriteFile(path, []byte(content), 0644); err != nil { - t.Fatalf("failed to write file %s: %v", filename, err) - } - filesConsideredChanged = append(filesConsideredChanged, filename) - } - - if test.setup != nil { - if err := test.setup(tmpDir); err != nil { - t.Fatalf("setup failed: %v", err) - } - } - - mockRepo := &MockRepository{ - Dir: tmpDir, - ChangedFilesValue: filesConsideredChanged, - NewAndDeletedFilesValue: test.newAndDeletedFiles, - } - if test.changedFiles != nil { - mockRepo.ChangedFilesValue = test.changedFiles - } - - runner := &testGenerateRunner{ - repo: mockRepo, - checkUnexpectedChanges: test.checkUnexpectedChanges, - } - libraryState := test.libraryState - if libraryState == nil { - // Default to the root directory if not specified. - libraryState = &legacyconfig.LibraryState{SourceRoots: []string{""}} - } - err := runner.validateGenerateTest(nil, test.protoFileToGUIDs, libraryState) - - if test.wantErrMsg != "" { - if err == nil { - t.Fatalf("validateGenerateTest() did not return an error, but one was expected") - } - if !strings.Contains(err.Error(), test.wantErrMsg) { - t.Errorf("validateGenerateTest() returned error %q, want error containing %q", err.Error(), test.wantErrMsg) - } - } else if err != nil { - t.Fatalf("validateGenerateTest() returned unexpected error: %v", err) - } - }) - } -} - -func TestPrepareForGenerateTest(t *testing.T) { - t.Parallel() - - // Common setup for all test cases - const ( - protoPath = "google/cloud/aiplatform/v1" - protoFilename = "prediction_service.proto" - initialCommit = "abcdef1234567890abcdef1234567890abcdef12" - libraryID = "google-cloud-aiplatform-v1" - checkoutErrStr = "checkout error" - addAllErrStr = "add all error" - commitErrStr = "commit error" - ) - protoWithServiceAndMessage := ` -syntax = "proto3"; -package google.cloud.aiplatform.v1; -import "google/api/annotations.proto"; -service PredictionService { - rpc Predict(PredictRequest) returns (PredictResponse) { - option (google.api.http) = { - post: "/v1/{endpoint=projects/*/locations/*/endpoints/*}:predict" - body: "*" - }; - } -} -message PredictRequest {} -message PredictResponse {} -` - protoWithServiceOnly := ` -syntax = "proto3"; -package google.cloud.aiplatform.v1; -service PredictionService {} -` - protoWithMessageOnly := ` -syntax = "proto3"; -package google.cloud.aiplatform.v1; -message PredictRequest {} -` - - for _, test := range []struct { - name string - libraryState *legacyconfig.LibraryState - mockRepo *MockRepository - protoContent string - setup func(repoDir string) error - wantErrMsg string - wantCommitCalls int - wantGUIDCount int - }{ - { - name: "success with service and message", - libraryState: &legacyconfig.LibraryState{ - ID: libraryID, - LastGeneratedCommit: initialCommit, - APIs: []*legacyconfig.API{{Path: protoPath}}, - }, - mockRepo: &MockRepository{}, - protoContent: protoWithServiceAndMessage, - wantErrMsg: "", - wantCommitCalls: 1, - wantGUIDCount: 2, - }, - { - name: "success with service only", - libraryState: &legacyconfig.LibraryState{ - ID: libraryID, - LastGeneratedCommit: initialCommit, - APIs: []*legacyconfig.API{{Path: protoPath}}, - }, - mockRepo: &MockRepository{}, - protoContent: protoWithServiceOnly, - wantErrMsg: "", - wantCommitCalls: 1, - wantGUIDCount: 1, - }, - { - name: "success with message only", - libraryState: &legacyconfig.LibraryState{ - ID: libraryID, - LastGeneratedCommit: initialCommit, - APIs: []*legacyconfig.API{{Path: protoPath}}, - }, - mockRepo: &MockRepository{}, - protoContent: protoWithMessageOnly, - wantErrMsg: "", - wantCommitCalls: 1, - wantGUIDCount: 1, - }, - { - name: "missing last generated commit", - libraryState: &legacyconfig.LibraryState{ - ID: libraryID, - LastGeneratedCommit: "", // Missing commit - APIs: []*legacyconfig.API{{Path: protoPath}}, - }, - mockRepo: &MockRepository{}, - protoContent: protoWithServiceAndMessage, - wantErrMsg: "last_generated_commit is not set", - wantCommitCalls: 0, - wantGUIDCount: 0, - }, - { - name: "checkout commit error", - libraryState: &legacyconfig.LibraryState{ - ID: libraryID, - LastGeneratedCommit: initialCommit, - APIs: []*legacyconfig.API{{Path: protoPath}}, - }, - mockRepo: &MockRepository{ - CheckoutCommitAndCreateBranchError: errors.New(checkoutErrStr), - }, - protoContent: protoWithServiceAndMessage, - wantErrMsg: checkoutErrStr, - wantCommitCalls: 0, - wantGUIDCount: 0, - }, - { - name: "add all error", - libraryState: &legacyconfig.LibraryState{ - ID: libraryID, - LastGeneratedCommit: initialCommit, - APIs: []*legacyconfig.API{{Path: protoPath}}, - }, - mockRepo: &MockRepository{ - AddAllError: errors.New(addAllErrStr), - }, - protoContent: protoWithServiceAndMessage, - wantErrMsg: addAllErrStr, - wantCommitCalls: 0, - wantGUIDCount: 0, - }, - { - name: "commit error", - libraryState: &legacyconfig.LibraryState{ - ID: libraryID, - LastGeneratedCommit: initialCommit, - APIs: []*legacyconfig.API{{Path: protoPath}}, - }, - mockRepo: &MockRepository{ - CommitError: errors.New(commitErrStr), - }, - protoContent: protoWithServiceAndMessage, - wantErrMsg: commitErrStr, - wantCommitCalls: 1, // Commit is still called - wantGUIDCount: 0, - }, - { - name: "empty proto file", - libraryState: &legacyconfig.LibraryState{ - ID: libraryID, - LastGeneratedCommit: initialCommit, - APIs: []*legacyconfig.API{{Path: protoPath}}, - }, - mockRepo: &MockRepository{}, - protoContent: "", // Empty content - wantErrMsg: "configured to generate, but nothing to generate", - wantCommitCalls: 0, - wantGUIDCount: 0, - }, - { - name: "proto file with no insertion point", - libraryState: &legacyconfig.LibraryState{ - ID: libraryID, - LastGeneratedCommit: initialCommit, - APIs: []*legacyconfig.API{{Path: protoPath}}, - }, - mockRepo: &MockRepository{}, - protoContent: ` -syntax = "proto3"; -package google.cloud.aiplatform.v1; -import "google/api/annotations.proto"; -// no message, service or enum -`, - wantErrMsg: "configured to generate, but nothing to generate", - wantCommitCalls: 0, - wantGUIDCount: 0, - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - repoDir := t.TempDir() - test.mockRepo.Dir = repoDir - - // Setup proto files - fullProtoDir := filepath.Join(repoDir, protoPath) - if err := os.MkdirAll(fullProtoDir, 0755); err != nil { - t.Fatalf("os.MkdirAll() error = %v", err) - } - fullProtoPath := filepath.Join(fullProtoDir, protoFilename) - if err := os.WriteFile(fullProtoPath, []byte(test.protoContent), 0644); err != nil { - t.Fatalf("os.WriteFile() error = %v", err) - } - - if test.setup != nil { - if err := test.setup(repoDir); err != nil { - t.Fatalf("setup() error = %v", err) - } - } - - runner := &testGenerateRunner{ - sourceRepo: test.mockRepo, - } - protoFileToGUIDs, err := runner.prepareForGenerateTest(test.libraryState, test.libraryState.ID) - - // Check for error - if test.wantErrMsg != "" { - if err == nil { - t.Fatalf("prepareForGenerateTest() did not return an error, but one was expected") - } - if !strings.Contains(err.Error(), test.wantErrMsg) { - t.Errorf("prepareForGenerateTest() returned error %q, want error containing %q", err.Error(), test.wantErrMsg) - } - } else if err != nil { - t.Fatalf("prepareForGenerateTest() returned unexpected error: %v", err) - } - - // Check if a GUID was expected to be injected. - if test.wantGUIDCount > 0 { - if len(protoFileToGUIDs) != 1 { - t.Fatalf("len(protoFileToGUIDs) = %d, want 1", len(protoFileToGUIDs)) - } - // Check the number of GUIDs injected into the single proto file. - injectedGUIDs := protoFileToGUIDs[filepath.Join(protoPath, protoFilename)] - if len(injectedGUIDs) != test.wantGUIDCount { - t.Errorf("len(injectedGUIDs) = %d, want %d", len(injectedGUIDs), test.wantGUIDCount) - } - } else { - if len(protoFileToGUIDs) != 0 { - t.Fatalf("len(protoFileToGUIDs) = %d, want 0", len(protoFileToGUIDs)) - } - } - - // Check if the expected number of commits were made. - if test.mockRepo.CommitCalls != test.wantCommitCalls { - t.Errorf("mockRepo.CommitCalls = %d, want %d", test.mockRepo.CommitCalls, test.wantCommitCalls) - } - }) - } -} - -func TestTestGenerateRunnerRun(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - state *legacyconfig.LibrarianState - librarianConfig *legacyconfig.LibrarianConfig - libraryID string - prepareErr error - generateErr error - validateErr error - wantErrMsg string - checkUnexpectedChanges bool - repoChangedFiles []string - }{ - { - name: "library not found", - state: &legacyconfig.LibrarianState{}, - libraryID: "non-existent-library", - wantErrMsg: "library \"non-existent-library\" not found in state", - }, - { - name: "generate blocked library is skipped", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "blocked-lib", - LastGeneratedCommit: "initial-commit", - APIs: []*legacyconfig.API{{Path: "google/blocked/v1"}}, - }, - }, - }, - librarianConfig: &legacyconfig.LibrarianConfig{ - Libraries: []*legacyconfig.LibraryConfig{ - { - LibraryID: "blocked-lib", - GenerateBlocked: true, - }, - }, - }, - libraryID: "blocked-lib", - wantErrMsg: "", // No error expected, as it should be skipped - }, - { - name: "prepareForGenerateTest error", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "google-cloud-aiplatform-v1", - LastGeneratedCommit: "initial-commit", - APIs: []*legacyconfig.API{ - { - Path: "google/cloud/aiplatform/v1", - }, - }, - }, - }, - }, - libraryID: "google-cloud-aiplatform-v1", - prepareErr: fmt.Errorf("checkout error"), - wantErrMsg: "checkout error", - }, - { - name: "generateSingleLibrary error", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "google-cloud-aiplatform-v1", - LastGeneratedCommit: "initial-commit", - APIs: []*legacyconfig.API{ - { - Path: "google/cloud/aiplatform/v1", - }, - }, - }, - }, - }, - libraryID: "google-cloud-aiplatform-v1", - generateErr: fmt.Errorf("generate error"), - wantErrMsg: "generation command failed: generate error", - }, - { - name: "validateGenerateTest error", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "google-cloud-aiplatform-v1", - LastGeneratedCommit: "initial-commit", - APIs: []*legacyconfig.API{ - { - Path: "google/cloud/aiplatform/v1", - }, - }, - }, - }, - }, - libraryID: "google-cloud-aiplatform-v1", - checkUnexpectedChanges: true, - repoChangedFiles: []string{"unrelated.txt"}, - wantErrMsg: "produced no corresponding generated file changes", - }, - { - name: "multiple library failures", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - { - ID: "lib1", - LastGeneratedCommit: "initial-commit", - APIs: []*legacyconfig.API{ - {Path: "google/lib1/v1"}, - }, - }, - { - ID: "lib2", - LastGeneratedCommit: "initial-commit", - APIs: []*legacyconfig.API{ - {Path: "google/lib2/v1"}, - }, - }, - }, - }, - libraryID: "", // Run for all libraries. - generateErr: fmt.Errorf("generate error"), - wantErrMsg: "generation tests failed for 2 libraries", - }, - { - name: "multiple libraries, some skipped", - state: &legacyconfig.LibrarianState{ - Libraries: []*legacyconfig.LibraryState{ - - { - ID: "lib1", - LastGeneratedCommit: "initial-commit", - APIs: []*legacyconfig.API{{Path: "google/lib1/v1"}}, - }, - { - ID: "lib2", - LastGeneratedCommit: "initial-commit", - APIs: []*legacyconfig.API{{Path: "google/lib2/v1"}}, - }, - }, - }, - librarianConfig: &legacyconfig.LibrarianConfig{ - Libraries: []*legacyconfig.LibraryConfig{ - { - LibraryID: "lib2", - GenerateBlocked: true, // lib2 will be skipped - }, - }, - }, - libraryID: "", // Run for all libraries - wantErrMsg: "generation tests failed for 1 libraries", - }, - } { - t.Run(test.name, func(t *testing.T) { - // 1. Setup the runner with mocked dependencies based on the test case. - // Create a temporary directory to act as a mock git repository. - repoDir := t.TempDir() - - // Create dummy proto files within the mock repository if the test case needs them. - // This is needed because the prepare step searches for .proto files to modify. - if test.state != nil { - for _, lib := range test.state.Libraries { - if len(lib.APIs) > 0 { - protoPath := lib.APIs[0].Path - protoFilename := "service.proto" - fullProtoDir := filepath.Join(repoDir, protoPath) - if err := os.MkdirAll(fullProtoDir, 0755); err != nil { - t.Fatalf("os.MkdirAll() error = %v", err) - } - protoContent := "service MyService {}" - if err := os.WriteFile(filepath.Join(fullProtoDir, protoFilename), []byte(protoContent), 0644); err != nil { - t.Fatalf("os.WriteFile() error = %v", err) - } - } - } - } - - // Set up the mock repositories and clients with behavior defined by the test case. - mockSourceRepo := &MockRepository{ - Dir: repoDir, - CheckoutCommitAndCreateBranchError: test.prepareErr, - } - mockRepoDir := t.TempDir() - for _, f := range test.repoChangedFiles { - if err := os.WriteFile(filepath.Join(mockRepoDir, f), []byte("some content"), 0644); err != nil { - t.Fatalf("failed to write file: %v", err) - } - } - mockRepo := &MockRepository{ - Dir: mockRepoDir, - ChangedFilesValue: test.repoChangedFiles, - } - mockContainerClient := &mockContainerClient{ - generateErr: test.generateErr, - } - - // Create testGenerateRunner with the mocked dependencies. - - runner := &testGenerateRunner{ - library: test.libraryID, - repo: mockRepo, - sourceRepo: mockSourceRepo, - state: test.state, - librarianConfig: test.librarianConfig, - workRoot: t.TempDir(), - containerClient: mockContainerClient, - checkUnexpectedChanges: test.checkUnexpectedChanges, - } - - // 2. Execute the run method. - err := runner.run(context.Background()) - - // 3. Assert the outcome. - if test.wantErrMsg != "" { - if err == nil { - t.Fatal("runner.run() did not return an error, but one was expected") - } - if !strings.Contains(err.Error(), test.wantErrMsg) { - t.Errorf("runner.run() returned error %q, want error containing %q", err.Error(), test.wantErrMsg) - } - } else if err != nil { - t.Fatalf("runner.run() returned unexpected error: %v", err) - } - }) - } -} - -func TestCleanup(t *testing.T) { - t.Parallel() - - for _, test := range []struct { - name string - sourceRepo *MockRepository - repo *MockRepository - branchesToDelete []string - wantDeleteLocalBranchesCalls int - wantResetHardCalls int - wantErrMsg string - }{ - { - name: "successful cleanup", - sourceRepo: &MockRepository{}, - repo: &MockRepository{}, - branchesToDelete: []string{"test-branch-1"}, - wantDeleteLocalBranchesCalls: 1, - wantResetHardCalls: 1, - }, - { - name: "DeleteLocalBranches returns error", - sourceRepo: &MockRepository{ - DeleteLocalBranchesError: errors.New("delete branch error"), - }, - repo: &MockRepository{}, - branchesToDelete: []string{"test-branch-2"}, - wantDeleteLocalBranchesCalls: 1, - wantResetHardCalls: 0, // ResetHard should not be called - wantErrMsg: "failed to delete branch", - }, - { - name: "ResetHard returns error", - sourceRepo: &MockRepository{}, - repo: &MockRepository{ - ResetHardError: errors.New("reset hard error"), - }, - branchesToDelete: []string{"test-branch-3"}, - wantDeleteLocalBranchesCalls: 1, - wantResetHardCalls: 1, - wantErrMsg: "failed to reset repo", - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - - runner := &testGenerateRunner{ - sourceRepo: test.sourceRepo, - repo: test.repo, - branchesToDelete: test.branchesToDelete, - } - - err := runner.cleanup() - - if test.wantErrMsg != "" { - if err == nil { - t.Fatalf("cleanup() did not return an error, but one was expected containing %q", test.wantErrMsg) - } - if !strings.Contains(err.Error(), test.wantErrMsg) { - t.Errorf("cleanup() returned error %q, want error containing %q", err.Error(), test.wantErrMsg) - } - } else if err != nil { - t.Fatalf("cleanup() returned unexpected error: %v", err) - } - - if test.sourceRepo.DeleteLocalBranchesCalls != test.wantDeleteLocalBranchesCalls { - t.Errorf("DeleteLocalBranches was called %d times, want %d", test.sourceRepo.DeleteLocalBranchesCalls, test.wantDeleteLocalBranchesCalls) - } - if test.repo.ResetHardCalls != test.wantResetHardCalls { - t.Errorf("ResetHard was called %d times, want %d", test.repo.ResetHardCalls, test.wantResetHardCalls) - } - }) - } -} diff --git a/internal/legacylibrarian/legacylibrarian/testdata/find_a_service_config/another_config.yaml b/internal/legacylibrarian/legacylibrarian/testdata/find_a_service_config/another_config.yaml deleted file mode 100644 index d1d0728b96d..00000000000 --- a/internal/legacylibrarian/legacylibrarian/testdata/find_a_service_config/another_config.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -something: someValue diff --git a/internal/legacylibrarian/legacylibrarian/testdata/find_a_service_config/random_file.txt b/internal/legacylibrarian/legacylibrarian/testdata/find_a_service_config/random_file.txt deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/internal/legacylibrarian/legacylibrarian/testdata/find_a_service_config/service_config.yaml b/internal/legacylibrarian/legacylibrarian/testdata/find_a_service_config/service_config.yaml deleted file mode 100644 index 706688903cd..00000000000 --- a/internal/legacylibrarian/legacylibrarian/testdata/find_a_service_config/service_config.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -type: google.api.Service -anything: any_value diff --git a/internal/legacylibrarian/legacylibrarian/testdata/invalid_yaml/invalid_yaml.yaml b/internal/legacylibrarian/legacylibrarian/testdata/invalid_yaml/invalid_yaml.yaml deleted file mode 100644 index 1ee26d7c815..00000000000 --- a/internal/legacylibrarian/legacylibrarian/testdata/invalid_yaml/invalid_yaml.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -random diff --git a/internal/legacylibrarian/legacylibrarian/testdata/no_service_config/some_config.yaml b/internal/legacylibrarian/legacylibrarian/testdata/no_service_config/some_config.yaml deleted file mode 100644 index a9e5029b4e6..00000000000 --- a/internal/legacylibrarian/legacylibrarian/testdata/no_service_config/some_config.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -randomKey: randomValue diff --git a/internal/legacylibrarian/legacylibrarian/testdata/populate_service_config/example/api/example_api_config.yaml b/internal/legacylibrarian/legacylibrarian/testdata/populate_service_config/example/api/example_api_config.yaml deleted file mode 100644 index 706688903cd..00000000000 --- a/internal/legacylibrarian/legacylibrarian/testdata/populate_service_config/example/api/example_api_config.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -type: google.api.Service -anything: any_value diff --git a/internal/legacylibrarian/legacylibrarian/testdata/test-parse-global-config/invalid-global-config.yaml b/internal/legacylibrarian/legacylibrarian/testdata/test-parse-global-config/invalid-global-config.yaml deleted file mode 100644 index 425c1da09cb..00000000000 --- a/internal/legacylibrarian/legacylibrarian/testdata/test-parse-global-config/invalid-global-config.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -global_files_allowlist: - - path: a/path - permissions: wrong-permission diff --git a/internal/legacylibrarian/legacylibrarian/testdata/test-parse-global-config/successful-parsing-config.yaml b/internal/legacylibrarian/legacylibrarian/testdata/test-parse-global-config/successful-parsing-config.yaml deleted file mode 100644 index 84d1ae6d820..00000000000 --- a/internal/legacylibrarian/legacylibrarian/testdata/test-parse-global-config/successful-parsing-config.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -global_files_allowlist: - - path: a/path - permissions: read-only - - path: another/path - permissions: read-write -libraries: - - id: example-library diff --git a/internal/legacylibrarian/legacylibrarian/testdata/test-read-library-state/empty-libraryState.json b/internal/legacylibrarian/legacylibrarian/testdata/test-read-library-state/empty-libraryState.json deleted file mode 100644 index 0967ef424bc..00000000000 --- a/internal/legacylibrarian/legacylibrarian/testdata/test-read-library-state/empty-libraryState.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/internal/legacylibrarian/legacylibrarian/testdata/test-read-library-state/successful-unmarshal-libraryState.json b/internal/legacylibrarian/legacylibrarian/testdata/test-read-library-state/successful-unmarshal-libraryState.json deleted file mode 100644 index 0981090f0d0..00000000000 --- a/internal/legacylibrarian/legacylibrarian/testdata/test-read-library-state/successful-unmarshal-libraryState.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "id": "google-cloud-go", - "version": "1.0.0", - "apis": [ - { - "path": "google/cloud/compute/v1", - "service_config": "example_service_config.yaml" - } - ], - "source_roots": [ - "src/example/path" - ], - "preserve_regex": [ - "example-preserve-regex" - ], - "remove_regex": [ - "example-remove-regex" - ] -} \ No newline at end of file diff --git a/internal/legacylibrarian/legacylibrarian/testdata/test-read-library-state/unmarshal-libraryState-with-error-msg.json b/internal/legacylibrarian/legacylibrarian/testdata/test-read-library-state/unmarshal-libraryState-with-error-msg.json deleted file mode 100644 index 7006b853161..00000000000 --- a/internal/legacylibrarian/legacylibrarian/testdata/test-read-library-state/unmarshal-libraryState-with-error-msg.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "id": "google-cloud-go", - "version": "1.0.0", - "apis": [ - { - "path": "google/cloud/compute/v1", - "service_config": "example_service_config.yaml" - } - ], - "source_roots": [], - "preserve_regex": [], - "remove_regex": [], - "error": "simulated error message" -} \ No newline at end of file diff --git a/internal/legacylibrarian/legacylibrarian/update_image.go b/internal/legacylibrarian/legacylibrarian/update_image.go deleted file mode 100644 index 4912e8dcad2..00000000000 --- a/internal/legacylibrarian/legacylibrarian/update_image.go +++ /dev/null @@ -1,295 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacylibrarian - -import ( - "bytes" - "context" - "errors" - "fmt" - "html/template" - "log/slog" - "path/filepath" - "strings" - "time" - - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" - "github.com/googleapis/librarian/internal/legacylibrarian/legacygitrepo" - "github.com/googleapis/librarian/internal/legacylibrarian/legacyimages" -) - -const ( - updateImageCmdName = "update-image" -) - -type updateImageRunner struct { - branch string - containerClient ContainerClient - imagesClient ImageRegistryClient - ghClient GitHubClient - librarianConfig *legacyconfig.LibrarianConfig - repo legacygitrepo.Repository - sourceRepo legacygitrepo.Repository - state *legacyconfig.LibrarianState - build bool - push bool - commit bool - image string - workRoot string - test bool - libraryToTest string - checkUnexpectedChanges bool -} - -// ImageRegistryClient is an abstraction around interacting with image. -type ImageRegistryClient interface { - FindLatest(ctx context.Context, imageName string) (string, error) -} - -func newUpdateImageRunner(cfg *legacyconfig.Config) (*updateImageRunner, error) { - runner, err := newCommandRunner(cfg) - if err != nil { - return nil, err - } - return &updateImageRunner{ - branch: cfg.Branch, - containerClient: runner.containerClient, - ghClient: runner.ghClient, - librarianConfig: runner.librarianConfig, - repo: runner.repo, - sourceRepo: runner.sourceRepo, - state: runner.state, - build: cfg.Build, - commit: cfg.Commit, - push: cfg.Push, - image: cfg.Image, - workRoot: runner.workRoot, - test: cfg.Test, - libraryToTest: cfg.LibraryToTest, - checkUnexpectedChanges: cfg.CheckUnexpectedChanges, - }, nil -} - -func (r *updateImageRunner) run(ctx context.Context) error { - if r.librarianConfig != nil && r.librarianConfig.ReleaseOnlyMode { - return errGenerateInReleaseOnlyMode - } - imagesClient := r.imagesClient - if imagesClient == nil { - slog.Info("no imagesClient provided, defaulting to ArtifactRegistry implementation") - client, err := legacyimages.NewArtifactRegistryClient(ctx) - if err != nil { - return err - } - defer client.Close() - imagesClient = client - } - - // Update `image` entry in state.yaml - if r.image == "" { - slog.Info("no image found, looking up latest") - latestImage, err := imagesClient.FindLatest(ctx, r.state.Image) - if err != nil { - slog.Error("unable to determine latest image to use", "image", r.state.Image) - return err - } - r.image = latestImage - } - - if r.image == r.state.Image { - slog.Info("no update to the image; assuming diagnostic run") - } - - r.state.Image = r.image - - if err := saveLibrarianState(r.repo.GetDir(), r.state); err != nil { - return err - } - - // For each library, run generation at the previous commit - var failedGenerations []*legacyconfig.LibraryState - var successfulGenerations []*legacyconfig.LibraryState - var skippedGenerationsCount int - sourceHead, err := r.sourceRepo.HeadHash() - if err != nil { - return err - } - outputDir := filepath.Join(r.workRoot, "output") - timings := map[string]time.Duration{} - for _, libraryState := range r.state.Libraries { - if r.librarianConfig.IsGenerationBlocked(libraryState.ID) { - slog.Debug("skipping generation for library due to generate_blocked", "library", libraryState.ID) - skippedGenerationsCount++ - continue - } - startTime := time.Now() - err := r.regenerateSingleLibrary(ctx, libraryState, outputDir) - if err != nil { - slog.Error(err.Error(), "library", libraryState.ID, "commit", libraryState.LastGeneratedCommit) - failedGenerations = append(failedGenerations, libraryState) - continue - } else { - successfulGenerations = append(successfulGenerations, libraryState) - } - timings[libraryState.ID] = time.Since(startTime) - } - if len(failedGenerations) > 0 { - slog.Warn("failed generations", "num", len(failedGenerations)) - } - slog.Info( - "generation statistics", - "all", len(r.state.Libraries), - "successes", len(successfulGenerations), - "skipped", skippedGenerationsCount, - "failures", len(failedGenerations)) - if err := writeTiming(r.workRoot, timings); err != nil { - return err - } - - // Restore api source repo - if err := r.sourceRepo.Checkout(sourceHead); err != nil { - slog.Error(err.Error(), "repository", r.sourceRepo, "HEAD", sourceHead) - } - if r.test { - slog.Info("running container tests") - testRunner := &testGenerateRunner{ - library: r.libraryToTest, - repo: r.repo, - sourceRepo: r.sourceRepo, - state: r.state, - librarianConfig: r.librarianConfig, - workRoot: r.workRoot, - containerClient: r.containerClient, - checkUnexpectedChanges: r.checkUnexpectedChanges, - branchesToDelete: []string{}, - } - if err := runContainerGenerateTest(ctx, r.repo, sourceHead, testRunner); err != nil { - return fmt.Errorf("container generate test failed: %w", err) - } - } - prBodyBuilder := func() (string, error) { - return formatUpdateImagePRBody(r.image, failedGenerations) - } - commitMessage := fmt.Sprintf("feat: update image to %s", r.image) - return commitAndPush(ctx, &commitInfo{ - branch: r.branch, - commit: r.commit, - commitMessage: commitMessage, - prType: pullRequestUpdateImage, - ghClient: r.ghClient, - pullRequestLabels: []string{}, - push: r.push, - languageRepo: r.repo, - sourceRepo: r.sourceRepo, - state: r.state, - workRoot: r.workRoot, - failedGenerations: len(failedGenerations), - prBodyBuilder: prBodyBuilder, - isDraft: len(failedGenerations) > 0, - }) -} - -func (r *updateImageRunner) regenerateSingleLibrary(ctx context.Context, libraryState *legacyconfig.LibraryState, outputDir string) error { - if len(libraryState.APIs) == 0 { - slog.Info("library has no APIs; skipping generation", "library", libraryState.ID) - return nil - } - - slog.Info("checking out apiSource", "commit", libraryState.LastGeneratedCommit) - if err := r.sourceRepo.Checkout(libraryState.LastGeneratedCommit); err != nil { - return fmt.Errorf("error checking out from sourceRepo %w", err) - } - - if err := generateSingleLibrary(ctx, r.containerClient, r.state, libraryState, r.repo, r.sourceRepo, r.state.Image, outputDir); err != nil { - slog.Error("failed to regenerate a single library", "error", err, "ID", libraryState.ID) - return err - } - - if !r.build { - slog.Info("build not specified, skipping build") - return nil - } - if err := buildSingleLibrary(ctx, r.containerClient, r.state, libraryState, r.repo); err != nil { - slog.Error("failed to build a single library", "error", err, "ID", libraryState.ID) - return err - } - - return nil -} - -// runContainerGenerateTest creates a temporary commit to ensure a clean -// repo state for the test runner, runs the tests, and then soft-resets -// the commit to leave the working directory in its original dirty state -// for the final commit operation. -func runContainerGenerateTest(ctx context.Context, repo legacygitrepo.Repository, sourceHead string, testRunner *testGenerateRunner) error { - slog.Debug("creating temporary commit for testing") - committed := true - if err := repo.AddAll(); err != nil { - return fmt.Errorf("failed to stage changes for temporary commit: %w", err) - } - - // Commit the generated changes so the repo is clean for the test runner. - if err := repo.Commit("chore: temporary commit for update-image test"); err != nil { - if !errors.Is(err, legacygitrepo.ErrNoModificationsToCommit) { - return fmt.Errorf("failed to create temporary commit for test: %w", err) - } - slog.Debug("no changes to commit for test, proceeding without temporary commit") - committed = false - } - - if err := testRunner.runTests(ctx, sourceHead); err != nil { - // If tests fail, leave the temporary commit in place for diagnostics and return the error. - return fmt.Errorf("failure in container generate test: %w", err) - } - - // If tests pass and temporary commit was made, reset it to restore the dirty state for the final commit. - if committed { - slog.Debug("tests passed, resetting temporary commit") - if err := repo.ResetSoft("HEAD~1"); err != nil { - return fmt.Errorf("failed to reset temporary commit after successful test: %w", err) - } - } - return nil -} - -var updateImageTemplate = template.Must(template.New("updateImage").Parse(`feat: update image to {{.Image}} -{{ if .FailedLibraries }} -## Generation failed for -{{- range .FailedLibraries }} -- {{ . }} -{{- end -}} -{{- end }} -`)) - -type updateImagePRBody struct { - Image string - FailedLibraries []string -} - -func formatUpdateImagePRBody(image string, failedGenerations []*legacyconfig.LibraryState) (string, error) { - failedLibraries := make([]string, 0, len(failedGenerations)) - for _, failedGeneration := range failedGenerations { - failedLibraries = append(failedLibraries, failedGeneration.ID) - } - data := &updateImagePRBody{ - Image: image, - FailedLibraries: failedLibraries, - } - var out bytes.Buffer - if err := updateImageTemplate.Execute(&out, data); err != nil { - return "", fmt.Errorf("error executing template %w", err) - } - return strings.TrimSpace(out.String()), nil -} diff --git a/internal/legacylibrarian/legacylibrarian/update_image_test.go b/internal/legacylibrarian/legacylibrarian/update_image_test.go deleted file mode 100644 index e934798bf55..00000000000 --- a/internal/legacylibrarian/legacylibrarian/update_image_test.go +++ /dev/null @@ -1,1012 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package legacylibrarian - -import ( - "fmt" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/google/go-cmp/cmp" - "github.com/googleapis/librarian/internal/legacylibrarian/legacyconfig" - "github.com/googleapis/librarian/internal/legacylibrarian/legacygithub" - "github.com/googleapis/librarian/internal/legacylibrarian/legacygitrepo" -) - -func TestNewUpdateImageRunner(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - cfg *legacyconfig.Config - wantErr bool - wantErrMsg string - setupFunc func(*legacyconfig.Config) error - }{ - { - name: "valid config", - cfg: &legacyconfig.Config{ - API: "some/api", - APISource: newTestGitRepo(t).GetDir(), - Branch: "test-branch", - Repo: newTestGitRepo(t).GetDir(), - WorkRoot: t.TempDir(), - Image: "gcr.io/test/test-image", - CommandName: updateImageCmdName, - }, - }, - - { - name: "invalid api source", - cfg: &legacyconfig.Config{ - API: "some/api", - APISource: t.TempDir(), // Not a git repo - Repo: newTestGitRepo(t).GetDir(), - WorkRoot: t.TempDir(), - Image: "gcr.io/test/test-image", - CommandName: updateImageCmdName, - }, - wantErr: true, - wantErrMsg: "repository does not exist", - }, - { - name: "missing image", - cfg: &legacyconfig.Config{ - API: "some/api", - APISource: t.TempDir(), - Branch: "test-branch", - Repo: "https://github.com/googleapis/librarian.git", - WorkRoot: t.TempDir(), - CommandName: updateImageCmdName, - }, - wantErr: true, - }, - { - name: "valid config with github token", - cfg: &legacyconfig.Config{ - API: "some/api", - APISource: newTestGitRepo(t).GetDir(), - Branch: "test-branch", - Repo: newTestGitRepo(t).GetDir(), - WorkRoot: t.TempDir(), - Image: "gcr.io/test/test-image", - GitHubToken: "gh-token", - CommandName: updateImageCmdName, - }, - }, - { - name: "empty API source", - cfg: &legacyconfig.Config{ - API: "some/api", - APISource: "https://github.com/googleapis/googleapis", // This will trigger the clone of googleapis - APISourceBranch: "master", - APISourceDepth: 1, - Branch: "test-branch", - Repo: newTestGitRepo(t).GetDir(), - WorkRoot: t.TempDir(), - Image: "gcr.io/test/test-image", - CommandName: updateImageCmdName, - }, - }, - { - name: "clone googleapis fails", - cfg: &legacyconfig.Config{ - API: "some/api", - APISource: "https://github.com/googleapis/googleapis", // This will trigger the clone of googleapis - APISourceDepth: 1, - Repo: newTestGitRepo(t).GetDir(), - WorkRoot: t.TempDir(), - Image: "gcr.io/test/test-image", - CommandName: updateImageCmdName, - }, - wantErr: true, - wantErrMsg: "repository does not exist", - setupFunc: func(cfg *legacyconfig.Config) error { - // The function will try to clone googleapis into the current work directory. - // To make it fail, create a non-empty, non-git directory. - googleapisDir := filepath.Join(cfg.WorkRoot, "googleapis") - if err := os.MkdirAll(googleapisDir, 0755); err != nil { - return err - } - if err := os.WriteFile(filepath.Join(googleapisDir, "some-file"), []byte("foo"), 0644); err != nil { - return err - } - return nil - }, - }, - { - name: "valid config with local repo", - cfg: &legacyconfig.Config{ - API: "some/api", - APISource: newTestGitRepo(t).GetDir(), - Branch: "test-branch", - Repo: newTestGitRepo(t).GetDir(), - WorkRoot: t.TempDir(), - Image: "gcr.io/test/test-image", - CommandName: updateImageCmdName, - }, - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - - // custom setup - if test.setupFunc != nil { - if err := test.setupFunc(test.cfg); err != nil { - t.Fatalf("error in setup %v", err) - } - } - - r, err := newUpdateImageRunner(test.cfg) - if test.wantErr { - if err == nil { - t.Fatalf("newUpdateImageRunner() error = %v, wantErr %v", err, test.wantErr) - } - - if !strings.Contains(err.Error(), test.wantErrMsg) { - t.Fatalf("want error message: %s, got: %s", test.wantErrMsg, err.Error()) - } - - return - } - - if err != nil { - t.Fatalf("newUpdateImageRunner() got error: %v", err) - } - - if r.branch == "" { - t.Errorf("newUpdateImageRunner() branch is not set") - } - - if r.ghClient == nil { - t.Errorf("newUpdateImageRunner() ghClient is nil") - } - if r.containerClient == nil { - t.Errorf("newUpdateImageRunner() containerClient is nil") - } - if r.repo == nil { - t.Errorf("newUpdateImageRunner() repo is nil") - } - if r.sourceRepo == nil { - t.Errorf("newUpdateImageRunner() sourceRepo is nil") - } - }) - } -} - -func TestUpdateImageRunnerRun(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - imagesClient *mockImagesClient - containerClient *mockContainerClient - ghClient *mockGitHubClient - state *legacyconfig.LibrarianState - librarianConfig *legacyconfig.LibrarianConfig - image string - build bool - commit bool - push bool - test bool - libraryToTest string - wantErr bool - wantErrMsg string - wantFindLatestCalls int - wantGenerateCalls int - wantBuildCalls int - wantCheckoutCalls int - wantCreatePullRequestCalls int - wantCreateIssueCalls int - wantCommitMsg string - checkoutError error - wantImage string - }{ - { - name: "specific image", - image: "some-image", - state: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "lib1", - APIs: []*legacyconfig.API{{Path: "some/api1"}}, - SourceRoots: []string{ - "src/a", - }, - LastGeneratedCommit: "abcd1234", - }, - }, - }, - containerClient: &mockContainerClient{}, - imagesClient: &mockImagesClient{}, - ghClient: &mockGitHubClient{}, - wantFindLatestCalls: 0, - wantGenerateCalls: 1, - wantBuildCalls: 0, // no -build flag - wantCheckoutCalls: 2, - wantImage: "some-image", - }, - { - name: "no change image", - image: "gcr.io/test/image:v1.2.3", - state: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "lib1", - APIs: []*legacyconfig.API{{Path: "some/api1"}}, - SourceRoots: []string{ - "src/a", - }, - LastGeneratedCommit: "abcd1234", - }, - }, - }, - containerClient: &mockContainerClient{}, - imagesClient: &mockImagesClient{}, - ghClient: &mockGitHubClient{}, - wantFindLatestCalls: 0, - wantGenerateCalls: 1, - wantBuildCalls: 0, - wantCheckoutCalls: 2, - wantImage: "gcr.io/test/image:v1.2.3", - }, - { - name: "finds latest image", - state: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "lib1", - APIs: []*legacyconfig.API{{Path: "some/api1"}}, - SourceRoots: []string{ - "src/a", - }, - LastGeneratedCommit: "abcd1234", - }, - }, - }, - containerClient: &mockContainerClient{}, - imagesClient: &mockImagesClient{ - latestImage: "gcr.io/test/image@sha256:abc123", - }, - ghClient: &mockGitHubClient{}, - wantFindLatestCalls: 1, - wantGenerateCalls: 1, - wantBuildCalls: 0, // no -build flag - wantCheckoutCalls: 2, - wantImage: "gcr.io/test/image@sha256:abc123", - }, - { - name: "finds image error", - state: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "lib1", - APIs: []*legacyconfig.API{{Path: "some/api1"}}, - SourceRoots: []string{ - "src/a", - }, - LastGeneratedCommit: "abcd1234", - }, - }, - }, - containerClient: &mockContainerClient{}, - imagesClient: &mockImagesClient{ - err: fmt.Errorf("some lookup error"), - }, - ghClient: &mockGitHubClient{}, - wantFindLatestCalls: 1, - wantGenerateCalls: 0, - wantBuildCalls: 0, - wantCheckoutCalls: 0, - wantErr: true, - wantErrMsg: "some lookup error", - }, - { - name: "runs build", - state: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "lib1", - APIs: []*legacyconfig.API{{Path: "some/api1"}}, - SourceRoots: []string{ - "src/a", - }, - LastGeneratedCommit: "abcd1234", - }, - }, - }, - containerClient: &mockContainerClient{}, - imagesClient: &mockImagesClient{ - latestImage: "gcr.io/test/image@sha256:abc123", - }, - ghClient: &mockGitHubClient{}, - build: true, - wantFindLatestCalls: 1, - wantGenerateCalls: 1, - wantBuildCalls: 1, - wantCheckoutCalls: 2, - }, - { - name: "updates multiple", - state: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "lib1", - APIs: []*legacyconfig.API{{Path: "some/api1"}}, - SourceRoots: []string{ - "src/a", - }, - LastGeneratedCommit: "abcd1234", - }, - { - ID: "lib2", - APIs: []*legacyconfig.API{{Path: "some/api2"}}, - SourceRoots: []string{ - "src/b", - }, - LastGeneratedCommit: "abcd1235", - }, - }, - }, - containerClient: &mockContainerClient{}, - imagesClient: &mockImagesClient{ - latestImage: "gcr.io/test/image@sha256:abc123", - }, - ghClient: &mockGitHubClient{}, - build: true, - wantFindLatestCalls: 1, - wantGenerateCalls: 2, - wantBuildCalls: 2, - wantCheckoutCalls: 3, - }, - { - name: "skips libraries without APIs", - state: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "lib1", - APIs: []*legacyconfig.API{{Path: "some/api1"}}, - SourceRoots: []string{ - "src/a", - }, - LastGeneratedCommit: "abcd1234", - }, - { - ID: "lib2", - APIs: []*legacyconfig.API{}, - SourceRoots: []string{ - "src/b", - }, - LastGeneratedCommit: "abcd1235", - }, - }, - }, - containerClient: &mockContainerClient{}, - imagesClient: &mockImagesClient{ - latestImage: "gcr.io/test/image@sha256:abc123", - }, - ghClient: &mockGitHubClient{}, - build: true, - wantFindLatestCalls: 1, - wantGenerateCalls: 1, - wantBuildCalls: 1, - wantCheckoutCalls: 2, - }, - { - name: "partial generate success", - state: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "lib1", - APIs: []*legacyconfig.API{{Path: "some/api1"}}, - SourceRoots: []string{ - "src/a", - }, - LastGeneratedCommit: "abcd1234", - }, - { - ID: "lib2", - APIs: []*legacyconfig.API{{Path: "some/api2"}}, - SourceRoots: []string{ - "src/b", - }, - LastGeneratedCommit: "abcd1235", - }, - }, - }, - containerClient: &mockContainerClient{ - failGenerateForID: "lib1", - generateErrForID: fmt.Errorf("error generating lib1"), - }, - imagesClient: &mockImagesClient{ - latestImage: "gcr.io/test/image@sha256:abc123", - }, - ghClient: &mockGitHubClient{}, - build: true, - wantFindLatestCalls: 1, - wantGenerateCalls: 2, - wantBuildCalls: 1, // build for failed generate should not run - wantCheckoutCalls: 3, - }, - { - name: "partial build success", - state: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "lib1", - APIs: []*legacyconfig.API{{Path: "some/api1"}}, - SourceRoots: []string{ - "src/a", - }, - LastGeneratedCommit: "abcd1234", - }, - { - ID: "lib2", - APIs: []*legacyconfig.API{{Path: "some/api2"}}, - SourceRoots: []string{ - "src/b", - }, - LastGeneratedCommit: "abcd1235", - }, - }, - }, - containerClient: &mockContainerClient{ - failBuildForID: "lib1", - buildErrForID: fmt.Errorf("error building lib1"), - }, - imagesClient: &mockImagesClient{ - latestImage: "gcr.io/test/image@sha256:abc123", - }, - ghClient: &mockGitHubClient{}, - build: true, - wantFindLatestCalls: 1, - wantGenerateCalls: 2, - wantBuildCalls: 2, - wantCheckoutCalls: 3, - }, - { - name: "checkout error", - state: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "lib1", - APIs: []*legacyconfig.API{{Path: "some/api1"}}, - SourceRoots: []string{ - "src/a", - }, - LastGeneratedCommit: "abcd1234", - }, - { - ID: "lib2", - APIs: []*legacyconfig.API{{Path: "some/api2"}}, - SourceRoots: []string{ - "src/b", - }, - LastGeneratedCommit: "abcd1235", - }, - }, - }, - containerClient: &mockContainerClient{}, - imagesClient: &mockImagesClient{ - latestImage: "gcr.io/test/image@sha256:abc123", - }, - ghClient: &mockGitHubClient{}, - build: true, - wantFindLatestCalls: 1, - wantGenerateCalls: 0, - wantBuildCalls: 0, - wantCheckoutCalls: 3, - checkoutError: fmt.Errorf("some checkout error"), - }, - { - name: "updates multiple with commit", - state: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "lib1", - APIs: []*legacyconfig.API{{Path: "some/api1"}}, - SourceRoots: []string{ - "src/a", - }, - LastGeneratedCommit: "abcd1234", - }, - { - ID: "lib2", - APIs: []*legacyconfig.API{{Path: "some/api2"}}, - SourceRoots: []string{ - "src/b", - }, - LastGeneratedCommit: "abcd1235", - }, - }, - }, - containerClient: &mockContainerClient{}, - imagesClient: &mockImagesClient{ - latestImage: "gcr.io/test/image@sha256:abc123", - }, - ghClient: &mockGitHubClient{}, - build: true, - commit: true, - wantFindLatestCalls: 1, - wantGenerateCalls: 2, - wantBuildCalls: 2, - wantCheckoutCalls: 3, - wantCommitMsg: "feat: update image to gcr.io/test/image@sha256:abc123", - }, - { - name: "push failure", - state: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "lib1", - APIs: []*legacyconfig.API{{Path: "some/api1"}}, - SourceRoots: []string{ - "src/a", - }, - LastGeneratedCommit: "abcd1234", - }, - { - ID: "lib2", - APIs: []*legacyconfig.API{{Path: "some/api2"}}, - SourceRoots: []string{ - "src/b", - }, - LastGeneratedCommit: "abcd1235", - }, - }, - }, - containerClient: &mockContainerClient{}, - imagesClient: &mockImagesClient{ - latestImage: "gcr.io/test/image@sha256:abc123", - }, - ghClient: &mockGitHubClient{ - createPullRequestErr: fmt.Errorf("some API error"), - }, - build: true, - commit: true, - push: true, - wantFindLatestCalls: 1, - wantGenerateCalls: 2, - wantBuildCalls: 2, - wantCheckoutCalls: 3, - wantCreatePullRequestCalls: 1, - wantCommitMsg: "feat: update image to gcr.io/test/image@sha256:abc123", - wantErr: true, - wantErrMsg: "some API error", - }, - { - name: "updates multiple with push", - state: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "lib1", - APIs: []*legacyconfig.API{{Path: "some/api1"}}, - SourceRoots: []string{ - "src/a", - }, - LastGeneratedCommit: "abcd1234", - }, - { - ID: "lib2", - APIs: []*legacyconfig.API{{Path: "some/api2"}}, - SourceRoots: []string{ - "src/b", - }, - LastGeneratedCommit: "abcd1235", - }, - }, - }, - containerClient: &mockContainerClient{}, - imagesClient: &mockImagesClient{ - latestImage: "gcr.io/test/image@sha256:abc123", - }, - ghClient: &mockGitHubClient{ - createdPR: &legacygithub.PullRequestMetadata{ - Number: 1234, - Repo: &legacygithub.Repository{ - Owner: "googleapis", - Name: "google-cloud-go", - }, - }, - }, - build: true, - commit: true, - push: true, - wantFindLatestCalls: 1, - wantGenerateCalls: 2, - wantBuildCalls: 2, - wantCheckoutCalls: 3, - wantCreatePullRequestCalls: 1, - wantCommitMsg: "feat: update image to gcr.io/test/image@sha256:abc123", - }, - { - name: "runs test", - state: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "lib1", - APIs: []*legacyconfig.API{{Path: "some/api1"}}, - SourceRoots: []string{ - "src/a", - }, - LastGeneratedCommit: "abcd1234", - }, - }, - }, - containerClient: &mockContainerClient{}, - imagesClient: &mockImagesClient{ - latestImage: "gcr.io/test/image@sha256:abc123", - }, - ghClient: &mockGitHubClient{ - createdPR: &legacygithub.PullRequestMetadata{ - Number: 1234, - Repo: &legacygithub.Repository{ - Owner: "googleapis", - Name: "google-cloud-go", - }, - }, - }, - test: true, - libraryToTest: "lib1", - wantFindLatestCalls: 1, - wantGenerateCalls: 1, - wantCheckoutCalls: 3, - wantErr: true, - // The test setup does not have protos, so the test fails in the preparation step. - wantErrMsg: "failed in test preparing steps", - }, - { - name: "partial updates with push", - state: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{{ - ID: "lib1", - APIs: []*legacyconfig.API{{Path: "some/api1"}}, - SourceRoots: []string{ - "src/a", - }, - LastGeneratedCommit: "abcd1234", - }, - { - ID: "lib2", - APIs: []*legacyconfig.API{{Path: "some/api2"}}, - SourceRoots: []string{ - "src/b", - }, - LastGeneratedCommit: "abcd1235", - }, - }, - }, - containerClient: &mockContainerClient{ - failBuildForID: "lib1", - buildErrForID: fmt.Errorf("error building lib1"), - }, - imagesClient: &mockImagesClient{ - latestImage: "gcr.io/test/image@sha256:abc123", - }, - ghClient: &mockGitHubClient{ - createdPR: &legacygithub.PullRequestMetadata{ - Number: 1234, - Repo: &legacygithub.Repository{ - Owner: "googleapis", - Name: "google-cloud-go", - }, - }, - }, - build: true, - commit: true, - push: true, - wantFindLatestCalls: 1, - wantGenerateCalls: 2, - wantBuildCalls: 2, - wantCheckoutCalls: 3, - wantCreatePullRequestCalls: 1, - wantCreateIssueCalls: 1, - wantCommitMsg: "feat: update image to gcr.io/test/image@sha256:abc123", - }, - { - name: "skip generation for library", - state: &legacyconfig.LibrarianState{ - Image: "gcr.io/test/image:v1.2.3", - Libraries: []*legacyconfig.LibraryState{ - { - ID: "blocked-lib", - APIs: []*legacyconfig.API{{Path: "some/api1"}}, - SourceRoots: []string{"src/a"}, - LastGeneratedCommit: "abcd1234", - }, - }, - }, - - librarianConfig: &legacyconfig.LibrarianConfig{ - Libraries: []*legacyconfig.LibraryConfig{ - { - LibraryID: "blocked-lib", - GenerateBlocked: true, - }, - }, - }, - containerClient: &mockContainerClient{}, - imagesClient: &mockImagesClient{}, - ghClient: &mockGitHubClient{}, - wantFindLatestCalls: 1, - wantGenerateCalls: 0, - wantBuildCalls: 0, - wantCheckoutCalls: 1, - }, - { - name: "update-image in release only mode returns error", - librarianConfig: &legacyconfig.LibrarianConfig{ - ReleaseOnlyMode: true, - }, - containerClient: &mockContainerClient{}, - imagesClient: &mockImagesClient{}, - ghClient: &mockGitHubClient{}, - wantFindLatestCalls: 0, - wantGenerateCalls: 0, - wantBuildCalls: 0, - wantCheckoutCalls: 0, - wantErr: true, - wantErrMsg: "generate in release only mode", - }, - } { - t.Run(test.name, func(t *testing.T) { - testRepo := newTestGitRepoWithState(t, test.state) - repo := &MockRepository{ - Dir: testRepo.GetDir(), - RemotesValue: []*legacygitrepo.Remote{ - { - Name: "origin", - URLs: []string{"https://github.com/googleapis/google-cloud-go.git"}, - }, - }, - } - sourceRepo := &MockRepository{ - CheckoutError: test.checkoutError, - } - r := &updateImageRunner{ - branch: "main", - build: test.build, - commit: test.commit, - push: test.push, - test: test.test, - libraryToTest: test.libraryToTest, - image: test.image, - containerClient: test.containerClient, - imagesClient: test.imagesClient, - ghClient: test.ghClient, - state: test.state, - librarianConfig: test.librarianConfig, - workRoot: t.TempDir(), - repo: repo, - sourceRepo: sourceRepo, - } - - err := r.run(t.Context()) - - if diff := cmp.Diff(test.wantGenerateCalls, test.containerClient.generateCalls); diff != "" { - t.Errorf("%s: run() generateCalls mismatch (-want +got):%s", test.name, diff) - } - if test.wantImage != "" { - if diff := cmp.Diff(test.wantImage, test.containerClient.generateRequest.Image); diff != "" { - t.Errorf("%s: run() image mismatch (-want +got):%s", test.name, diff) - } - } - if diff := cmp.Diff(test.wantBuildCalls, test.containerClient.buildCalls); diff != "" { - t.Errorf("%s: run() buildCalls mismatch (-want +got):%s", test.name, diff) - } - if diff := cmp.Diff(test.wantFindLatestCalls, test.imagesClient.findLatestCalls); diff != "" { - t.Errorf("%s: run() findLatestCalls mismatch (-want +got):%s", test.name, diff) - } - if diff := cmp.Diff(test.wantCheckoutCalls, sourceRepo.CheckoutCalls); diff != "" { - t.Errorf("%s: run() checkoutCalls mismatch (-want +got):%s", test.name, diff) - } - if diff := cmp.Diff(test.wantCreatePullRequestCalls, test.ghClient.createPullRequestCalls); diff != "" { - t.Errorf("%s: run() createPullRequestCalls mismatch (-want +got):%s", test.name, diff) - } - if diff := cmp.Diff(test.wantCreateIssueCalls, test.ghClient.createIssueCalls); diff != "" { - t.Errorf("%s: run() createIssueCalls mismatch (-want +got):%s", test.name, diff) - } - - if test.wantErr { - if err == nil { - t.Fatalf("%s should return error", test.name) - } - - if !strings.Contains(err.Error(), test.wantErrMsg) { - t.Errorf("want error message %s, got %s", test.wantErrMsg, err.Error()) - } - return - } else { - if err != nil { - t.Fatal(err) - } - } - - if test.wantCommitMsg != "" { - if diff := cmp.Diff(test.wantCommitMsg, repo.LastCommitMessage); diff != "" { - t.Errorf("%s: run() commit message mismatch (-want +got):%s", test.name, diff) - } - } - }) - } -} - -func TestFormatUpdateImagePRBody(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - image string - failedGenerations []*legacyconfig.LibraryState - want string - wantErr bool - wantErrMsg string - }{ - { - name: "success", - image: "some-image", - want: "feat: update image to some-image", - }, - { - name: "multiple errors", - image: "some-image", - failedGenerations: []*legacyconfig.LibraryState{ - { - ID: "library-id-1", - }, - { - ID: "library-id-2", - }, - }, - want: `feat: update image to some-image - -## Generation failed for -- library-id-1 -- library-id-2`, - }, - } { - t.Run(test.name, func(t *testing.T) { - t.Parallel() - got, err := formatUpdateImagePRBody(test.image, test.failedGenerations) - - if test.wantErr { - if err == nil { - t.Fatalf("formatUpdateImagePRBody() error = %v, wantErr %v", err, test.wantErr) - } - - if !strings.Contains(err.Error(), test.wantErrMsg) { - t.Fatalf("want error message: %s, got: %s", test.wantErrMsg, err.Error()) - } - return - } - if diff := cmp.Diff(got, test.want); diff != "" { - t.Errorf("%s: formatUpdateImagePRBody() mismatch (-want +got):%s", test.name, diff) - } - }) - } -} - -func TestRunContainerGenerateTest(t *testing.T) { - t.Parallel() - for _, test := range []struct { - name string - mockRepo *MockRepository - testRunner *testGenerateRunner - wantErrMsg string - wantResetCalls int - }{ - { - name: "AddAll fails", - mockRepo: &MockRepository{ - AddAllError: fmt.Errorf("add all failed"), - }, - testRunner: &testGenerateRunner{ - workRoot: t.TempDir(), - }, - wantErrMsg: "failed to stage changes", - }, - { - name: "Commit fails with unexpected error", - mockRepo: &MockRepository{ - CommitError: fmt.Errorf("unexpected commit error"), - }, - testRunner: &testGenerateRunner{ - workRoot: t.TempDir(), - }, - wantErrMsg: "failed to create temporary commit", - }, - { - name: "ResetSoft fails after successful test", - mockRepo: &MockRepository{ - CommitError: nil, // Commit succeeds - ResetSoftError: fmt.Errorf("reset soft failed"), - }, - testRunner: &testGenerateRunner{ - // Mocking a successful test run. - containerClient: &mockContainerClient{noGenerateResponse: true}, - repo: &MockRepository{}, - sourceRepo: &MockRepository{}, - librarianConfig: &legacyconfig.LibrarianConfig{}, - state: &legacyconfig.LibrarianState{}, - workRoot: t.TempDir(), - }, - wantErrMsg: "failed to reset temporary commit", - wantResetCalls: 1, - }, - { - name: "Success with commit", - mockRepo: &MockRepository{ - CommitError: nil, - }, - testRunner: &testGenerateRunner{ - containerClient: &mockContainerClient{noGenerateResponse: true}, - sourceRepo: &MockRepository{}, - librarianConfig: &legacyconfig.LibrarianConfig{}, - repo: &MockRepository{}, - state: &legacyconfig.LibrarianState{}, - workRoot: t.TempDir(), - }, - wantResetCalls: 1, - }, - { - name: "Success with no changes to commit", - mockRepo: &MockRepository{ - CommitError: legacygitrepo.ErrNoModificationsToCommit, - }, - testRunner: &testGenerateRunner{ - containerClient: &mockContainerClient{noGenerateResponse: true}, - sourceRepo: &MockRepository{}, - librarianConfig: &legacyconfig.LibrarianConfig{}, - repo: &MockRepository{}, - state: &legacyconfig.LibrarianState{}, - workRoot: t.TempDir(), - }, - wantResetCalls: 0, - }, - } { - t.Run(test.name, func(t *testing.T) { - err := runContainerGenerateTest(t.Context(), test.mockRepo, "fake-head", test.testRunner) - - if test.wantErrMsg != "" { - if err == nil { - t.Fatalf("runContainerGenerateTest() expected an error, but got nil") - } - if !strings.Contains(err.Error(), test.wantErrMsg) { - t.Errorf("runContainerGenerateTest() error = %q, want error containing %q", err.Error(), test.wantErrMsg) - } - } else if err != nil { - t.Fatalf("runContainerGenerateTest() returned unexpected error: %v", err) - } - - if test.mockRepo.ResetSoftCalls != test.wantResetCalls { - t.Errorf("ResetSoft was called %d times, want %d", test.mockRepo.ResetSoftCalls, test.wantResetCalls) - } - }) - } -} From 75699276c8e11dd9c03263c4619dcd3d685c46be Mon Sep 17 00:00:00 2001 From: Joe Wang <106995533+JoeWang1127@users.noreply.github.com> Date: Mon, 29 Jun 2026 20:40:39 +0000 Subject: [PATCH 083/108] chore(.github): update node and pnpm in nodejs workflow (#6577) The `node` and `pnpm` version are updated in https://github.com/googleapis/librarian/pull/6494, we should update these versions in workflow. --- .github/workflows/nodejs.yaml | 4 ++-- internal/librarian/nodejs/install.go | 1 + internal/librarian/nodejs/install_test.go | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nodejs.yaml b/.github/workflows/nodejs.yaml index b722226456a..8a009bb335d 100644 --- a/.github/workflows/nodejs.yaml +++ b/.github/workflows/nodejs.yaml @@ -27,11 +27,11 @@ jobs: - uses: ./.github/actions/setup-librarian - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 with: - node-version: "18" + node-version: "22" - name: "Set up pnpm via corepack" run: | corepack enable pnpm - corepack prepare pnpm@7.32.2 --activate + corepack prepare pnpm@11.7.0 --activate - name: "Install Node.js dependencies (resolve cache paths)" id: tool-paths run: | diff --git a/internal/librarian/nodejs/install.go b/internal/librarian/nodejs/install.go index a4e50e241ad..55ebc34af71 100644 --- a/internal/librarian/nodejs/install.go +++ b/internal/librarian/nodejs/install.go @@ -98,6 +98,7 @@ func getPNPMEnv(ctx context.Context) ([]string, error) { env = append(env, "PNPM_CONFIG_GLOBAL_BIN_DIR="+globalBin) env = append(env, "PNPM_CONFIG_GLOBAL_DIR="+filepath.Join(globalBin, "pnpm-global")) env = append(env, "PNPM_CONFIG_STORE_DIR="+filepath.Join(globalBin, "pnpm-store")) + env = append(env, "PNPM_CONFIG_DANGEROUSLY_ALLOW_ALL_BUILDS=true") return env, nil } diff --git a/internal/librarian/nodejs/install_test.go b/internal/librarian/nodejs/install_test.go index 8017ab77dff..805bdaa5bfc 100644 --- a/internal/librarian/nodejs/install_test.go +++ b/internal/librarian/nodejs/install_test.go @@ -56,7 +56,8 @@ func TestInstall(t *testing.T) { bin := t.TempDir() pnpmStub := `#!/bin/sh # Assert that transient environmental variables are set dynamically for process lifetime -if [ -z "$PNPM_HOME" ] || [ -z "$PNPM_CONFIG_GLOBAL_BIN_DIR" ] || [ -z "$PNPM_CONFIG_GLOBAL_DIR" ] || [ -z "$PNPM_CONFIG_STORE_DIR" ]; then +if [ -z "$PNPM_HOME" ] || [ -z "$PNPM_CONFIG_GLOBAL_BIN_DIR" ] || [ -z "$PNPM_CONFIG_GLOBAL_DIR" ] || [ -z "$PNPM_CONFIG_STORE_DIR" ] || \ + [ -z "$PNPM_CONFIG_DANGEROUSLY_ALLOW_ALL_BUILDS" ]; then echo "Error: Required transient PNPM environment variables are missing!" >&2 exit 1 fi From 5be31dbc19f4a15b1e4642932f743086d204afdb Mon Sep 17 00:00:00 2001 From: Joe Wang <106995533+JoeWang1127@users.noreply.github.com> Date: Mon, 29 Jun 2026 20:48:24 +0000 Subject: [PATCH 084/108] chore(internal/librarian/nodejs): update node generator (#6575) Update nodejs generator to [v4.12.1](https://github.com/googleapis/google-cloud-node/releases/tag/gapic-generator-v4.12.1). --- internal/librarian/nodejs/librarian.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/librarian/nodejs/librarian.yaml b/internal/librarian/nodejs/librarian.yaml index 2cac2d4ace6..2e6d7ffd72a 100644 --- a/internal/librarian/nodejs/librarian.yaml +++ b/internal/librarian/nodejs/librarian.yaml @@ -19,9 +19,9 @@ language: nodejs tools: pnpm: - name: gapic-generator-typescript - version: "gapic-generator-v4.12.0" - package: "https://github.com/googleapis/google-cloud-node/archive/gapic-generator-v4.12.0.tar.gz" - checksum: "7db96e31b8c6ac15d40e92654a731ec7aa2b0a3425708ac68369b374688112e8" + version: "gapic-generator-v4.12.1" + package: "https://github.com/googleapis/google-cloud-node/archive/gapic-generator-v4.12.1.tar.gz" + checksum: "477e4a9b3442457667846bd5c629d1c059e7caf2d5a3ebce95da626a2d42ea80" build: - "pnpm install" # tsc only compiles TypeScript; the generator also needs templates/ From 500ba22a7de65f5ab0e84112daa7d16ff76ff841 Mon Sep 17 00:00:00 2001 From: yangyzs <171981480+yangyzs@users.noreply.github.com> Date: Mon, 29 Jun 2026 16:49:16 -0400 Subject: [PATCH 085/108] feat(internal/librarian/java): add README sample extraction helpers (#6574) Add helper functions collectSampleFiles and parseCodeSample to scan samples/src/main/java and extract sample metadata. Add codeSample struct and unit tests. For #6515 --- internal/librarian/java/readme.go | 58 +++++++++ internal/librarian/java/readme_test.go | 166 +++++++++++++++++++++++++ 2 files changed, 224 insertions(+) diff --git a/internal/librarian/java/readme.go b/internal/librarian/java/readme.go index 2058f7334e7..15a0ce22606 100644 --- a/internal/librarian/java/readme.go +++ b/internal/librarian/java/readme.go @@ -18,6 +18,7 @@ import ( "bytes" "errors" "fmt" + "io/fs" "os" "path/filepath" "regexp" @@ -38,6 +39,63 @@ var ( errEmptyTitle = errors.New("title value cannot be empty") ) +// codeSample represents a discovered Java code sample along with its derived title. +type codeSample struct { + Title string + File string +} + +// collectSampleFiles recursively scans dir/samples for Java production files. +func collectSampleFiles(dir string) ([]string, error) { + samplesDir := filepath.Join(dir, "samples") + if _, err := os.Stat(samplesDir); err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + return nil, fmt.Errorf("failed to stat samples directory: %w", err) + } + var files []string + err := filepath.WalkDir(samplesDir, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if !d.Type().IsRegular() { + return nil + } + rel, err := filepath.Rel(dir, path) + if err != nil { + return err + } + if isProductionSample(rel) { + files = append(files, rel) + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("failed to walk samples directory: %w", err) + } + return files, nil +} + +// parseCodeSample reads a Java sample file and constructs a codeSample struct with its title and relative path. +func parseCodeSample(dir, file string) (*codeSample, error) { + // Derive default title by stripping extension and converting CamelCase to space-separated words. + base := strings.TrimSuffix(filepath.Base(file), ".java") + title := decamelize(base) + titleOverride, err := extractTitle(filepath.Join(dir, file)) + if err != nil { + return nil, fmt.Errorf("failed to extract title from %s: %w", file, err) + } + if titleOverride != "" { + title = titleOverride + } + return &codeSample{ + Title: title, + // Normalize path separators to forward slashes for Markdown links in README. + File: filepath.ToSlash(file), + }, nil +} + // decamelize converts CamelCase string to space-separated string (e.g. "CamelCase" -> "Camel Case"). func decamelize(value string) string { return strings.TrimSpace(camelCaseRegexp.ReplaceAllString(value, `$1 $2`)) diff --git a/internal/librarian/java/readme_test.go b/internal/librarian/java/readme_test.go index a2261c597f2..d4ba75a3355 100644 --- a/internal/librarian/java/readme_test.go +++ b/internal/librarian/java/readme_test.go @@ -23,6 +23,172 @@ import ( "github.com/google/go-cmp/cmp" ) +func TestCollectSampleFiles(t *testing.T) { + for _, test := range []struct { + name string + setupFiles func(t *testing.T, dir string) + want []string + }{ + { + name: "missing samples directory", + setupFiles: func(t *testing.T, dir string) { + // Do nothing, temp dir is empty. + }, + want: nil, + }, + { + name: "collects production java files only", + setupFiles: func(t *testing.T, dir string) { + samplesDir := filepath.Join(dir, "samples", "src", "main", "java", "com", "example") + if err := os.MkdirAll(samplesDir, 0755); err != nil { + t.Fatal(err) + } + testDir := filepath.Join(dir, "samples", "src", "test", "java", "com", "example") + if err := os.MkdirAll(testDir, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(samplesDir, "SampleA.java"), []byte("public class SampleA {}"), 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(samplesDir, "README.md"), []byte("# Docs"), 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(testDir, "SampleATest.java"), []byte("public class SampleATest {}"), 0644); err != nil { + t.Fatal(err) + } + }, + want: []string{ + filepath.Join("samples", "src", "main", "java", "com", "example", "SampleA.java"), + }, + }, + { + name: "collects production java files across nested packages and ignores directories", + setupFiles: func(t *testing.T, dir string) { + pkg1 := filepath.Join(dir, "samples", "src", "main", "java", "com", "example") + pkg2 := filepath.Join(dir, "samples", "src", "main", "java", "com", "example", "subpkg") + fakeJavaDir := filepath.Join(dir, "samples", "src", "main", "java", "com", "example", "FakeDir.java") + if err := os.MkdirAll(pkg1, 0755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(pkg2, 0755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(fakeJavaDir, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(pkg1, "Alpha.java"), []byte("public class Alpha {}"), 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(pkg2, "Beta.java"), []byte("package com.example.subpkg; public class Beta {}"), 0644); err != nil { + t.Fatal(err) + } + }, + want: []string{ + filepath.Join("samples", "src", "main", "java", "com", "example", "Alpha.java"), + filepath.Join("samples", "src", "main", "java", "com", "example", "subpkg", "Beta.java"), + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + tempDir := t.TempDir() + test.setupFiles(t, tempDir) + + got, err := collectSampleFiles(tempDir) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestParseCodeSample(t *testing.T) { + for _, test := range []struct { + name string + relPath string + content string + want *codeSample + }{ + { + name: "default title derived from filename", + relPath: filepath.Join("samples", "src", "main", "java", "DemoSample.java"), + content: "public class DemoSample {}", + want: &codeSample{ + Title: "Demo Sample", + File: "samples/src/main/java/DemoSample.java", + }, + }, + { + name: "custom title override from metadata", + relPath: filepath.Join("samples", "src", "main", "java", "RequesterPays.java"), + content: "// sample-metadata:\n// title: Custom Title Override\npublic class RequesterPays {}", + want: &codeSample{ + Title: "Custom Title Override", + File: "samples/src/main/java/RequesterPays.java", + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + tempDir := t.TempDir() + absPath := filepath.Join(tempDir, test.relPath) + if err := os.MkdirAll(filepath.Dir(absPath), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(absPath, []byte(test.content), 0644); err != nil { + t.Fatal(err) + } + + got, err := parseCodeSample(tempDir, test.relPath) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestParseCodeSample_Error(t *testing.T) { + for _, test := range []struct { + name string + relPath string + content string + wantErr error + }{ + { + name: "empty title returns error", + relPath: filepath.Join("samples", "src", "main", "java", "InvalidSample.java"), + content: "// sample-metadata:\n// title: \"\"\npublic class InvalidSample {}", + wantErr: errEmptyTitle, + }, + { + name: "missing title line returns error", + relPath: filepath.Join("samples", "src", "main", "java", "MissingTitle.java"), + content: "// sample-metadata:\n// description: missing\npublic class MissingTitle {}", + wantErr: errMissingTitle, + }, + } { + t.Run(test.name, func(t *testing.T) { + tempDir := t.TempDir() + absPath := filepath.Join(tempDir, test.relPath) + if err := os.MkdirAll(filepath.Dir(absPath), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(absPath, []byte(test.content), 0644); err != nil { + t.Fatal(err) + } + + _, err := parseCodeSample(tempDir, test.relPath) + if !errors.Is(err, test.wantErr) { + t.Errorf("parseCodeSample() error = %v, wantErr %v", err, test.wantErr) + } + }) + } +} + func TestDecamelize(t *testing.T) { for _, test := range []struct { name string From 97ed951a4d7ac6e3436a903e78cd1c1652003a80 Mon Sep 17 00:00:00 2001 From: Joe Wang <106995533+JoeWang1127@users.noreply.github.com> Date: Mon, 29 Jun 2026 21:03:16 +0000 Subject: [PATCH 086/108] chore(.github): rm legacylibrarian from workflow (#6579) Remove legacylibrarian from workflow. --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 433672ac8df..69259e1fef6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -34,13 +34,13 @@ jobs: - name: Run tests and check coverage run: | go run ./tool/cmd/coverage -target=80 \ - $(go list ./... | grep -v -E 'github.com/googleapis/librarian$|internal/librarian/(dart|golang|java|nodejs|python|rust|swift)|internal/sidekick|internal/legacylibrarian|internal/sample|internal/testhelper') + $(go list ./... | grep -v -E 'github.com/googleapis/librarian$|internal/librarian/(dart|golang|java|nodejs|python|rust|swift)|internal/sidekick|internal/sample|internal/testhelper') create-issue-on-failure: runs-on: ubuntu-24.04 permissions: contents: read issues: write - needs: [legacylibrarian, test] + needs: [test] if: ${{ always() && contains(needs.*.result, 'failure') && github.event_name == 'push' && github.ref == 'refs/heads/main' }} steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 From 3ee938a275e1f34480d3119331c8c97790f70aab Mon Sep 17 00:00:00 2001 From: g-husam Date: Mon, 29 Jun 2026 17:05:31 -0400 Subject: [PATCH 087/108] fix(internal/librarian): gracefully handle nil defaults in applyDefaults (#6571) The applyDefaults function is updated to safely handle a nil defaults parameter to avoid a runtime nil pointer panic when attempting to access defaults.Output. A test case is added to verify this safety. --- internal/librarian/library.go | 6 +++++- internal/librarian/library_test.go | 14 ++++++++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/internal/librarian/library.go b/internal/librarian/library.go index 13a56f6651e..d3304620711 100644 --- a/internal/librarian/library.go +++ b/internal/librarian/library.go @@ -299,7 +299,11 @@ func applyDefaults(language string, lib *config.Library, defaults *config.Defaul if len(lib.APIs) > 0 { apiPath = lib.APIs[0].Path } - lib.Output = defaultOutput(language, lib.Name, apiPath, defaults.Output) + var defaultOut string + if defaults != nil { + defaultOut = defaults.Output + } + lib.Output = defaultOutput(language, lib.Name, apiPath, defaultOut) } return fillLibraryDefaults(language, fillDefaults(lib, defaults)) } diff --git a/internal/librarian/library_test.go b/internal/librarian/library_test.go index d79eeb2e891..464e97654d4 100644 --- a/internal/librarian/library_test.go +++ b/internal/librarian/library_test.go @@ -636,6 +636,7 @@ func TestApplyDefaults(t *testing.T) { apis []*config.API wantOutput string wantAPIPath string + nilDefaults bool }{ { name: "empty output derives path from api", @@ -690,6 +691,12 @@ func TestApplyDefaults(t *testing.T) { language: config.LanguageGo, wantOutput: "src/generated/google-cloud-secretmanager-v1", }, + { + name: "nil defaults is handled safely", + language: config.LanguageGo, + nilDefaults: true, + wantOutput: "google-cloud-secretmanager-v1", + }, } { t.Run(test.name, func(t *testing.T) { lib := &config.Library{ @@ -698,8 +705,11 @@ func TestApplyDefaults(t *testing.T) { APIs: test.apis, Rust: test.rust, } - defaults := &config.Default{ - Output: "src/generated", + var defaults *config.Default + if !test.nilDefaults { + defaults = &config.Default{ + Output: "src/generated", + } } got, err := applyDefaults(test.language, lib, defaults) if err != nil { From bfb6df9debe0a2f89c0f38e50eaf480525a75c99 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Tue, 30 Jun 2026 05:18:24 +0800 Subject: [PATCH 088/108] chore(main): release 0.23.0 (#6536) :robot: I have created a release *beep* *boop* --- ## [0.23.0](https://github.com/googleapis/librarian/compare/v0.22.0...v0.23.0) (2026-06-29) ### Features * **internal/librarian/java:** add decamelize utility for README generation ([#6538](https://github.com/googleapis/librarian/issues/6538)) ([a6d950a](https://github.com/googleapis/librarian/commit/a6d950a157338083086d2b3d8c03150179628b10)), closes [#6515](https://github.com/googleapis/librarian/issues/6515) * **internal/librarian/java:** Add JSpecify dep to proto module template ([#6564](https://github.com/googleapis/librarian/issues/6564)) ([660ab2b](https://github.com/googleapis/librarian/commit/660ab2bcbb654ad546945b4a107fd77923ee57a9)) * **internal/librarian/java:** add production sample filter ([#6545](https://github.com/googleapis/librarian/issues/6545)) ([df213ec](https://github.com/googleapis/librarian/commit/df213ecb97834adb13c62b902fe090955845ba19)), closes [#6515](https://github.com/googleapis/librarian/issues/6515) * **internal/librarian/java:** add README sample extraction helpers ([#6574](https://github.com/googleapis/librarian/issues/6574)) ([500ba22](https://github.com/googleapis/librarian/commit/500ba22a7de65f5ab0e84112daa7d16ff76ff841)), closes [#6515](https://github.com/googleapis/librarian/issues/6515) * **internal/librarian/java:** add title override extractor ([#6546](https://github.com/googleapis/librarian/issues/6546)) ([0986d6a](https://github.com/googleapis/librarian/commit/0986d6a2cd5d3e4736b9e336a47dd85c9240e14b)), closes [#6515](https://github.com/googleapis/librarian/issues/6515) * **internal/librarian/python:** update gapic-generator to 1.36.0 ([#6548](https://github.com/googleapis/librarian/issues/6548)) ([a0930f4](https://github.com/googleapis/librarian/commit/a0930f4b0f52383ee4b19f23ae666ae4017f40f6)) * **nodejs:** bump pnpm version to 11.7.0 and support v11+ global bin layout ([#6494](https://github.com/googleapis/librarian/issues/6494)) ([1373628](https://github.com/googleapis/librarian/commit/1373628c9c8c639a666fe17813a4f313fe0f8c40)), closes [#6480](https://github.com/googleapis/librarian/issues/6480) * **sidekick/rust:** track recording error info for discovery LROs ([#6304](https://github.com/googleapis/librarian/issues/6304)) ([feb2ef2](https://github.com/googleapis/librarian/commit/feb2ef22fa2b45f2bf216a7da95c106eeaacbad8)), closes [#6286](https://github.com/googleapis/librarian/issues/6286) * **sidekick/swift:** generate synthetic messages ([#6530](https://github.com/googleapis/librarian/issues/6530)) ([7303648](https://github.com/googleapis/librarian/commit/730364823f86ac636ddea676bb439fce8f8e5a6c)) * **sidekick/swift:** handle clashing names ([#6543](https://github.com/googleapis/librarian/issues/6543)) ([56bf365](https://github.com/googleapis/librarian/commit/56bf365d10860f00bc9059a6d78f5f554c59406c)) * **sidekick/swift:** swap client vs. protocol ([#6566](https://github.com/googleapis/librarian/issues/6566)) ([14e186c](https://github.com/googleapis/librarian/commit/14e186cb2cdc1d8c545d022aba24bbd864e13ea0)) ### Bug Fixes * **internal/librarian:** gracefully handle nil defaults in applyDefaults ([#6571](https://github.com/googleapis/librarian/issues/6571)) ([3ee938a](https://github.com/googleapis/librarian/commit/3ee938a275e1f34480d3119331c8c97790f70aab)) --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- .release-please-manifest.json | 2 +- CHANGELOG.md | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 788b0fa7632..97bce112d22 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "0.22.0" + ".": "0.23.0" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 25fad1c6fa0..1a773dcf5b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,27 @@ # Changelog +## [0.23.0](https://github.com/googleapis/librarian/compare/v0.22.0...v0.23.0) (2026-06-29) + + +### Features + +* **internal/librarian/java:** add decamelize utility for README generation ([#6538](https://github.com/googleapis/librarian/issues/6538)) ([a6d950a](https://github.com/googleapis/librarian/commit/a6d950a157338083086d2b3d8c03150179628b10)), closes [#6515](https://github.com/googleapis/librarian/issues/6515) +* **internal/librarian/java:** Add JSpecify dep to proto module template ([#6564](https://github.com/googleapis/librarian/issues/6564)) ([660ab2b](https://github.com/googleapis/librarian/commit/660ab2bcbb654ad546945b4a107fd77923ee57a9)) +* **internal/librarian/java:** add production sample filter ([#6545](https://github.com/googleapis/librarian/issues/6545)) ([df213ec](https://github.com/googleapis/librarian/commit/df213ecb97834adb13c62b902fe090955845ba19)), closes [#6515](https://github.com/googleapis/librarian/issues/6515) +* **internal/librarian/java:** add README sample extraction helpers ([#6574](https://github.com/googleapis/librarian/issues/6574)) ([500ba22](https://github.com/googleapis/librarian/commit/500ba22a7de65f5ab0e84112daa7d16ff76ff841)), closes [#6515](https://github.com/googleapis/librarian/issues/6515) +* **internal/librarian/java:** add title override extractor ([#6546](https://github.com/googleapis/librarian/issues/6546)) ([0986d6a](https://github.com/googleapis/librarian/commit/0986d6a2cd5d3e4736b9e336a47dd85c9240e14b)), closes [#6515](https://github.com/googleapis/librarian/issues/6515) +* **internal/librarian/python:** update gapic-generator to 1.36.0 ([#6548](https://github.com/googleapis/librarian/issues/6548)) ([a0930f4](https://github.com/googleapis/librarian/commit/a0930f4b0f52383ee4b19f23ae666ae4017f40f6)) +* **nodejs:** bump pnpm version to 11.7.0 and support v11+ global bin layout ([#6494](https://github.com/googleapis/librarian/issues/6494)) ([1373628](https://github.com/googleapis/librarian/commit/1373628c9c8c639a666fe17813a4f313fe0f8c40)), closes [#6480](https://github.com/googleapis/librarian/issues/6480) +* **sidekick/rust:** track recording error info for discovery LROs ([#6304](https://github.com/googleapis/librarian/issues/6304)) ([feb2ef2](https://github.com/googleapis/librarian/commit/feb2ef22fa2b45f2bf216a7da95c106eeaacbad8)), closes [#6286](https://github.com/googleapis/librarian/issues/6286) +* **sidekick/swift:** generate synthetic messages ([#6530](https://github.com/googleapis/librarian/issues/6530)) ([7303648](https://github.com/googleapis/librarian/commit/730364823f86ac636ddea676bb439fce8f8e5a6c)) +* **sidekick/swift:** handle clashing names ([#6543](https://github.com/googleapis/librarian/issues/6543)) ([56bf365](https://github.com/googleapis/librarian/commit/56bf365d10860f00bc9059a6d78f5f554c59406c)) +* **sidekick/swift:** swap client vs. protocol ([#6566](https://github.com/googleapis/librarian/issues/6566)) ([14e186c](https://github.com/googleapis/librarian/commit/14e186cb2cdc1d8c545d022aba24bbd864e13ea0)) + + +### Bug Fixes + +* **internal/librarian:** gracefully handle nil defaults in applyDefaults ([#6571](https://github.com/googleapis/librarian/issues/6571)) ([3ee938a](https://github.com/googleapis/librarian/commit/3ee938a275e1f34480d3119331c8c97790f70aab)) + ## [0.22.0](https://github.com/googleapis/librarian/compare/v0.21.0...v0.22.0) (2026-06-22) From 0bf03bf4006b77ba2469613ed327e4db73792651 Mon Sep 17 00:00:00 2001 From: Noah Dietz Date: Mon, 29 Jun 2026 18:46:07 -0400 Subject: [PATCH 089/108] chore(ci): exclude CHANGELOG.md from typos check (#6581) --- .typos.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.typos.toml b/.typos.toml index 60f691af0f1..08078a5544f 100644 --- a/.typos.toml +++ b/.typos.toml @@ -16,6 +16,8 @@ extend-exclude = [ # The test data have typos, or at least uncommon spelling. Not worth fixing. "**/testdata/**", + # The CHANGELOG has commit hashes which can be flagged as typos. + "CHANGELOG.md", ] [type.mustache] From 2ed15d75e98df272531266688390b966ac6ebe41 Mon Sep 17 00:00:00 2001 From: sofisl <55454395+sofisl@users.noreply.github.com> Date: Mon, 29 Jun 2026 16:59:06 -0700 Subject: [PATCH 090/108] Revert "feat(internal/librarian/java): remove redundant keep items in librarian.yaml" (#6512) Reverts googleapis/librarian#6291 due to regression in which tidy removes too many files, many of which are necessary. Fixes https://github.com/googleapis/librarian/issues/6510 --- internal/librarian/java/clean.go | 28 +++++------------------- internal/librarian/java/defaults.go | 28 ++++-------------------- internal/librarian/java/defaults_test.go | 21 ++++++------------ 3 files changed, 17 insertions(+), 60 deletions(-) diff --git a/internal/librarian/java/clean.go b/internal/librarian/java/clean.go index 7d36826493e..34264dedbf7 100644 --- a/internal/librarian/java/clean.go +++ b/internal/librarian/java/clean.go @@ -85,9 +85,6 @@ func cleanPatterns(library *config.Library) map[string]bool { } for _, api := range library.APIs { javaAPI := api.Java - if javaAPI == nil { - javaAPI = &config.JavaAPI{} - } version := filepath.Base(api.Path) apiCoordinates := DeriveAPICoordinates(libraryCoordinates, version, javaAPI) if apiCoordinates.Proto.ArtifactID != "" && shouldGenerateProto(javaAPI) { @@ -176,29 +173,16 @@ func isDefaultPreserved(path string) bool { return itTestRegexp.MatchString(path) || versionRegexp.MatchString(path) } -// isManualFile checks if the file at the given path is manually maintained. -func isManualFile(path string, name string) (bool, error) { - if slices.Contains(generatedNonJavaFiles, name) { - return false, nil - } - if filepath.Ext(path) != ".java" { - return true, nil - } - hasGenMarker, err := hasMarker(path) - if err != nil { - return false, err - } - return !hasGenMarker, nil -} - // shouldCleanMarkerPath returns true if the file at path should be cleaned based on // its name or whether it contains the auto-generated marker. func shouldCleanMarkerPath(path string, d os.DirEntry) (bool, error) { - isManual, err := isManualFile(path, d.Name()) - if err != nil { - return false, err + if slices.Contains(generatedNonJavaFiles, d.Name()) { + return true, nil + } + if filepath.Ext(path) != ".java" { + return false, nil } - return !isManual, nil + return hasMarker(path) } // hasMarker checks if the file at path contains the auto-generated marker. diff --git a/internal/librarian/java/defaults.go b/internal/librarian/java/defaults.go index 76fa16047b3..c1a250fa75b 100644 --- a/internal/librarian/java/defaults.go +++ b/internal/librarian/java/defaults.go @@ -89,10 +89,7 @@ func Fill(library *config.Library) (*config.Library, error) { // Tidy tidies the Java-specific configuration for a library by removing default // values. func Tidy(library *config.Library) (*config.Library, error) { - if _, err := Fill(library); err != nil { - return nil, err - } - library.Keep = tidyKeep(library) + library.Keep = tidyKeep(library.Keep) if library.Output == deriveOutput(library.Name) { library.Output = "" } @@ -146,19 +143,16 @@ func Tidy(library *config.Library) (*config.Library, error) { } // tidyKeep removes default files from the library's keep configuration. -func tidyKeep(library *config.Library) []string { - if len(library.Keep) == 0 { +func tidyKeep(keep []string) []string { + if len(keep) == 0 { return nil } var filteredKeepPaths []string - for _, keepPath := range library.Keep { + for _, keepPath := range keep { keepPathSlash := filepath.ToSlash(keepPath) if isDefaultPreserved(keepPathSlash) { continue } - if isInGAPICModule(library, keepPath) { - continue - } filteredKeepPaths = append(filteredKeepPaths, keepPath) } slices.Sort(filteredKeepPaths) @@ -169,20 +163,6 @@ func tidyKeep(library *config.Library) []string { return filteredKeepPaths } -// isInGAPICModule checks if the given keepPath belongs to a generated GAPIC module of the library. -func isInGAPICModule(library *config.Library, keepPath string) bool { - keepPathSlash := filepath.ToSlash(keepPath) - for pattern, isGAPIC := range cleanPatterns(library) { - if isGAPIC { - moduleName, _, _ := strings.Cut(filepath.ToSlash(pattern), "/") - if strings.HasPrefix(keepPathSlash, moduleName+"/") { - return true - } - } - } - return false -} - var ( // ErrOmitCommonResourcesConflict is returned when OmitCommonResources is true // but common_resources.proto is also explicitly listed in AdditionalProtos. diff --git a/internal/librarian/java/defaults_test.go b/internal/librarian/java/defaults_test.go index 48b1d577134..98a8b8d4b83 100644 --- a/internal/librarian/java/defaults_test.go +++ b/internal/librarian/java/defaults_test.go @@ -446,6 +446,9 @@ func TestTidy(t *testing.T) { }, }, Keep: []string{ + "google-cloud-vision/src/main/resources/META-INF/native-image/reflect-config.json", + "google-cloud-vision/src/test/java/com/google/cloud/vision/it/ITSystemTest.java", + "google-cloud-vision/src/test/resources/placeholder.txt", "proto-google-cloud-vision-v1/src/main/java/com/google/cloud/vision/v1/ImageName.java", }, }, @@ -644,25 +647,15 @@ func TestTidyKeep(t *testing.T) { "proto-google-cloud-vision-v1/src/main/java/com/google/cloud/vision/v1/ImageName.java", }, want: []string{ + "google-cloud-vision/src/main/resources/META-INF/native-image/reflect-config.json", + "google-cloud-vision/src/test/java/com/google/cloud/vision/it/ITSystemTest.java", + "google-cloud-vision/src/test/resources/placeholder.txt", "proto-google-cloud-vision-v1/src/main/java/com/google/cloud/vision/v1/ImageName.java", }, }, } { t.Run(test.name, func(t *testing.T) { - lib := &config.Library{ - Name: "vision", - APIs: []*config.API{ - { - Path: "google/cloud/vision/v1", - }, - }, - Java: &config.JavaModule{ - GroupID: "com.google.cloud", - ArtifactID: "google-cloud-vision", - }, - Keep: test.keep, - } - got := tidyKeep(lib) + got := tidyKeep(test.keep) if diff := cmp.Diff(test.want, got); diff != "" { t.Errorf("mismatch (-want +got):\n%s", diff) } From b5e3d4519d4a747fa8f09fdababb3bf77e78611c Mon Sep 17 00:00:00 2001 From: yangyzs <171981480+yangyzs@users.noreply.github.com> Date: Tue, 30 Jun 2026 10:58:27 -0400 Subject: [PATCH 091/108] feat(internal/librarian/java): add extractSamples for README generation (#6578) Add extractSamples main function to locate Java sample files and return parsed codeSample structs. Add unit tests in readme_test.go. For #6515 --- internal/librarian/java/readme.go | 24 +++++ internal/librarian/java/readme_test.go | 121 +++++++++++++++++++++++++ 2 files changed, 145 insertions(+) diff --git a/internal/librarian/java/readme.go b/internal/librarian/java/readme.go index 15a0ce22606..8ca97817b8f 100644 --- a/internal/librarian/java/readme.go +++ b/internal/librarian/java/readme.go @@ -37,6 +37,9 @@ var ( // errEmptyTitle indicates the extracted title value is empty. errEmptyTitle = errors.New("title value cannot be empty") + + // errEmptyDir indicates the provided directory path is empty. + errEmptyDir = errors.New("dir cannot be empty") ) // codeSample represents a discovered Java code sample along with its derived title. @@ -45,6 +48,27 @@ type codeSample struct { File string } +// extractSamples locates production Java sample files and returns parsed codeSample structs +// containing display titles and relative paths for README rendering. +func extractSamples(dir string) ([]codeSample, error) { + if dir == "" { + return nil, errEmptyDir + } + files, err := collectSampleFiles(dir) + if err != nil { + return nil, err + } + var samples []codeSample + for _, file := range files { + sample, err := parseCodeSample(dir, file) + if err != nil { + return nil, err + } + samples = append(samples, *sample) + } + return samples, nil +} + // collectSampleFiles recursively scans dir/samples for Java production files. func collectSampleFiles(dir string) ([]string, error) { samplesDir := filepath.Join(dir, "samples") diff --git a/internal/librarian/java/readme_test.go b/internal/librarian/java/readme_test.go index d4ba75a3355..ea83f04e1d4 100644 --- a/internal/librarian/java/readme_test.go +++ b/internal/librarian/java/readme_test.go @@ -23,6 +23,127 @@ import ( "github.com/google/go-cmp/cmp" ) +func TestExtractSamples(t *testing.T) { + for _, test := range []struct { + name string + setupFiles func(t *testing.T, dir string) + want []codeSample + }{ + { + name: "missing samples directory", + setupFiles: func(t *testing.T, dir string) { + // Do nothing, tempDir is empty. + }, + want: nil, + }, + { + name: "extract successfully", + setupFiles: func(t *testing.T, dir string) { + samplesDir := filepath.Join(dir, "samples", "src", "main", "java") + if err := os.MkdirAll(samplesDir, 0755); err != nil { + t.Fatal(err) + } + file1 := filepath.Join(samplesDir, "RequesterPays.java") + content1 := `// sample-metadata: +// title: Custom Title Override +public class RequesterPays {}` + if err := os.WriteFile(file1, []byte(content1), 0644); err != nil { + t.Fatal(err) + } + file2 := filepath.Join(samplesDir, "DemoSample.java") + content2 := `public class DemoSample {}` + if err := os.WriteFile(file2, []byte(content2), 0644); err != nil { + t.Fatal(err) + } + }, + want: []codeSample{ + { + Title: "Demo Sample", + File: "samples/src/main/java/DemoSample.java", + }, + { + Title: "Custom Title Override", + File: "samples/src/main/java/RequesterPays.java", + }, + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + tempDir := t.TempDir() + test.setupFiles(t, tempDir) + + samples, err := extractSamples(tempDir) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(test.want, samples); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestExtractSamples_Error(t *testing.T) { + for _, test := range []struct { + name string + setupFiles func(t *testing.T, dir string) + dir string + wantErr error + }{ + { + name: "error on empty directory", + dir: "", + wantErr: errEmptyDir, + }, + { + name: "error on empty title override", + setupFiles: func(t *testing.T, dir string) { + samplesDir := filepath.Join(dir, "samples", "src", "main", "java") + if err := os.MkdirAll(samplesDir, 0755); err != nil { + t.Fatal(err) + } + file := filepath.Join(samplesDir, "Sample.java") + content := `// sample-metadata: +// title: "" +public class Invalid {}` + if err := os.WriteFile(file, []byte(content), 0644); err != nil { + t.Fatal(err) + } + }, + wantErr: errEmptyTitle, + }, + { + name: "error on missing title line immediately following sample-metadata", + setupFiles: func(t *testing.T, dir string) { + samplesDir := filepath.Join(dir, "samples", "src", "main", "java") + if err := os.MkdirAll(samplesDir, 0755); err != nil { + t.Fatal(err) + } + file := filepath.Join(samplesDir, "Sample.java") + content := `// sample-metadata: +// description: missing title line +public class Invalid {}` + if err := os.WriteFile(file, []byte(content), 0644); err != nil { + t.Fatal(err) + } + }, + wantErr: errMissingTitle, + }, + } { + t.Run(test.name, func(t *testing.T) { + dir := test.dir + if test.setupFiles != nil { + dir = t.TempDir() + test.setupFiles(t, dir) + } + _, err := extractSamples(dir) + if !errors.Is(err, test.wantErr) { + t.Errorf("extractSamples() err = %v, want %v", err, test.wantErr) + } + }) + } +} + func TestCollectSampleFiles(t *testing.T) { for _, test := range []struct { name string From bb40e83a4e605ac0cc9b1719f8d965b4a68bb908 Mon Sep 17 00:00:00 2001 From: Carlos O'Ryan Date: Tue, 30 Jun 2026 11:52:02 -0400 Subject: [PATCH 092/108] feat(sidekick/discovery): signatures without path params (#6588) For discovery-based services, most methods get a method signature with all the path parameters. This does not work for methods without any path parameters. Such methods typically require one or more query parameters, and there is no way to tell which ones are required. --- internal/sidekick/parser/discovery/methods.go | 23 +++-- .../sidekick/parser/discovery/methods_test.go | 95 +++++++++++++------ 2 files changed, 84 insertions(+), 34 deletions(-) diff --git a/internal/sidekick/parser/discovery/methods.go b/internal/sidekick/parser/discovery/methods.go index 1f8cd78a7ca..bc34c7dc014 100644 --- a/internal/sidekick/parser/discovery/methods.go +++ b/internal/sidekick/parser/discovery/methods.go @@ -107,9 +107,18 @@ func makeMethod(model *api.API, parent *api.Message, doc *document, input *metho fieldNames[field.Name] = true } - // Every discovery-based method has one additional signature formed by the - // path parameters, in the order specified by the discovery doc. - signature := &api.MethodSignature{Names: input.ParameterOrder} + // Methods without path parameters don't get an overload. Typically one of + // the query parameter is "required" for them. Otherwise the request is some + // kind of singleton, these are sufficiently rare that missing the + // convenience overload is not a big problem. + var signature *api.MethodSignature + signatures := []*api.MethodSignature{} + if len(input.ParameterOrder) != 0 { + // If there are path parameters, then create an additional signature + // formed by them, in the order specified by the discovery doc. + signature = &api.MethodSignature{Names: input.ParameterOrder} + signatures = append(signatures, signature) + } bodyPathField := "" if bodyID != ".google.protobuf.Empty" { @@ -128,8 +137,10 @@ func makeMethod(model *api.API, parent *api.Message, doc *document, input *metho } requestMessage.Fields = append(requestMessage.Fields, body) bodyPathField = name - // The body, if present, is required for signature requests. - signature.Names = append(signature.Names, name) + if signature != nil { + // The body, if present, is required for signature requests. + signature.Names = append(signature.Names, name) + } } method := &api.Method{ @@ -144,7 +155,7 @@ func makeMethod(model *api.API, parent *api.Message, doc *document, input *metho Bindings: []*api.PathBinding{binding}, BodyFieldPath: bodyPathField, }, - Signatures: []*api.MethodSignature{signature}, + Signatures: signatures, APIVersion: input.APIVersion, } return method, nil diff --git a/internal/sidekick/parser/discovery/methods_test.go b/internal/sidekick/parser/discovery/methods_test.go index b635b6b32ba..fb2541b3011 100644 --- a/internal/sidekick/parser/discovery/methods_test.go +++ b/internal/sidekick/parser/discovery/methods_test.go @@ -27,37 +27,76 @@ func TestMakeServiceMethods(t *testing.T) { if err != nil { t.Fatal(err) } - id := "..zones.get" - got := model.Method(id) - if got == nil { - t.Fatalf("expected method %s in the API model", id) - } - want := &api.Method{ - ID: "..zones.get", - Name: "get", - Documentation: "Returns the specified Zone resource.", - InputTypeID: "..zones.getRequest", - OutputTypeID: "..Zone", - PathInfo: &api.PathInfo{ - Bindings: []*api.PathBinding{ - { - Verb: "GET", - PathTemplate: (&api.PathTemplate{}). - WithLiteral("compute"). - WithLiteral("v1"). - WithLiteral("projects"). - WithVariableNamed("project"). - WithLiteral("zones"). - WithVariableNamed("zone"), - QueryParameters: map[string]bool{}, + for _, test := range []struct { + id string + want *api.Method + }{ + { + id: "..zones.get", + want: &api.Method{ + ID: "..zones.get", + Name: "get", + Documentation: "Returns the specified Zone resource.", + InputTypeID: "..zones.getRequest", + OutputTypeID: "..Zone", + PathInfo: &api.PathInfo{ + Bindings: []*api.PathBinding{ + { + Verb: "GET", + PathTemplate: (&api.PathTemplate{}). + WithLiteral("compute"). + WithLiteral("v1"). + WithLiteral("projects"). + WithVariableNamed("project"). + WithLiteral("zones"). + WithVariableNamed("zone"), + QueryParameters: map[string]bool{}, + }, + }, + BodyFieldPath: "", }, + Signatures: []*api.MethodSignature{{Names: []string{"project", "zone"}}}, }, - BodyFieldPath: "", }, - Signatures: []*api.MethodSignature{{Names: []string{"project", "zone"}}}, - } - if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + { + id: "..firewallPolicies.insert", + want: &api.Method{ + ID: "..firewallPolicies.insert", + Name: "insert", + Documentation: "Creates a new policy in the specified project using the data included in the request.", + InputTypeID: "..firewallPolicies.insertRequest", + OutputTypeID: "..Operation", + PathInfo: &api.PathInfo{ + Bindings: []*api.PathBinding{ + { + Verb: "POST", + PathTemplate: (&api.PathTemplate{}). + WithLiteral("compute"). + WithLiteral("v1"). + WithLiteral("locations"). + WithLiteral("global"). + WithLiteral("firewallPolicies"), + QueryParameters: map[string]bool{ + "parentId": true, + "requestId": true, + }, + }, + }, + BodyFieldPath: "body", + }, + Signatures: []*api.MethodSignature{}, + }, + }, + } { + t.Run(test.id, func(t *testing.T) { + got := model.Method(test.id) + if got == nil { + t.Fatalf("expected method %s in the API model", test.id) + } + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + }) } } From 12fc3af5a499bf141238616a1393034f64873b33 Mon Sep 17 00:00:00 2001 From: Carlos O'Ryan Date: Tue, 30 Jun 2026 12:15:50 -0400 Subject: [PATCH 093/108] cleanup(sidekick): follow test guidelines (#6590) I got tired of refactoring code only to be told I was not following the guidelines in `howwewritego.md`. Change all the `t.Errorf(, diff)` to use the prescribed format. --- internal/command/command_test.go | 4 +- internal/git/git_test.go | 12 +-- internal/sidekick/api/apitest/apitest.go | 16 ++-- internal/sidekick/api/dependencies_test.go | 30 +++--- internal/sidekick/api/discovery_test.go | 9 +- internal/sidekick/api/documentation_test.go | 12 +-- internal/sidekick/api/model_test.go | 8 +- internal/sidekick/api/pagination_test.go | 12 +-- .../api/resource_name_heuristic_test.go | 2 +- internal/sidekick/api/scopes_test.go | 14 +-- .../sidekick/api/service_dependencies_test.go | 6 +- internal/sidekick/api/skip_test.go | 22 ++--- internal/sidekick/api/xref_test.go | 30 +++--- internal/sidekick/dart/annotate_test.go | 92 ++++++++++--------- internal/sidekick/dart/dart_test.go | 12 +-- internal/sidekick/language/codec_test.go | 10 +- .../language/walk_templates_dir_test.go | 2 +- internal/sidekick/parser/disco_test.go | 4 +- .../parser/discovery/discovery_test.go | 6 +- .../parser/discovery/enum_field_test.go | 6 +- .../discovery/inline_object_field_test.go | 4 +- .../sidekick/parser/discovery/lro_test.go | 4 +- .../parser/discovery/map_field_test.go | 4 +- .../sidekick/parser/discovery/methods_test.go | 4 +- .../parser/discovery/uritemplate_test.go | 71 +++++++------- internal/sidekick/parser/mixin_test.go | 4 +- internal/sidekick/parser/openapi_test.go | 4 +- .../sidekick/parser/protobuf_imports_test.go | 2 +- internal/sidekick/parser/protobuf_test.go | 14 +-- internal/sidekick/parser/routing_info_test.go | 4 +- .../sidekick/protobuf/input_files_test.go | 4 +- internal/sidekick/rust/bigquery_test.go | 2 +- internal/sidekick/rust_prost/annotate_test.go | 6 +- internal/sidekick/rust_prost/codec_test.go | 2 +- .../swift/annotate_enum_value_test.go | 2 +- .../sidekick/swift/annotate_message_test.go | 20 ++-- .../sidekick/swift/annotate_method_test.go | 12 +-- internal/sidekick/swift/codec_test.go | 6 +- .../swift/format_documentation_test.go | 4 +- internal/sidekick/swift/format_path_test.go | 4 +- .../swift/generate_package_swift_test.go | 4 +- internal/sidekick/swift/lookup_test.go | 6 +- 42 files changed, 253 insertions(+), 243 deletions(-) diff --git a/internal/command/command_test.go b/internal/command/command_test.go index 1403dfc01b7..43c2033f7dd 100644 --- a/internal/command/command_test.go +++ b/internal/command/command_test.go @@ -230,10 +230,10 @@ func TestRunStreaming(t *testing.T) { t.Fatal(err) } if diff := cmp.Diff(test.wantOut, outBuf.String()); diff != "" { - t.Errorf("mismatch of stdout (-want +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } if diff := cmp.Diff(test.wantErr, errBuf.String()); diff != "" { - t.Errorf("mismatch of stderr (-want +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }) } diff --git a/internal/git/git_test.go b/internal/git/git_test.go index 935536bcfd9..0da5c732b2d 100644 --- a/internal/git/git_test.go +++ b/internal/git/git_test.go @@ -110,7 +110,7 @@ func TestFilesChangedSuccess(t *testing.T) { } want := []string{path.Join("src", "storage", "src", "lib.rs")} if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -137,7 +137,7 @@ func TestFilterNoFilter(t *testing.T) { got := filesFilter(nil, input) want := input if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -163,7 +163,7 @@ func TestFilterBasic(t *testing.T) { "src/generated/cloud/secretmanager/v1/src/model.rs", } if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -181,7 +181,7 @@ func TestFilterSomeGlobs(t *testing.T) { }, input) want := []string{} if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -263,7 +263,7 @@ func TestShowFileAtRemoteBranch(t *testing.T) { t.Fatal(err) } if diff := cmp.Diff(testhelper.ReadmeContents, got); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -315,7 +315,7 @@ func TestShowFileAtRevision(t *testing.T) { t.Fatal(err) } if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }) } diff --git a/internal/sidekick/api/apitest/apitest.go b/internal/sidekick/api/apitest/apitest.go index bda4f8ce68f..2da541a8b8b 100644 --- a/internal/sidekick/api/apitest/apitest.go +++ b/internal/sidekick/api/apitest/apitest.go @@ -28,15 +28,15 @@ func CheckMessage(t *testing.T, got *api.Message, want *api.Message) { t.Helper() // Checking Parent, Messages, Fields, and OneOfs requires special handling. if diff := cmp.Diff(want, got, cmpopts.IgnoreFields(api.Message{}, "Fields", "OneOfs", "Parent", "Messages", "Enums", "Resource")); diff != "" { - t.Errorf("message attributes mismatch (-want +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } less := func(a, b *api.Field) bool { return a.Name < b.Name } if diff := cmp.Diff(want.Fields, got.Fields, cmpopts.SortSlices(less)); diff != "" { - t.Errorf("field mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } // Ignore parent because types are cyclic if diff := cmp.Diff(want.OneOfs, got.OneOfs, cmpopts.SortSlices(less)); diff != "" { - t.Errorf("oneofs mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -44,11 +44,11 @@ func CheckMessage(t *testing.T, got *api.Message, want *api.Message) { func CheckEnum(t *testing.T, got api.Enum, want api.Enum) { t.Helper() if diff := cmp.Diff(want, got, cmpopts.IgnoreFields(api.Enum{}, "Values", "UniqueNumberValues", "Parent")); diff != "" { - t.Errorf("mismatched service attributes (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } less := func(a, b *api.EnumValue) bool { return a.Name < b.Name } if diff := cmp.Diff(want.Values, got.Values, cmpopts.SortSlices(less), cmpopts.IgnoreFields(api.EnumValue{}, "Parent")); diff != "" { - t.Errorf("method mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -56,11 +56,11 @@ func CheckEnum(t *testing.T, got api.Enum, want api.Enum) { func CheckService(t *testing.T, got *api.Service, want *api.Service) { t.Helper() if diff := cmp.Diff(want, got, cmpopts.IgnoreFields(api.Service{}, "Methods")); diff != "" { - t.Errorf("mismatched service attributes (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } less := func(a, b *api.Method) bool { return a.Name < b.Name } if diff := cmp.Diff(want.Methods, got.Methods, cmpopts.SortSlices(less)); diff != "" { - t.Errorf("method mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -80,6 +80,6 @@ func CheckMethod(t *testing.T, service *api.Service, name string, want *api.Meth t.Errorf("missing method %s", name) } if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("mismatched data for method %s (-want, +got):\n%s", name, diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } diff --git a/internal/sidekick/api/dependencies_test.go b/internal/sidekick/api/dependencies_test.go index 8d3f62344d1..c98274c8d84 100644 --- a/internal/sidekick/api/dependencies_test.go +++ b/internal/sidekick/api/dependencies_test.go @@ -71,7 +71,7 @@ func TestFindDependenciesEnumFields(t *testing.T) { // Note that `MessageWithEnumField` is not included. want := []string{".test.OrphanEnum"} if diff := cmp.Diff(want, flatten(got), cmpopts.SortSlices(less)); diff != "" { - t.Errorf("dependencies mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } // Verify that a message with an enum field depends on the enum. @@ -81,7 +81,7 @@ func TestFindDependenciesEnumFields(t *testing.T) { } want = []string{".test.OrphanEnum", ".test.MessageWithEnumField"} if diff := cmp.Diff(want, flatten(got), cmpopts.SortSlices(less)); diff != "" { - t.Errorf("dependencies mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -110,7 +110,7 @@ func TestFindDependenciesNestedEnum(t *testing.T) { } want := []string{parent, child} if diff := cmp.Diff(want, flatten(got), cmpopts.SortSlices(less)); diff != "" { - t.Errorf("dependencies mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } got, err = FindDependencies(model, []string{parent}) @@ -119,7 +119,7 @@ func TestFindDependenciesNestedEnum(t *testing.T) { } want = []string{parent} if diff := cmp.Diff(want, flatten(got), cmpopts.SortSlices(less)); diff != "" { - t.Errorf("dependencies mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -169,7 +169,7 @@ func TestFindDependenciesNestedMessage(t *testing.T) { t.Fatal(err) } if diff := cmp.Diff(test.Want, flatten(got), cmpopts.SortSlices(less)); diff != "" { - t.Errorf("dependencies mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -205,7 +205,7 @@ func TestFindDependenciesMessage(t *testing.T) { } want := []string{".test.MessageWithMessageField", ".test.Orphan"} if diff := cmp.Diff(want, flatten(got), cmpopts.SortSlices(less)); diff != "" { - t.Errorf("dependencies mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } got, err = FindDependencies(model, []string{".test.Orphan"}) @@ -215,7 +215,7 @@ func TestFindDependenciesMessage(t *testing.T) { // Note that `MessageWithMessageField` is not included. want = []string{".test.Orphan"} if diff := cmp.Diff(want, flatten(got), cmpopts.SortSlices(less)); diff != "" { - t.Errorf("dependencies mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -244,7 +244,7 @@ func TestFindDependenciesHandlesCycles1(t *testing.T) { } want := []string{".test.Recursive"} if diff := cmp.Diff(want, flatten(got), cmpopts.SortSlices(less)); diff != "" { - t.Errorf("dependencies mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -294,7 +294,7 @@ func TestFindDependenciesHandlesCycles2(t *testing.T) { } want := []string{".test.A", ".test.B"} if diff := cmp.Diff(want, flatten(got), cmpopts.SortSlices(less)); diff != "" { - t.Errorf("dependencies mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } } @@ -346,7 +346,7 @@ func TestFindDependenciesHandlesCycles3(t *testing.T) { } want := []string{".test.Triangle2.Triangle1", ".test.Triangle2", ".test.Triangle3"} if diff := cmp.Diff(want, flatten(got), cmpopts.SortSlices(less)); diff != "" { - t.Errorf("dependencies mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } } @@ -394,7 +394,7 @@ func TestFindDependenciesMethod(t *testing.T) { // Note that `Sibling` is not included want := []string{".test.Service", ".test.Service.Method", ".test.Request", ".test.Response"} if diff := cmp.Diff(want, flatten(got), cmpopts.SortSlices(less)); diff != "" { - t.Errorf("dependencies mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } // Verify that messages don't imply methods @@ -404,7 +404,7 @@ func TestFindDependenciesMethod(t *testing.T) { } want = []string{".test.Request"} if diff := cmp.Diff(want, flatten(got), cmpopts.SortSlices(less)); diff != "" { - t.Errorf("dependencies mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -452,7 +452,7 @@ func TestFindDependenciesLroMethod(t *testing.T) { } want := []string{".test.Service", ".test.Service.Lro", ".test.Empty", ".test.OpMetadata", ".test.OpResponse"} if diff := cmp.Diff(want, flatten(got), cmpopts.SortSlices(less)); diff != "" { - t.Errorf("dependencies mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -557,7 +557,7 @@ func TestFindDependenciesService(t *testing.T) { ".test.Enum", } if diff := cmp.Diff(want, flatten(got), cmpopts.SortSlices(less)); diff != "" { - t.Errorf("dependencies mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } got, err = FindDependencies(model, []string{".test.Service", ".test.OtherService"}) @@ -579,7 +579,7 @@ func TestFindDependenciesService(t *testing.T) { ".test.OtherResponse", } if diff := cmp.Diff(want, flatten(got), cmpopts.SortSlices(less)); diff != "" { - t.Errorf("dependencies mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } diff --git a/internal/sidekick/api/discovery_test.go b/internal/sidekick/api/discovery_test.go index deca05c2992..0cd22e47fae 100644 --- a/internal/sidekick/api/discovery_test.go +++ b/internal/sidekick/api/discovery_test.go @@ -34,12 +34,12 @@ func TestLroServices(t *testing.T) { } got := d.LroServices() if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("LroServices() mismatch (-want +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } func TestPathParameters(t *testing.T) { - tests := []struct { + for _, test := range []struct { name string input *Poller want []string @@ -59,12 +59,11 @@ func TestPathParameters(t *testing.T) { input: &Poller{Prefix: "a/{b}"}, want: []string{"b"}, }, - } - for _, test := range tests { + } { t.Run(test.name, func(t *testing.T) { got := test.input.PathParameters() if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("PathParameters(%q) mismatch (-want +got):\n%s", test.input.Prefix, diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }) } diff --git a/internal/sidekick/api/documentation_test.go b/internal/sidekick/api/documentation_test.go index a6373fcfd2b..68e8781d1aa 100644 --- a/internal/sidekick/api/documentation_test.go +++ b/internal/sidekick/api/documentation_test.go @@ -82,7 +82,7 @@ func TestPatchCommentsMessage(t *testing.T) { } if diff := cmp.Diff(m0.Documentation, Want); diff != "" { - t.Errorf("mismatch in enums (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -203,7 +203,7 @@ func TestPatchCommentsField(t *testing.T) { } if diff := cmp.Diff(f0.Documentation, Want); diff != "" { - t.Errorf("mismatch in enums (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -227,7 +227,7 @@ func TestPatchCommentsEnum(t *testing.T) { } if diff := cmp.Diff(e0.Documentation, Want); diff != "" { - t.Errorf("mismatch in enums (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -257,7 +257,7 @@ func TestPatchCommentsEnumValue(t *testing.T) { } if diff := cmp.Diff(v0.Documentation, Want); diff != "" { - t.Errorf("mismatch in enums (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -281,7 +281,7 @@ func TestPatchCommentsService(t *testing.T) { } if diff := cmp.Diff(s0.Documentation, Want); diff != "" { - t.Errorf("mismatch in enums (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -310,6 +310,6 @@ func TestPatchCommentsMethod(t *testing.T) { } if diff := cmp.Diff(m0.Documentation, Want); diff != "" { - t.Errorf("mismatch in enums (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } diff --git a/internal/sidekick/api/model_test.go b/internal/sidekick/api/model_test.go index b70963b1756..a41c1885cca 100644 --- a/internal/sidekick/api/model_test.go +++ b/internal/sidekick/api/model_test.go @@ -54,7 +54,7 @@ func TestRoutingCombosSimpleOr(t *testing.T) { }, } if diff := cmp.Diff(want, method.RoutingCombos()); diff != "" { - t.Errorf("Incorrect routing combos (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -92,7 +92,7 @@ func TestRoutingCombosSimpleAnd(t *testing.T) { }, } if diff := cmp.Diff(want, method.RoutingCombos()); diff != "" { - t.Errorf("Incorrect routing combos (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -161,7 +161,7 @@ func TestRoutingCombosFull(t *testing.T) { make_combo(va3, vb2, vc1), } if diff := cmp.Diff(want, method.RoutingCombos()); diff != "" { - t.Errorf("Incorrect routing combos (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -229,7 +229,7 @@ func TestPathTemplateBuilder(t *testing.T) { Verb: verb, } if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("bad builder result (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } diff --git a/internal/sidekick/api/pagination_test.go b/internal/sidekick/api/pagination_test.go index 2e297e0dd3e..fdc3a859f7c 100644 --- a/internal/sidekick/api/pagination_test.go +++ b/internal/sidekick/api/pagination_test.go @@ -90,7 +90,7 @@ func TestPageSimple(t *testing.T) { PageableItem: response.Fields[1], } if diff := cmp.Diff(want, response.Pagination); diff != "" { - t.Errorf("mismatch, (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -175,7 +175,7 @@ func TestPageWithOverride(t *testing.T) { PageableItem: response.Fields[2], } if diff := cmp.Diff(want, response.Pagination); diff != "" { - t.Errorf("mismatch, (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -429,7 +429,7 @@ func TestPaginationRequestPageSizeSuccess(t *testing.T) { } got := paginationRequestPageSize(response) if diff := cmp.Diff(response.Fields[0], got); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } } @@ -559,7 +559,7 @@ func TestPaginationResponseItemMatching(t *testing.T) { } got := paginationResponseItem(nil, "package.Service.List", response) if diff := cmp.Diff(response.Fields[0], got); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } } @@ -594,7 +594,7 @@ func TestPaginationResponseItemMatchingMany(t *testing.T) { } got := paginationResponseItem(nil, "package.Service.List", response) if diff := cmp.Diff(response.Fields[0], got); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } } @@ -620,7 +620,7 @@ func TestPaginationResponseItemMatchingPreferRepeatedOverMap(t *testing.T) { } got := paginationResponseItem(nil, "package.Service.List", response) if diff := cmp.Diff(response.Fields[1], got); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } diff --git a/internal/sidekick/api/resource_name_heuristic_test.go b/internal/sidekick/api/resource_name_heuristic_test.go index ea2e8942f38..bdd607848a0 100644 --- a/internal/sidekick/api/resource_name_heuristic_test.go +++ b/internal/sidekick/api/resource_name_heuristic_test.go @@ -162,7 +162,7 @@ func TestBuildHeuristicVocabulary(t *testing.T) { } got := BuildHeuristicVocabulary(model) if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("BuildHeuristicVocabulary() mismatch (-want +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }) } diff --git a/internal/sidekick/api/scopes_test.go b/internal/sidekick/api/scopes_test.go index 7400468b6e0..99bcc082715 100644 --- a/internal/sidekick/api/scopes_test.go +++ b/internal/sidekick/api/scopes_test.go @@ -29,7 +29,7 @@ func TestScopesService(t *testing.T) { got := service.Scopes() want := []string{"test.Service", "test"} if diff := cmp.Diff(want, got); len(diff) != 0 { - t.Errorf("mismatched service.Scopes() (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -50,13 +50,13 @@ func TestScopesMessage(t *testing.T) { got := parent.Scopes() want := []string{"test.Parent", "test"} if diff := cmp.Diff(want, got); len(diff) != 0 { - t.Errorf("mismatched parent.Scopes() (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } got = child.Scopes() want = []string{"test.Parent.Child", "test.Parent", "test"} if diff := cmp.Diff(want, got); len(diff) != 0 { - t.Errorf("mismatched child.Scopes() (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -70,7 +70,7 @@ func TestScopesEnum(t *testing.T) { got := enum.Scopes() want := []string{"test.Enum", "test"} if diff := cmp.Diff(want, got); len(diff) != 0 { - t.Errorf("mismatched enum.Scopes() (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -91,7 +91,7 @@ func TestScopesEnumInMessage(t *testing.T) { got := child.Scopes() want := []string{"test.Parent.Child", "test.Parent", "test"} if diff := cmp.Diff(want, got); len(diff) != 0 { - t.Errorf("mismatched child.Scopes() (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -111,7 +111,7 @@ func TestScopesEnumValue(t *testing.T) { got := enumValue.Scopes() want := []string{"test.Enum", "test"} if diff := cmp.Diff(want, got); len(diff) != 0 { - t.Errorf("mismatched enum.Scopes() (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -138,7 +138,7 @@ func TestScopesEnumValueInMessage(t *testing.T) { got := enumValue.Scopes() want := []string{"test.Parent.Enum", "test.Parent", "test"} if diff := cmp.Diff(want, got); len(diff) != 0 { - t.Errorf("mismatched child.Scopes() (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } diff --git a/internal/sidekick/api/service_dependencies_test.go b/internal/sidekick/api/service_dependencies_test.go index 9520a48f6ef..7308a854728 100644 --- a/internal/sidekick/api/service_dependencies_test.go +++ b/internal/sidekick/api/service_dependencies_test.go @@ -123,7 +123,7 @@ func TestFindServiceDependencies(t *testing.T) { got := FindServiceDependencies(model, ".test.NotFound") want := &ServiceDependencies{} if diff := cmp.Diff(want, got, cmpopts.SortSlices(less)); diff != "" { - t.Errorf("dependencies mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } got = FindServiceDependencies(model, ".test.Service1") @@ -132,7 +132,7 @@ func TestFindServiceDependencies(t *testing.T) { Enums: []string{".test.SomeEnum"}, } if diff := cmp.Diff(want, got, cmpopts.SortSlices(less)); diff != "" { - t.Errorf("dependencies mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } got = FindServiceDependencies(model, ".test.Service2") @@ -140,6 +140,6 @@ func TestFindServiceDependencies(t *testing.T) { Messages: []string{".test.Empty", ".test.OpMetadata", ".test.OpResponse"}, } if diff := cmp.Diff(want, got, cmpopts.SortSlices(less)); diff != "" { - t.Errorf("dependencies mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } diff --git a/internal/sidekick/api/skip_test.go b/internal/sidekick/api/skip_test.go index e3c480bcdfe..9de81a2d68b 100644 --- a/internal/sidekick/api/skip_test.go +++ b/internal/sidekick/api/skip_test.go @@ -46,7 +46,7 @@ func TestSkipMessages(t *testing.T) { want := []*Message{m0, m2} if diff := cmp.Diff(want, model.Messages); diff != "" { - t.Errorf("mismatch in messages (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -75,7 +75,7 @@ func TestSkipEnums(t *testing.T) { want := []*Enum{e0, e2} if diff := cmp.Diff(want, model.Enums); diff != "" { - t.Errorf("mismatch in enums (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -103,7 +103,7 @@ func TestSkipNestedMessages(t *testing.T) { }) want := []*Message{m0} if diff := cmp.Diff(want, m2.Messages); diff != "" { - t.Errorf("mismatch in messages (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -137,7 +137,7 @@ func TestSkipNestedEnums(t *testing.T) { want := []*Enum{e0, e2} if diff := cmp.Diff(want, m.Enums); diff != "" { - t.Errorf("mismatch in enums (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -166,7 +166,7 @@ func TestSkipServices(t *testing.T) { want := []*Service{s0, s2} if diff := cmp.Diff(want, model.Services, cmpopts.IgnoreFields(Service{}, "Model")); diff != "" { - t.Errorf("mismatch in services (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -208,7 +208,7 @@ func TestSkipMethods(t *testing.T) { wantServices := []*Service{s0, s1, s2} if diff := cmp.Diff(wantServices, model.Services, cmpopts.IgnoreFields(Service{}, "Model")); diff != "" { - t.Errorf("mismatch in services (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } wantMethods := []*Method{ @@ -222,7 +222,7 @@ func TestSkipMethods(t *testing.T) { }, } if diff := cmp.Diff(wantMethods, s1.Methods); diff != "" { - t.Errorf("mismatch in methods (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -270,7 +270,7 @@ func TestIncludeNestedEnums(t *testing.T) { want := []*Enum{e0} if diff := cmp.Diff(want, m.Enums, cmpopts.IgnoreFields(Message{}, "Enums")); diff != "" { - t.Errorf("mismatch in enums (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -297,7 +297,7 @@ func TestIncludeNestedMessages(t *testing.T) { }) want := []*Message{m0} if diff := cmp.Diff(want, m2.Messages, cmpopts.IgnoreFields(Message{}, "Messages")); diff != "" { - t.Errorf("mismatch in messages (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -373,7 +373,7 @@ func TestIncludeMethods(t *testing.T) { wantServices := []*Service{s1} if diff := cmp.Diff(wantServices, model.Services, cmpopts.IgnoreFields(Method{}, methodIgnoreFields...), cmpopts.IgnoreFields(Service{}, "Model", "QuickstartMethod")); diff != "" { - t.Errorf("mismatch in services (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } wantMethods := []*Method{ @@ -387,6 +387,6 @@ func TestIncludeMethods(t *testing.T) { }, } if diff := cmp.Diff(wantMethods, s1.Methods, cmpopts.IgnoreFields(Method{}, methodIgnoreFields...)); diff != "" { - t.Errorf("mismatch in methods (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } diff --git a/internal/sidekick/api/xref_test.go b/internal/sidekick/api/xref_test.go index 260f7ca14ff..14880aa7693 100644 --- a/internal/sidekick/api/xref_test.go +++ b/internal/sidekick/api/xref_test.go @@ -286,7 +286,7 @@ func TestEnrichSamplesEnumValues(t *testing.T) { got := enum.ValuesForExamples if diff := cmp.Diff(tc.wantExamples, got, cmpopts.IgnoreFields(EnumValue{}, "Parent")); diff != "" { - t.Errorf("mismatch in ValuesForExamples (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }) } @@ -413,7 +413,7 @@ func TestEnrichSamplesOneOfExampleField(t *testing.T) { got := group.ExampleField if diff := cmp.Diff(tc.want, got); diff != "" { - t.Errorf("mismatch in ExampleField (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }) } @@ -1144,7 +1144,7 @@ func TestAIPStandardGetInfo(t *testing.T) { enrichMethodSamples(tc.method) got := tc.method.SampleInfo if diff := cmp.Diff(tc.want, got); diff != "" { - t.Errorf("SampleInfo() mismatch (-want +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }) } @@ -1277,7 +1277,7 @@ func TestAIPStandardDeleteInfo(t *testing.T) { enrichMethodSamples(tc.method) got := tc.method.SampleInfo if diff := cmp.Diff(tc.want, got); diff != "" { - t.Errorf("SampleInfo() mismatch (-want +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }) } @@ -1418,7 +1418,7 @@ func TestAIPStandardUndeleteInfo(t *testing.T) { enrichMethodSamples(tc.method) got := tc.method.SampleInfo if diff := cmp.Diff(tc.want, got); diff != "" { - t.Errorf("SampleInfo() mismatch (-want +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }) } @@ -1516,7 +1516,7 @@ func TestAIPStandardCreateInfo(t *testing.T) { enrichMethodSamples(tc.method) got := tc.method.SampleInfo if diff := cmp.Diff(tc.want, got); diff != "" { - t.Errorf("SampleInfo() mismatch (-want +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }) } @@ -1642,7 +1642,7 @@ func TestAIPStandardUpdateInfo(t *testing.T) { enrichMethodSamples(tc.method) got := tc.method.SampleInfo if diff := cmp.Diff(tc.want, got); diff != "" { - t.Errorf("SampleInfo() mismatch (-want +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }) } @@ -1776,7 +1776,7 @@ func TestAIPStandardListInfo(t *testing.T) { enrichMethodSamples(tc.method) got := tc.method.SampleInfo if diff := cmp.Diff(tc.want, got); diff != "" { - t.Errorf("SampleInfo() mismatch (-want +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }) } @@ -1826,7 +1826,7 @@ func TestFindBestResourceFieldByType(t *testing.T) { msg := &Message{Fields: tc.fields} got := findBestResourceFieldByType(msg, f.model, targetType) if diff := cmp.Diff(tc.want, got); diff != "" { - t.Errorf("findBestResourceFieldByType() mismatch (-want +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }) } @@ -1881,7 +1881,7 @@ func TestFindBestResourceFieldBySingular(t *testing.T) { msg := &Message{Fields: tc.fields} got := findBestResourceFieldBySingular(msg, f.model, targetSingular) if diff := cmp.Diff(tc.want, got); diff != "" { - t.Errorf("findBestResourceFieldBySingular() mismatch (-want +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }) } @@ -1942,7 +1942,7 @@ func TestFindBestParentFieldByType(t *testing.T) { msg := &Message{Fields: tc.fields} got := findBestParentFieldByType(msg, childType) if diff := cmp.Diff(tc.want, got); diff != "" { - t.Errorf("findBestParentFieldByType() mismatch (-want +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }) } @@ -2132,7 +2132,7 @@ func TestFindBodyField(t *testing.T) { t.Run(tc.name, func(t *testing.T) { got := findBodyField(tc.message, tc.pathInfo, tc.targetType, tc.singular) if diff := cmp.Diff(tc.want, got); diff != "" { - t.Errorf("findBodyField() mismatch (-want +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }) } @@ -2166,7 +2166,7 @@ func TestFindResourceIDField(t *testing.T) { t.Run(tc.name, func(t *testing.T) { got := findResourceIDField(tc.message, tc.singular) if diff := cmp.Diff(tc.want, got); diff != "" { - t.Errorf("findResourceIDField() mismatch (-want +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }) } @@ -2237,7 +2237,7 @@ func TestFindQuickstartMethod(t *testing.T) { t.Run(tc.name, func(t *testing.T) { got := findQuickstartMethod(tc.service) if diff := cmp.Diff(tc.want, got, cmpopts.IgnoreFields(Method{}, "Service", "Model")); diff != "" { - t.Errorf("findQuickstartMethod() mismatch (-want +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }) } @@ -2285,7 +2285,7 @@ func TestFindQuickstartService(t *testing.T) { t.Run(tc.name, func(t *testing.T) { got := findQuickstartService(tc.api) if diff := cmp.Diff(tc.want, got); diff != "" { - t.Errorf("findQuickstartService() mismatch (-want +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }) } diff --git a/internal/sidekick/dart/annotate_test.go b/internal/sidekick/dart/annotate_test.go index 3402cbb6624..21b5f582c6e 100644 --- a/internal/sidekick/dart/annotate_test.go +++ b/internal/sidekick/dart/annotate_test.go @@ -15,6 +15,7 @@ package dart import ( + "fmt" "maps" "slices" "testing" @@ -50,10 +51,10 @@ func TestAnnotateModel(t *testing.T) { codec := model.Codec.(*modelAnnotations) if diff := cmp.Diff("google_cloud_test", codec.PackageName); diff != "" { - t.Errorf("mismatch in Codec.PackageName (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } if diff := cmp.Diff("test.dart", codec.MainFileNameWithExtension); diff != "" { - t.Errorf("mismatch in Codec.MainFileNameWithExtension (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -105,7 +106,7 @@ func TestAnnotateModel_FakeList(t *testing.T) { want := "FakeAccessApprovalService, FakeSecretManagerService" if diff := cmp.Diff(want, codec.FakeList); diff != "" { - t.Errorf("mismatch in Codec.FakeList (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -121,7 +122,7 @@ func TestAnnotateModel_Options(t *testing.T) { func(t *testing.T, am *annotateModel) { codec := model.Codec.(*modelAnnotations) if diff := cmp.Diff("src/buffers.dart", codec.MainFileNameWithExtension); diff != "" { - t.Errorf("mismatch in Codec.MainFileNameWithExtension (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }, }, @@ -130,7 +131,7 @@ func TestAnnotateModel_Options(t *testing.T) { func(t *testing.T, am *annotateModel) { codec := model.Codec.(*modelAnnotations) if diff := cmp.Diff("google-cloud-type", codec.PackageName); diff != "" { - t.Errorf("mismatch in Codec.PackageName (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }, }, @@ -139,7 +140,7 @@ func TestAnnotateModel_Options(t *testing.T) { func(t *testing.T, am *annotateModel) { codec := model.Codec.(*modelAnnotations) if diff := cmp.Diff([]string{"mockito", "test"}, codec.DevDependencies); diff != "" { - t.Errorf("mismatch in Codec.PackageName (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }, }, @@ -165,7 +166,7 @@ func TestAnnotateModel_Options(t *testing.T) { if diff := cmp.Diff([]string{ "export 'package:google_cloud_gax/gax.dart' show Any", "export 'package:google_cloud_gax/gax.dart' show Status"}, codec.Exports); diff != "" { - t.Errorf("mismatch in Codec.Exports (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }, }, @@ -186,7 +187,7 @@ func TestAnnotateModel_Options(t *testing.T) { func(t *testing.T, am *annotateModel) { codec := model.Codec.(*modelAnnotations) if diff := cmp.Diff("1.2.3", codec.PackageVersion); diff != "" { - t.Errorf("mismatch in Codec.PackageVersion (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }, }, @@ -195,7 +196,7 @@ func TestAnnotateModel_Options(t *testing.T) { func(t *testing.T, am *annotateModel) { codec := model.Codec.(*modelAnnotations) if diff := cmp.Diff("src/test.p.dart", codec.PartFileReference); diff != "" { - t.Errorf("mismatch in Codec.PartFileReference (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }, }, @@ -204,7 +205,7 @@ func TestAnnotateModel_Options(t *testing.T) { func(t *testing.T, am *annotateModel) { codec := model.Codec.(*modelAnnotations) if diff := cmp.Diff("> [!TIP] Still beta!", codec.ReadMeAfterTitleText); diff != "" { - t.Errorf("mismatch in Codec.ReadMeAfterTitleText (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }, }, @@ -213,7 +214,7 @@ func TestAnnotateModel_Options(t *testing.T) { func(t *testing.T, am *annotateModel) { codec := model.Codec.(*modelAnnotations) if diff := cmp.Diff("## Getting Started\n...", codec.ReadMeQuickstartText); diff != "" { - t.Errorf("mismatch in Codec.ReadMeQuickstartText (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }, }, @@ -222,7 +223,7 @@ func TestAnnotateModel_Options(t *testing.T) { func(t *testing.T, am *annotateModel) { codec := model.Codec.(*modelAnnotations) if diff := cmp.Diff("http://example.com/repo", codec.RepositoryURL); diff != "" { - t.Errorf("mismatch in Codec.RepositoryURL (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }, }, @@ -231,7 +232,7 @@ func TestAnnotateModel_Options(t *testing.T) { func(t *testing.T, am *annotateModel) { codec := model.Codec.(*modelAnnotations) if diff := cmp.Diff("http://example.com/issues", codec.IssueTrackerURL); diff != "" { - t.Errorf("mismatch in Codec.IssueTrackerURL (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }, }, @@ -243,7 +244,7 @@ func TestAnnotateModel_Options(t *testing.T) { "google_cloud_protobuf": "^7.8.9", "http": "1.2.0"}, am.dependencyConstraints); diff != "" { - t.Errorf("mismatch in annotateModel.dependencyConstraints (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }, }, @@ -628,7 +629,7 @@ func TestCalculateImports(t *testing.T) { got := calculateImports(deps, test.packageName, test.mainFileName) if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("mismatch in calculateImports (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }) } @@ -914,7 +915,7 @@ func TestBuildQueryLines_Primitives(t *testing.T) { got := annotate.buildQueryLines([]string{}, "result.", false, "", test.field) if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("mismatch in TestBuildQueryLines (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }) } @@ -979,7 +980,7 @@ func TestBuildQueryLines_Enums(t *testing.T) { t.Run(test.enumField.Name, func(t *testing.T) { got := annotate.buildQueryLines([]string{}, "result.", false, "", test.enumField) if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("mismatch in TestBuildQueryLinesEnums (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }) } @@ -1046,7 +1047,7 @@ func TestBuildQueryLines_Messages(t *testing.T) { "if (result.message1?.state case final $1? when $1.isNotDefault) 'message1.state': $1.value", } if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("mismatch in TestBuildQueryLines (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } got = annotate.buildQueryLines([]string{}, "result.", false, "", messageField2) @@ -1055,7 +1056,7 @@ func TestBuildQueryLines_Messages(t *testing.T) { "if (result.message2?.dataCrc32C case final $1?) 'message2.dataCrc32c': '${$1}'", } if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("mismatch in TestBuildQueryLines (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } // nested messages @@ -1065,7 +1066,7 @@ func TestBuildQueryLines_Messages(t *testing.T) { "if (result.message3?.fieldMask case final $1?) 'message3.fieldMask': $1.toJson()", } if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("mismatch in TestBuildQueryLines (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } // custom encoded messages @@ -1074,7 +1075,7 @@ func TestBuildQueryLines_Messages(t *testing.T) { "if (result.fieldMask case final $1?) 'fieldMask': $1.toJson()", } if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("mismatch in TestBuildQueryLines (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } got = annotate.buildQueryLines([]string{}, "result.", false, "", durationField) @@ -1082,7 +1083,7 @@ func TestBuildQueryLines_Messages(t *testing.T) { "if (result.duration case final $1?) 'duration': $1.toJson()", } if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("mismatch in TestBuildQueryLines (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } got = annotate.buildQueryLines([]string{}, "result.", false, "", timestampField) @@ -1090,7 +1091,7 @@ func TestBuildQueryLines_Messages(t *testing.T) { "if (result.time case final $1?) 'time': $1.toJson()", } if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("mismatch in TestBuildQueryLines (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -1349,7 +1350,7 @@ func TestCreateFromJsonLine(t *testing.T) { got := annotate.createFromJsonLine(test.field, codec.Required) if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("mismatch in TestBuildQueryLines (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }) } @@ -1721,7 +1722,7 @@ func TestToJson(t *testing.T) { annotate.annotateField(test.field) got := test.field.Codec.(*fieldAnnotation).ToJson if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("mismatch in TestToJson (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }) } @@ -1796,25 +1797,30 @@ func TestAnnotateEnum(t *testing.T) { wantValueAnnotations: []wantedValueAnnotation{{"NAME"}, {"name"}}, }, } { - annotate.annotateEnum(test.enum) - codec := test.enum.Codec.(*enumAnnotation) - gotEnumName := codec.Name - gotEnumDefaultValue := codec.DefaultValue - - if diff := cmp.Diff(test.wantEnumName, gotEnumName); diff != "" { - t.Errorf("mismatch in TestAnnotateEnum(%q) (-want, +got)\n:%s", test.enum.Name, diff) - } - if diff := cmp.Diff(test.wantEnumDefaultValue, gotEnumDefaultValue); diff != "" { - t.Errorf("mismatch in TestAnnotateEnum(%q) (-want, +got)\n:%s", test.enum.Name, diff) - } + t.Run(test.wantEnumName, func(t *testing.T) { + annotate.annotateEnum(test.enum) + codec := test.enum.Codec.(*enumAnnotation) + gotEnumName := codec.Name + gotEnumDefaultValue := codec.DefaultValue + + if diff := cmp.Diff(test.wantEnumName, gotEnumName); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(test.wantEnumDefaultValue, gotEnumDefaultValue); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } - for i, value := range test.enum.Values { - wantValueAnnotation := test.wantValueAnnotations[i] - gotValueAnnotation := value.Codec.(*enumValueAnnotation) - if diff := cmp.Diff(wantValueAnnotation.wantValueName, gotValueAnnotation.Name); diff != "" { - t.Errorf("mismatch in TestAnnotateEnum(%q) [value annotation %d] (-want, +got)\n:%s", test.enum.Name, i, diff) + for i, value := range test.enum.Values { + testName := fmt.Sprintf("TestAnnotateEnum(%q) [value annotation %d]", test.enum.Name, i) + t.Run(testName, func(t *testing.T) { + wantValueAnnotation := test.wantValueAnnotations[i] + gotValueAnnotation := value.Codec.(*enumValueAnnotation) + if diff := cmp.Diff(wantValueAnnotation.wantValueName, gotValueAnnotation.Name); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + }) } - } + }) } } @@ -2069,7 +2075,7 @@ func TestAnnotateField(t *testing.T) { got.ToJson = "" if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("mismatch in TestAnnotateField(-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }) } diff --git a/internal/sidekick/dart/dart_test.go b/internal/sidekick/dart/dart_test.go index 9053f4290d1..1e4f4bd88b9 100644 --- a/internal/sidekick/dart/dart_test.go +++ b/internal/sidekick/dart/dart_test.go @@ -510,7 +510,7 @@ We want to respect whitespace at the beginning, because it important in Markdown } got := formatDocComments(input, nil) if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("mismatch in FormatDocComments (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -520,13 +520,13 @@ func TestFormatDocCommentsEmpty(t *testing.T) { want := []string{} got := formatDocComments(input, nil) if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("mismatch in FormatDocComments (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } func TestFormatDocCommentsTrimTrailingSpaces(t *testing.T) { input := `The next line contains spaces. - + This line has trailing spaces. ` want := []string{ @@ -536,7 +536,7 @@ This line has trailing spaces. ` } got := formatDocComments(input, nil) if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("mismatch in FormatDocComments (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -554,7 +554,7 @@ Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris. } got := formatDocComments(input, nil) if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("mismatch in FormatDocComments (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -603,7 +603,7 @@ is set to true, this field will contain the sentiment for the sentence.`, gotLines := formatDocComments(test.input, nil) got := strings.Join(gotLines, "\n") if diff := cmp.Diff(test.output, got); diff != "" { - t.Errorf("mismatch in FormatDocComments (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }) } diff --git a/internal/sidekick/language/codec_test.go b/internal/sidekick/language/codec_test.go index 05e763ffc2e..2abd60270de 100644 --- a/internal/sidekick/language/codec_test.go +++ b/internal/sidekick/language/codec_test.go @@ -64,7 +64,7 @@ func TestQueryParams(t *testing.T) { want := []*api.Field{field1, field2} less := func(a, b *api.Field) bool { return a.Name < b.Name } if diff := cmp.Diff(want, got, cmpopts.SortSlices(less)); diff != "" { - t.Errorf("mismatched query parameters (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -83,7 +83,7 @@ func TestPathParams(t *testing.T) { } want := []*api.Field{sample.CreateRequest().Fields[0]} if diff := cmp.Diff(want, got, cmpopts.SortSlices(less)); diff != "" { - t.Errorf("mismatched query parameters (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } got, err = PathParams(sample.MethodUpdate(), test) @@ -92,7 +92,7 @@ func TestPathParams(t *testing.T) { } want = []*api.Field{sample.UpdateRequest().Fields[0]} if diff := cmp.Diff(want, got, cmpopts.SortSlices(less)); diff != "" { - t.Errorf("mismatched query parameters (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -100,7 +100,7 @@ func TestFilterSlice(t *testing.T) { got := FilterSlice([]string{"a.1", "b.1", "a.2", "b.2"}, func(s string) bool { return strings.HasPrefix(s, "a.") }) want := []string{"a.1", "a.2"} if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("mismatched FilterSlice result (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -108,7 +108,7 @@ func TestMapSlice(t *testing.T) { got := MapSlice([]string{"a", "aa", "aaa"}, func(s string) int { return len(s) }) want := []int{1, 2, 3} if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("mismatched FilterSlice result (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } diff --git a/internal/sidekick/language/walk_templates_dir_test.go b/internal/sidekick/language/walk_templates_dir_test.go index 521ea6ca53f..f4c87f658a4 100644 --- a/internal/sidekick/language/walk_templates_dir_test.go +++ b/internal/sidekick/language/walk_templates_dir_test.go @@ -35,6 +35,6 @@ func TestWalkDir(t *testing.T) { }, } if diff := cmp.Diff(want, got); len(diff) != 0 { - t.Errorf("mismatched config from LoadConfig (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } diff --git a/internal/sidekick/parser/disco_test.go b/internal/sidekick/parser/disco_test.go index a77c00df1a0..7117def8ccb 100644 --- a/internal/sidekick/parser/disco_test.go +++ b/internal/sidekick/parser/disco_test.go @@ -125,7 +125,7 @@ func TestDisco_ParsePagination(t *testing.T) { Optional: true, } if diff := cmp.Diff(wantPagination, got.Pagination, cmpopts.IgnoreFields(api.Field{}, "Documentation")); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -152,7 +152,7 @@ func TestDisco_ParsePaginationAggregate(t *testing.T) { Optional: true, } if diff := cmp.Diff(wantPagination, got.Pagination, cmpopts.IgnoreFields(api.Field{}, "Documentation")); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } diff --git a/internal/sidekick/parser/discovery/discovery_test.go b/internal/sidekick/parser/discovery/discovery_test.go index a5b468a4cac..5d6d18eb98e 100644 --- a/internal/sidekick/parser/discovery/discovery_test.go +++ b/internal/sidekick/parser/discovery/discovery_test.go @@ -55,7 +55,7 @@ func TestInfo(t *testing.T) { Revision: "20250810", } if diff := cmp.Diff(want, got, cmpopts.IgnoreFields(api.API{}, "Services", "Messages", "Enums"), cmpopts.IgnoreUnexported(api.API{})); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -77,7 +77,7 @@ func TestServiceConfigOverridesInfo(t *testing.T) { PackageName: "google.cloud.secretmanager.v1", } if diff := cmp.Diff(want, got, cmpopts.IgnoreFields(api.API{}, "Services", "Messages", "Enums"), cmpopts.IgnoreUnexported(api.API{})); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } if len(sc.Apis) != 2 { t.Fatalf("expected 2 APIs in service config") @@ -192,7 +192,7 @@ func TestDeprecatedField(t *testing.T) { Optional: true, } if diff := cmp.Diff(wantField, gotField, cmpopts.IgnoreFields(api.Field{}, "Parent")); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } diff --git a/internal/sidekick/parser/discovery/enum_field_test.go b/internal/sidekick/parser/discovery/enum_field_test.go index d295efd7cee..e03fde5c7e4 100644 --- a/internal/sidekick/parser/discovery/enum_field_test.go +++ b/internal/sidekick/parser/discovery/enum_field_test.go @@ -117,7 +117,7 @@ func TestMakeEnumFields(t *testing.T) { apitest.CheckMessage(t, message, want) wantEnums := []*api.Enum{wantEnum} if diff := cmp.Diff(wantEnums, message.Enums, cmpopts.IgnoreFields(api.Enum{}, "Parent"), cmpopts.IgnoreFields(api.EnumValue{}, "Parent")); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -202,7 +202,7 @@ func TestMakeEnumFieldsDeprecated(t *testing.T) { apitest.CheckMessage(t, message, want) wantEnums := []*api.Enum{wantEnum} if diff := cmp.Diff(wantEnums, message.Enums, cmpopts.IgnoreFields(api.Enum{}, "Parent"), cmpopts.IgnoreFields(api.EnumValue{}, "Parent")); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -308,7 +308,7 @@ func TestMakeEnumFieldsWithDeprecatedValues(t *testing.T) { apitest.CheckMessage(t, message, want) wantEnums := []*api.Enum{wantEnum} if diff := cmp.Diff(wantEnums, message.Enums, cmpopts.IgnoreFields(api.Enum{}, "Parent"), cmpopts.IgnoreFields(api.EnumValue{}, "Parent")); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } diff --git a/internal/sidekick/parser/discovery/inline_object_field_test.go b/internal/sidekick/parser/discovery/inline_object_field_test.go index a1a04c43ef8..7b999dd24d0 100644 --- a/internal/sidekick/parser/discovery/inline_object_field_test.go +++ b/internal/sidekick/parser/discovery/inline_object_field_test.go @@ -64,7 +64,7 @@ func TestMaybeInlineObject(t *testing.T) { TypezID: ".package.Message.inline", } if diff := cmp.Diff(wantField, field); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } wantInlineMessage := &api.Message{ @@ -149,7 +149,7 @@ func TestArrayWithInlineObject(t *testing.T) { TypezID: ".package.Message.arrayWithObject", } if diff := cmp.Diff(wantField, field); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } wantInlineMessage := &api.Message{ diff --git a/internal/sidekick/parser/discovery/lro_test.go b/internal/sidekick/parser/discovery/lro_test.go index 10fc7686f9e..875b45523b5 100644 --- a/internal/sidekick/parser/discovery/lro_test.go +++ b/internal/sidekick/parser/discovery/lro_test.go @@ -71,7 +71,7 @@ func TestLroAnnotations(t *testing.T) { t.Fatalf("missing method %s in model", want.ID) } if diff := cmp.Diff(want, got, cmpopts.IgnoreFields(api.Method{}, "Documentation")); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } // The parser should have injected a mixin method. @@ -105,7 +105,7 @@ func TestLroAnnotations(t *testing.T) { t.Fatalf("missing method %s in model", wantMixin.ID) } if diff := cmp.Diff(wantMixin, gotMixin, cmpopts.IgnoreFields(api.Method{}, "Documentation", "Service")); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } diff --git a/internal/sidekick/parser/discovery/map_field_test.go b/internal/sidekick/parser/discovery/map_field_test.go index 6ff19aa4e06..9e9301da89b 100644 --- a/internal/sidekick/parser/discovery/map_field_test.go +++ b/internal/sidekick/parser/discovery/map_field_test.go @@ -322,7 +322,7 @@ func TestMapScalarTypes(t *testing.T) { }, } if diff := cmp.Diff(wantFields, message.Fields, cmpopts.IgnoreFields(api.Field{}, "TypezID")); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) continue } mapMessage := model.Message(message.Fields[0].TypezID) @@ -342,7 +342,7 @@ func TestMapScalarTypes(t *testing.T) { TypezID: test.WantTypeID, } if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("mismatch on value field (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } } diff --git a/internal/sidekick/parser/discovery/methods_test.go b/internal/sidekick/parser/discovery/methods_test.go index fb2541b3011..0fad5d1bc9a 100644 --- a/internal/sidekick/parser/discovery/methods_test.go +++ b/internal/sidekick/parser/discovery/methods_test.go @@ -138,7 +138,7 @@ func TestMakeServiceMethodsReturnsEmpty(t *testing.T) { Signatures: []*api.MethodSignature{{Names: []string{"project", "zone", "operation"}}}, } if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -177,7 +177,7 @@ func TestMakeServiceMethodsDeprecated(t *testing.T) { Signatures: []*api.MethodSignature{{Names: []string{"project", "body"}}}, } if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } diff --git a/internal/sidekick/parser/discovery/uritemplate_test.go b/internal/sidekick/parser/discovery/uritemplate_test.go index f1b1662ec9b..44c21e7e961 100644 --- a/internal/sidekick/parser/discovery/uritemplate_test.go +++ b/internal/sidekick/parser/discovery/uritemplate_test.go @@ -53,14 +53,15 @@ func TestParseUriTemplateSuccess(t *testing.T) { WithVariable(api.NewPathVariable("resource").WithAllowReserved().WithMatchRecursive()). WithVerb("getIamPolicy")}, } { - got, err := ParseUriTemplate(test.input) - if err != nil { - t.Errorf("expected a successful parse with input=%s, err=%v", test.input, err) - continue - } - if diff := cmp.Diff(test.want, got, cmpopts.EquateEmpty()); diff != "" { - t.Errorf("mismatch [%s] (-want, +got):\n%s", test.input, diff) - } + t.Run(test.input, func(t *testing.T) { + got, err := ParseUriTemplate(test.input) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(test.want, got, cmpopts.EquateEmpty()); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + }) } } @@ -78,9 +79,11 @@ func TestParseUriTemplateError(t *testing.T) { {"dns/v1/{+resource}:verb/should/not/have/slashes"}, {"dns/v1/{+emptyVerb}:"}, } { - if got, err := ParseUriTemplate(test.input); err == nil { - t.Errorf("expected a parsing error with input=%s, got=%v", test.input, got) - } + t.Run(test.input, func(t *testing.T) { + if got, err := ParseUriTemplate(test.input); err == nil { + t.Fatalf("expected error, got=%v", got) + } + }) } } @@ -95,17 +98,18 @@ func TestParseExpression(t *testing.T) { {"{abc_012}", "abc_012"}, {"{abc_012}/foo/{bar}", "abc_012"}, } { - gotSegment, gotWidth, err := parseExpression(test.input) - if err != nil { - t.Errorf("expected a successful parse with input=%s, err=%v", test.input, err) - continue - } - if diff := cmp.Diff(&api.PathSegment{Variable: api.NewPathVariable(test.want).WithMatch()}, gotSegment); diff != "" { - t.Errorf("mismatch [%s] (-want, +got):\n%s", test.input, diff) - } - if len(test.want)+2 != gotWidth { - t.Errorf("mismatch want=%d, got=%d", len(test.want), gotWidth) - } + t.Run(test.input, func(t *testing.T) { + gotSegment, gotWidth, err := parseExpression(test.input) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(&api.PathSegment{Variable: api.NewPathVariable(test.want).WithMatch()}, gotSegment); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + if len(test.want)+2 != gotWidth { + t.Errorf("mismatch want=%d, got=%d", len(test.want), gotWidth) + } + }) } } @@ -136,17 +140,18 @@ func TestParseLiteral(t *testing.T) { {"abcde/f", "abcde"}, {"abcdef", "abcdef"}, } { - gotSegment, gotWidth, err := parseLiteral(test.input) - if err != nil { - t.Errorf("expected a successful parse with input=%s, err=%v", test.input, err) - continue - } - if diff := cmp.Diff(&api.PathSegment{Literal: test.want}, gotSegment); diff != "" { - t.Errorf("mismatch [%s] (-want, +got):\n%s", test.input, diff) - } - if len(test.want) != gotWidth { - t.Errorf("mismatch want=%d, got=%d", len(test.want), gotWidth) - } + t.Run(test.input, func(t *testing.T) { + gotSegment, gotWidth, err := parseLiteral(test.input) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(&api.PathSegment{Literal: test.want}, gotSegment); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + if len(test.want) != gotWidth { + t.Errorf("mismatch want=%d, got=%d", len(test.want), gotWidth) + } + }) } } diff --git a/internal/sidekick/parser/mixin_test.go b/internal/sidekick/parser/mixin_test.go index 53a14b855d1..d5f6641add0 100644 --- a/internal/sidekick/parser/mixin_test.go +++ b/internal/sidekick/parser/mixin_test.go @@ -47,7 +47,7 @@ func TestProtobuf_ForceLongrunning(t *testing.T) { } gotMethods, gotDescriptors := loadMixins(sc, true) if diff := cmp.Diff(wantMethods, gotMethods); diff != "" { - t.Errorf("mismatched operations (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } names := map[string]bool{} @@ -68,7 +68,7 @@ func TestProtobuf_ForceLongrunningNoRules(t *testing.T) { } gotMethods, gotDescriptors := loadMixins(sc, true) if diff := cmp.Diff(wantMethods, gotMethods); diff != "" { - t.Errorf("mismatched operations (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } names := map[string]bool{} diff --git a/internal/sidekick/parser/openapi_test.go b/internal/sidekick/parser/openapi_test.go index 2ed8a9ca2ea..a811d89459c 100644 --- a/internal/sidekick/parser/openapi_test.go +++ b/internal/sidekick/parser/openapi_test.go @@ -717,7 +717,7 @@ func TestOpenAPI_MakeAPI(t *testing.T) { wantService.Name = "Service" wantService.ID = "..Service" if diff := cmp.Diff(wantService, service, cmpopts.IgnoreFields(api.Service{}, "Methods")); diff != "" { - t.Errorf("mismatched service attributes (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } apitest.CheckMethod(t, service, "ListLocations", &api.Method{ @@ -1156,7 +1156,7 @@ func TestOpenAPI_AutoPopulated(t *testing.T) { } wantField := []*api.Field{request_id, request_id_explicit} if diff := cmp.Diff(wantField, method.AutoPopulated); diff != "" { - t.Errorf("incorrect auto-populated fields on method (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } diff --git a/internal/sidekick/parser/protobuf_imports_test.go b/internal/sidekick/parser/protobuf_imports_test.go index 431c599fd7d..c58cc151bb4 100644 --- a/internal/sidekick/parser/protobuf_imports_test.go +++ b/internal/sidekick/parser/protobuf_imports_test.go @@ -33,7 +33,7 @@ func TestProtobufImportsSync(t *testing.T) { sort.Strings(g3Names) if diff := cmp.Diff(ossNames, g3Names); diff != "" { - t.Errorf("protobuf_imports_oss.go and protobuf_imports_google3.go are out of sync (-oss +g3):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } diff --git a/internal/sidekick/parser/protobuf_test.go b/internal/sidekick/parser/protobuf_test.go index 6bd83a01fa9..0b064a76d14 100644 --- a/internal/sidekick/parser/protobuf_test.go +++ b/internal/sidekick/parser/protobuf_test.go @@ -548,10 +548,10 @@ func TestProtobuf_UniqueEnumValues(t *testing.T) { less := func(a, b *api.EnumValue) bool { return a.Name < b.Name } if diff := cmp.Diff(fullList, withAlias.Values, cmpopts.SortSlices(less), cmpopts.IgnoreFields(api.EnumValue{}, "Parent")); diff != "" { - t.Errorf("enum values mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } if diff := cmp.Diff(uniqueList, withAlias.UniqueNumberValues, cmpopts.SortSlices(less), cmpopts.IgnoreFields(api.EnumValue{}, "Parent")); diff != "" { - t.Errorf("enum values mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -1132,7 +1132,7 @@ func TestProtobuf_TrimLeadingSpacesInDocumentation(t *testing.T) { got := trimLeadingSpacesInDocumentation(input) if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("mismatch in trimLeadingSpacesInDocumentation (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -1755,7 +1755,7 @@ func TestProtobuf_AutoPopulated(t *testing.T) { } want := []*api.Field{request_id, request_id_optional, request_id_with_field_behavior} if diff := cmp.Diff(want, method.AutoPopulated); diff != "" { - t.Errorf("incorrect auto-populated fields on method (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -1919,7 +1919,7 @@ func TestProtobuf_ResourceAnnotations(t *testing.T) { t.Fatalf("Expected ResourceDefinition for 'library.googleapis.com/Shelf' not found") } if diff := cmp.Diff(shelfResourceDef, foundShelf, cmpopts.IgnoreFields(api.Resource{}, "Self", "Codec", "Plural", "Singular")); diff != "" { - t.Errorf("ResourceDefinition (Shelf) mismatch (-want +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } // Verify Book @@ -1952,7 +1952,7 @@ func TestProtobuf_ResourceAnnotations(t *testing.T) { // Note: Book resource has 'Self' populated because it's a message resource. // Ignoring Self/Codec for comparison. if diff := cmp.Diff(bookResourceDef, foundBook, cmpopts.IgnoreFields(api.Resource{}, "Self", "Codec")); diff != "" { - t.Errorf("ResourceDefinition (Book) mismatch (-want +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }) @@ -1997,7 +1997,7 @@ func TestProtobuf_ResourceAnnotations(t *testing.T) { } if diff := cmp.Diff(wantBookResource, bookMessage.Resource, cmpopts.IgnoreFields(api.Resource{}, "Self", "Codec")); diff != "" { - t.Errorf("Book message Resource mismatch (-want +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } apitest.CheckMessage(t, bookMessage, &api.Message{ diff --git a/internal/sidekick/parser/routing_info_test.go b/internal/sidekick/parser/routing_info_test.go index eb5a3d4037b..81c8b787aa7 100644 --- a/internal/sidekick/parser/routing_info_test.go +++ b/internal/sidekick/parser/routing_info_test.go @@ -252,7 +252,7 @@ func TestExamples(t *testing.T) { t.Fatalf("Cannot find method %s in API State", test.methodID) } if diff := cmp.Diff(test.want, got.Routing); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }) } @@ -531,7 +531,7 @@ func TestParseRoutingPathSpecSuccess(t *testing.T) { t.Run(test.path, func(t *testing.T) { got, width := parseRoutingPathSpec(test.path) if diff := cmp.Diff(test.wantSegments, got.Segments); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s\n", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } if test.path[width:] != test.wantTrailer { t.Errorf("trailer segment mismatch, want=%s, got=%s", test.wantTrailer, test.path[width:]) diff --git a/internal/sidekick/protobuf/input_files_test.go b/internal/sidekick/protobuf/input_files_test.go index b019d1012b4..d77a83542f6 100644 --- a/internal/sidekick/protobuf/input_files_test.go +++ b/internal/sidekick/protobuf/input_files_test.go @@ -51,7 +51,7 @@ func TestBasic(t *testing.T) { filepath.ToSlash(path.Join(testdataDir, source, "service.proto")), } if diff := cmp.Diff(want, got); len(diff) != 0 { - t.Errorf("mismatched merged config (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -75,6 +75,6 @@ func TestIncludeList(t *testing.T) { filepath.ToSlash(path.Join(testdataDir, source, "resources.proto")), } if diff := cmp.Diff(want, got); len(diff) != 0 { - t.Errorf("mismatched merged config (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } diff --git a/internal/sidekick/rust/bigquery_test.go b/internal/sidekick/rust/bigquery_test.go index d077b733a5d..8744471b9dc 100644 --- a/internal/sidekick/rust/bigquery_test.go +++ b/internal/sidekick/rust/bigquery_test.go @@ -111,7 +111,7 @@ func TestBigQueryFiltering(t *testing.T) { // "output_only" and "skip" must be skipped; "foo" must be present. want := []string{"foo"} if diff := cmp.Diff(want, fieldNames); diff != "" { - t.Errorf("unexpected field list (-want +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } diff --git a/internal/sidekick/rust_prost/annotate_test.go b/internal/sidekick/rust_prost/annotate_test.go index 5dc25db294b..d42fc83c8d6 100644 --- a/internal/sidekick/rust_prost/annotate_test.go +++ b/internal/sidekick/rust_prost/annotate_test.go @@ -52,7 +52,7 @@ func TestModelAnnotations(t *testing.T) { }, } if diff := cmp.Diff(want, model.Codec, cmpopts.IgnoreFields(modelAnnotations{}, "BoilerPlate")); diff != "" { - t.Errorf("mismatch in model annotations (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -88,7 +88,7 @@ func TestServiceAnnotations(t *testing.T) { t.Fatalf("cannot find service %s", ".google.cloud.workflows.v1.Workflows") } if diff := cmp.Diff(want, got.Codec); diff != "" { - t.Errorf("mismatch in service annotations (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -130,6 +130,6 @@ func TestMethodAnnotations(t *testing.T) { t.Fatalf("cannot find service %s", ".google.cloud.workflows.v1.Workflows.GetWorkflow") } if diff := cmp.Diff(want, got.Codec); diff != "" { - t.Errorf("mismatch in method annotations (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } diff --git a/internal/sidekick/rust_prost/codec_test.go b/internal/sidekick/rust_prost/codec_test.go index 595f29bf7a6..c5c052533ca 100644 --- a/internal/sidekick/rust_prost/codec_test.go +++ b/internal/sidekick/rust_prost/codec_test.go @@ -43,6 +43,6 @@ func TestParseOptions(t *testing.T) { RootName: "test-root", } if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("mismatch in codec (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } diff --git a/internal/sidekick/swift/annotate_enum_value_test.go b/internal/sidekick/swift/annotate_enum_value_test.go index 656824bc555..bfbc54aff6a 100644 --- a/internal/sidekick/swift/annotate_enum_value_test.go +++ b/internal/sidekick/swift/annotate_enum_value_test.go @@ -119,6 +119,6 @@ func TestAnnotateEnumValue_Aliases(t *testing.T) { }, } if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("mismatch in Values (-want +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } diff --git a/internal/sidekick/swift/annotate_message_test.go b/internal/sidekick/swift/annotate_message_test.go index 0f6e9d6862d..e0940f34c4c 100644 --- a/internal/sidekick/swift/annotate_message_test.go +++ b/internal/sidekick/swift/annotate_message_test.go @@ -159,10 +159,10 @@ func TestAnnotateMessage(t *testing.T) { t.Fatal(err) } if diff := cmp.Diff(test.want, test.message.Codec, cmpopts.IgnoreFields(messageAnnotations{}, "Model", "DependsOn")); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } if diff := cmp.Diff(test.wantImports, test.message.Codec.(*messageAnnotations).MessageImports()); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }) } @@ -294,7 +294,7 @@ func TestAnnotateMessage_Discovery(t *testing.T) { t.Fatal(err) } if diff := cmp.Diff(test.want, test.message.Codec, cmpopts.IgnoreFields(messageAnnotations{}, "Model", "DependsOn")); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }) } @@ -349,7 +349,7 @@ func TestAnnotateMessage_DiscoveryRequests(t *testing.T) { t.Fatal(err) } if diff := cmp.Diff(test.want, test.request.Codec, cmpopts.IgnoreFields(messageAnnotations{}, "Model", "DependsOn")); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }) } @@ -427,11 +427,11 @@ func TestAnnotateMessage_Pagination(t *testing.T) { ParameterTypeName: "ListSecretsRequest", } if diff := cmp.Diff(wantRequest, gotRequest, cmpopts.IgnoreFields(messageAnnotations{}, "Model", "DependsOn")); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } wantRequestImports := []string{"GoogleCloudWkt"} if diff := cmp.Diff(wantRequestImports, gotRequest.MessageImports()); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } // Verify annotations on response message @@ -446,11 +446,11 @@ func TestAnnotateMessage_Pagination(t *testing.T) { ParameterTypeName: "ListSecretsResponse", } if diff := cmp.Diff(wantResponse, gotResponse, cmpopts.IgnoreFields(messageAnnotations{}, "Model", "DependsOn")); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } wantResponseImports := []string{"GoogleCloudGax", "GoogleCloudWkt"} if diff := cmp.Diff(wantResponseImports, gotResponse.MessageImports()); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -499,12 +499,12 @@ func TestAnnotateMessage_RecursiveNested(t *testing.T) { ParameterTypeName: "OuterMessage", } if diff := cmp.Diff(wantOuter, gotOuter, cmpopts.IgnoreFields(messageAnnotations{}, "Model", "DependsOn")); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } wantImports := []string{"GoogleCloudGax", "GoogleCloudWkt"} if diff := cmp.Diff(wantImports, gotOuter.MessageImports()); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } diff --git a/internal/sidekick/swift/annotate_method_test.go b/internal/sidekick/swift/annotate_method_test.go index fc4b6c8de6e..36fd9879fca 100644 --- a/internal/sidekick/swift/annotate_method_test.go +++ b/internal/sidekick/swift/annotate_method_test.go @@ -364,7 +364,7 @@ func TestAnnotateMethod_Pagination(t *testing.T) { ReturnType: "GoogleTest.ListResponse", } if diff := cmp.Diff(wantMethod, gotMethod); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } if gotMethod.PlainRPC() { t.Errorf("gotMethod.PlainRPC() == false, want true\ngotMethod=%+v", gotMethod) @@ -379,11 +379,11 @@ func TestAnnotateMethod_Pagination(t *testing.T) { ParameterTypeName: "ListRequest", } if diff := cmp.Diff(wantRequest, gotRequest, cmpopts.IgnoreFields(messageAnnotations{}, "Model", "DependsOn")); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } wantRequestImports := []string{"GoogleCloudWkt"} if diff := cmp.Diff(wantRequestImports, gotRequest.MessageImports()); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } // Verify response message annotations @@ -398,12 +398,12 @@ func TestAnnotateMethod_Pagination(t *testing.T) { ParameterTypeName: "ListResponse", } if diff := cmp.Diff(wantResponse, gotResponse, cmpopts.IgnoreFields(messageAnnotations{}, "Model", "DependsOn")); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } // Response type is a paginated response which depends on gax wantResponseImports := []string{"GoogleCloudGax", "GoogleCloudWkt"} if diff := cmp.Diff(wantResponseImports, gotResponse.MessageImports()); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -547,7 +547,7 @@ func TestAnnotateMethod_LRO_Empty(t *testing.T) { ReturnType: "GoogleTest.Operation", } if diff := cmp.Diff(wantMethod, gotMethod); diff != "" { - t.Errorf("mismatch (-want, +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } if gotMethod.PlainRPC() { t.Errorf("gotMethod.PlainRPC() == false, want true\ngotMethod=%+v", gotMethod) diff --git a/internal/sidekick/swift/codec_test.go b/internal/sidekick/swift/codec_test.go index e371009f605..9cdfff2d65e 100644 --- a/internal/sidekick/swift/codec_test.go +++ b/internal/sidekick/swift/codec_test.go @@ -78,7 +78,7 @@ func TestParseOptions(t *testing.T) { t.Fatal(err) } if diff := cmp.Diff(test.want, got, cmpopts.IgnoreUnexported(api.API{})); diff != "" { - t.Errorf("mismatch (-want, +got)\n:%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }) } @@ -105,14 +105,14 @@ func TestNewCodec_WithSwiftCfg(t *testing.T) { {SwiftDependency: swiftCfg.Dependencies[1]}, } if diff := cmp.Diff(wantDeps, got.Dependencies); diff != "" { - t.Errorf("mismatch in Dependencies (-want +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } wantApiPackages := map[string]*Dependency{ "google.cloud.location": {SwiftDependency: swiftCfg.Dependencies[1]}, } if diff := cmp.Diff(wantApiPackages, got.ApiPackages); diff != "" { - t.Errorf("mismatch in ApiPackages (-want +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } diff --git a/internal/sidekick/swift/format_documentation_test.go b/internal/sidekick/swift/format_documentation_test.go index 291156d6b76..6ad17872a54 100644 --- a/internal/sidekick/swift/format_documentation_test.go +++ b/internal/sidekick/swift/format_documentation_test.go @@ -56,7 +56,7 @@ func TestFormatDocumentation(t *testing.T) { t.Fatal(err) } if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("formatDocumentation() mismatch (-want +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }) } @@ -83,6 +83,6 @@ func TestFormatDocumentationWithLinks(t *testing.T) { t.Fatal(err) } if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("formatDocumentation() mismatch (-want +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } diff --git a/internal/sidekick/swift/format_path_test.go b/internal/sidekick/swift/format_path_test.go index 9e18e14fd6c..30acd80756b 100644 --- a/internal/sidekick/swift/format_path_test.go +++ b/internal/sidekick/swift/format_path_test.go @@ -152,7 +152,7 @@ func TestPathVariables(t *testing.T) { return } if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("pathVariables() mismatch (-want +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }) } @@ -318,7 +318,7 @@ func TestNewPathVariable(t *testing.T) { return } if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("newPathVariable() mismatch (-want +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } }) } diff --git a/internal/sidekick/swift/generate_package_swift_test.go b/internal/sidekick/swift/generate_package_swift_test.go index ee31cccbeaa..50f9c842485 100644 --- a/internal/sidekick/swift/generate_package_swift_test.go +++ b/internal/sidekick/swift/generate_package_swift_test.go @@ -73,7 +73,7 @@ func TestGeneratePackageSwift_WithDependencies(t *testing.T) { .package(path: "../../packages/wkt"), ],` if diff := cmp.Diff(wantPackageDeps, gotPackageDeps); diff != "" { - t.Errorf("mismatch in package dependencies (-want +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } gotTargetDeps := extractBlock(t, contentStr, " dependencies: [", "\n ]") @@ -83,7 +83,7 @@ func TestGeneratePackageSwift_WithDependencies(t *testing.T) { .product(name: "wkt", package: "wkt"), ]` if diff := cmp.Diff(wantTargetDeps, gotTargetDeps); diff != "" { - t.Errorf("mismatch in target dependencies (-want +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } diff --git a/internal/sidekick/swift/lookup_test.go b/internal/sidekick/swift/lookup_test.go index 56c10d84f10..480734af808 100644 --- a/internal/sidekick/swift/lookup_test.go +++ b/internal/sidekick/swift/lookup_test.go @@ -30,7 +30,7 @@ func TestLookupMessage(t *testing.T) { t.Fatal(err) } if diff := cmp.Diff(msg, got); diff != "" { - t.Errorf("lookupMessage() mismatch (-want +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -52,7 +52,7 @@ func TestLookupEnum(t *testing.T) { t.Fatal(err) } if diff := cmp.Diff(enum, got); diff != "" { - t.Errorf("lookupEnum() mismatch (-want +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } @@ -78,7 +78,7 @@ func TestLookupField(t *testing.T) { t.Fatal(err) } if diff := cmp.Diff(field, got); diff != "" { - t.Errorf("lookupField() mismatch (-want +got):\n%s", diff) + t.Errorf("mismatch (-want +got):\n%s", diff) } } From e20f77a70df502836fedda0f302fda66b09f1452 Mon Sep 17 00:00:00 2001 From: sofisl <55454395+sofisl@users.noreply.github.com> Date: Tue, 30 Jun 2026 09:26:19 -0700 Subject: [PATCH 094/108] fix(internal/serviceconfig): normalize transport name for Java repo-metadata (#6582) Remove the Java-specific mapping in RepoMetadataTransport that translated transport values to "http" or "both". Now it returns the standard values ("grpc", "rest", "grpc+rest") directly, aligning Java with other languages. After merging, we will merge https://github.com/googleapis/google-cloud-java/pull/13586, once we update the librarian pseudo-version and regenerate the repo-metadata values. Fixes https://github.com/googleapis/librarian/issues/4854 --- internal/librarian/java/repometadata_test.go | 6 +++--- internal/serviceconfig/api.go | 18 ++---------------- internal/serviceconfig/api_test.go | 10 ++++++++-- 3 files changed, 13 insertions(+), 21 deletions(-) diff --git a/internal/librarian/java/repometadata_test.go b/internal/librarian/java/repometadata_test.go index c1f42a8ac91..4eb4a1f2e68 100644 --- a/internal/librarian/java/repometadata_test.go +++ b/internal/librarian/java/repometadata_test.go @@ -112,7 +112,7 @@ func TestDeriveRepoMetadata_Overrides(t *testing.T) { APIDescription: "Custom description", ClientDocumentation: "https://custom.client.docs", ReleaseLevel: s.ReleaseLevel, - Transport: "both", + Transport: "grpc+rest", Language: cfg.Language, Repo: cfg.Repo, RepoShort: "java-secretmanager", @@ -137,7 +137,7 @@ func TestDeriveRepoMetadata_Overrides(t *testing.T) { APIDescription: wantAPIDescription, ClientDocumentation: "https://cloud.google.com/java/docs/reference/google-cloud-secretmanager/latest/overview", ReleaseLevel: "stable", - Transport: "both", + Transport: "grpc+rest", Language: "java", Repo: "googleapis/google-cloud-java", RepoShort: "java-secretmanager", @@ -162,7 +162,7 @@ func TestDeriveRepoMetadata_Overrides(t *testing.T) { APIDescription: wantAPIDescription, ClientDocumentation: "https://cloud.google.com/java/docs/reference/google-cloud-secretmanager/latest/overview", ReleaseLevel: "stable", - Transport: "http", + Transport: "rest", Language: "java", Repo: "googleapis/google-cloud-java", RepoShort: "java-secretmanager", diff --git a/internal/serviceconfig/api.go b/internal/serviceconfig/api.go index 16878f488a7..4302e653df2 100644 --- a/internal/serviceconfig/api.go +++ b/internal/serviceconfig/api.go @@ -167,24 +167,10 @@ func (api *API) ReleaseLevel(language, version string) string { } // RepoMetadataTransport returns the transport for repo metadata. -// -// TODO(https://github.com/googleapis/librarian/issues/4854): delete -// once the issue is resolved. -// For Java, it maps the transport to "grpc", "http", or "both". func (api *API) RepoMetadataTransport(language string, library *config.Library) string { transport := api.Transport(language) - if language == config.LanguageJava { - if library != nil && library.Java != nil && library.Java.TransportOverride != "" { - transport = Transport(library.Java.TransportOverride) - } - switch transport { - case GRPC: - return "grpc" - case Rest: - return "http" - default: - return "both" - } + if language == config.LanguageJava && library != nil && library.Java != nil && library.Java.TransportOverride != "" { + transport = Transport(library.Java.TransportOverride) } return string(transport) } diff --git a/internal/serviceconfig/api_test.go b/internal/serviceconfig/api_test.go index 21d75d00cce..070922f0ccf 100644 --- a/internal/serviceconfig/api_test.go +++ b/internal/serviceconfig/api_test.go @@ -330,7 +330,13 @@ func TestRepoMetadataTransport(t *testing.T) { Transports: map[string]Transport{config.LanguageJava: Rest}, }, language: config.LanguageJava, - want: "http", + want: "rest", + }, + { + name: "java, default", + sc: &API{}, + language: config.LanguageJava, + want: "grpc+rest", }, { name: "non-java, default", @@ -365,7 +371,7 @@ func TestRepoMetadataTransport(t *testing.T) { TransportOverride: "rest", }, }, - want: "http", + want: "rest", }, } { t.Run(test.name, func(t *testing.T) { From 027103b817777f101d72ad7df3af2bf7255e7b6a Mon Sep 17 00:00:00 2001 From: hongdaj Date: Tue, 30 Jun 2026 12:39:52 -0400 Subject: [PATCH 095/108] feat(internal/librarian): add debug command with env subcommand (#6576) A new `debug` command with an `env` subcommand is added to the CLI to print the librarian environment. This includes resolved paths for `LIBRARIAN_CACHE`, `LIBRARIAN_BIN`, and language-specific tool installation directories. To support accessing the tool installation directories from the new command, the internal `getInstallDir` functions in the golang and java packages are exported as `InstallDir`. __Local Testing__ run `go run ./cmd/librarian debug env`, result: ``` LIBRARIAN_CACHE=/usr/local/google/home/hongdaj/.cache/librarian LIBRARIAN_BIN=/usr/local/google/home/hongdaj/.cache/librarian/bin Language-specific tool installation directories: golang: /usr/local/google/home/hongdaj/.cache/librarian/bin/go_tools java: /usr/local/google/home/hongdaj/.cache/librarian/bin/java_tools ``` Fixes #6374 --- cmd/librarian/doc.go | 16 +++++ internal/librarian/debug.go | 79 +++++++++++++++++++++++ internal/librarian/debug_test.go | 70 ++++++++++++++++++++ internal/librarian/golang/command.go | 2 +- internal/librarian/golang/install.go | 20 +++--- internal/librarian/golang/install_test.go | 2 +- internal/librarian/java/install.go | 22 +++---- internal/librarian/librarian.go | 1 + 8 files changed, 189 insertions(+), 23 deletions(-) create mode 100644 internal/librarian/debug.go create mode 100644 internal/librarian/debug_test.go diff --git a/cmd/librarian/doc.go b/cmd/librarian/doc.go index face1f6cdef..1a62c4d5ea3 100644 --- a/cmd/librarian/doc.go +++ b/cmd/librarian/doc.go @@ -187,5 +187,21 @@ Usage: version prints the librarian binary version and exits. The version is embedded at build time and follows the conventions described at https://go.dev/ref/mod#versions. + +# Various debugging commands + +Usage: + + librarian debug [command] + +# Print environment variables for the librarian command line interface. + +Usage: + + librarian debug env + +env prints the librarian interpretation of the environment it is run in. +This includes the resolved LIBRARIAN_CACHE and LIBRARIAN_BIN paths, +as well as the language-specific tool installation directories. */ package main diff --git a/internal/librarian/debug.go b/internal/librarian/debug.go new file mode 100644 index 00000000000..2c2d9fbbdd3 --- /dev/null +++ b/internal/librarian/debug.go @@ -0,0 +1,79 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package librarian + +import ( + "context" + "fmt" + "io" + "strings" + + "github.com/googleapis/librarian/internal/cache" + "github.com/googleapis/librarian/internal/librarian/golang" + "github.com/googleapis/librarian/internal/librarian/java" + "github.com/urfave/cli/v3" +) + +// debugCommand returns the CLI command for librarian debugging tools. +func debugCommand() *cli.Command { + return &cli.Command{ + Name: "debug", + Usage: "various debugging commands", + UsageText: "librarian debug [command]", + Commands: []*cli.Command{ + envCommand(), + }, + } +} + +// envCommand returns the CLI command for printing the librarian environment. +func envCommand() *cli.Command { + return &cli.Command{ + Name: "env", + Usage: "print environment variables for the librarian command line interface.", + UsageText: "librarian debug env", + Description: `env prints the librarian interpretation of the environment it is run in. +This includes the resolved LIBRARIAN_CACHE and LIBRARIAN_BIN paths, +as well as the language-specific tool installation directories.`, + Action: func(ctx context.Context, cmd *cli.Command) error { + return runEnv(cmd.Root().Writer) + }, + } +} + +func runEnv(w io.Writer) error { + cacheDir := dirOrErr(cache.Directory()) + buildDir := dirOrErr(cache.BinDirectory()) + goToolsDir := dirOrErr(golang.InstallDir()) + javaToolsDir := dirOrErr(java.InstallDir()) + var b strings.Builder + fmt.Fprintf(&b, "LIBRARIAN_CACHE=%s\n", cacheDir) + fmt.Fprintf(&b, "LIBRARIAN_BIN=%s\n", buildDir) + fmt.Fprintln(&b) + fmt.Fprintln(&b, "Language-specific tool installation directories:") + fmt.Fprintf(&b, " golang: %s\n", goToolsDir) + fmt.Fprintf(&b, " java: %s\n", javaToolsDir) + _, err := io.WriteString(w, b.String()) + return err +} + +// dirOrErr converts a directory path and potential error into a string. If an error +// occurred, it returns a formatted error string; otherwise, it returns the directory path. +func dirOrErr(dir string, err error) string { + if err != nil { + return fmt.Sprintf("", err) + } + return dir +} diff --git a/internal/librarian/debug_test.go b/internal/librarian/debug_test.go new file mode 100644 index 00000000000..7ce9c768c96 --- /dev/null +++ b/internal/librarian/debug_test.go @@ -0,0 +1,70 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package librarian + +import ( + "bytes" + "fmt" + "path/filepath" + "strings" + "testing" +) + +func TestRunEnv(t *testing.T) { + cacheDir := t.TempDir() + binDir := t.TempDir() + t.Setenv("LIBRARIAN_CACHE", cacheDir) + t.Setenv("LIBRARIAN_BIN", binDir) + var buf bytes.Buffer + if err := runEnv(&buf); err != nil { + t.Fatal(err) + } + got := buf.String() + wants := []string{ + fmt.Sprintf("LIBRARIAN_CACHE=%s", cacheDir), + fmt.Sprintf("LIBRARIAN_BIN=%s", binDir), + fmt.Sprintf("golang: %s", filepath.Join(binDir, "go_tools")), + fmt.Sprintf("java: %s", filepath.Join(binDir, "java_tools")), + } + for _, want := range wants { + if !strings.Contains(got, want) { + t.Errorf("runEnv() output missing %q\ngot:\n%s", want, got) + } + } +} + +func TestRunEnv_Error(t *testing.T) { + // Unset environment variables to force path resolution errors. + t.Setenv("LIBRARIAN_CACHE", "") + t.Setenv("LIBRARIAN_BIN", "") + t.Setenv("HOME", "") + t.Setenv("XDG_CACHE_HOME", "") + var buf bytes.Buffer + if err := runEnv(&buf); err != nil { + t.Fatal(err) + } + got := buf.String() + wants := []string{ + "LIBRARIAN_CACHE= Date: Tue, 30 Jun 2026 14:13:39 -0400 Subject: [PATCH 096/108] feat(add): handle Release Please config for google-cloud-node (#6569) We had to manally add Release Please entry in the configuration files when we onboard a new library to google-cloud-node. https://github.com/googleapis/google-cloud-node/pull/8693 is the example. Let's have Librarian to add the entries in the Release Please files. Python and Go already have the feature with the "bulk" files. Let's make the logic work for google-cloud-node's configuration files. They use the default names: release-please-config.json and .release-please-manifest.json. Note that when Librarian starts to touch the Release Please files, it sorts the JSON keys. https://github.com/googleapis/google-cloud-node/pull/8777 is the outcome of an example invocation for the agentregistry package using the source tree before the package was introduced. The JSON keys are sorted. (The irrelevant changes in the mixin fields are due to recent change in Librarian for NodeJS.) Fixes https://github.com/googleapis/librarian/issues/6528 --- internal/librarian/add.go | 4 +- internal/librarian/release_please.go | 56 ++++++++++++---- internal/librarian/release_please_test.go | 78 ++++++++++++++++++++--- 3 files changed, 115 insertions(+), 23 deletions(-) diff --git a/internal/librarian/add.go b/internal/librarian/add.go index 4c2e5d995d9..b09f7aa372a 100644 --- a/internal/librarian/add.go +++ b/internal/librarian/add.go @@ -100,8 +100,8 @@ func runAdd(ctx context.Context, cfg *config.Config, api string) error { if err != nil { return err } - if cfg.Language == config.LanguageGo || cfg.Language == config.LanguagePython { - if hasBulkReleasePleaseConfigs(".") { + if cfg.Language == config.LanguageGo || cfg.Language == config.LanguagePython || cfg.Language == config.LanguageNodejs { + if hasBulkReleasePleaseConfigs(".", cfg) { if err := syncToReleasePlease(".", cfg, name); err != nil { return err } diff --git a/internal/librarian/release_please.go b/internal/librarian/release_please.go index e01f76b9bf4..a921f1fafc5 100644 --- a/internal/librarian/release_please.go +++ b/internal/librarian/release_please.go @@ -31,15 +31,33 @@ import ( const ( bulkManifestFile = ".release-please-bulk-manifest.json" bulkConfigFile = "release-please-bulk-config.json" + defaultManifestFile = ".release-please-manifest.json" + defaultConfigFile = "release-please-config.json" defaultReleasePleaseVersion = "0.0.0" ) -func hasBulkReleasePleaseConfigs(dir string) bool { - _, errM := os.Stat(filepath.Join(dir, bulkManifestFile)) - _, errC := os.Stat(filepath.Join(dir, bulkConfigFile)) +func hasBulkReleasePleaseConfigs(dir string, cfg *config.Config) bool { + manifestFile, configFile := releasePleaseFiles(cfg) + _, errM := os.Stat(filepath.Join(dir, manifestFile)) + _, errC := os.Stat(filepath.Join(dir, configFile)) return !errors.Is(errM, fs.ErrNotExist) && !errors.Is(errC, fs.ErrNotExist) } +// releasePleaseFiles returns the file names for the Release Please manifest file +// and config file in this order, depending on the SDK language. +func releasePleaseFiles(cfg *config.Config) (string, string) { + // google-cloud-node uses the default Release Please files to add a new library. + // google-cloud-python and google-cloud-go use the "-bulk-" files. + manifestFile := bulkManifestFile + configFile := bulkConfigFile + if cfg.Language == config.LanguageNodejs { + // google-cloud-node uses the default files + manifestFile = defaultManifestFile + configFile = defaultConfigFile + } + return manifestFile, configFile +} + // syncToReleasePlease updates the release-please configuration files with the // onboarded library's package name, initial version, and language-specific // extra files to track for release version bumps. @@ -49,19 +67,20 @@ func syncToReleasePlease(dir string, cfg *config.Config, name string) error { return err } - manifestPath := filepath.Join(dir, bulkManifestFile) + manifestFile, configFile := releasePleaseFiles(cfg) + manifestPath := filepath.Join(dir, manifestFile) manifest, err := readJSONFile[map[string]string](manifestPath) if err != nil { - return fmt.Errorf("failed to read bulk manifest file: %w", err) + return fmt.Errorf("failed to read manifest file: %w", err) } if manifest == nil { manifest = make(map[string]string) } - configPath := filepath.Join(dir, bulkConfigFile) + configPath := filepath.Join(dir, configFile) bulkConfig, err := readJSONFile[map[string]any](configPath) if err != nil { - return fmt.Errorf("failed to read bulk config file: %w", err) + return fmt.Errorf("failed to read config file: %w", err) } if bulkConfig == nil { bulkConfig = make(map[string]any) @@ -69,7 +88,8 @@ func syncToReleasePlease(dir string, cfg *config.Config, name string) error { packagesRaw, pkgsExist := bulkConfig["packages"] packages, isMap := packagesRaw.(map[string]any) if pkgsExist && !isMap { - return fmt.Errorf("'packages' in bulk config is not an object: %v", packagesRaw) + return fmt.Errorf("'packages' in %s is not an object: %v", + configPath, packagesRaw) } if !isMap || packages == nil { packages = make(map[string]any) @@ -78,12 +98,22 @@ func syncToReleasePlease(dir string, cfg *config.Config, name string) error { var extraFiles []any pkgPath := lib.Name - if cfg.Language == config.LanguagePython { + switch cfg.Language { + case config.LanguagePython: pkgPath = python.ReleasePleasePkgPrefix + lib.Name extraFiles = python.ReleasePleaseExtraFiles(lib) + case config.LanguageNodejs: + pkgPath = "packages/" + lib.Name } - if err := syncPackageToReleasePlease(manifest, packages, pkgPath, lib.Version, lib.Name, extraFiles); err != nil { + component := lib.Name + if cfg.Language == config.LanguageNodejs { + // google-cloud-node does not need to override + // component value in package. + component = "" + } + + if err := syncPackageToReleasePlease(manifest, packages, pkgPath, lib.Version, component, extraFiles); err != nil { return err } @@ -140,7 +170,11 @@ func syncPackageToReleasePlease(manifest map[string]string, packages map[string] packages[pkgPath] = pkgCfg } - pkgCfg["component"] = component + if component != "" { + // Python and Go set component names for packages in the config file. + // NodeJS does not do this and passes an empty string in the argument. + pkgCfg["component"] = component + } if len(extraFiles) > 0 { var existing []any diff --git a/internal/librarian/release_please_test.go b/internal/librarian/release_please_test.go index 4d999855770..f9529ab9ef2 100644 --- a/internal/librarian/release_please_test.go +++ b/internal/librarian/release_please_test.go @@ -27,30 +27,63 @@ import ( func TestHasBulkReleasePleaseConfigs(t *testing.T) { for _, test := range []struct { name string + language string createConfig bool createManifest bool want bool }{ { - name: "both missing", + name: "both missing (Go)", + language: config.LanguageGo, createConfig: false, createManifest: false, want: false, }, { - name: "config missing", + name: "config missing (Go)", + language: config.LanguageGo, createConfig: false, createManifest: true, want: false, }, { - name: "manifest missing", + name: "manifest missing (Go)", + language: config.LanguageGo, createConfig: true, createManifest: false, want: false, }, { - name: "both exist", + name: "both exist (Go)", + language: config.LanguageGo, + createConfig: true, + createManifest: true, + want: true, + }, + { + name: "both missing (Nodejs)", + language: config.LanguageNodejs, + createConfig: false, + createManifest: false, + want: false, + }, + { + name: "config missing (Nodejs)", + language: config.LanguageNodejs, + createConfig: false, + createManifest: true, + want: false, + }, + { + name: "manifest missing (Nodejs)", + language: config.LanguageNodejs, + createConfig: true, + createManifest: false, + want: false, + }, + { + name: "both exist (Nodejs)", + language: config.LanguageNodejs, createConfig: true, createManifest: true, want: true, @@ -58,19 +91,24 @@ func TestHasBulkReleasePleaseConfigs(t *testing.T) { } { t.Run(test.name, func(t *testing.T) { tmp := t.TempDir() + manifestFile, configFile := releasePleaseFiles( + &config.Config{ + Language: test.language, + }, + ) if test.createConfig { - if err := os.WriteFile(filepath.Join(tmp, "release-please-bulk-config.json"), []byte("{}"), 0644); err != nil { + if err := os.WriteFile(filepath.Join(tmp, configFile), []byte("{}"), 0644); err != nil { t.Fatal(err) } } if test.createManifest { - if err := os.WriteFile(filepath.Join(tmp, ".release-please-bulk-manifest.json"), []byte("{}"), 0644); err != nil { + if err := os.WriteFile(filepath.Join(tmp, manifestFile), []byte("{}"), 0644); err != nil { t.Fatal(err) } } - got := hasBulkReleasePleaseConfigs(tmp) + got := hasBulkReleasePleaseConfigs(tmp, &config.Config{Language: test.language}) if got != test.want { - t.Errorf("hasBulkReleasePleaseConfigs(%s) = %t, want %t", tmp, got, test.want) + t.Errorf("hasBulkReleasePleaseConfigs(%s, %s) = %t, want %t", tmp, test.language, got, test.want) } }) } @@ -101,6 +139,21 @@ func TestSyncToReleasePlease(t *testing.T) { wantManifest: `{"secretmanager":"1.0.0"}`, wantConfig: `{"packages":{"secretmanager":{"component":"secretmanager"}}}`, }, + { + name: "new nodejs library", + language: config.LanguageNodejs, + initialManifest: `{}`, + initialConfig: `{"packages": {}}`, + library: &config.Library{ + Name: "google-cloud-secretmanager", + Version: "1.0.0", + APIs: []*config.API{ + {Path: "google/cloud/secretmanager/v1"}, + }, + }, + wantManifest: `{"packages/google-cloud-secretmanager":"1.0.0"}`, + wantConfig: `{"packages":{"packages/google-cloud-secretmanager":{}}}`, + }, { name: "new python library", @@ -258,8 +311,13 @@ func TestSyncToReleasePlease(t *testing.T) { } { t.Run(test.name, func(t *testing.T) { tmp := t.TempDir() - manifestPath := filepath.Join(tmp, ".release-please-bulk-manifest.json") - configPath := filepath.Join(tmp, "release-please-bulk-config.json") + manifestFile, configFile := releasePleaseFiles( + &config.Config{ + Language: test.language, + }, + ) + manifestPath := filepath.Join(tmp, manifestFile) + configPath := filepath.Join(tmp, configFile) if err := os.WriteFile(manifestPath, []byte(test.initialManifest), 0644); err != nil { t.Fatal(err) } From 986aee41ec200699cf727d51d17def82e56b91ec Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Tue, 30 Jun 2026 14:46:55 -0400 Subject: [PATCH 097/108] WIP: backup full README generator implementation --- internal/librarian/java/readme.go | 430 ++++++++- internal/librarian/java/readme_test.go | 833 +++++++++++++++++- .../librarian/java/template/README.md.go.tmpl | 255 ++++++ 3 files changed, 1512 insertions(+), 6 deletions(-) create mode 100644 internal/librarian/java/template/README.md.go.tmpl diff --git a/internal/librarian/java/readme.go b/internal/librarian/java/readme.go index 9a7db30a34b..7ed2eec0052 100644 --- a/internal/librarian/java/readme.go +++ b/internal/librarian/java/readme.go @@ -15,13 +15,32 @@ package java import ( + "bufio" + "bytes" + _ "embed" "errors" + "fmt" + "io/fs" + "os" "path/filepath" "regexp" + "sort" "strings" + "text/template" + "unicode" + + "github.com/googleapis/librarian/internal/yaml" ) var ( + //go:embed template/README.md.go.tmpl + readmeTmpl string + readmeTmplParsed = template.Must(template.New("README").Parse(readmeTmpl)) + openSnippetRegex = regexp.MustCompile(`\[START ([a-zA-Z0-9_-]+)\]`) + closeSnippetRegex = regexp.MustCompile(`\[END ([a-zA-Z0-9_-]+)\]`) + openExcludeRegex = regexp.MustCompile(`\[START_EXCLUDE\]`) + closeExcludeRegex = regexp.MustCompile(`\[END_EXCLUDE\]`) + // Matches lowercase/digit followed by uppercase (e.g., "FooBar" -> "Foo Bar"). camelCaseRegexp = regexp.MustCompile(`([a-z0-9])([A-Z])`) @@ -33,8 +52,164 @@ var ( // errEmptyTitle indicates the extracted title value is empty. errEmptyTitle = errors.New("title value cannot be empty") + + // errEmptyDir indicates an empty directory string was provided. + errEmptyDir = errors.New("dir cannot be empty") + + // errEmptyFile indicates an empty file path was provided. + errEmptyFile = errors.New("file cannot be empty") + + // errNilMetadata indicates a nil repoMetadata pointer was provided. + errNilMetadata = errors.New("metadata cannot be nil") ) +// codeSample represents a discovered Java code sample along with its derived title. +type codeSample struct { + Title string + File string +} + +// readmeData represents the top-level template execution context passed to README.md.go.tmpl. +type readmeData struct { + Metadata map[string]interface{} // Contains Repo, LibraryVersion, Samples, Snippets, Partials. // TODO delete these comments for readmeData + GroupID string // Maven Group ID (e.g. com.google.cloud), required for Maven/Gradle dependency blocks. + ArtifactID string // Maven Artifact ID (e.g. google-cloud-storage), required for dependency blocks. + Version string // Current library version. + RepoShort string // Short repository name used in GitHub archive migration notices. + MigratedSplitRepo bool // Flag indicating if repository moved to monorepo. + Monorepo bool // Flag indicating if library is part of google-cloud-java monorepo. + BOMVersion string // Version of libraries-bom for dependencyManagement block. +} + +// renderREADME generates README.md in dir using the embedded Markdown template. +// It injects repository metadata, versions, samples, and snippets, skipping rendering if protected by keepSet. +func renderREADME(dir string, metadata *repoMetadata, bomVersion, libraryVersion string, keepSet map[string]bool) error { + if dir == "" { + return errEmptyDir + } + if metadata == nil { + return errNilMetadata + } + if keepSet["README.md"] { + return nil + } + partials, err := loadReadmePartials(dir) + if err != nil { + return err + } + groupID, artifactID := parseGroupIDArtifactID(metadata.DistributionName) + repoShort := parseRepoShortName(metadata.Repo) + minJavaVersion := metadata.MinJavaVersion + if minJavaVersion == 0 { + minJavaVersion = 8 + } + samples, err := extractSamples(dir) + if err != nil { + return fmt.Errorf("failed to extract samples: %w", err) + } + snippets, err := extractSnippets(dir) + if err != nil { + return fmt.Errorf("failed to extract snippets: %w", err) + } + templateMetadata := map[string]interface{}{ + "Repo": metadata, + "Samples": samples, + "Snippets": snippets, + "MinJavaVersion": minJavaVersion, + } + if len(partials) > 0 { + templateMetadata["Partials"] = partials + } + data := readmeData{ + Metadata: templateMetadata, + GroupID: groupID, + ArtifactID: artifactID, + Version: libraryVersion, + RepoShort: repoShort, + MigratedSplitRepo: false, + Monorepo: true, + BOMVersion: bomVersion, + } + var buf strings.Builder + if err := readmeTmplParsed.Execute(&buf, data); err != nil { + return fmt.Errorf("failed to execute template: %w", err) + } + outputPath := filepath.Join(dir, "README.md") + return os.WriteFile(outputPath, []byte(buf.String()), 0644) +} + +// extractSamples locates standard Java example files under the "samples" directory. +// It returns a codeSample struct containing the display title and relative path of each sample for README rendering. +func extractSamples(dir string) ([]codeSample, error) { + if dir == "" { + return nil, errEmptyDir + } + files, err := collectSampleFiles(dir) + if err != nil { + return nil, err + } + var samples []codeSample + for _, file := range files { + sample, err := parseCodeSample(dir, file) + if err != nil { + return nil, err + } + samples = append(samples, *sample) + } + return samples, nil +} + +// collectSampleFiles recursively scans dir/samples for Java production files. +func collectSampleFiles(dir string) ([]string, error) { + samplesDir := filepath.Join(dir, "samples") + if _, err := os.Stat(samplesDir); err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + return nil, fmt.Errorf("failed to stat samples directory: %w", err) + } + var files []string + err := filepath.WalkDir(samplesDir, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if !d.Type().IsRegular() { + return nil + } + rel, err := filepath.Rel(dir, path) + if err != nil { + return err + } + if isProductionSample(rel) { + files = append(files, rel) + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("failed to walk samples directory: %w", err) + } + return files, nil +} + +// parseCodeSample reads a Java sample file and constructs a codeSample struct with its title and relative path. +func parseCodeSample(dir, file string) (*codeSample, error) { + // Derive default title by stripping extension and converting CamelCase to space-separated words. + base := strings.TrimSuffix(filepath.Base(file), ".java") + title := decamelize(base) + titleOverride, err := extractTitle(filepath.Join(dir, file)) + if err != nil { + return nil, fmt.Errorf("failed to extract title from %s: %w", file, err) + } + if titleOverride != "" { + title = titleOverride + } + return &codeSample{ + Title: title, + // Normalize path separators to forward slashes for Markdown links in README. + File: filepath.ToSlash(file), + }, nil +} + // decamelize converts CamelCase string to space-separated string (e.g. "CamelCase" -> "Camel Case"). func decamelize(value string) string { return strings.TrimSpace(camelCaseRegexp.ReplaceAllString(value, `$1 $2`)) @@ -48,13 +223,19 @@ func isProductionSample(path string) bool { (strings.HasPrefix(slashed, "src/main/java/") || strings.Contains(slashed, "/src/main/java/")) } -// extractTitle extracts and validates the title override from Java comment blocks. +// extractTitle reads a file from disk and extracts the title override from Java comment blocks. // It expects a "title:" line to immediately follow the "sample-metadata:" marker. -// Returns an error if the marker is present but the title line is missing, malformed, or empty. -func extractTitle(content string) (string, error) { - if !strings.Contains(content, "sample-metadata:") { +// Returns an error if the file cannot be read, or if the marker is present but the title line +// is missing, malformed, or empty. +func extractTitle(filePath string) (string, error) { + contentBytes, err := os.ReadFile(filePath) + if err != nil { + return "", fmt.Errorf("failed to read file: %w", err) + } + if !bytes.Contains(contentBytes, []byte("sample-metadata:")) { return "", nil } + content := string(contentBytes) matches := reTitle.FindStringSubmatch(content) if len(matches) < 2 { return "", errMissingTitle @@ -66,3 +247,244 @@ func extractTitle(content string) (string, error) { } return title, nil } + +// extractSnippets walks the "samples" directory locating *.java and *.xml files. +// It line-scans for START and END tags while supporting START_EXCLUDE blocks. +func extractSnippets(dir string) (map[string]string, error) { + if dir == "" { + return nil, errEmptyDir + } + files, err := collectSnippetFiles(dir) + if err != nil { + return nil, err + } + if len(files) == 0 { + return nil, nil + } + sort.Strings(files) // TODO: check- do we need this? check the legacy owlbot and see if we sort the files? + snippetLines := make(map[string][]string) + for _, file := range files { + fileSnippets, err := extractSnippetsFromFile(file) + if err != nil { + return nil, err + } + for name, lines := range fileSnippets { + snippetLines[name] = append(snippetLines[name], lines...) + } + } + if len(snippetLines) == 0 { + return nil, nil + } + result := make(map[string]string) + for snippet, lines := range snippetLines { + result[snippet] = trimLeadingWhitespace(lines) + } + return result, nil +} + +// collectSnippetFiles recursively scans dir/samples for Java and XML files containing snippets. +func collectSnippetFiles(dir string) ([]string, error) { + samplesDir := filepath.Join(dir, "samples") + if _, err := os.Stat(samplesDir); err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + return nil, fmt.Errorf("failed to stat samples directory: %w", err) + } + var files []string + err := filepath.WalkDir(samplesDir, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + if d.Name() == "test" || (d.Name() == "generated" && filepath.Base(filepath.Dir(path)) == "snippets") { + return filepath.SkipDir + } + return nil + } + if !d.Type().IsRegular() { + return nil + } + ext := filepath.Ext(path) + if ext == ".java" || ext == ".xml" { + files = append(files, path) + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("failed to walk samples directory: %w", err) + } + return files, nil +} + +// extractSnippetsFromFile parses a single file to return a map of tagged code snippets. +// Reads code between START and END markers while omitting EXCLUDE blocks. +// START and END markers must be named (e.g. [START ]). +func extractSnippetsFromFile(file string) (map[string][]string, error) { + if file == "" { + return nil, errEmptyFile + } + f, err := os.Open(file) + if err != nil { + return nil, fmt.Errorf("failed to open file %s: %w", file, err) + } + defer f.Close() + scanner := bufio.NewScanner(f) + // 10 MB sanity limit to protect system memory. + scanner.Buffer(make([]byte, 64*1024), 10*1024*1024) + // Scan through file line by line and capture all open snippets. + // More than one open block might exist for a given line, so we + // need to track open snippets in a map. + snippetLines := make(map[string][]string) + openSnippets := make(map[string]bool) + excluding := false + for scanner.Scan() { + line := scanner.Text() + if openExcludeRegex.MatchString(line) { + excluding = true + continue + } + if closeExcludeRegex.MatchString(line) { + excluding = false + continue + } + if excluding { + continue + } + openMatch := openSnippetRegex.FindStringSubmatch(line) + closeMatch := closeSnippetRegex.FindStringSubmatch(line) + if len(openMatch) > 1 { + name := openMatch[1] + openSnippets[name] = true + if _, exists := snippetLines[name]; !exists { + snippetLines[name] = []string{} + } + continue + } + if len(closeMatch) > 1 { + delete(openSnippets, closeMatch[1]) + continue + } + for s := range openSnippets { + snippetLines[s] = append(snippetLines[s], line) + } + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("failed scanning file %s: %w", file, err) + } + return snippetLines, nil +} + +// minLeadingSpaces finds the minimum number of leading spaces across non-empty lines. +func minLeadingSpaces(lines []string) int { + if len(lines) == 0 { + return 0 + } + minSpaces := -1 + for _, line := range lines { + if strings.TrimSpace(line) == "" { + continue + } + spaces := len(line) - len(strings.TrimLeft(line, " ")) + if minSpaces == -1 || spaces < minSpaces { + minSpaces = spaces + } + } + if minSpaces == -1 { + return 0 + } + return minSpaces +} + +// trimLeadingWhitespace computes minimum leading space indentation and trims it. +// Used to clean up snippet lines so code formatting looks natural in README blocks. +func trimLeadingWhitespace(lines []string) string { + if len(lines) == 0 { + return "" + } + minSpaces := minLeadingSpaces(lines) + var sb strings.Builder + for _, line := range lines { + if strings.TrimSpace(line) == "" { + sb.WriteString("\n") + continue + } + if len(line) >= minSpaces { + sb.WriteString(line[minSpaces:]) + } else { + sb.WriteString(line) + } + sb.WriteString("\n") + } + return sb.String() +} + +// loadReadmePartials loads and camel-cases README partials from .readme-partials.yaml or .yml. +func loadReadmePartials(dir string) (map[string]interface{}, error) { + if dir == "" { + return nil, errEmptyDir + } + partialsPath := filepath.Join(dir, ".readme-partials.yaml") + if _, err := os.Stat(partialsPath); err != nil { + if !errors.Is(err, fs.ErrNotExist) { + return nil, fmt.Errorf("failed to stat partials file: %w", err) + } + partialsPath = filepath.Join(dir, ".readme-partials.yml") + if _, err = os.Stat(partialsPath); err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + return nil, fmt.Errorf("failed to stat partials file: %w", err) + } + } + partialsBytes, err := os.ReadFile(partialsPath) + if err != nil { + return nil, fmt.Errorf("failed to read partials file: %w", err) + } + if partialsBytes == nil { + return nil, nil + } + rawPartials, err := yaml.Unmarshal[map[string]interface{}](partialsBytes) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal partials: %w", err) + } + result := make(map[string]interface{}) + for k, v := range *rawPartials { + // Convert to camel case because jinja templates use snake_case but go template fields should use camelcase. + result[toCamelCase(k)] = v + } + return result, nil +} + +// toCamelCase converts snake_case, kebab-case, or lower word strings to CamelCase. +func toCamelCase(s string) string { + parts := strings.FieldsFunc(s, func(r rune) bool { + return r == '_' || r == '-' || r == ' ' + }) + var sb strings.Builder + for _, p := range parts { + if len(p) > 0 { + r := []rune(p) + r[0] = unicode.ToUpper(r[0]) + sb.WriteString(string(r)) + } + } + return sb.String() +} + +// parseGroupIDArtifactID extracts GroupID and ArtifactID from a Maven distribution name. +func parseGroupIDArtifactID(distributionName string) (string, string) { + groupID, artifactID, _ := strings.Cut(distributionName, ":") + return groupID, artifactID +} + +// parseRepoShortName extracts the short repository name from the full repo path. +func parseRepoShortName(repo string) string { + if repo == "" { + return "" + } + if i := strings.LastIndexByte(repo, '/'); i >= 0 { + return repo[i+1:] + } + return repo +} diff --git a/internal/librarian/java/readme_test.go b/internal/librarian/java/readme_test.go index 595c87cb645..6a24c1d53d4 100644 --- a/internal/librarian/java/readme_test.go +++ b/internal/librarian/java/readme_test.go @@ -16,6 +16,10 @@ package java import ( "errors" + "io/fs" + "os" + "path/filepath" + "strings" "testing" "github.com/google/go-cmp/cmp" @@ -141,7 +145,11 @@ public class Normal {}`, }, } { t.Run(test.name, func(t *testing.T) { - got, err := extractTitle(test.content) + tmpPath := filepath.Join(t.TempDir(), "Sample.java") + if err := os.WriteFile(tmpPath, []byte(test.content), 0644); err != nil { + t.Fatal(err) + } + got, err := extractTitle(tmpPath) if err != nil { t.Fatal(err) } @@ -172,10 +180,831 @@ func TestExtractTitle_Error(t *testing.T) { }, } { t.Run(test.name, func(t *testing.T) { - _, gotErr := extractTitle(test.content) + tmpPath := filepath.Join(t.TempDir(), "Sample.java") + if err := os.WriteFile(tmpPath, []byte(test.content), 0644); err != nil { + t.Fatal(err) + } + _, gotErr := extractTitle(tmpPath) if !errors.Is(gotErr, test.wantErr) { t.Errorf("extractTitle() error = %v, wantErr %v", gotErr, test.wantErr) } }) } } + +func TestCollectSampleFiles(t *testing.T) { + for _, test := range []struct { + name string + setupFiles func(t *testing.T, dir string) + want []string + }{ + { + name: "missing samples directory", + setupFiles: func(t *testing.T, dir string) { + // Do nothing, temp dir is empty. + }, + want: nil, + }, + { + name: "collects production java files only", + setupFiles: func(t *testing.T, dir string) { + samplesDir := filepath.Join(dir, "samples", "src", "main", "java", "com", "example") + if err := os.MkdirAll(samplesDir, 0755); err != nil { + t.Fatal(err) + } + testDir := filepath.Join(dir, "samples", "src", "test", "java", "com", "example") + if err := os.MkdirAll(testDir, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(samplesDir, "SampleA.java"), []byte("public class SampleA {}"), 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(samplesDir, "README.md"), []byte("# Docs"), 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(testDir, "SampleATest.java"), []byte("public class SampleATest {}"), 0644); err != nil { + t.Fatal(err) + } + }, + want: []string{ + filepath.Join("samples", "src", "main", "java", "com", "example", "SampleA.java"), + }, + }, + { + name: "collects production java files across nested packages and ignores directories", + setupFiles: func(t *testing.T, dir string) { + pkg1 := filepath.Join(dir, "samples", "src", "main", "java", "com", "example") + pkg2 := filepath.Join(dir, "samples", "src", "main", "java", "com", "example", "subpkg") + fakeJavaDir := filepath.Join(dir, "samples", "src", "main", "java", "com", "example", "FakeDir.java") + if err := os.MkdirAll(pkg1, 0755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(pkg2, 0755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(fakeJavaDir, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(pkg1, "Alpha.java"), []byte("public class Alpha {}"), 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(pkg2, "Beta.java"), []byte("package com.example.subpkg; public class Beta {}"), 0644); err != nil { + t.Fatal(err) + } + }, + want: []string{ + filepath.Join("samples", "src", "main", "java", "com", "example", "Alpha.java"), + filepath.Join("samples", "src", "main", "java", "com", "example", "subpkg", "Beta.java"), + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + tempDir := t.TempDir() + test.setupFiles(t, tempDir) + + got, err := collectSampleFiles(tempDir) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestParseCodeSample(t *testing.T) { + for _, test := range []struct { + name string + relPath string + content string + want *codeSample + }{ + { + name: "default title derived from filename", + relPath: filepath.Join("samples", "src", "main", "java", "DemoSample.java"), + content: "public class DemoSample {}", + want: &codeSample{ + Title: "Demo Sample", + File: "samples/src/main/java/DemoSample.java", + }, + }, + { + name: "custom title override from metadata", + relPath: filepath.Join("samples", "src", "main", "java", "RequesterPays.java"), + content: "// sample-metadata:\n// title: Custom Title Override\npublic class RequesterPays {}", + want: &codeSample{ + Title: "Custom Title Override", + File: "samples/src/main/java/RequesterPays.java", + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + tempDir := t.TempDir() + absPath := filepath.Join(tempDir, test.relPath) + if err := os.MkdirAll(filepath.Dir(absPath), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(absPath, []byte(test.content), 0644); err != nil { + t.Fatal(err) + } + + got, err := parseCodeSample(tempDir, test.relPath) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestParseCodeSample_Error(t *testing.T) { + for _, test := range []struct { + name string + relPath string + content string + wantErr error + }{ + { + name: "empty title returns error", + relPath: filepath.Join("samples", "src", "main", "java", "InvalidSample.java"), + content: "// sample-metadata:\n// title: \"\"\npublic class InvalidSample {}", + wantErr: errEmptyTitle, + }, + { + name: "missing title line returns error", + relPath: filepath.Join("samples", "src", "main", "java", "MissingTitle.java"), + content: "// sample-metadata:\n// description: missing\npublic class MissingTitle {}", + wantErr: errMissingTitle, + }, + } { + t.Run(test.name, func(t *testing.T) { + tempDir := t.TempDir() + absPath := filepath.Join(tempDir, test.relPath) + if err := os.MkdirAll(filepath.Dir(absPath), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(absPath, []byte(test.content), 0644); err != nil { + t.Fatal(err) + } + + _, err := parseCodeSample(tempDir, test.relPath) + if !errors.Is(err, test.wantErr) { + t.Errorf("parseCodeSample() error = %v, wantErr %v", err, test.wantErr) + } + }) + } +} + +func TestExtractSamples(t *testing.T) { + for _, test := range []struct { + name string + setupFiles func(t *testing.T, dir string) + want []codeSample + }{ + { + name: "missing samples directory", + setupFiles: func(t *testing.T, dir string) { + // Do nothing, tempDir is empty. + }, + want: nil, + }, + { + name: "extract successfully", + setupFiles: func(t *testing.T, dir string) { + samplesDir := filepath.Join(dir, "samples", "src", "main", "java") + if err := os.MkdirAll(samplesDir, 0755); err != nil { + t.Fatal(err) + } + file1 := filepath.Join(samplesDir, "RequesterPays.java") + content1 := `// sample-metadata: +// title: Custom Title Override +public class RequesterPays {}` + if err := os.WriteFile(file1, []byte(content1), 0644); err != nil { + t.Fatal(err) + } + file2 := filepath.Join(samplesDir, "DemoSample.java") + content2 := `public class DemoSample {}` + if err := os.WriteFile(file2, []byte(content2), 0644); err != nil { + t.Fatal(err) + } + }, + want: []codeSample{ + { + Title: "Demo Sample", + File: "samples/src/main/java/DemoSample.java", + }, + { + Title: "Custom Title Override", + File: "samples/src/main/java/RequesterPays.java", + }, + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + tempDir := t.TempDir() + test.setupFiles(t, tempDir) + + samples, err := extractSamples(tempDir) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(test.want, samples); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestExtractSamples_Error(t *testing.T) { + for _, test := range []struct { + name string + setupFiles func(t *testing.T, dir string) + dir string + wantErr error + }{ + { + name: "error on empty directory", + dir: "", + wantErr: errEmptyDir, + }, + { + name: "error on empty title override", + setupFiles: func(t *testing.T, dir string) { + samplesDir := filepath.Join(dir, "samples", "src", "main", "java") + if err := os.MkdirAll(samplesDir, 0755); err != nil { + t.Fatal(err) + } + file := filepath.Join(samplesDir, "Sample.java") + content := `// sample-metadata: +// title: "" +public class Invalid {}` + if err := os.WriteFile(file, []byte(content), 0644); err != nil { + t.Fatal(err) + } + }, + wantErr: errEmptyTitle, + }, + { + name: "error on missing title line immediately following sample-metadata", + setupFiles: func(t *testing.T, dir string) { + samplesDir := filepath.Join(dir, "samples", "src", "main", "java") + if err := os.MkdirAll(samplesDir, 0755); err != nil { + t.Fatal(err) + } + file := filepath.Join(samplesDir, "Sample.java") + content := `// sample-metadata: +// description: missing title line +public class Invalid {}` + if err := os.WriteFile(file, []byte(content), 0644); err != nil { + t.Fatal(err) + } + }, + wantErr: errMissingTitle, + }, + } { + t.Run(test.name, func(t *testing.T) { + dir := test.dir + if test.setupFiles != nil { + dir = t.TempDir() + test.setupFiles(t, dir) + } + _, err := extractSamples(dir) + if !errors.Is(err, test.wantErr) { + t.Errorf("extractSamples() error = %v, wantErr %v", err, test.wantErr) + } + }) + } +} + +func TestCollectSnippetFiles(t *testing.T) { + for _, test := range []struct { + name string + setupFiles func(t *testing.T, dir string) + want []string + }{ + { + name: "missing samples directory returns nil", + setupFiles: func(t *testing.T, dir string) { + // Empty directory. + }, + want: nil, + }, + { + name: "collects java and xml files while ignoring excluded directories", + setupFiles: func(t *testing.T, dir string) { + validJavaDir := filepath.Join(dir, "samples", "src", "main", "java") + validXMLDir := filepath.Join(dir, "samples", "src", "main", "resources") + testDir := filepath.Join(dir, "samples", "src", "test", "java") + genDir := filepath.Join(dir, "samples", "snippets", "generated") + if err := os.MkdirAll(validJavaDir, 0755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(validXMLDir, 0755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(testDir, 0755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(genDir, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(validJavaDir, "Sample.java"), []byte("public class Sample {}"), 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(validXMLDir, "pom.xml"), []byte(""), 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(validJavaDir, "README.md"), []byte("# Ignore"), 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(testDir, "TestSample.java"), []byte("public class TestSample {}"), 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(genDir, "GenSnippet.java"), []byte("public class GenSnippet {}"), 0644); err != nil { + t.Fatal(err) + } + }, + want: []string{ + filepath.Join("samples", "src", "main", "java", "Sample.java"), + filepath.Join("samples", "src", "main", "resources", "pom.xml"), + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + tempDir := t.TempDir() + test.setupFiles(t, tempDir) + + got, err := collectSnippetFiles(tempDir) + if err != nil { + t.Fatal(err) + } + var relGot []string + for _, p := range got { + rel, err := filepath.Rel(tempDir, p) + if err != nil { + t.Fatal(err) + } + relGot = append(relGot, rel) + } + if diff := cmp.Diff(test.want, relGot); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestMinLeadingSpaces(t *testing.T) { + for _, test := range []struct { + name string + lines []string + want int + }{ + {"two lines indented", []string{" foo", " bar"}, 2}, + {"zero indented line", []string{"foo", " bar"}, 0}, + {"empty slice", nil, 0}, + {"only whitespace lines", []string{" ", "\t"}, 0}, + } { + t.Run(test.name, func(t *testing.T) { + got := minLeadingSpaces(test.lines) + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestTrimLeadingWhitespace(t *testing.T) { + for _, test := range []struct { + name string + lines []string + want string + }{ + { + name: "standard indentation", + lines: []string{" int x = 1;", " int y = 2;"}, + want: "int x = 1;\n int y = 2;\n", + }, + { + name: "with blank line", + lines: []string{" int x = 1;", "", " int y = 2;"}, + want: "int x = 1;\n\nint y = 2;\n", + }, + { + name: "empty input", + lines: nil, + want: "", + }, + } { + t.Run(test.name, func(t *testing.T) { + got := trimLeadingWhitespace(test.lines) + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestExtractSnippetsFromFile(t *testing.T) { + for _, test := range []struct { + name string + content string + want map[string][]string + }{ + { + name: "extracts snippets and respects exclude blocks", + content: `public class Example { + // [START my_snippet] + public void run() { + // [START_EXCLUDE] + secretInit(); + // [END_EXCLUDE] + doWork(); + } + // [END my_snippet] +}`, + want: map[string][]string{ + "my_snippet": { + " public void run() {", + " doWork();", + " }", + }, + }, + }, + { + name: "no snippets in file", + content: "public class Simple {}", + want: map[string][]string{}, + }, + } { + t.Run(test.name, func(t *testing.T) { + tmpPath := filepath.Join(t.TempDir(), "SampleSnippet.java") + if err := os.WriteFile(tmpPath, []byte(test.content), 0644); err != nil { + t.Fatal(err) + } + got, err := extractSnippetsFromFile(tmpPath) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestExtractSnippetsFromFile_Error(t *testing.T) { + for _, test := range []struct { + name string + file string + wantErr error + }{ + { + name: "empty file parameter returns error", + file: "", + wantErr: errEmptyFile, + }, + { + name: "non-existent file returns error", + file: "non-existent-file.java", + wantErr: fs.ErrNotExist, + }, + } { + t.Run(test.name, func(t *testing.T) { + _, err := extractSnippetsFromFile(test.file) + if !errors.Is(err, test.wantErr) { + t.Errorf("extractSnippetsFromFile(%q) error = %v, wantErr %v", test.file, err, test.wantErr) + } + }) + } +} + +func TestExtractSnippets(t *testing.T) { + tempDir := t.TempDir() + samplesDir := filepath.Join(tempDir, "samples") + if err := os.MkdirAll(samplesDir, 0755); err != nil { + t.Fatal(err) + } + + pomPath := filepath.Join(samplesDir, "pom.xml") + pomContent := ` + + + com.google.cloud + + +` + if err := os.WriteFile(pomPath, []byte(pomContent), 0644); err != nil { + t.Fatal(err) + } + + javaPath := filepath.Join(samplesDir, "Demo.java") + javaContent := `public class Demo { + // [START quickstart] + public void run() { + // [START_EXCLUDE] + System.out.println("hidden"); + // [END_EXCLUDE] + System.out.println("visible"); + } + // [END quickstart] +}` + if err := os.WriteFile(javaPath, []byte(javaContent), 0644); err != nil { + t.Fatal(err) + } + + snippets, err := extractSnippets(tempDir) + if err != nil { + t.Fatal(err) + } + + if len(snippets) != 2 { + t.Fatalf("Expected 2 snippets, got %d", len(snippets)) + } + + expectedDep := ` + com.google.cloud + +` + if diff := cmp.Diff(expectedDep, snippets["dependency_snippet"]); diff != "" { + t.Errorf("dependency_snippet mismatch (-want +got):\n%s", diff) + } + + expectedQuick := `public void run() { + System.out.println("visible"); +} +` + if diff := cmp.Diff(expectedQuick, snippets["quickstart"]); diff != "" { + t.Errorf("quickstart mismatch (-want +got):\n%s", diff) + } +} + +func TestExtractSnippets_Error(t *testing.T) { + for _, test := range []struct { + name string + dir string + wantErr error + }{ + { + name: "empty directory parameter returns error", + dir: "", + wantErr: errEmptyDir, + }, + } { + t.Run(test.name, func(t *testing.T) { + _, err := extractSnippets(test.dir) + if !errors.Is(err, test.wantErr) { + t.Errorf("extractSnippets(%q) error = %v, wantErr %v", test.dir, err, test.wantErr) + } + }) + } +} +func TestLoadReadmePartials(t *testing.T) { + for _, test := range []struct { + name string + setupFiles func(t *testing.T, dir string) + want map[string]interface{} + }{ + { + name: "loads yaml partials with camel case conversion", + setupFiles: func(t *testing.T, dir string) { + path := filepath.Join(dir, ".readme-partials.yaml") + content := `about_text: "Custom about"` + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatal(err) + } + }, + want: map[string]interface{}{"AboutText": "Custom about"}, + }, + { + name: "loads yml fallback partials", + setupFiles: func(t *testing.T, dir string) { + path := filepath.Join(dir, ".readme-partials.yml") + content := `introduction: "Intro text"` + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatal(err) + } + }, + want: map[string]interface{}{"Introduction": "Intro text"}, + }, + { + name: "missing partials file returns nil", + setupFiles: func(t *testing.T, dir string) { + // No file written. + }, + want: nil, + }, + } { + t.Run(test.name, func(t *testing.T) { + dir := t.TempDir() + test.setupFiles(t, dir) + got, err := loadReadmePartials(dir) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestLoadReadmePartials_Error(t *testing.T) { + for _, test := range []struct { + name string + dir string + setupFiles func(t *testing.T, dir string) + wantErr error + }{ + { + name: "empty directory parameter returns error", + dir: "", + wantErr: errEmptyDir, + }, + { + name: "invalid yaml syntax", + setupFiles: func(t *testing.T, dir string) { + path := filepath.Join(dir, ".readme-partials.yaml") + content := `key: [unclosed list` + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatal(err) + } + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + dir := test.dir + if test.setupFiles != nil { + dir = t.TempDir() + test.setupFiles(t, dir) + } + _, err := loadReadmePartials(dir) + if err == nil { + t.Errorf("expected error, got nil") + } else if test.wantErr != nil && !errors.Is(err, test.wantErr) { + t.Errorf("loadReadmePartials() error = %v, wantErr %v", err, test.wantErr) + } + }) + } +} + +func TestToCamelCase(t *testing.T) { + for _, test := range []struct { + name string + input string + want string + }{ + {"snake case", "custom_content", "CustomContent"}, + {"kebab case", "readme-partials", "ReadmePartials"}, + {"space separated", "about us", "AboutUs"}, + {"already camel", "About", "About"}, + {"empty string", "", ""}, + } { + t.Run(test.name, func(t *testing.T) { + got := toCamelCase(test.input) + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestParseGroupIDArtifactID(t *testing.T) { + for _, test := range []struct { + name string + input string + wantGroupID string + wantArtifactID string + }{ + {"standard coordinates", "com.google.cloud:google-cloud-storage", "com.google.cloud", "google-cloud-storage"}, + {"missing artifact id", "com.google.cloud", "com.google.cloud", ""}, + {"empty distribution name", "", "", ""}, + } { + t.Run(test.name, func(t *testing.T) { + gotGroup, gotArtifact := parseGroupIDArtifactID(test.input) + if diff := cmp.Diff(test.wantGroupID, gotGroup); diff != "" { + t.Errorf("group ID mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(test.wantArtifactID, gotArtifact); diff != "" { + t.Errorf("artifact ID mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestParseRepoShortName(t *testing.T) { + for _, test := range []struct { + name string + input string + want string + }{ + {"full repo path", "googleapis/google-cloud-java", "google-cloud-java"}, + {"short repo name only", "google-cloud-java", "google-cloud-java"}, + {"empty repo string", "", ""}, + } { + t.Run(test.name, func(t *testing.T) { + got := parseRepoShortName(test.input) + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestRenderREADME(t *testing.T) { + tmpDir := t.TempDir() + metadata := &repoMetadata{ + NamePretty: "My API", + DistributionName: "com.google.cloud:google-cloud-myapi", + Repo: "googleapis/google-cloud-java", + APIShortname: "myapi", + MinJavaVersion: 8, + } + + // Test case 1: Without partials + if err := renderREADME(tmpDir, metadata, "1.0.0-BOM", "1.2.3-LIB", nil); err != nil { + t.Fatal(err) + } + outputPath := filepath.Join(tmpDir, "README.md") + outputContent, err := os.ReadFile(outputPath) + if err != nil { + t.Fatal(err) + } + content := string(outputContent) + for _, expectedSubstring := range []string{ + "# Google My API Client for Java", + "com.google.cloud", + "google-cloud-myapi", + "1.2.3-LIB", + } { + if !strings.Contains(content, expectedSubstring) { + t.Errorf("generated README missing expected substring %q", expectedSubstring) + } + } + + // Test case 2: With partials + partialsPath := filepath.Join(tmpDir, ".readme-partials.yaml") + partialsContent := `about: "This is a great API."` + if err := os.WriteFile(partialsPath, []byte(partialsContent), 0644); err != nil { + t.Fatal(err) + } + if err := renderREADME(tmpDir, metadata, "1.0.0-BOM", "1.2.3-LIB", nil); err != nil { + t.Fatal(err) + } + outputContent, err = os.ReadFile(outputPath) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(outputContent), "This is a great API.") { + t.Errorf("generated README missing partial content %q", "This is a great API.") + } + + // Test case 3: With README.md in keep list + keepSet := map[string]bool{"README.md": true} + customContent := "Custom README content" + if err := os.WriteFile(outputPath, []byte(customContent), 0644); err != nil { + t.Fatal(err) + } + if err := renderREADME(tmpDir, metadata, "1.0.0-BOM", "1.2.3-LIB", keepSet); err != nil { + t.Fatal(err) + } + outputContent, err = os.ReadFile(outputPath) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(customContent, string(outputContent)); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } +} + +func TestRenderREADME_Error(t *testing.T) { + for _, test := range []struct { + name string + fn func() error + wantErr error + }{ + { + name: "empty directory parameter returns error", + fn: func() error { + return renderREADME("", &repoMetadata{}, "1.0", "1.0", nil) + }, + wantErr: errEmptyDir, + }, + { + name: "nil metadata returns error", + fn: func() error { + return renderREADME("dir", nil, "1.0", "1.0", nil) + }, + wantErr: errNilMetadata, + }, + } { + t.Run(test.name, func(t *testing.T) { + err := test.fn() + if !errors.Is(err, test.wantErr) { + t.Errorf("renderREADME() error = %v, wantErr %v", err, test.wantErr) + } + }) + } +} diff --git a/internal/librarian/java/template/README.md.go.tmpl b/internal/librarian/java/template/README.md.go.tmpl new file mode 100644 index 00000000000..6bc2d67ddcb --- /dev/null +++ b/internal/librarian/java/template/README.md.go.tmpl @@ -0,0 +1,255 @@ +# Google {{ .Metadata.Repo.NamePretty }} Client for Java + +Java idiomatic client for [{{ .Metadata.Repo.NamePretty }}][product-docs]. + +[![Maven][maven-version-image]][maven-version-link] +![Stability][stability-image] + +- [Product Documentation][product-docs] +- [Client Library Documentation][javadocs] + +{{ if and .Metadata.Partials .Metadata.Partials.DeprecationWarning }}{{ .Metadata.Partials.DeprecationWarning }} +{{ else if eq .Metadata.Repo.ReleaseLevel "preview" }}> Note: This client is a work-in-progress, and may occasionally +> make backwards-incompatible changes. + +{{ end }}{{ if .MigratedSplitRepo }}:bus: In October 2022, this library has moved to +[google-cloud-java/{{ .Metadata.Repo.RepoShort }}]( +https://github.com/googleapis/google-cloud-java/tree/main/{{ .Metadata.Repo.RepoShort }}). +This repository will be archived in the future. +Future releases will appear in the new repository (https://github.com/googleapis/google-cloud-java/releases). +The Maven artifact coordinates (`{{ .GroupID }}:{{ .ArtifactID }}`) remain the same. +{{ end }} +## Quickstart + +{{ if and .Metadata.Snippets (index .Metadata.Snippets (print .Metadata.Repo.APIShortname "_install_with_bom")) -}} +If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file: + +```xml +{{ index .Metadata.Snippets (print .Metadata.Repo.APIShortname "_install_with_bom") }} +``` + +If you are using Maven without the BOM, add this to your dependencies: +{{ else if and .Monorepo (or (eq .GroupID "com.google.cloud") (eq .GroupID "com.google.analytics") (eq .GroupID "com.google.area120")) }} +If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file: + +```xml + + + + com.google.cloud + libraries-bom + {{ .BOMVersion }} + pom + import + + + + + + + {{ .GroupID }} + {{ .ArtifactID }} + + +``` + +If you are using Maven without the BOM, add this to your dependencies: +{{ else }}If you are using Maven, add this to your pom.xml file: +{{ end }} + +```xml +{{ if and .Metadata.Snippets (index .Metadata.Snippets (print .Metadata.Repo.APIShortname "_install_without_bom")) -}} +{{ index .Metadata.Snippets (print .Metadata.Repo.APIShortname "_install_without_bom") }} +{{- else -}} + + {{ .GroupID }} + {{ .ArtifactID }} + {{ .Version }} + +{{- end }} +``` + +{{ if and .Metadata.Snippets (index .Metadata.Snippets (print .Metadata.Repo.APIShortname "_install_with_bom")) -}} +If you are using Gradle 5.x or later, add this to your dependencies: + +```Groovy +implementation platform('com.google.cloud:libraries-bom:{{ .BOMVersion }}') + +implementation '{{ .GroupID }}:{{ .ArtifactID }}' +``` +{{ end }}If you are using Gradle without BOM, add this to your dependencies: + +```Groovy +implementation '{{ .GroupID }}:{{ .ArtifactID }}:{{ .Version }}' +``` + +If you are using SBT, add this to your dependencies: + +```Scala +libraryDependencies += "{{ .GroupID }}" % "{{ .ArtifactID }}" % "{{ .Version }}" +``` + +## Authentication + +See the [Authentication][authentication] section in the base directory's README. + +## Authorization + +The client application making API calls must be granted [authorization scopes][auth-scopes] required for the desired {{ .Metadata.Repo.NamePretty }} APIs, and the authenticated principal must have the [IAM role(s)][predefined-iam-roles] required to access GCP resources using the {{ .Metadata.Repo.NamePretty }} API calls. + +## Getting Started + +### Prerequisites + +You will need a [Google Cloud Platform Console][developer-console] project with the {{ .Metadata.Repo.NamePretty }} [API enabled][enable-api]. +{{ if .Metadata.Repo.RequiresBilling }}You will need to [enable billing][enable-billing] to use Google {{ .Metadata.Repo.NamePretty }}.{{ end }} +[Follow these instructions][create-project] to get your project set up. You will also need to set up the local development environment by +[installing the Google Cloud Command Line Interface][cloud-cli] and running the following commands in command line: +`gcloud auth login` and `gcloud config set project [YOUR PROJECT ID]`. + +### Installation and setup + +You'll need to obtain the `{{ .ArtifactID }}` library. See the [Quickstart](#quickstart) section +to add `{{ .ArtifactID }}` as a dependency in your code. + +## About {{ .Metadata.Repo.NamePretty }} + +{{ if and .Metadata.Partials .Metadata.Partials.About }} +{{ .Metadata.Partials.About }} +{{ else }} +[{{ .Metadata.Repo.NamePretty }}][product-docs] {{ .Metadata.Repo.APIDescription }} + +See the [{{ .Metadata.Repo.NamePretty }} client library docs][javadocs] to learn how to +use this {{ .Metadata.Repo.NamePretty }} Client Library. +{{ end -}} +{{ if and .Metadata.Partials .Metadata.Partials.CustomContent }} + +{{ .Metadata.Partials.CustomContent }} +{{ end }} + +{{ if .Metadata.Samples }} +## Samples + +Samples are in the [`samples/`](https://github.com/{{ .Metadata.Repo.Repo }}/tree/main/samples) directory. + +| Sample | Source Code | Try it | +| --------------------------- | --------------------------------- | ------ | +{{ range .Metadata.Samples }}| {{ .Title }} | [source code](https://github.com/{{ $.Metadata.Repo.Repo }}/blob/main/{{ .File }}) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/{{ $.Metadata.Repo.Repo }}&page=editor&open_in_editor={{ .File }}) | +{{ end }} +{{ end }} + +{{ if not .Metadata.Samples }} + +{{ end -}} +## Troubleshooting + +To get help, follow the instructions in the [shared Troubleshooting document][troubleshooting]. + +{{ if .Metadata.Repo.Transport }}## Transport + +{{ if eq .Metadata.Repo.Transport "grpc" }}{{ .Metadata.Repo.NamePretty }} uses gRPC for the transport layer. +{{ else if eq .Metadata.Repo.Transport "http" }}{{ .Metadata.Repo.NamePretty }} uses HTTP/JSON for the transport layer. +{{ else if eq .Metadata.Repo.Transport "both" }}{{ .Metadata.Repo.NamePretty }} uses both gRPC and HTTP/JSON for the transport layer. +{{ end }} +{{ end }}## Supported Java Versions + +Java {{ .Metadata.MinJavaVersion }} or above is required for using this client. + +Google's Java client libraries, +[Google Cloud Client Libraries][cloudlibs] +and +[Google Cloud API Libraries][apilibs], +follow the +[Oracle Java SE support roadmap][oracle] +(see the Oracle Java SE Product Releases section). + +### For new development + +In general, new feature development occurs with support for the lowest Java +LTS version covered by Oracle's Premier Support (which typically lasts 5 years +from initial General Availability). If the minimum required JVM for a given +library is changed, it is accompanied by a [semver][semver] major release. + +Java 11 and (in September 2021) Java 17 are the best choices for new +development. + +### Keeping production systems current + +Google tests its client libraries with all current LTS versions covered by +Oracle's Extended Support (which typically lasts 8 years from initial +General Availability). + +#### Legacy support + +Google's client libraries support legacy versions of Java runtimes with long +term stable libraries that don't receive feature updates on a best efforts basis +as it may not be possible to backport all patches. + +Google provides updates on a best efforts basis to apps that continue to use +Java 7, though apps might need to upgrade to current versions of the library +that supports their JVM. + +#### Where to find specific information + +The latest versions and the supported Java versions are identified on +the individual GitHub repository `github.com/GoogleAPIs/java-SERVICENAME` +and on [google-cloud-java][g-c-j]. + +## Versioning + +{{- if and .Metadata.Partials .Metadata.Partials.Versioning }} +{{ .Metadata.Partials.Versioning }} +{{ else }} +This library follows [Semantic Versioning](http://semver.org/). + +{{ if eq .Metadata.Repo.ReleaseLevel "preview" }} +It is currently in major version zero (``0.y.z``), which means that anything may change at any time +and the public API should not be considered stable. +{{ end }}{{ end }} + +## Contributing + +{{ if and .Metadata.Partials .Metadata.Partials.Contributing }} +{{ .Metadata.Partials.Contributing }} +{{ else }} +Contributions to this library are always welcome and highly encouraged. + +See [CONTRIBUTING][contributing] for more information how to get started. + +Please note that this project is released with a Contributor Code of Conduct. By participating in +this project you agree to abide by its terms. See [Code of Conduct][code-of-conduct] for more +information. +{{ end }} + +## License + +Apache 2.0 - See [LICENSE][license] for more information. + +Java is a registered trademark of Oracle and/or its affiliates. + +[product-docs]: {{ .Metadata.Repo.ProductDocumentation }} +[javadocs]: {{ .Metadata.Repo.ClientDocumentation }} +[stability-image]: https://img.shields.io/badge/stability-{{ if eq .Metadata.Repo.ReleaseLevel "stable" }}stable-green{{ else if eq .Metadata.Repo.ReleaseLevel "preview" }}preview-yellow{{ else }}unknown-red{{ end }} +[maven-version-image]: https://img.shields.io/maven-central/v/{{ .GroupID }}/{{ .ArtifactID }}.svg +[maven-version-link]: https://central.sonatype.com/artifact/{{ .GroupID }}/{{ .ArtifactID }}/{{ .Version }} +[authentication]: https://github.com/googleapis/google-cloud-java#authentication +[auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes +[predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles +[iam-policy]: https://cloud.google.com/iam/docs/overview#cloud-iam-policy +[developer-console]: https://console.developers.google.com/ +[create-project]: https://cloud.google.com/resource-manager/docs/creating-managing-projects +[cloud-cli]: https://cloud.google.com/cli +[troubleshooting]: https://github.com/googleapis/google-cloud-java/blob/main/TROUBLESHOOTING.md +[contributing]: https://github.com/{{ .Metadata.Repo.Repo }}/blob/main/CONTRIBUTING.md +[code-of-conduct]: https://github.com/{{ .Metadata.Repo.Repo }}/blob/main/CODE_OF_CONDUCT.md#contributor-code-of-conduct +[license]: https://github.com/{{ .Metadata.Repo.Repo }}/blob/main/LICENSE +{{ if .Metadata.Repo.RequiresBilling }}[enable-billing]: https://cloud.google.com/apis/docs/getting-started#enabling_billing{{ end }} +{{ if .Metadata.Repo.APIID }}[enable-api]: https://console.cloud.google.com/flows/enableapi?apiid={{ .Metadata.Repo.APIID }}{{ end }} +[libraries-bom]: https://github.com/GoogleCloudPlatform/cloud-opensource-java/wiki/The-Google-Cloud-Platform-Libraries-BOM +[shell_img]: https://gstatic.com/cloudssh/images/open-btn.png + +[semver]: https://semver.org/ +[cloudlibs]: https://cloud.google.com/apis/docs/client-libraries-explained +[apilibs]: https://cloud.google.com/apis/docs/client-libraries-explained#google_api_client_libraries +[oracle]: https://www.oracle.com/java/technologies/java-se-support-roadmap.html +[g-c-j]: http://github.com/googleapis/google-cloud-java From 20875141693fa2453f9ad6d3de58dffea3386cdf Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Tue, 30 Jun 2026 14:48:45 -0400 Subject: [PATCH 098/108] refactor(internal/librarian/java): remove unreachable branch in trimLeadingWhitespace --- internal/librarian/java/readme.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/internal/librarian/java/readme.go b/internal/librarian/java/readme.go index 7ed2eec0052..17636dda0c5 100644 --- a/internal/librarian/java/readme.go +++ b/internal/librarian/java/readme.go @@ -409,11 +409,7 @@ func trimLeadingWhitespace(lines []string) string { sb.WriteString("\n") continue } - if len(line) >= minSpaces { - sb.WriteString(line[minSpaces:]) - } else { - sb.WriteString(line) - } + sb.WriteString(line[minSpaces:]) sb.WriteString("\n") } return sb.String() From 299c9bcc013877545b920fbd7451885483bf6171 Mon Sep 17 00:00:00 2001 From: Noah Dietz Date: Tue, 30 Jun 2026 13:25:05 -0700 Subject: [PATCH 099/108] refactor(java): simplify transport lookup (#6580) --- internal/librarian/generate.go | 2 +- internal/librarian/java/pom.go | 15 +++--- internal/librarian/java/pom_test.go | 13 +++-- internal/serviceconfig/api.go | 12 +++++ internal/serviceconfig/api_test.go | 68 +++++++++++++++++++++++++ internal/serviceconfig/serviceconfig.go | 48 ++++++++++------- 6 files changed, 123 insertions(+), 35 deletions(-) diff --git a/internal/librarian/generate.go b/internal/librarian/generate.go index 43833550b33..cc584e08ddd 100644 --- a/internal/librarian/generate.go +++ b/internal/librarian/generate.go @@ -233,7 +233,7 @@ func generateLibraries(ctx context.Context, cfg *config.Config, libraries []*con case config.LanguageJava: var allMissingArtifacts []java.MissingArtifact for _, library := range libraries { - missingArtifactIDs, err := java.IdentifyMissingModules(library, library.Output, src) + missingArtifactIDs, err := java.IdentifyMissingModules(library, library.Output) if err != nil { return fmt.Errorf("failed to identify missing modules for %q: %w", library.Name, err) } diff --git a/internal/librarian/java/pom.go b/internal/librarian/java/pom.go index 8a0abc3f790..3fdf6400686 100644 --- a/internal/librarian/java/pom.go +++ b/internal/librarian/java/pom.go @@ -26,7 +26,6 @@ import ( "github.com/googleapis/librarian/internal/config" "github.com/googleapis/librarian/internal/serviceconfig" - "github.com/googleapis/librarian/internal/sources" ) const ( @@ -103,14 +102,14 @@ type expectedModule struct { APICoords *APICoordinate } -func loadTransports(library *config.Library, googleapisDir string) (map[string]serviceconfig.Transport, error) { +func loadTransports(library *config.Library) (map[string]serviceconfig.Transport, error) { transports := make(map[string]serviceconfig.Transport) for _, api := range library.APIs { - apiCfg, err := serviceconfig.Find(googleapisDir, api.Path, config.LanguageJava) + transport, err := serviceconfig.FindTransport(api.Path, config.LanguageJava) if err != nil { - return nil, fmt.Errorf("failed to find api config for %s: %w", api.Path, err) + return nil, err } - transports[api.Path] = apiCfg.Transport(config.LanguageJava) + transports[api.Path] = transport } return transports, nil } @@ -263,10 +262,8 @@ func syncPOMs(library *config.Library, libraryDir, monorepoVersion string, metad // IdentifyMissingModules identifies all expected proto-*, grpc-*, client, BOM and Parent modules // for the given library based on its configuration and checks for pom.xml presence // on the filesystem. It returns a list of artifact IDs for the missing modules. -func IdentifyMissingModules(library *config.Library, libraryDir string, srcs *sources.Sources) ([]string, error) { - srcCfg := sources.NewSourceConfig(srcs, library.Roots) - primaryDir := srcCfg.Root(srcCfg.ActiveRoots[0]) - transports, err := loadTransports(library, primaryDir) +func IdentifyMissingModules(library *config.Library, libraryDir string) ([]string, error) { + transports, err := loadTransports(library) if err != nil { return nil, err } diff --git a/internal/librarian/java/pom_test.go b/internal/librarian/java/pom_test.go index 82163d05325..16de2c02de0 100644 --- a/internal/librarian/java/pom_test.go +++ b/internal/librarian/java/pom_test.go @@ -25,7 +25,6 @@ import ( "github.com/google/go-cmp/cmp/cmpopts" "github.com/googleapis/librarian/internal/config" "github.com/googleapis/librarian/internal/serviceconfig" - "github.com/googleapis/librarian/internal/sources" ) // update is used to refresh the golden files in testdata/ when template @@ -68,7 +67,7 @@ func TestSyncPOMs_Golden(t *testing.T) { NamePretty: "Secret Manager", APIDescription: "Stores sensitive data such as API keys, passwords, and certificates.\nProvides convenience while improving security.", } - gotVersions, err := IdentifyMissingModules(library, tmpDir, &sources.Sources{Googleapis: googleapisDir}) + gotVersions, err := IdentifyMissingModules(library, tmpDir) if err != nil { t.Fatal(err) } @@ -751,7 +750,7 @@ func TestIdentifyMissingModules(t *testing.T) { if test.setup != nil { test.setup(t, tmpDir) } - got, err := IdentifyMissingModules(library, tmpDir, &sources.Sources{Googleapis: googleapisDir}) + got, err := IdentifyMissingModules(library, tmpDir) if err != nil { t.Fatal(err) } @@ -777,7 +776,7 @@ func TestIdentifyMissingModules_SkipPOMUpdates(t *testing.T) { }, } tmpDir := t.TempDir() - got, err := IdentifyMissingModules(library, tmpDir, &sources.Sources{Googleapis: "invalid-dir"}) + got, err := IdentifyMissingModules(library, tmpDir) if err != nil { t.Fatal(err) } @@ -815,7 +814,7 @@ func TestIdentifyMissingModules_ExcludedPOMs(t *testing.T) { t.Fatal(err) } } - got, err := IdentifyMissingModules(library, tmpDir, &sources.Sources{Googleapis: googleapisDir}) + got, err := IdentifyMissingModules(library, tmpDir) if err != nil { t.Fatal(err) } @@ -861,7 +860,7 @@ func TestIdentifyMissingModules_GenerateProtoFalse(t *testing.T) { t.Fatal(err) } } - got, err := IdentifyMissingModules(library, tmpDir, &sources.Sources{Googleapis: googleapisDir}) + got, err := IdentifyMissingModules(library, tmpDir) if err != nil { t.Fatal(err) } @@ -907,7 +906,7 @@ func TestIdentifyMissingModules_GenerateGAPICFalse(t *testing.T) { t.Fatal(err) } } - got, err := IdentifyMissingModules(library, tmpDir, &sources.Sources{Googleapis: googleapisDir}) + got, err := IdentifyMissingModules(library, tmpDir) if err != nil { t.Fatal(err) } diff --git a/internal/serviceconfig/api.go b/internal/serviceconfig/api.go index 4302e653df2..14ea1013635 100644 --- a/internal/serviceconfig/api.go +++ b/internal/serviceconfig/api.go @@ -214,6 +214,18 @@ func HasAPIPath(path, language string) bool { return slices.Contains(api.Languages, config.LanguageAll) || slices.Contains(api.Languages, language) } +// FindTransport looks up the API by path in sdk.yaml, validates that it is +// allowed for the specified language, and returns its configured transport. +// If the API is not explicitly configured in sdk.yaml, it is assumed to be +// allowed and defaults to GRPCRest. +func FindTransport(path, language string) (Transport, error) { + api, err := findAPI(path, language) + if err != nil { + return "", err + } + return api.Transport(language), nil +} + func unmarshalAPIsOrPanic() []API { apis, err := yaml.Unmarshal[[]API](sdkYaml) if err != nil { diff --git a/internal/serviceconfig/api_test.go b/internal/serviceconfig/api_test.go index 070922f0ccf..3e57e8c5422 100644 --- a/internal/serviceconfig/api_test.go +++ b/internal/serviceconfig/api_test.go @@ -15,6 +15,7 @@ package serviceconfig import ( + "errors" "testing" "github.com/google/go-cmp/cmp" @@ -382,3 +383,70 @@ func TestRepoMetadataTransport(t *testing.T) { }) } } + +func TestFindTransport(t *testing.T) { + for _, test := range []struct { + name string + path string + language string + want Transport + }{ + { + name: "matching path and allowed language with custom transport", + path: "google/ads/admanager/v1", + language: config.LanguageJava, + want: Rest, + }, + { + name: "matching path and allowed language with default transport", + path: "google/ads/datamanager/v1", + language: config.LanguageGo, + want: GRPCRest, + }, + { + name: "unknown path defaults to GRPCRest", + path: "google/does/not/exist/v1", + language: config.LanguageGo, + want: GRPCRest, + }, + { + name: "empty path defaults to GRPCRest", + path: "", + language: config.LanguageGo, + want: GRPCRest, + }, + } { + t.Run(test.name, func(t *testing.T) { + got, err := FindTransport(test.path, test.language) + if err != nil { + t.Fatal(err) + } + if got != test.want { + t.Errorf("FindTransport(%q, %q) = %q, want %q", test.path, test.language, got, test.want) + } + }) + } +} + +func TestFindTransport_Error(t *testing.T) { + for _, test := range []struct { + name string + path string + language string + want error + }{ + { + name: "matching path but not allowed language", + path: "google/ads/admanager/v1", + language: config.LanguageGo, + want: errNotAllowed, + }, + } { + t.Run(test.name, func(t *testing.T) { + _, err := FindTransport(test.path, test.language) + if !errors.Is(err, test.want) { + t.Errorf("FindTransport(%q, %q) expected %v, got %v", test.path, test.language, test.want, err) + } + }) + } +} diff --git a/internal/serviceconfig/serviceconfig.go b/internal/serviceconfig/serviceconfig.go index 02c7d632e94..2707603aee1 100644 --- a/internal/serviceconfig/serviceconfig.go +++ b/internal/serviceconfig/serviceconfig.go @@ -45,6 +45,10 @@ type ( OAuthRequirements = serviceconfig.OAuthRequirements ) +var ( + errNotAllowed = errors.New("API is not allowlisted") +) + // Read reads a service config from a YAML file and returns it as a Service // proto. The file is parsed as YAML, converted to JSON, and then unmarshaled // into a Service proto. @@ -76,21 +80,13 @@ func Read(serviceConfigPath string) (*Service, error) { return cfg, nil } -// Find looks up the service config path and title override for a given API path, -// and validates that the API is allowed for the specified language. -// -// It first checks the API list for overrides and language restrictions, -// then searches for YAML files containing "type: google.api.Service", -// skipping any files ending in _gapic.yaml. -// -// The path should be relative to googleapisDir (e.g., "google/cloud/secretmanager/v1"). -// Returns an API struct with Path, ServiceConfig, and Title fields populated. -// ServiceConfig and Title may be empty strings if not found or not configured. -// -// The Showcase API ("schema/google/showcase/v1beta1") is a special case: -// it does not live under https://github.com/googleapis/googleapis. -// For this API only, googleapisDir should point to showcase source dir instead. -func Find(googleapisDir, path string, language string) (*API, error) { +// findAPI looks up the API by path in sdk.yaml and validates that it is +// allowed for the specified language. If the API is not explicitly +// configured in sdk.yaml, it is assumed to be allowed and an entry is returned. +func findAPI(path, language string) (*API, error) { + if path == "" { + return &API{}, nil + } var result *API for _, api := range APIs { // The path for OpenAPI and discovery documents are in @@ -105,9 +101,25 @@ func Find(googleapisDir, path string, language string) (*API, error) { break } } + return validateAPI(path, language, result) +} - var err error - result, err = validateAPI(path, language, result) +// Find looks up the service config path and title override for a given API path, +// and validates that the API is allowed for the specified language. +// +// It first checks the API list for overrides and language restrictions, +// then searches for YAML files containing "type: google.api.Service", +// skipping any files ending in _gapic.yaml. +// +// The path should be relative to googleapisDir (e.g., "google/cloud/secretmanager/v1"). +// Returns an API struct with Path, ServiceConfig, and Title fields populated. +// ServiceConfig and Title may be empty strings if not found or not configured. +// +// The Showcase API ("schema/google/showcase/v1beta1") is a special case: +// it does not live under https://github.com/googleapis/googleapis. +// For this API only, googleapisDir should point to showcase source dir instead. +func Find(googleapisDir, path string, language string) (*API, error) { + result, err := findAPI(path, language) if err != nil { return nil, err } @@ -222,7 +234,7 @@ func validateAPI(path string, language string, api *API) (*API, error) { return api, nil } } - return nil, fmt.Errorf("API %s is not allowed for language %s", path, language) + return nil, fmt.Errorf("%s for language %s: %w", path, language, errNotAllowed) } // isServiceConfigFile checks if the file contains "type: google.api.Service". From 96f6925fb95a81bdaa95e3405d4a09701a70178b Mon Sep 17 00:00:00 2001 From: yangyzs <171981480+yangyzs@users.noreply.github.com> Date: Tue, 30 Jun 2026 21:12:37 -0400 Subject: [PATCH 100/108] feat(internal/librarian/java): add code snippet extraction helpers for README rendering (#6593) Add collectSnippetFiles and extractSnippetsFromFile helpers to scan Java and XML sample files for code snippets. Add unit tests in readme_test.go. For #6515 --- internal/librarian/java/readme.go | 123 ++++++++++++++++ internal/librarian/java/readme_test.go | 195 +++++++++++++++++++++++++ 2 files changed, 318 insertions(+) diff --git a/internal/librarian/java/readme.go b/internal/librarian/java/readme.go index 8ca97817b8f..d1e6875c128 100644 --- a/internal/librarian/java/readme.go +++ b/internal/librarian/java/readme.go @@ -15,6 +15,7 @@ package java import ( + "bufio" "bytes" "errors" "fmt" @@ -26,6 +27,11 @@ import ( ) var ( + openSnippetRegex = regexp.MustCompile(`\[START ([a-zA-Z0-9_-]+)\]`) + closeSnippetRegex = regexp.MustCompile(`\[END ([a-zA-Z0-9_-]+)\]`) + openExcludeRegex = regexp.MustCompile(`\[START_EXCLUDE\]`) + closeExcludeRegex = regexp.MustCompile(`\[END_EXCLUDE\]`) + // Matches lowercase/digit followed by uppercase (e.g., "FooBar" -> "Foo Bar"). camelCaseRegexp = regexp.MustCompile(`([a-z0-9])([A-Z])`) @@ -40,6 +46,9 @@ var ( // errEmptyDir indicates the provided directory path is empty. errEmptyDir = errors.New("dir cannot be empty") + + // errEmptyFile indicates an empty file path was provided. + errEmptyFile = errors.New("file cannot be empty") ) // codeSample represents a discovered Java code sample along with its derived title. @@ -157,3 +166,117 @@ func extractTitle(filePath string) (string, error) { } return title, nil } + +// collectSnippetFiles recursively scans dir/samples for Java and XML files containing snippets. +func collectSnippetFiles(dir string) ([]string, error) { + samplesDir := filepath.Join(dir, "samples") + if _, err := os.Stat(samplesDir); err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + return nil, fmt.Errorf("failed to stat samples directory: %w", err) + } + var files []string + err := filepath.WalkDir(samplesDir, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + // Skip unit test directories and generated snippet output directories. + if d.Name() == "test" || (d.Name() == "generated" && filepath.Base(filepath.Dir(path)) == "snippets") { + return filepath.SkipDir + } + return nil + } + if !d.Type().IsRegular() { + return nil + } + // Include .xml files since non-POM configs (e.g., logback.xml) also contain snippets. + ext := filepath.Ext(path) + if ext == ".java" || ext == ".xml" { + files = append(files, path) + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("failed to walk samples directory: %w", err) + } + return files, nil +} + +// extractSnippetsFromFile parses a single file to return a map of tagged code snippets. +// Code between [START ] and [END ] tags is captured line by line. +// Any code inside [START_EXCLUDE] and [END_EXCLUDE] blocks is omitted. +// Example: +// +// Input file content: +// // [START my_snippet] +// void run() { +// // [START_EXCLUDE] +// secretInit(); +// // [END_EXCLUDE] +// doWork(); +// } +// // [END my_snippet] +// +// Resulting map entry for "my_snippet": +// void run() { +// doWork(); +// } +func extractSnippetsFromFile(file string) (map[string][]string, error) { + if file == "" { + return nil, errEmptyFile + } + f, err := os.Open(file) + if err != nil { + return nil, fmt.Errorf("failed to open file %s: %w", file, err) + } + defer f.Close() + scanner := bufio.NewScanner(f) + // 10 MB sanity limit to protect system memory. + scanner.Buffer(make([]byte, 64*1024), 10*1024*1024) + // Scan through file line by line and capture all open snippets. + // More than one open block might exist for a given line, so we + // need openSnippets to track currently active snippet blocks by name + snippetLines := make(map[string][]string) + openSnippets := make(map[string]bool) + excluding := false + for scanner.Scan() { + line := scanner.Text() + // Check for exclusion blocks first; code within EXCLUDE tags is completely skipped. + if openExcludeRegex.MatchString(line) { + excluding = true + continue + } + if closeExcludeRegex.MatchString(line) { + excluding = false + continue + } + if excluding { + continue + } + // Check for snippet start/end tags. Tag lines themselves are not saved. + openMatch := openSnippetRegex.FindStringSubmatch(line) + closeMatch := closeSnippetRegex.FindStringSubmatch(line) + if len(openMatch) > 1 { + name := openMatch[1] + openSnippets[name] = true + if _, exists := snippetLines[name]; !exists { + snippetLines[name] = nil + } + continue + } + if len(closeMatch) > 1 { + delete(openSnippets, closeMatch[1]) + continue + } + // Append this line of code to every snippet block currently open. + for s := range openSnippets { + snippetLines[s] = append(snippetLines[s], line) + } + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("failed scanning file %s: %w", file, err) + } + return snippetLines, nil +} diff --git a/internal/librarian/java/readme_test.go b/internal/librarian/java/readme_test.go index ea83f04e1d4..177622ad44b 100644 --- a/internal/librarian/java/readme_test.go +++ b/internal/librarian/java/readme_test.go @@ -16,6 +16,7 @@ package java import ( "errors" + "io/fs" "os" "path/filepath" "testing" @@ -476,3 +477,197 @@ func TestExtractTitle_Error(t *testing.T) { }) } } + +func TestCollectSnippetFiles(t *testing.T) { + for _, test := range []struct { + name string + setupFiles func(t *testing.T, dir string) + want []string + }{ + { + name: "missing samples directory returns nil", + setupFiles: func(t *testing.T, dir string) { + // Empty directory. + }, + want: nil, + }, + { + name: "collects java and xml files while ignoring excluded directories", + setupFiles: func(t *testing.T, dir string) { + validJavaDir := filepath.Join(dir, "samples", "src", "main", "java") + validXMLDir := filepath.Join(dir, "samples", "src", "main", "resources") + testDir := filepath.Join(dir, "samples", "src", "test", "java") + genDir := filepath.Join(dir, "samples", "snippets", "generated") + for _, d := range []string{validJavaDir, validXMLDir, testDir, genDir} { + if err := os.MkdirAll(d, 0755); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(validJavaDir, "Sample.java"), []byte("public class Sample {}"), 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(validXMLDir, "pom.xml"), []byte(""), 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(validJavaDir, "README.md"), []byte("# Ignore"), 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(testDir, "TestSample.java"), []byte("public class TestSample {}"), 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(genDir, "GenSnippet.java"), []byte("public class GenSnippet {}"), 0644); err != nil { + t.Fatal(err) + } + }, + want: []string{ + filepath.Join("samples", "src", "main", "java", "Sample.java"), + filepath.Join("samples", "src", "main", "resources", "pom.xml"), + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + tempDir := t.TempDir() + test.setupFiles(t, tempDir) + + got, err := collectSnippetFiles(tempDir) + if err != nil { + t.Fatal(err) + } + var relGot []string + for _, p := range got { + rel, err := filepath.Rel(tempDir, p) + if err != nil { + t.Fatal(err) + } + relGot = append(relGot, rel) + } + if diff := cmp.Diff(test.want, relGot); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestExtractSnippetsFromFile(t *testing.T) { + t.Parallel() + for _, test := range []struct { + name string + content string + want map[string][]string + }{ + { + name: "extracts snippets and respects exclude blocks", + content: `public class Example { + // [START my_snippet] + public void run() { + // [START_EXCLUDE] + secretInit(); + // [END_EXCLUDE] + doWork(); + } + // [END my_snippet] +}`, + want: map[string][]string{ + "my_snippet": { + " public void run() {", + " doWork();", + " }", + }, + }, + }, + { + name: "multiple independent snippets in single file", + content: `public class Multi { + // [START first] + void a() {} + // [END first] + // [START second] + void b() {} + // [END second] +}`, + want: map[string][]string{ + "first": { + " void a() {}", + }, + "second": { + " void b() {}", + }, + }, + }, + { + name: "nested snippets with exclude block", + content: `public class Nested { + // [START outer] + void start() { + // [START inner] + doInner(); + // [START_EXCLUDE] + logDebug(); + // [END_EXCLUDE] + finishInner(); + // [END inner] + } + // [END outer] +}`, + want: map[string][]string{ + "outer": { + " void start() {", + " doInner();", + " finishInner();", + " }", + }, + "inner": { + " doInner();", + " finishInner();", + }, + }, + }, + { + name: "no snippets in file", + content: "public class Simple {}", + want: map[string][]string{}, + }, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + tmpPath := filepath.Join(t.TempDir(), "SampleSnippet.java") + if err := os.WriteFile(tmpPath, []byte(test.content), 0644); err != nil { + t.Fatal(err) + } + got, err := extractSnippetsFromFile(tmpPath) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestExtractSnippetsFromFile_Error(t *testing.T) { + for _, test := range []struct { + name string + file string + wantErr error + }{ + { + name: "empty file parameter returns error", + file: "", + wantErr: errEmptyFile, + }, + // Triggers os.Open error when target file does not exist on disk. + { + name: "non-existent file returns error", + file: "non-existent-file.java", + wantErr: fs.ErrNotExist, + }, + } { + t.Run(test.name, func(t *testing.T) { + _, err := extractSnippetsFromFile(test.file) + if !errors.Is(err, test.wantErr) { + t.Errorf("extractSnippetsFromFile(%q) error = %v, wantErr %v", test.file, err, test.wantErr) + } + }) + } +} From b8e50a51a11cb07192f209c0e7fadeb5f3ce77c7 Mon Sep 17 00:00:00 2001 From: Min Zhu Date: Wed, 1 Jul 2026 08:55:07 -0400 Subject: [PATCH 101/108] fix(internal/librarian/java): exclude google-cloud-bom and libraries-bom when generating gapic-libraries-bom/pom.xml (#6601) Exclude google-cloud-bom and libraries-bom when generating gapic-libraries-bom/pom.xml. This is needed because https://github.com/googleapis/google-cloud-java/pull/13498 added new modules, among them are these two BOMs. Fixes https://github.com/googleapis/librarian/issues/6600 For https://github.com/googleapis/google-cloud-java/issues/13609 --- internal/librarian/java/postgenerate.go | 2 ++ internal/librarian/java/postgenerate_test.go | 19 +++++++++---------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/internal/librarian/java/postgenerate.go b/internal/librarian/java/postgenerate.go index 8c8f95f8f28..cdc6423e796 100644 --- a/internal/librarian/java/postgenerate.go +++ b/internal/librarian/java/postgenerate.go @@ -49,6 +49,8 @@ var ( // excludedBOMs is a set of artifact IDs to exclude from the generated GAPIC BOM. excludedBOMs = map[string]bool{ "google-cloud-bigtable-deps-bom": true, + "google-cloud-bom": true, + "libraries-bom": true, } ignoredDirs = map[string]bool{ gapicBOM: true, diff --git a/internal/librarian/java/postgenerate_test.go b/internal/librarian/java/postgenerate_test.go index e097fe25d5b..519bcb0da35 100644 --- a/internal/librarian/java/postgenerate_test.go +++ b/internal/librarian/java/postgenerate_test.go @@ -126,16 +126,15 @@ func verifyBOM(t *testing.T, path string, wantVersion string, wantDeps []bomDepe t.Errorf("mismatch (-want +got):\n%s", diff) } // Verify that libraries like java-maps-places are excluded because their - // GroupID (com.google.maps) is not in the allowed groupInclusions list. - if slices.ContainsFunc(p.Dependencies, func(d bomDependency) bool { - return d.ArtifactID == "google-maps-places-bom" - }) { - t.Errorf("%s should NOT contain google-maps-places-bom", path) - } - if slices.ContainsFunc(p.Dependencies, func(d bomDependency) bool { - return d.ArtifactID == "google-cloud-bigtable-deps-bom" - }) { - t.Errorf("%s should NOT contain google-cloud-bigtable-deps-bom", path) + // GroupID (com.google.maps) is not in the allowed groupInclusions list, and + // other BOMs explicitly excluded in excludedBOMs are also absent. + for _, excluded := range []string{"google-maps-places-bom", + "google-cloud-bigtable-deps-bom", "google-cloud-bom", "libraries-bom"} { + if slices.ContainsFunc(p.Dependencies, func(d bomDependency) bool { + return d.ArtifactID == excluded + }) { + t.Errorf("%s should NOT contain %s", path, excluded) + } } } From 3f6cfed0b7d4c24c2d5aaa4750b4cc32db34cc51 Mon Sep 17 00:00:00 2001 From: Santiago Quiroga <22756465+quirogas@users.noreply.github.com> Date: Wed, 1 Jul 2026 07:06:26 -0700 Subject: [PATCH 102/108] feat(internal/librarian/nodejs): add metadata_name_override and name_pretty_override support (#6603) This pull request adds metadata_name_override and name_pretty_override configuration fields to NodejsPackage in librarian.yaml . Currently, Librarian infers name and name_pretty in `.repo-metadata.json` directly from the backend service config ( api.ShortName and api.Title ). When multiple packages share a single service authority (e.g., @google-cloud/dialogflow and @google-cloud/dialogflow-cx both generating from dialogflow.googleapis.com ), Librarian overwrites both `.repo-metadata.json` files with identical values ( "dialogflow" / "Dialogflow" ), causing config collisions. Adding these override fields aligns Node.js with Java ( name_pretty_override ) and Python ( metadata_name_override ), allowing packages with shared service authorities to explicitly preserve their unique metadata names. For #6453 --- doc/config-schema.md | 2 ++ internal/config/language.go | 6 ++++++ internal/librarian/library.go | 6 ++++++ internal/librarian/nodejs/generate_test.go | 22 ++++++++++++++++++++++ internal/librarian/nodejs/repometadata.go | 12 ++++++++++-- 5 files changed, 46 insertions(+), 2 deletions(-) diff --git a/doc/config-schema.md b/doc/config-schema.md index 351c8d34076..4d507c0ac26 100644 --- a/doc/config-schema.md +++ b/doc/config-schema.md @@ -364,6 +364,8 @@ This document describes the schema for the librarian.yaml. | `nodejs_apis` | list of [NodejsAPI](#nodejsapi-configuration) (optional) | Is a list of Node.js-specific API configurations. | | `package_name` | string | Is the npm package name (e.g., "@google-cloud/access-approval"). | | `client_documentation_override` | string | Allows the client_documentation field in .repo-metadata.json to be overridden from the default that's inferred. | +| `metadata_name_override` | string | Allows the name field in .repo-metadata.json to be overridden. | +| `name_pretty_override` | string | Allows the name_pretty field in .repo-metadata.json to be overridden. | ## PythonDefault Configuration diff --git a/internal/config/language.go b/internal/config/language.go index 7a2d803cdba..bd72cb1e373 100644 --- a/internal/config/language.go +++ b/internal/config/language.go @@ -777,6 +777,12 @@ type NodejsPackage struct { // ClientDocumentationOverride allows the client_documentation field in // .repo-metadata.json to be overridden from the default that's inferred. ClientDocumentationOverride string `yaml:"client_documentation_override,omitempty"` + + // MetadataNameOverride allows the name field in .repo-metadata.json to be overridden. + MetadataNameOverride string `yaml:"metadata_name_override,omitempty"` + + // NamePrettyOverride allows the name_pretty field in .repo-metadata.json to be overridden. + NamePrettyOverride string `yaml:"name_pretty_override,omitempty"` } // NodejsAPI represents configuration for a single API within a Node.js package. diff --git a/internal/librarian/library.go b/internal/librarian/library.go index d3304620711..140ab30e849 100644 --- a/internal/librarian/library.go +++ b/internal/librarian/library.go @@ -674,6 +674,12 @@ func mergeNodejs(dst, src *config.NodejsPackage) *config.NodejsPackage { if src.ClientDocumentationOverride != "" { res.ClientDocumentationOverride = src.ClientDocumentationOverride } + if src.MetadataNameOverride != "" { + res.MetadataNameOverride = src.MetadataNameOverride + } + if src.NamePrettyOverride != "" { + res.NamePrettyOverride = src.NamePrettyOverride + } return &res } diff --git a/internal/librarian/nodejs/generate_test.go b/internal/librarian/nodejs/generate_test.go index 9c31969dd0f..db3f7a56a9b 100644 --- a/internal/librarian/nodejs/generate_test.go +++ b/internal/librarian/nodejs/generate_test.go @@ -1125,6 +1125,28 @@ func TestWriteRepoMetadata(t *testing.T) { return w }, }, + { + name: "metadata name and name pretty overrides", + library: &config.Library{ + Name: "google-cloud-dialogflow-cx", + APIs: []*config.API{{Path: "google/cloud/secretmanager/v1"}}, + Nodejs: &config.NodejsPackage{ + MetadataNameOverride: "dialogflow-cx", + NamePrettyOverride: "Dialogflow CX API", + }, + }, + want: func() *repometadata.RepoMetadata { + w := sample.RepoMetadata() + w.DistributionName = "@google-cloud/dialogflow-cx" + w.Language = cfg.Language + w.Repo = cfg.Repo + w.ClientDocumentation = "https://cloud.google.com/nodejs/docs/reference/dialogflow-cx/latest" + w.ProductDocumentation = "https://cloud.google.com/secret-manager/docs" + w.Name = "dialogflow-cx" + w.NamePretty = "Dialogflow CX API" + return w + }, + }, } { t.Run(test.name, func(t *testing.T) { outDir := t.TempDir() diff --git a/internal/librarian/nodejs/repometadata.go b/internal/librarian/nodejs/repometadata.go index ed898366b4b..8457c792b32 100644 --- a/internal/librarian/nodejs/repometadata.go +++ b/internal/librarian/nodejs/repometadata.go @@ -35,8 +35,16 @@ func generateRepoMetadata(cfg *config.Config, library *config.Library, googleapi metadata.ClientDocumentation = fmt.Sprintf("https://cloud.google.com/nodejs/docs/reference/%s/latest", pkgSuffix) } - if library.Nodejs != nil && library.Nodejs.ClientDocumentationOverride != "" { - metadata.ClientDocumentation = library.Nodejs.ClientDocumentationOverride + if library.Nodejs != nil { + if library.Nodejs.ClientDocumentationOverride != "" { + metadata.ClientDocumentation = library.Nodejs.ClientDocumentationOverride + } + if library.Nodejs.MetadataNameOverride != "" { + metadata.Name = library.Nodejs.MetadataNameOverride + } + if library.Nodejs.NamePrettyOverride != "" { + metadata.NamePretty = library.Nodejs.NamePrettyOverride + } } if strings.HasPrefix(metadata.ProductDocumentation, "https://cloud.google.com/") { From 5a44ed7b04f06fefcb1cf78c2e46b0f8bedeb035 Mon Sep 17 00:00:00 2001 From: Carlos O'Ryan Date: Wed, 1 Jul 2026 10:42:33 -0400 Subject: [PATCH 103/108] feat(librarian/swift): use discovery config (#6604) Use the discovery configuration to generate `getOperation()` mixins in discovery-based APIs. --- internal/librarian/swift/generate.go | 25 ++++++++++--- internal/librarian/swift/generate_test.go | 44 +++++++++++++++++++++++ internal/sidekick/api/xref.go | 12 +++---- internal/sidekick/api/xref_test.go | 18 ++++++---- 4 files changed, 80 insertions(+), 19 deletions(-) diff --git a/internal/librarian/swift/generate.go b/internal/librarian/swift/generate.go index a5ca234e2b1..9d82ec2fe85 100644 --- a/internal/librarian/swift/generate.go +++ b/internal/librarian/swift/generate.go @@ -23,6 +23,7 @@ import ( "github.com/googleapis/librarian/internal/command" "github.com/googleapis/librarian/internal/config" "github.com/googleapis/librarian/internal/serviceconfig" + "github.com/googleapis/librarian/internal/sidekick/api" "github.com/googleapis/librarian/internal/sidekick/parser" sidekickswift "github.com/googleapis/librarian/internal/sidekick/swift" "github.com/googleapis/librarian/internal/sources" @@ -74,8 +75,8 @@ func camelLibraryName(api string) string { return name.String() } -func libraryToModelConfig(library *config.Library, api *config.API, src *sources.Sources) (*parser.ModelConfig, error) { - svcConfig, err := serviceconfig.Find(src.Googleapis, api.Path, config.LanguageSwift) +func libraryToModelConfig(library *config.Library, apiCfg *config.API, src *sources.Sources) (*parser.ModelConfig, error) { + svcConfig, err := serviceconfig.Find(src.Googleapis, apiCfg.Path, config.LanguageSwift) if err != nil { return nil, err } @@ -89,15 +90,29 @@ func libraryToModelConfig(library *config.Library, api *config.API, src *sources specFormat = library.SpecificationFormat } - return &parser.ModelConfig{ + modelCfg := &parser.ModelConfig{ Language: config.LanguageSwift, SpecificationFormat: specFormat, ServiceConfig: svcConfig.ServiceConfig, - SpecificationSource: api.Path, + SpecificationSource: apiCfg.Path, Source: sourceConfig, Codec: map[string]string{ "copyright-year": library.CopyrightYear, "version": library.Version, }, - }, nil + } + if library.Swift != nil && library.Swift.Discovery != nil { + pollers := make([]*api.Poller, len(library.Swift.Discovery.Pollers)) + for i, poller := range library.Swift.Discovery.Pollers { + pollers[i] = &api.Poller{ + Prefix: poller.Prefix, + MethodID: poller.MethodID, + } + } + modelCfg.Discovery = &api.Discovery{ + OperationID: library.Swift.Discovery.OperationID, + Pollers: pollers, + } + } + return modelCfg, nil } diff --git a/internal/librarian/swift/generate_test.go b/internal/librarian/swift/generate_test.go index 3fb948d47b5..e4abae4766b 100644 --- a/internal/librarian/swift/generate_test.go +++ b/internal/librarian/swift/generate_test.go @@ -21,6 +21,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/googleapis/librarian/internal/config" + "github.com/googleapis/librarian/internal/sidekick/api" "github.com/googleapis/librarian/internal/sidekick/parser" "github.com/googleapis/librarian/internal/sources" "github.com/googleapis/librarian/internal/testhelper" @@ -207,6 +208,49 @@ func TestLibraryToModelConfig(t *testing.T) { }, }, }, + { + name: "discovery config with LRO", + library: &config.Library{ + Name: "google-cloud-compute-v1", + CopyrightYear: "2038", + Version: "1.2.3", + Roots: []string{"discovery", "googleapis"}, + SpecificationFormat: config.SpecDiscovery, + Swift: &config.SwiftPackage{ + Discovery: &config.SwiftDiscovery{ + OperationID: ".google.cloud.compute.v1.", + Pollers: []config.SwiftPoller{ + { + Prefix: "compute/v1/projects/{project}/zones/{zone}", + MethodID: ".google.cloud.compute.v1.zoneOperations.get", + }, + }, + }, + }, + }, + api: &config.API{ + Path: "discoveries/compute.v1.json", + }, + want: &parser.ModelConfig{ + Language: config.LanguageSwift, + SpecificationFormat: config.SpecDiscovery, + SpecificationSource: "discoveries/compute.v1.json", + ServiceConfig: "google/cloud/compute/v1/compute_v1.yaml", + Codec: map[string]string{"copyright-year": "2038", "version": "1.2.3"}, + Source: &sources.SourceConfig{ + ActiveRoots: []string{"discovery", "googleapis"}, + }, + Discovery: &api.Discovery{ + OperationID: ".google.cloud.compute.v1.", + Pollers: []*api.Poller{ + { + Prefix: "compute/v1/projects/{project}/zones/{zone}", + MethodID: ".google.cloud.compute.v1.zoneOperations.get", + }, + }, + }, + }, + }, } { t.Run(test.name, func(t *testing.T) { srcs := &sources.Sources{ diff --git a/internal/sidekick/api/xref.go b/internal/sidekick/api/xref.go index 958a73624d7..44180fe9f31 100644 --- a/internal/sidekick/api/xref.go +++ b/internal/sidekick/api/xref.go @@ -361,11 +361,11 @@ func sortOneOfFieldForExamples(f1, f2 *Field) int { } func enrichMethodSamples(m *Method) { - m.IsSimple = m.Pagination == nil && - !m.ClientSideStreaming && !m.ServerSideStreaming && - m.OperationInfo == nil && m.DiscoveryLro == nil - - m.IsLRO = m.OperationInfo != nil + // Methods with AIP-151 LRO annotations *OR* discovery LRO annotations are LROs. + m.IsLRO = m.OperationInfo != nil || m.DiscoveryLro != nil + m.IsStreaming = m.ClientSideStreaming || m.ServerSideStreaming + // A simple method is not paginated, not streaming and not an LRO. + m.IsSimple = m.Pagination == nil && !m.IsStreaming && !m.IsLRO if m.SourceServiceID == ".google.longrunning.Operations" && m.Name == "GetOperation" && @@ -381,8 +381,6 @@ func enrichMethodSamples(m *Method) { m.IsList = m.OutputType != nil && m.OutputType.Pagination != nil - m.IsStreaming = m.ClientSideStreaming || m.ServerSideStreaming - if m.SampleInfo = aipStandardGetInfo(m); m.SampleInfo != nil { m.IsAIPStandardGet = true } else if m.SampleInfo = aipStandardDeleteInfo(m); m.SampleInfo != nil { diff --git a/internal/sidekick/api/xref_test.go b/internal/sidekick/api/xref_test.go index 14880aa7693..5e388a9572d 100644 --- a/internal/sidekick/api/xref_test.go +++ b/internal/sidekick/api/xref_test.go @@ -672,7 +672,7 @@ func TestIsSimpleMethod(t *testing.T) { } func TestIsLRO(t *testing.T) { - testCases := []struct { + for _, test := range []struct { name string method *Method want bool @@ -687,12 +687,16 @@ func TestIsLRO(t *testing.T) { method: &Method{OperationInfo: &OperationInfo{}}, want: true, }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - enrichMethodSamples(tc.method) - if got := tc.method.IsLRO; got != tc.want { - t.Errorf("IsLRO() = %v, want %v", got, tc.want) + { + name: "LRO method is discovery LRO", + method: &Method{DiscoveryLro: &DiscoveryLro{}}, + want: true, + }, + } { + t.Run(test.name, func(t *testing.T) { + enrichMethodSamples(test.method) + if got := test.method.IsLRO; got != test.want { + t.Errorf("IsLRO() = %v, want %v", got, test.want) } }) } From 0efb6e71ddbdb5fd6d121a02ecbf02138235e102 Mon Sep 17 00:00:00 2001 From: Noah Dietz Date: Wed, 1 Jul 2026 09:35:50 -0700 Subject: [PATCH 104/108] fix(sdk.yaml): allow rust for many non-cloud apis (#6598) --- internal/serviceconfig/api_test.go | 2 +- internal/serviceconfig/sdk.yaml | 147 +---------------------------- 2 files changed, 6 insertions(+), 143 deletions(-) diff --git a/internal/serviceconfig/api_test.go b/internal/serviceconfig/api_test.go index 3e57e8c5422..a2896920cc0 100644 --- a/internal/serviceconfig/api_test.go +++ b/internal/serviceconfig/api_test.go @@ -50,7 +50,7 @@ func TestHasAPIPath(t *testing.T) { want bool }{ {"matching path and language", "google/cloud/accessapproval/v1", config.LanguageRust, true}, - {"matching path but not language", "google/ads/admanager/v1", config.LanguageRust, false}, + {"matching path but not language", "google/ads/admanager/v1", config.LanguageFake, false}, {"unknown path", "google/does/not/exist/v1", config.LanguageRust, false}, {"empty path", "", config.LanguageRust, false}, } { diff --git a/internal/serviceconfig/sdk.yaml b/internal/serviceconfig/sdk.yaml index 8a314c151d9..4d6e49b9f28 100644 --- a/internal/serviceconfig/sdk.yaml +++ b/internal/serviceconfig/sdk.yaml @@ -16,16 +16,12 @@ - java - nodejs - python + - rust release_level: java: preview transports: all: rest - path: google/ads/datamanager/v1 - languages: - - go - - java - - nodejs - - python release_level: java: preview - path: google/ai/generativelanguage/v1 @@ -148,15 +144,11 @@ languages: - java - python + - rust skip_rest_numeric_enums: - python title: Google Apps Card - path: google/apps/events/subscriptions/v1 - languages: - - go - - java - - nodejs - - python release_level: java: preview - path: google/apps/events/subscriptions/v1beta @@ -168,11 +160,6 @@ release_level: java: preview - path: google/apps/meet/v2 - languages: - - go - - java - - nodejs - - python release_level: java: preview - path: google/apps/meet/v2beta @@ -273,11 +260,6 @@ transports: java: grpc - path: google/chat/v1 - languages: - - go - - java - - nodejs - - python release_level: java: preview - path: google/cloud @@ -2807,6 +2789,7 @@ languages: - java - python + - rust skip_rest_numeric_enums: - python transports: @@ -2921,39 +2904,19 @@ go: stable - path: google/maps/addressvalidation/v1 documentation_uri: https://mapsplatform.google.com/maps-products/address-validation/ - languages: - - go - - java - - nodejs - - python release_level: java: preview transports: csharp: grpc ruby: grpc - path: google/maps/areainsights/v1 - languages: - - go - - java - - nodejs - - python release_level: go: preview java: preview - path: google/maps/fleetengine/delivery/v1 - languages: - - go - - java - - nodejs - - python release_level: java: preview - path: google/maps/fleetengine/v1 - languages: - - go - - java - - nodejs - - python release_level: java: preview transports: @@ -2961,12 +2924,6 @@ java: grpc nodejs: grpc python: grpc -- path: google/maps/geocode/v4 - languages: - - go - - java - - nodejs - - python - path: google/maps/mapmanagement/v2beta languages: - go @@ -2974,10 +2931,6 @@ - nodejs - python - path: google/maps/mapsplatformdatasets/v1 - languages: - - java - - nodejs - - python release_level: java: preview - path: google/maps/navconnect/v1 @@ -2985,38 +2938,18 @@ - nodejs - python - path: google/maps/places/v1 - languages: - - go - - java - - nodejs - - python sample_uris: go: https://developers.google.com/maps/documentation/places/web-service/client-library-examples#go release_level: go: preview java: preview - path: google/maps/routeoptimization/v1 - languages: - - go - - java - - nodejs - - python release_level: go: preview java: preview - path: google/maps/routing/v2 documentation_uri: https://mapsplatform.google.com/maps-products/#routes-section - languages: - - go - - java - - nodejs - - python - path: google/maps/solar/v1 - languages: - - go - - java - - nodejs - - python release_level: java: preview - path: google/marketingplatform/admin/v1alpha @@ -3080,108 +3013,54 @@ release_level: go: preview java: preview -- path: google/shopping/merchant/accounts/v1 - languages: - - go - - java - - nodejs - - python - path: google/shopping/merchant/accounts/v1beta languages: - go - java - nodejs - python -- path: google/shopping/merchant/conversions/v1 - languages: - - go - - java - - nodejs - - python - path: google/shopping/merchant/conversions/v1beta languages: - go - java - nodejs - python -- path: google/shopping/merchant/datasources/v1 - languages: - - go - - java - - nodejs - - python - path: google/shopping/merchant/datasources/v1beta languages: - go - java - nodejs - python -- path: google/shopping/merchant/inventories/v1 - languages: - - go - - java - - nodejs - - python - path: google/shopping/merchant/inventories/v1beta languages: - go - java - nodejs - python -- path: google/shopping/merchant/issueresolution/v1 - languages: - - go - - java - - nodejs - - python - path: google/shopping/merchant/issueresolution/v1beta languages: - go - java - nodejs - python -- path: google/shopping/merchant/lfp/v1 - languages: - - go - - java - - nodejs - - python - path: google/shopping/merchant/lfp/v1beta languages: - go - java - nodejs - python -- path: google/shopping/merchant/notifications/v1 - languages: - - go - - java - - nodejs - - python - path: google/shopping/merchant/notifications/v1beta languages: - go - java - nodejs - python -- path: google/shopping/merchant/ordertracking/v1 - languages: - - go - - java - - nodejs - - python - path: google/shopping/merchant/ordertracking/v1beta languages: - go - java - nodejs - python -- path: google/shopping/merchant/products/v1 - languages: - - go - - java - - nodejs - - python - path: google/shopping/merchant/products/v1beta languages: - go @@ -3195,36 +3074,18 @@ - python release_level: java: preview -- path: google/shopping/merchant/promotions/v1 - languages: - - go - - java - - nodejs - - python - path: google/shopping/merchant/promotions/v1beta languages: - go - java - nodejs - python -- path: google/shopping/merchant/quota/v1 - languages: - - go - - java - - nodejs - - python - path: google/shopping/merchant/quota/v1beta languages: - go - java - nodejs - python -- path: google/shopping/merchant/reports/v1 - languages: - - go - - java - - nodejs - - python - path: google/shopping/merchant/reports/v1alpha languages: - java @@ -3253,6 +3114,7 @@ - go - java - python + - rust skip_rest_numeric_enums: - python title: Shopping Common Types @@ -3289,6 +3151,7 @@ languages: - go - nodejs + - rust - path: grafeas/v1 documentation_uri: https://grafeas.io languages: From 003b443739809302d9b43844dd95ce358a2eab1a Mon Sep 17 00:00:00 2001 From: sofisl <55454395+sofisl@users.noreply.github.com> Date: Wed, 1 Jul 2026 10:54:39 -0700 Subject: [PATCH 105/108] refactor(internal/config): remove deprecated libraries_bom_version from JavaModule (#6584) Finalizes the migration to JavaDefault by removing the deprecated libraries_bom_version field from the JavaModule struct, its merging logic, and its library-level fallback check in the generator. This is backwards compatibile because it is only defined once at the global default.java level in librarian.yaml in google-cloud-java. Follow up to https://github.com/googleapis/librarian/pull/6447. Fixes https://github.com/googleapis/librarian/issues/5171 --- doc/config-schema.md | 1 - internal/config/language.go | 3 --- internal/librarian/java/generate.go | 5 +---- internal/librarian/java/postprocess.go | 2 +- internal/librarian/library.go | 3 --- internal/librarian/library_test.go | 2 -- 6 files changed, 2 insertions(+), 14 deletions(-) diff --git a/doc/config-schema.md b/doc/config-schema.md index 4d507c0ac26..0db8c097a5f 100644 --- a/doc/config-schema.md +++ b/doc/config-schema.md @@ -325,7 +325,6 @@ This document describes the schema for the librarian.yaml. | `extra_versioned_modules` | string | Is a list of extra versioned modules. | | `group_id` | string | Is the Maven group ID, defaults to "com.google.cloud". | | `issue_tracker_override` | string | Allows the "issue_tracker" field in .repo-metadata.json to be overridden. | -| `libraries_bom_version` | string | Is the version of the libraries-bom to use for Java. | | `released_version` | string | Is the last released version of the library. If omitted, it will be derived from the library version. Note: It assumes a minor bump from the previous '.0' version (e.g., '1.2.0-SNAPSHOT' -> '1.1.0') and does not support deriving previous patch releases (e.g., '1.1.1'). | | `library_type_override` | string | Allows the "library_type" field in .repo-metadata.json to be overridden. | | `min_java_version` | int | Is the minimum Java version required. | diff --git a/internal/config/language.go b/internal/config/language.go index bd72cb1e373..d6d85781f82 100644 --- a/internal/config/language.go +++ b/internal/config/language.go @@ -512,9 +512,6 @@ type JavaModule struct { // to be overridden. IssueTrackerOverride string `yaml:"issue_tracker_override,omitempty"` - // LibrariesBOMVersion is the version of the libraries-bom to use for Java. - LibrariesBOMVersion string `yaml:"libraries_bom_version,omitempty"` - // ReleasedVersion is the last released version of the library. // If omitted, it will be derived from the library version. // Note: It assumes a minor bump from the previous '.0' version diff --git a/internal/librarian/java/generate.go b/internal/librarian/java/generate.go index 26972398e5c..9faf88c071f 100644 --- a/internal/librarian/java/generate.go +++ b/internal/librarian/java/generate.go @@ -337,10 +337,7 @@ func gapicOpt(key, value string) string { // TODO(https://github.com/googleapis/librarian/issues/5152): // BOM version should be required and pre-validated, remove this and inline when done. -func findBOMVersion(cfg *config.Config, library *config.Library) (string, error) { - if library != nil && library.Java != nil && library.Java.LibrariesBOMVersion != "" { - return library.Java.LibrariesBOMVersion, nil - } +func findBOMVersion(cfg *config.Config) (string, error) { if cfg.Default != nil && cfg.Default.Java != nil && cfg.Default.Java.LibrariesBOMVersion != "" { return cfg.Default.Java.LibrariesBOMVersion, nil } diff --git a/internal/librarian/java/postprocess.go b/internal/librarian/java/postprocess.go index 1c396a6d37a..bfeb28715dc 100644 --- a/internal/librarian/java/postprocess.go +++ b/internal/librarian/java/postprocess.go @@ -71,7 +71,7 @@ func postProcessLibrary(ctx context.Context, params libraryPostProcessParams) er if err := createOrVerifyOwlbotPy(params.outDir); err != nil { return err } - bomVersion, err := findBOMVersion(params.cfg, params.library) + bomVersion, err := findBOMVersion(params.cfg) if err != nil { return err } diff --git a/internal/librarian/library.go b/internal/librarian/library.go index 140ab30e849..8fbae9a3671 100644 --- a/internal/librarian/library.go +++ b/internal/librarian/library.go @@ -600,9 +600,6 @@ func mergeJava(dst, src *config.JavaModule) *config.JavaModule { if src.IssueTrackerOverride != "" { res.IssueTrackerOverride = src.IssueTrackerOverride } - if src.LibrariesBOMVersion != "" { - res.LibrariesBOMVersion = src.LibrariesBOMVersion - } if src.ReleasedVersion != "" { res.ReleasedVersion = src.ReleasedVersion } diff --git a/internal/librarian/library_test.go b/internal/librarian/library_test.go index 464e97654d4..d052702b83c 100644 --- a/internal/librarian/library_test.go +++ b/internal/librarian/library_test.go @@ -1262,7 +1262,6 @@ func TestMergeJava(t *testing.T) { ExtraVersionedModules: "extra", GroupID: "com.new", IssueTrackerOverride: "issue", - LibrariesBOMVersion: "bom", LibraryTypeOverride: "type", MinJavaVersion: 11, NamePrettyOverride: "pretty", @@ -1288,7 +1287,6 @@ func TestMergeJava(t *testing.T) { ExtraVersionedModules: "extra", GroupID: "com.new", IssueTrackerOverride: "issue", - LibrariesBOMVersion: "bom", LibraryTypeOverride: "type", MinJavaVersion: 11, NamePrettyOverride: "pretty", From accb8adb99c8905d91489d59e6ed112159630874 Mon Sep 17 00:00:00 2001 From: Joe Wang <106995533+JoeWang1127@users.noreply.github.com> Date: Wed, 1 Jul 2026 18:17:23 +0000 Subject: [PATCH 106/108] feat(internal/librarian): populate Java Maven coordinates from defaults (#6554) The library configuration loader is updated to automatically populate Java Maven coordinates (GroupID and ArtifactID) based on API path prefix mappings defined in default configuration. When a library's Java module configuration is missing or its GroupID is empty, the loader scans the library's API paths against the default api_path_to_group_id map to resolve the appropriate GroupID and sets the ArtifactID using the standard "google-" naming convention. Add default mappings in follow up PRs. For #6513 --- doc/config-schema.md | 1 + internal/config/language.go | 5 + internal/librarian/library.go | 46 +++++-- internal/librarian/library_test.go | 211 +++++++++++++++++++++++++++++ 4 files changed, 253 insertions(+), 10 deletions(-) diff --git a/doc/config-schema.md b/doc/config-schema.md index 0db8c097a5f..01b762b7f58 100644 --- a/doc/config-schema.md +++ b/doc/config-schema.md @@ -299,6 +299,7 @@ This document describes the schema for the librarian.yaml. | Field | Type | Description | | :--- | :--- | :--- | +| `custom_group_ids` | map[string]string | Maps API path prefixes (e.g., "google/shopping") to their corresponding Maven Group IDs (e.g., "com.google.shopping"). Use this to override the default "com.google.cloud" Group ID for specific API paths (e.g., maps, ads, shopping). | | `libraries_bom_version` | string | Is the version of the libraries-bom to use for Java. | ## JavaFileCopy Configuration diff --git a/internal/config/language.go b/internal/config/language.go index d6d85781f82..66660f22bc9 100644 --- a/internal/config/language.go +++ b/internal/config/language.go @@ -458,6 +458,11 @@ type DartPackage struct { // JavaDefault contains Java-specific default configuration. type JavaDefault struct { + // CustomGroupIDs maps API path prefixes (e.g., "google/shopping") to their + // corresponding Maven Group IDs (e.g., "com.google.shopping"). + // Use this to override the default "com.google.cloud" Group ID for specific API + // paths (e.g., maps, ads, shopping). + CustomGroupIDs map[string]string `yaml:"custom_group_ids,omitempty"` // LibrariesBOMVersion is the version of the libraries-bom to use for Java. LibrariesBOMVersion string `yaml:"libraries_bom_version,omitempty"` } diff --git a/internal/librarian/library.go b/internal/librarian/library.go index 8fbae9a3671..85798e95006 100644 --- a/internal/librarian/library.go +++ b/internal/librarian/library.go @@ -44,22 +44,22 @@ func fillDefaults(lib *config.Library, d *config.Default) *config.Library { if lib.Output == "" { lib.Output = d.Output } - if d.Go != nil { + switch { + case d.Go != nil: return fillGo(lib, d) - } - if d.Rust != nil { + case d.Java != nil: + return fillJava(lib, d) + case d.Rust != nil: return fillRust(lib, d) - } - if d.Dart != nil { + case d.Dart != nil: return fillDart(lib, d) - } - if d.Python != nil { + case d.Python != nil: return fillPython(lib, d) - } - if d.Swift != nil { + case d.Swift != nil: return fillSwift(lib, d) + default: + return lib } - return lib } // fillGo populates empty Go-specific fields in lib from the provided default. @@ -97,6 +97,32 @@ func union(a, b []string) []string { return res } +// fillJava populates empty Java-specific fields in lib from the provided default. +func fillJava(lib *config.Library, d *config.Default) *config.Library { + if lib.Java == nil { + lib.Java = &config.JavaModule{} + } + fillGroupIDIfEmpty(lib, d) + return lib +} + +// fillGroupIDIfEmpty sets the Java group ID on lib if one is not already configured. +// It matches the library's API paths against the custom group ID prefixes in default +// and assigns the first matching group ID. +func fillGroupIDIfEmpty(lib *config.Library, d *config.Default) { + if lib.Java.GroupID != "" || d.Java.CustomGroupIDs == nil { + return + } + for _, api := range lib.APIs { + for apiPrefix, groupID := range d.Java.CustomGroupIDs { + if api.Path == apiPrefix || strings.HasPrefix(api.Path, apiPrefix+"/") { + lib.Java.GroupID = groupID + return + } + } + } +} + // fillRust populates empty Rust-specific fields in lib from the provided default. func fillRust(lib *config.Library, d *config.Default) *config.Library { if lib.Rust == nil { diff --git a/internal/librarian/library_test.go b/internal/librarian/library_test.go index d052702b83c..1e1c9cd9430 100644 --- a/internal/librarian/library_test.go +++ b/internal/librarian/library_test.go @@ -233,6 +233,217 @@ func TestFillDefaults(t *testing.T) { } } +func TestFillDefaults_Java(t *testing.T) { + defaults := &config.Default{ + Java: &config.JavaDefault{ + CustomGroupIDs: map[string]string{ + "google/shopping": "com.google.shopping", + "google/exact/v1": "com.google.exact", + "google/maps": "com.google.maps", + "google/ads": "com.google.api-ads", + "google/analytics": "com.google.analytics", + }, + }, + } + for _, test := range []struct { + name string + lib *config.Library + defaults *config.Default + want *config.Library + }{ + { + name: "shopping library", + lib: &config.Library{ + Name: "shopping-merchant-issue-resolution", + APIs: []*config.API{ + {Path: "google/shopping/merchant/issueresolution/v1"}, + {Path: "google/shopping/merchant/issueresolution/v1beta"}, + }, + }, + defaults: defaults, + want: &config.Library{ + Name: "shopping-merchant-issue-resolution", + APIs: []*config.API{ + {Path: "google/shopping/merchant/issueresolution/v1"}, + {Path: "google/shopping/merchant/issueresolution/v1beta"}, + }, + Java: &config.JavaModule{ + GroupID: "com.google.shopping", + }, + }, + }, + { + name: "do not override custom artifact id", + lib: &config.Library{ + Name: "custom-shopping", + APIs: []*config.API{ + {Path: "google/shopping/merchant/issueresolution/v1"}, + {Path: "google/shopping/merchant/issueresolution/v1beta"}, + }, + Java: &config.JavaModule{ + ArtifactID: "custom-shopping-id", + }, + }, + defaults: defaults, + want: &config.Library{ + Name: "custom-shopping", + APIs: []*config.API{ + {Path: "google/shopping/merchant/issueresolution/v1"}, + {Path: "google/shopping/merchant/issueresolution/v1beta"}, + }, + Java: &config.JavaModule{ + ArtifactID: "custom-shopping-id", + GroupID: "com.google.shopping", + }, + }, + }, + { + name: "maps library", + lib: &config.Library{ + Name: "maps-routeoptimization", + APIs: []*config.API{{Path: "google/maps/routeoptimization/v1"}}, + }, + defaults: defaults, + want: &config.Library{ + Name: "maps-routeoptimization", + APIs: []*config.API{{Path: "google/maps/routeoptimization/v1"}}, + Java: &config.JavaModule{ + GroupID: "com.google.maps", + }, + }, + }, + { + name: "ads library", + lib: &config.Library{ + Name: "admanager", + APIs: []*config.API{{Path: "google/ads/admanager/v1"}}, + }, + defaults: defaults, + want: &config.Library{ + Name: "admanager", + APIs: []*config.API{{Path: "google/ads/admanager/v1"}}, + Java: &config.JavaModule{ + GroupID: "com.google.api-ads", + }, + }, + }, + { + name: "analytics library", + lib: &config.Library{ + Name: "analytics-admin", + APIs: []*config.API{ + {Path: "google/analytics/admin/v1beta"}, + {Path: "google/analytics/admin/v1alpha"}, + }, + }, + defaults: defaults, + want: &config.Library{ + Name: "analytics-admin", + APIs: []*config.API{ + {Path: "google/analytics/admin/v1beta"}, + {Path: "google/analytics/admin/v1alpha"}, + }, + Java: &config.JavaModule{ + GroupID: "com.google.analytics", + }, + }, + }, + { + name: "do not fill if group id already set", + lib: &config.Library{ + Name: "common-protos", + APIs: []*config.API{ + {Path: "google/shopping/type"}, + }, + Java: &config.JavaModule{ + GroupID: "com.google.api.grpc", + }, + }, + defaults: defaults, + want: &config.Library{ + Name: "common-protos", + APIs: []*config.API{ + {Path: "google/shopping/type"}, + }, + Java: &config.JavaModule{ + GroupID: "com.google.api.grpc", + }, + }, + }, + { + name: "prefix match respects path segment boundaries", + lib: &config.Library{ + Name: "shopping-foo", + APIs: []*config.API{ + {Path: "google/shopping-foo/v1"}, + }, + }, + defaults: defaults, + want: &config.Library{ + Name: "shopping-foo", + APIs: []*config.API{ + {Path: "google/shopping-foo/v1"}, + }, + Java: &config.JavaModule{}, + }, + }, + { + name: "no matching api prefix leaves group id empty", + lib: &config.Library{ + Name: "unknown", + APIs: []*config.API{{Path: "google/unknown/v1"}}, + }, + defaults: defaults, + want: &config.Library{ + Name: "unknown", + APIs: []*config.API{{Path: "google/unknown/v1"}}, + Java: &config.JavaModule{}, + }, + }, + { + name: "library does not change with nil map", + lib: &config.Library{ + Name: "lib", + APIs: []*config.API{{Path: "google/example/v1"}}, + }, + defaults: &config.Default{ + Java: &config.JavaDefault{}, + }, + want: &config.Library{ + Name: "lib", + APIs: []*config.API{{Path: "google/example/v1"}}, + Java: &config.JavaModule{}, + }, + }, + { + name: "api path exact match", + lib: &config.Library{ + Name: "exact", + APIs: []*config.API{ + {Path: "google/exact/v1"}, + }, + }, + defaults: defaults, + want: &config.Library{ + Name: "exact", + APIs: []*config.API{ + {Path: "google/exact/v1"}, + }, + Java: &config.JavaModule{ + GroupID: "com.google.exact", + }, + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + got := fillJava(test.lib, test.defaults) + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + }) + } +} + func TestFillDefaults_Rust(t *testing.T) { defaults := &config.Default{ Rust: &config.RustDefault{ From 95a80c135ce8c72400563505a1ffabce6b7e9987 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Wed, 1 Jul 2026 11:30:51 -0700 Subject: [PATCH 107/108] chore(main): release 0.24.0 (#6586) :robot: I have created a release *beep* *boop* --- ## [0.24.0](https://github.com/googleapis/librarian/compare/v0.23.0...v0.24.0) (2026-07-01) ### Features * **add:** handle Release Please config for google-cloud-node ([#6569](https://github.com/googleapis/librarian/issues/6569)) ([1f8ee00](https://github.com/googleapis/librarian/commit/1f8ee00cfecad14cfb77b8b93d4e3b73b00100d5)) * **internal/librarian/java:** add code snippet extraction helpers for README rendering ([#6593](https://github.com/googleapis/librarian/issues/6593)) ([96f6925](https://github.com/googleapis/librarian/commit/96f6925fb95a81bdaa95e3405d4a09701a70178b)), closes [#6515](https://github.com/googleapis/librarian/issues/6515) * **internal/librarian/java:** add extractSamples for README generation ([#6578](https://github.com/googleapis/librarian/issues/6578)) ([b5e3d45](https://github.com/googleapis/librarian/commit/b5e3d4519d4a747fa8f09fdababb3bf77e78611c)), closes [#6515](https://github.com/googleapis/librarian/issues/6515) * **internal/librarian/nodejs:** add metadata_name_override and name_pretty_override support ([#6603](https://github.com/googleapis/librarian/issues/6603)) ([3f6cfed](https://github.com/googleapis/librarian/commit/3f6cfed0b7d4c24c2d5aaa4750b4cc32db34cc51)), closes [#6453](https://github.com/googleapis/librarian/issues/6453) * **internal/librarian:** add debug command with env subcommand ([#6576](https://github.com/googleapis/librarian/issues/6576)) ([027103b](https://github.com/googleapis/librarian/commit/027103b817777f101d72ad7df3af2bf7255e7b6a)), closes [#6374](https://github.com/googleapis/librarian/issues/6374) * **internal/librarian:** populate Java Maven coordinates from defaults ([#6554](https://github.com/googleapis/librarian/issues/6554)) ([accb8ad](https://github.com/googleapis/librarian/commit/accb8adb99c8905d91489d59e6ed112159630874)), closes [#6513](https://github.com/googleapis/librarian/issues/6513) * **librarian/swift:** use discovery config ([#6604](https://github.com/googleapis/librarian/issues/6604)) ([5a44ed7](https://github.com/googleapis/librarian/commit/5a44ed7b04f06fefcb1cf78c2e46b0f8bedeb035)) * **sidekick/discovery:** signatures without path params ([#6588](https://github.com/googleapis/librarian/issues/6588)) ([bb40e83](https://github.com/googleapis/librarian/commit/bb40e83a4e605ac0cc9b1719f8d965b4a68bb908)) ### Bug Fixes * **internal/librarian/java:** exclude google-cloud-bom and libraries-bom when generating gapic-libraries-bom/pom.xml ([#6601](https://github.com/googleapis/librarian/issues/6601)) ([b8e50a5](https://github.com/googleapis/librarian/commit/b8e50a51a11cb07192f209c0e7fadeb5f3ce77c7)) * **internal/serviceconfig:** normalize transport name for Java repo-metadata ([#6582](https://github.com/googleapis/librarian/issues/6582)) ([e20f77a](https://github.com/googleapis/librarian/commit/e20f77a70df502836fedda0f302fda66b09f1452)) * **sdk.yaml:** allow rust for many non-cloud apis ([#6598](https://github.com/googleapis/librarian/issues/6598)) ([0efb6e7](https://github.com/googleapis/librarian/commit/0efb6e71ddbdb5fd6d121a02ecbf02138235e102)) --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- .release-please-manifest.json | 2 +- CHANGELOG.md | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 97bce112d22..fc5553b079e 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "0.23.0" + ".": "0.24.0" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a773dcf5b1..7261e45b447 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,26 @@ # Changelog +## [0.24.0](https://github.com/googleapis/librarian/compare/v0.23.0...v0.24.0) (2026-07-01) + + +### Features + +* **add:** handle Release Please config for google-cloud-node ([#6569](https://github.com/googleapis/librarian/issues/6569)) ([1f8ee00](https://github.com/googleapis/librarian/commit/1f8ee00cfecad14cfb77b8b93d4e3b73b00100d5)) +* **internal/librarian/java:** add code snippet extraction helpers for README rendering ([#6593](https://github.com/googleapis/librarian/issues/6593)) ([96f6925](https://github.com/googleapis/librarian/commit/96f6925fb95a81bdaa95e3405d4a09701a70178b)), closes [#6515](https://github.com/googleapis/librarian/issues/6515) +* **internal/librarian/java:** add extractSamples for README generation ([#6578](https://github.com/googleapis/librarian/issues/6578)) ([b5e3d45](https://github.com/googleapis/librarian/commit/b5e3d4519d4a747fa8f09fdababb3bf77e78611c)), closes [#6515](https://github.com/googleapis/librarian/issues/6515) +* **internal/librarian/nodejs:** add metadata_name_override and name_pretty_override support ([#6603](https://github.com/googleapis/librarian/issues/6603)) ([3f6cfed](https://github.com/googleapis/librarian/commit/3f6cfed0b7d4c24c2d5aaa4750b4cc32db34cc51)), closes [#6453](https://github.com/googleapis/librarian/issues/6453) +* **internal/librarian:** add debug command with env subcommand ([#6576](https://github.com/googleapis/librarian/issues/6576)) ([027103b](https://github.com/googleapis/librarian/commit/027103b817777f101d72ad7df3af2bf7255e7b6a)), closes [#6374](https://github.com/googleapis/librarian/issues/6374) +* **internal/librarian:** populate Java Maven coordinates from defaults ([#6554](https://github.com/googleapis/librarian/issues/6554)) ([accb8ad](https://github.com/googleapis/librarian/commit/accb8adb99c8905d91489d59e6ed112159630874)), closes [#6513](https://github.com/googleapis/librarian/issues/6513) +* **librarian/swift:** use discovery config ([#6604](https://github.com/googleapis/librarian/issues/6604)) ([5a44ed7](https://github.com/googleapis/librarian/commit/5a44ed7b04f06fefcb1cf78c2e46b0f8bedeb035)) +* **sidekick/discovery:** signatures without path params ([#6588](https://github.com/googleapis/librarian/issues/6588)) ([bb40e83](https://github.com/googleapis/librarian/commit/bb40e83a4e605ac0cc9b1719f8d965b4a68bb908)) + + +### Bug Fixes + +* **internal/librarian/java:** exclude google-cloud-bom and libraries-bom when generating gapic-libraries-bom/pom.xml ([#6601](https://github.com/googleapis/librarian/issues/6601)) ([b8e50a5](https://github.com/googleapis/librarian/commit/b8e50a51a11cb07192f209c0e7fadeb5f3ce77c7)) +* **internal/serviceconfig:** normalize transport name for Java repo-metadata ([#6582](https://github.com/googleapis/librarian/issues/6582)) ([e20f77a](https://github.com/googleapis/librarian/commit/e20f77a70df502836fedda0f302fda66b09f1452)) +* **sdk.yaml:** allow rust for many non-cloud apis ([#6598](https://github.com/googleapis/librarian/issues/6598)) ([0efb6e7](https://github.com/googleapis/librarian/commit/0efb6e71ddbdb5fd6d121a02ecbf02138235e102)) + ## [0.23.0](https://github.com/googleapis/librarian/compare/v0.22.0...v0.23.0) (2026-06-29) From ca68a4bfc00f835497af34d4b1d4220970b5c152 Mon Sep 17 00:00:00 2001 From: Sophia Yang <171981480+yangyzs@users.noreply.github.com> Date: Wed, 1 Jul 2026 14:35:52 -0400 Subject: [PATCH 108/108] feat(java): assemble complete README generator stack --- internal/librarian/java/readme.go | 113 ++++--- internal/librarian/java/readme_test.go | 391 ++++++++++++++++++------- 2 files changed, 342 insertions(+), 162 deletions(-) diff --git a/internal/librarian/java/readme.go b/internal/librarian/java/readme.go index 678af5de36b..e6dc1cfdd0a 100644 --- a/internal/librarian/java/readme.go +++ b/internal/librarian/java/readme.go @@ -28,6 +28,7 @@ import ( "strings" "text/template" "unicode" + "unicode/utf8" "github.com/googleapis/librarian/internal/yaml" ) @@ -71,7 +72,7 @@ type codeSample struct { // readmeData represents the top-level template execution context passed to README.md.go.tmpl. type readmeData struct { - Metadata map[string]interface{} // Contains Repo, LibraryVersion, Samples, Snippets, Partials. // TODO delete these comments for readmeData + Metadata map[string]interface{} // Contains Repo, LibraryVersion, Samples, Snippets, Partials. GroupID string // Maven Group ID (e.g. com.google.cloud), required for Maven/Gradle dependency blocks. ArtifactID string // Maven Artifact ID (e.g. google-cloud-storage), required for dependency blocks. Version string // Current library version. @@ -248,40 +249,6 @@ func extractTitle(filePath string) (string, error) { return title, nil } -// extractSnippets walks the "samples" directory locating *.java and *.xml files. -// It line-scans for START and END tags while supporting START_EXCLUDE blocks. -func extractSnippets(dir string) (map[string]string, error) { - if dir == "" { - return nil, errEmptyDir - } - files, err := collectSnippetFiles(dir) - if err != nil { - return nil, err - } - if len(files) == 0 { - return nil, nil - } - sort.Strings(files) // TODO: check- do we need this? check the legacy owlbot and see if we sort the files? - snippetLines := make(map[string][]string) - for _, file := range files { - fileSnippets, err := extractSnippetsFromFile(file) - if err != nil { - return nil, err - } - for name, lines := range fileSnippets { - snippetLines[name] = append(snippetLines[name], lines...) - } - } - if len(snippetLines) == 0 { - return nil, nil - } - result := make(map[string]string) - for snippet, lines := range snippetLines { - result[snippet] = trimLeadingWhitespace(lines) - } - return result, nil -} - // collectSnippetFiles recursively scans dir/samples for Java and XML files containing snippets. func collectSnippetFiles(dir string) ([]string, error) { samplesDir := filepath.Join(dir, "samples") @@ -350,10 +317,9 @@ func extractSnippetsFromFile(file string) (map[string][]string, error) { scanner := bufio.NewScanner(f) // 10 MB sanity limit to protect system memory. scanner.Buffer(make([]byte, 64*1024), 10*1024*1024) - // Scan through file line by line and capture all open snippets. - // More than one open block might exist for a given line, so we - // need openSnippets to track currently active snippet blocks by name + // snippetLines maps snippet names to their captured code lines. snippetLines := make(map[string][]string) + // openSnippets tracks currently active snippet blocks by name, allowing nested or overlapping blocks. openSnippets := make(map[string]bool) excluding := false for scanner.Scan() { @@ -370,6 +336,7 @@ func extractSnippetsFromFile(file string) (map[string][]string, error) { if excluding { continue } + // Check for snippet start/end tags. Tag lines themselves are not saved. openMatch := openSnippetRegex.FindStringSubmatch(line) closeMatch := closeSnippetRegex.FindStringSubmatch(line) @@ -385,6 +352,7 @@ func extractSnippetsFromFile(file string) (map[string][]string, error) { delete(openSnippets, closeMatch[1]) continue } + // Append this line of code to every snippet block currently open. for s := range openSnippets { snippetLines[s] = append(snippetLines[s], line) @@ -418,7 +386,6 @@ func minLeadingSpaces(lines []string) int { } // trimLeadingWhitespace computes minimum leading space indentation and trims it. -// Used to clean up snippet lines so code formatting looks natural in README blocks. func trimLeadingWhitespace(lines []string) string { if len(lines) == 0 { return "" @@ -436,55 +403,85 @@ func trimLeadingWhitespace(lines []string) string { return sb.String() } +// extractSnippets recursively aggregates tagged code snippets across all Java and XML files in dir/samples. +func extractSnippets(dir string) (map[string]string, error) { + if dir == "" { + return nil, errEmptyDir + } + files, err := collectSnippetFiles(dir) + if err != nil { + return nil, err + } + if len(files) == 0 { + return nil, nil + } + sort.Strings(files) + snippetLines := make(map[string][]string) + for _, file := range files { + fileSnippets, err := extractSnippetsFromFile(file) + if err != nil { + return nil, err + } + for name, lines := range fileSnippets { + snippetLines[name] = append(snippetLines[name], lines...) + } + } + if len(snippetLines) == 0 { + return nil, nil + } + result := make(map[string]string, len(snippetLines)) + for snippet, lines := range snippetLines { + result[snippet] = trimLeadingWhitespace(lines) + } + return result, nil +} + // loadReadmePartials loads and camel-cases README partials from .readme-partials.yaml or .yml. func loadReadmePartials(dir string) (map[string]interface{}, error) { if dir == "" { return nil, errEmptyDir } - partialsPath := filepath.Join(dir, ".readme-partials.yaml") - if _, err := os.Stat(partialsPath); err != nil { + partialsBytes, err := os.ReadFile(filepath.Join(dir, ".readme-partials.yaml")) + if err != nil { if !errors.Is(err, fs.ErrNotExist) { - return nil, fmt.Errorf("failed to stat partials file: %w", err) + return nil, fmt.Errorf("failed to read partials file: %w", err) } - partialsPath = filepath.Join(dir, ".readme-partials.yml") - if _, err = os.Stat(partialsPath); err != nil { + partialsBytes, err = os.ReadFile(filepath.Join(dir, ".readme-partials.yml")) + if err != nil { if errors.Is(err, fs.ErrNotExist) { return nil, nil } - return nil, fmt.Errorf("failed to stat partials file: %w", err) + return nil, fmt.Errorf("failed to read partials file: %w", err) } } - partialsBytes, err := os.ReadFile(partialsPath) - if err != nil { - return nil, fmt.Errorf("failed to read partials file: %w", err) - } - if partialsBytes == nil { + if len(partialsBytes) == 0 { return nil, nil } rawPartials, err := yaml.Unmarshal[map[string]interface{}](partialsBytes) if err != nil { return nil, fmt.Errorf("failed to unmarshal partials: %w", err) } - result := make(map[string]interface{}) + if rawPartials == nil || len(*rawPartials) == 0 { + return nil, nil + } + result := make(map[string]interface{}, len(*rawPartials)) for k, v := range *rawPartials { - // Convert to camel case because jinja templates use snake_case but go template fields should use camelcase. result[toCamelCase(k)] = v } return result, nil } -// toCamelCase converts snake_case, kebab-case, or lower word strings to CamelCase. +// toCamelCase converts snake_case, kebab-case, or space-separated strings into CamelCase identifiers. func toCamelCase(s string) string { parts := strings.FieldsFunc(s, func(r rune) bool { return r == '_' || r == '-' || r == ' ' }) var sb strings.Builder for _, p := range parts { - if len(p) > 0 { - r := []rune(p) - r[0] = unicode.ToUpper(r[0]) - sb.WriteString(string(r)) - } + // Decode and capitalize the first rune, then append the remaining subslice without copying. + r, size := utf8.DecodeRuneInString(p) + sb.WriteRune(unicode.ToUpper(r)) + sb.WriteString(p[size:]) } return sb.String() } diff --git a/internal/librarian/java/readme_test.go b/internal/librarian/java/readme_test.go index d7ddd1e0aa7..a2a6ab069ac 100644 --- a/internal/librarian/java/readme_test.go +++ b/internal/librarian/java/readme_test.go @@ -549,57 +549,6 @@ func TestCollectSnippetFiles(t *testing.T) { } } -func TestMinLeadingSpaces(t *testing.T) { - for _, test := range []struct { - name string - lines []string - want int - }{ - {"two lines indented", []string{" foo", " bar"}, 2}, - {"zero indented line", []string{"foo", " bar"}, 0}, - {"empty slice", nil, 0}, - {"only whitespace lines", []string{" ", "\t"}, 0}, - } { - t.Run(test.name, func(t *testing.T) { - got := minLeadingSpaces(test.lines) - if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestTrimLeadingWhitespace(t *testing.T) { - for _, test := range []struct { - name string - lines []string - want string - }{ - { - name: "standard indentation", - lines: []string{" int x = 1;", " int y = 2;"}, - want: "int x = 1;\n int y = 2;\n", - }, - { - name: "with blank line", - lines: []string{" int x = 1;", "", " int y = 2;"}, - want: "int x = 1;\n\nint y = 2;\n", - }, - { - name: "empty input", - lines: nil, - want: "", - }, - } { - t.Run(test.name, func(t *testing.T) { - got := trimLeadingWhitespace(test.lines) - if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("mismatch (-want +got):\n%s", diff) - } - }) - } -} - func TestExtractSnippetsFromFile(t *testing.T) { t.Parallel() for _, test := range []struct { @@ -724,38 +673,134 @@ func TestExtractSnippetsFromFile_Error(t *testing.T) { } } -func TestLoadReadmePartials(t *testing.T) { +func TestMinLeadingSpaces(t *testing.T) { + for _, test := range []struct { + name string + lines []string + want int + }{ + { + name: "standard indentation", + lines: []string{" int x = 1;", " int y = 2;"}, + want: 2, + }, + { + name: "ignores empty and whitespace lines", + lines: []string{"", " ", " int z = 3;"}, + want: 4, + }, + { + name: "zero indentation present", + lines: []string{"int a = 0;", " int b = 1;"}, + want: 0, + }, + { + name: "empty slice", + lines: nil, + want: 0, + }, + } { + t.Run(test.name, func(t *testing.T) { + got := minLeadingSpaces(test.lines) + if got != test.want { + t.Errorf("minLeadingSpaces() = %d, want %d", got, test.want) + } + }) + } +} + +func TestTrimLeadingWhitespace(t *testing.T) { + for _, test := range []struct { + name string + lines []string + want string + }{ + { + name: "standard indentation", + lines: []string{" int x = 1;", " int y = 2;"}, + want: "int x = 1;\n int y = 2;\n", + }, + { + name: "with blank line", + lines: []string{" int x = 1;", "", " int y = 2;"}, + want: "int x = 1;\n\nint y = 2;\n", + }, + { + name: "empty input", + lines: nil, + want: "", + }, + } { + t.Run(test.name, func(t *testing.T) { + got := trimLeadingWhitespace(test.lines) + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestExtractSnippets(t *testing.T) { for _, test := range []struct { name string setupFiles func(t *testing.T, dir string) - want map[string]interface{} + want map[string]string }{ { - name: "loads yaml partials with camel case conversion", + name: "extracts snippets across java and xml files with indentation trimmed", setupFiles: func(t *testing.T, dir string) { - path := filepath.Join(dir, ".readme-partials.yaml") - content := `about_text: "Custom about"` - if err := os.WriteFile(path, []byte(content), 0644); err != nil { + samplesDir := filepath.Join(dir, "samples") + if err := os.MkdirAll(samplesDir, 0755); err != nil { + t.Fatal(err) + } + pomPath := filepath.Join(samplesDir, "pom.xml") + pomContent := ` + + + com.google.cloud + + +` + if err := os.WriteFile(pomPath, []byte(pomContent), 0644); err != nil { + t.Fatal(err) + } + javaPath := filepath.Join(samplesDir, "Demo.java") + javaContent := `public class Demo { + // [START quickstart] + public void run() { + // [START_EXCLUDE] + System.out.println("hidden"); + // [END_EXCLUDE] + System.out.println("visible"); + } + // [END quickstart] +}` + if err := os.WriteFile(javaPath, []byte(javaContent), 0644); err != nil { t.Fatal(err) } }, - want: map[string]interface{}{"AboutText": "Custom about"}, + want: map[string]string{ + "dependency_snippet": "\n com.google.cloud\n\n", + "quickstart": "public void run() {\n System.out.println(\"visible\");\n}\n", + }, }, { - name: "loads yml fallback partials", + name: "missing samples directory returns nil", setupFiles: func(t *testing.T, dir string) { - path := filepath.Join(dir, ".readme-partials.yml") - content := `introduction: "Intro text"` - if err := os.WriteFile(path, []byte(content), 0644); err != nil { - t.Fatal(err) - } + // No samples dir. }, - want: map[string]interface{}{"Introduction": "Intro text"}, + want: nil, }, { - name: "missing partials file returns nil", + name: "no snippet tags in java files returns nil", setupFiles: func(t *testing.T, dir string) { - // No file written. + samplesDir := filepath.Join(dir, "samples") + if err := os.MkdirAll(samplesDir, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(samplesDir, "Plain.java"), []byte("public class Plain {}"), 0644); err != nil { + t.Fatal(err) + } }, want: nil, }, @@ -763,7 +808,7 @@ func TestLoadReadmePartials(t *testing.T) { t.Run(test.name, func(t *testing.T) { dir := t.TempDir() test.setupFiles(t, dir) - got, err := loadReadmePartials(dir) + got, err := extractSnippets(dir) if err != nil { t.Fatal(err) } @@ -774,40 +819,22 @@ func TestLoadReadmePartials(t *testing.T) { } } -func TestLoadReadmePartials_Error(t *testing.T) { +func TestExtractSnippets_Error(t *testing.T) { for _, test := range []struct { - name string - dir string - setupFiles func(t *testing.T, dir string) - wantErr error + name string + dir string + wantErr error }{ { name: "empty directory parameter returns error", dir: "", wantErr: errEmptyDir, }, - { - name: "invalid yaml syntax", - setupFiles: func(t *testing.T, dir string) { - path := filepath.Join(dir, ".readme-partials.yaml") - content := `key: [unclosed list` - if err := os.WriteFile(path, []byte(content), 0644); err != nil { - t.Fatal(err) - } - }, - }, } { t.Run(test.name, func(t *testing.T) { - dir := test.dir - if test.setupFiles != nil { - dir = t.TempDir() - test.setupFiles(t, dir) - } - _, err := loadReadmePartials(dir) - if err == nil { - t.Errorf("expected error, got nil") - } else if test.wantErr != nil && !errors.Is(err, test.wantErr) { - t.Errorf("loadReadmePartials() error = %v, wantErr %v", err, test.wantErr) + _, err := extractSnippets(test.dir) + if !errors.Is(err, test.wantErr) { + t.Errorf("extractSnippets(%q) error = %v, wantErr %v", test.dir, err, test.wantErr) } }) } @@ -819,11 +846,31 @@ func TestToCamelCase(t *testing.T) { input string want string }{ - {"snake case", "custom_content", "CustomContent"}, - {"kebab case", "readme-partials", "ReadmePartials"}, - {"space separated", "about us", "AboutUs"}, - {"already camel", "About", "About"}, - {"empty string", "", ""}, + { + name: "snake case", + input: "custom_content", + want: "CustomContent", + }, + { + name: "kebab case", + input: "readme-partials", + want: "ReadmePartials", + }, + { + name: "space separated", + input: "about us", + want: "AboutUs", + }, + { + name: "already camel", + input: "About", + want: "About", + }, + { + name: "empty string", + input: "", + want: "", + }, } { t.Run(test.name, func(t *testing.T) { got := toCamelCase(test.input) @@ -841,9 +888,24 @@ func TestParseGroupIDArtifactID(t *testing.T) { wantGroupID string wantArtifactID string }{ - {"standard coordinates", "com.google.cloud:google-cloud-storage", "com.google.cloud", "google-cloud-storage"}, - {"missing artifact id", "com.google.cloud", "com.google.cloud", ""}, - {"empty distribution name", "", "", ""}, + { + name: "standard coordinates", + input: "com.google.cloud:google-cloud-storage", + wantGroupID: "com.google.cloud", + wantArtifactID: "google-cloud-storage", + }, + { + name: "missing artifact id", + input: "com.google.cloud", + wantGroupID: "com.google.cloud", + wantArtifactID: "", + }, + { + name: "empty distribution name", + input: "", + wantGroupID: "", + wantArtifactID: "", + }, } { t.Run(test.name, func(t *testing.T) { gotGroup, gotArtifact := parseGroupIDArtifactID(test.input) @@ -863,9 +925,21 @@ func TestParseRepoShortName(t *testing.T) { input string want string }{ - {"full repo path", "googleapis/google-cloud-java", "google-cloud-java"}, - {"short repo name only", "google-cloud-java", "google-cloud-java"}, - {"empty repo string", "", ""}, + { + name: "full repo path", + input: "googleapis/google-cloud-java", + want: "google-cloud-java", + }, + { + name: "short repo name only", + input: "google-cloud-java", + want: "google-cloud-java", + }, + { + name: "empty repo string", + input: "", + want: "", + }, } { t.Run(test.name, func(t *testing.T) { got := parseRepoShortName(test.input) @@ -876,6 +950,115 @@ func TestParseRepoShortName(t *testing.T) { } } +func TestLoadReadmePartials(t *testing.T) { + for _, test := range []struct { + name string + setupFiles func(t *testing.T, dir string) + want map[string]interface{} + }{ + { + name: "loads yaml partials with camel case conversion", + setupFiles: func(t *testing.T, dir string) { + path := filepath.Join(dir, ".readme-partials.yaml") + content := `about_text: "Custom about"` + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatal(err) + } + }, + want: map[string]interface{}{"AboutText": "Custom about"}, + }, + { + name: "loads yml fallback partials", + setupFiles: func(t *testing.T, dir string) { + path := filepath.Join(dir, ".readme-partials.yml") + content := `introduction: "Intro text"` + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatal(err) + } + }, + want: map[string]interface{}{"Introduction": "Intro text"}, + }, + { + name: "missing partials file returns nil", + setupFiles: func(t *testing.T, dir string) { + // No file written. + }, + want: nil, + }, + { + name: "empty partials file returns nil", + setupFiles: func(t *testing.T, dir string) { + path := filepath.Join(dir, ".readme-partials.yaml") + if err := os.WriteFile(path, []byte(""), 0644); err != nil { + t.Fatal(err) + } + }, + want: nil, + }, + { + name: "partials file with only comments returns nil", + setupFiles: func(t *testing.T, dir string) { + path := filepath.Join(dir, ".readme-partials.yaml") + if err := os.WriteFile(path, []byte("# only comments\n# no keys defined"), 0644); err != nil { + t.Fatal(err) + } + }, + want: nil, + }, + } { + t.Run(test.name, func(t *testing.T) { + dir := t.TempDir() + test.setupFiles(t, dir) + got, err := loadReadmePartials(dir) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestLoadReadmePartials_Error(t *testing.T) { + for _, test := range []struct { + name string + dir string + setupFiles func(t *testing.T, dir string) + wantErr error + }{ + { + name: "empty directory parameter returns error", + dir: "", + wantErr: errEmptyDir, + }, + { + name: "invalid yaml syntax", + setupFiles: func(t *testing.T, dir string) { + path := filepath.Join(dir, ".readme-partials.yaml") + content := `key: [unclosed list` + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatal(err) + } + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + dir := test.dir + if test.setupFiles != nil { + dir = t.TempDir() + test.setupFiles(t, dir) + } + _, err := loadReadmePartials(dir) + if err == nil { + t.Errorf("expected error, got nil") + } else if test.wantErr != nil && !errors.Is(err, test.wantErr) { + t.Errorf("loadReadmePartials() error = %v, wantErr %v", err, test.wantErr) + } + }) + } +} + func TestRenderREADME(t *testing.T) { tmpDir := t.TempDir() metadata := &repoMetadata{