diff --git a/cmd/amazon-cloudwatch-agent/amazon-cloudwatch-agent.go b/cmd/amazon-cloudwatch-agent/amazon-cloudwatch-agent.go index 97c1f123cf7..4d17cddc6d7 100644 --- a/cmd/amazon-cloudwatch-agent/amazon-cloudwatch-agent.go +++ b/cmd/amazon-cloudwatch-agent/amazon-cloudwatch-agent.go @@ -434,12 +434,19 @@ func mergeConfigs(configPaths []string) (*confmap.Conf, error) { for _, configPath := range configPaths { loaders = append(loaders, confmap.NewFileLoader(configPath)) } - result := confmap.New() + var result *confmap.Conf for _, loader := range loaders { conf, err := loader.Load() if err != nil { + if errors.Is(err, os.ErrNotExist) { + log.Printf("D! Skipping non-existent OTEL config: %s", loader.ID()) + continue + } return nil, fmt.Errorf("failed to load OTEL configs: %w", err) } + if result == nil { + result = confmap.New() + } if err = result.Merge(conf); err != nil { return nil, fmt.Errorf("failed to merge OTEL configs: %w", err) } diff --git a/cmd/amazon-cloudwatch-agent/amazon-cloudwatch-agent_test.go b/cmd/amazon-cloudwatch-agent/amazon-cloudwatch-agent_test.go index 548ba239a64..c254f05f99f 100644 --- a/cmd/amazon-cloudwatch-agent/amazon-cloudwatch-agent_test.go +++ b/cmd/amazon-cloudwatch-agent/amazon-cloudwatch-agent_test.go @@ -89,9 +89,17 @@ service: wantErr bool }{ "WithInvalidFile": { - input: []string{filepath.Join("not", "a", "file"), filepath.Join("testdata", "base.yaml")}, + input: []string{filepath.Join("testdata", "invalid.yaml"), filepath.Join("testdata", "base.yaml")}, wantErr: true, }, + "WithAllMissingFiles": { + input: []string{filepath.Join("not", "a", "file"), filepath.Join("also", "not", "a", "file")}, + want: nil, + }, + "WithMissingFile": { + input: []string{filepath.Join("not", "a", "file"), filepath.Join("testdata", "base.yaml")}, + want: mustLoadFromFile(t, filepath.Join("testdata", "base.yaml")), + }, "WithNoMerge": { input: []string{filepath.Join("testdata", "base.yaml")}, wantErr: false, diff --git a/cmd/amazon-cloudwatch-agent/testdata/invalid.yaml b/cmd/amazon-cloudwatch-agent/testdata/invalid.yaml new file mode 100644 index 00000000000..e1bf27f57ae --- /dev/null +++ b/cmd/amazon-cloudwatch-agent/testdata/invalid.yaml @@ -0,0 +1 @@ +invalid: yaml: content: [ diff --git a/internal/merge/confmap/load.go b/internal/merge/confmap/load.go index c6965e72311..c983e5638db 100644 --- a/internal/merge/confmap/load.go +++ b/internal/merge/confmap/load.go @@ -13,6 +13,7 @@ import ( type Loader interface { Load() (*Conf, error) + ID() string } type FileLoader struct { @@ -23,6 +24,10 @@ func NewFileLoader(path string) *FileLoader { return &FileLoader{path: path} } +func (f *FileLoader) ID() string { + return f.path +} + func (f *FileLoader) Load() (*Conf, error) { // Clean the path before using it. content, err := os.ReadFile(filepath.Clean(f.path)) @@ -41,6 +46,10 @@ func NewByteLoader(id string, content []byte) *ByteLoader { return &ByteLoader{id: id, content: content} } +func (b *ByteLoader) ID() string { + return b.id +} + func (b *ByteLoader) Load() (*Conf, error) { var rawConf map[string]any if err := yaml.Unmarshal(b.content, &rawConf); err != nil { diff --git a/internal/merge/confmap/load_test.go b/internal/merge/confmap/load_test.go index df9b7735395..33759cf4bee 100644 --- a/internal/merge/confmap/load_test.go +++ b/internal/merge/confmap/load_test.go @@ -12,6 +12,7 @@ import ( func TestFileLoader(t *testing.T) { loader := NewFileLoader(filepath.Join("not", "a", "file")) + assert.Equal(t, filepath.Join("not", "a", "file"), loader.ID()) got, err := loader.Load() assert.Error(t, err) assert.Nil(t, got) @@ -26,6 +27,7 @@ func TestByteLoader(t *testing.T) { nop/1: ` loader := NewByteLoader("invalid-yaml", []byte("string")) + assert.Equal(t, "invalid-yaml", loader.ID()) got, err := loader.Load() assert.Error(t, err) assert.Nil(t, got) diff --git a/tool/translator/translator.go b/tool/translator/translator.go index a4e5c0c3cd0..e2b3559852b 100644 --- a/tool/translator/translator.go +++ b/tool/translator/translator.go @@ -23,6 +23,7 @@ import ( const ( exitErrorMessage = "configuration validation first phase failed. Agent version: %v. Verify the JSON input is only using features supported by this version" exitSuccessMessage = "Configuration validation first phase succeeded" + exitSkipMessage = "Configuration validation first phase skipped. No JSON files found" version = "1.0" envConfigFileName = "env-config.json" yamlConfigFileName = "amazon-cloudwatch-agent.yaml" @@ -98,12 +99,23 @@ func (ct *ConfigTranslator) Translate() (err error) { } }() + tomlConfigPath := cmdutil.GetTomlConfigPath(ct.ctx.OutputTomlFilePath()) + tomlConfigDir := filepath.Dir(tomlConfigPath) + yamlConfigPath := filepath.Join(tomlConfigDir, yamlConfigFileName) + mergedJSONConfigMap, err := cmdutil.GenerateMergedJsonConfigMap(ct.ctx) - if err != nil { + onlyYAML := errors.Is(err, cmdutil.ErrOnlyYAML) + if err != nil && !onlyYAML { return fmt.Errorf("failed to generate merged json config: %v", err) } + if onlyYAML { + log.Println(exitSkipMessage) + // YAML-only configs still require a generated TOML for agent and logging configuration. + // TOML translation requires a non-nil map. + mergedJSONConfigMap = map[string]any{} + } - if !ct.ctx.RunInContainer() { + if !onlyYAML && !ct.ctx.RunInContainer() { current, err := user.Current() if err == nil && current.Name == "****" { runAsUser, err := userutil.DetectRunAsUser(mergedJSONConfigMap) @@ -114,24 +126,30 @@ func (ct *ConfigTranslator) Translate() (err error) { } } - tomlConfigPath := cmdutil.GetTomlConfigPath(ct.ctx.OutputTomlFilePath()) - tomlConfigDir := filepath.Dir(tomlConfigPath) - yamlConfigPath := filepath.Join(tomlConfigDir, yamlConfigFileName) tomlConfig, err := cmdutil.TranslateJsonMapToTomlConfig(mergedJSONConfigMap) if err != nil { return fmt.Errorf("failed to generate TOML configuration validation content: %v", err) } - yamlConfig, err := cmdutil.TranslateJsonMapToYamlConfig(mergedJSONConfigMap) - if err != nil && !errors.Is(err, pipeline.ErrNoPipelines) { - return fmt.Errorf("failed to generate YAML configuration validation content: %v", err) + var yamlConfig any + if !onlyYAML { + yamlConfig, err = cmdutil.TranslateJsonMapToYamlConfig(mergedJSONConfigMap) + if err != nil && !errors.Is(err, pipeline.ErrNoPipelines) { + return fmt.Errorf("failed to generate YAML configuration validation content: %v", err) + } } if err = cmdutil.ConfigToTomlFile(tomlConfig, tomlConfigPath); err != nil { return fmt.Errorf("failed to create the configuration TOML validation file: %v", err) } - if err = cmdutil.ConfigToYamlFile(yamlConfig, yamlConfigPath); err != nil { - return fmt.Errorf("failed to create the configuration YAML validation file: %v", err) + if yamlConfig != nil { + if err = cmdutil.ConfigToYamlFile(yamlConfig, yamlConfigPath); err != nil { + return fmt.Errorf("failed to create the configuration YAML validation file: %v", err) + } + } else { + _ = os.Remove(yamlConfigPath) + } + if !onlyYAML { + log.Println(exitSuccessMessage) } - log.Println(exitSuccessMessage) envConfigPath := filepath.Join(tomlConfigDir, envConfigFileName) cmdutil.TranslateJsonMapToEnvConfigFile(mergedJSONConfigMap, envConfigPath) diff --git a/tool/translator/translator_test.go b/tool/translator/translator_test.go new file mode 100644 index 00000000000..ec066e3d8e4 --- /dev/null +++ b/tool/translator/translator_test.go @@ -0,0 +1,43 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package translator + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/aws/amazon-cloudwatch-agent/translator" + translatorcontext "github.com/aws/amazon-cloudwatch-agent/translator/context" + translatorutil "github.com/aws/amazon-cloudwatch-agent/translator/util" +) + +func TestTranslate_OnlyYAML(t *testing.T) { + orig := translatorutil.DetectRegion + translatorutil.DetectRegion = func(string, map[string]string) (string, string) { + return "us-east-1", "mock" + } + defer func() { translatorutil.DetectRegion = orig }() + + translator.ResetMessages() + translatorcontext.ResetContext() + + tmpDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(tmpDir, "config.yaml"), nil, 0600)) + + tomlPath := filepath.Join(tmpDir, "output.toml") + // Pre-create the YAML output file to verify it gets removed. + yamlPath := filepath.Join(tmpDir, yamlConfigFileName) + require.NoError(t, os.WriteFile(yamlPath, nil, 0600)) + + ct, err := NewConfigTranslator("linux", "", tmpDir, tomlPath, "ec2", "", "default") + require.NoError(t, err) + + assert.NoError(t, ct.Translate()) + assert.FileExists(t, tomlPath) + assert.NoFileExists(t, yamlPath) +} diff --git a/translator/cmdutil/translatorutil.go b/translator/cmdutil/translatorutil.go index 167fe7e8e03..883015d8d13 100644 --- a/translator/cmdutil/translatorutil.go +++ b/translator/cmdutil/translatorutil.go @@ -4,6 +4,7 @@ package cmdutil import ( + "errors" "fmt" "log" "os" @@ -36,6 +37,8 @@ const ( defaultTomlConfigName = "CWAgent.conf" ) +var ErrOnlyYAML = errors.New("only YAML files detected") + // TranslateJsonMapToEnvConfigFile populates env-config.json based on the input json config. func TranslateJsonMapToEnvConfigFile(jsonConfigValue map[string]interface{}, envConfigPath string) { if envConfigPath == "" { @@ -115,6 +118,7 @@ func GenerateMergedJsonConfigMap(ctx *context.Context) (map[string]interface{}, // for the append operation when the existing file name and new .tmp file name have diff // only for the ".tmp" suffix, i.e. it is override operation even it says append. var jsonConfigMapMap = make(map[string]map[string]interface{}) + var foundYAML bool if ctx.MultiConfig() == "append" || ctx.MultiConfig() == "remove" { // backwards compatible for the old json config file @@ -163,6 +167,7 @@ func GenerateMergedJsonConfigMap(ctx *context.Context) (map[string]interface{}, ext = filepath.Ext(key) // skip .yaml files if ext == constants.FileSuffixYAML { + foundYAML = true return nil } if ctx.MultiConfig() == "default" || ctx.MultiConfig() == "append" { @@ -176,6 +181,7 @@ func GenerateMergedJsonConfigMap(ctx *context.Context) (map[string]interface{}, } } else if ext == constants.FileSuffixYAML { // skip .yaml files + foundYAML = true return nil } else { // non .tmp / existing files @@ -208,6 +214,9 @@ func GenerateMergedJsonConfigMap(ctx *context.Context) (map[string]interface{}, } jsonConfigMapMap[config.CWConfigContent] = jm } + if foundYAML && len(jsonConfigMapMap) == 0 { + return nil, ErrOnlyYAML + } } defaultConfig, err := translatorUtil.GetDefaultJsonConfigMap(ctx.Os(), ctx.Mode()) diff --git a/translator/cmdutil/translatorutil_test.go b/translator/cmdutil/translatorutil_test.go index 89c783a7407..bb88cf81761 100644 --- a/translator/cmdutil/translatorutil_test.go +++ b/translator/cmdutil/translatorutil_test.go @@ -6,13 +6,15 @@ package cmdutil import ( "encoding/json" "os" - "path" + "path/filepath" "regexp" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/aws/amazon-cloudwatch-agent/cfg/envconfig" + translatorcontext "github.com/aws/amazon-cloudwatch-agent/translator/context" "github.com/aws/amazon-cloudwatch-agent/translator/util" ) @@ -24,7 +26,7 @@ func TestTranslateJsonMapToEnvConfigFile(t *testing.T) { "aws_sdk_log_level": "loglevel", }, } - envConfigPath := path.Join(t.TempDir(), "env-config.json") + envConfigPath := filepath.Join(t.TempDir(), "env-config.json") expectedFile := "testdata/env-config.json" TranslateJsonMapToEnvConfigFile(jsonConfigValue, envConfigPath) @@ -233,3 +235,80 @@ func checkIfSchemaValidateAsExpected(t *testing.T, jsonInputPath string, shouldS assert.False(t, shouldSuccess, "It should pass the schemaValidation!") } } + +func TestGenerateMergedJsonConfigMap_OnlyYAML(t *testing.T) { + testCases := map[string]string{ + "WithYAMLFile": "config.yaml", + "WithYAMLTmpFile": "config.yaml.tmp", + } + for name, file := range testCases { + t.Run(name, func(t *testing.T) { + tmpDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(tmpDir, file), nil, 0600)) + + translatorcontext.ResetContext() + ctx := translatorcontext.CurrentContext() + ctx.SetInputJsonDirPath(tmpDir) + ctx.SetMultiConfig("default") + + got, err := GenerateMergedJsonConfigMap(ctx) + + assert.ErrorIs(t, err, ErrOnlyYAML) + assert.Nil(t, got) + }) + } +} + +func TestGenerateMergedJsonConfigMap_MixedJSONAndYAML(t *testing.T) { + for _, mode := range []string{"append", "remove"} { + t.Run(mode, func(t *testing.T) { + tmpDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(tmpDir, "config.json"), []byte(`{"agent":{"debug":true}}`), 0600)) + require.NoError(t, os.WriteFile(filepath.Join(tmpDir, "otel.yaml"), nil, 0600)) + + translatorcontext.ResetContext() + ctx := translatorcontext.CurrentContext() + ctx.SetInputJsonDirPath(tmpDir) + ctx.SetMultiConfig(mode) + + result, err := GenerateMergedJsonConfigMap(ctx) + + assert.NoError(t, err) + assert.NotNil(t, result) + }) + } +} + +func TestGenerateMergedJsonConfigMap_DefaultModeWithTmpJSONAndYAML(t *testing.T) { + tmpDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(tmpDir, "config.json.tmp"), []byte(`{"agent":{"debug":true}}`), 0600)) + require.NoError(t, os.WriteFile(filepath.Join(tmpDir, "otel.yaml"), nil, 0600)) + + translatorcontext.ResetContext() + ctx := translatorcontext.CurrentContext() + ctx.SetInputJsonDirPath(tmpDir) + ctx.SetMultiConfig("default") + + result, err := GenerateMergedJsonConfigMap(ctx) + + assert.NoError(t, err) + assert.NotNil(t, result) +} + +func TestGenerateMergedJsonConfigMap_EnvVarJSONWithYAML(t *testing.T) { + tmpDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(tmpDir, "otel.yaml"), nil, 0600)) + + t.Setenv(envconfig.RunInContainer, envconfig.TrueValue) + t.Setenv(envconfig.CWConfigContent, `{"agent":{"debug":true}}`) + + translatorcontext.ResetContext() + ctx := translatorcontext.CurrentContext() + ctx.SetInputJsonDirPath(tmpDir) + ctx.SetMultiConfig("default") + + result, err := GenerateMergedJsonConfigMap(ctx) + + assert.NoError(t, err) + assert.NotNil(t, result) +}