diff --git a/apps/builtinconf.go b/apps/builtinconf.go index 361f2f2063..558d33d2b6 100644 --- a/apps/builtinconf.go +++ b/apps/builtinconf.go @@ -16,8 +16,8 @@ package apps import cg "github.com/GoogleCloudPlatform/ops-agent/confgenerator" -var ( - BuiltInConfStructs = map[string]*cg.UnifiedConfig{ +func init() { + cg.BuiltInConfStructs = map[string]*cg.UnifiedConfig{ "linux": { Logging: &cg.Logging{ Receivers: map[string]cg.LoggingReceiver{ @@ -105,4 +105,4 @@ var ( }, }, } -) +} diff --git a/cmd/agent_wrapper/main.go b/cmd/agent_wrapper/main.go index 44430c9398..ea0dd83c36 100644 --- a/cmd/agent_wrapper/main.go +++ b/cmd/agent_wrapper/main.go @@ -21,7 +21,7 @@ import ( "os" "os/exec" - "github.com/GoogleCloudPlatform/ops-agent/apps" + _ "github.com/GoogleCloudPlatform/ops-agent/apps" "github.com/GoogleCloudPlatform/ops-agent/confgenerator" "gopkg.in/natefinch/lumberjack.v2" ) @@ -34,7 +34,7 @@ func getLogFileRotation(config *confgenerator.UnifiedConfig) confgenerator.LogFi } func run(logFilename, configurationPath string, cmd *exec.Cmd) error { - ucConfig, err := confgenerator.MergeConfFiles(context.Background(), configurationPath, apps.BuiltInConfStructs) + ucConfig, err := confgenerator.MergeConfFiles(context.Background(), configurationPath) if err != nil { return err } diff --git a/cmd/google_cloud_ops_agent_engine/main.go b/cmd/google_cloud_ops_agent_engine/main.go index a263f7c3a3..6fce0c5916 100644 --- a/cmd/google_cloud_ops_agent_engine/main.go +++ b/cmd/google_cloud_ops_agent_engine/main.go @@ -20,7 +20,7 @@ import ( "log" "os" - "github.com/GoogleCloudPlatform/ops-agent/apps" + _ "github.com/GoogleCloudPlatform/ops-agent/apps" "github.com/GoogleCloudPlatform/ops-agent/confgenerator" "github.com/GoogleCloudPlatform/ops-agent/internal/healthchecks" "github.com/GoogleCloudPlatform/ops-agent/internal/logs" @@ -55,7 +55,7 @@ func main() { func run() error { ctx := context.Background() // TODO(lingshi) Move this to a shared place across Linux and Windows. - uc, err := confgenerator.MergeConfFiles(ctx, *input, apps.BuiltInConfStructs) + uc, err := confgenerator.MergeConfFiles(ctx, *input) if err != nil { return err } @@ -63,7 +63,7 @@ func run() error { // Log the built-in and merged config files to STDOUT. These are then written // by journald to var/log/syslog and so to Cloud Logging once the ops-agent is // running. - log.Printf("Built-in config:\n%s", apps.BuiltInConfStructs["linux"]) + log.Printf("Built-in config:\n%s", confgenerator.BuiltInConfStructs["linux"]) log.Printf("Merged config:\n%s", uc) switch *service { diff --git a/cmd/ops_agent_uap_plugin/plugin_test.go b/cmd/ops_agent_uap_plugin/plugin_test.go index a6e0e038e8..50dc1f5df3 100644 --- a/cmd/ops_agent_uap_plugin/plugin_test.go +++ b/cmd/ops_agent_uap_plugin/plugin_test.go @@ -23,7 +23,7 @@ import ( "buf.build/go/protoyaml" // Import the protoyaml-go package pb "github.com/GoogleCloudPlatform/google-guest-agent/pkg/proto/plugin_comm" - "github.com/GoogleCloudPlatform/ops-agent/apps" + _ "github.com/GoogleCloudPlatform/ops-agent/apps" "github.com/GoogleCloudPlatform/ops-agent/confgenerator" "github.com/GoogleCloudPlatform/ops-agent/internal/platform" spb "google.golang.org/protobuf/types/known/structpb" @@ -188,7 +188,7 @@ func TestWriteCustomConfigToFile(t *testing.T) { t.Errorf("%v: writeCustomConfigToFile got error: %v, want nil error", tc.name, err) } - _, err = confgenerator.MergeConfFiles(context.Background(), configPath, apps.BuiltInConfStructs) + _, err = confgenerator.MergeConfFiles(context.Background(), configPath) if err != nil { t.Errorf("%v: conf generator fails to validate the output Ops agent yaml: %v", tc.name, err) } diff --git a/cmd/ops_agent_uap_plugin/service_windows.go b/cmd/ops_agent_uap_plugin/service_windows.go index b8e9720884..687a475130 100644 --- a/cmd/ops_agent_uap_plugin/service_windows.go +++ b/cmd/ops_agent_uap_plugin/service_windows.go @@ -29,7 +29,7 @@ import ( "sync" "unsafe" - "github.com/GoogleCloudPlatform/ops-agent/apps" + _ "github.com/GoogleCloudPlatform/ops-agent/apps" "github.com/GoogleCloudPlatform/ops-agent/confgenerator" "github.com/GoogleCloudPlatform/ops-agent/internal/healthchecks" "github.com/GoogleCloudPlatform/ops-agent/internal/logs" @@ -207,12 +207,12 @@ func findPreExistentAgents(mgr serviceManager, agentWindowsServiceNames []string } func generateSubAgentConfigs(ctx context.Context, userConfigPath string, pluginStateDir string) error { - uc, err := confgenerator.MergeConfFiles(ctx, userConfigPath, apps.BuiltInConfStructs) + uc, err := confgenerator.MergeConfFiles(ctx, userConfigPath) if err != nil { return err } - log.Printf("Built-in config:\n%s\n", apps.BuiltInConfStructs["windows"]) + log.Printf("Built-in config:\n%s\n", confgenerator.BuiltInConfStructs["windows"]) log.Printf("Merged config:\n%s\n", uc) // The generated otlp metric json files are used only by the otel service. diff --git a/cmd/ops_agent_windows/run_windows.go b/cmd/ops_agent_windows/run_windows.go index 22851f5341..17b3df7206 100644 --- a/cmd/ops_agent_windows/run_windows.go +++ b/cmd/ops_agent_windows/run_windows.go @@ -23,7 +23,7 @@ import ( "os" "path/filepath" - "github.com/GoogleCloudPlatform/ops-agent/apps" + _ "github.com/GoogleCloudPlatform/ops-agent/apps" "github.com/GoogleCloudPlatform/ops-agent/confgenerator" "github.com/GoogleCloudPlatform/ops-agent/internal/healthchecks" "github.com/GoogleCloudPlatform/ops-agent/internal/logs" @@ -158,12 +158,12 @@ func (srv *service) runHealthChecks() { func (s *service) generateConfigs(ctx context.Context) error { // TODO(lingshi) Move this to a shared place across Linux and Windows. - uc, err := confgenerator.MergeConfFiles(ctx, s.userConf, apps.BuiltInConfStructs) + uc, err := confgenerator.MergeConfFiles(ctx, s.userConf) if err != nil { return err } - s.log.Info(EngineEventID, fmt.Sprintf("Built-in config:\n%s\n", apps.BuiltInConfStructs["windows"])) + s.log.Info(EngineEventID, fmt.Sprintf("Built-in config:\n%s\n", confgenerator.BuiltInConfStructs["windows"])) s.log.Info(EngineEventID, fmt.Sprintf("Merged config:\n%s\n", uc)) if err := s.checkForStandaloneAgents(uc); err != nil { return err diff --git a/cmd/simulacra/simulacra.go b/cmd/simulacra/simulacra.go index 1d5ea8994f..5461148443 100644 --- a/cmd/simulacra/simulacra.go +++ b/cmd/simulacra/simulacra.go @@ -30,7 +30,7 @@ import ( "github.com/GoogleCloudPlatform/opentelemetry-operations-collector/integration_test/gce-testing-internal/gce" "github.com/GoogleCloudPlatform/opentelemetry-operations-collector/integration_test/gce-testing-internal/logging" - "github.com/GoogleCloudPlatform/ops-agent/apps" + _ "github.com/GoogleCloudPlatform/ops-agent/apps" "github.com/GoogleCloudPlatform/ops-agent/confgenerator" "github.com/GoogleCloudPlatform/ops-agent/integration_test/agents" "github.com/binxio/gcloudconfig" @@ -166,7 +166,7 @@ func getReceiversFromConfig(ctx context.Context, vm *gce.VM, logger *logging.Dir return []string{}, nil } - config, err := confgenerator.MergeConfFiles(ctx, configFilePath, apps.BuiltInConfStructs) + config, err := confgenerator.MergeConfFiles(ctx, configFilePath) if err != nil { return nil, err } diff --git a/confgenerator/agentmetrics.go b/confgenerator/agentmetrics.go index c46ec11642..649bf18d7f 100644 --- a/confgenerator/agentmetrics.go +++ b/confgenerator/agentmetrics.go @@ -33,7 +33,6 @@ type AgentSelfMetrics struct { FluentBitPort int OtelPort int OtelRuntimeDir string - OtelLogging bool } // Following reference : https://github.com/googleapis/googleapis/blob/master/google/rpc/code.proto diff --git a/confgenerator/confgenerator.go b/confgenerator/confgenerator.go index ac4bf13d31..1b9ef31e51 100644 --- a/confgenerator/confgenerator.go +++ b/confgenerator/confgenerator.go @@ -231,7 +231,6 @@ func (uc *UnifiedConfig) GenerateOtelConfig(ctx context.Context, outDir, stateDi FluentBitPort: fluentbit.MetricsPort, OtelPort: otel.MetricsPort, OtelRuntimeDir: outDir, - OtelLogging: uc.Logging.Service.OTelLogging, } agentSelfMetrics.AddSelfMetricsPipelines(receiverPipelines, pipelines, ctx) resource, err := p.GetResource() @@ -412,6 +411,10 @@ func (p PipelineInstance) OTelComponents(ctx context.Context) (map[string]otel.R if err != nil { return nil, nil, fmt.Errorf("receiver %q has invalid configuration: %w", p.RID, err) } + gceMetadataAttributesProcessors, err := addGceMetadataAttributesProcessor(ctx).Processors(ctx) + if err != nil { + panic("Failed to generate static ModifyFields") + } for i, receiverPipeline := range receiverPipelines { receiverPipelineName := strings.ReplaceAll(p.RID, "_", "__") if i > 0 { @@ -435,6 +438,10 @@ func (p PipelineInstance) OTelComponents(ctx context.Context) (map[string]otel.R otelFluentForwardSetLogNameComponents()..., ) } + receiverPipeline.Processors["logs"] = append( + receiverPipeline.Processors["logs"], + gceMetadataAttributesProcessors..., + ) } outR[receiverPipelineName] = receiverPipeline @@ -524,42 +531,38 @@ const ( attributeLabelPrefix string = "compute.googleapis.com/attributes/" ) -// addGceMetadataAttributesComponents annotates logs with labels corresponding -// to instance attributes from the GCE metadata server. -func addGceMetadataAttributesComponents(ctx context.Context, attributes []string, tag, uid string) []fluentbit.Component { - processorName := fmt.Sprintf("%s.%s.gce_metadata", tag, uid) +// addGceMetadataAttributesProcessor annotates logs with labels corresponding +// to specific instance attributes from the GCE metadata server. +func addGceMetadataAttributesProcessor(ctx context.Context) LoggingProcessorModifyFields { + attributes := []string{ + "dataproc-cluster-name", + "dataproc-cluster-uuid", + "dataproc-region", + } + + modifications := map[string]*ModifyField{} + p := LoggingProcessorModifyFields{ + Fields: modifications, + } resource, err := platform.FromContext(ctx).GetResource() if err != nil { log.Printf("can't get resource metadata: %v", err) - return nil + return p } gceMetadata, ok := resource.(resourcedetector.GCEResource) if !ok { // Not on GCE; no attributes to detect. log.Printf("ignoring the gce_metadata_attributes processor outside of GCE: %T", resource) - return nil - } - modifications := map[string]*ModifyField{} - var attributeKeys []string - for k, _ := range gceMetadata.Metadata { - attributeKeys = append(attributeKeys, k) + return p } - sort.Strings(attributeKeys) - for _, k := range attributeKeys { - if !sliceContains(attributes, k) { - continue - } - v := gceMetadata.Metadata[k] - modifications[fmt.Sprintf(`labels."%s%s"`, attributeLabelPrefix, k)] = &ModifyField{ - StaticValue: &v, + for _, k := range attributes { + if v, ok := gceMetadata.Metadata[k]; ok { + modifications[fmt.Sprintf(`labels."%s%s"`, attributeLabelPrefix, k)] = &ModifyField{ + StaticValue: &v, + } } } - if len(modifications) == 0 { - return nil - } - return LoggingProcessorModifyFields{ - Fields: modifications, - }.Components(ctx, tag, processorName) + return p } type fbSource struct { @@ -578,7 +581,7 @@ func (uc *UnifiedConfig) generateFluentbitComponents(ctx context.Context, userAg out = append(out, service.Component()) out = append(out, fluentbit.MetricsInputComponent()) - if l != nil && l.Service != nil && !l.Service.OTelLogging { + if l != nil && l.Service != nil && (l.Service.OTelLogging == nil || !*l.Service.OTelLogging) { // Type for sorting. var sources []fbSource var tags []string @@ -606,11 +609,7 @@ func (uc *UnifiedConfig) generateFluentbitComponents(ctx context.Context, userAg if len(tags) > 0 { out = append(out, stackdriverOutputComponent(ctx, strings.Join(tags, "|"), userAgent, "2G", l.Service.Compress)) } - out = append(out, addGceMetadataAttributesComponents(ctx, []string{ - "dataproc-cluster-name", - "dataproc-cluster-uuid", - "dataproc-region", - }, "*", "default-dataproc")...) + out = append(out, addGceMetadataAttributesProcessor(ctx).Components(ctx, "*", "*.default-data-proc.gce_metadata")...) } out = append(out, uc.generateSelfLogsComponents(ctx, userAgent)...) out = append(out, fluentbit.MetricsOutputComponent()) diff --git a/confgenerator/confgenerator_test.go b/confgenerator/confgenerator_test.go index 90fe8449b3..6335540f2c 100644 --- a/confgenerator/confgenerator_test.go +++ b/confgenerator/confgenerator_test.go @@ -25,7 +25,7 @@ import ( "testing" "github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp" - "github.com/GoogleCloudPlatform/ops-agent/apps" + _ "github.com/GoogleCloudPlatform/ops-agent/apps" "github.com/GoogleCloudPlatform/ops-agent/confgenerator" "github.com/GoogleCloudPlatform/ops-agent/confgenerator/resourcedetector" "github.com/GoogleCloudPlatform/ops-agent/internal/platform" @@ -261,12 +261,11 @@ func generateConfigs(pc platformConfig, testDir string) (got map[string]string, mergedUc, err := confgenerator.MergeConfFiles( ctx, filepath.Join("testdata", testDir, inputFileName), - apps.BuiltInConfStructs, ) if err != nil { return } - got[builtinConfigFileName] = apps.BuiltInConfStructs[pc.platform.Name()].String() + got[builtinConfigFileName] = confgenerator.BuiltInConfStructs[pc.platform.Name()].String() // Fluent Bit configs flbGeneratedConfigs, err := mergedUc.GenerateFluentBitConfigs(ctx, @@ -351,7 +350,6 @@ func generateOtelConfigWithOtlpExporterEnabled(got map[string]string, experiment mergedUcOtlp, err := confgenerator.MergeConfFiles( ctxOtlp, filepath.Join("testdata", testDir, inputFileName), - apps.BuiltInConfStructs, ) if err == nil { otelGeneratedConfigOtlp, err := mergedUcOtlp.GenerateOtelConfig(ctxOtlp, "", "") diff --git a/confgenerator/config.go b/confgenerator/config.go index 401b596bd4..531f76f1f4 100644 --- a/confgenerator/config.go +++ b/confgenerator/config.go @@ -21,6 +21,7 @@ import ( "path/filepath" "reflect" "runtime" + "slices" "sort" "strings" "time" @@ -38,6 +39,10 @@ import ( "go.uber.org/multierr" ) +// BuiltInConfStructs contains the default configuration for each platform. +// It is populated by an initializer in the `apps` package. +var BuiltInConfStructs map[string]*UnifiedConfig + // Ops Agent config. type UnifiedConfig struct { Combined *Combined `yaml:"combined,omitempty"` @@ -583,7 +588,7 @@ type LoggingService struct { Compress string `yaml:"compress,omitempty" validate:"omitempty,oneof=gzip,experimental=log_compression"` LogLevel string `yaml:"log_level,omitempty" validate:"omitempty,oneof=error warn info debug trace"` Pipelines map[string]*Pipeline `validate:"dive,keys,startsnotwith=lib:"` - OTelLogging bool `yaml:"experimental_otel_logging,omitempty" validate:"omitempty,experimental=otel_logging"` + OTelLogging *bool `yaml:"experimental_otel_logging,omitempty" validate:"omitempty"` } type Pipeline struct { @@ -1084,16 +1089,25 @@ func (uc *UnifiedConfig) loggingPipelines(ctx context.Context) ([]PipelineInstan if err != nil { return nil, err } + platformDefaultConfig := BuiltInConfStructs[platform.FromContext(ctx).Name()].Logging exp_otlp := experimentsFromContext(ctx)["otlp_logging"] - exp_otel := l.Service.OTelLogging + // N.B. Temporarily gate the "auto" otel logging behind an experiment flag. + exp_otel := experimentsFromContext(ctx)["otel_logging"] + force_otel := l.Service.OTelLogging var out []PipelineInstance for _, pID := range otel.SortedKeys(l.Service.Pipelines) { p := l.Service.Pipelines[pID] + defaultP, ok := platformDefaultConfig.Service.Pipelines[pID] + isDefaultPipeline := ok && slices.Equal(p.ReceiverIDs, defaultP.ReceiverIDs) && slices.Equal(p.ProcessorIDs, defaultP.ProcessorIDs) for _, rID := range p.ReceiverIDs { receiver, ok := receivers[rID] if !ok { return nil, fmt.Errorf("logging receiver %q not found", rID) } + defaultReceiver, ok := platformDefaultConfig.Receivers[rID] + if !ok || !reflect.DeepEqual(receiver, defaultReceiver) { + isDefaultPipeline = false + } var processors []struct { ID string Component @@ -1120,7 +1134,9 @@ func (uc *UnifiedConfig) loggingPipelines(ctx context.Context) ([]PipelineInstan Receiver: receiver, Processors: processors, } - if exp_otel || (receiver.Type() == "otlp" && exp_otlp) { + if (force_otel != nil && *force_otel) || // User asked for OTel logging + (exp_otel && force_otel == nil && isDefaultPipeline) || // Unmodified default pipeline + (receiver.Type() == "otlp" && exp_otlp) { // OTLP receiver instance.Backend = BackendOTel } out = append(out, instance) @@ -1195,7 +1211,8 @@ func (uc *UnifiedConfig) OTelLoggingSupported(ctx context.Context) bool { if ucLoggingCopy.Logging.Service == nil { ucLoggingCopy.Logging.Service = &LoggingService{} } - ucLoggingCopy.Logging.Service.OTelLogging = true + t := true + ucLoggingCopy.Logging.Service.OTelLogging = &t _, err = ucLoggingCopy.GenerateOtelConfig(ctx, "", "") return err == nil } diff --git a/confgenerator/confmerger.go b/confgenerator/confmerger.go index 666b0b4d52..9369e51cb7 100644 --- a/confgenerator/confmerger.go +++ b/confgenerator/confmerger.go @@ -23,8 +23,8 @@ import ( ) // MergeConfFiles merges the user provided config with the built-in config struct for the platform. -func MergeConfFiles(ctx context.Context, userConfPath string, builtInConfStructs map[string]*UnifiedConfig) (*UnifiedConfig, error) { - builtInStruct := builtInConfStructs[platform.FromContext(ctx).Name()] +func MergeConfFiles(ctx context.Context, userConfPath string) (*UnifiedConfig, error) { + builtInStruct := BuiltInConfStructs[platform.FromContext(ctx).Name()] // Start with the built-in config. result, err := builtInStruct.DeepCopy(ctx) diff --git a/confgenerator/feature_tracking_test.go b/confgenerator/feature_tracking_test.go index d6dfe1c58e..51a41df47f 100644 --- a/confgenerator/feature_tracking_test.go +++ b/confgenerator/feature_tracking_test.go @@ -26,7 +26,7 @@ import ( "testing" "time" - "github.com/GoogleCloudPlatform/ops-agent/apps" + _ "github.com/GoogleCloudPlatform/ops-agent/apps" "github.com/GoogleCloudPlatform/ops-agent/confgenerator" "github.com/GoogleCloudPlatform/ops-agent/confgenerator/fluentbit" "github.com/GoogleCloudPlatform/ops-agent/confgenerator/otel" @@ -42,7 +42,7 @@ import ( ) var emptyUc = confgenerator.UnifiedConfig{} -var builtInConfLinux = apps.BuiltInConfStructs["linux"] +var builtInConfLinux = confgenerator.BuiltInConfStructs["linux"] var expectedFeatureBase = []confgenerator.Feature{ { diff --git a/confgenerator/logging_modify_fields.go b/confgenerator/logging_modify_fields.go index 941a1ca9ef..c2b3f9109d 100644 --- a/confgenerator/logging_modify_fields.go +++ b/confgenerator/logging_modify_fields.go @@ -74,6 +74,10 @@ func (p LoggingProcessorModifyFields) Components(ctx context.Context, tag, uid s return c } func (p LoggingProcessorModifyFields) components(tag, uid string) ([]fluentbit.Component, error) { + if len(p.Fields) == 0 { + // Don't bother generating anything if we're a noop. + return nil, nil + } var lua strings.Builder lua.WriteString(` function process(tag, timestamp, record) @@ -245,6 +249,9 @@ func (p LoggingProcessorModifyFields) Processors(ctx context.Context) ([]otel.Co if err != nil { return nil, err } + if len(out) == 0 { + return nil, nil + } return []otel.Component{otel.Transform( "log", "log", out, diff --git a/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/0f15dbe303dc7122d43443c9a4c31632.lua b/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/0f15dbe303dc7122d43443c9a4c31632.lua new file mode 100644 index 0000000000..dda27aee97 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/0f15dbe303dc7122d43443c9a4c31632.lua @@ -0,0 +1,28 @@ + +function process(tag, timestamp, record) +local v = "ops-agent"; +(function(value) +if record["logging.googleapis.com/labels"] == nil +then +record["logging.googleapis.com/labels"] = {} +end +record["logging.googleapis.com/labels"]["agent.googleapis.com/health/agentKind"] = value +end)(v) +local v = "latest"; +(function(value) +if record["logging.googleapis.com/labels"] == nil +then +record["logging.googleapis.com/labels"] = {} +end +record["logging.googleapis.com/labels"]["agent.googleapis.com/health/agentVersion"] = value +end)(v) +local v = "v1"; +(function(value) +if record["logging.googleapis.com/labels"] == nil +then +record["logging.googleapis.com/labels"] = {} +end +record["logging.googleapis.com/labels"]["agent.googleapis.com/health/schemaVersion"] = value +end)(v) +return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/4d6012ff003886818fb9b9285b4af962.lua b/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/4d6012ff003886818fb9b9285b4af962.lua new file mode 100644 index 0000000000..c225be24d5 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/4d6012ff003886818fb9b9285b4af962.lua @@ -0,0 +1,19 @@ + +function process(tag, timestamp, record) +local __field_0 = (function() +return record["severity"] +end)(); +(function(value) +record["severity"] = value +end)(nil); +local v = __field_0; +if v == "debug" then v = "DEBUG" +elseif v == "error" then v = "ERROR" +elseif v == "info" then v = "INFO" +elseif v == "warn" then v = "WARNING" +end +(function(value) +record["logging.googleapis.com/severity"] = value +end)(v) +return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/5fc5f42c16c9e1ab8292e3d42f74f3be.lua b/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/5fc5f42c16c9e1ab8292e3d42f74f3be.lua new file mode 100644 index 0000000000..c5465182b5 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/5fc5f42c16c9e1ab8292e3d42f74f3be.lua @@ -0,0 +1,49 @@ + + function shallow_merge(record, parsedRecord) + -- If no exiting record exists + if (record == nil) then + return parsedRecord + end + + for k, v in pairs(parsedRecord) do + record[k] = v + end + + return record +end + +function merge(record, parsedRecord) + -- If no exiting record exists + if record == nil then + return parsedRecord + end + + -- Potentially overwrite or merge the original records. + for k, v in pairs(parsedRecord) do + -- If there is no conflict + if k == "logging.googleapis.com/logName" then + -- Ignore the parsed payload since the logName is controlled + -- by the OpsAgent. + elseif k == "logging.googleapis.com/labels" then + -- LogEntry.labels are basically a map[string]string and so only require a + -- shallow merge (one level deep merge). + record[k] = shallow_merge(record[k], v) + else + record[k] = v + end + end + + return record +end + +function parser_merge_record(tag, timestamp, record) + originalPayload = record["logging.googleapis.com/__tmp"] + if originalPayload == nil then + return 0, timestamp, record + end + + -- Remove original payload + record["logging.googleapis.com/__tmp"] = nil + record = merge(originalPayload, record) + return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/adea349dc2d92cd07daa1d7847f5e96a.lua b/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/adea349dc2d92cd07daa1d7847f5e96a.lua new file mode 100644 index 0000000000..b2a60a042b --- /dev/null +++ b/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/adea349dc2d92cd07daa1d7847f5e96a.lua @@ -0,0 +1,74 @@ + +function process(tag, timestamp, record) +local __field_0 = (function() +return record["agent.googleapis.com/log_file_path"] +end)(); +local __field_1 = (function() +if record["logging.googleapis.com/labels"] == nil +then +return nil +end +return record["logging.googleapis.com/labels"]["compute.googleapis.com/instance_group_manager/name"] +end)(); +local __field_2 = (function() +if record["logging.googleapis.com/labels"] == nil +then +return nil +end +return record["logging.googleapis.com/labels"]["compute.googleapis.com/instance_group_manager/zone"] +end)(); +local __field_3 = (function() +if record["logging.googleapis.com/labels"] == nil +then +return nil +end +return record["logging.googleapis.com/labels"]["compute.googleapis.com/resource_name"] +end)(); +local __field_4 = (function() +return record["logging.googleapis.com/logName"] +end)(); +(function(value) +record["agent.googleapis.com/log_file_path"] = value +end)(nil); +local v = __field_0; +(function(value) +if record["logging.googleapis.com/labels"] == nil +then +record["logging.googleapis.com/labels"] = {} +end +record["logging.googleapis.com/labels"]["agent.googleapis.com/log_file_path"] = value +end)(v) +local v = __field_1; +if v == nil then v = "test-mig" end; +(function(value) +if record["logging.googleapis.com/labels"] == nil +then +record["logging.googleapis.com/labels"] = {} +end +record["logging.googleapis.com/labels"]["compute.googleapis.com/instance_group_manager/name"] = value +end)(v) +local v = __field_2; +if v == nil then v = "test-zone" end; +(function(value) +if record["logging.googleapis.com/labels"] == nil +then +record["logging.googleapis.com/labels"] = {} +end +record["logging.googleapis.com/labels"]["compute.googleapis.com/instance_group_manager/zone"] = value +end)(v) +local v = __field_3; +if v == nil then v = "" end; +(function(value) +if record["logging.googleapis.com/labels"] == nil +then +record["logging.googleapis.com/labels"] = {} +end +record["logging.googleapis.com/labels"]["compute.googleapis.com/resource_name"] = value +end)(v) +local v = __field_4; +if v == nil then v = "syslog" end; +(function(value) +record["logging.googleapis.com/logName"] = value +end)(v) +return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/b4a0dead382dce7b4fe011d3f59fdb6d.lua b/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/b4a0dead382dce7b4fe011d3f59fdb6d.lua new file mode 100644 index 0000000000..6263563b66 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/b4a0dead382dce7b4fe011d3f59fdb6d.lua @@ -0,0 +1,17 @@ + +function parser_nest(tag, timestamp, record) + local nestedRecord = {} + local parseKey = "message" + for k, v in pairs(record) do + if k ~= parseKey then + nestedRecord[k] = v + end + end + + local result = {} + result[parseKey] = record[parseKey] + result["logging.googleapis.com/__tmp"] = nestedRecord + + return 2, timestamp, result +end + diff --git a/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/enabled_receivers_otlp.json b/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/enabled_receivers_otlp.json new file mode 100644 index 0000000000..9cf0a64e2d --- /dev/null +++ b/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/enabled_receivers_otlp.json @@ -0,0 +1 @@ +{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/ops_agent/enabled_receivers","gauge":{"dataPoints":[{"attributes":[{"key":"telemetry_type","value":{"stringValue":"metrics"}},{"key":"receiver_type","value":{"stringValue":"hostmetrics"}}],"asInt":"1"},{"attributes":[{"key":"telemetry_type","value":{"stringValue":"logs"}},{"key":"receiver_type","value":{"stringValue":"files"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/feature_tracking_otlp.json b/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/feature_tracking_otlp.json new file mode 100644 index 0000000000..4243dee2e7 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/feature_tracking_otlp.json @@ -0,0 +1 @@ +{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"exporters:otlp"}},{"key":"key","value":{"stringValue":"otlp_exporter"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"exporters:otlp"}},{"key":"key","value":{"stringValue":"otlp_exporter"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/features.yaml b/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/features.yaml new file mode 100644 index 0000000000..23bb718b20 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/features.yaml @@ -0,0 +1,24 @@ +- module: logging + feature: service:pipelines + key: default_pipeline_overridden + value: "false" +- module: metrics + feature: service:pipelines + key: default_pipeline_overridden + value: "false" +- module: global + feature: default:self_log + key: default_self_log_file_collection + value: "true" +- module: logging + feature: service:otel_logging + key: otel_logging_supported_config + value: "true" +- module: metrics + feature: exporters:otlp + key: otlp_exporter + value: "false" +- module: logging + feature: exporters:otlp + key: otlp_exporter + value: "false" diff --git a/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/fluent_bit_main.conf new file mode 100644 index 0000000000..a41f7f5311 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/fluent_bit_main.conf @@ -0,0 +1,187 @@ +@SET buffers_dir=/var/lib/google-cloud-ops-agent/fluent-bit/buffers +@SET logs_dir=/var/log/google-cloud-ops-agent + +[SERVICE] + Daemon off + Flush 1 + Log_Level info + dns.resolver legacy + storage.backlog.mem_limit 50M + storage.checksum off + storage.max_chunks_up 128 + storage.metrics on + storage.sync normal + +[INPUT] + Name fluentbit_metrics + Scrape_Interval 60 + Scrape_On_Start True + +[INPUT] + Buffer_Chunk_Size 512k + Buffer_Max_Size 2M + DB ${buffers_dir}/default_pipeline_syslog + DB.locking true + Key message + Mem_Buf_Limit 10M + Name tail + Path /var/log/messages,/var/log/syslog + Read_from_Head True + Rotate_Wait 30 + Skip_Long_Lines On + Tag default_pipeline.syslog + storage.type filesystem + +[INPUT] + Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} + Interval_NSec 0 + Interval_Sec 600 + Name dummy + Tag ops-agent-health + +[INPUT] + Buffer_Chunk_Size 512k + Buffer_Max_Size 2M + DB ${buffers_dir}/ops-agent-fluent-bit + DB.locking true + Key message + Mem_Buf_Limit 10M + Name tail + Path ${logs_dir}/subagents/logging-module.log + Read_from_Head True + Rotate_Wait 30 + Skip_Long_Lines On + Tag ops-agent-fluent-bit + storage.type memory + +[INPUT] + Buffer_Chunk_Size 512k + Buffer_Max_Size 2M + DB ${buffers_dir}/ops-agent-health + DB.locking true + Key message + Mem_Buf_Limit 10M + Name tail + Path ${logs_dir}/health-checks.log + Read_from_Head True + Rotate_Wait 30 + Skip_Long_Lines On + Tag ops-agent-health + storage.type memory + +[FILTER] + Match default_pipeline.syslog + Name lua + call process + script adea349dc2d92cd07daa1d7847f5e96a.lua + +[FILTER] + Match ops-agent-fluent-bit + Name lua + call parser_nest + script b4a0dead382dce7b4fe011d3f59fdb6d.lua + +[FILTER] + Key_Name message + Match ops-agent-fluent-bit + Name parser + Preserve_Key True + Reserve_Data True + Parser ops-agent-fluent-bit.fluent-bit-self-log-regex-parsing + +[FILTER] + Match ops-agent-fluent-bit + Name lua + call parser_merge_record + script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua + +[FILTER] + Match ops-agent-health + Name lua + call parser_nest + script b4a0dead382dce7b4fe011d3f59fdb6d.lua + +[FILTER] + Key_Name message + Match ops-agent-health + Name parser + Reserve_Data True + Parser ops-agent-health.health-checks-json + +[FILTER] + Match ops-agent-health + Name lua + call parser_merge_record + script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua + +[FILTER] + Match ops-agent-health + Name grep + Regex severity INFO|ERROR|WARNING|DEBUG|info|error|warning|debug + +[FILTER] + Match ops-agent-fluent-bit + Name rewrite_tag + Rule message \[error\]\s\[lib\]\sbackend\sfailed ops-agent-health true + +[FILTER] + Name modify + Match ops-agent-health + Condition Key_value_matches message \[error\]\s\[lib\]\sbackend\sfailed + Set code LogPipelineErr + Set message "[Runtime Check] Result: FAIL, Error code: LogPipelineErr, Failure: Ops Agent logging pipeline failed, Solution: Refer to provided documentation link., Resource: https://cloud.google.com/stackdriver/docs/solutions/agents/ops-agent/troubleshoot-find-info" + +[FILTER] + Match ops-agent-fluent-bit + Name rewrite_tag + Rule message \[error\]\s\[parser\]\scannot\sparse ops-agent-health true + +[FILTER] + Name modify + Match ops-agent-health + Condition Key_value_matches message \[error\]\s\[parser\]\scannot\sparse + Set code LogParseErr + Set message "[Runtime Check] Result: WARNING, Error code: LogParseErr, Failure: Ops Agent failed to parse logs, Solution: Refer to provided documentation link., Resource: https://cloud.google.com/stackdriver/docs/solutions/agents/ops-agent/troubleshoot-find-info" + +[FILTER] + Match ops-agent-health + Name lua + call process + script 0f15dbe303dc7122d43443c9a4c31632.lua + +[FILTER] + Match ops-agent-* + Name lua + call process + script 4d6012ff003886818fb9b9285b4af962.lua + +[OUTPUT] + Match_Regex ^(default_pipeline\.syslog)$ + Name stackdriver + Retry_Limit 3 + http_request_key logging.googleapis.com/httpRequest + net.connect_timeout_log_error False + resource gce_instance + stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) + storage.total_limit_size 2G + tls On + tls.verify Off + workers 8 + +[OUTPUT] + Match_Regex ^(ops-agent-health|ops-agent-fluent-bit)$ + Name stackdriver + Retry_Limit 3 + http_request_key logging.googleapis.com/httpRequest + net.connect_timeout_log_error False + resource gce_instance + stackdriver_agent Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) + tls On + tls.verify Off + workers 8 + +[OUTPUT] + Match * + Name prometheus_exporter + host 0.0.0.0 + port 20202 diff --git a/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/fluent_bit_parser.conf new file mode 100644 index 0000000000..3c603780bd --- /dev/null +++ b/confgenerator/testdata/goldens/logging-default_no_otel/golden/linux-gpu/fluent_bit_parser.conf @@ -0,0 +1,13 @@ +[PARSER] + Format regex + Name ops-agent-fluent-bit.fluent-bit-self-log-regex-parsing + Regex (?\[[ ]*(?