diff --git a/.gitignore b/.gitignore index 66df0f33..aca7edef 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ /vendor /bin *.pprof +*.csv +/local diff --git a/.golangci.yml b/.golangci.yml index 6d4bdc02..bd213c55 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -57,7 +57,7 @@ linters: exhaustruct: # Structs that must have all fields explicitly set include: - - github.com/form3tech-oss/f1/v2/internal/run/views.* + - github.com/form3tech-oss/f1/v3/internal/run/views.* nolintlint: require-explanation: true # nolint directives must explain why @@ -79,14 +79,12 @@ linters: - std-error-handling rules: - # Deprecated Logger/WithLogrusLogger - legacy API support - - linters: [staticcheck] - path: \.go - text: "SA1019: (.*.Logger|testing.WithLogrusLogger)" - # Test files - relaxed rules for readability - linters: [dupword, lll, unparam, wrapcheck] path: _test\.go + # cobra.AddTemplateFunc modifies a global map; sync.Once is required + - linters: [gochecknoglobals] + path: internal/run/help\.go - linters: [staticcheck] path: _test\.go text: ST1003 diff --git a/README.md b/README.md index 52a1d769..045f9146 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -Go Reference +Go Reference # f1 `f1` is a flexible load testing framework using the `go` language for test scenarios. This allows test scenarios to be developed as code, utilising full development principles such as test driven development. Test scenarios with multiple stages and multiple modes are ideally suited to this environment. @@ -15,12 +15,12 @@ These `ScenarioFn` and `RunFn` functions are defined as types in `f1`: ```golang // ScenarioFn initialises a scenario and returns the iteration function (RunFn) to be invoked for every iteration -// of the tests. -type ScenarioFn func(t *T) RunFn +// of the tests. ctx is cancelled when the run is interrupted or times out. +type ScenarioFn func(ctx context.Context, t *T) RunFn // RunFn performs a single iteration of the scenario. 't' may be used for asserting -// results or failing the scenario. -type RunFn func(t *T) +// results or failing the scenario. ctx is cancelled when the run is stopped; check ctx.Done() for cancellation. +type RunFn func(ctx context.Context, t *T) ``` Writing tests is simply a case of implementing the types and registering them with `f1`: @@ -29,31 +29,32 @@ Writing tests is simply a case of implementing the types and registering them wi package main import ( + "context" "fmt" - "github.com/form3tech-oss/f1/v2/pkg/f1" - "github.com/form3tech-oss/f1/v2/pkg/f1/testing" + "github.com/form3tech-oss/f1/v3/pkg/f1" + "github.com/form3tech-oss/f1/v3/pkg/f1/f1testing" ) func main() { // Create a new f1 instance, add all the scenarios and execute the f1 tool. // Any scenario that is added here can be executed like: `go run main.go run constant mySuperFastLoadTest` - f1.New().Add("mySuperFastLoadTest", setupMySuperFastLoadTest).Execute() + f1.New().AddScenario("mySuperFastLoadTest", setupMySuperFastLoadTest).Execute() } // Performs any setup steps and returns a function to run on every iteration of the scenario -func setupMySuperFastLoadTest(t *testing.T) testing.RunFn { +func setupMySuperFastLoadTest(ctx context.Context, t *f1testing.T) f1testing.RunFn { fmt.Println("Setup the scenario") - + // Register clean up function which will be invoked at the end of the scenario execution to clean up the setup t.Cleanup(func() { fmt.Println("Clean up the setup of the scenario") }) - - runFn := func(t *testing.T) { - fmt.Println("Run the test") - // Register clean up function for each test which will be invoked in LIFO order after each iteration + runFn := func(ctx context.Context, t *f1testing.T) { + fmt.Println("Run the test") + + // Register clean up function for each test which will be invoked in LIFO order after each iteration t.Cleanup(func() { fmt.Println("Clean up the test execution") }) @@ -87,16 +88,83 @@ It provides the following information: - `(20/s)` (attempted) rate, - `avg: 72ns, min: 125ns, max: 27.590042ms` average, min and max iteration times. -### Environment variables +### Configuration + +f1 can be configured via environment variables, programmatic options, or both. By default, environment variables are read at construction time. Programmatic options override env vars for the fields they set. + +#### Settings reference -| Name | Format | Default | Description | +| Setting | Environment variable | Programmatic option | Default | | --- | --- | --- | --- | -| `PROMETHEUS_PUSH_GATEWAY` | string - `host:port` or `ip:port` | `""` | Configures the address of a [Prometheus Push Gateway](https://prometheus.io/docs/instrumenting/pushing/) for exposing metrics. The prometheus job name configured will be `f1-{scenario_name}`. Disabled by default.| -| `PROMETHEUS_NAMESPACE` | string | `""` | Sets the metric label `namespace` to the specified value. Label is omitted if the value provided is empty.| -| `PROMETHEUS_LABEL_ID` | string | `""` | Sets the metric label `id` to the specified value. Label is omitted if the value provided is empty.| -| `LOG_FILE_PATH` | string | `""`| Specify the log file path used if `--verbose` is disabled. The logfile path will be an automatically generated temp file if not specified. | -| `F1_LOG_LEVEL` | string | `"info"`| Specify the log level of the default logger, one of: `debug`, `warn`, `error` | -| `F1_LOG_FORMAT` | string | `""`| Specify the log format of the default logger, defaults to `text` formatter, allows `json` | +| Prometheus push gateway | `PROMETHEUS_PUSH_GATEWAY` | `f1.WithPrometheusPushGateway(url)` | disabled | +| Prometheus namespace label | `PROMETHEUS_NAMESPACE` | `f1.WithPrometheusNamespace(ns)` | `""` | +| Prometheus ID label | `PROMETHEUS_LABEL_ID` | `f1.WithPrometheusLabelID(id)` | `""` | +| Log file path | `LOG_FILE_PATH` | `f1.WithLogFilePath(path)` | auto temp file | +| Log level | `F1_LOG_LEVEL` | `f1.WithLogLevel(slog.LevelDebug)` | `slog.LevelInfo` | +| Log format | `F1_LOG_FORMAT` | `f1.WithLogFormat(f1.LogFormatJSON)` | `f1.LogFormatText` | + +Log level and format options use Go's standard `slog.Level` and f1's `LogFormat` type for compile-time safety. Use `f1.ParseLogLevel(string)` and `f1.ParseLogFormat(string)` to convert from strings (e.g. from config files). + +#### Configuring without environment variables + +Use `f1.WithSettings(f1.Settings{})` to start from zero values, ignoring all environment variables. Fine-grained options (`WithLogLevel`, `WithPrometheusPushGateway`, etc.) still apply after the baseline: + +```golang +f1.New( + f1.WithSettings(f1.Settings{}), + f1.WithLogLevel(slog.LevelWarn), + f1.WithLogFormat(f1.LogFormatJSON), +).AddScenario("myScenario", mySetup).Execute() +``` + +For full control, pass a complete `f1.Settings` struct: + +```golang +f1.New( + f1.WithSettings(f1.Settings{ + Prometheus: f1.PrometheusSettings{ + PushGateway: "http://pushgateway:9091", + Namespace: "my-namespace", + }, + Logging: f1.LoggingSettings{ + Level: slog.LevelDebug, + Format: f1.LogFormatJSON, + }, + }), +).AddScenario("myScenario", mySetup).Execute() +``` + +#### Precedence + +Settings are resolved in this order (highest priority first): + +1. **Programmatic options** — values passed to `f1.New()` (applied in order) +2. **Environment variables** — read at construction time (baseline when no `WithSettings` is used) +3. **Defaults** — `slog.LevelInfo`, `LogFormatText`, no Prometheus push + +When `f1.WithLogger(logger)` is used, the caller owns the logger entirely. `WithLogLevel`, `WithLogFormat`, and the corresponding env vars have no effect: + +```golang +logger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelDebug})) +f1.New( + f1.WithLogger(logger), +).AddScenario("myScenario", mySetup).Execute() +``` + +#### Default env-backed behaviour + +When no `WithSettings` is provided, environment variables are used as the baseline (backward-compatible with previous releases): + +```golang +// Env vars like PROMETHEUS_PUSH_GATEWAY are read automatically +f1.New().AddScenario("myScenario", mySetup).Execute() + +// Fine-grained options override individual env var values +f1.New( + f1.WithPrometheusPushGateway("http://pushgateway:9091"), + f1.WithLogLevel(slog.LevelDebug), +).AddScenario("myScenario", mySetup).Execute() +``` ## Contributions If you'd like to help improve `f1`, please fork this repo and raise a PR! diff --git a/benchcmd/main.go b/benchcmd/main.go index 9883959e..12a9f0fd 100644 --- a/benchcmd/main.go +++ b/benchcmd/main.go @@ -1,60 +1,57 @@ package main import ( + "context" "os" "strconv" "time" - "github.com/form3tech-oss/f1/v2/pkg/f1" - "github.com/form3tech-oss/f1/v2/pkg/f1/testing" + "github.com/form3tech-oss/f1/v3/pkg/f1" + "github.com/form3tech-oss/f1/v3/pkg/f1/f1testing" ) func main() { f1.New(). - Add("emptyScenario", emptyScenario). - Add("failingScenario", failingScenario). - Add("sleepScenario", sleepScenario). - Add("logScenario", logScenario). + AddScenario("emptyScenario", emptyScenario). + AddScenario("failingScenario", failingScenario). + AddScenario("sleepScenario", sleepScenario). + AddScenario("logScenario", logScenario). Execute() } -func emptyScenario(*testing.T) testing.RunFn { - runFn := func(t *testing.T) { +func emptyScenario(context.Context, *f1testing.T) f1testing.RunFn { + return func(_ context.Context, t *f1testing.T) { t.Require().True(true) } - - return runFn } -func sleepScenario(t *testing.T) testing.RunFn { +func sleepScenario(_ context.Context, t *f1testing.T) f1testing.RunFn { msString := os.Getenv("MS_SLEEP") ms, err := strconv.ParseInt(msString, 10, 64) t.Require().NoError(err) - runFn := func(*testing.T) { + runFn := func(_ context.Context, _ *f1testing.T) { time.Sleep(time.Duration(ms) * time.Millisecond) } return runFn } -func failingScenario(*testing.T) testing.RunFn { - runFn := func(t *testing.T) { +func failingScenario(context.Context, *f1testing.T) f1testing.RunFn { + return func(_ context.Context, t *f1testing.T) { t.Require().True(false) } - - return runFn } -func logScenario(t *testing.T) testing.RunFn { +func logScenario(_ context.Context, t *f1testing.T) f1testing.RunFn { t.Log("Setup") - runFn := func(t *testing.T) { - t.Logf("Iteration: %s", t.Iteration) - t.Logger().WithField("iteration", t.Iteration).Trace("Trace log") - t.Logger().WithField("iteration", t.Iteration).Debug("Debug log") - t.Logger().WithField("iteration", t.Iteration).Info("Info log") - t.Logger().WithField("iteration", t.Iteration).Warn("Warn log") - t.Logger().WithField("iteration", t.Iteration).Error("Error log") + runFn := func(_ context.Context, t *f1testing.T) { + t.Logf("Iteration: %d", t.Iteration) + t.Logger().With("iteration", t.Iteration).Debug("Trace log") + t.Logger().With("iteration", t.Iteration).Debug("Debug log") + t.Logger().With("iteration", t.Iteration).Info("Info log") + t.Logger().With("iteration", t.Iteration).Warn("Warn log") + t.Logger().With("iteration", t.Iteration).Error("Error log") panic("panic message") } diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md new file mode 100644 index 00000000..083a0f7b --- /dev/null +++ b/docs/MIGRATION.md @@ -0,0 +1,570 @@ +# Migration Guide: F1 v2 → v3 + +This guide documents all breaking changes in F1 v3 and how to migrate your code. The v3 release modernises the API, adds context support, and removes deprecated features. + +--- + +## Table of Contents + +1. [Overview](#1-overview) +2. [Module Path and go.mod](#2-module-path-and-gomod) +3. [Import Path Changes](#3-import-path-changes) +4. [Entry Points: Execute and Run](#4-entry-points-execute-and-run) +5. [F1 Construction and Options](#5-f1-construction-and-options) +6. [Scenario Registration](#6-scenario-registration) +7. [Scenario Function Signatures](#7-scenario-function-signatures) +8. [Testing Package Rename](#8-testing-package-rename) +9. [T Type Changes](#9-t-type-changes) +10. [Metrics Package Removal](#10-metrics-package-removal) +11. [CLI and Flag Changes](#11-cli-and-flag-changes) +12. [Removed Features](#12-removed-features) +13. [Configuration Changes](#13-configuration-changes) +14. [Complete Before/After Example](#14-complete-beforeafter-example) +15. [Migration Checklist](#15-migration-checklist) + +--- + +## 1. Overview + +| Area | v2 | v3 | +|------|-----|-----| +| Module | `github.com/form3tech-oss/f1/v2` | `github.com/form3tech-oss/f1/v3` | +| Testing package | `pkg/f1/testing` | `pkg/f1/f1testing` | +| Entry point (library) | `ExecuteWithArgs(args)` | `Run(ctx, args)` | +| F1 construction | `New().WithLogger(l)` | `New(WithLogger(l))` | +| Scenario registration | `Add(name, fn)` | `AddScenario(name, fn)` | +| Scenario options | `Description(d)`, `Parameter(p)` | `WithDescription(d)`, `WithParameter(p)` | +| Scenario/Run signatures | `func(t *T)` | `func(ctx context.Context, t *T)` | +| T.Error / T.Fatal | `Error(err error)`, `Fatal(err error)` | `Error(args ...any)`, `Fatal(args ...any)` | +| T.Logger | `*logrus.Logger` | `*slog.Logger` | +| T.Iteration | `string` | `uint64` (`IterationSetup=0` for setup) | +| T.VUID | not available | `int` (Virtual User ID: -1 for setup, 0-based for workers) | +| Metrics | `metrics.GetMetrics()` | Removed; use `WithStaticMetrics` | +| Logging | logrus | `log/slog` | + +--- + +## 2. Module Path and go.mod + +**Change**: Update the module path from `v2` to `v3`. + +```diff +# go.mod +- module github.com/form3tech-oss/f1/v2 ++ module github.com/form3tech-oss/f1/v3 +``` + +Update your dependency: + +```bash +go get github.com/form3tech-oss/f1/v3@latest +``` + +--- + +## 3. Import Path Changes + +**Change**: All imports must use the `v3` path and the renamed `f1testing` package. + +```diff +import ( +- "github.com/form3tech-oss/f1/v2/pkg/f1" +- "github.com/form3tech-oss/f1/v2/pkg/f1/testing" ++ "github.com/form3tech-oss/f1/v3/pkg/f1" ++ "github.com/form3tech-oss/f1/v3/pkg/f1/f1testing" +) +``` + +**Package rename**: `pkg/f1/testing` → `pkg/f1/f1testing`. The name `testing` clashed with Go's standard `testing` package; `f1testing` avoids this and makes the origin explicit. + +--- + +## 4. Entry Points: Execute and Run + +### Execute (unchanged) + +`Execute()` remains the same for typical CLI usage from `main()`: + +```go +// v2 and v3 — no change +f1.New().AddScenario("myTest", myScenario).Execute() +``` + +### ExecuteWithArgs → Run + +**Change**: `ExecuteWithArgs(args)` is removed. Use `Run(ctx, args)` instead. `Run` returns an `error` and never exits; it accepts `context.Context` for cancellation and timeouts. + +| v2 | v3 | +|----|-----| +| `f.ExecuteWithArgs(args)` | `f.Run(context.Background(), args)` | +| `err := f.ExecuteWithArgs(args)` | `err := f.Run(ctx, args)` | + +```diff +// v2 +- err := f.ExecuteWithArgs([]string{"run", "constant", "-r", "1/s", "-d", "10s", "myTest"}) + +// v3 ++ err := f.Run(context.Background(), []string{"run", "constant", "-r", "1/s", "-d", "10s", "myTest"}) +``` + +**With cancellation** (e.g. in tests): + +```go +ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) +defer cancel() +err := f.Run(ctx, []string{"run", "constant", "myTest"}) +``` + +**Pass `nil` for args** to use `os.Args` (e.g. when called from `main`): + +```go +f.Run(context.Background(), nil) // equivalent to Execute() but returns error +``` + +--- + +## 5. F1 Construction and Options + +**Change**: `New()` now accepts functional options. Fluent methods `WithLogger` and `WithStaticMetrics` are removed in favour of constructor options. + +| v2 | v3 | +|----|-----| +| `New()` | `New()` — unchanged | +| `New().WithLogger(logger)` | `New(WithLogger(logger))` | +| `New().WithStaticMetrics(labels)` | `New(WithStaticMetrics(labels))` | + +```diff +// v2 +- f := f1.New().WithLogger(myLogger).WithStaticMetrics(map[string]string{"env": "prod"}) + +// v3 ++ f := f1.New( ++ f1.WithLogger(myLogger), ++ f1.WithStaticMetrics(map[string]string{"env": "prod"}), ++ ) +``` + +**New in v3**: Additional constructor options for programmatic configuration: + +```go +f1.New( + f1.WithLogLevel(slog.LevelDebug), + f1.WithLogFormat(f1.LogFormatJSON), + f1.WithPrometheusPushGateway("http://pushgateway:9091"), +) +``` + +--- + +## 6. Scenario Registration + +### Add → AddScenario + +**Change**: `Add` is renamed to `AddScenario` for clarity. + +```diff +// v2 +- f.Add("myTest", myScenario) + +// v3 ++ f.AddScenario("myTest", myScenario) +``` + +### Scenario Options: Description and Parameter + +**Change**: `Description(d)` and `Parameter(p)` are renamed to `WithDescription(d)` and `WithParameter(p)` for consistency with the functional options pattern. + +```diff +// v2 +- f.Add("myTest", myScenario, +- scenarios.Description("Load test for API X"), +- scenarios.Parameter(scenarios.ScenarioParameter{Name: "rate", Default: "1/s"}), +- ) + +// v3 ++ f.AddScenario("myTest", myScenario, ++ scenarios.WithDescription("Load test for API X"), ++ scenarios.WithParameter(scenarios.ScenarioParameter{Name: "rate", Default: "1/s"}), ++ ) +``` + +--- + +## 7. Scenario Function Signatures + +**Change**: `ScenarioFn` and `RunFn` now receive `context.Context` as the first parameter. The context is cancelled when the run is interrupted (SIGINT/SIGTERM), times out (`--max-duration`), or reaches max iterations. + +### ScenarioFn + +| v2 | v3 | +|----|-----| +| `func(t *T) RunFn` | `func(ctx context.Context, t *T) RunFn` | + +### RunFn + +| v2 | v3 | +|----|-----| +| `func(t *T)` | `func(ctx context.Context, t *T)` | + +```diff +// v2 +- func myScenario(t *f1testing.T) f1testing.RunFn { +- runFn := func(t *f1testing.T) { ... } ++ func myScenario(ctx context.Context, t *f1testing.T) f1testing.RunFn { ++ runFn := func(ctx context.Context, t *f1testing.T) { ... } + return runFn + } +``` + +**Using context** for cancellation or timeouts: + +```go +func myScenario(ctx context.Context, t *f1testing.T) f1testing.RunFn { + return func(ctx context.Context, t *f1testing.T) { + req, _ := http.NewRequestWithContext(ctx, "GET", url, nil) + resp, err := http.DefaultClient.Do(req) // respects ctx cancellation + // ... + } +} +``` + +--- + +## 8. Testing Package Rename + +**Change**: The package `pkg/f1/testing` is renamed to `pkg/f1/f1testing`. Update all references. + +```diff +- import "github.com/form3tech-oss/f1/v2/pkg/f1/testing" ++ import "github.com/form3tech-oss/f1/v3/pkg/f1/f1testing" + +- func myScenario(t *testing.T) testing.RunFn { ++ func myScenario(ctx context.Context, t *f1testing.T) f1testing.RunFn { +``` + +Type references: + +| v2 | v3 | +|----|-----| +| `*testing.T` | `*f1testing.T` | +| `testing.ScenarioFn` | `f1testing.ScenarioFn` | +| `testing.RunFn` | `f1testing.RunFn` | + +--- + +## 9. T Type Changes + +### 9.1 Error and Fatal Signatures + +**Change**: `Error` and `Fatal` now accept `args ...any` (matching `testing.T`), instead of `err error`. + +| v2 | v3 | +|----|-----| +| `Error(err error)` | `Error(args ...any)` | +| `Fatal(err error)` | `Fatal(args ...any)` | + +**Backward compatibility**: Existing `Error(err)` and `Fatal(err)` calls still work — a single `error` is passed as the first argument and formatted with `fmt.Sprintln`. + +```go +// All of these work in v3 +t.Error(err) +t.Error("failed:", err) +t.Fatal(err) +t.Fatalf("iteration %d failed: %v", t.Iteration, err) +``` + +### 9.2 Log Levels + +**Change**: `Error`, `Errorf`, `Fatal`, and `Fatalf` now log at ERROR level. `Log` and `Logf` log at INFO level. In v2, Error/Fatal delegated to Log (INFO); v3 aligns with `testing.T` semantics. + +### 9.3 Logger() Return Type + +**Change**: `T.Logger()` now returns `*slog.Logger` instead of `*logrus.Logger`. Logrus is removed as a dependency. + +```diff +// v2 — Logger() returned *logrus.Logger +- logger := t.Logger() +- logger.WithField("key", "value").Info("msg") + +// v3 — Logger() returns *slog.Logger ++ logger := t.Logger() ++ logger.With("key", "value").Info("msg") +``` + +### 9.4 T.Time() Removed + +**Change**: `T.Time(stageName string, f func())` is removed. Internal metrics are no longer exposed via the testing package. + +```diff +// v2 +- t.Time("http_request", func() { doRequest() }) + +// v3 — record timing yourself if needed ++ start := time.Now() ++ doRequest() ++ duration := time.Since(start) +``` + +### 9.5 NewT() Removed + +**Change**: `NewT(iter, scenarioName string)` is removed. Use `NewTWithOptions` only. + +```diff +// v2 +- t, teardown := testing.NewT("1", "myScenario") + +// v3 ++ t, teardown := f1testing.NewTWithOptions("myScenario", f1testing.WithIteration(1)) +``` + +### 9.6 T.Iteration Type Change + +**Change**: `T.Iteration` is now `uint64` instead of `string`. Use `f1testing.IterationSetup` (0) for the setup phase; iteration numbers are 1-based. + +```diff +// v2 +- t.Logf("Iteration: %s", t.Iteration) + +// v3 ++ t.Logf("Iteration: %d", t.Iteration) +``` + +### 9.7 WithLogrusLogger Removed + +**Change**: `WithLogrusLogger(logrusLogger *logrus.Logger)` is removed. Use `WithLogger(*slog.Logger)`. + +```diff +// v2 +- t, teardown := testing.NewTWithOptions("myScenario", +- testing.WithLogrusLogger(logrusLogger), +- ) + +// v3 ++ t, teardown := f1testing.NewTWithOptions("myScenario", ++ f1testing.WithLogger(slogLogger), ++ ) +``` + +### 9.8 T.VUID (New in v3) + +**New**: `T.VUID` is a `int` field representing the Virtual User ID. It's -1 during setup and 0-based for pool workers. Useful for per-user state in `users` trigger mode. + +```go +func myScenario(ctx context.Context, t *f1testing.T) f1testing.RunFn { + userAccounts := loadUserAccounts() + return func(ctx context.Context, t *f1testing.T) { + account := userAccounts[t.VUID] + // use account for this virtual user + } +} +``` + +--- + +## 10. Metrics Package Removal + +**Change**: `pkg/f1/metrics.GetMetrics()` is removed. Internal metrics are no longer exposed. Use `WithStaticMetrics` on the F1 instance for custom labels. + +```diff +// v2 — direct access to internal metrics (deprecated) +- m := metrics.GetMetrics() +- m.RecordIterationStage(...) + +// v3 — no replacement for GetMetrics ++ // Use f1.New(WithStaticMetrics(map[string]string{"env": "prod"})) for labels +``` + +--- + +## 11. CLI and Flag Changes + +### 11.1 Renamed Flags + +| v2 | v3 | +|----|-----| +| `--cpuprofile` | `--cpu-profile` | +| `--memprofile` | `--mem-profile` | +| `--iterationFrequency` (staged, gaussian) | `--iteration-frequency` | + +### 11.2 Flag Grouping + +Run command flags are now grouped in help output (Output, Duration & limits, Concurrency, Failure handling, Shutdown, Trigger options). Behaviour is unchanged. + +### 11.3 Removed Flags + +| Flag | Action | +|------|--------| +| `--verbose-fail` | Removed (was deprecated) | + +--- + +## 12. Removed Features + +### 12.1 Chart Command + +**Change**: The `chart` subcommand (`f1 chart ...`) is removed. The go-chart and asciigraph dependencies are removed. Use external tools (e.g. Grafana, Prometheus) for visualisation. + +### 12.2 Fluentd Integration + +**Change**: Fluentd environment variables (`FluentdHost`, `FluentdPort`) and related code are removed. Use structured logs (slog) and your log aggregation pipeline instead. + +### 12.3 Logrus + +**Change**: Logrus is removed. All logging uses `log/slog`. Migrate `WithLogrusLogger` to `WithLogger(*slog.Logger)`. + +--- + +## 13. Configuration Changes + +### 13.1 Programmatic Configuration (New in v3) + +v3 adds typed, programmatic configuration options as an alternative to environment variables: + +```go +f1.New( + f1.WithLogLevel(slog.LevelDebug), + f1.WithLogFormat(f1.LogFormatJSON), + f1.WithPrometheusPushGateway("http://pushgateway:9091"), + f1.WithPrometheusNamespace("my-namespace"), +) +``` + +Environment variables continue to work as the default baseline (backward compatible). + +### 13.2 `WithSettings` for Full Control + +Use `WithSettings(Settings{})` to ignore all environment variables: + +```go +f1.New( + f1.WithSettings(f1.Settings{}), + f1.WithLogLevel(slog.LevelWarn), +) +``` + +### 13.3 Precedence + +Settings are resolved in this order (highest to lowest): +1. Programmatic options (applied in order) +2. Environment variables (baseline when no `WithSettings` is used) +3. Defaults (`slog.LevelInfo`, `LogFormatText`, no Prometheus push) + +`WithLogger` takes absolute precedence for logging. + +--- + +## 14. Complete Before/After Example + +### v2 + +```go +package main + +import ( + "fmt" + "log/slog" + + "github.com/form3tech-oss/f1/v2/pkg/f1" + "github.com/form3tech-oss/f1/v2/pkg/f1/scenarios" + "github.com/form3tech-oss/f1/v2/pkg/f1/testing" +) + +func main() { + f := f1.New(). + WithLogger(slog.Default()). + Add("myLoadTest", myScenario, + scenarios.Description("API load test"), + scenarios.Parameter(scenarios.ScenarioParameter{Name: "rate", Default: "1/s"}), + ) + f.Execute() +} + +func myScenario(t *testing.T) testing.RunFn { + t.Cleanup(func() { fmt.Println("cleanup") }) + return func(t *testing.T) { + if err := doWork(); err != nil { + t.Error(err) + } + } +} +``` + +### v3 + +```go +package main + +import ( + "context" + "fmt" + "log/slog" + + "github.com/form3tech-oss/f1/v3/pkg/f1" + "github.com/form3tech-oss/f1/v3/pkg/f1/f1testing" + "github.com/form3tech-oss/f1/v3/pkg/f1/scenarios" +) + +func main() { + f := f1.New( + f1.WithLogger(slog.Default()), + ). + AddScenario("myLoadTest", myScenario, + scenarios.WithDescription("API load test"), + scenarios.WithParameter(scenarios.ScenarioParameter{Name: "rate", Default: "1/s"}), + ) + f.Execute() +} + +func myScenario(ctx context.Context, t *f1testing.T) f1testing.RunFn { + t.Cleanup(func() { fmt.Println("cleanup") }) + return func(ctx context.Context, t *f1testing.T) { + if err := doWork(); err != nil { + t.Error(err) + } + } +} +``` + +--- + +## 15. Migration Checklist + +Use this checklist when migrating from v2 to v3: + +- [ ] Update `go.mod`: `github.com/form3tech-oss/f1/v2` → `github.com/form3tech-oss/f1/v3` +- [ ] Update imports: `pkg/f1/testing` → `pkg/f1/f1testing` +- [ ] Update scenario signature: add `ctx context.Context` as first param to `ScenarioFn` and `RunFn` +- [ ] Replace `Add(` → `AddScenario(` +- [ ] Replace `Description(` → `WithDescription(`, `Parameter(` → `WithParameter(` +- [ ] Replace `New().WithLogger(l)` → `New(WithLogger(l))`, `New().WithStaticMetrics(m)` → `New(WithStaticMetrics(m))` +- [ ] Replace `ExecuteWithArgs(args)` → `Run(context.Background(), args)` (or `Run(ctx, args)` with a context) +- [ ] Replace `NewT()` with `NewTWithOptions()` if used +- [ ] Replace `WithLogrusLogger()` with `WithLogger(*slog.Logger)` if used +- [ ] Remove `metrics.GetMetrics()` usage; use `WithStaticMetrics` for labels +- [ ] Remove `T.Time()` usage; record timing manually if needed +- [ ] Update `T.Logger()` call sites: returns `*slog.Logger` (not `*logrus.Logger`) +- [ ] (Optional) `Error`/`Fatal` now use `args ...any`; existing `Error(err)`/`Fatal(err)` calls remain valid +- [ ] Replace `t.Logf("Iteration: %s", t.Iteration)` with `t.Logf("Iteration: %d", t.Iteration)` if used +- [ ] Update CLI invocations: `--cpuprofile` → `--cpu-profile`, `--memprofile` → `--mem-profile`, `--iterationFrequency` → `--iteration-frequency` +- [ ] Remove `--verbose-fail` if used +- [ ] Remove `f1 chart` usage; use external visualisation tools + +--- + +## Quick Reference: Search and Replace + +| Find | Replace | +|------|---------| +| `f1/v2` | `f1/v3` | +| `pkg/f1/testing` | `pkg/f1/f1testing` | +| `*testing.T` | `*f1testing.T` | +| `testing.ScenarioFn` | `f1testing.ScenarioFn` | +| `testing.RunFn` | `f1testing.RunFn` | +| `f.Add(` | `f.AddScenario(` | +| `scenarios.Description(` | `scenarios.WithDescription(` | +| `scenarios.Parameter(` | `scenarios.WithParameter(` | +| `ExecuteWithArgs(` | `Run(context.Background(), ` | +| `New().WithLogger(` | `New(WithLogger(` | +| `New().WithStaticMetrics(` | `New(WithStaticMetrics(` | + +**Note**: Scenario and Run function signatures require manual edits to add `ctx context.Context` as the first parameter. diff --git a/go.mod b/go.mod index da82af39..d09cd46c 100644 --- a/go.mod +++ b/go.mod @@ -1,18 +1,15 @@ -module github.com/form3tech-oss/f1/v2 +module github.com/form3tech-oss/f1/v3 go 1.26 require ( - github.com/guptarohit/asciigraph v0.7.3 github.com/mattn/go-isatty v0.0.20 github.com/prometheus/client_golang v1.23.2 github.com/prometheus/client_model v0.6.2 github.com/prometheus/common v0.67.5 - github.com/sirupsen/logrus v1.9.4 github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 github.com/stretchr/testify v1.11.1 - github.com/wcharczuk/go-chart/v2 v2.1.2 go.uber.org/goleak v1.3.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -21,7 +18,6 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/kr/text v0.2.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect @@ -29,7 +25,6 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/procfs v0.20.1 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect - golang.org/x/image v0.36.0 // indirect golang.org/x/sys v0.41.0 // indirect google.golang.org/protobuf v1.36.11 // indirect ) diff --git a/go.sum b/go.sum index 36c050d4..314549f8 100644 --- a/go.sum +++ b/go.sum @@ -6,13 +6,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6N github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= -github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/guptarohit/asciigraph v0.7.3 h1:p05XDDn7cBTWiBqWb30mrwxd6oU0claAjqeytllnsPY= -github.com/guptarohit/asciigraph v0.7.3/go.mod h1:dYl5wwK4gNsnFf9Zp+l06rFiDZ5YtXM6x7SRWZ3KGag= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -38,8 +33,6 @@ github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4Ul github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= -github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -47,78 +40,14 @@ github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/wcharczuk/go-chart/v2 v2.1.2 h1:Y17/oYNuXwZg6TFag06qe8sBajwwsuvPiJJXcUcLL6E= -github.com/wcharczuk/go-chart/v2 v2.1.2/go.mod h1:Zi4hbaqlWpYajnXB2K22IUYVXRXaLfSGNNR7P4ukyyQ= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= -golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= -golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E= -golang.org/x/image v0.36.0 h1:Iknbfm1afbgtwPTmHnS2gTM/6PPZfH+z2EFuOkSbqwc= -golang.org/x/image v0.36.0/go.mod h1:YsWD2TyyGKiIX1kZlu9QfKIsQ4nAAK9bdgdrIsE7xy4= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= -golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/chart/chart_cmd.go b/internal/chart/chart_cmd.go deleted file mode 100644 index c4b864f6..00000000 --- a/internal/chart/chart_cmd.go +++ /dev/null @@ -1,146 +0,0 @@ -package chart - -import ( - "fmt" - "os" - "time" - - "github.com/guptarohit/asciigraph" - "github.com/spf13/cobra" - "github.com/wcharczuk/go-chart/v2" - - "github.com/form3tech-oss/f1/v2/internal/trigger/api" - "github.com/form3tech-oss/f1/v2/internal/ui" -) - -const ( - flagChartStart = "chart-start" - flagChartDuration = "chart-duration" - flagFilename = "filename" -) - -func Cmd(builders []api.Builder, output *ui.Output) *cobra.Command { - chartCmd := &cobra.Command{ - Use: "chart ", - Short: "plots a chart of the test scenarios that would be triggered over time with the provided run function", - } - - for _, t := range builders { - triggerCmd := &cobra.Command{ - Use: t.Name, - Short: t.Description, - RunE: chartCmdExecute(t, output), - } - triggerCmd.Flags().String(flagChartStart, time.Now().Format(time.RFC3339), "Optional start time for the chart") - triggerCmd.Flags().Duration(flagChartDuration, 10*time.Minute, "Duration for the chart") - triggerCmd.Flags().String(flagFilename, "", fmt.Sprintf("Filename for optional detailed chart, e.g. %s.png", t.Name)) - triggerCmd.Flags().AddFlagSet(t.Flags) - chartCmd.AddCommand(triggerCmd) - } - - return chartCmd -} - -func chartCmdExecute( - t api.Builder, - output *ui.Output, -) func(cmd *cobra.Command, args []string) error { - return func(cmd *cobra.Command, _ []string) error { - cmd.SilenceUsage = true - - startString, err := cmd.Flags().GetString(flagChartStart) - if err != nil { - return fmt.Errorf("getting flag: %w", err) - } - start, err := time.Parse(time.RFC3339, startString) - if err != nil { - return fmt.Errorf("parsing start time: %w", err) - } - duration, err := cmd.Flags().GetDuration(flagChartDuration) - if err != nil { - return fmt.Errorf("getting flag: %w", err) - } - filename, err := cmd.Flags().GetString(flagFilename) - if err != nil { - return fmt.Errorf("getting flag: %w", err) - } - - trig, err := t.New(cmd.Flags()) - if err != nil { - return fmt.Errorf("creating builder: %w", err) - } - - if trig.DryRun == nil { - return fmt.Errorf("%s does not support charting predicted load", cmd.Name()) - } - - current := start - end := current.Add(duration) - width := 160 - sampleInterval := duration / time.Duration(width-1) - - rates := []float64{0.0} - times := []time.Time{current} - for ; current.Add(sampleInterval).Before(end); current = current.Add(sampleInterval) { - rate := trig.DryRun(current) - rates = append(rates, float64(rate)) - times = append(times, current) - } - - output.Display(ui.InteractiveMessage{ - Message: asciigraph.Plot(rates, asciigraph.Height(15), asciigraph.Width(width)), - }) - - if filename == "" { - return nil - } - graph := chart.Chart{ - Title: trig.Description, - TitleStyle: chart.StyleTextDefaults(), - Width: 1920, - Height: 1024, - YAxis: chart.YAxis{ - Name: "Triggered Test Iterations", - NameStyle: chart.StyleTextDefaults(), - Style: chart.StyleTextDefaults(), - AxisType: chart.YAxisSecondary, - }, - XAxis: chart.XAxis{ - Name: "Time", - NameStyle: chart.StyleTextDefaults(), - ValueFormatter: chart.TimeMinuteValueFormatter, - Style: chart.StyleTextDefaults(), - }, - Series: []chart.Series{ - chart.TimeSeries{ - Style: chart.Style{ - StrokeColor: chart.GetDefaultColor(0).WithAlpha(64), - }, - Name: "testing", - XValues: times, - YValues: rates, - }, - }, - } - - f, err := os.Create(filename) - if err != nil { - return fmt.Errorf("creting file: %w", err) - } - defer func() { - if err = f.Close(); err != nil { - output.Display(ui.ErrorMessage{ - Message: "unable to close the chart file", - Error: err, - }) - } - }() - - err = graph.Render(chart.PNG, f) - if err != nil { - return fmt.Errorf("rendering graph: %w", err) - } - output.Display(ui.InteractiveMessage{Message: "Detailed chart written to " + filename}) - return nil - } -} diff --git a/internal/chart/chart_cmd_stage_test.go b/internal/chart/chart_cmd_stage_test.go deleted file mode 100644 index 108d35d9..00000000 --- a/internal/chart/chart_cmd_stage_test.go +++ /dev/null @@ -1,82 +0,0 @@ -package chart_test - -import ( - "strconv" - "testing" - "time" - - "github.com/stretchr/testify/assert" - - "github.com/form3tech-oss/f1/v2/internal/chart" - "github.com/form3tech-oss/f1/v2/internal/trigger" - "github.com/form3tech-oss/f1/v2/internal/ui" -) - -type ChartTestStage struct { - t *testing.T - assert *assert.Assertions - err error - args []string -} - -func NewChartTestStage(t *testing.T) (*ChartTestStage, *ChartTestStage, *ChartTestStage) { - t.Helper() - - stage := &ChartTestStage{ - t: t, - assert: assert.New(t), - } - return stage, stage, stage -} - -func (s *ChartTestStage) and() *ChartTestStage { - return s -} - -func (s *ChartTestStage) i_execute_the_chart_command() *ChartTestStage { - outputer := ui.NewDiscardOutput() - cmd := chart.Cmd(trigger.GetBuilders(outputer), outputer) - cmd.SetArgs(s.args) - s.err = cmd.Execute() - return s -} - -func (s *ChartTestStage) the_command_is_successful() *ChartTestStage { - s.assert.NoError(s.err) - return s -} - -func (s *ChartTestStage) the_load_style_is_constant() *ChartTestStage { - s.args = append(s.args, "constant", "--rate", "10/s", "--distribution", "none") - return s -} - -func (s *ChartTestStage) jitter_is_applied() *ChartTestStage { - s.args = append(s.args, "--jitter", "20", "--distribution", "none") - return s -} - -func (s *ChartTestStage) the_load_style_is_staged(stages string) *ChartTestStage { - s.args = append(s.args, "staged", "--stages", stages, "--distribution", "none") - return s -} - -func (s *ChartTestStage) the_load_style_is_ramp() *ChartTestStage { - s.args = append(s.args, "ramp", "--start-rate", "0/s", "--end-rate", "10/s", "--ramp-duration", "10s", "--chart-duration", "10s", "--distribution", "none") - return s -} - -func (s *ChartTestStage) the_load_style_is_gaussian_with_a_volume_of(volume int) *ChartTestStage { - s.args = append(s.args, "gaussian", "--peak", "5m", "--repeat", "10m", "--volume", strconv.Itoa(volume), "--standard-deviation", "1m", "--distribution", "none") - return s -} - -func (s *ChartTestStage) the_chart_starts_at_a_fixed_time() *ChartTestStage { - s.args = append(s.args, "--chart-start", time.Now().Truncate(10*time.Minute).Format(time.RFC3339)) - return s -} - -func (s *ChartTestStage) the_load_style_is_defined_in_the_config_file(filename string) *ChartTestStage { - s.args = append(s.args, "file", filename, "--chart-duration", "5s") - return s -} diff --git a/internal/chart/chart_cmd_test.go b/internal/chart/chart_cmd_test.go deleted file mode 100644 index 31154f1a..00000000 --- a/internal/chart/chart_cmd_test.go +++ /dev/null @@ -1,114 +0,0 @@ -package chart_test - -import ( - "testing" -) - -func TestChartConstant(t *testing.T) { - t.Parallel() - - given, when, then := NewChartTestStage(t) - - given. - the_load_style_is_constant().and(). - jitter_is_applied() - - when. - i_execute_the_chart_command() - - then. - the_command_is_successful() -} - -func TestChartConstantNoJitter(t *testing.T) { - t.Parallel() - - given, when, then := NewChartTestStage(t) - - given. - the_load_style_is_constant() - - when. - i_execute_the_chart_command() - - then. - the_command_is_successful() -} - -func TestChartStaged(t *testing.T) { - t.Parallel() - - given, when, then := NewChartTestStage(t) - - given. - the_load_style_is_staged("5m:100,2m:0,10s:100") - - when. - i_execute_the_chart_command() - - then. - the_command_is_successful() -} - -func TestChartGaussian(t *testing.T) { - t.Parallel() - - given, when, then := NewChartTestStage(t) - - given. - the_load_style_is_gaussian_with_a_volume_of(100000).and(). - the_chart_starts_at_a_fixed_time() - - when. - i_execute_the_chart_command() - - then. - the_command_is_successful() -} - -func TestChartGaussianWithJitter(t *testing.T) { - t.Parallel() - - given, when, then := NewChartTestStage(t) - - given. - the_load_style_is_gaussian_with_a_volume_of(100000).and(). - jitter_is_applied().and(). - the_chart_starts_at_a_fixed_time() - - when. - i_execute_the_chart_command() - - then. - the_command_is_successful() -} - -func TestChartRamp(t *testing.T) { - t.Parallel() - - given, when, then := NewChartTestStage(t) - - given. - the_load_style_is_ramp() - - when. - i_execute_the_chart_command() - - then. - the_command_is_successful() -} - -func TestChartFileConfig(t *testing.T) { - t.Parallel() - - given, when, then := NewChartTestStage(t) - - given. - the_load_style_is_defined_in_the_config_file("../testdata/config-file.yaml") - - when. - i_execute_the_chart_command() - - then. - the_command_is_successful() -} diff --git a/internal/envsettings/env.go b/internal/envsettings/env.go index 43a98ee4..557782bf 100644 --- a/internal/envsettings/env.go +++ b/internal/envsettings/env.go @@ -14,9 +14,6 @@ const ( EnvLogFilePath = "LOG_FILE_PATH" EnvLogFormat = "F1_LOG_FORMAT" EnvLogLevel = "F1_LOG_LEVEL" - - EnvFluentdHost = "FLUENTD_HOST" - EnvFluentdPort = "FLUENTD_PORT" ) type Prometheus struct { @@ -25,15 +22,6 @@ type Prometheus struct { PushGateway string } -type Fluentd struct { - Host string - Port string -} - -func (f Fluentd) Present() bool { - return f.Host != "" || f.Port != "" -} - type Log struct { FilePath string Level string @@ -60,7 +48,6 @@ func (l Log) IsFormatJSON() bool { type Settings struct { Prometheus Prometheus - Fluentd Fluentd Log Log } @@ -75,10 +62,6 @@ func Get() Settings { Level: os.Getenv(EnvLogLevel), Format: os.Getenv(EnvLogFormat), }, - Fluentd: Fluentd{ - Host: os.Getenv(EnvFluentdHost), - Port: os.Getenv(EnvFluentdPort), - }, Prometheus: Prometheus{ LabelID: os.Getenv(EnvPrometheusLabelID), Namespace: os.Getenv(EnvPrometheusNamespace), diff --git a/internal/log/attrs.go b/internal/log/attrs.go index 181d31fc..c9c3fcfc 100644 --- a/internal/log/attrs.go +++ b/internal/log/attrs.go @@ -25,8 +25,8 @@ func ScenarioAttr(scenarioName string) slog.Attr { return slog.String("scenario", scenarioName) } -func IterationAttr(iteration string) slog.Attr { - return slog.String("iteration", iteration) +func IterationAttr(iteration uint64) slog.Attr { + return slog.Uint64("iteration", iteration) } func VUIDAttr(vuid int) slog.Attr { diff --git a/internal/log/config.go b/internal/log/config.go index d757fbef..901b5eb4 100644 --- a/internal/log/config.go +++ b/internal/log/config.go @@ -50,7 +50,6 @@ func (c *Config) JSONHandlerOptions() *slog.HandlerOptions { case slog.MessageKey: a.Key = "message" case slog.LevelKey: - // logrus compatibility if l, ok := a.Value.Any().(slog.Level); ok { switch l { case slog.LevelDebug: diff --git a/internal/log/logrus.go b/internal/log/logrus.go deleted file mode 100644 index 4d3d4c2b..00000000 --- a/internal/log/logrus.go +++ /dev/null @@ -1,67 +0,0 @@ -package log - -import ( - "context" - "io" - "log/slog" - - "github.com/sirupsen/logrus" -) - -var _ logrus.Hook = (*slogHook)(nil) - -// NewSlogLogrusLogger returns a logrus.Logger that will use slog as logging backend. -func NewSlogLogrusLogger(logger *slog.Logger) *logrus.Logger { - l := logrus.New() - l.AddHook(newSlogHook(logger)) - l.SetOutput(io.Discard) - - return l -} - -// slogHook converts logurs entries to slog -// -// This is needed for backwards compatibility with externally exposed logrus logger. -type slogHook struct { - logger *slog.Logger -} - -func newSlogHook(logger *slog.Logger) *slogHook { - return &slogHook{ - logger: logger, - } -} - -func (*slogHook) Levels() []logrus.Level { - return logrus.AllLevels -} - -func (h *slogHook) Fire(entry *logrus.Entry) error { - level := convertLevel(entry.Level) - msg := entry.Message - - fields := make([]slog.Attr, 0, len(entry.Data)) - for k, v := range entry.Data { - fields = append(fields, slog.Any(k, v)) - } - - h.logger.LogAttrs(context.Background(), level, msg, fields...) - return nil -} - -func convertLevel(l logrus.Level) slog.Level { - switch l { - case logrus.TraceLevel, logrus.DebugLevel: - return slog.LevelDebug - case logrus.InfoLevel: - return slog.LevelInfo - case logrus.WarnLevel: - return slog.LevelWarn - case logrus.ErrorLevel: - return slog.LevelError - case logrus.FatalLevel, logrus.PanicLevel: - return slog.LevelError - default: - return slog.LevelInfo - } -} diff --git a/internal/logutils/logutils.go b/internal/logutils/logutils.go index 5ab8b890..77176d05 100644 --- a/internal/logutils/logutils.go +++ b/internal/logutils/logutils.go @@ -1,8 +1,8 @@ package logutils import ( - "github.com/form3tech-oss/f1/v2/internal/envsettings" - "github.com/form3tech-oss/f1/v2/internal/log" + "github.com/form3tech-oss/f1/v3/internal/envsettings" + "github.com/form3tech-oss/f1/v3/internal/log" ) func NewLogConfigFromSettings(settings envsettings.Settings) *log.Config { diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 26f68dc8..4bac30b4 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -1,9 +1,7 @@ package metrics import ( - "errors" "sort" - "sync" "github.com/prometheus/client_golang/prometheus" ) @@ -22,33 +20,27 @@ const ( const IterationStage = "iteration" type Metrics struct { - Setup *prometheus.SummaryVec - Iteration *prometheus.SummaryVec - Registry *prometheus.Registry - IterationMetricsEnabled bool + setup *prometheus.SummaryVec + iteration *prometheus.SummaryVec + registry *prometheus.Registry + iterationMetricsEnabled bool staticMetricLabelValues []string } -//nolint:gochecknoglobals // removing the global Instance is a breaking change -var ( - m *Metrics - once sync.Once -) - func buildMetrics(staticMetrics map[string]string) *Metrics { percentileObjectives := map[float64]float64{ 0.5: 0.05, 0.75: 0.05, 0.9: 0.01, 0.95: 0.001, 0.99: 0.001, 0.9999: 0.00001, 1.0: 0.00001, } labelKeys := getStaticMetricLabelKeys(staticMetrics) return &Metrics{ - Setup: prometheus.NewSummaryVec(prometheus.SummaryOpts{ + setup: prometheus.NewSummaryVec(prometheus.SummaryOpts{ Namespace: metricNamespace, Subsystem: metricSubsystem, Name: "setup", Help: "Duration of setup functions.", Objectives: percentileObjectives, }, append([]string{TestNameLabel, ResultLabel}, labelKeys...)), - Iteration: prometheus.NewSummaryVec(prometheus.SummaryOpts{ + iteration: prometheus.NewSummaryVec(prometheus.SummaryOpts{ Namespace: metricNamespace, Subsystem: metricSubsystem, Name: "iteration", @@ -63,59 +55,43 @@ func NewInstance(registry *prometheus.Registry, staticMetrics map[string]string, ) *Metrics { i := buildMetrics(staticMetrics) - i.Registry = registry + i.registry = registry - i.Registry.MustRegister( - i.Setup, - i.Iteration, + i.registry.MustRegister( + i.setup, + i.iteration, ) - i.IterationMetricsEnabled = iterationMetricsEnabled + i.iterationMetricsEnabled = iterationMetricsEnabled i.staticMetricLabelValues = getStaticMetricLabelValues(staticMetrics) return i } -func Init(iterationMetricsEnabled bool) { - InitWithStaticMetrics(iterationMetricsEnabled, nil) -} - -func InitWithStaticMetrics(iterationMetricsEnabled bool, staticMetrics map[string]string) { - once.Do(func() { - defaultRegistry, ok := prometheus.DefaultRegisterer.(*prometheus.Registry) - if !ok { - panic(errors.New("casting prometheus.DefaultRegisterer to Registry")) - } - m = NewInstance(defaultRegistry, iterationMetricsEnabled, staticMetrics) - }) +// Gatherer returns the Prometheus registry for pushing metrics to a gateway. +func (m *Metrics) Gatherer() *prometheus.Registry { + return m.registry } -func Instance() *Metrics { - return m +// IterationCollector returns the iteration metric for testing. +func (m *Metrics) IterationCollector() *prometheus.SummaryVec { + return m.iteration } -func (metrics *Metrics) Reset() { - metrics.Iteration.Reset() - metrics.Setup.Reset() +func (m *Metrics) Reset() { + m.iteration.Reset() + m.setup.Reset() } -func (metrics *Metrics) RecordSetupResult(name string, result ResultType, nanoseconds int64) { - labels := append([]string{name, result.String()}, metrics.staticMetricLabelValues...) - metrics.Setup.WithLabelValues(labels...).Observe(float64(nanoseconds)) -} - -func (metrics *Metrics) RecordIterationResult(name string, result ResultType, nanoseconds int64) { - if !metrics.IterationMetricsEnabled { - return - } - labels := append([]string{name, IterationStage, result.String()}, metrics.staticMetricLabelValues...) - metrics.Iteration.WithLabelValues(labels...).Observe(float64(nanoseconds)) +func (m *Metrics) RecordSetupResult(name string, result ResultType, nanoseconds int64) { + labels := append([]string{name, result.String()}, m.staticMetricLabelValues...) + m.setup.WithLabelValues(labels...).Observe(float64(nanoseconds)) } -func (metrics *Metrics) RecordIterationStage(name string, stage string, result ResultType, nanoseconds int64) { - if !metrics.IterationMetricsEnabled { +func (m *Metrics) RecordIterationResult(name string, result ResultType, nanoseconds int64) { + if !m.iterationMetricsEnabled { return } - labels := append([]string{name, stage, result.String()}, metrics.staticMetricLabelValues...) - metrics.Iteration.WithLabelValues(labels...).Observe(float64(nanoseconds)) + labels := append([]string{name, IterationStage, result.String()}, m.staticMetricLabelValues...) + m.iteration.WithLabelValues(labels...).Observe(float64(nanoseconds)) } func getStaticMetricLabelKeys(staticMetrics map[string]string) []string { diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go index 1c001bbd..23ed0018 100644 --- a/internal/metrics/metrics_test.go +++ b/internal/metrics/metrics_test.go @@ -6,14 +6,15 @@ import ( "strings" "testing" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/form3tech-oss/f1/v2/internal/metrics" + "github.com/form3tech-oss/f1/v3/internal/metrics" ) -func TestMetrics_Init_IsSafe(t *testing.T) { +func TestMetrics_RecordIterationResult(t *testing.T) { t.Parallel() labels := map[string]string{ "product": "fps", @@ -21,15 +22,9 @@ func TestMetrics_Init_IsSafe(t *testing.T) { "f1_id": "myid", "labelx": "vx", } - metrics.InitWithStaticMetrics(true, labels) // race detector assertion - for range 10 { - go func() { - metrics.InitWithStaticMetrics(true, labels) - }() - } - assert.True(t, metrics.Instance().IterationMetricsEnabled) - metrics.Instance().RecordIterationResult("test1", metrics.SuccessResult, 1) - assert.Equal(t, 1, testutil.CollectAndCount(metrics.Instance().Iteration, "form3_loadtest_iteration")) + m := metrics.NewInstance(prometheus.NewRegistry(), true, labels) + m.RecordIterationResult("test1", metrics.SuccessResult, 1) + assert.Equal(t, 1, testutil.CollectAndCount(m.IterationCollector(), "form3_loadtest_iteration")) var expected strings.Builder expected.WriteString(` @@ -48,5 +43,5 @@ func TestMetrics_Init_IsSafe(t *testing.T) { form3_loadtest_iteration_count{customer="fake-customer",f1_id="myid",labelx="vx",product="fps",result="success",stage="iteration",test="test1"} 1 `) r := bytes.NewReader([]byte(expected.String())) - require.NoError(t, testutil.CollectAndCompare(metrics.Instance().Iteration, r)) + require.NoError(t, testutil.CollectAndCompare(m.IterationCollector(), r)) } diff --git a/internal/options/run_options.go b/internal/options/run_options.go index 1f66735a..187e6670 100644 --- a/internal/options/run_options.go +++ b/internal/options/run_options.go @@ -16,6 +16,52 @@ type RunOptions struct { WaitForCompletionTimeout time.Duration } +type RunOption func(*RunOptions) + +func WithScenario(s string) RunOption { + return func(o *RunOptions) { o.Scenario = s } +} + +func WithMaxDuration(d time.Duration) RunOption { + return func(o *RunOptions) { o.MaxDuration = d } +} + +func WithConcurrency(n int) RunOption { + return func(o *RunOptions) { o.Concurrency = n } +} + +func WithMaxIterations(n uint64) RunOption { + return func(o *RunOptions) { o.MaxIterations = n } +} + +func WithMaxFailures(n uint64) RunOption { + return func(o *RunOptions) { o.MaxFailures = n } +} + +func WithMaxFailuresRate(n int) RunOption { + return func(o *RunOptions) { o.MaxFailuresRate = n } +} + +func WithVerbose(v bool) RunOption { + return func(o *RunOptions) { o.Verbose = v } +} + +func WithIgnoreDropped(v bool) RunOption { + return func(o *RunOptions) { o.IgnoreDropped = v } +} + +func WithWaitForCompletionTimeout(d time.Duration) RunOption { + return func(o *RunOptions) { o.WaitForCompletionTimeout = d } +} + +func DefaultRunOptions() RunOptions { + return RunOptions{ + MaxDuration: time.Second, + Concurrency: 100, + WaitForCompletionTimeout: 10 * time.Second, + } +} + func (o *RunOptions) LogToFile() bool { return !o.Verbose } diff --git a/internal/progress/stats.go b/internal/progress/stats.go index 85aa9d2b..71015e4b 100644 --- a/internal/progress/stats.go +++ b/internal/progress/stats.go @@ -4,7 +4,7 @@ import ( "sync/atomic" "time" - "github.com/form3tech-oss/f1/v2/internal/metrics" + "github.com/form3tech-oss/f1/v3/internal/metrics" ) type Stats struct { diff --git a/internal/raterun/runner_stage_test.go b/internal/raterun/runner_stage_test.go index ccbe2034..82771a17 100644 --- a/internal/raterun/runner_stage_test.go +++ b/internal/raterun/runner_stage_test.go @@ -10,7 +10,7 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/goleak" - "github.com/form3tech-oss/f1/v2/internal/raterun" + "github.com/form3tech-oss/f1/v3/internal/raterun" ) type RatedRunnerStage struct { diff --git a/internal/raterun/runner_test.go b/internal/raterun/runner_test.go index d2a5924b..a722fb38 100644 --- a/internal/raterun/runner_test.go +++ b/internal/raterun/runner_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - "github.com/form3tech-oss/f1/v2/internal/raterun" + "github.com/form3tech-oss/f1/v3/internal/raterun" ) func Test_FunctionIsExecutedAtSpecifiedRates(t *testing.T) { diff --git a/internal/run/help.go b/internal/run/help.go new file mode 100644 index 00000000..d4285f70 --- /dev/null +++ b/internal/run/help.go @@ -0,0 +1,67 @@ +package run + +import ( + "fmt" + "strings" + "sync" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "github.com/form3tech-oss/f1/v3/internal/triggerflags" +) + +func flagGroupOrder() []string { + return []string{ + "Output", "Duration & limits", "Concurrency", "Failure handling", "Shutdown", + "Trigger options", "Help", + } +} + +func commonFlagGroups() map[string]string { + return map[string]string{ + triggerflags.FlagVerbose: "Output", + triggerflags.FlagMaxDuration: "Duration & limits", + triggerflags.FlagMaxIterations: "Duration & limits", + triggerflags.FlagConcurrency: "Concurrency", + triggerflags.FlagMaxFailures: "Failure handling", + triggerflags.FlagMaxFailuresRate: "Failure handling", + triggerflags.FlagIgnoreDropped: "Failure handling", + triggerflags.FlagWaitForCompletionTimeout: "Shutdown", + "help": "Help", + } +} + +var registerHelpTemplateFunc = sync.OnceFunc(func() { + cobra.AddTemplateFunc("groupedFlagUsages", groupedFlagUsages) +}) + +func groupedFlagUsages(cmd *cobra.Command) string { + if cmd == nil || !cmd.HasAvailableLocalFlags() { + return "" + } + fs := cmd.LocalFlags() + groups := commonFlagGroups() + + var out strings.Builder + for _, groupName := range flagGroupOrder() { + groupFS := pflag.NewFlagSet("", pflag.ContinueOnError) + groupFS.SortFlags = false + fs.VisitAll(func(flag *pflag.Flag) { + if flag.Hidden { + return + } + g := groups[flag.Name] + if g == "" { + g = "Trigger options" + } + if g == groupName { + groupFS.AddFlag(flag) + } + }) + if groupFS.HasFlags() { + fmt.Fprintf(&out, "\n%s:\n%s", groupName, groupFS.FlagUsages()) + } + } + return strings.TrimSpace(out.String()) + "\n" +} diff --git a/internal/run/log_file_path_test.go b/internal/run/log_file_path_test.go index 251ee692..2859d017 100644 --- a/internal/run/log_file_path_test.go +++ b/internal/run/log_file_path_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/assert" - "github.com/form3tech-oss/f1/v2/internal/run" + "github.com/form3tech-oss/f1/v3/internal/run" ) func TestLogFilePathOrDefault(t *testing.T) { diff --git a/internal/run/result.go b/internal/run/result.go index fd477a2b..d282eb38 100644 --- a/internal/run/result.go +++ b/internal/run/result.go @@ -7,9 +7,9 @@ import ( "sync" "time" - "github.com/form3tech-oss/f1/v2/internal/options" - "github.com/form3tech-oss/f1/v2/internal/progress" - "github.com/form3tech-oss/f1/v2/internal/run/views" + "github.com/form3tech-oss/f1/v3/internal/options" + "github.com/form3tech-oss/f1/v3/internal/progress" + "github.com/form3tech-oss/f1/v3/internal/run/views" ) type Result struct { diff --git a/internal/run/run_cmd.go b/internal/run/run_cmd.go index b5bf9e31..3ef1dfa4 100644 --- a/internal/run/run_cmd.go +++ b/internal/run/run_cmd.go @@ -1,28 +1,63 @@ package run import ( + "context" "errors" "fmt" "time" "github.com/spf13/cobra" - "github.com/form3tech-oss/f1/v2/internal/envsettings" - "github.com/form3tech-oss/f1/v2/internal/metrics" - "github.com/form3tech-oss/f1/v2/internal/options" - "github.com/form3tech-oss/f1/v2/internal/trigger/api" - "github.com/form3tech-oss/f1/v2/internal/triggerflags" - "github.com/form3tech-oss/f1/v2/internal/ui" - "github.com/form3tech-oss/f1/v2/pkg/f1/scenarios" + "github.com/form3tech-oss/f1/v3/internal/envsettings" + "github.com/form3tech-oss/f1/v3/internal/metrics" + "github.com/form3tech-oss/f1/v3/internal/options" + "github.com/form3tech-oss/f1/v3/internal/trigger/api" + "github.com/form3tech-oss/f1/v3/internal/triggerflags" + "github.com/form3tech-oss/f1/v3/internal/ui" + "github.com/form3tech-oss/f1/v3/pkg/f1/scenarios" ) +// runTriggerUsageTemplate uses groupedFlagUsages for the Flags section. +const runTriggerUsageTemplate = `Usage:{{if .Runnable}} + {{.UseLine}}{{end}}{{if .HasAvailableSubCommands}} + {{.CommandPath}} [command]{{end}}{{if gt (len .Aliases) 0}} + +Aliases: + {{.NameAndAliases}}{{end}}{{if .HasExample}} + +Examples: +{{.Example}}{{end}}{{if .HasAvailableSubCommands}}{{$cmds := .Commands}}{{if eq (len .Groups) 0}} + +Available Commands:{{range $cmds}}{{if (or .IsAvailableCommand (eq .Name "help"))}} + {{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{else}}{{range $group := .Groups}} + +{{.Title}}{{range $cmds}}{{if (and (eq .GroupID $group.ID) (or .IsAvailableCommand (eq .Name "help")))}} + {{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{end}}{{if not .AllChildCommandsHaveGroup}} + +Additional Commands:{{range $cmds}}{{if (and (eq .GroupID "") (or .IsAvailableCommand (eq .Name "help")))}} + {{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{end}}{{end}}{{end}}{{if .HasAvailableLocalFlags}} + +{{groupedFlagUsages . | trimTrailingWhitespaces}}{{end}}{{if .HasAvailableInheritedFlags}} + +Global Flags: +{{.InheritedFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasHelpSubCommands}} + +Additional help topics:{{range .Commands}}{{if .IsAdditionalHelpTopicCommand}} + {{rpad .CommandPath .CommandPathPadding}} {{.Short}}{{end}}{{end}}{{end}}{{if .HasAvailableSubCommands}} + +Use "{{.CommandPath}} [command] --help" for more information about a command.{{end}} +` + func Cmd( + ctx context.Context, s *scenarios.Scenarios, builders []api.Builder, settings envsettings.Settings, metricsInstance *metrics.Metrics, output *ui.Output, ) *cobra.Command { + registerHelpTemplateFunc() + runCmd := &cobra.Command{ Use: "run ", Short: "Runs a test scenario", @@ -32,32 +67,44 @@ func Cmd( triggerCmd := &cobra.Command{ Use: t.Name, Short: t.Description, - RunE: runCmdExecute(s, t, settings, metricsInstance, output), + Long: t.Long, + RunE: runCmdExecute(ctx, s, t, settings, metricsInstance, output), Args: cobra.MatchAll(cobra.ExactArgs(1)), } - triggerCmd.Flags().BoolP(triggerflags.FlagVerbose, "v", false, "enables log output to stdout") - triggerCmd.Flags().Bool(triggerflags.FlagVerboseFail, false, "DEPRECATED: log output to stdout on failure") + triggerCmd.Flags().SortFlags = false + + // Output + triggerCmd.Flags().BoolP(triggerflags.FlagVerbose, "v", false, "enable log output to stdout") if !t.IgnoreCommonFlags { triggerCmd.ValidArgs = s.GetScenarioNames() - triggerCmd.Flags().Bool(triggerflags.FlagIgnoreDropped, false, "dropped requests will not fail the run") + // Duration & limits triggerCmd.Flags().DurationP(triggerflags.FlagMaxDuration, "d", time.Second, - "--max-duration 1s (stop after 1 second)") - triggerCmd.Flags().IntP(triggerflags.FlagConcurrency, "c", 100, - "--concurrency 2 (allow at most 2 groups of iterations to run concurrently)") + "stop after duration (e.g. 1s, 5m)") triggerCmd.Flags().Uint64P(triggerflags.FlagMaxIterations, "i", 0, - "--max-iterations 100 (stop after 100 iterations, regardless of remaining duration)") + "stop after N iterations (0 = unlimited)") + + // Concurrency + triggerCmd.Flags().IntP(triggerflags.FlagConcurrency, "c", 100, + "max concurrent iteration groups (e.g. 2, 100)") + + // Failure handling triggerCmd.Flags().Uint64(triggerflags.FlagMaxFailures, 0, - "--max-failures 10 (load test will fail if more than 10 errors occurred, default is 0)") + "fail run if error count exceeds N (0 = disabled)") triggerCmd.Flags().Int(triggerflags.FlagMaxFailuresRate, 0, - "--max-failures-rate 5 (load test will fail if more than 5\\% requests failed, default is 0)") + "fail run if error rate exceeds N%% (0 = disabled)") + triggerCmd.Flags().Bool(triggerflags.FlagIgnoreDropped, false, + "do not fail run when requests are dropped") + + // Shutdown triggerCmd.Flags().Duration(triggerflags.FlagWaitForCompletionTimeout, 10*time.Second, - "--wait-for-completion-timeout 10s (wait for completion for 10 seconds)") + "wait for active iterations before exit (e.g. 10s)") } triggerCmd.Flags().AddFlagSet(t.Flags) + triggerCmd.SetUsageTemplate(runTriggerUsageTemplate) runCmd.AddCommand(triggerCmd) } @@ -65,6 +112,7 @@ func Cmd( } func runCmdExecute( + ctx context.Context, s *scenarios.Scenarios, t api.Builder, settings envsettings.Settings, @@ -138,39 +186,21 @@ func runCmdExecute( return fmt.Errorf("getting flag: %w", err) } - verboseFail, err := cmd.Flags().GetBool(triggerflags.FlagVerboseFail) - if err != nil { - return fmt.Errorf("getting flag: %w", err) - } - if verboseFail { - output.Display(ui.WarningMessage{Message: "--verbose-fail option has been removed"}) - } - - if settings.Fluentd.Present() { - output.Display(ui.WarningMessage{ - Message: fmt.Sprintf("WARNING: fluentd integration has been removed. %s and %s have no effect.", - envsettings.EnvFluentdHost, - envsettings.EnvFluentdPort, - ), - }, - ) - } - - run, err := NewRun(options.RunOptions{ - Scenario: scenarioName, - MaxDuration: duration, - Concurrency: concurrency, - Verbose: verbose, - MaxIterations: maxIterations, - MaxFailures: maxFailures, - MaxFailuresRate: maxFailuresRate, - IgnoreDropped: ignoreDropped, - WaitForCompletionTimeout: waitForCompletionTimeout, - }, s, trig, settings, metricsInstance, output) + run, err := NewRun(s, trig, settings, metricsInstance, output, + options.WithScenario(scenarioName), + options.WithMaxDuration(duration), + options.WithConcurrency(concurrency), + options.WithVerbose(verbose), + options.WithMaxIterations(maxIterations), + options.WithMaxFailures(maxFailures), + options.WithMaxFailuresRate(maxFailuresRate), + options.WithIgnoreDropped(ignoreDropped), + options.WithWaitForCompletionTimeout(waitForCompletionTimeout), + ) if err != nil { return fmt.Errorf("new run: %w", err) } - result, err := run.Do(cmd.Context()) + result, err := run.Do(ctx) if err != nil { return fmt.Errorf("internal error on run: %w", err) } diff --git a/internal/run/run_cmd_test.go b/internal/run/run_cmd_test.go index 2ccdd2d8..4a341719 100644 --- a/internal/run/run_cmd_test.go +++ b/internal/run/run_cmd_test.go @@ -831,26 +831,30 @@ func TestOutput_JSONLogging(t *testing.T) { scenarioOnlyLogs := []logFieldMatchers{ { - "message": "setup", - "level": "info", - "scenario": "scenario_where_each_iteration_takes_200ms", + "message": "setup", + "level": "info", + "scenario": "scenario_where_each_iteration_takes_200ms", + "iteration": float64(0), + "vuid": float64(-1), }, { - "message": "logrus - setup", + "message": "slog - setup", "level": "info", - "logger": "logrus", + "logger": "slog", "scenario": "scenario_where_each_iteration_takes_200ms", }, { - "message": "first iteration", - "level": "info", - "scenario": "scenario_where_each_iteration_takes_200ms", + "message": "first iteration", + "level": "info", + "scenario": "scenario_where_each_iteration_takes_200ms", + "iteration": float64(0), + "vuid": float64(-1), }, { - "message": "logrus - first iteration", + "message": "slog - first iteration", "level": "info", - "logger": "logrus", + "logger": "slog", "scenario": "scenario_where_each_iteration_takes_200ms", }, } @@ -862,26 +866,30 @@ func TestOutput_JSONLogging(t *testing.T) { "scenario": "scenario_where_each_iteration_takes_200ms", }, { - "message": "setup", - "level": "info", - "scenario": "scenario_where_each_iteration_takes_200ms", + "message": "setup", + "level": "info", + "scenario": "scenario_where_each_iteration_takes_200ms", + "iteration": float64(0), + "vuid": float64(-1), }, { - "message": "logrus - setup", + "message": "slog - setup", "level": "info", - "logger": "logrus", + "logger": "slog", "scenario": "scenario_where_each_iteration_takes_200ms", }, { - "message": "first iteration", - "level": "info", - "scenario": "scenario_where_each_iteration_takes_200ms", + "message": "first iteration", + "level": "info", + "scenario": "scenario_where_each_iteration_takes_200ms", + "iteration": float64(0), + "vuid": float64(-1), }, { - "message": "logrus - first iteration", + "message": "slog - first iteration", "level": "info", - "logger": "logrus", + "logger": "slog", "scenario": "scenario_where_each_iteration_takes_200ms", }, diff --git a/internal/run/run_stage_test.go b/internal/run/run_stage_test.go index 26485589..d77769a9 100644 --- a/internal/run/run_stage_test.go +++ b/internal/run/run_stage_test.go @@ -21,21 +21,21 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/form3tech-oss/f1/v2/internal/envsettings" - "github.com/form3tech-oss/f1/v2/internal/log" - "github.com/form3tech-oss/f1/v2/internal/logutils" - "github.com/form3tech-oss/f1/v2/internal/metrics" - "github.com/form3tech-oss/f1/v2/internal/options" - "github.com/form3tech-oss/f1/v2/internal/run" - "github.com/form3tech-oss/f1/v2/internal/trigger/api" - "github.com/form3tech-oss/f1/v2/internal/trigger/constant" - "github.com/form3tech-oss/f1/v2/internal/trigger/file" - "github.com/form3tech-oss/f1/v2/internal/trigger/ramp" - "github.com/form3tech-oss/f1/v2/internal/trigger/staged" - "github.com/form3tech-oss/f1/v2/internal/trigger/users" - "github.com/form3tech-oss/f1/v2/internal/ui" - "github.com/form3tech-oss/f1/v2/pkg/f1" - f1_testing "github.com/form3tech-oss/f1/v2/pkg/f1/testing" + "github.com/form3tech-oss/f1/v3/internal/envsettings" + "github.com/form3tech-oss/f1/v3/internal/log" + "github.com/form3tech-oss/f1/v3/internal/logutils" + "github.com/form3tech-oss/f1/v3/internal/metrics" + "github.com/form3tech-oss/f1/v3/internal/options" + "github.com/form3tech-oss/f1/v3/internal/run" + "github.com/form3tech-oss/f1/v3/internal/trigger/api" + "github.com/form3tech-oss/f1/v3/internal/trigger/constant" + "github.com/form3tech-oss/f1/v3/internal/trigger/file" + "github.com/form3tech-oss/f1/v3/internal/trigger/ramp" + "github.com/form3tech-oss/f1/v3/internal/trigger/staged" + "github.com/form3tech-oss/f1/v3/internal/trigger/users" + "github.com/form3tech-oss/f1/v3/internal/ui" + "github.com/form3tech-oss/f1/v3/pkg/f1" + "github.com/form3tech-oss/f1/v3/pkg/f1/f1testing" ) const ( @@ -64,7 +64,7 @@ type parsedLogLine struct { } type ( - logFieldMatchers map[string]string + logFieldMatchers map[string]any ) type RunTestStage struct { @@ -197,16 +197,16 @@ func (s *RunTestStage) setupRun() { logger := log.NewLogger(&s.stdout, logutils.NewLogConfigFromSettings(s.settings)) outputer := ui.NewOutput(logger, printer, s.interactive, false) - r, err := run.NewRun(options.RunOptions{ - Scenario: s.scenario, - MaxDuration: s.duration, - Concurrency: s.concurrency, - MaxIterations: s.maxIterations, - MaxFailures: s.maxFailures, - MaxFailuresRate: s.maxFailuresRate, - Verbose: s.verbose, - WaitForCompletionTimeout: s.waitForCompletionTimeout, - }, s.f1.GetScenarios(), s.build_trigger(), s.settings, s.metrics, outputer) + r, err := run.NewRun(s.f1.GetScenarios(), s.build_trigger(), s.settings, s.metrics, outputer, + options.WithScenario(s.scenario), + options.WithMaxDuration(s.duration), + options.WithConcurrency(s.concurrency), + options.WithMaxIterations(s.maxIterations), + options.WithMaxFailures(s.maxFailures), + options.WithMaxFailuresRate(s.maxFailuresRate), + options.WithVerbose(s.verbose), + options.WithWaitForCompletionTimeout(s.waitForCompletionTimeout), + ) s.require.NoError(err) s.runInstance = r @@ -279,10 +279,10 @@ func (s *RunTestStage) the_command_should_fail() *RunTestStage { func (s *RunTestStage) a_test_scenario_that_always_fails() *RunTestStage { s.scenario = "scenario_that_always_fails" - s.f1.Add(s.scenario, func(scenarioT *f1_testing.T) f1_testing.RunFn { + s.f1.AddScenario(s.scenario, func(_ context.Context, scenarioT *f1testing.T) f1testing.RunFn { scenarioT.Cleanup(s.scenarioCleanup) - return func(iterationT *f1_testing.T) { + return func(_ context.Context, iterationT *f1testing.T) { iterationT.Cleanup(s.iterationCleanup) iterationT.FailNow() @@ -293,10 +293,10 @@ func (s *RunTestStage) a_test_scenario_that_always_fails() *RunTestStage { func (s *RunTestStage) a_test_scenario_that_always_panics() *RunTestStage { s.scenario = "scenario_that_always_panics" - s.f1.Add(s.scenario, func(scenarioT *f1_testing.T) f1_testing.RunFn { + s.f1.AddScenario(s.scenario, func(_ context.Context, scenarioT *f1testing.T) f1testing.RunFn { scenarioT.Cleanup(s.scenarioCleanup) - return func(iterationT *f1_testing.T) { + return func(_ context.Context, iterationT *f1testing.T) { iterationT.Cleanup(s.iterationCleanup) panic("test panic in scenario iteration") @@ -307,10 +307,10 @@ func (s *RunTestStage) a_test_scenario_that_always_panics() *RunTestStage { func (s *RunTestStage) a_test_scenario_that_always_fails_an_assertion() *RunTestStage { s.scenario = "scenario_that_always_fails_an_assertion" - s.f1.Add(s.scenario, func(scenarioT *f1_testing.T) f1_testing.RunFn { + s.f1.AddScenario(s.scenario, func(_ context.Context, scenarioT *f1testing.T) f1testing.RunFn { scenarioT.Cleanup(s.scenarioCleanup) - return func(iterationT *f1_testing.T) { + return func(_ context.Context, iterationT *f1testing.T) { iterationT.Cleanup(s.iterationCleanup) assert.Fail(iterationT, "fail") @@ -321,7 +321,7 @@ func (s *RunTestStage) a_test_scenario_that_always_fails_an_assertion() *RunTest func (s *RunTestStage) a_test_scenario_that_always_fails_setup() *RunTestStage { s.scenario = "scenario_that_always_fails_setup" - s.f1.Add(s.scenario, func(scenarioT *f1_testing.T) f1_testing.RunFn { + s.f1.AddScenario(s.scenario, func(_ context.Context, scenarioT *f1testing.T) f1testing.RunFn { scenarioT.Cleanup(s.scenarioCleanup) scenarioT.FailNow() @@ -332,18 +332,18 @@ func (s *RunTestStage) a_test_scenario_that_always_fails_setup() *RunTestStage { func (s *RunTestStage) a_scenario_where_each_iteration_takes(duration time.Duration) *RunTestStage { s.scenario = "scenario_where_each_iteration_takes_" + duration.String() - s.f1.Add(s.scenario, func(scenarioT *f1_testing.T) f1_testing.RunFn { + s.f1.AddScenario(s.scenario, func(_ context.Context, scenarioT *f1testing.T) f1testing.RunFn { scenarioT.Cleanup(s.scenarioCleanup) scenarioT.Log("setup") - scenarioT.Logger().WithField("logger", "logrus").Info("logrus - setup") + scenarioT.Logger().With("logger", "slog").Info("slog - setup") s.runCount.Store(0) - return func(iterationT *f1_testing.T) { + return func(_ context.Context, iterationT *f1testing.T) { if s.runCount.Load() == 0 { scenarioT.Log("first iteration") - scenarioT.Logger().WithField("logger", "logrus").Info("logrus - first iteration") + scenarioT.Logger().With("logger", "slog").Info("slog - first iteration") } iterationT.Cleanup(s.iterationCleanup) @@ -371,11 +371,11 @@ func (s *RunTestStage) iteration_teardown_is_called_n_times(n int64) *RunTestSta func (s *RunTestStage) a_test_scenario_that_fails_intermittently() *RunTestStage { s.scenario = "scenario_that_fails_intermittently" - s.f1.Add(s.scenario, func(scenarioT *f1_testing.T) f1_testing.RunFn { + s.f1.AddScenario(s.scenario, func(_ context.Context, scenarioT *f1testing.T) f1testing.RunFn { scenarioT.Cleanup(s.scenarioCleanup) s.runCount.Store(0) - return func(t *f1_testing.T) { + return func(_ context.Context, t *f1testing.T) { t.Cleanup(s.iterationCleanup) count := s.runCount.Add(1) @@ -476,7 +476,7 @@ func (s *RunTestStage) build_trigger() *api.Trigger { err = flags.Set("stages", s.stages) require.NoError(s.t, err) - err = flags.Set("iterationFrequency", s.frequency) + err = flags.Set("iteration-frequency", s.frequency) require.NoError(s.t, err) if s.distributionType != "" { @@ -562,12 +562,12 @@ func (s *RunTestStage) metrics_are_pushed_to_prometheus() *RunTestStage { func (s *RunTestStage) a_scenario_where_iteration_n_takes_100ms(n uint32) *RunTestStage { s.scenario = fmt.Sprintf("scenario_where_iteration_%d_takes_100ms", n) - s.f1.Add(s.scenario, func(scenarioT *f1_testing.T) f1_testing.RunFn { + s.f1.AddScenario(s.scenario, func(_ context.Context, scenarioT *f1testing.T) f1testing.RunFn { scenarioT.Cleanup(s.scenarioCleanup) s.runCount.Store(0) - return func(iterationT *f1_testing.T) { + return func(_ context.Context, iterationT *f1testing.T) { iterationT.Cleanup(s.iterationCleanup) current := s.runCount.Add(1) @@ -739,7 +739,7 @@ func (s *RunTestStage) assertJSONLogMatches(t *testing.T, output string, expecte matchers := expectedLogLines[lineIndex] for key, value := range matchers { if value != anyValue { - s.assert.Equal(value, parsedLine.parsed[key]) + s.assert.Equalf(value, parsedLine.parsed[key], "field %q: expected %v, got %v in %s", key, value, parsedLine.parsed[key], parsedLine.raw) } } diff --git a/internal/run/scenario_logger.go b/internal/run/scenario_logger.go index d904b929..c6e5531b 100644 --- a/internal/run/scenario_logger.go +++ b/internal/run/scenario_logger.go @@ -5,8 +5,8 @@ import ( "log/slog" "os" - "github.com/form3tech-oss/f1/v2/internal/log" - "github.com/form3tech-oss/f1/v2/internal/ui" + "github.com/form3tech-oss/f1/v3/internal/log" + "github.com/form3tech-oss/f1/v3/internal/ui" ) type ScenarioLogger struct { diff --git a/internal/run/test_runner.go b/internal/run/test_runner.go index 0e7bfee0..425bf52f 100644 --- a/internal/run/test_runner.go +++ b/internal/run/test_runner.go @@ -9,19 +9,19 @@ import ( "github.com/prometheus/client_golang/prometheus/push" - "github.com/form3tech-oss/f1/v2/internal/envsettings" - "github.com/form3tech-oss/f1/v2/internal/log" - "github.com/form3tech-oss/f1/v2/internal/logutils" - "github.com/form3tech-oss/f1/v2/internal/metrics" - "github.com/form3tech-oss/f1/v2/internal/options" - "github.com/form3tech-oss/f1/v2/internal/progress" - "github.com/form3tech-oss/f1/v2/internal/raterun" - "github.com/form3tech-oss/f1/v2/internal/run/views" - "github.com/form3tech-oss/f1/v2/internal/trigger/api" - "github.com/form3tech-oss/f1/v2/internal/ui" - "github.com/form3tech-oss/f1/v2/internal/workers" - "github.com/form3tech-oss/f1/v2/internal/xcontext" - "github.com/form3tech-oss/f1/v2/pkg/f1/scenarios" + "github.com/form3tech-oss/f1/v3/internal/envsettings" + "github.com/form3tech-oss/f1/v3/internal/log" + "github.com/form3tech-oss/f1/v3/internal/logutils" + "github.com/form3tech-oss/f1/v3/internal/metrics" + "github.com/form3tech-oss/f1/v3/internal/options" + "github.com/form3tech-oss/f1/v3/internal/progress" + "github.com/form3tech-oss/f1/v3/internal/raterun" + "github.com/form3tech-oss/f1/v3/internal/run/views" + "github.com/form3tech-oss/f1/v3/internal/trigger/api" + "github.com/form3tech-oss/f1/v3/internal/ui" + "github.com/form3tech-oss/f1/v3/internal/workers" + "github.com/form3tech-oss/f1/v3/internal/xcontext" + "github.com/form3tech-oss/f1/v3/pkg/f1/scenarios" ) const ( @@ -43,28 +43,33 @@ type Run struct { } func NewRun( - options options.RunOptions, scenarios *scenarios.Scenarios, trigger *api.Trigger, settings envsettings.Settings, metricsInstance *metrics.Metrics, parentOutput *ui.Output, + opts ...options.RunOption, ) (*Run, error) { + runOptions := options.DefaultRunOptions() + for _, opt := range opts { + opt(&runOptions) + } + progressStats := &progress.Stats{} viewsInstance := views.New() - scenario := scenarios.GetScenario(options.Scenario) + scenario := scenarios.GetScenario(runOptions.Scenario) if scenario == nil { - return nil, fmt.Errorf("scenario not defined: %s", options.Scenario) + return nil, fmt.Errorf("scenario not defined: %s", runOptions.Scenario) } - result := NewResult(options, viewsInstance, progressStats) + result := NewResult(runOptions, viewsInstance, progressStats) outputer := ui.NewOutput( parentOutput.Logger.With(log.ScenarioAttr(scenario.Name)), parentOutput.Printer, parentOutput.Interactive, - options.LogToFile(), + runOptions.LogToFile(), ) scenarioLogger := NewScenarioLogger(outputer) @@ -72,7 +77,7 @@ func NewRun( LogFilePathOrDefault(settings.Log.FilePath, scenario.Name), logutils.NewLogConfigFromSettings(settings), scenario.Name, - options.LogToFile(), + runOptions.LogToFile(), ) progressRunner, err := newProgressRunner(result, outputer) @@ -85,13 +90,12 @@ func NewRun( metricsInstance, progressStats, scenarioLogger.Logger, - log.NewSlogLogrusLogger(scenarioLogger.Logger), ) pusher := newMetricsPusher(settings, scenario.Name, metricsInstance) return &Run{ - options: options, + options: runOptions, trigger: trigger, metrics: metricsInstance, views: viewsInstance, @@ -114,7 +118,7 @@ func newMetricsPusher( } pusher := push.New(settings.Prometheus.PushGateway, "f1-"+scenarioName). - Gatherer(metricsInstance.Registry) + Gatherer(metricsInstance.Gatherer()) if settings.Prometheus.Namespace != "" { pusher = pusher.Grouping("namespace", settings.Prometheus.Namespace) @@ -170,7 +174,7 @@ func (r *Run) Do(ctx context.Context) (*Result, error) { r.metrics.Reset() - r.activeScenario.Setup() + r.activeScenario.Setup(ctx) r.pushMetrics(ctx) diff --git a/internal/run/views/exit.go b/internal/run/views/exit.go index dae7bd28..7946c3f7 100644 --- a/internal/run/views/exit.go +++ b/internal/run/views/exit.go @@ -4,8 +4,8 @@ import ( "log/slog" "time" - "github.com/form3tech-oss/f1/v2/internal/log" - "github.com/form3tech-oss/f1/v2/internal/ui" + "github.com/form3tech-oss/f1/v3/internal/log" + "github.com/form3tech-oss/f1/v3/internal/ui" ) //nolint:lll // templates read better with long lines diff --git a/internal/run/views/exit_test.go b/internal/run/views/exit_test.go index 3a5661ac..d41bdb1f 100644 --- a/internal/run/views/exit_test.go +++ b/internal/run/views/exit_test.go @@ -7,8 +7,8 @@ import ( "github.com/stretchr/testify/assert" - "github.com/form3tech-oss/f1/v2/internal/log" - "github.com/form3tech-oss/f1/v2/internal/run/views" + "github.com/form3tech-oss/f1/v3/internal/log" + "github.com/form3tech-oss/f1/v3/internal/run/views" ) func Test_RenderTimeout(t *testing.T) { diff --git a/internal/run/views/progress.go b/internal/run/views/progress.go index ffcedc5a..8772d373 100644 --- a/internal/run/views/progress.go +++ b/internal/run/views/progress.go @@ -4,9 +4,9 @@ import ( "log/slog" "time" - "github.com/form3tech-oss/f1/v2/internal/log" - "github.com/form3tech-oss/f1/v2/internal/progress" - "github.com/form3tech-oss/f1/v2/internal/ui" + "github.com/form3tech-oss/f1/v3/internal/log" + "github.com/form3tech-oss/f1/v3/internal/progress" + "github.com/form3tech-oss/f1/v3/internal/ui" ) //nolint:lll // templates read better with long lines diff --git a/internal/run/views/progress_test.go b/internal/run/views/progress_test.go index 3aff1cb1..06fe2286 100644 --- a/internal/run/views/progress_test.go +++ b/internal/run/views/progress_test.go @@ -7,9 +7,9 @@ import ( "github.com/stretchr/testify/assert" - "github.com/form3tech-oss/f1/v2/internal/log" - "github.com/form3tech-oss/f1/v2/internal/progress" - "github.com/form3tech-oss/f1/v2/internal/run/views" + "github.com/form3tech-oss/f1/v3/internal/log" + "github.com/form3tech-oss/f1/v3/internal/progress" + "github.com/form3tech-oss/f1/v3/internal/run/views" ) func Test_RenderProgress(t *testing.T) { diff --git a/internal/run/views/result.go b/internal/run/views/result.go index f10d0886..18896320 100644 --- a/internal/run/views/result.go +++ b/internal/run/views/result.go @@ -4,9 +4,9 @@ import ( "log/slog" "time" - "github.com/form3tech-oss/f1/v2/internal/log" - "github.com/form3tech-oss/f1/v2/internal/progress" - "github.com/form3tech-oss/f1/v2/internal/ui" + "github.com/form3tech-oss/f1/v3/internal/log" + "github.com/form3tech-oss/f1/v3/internal/progress" + "github.com/form3tech-oss/f1/v3/internal/ui" ) //nolint:lll // templates read better with long lines diff --git a/internal/run/views/result_test.go b/internal/run/views/result_test.go index 308055c9..c218f9cc 100644 --- a/internal/run/views/result_test.go +++ b/internal/run/views/result_test.go @@ -8,9 +8,9 @@ import ( "github.com/stretchr/testify/assert" - "github.com/form3tech-oss/f1/v2/internal/log" - "github.com/form3tech-oss/f1/v2/internal/progress" - "github.com/form3tech-oss/f1/v2/internal/run/views" + "github.com/form3tech-oss/f1/v3/internal/log" + "github.com/form3tech-oss/f1/v3/internal/progress" + "github.com/form3tech-oss/f1/v3/internal/run/views" ) func Test_RenderResult(t *testing.T) { diff --git a/internal/run/views/stage.go b/internal/run/views/stage.go index 135a46fc..55446515 100644 --- a/internal/run/views/stage.go +++ b/internal/run/views/stage.go @@ -3,8 +3,8 @@ package views import ( "log/slog" - "github.com/form3tech-oss/f1/v2/internal/log" - "github.com/form3tech-oss/f1/v2/internal/ui" + "github.com/form3tech-oss/f1/v3/internal/log" + "github.com/form3tech-oss/f1/v3/internal/ui" ) const ( diff --git a/internal/run/views/stage_test.go b/internal/run/views/stage_test.go index 3c475369..c77f926d 100644 --- a/internal/run/views/stage_test.go +++ b/internal/run/views/stage_test.go @@ -7,8 +7,8 @@ import ( "github.com/stretchr/testify/assert" - "github.com/form3tech-oss/f1/v2/internal/log" - "github.com/form3tech-oss/f1/v2/internal/run/views" + "github.com/form3tech-oss/f1/v3/internal/log" + "github.com/form3tech-oss/f1/v3/internal/run/views" ) func Test_RenderSetup(t *testing.T) { diff --git a/internal/run/views/start.go b/internal/run/views/start.go index 53a8a8a0..e8e5a186 100644 --- a/internal/run/views/start.go +++ b/internal/run/views/start.go @@ -5,7 +5,7 @@ import ( "strconv" "time" - "github.com/form3tech-oss/f1/v2/internal/ui" + "github.com/form3tech-oss/f1/v3/internal/ui" ) //nolint:lll // templates read better with long lines diff --git a/internal/run/views/start_test.go b/internal/run/views/start_test.go index bcab06d4..09a3982a 100644 --- a/internal/run/views/start_test.go +++ b/internal/run/views/start_test.go @@ -7,8 +7,8 @@ import ( "github.com/stretchr/testify/assert" - "github.com/form3tech-oss/f1/v2/internal/log" - "github.com/form3tech-oss/f1/v2/internal/run/views" + "github.com/form3tech-oss/f1/v3/internal/log" + "github.com/form3tech-oss/f1/v3/internal/run/views" ) func Test_RenderStart(t *testing.T) { diff --git a/internal/run/views/templates.go b/internal/run/views/templates.go index 5686bd04..576d477a 100644 --- a/internal/run/views/templates.go +++ b/internal/run/views/templates.go @@ -6,7 +6,7 @@ import ( "text/template" "time" - "github.com/form3tech-oss/f1/v2/internal/termcolor" + "github.com/form3tech-oss/f1/v3/internal/termcolor" ) type renderTermColorsType bool diff --git a/internal/run/views/views.go b/internal/run/views/views.go index 7492949a..371dd39c 100644 --- a/internal/run/views/views.go +++ b/internal/run/views/views.go @@ -7,8 +7,8 @@ import ( "github.com/mattn/go-isatty" - "github.com/form3tech-oss/f1/v2/internal/log" - "github.com/form3tech-oss/f1/v2/internal/ui" + "github.com/form3tech-oss/f1/v3/internal/log" + "github.com/form3tech-oss/f1/v3/internal/ui" ) type ViewContext[T log.Loggable] struct { diff --git a/internal/trigger/api/api.go b/internal/trigger/api/api.go index c32b686c..39a70658 100644 --- a/internal/trigger/api/api.go +++ b/internal/trigger/api/api.go @@ -6,9 +6,9 @@ import ( "github.com/spf13/pflag" - "github.com/form3tech-oss/f1/v2/internal/options" - "github.com/form3tech-oss/f1/v2/internal/ui" - "github.com/form3tech-oss/f1/v2/internal/workers" + "github.com/form3tech-oss/f1/v3/internal/options" + "github.com/form3tech-oss/f1/v3/internal/ui" + "github.com/form3tech-oss/f1/v3/internal/workers" ) type ( @@ -28,6 +28,7 @@ type Builder struct { Flags *pflag.FlagSet Name string Description string + Long string // optional long description (e.g. short-flag meanings) IgnoreCommonFlags bool } @@ -49,7 +50,6 @@ type Options struct { MaxFailures uint64 MaxFailuresRate int Verbose bool - VerboseFail bool IgnoreDropped bool WaitForCompletionTimeout time.Duration } diff --git a/internal/trigger/api/iteration_distribution_test.go b/internal/trigger/api/iteration_distribution_test.go index aece0c0e..17fa7372 100644 --- a/internal/trigger/api/iteration_distribution_test.go +++ b/internal/trigger/api/iteration_distribution_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/require" - "github.com/form3tech-oss/f1/v2/internal/trigger/api" + "github.com/form3tech-oss/f1/v3/internal/trigger/api" ) func TestRegularRateDistribution(t *testing.T) { diff --git a/internal/trigger/api/iteration_worker.go b/internal/trigger/api/iteration_worker.go index 55caa922..e7dc56e5 100644 --- a/internal/trigger/api/iteration_worker.go +++ b/internal/trigger/api/iteration_worker.go @@ -4,9 +4,9 @@ import ( "context" "time" - "github.com/form3tech-oss/f1/v2/internal/options" - "github.com/form3tech-oss/f1/v2/internal/ui" - "github.com/form3tech-oss/f1/v2/internal/workers" + "github.com/form3tech-oss/f1/v3/internal/options" + "github.com/form3tech-oss/f1/v3/internal/ui" + "github.com/form3tech-oss/f1/v3/internal/workers" ) // NewIterationWorker produces a WorkTriggerer which triggers work at fixed intervals. diff --git a/internal/trigger/configure.go b/internal/trigger/configure.go index 360634d2..e7c0a166 100644 --- a/internal/trigger/configure.go +++ b/internal/trigger/configure.go @@ -1,14 +1,14 @@ package trigger import ( - "github.com/form3tech-oss/f1/v2/internal/trigger/api" - "github.com/form3tech-oss/f1/v2/internal/trigger/constant" - "github.com/form3tech-oss/f1/v2/internal/trigger/file" - "github.com/form3tech-oss/f1/v2/internal/trigger/gaussian" - "github.com/form3tech-oss/f1/v2/internal/trigger/ramp" - "github.com/form3tech-oss/f1/v2/internal/trigger/staged" - "github.com/form3tech-oss/f1/v2/internal/trigger/users" - "github.com/form3tech-oss/f1/v2/internal/ui" + "github.com/form3tech-oss/f1/v3/internal/trigger/api" + "github.com/form3tech-oss/f1/v3/internal/trigger/constant" + "github.com/form3tech-oss/f1/v3/internal/trigger/file" + "github.com/form3tech-oss/f1/v3/internal/trigger/gaussian" + "github.com/form3tech-oss/f1/v3/internal/trigger/ramp" + "github.com/form3tech-oss/f1/v3/internal/trigger/staged" + "github.com/form3tech-oss/f1/v3/internal/trigger/users" + "github.com/form3tech-oss/f1/v3/internal/ui" ) func GetBuilders(output *ui.Output) []api.Builder { diff --git a/internal/trigger/constant/constant_rate.go b/internal/trigger/constant/constant_rate.go index dca65abc..b3aa7c45 100644 --- a/internal/trigger/constant/constant_rate.go +++ b/internal/trigger/constant/constant_rate.go @@ -6,9 +6,9 @@ import ( "github.com/spf13/pflag" - "github.com/form3tech-oss/f1/v2/internal/trigger/api" - "github.com/form3tech-oss/f1/v2/internal/trigger/rate" - "github.com/form3tech-oss/f1/v2/internal/triggerflags" + "github.com/form3tech-oss/f1/v3/internal/trigger/api" + "github.com/form3tech-oss/f1/v3/internal/trigger/rate" + "github.com/form3tech-oss/f1/v3/internal/triggerflags" ) const ( @@ -18,7 +18,7 @@ const ( func Rate() api.Builder { flags := pflag.NewFlagSet("constant", pflag.ContinueOnError) flags.StringP(flagRate, "r", "1/s", - "number of iterations to start per interval, in the form /") + "iterations per interval, e.g. 10/s, 100/m") triggerflags.JitterFlag(flags) triggerflags.DistributionFlag(flags) @@ -26,6 +26,7 @@ func Rate() api.Builder { return api.Builder{ Name: "constant ", Description: "triggers test iterations at a constant rate", + Long: "Short flags: -r rate, -j jitter", Flags: flags, New: func(params *pflag.FlagSet) (*api.Trigger, error) { rateArg, err := params.GetString(flagRate) diff --git a/internal/trigger/file/file_parser.go b/internal/trigger/file/file_parser.go index b5ecf328..9b49d76e 100644 --- a/internal/trigger/file/file_parser.go +++ b/internal/trigger/file/file_parser.go @@ -7,10 +7,10 @@ import ( "gopkg.in/yaml.v3" - "github.com/form3tech-oss/f1/v2/internal/trigger/constant" - "github.com/form3tech-oss/f1/v2/internal/trigger/gaussian" - "github.com/form3tech-oss/f1/v2/internal/trigger/ramp" - "github.com/form3tech-oss/f1/v2/internal/trigger/staged" + "github.com/form3tech-oss/f1/v3/internal/trigger/constant" + "github.com/form3tech-oss/f1/v3/internal/trigger/gaussian" + "github.com/form3tech-oss/f1/v3/internal/trigger/ramp" + "github.com/form3tech-oss/f1/v3/internal/trigger/staged" ) type ConfigFile struct { diff --git a/internal/trigger/file/file_parser_test.go b/internal/trigger/file/file_parser_test.go index 1b47d769..a1b75b9b 100644 --- a/internal/trigger/file/file_parser_test.go +++ b/internal/trigger/file/file_parser_test.go @@ -6,7 +6,7 @@ import ( "github.com/stretchr/testify/require" - "github.com/form3tech-oss/f1/v2/internal/trigger/file" + "github.com/form3tech-oss/f1/v3/internal/trigger/file" ) func TestFileRate_SingleStages(t *testing.T) { diff --git a/internal/trigger/file/file_rate.go b/internal/trigger/file/file_rate.go index d27dfb42..00b23d81 100644 --- a/internal/trigger/file/file_rate.go +++ b/internal/trigger/file/file_rate.go @@ -9,8 +9,8 @@ import ( "github.com/spf13/pflag" - "github.com/form3tech-oss/f1/v2/internal/trigger/api" - "github.com/form3tech-oss/f1/v2/internal/ui" + "github.com/form3tech-oss/f1/v3/internal/trigger/api" + "github.com/form3tech-oss/f1/v3/internal/ui" ) type RunnableStages struct { diff --git a/internal/trigger/file/stages_worker.go b/internal/trigger/file/stages_worker.go index d00aca8e..471d6347 100644 --- a/internal/trigger/file/stages_worker.go +++ b/internal/trigger/file/stages_worker.go @@ -5,11 +5,11 @@ import ( "os" "time" - "github.com/form3tech-oss/f1/v2/internal/options" - "github.com/form3tech-oss/f1/v2/internal/trigger/api" - "github.com/form3tech-oss/f1/v2/internal/trigger/users" - "github.com/form3tech-oss/f1/v2/internal/ui" - "github.com/form3tech-oss/f1/v2/internal/workers" + "github.com/form3tech-oss/f1/v3/internal/options" + "github.com/form3tech-oss/f1/v3/internal/trigger/api" + "github.com/form3tech-oss/f1/v3/internal/trigger/users" + "github.com/form3tech-oss/f1/v3/internal/ui" + "github.com/form3tech-oss/f1/v3/internal/workers" ) const safeDurationBeforeNextStage = 20 * time.Millisecond @@ -67,7 +67,7 @@ func setEnvs(envs map[string]string, output *ui.Output) { err := os.Setenv(key, value) if err != nil { output.Display(ui.ErrorMessage{ - Message: "unable set environment variables for given scenario", + Message: "unable to set environment variables for given scenario", Error: err, }) } @@ -79,7 +79,7 @@ func unsetEnvs(envs map[string]string, output *ui.Output) { err := os.Unsetenv(key) if err != nil { output.Display(ui.ErrorMessage{ - Message: "unable unset environment variables for given scenario", + Message: "unable to unset environment variables for given scenario", Error: err, }) } diff --git a/internal/trigger/gaussian/gaussian_rate.go b/internal/trigger/gaussian/gaussian_rate.go index 97b047d2..3c89a0cd 100644 --- a/internal/trigger/gaussian/gaussian_rate.go +++ b/internal/trigger/gaussian/gaussian_rate.go @@ -9,11 +9,11 @@ import ( "github.com/spf13/pflag" - "github.com/form3tech-oss/f1/v2/internal/gaussian" - "github.com/form3tech-oss/f1/v2/internal/trigger/api" - "github.com/form3tech-oss/f1/v2/internal/trigger/rate" - "github.com/form3tech-oss/f1/v2/internal/triggerflags" - "github.com/form3tech-oss/f1/v2/internal/ui" + "github.com/form3tech-oss/f1/v3/internal/gaussian" + "github.com/form3tech-oss/f1/v3/internal/trigger/api" + "github.com/form3tech-oss/f1/v3/internal/trigger/rate" + "github.com/form3tech-oss/f1/v3/internal/triggerflags" + "github.com/form3tech-oss/f1/v3/internal/ui" ) const defaultVolume = 24 * 60 * 60 @@ -31,24 +31,19 @@ const ( func Rate(output *ui.Output) api.Builder { flags := pflag.NewFlagSet("gaussian", pflag.ContinueOnError) flags.Float64(flagVolume, defaultVolume, - "The desired volume to be achieved with the calculated load profile. "+ - "Will be ignored if --peak-rate is also provided.") + "desired volume for load profile (ignored if --peak-rate is set)") flags.Duration(flagRepeat, 24*time.Hour, - "How often the cycle should repeat") + "cycle repeat interval (default 24h)") flags.Duration(flagIterationFrequency, 1*time.Second, - "How frequently iterations should be started") + "how often to start iterations (default 1s)") flags.String(flagWeights, "", - "Optional scaling factor to apply per repetition. "+ - "This can be used for example with daily repetitions to set different weights per day of the week") + "comma-separated scaling factors per repetition (e.g. per day of week)") flags.Duration(flagPeak, 14*time.Hour, - "The offset within the repetition window when the load should reach its maximum. "+ - "Default 14 hours (with 24 hour default repeat)") + "offset within repeat window when load peaks (default 14h)") flags.StringP(flagPeakRate, "r", "", - "number of iterations per interval in peak time, "+ - "in the form / (e.g. 1/s). If --peak-rate is provided, "+ - "the value given for --volume will be ignored.") + "peak rate, e.g. 100/s (overrides --volume)") flags.Duration(flagStandardDeviation, 150*time.Minute, - "The standard deviation to use for the distribution of load") + "standard deviation for load distribution (default 150m)") triggerflags.JitterFlag(flags) triggerflags.DistributionFlag(flags) @@ -56,6 +51,7 @@ func Rate(output *ui.Output) api.Builder { return api.Builder{ Name: "gaussian ", Description: "distributes load to match a desired monthly volume", + Long: "Short flags: -r peak-rate", Flags: flags, New: func(flags *pflag.FlagSet) (*api.Trigger, error) { volume, err := flags.GetFloat64(flagVolume) diff --git a/internal/trigger/gaussian/gaussian_rate_bench_test.go b/internal/trigger/gaussian/gaussian_rate_bench_test.go index 18af1a66..7483e7fa 100644 --- a/internal/trigger/gaussian/gaussian_rate_bench_test.go +++ b/internal/trigger/gaussian/gaussian_rate_bench_test.go @@ -6,7 +6,7 @@ import ( "github.com/stretchr/testify/require" - "github.com/form3tech-oss/f1/v2/internal/trigger/gaussian" + "github.com/form3tech-oss/f1/v3/internal/trigger/gaussian" ) func Benchmark_calculateVolume(b *testing.B) { diff --git a/internal/trigger/gaussian/gaussian_rate_test.go b/internal/trigger/gaussian/gaussian_rate_test.go index 26bf5512..506d5d37 100644 --- a/internal/trigger/gaussian/gaussian_rate_test.go +++ b/internal/trigger/gaussian/gaussian_rate_test.go @@ -9,8 +9,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/form3tech-oss/f1/v2/internal/trigger/api" - "github.com/form3tech-oss/f1/v2/internal/trigger/gaussian" + "github.com/form3tech-oss/f1/v3/internal/trigger/api" + "github.com/form3tech-oss/f1/v3/internal/trigger/gaussian" ) func TestTotalVolumes(t *testing.T) { diff --git a/internal/trigger/ramp/ramp_rate.go b/internal/trigger/ramp/ramp_rate.go index 09851237..8f12c6ac 100644 --- a/internal/trigger/ramp/ramp_rate.go +++ b/internal/trigger/ramp/ramp_rate.go @@ -7,9 +7,9 @@ import ( "github.com/spf13/pflag" - "github.com/form3tech-oss/f1/v2/internal/trigger/api" - "github.com/form3tech-oss/f1/v2/internal/trigger/rate" - "github.com/form3tech-oss/f1/v2/internal/triggerflags" + "github.com/form3tech-oss/f1/v3/internal/trigger/api" + "github.com/form3tech-oss/f1/v3/internal/trigger/rate" + "github.com/form3tech-oss/f1/v3/internal/triggerflags" ) const ( @@ -21,11 +21,11 @@ const ( func Rate() api.Builder { flags := pflag.NewFlagSet("ramp", pflag.ContinueOnError) flags.StringP(flagStartRate, "s", "1/s", - "number of iterations to start per interval, in the form /") + "initial rate, e.g. 1/s, 10/m") flags.StringP(flagEndRate, "e", "1/s", - "number of iterations to end per interval, in the form /") + "target rate at end of ramp, e.g. 100/s") flags.DurationP(flagRampDuration, "r", 1*time.Second, - "ramp duration, if not provided then --max-duration will be used") + "ramp duration (default: --max-duration)") triggerflags.JitterFlag(flags) triggerflags.DistributionFlag(flags) @@ -33,6 +33,7 @@ func Rate() api.Builder { return api.Builder{ Name: "ramp ", Description: "ramp up or down requests for a certain duration", + Long: "Short flags: -s start-rate, -e end-rate, -r ramp-duration", Flags: flags, New: func(flags *pflag.FlagSet) (*api.Trigger, error) { startRateArg, err := flags.GetString(flagStartRate) diff --git a/internal/trigger/staged/calculator_test.go b/internal/trigger/staged/calculator_test.go index 4ed39489..2fa9fee1 100644 --- a/internal/trigger/staged/calculator_test.go +++ b/internal/trigger/staged/calculator_test.go @@ -6,7 +6,7 @@ import ( "github.com/stretchr/testify/assert" - "github.com/form3tech-oss/f1/v2/internal/trigger/staged" + "github.com/form3tech-oss/f1/v3/internal/trigger/staged" ) func TestCalculatorWithDefaultStartTime(t *testing.T) { diff --git a/internal/trigger/staged/stage_test.go b/internal/trigger/staged/stage_test.go index d57286cf..2ec6543b 100644 --- a/internal/trigger/staged/stage_test.go +++ b/internal/trigger/staged/stage_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/form3tech-oss/f1/v2/internal/trigger/staged" + "github.com/form3tech-oss/f1/v3/internal/trigger/staged" ) func TestParseStages_With_Valid_String(t *testing.T) { diff --git a/internal/trigger/staged/staged_rate.go b/internal/trigger/staged/staged_rate.go index 5c7edb61..92fe1bb1 100644 --- a/internal/trigger/staged/staged_rate.go +++ b/internal/trigger/staged/staged_rate.go @@ -6,24 +6,23 @@ import ( "github.com/spf13/pflag" - "github.com/form3tech-oss/f1/v2/internal/trigger/api" - "github.com/form3tech-oss/f1/v2/internal/triggerflags" + "github.com/form3tech-oss/f1/v3/internal/trigger/api" + "github.com/form3tech-oss/f1/v3/internal/triggerflags" ) const ( flagStages = "stages" - flagIterationFrequency = "iterationFrequency" + flagIterationFrequency = "iteration-frequency" flagStartTime = "startTime" ) func Rate() api.Builder { flags := pflag.NewFlagSet("staged", pflag.ContinueOnError) - flags.StringP("stages", "s", "0s:1, 10s:1", - "Comma separated list of :. "+ - "During the stage, the number of concurrent iterations will ramp up or down to the target.") + flags.StringP(flagStages, "s", "0s:1, 10s:1", + "comma-separated : pairs, e.g. 0s:1, 10s:5, 20s:10") flags.DurationP(flagIterationFrequency, "f", 1*time.Second, - "How frequently iterations should be started") - flags.String(flagStartTime, "", "Starting point of stage calculation, defaults to now") + "how often to start iterations (e.g. 1s)") + flags.String(flagStartTime, "", "start time for stage calculation (default: now)") triggerflags.JitterFlag(flags) triggerflags.DistributionFlag(flags) @@ -31,6 +30,7 @@ func Rate() api.Builder { return api.Builder{ Name: "staged ", Description: "triggers iterations at varying rates", + Long: "Short flags: -s stages, -f iteration-frequency", Flags: flags, New: func(params *pflag.FlagSet) (*api.Trigger, error) { jitterArg, err := params.GetFloat64(triggerflags.FlagJitter) diff --git a/internal/trigger/users/users_rate.go b/internal/trigger/users/users_rate.go index 2cf104fe..af89dbb8 100644 --- a/internal/trigger/users/users_rate.go +++ b/internal/trigger/users/users_rate.go @@ -6,10 +6,10 @@ import ( "github.com/spf13/pflag" - "github.com/form3tech-oss/f1/v2/internal/options" - "github.com/form3tech-oss/f1/v2/internal/trigger/api" - "github.com/form3tech-oss/f1/v2/internal/ui" - "github.com/form3tech-oss/f1/v2/internal/workers" + "github.com/form3tech-oss/f1/v3/internal/options" + "github.com/form3tech-oss/f1/v3/internal/trigger/api" + "github.com/form3tech-oss/f1/v3/internal/ui" + "github.com/form3tech-oss/f1/v3/internal/workers" ) func Rate() api.Builder { @@ -31,15 +31,11 @@ func Rate() api.Builder { } return &api.Trigger{ - Trigger: trigger, - Description: "Makes requests from a set of users specified by --concurrency", - // The rate function used by the `users` mode, is actually dependent - // on the number of users specified in the `--concurrency` flag. - // This flag is not required for the `chart` command, which uses the `DryRun` - // function, so its not possible to provide an accurate rate function here. - DryRun: func(time.Time) int { return 1 }, - }, - nil + Trigger: trigger, + Description: "Makes requests from a set of users specified by --concurrency", + // The rate for users mode depends on --concurrency; DryRun returns 1 as a placeholder. + DryRun: func(time.Time) int { return 1 }, + }, nil }, } } diff --git a/internal/triggerflags/flags.go b/internal/triggerflags/flags.go index 67e42ce4..641f8db0 100644 --- a/internal/triggerflags/flags.go +++ b/internal/triggerflags/flags.go @@ -5,12 +5,11 @@ import ( "github.com/spf13/pflag" - "github.com/form3tech-oss/f1/v2/internal/trigger/api" + "github.com/form3tech-oss/f1/v3/internal/trigger/api" ) const ( FlagVerbose = "verbose" - FlagVerboseFail = "verbose-fail" FlagIgnoreDropped = "ignore-dropped" FlagMaxDuration = "max-duration" FlagMaxIterations = "max-iterations" @@ -31,12 +30,12 @@ func DistributionFlag(flagSet *pflag.FlagSet) { distributions := strings.Join(distributionTypes, "|") flagSet.String(FlagDistribution, string(api.RegularDistribution), - "optional parameter to distribute the rate over steps of 100ms, which can be "+distributions) + "rate distribution: "+distributions) } const FlagJitter = "jitter" func JitterFlag(flagSet *pflag.FlagSet) { flagSet.Float64P(FlagJitter, "j", 0.0, - "vary the rate randomly by up to jitter percent") + "random rate variation, e.g. 5 for ±5%") } diff --git a/internal/ui/messages.go b/internal/ui/messages.go index 98bed6b1..3da2eba0 100644 --- a/internal/ui/messages.go +++ b/internal/ui/messages.go @@ -4,7 +4,7 @@ import ( "fmt" "log/slog" - "github.com/form3tech-oss/f1/v2/internal/log" + "github.com/form3tech-oss/f1/v3/internal/log" ) type ErrorMessage struct { diff --git a/internal/ui/output.go b/internal/ui/output.go index 01695044..531ad7c4 100644 --- a/internal/ui/output.go +++ b/internal/ui/output.go @@ -6,11 +6,11 @@ import ( "github.com/mattn/go-isatty" - "github.com/form3tech-oss/f1/v2/internal/log" + "github.com/form3tech-oss/f1/v3/internal/log" ) // Outputable may be a type of message (like [ErrorMessage], [InfoMessage], etc) or -// [github.com/form3tech-oss/f1/v2/internal/run/views.ViewContext] +// [github.com/form3tech-oss/f1/v3/internal/run/views.ViewContext] type Outputable interface { Print(printer *Printer) Log(logger *slog.Logger) diff --git a/internal/workers/active_scenario.go b/internal/workers/active_scenario.go index 99cb6622..78fd2120 100644 --- a/internal/workers/active_scenario.go +++ b/internal/workers/active_scenario.go @@ -1,25 +1,23 @@ package workers import ( + "context" "log/slog" - "github.com/sirupsen/logrus" - - "github.com/form3tech-oss/f1/v2/internal/metrics" - "github.com/form3tech-oss/f1/v2/internal/progress" - "github.com/form3tech-oss/f1/v2/internal/xtime" - "github.com/form3tech-oss/f1/v2/pkg/f1/scenarios" - "github.com/form3tech-oss/f1/v2/pkg/f1/testing" + "github.com/form3tech-oss/f1/v3/internal/metrics" + "github.com/form3tech-oss/f1/v3/internal/progress" + "github.com/form3tech-oss/f1/v3/internal/xtime" + "github.com/form3tech-oss/f1/v3/pkg/f1/f1testing" + "github.com/form3tech-oss/f1/v3/pkg/f1/scenarios" ) type ActiveScenario struct { - scenario *scenarios.Scenario - m *metrics.Metrics - progress *progress.Stats - t *testing.T - Teardown func() - logger *slog.Logger - logrusLogger *logrus.Logger + scenario *scenarios.Scenario + m *metrics.Metrics + progress *progress.Stats + t *f1testing.T + Teardown func() + logger *slog.Logger } const instantDuration = 0 @@ -29,34 +27,31 @@ func NewActiveScenario( metricsInstance *metrics.Metrics, stats *progress.Stats, logger *slog.Logger, - logrusLogger *logrus.Logger, ) *ActiveScenario { - t, teardown := testing.NewTWithOptions(scenario.Name, - testing.WithIteration("setup"), - testing.WithVUID(-1), - testing.WithLogger(logger), - testing.WithLogrusLogger(logrusLogger), + t, teardown := f1testing.NewTWithOptions(scenario.Name, + f1testing.WithIteration(f1testing.IterationSetup), + f1testing.WithVUID(-1), + f1testing.WithLogger(logger), ) s := &ActiveScenario{ - scenario: scenario, - m: metricsInstance, - t: t, - Teardown: teardown, - progress: stats, - logger: logger, - logrusLogger: logrusLogger, + scenario: scenario, + m: metricsInstance, + t: t, + Teardown: teardown, + progress: stats, + logger: logger, } return s } -func (s *ActiveScenario) Setup() { +func (s *ActiveScenario) Setup(ctx context.Context) { start := xtime.NanoTime() func() { - defer testing.CheckResults(s.t, nil) + defer f1testing.CheckResults(s.t, nil) - s.scenario.RunFn = s.scenario.ScenarioFn(s.t) + s.scenario.RunFn = s.scenario.ScenarioFn(ctx, s.t) }() duration := xtime.NanoTime() - start @@ -73,13 +68,13 @@ func (s *ActiveScenario) Failed() bool { } // Run performs a single iteration of the test. -func (s *ActiveScenario) Run(state *iterationState) { +func (s *ActiveScenario) Run(ctx context.Context, state *iterationState) { defer state.teardown() start := xtime.NanoTime() func() { - defer testing.CheckResults(state.t, nil) - s.scenario.RunFn(state.t) + defer f1testing.CheckResults(state.t, nil) + s.scenario.RunFn(ctx, state.t) }() failed := state.t.Failed() @@ -95,10 +90,9 @@ func (s *ActiveScenario) RecordDroppedIteration() { } func (s *ActiveScenario) newIterationState(id int) *iterationState { - t, teardown := testing.NewTWithOptions(s.scenario.Name, - testing.WithVUID(id), - testing.WithLogger(s.logger), - testing.WithLogrusLogger(s.logrusLogger), + t, teardown := f1testing.NewTWithOptions(s.scenario.Name, + f1testing.WithVUID(id), + f1testing.WithLogger(s.logger), ) return &iterationState{ diff --git a/internal/workers/continuous_pool.go b/internal/workers/continuous_pool.go index 73805840..6d68044e 100644 --- a/internal/workers/continuous_pool.go +++ b/internal/workers/continuous_pool.go @@ -2,7 +2,6 @@ package workers import ( "context" - "strconv" "sync" "sync/atomic" ) @@ -32,7 +31,7 @@ func (p *ContinuousPool) Start(ctx context.Context) { workersStarted.Add(p.numWorkers) p.manager.runningWorkers.Add(p.numWorkers) for _, iterationState := range p.iterationStatePool { - go p.startWorker(iterationState, &workersStarted) + go p.startWorker(workerCtx, iterationState, &workersStarted) } // context.Done() and context.Err() for context that can be cancelled use a Lock. @@ -49,6 +48,7 @@ func (p *ContinuousPool) maxIterationsReached() { } func (p *ContinuousPool) startWorker( + ctx context.Context, iterationState *iterationState, workersStarted *sync.WaitGroup, ) { @@ -67,7 +67,7 @@ func (p *ContinuousPool) startWorker( return } - iterationState.t.Reset(strconv.FormatUint(iteration, 10)) - p.manager.activeScenario.Run(iterationState) + iterationState.t.Reset(iteration) + p.manager.activeScenario.Run(ctx, iterationState) } } diff --git a/internal/workers/pool_manager.go b/internal/workers/pool_manager.go index 61227c36..8a842177 100644 --- a/internal/workers/pool_manager.go +++ b/internal/workers/pool_manager.go @@ -5,12 +5,12 @@ import ( "sync" "sync/atomic" - "github.com/form3tech-oss/f1/v2/pkg/f1/testing" + "github.com/form3tech-oss/f1/v3/pkg/f1/f1testing" ) type iterationState struct { teardown func() - t *testing.T + t *f1testing.T } type PoolManager struct { diff --git a/internal/workers/trigger_pool.go b/internal/workers/trigger_pool.go index 26f3faf4..b67f98d0 100644 --- a/internal/workers/trigger_pool.go +++ b/internal/workers/trigger_pool.go @@ -2,7 +2,6 @@ package workers import ( "context" - "strconv" "sync" "sync/atomic" ) @@ -47,7 +46,7 @@ func (p *TriggerPool) Start(ctx context.Context) context.Context { p.workerCtxCancel = cancel for _, statePool := range p.iterationStatePool { - go p.run(statePool, &startedWg) + go p.run(workerCtx, statePool, &startedWg) } // wait for all workers to start, to make sure we have the concurrency requested, @@ -102,6 +101,7 @@ func (p *TriggerPool) waitForNewJobs() { } func (p *TriggerPool) run( + ctx context.Context, iterationState *iterationState, startWg *sync.WaitGroup, ) { @@ -120,8 +120,8 @@ func (p *TriggerPool) run( return } - iterationState.t.Reset(strconv.FormatUint(iteration, 10)) - p.manager.activeScenario.Run(iterationState) + iterationState.t.Reset(iteration) + p.manager.activeScenario.Run(ctx, iterationState) } } } diff --git a/internal/xtime/nanotime_test.go b/internal/xtime/nanotime_test.go index 72cd6fe4..1596da5f 100644 --- a/internal/xtime/nanotime_test.go +++ b/internal/xtime/nanotime_test.go @@ -5,7 +5,7 @@ import ( "github.com/stretchr/testify/assert" - "github.com/form3tech-oss/f1/v2/internal/xtime" + "github.com/form3tech-oss/f1/v3/internal/xtime" ) func TestNanoTime(t *testing.T) { diff --git a/pkg/f1/doc.go b/pkg/f1/doc.go index d8bf54b5..9bd4843b 100644 --- a/pkg/f1/doc.go +++ b/pkg/f1/doc.go @@ -22,33 +22,35 @@ Cleanup functions can also be provided for both stages, and are executed in LIFO Types are provided for setup and iteration/run functions as below: // ScenarioFn initialises a scenario and returns the iteration function (RunFn) to be - // invoked for every iteration of the tests. - type ScenarioFn func(t *T) RunFn + // invoked for every iteration of the tests. ctx is cancelled when the run is interrupted or times out. + type ScenarioFn func(ctx context.Context, t *T) RunFn // RunFn performs a single iteration of the scenario. 't' may be used for asserting - // results or failing the scenario. - type RunFn func(t *T) + // results or failing the scenario. ctx is cancelled when the run is stopped; pass it to + // context-aware operations or check ctx.Done() to abort early. + type RunFn func(ctx context.Context, t *T) Writing tests is simply a case of implementing the types and registering them with F1: package main import ( + "context" "fmt" - "github.com/form3tech-oss/f1/v2/pkg/f1" - "github.com/form3tech-oss/f1/v2/pkg/f1/testing" + "github.com/form3tech-oss/f1/v3/pkg/f1" + "github.com/form3tech-oss/f1/v3/pkg/f1/f1testing" ) func main() { // Create a new f1 instance, add all the scenarios and execute the f1 tool. // Any scenario that is added here can be executed like: // `go run main.go run constant mySuperFastLoadTest` - f1.New().Add("mySuperFastLoadTest", setupMySuperFastLoadTest).Execute() + f1.New().AddScenario("mySuperFastLoadTest", setupMySuperFastLoadTest).Execute() } // Performs any setup steps and returns a function to run on every iteration of the scenario - func setupMySuperFastLoadTest(t *testing.T) testing.RunFn { + func setupMySuperFastLoadTest(ctx context.Context, t *f1testing.T) f1testing.RunFn { fmt.Println("Setup the scenario") // Register clean up function which will be invoked at the end of the scenario @@ -57,7 +59,7 @@ Writing tests is simply a case of implementing the types and registering them wi fmt.Println("Clean up the setup of the scenario") }) - runFn := func(t *testing.T) { + runFn := func(ctx context.Context, t *f1testing.T) { fmt.Println("Run the test") // Register clean up function for each test which will be invoked in LIFO diff --git a/pkg/f1/f1.go b/pkg/f1/f1.go index 8ca625c7..09b7051a 100644 --- a/pkg/f1/f1.go +++ b/pkg/f1/f1.go @@ -4,15 +4,13 @@ import ( "context" "errors" "fmt" - "log/slog" "os" "os/signal" "syscall" - "github.com/form3tech-oss/f1/v2/internal/envsettings" - "github.com/form3tech-oss/f1/v2/internal/ui" - "github.com/form3tech-oss/f1/v2/pkg/f1/scenarios" - "github.com/form3tech-oss/f1/v2/pkg/f1/testing" + "github.com/form3tech-oss/f1/v3/internal/ui" + "github.com/form3tech-oss/f1/v3/pkg/f1/f1testing" + "github.com/form3tech-oss/f1/v3/pkg/f1/scenarios" ) const ( @@ -29,55 +27,49 @@ const ( type F1 struct { scenarios *scenarios.Scenarios profiling *profiling - settings envsettings.Settings + settings Settings options *f1Options } type f1Options struct { - output *ui.Output - staticMetrics map[string]string + output *ui.Output + staticMetrics map[string]string + loggerExplicit bool } -// New instantiates a new instance of an F1 CLI. -func New() *F1 { - settings := envsettings.Get() - - return &F1{ +// New instantiates a new F1 CLI. Pass options to configure logger, metrics, etc. +// +// Construction order: +// 1. Load default settings from environment variables (see DefaultSettings) +// 2. Apply options (may replace settings via WithSettings or override individual fields) +// 3. Build default output from final settings unless WithLogger was used +func New(opts ...Option) *F1 { + f := &F1{ scenarios: scenarios.New(), profiling: &profiling{}, - settings: settings, - options: &f1Options{ - output: ui.NewDefaultOutput(settings.Log.SlogLevel(), settings.Log.IsFormatJSON()), - }, + settings: DefaultSettings(), + options: &f1Options{}, + } + for _, opt := range opts { + opt(f) } -} -// WithLogger allows specifying logger to be used for all internal and scenario logs -// -// This will disable the F1_LOG_LEVEL and F1_LOG_FORMAT options, as they only relate to the built-in -// logger. -// -// The logger will be used for non-interactive output, file logs or when `--verbose` is specified. -func (f *F1) WithLogger(logger *slog.Logger) *F1 { - f.options.output = ui.NewDefaultOutputWithLogger(logger) - return f -} + if !f.options.loggerExplicit { + f.options.output = ui.NewDefaultOutput(f.settings.Logging.Level, f.settings.Logging.Format == LogFormatJSON) + } -// WithStaticMetrics registers additional labels with fixed values to the f1 metrics -func (f *F1) WithStaticMetrics(labels map[string]string) *F1 { - f.options.staticMetrics = labels return f } -// Add registers a new test scenario with the given name. This is the name used when running +// AddScenario registers a new test scenario with the given name. This is the name used when running // load test scenarios. For example, calling the function with the following arguments: // -// f.Add("myTest", myScenario) +// f.AddScenario("myTest", myScenario) // // will result in the test "myTest" being runnable from the command line: // // f1 run constant -r 1/s -d 10s myTest -func (f *F1) Add(name string, scenarioFn testing.ScenarioFn, options ...scenarios.ScenarioOption) *F1 { +func (f *F1) AddScenario(name string, scenarioFn f1testing.ScenarioFn, options ...scenarios.ScenarioOption) *F1 { info := &scenarios.Scenario{ Name: name, ScenarioFn: scenarioFn, @@ -87,15 +79,14 @@ func (f *F1) Add(name string, scenarioFn testing.ScenarioFn, options ...scenario opt(info) } - f.scenarios.Add(info) + f.scenarios.AddScenario(info) return f } -// NewSignalContext returns a context.Context that is cancelled whenever -// 'SIGINT' or 'SIGTERM' are received. -// If one of these two signals is received a second time, the application exits. -func newSignalContext(stopCh <-chan struct{}) context.Context { - ctx, cancel := context.WithCancel(context.Background()) +// newSignalContext returns a context that is cancelled when parent is cancelled or +// when SIGINT/SIGTERM is received. If a signal is received a second time, the app exits. +func newSignalContext(parent context.Context, stopCh <-chan struct{}) context.Context { + ctx, cancel := context.WithCancel(parent) c := make(chan os.Signal, signalChanBufferSize) signal.Notify(c, os.Interrupt, syscall.SIGTERM) @@ -104,6 +95,8 @@ func newSignalContext(stopCh <-chan struct{}) context.Context { select { case <-c: cancel() + case <-parent.Done(): + return case <-stopCh: return } @@ -119,24 +112,22 @@ func newSignalContext(stopCh <-chan struct{}) context.Context { return ctx } -// Execute synchronously runs the F1 CLI. This function is the blocking entrypoint to the CLI, -// so you should register your test scenarios with the Add function prior to calling this -// function. -func (f *F1) Execute() { - if err := f.execute(nil); err != nil { - f.options.output.Display(ui.ErrorMessage{Message: "f1 failed", Error: err}) - os.Exit(1) +// Run runs the CLI with the given args. Returns error on failure; never exits. +// ctx controls cancellation; SIGINT/SIGTERM also cancel via internal signal handling. +// Pass nil for args to use os.Args (e.g. when called from main). +func (f *F1) Run(ctx context.Context, args []string) error { + if err := f.execute(ctx, args); err != nil { + return fmt.Errorf("run: %w", err) } + return nil } -// ExecuteWithArgs is similar to Execute, but takes command line arguments from the args array. -// Useful for testing F1 test scenarios. -func (f *F1) ExecuteWithArgs(args []string) error { - if err := f.execute(args); err != nil { - return fmt.Errorf("execute with args: %w", err) +// Execute runs the CLI and exits with code 1 on error. Convenience for main(). +func (f *F1) Execute() { + if err := f.Run(context.Background(), nil); err != nil { + f.options.output.Display(ui.ErrorMessage{Message: "f1 failed", Error: err}) + os.Exit(1) } - - return nil } // GetScenarios returns the list of registered scenarios. @@ -144,8 +135,15 @@ func (f *F1) GetScenarios() *scenarios.Scenarios { return f.scenarios } -func (f *F1) execute(args []string) error { - rootCmd, err := buildRootCmd(f.scenarios, f.settings, f.profiling, f.options.output, f.options.staticMetrics) +func (f *F1) execute(ctx context.Context, args []string) error { + stopCh := make(chan struct{}) + defer close(stopCh) + execCtx := newSignalContext(ctx, stopCh) + + rootCmd, err := buildRootCmd( + execCtx, f.scenarios, f.settings.toInternal(), + f.profiling, f.options.output, f.options.staticMetrics, + ) if err != nil { return fmt.Errorf("building root command: %w", err) } @@ -154,18 +152,14 @@ func (f *F1) execute(args []string) error { rootCmd.SetArgs(args) } - stopCh := make(chan struct{}) - defer close(stopCh) - ctx := newSignalContext(stopCh) - - err = rootCmd.ExecuteContext(ctx) + err = rootCmd.ExecuteContext(execCtx) // stop profiling regardless of err profilingErr := f.profiling.stop() errs := errors.Join(err, profilingErr) if errs != nil { - return fmt.Errorf("command execution: %w", err) + return fmt.Errorf("command execution: %w", errs) } return nil diff --git a/pkg/f1/f1_scenarios.go b/pkg/f1/f1_scenarios.go index 82c2c395..466ad3d3 100644 --- a/pkg/f1/f1_scenarios.go +++ b/pkg/f1/f1_scenarios.go @@ -1,22 +1,24 @@ package f1 import ( - "github.com/form3tech-oss/f1/v2/pkg/f1/testing" + "context" + + "github.com/form3tech-oss/f1/v3/pkg/f1/f1testing" ) // CombineScenarios creates a single scenario that will call each ScenarioFn -// sequentially and return a testing.RunFn that will call each scenario's RunFn +// sequentially and return a f1testing.RunFn that will call each scenario's RunFn // every iteration. -func CombineScenarios(scenarios ...testing.ScenarioFn) testing.ScenarioFn { - return func(t *testing.T) testing.RunFn { - run := make([]testing.RunFn, 0, len(scenarios)) +func CombineScenarios(scenarios ...f1testing.ScenarioFn) f1testing.ScenarioFn { + return func(ctx context.Context, t *f1testing.T) f1testing.RunFn { + run := make([]f1testing.RunFn, 0, len(scenarios)) for _, s := range scenarios { - run = append(run, s(t)) + run = append(run, s(ctx, t)) } - return func(t *testing.T) { + return func(ctx context.Context, t *f1testing.T) { for _, r := range run { - r(t) + r(ctx, t) } } } diff --git a/pkg/f1/f1_scenarios_stage_test.go b/pkg/f1/f1_scenarios_stage_test.go index 476e0b16..0bb9fe77 100644 --- a/pkg/f1/f1_scenarios_stage_test.go +++ b/pkg/f1/f1_scenarios_stage_test.go @@ -1,14 +1,15 @@ package f1_test import ( + "context" "sync/atomic" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/form3tech-oss/f1/v2/pkg/f1" - f1_testing "github.com/form3tech-oss/f1/v2/pkg/f1/testing" + "github.com/form3tech-oss/f1/v3/pkg/f1" + "github.com/form3tech-oss/f1/v3/pkg/f1/f1testing" ) type f1ScenariosStage struct { @@ -22,10 +23,10 @@ type scenario struct { iterations atomic.Uint32 } -func (s *scenario) scenariofn(*f1_testing.T) f1_testing.RunFn { +func (s *scenario) scenariofn(context.Context, *f1testing.T) f1testing.RunFn { s.setups.Add(1) - return func(*f1_testing.T) { + return func(context.Context, *f1testing.T) { s.iterations.Add(1) } } @@ -54,17 +55,17 @@ func newF1ScenarioStage(t *testing.T) (*f1ScenariosStage, *f1ScenariosStage, *f1 } func (s *f1ScenariosStage) f1_is_configured_to_run_a_combined_scenario() { - scenarios := make([]f1_testing.ScenarioFn, len(s.scenarios)) + scenarios := make([]f1testing.ScenarioFn, len(s.scenarios)) for i, scn := range s.scenarios { fn := scn.scenariofn scenarios[i] = fn } - s.runner = f1.New().Add("combined", f1.CombineScenarios(scenarios...)) + s.runner = f1.New().AddScenario("combined", f1.CombineScenarios(scenarios...)) } func (s *f1ScenariosStage) the_f1_scenario_is_executed() { - err := s.runner.ExecuteWithArgs([]string{ + err := s.runner.Run(context.TODO(), []string{ "run", "constant", "combined", "--rate", "5/1s", "--max-duration", "1s", diff --git a/pkg/f1/f1_stage_test.go b/pkg/f1/f1_stage_test.go index f23da32e..403c1789 100644 --- a/pkg/f1/f1_stage_test.go +++ b/pkg/f1/f1_stage_test.go @@ -2,6 +2,7 @@ package f1_test import ( "bytes" + "context" "fmt" "os" "strings" @@ -14,9 +15,9 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/goleak" - "github.com/form3tech-oss/f1/v2/internal/log" - "github.com/form3tech-oss/f1/v2/pkg/f1" - f1_testing "github.com/form3tech-oss/f1/v2/pkg/f1/testing" + "github.com/form3tech-oss/f1/v3/internal/log" + "github.com/form3tech-oss/f1/v3/pkg/f1" + "github.com/form3tech-oss/f1/v3/pkg/f1/f1testing" ) type f1Stage struct { @@ -51,7 +52,7 @@ func (s *f1Stage) and() *f1Stage { func (s *f1Stage) a_custom_logger_is_configured_with_attr(key, value string) *f1Stage { logger := log.NewTestLogger(&s.logOutput).With(key, value) - s.f1 = f1.New().WithLogger(logger) + s.f1 = f1.New(f1.WithLogger(logger)) return s } @@ -74,8 +75,8 @@ func (s *f1Stage) after_duration_signal_will_be_sent(duration time.Duration, sig func (s *f1Stage) a_scenario_where_each_iteration_takes(duration time.Duration) *f1Stage { s.scenario = "scenario_where_each_iteration_takes_" + duration.String() - s.f1.Add(s.scenario, func(*f1_testing.T) f1_testing.RunFn { - return func(*f1_testing.T) { + s.f1.AddScenario(s.scenario, func(context.Context, *f1testing.T) f1testing.RunFn { + return func(context.Context, *f1testing.T) { s.runCount.Add(1) time.Sleep(duration) } @@ -86,12 +87,12 @@ func (s *f1Stage) a_scenario_where_each_iteration_takes(duration time.Duration) func (s *f1Stage) a_scenario_that_logs() *f1Stage { s.scenario = "logging_scenario" - s.f1.Add(s.scenario, func(sceanrioT *f1_testing.T) f1_testing.RunFn { - sceanrioT.Log("scenario") + s.f1.AddScenario(s.scenario, func(_ context.Context, scenarioT *f1testing.T) f1testing.RunFn { + scenarioT.Log("scenario") - return func(*f1_testing.T) { - sceanrioT.Log("iteration") - sceanrioT.Logger().Info("iteration") + return func(_ context.Context, t *f1testing.T) { + t.Log("iteration") + t.Logger().Info("iteration") } }) @@ -99,7 +100,7 @@ func (s *f1Stage) a_scenario_that_logs() *f1Stage { } func (s *f1Stage) the_f1_scenario_is_executed_with_constant_rate_and_args(args ...string) *f1Stage { - err := s.f1.ExecuteWithArgs(append([]string{ + err := s.f1.Run(context.TODO(), append([]string{ "run", "constant", s.scenario, }, args...)) s.require.NoError(err, "error executing scenarios") @@ -108,7 +109,7 @@ func (s *f1Stage) the_f1_scenario_is_executed_with_constant_rate_and_args(args . } func (s *f1Stage) an_unknown_f1_scenario_is_executed() *f1Stage { - s.executeErr = s.f1.ExecuteWithArgs([]string{ + s.executeErr = s.f1.Run(context.TODO(), []string{ "run", "constant", "unknownScenario", }) diff --git a/pkg/f1/f1_test.go b/pkg/f1/f1_test.go index 1e6a96ce..5ce9fd78 100644 --- a/pkg/f1/f1_test.go +++ b/pkg/f1/f1_test.go @@ -5,6 +5,8 @@ import ( "syscall" "testing" "time" + + "github.com/stretchr/testify/require" ) func TestSignalHandling(t *testing.T) { @@ -46,6 +48,17 @@ func TestMissingScenario(t *testing.T) { the_execute_command_returns_an_error("scenario not defined: unknownScenario") } +func TestEnvVarsUsedByDefault(t *testing.T) { + ts, count := newPushGatewayServer(t) + t.Setenv("PROMETHEUS_PUSH_GATEWAY", ts.URL) + + inst := newF1WithScenario("env_default") + runConstant(t, inst, "env_default") + + require.Positive(t, count.Load(), + "PROMETHEUS_PUSH_GATEWAY env var should trigger metrics push") +} + func TestWithCustomLogger(t *testing.T) { given, when, then := newF1Stage(t) diff --git a/pkg/f1/f1testing/api.go b/pkg/f1/f1testing/api.go new file mode 100644 index 00000000..1a77bc14 --- /dev/null +++ b/pkg/f1/f1testing/api.go @@ -0,0 +1,17 @@ +package f1testing + +import "context" + +// ScenarioFn initialises a scenario and returns the iteration function (RunFn) to be invoked for every iteration +// of the tests. +// +// ctx is cancelled when the run is interrupted (SIGINT/SIGTERM), times out (--max-duration), or reaches +// max iterations. Pass it to context-aware operations or check ctx.Done() to abort long-running setup. +type ScenarioFn func(ctx context.Context, t *T) RunFn + +// RunFn performs a single iteration of the scenario. 't' may be used for asserting +// results or failing the scenario. +// +// ctx is cancelled when the run is stopped. Pass it to context-aware operations or check ctx.Done() +// to exit early when the user interrupts. +type RunFn func(ctx context.Context, t *T) diff --git a/pkg/f1/f1testing/doc.go b/pkg/f1/f1testing/doc.go new file mode 100644 index 00000000..a5d72a28 --- /dev/null +++ b/pkg/f1/f1testing/doc.go @@ -0,0 +1,11 @@ +/* +Package f1testing provides the scenario execution context, analogous to Go's testing package. +It provides a T type which is injected into setup and iteration run functions, with common +functionality such as assertions and cleanup. + +Both ScenarioFn and RunFn receive a context.Context as their first parameter. The context +is cancelled when the run is interrupted (SIGINT/SIGTERM), times out (--max-duration), or +reaches max iterations. Pass it to context-aware operations or check ctx.Done() to abort +long-running work when the run stops. +*/ +package f1testing diff --git a/pkg/f1/testing/t.go b/pkg/f1/f1testing/t.go similarity index 65% rename from pkg/f1/testing/t.go rename to pkg/f1/f1testing/t.go index aa470081..35b210d0 100644 --- a/pkg/f1/testing/t.go +++ b/pkg/f1/f1testing/t.go @@ -1,33 +1,35 @@ -package testing +package f1testing import ( "errors" "fmt" "log/slog" "runtime/debug" + "strings" "sync/atomic" - "time" - "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" - "github.com/form3tech-oss/f1/v2/internal/log" - "github.com/form3tech-oss/f1/v2/internal/metrics" + "github.com/form3tech-oss/f1/v3/internal/log" ) var errFailNow = errors.New("FailNow") +// IterationSetup is the value of T.Iteration during the setup phase. +// Iteration numbers from the run phase are 1-based, so 0 is never used for iterations. +const IterationSetup uint64 = 0 + // T is a type passed to Scenario functions to manage test state and support formatted test logs. A // test ends when its Scenario function returns or calls any of the methods FailNow, Fatal, Fatalf. // Those methods must be called only from the goroutine running the Scenario function. The other // reporting methods, such as the variations of Log and Error, may be called simultaneously from // multiple goroutines. type T struct { - logrusLogger *logrus.Logger - logger *slog.Logger - require *require.Assertions - Iteration string // iteration number or "setup" - Scenario string + logger *slog.Logger + require *require.Assertions + // Iteration is the iteration index (1-based) or IterationSetup (0) for the setup phase. + Iteration uint64 + Scenario string // VUID is the Virtual User ID - a stable identifier for the pool worker running this iteration. // Useful for correlating iterations with user-specific test data (e.g. in the "users" trigger mode). // VUID is -1 for setup; 0-based for pool workers. @@ -40,22 +42,13 @@ type T struct { type TOption func(*T) -// WithLogrusLogger will be removed in future versions, needed for backwards compatibility -// -// Deprecated: Will be removed in future versions. -func WithLogrusLogger(logrusLogger *logrus.Logger) TOption { - return func(t *T) { - t.logrusLogger = logrusLogger - } -} - func WithLogger(logger *slog.Logger) TOption { return func(t *T) { t.logger = logger } } -func WithIteration(iteration string) TOption { +func WithIteration(iteration uint64) TOption { return func(t *T) { t.Iteration = iteration } @@ -69,21 +62,6 @@ func WithVUID(id int) TOption { } } -// NewT returns a new T state -// -// Deprecated: Will be removed in favour of NewTWithOptions -func NewT(iter, scenarioName string) (*T, func()) { - logger := slog.Default() - - t, teardown := NewTWithOptions(scenarioName, - WithIteration(iter), - WithLogrusLogger(log.NewSlogLogrusLogger(logger)), - WithLogger(logger), - ) - - return t, teardown -} - func NewTWithOptions(scenarioName string, options ...TOption) (*T, func()) { t := &T{ Scenario: scenarioName, @@ -94,11 +72,14 @@ func NewTWithOptions(scenarioName string, options ...TOption) (*T, func()) { for _, opt := range options { opt(t) } + if t.logger == nil { + t.logger = slog.Default() + } return t, t.teardown } -func (t *T) Reset(iter string) { +func (t *T) Reset(iter uint64) { t.Iteration = iter t.failed.Store(false) t.teardownFailed.Store(false) @@ -106,17 +87,7 @@ func (t *T) Reset(iter string) { t.teardownStack = []func(){} } -// Logger returns a logrus logger, needed for backwards compatibility. Use StandardLogger -// instead. -// -// Internally it uses slog as a logging backend. -// -// Deprecated: logrus will be removed in future versions. -func (t *T) Logger() *logrus.Logger { - return t.logrusLogger -} - -func (t *T) StandardLogger() *slog.Logger { +func (t *T) Logger() *slog.Logger { return t.logger } @@ -152,40 +123,45 @@ func (t *T) Fail() { } } -// Errorf is equivalent to Logf followed by Fail. +// Errorf is equivalent to Logf followed by Fail. Logs at Error level. func (t *T) Errorf(format string, args ...any) { - t.logger.Error(fmt.Sprintf(format, args...)) + t.logger.With(log.IterationAttr(t.Iteration), log.VUIDAttr(t.VUID)).Error(fmt.Sprintf(format, args...)) t.Fail() } -// Error is equivalent to Log followed by Fail. -func (t *T) Error(err error) { - t.logger.Error("iteration failed", log.IterationAttr(t.Iteration), log.VUIDAttr(t.VUID), log.ErrorAttr(err)) +// Error is equivalent to Log followed by Fail. Logs at Error level. +func (t *T) Error(args ...any) { + msg := strings.TrimSuffix(fmt.Sprintln(args...), "\n") + t.logger.With(log.IterationAttr(t.Iteration), log.VUIDAttr(t.VUID)).Error(msg) t.Fail() } -// Fatalf is equivalent to Logf followed by FailNow. +// Fatalf is equivalent to Logf followed by FailNow. Logs at Error level. func (t *T) Fatalf(format string, args ...any) { - t.logger.Error(fmt.Sprintf(format, args...)) + t.logger.With(log.IterationAttr(t.Iteration), log.VUIDAttr(t.VUID)).Error(fmt.Sprintf(format, args...)) t.FailNow() } -// Fatal is equivalent to Log followed by FailNow. -func (t *T) Fatal(err error) { - t.logger.Error("iteration failed", log.IterationAttr(t.Iteration), log.VUIDAttr(t.VUID), log.ErrorAttr(err)) +// Fatal is equivalent to Log followed by FailNow. Logs at Error level. +func (t *T) Fatal(args ...any) { + msg := strings.TrimSuffix(fmt.Sprintln(args...), "\n") + t.logger.With(log.IterationAttr(t.Iteration), log.VUIDAttr(t.VUID)).Error(msg) t.FailNow() } // Log formats its arguments using default formatting, analogous to Println, and records the text in the error log. // The text will be printed only if f1 is running in verbose mode. +// Aligns with testing.T: uses fmt.Sprintln for space-separated args (trailing newline trimmed for structured logs). func (t *T) Log(args ...any) { - t.logger.Info(fmt.Sprint(args...)) + msg := strings.TrimSuffix(fmt.Sprintln(args...), "\n") + t.logger.With(log.IterationAttr(t.Iteration), log.VUIDAttr(t.VUID)).Info(msg) } // Logf formats its arguments according to the format, analogous to Printf, and records the text in the error log. // A final newline is added if not provided. The text will be printed only if f1 is running in verbose mode. +// Aligns with testing.T: uses fmt.Sprintf. func (t *T) Logf(format string, args ...any) { - t.logger.Info(fmt.Sprintf(format, args...)) + t.logger.With(log.IterationAttr(t.Iteration), log.VUIDAttr(t.VUID)).Info(fmt.Sprintf(format, args...)) } // Failed reports whether the function has failed. @@ -197,13 +173,6 @@ func (t *T) TeardownFailed() bool { return t.teardownFailed.Load() } -// Time records a metric for the duration of the given function -func (t *T) Time(stageName string, f func()) { - start := time.Now() - defer recordTime(t, stageName, start) - f() -} - // Cleanup registers a function to be called when the scenario or the iteration completes. // Cleanup functions will be called in last added, first called order. func (t *T) Cleanup(f func()) { @@ -258,12 +227,3 @@ func (t *T) teardown() { }() } } - -func recordTime(t *T, stageName string, start time.Time) { - metrics.Instance().RecordIterationStage( - t.Scenario, - stageName, - metrics.Result(t.Failed()), - time.Since(start).Nanoseconds(), - ) -} diff --git a/pkg/f1/f1testing/t_test.go b/pkg/f1/f1testing/t_test.go new file mode 100644 index 00000000..b9e75478 --- /dev/null +++ b/pkg/f1/f1testing/t_test.go @@ -0,0 +1,382 @@ +package f1testing_test + +import ( + "bytes" + "encoding/json" + "errors" + "log/slog" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/form3tech-oss/f1/v3/internal/log" + "github.com/form3tech-oss/f1/v3/pkg/f1/f1testing" +) + +// commonTInterface is the subset of methods that f1testing.T and testing.T share with identical +// signatures. This compile-time check ensures f1testing.T stays compatible with testing.T for +// these methods, enabling users to share test helpers between f1 scenarios and standard go tests. +// The interface is test-only and not exposed in the package. +var ( + _ commonTInterface = (*f1testing.T)(nil) + _ commonTInterface = (*testing.T)(nil) +) + +//nolint:interfacebloat,inamedparam // Single interface for compile-time verification; Cleanup matches testing.T signature +type commonTInterface interface { + Cleanup(func()) + Error(args ...any) + Errorf(format string, args ...any) + Fail() + FailNow() + Failed() bool + Fatal(args ...any) + Fatalf(format string, args ...any) + Log(args ...any) + Logf(format string, args ...any) + Name() string +} + +func parseJSONLogLine(t *testing.T, line string) map[string]any { + t.Helper() + var m map[string]any + require.NoError(t, json.Unmarshal([]byte(line), &m)) + return m +} + +func assertLogFormat(t *testing.T, line string, wantLevel, wantMsg string, wantIteration float64, wantVUID float64) { + t.Helper() + m := parseJSONLogLine(t, line) + require.Contains(t, m, "time", "log must have time field") + require.Contains(t, m, "vuid", "log must have vuid field") + require.Equal(t, wantLevel, m["level"], "level must match") + require.Equal(t, wantMsg, m["msg"], "msg must match") + iter, ok := m["iteration"].(float64) + require.True(t, ok, "iteration must be float64 (JSON number)") + require.InDelta(t, wantIteration, iter, 0, "iteration must match") + vuid, ok := m["vuid"].(float64) + require.True(t, ok, "vuid must be float64 (JSON number)") + require.InDelta(t, wantVUID, vuid, 0, "vuid must match") +} + +func TestNewTIsNotFailed(t *testing.T) { + t.Parallel() + + newT, teardown := newT() + defer teardown() + + require.False(t, newT.Failed()) +} + +func TestReportsPanicReasonWhenCleanupFails(t *testing.T) { + t.Parallel() + + var buf bytes.Buffer + logger := slog.New(slog.NewTextHandler(&buf, nil)) + newT, teardown := f1testing.NewTWithOptions("test", f1testing.WithLogger(logger)) + newT.Cleanup(func() { + panic("boom") + }) + teardown() + require.Contains(t, buf.String(), "recovered panic in scenario") +} + +func TestReportsErrorMessageWhenCleanupFails(t *testing.T) { + t.Parallel() + + var buf bytes.Buffer + logger := slog.New(slog.NewTextHandler(&buf, nil)) + newT, teardown := f1testing.NewTWithOptions("test", f1testing.WithLogger(logger)) + newT.Cleanup(func() { + panic(errors.New("boom")) + }) + teardown() + logs := buf.String() + require.Contains(t, logs, "recovered panic in scenario") + require.Regexp(t, "stack_trace=\"goroutine", logs) +} + +func TestCleanupCalledInReverseOrder(t *testing.T) { + t.Parallel() + + var actual []int + newT, teardown := newT() + + newT.Cleanup(func() { + actual = append(actual, 1) + }) + + newT.Cleanup(func() { + actual = append(actual, 2) + }) + + teardown() + + expected := []int{2, 1} + require.Equal(t, expected, actual) +} + +func TestFailNowSetsTheFailedState(t *testing.T) { + t.Parallel() + + newT, teardown := newT() + defer teardown() + + done := make(chan struct{}) + go func() { + defer catchPanics(done) + newT.FailNow() + }() + <-done + + require.True(t, newT.Failed()) +} + +func TestFailSetsTheFailedState(t *testing.T) { + t.Parallel() + + newT, teardown := newT() + defer teardown() + + done := make(chan struct{}) + go func() { + defer catchPanics(done) + newT.Fail() + }() + <-done + + require.True(t, newT.Failed()) +} + +func TestError(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + args []any + wantMsg string + wantIteration float64 + wantVUID float64 + }{ + "error argument": { + args: []any{errors.New("boom")}, + wantMsg: "boom", + wantIteration: 0, + wantVUID: 0, + }, + "no arguments": { + args: []any{}, + wantMsg: "", + wantIteration: 0, + wantVUID: 0, + }, + "multiple arguments": { + args: []any{"expected", 42, "got", 0}, + wantMsg: "expected 42 got 0", + wantIteration: 0, + wantVUID: 0, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + var buf bytes.Buffer + logger := slog.New(slog.NewJSONHandler(&buf, nil)) + newT, teardown := f1testing.NewTWithOptions("test", + f1testing.WithIteration(uint64(tc.wantIteration)), + f1testing.WithVUID(int(tc.wantVUID)), + f1testing.WithLogger(logger), + ) + defer teardown() + + newT.Error(tc.args...) + require.True(t, newT.Failed()) + assertLogFormat(t, strings.TrimSpace(buf.String()), "ERROR", tc.wantMsg, tc.wantIteration, tc.wantVUID) + }) + } +} + +func TestErrorf(t *testing.T) { + t.Parallel() + + var buf bytes.Buffer + logger := slog.New(slog.NewJSONHandler(&buf, nil)) + newT, teardown := f1testing.NewTWithOptions("test", + f1testing.WithIteration(0), + f1testing.WithVUID(0), + f1testing.WithLogger(logger), + ) + defer teardown() + + newT.Errorf("got %d errors", 3) + require.True(t, newT.Failed()) + assertLogFormat(t, strings.TrimSpace(buf.String()), "ERROR", "got 3 errors", 0, 0) +} + +func TestFatal(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + args []any + wantMsg string + wantIteration float64 + wantVUID float64 + }{ + "error argument": { + args: []any{errors.New("boom")}, + wantMsg: "boom", + wantIteration: 0, + wantVUID: 0, + }, + "no arguments": { + args: []any{}, + wantMsg: "", + wantIteration: 0, + wantVUID: 0, + }, + "multiple arguments": { + args: []any{"boom", 1, 2.0}, + wantMsg: "boom 1 2", + wantIteration: 0, + wantVUID: 0, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + var buf bytes.Buffer + logger := slog.New(slog.NewJSONHandler(&buf, nil)) + newT, teardown := f1testing.NewTWithOptions("test", + f1testing.WithIteration(uint64(tc.wantIteration)), + f1testing.WithVUID(int(tc.wantVUID)), + f1testing.WithLogger(logger), + ) + defer teardown() + + done := make(chan struct{}) + go func() { + defer catchPanics(done) + newT.Fatal(tc.args...) + }() + <-done + + require.True(t, newT.Failed()) + assertLogFormat(t, strings.TrimSpace(buf.String()), "ERROR", tc.wantMsg, tc.wantIteration, tc.wantVUID) + }) + } +} + +func TestFatalf(t *testing.T) { + t.Parallel() + + var buf bytes.Buffer + logger := slog.New(slog.NewJSONHandler(&buf, nil)) + newT, teardown := f1testing.NewTWithOptions("test", + f1testing.WithIteration(0), + f1testing.WithVUID(0), + f1testing.WithLogger(logger), + ) + defer teardown() + + done := make(chan struct{}) + go func() { + defer catchPanics(done) + newT.Fatalf("fatal: %s", "boom") + }() + <-done + + require.True(t, newT.Failed()) + assertLogFormat(t, strings.TrimSpace(buf.String()), "ERROR", "fatal: boom", 0, 0) +} + +func TestNameReturnsScenarioName(t *testing.T) { + t.Parallel() + + newT, teardown := newT() + defer teardown() + + require.Equal(t, "test", newT.Name()) +} + +func TestWithVUIDSetsVirtualUserID(t *testing.T) { + t.Parallel() + + newT, teardown := f1testing.NewTWithOptions("test", f1testing.WithVUID(42)) + defer teardown() + + require.Equal(t, 42, newT.VUID) +} + +func TestLog(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + call func(*f1testing.T) + wantMsg string + wantIteration float64 + wantVUID float64 + }{ + "single argument": { + call: func(t *f1testing.T) { t.Log("info message") }, + wantMsg: "info message", + wantIteration: 0, + wantVUID: 0, + }, + "multiple arguments": { + call: func(t *f1testing.T) { t.Log("step", 1, "of", 3) }, + wantMsg: "step 1 of 3", + wantIteration: 0, + wantVUID: 0, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + var buf bytes.Buffer + logger := slog.New(slog.NewJSONHandler(&buf, nil)) + newT, teardown := f1testing.NewTWithOptions("test", + f1testing.WithIteration(uint64(tc.wantIteration)), + f1testing.WithVUID(int(tc.wantVUID)), + f1testing.WithLogger(logger), + ) + defer teardown() + + tc.call(newT) + assertLogFormat(t, strings.TrimSpace(buf.String()), "INFO", tc.wantMsg, tc.wantIteration, tc.wantVUID) + }) + } +} + +func TestLogf(t *testing.T) { + t.Parallel() + + var buf bytes.Buffer + logger := slog.New(slog.NewJSONHandler(&buf, nil)) + newT, teardown := f1testing.NewTWithOptions("test", + f1testing.WithIteration(0), + f1testing.WithVUID(0), + f1testing.WithLogger(logger), + ) + defer teardown() + + newT.Logf("progress: %d%%", 50) + assertLogFormat(t, strings.TrimSpace(buf.String()), "INFO", "progress: 50%", 0, 0) +} + +func catchPanics(done chan<- struct{}) { + _ = recover() + close(done) +} + +func newT() (*f1testing.T, func()) { + logger := log.NewDiscardLogger() + + return f1testing.NewTWithOptions( + "test", + f1testing.WithIteration(0), + f1testing.WithLogger(logger), + ) +} diff --git a/pkg/f1/metrics/metrics.go b/pkg/f1/metrics/metrics.go deleted file mode 100644 index eec42f38..00000000 --- a/pkg/f1/metrics/metrics.go +++ /dev/null @@ -1,10 +0,0 @@ -package metrics - -import ( - internal_metrics "github.com/form3tech-oss/f1/v2/internal/metrics" -) - -// Deprecated: internal metrics will not be exposed in future versions -func GetMetrics() *internal_metrics.Metrics { - return internal_metrics.Instance() -} diff --git a/pkg/f1/options.go b/pkg/f1/options.go new file mode 100644 index 00000000..957c41f6 --- /dev/null +++ b/pkg/f1/options.go @@ -0,0 +1,87 @@ +package f1 + +import ( + "log/slog" + + "github.com/form3tech-oss/f1/v3/internal/ui" +) + +// Option configures an F1 instance at construction. +type Option func(*F1) + +// WithSettings replaces the settings baseline entirely. By default, settings +// are loaded from environment variables (see DefaultSettings). Pass Settings{} +// to start from zero values and ignore all environment variables. +// +// Individual field options (WithLogLevel, WithPrometheusPushGateway, etc.) +// still apply when placed after WithSettings in the option list. +func WithSettings(s Settings) Option { + return func(f *F1) { + f.settings = s + } +} + +// WithLogger specifies the logger for internal and scenario logs. +// When used, logging settings (WithLogLevel, WithLogFormat, F1_LOG_LEVEL, +// F1_LOG_FORMAT) have no effect because the caller controls the logger. +func WithLogger(logger *slog.Logger) Option { + return func(f *F1) { + f.options.output = ui.NewDefaultOutputWithLogger(logger) + f.options.loggerExplicit = true + } +} + +// WithStaticMetrics registers additional labels with fixed values for f1 metrics. +func WithStaticMetrics(labels map[string]string) Option { + return func(f *F1) { + f.options.staticMetrics = labels + } +} + +// WithPrometheusPushGateway sets the Prometheus push gateway URL, +// overriding the PROMETHEUS_PUSH_GATEWAY environment variable. +func WithPrometheusPushGateway(url string) Option { + return func(f *F1) { + f.settings.Prometheus.PushGateway = url + } +} + +// WithPrometheusNamespace sets the Prometheus namespace label, +// overriding the PROMETHEUS_NAMESPACE environment variable. +func WithPrometheusNamespace(ns string) Option { + return func(f *F1) { + f.settings.Prometheus.Namespace = ns + } +} + +// WithPrometheusLabelID sets the Prometheus label ID, +// overriding the PROMETHEUS_LABEL_ID environment variable. +func WithPrometheusLabelID(id string) Option { + return func(f *F1) { + f.settings.Prometheus.LabelID = id + } +} + +// WithLogFilePath sets the log file path, +// overriding the LOG_FILE_PATH environment variable. +func WithLogFilePath(path string) Option { + return func(f *F1) { + f.settings.Logging.FilePath = path + } +} + +// WithLogLevel sets the log level for the default logger. +// Has no effect when WithLogger is also used. +func WithLogLevel(level slog.Level) Option { + return func(f *F1) { + f.settings.Logging.Level = level + } +} + +// WithLogFormat sets the log output format for the default logger. +// Has no effect when WithLogger is also used. +func WithLogFormat(format LogFormat) Option { + return func(f *F1) { + f.settings.Logging.Format = format + } +} diff --git a/pkg/f1/options_test.go b/pkg/f1/options_test.go new file mode 100644 index 00000000..cfb7f8d9 --- /dev/null +++ b/pkg/f1/options_test.go @@ -0,0 +1,260 @@ +package f1_test + +import ( + "bytes" + "context" + "log/slog" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/form3tech-oss/f1/v3/internal/log" + "github.com/form3tech-oss/f1/v3/pkg/f1" + "github.com/form3tech-oss/f1/v3/pkg/f1/f1testing" +) + +func newPushGatewayServer(t *testing.T) (*httptest.Server, *atomic.Int32) { + t.Helper() + + var count atomic.Int32 + ts := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) { + count.Add(1) + })) + t.Cleanup(ts.Close) + + return ts, &count +} + +func newF1WithScenario(name string, opts ...f1.Option) *f1.F1 { + inst := f1.New(opts...) + inst.AddScenario(name, func(_ context.Context, _ *f1testing.T) f1testing.RunFn { + return func(_ context.Context, _ *f1testing.T) {} + }) + + return inst +} + +func runConstant(t *testing.T, inst *f1.F1, scenario string) { + t.Helper() + + err := inst.Run(context.Background(), []string{ + "run", "constant", scenario, + "--rate", "1/1s", + "--max-duration", "1s", + "--max-iterations", "1", + }) + require.NoError(t, err) +} + +func TestWithSettingsReplacesPushGateway(t *testing.T) { + t.Parallel() + + ts, count := newPushGatewayServer(t) + inst := newF1WithScenario("settings_push", + f1.WithSettings(f1.Settings{ + Prometheus: f1.PrometheusSettings{PushGateway: ts.URL}, + }), + ) + runConstant(t, inst, "settings_push") + + require.Positive(t, count.Load(), + "WithSettings should configure push gateway without env vars") +} + +func TestWithSettingsEmptyDisablesAllSettings(t *testing.T) { + t.Parallel() + + ts, count := newPushGatewayServer(t) + _ = ts + + inst := newF1WithScenario("empty_settings", f1.WithSettings(f1.Settings{})) + runConstant(t, inst, "empty_settings") + + require.Equal(t, int32(0), count.Load(), + "WithSettings(Settings{}) should start from zero values; no push gateway") +} + +func TestFineGrainedOverridesAfterWithSettings(t *testing.T) { + t.Parallel() + + ts, count := newPushGatewayServer(t) + inst := newF1WithScenario("fine_grained", + f1.WithSettings(f1.Settings{}), + f1.WithPrometheusPushGateway(ts.URL), + ) + runConstant(t, inst, "fine_grained") + + require.Positive(t, count.Load(), + "fine-grained options should apply after WithSettings") +} + +func TestWithSettingsOverridesFinegrained(t *testing.T) { + t.Parallel() + + ts, count := newPushGatewayServer(t) + + inst := newF1WithScenario("settings_last", + f1.WithPrometheusPushGateway(ts.URL), + f1.WithSettings(f1.Settings{}), + ) + runConstant(t, inst, "settings_last") + + require.Equal(t, int32(0), count.Load(), + "WithSettings placed after fine-grained options should replace them") +} + +func TestWithLogLevelAndFormat(t *testing.T) { + t.Parallel() + + inst := newF1WithScenario("log_opts", + f1.WithSettings(f1.Settings{}), + f1.WithLogLevel(slog.LevelDebug), + f1.WithLogFormat(f1.LogFormatJSON), + ) + runConstant(t, inst, "log_opts") +} + +func TestWithLoggerTakesPrecedenceOverLogOptions(t *testing.T) { + t.Parallel() + + var buf bytes.Buffer + logger := log.NewTestLogger(&buf) + + inst := newF1WithScenario("logger_precedence", + f1.WithLogger(logger), + f1.WithLogLevel(slog.LevelError), + f1.WithLogFormat(f1.LogFormatJSON), + ) + runConstant(t, inst, "logger_precedence") + + output := buf.String() + require.NotEmpty(t, output, "WithLogger's logger should capture output") + require.NotContains(t, output, `"level"`, + "explicit logger format (text) should be used, not JSON from WithLogFormat") +} + +func TestWithLoggerTakesPrecedenceOverWithSettings(t *testing.T) { + t.Parallel() + + var buf bytes.Buffer + logger := log.NewTestLogger(&buf) + + inst := newF1WithScenario("logger_over_settings", + f1.WithSettings(f1.Settings{ + Logging: f1.LoggingSettings{ + Level: slog.LevelError, + Format: f1.LogFormatJSON, + }, + }), + f1.WithLogger(logger), + ) + runConstant(t, inst, "logger_over_settings") + + output := buf.String() + require.NotEmpty(t, output, "WithLogger should capture output") + require.NotContains(t, output, `"level"`, + "WithLogger text format should override Settings JSON format") +} + +func TestWithSettingsAllFields(t *testing.T) { + t.Parallel() + + ts, count := newPushGatewayServer(t) + inst := newF1WithScenario("all_fields", + f1.WithSettings(f1.Settings{ + Prometheus: f1.PrometheusSettings{ + PushGateway: ts.URL, + Namespace: "test-ns", + LabelID: "test-id", + }, + Logging: f1.LoggingSettings{ + Level: slog.LevelDebug, + Format: f1.LogFormatJSON, + }, + }), + ) + runConstant(t, inst, "all_fields") + + require.Positive(t, count.Load(), + "WithSettings with PushGateway should trigger metrics push") +} + +func TestParseLogLevel(t *testing.T) { + t.Parallel() + + tests := []struct { + input string + want slog.Level + }{ + {"debug", slog.LevelDebug}, + {"DEBUG", slog.LevelDebug}, + {"trace", slog.LevelDebug}, + {"info", slog.LevelInfo}, + {"INFO", slog.LevelInfo}, + {"", slog.LevelInfo}, + {"warn", slog.LevelWarn}, + {"warning", slog.LevelWarn}, + {"error", slog.LevelError}, + {"fatal", slog.LevelError}, + {"panic", slog.LevelError}, + } + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + t.Parallel() + + got, err := f1.ParseLogLevel(tt.input) + require.NoError(t, err) + require.Equal(t, tt.want, got) + }) + } +} + +func TestParseLogLevelInvalid(t *testing.T) { + t.Parallel() + + _, err := f1.ParseLogLevel("invalid") + require.Error(t, err) + require.ErrorContains(t, err, "unknown log level") +} + +func TestParseLogFormat(t *testing.T) { + t.Parallel() + + tests := []struct { + input string + want f1.LogFormat + }{ + {"text", f1.LogFormatText}, + {"TEXT", f1.LogFormatText}, + {"", f1.LogFormatText}, + {"json", f1.LogFormatJSON}, + {"JSON", f1.LogFormatJSON}, + } + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + t.Parallel() + + got, err := f1.ParseLogFormat(tt.input) + require.NoError(t, err) + require.Equal(t, tt.want, got) + }) + } +} + +func TestParseLogFormatInvalid(t *testing.T) { + t.Parallel() + + _, err := f1.ParseLogFormat("yaml") + require.Error(t, err) + require.ErrorContains(t, err, "unknown log format") +} + +func TestLogFormatString(t *testing.T) { + t.Parallel() + + require.Equal(t, "text", f1.LogFormatText.String()) + require.Equal(t, "json", f1.LogFormatJSON.String()) +} diff --git a/pkg/f1/root_cmd.go b/pkg/f1/root_cmd.go index 5393271c..4b9fc6b6 100644 --- a/pkg/f1/root_cmd.go +++ b/pkg/f1/root_cmd.go @@ -1,27 +1,29 @@ package f1 import ( + "context" "fmt" "os" "path" + "github.com/prometheus/client_golang/prometheus" "github.com/spf13/cobra" - "github.com/form3tech-oss/f1/v2/internal/chart" - "github.com/form3tech-oss/f1/v2/internal/envsettings" - "github.com/form3tech-oss/f1/v2/internal/metrics" - "github.com/form3tech-oss/f1/v2/internal/run" - "github.com/form3tech-oss/f1/v2/internal/trigger" - "github.com/form3tech-oss/f1/v2/internal/ui" - "github.com/form3tech-oss/f1/v2/pkg/f1/scenarios" + "github.com/form3tech-oss/f1/v3/internal/envsettings" + "github.com/form3tech-oss/f1/v3/internal/metrics" + "github.com/form3tech-oss/f1/v3/internal/run" + "github.com/form3tech-oss/f1/v3/internal/trigger" + "github.com/form3tech-oss/f1/v3/internal/ui" + "github.com/form3tech-oss/f1/v3/pkg/f1/scenarios" ) const ( - flagCPUProfile = "cpuprofile" - flagMemProfile = "memprofile" + flagCPUProfile = "cpu-profile" + flagMemProfile = "mem-profile" ) func buildRootCmd( + ctx context.Context, scenarioList *scenarios.Scenarios, settings envsettings.Settings, p *profiling, @@ -44,19 +46,19 @@ func buildRootCmd( return nil, fmt.Errorf("marking flag as filename: %w", err) } - metrics.InitWithStaticMetrics(settings.PrometheusEnabled(), staticMetrics) - metricsInstance := metrics.Instance() + registry := prometheus.NewRegistry() + metricsInstance := metrics.NewInstance(registry, settings.PrometheusEnabled(), staticMetrics) builders := trigger.GetBuilders(output) rootCmd.AddCommand(run.Cmd( + ctx, scenarioList, builders, settings, metricsInstance, output, )) - rootCmd.AddCommand(chart.Cmd(builders, output)) rootCmd.AddCommand(scenarios.Cmd(scenarioList)) rootCmd.AddCommand(completionsCmd(rootCmd)) return rootCmd, nil diff --git a/pkg/f1/scenarios/scenario_builder.go b/pkg/f1/scenarios/scenario_builder.go index e3ac4b0a..7b523c92 100644 --- a/pkg/f1/scenarios/scenario_builder.go +++ b/pkg/f1/scenarios/scenario_builder.go @@ -3,7 +3,7 @@ package scenarios import ( "sort" - "github.com/form3tech-oss/f1/v2/pkg/f1/testing" + "github.com/form3tech-oss/f1/v3/pkg/f1/f1testing" ) // Scenarios represents a list of test scenarios. @@ -17,9 +17,9 @@ type Scenario struct { Name string Description string Parameters []ScenarioParameter - ScenarioFn testing.ScenarioFn + ScenarioFn f1testing.ScenarioFn // The function that is invoked on each iteration of the test scenario. - RunFn testing.RunFn + RunFn f1testing.RunFn } type ScenarioParameter struct { @@ -30,13 +30,13 @@ type ScenarioParameter struct { type ScenarioOption func(info *Scenario) -func Description(d string) ScenarioOption { +func WithDescription(d string) ScenarioOption { return func(i *Scenario) { i.Description = d } } -func Parameter(parameter ScenarioParameter) ScenarioOption { +func WithParameter(parameter ScenarioParameter) ScenarioOption { return func(i *Scenario) { i.Parameters = append(i.Parameters, parameter) } @@ -48,7 +48,8 @@ func New() *Scenarios { } } -func (s *Scenarios) Add(scenario *Scenario) *Scenarios { +// AddScenario adds a scenario to the collection. +func (s *Scenarios) AddScenario(scenario *Scenario) *Scenarios { s.scenarios[scenario.Name] = scenario return s } diff --git a/pkg/f1/scenarios/scenario_builder_test.go b/pkg/f1/scenarios/scenario_builder_test.go new file mode 100644 index 00000000..305981fe --- /dev/null +++ b/pkg/f1/scenarios/scenario_builder_test.go @@ -0,0 +1,79 @@ +package scenarios_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/form3tech-oss/f1/v3/pkg/f1/f1testing" + "github.com/form3tech-oss/f1/v3/pkg/f1/scenarios" +) + +func TestWithDescription(t *testing.T) { + t.Parallel() + + info := &scenarios.Scenario{Name: "test"} + scenarios.WithDescription("a load test")(info) + require.Equal(t, "a load test", info.Description) +} + +func TestWithParameter(t *testing.T) { + t.Parallel() + + info := &scenarios.Scenario{Name: "test"} + param := scenarios.ScenarioParameter{Name: "rate", Description: "requests per second", Default: "1/s"} + scenarios.WithParameter(param)(info) + require.Len(t, info.Parameters, 1) + require.Equal(t, "rate", info.Parameters[0].Name) + require.Equal(t, "requests per second", info.Parameters[0].Description) + require.Equal(t, "1/s", info.Parameters[0].Default) + + scenarios.WithParameter(scenarios.ScenarioParameter{Name: "duration", Default: "10s"})(info) + require.Len(t, info.Parameters, 2) + require.Equal(t, "duration", info.Parameters[1].Name) +} + +func TestAddScenarioAndGetScenario(t *testing.T) { + t.Parallel() + + s := scenarios.New() + scenario := &scenarios.Scenario{ + Name: "myScenario", + ScenarioFn: func(context.Context, *f1testing.T) f1testing.RunFn { return func(context.Context, *f1testing.T) {} }, + } + + s.AddScenario(scenario) + + got := s.GetScenario("myScenario") + require.NotNil(t, got) + require.Equal(t, "myScenario", got.Name) + require.Nil(t, s.GetScenario("nonexistent")) +} + +func TestGetScenarioNames(t *testing.T) { + t.Parallel() + + s := scenarios.New() + mkScenario := func(name string) *scenarios.Scenario { + return &scenarios.Scenario{ + Name: name, + ScenarioFn: func(context.Context, *f1testing.T) f1testing.RunFn { return func(context.Context, *f1testing.T) {} }, + } + } + + s.AddScenario(mkScenario("zebra")). + AddScenario(mkScenario("alpha")). + AddScenario(mkScenario("beta")) + + names := s.GetScenarioNames() + require.Equal(t, []string{"alpha", "beta", "zebra"}, names) +} + +func TestGetScenarioNamesEmpty(t *testing.T) { + t.Parallel() + + s := scenarios.New() + names := s.GetScenarioNames() + require.Empty(t, names) +} diff --git a/pkg/f1/scenarios/scenarios_cmd.go b/pkg/f1/scenarios/scenarios_cmd.go index 3f52ae3d..5090772b 100644 --- a/pkg/f1/scenarios/scenarios_cmd.go +++ b/pkg/f1/scenarios/scenarios_cmd.go @@ -2,8 +2,6 @@ package scenarios import ( "fmt" - "os" - "sort" "github.com/spf13/cobra" ) @@ -19,19 +17,20 @@ func Cmd(s *Scenarios) *cobra.Command { } func lsCmd(s *Scenarios) *cobra.Command { - lsCmd := &cobra.Command{ - Use: "ls", - Run: lsCmdExecute(s), + return &cobra.Command{ + Use: "ls", + Short: "List available scenario names", + Long: "List all registered scenario names, one per line, sorted alphabetically.", + Run: lsCmdExecute(s), } - return lsCmd } func lsCmdExecute(s *Scenarios) func(*cobra.Command, []string) { - return func(*cobra.Command, []string) { - scenarios := s.GetScenarioNames() - sort.Strings(scenarios) - for _, scenario := range scenarios { - fmt.Fprintln(os.Stdout, scenario) + return func(cmd *cobra.Command, _ []string) { + names := s.GetScenarioNames() + out := cmd.OutOrStdout() + for _, name := range names { + fmt.Fprintln(out, name) } } } diff --git a/pkg/f1/scenarios/scenarios_cmd_test.go b/pkg/f1/scenarios/scenarios_cmd_test.go new file mode 100644 index 00000000..5a9ec204 --- /dev/null +++ b/pkg/f1/scenarios/scenarios_cmd_test.go @@ -0,0 +1,66 @@ +package scenarios_test + +import ( + "bytes" + "context" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/form3tech-oss/f1/v3/pkg/f1/f1testing" + "github.com/form3tech-oss/f1/v3/pkg/f1/scenarios" +) + +func TestScenariosLsOutput(t *testing.T) { + t.Parallel() + + s := scenarios.New() + mkScenario := func(name string) *scenarios.Scenario { + return &scenarios.Scenario{ + Name: name, + ScenarioFn: func(context.Context, *f1testing.T) f1testing.RunFn { return func(context.Context, *f1testing.T) {} }, + } + } + s.AddScenario(mkScenario("zebra")). + AddScenario(mkScenario("alpha")). + AddScenario(mkScenario("beta")) + + cmd := scenarios.Cmd(s) + cmd.SetArgs([]string{"ls"}) + var buf bytes.Buffer + cmd.SetOut(&buf) + + err := cmd.Execute() + require.NoError(t, err) + + output := buf.String() + lines := strings.Split(strings.TrimSuffix(output, "\n"), "\n") + require.Equal(t, []string{"alpha", "beta", "zebra"}, lines, "output should be newline-delimited and sorted") +} + +func TestScenariosLsOutputEmpty(t *testing.T) { + t.Parallel() + + s := scenarios.New() + cmd := scenarios.Cmd(s) + cmd.SetArgs([]string{"ls"}) + var buf bytes.Buffer + cmd.SetOut(&buf) + + err := cmd.Execute() + require.NoError(t, err) + + require.Empty(t, strings.TrimSpace(buf.String()), "empty registry should produce no output") +} + +func TestScenariosLsHelpText(t *testing.T) { + t.Parallel() + + s := scenarios.New() + cmd := scenarios.Cmd(s) + lsCmd := cmd.Commands()[0] + + require.NotEmpty(t, lsCmd.Short, "ls command should have Short help") + require.NotEmpty(t, lsCmd.Long, "ls command should have Long help") +} diff --git a/pkg/f1/settings.go b/pkg/f1/settings.go new file mode 100644 index 00000000..12d8f3ef --- /dev/null +++ b/pkg/f1/settings.go @@ -0,0 +1,157 @@ +package f1 + +import ( + "fmt" + "log/slog" + "strings" + + "github.com/form3tech-oss/f1/v3/internal/envsettings" +) + +// LogFormat specifies the output format for the default logger. +type LogFormat uint8 + +const ( + // LogFormatText selects plain-text log output (default). + LogFormatText LogFormat = iota + // LogFormatJSON selects JSON-structured log output. + LogFormatJSON +) + +const ( + logFormatTextStr = "text" + logFormatJSONStr = "json" + logLevelInfoStr = "info" +) + +// String returns "text" or "json". +func (f LogFormat) String() string { + if f == LogFormatJSON { + return logFormatJSONStr + } + + return logFormatTextStr +} + +// ParseLogLevel parses a log level string into slog.Level. +// Accepted values (case-insensitive): "debug", "info", "warn", "error". +// Also accepts legacy aliases: "trace" (→ Debug), "warning" (→ Warn), +// "fatal"/"panic" (→ Error). Empty string defaults to Info. +func ParseLogLevel(s string) (slog.Level, error) { + switch strings.ToLower(strings.TrimSpace(s)) { + case "debug", "trace": + return slog.LevelDebug, nil + case logLevelInfoStr, "": + return slog.LevelInfo, nil + case "warn", "warning": + return slog.LevelWarn, nil + case "error", "fatal", "panic": + return slog.LevelError, nil + default: + return 0, fmt.Errorf("unknown log level %q: use debug, info, warn, or error", s) + } +} + +// ParseLogFormat parses a log format string into LogFormat. +// Accepted values (case-insensitive): "text", "json". +// Empty string defaults to LogFormatText. +func ParseLogFormat(s string) (LogFormat, error) { + switch strings.ToLower(strings.TrimSpace(s)) { + case logFormatTextStr, "": + return LogFormatText, nil + case logFormatJSONStr: + return LogFormatJSON, nil + default: + return 0, fmt.Errorf("unknown log format %q: use %s or %s", s, logFormatTextStr, logFormatJSONStr) + } +} + +// LoggingSettings configures the default logger built by f1. +// These settings have no effect when WithLogger is used. +type LoggingSettings struct { + FilePath string + Level slog.Level + Format LogFormat +} + +// PrometheusSettings configures Prometheus metrics push. +type PrometheusSettings struct { + PushGateway string + Namespace string + LabelID string +} + +// Settings configures f1 infrastructure (logging, Prometheus). +// Use DefaultSettings to obtain the env-backed baseline, +// or construct a zero-value Settings{} to start from scratch. +type Settings struct { + Prometheus PrometheusSettings + Logging LoggingSettings +} + +// DefaultSettings returns settings loaded from environment variables. +// This is the baseline used by New when no WithSettings option is provided. +// +// Environment variables read: +// +// PROMETHEUS_PUSH_GATEWAY, PROMETHEUS_NAMESPACE, PROMETHEUS_LABEL_ID +// LOG_FILE_PATH, F1_LOG_LEVEL, F1_LOG_FORMAT +func DefaultSettings() Settings { + es := envsettings.Get() + + return Settings{ + Prometheus: PrometheusSettings{ + PushGateway: es.Prometheus.PushGateway, + Namespace: es.Prometheus.Namespace, + LabelID: es.Prometheus.LabelID, + }, + Logging: LoggingSettings{ + FilePath: es.Log.FilePath, + Level: es.Log.SlogLevel(), + Format: logFormatFromEnv(es.Log.Format), + }, + } +} + +func (s Settings) toInternal() envsettings.Settings { + var format string + if s.Logging.Format == LogFormatJSON { + format = logFormatJSONStr + } + + return envsettings.Settings{ + Prometheus: envsettings.Prometheus{ + PushGateway: s.Prometheus.PushGateway, + Namespace: s.Prometheus.Namespace, + LabelID: s.Prometheus.LabelID, + }, + Log: envsettings.Log{ + FilePath: s.Logging.FilePath, + Level: slogLevelToString(s.Logging.Level), + Format: format, + }, + } +} + +func logFormatFromEnv(s string) LogFormat { + if strings.EqualFold(s, logFormatJSONStr) { + return LogFormatJSON + } + + return LogFormatText +} + +func slogLevelToString(level slog.Level) string { + switch level { + case slog.LevelDebug: + return "debug" + case slog.LevelInfo: + return logLevelInfoStr + case slog.LevelWarn: + return "warn" + case slog.LevelError: + return "error" + default: + return logLevelInfoStr + } +} diff --git a/pkg/f1/testing/api.go b/pkg/f1/testing/api.go deleted file mode 100644 index a60dbe79..00000000 --- a/pkg/f1/testing/api.go +++ /dev/null @@ -1,9 +0,0 @@ -package testing - -// ScenarioFn initialises a scenario and returns the iteration function (RunFn) to be invoked for every iteration -// of the tests. -type ScenarioFn func(t *T) RunFn - -// RunFn performs a single iteration of the scenario. 't' may be used for asserting -// results or failing the scenario. -type RunFn func(t *T) diff --git a/pkg/f1/testing/doc.go b/pkg/f1/testing/doc.go deleted file mode 100644 index 6b346cb5..00000000 --- a/pkg/f1/testing/doc.go +++ /dev/null @@ -1,6 +0,0 @@ -/* -Package testing is analogous to Go's built-in testing package. It provides a "t" type which is -injected into setup and iteration run functions, which also provides some common testing -functionality such as the ability to perform assertions. -*/ -package testing diff --git a/pkg/f1/testing/t_test.go b/pkg/f1/testing/t_test.go deleted file mode 100644 index f67b49a8..00000000 --- a/pkg/f1/testing/t_test.go +++ /dev/null @@ -1,214 +0,0 @@ -package testing_test - -import ( - "bytes" - "errors" - "log/slog" - "testing" - - "github.com/stretchr/testify/require" - - "github.com/form3tech-oss/f1/v2/internal/log" - f1testing "github.com/form3tech-oss/f1/v2/pkg/f1/testing" -) - -func TestNewTIsNotFailed(t *testing.T) { - t.Parallel() - - newT, teardown := newT() - defer teardown() - - require.False(t, newT.Failed()) -} - -func TestReportsPanicReasonWhenCleanupFails(t *testing.T) { - t.Parallel() - - var buf bytes.Buffer - logger := slog.New(slog.NewTextHandler(&buf, nil)) - - newT, teardown := f1testing.NewTWithOptions("test", f1testing.WithLogger(logger)) - - newT.Cleanup(func() { - panic("boom") - }) - - teardown() - logs := buf.String() - require.Contains(t, logs, "recovered panic in scenario") -} - -func TestReportsErrorMessageWhenCleanupFails(t *testing.T) { - t.Parallel() - - var buf bytes.Buffer - logger := slog.New(slog.NewTextHandler(&buf, nil)) - - newT, teardown := f1testing.NewTWithOptions("test", f1testing.WithLogger(logger)) - - newT.Cleanup(func() { - panic(errors.New("boom")) - }) - - teardown() - logs := buf.String() - require.Contains(t, logs, "recovered panic in scenario") - require.Regexp(t, "stack_trace=\"goroutine", logs) -} - -func TestCleanupCalledInReverseOrder(t *testing.T) { - t.Parallel() - - var actual []int - newT, teardown := newT() - - newT.Cleanup(func() { - actual = append(actual, 1) - }) - - newT.Cleanup(func() { - actual = append(actual, 2) - }) - - teardown() - - expected := []int{2, 1} - require.Equal(t, expected, actual) -} - -func TestFailNowSetsTheFailedState(t *testing.T) { - t.Parallel() - - newT, teardown := newT() - defer teardown() - - done := make(chan struct{}) - go func() { - defer catchPanics(done) - newT.FailNow() - }() - <-done - - require.True(t, newT.Failed()) -} - -func TestFailSetsTheFailedState(t *testing.T) { - t.Parallel() - - newT, teardown := newT() - defer teardown() - - done := make(chan struct{}) - go func() { - defer catchPanics(done) - newT.Fail() - }() - <-done - - require.True(t, newT.Failed()) -} - -func TestErrorSetsTheFailedState(t *testing.T) { - t.Parallel() - - newT, teardown := newT() - defer teardown() - - newT.Error(errors.New("boom")) - require.True(t, newT.Failed()) -} - -func TestErrorfSetsTheFailedState(t *testing.T) { - t.Parallel() - - newT, teardown := newT() - defer teardown() - - newT.Errorf("boom") - require.True(t, newT.Failed()) -} - -func TestFatalSetsTheFailedState(t *testing.T) { - t.Parallel() - - newT, teardown := newT() - defer teardown() - - done := make(chan struct{}) - go func() { - defer catchPanics(done) - newT.Fatal(errors.New("boom")) - }() - <-done - - require.True(t, newT.Failed()) -} - -func TestFatalfSetsTheFailedState(t *testing.T) { - t.Parallel() - - newT, teardown := newT() - defer teardown() - - done := make(chan struct{}) - go func() { - defer catchPanics(done) - newT.Fatalf("boom") - }() - <-done - - require.True(t, newT.Failed()) -} - -func TestNameReturnsScenarioName(t *testing.T) { - t.Parallel() - - newT, teardown := newT() - defer teardown() - - require.Equal(t, "test", newT.Name()) -} - -func TestWithVUIDSetsVirtualUserID(t *testing.T) { - t.Parallel() - - newT, teardown := f1testing.NewTWithOptions("test", f1testing.WithVUID(42)) - defer teardown() - - require.Equal(t, 42, newT.VUID) -} - -func TestWithVUIDIncludedInErrorLogs(t *testing.T) { - t.Parallel() - - var buf bytes.Buffer - logger := slog.New(slog.NewTextHandler(&buf, nil)) - - newT, teardown := f1testing.NewTWithOptions("test", - f1testing.WithVUID(7), - f1testing.WithLogger(logger), - ) - defer teardown() - - newT.Error(errors.New("test error")) - logs := buf.String() - require.Contains(t, logs, "vuid=7") - require.Contains(t, logs, "test error") -} - -func catchPanics(done chan<- struct{}) { - _ = recover() - close(done) -} - -func newT() (*f1testing.T, func()) { - logger := log.NewDiscardLogger() - logrus := log.NewSlogLogrusLogger(logger) - - return f1testing.NewTWithOptions( - "test", - f1testing.WithIteration("iteration 0"), - f1testing.WithLogger(logger), - f1testing.WithLogrusLogger(logrus), - ) -}