Feature/improvements on using it in other cde

README.md

@@ -82,6 +82,7 @@ go run main.go
 | `min` | Minimum value (numbers, durations) | `min:"1024"` |
 | `max` | Maximum value (numbers, durations) | `max:"65535"` |
 | `pattern` | Regex pattern for strings | `pattern:"^[a-z]+$"` |
+| `scheme` | Command separated list of schemes for `*url.URL` |  `scheme:"http,https"` |
 | `required` | Must be present and non-empty | `required:"true"` |
 | `keyRequired` | Must be present (can be empty) | `keyRequired:"true"` |
 
@@ -172,6 +173,11 @@ export MODEL_PARAMS='{"temperature":0.7,"max_tokens":1000}'
 
 📚 **[JSON Guide](docs/json.md)**
 
+## Troubleshooting
+
+If you see an error about parsing JSON when you are not expecting a JSON value, check that the type is recognized.
+The JSON handling for struct types catches various types (such as `url.URL` before I added support for it)
+
 ## Documentation
 
 - 📖 **[Documentation Index](docs/)** - Complete guides and reference

custom_types.go

@@ -16,6 +16,8 @@ type Wrapper[T any] = readpipeline.Wrapper[T]
 
 type TypedHandler[T any] = readpipeline.TypedHandler[T]
 
+type Transform[T, U any] = customtypes.Transform[T, U]
+
 func RegisterCustomType[T any](handler TypedHandler[T]) {
 	readpipeline.RegisterType[T](handler)
 }
@@ -39,16 +41,39 @@ func AddValidators[T any](baseHandler TypedHandler[T], customValidators ...Valid
 	return baseHandler
 }
 
+// AddDynamicValidation allows a TypedHandler to add validation (or other logic) to the process pipeline dependent
+// on struct tags present on the target field.
+// See the AddValidatorToPipeline function and the example/custom_tags example for more details.
 func AddDynamicValidation[T any](baseHandler TypedHandler[T], wrapper Wrapper[T]) TypedHandler[T] {
 	return customtypes.AddWrapper(baseHandler, wrapper)
 }
 
+// AddValidatorToPipeline adds a validator to a pipeline. This is used as part of pipeline building in the TypedHandler.
+func AddValidatorToPipeline[T any](pipeline FieldProcessor[T], validator Validator[T]) FieldProcessor[T] {
+	return func(rawValue string) (T, error) {
+		value, err := pipeline(rawValue)
+		if err != nil {
+			return value, err
+		}
+		if err = validator(value); err != nil {
+			return value, err
+		}
+		return value, nil
+	}
+}
+
 func CastCustomType[T, U any](baseHandler TypedHandler[T]) TypedHandler[U] {
-	return customtypes.NewTransformer[T, U](baseHandler)
+	return customtypes.NewCastingTransformer[T, U](baseHandler)
+}
+
+// TransformCustomType creates a TypedHandler that applies a Transform function to process data from a base handler.
+func TransformCustomType[T, U any](baseHandler TypedHandler[T], transform Transform[T, U]) TypedHandler[U] {
+	return customtypes.NewTransformer(baseHandler, transform)
 }
 
-func DefaultStringType() TypedHandler[string] {
-	return readpipeline.NewTypedStringHandler()
+func DefaultStringType[T ~string]() TypedHandler[T] {
+	pipeline := readpipeline.NewTypedStringHandler()
+	return CastCustomType[string, T](pipeline)
 }
 
 func DefaultIntegerType[T ~int | ~int8 | ~int16 | ~int32 | ~int64]() TypedHandler[T] {

custom_types_test.go

@@ -142,7 +142,7 @@ func TestAddDynamicValidation(t *testing.T) {
 		Val string `key:"VAL" check:"true"`
 	}
 
-	baseHandler := DefaultStringType()
+	baseHandler := DefaultStringType[string]()
 	handler := AddDynamicValidation(baseHandler, func(tags reflect.StructTag, inputProcess FieldProcessor[string]) (FieldProcessor[string], error) {
 		if tags.Get("check") == "true" {
 			return func(rawValue string) (string, error) {
@@ -196,7 +196,7 @@ func TestCastCustomType(t *testing.T) {
 
 func TestDefaultTypeHandlers(t *testing.T) {
 	t.Run("String", func(t *testing.T) {
-		handler := DefaultStringType()
+		handler := DefaultStringType[string]()
 		p, _ := handler.BuildPipeline("")
 		val, _ := p("hello")
 		if any(val).(string) != "hello" {

envfile_keystore.go

@@ -0,0 +1,76 @@
+package goconfig
+
+import (
+	"bufio"
+	"context"
+	"os"
+	"strings"
+)
+
+// NewEnvFileKeyStore returns a KeyStore that reads values from a list of environment files.
+// If no filenames are provided, it defaults to ".env".
+// Files are processed in the order they are provided. If multiple files contain the same key,
+// the first one encountered wins.
+func NewEnvFileKeyStore(filenames ...string) KeyStore {
+	if len(filenames) == 0 {
+		filenames = []string{".env"}
+	}
+
+	// Pre-load all files into a map
+	values := make(map[string]string)
+	for _, filename := range filenames {
+		fileValues, err := readEnvFile(filename)
+		if err != nil {
+			// If a file doesn't exist or can't be read, we just skip it as per typical .env behavior
+			continue
+		}
+		for k, v := range fileValues {
+			if _, exists := values[k]; !exists {
+				values[k] = v
+			}
+		}
+	}
+
+	return func(ctx context.Context, key string) (string, bool, error) {
+		val, ok := values[key]
+		return val, ok, nil
+	}
+}
+
+func readEnvFile(filename string) (map[string]string, error) {
+	file, err := os.Open(filename)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	values := make(map[string]string)
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		// Skip empty lines and comments
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+
+		// Split by first '='
+		parts := strings.SplitN(line, "=", 2)
+		if len(parts) != 2 {
+			continue
+		}
+
+		key := strings.TrimSpace(parts[0])
+		value := strings.TrimSpace(parts[1])
+
+		// Remove quotes if present
+		if len(value) >= 2 {
+			if (value[0] == '"' && value[len(value)-1] == '"') || (value[0] == '\'' && value[len(value)-1] == '\'') {
+				value = value[1 : len(value)-1]
+			}
+		}
+
+		values[key] = value
+	}
+
+	return values, scanner.Err()
+}

envfile_keystore_test.go

@@ -0,0 +1,93 @@
+package goconfig
+
+import (
+	"context"
+	"os"
+	"testing"
+)
+
+func TestNewEnvFileKeyStore(t *testing.T) {
+	// Create a temporary .env file
+	content := `
+PORT=9000
+DB_HOST=localhost
+# This is a comment
+EMPTY=
+QUOTED="quoted value"
+SINGLE_QUOTED='single quoted'
+`
+	err := os.WriteFile(".env", []byte(content), 0644)
+	if err != nil {
+		t.Fatalf("Failed to create .env file: %v", err)
+	}
+	defer os.Remove(".env")
+
+	t.Run("Default .env", func(t *testing.T) {
+		store := NewEnvFileKeyStore()
+		ctx := context.Background()
+
+		tests := []struct {
+			key     string
+			wantVal string
+			wantOk  bool
+		}{
+			{"PORT", "9000", true},
+			{"DB_HOST", "localhost", true},
+			{"EMPTY", "", true},
+			{"QUOTED", "quoted value", true},
+			{"SINGLE_QUOTED", "single quoted", true},
+			{"MISSING", "", false},
+		}
+
+		for _, tt := range tests {
+			val, ok, err := store(ctx, tt.key)
+			if err != nil {
+				t.Errorf("Unexpected error for %s: %v", tt.key, err)
+			}
+			if ok != tt.wantOk {
+				t.Errorf("%s: got ok %v, want %v", tt.key, ok, tt.wantOk)
+			}
+			if val != tt.wantVal {
+				t.Errorf("%s: got val %q, want %q", tt.key, val, tt.wantVal)
+			}
+		}
+	})
+
+	t.Run("Specific files", func(t *testing.T) {
+		f1 := "test1.env"
+		f2 := "test2.env"
+		os.WriteFile(f1, []byte("KEY1=VAL1\nKEY2=VAL2_F1"), 0644)
+		os.WriteFile(f2, []byte("KEY2=VAL2_F2\nKEY3=VAL3"), 0644)
+		defer os.Remove(f1)
+		defer os.Remove(f2)
+
+		store := NewEnvFileKeyStore(f1, f2)
+		ctx := context.Background()
+
+		tests := []struct {
+			key     string
+			wantVal string
+			wantOk  bool
+		}{
+			{"KEY1", "VAL1", true},
+			{"KEY2", "VAL2_F1", true}, // First one wins in my implementation
+			{"KEY3", "VAL3", true},
+		}
+
+		for _, tt := range tests {
+			val, ok, _ := store(ctx, tt.key)
+			if ok != tt.wantOk || val != tt.wantVal {
+				t.Errorf("%s: got (%q, %v), want (%q, %v)", tt.key, val, ok, tt.wantVal, tt.wantOk)
+			}
+		}
+	})
+
+	t.Run("Non-existent file", func(t *testing.T) {
+		store := NewEnvFileKeyStore("nonexistent.env")
+		ctx := context.Background()
+		_, ok, _ := store(ctx, "ANY")
+		if ok {
+			t.Error("Expected ok=false for nonexistent file")
+		}
+	})
+}

example/custom_tags/README.md

@@ -0,0 +1,37 @@
+# Custom Tags
+
+This takes the [URL example](../custom_types/README.md) and reimplements the `SecureURL` type using a custom tag. This change
+allows the `net.URL` type to be used throughout the config so avoiding the need to cast. This is a great improvement on
+the URL sample.
+
+By the time this is released I will have added native support for `net.URL` into `goconfig` with the ability to use struct
+tags to validate the scheme for `https`. This sample remains as a demonstration of how to use custom tags.
+
+This sample will fall back to `env.example` in the current working directory.
+
+## Testing
+
+If you run the sample from this working directory, it will output the values from `env.example`.
+
+You can demonstrate the custom Secure URL validation using
+
+```bash
+export WHATSAPP_SERVER_URL=http://localhost:3000/
+go run .
+```
+
+You can demonstrate that explicitly blank values can override both defaults and later keystore (the `env.example` file)
+
+```bash
+export WHATSAPP_SERVER_URL=
+go run .
+```
+
+You can demonstrate that the `SecureURL` type is based on, but does not change, the `net.URL` type by providing a non-secure
+URL for `MY_BASE_URL`
+
+```bash
+export MY_BASE_URL=http://localhost:3000
+go run .
+```
+

example/custom_tags/config/config.go

@@ -0,0 +1,42 @@
+package config
+
+import (
+	"context"
+	"net/url"
+
+	"github.com/m0rjc/goconfig"
+)
+
+type Config struct {
+	// MyBaseURL is the base URL for my web interface. This is used when sharing links in messages to direct the user
+	// to my web interface.
+	MyBaseURL *url.URL `key:"MY_BASE_URL" required:"false" default:"http://localhost:8080"`
+	// WhatsAppPhoneId is the phone number ID for my phone number
+	WhatsAppPhoneId string `key:"WHATSAPP_PHONE_ID" required:"true" pattern:"^[0-9]+$"`
+	// WhatsAppServerUrl is the URL of the WhatsApp Business API server. This uses the custom "secure" tag defined by this example
+	WhatsAppServerUrl *url.URL `key:"WHATSAPP_SERVER_URL" required:"true" secure:"true" default:"https://api.whatsapp.com"`
+	// WhatsAppAuthToken is the authentication token for the WhatsApp Business API.
+	WhatsAppAuthToken string `key:"WHATSAPP_AUTH_TOKEN" required:"true"`
+	// WhatsAppChallenge is the challenge token sent by the WhatsApp Business API.
+	WhatsAppChallenge string `key:"WHATSAPP_CHALLENGE"`
+	// ServerPort is the port on which the server will listen for incoming public requests.
+	ServerPort int `key:"SERVER_PORT" required:"true" default:"8080" min:"1024" max:"65535"`
+	// HealthPort is the port on which the server will listen for health checks.
+	// This can be forced off by setting the key to the empty string.
+	HealthPort *int `key:"HEALTH_PORT" default:"8081" min:"1024" max:"65535"`
+}
+
+func LoadConfig() (*Config, error) {
+	var config Config
+	keystore := goconfig.CompositeStore(
+		fakeSecretsKeyStore,
+		goconfig.EnvironmentKeyStore,
+		goconfig.NewEnvFileKeyStore("env.example"))
+
+	if err := goconfig.Load(context.Background(), &config,
+		goconfig.WithKeyStore(keystore),
+		goconfig.WithCustomType(newUrlCustomType())); err != nil {
+		return nil, err
+	}
+	return &config, nil
+}

example/custom_tags/config/fake_secrets_keystore.go

@@ -0,0 +1,15 @@
+package config
+
+import (
+	"context"
+	"os"
+)
+
+// fakeSecretsKeyStore is a fake keystore that reads secrets from environment variables.
+// The environment variables must be prefixed with SECRET_.
+// A production keystore could read secrets from a secure store, though I've just had Kubernetes inject them into
+// the environment and used the normal keystore.
+func fakeSecretsKeyStore(ctx context.Context, key string) (string, bool, error) {
+	value, ok := os.LookupEnv("SECRET_" + key)
+	return value, ok, nil
+}