simulator.example.yaml

# ============================================================================
# LLM-Simulator Example Configuration
# ============================================================================
# This is a complete example configuration file demonstrating all features.
# Copy this to simulator.yaml and customize for your needs.
#
# Configuration precedence (highest to lowest):
#   1. CLI arguments (--port, --host, etc.)
#   2. Environment variables (LLM_SIM_*)
#   3. simulator.local.yaml (gitignored, for local overrides)
#   4. simulator.yaml (this file)
#   5. Built-in defaults
# ============================================================================

version: "1.0"

# ============================================================================
# Server Configuration
# ============================================================================
server:
  # Bind address (use "0.0.0.0" for all interfaces)
  host: "127.0.0.1"

  # Listen port
  port: 8080

  # Maximum concurrent connections
  max_connections: 10000

  # Request timeout in seconds
  request_timeout_secs: 300

  # Keep-alive timeout in seconds
  keepalive_timeout_secs: 75

  # TLS/SSL Configuration (optional)
  # Uncomment to enable HTTPS
  # tls:
  #   cert_path: "/etc/ssl/certs/llm-simulator.pem"
  #   key_path: "/etc/ssl/private/llm-simulator.key"
  #   min_version: "1.2"  # TLS 1.2 or 1.3
  #   client_auth: "none"  # none, optional, required

  # CORS Configuration
  cors:
    enabled: true
    allowed_origins: ["*"]
    allowed_methods: ["GET", "POST", "OPTIONS"]
    allowed_headers: ["Content-Type", "Authorization"]
    max_age_secs: 3600

  # Rate Limiting
  rate_limit:
    enabled: false
    requests_per_second: 100
    burst_capacity: 200
    # rate_limit_header: "X-API-Key"  # Optional: rate limit by header

  # Graceful shutdown timeout
  shutdown_timeout_secs: 30

# ============================================================================
# Provider Profiles
# ============================================================================
# Define different LLM provider/model profiles with realistic latency,
# error patterns, and response behaviors.
# ============================================================================
providers:
  # OpenAI GPT-4 Turbo
  gpt-4-turbo:
    name: "GPT-4 Turbo"
    provider: "openai"
    model: "gpt-4-turbo"
    enabled: true

    # Latency simulation
    latency:
      # Time to First Token (TTFT)
      ttft:
        type: "log_normal"
        p50_ms: 800.0    # Median: 800ms
        p99_ms: 2500.0   # 99th percentile: 2.5s

      # Inter-Token Latency (ITL)
      itl:
        type: "normal"
        mean_ms: 20.0
        std_dev_ms: 5.0
        min_clamp_ms: 0.0

      # Network jitter (optional)
      network_jitter:
        type: "normal"
        mean_ms: 0.0
        std_dev_ms: 2.0

      # Load-based degradation
      degradation:
        type: "exponential"
        alpha: 0.5
        baseline_qps: 10.0

      # Validation metrics (optional, for testing accuracy)
      validation_metrics:
        ttft_p50_ms: 800.0
        ttft_p99_ms: 2500.0
        itl_mean_ms: 20.0
        itl_p99_ms: 35.0
        tokens_per_sec: 50.0
        measurement_date: "2024-01"
        sample_size: 1000

    # Error injection (optional)
    errors:
      enabled: false
      base_rate: 0.0

      # Uncomment to enable error simulation
      # error_types:
      #   rate_limit:
      #     probability: 0.6
      #     status_code: 429
      #     message: "Rate limit exceeded"
      #     retry_after: 60
      #   server_error:
      #     probability: 0.4
      #     status_code: 500
      #     message: "Internal server error"

    # Response generation
    response:
      content_strategy: "random"  # random, pattern, markov, echo, template
      max_tokens: 4096
      vocab_size: 50000
      streaming_chunk_size: 1
      include_usage: true

    # Cost simulation (optional)
    cost:
      input_tokens_per_million: 10.0   # $10 per 1M input tokens
      output_tokens_per_million: 30.0  # $30 per 1M output tokens
      include_in_response: true

  # Anthropic Claude 3 Opus
  claude-3-opus:
    name: "Claude 3 Opus"
    provider: "anthropic"
    model: "claude-3-opus"
    enabled: true

    latency:
      ttft:
        type: "log_normal"
        p50_ms: 1200.0
        p99_ms: 3000.0

      itl:
        type: "normal"
        mean_ms: 25.0
        std_dev_ms: 6.0

      degradation:
        type: "mm_one"
        service_rate: 8.0
        baseline_qps: 5.0

    errors:
      enabled: false

    response:
      content_strategy: "random"
      max_tokens: 4096

    cost:
      input_tokens_per_million: 15.0
      output_tokens_per_million: 75.0
      include_in_response: true

  # Fast model with cache simulation
  gpt-3.5-turbo:
    name: "GPT-3.5 Turbo"
    provider: "openai"
    model: "gpt-3.5-turbo"
    enabled: true

    latency:
      # Bimodal distribution for cache hit/miss
      ttft:
        type: "bimodal"
        fast_mean_ms: 100.0    # Cache hit
        fast_std_ms: 20.0
        slow_mean_ms: 500.0    # Cache miss
        slow_std_ms: 100.0
        fast_probability: 0.9  # 90% cache hit rate

      itl:
        type: "normal"
        mean_ms: 10.0
        std_dev_ms: 3.0

      degradation:
        type: "linear"
        slope: 0.3
        baseline_qps: 50.0

    response:
      content_strategy: "random"
      max_tokens: 4096

    cost:
      input_tokens_per_million: 0.5
      output_tokens_per_million: 1.5
      include_in_response: true

# ============================================================================
# Simulation Configuration
# ============================================================================
simulation:
  # Random seed for reproducibility (optional)
  # seed: 42

  # Maximum concurrent sessions
  max_concurrent_sessions: 1000

  # Session timeout in seconds
  session_timeout_secs: 3600

  # Persist sessions to disk
  persist_sessions: false
  # session_storage_path: "/var/lib/llm-simulator/sessions"

  # Concurrency settings
  concurrency:
    worker_threads: 0  # 0 = auto-detect (number of CPU cores)
    blocking_threads: 512
    task_queue_size: 10000
    backpressure: "reject"  # drop, block, reject

  # Resource limits
  limits:
    # max_memory_mb: 4096
    # max_cpu_percent: 80
    # max_disk_mb: 10240
    max_body_size_mb: 10
    max_response_size_mb: 100

  # Warmup phase (optional)
  # warmup:
  #   enabled: true
  #   requests_per_profile: 100
  #   duration_secs: 30

# ============================================================================
# Telemetry Configuration
# ============================================================================
telemetry:
  enabled: true
  service_name: "llm-simulator"

  # Logging
  logging:
    level: "info"      # trace, debug, info, warn, error
    format: "json"     # json, text, pretty
    output: "stdout"   # stdout, stderr, file

    # File output (when output = "file")
    # file_path: "/var/log/llm-simulator/app.log"

    # Log rotation (optional)
    # rotation:
    #   max_size_mb: 100
    #   max_backups: 10
    #   compress: true

    include_location: false
    include_thread_id: false

  # Metrics
  metrics:
    enabled: true
    exporter: "prometheus"  # prometheus, otlp, statsd
    endpoint: "/metrics"
    interval_secs: 60

    # Histogram buckets (in seconds)
    histogram_buckets:
      - 0.001  # 1ms
      - 0.005  # 5ms
      - 0.01   # 10ms
      - 0.025  # 25ms
      - 0.05   # 50ms
      - 0.1    # 100ms
      - 0.25   # 250ms
      - 0.5    # 500ms
      - 1.0    # 1s
      - 2.5    # 2.5s
      - 5.0    # 5s
      - 10.0   # 10s

  # Distributed tracing (optional)
  tracing:
    enabled: false
    exporter: "otlp"  # otlp, jaeger, zipkin
    # otlp_endpoint: "http://localhost:4317"
    sampling_rate: 1.0  # 0.0 - 1.0
    trace_id_strategy: "random"  # random, deterministic
    propagation_format: "w3c"    # w3c, b3, jaeger

# ============================================================================
# Scenario Configuration
# ============================================================================
# Define load testing scenarios with patterns and assertions
# ============================================================================
scenarios:
  # Baseline performance test
  baseline:
    name: "Baseline Performance Test"
    description: "Constant load to measure baseline performance"
    enabled: true
    profile: "gpt-4-turbo"

    pattern:
      type: "constant"
      qps: 10.0
      duration_secs: 300

    assertions:
      - metric: "latency_p99"
        operator: "less_than"
        threshold: 3000.0  # 3 seconds
      - metric: "error_rate"
        operator: "less_than"
        threshold: 0.01    # 1%

    tags: ["baseline", "regression"]

  # Stress test
  stress:
    name: "Stress Test"
    description: "Ramp up load to find breaking point"
    enabled: false
    profile: "gpt-4-turbo"

    pattern:
      type: "ramp"
      start_qps: 1.0
      end_qps: 100.0
      duration_secs: 600

    assertions:
      - metric: "latency_p95"
        operator: "less_than"
        threshold: 5000.0

    tags: ["stress", "capacity-planning"]

  # Spike test
  spike:
    name: "Spike Test"
    description: "Sudden traffic spike"
    enabled: false
    profile: "gpt-4-turbo"

    pattern:
      type: "spike"
      baseline_qps: 10.0
      spike_qps: 100.0
      spike_duration_secs: 60
      total_duration_secs: 300

    tags: ["spike", "resilience"]

# ============================================================================
# Security Configuration
# ============================================================================
security:
  # Authentication (optional)
  # authentication:
  #   method: "api_key"  # api_key, jwt, oauth2

  # API keys (optional)
  # api_keys:
  #   - key: "sk-test-key-1"
  #     name: "Test Key 1"
  #     permissions: ["read", "write"]
  #     # expires_at: "2025-12-31T23:59:59Z"
  #     rate_limit:
  #       requests_per_second: 50
  #       burst_capacity: 100

  # IP filtering (optional)
  # ip_allowlist:
  #   - "192.168.1.0/24"
  #   - "10.0.0.0/8"
  # ip_blocklist:
  #   - "1.2.3.4"

  # Request signing (optional)
  request_signing: false
  # signing_secret: "your-secret-key"

# ============================================================================
# Feature Flags
# ============================================================================
features:
  # Core features
  streaming: true
  function_calling: false
  vision: false
  embeddings: false

  # Advanced features
  fine_tuning: false
  batch_processing: false

  # Optimization features
  caching: false
  prompt_caching: false

  # Custom flags
  # custom:
  #   experimental_features: false
  #   debug_mode: false

# ============================================================================
# Usage Examples
# ============================================================================
# 1. Start with defaults:
#    ./llm-simulator
#
# 2. Override with environment variables:
#    export LLM_SIM_PORT=9090
#    export LLM_SIM_LOG_LEVEL=debug
#    ./llm-simulator
#
# 3. Override with CLI arguments:
#    ./llm-simulator --port 9090 --log-level debug
#
# 4. Use custom config file:
#    ./llm-simulator --config /path/to/custom.yaml
#
# 5. Validate configuration:
#    ./llm-simulator --validate-only
#
# 6. Dump effective configuration:
#    ./llm-simulator --dump-config
# ============================================================================