Refactoring: Tests and project improvements

.drone/drone.yml

@@ -0,0 +1,195 @@
+kind: pipeline
+type: docker
+name: watchtower-pr
+
+trigger:
+  event:
+    - pull_request
+  actions:
+    - synchronized
+
+environment:
+  GO_VERSION: 1.25
+
+services:
+  - name: redis
+    image: redis:alpine
+
+  - name: rabbitmq
+    image: rabbitmq:4-management
+
+  - name: minio
+    image: minio/minio
+    commands:
+      - minio server /data
+    environment:
+      MINIO_ROOT_USER: "minio-root"
+      MINIO_ROOT_PASSWORD: "minio-root"
+
+steps:
+  - name: cache-built-go
+    image: drillster/drone-volume-cache
+    settings:
+      restore: true
+      mount:
+        - /usr/local/go
+        - /drone/src
+    volumes:
+      - name: go-targets
+        path: /cache
+
+  - name: build
+    image: golang:1.25
+    depends_on:
+      - cache-built-go
+    commands:
+      - go build watchtower/cmd/watchtower
+    volumes:
+      - name: go-targets
+        path: /cache
+
+  - name: lint
+    image: golang:1.25
+    depends_on:
+      - build
+    commands:
+      - go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@latest
+      - golangci-lint run ./...
+    volumes:
+      - name: go-targets
+        path: /cache
+
+  - name: test
+    image: golang:1.25
+    depends_on:
+      - build
+    commands:
+      - sleep 15
+      - curl -v -u guest:guest -K /drone/src/tests/config/rmq/headers.conf --data-binary '@/drone/src/tests/config/rmq/definitions.json' http://rabbitmq:15672/api/definitions
+      - sleep 10
+      - go test watchtower/tests
+    volumes:
+      - name: go-targets
+        path: /cache
+
+volumes:
+  - name: go-targets
+    temp: {}
+
+
+# On merged pull request to main branch or created release branch
+---
+kind: pipeline
+type: docker
+name: watchtower-merged
+
+trigger:
+  event:
+    - push
+  branch:
+    - main
+    - master
+    - release/*
+
+environment:
+  GO_VERSION: 1.25
+
+services:
+  - name: redis
+    image: redis:alpine
+
+  - name: rabbitmq
+    image: rabbitmq:4-management
+
+  - name: minio
+    image: minio/minio
+    commands:
+      - minio server /data
+    environment:
+      MINIO_ROOT_USER: "minio-root"
+      MINIO_ROOT_PASSWORD: "minio-root"
+
+steps:
+  - name: cache-built-go
+    image: drillster/drone-volume-cache
+    settings:
+      restore: true
+      mount:
+        - /usr/local/go
+        - /drone/src
+    volumes:
+      - name: go-targets
+        path: /cache
+
+  - name: build
+    image: golang:1.25
+    depends_on:
+      - cache-built-go
+    commands:
+      - go build watchtower/cmd/watchtower
+    volumes:
+      - name: go-targets
+        path: /cache
+
+  - name: lint
+    image: golang:1.25
+    depends_on:
+      - build
+    commands:
+      - go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@latest
+      - golangci-lint run ./...
+    volumes:
+      - name: go-targets
+        path: /cache
+
+  - name: test
+    image: golang:1.25
+    depends_on:
+      - build
+    commands:
+      - sleep 15
+      - curl -v -u guest:guest -K /drone/src/tests/config/rmq/headers.conf --data-binary '@/drone/src/tests/config/rmq/definitions.json' http://rabbitmq:15672/api/definitions
+      - sleep 10
+      - go test watchtower/tests
+    volumes:
+      - name: go-targets
+        path: /cache
+
+volumes:
+  - name: go-targets
+    temp: {}
+
+
+# Build release on new tag creating event
+---
+kind: pipeline
+type: docker
+name: watchtower-release
+
+trigger:
+  event:
+    - tag
+
+environment:
+  REGISTRY_ADDRESS: ${REGISTRY_ADDRESS}
+
+steps:
+  - name: build-and-push-docker
+    image: plugins/docker
+    settings:
+      repo: git.sova.local:3000/sova-core/watchtower
+      insecure: true
+      dockerfile: Dockerfile
+      registry: git.sova.local:5000
+      tags:
+        - ${DRONE_COMMIT_SHA:0:8}
+        - ${DRONE_TAG}
+        - latest
+      username:
+        from_secret: GITEA_USERNAME
+      password:
+        from_secret: GITEA_TOKEN
+
+volumes:
+  - name: go-targets
+    temp: {}

.env

@@ -1,3 +1,5 @@
+WATCHTOWER__RUN_MODE=development
+
 WATCHTOWER__ORCHESTRATOR__SEMAPHORE_SIZE=10
 
 WATCHTOWER__OTLP__LOGGER__LEVEL=DEBUG
@@ -25,8 +27,8 @@ WATCHTOWER__TASK__QUEUE__RMQ__EXCHANGE=watchtower
 WATCHTOWER__TASK__QUEUE__RMQ__ROUTING_KEY=task
 WATCHTOWER__TASK__QUEUE__RMQ__QUEUE=watchtower-tasks
 
-WATCHTOWER__TASK__PROCESSOR__DOCPARSER__ADDRESS=http://localhost:8012/api/v1
+WATCHTOWER__TASK__PROCESSOR__DOCPARSER__ADDRESS=http://localhost:8012
 WATCHTOWER__TASK__PROCESSOR__DOCPARSER__TIMEOUT=100s
 
-WATCHTOWER__TASK__PROCESSOR__DOCSTORAGE__ADDRESS=http://localhost:2892/api/v1
+WATCHTOWER__TASK__PROCESSOR__DOCSTORAGE__ADDRESS=http://localhost:2892
 WATCHTOWER__TASK__PROCESSOR__DOCSTORAGE__TIMEOUT=100s

.github/workflows/pull-request.yml

@@ -26,11 +26,13 @@ jobs:
       - uses: actions/checkout@v5
       - uses: actions/setup-go@v6
         with:
-          go-version: stable
+          go-version: 1.25.7
+          check-latest: 'false'
+
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v8
         with:
-          version: v2.6.0
+          version: latest
 
   test:
     runs-on: ubuntu-latest

Makefile

@@ -7,7 +7,7 @@ build:
 	go build -v -o $(SERVICE_BIN_FILE_PATH) ./cmd/watchtower
 
 run: build
-	$(SERVICE_BIN_FILE_PATH) -c ./configs/config.toml
+	$(SERVICE_BIN_FILE_PATH) -d
 
 test:
 	go test -race ./tests/...

cmd/config.go

@@ -0,0 +1,142 @@
+package cmd
+
+import (
+	"fmt"
+	"log/slog"
+	"os"
+	"strings"
+
+	otlp_go "github.com/breadrock1/otlp-go/otlp"
+	"github.com/spf13/viper"
+
+	"watchtower/cmd/watchtower/httpserver"
+	"watchtower/internal/core/cloud/infrastructure/s3"
+	"watchtower/internal/process"
+	"watchtower/internal/support/task/infrastructure/docparser"
+	"watchtower/internal/support/task/infrastructure/docsearch"
+	"watchtower/internal/support/task/infrastructure/redis"
+	"watchtower/internal/support/task/infrastructure/rmq"
+)
+
+type Config struct {
+	Otlp         otlp_go.OtlpConfig `mapstructure:"otlp"`
+	Orchestrator process.Config     `mapstructure:"orchestrator"`
+	Server       ServerConfig       `mapstructure:"server"`
+	Storage      StorageConfig      `mapstructure:"storage"`
+	Task         TaskConfig         `mapstructure:"task"`
+}
+
+type ServerConfig struct {
+	Http httpserver.Config `mapstructure:"http"`
+}
+
+type StorageConfig struct {
+	S3 s3.Config `mapstructure:"s3"`
+}
+
+type TaskConfig struct {
+	TaskStorage TaskStorageConfig `mapstructure:"storage"`
+	TaskQueue   TaskQueueConfig   `mapstructure:"queue"`
+	Processor   ProcessorConfig   `mapstructure:"processor"`
+}
+
+type TaskStorageConfig struct {
+	Redis redis.Config `mapstructure:"redis"`
+}
+
+type TaskQueueConfig struct {
+	Rmq rmq.Config `mapstructure:"rmq"`
+}
+
+type ProcessorConfig struct {
+	DocParser  docparser.Config `mapstructure:"docparser"`
+	DocStorage docsearch.Config `mapstructure:"docstorage"`
+}
+
+const (
+	launchModeEnvKey  = "WATCHTOWER__RUN_MODE"
+	defaultLaunchMode = "development"
+	serviceEnvPrefix  = "WATCHTOWER"
+)
+
+func InitConfig() (*Config, error) {
+	launchMode := os.Getenv(launchModeEnvKey)
+	if launchMode == "" {
+		launchMode = defaultLaunchMode
+	}
+
+	viperInst := viper.New()
+
+	viperInst.SetConfigName(launchMode)
+	viperInst.SetConfigType("toml")
+
+	viperInst.AddConfigPath(".")
+	viperInst.AddConfigPath("./configs")
+	viperInst.AddConfigPath("../configs")
+
+	if launchMode == defaultLaunchMode {
+		// Used to include config from integration tests
+		viperInst.AddConfigPath("../../configs")
+	}
+
+	if err := viperInst.ReadInConfig(); err != nil {
+		//nolint
+		if _, ok := err.(viper.ConfigFileNotFoundError); ok {
+			slog.Info("config file not found, using env vars")
+		} else {
+			return nil, fmt.Errorf("error reading config file: %w", err)
+		}
+	}
+
+	setupEnv(viperInst)
+
+	config := &Config{}
+	if err := viperInst.Unmarshal(config); err != nil {
+		confErr := fmt.Errorf("failed while unmarshaling config: %w", err)
+		return config, confErr
+	}
+
+	return config, nil
+}
+
+func setupEnv(viperInst *viper.Viper) {
+	viperInst.AutomaticEnv()
+	viperInst.SetEnvPrefix(serviceEnvPrefix)
+	viperInst.SetEnvKeyReplacer(strings.NewReplacer(".", "__"))
+
+	//nolint
+	envMappings := map[string]string{
+		"orchestrator.semaphore_size":       "ORCHESTRATOR__SEMAPHORE_SIZE",
+		"otlp.logger.level":                 "OTLP__LOGGER__LEVEL",
+		"otlp.logger.address":               "OTLP__LOGGER__ADDRESS",
+		"otlp.logger.enable_loki":           "OTLP__LOGGER__ENABLE_LOKI",
+		"otlp.tracer.address":               "OTLP__TRACER__ADDRESS",
+		"otlp.tracer.enable_jaeger":         "OTLP__TRACER__ENABLE_JAEGER",
+		"server.http.address":               "SERVER__HTTP__ADDRESS",
+		"storage.s3.address":                "STORAGE__S3__ADDRESS",
+		"storage.s3.access_id":              "STORAGE__S3__ACCESS_ID",
+		"storage.s3.secret_key":             "STORAGE__S3__SECRET_KEY",
+		"storage.s3.enable_ssl":             "STORAGE__S3__ENABLE_SSL",
+		"storage.s3.token":                  "STORAGE__S3__TOKEN",
+		"task.storage.redis.address":        "TASK__STORAGE__REDIS__ADDRESS",
+		"task.storage.redis.username":       "TASK__STORAGE__REDIS__USERNAME",
+		"task.storage.redis.password":       "TASK__STORAGE__REDIS__PASSWORD",
+		"task.storage.redis.expired":        "TASK__STORAGE__REDIS__EXPIRED",
+		"task.queue.rmq.address":            "TASK__QUEUE__RMQ__ADDRESS",
+		"task.queue.rmq.exchange":           "TASK__QUEUE__RMQ__EXCHANGE",
+		"task.queue.rmq.routing_key":        "TASK__QUEUE__RMQ__ROUTING_KEY",
+		"task.queue.rmq.queue":              "TASK__QUEUE__RMQ__QUEUE",
+		"task.processor.docstorage.address": "TASK__PROCESSOR__DOCSTORAGE__ADDRESS",
+		"task.processor.docstorage.timeout": "TASK__PROCESSOR__DOCSTORAGE__TIMEOUT",
+		"task.processor.docparser.address":  "TASK__PROCESSOR__DOCPARSER__ADDRESS",
+		"task.processor.docparser.timeout":  "TASK__PROCESSOR__DOCPARSER__TIMEOUT",
+	}
+
+	var bindErr error
+	for key, value := range envMappings {
+		bindErr = viperInst.BindEnv(key, fmt.Sprintf("%s__%s", serviceEnvPrefix, value))
+		if bindErr != nil {
+			slog.Warn("failed to bind env var", slog.String("err", bindErr.Error()))
+		}
+	}
+}