diff --git a/.env.example b/.env.example index 9f7bb76..4c5ccb6 100644 --- a/.env.example +++ b/.env.example @@ -7,4 +7,6 @@ POSTGRES_PASSWORD=XXXXXXXXX POSTGRES_DB=XXXXXXXXX DATABASE_URL=XXXXXXXXX REDIS_HOST=XXXXXXXXX -REDIS_PORT=XXXXXXXXX \ No newline at end of file +REDIS_PORT=XXXXXXXXX +GRAFANA_ADMIN_USER=admin +GRAFANA_ADMIN_PASSWORD=admin diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index aabc68d..49d9afb 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -75,6 +75,15 @@ jobs: push: true tags: ghcr.io/${{ env.REPO_LOWER }}/user-service:latest + - name: Scan userService image with Trivy + uses: aquasecurity/trivy-action@0.28.0 + with: + image-ref: ghcr.io/${{ env.REPO_LOWER }}/user-service:latest + format: table + severity: CRITICAL,HIGH + exit-code: '1' + ignore-unfixed: true + # Build authService build-authService: needs: detect-changes @@ -104,6 +113,15 @@ jobs: push: true tags: ghcr.io/${{ env.REPO_LOWER }}/auth-service:latest + - name: Scan authService image with Trivy + uses: aquasecurity/trivy-action@0.28.0 + with: + image-ref: ghcr.io/${{ env.REPO_LOWER }}/auth-service:latest + format: table + severity: CRITICAL,HIGH + exit-code: '1' + ignore-unfixed: true + # Build mailService build-mailService: needs: detect-changes @@ -133,6 +151,15 @@ jobs: push: true tags: ghcr.io/${{ env.REPO_LOWER }}/mail-service:latest + - name: Scan mailService image with Trivy + uses: aquasecurity/trivy-action@0.28.0 + with: + image-ref: ghcr.io/${{ env.REPO_LOWER }}/mail-service:latest + format: table + severity: CRITICAL,HIGH + exit-code: '1' + ignore-unfixed: true + # Build eureka build-eureka: needs: detect-changes @@ -162,6 +189,15 @@ jobs: push: true tags: ghcr.io/${{ env.REPO_LOWER }}/eureka:latest + - name: Scan eureka image with Trivy + uses: aquasecurity/trivy-action@0.28.0 + with: + image-ref: ghcr.io/${{ env.REPO_LOWER }}/eureka:latest + format: table + severity: CRITICAL,HIGH + exit-code: '1' + ignore-unfixed: true + # Build configServer build-configServer: needs: detect-changes @@ -191,6 +227,15 @@ jobs: push: true tags: ghcr.io/${{ env.REPO_LOWER }}/config-server:latest + - name: Scan configServer image with Trivy + uses: aquasecurity/trivy-action@0.28.0 + with: + image-ref: ghcr.io/${{ env.REPO_LOWER }}/config-server:latest + format: table + severity: CRITICAL,HIGH + exit-code: '1' + ignore-unfixed: true + # Build gateway build-gateway: needs: detect-changes @@ -218,4 +263,12 @@ jobs: with: context: ./gateway push: true - tags: ghcr.io/${{ env.REPO_LOWER }}/gateway:latest \ No newline at end of file + tags: ghcr.io/${{ env.REPO_LOWER }}/gateway:latest + - name: Scan gateway image with Trivy + uses: aquasecurity/trivy-action@0.28.0 + with: + image-ref: ghcr.io/${{ env.REPO_LOWER }}/gateway:latest + format: table + severity: CRITICAL,HIGH + exit-code: '1' + ignore-unfixed: true diff --git a/README.md b/README.md index 115710c..d82eadf 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ A production-ready Spring Boot 3 microservices template. Includes service discov - [Kafka Topics](#kafka-topics) - [Redis Caching](#redis-caching) - [Health Checks](#health-checks) +- [Centralized Logging](#centralized-logging) - [Adding a New Service](#adding-a-new-service) - [Kubernetes](#kubernetes) - [Common Issues](#common-issues) @@ -116,7 +117,7 @@ A production-ready Spring Boot 3 microservices template. Includes service discov | Messaging | Apache Kafka 7.5.0, Spring Kafka | | Security | Spring Security, JWT (jjwt 0.12.5) | | Resilience | Resilience4j Circuit Breaker, Spring Retry | -| Monitoring | Spring Boot Actuator | +| Monitoring | Spring Boot Actuator, Loki, Promtail, Grafana | | Load Testing | Gatling 3.10.5 | | Containerization | Docker, Docker Compose | | Orchestration | Kubernetes (Deployments, StatefulSets, HPA) | @@ -382,6 +383,39 @@ Startup order: `zookeeper` → `kafka` → `db` → `eureka` → `config-server` --- +## Centralized Logging + +Logs are centralized with **Promtail → Loki → Grafana**. + +- **Promtail** tails Docker container logs and forwards them to Loki. +- **Loki** stores and indexes logs. +- **Grafana** provides querying and visualization. + +Start the stack (logging services are already included in compose): + +```bash +docker compose -f compose.dev.yml up --build -d +``` + +Access: + +- Grafana: `http://localhost:3000` +- Loki: `http://localhost:3100` + +Default Grafana credentials are `admin/admin` (override in `.env` with `GRAFANA_ADMIN_USER` and `GRAFANA_ADMIN_PASSWORD`). + +Example LogQL queries in Grafana Explore: + +```logql +{compose_service="gateway"} +``` + +```logql +{compose_project="springboot_microservice"} +``` + +--- + ## Adding a New Service **1. Create Spring Boot project** with dependencies: `web`, `actuator`, `eureka-client`, `spring-cloud-config` @@ -505,6 +539,8 @@ Kafka container listens on `29092` internally. Host port `19092` maps to contain | Redis | 6379 | 6379 | | Kafka | 19092 | 29092 | | Zookeeper | 2181 | 2181 | +| Grafana | 3000 | 3000 | +| Loki | 3100 | 3100 | --- diff --git a/compose.dev.yml b/compose.dev.yml index 6a36191..9d76761 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -84,7 +84,7 @@ services: kafka: condition: service_healthy healthcheck: - test: ["CMD-SHELL", "curl -f http://localhost:8084/actuator/health || exit 1"] + test: ["CMD-SHELL", "curl -f http://localhost:${MANAGEMENT_PORT}/actuator/health || exit 1"] interval: 10s timeout: 5s retries: 5 @@ -123,6 +123,10 @@ services: context: ./configServer dockerfile: Dockerfile.dev container_name: config-server + volumes: + - ./configServer/target:/app/target + - ./configServer/src:/app/src + - ~/.m2:/root/.m2 restart: always ports: - "8888:8888" @@ -134,7 +138,7 @@ services: interval: 10s timeout: 5s retries: 5 - start_period: 40s + start_period: 50s networks: - micro-net mem_limit: 300m @@ -147,6 +151,10 @@ services: context: ./gateway dockerfile: Dockerfile.dev container_name: gateway + volumes: + - ./gateway/target:/app/target + - ./gateway/src:/app/src + - ~/.m2:/root/.m2 restart: always ports: - "8080:8080" @@ -155,8 +163,10 @@ services: condition: service_healthy config-server: condition: service_healthy + redis: + condition: service_healthy healthcheck: - test: ["CMD-SHELL", "curl -f http://localhost:8080/actuator/health || exit 1"] + test: ["CMD-SHELL", "curl -f http://localhost:${MANAGEMENT_PORT}/actuator/health || exit 1"] interval: 10s timeout: 5s retries: 5 @@ -260,6 +270,63 @@ services: interval: 10s timeout: 5s retries: 5 + loki: + image: grafana/loki:3.1.1 + container_name: loki + restart: always + ports: + - "3100:3100" + command: [ "-config.file=/etc/loki/config.yml" ] + volumes: + - ./observability/loki-config.yml:/etc/loki/config.yml:ro + - loki-data:/loki + networks: + - micro-net + healthcheck: + test: [ "CMD-SHELL", "wget -qO- http://localhost:3100/ready || exit 1" ] + interval: 15s + timeout: 5s + retries: 5 + + promtail: + image: grafana/promtail:3.1.1 + container_name: promtail + restart: always + command: [ "-config.file=/etc/promtail/config.yml" ] + depends_on: + loki: + condition: service_healthy + volumes: + - ./observability/promtail-config.yml:/etc/promtail/config.yml:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + - /var/lib/docker/containers:/var/lib/docker/containers:ro + - promtail-data:/tmp + networks: + - micro-net + + grafana: + image: grafana/grafana:11.2.2 + container_name: grafana + restart: always + ports: + - "3000:3000" + depends_on: + loki: + condition: service_healthy + environment: + - GF_SECURITY_ADMIN_USER=${GRAFANA_ADMIN_USER:-admin} + - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:-admin} + - GF_USERS_ALLOW_SIGN_UP=false + volumes: + - grafana-data:/var/lib/grafana + - ./observability/grafana/datasources:/etc/grafana/provisioning/datasources:ro + networks: + - micro-net + healthcheck: + test: [ "CMD-SHELL", "wget -qO- http://localhost:3000/api/health | grep -q ok || exit 1" ] + interval: 15s + timeout: 5s + retries: 5 networks: micro-net: @@ -271,3 +338,6 @@ volumes: zookeeper-logs: kafka-data: redis-data: + loki-data: + promtail-data: + grafana-data: diff --git a/compose.yml b/compose.yml index 2739307..4d869ad 100644 --- a/compose.yml +++ b/compose.yml @@ -40,7 +40,7 @@ services: config-server: condition: service_healthy healthcheck: - test: ["CMD-SHELL", "curl -f http://localhost:{MANAGEMENT_PORT}/actuator/health || exit 1"] + test: ["CMD-SHELL", "curl -f http://localhost:${MANAGEMENT_PORT}/actuator/health || exit 1"] interval: 10s timeout: 5s retries: 5 @@ -66,7 +66,7 @@ services: kafka: condition: service_healthy healthcheck: - test: ["CMD-SHELL", "curl -f http://localhost:{MANAGEMENT_PORT}/actuator/health || exit 1"] + test: ["CMD-SHELL", "curl -f http://localhost:${MANAGEMENT_PORT}/actuator/health || exit 1"] interval: 10s timeout: 5s retries: 5 @@ -123,15 +123,15 @@ services: gateway: image: ghcr.io/goal651/springboot_microservice/gateway:latest restart: always - deploy: - replicas: 2 + ports: + - "8080:8080" depends_on: eureka: condition: service_healthy config-server: condition: service_healthy healthcheck: - test: ["CMD-SHELL", "curl -f http://localhost:{MANAGEMENT_PORT}/actuator/health || exit 1"] + test: ["CMD-SHELL", "curl -f http://localhost:${MANAGEMENT_PORT}/actuator/health || exit 1"] interval: 10s timeout: 5s retries: 5 @@ -235,6 +235,63 @@ services: interval: 10s timeout: 5s retries: 5 + loki: + image: grafana/loki:3.1.1 + container_name: loki + restart: always + ports: + - "3100:3100" + command: [ "-config.file=/etc/loki/config.yml" ] + volumes: + - ./observability/loki-config.yml:/etc/loki/config.yml:ro + - loki-data:/loki + networks: + - micro-net + healthcheck: + test: [ "CMD-SHELL", "wget -qO- http://localhost:3100/ready || exit 1" ] + interval: 15s + timeout: 5s + retries: 5 + + promtail: + image: grafana/promtail:3.1.1 + container_name: promtail + restart: always + command: [ "-config.file=/etc/promtail/config.yml" ] + depends_on: + loki: + condition: service_healthy + volumes: + - ./observability/promtail-config.yml:/etc/promtail/config.yml:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + - /var/lib/docker/containers:/var/lib/docker/containers:ro + - promtail-data:/tmp + networks: + - micro-net + + grafana: + image: grafana/grafana:11.2.2 + container_name: grafana + restart: always + ports: + - "3000:3000" + depends_on: + loki: + condition: service_healthy + environment: + - GF_SECURITY_ADMIN_USER=${GRAFANA_ADMIN_USER:-admin} + - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:-admin} + - GF_USERS_ALLOW_SIGN_UP=false + volumes: + - grafana-data:/var/lib/grafana + - ./observability/grafana/datasources:/etc/grafana/provisioning/datasources:ro + networks: + - micro-net + healthcheck: + test: [ "CMD-SHELL", "wget -qO- http://localhost:3000/api/health | grep -q ok || exit 1" ] + interval: 15s + timeout: 5s + retries: 5 networks: micro-net: @@ -245,3 +302,6 @@ volumes: zookeeper-logs: kafka-data: redis-data: + loki-data: + promtail-data: + grafana-data: diff --git a/configServer/src/main/resources/configs/api-gateway.properties b/configServer/src/main/resources/configs/api-gateway.properties new file mode 100644 index 0000000..74b4be4 --- /dev/null +++ b/configServer/src/main/resources/configs/api-gateway.properties @@ -0,0 +1,4 @@ +server.port=8080 +spring.application.name=api-gateway +spring.data.redis.host=redis +spring.data.redis.port=${REDIS_PORT} \ No newline at end of file diff --git a/configServer/src/main/resources/configs/gateway.properties b/configServer/src/main/resources/configs/gateway.properties deleted file mode 100644 index 5654337..0000000 --- a/configServer/src/main/resources/configs/gateway.properties +++ /dev/null @@ -1,2 +0,0 @@ -server.port=8080 -spring.application.name=api-gateway diff --git a/configServer/src/main/resources/configs/mail-service.properties b/configServer/src/main/resources/configs/mail-service.properties index 671e2b2..92b2e71 100644 --- a/configServer/src/main/resources/configs/mail-service.properties +++ b/configServer/src/main/resources/configs/mail-service.properties @@ -29,4 +29,6 @@ spring.kafka.producer.retries=3 spring.kafka.producer.properties.spring.json.add.type.headers=false spring.kafka.consumer.properties.spring.json.trusted.packages=* spring.kafka.consumer.properties.spring.json.use.type.headers=false -spring.kafka.consumer.properties.spring.json.value.default.type=com.tutorial.mailService.dto.AuthEvent \ No newline at end of file +spring.kafka.consumer.properties.spring.json.value.default.type=com.tutorial.mailService.dto.AuthEvent +app.kafka.topics.auth-events=auth-events +app.kafka.topics.auth-events-dlq=auth-events.dlq diff --git a/gateway/pom.xml b/gateway/pom.xml index be11cb2..10d30b6 100644 --- a/gateway/pom.xml +++ b/gateway/pom.xml @@ -36,6 +36,10 @@ org.springframework.cloud spring-cloud-starter-gateway + + org.springframework.boot + spring-boot-starter-data-redis-reactive + org.springframework.cloud spring-cloud-starter-config diff --git a/gateway/src/main/java/com/tutorial/gateway/GatewayApplication.java b/gateway/src/main/java/com/tutorial/gateway/GatewayApplication.java index 96098f4..903263c 100644 --- a/gateway/src/main/java/com/tutorial/gateway/GatewayApplication.java +++ b/gateway/src/main/java/com/tutorial/gateway/GatewayApplication.java @@ -3,9 +3,12 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.cloud.client.discovery.EnableDiscoveryClient; +import org.springframework.cloud.gateway.filter.ratelimit.KeyResolver; +import org.springframework.cloud.gateway.filter.ratelimit.RedisRateLimiter; import org.springframework.cloud.gateway.route.RouteLocator; import org.springframework.cloud.gateway.route.builder.RouteLocatorBuilder; import org.springframework.context.annotation.Bean; +import reactor.core.publisher.Mono; @SpringBootApplication @EnableDiscoveryClient @@ -19,9 +22,30 @@ public static void main(String[] args) { public RouteLocator customRouteLocator(RouteLocatorBuilder builder) { return builder.routes() .route("user-service", r -> r.path("/users/**") + .filters(f -> f.requestRateLimiter(c -> { + c.setRateLimiter(redisRateLimiter()); + c.setKeyResolver(ipKeyResolver()); + })) .uri("lb://user-service")) + .route("auth-service", r -> r.path("/auth/**") + .filters(f -> f.requestRateLimiter(c -> { + c.setRateLimiter(redisRateLimiter()); + c.setKeyResolver(ipKeyResolver()); + })) .uri("lb://auth-service")) .build(); - } + } + + @Bean + public RedisRateLimiter redisRateLimiter() { + return new RedisRateLimiter(10, 20, 1); + } + + @Bean + public KeyResolver ipKeyResolver() { + return exchange -> Mono.justOrEmpty(exchange.getRequest().getRemoteAddress()) + .map(remoteAddress -> remoteAddress.getAddress().getHostAddress()) + .defaultIfEmpty("unknown"); + } } \ No newline at end of file diff --git a/mailService/src/main/java/com/tutorial/mailService/config/KafkaDlqConfig.java b/mailService/src/main/java/com/tutorial/mailService/config/KafkaDlqConfig.java new file mode 100644 index 0000000..5aeded7 --- /dev/null +++ b/mailService/src/main/java/com/tutorial/mailService/config/KafkaDlqConfig.java @@ -0,0 +1,27 @@ +package com.tutorial.mailService.config; + +import org.apache.kafka.common.TopicPartition; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.listener.DeadLetterPublishingRecoverer; +import org.springframework.kafka.listener.DefaultErrorHandler; +import org.springframework.util.backoff.FixedBackOff; + +@Configuration +public class KafkaDlqConfig { + + @Bean + public DeadLetterPublishingRecoverer deadLetterPublishingRecoverer( + KafkaTemplate kafkaTemplate, + @Value("${app.kafka.topics.auth-events-dlq:auth-events.dlq}") String dlqTopic) { + return new DeadLetterPublishingRecoverer(kafkaTemplate, + (record, ex) -> new TopicPartition(dlqTopic, record.partition())); + } + + @Bean + public DefaultErrorHandler defaultErrorHandler(DeadLetterPublishingRecoverer recoverer) { + return new DefaultErrorHandler(recoverer, new FixedBackOff(1000L, 2L)); + } +} diff --git a/mailService/src/main/java/com/tutorial/mailService/config/KafkaProducerConfig.java b/mailService/src/main/java/com/tutorial/mailService/config/KafkaProducerConfig.java index 66f1a89..e068d03 100644 --- a/mailService/src/main/java/com/tutorial/mailService/config/KafkaProducerConfig.java +++ b/mailService/src/main/java/com/tutorial/mailService/config/KafkaProducerConfig.java @@ -15,4 +15,12 @@ public NewTopic mailEventsTopic() { .replicas(1) .build(); } + + @Bean + public NewTopic authEventsDlqTopic() { + return TopicBuilder.name("auth-events.dlq") + .partitions(3) + .replicas(1) + .build(); + } } \ No newline at end of file diff --git a/mailService/src/main/java/com/tutorial/mailService/service/KafkaListenerService.java b/mailService/src/main/java/com/tutorial/mailService/service/KafkaListenerService.java index 9a8c343..d7f3b48 100644 --- a/mailService/src/main/java/com/tutorial/mailService/service/KafkaListenerService.java +++ b/mailService/src/main/java/com/tutorial/mailService/service/KafkaListenerService.java @@ -13,7 +13,7 @@ public class KafkaListenerService { private final EmailService emailService; - @KafkaListener(topics = "auth-events", groupId = "mail-group-id") + @KafkaListener(topics = "${app.kafka.topics.auth-events:auth-events}", groupId = "mail-group-id") public void processAuthMessages(AuthEvent data) { emailService.sendSimpleEmail(data.getUserEmail(), "Auth Event Received ", "Hello " + data.getUserName() + ",\n\nWe received an authentication event for your account at " + data.getTimestamp() diff --git a/observability/grafana/datasources/loki.yml b/observability/grafana/datasources/loki.yml new file mode 100644 index 0000000..050b3c4 --- /dev/null +++ b/observability/grafana/datasources/loki.yml @@ -0,0 +1,9 @@ +apiVersion: 1 + +datasources: + - name: Loki + type: loki + access: proxy + url: http://loki:3100 + isDefault: true + editable: false diff --git a/observability/loki-config.yml b/observability/loki-config.yml new file mode 100644 index 0000000..16b4167 --- /dev/null +++ b/observability/loki-config.yml @@ -0,0 +1,32 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + +common: + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +query_range: + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 100 diff --git a/observability/promtail-config.yml b/observability/promtail-config.yml new file mode 100644 index 0000000..861c5ff --- /dev/null +++ b/observability/promtail-config.yml @@ -0,0 +1,31 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: + - job_name: docker + docker_sd_configs: + - host: unix:///var/run/docker.sock + refresh_interval: 5s + pipeline_stages: + - docker: {} + relabel_configs: + - source_labels: ['__meta_docker_container_name'] + regex: '/(.*)' + target_label: container + - source_labels: ['__meta_docker_container_id'] + target_label: container_id + - source_labels: ['__meta_docker_container_label_com_docker_compose_service'] + target_label: compose_service + - source_labels: ['__meta_docker_container_label_com_docker_compose_project'] + target_label: compose_project + - source_labels: ['__meta_docker_container_log_stream'] + target_label: stream + - source_labels: ['__meta_docker_container_name'] + target_label: job