From e626995899cda9a66e4e405a3ea6164c8fff8da3 Mon Sep 17 00:00:00 2001 From: AndrewGnev Date: Tue, 24 Feb 2026 08:20:06 +0300 Subject: [PATCH 1/4] WD-696 change MinIO to RustFS as target S3, update deserialization and tests --- .env | 4 ++-- docker-compose.yaml | 11 ++++----- .../storage/s3/policy/dto/Principal.kt | 24 +++++++++++++++++++ .../src/test/kotlin/S3GeneratePreviewTest.kt | 4 ++-- .../src/test/kotlin/S3StorageTest.kt | 8 +++---- 5 files changed, 37 insertions(+), 14 deletions(-) diff --git a/.env b/.env index 0d8bd66..b4e0d8a 100644 --- a/.env +++ b/.env @@ -3,8 +3,8 @@ APP_PORT=8080 S3_ENDPOINT=http://127.0.0.30:9000 S3_REGION=us-east-1 S3_BUCKET=test-bucket -MINIO_ACCESS_KEY=my_access_key -MINIO_SECRET_KEY=my_secret_key +RUSTFS_ACCESS_KEY=my_access_key +RUSTFS_SECRET_KEY=my_secret_key JPG_TEST_OBJECT=100.jpg PNG_TEST_OBJECT=100.png GIF_TEST_OBJECT=100.gif diff --git a/docker-compose.yaml b/docker-compose.yaml index 48dfa01..d341197 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,18 +1,17 @@ version: '3' services: - minio: - image: minio/minio + rustfs: + image: rustfs/rustfs restart: always logging: driver: "json-file" options: max-size: "20m" max-file: "5" - environment: - MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} - MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} ports: - ${APP_HOST}:9000:9000 + environment: + RUSTFS_ACCESS_KEY: ${RUSTFS_ACCESS_KEY} + RUSTFS_SECRET_KEY: ${RUSTFS_SECRET_KEY} volumes: - ./data:/data - command: 'server /data' diff --git a/storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/policy/dto/Principal.kt b/storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/policy/dto/Principal.kt index 6b88ef8..aa7cf20 100644 --- a/storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/policy/dto/Principal.kt +++ b/storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/policy/dto/Principal.kt @@ -2,10 +2,17 @@ package com.icerockdev.service.storage.s3.policy.dto import com.fasterxml.jackson.annotation.JsonInclude import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.core.JsonProcessingException +import com.fasterxml.jackson.databind.DeserializationContext +import com.fasterxml.jackson.databind.JsonDeserializer +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.annotation.JsonDeserialize @JsonInclude(JsonInclude.Include.NON_EMPTY) data class Principal( @JsonProperty("AWS") + @JsonDeserialize(using = PrincipalAwsDeserializer::class) val aws: List, @JsonProperty("CanonicalUser") val canonicalUser: String?, @@ -14,3 +21,20 @@ data class Principal( @JsonProperty("Service") val service: List = emptyList(), ) + +class PrincipalAwsDeserializer : JsonDeserializer>() { + override fun deserialize(p: JsonParser?, ctxt: DeserializationContext?): List { + try { + val path = p?.parsingContext?.pathAsPointer() + val node = p?.codec?.readTree(p) + return when { + node?.isNull == true -> emptyList() + node?.isArray == true -> p.codec?.treeToValue(node, Array::class.java)?.toList() ?: emptyList() + node?.isTextual == true -> p.codec?.treeToValue(node, String::class.java)?.let { listOf(it) } ?: emptyList() + else -> throw IllegalArgumentException("Principal deserialization error. Unexpected value: $node at path: $path") + } + } catch (e: JsonProcessingException) { + throw RuntimeException("Principal deserialization error happened at: ${e.location}", e) + } + } +} diff --git a/storage-service/src/test/kotlin/S3GeneratePreviewTest.kt b/storage-service/src/test/kotlin/S3GeneratePreviewTest.kt index 3750b8d..19d3814 100644 --- a/storage-service/src/test/kotlin/S3GeneratePreviewTest.kt +++ b/storage-service/src/test/kotlin/S3GeneratePreviewTest.kt @@ -42,7 +42,7 @@ class S3GeneratePreviewTest { .credentialsProvider( StaticCredentialsProvider.create( AwsBasicCredentials.create( - dotenv["MINIO_ACCESS_KEY"], dotenv["MINIO_SECRET_KEY"] + dotenv["RUSTFS_ACCESS_KEY"], dotenv["RUSTFS_SECRET_KEY"] ) ) ) @@ -55,7 +55,7 @@ class S3GeneratePreviewTest { .credentialsProvider( StaticCredentialsProvider.create( AwsBasicCredentials.create( - dotenv["MINIO_ACCESS_KEY"], dotenv["MINIO_SECRET_KEY"] + dotenv["RUSTFS_ACCESS_KEY"], dotenv["RUSTFS_SECRET_KEY"] ) ) ) diff --git a/storage-service/src/test/kotlin/S3StorageTest.kt b/storage-service/src/test/kotlin/S3StorageTest.kt index 490f905..5c38632 100644 --- a/storage-service/src/test/kotlin/S3StorageTest.kt +++ b/storage-service/src/test/kotlin/S3StorageTest.kt @@ -59,7 +59,7 @@ class S3StorageTest { .credentialsProvider( StaticCredentialsProvider.create( AwsBasicCredentials.create( - dotenv["MINIO_ACCESS_KEY"], dotenv["MINIO_SECRET_KEY"] + dotenv["RUSTFS_ACCESS_KEY"], dotenv["RUSTFS_SECRET_KEY"] ) ) ) @@ -72,7 +72,7 @@ class S3StorageTest { .credentialsProvider( StaticCredentialsProvider.create( AwsBasicCredentials.create( - dotenv["MINIO_ACCESS_KEY"], dotenv["MINIO_SECRET_KEY"] + dotenv["RUSTFS_ACCESS_KEY"], dotenv["RUSTFS_SECRET_KEY"] ) ) ) @@ -338,7 +338,7 @@ class S3StorageTest { val jpgObject = storage.get(bucketName, jpgFileName) - assertEquals(metadata, jpgObject?.response()?.metadata()) + assertTrue(jpgObject?.response()?.metadata()?.entries?.containsAll(metadata.entries) ?: false) // copy testing val copyFileName = storage.generateFileKey() @@ -349,7 +349,7 @@ class S3StorageTest { val copyObject = storage.get(bucketName, copyFileName) - assertEquals(metadata, copyObject?.response()?.metadata()) + assertTrue(copyObject?.response()?.metadata()?.entries?.containsAll(metadata.entries) ?: false) assertTrue { storage.deleteBucketWithObjects(bucketName) From 58bc3d6451be3934e67eb1c4a7c05d351bef4290 Mon Sep 17 00:00:00 2001 From: AndrewGnev Date: Fri, 27 Feb 2026 11:32:08 +0300 Subject: [PATCH 2/4] WD-696 mr fixes --- .env | 4 ++-- sample/src/main/kotlin/com/icerockdev/sample/Main.kt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.env b/.env index b4e0d8a..1c3d8cb 100644 --- a/.env +++ b/.env @@ -1,6 +1,6 @@ -APP_HOST=127.0.0.30 +APP_HOST=127.0.0.1 APP_PORT=8080 -S3_ENDPOINT=http://127.0.0.30:9000 +S3_ENDPOINT=http://127.0.0.1:9000 S3_REGION=us-east-1 S3_BUCKET=test-bucket RUSTFS_ACCESS_KEY=my_access_key diff --git a/sample/src/main/kotlin/com/icerockdev/sample/Main.kt b/sample/src/main/kotlin/com/icerockdev/sample/Main.kt index b19537b..1944d18 100644 --- a/sample/src/main/kotlin/com/icerockdev/sample/Main.kt +++ b/sample/src/main/kotlin/com/icerockdev/sample/Main.kt @@ -35,7 +35,7 @@ object Main { .credentialsProvider( StaticCredentialsProvider.create( AwsBasicCredentials.create( - dotenv["MINIO_ACCESS_KEY"], dotenv["MINIO_SECRET_KEY"] + dotenv["RUSTFS_ACCESS_KEY"], dotenv["RUSTFS_SECRET_KEY"] ) ) ) @@ -48,7 +48,7 @@ object Main { .credentialsProvider( StaticCredentialsProvider.create( AwsBasicCredentials.create( - dotenv["MINIO_ACCESS_KEY"], dotenv["MINIO_SECRET_KEY"] + dotenv["RUSTFS_ACCESS_KEY"], dotenv["RUSTFS_SECRET_KEY"] ) ) ) From a18db5e4676be5d6833bf8d9c87e021443a41871 Mon Sep 17 00:00:00 2001 From: AndrewGnev Date: Sun, 1 Mar 2026 01:53:48 +0300 Subject: [PATCH 3/4] WD-696 mr fix --- .env | 4 ++-- docker-compose.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.env b/.env index 1c3d8cb..14be57b 100644 --- a/.env +++ b/.env @@ -1,6 +1,6 @@ -APP_HOST=127.0.0.1 +APP_HOST=localhost APP_PORT=8080 -S3_ENDPOINT=http://127.0.0.1:9000 +S3_ENDPOINT=http://localhost:9001 S3_REGION=us-east-1 S3_BUCKET=test-bucket RUSTFS_ACCESS_KEY=my_access_key diff --git a/docker-compose.yaml b/docker-compose.yaml index d341197..eae999e 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -9,7 +9,7 @@ services: max-size: "20m" max-file: "5" ports: - - ${APP_HOST}:9000:9000 + - 9001:9000 environment: RUSTFS_ACCESS_KEY: ${RUSTFS_ACCESS_KEY} RUSTFS_SECRET_KEY: ${RUSTFS_SECRET_KEY} From 290c2cf2b88ce65cd0f6830683498ef81f6947e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=F0=9F=98=8B=20Mansur?= Date: Tue, 24 Mar 2026 17:19:40 +0300 Subject: [PATCH 4/4] WD-696 fix tests, add backwards compatibility --- .env | 12 ++++-- .gitignore | 2 + README.md | 2 +- docker-compose.yaml | 40 ++++++++++++++++--- .../main/kotlin/com/icerockdev/sample/Main.kt | 10 ++--- storage-service/build.gradle.kts | 2 +- .../service/storage/s3/IS3Storage.kt | 2 +- .../src/test/kotlin/S3GeneratePreviewTest.kt | 10 ++--- .../src/test/kotlin/S3StorageTest.kt | 10 ++--- 9 files changed, 63 insertions(+), 27 deletions(-) diff --git a/.env b/.env index 14be57b..bd44c79 100644 --- a/.env +++ b/.env @@ -1,10 +1,14 @@ -APP_HOST=localhost +STORAGE_PROVIDER=rustfs +# or +# STORAGE_PROVIDER=minio +COMPOSE_PROFILES=${STORAGE_PROVIDER} +APP_HOST=127.0.0.30 APP_PORT=8080 -S3_ENDPOINT=http://localhost:9001 +S3_ENDPOINT=http://127.0.0.30:9000 S3_REGION=us-east-1 S3_BUCKET=test-bucket -RUSTFS_ACCESS_KEY=my_access_key -RUSTFS_SECRET_KEY=my_secret_key +S3_ACCESS_KEY=my_access_key +S3_SECRET_KEY=my_secret_key JPG_TEST_OBJECT=100.jpg PNG_TEST_OBJECT=100.png GIF_TEST_OBJECT=100.gif diff --git a/.gitignore b/.gitignore index 54fcd05..f573da2 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ build /.gradle /.idea /data +/minio-data +/rustfs-data diff --git a/README.md b/README.md index d56aa02..487b665 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ repositories { } // Append dependency -implementation("com.icerockdev:storage-service:0.10.0") +implementation("com.icerockdev:storage-service:0.11.0") ```` ## Library usage diff --git a/docker-compose.yaml b/docker-compose.yaml index eae999e..ee4c02b 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -3,15 +3,45 @@ services: rustfs: image: rustfs/rustfs restart: always - logging: + logging: &logging driver: "json-file" options: max-size: "20m" max-file: "5" ports: - - 9001:9000 + - "${APP_HOST}:9000:9000" + - "${APP_HOST}:9001:9001" environment: - RUSTFS_ACCESS_KEY: ${RUSTFS_ACCESS_KEY} - RUSTFS_SECRET_KEY: ${RUSTFS_SECRET_KEY} + RUSTFS_ACCESS_KEY: ${S3_ACCESS_KEY} + RUSTFS_SECRET_KEY: ${S3_SECRET_KEY} volumes: - - ./data:/data + - ./rustfs-data:/data + profiles: + - rustfs + rustfs-volume-permission-helper: + image: alpine + volumes: + - ./rustfs-data:/data + command: > + sh -c " + chown -R 10001:10001 /data && + echo 'Volume Permissions fixed' && + exit 0 + " + restart: "no" + profiles: + - rustfs + minio: + image: minio/minio + restart: always + logging: *logging + environment: + MINIO_ACCESS_KEY: ${S3_ACCESS_KEY} + MINIO_SECRET_KEY: ${S3_SECRET_KEY} + ports: + - "${APP_HOST}:9000:9000" + volumes: + - ./minio-data:/data + command: 'server /data' + profiles: + - minio diff --git a/sample/src/main/kotlin/com/icerockdev/sample/Main.kt b/sample/src/main/kotlin/com/icerockdev/sample/Main.kt index 1944d18..e2cf2a3 100644 --- a/sample/src/main/kotlin/com/icerockdev/sample/Main.kt +++ b/sample/src/main/kotlin/com/icerockdev/sample/Main.kt @@ -5,7 +5,7 @@ package com.icerockdev.sample import com.icerockdev.service.storage.s3.S3StorageImpl -import com.icerockdev.service.storage.s3.minioConfBuilder +import com.icerockdev.service.storage.s3.s3Configuration import io.github.cdimascio.dotenv.dotenv import io.ktor.http.ContentType import io.ktor.http.content.PartData @@ -31,11 +31,11 @@ import software.amazon.awssdk.services.s3.presigner.S3Presigner object Main { private val dotenv = dotenv() private val s3 = S3Client.builder() - .serviceConfiguration(minioConfBuilder) + .serviceConfiguration(s3Configuration) .credentialsProvider( StaticCredentialsProvider.create( AwsBasicCredentials.create( - dotenv["RUSTFS_ACCESS_KEY"], dotenv["RUSTFS_SECRET_KEY"] + dotenv["S3_ACCESS_KEY"], dotenv["S3_SECRET_KEY"] ) ) ) @@ -44,11 +44,11 @@ object Main { .build() private val preSigner = S3Presigner.builder() - .serviceConfiguration(minioConfBuilder) + .serviceConfiguration(s3Configuration) .credentialsProvider( StaticCredentialsProvider.create( AwsBasicCredentials.create( - dotenv["RUSTFS_ACCESS_KEY"], dotenv["RUSTFS_SECRET_KEY"] + dotenv["S3_ACCESS_KEY"], dotenv["S3_SECRET_KEY"] ) ) ) diff --git a/storage-service/build.gradle.kts b/storage-service/build.gradle.kts index 8ab2cd5..ee332bd 100644 --- a/storage-service/build.gradle.kts +++ b/storage-service/build.gradle.kts @@ -17,7 +17,7 @@ apply(plugin = "java") apply(plugin = "kotlin") group = "com.icerockdev" -version = "0.10.0" +version = "0.11.0" val sourcesJar by tasks.registering(Jar::class) { archiveClassifier.set("sources") diff --git a/storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/IS3Storage.kt b/storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/IS3Storage.kt index e7b4951..1ce2d54 100644 --- a/storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/IS3Storage.kt +++ b/storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/IS3Storage.kt @@ -77,7 +77,7 @@ interface IS3Storage { fun buildResource(configure: ResourceBuilder.() -> Unit): String } -val minioConfBuilder: S3Configuration = +val s3Configuration: S3Configuration = S3Configuration .builder() .pathStyleAccessEnabled(true) diff --git a/storage-service/src/test/kotlin/S3GeneratePreviewTest.kt b/storage-service/src/test/kotlin/S3GeneratePreviewTest.kt index 19d3814..8493ffe 100644 --- a/storage-service/src/test/kotlin/S3GeneratePreviewTest.kt +++ b/storage-service/src/test/kotlin/S3GeneratePreviewTest.kt @@ -11,7 +11,7 @@ import com.icerockdev.service.storage.preview.boundImage import com.icerockdev.service.storage.preview.loadImage import com.icerockdev.service.storage.s3.IS3Storage import com.icerockdev.service.storage.s3.S3StorageImpl -import com.icerockdev.service.storage.s3.minioConfBuilder +import com.icerockdev.service.storage.s3.s3Configuration import io.github.cdimascio.dotenv.dotenv import kotlinx.coroutines.runBlocking import org.junit.After @@ -38,11 +38,11 @@ class S3GeneratePreviewTest { @Before fun init() { s3 = S3Client.builder() - .serviceConfiguration(minioConfBuilder) + .serviceConfiguration(s3Configuration) .credentialsProvider( StaticCredentialsProvider.create( AwsBasicCredentials.create( - dotenv["RUSTFS_ACCESS_KEY"], dotenv["RUSTFS_SECRET_KEY"] + dotenv["S3_ACCESS_KEY"], dotenv["S3_SECRET_KEY"] ) ) ) @@ -51,11 +51,11 @@ class S3GeneratePreviewTest { .build() preSigner = S3Presigner.builder() - .serviceConfiguration(minioConfBuilder) + .serviceConfiguration(s3Configuration) .credentialsProvider( StaticCredentialsProvider.create( AwsBasicCredentials.create( - dotenv["RUSTFS_ACCESS_KEY"], dotenv["RUSTFS_SECRET_KEY"] + dotenv["S3_ACCESS_KEY"], dotenv["S3_SECRET_KEY"] ) ) ) diff --git a/storage-service/src/test/kotlin/S3StorageTest.kt b/storage-service/src/test/kotlin/S3StorageTest.kt index 5c38632..f007e1c 100644 --- a/storage-service/src/test/kotlin/S3StorageTest.kt +++ b/storage-service/src/test/kotlin/S3StorageTest.kt @@ -7,7 +7,7 @@ import com.icerockdev.service.storage.exception.S3StorageException import com.icerockdev.service.storage.mime.MimeTypeDetector import com.icerockdev.service.storage.s3.IS3Storage import com.icerockdev.service.storage.s3.S3StorageImpl -import com.icerockdev.service.storage.s3.minioConfBuilder +import com.icerockdev.service.storage.s3.s3Configuration import com.icerockdev.service.storage.s3.dto.FileObjectDto import com.icerockdev.service.storage.s3.policy.dto.ActionEnum import com.icerockdev.service.storage.s3.policy.dto.EffectEnum @@ -55,11 +55,11 @@ class S3StorageTest { @Before fun init() { s3 = S3Client.builder() - .serviceConfiguration(minioConfBuilder) + .serviceConfiguration(s3Configuration) .credentialsProvider( StaticCredentialsProvider.create( AwsBasicCredentials.create( - dotenv["RUSTFS_ACCESS_KEY"], dotenv["RUSTFS_SECRET_KEY"] + dotenv["S3_ACCESS_KEY"], dotenv["S3_SECRET_KEY"] ) ) ) @@ -68,11 +68,11 @@ class S3StorageTest { .build() preSigner = S3Presigner.builder() - .serviceConfiguration(minioConfBuilder) + .serviceConfiguration(s3Configuration) .credentialsProvider( StaticCredentialsProvider.create( AwsBasicCredentials.create( - dotenv["RUSTFS_ACCESS_KEY"], dotenv["RUSTFS_SECRET_KEY"] + dotenv["S3_ACCESS_KEY"], dotenv["S3_SECRET_KEY"] ) ) )