WD-696 change MinIO to RustFS as target S3, update deserialization and tests

.env

@@ -1,10 +1,14 @@
+STORAGE_PROVIDER=rustfs
+# or
+# STORAGE_PROVIDER=minio
+COMPOSE_PROFILES=${STORAGE_PROVIDER}
 APP_HOST=127.0.0.30
 APP_PORT=8080
 S3_ENDPOINT=http://127.0.0.30:9000
 S3_REGION=us-east-1
 S3_BUCKET=test-bucket
-MINIO_ACCESS_KEY=my_access_key
-MINIO_SECRET_KEY=my_secret_key
+S3_ACCESS_KEY=my_access_key
+S3_SECRET_KEY=my_secret_key
 JPG_TEST_OBJECT=100.jpg
 PNG_TEST_OBJECT=100.png
 GIF_TEST_OBJECT=100.gif

.gitignore

@@ -2,3 +2,5 @@ build
 /.gradle
 /.idea
 /data
+/minio-data
+/rustfs-data

README.md

@@ -10,7 +10,7 @@ repositories {
 }
 
 // Append dependency
-implementation("com.icerockdev:storage-service:0.10.0")
+implementation("com.icerockdev:storage-service:0.11.0")
 ````
 
 ## Library usage

docker-compose.yaml

@@ -1,18 +1,47 @@
 version: '3'
 services:
-  minio:
-    image: minio/minio
+  rustfs:
+    image: rustfs/rustfs
     restart: always
-    logging:
+    logging: &logging
       driver: "json-file"
       options:
         max-size: "20m"
         max-file: "5"
+    ports:
+      - "${APP_HOST}:9000:9000"
+      - "${APP_HOST}:9001:9001"
+    environment:
+      RUSTFS_ACCESS_KEY: ${S3_ACCESS_KEY}
+      RUSTFS_SECRET_KEY: ${S3_SECRET_KEY}
+    volumes:
+      - ./rustfs-data:/data
+    profiles:
+      - rustfs
+  rustfs-volume-permission-helper:
+    image: alpine
+    volumes:
+      - ./rustfs-data:/data
+    command: >
+      sh -c "
+        chown -R 10001:10001 /data &&
+        echo 'Volume Permissions fixed' &&
+        exit 0
+      "
+    restart: "no"
+    profiles:
+      - rustfs
+  minio:
+    image: minio/minio
+    restart: always
+    logging: *logging
     environment:
-      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
-      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
+      MINIO_ACCESS_KEY: ${S3_ACCESS_KEY}
+      MINIO_SECRET_KEY: ${S3_SECRET_KEY}
     ports:
-      - ${APP_HOST}:9000:9000
+      - "${APP_HOST}:9000:9000"
     volumes:
-      - ./data:/data
+      - ./minio-data:/data
     command: 'server /data'
+    profiles:
+      - minio

sample/src/main/kotlin/com/icerockdev/sample/Main.kt

@@ -5,7 +5,7 @@
 package com.icerockdev.sample
 
 import com.icerockdev.service.storage.s3.S3StorageImpl
-import com.icerockdev.service.storage.s3.minioConfBuilder
+import com.icerockdev.service.storage.s3.s3Configuration
 import io.github.cdimascio.dotenv.dotenv
 import io.ktor.http.ContentType
 import io.ktor.http.content.PartData
@@ -31,11 +31,11 @@ import software.amazon.awssdk.services.s3.presigner.S3Presigner
 object Main {
     private val dotenv = dotenv()
     private val s3 = S3Client.builder()
-        .serviceConfiguration(minioConfBuilder)
+        .serviceConfiguration(s3Configuration)
         .credentialsProvider(
             StaticCredentialsProvider.create(
                 AwsBasicCredentials.create(
-                    dotenv["MINIO_ACCESS_KEY"], dotenv["MINIO_SECRET_KEY"]
+                    dotenv["S3_ACCESS_KEY"], dotenv["S3_SECRET_KEY"]
                 )
             )
         )
@@ -44,11 +44,11 @@ object Main {
         .build()
 
     private val preSigner = S3Presigner.builder()
-        .serviceConfiguration(minioConfBuilder)
+        .serviceConfiguration(s3Configuration)
         .credentialsProvider(
             StaticCredentialsProvider.create(
                 AwsBasicCredentials.create(
-                    dotenv["MINIO_ACCESS_KEY"], dotenv["MINIO_SECRET_KEY"]
+                    dotenv["S3_ACCESS_KEY"], dotenv["S3_SECRET_KEY"]
                 )
             )
         )

storage-service/build.gradle.kts

@@ -17,7 +17,7 @@ apply(plugin = "java")
 apply(plugin = "kotlin")
 
 group = "com.icerockdev"
-version = "0.10.0"
+version = "0.11.0"
 
 val sourcesJar by tasks.registering(Jar::class) {
     archiveClassifier.set("sources")

storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/IS3Storage.kt

@@ -77,7 +77,7 @@ interface IS3Storage {
     fun buildResource(configure: ResourceBuilder.() -> Unit): String
 }
 
-val minioConfBuilder: S3Configuration =
+val s3Configuration: S3Configuration =
     S3Configuration
         .builder()
         .pathStyleAccessEnabled(true)

storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/policy/dto/Principal.kt

@@ -2,10 +2,17 @@ package com.icerockdev.service.storage.s3.policy.dto
 
 import com.fasterxml.jackson.annotation.JsonInclude
 import com.fasterxml.jackson.annotation.JsonProperty
+import com.fasterxml.jackson.core.JsonParser
+import com.fasterxml.jackson.core.JsonProcessingException
+import com.fasterxml.jackson.databind.DeserializationContext
+import com.fasterxml.jackson.databind.JsonDeserializer
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize
 
 @JsonInclude(JsonInclude.Include.NON_EMPTY)
 data class Principal(
     @JsonProperty("AWS")
+    @JsonDeserialize(using = PrincipalAwsDeserializer::class)
     val aws: List<String>,
     @JsonProperty("CanonicalUser")
     val canonicalUser: String?,
@@ -14,3 +21,20 @@ data class Principal(
     @JsonProperty("Service")
     val service: List<String> = emptyList(),
 )
+
+class PrincipalAwsDeserializer : JsonDeserializer<List<String>>() {
+    override fun deserialize(p: JsonParser?, ctxt: DeserializationContext?): List<String> {
+        try {
+            val path = p?.parsingContext?.pathAsPointer()
+            val node = p?.codec?.readTree<JsonNode>(p)
+            return when {
+                node?.isNull == true -> emptyList()
+                node?.isArray == true -> p.codec?.treeToValue(node, Array<String>::class.java)?.toList() ?: emptyList()
+                node?.isTextual == true -> p.codec?.treeToValue(node, String::class.java)?.let { listOf(it) } ?: emptyList()
+                else -> throw IllegalArgumentException("Principal deserialization error. Unexpected value: $node at path: $path")
+            }
+        } catch (e: JsonProcessingException) {
+            throw RuntimeException("Principal deserialization error happened at: ${e.location}", e)
+        }
+    }
+}

storage-service/src/test/kotlin/S3GeneratePreviewTest.kt

@@ -11,7 +11,7 @@ import com.icerockdev.service.storage.preview.boundImage
 import com.icerockdev.service.storage.preview.loadImage
 import com.icerockdev.service.storage.s3.IS3Storage
 import com.icerockdev.service.storage.s3.S3StorageImpl
-import com.icerockdev.service.storage.s3.minioConfBuilder
+import com.icerockdev.service.storage.s3.s3Configuration
 import io.github.cdimascio.dotenv.dotenv
 import kotlinx.coroutines.runBlocking
 import org.junit.After
@@ -38,11 +38,11 @@ class S3GeneratePreviewTest {
     @Before
     fun init() {
         s3 = S3Client.builder()
-            .serviceConfiguration(minioConfBuilder)
+            .serviceConfiguration(s3Configuration)
             .credentialsProvider(
                 StaticCredentialsProvider.create(
                     AwsBasicCredentials.create(
-                        dotenv["MINIO_ACCESS_KEY"], dotenv["MINIO_SECRET_KEY"]
+                        dotenv["S3_ACCESS_KEY"], dotenv["S3_SECRET_KEY"]
                     )
                 )
             )
@@ -51,11 +51,11 @@ class S3GeneratePreviewTest {
             .build()
 
         preSigner = S3Presigner.builder()
-            .serviceConfiguration(minioConfBuilder)
+            .serviceConfiguration(s3Configuration)
             .credentialsProvider(
                 StaticCredentialsProvider.create(
                     AwsBasicCredentials.create(
-                        dotenv["MINIO_ACCESS_KEY"], dotenv["MINIO_SECRET_KEY"]
+                        dotenv["S3_ACCESS_KEY"], dotenv["S3_SECRET_KEY"]
                     )
                 )
             )

storage-service/src/test/kotlin/S3StorageTest.kt

@@ -7,7 +7,7 @@ import com.icerockdev.service.storage.exception.S3StorageException
 import com.icerockdev.service.storage.mime.MimeTypeDetector
 import com.icerockdev.service.storage.s3.IS3Storage
 import com.icerockdev.service.storage.s3.S3StorageImpl
-import com.icerockdev.service.storage.s3.minioConfBuilder
+import com.icerockdev.service.storage.s3.s3Configuration
 import com.icerockdev.service.storage.s3.dto.FileObjectDto
 import com.icerockdev.service.storage.s3.policy.dto.ActionEnum
 import com.icerockdev.service.storage.s3.policy.dto.EffectEnum
@@ -55,11 +55,11 @@ class S3StorageTest {
     @Before
     fun init() {
         s3 = S3Client.builder()
-            .serviceConfiguration(minioConfBuilder)
+            .serviceConfiguration(s3Configuration)
             .credentialsProvider(
                 StaticCredentialsProvider.create(
                     AwsBasicCredentials.create(
-                        dotenv["MINIO_ACCESS_KEY"], dotenv["MINIO_SECRET_KEY"]
+                        dotenv["S3_ACCESS_KEY"], dotenv["S3_SECRET_KEY"]
                     )
                 )
             )
@@ -68,11 +68,11 @@ class S3StorageTest {
             .build()
 
         preSigner = S3Presigner.builder()
-            .serviceConfiguration(minioConfBuilder)
+            .serviceConfiguration(s3Configuration)
             .credentialsProvider(
                 StaticCredentialsProvider.create(
                     AwsBasicCredentials.create(
-                        dotenv["MINIO_ACCESS_KEY"], dotenv["MINIO_SECRET_KEY"]
+                        dotenv["S3_ACCESS_KEY"], dotenv["S3_SECRET_KEY"]
                     )
                 )
             )
@@ -338,7 +338,7 @@ class S3StorageTest {
 
         val jpgObject = storage.get(bucketName, jpgFileName)
 
-        assertEquals(metadata, jpgObject?.response()?.metadata())
+        assertTrue(jpgObject?.response()?.metadata()?.entries?.containsAll(metadata.entries) ?: false)
 
         // copy testing
         val copyFileName = storage.generateFileKey()
@@ -349,7 +349,7 @@ class S3StorageTest {
 
         val copyObject = storage.get(bucketName, copyFileName)
 
-        assertEquals(metadata, copyObject?.response()?.metadata())
+        assertTrue(copyObject?.response()?.metadata()?.entries?.containsAll(metadata.entries) ?: false)
 
         assertTrue {
             storage.deleteBucketWithObjects(bucketName)