diff --git a/.env b/.env index 0d8bd66..bd44c79 100644 --- a/.env +++ b/.env @@ -1,10 +1,14 @@ +STORAGE_PROVIDER=rustfs +# or +# STORAGE_PROVIDER=minio +COMPOSE_PROFILES=${STORAGE_PROVIDER} APP_HOST=127.0.0.30 APP_PORT=8080 S3_ENDPOINT=http://127.0.0.30:9000 S3_REGION=us-east-1 S3_BUCKET=test-bucket -MINIO_ACCESS_KEY=my_access_key -MINIO_SECRET_KEY=my_secret_key +S3_ACCESS_KEY=my_access_key +S3_SECRET_KEY=my_secret_key JPG_TEST_OBJECT=100.jpg PNG_TEST_OBJECT=100.png GIF_TEST_OBJECT=100.gif diff --git a/.gitignore b/.gitignore index 54fcd05..f573da2 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ build /.gradle /.idea /data +/minio-data +/rustfs-data diff --git a/README.md b/README.md index d56aa02..487b665 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ repositories { } // Append dependency -implementation("com.icerockdev:storage-service:0.10.0") +implementation("com.icerockdev:storage-service:0.11.0") ```` ## Library usage diff --git a/docker-compose.yaml b/docker-compose.yaml index 48dfa01..ee4c02b 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,18 +1,47 @@ version: '3' services: - minio: - image: minio/minio + rustfs: + image: rustfs/rustfs restart: always - logging: + logging: &logging driver: "json-file" options: max-size: "20m" max-file: "5" + ports: + - "${APP_HOST}:9000:9000" + - "${APP_HOST}:9001:9001" + environment: + RUSTFS_ACCESS_KEY: ${S3_ACCESS_KEY} + RUSTFS_SECRET_KEY: ${S3_SECRET_KEY} + volumes: + - ./rustfs-data:/data + profiles: + - rustfs + rustfs-volume-permission-helper: + image: alpine + volumes: + - ./rustfs-data:/data + command: > + sh -c " + chown -R 10001:10001 /data && + echo 'Volume Permissions fixed' && + exit 0 + " + restart: "no" + profiles: + - rustfs + minio: + image: minio/minio + restart: always + logging: *logging environment: - MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} - MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_ACCESS_KEY: ${S3_ACCESS_KEY} + MINIO_SECRET_KEY: ${S3_SECRET_KEY} ports: - - ${APP_HOST}:9000:9000 + - "${APP_HOST}:9000:9000" volumes: - - ./data:/data + - ./minio-data:/data command: 'server /data' + profiles: + - minio diff --git a/sample/src/main/kotlin/com/icerockdev/sample/Main.kt b/sample/src/main/kotlin/com/icerockdev/sample/Main.kt index b19537b..e2cf2a3 100644 --- a/sample/src/main/kotlin/com/icerockdev/sample/Main.kt +++ b/sample/src/main/kotlin/com/icerockdev/sample/Main.kt @@ -5,7 +5,7 @@ package com.icerockdev.sample import com.icerockdev.service.storage.s3.S3StorageImpl -import com.icerockdev.service.storage.s3.minioConfBuilder +import com.icerockdev.service.storage.s3.s3Configuration import io.github.cdimascio.dotenv.dotenv import io.ktor.http.ContentType import io.ktor.http.content.PartData @@ -31,11 +31,11 @@ import software.amazon.awssdk.services.s3.presigner.S3Presigner object Main { private val dotenv = dotenv() private val s3 = S3Client.builder() - .serviceConfiguration(minioConfBuilder) + .serviceConfiguration(s3Configuration) .credentialsProvider( StaticCredentialsProvider.create( AwsBasicCredentials.create( - dotenv["MINIO_ACCESS_KEY"], dotenv["MINIO_SECRET_KEY"] + dotenv["S3_ACCESS_KEY"], dotenv["S3_SECRET_KEY"] ) ) ) @@ -44,11 +44,11 @@ object Main { .build() private val preSigner = S3Presigner.builder() - .serviceConfiguration(minioConfBuilder) + .serviceConfiguration(s3Configuration) .credentialsProvider( StaticCredentialsProvider.create( AwsBasicCredentials.create( - dotenv["MINIO_ACCESS_KEY"], dotenv["MINIO_SECRET_KEY"] + dotenv["S3_ACCESS_KEY"], dotenv["S3_SECRET_KEY"] ) ) ) diff --git a/storage-service/build.gradle.kts b/storage-service/build.gradle.kts index 8ab2cd5..ee332bd 100644 --- a/storage-service/build.gradle.kts +++ b/storage-service/build.gradle.kts @@ -17,7 +17,7 @@ apply(plugin = "java") apply(plugin = "kotlin") group = "com.icerockdev" -version = "0.10.0" +version = "0.11.0" val sourcesJar by tasks.registering(Jar::class) { archiveClassifier.set("sources") diff --git a/storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/IS3Storage.kt b/storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/IS3Storage.kt index e7b4951..1ce2d54 100644 --- a/storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/IS3Storage.kt +++ b/storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/IS3Storage.kt @@ -77,7 +77,7 @@ interface IS3Storage { fun buildResource(configure: ResourceBuilder.() -> Unit): String } -val minioConfBuilder: S3Configuration = +val s3Configuration: S3Configuration = S3Configuration .builder() .pathStyleAccessEnabled(true) diff --git a/storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/policy/dto/Principal.kt b/storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/policy/dto/Principal.kt index 6b88ef8..aa7cf20 100644 --- a/storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/policy/dto/Principal.kt +++ b/storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/policy/dto/Principal.kt @@ -2,10 +2,17 @@ package com.icerockdev.service.storage.s3.policy.dto import com.fasterxml.jackson.annotation.JsonInclude import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.core.JsonProcessingException +import com.fasterxml.jackson.databind.DeserializationContext +import com.fasterxml.jackson.databind.JsonDeserializer +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.annotation.JsonDeserialize @JsonInclude(JsonInclude.Include.NON_EMPTY) data class Principal( @JsonProperty("AWS") + @JsonDeserialize(using = PrincipalAwsDeserializer::class) val aws: List, @JsonProperty("CanonicalUser") val canonicalUser: String?, @@ -14,3 +21,20 @@ data class Principal( @JsonProperty("Service") val service: List = emptyList(), ) + +class PrincipalAwsDeserializer : JsonDeserializer>() { + override fun deserialize(p: JsonParser?, ctxt: DeserializationContext?): List { + try { + val path = p?.parsingContext?.pathAsPointer() + val node = p?.codec?.readTree(p) + return when { + node?.isNull == true -> emptyList() + node?.isArray == true -> p.codec?.treeToValue(node, Array::class.java)?.toList() ?: emptyList() + node?.isTextual == true -> p.codec?.treeToValue(node, String::class.java)?.let { listOf(it) } ?: emptyList() + else -> throw IllegalArgumentException("Principal deserialization error. Unexpected value: $node at path: $path") + } + } catch (e: JsonProcessingException) { + throw RuntimeException("Principal deserialization error happened at: ${e.location}", e) + } + } +} diff --git a/storage-service/src/test/kotlin/S3GeneratePreviewTest.kt b/storage-service/src/test/kotlin/S3GeneratePreviewTest.kt index 3750b8d..8493ffe 100644 --- a/storage-service/src/test/kotlin/S3GeneratePreviewTest.kt +++ b/storage-service/src/test/kotlin/S3GeneratePreviewTest.kt @@ -11,7 +11,7 @@ import com.icerockdev.service.storage.preview.boundImage import com.icerockdev.service.storage.preview.loadImage import com.icerockdev.service.storage.s3.IS3Storage import com.icerockdev.service.storage.s3.S3StorageImpl -import com.icerockdev.service.storage.s3.minioConfBuilder +import com.icerockdev.service.storage.s3.s3Configuration import io.github.cdimascio.dotenv.dotenv import kotlinx.coroutines.runBlocking import org.junit.After @@ -38,11 +38,11 @@ class S3GeneratePreviewTest { @Before fun init() { s3 = S3Client.builder() - .serviceConfiguration(minioConfBuilder) + .serviceConfiguration(s3Configuration) .credentialsProvider( StaticCredentialsProvider.create( AwsBasicCredentials.create( - dotenv["MINIO_ACCESS_KEY"], dotenv["MINIO_SECRET_KEY"] + dotenv["S3_ACCESS_KEY"], dotenv["S3_SECRET_KEY"] ) ) ) @@ -51,11 +51,11 @@ class S3GeneratePreviewTest { .build() preSigner = S3Presigner.builder() - .serviceConfiguration(minioConfBuilder) + .serviceConfiguration(s3Configuration) .credentialsProvider( StaticCredentialsProvider.create( AwsBasicCredentials.create( - dotenv["MINIO_ACCESS_KEY"], dotenv["MINIO_SECRET_KEY"] + dotenv["S3_ACCESS_KEY"], dotenv["S3_SECRET_KEY"] ) ) ) diff --git a/storage-service/src/test/kotlin/S3StorageTest.kt b/storage-service/src/test/kotlin/S3StorageTest.kt index 490f905..f007e1c 100644 --- a/storage-service/src/test/kotlin/S3StorageTest.kt +++ b/storage-service/src/test/kotlin/S3StorageTest.kt @@ -7,7 +7,7 @@ import com.icerockdev.service.storage.exception.S3StorageException import com.icerockdev.service.storage.mime.MimeTypeDetector import com.icerockdev.service.storage.s3.IS3Storage import com.icerockdev.service.storage.s3.S3StorageImpl -import com.icerockdev.service.storage.s3.minioConfBuilder +import com.icerockdev.service.storage.s3.s3Configuration import com.icerockdev.service.storage.s3.dto.FileObjectDto import com.icerockdev.service.storage.s3.policy.dto.ActionEnum import com.icerockdev.service.storage.s3.policy.dto.EffectEnum @@ -55,11 +55,11 @@ class S3StorageTest { @Before fun init() { s3 = S3Client.builder() - .serviceConfiguration(minioConfBuilder) + .serviceConfiguration(s3Configuration) .credentialsProvider( StaticCredentialsProvider.create( AwsBasicCredentials.create( - dotenv["MINIO_ACCESS_KEY"], dotenv["MINIO_SECRET_KEY"] + dotenv["S3_ACCESS_KEY"], dotenv["S3_SECRET_KEY"] ) ) ) @@ -68,11 +68,11 @@ class S3StorageTest { .build() preSigner = S3Presigner.builder() - .serviceConfiguration(minioConfBuilder) + .serviceConfiguration(s3Configuration) .credentialsProvider( StaticCredentialsProvider.create( AwsBasicCredentials.create( - dotenv["MINIO_ACCESS_KEY"], dotenv["MINIO_SECRET_KEY"] + dotenv["S3_ACCESS_KEY"], dotenv["S3_SECRET_KEY"] ) ) ) @@ -338,7 +338,7 @@ class S3StorageTest { val jpgObject = storage.get(bucketName, jpgFileName) - assertEquals(metadata, jpgObject?.response()?.metadata()) + assertTrue(jpgObject?.response()?.metadata()?.entries?.containsAll(metadata.entries) ?: false) // copy testing val copyFileName = storage.generateFileKey() @@ -349,7 +349,7 @@ class S3StorageTest { val copyObject = storage.get(bucketName, copyFileName) - assertEquals(metadata, copyObject?.response()?.metadata()) + assertTrue(copyObject?.response()?.metadata()?.entries?.containsAll(metadata.entries) ?: false) assertTrue { storage.deleteBucketWithObjects(bucketName)