From 1d97e2d59aaa4d9298be1d07bfe7c98dc6b10208 Mon Sep 17 00:00:00 2001 From: Hector Ventura Date: Sun, 29 Mar 2026 23:05:13 -0500 Subject: [PATCH 01/32] fix(storage): fix storage global config issue and memory s3 directory creation --- docs/configuration/application-yml.md | 11 --- docs/configuration/storage.md | 64 ++++++--------- .../floci/config/EmulatorConfig.java | 30 +++---- .../floci/core/storage/StorageFactory.java | 21 ++--- .../floci/services/s3/S3Service.java | 82 ++++++++++++++----- src/main/resources/application.yml | 12 --- src/test/resources/application.yml | 11 --- 7 files changed, 110 insertions(+), 121 deletions(-) diff --git a/docs/configuration/application-yml.md b/docs/configuration/application-yml.md index a4e3807e..e4256d5f 100644 --- a/docs/configuration/application-yml.md +++ b/docs/configuration/application-yml.md @@ -27,29 +27,18 @@ floci: compaction-interval-ms: 30000 services: ssm: - mode: memory flush-interval-ms: 5000 - sqs: - mode: memory - s3: - mode: memory dynamodb: - mode: memory flush-interval-ms: 5000 sns: - mode: memory flush-interval-ms: 5000 lambda: - mode: memory flush-interval-ms: 5000 cloudwatchlogs: - mode: memory flush-interval-ms: 5000 cloudwatchmetrics: - mode: memory flush-interval-ms: 5000 secretsmanager: - mode: memory flush-interval-ms: 5000 services: diff --git a/docs/configuration/storage.md b/docs/configuration/storage.md index ba375556..8b9a54c8 100644 --- a/docs/configuration/storage.md +++ b/docs/configuration/storage.md @@ -24,54 +24,44 @@ floci: ## Per-Service Override +When `mode` is omitted for a service, it inherits the global `storage.mode`. Only set a per-service mode when you need a different behaviour for that service. + ```yaml title="application.yml" floci: storage: - mode: memory + mode: memory # default for all services services: dynamodb: - mode: persistent + mode: persistent # DynamoDB uses persistent; everything else uses memory flush-interval-ms: 5000 s3: - mode: hybrid - sqs: - mode: memory + mode: hybrid # S3 uses hybrid; everything else uses memory ``` ## Per-Service Storage Overrides -Override the global mode for individual services via environment variables: - -| Variable | Default | Description | -|-----------------------------------------------------------------|----------|----------------------------------------| -| `FLOCI_STORAGE_SERVICES_SSM_MODE` | `memory` | SSM storage mode | -| `FLOCI_STORAGE_SERVICES_SSM_FLUSH_INTERVAL_MS` | `5000` | SSM flush interval (ms) | -| `FLOCI_STORAGE_SERVICES_SQS_MODE` | `memory` | SQS storage mode | -| `FLOCI_STORAGE_SERVICES_SQS_PERSIST_ON_SHUTDOWN` | `true` | Flush SQS messages to disk on shutdown | -| `FLOCI_STORAGE_SERVICES_S3_MODE` | `hybrid` | S3 storage mode | -| `FLOCI_STORAGE_SERVICES_S3_CACHE_SIZE_MB` | `100` | S3 in-memory cache size (MB) | -| `FLOCI_STORAGE_SERVICES_DYNAMODB_MODE` | `memory` | DynamoDB storage mode | -| `FLOCI_STORAGE_SERVICES_DYNAMODB_FLUSH_INTERVAL_MS` | `5000` | DynamoDB flush interval (ms) | -| `FLOCI_STORAGE_SERVICES_SNS_MODE` | `memory` | SNS storage mode | -| `FLOCI_STORAGE_SERVICES_SNS_FLUSH_INTERVAL_MS` | `5000` | SNS flush interval (ms) | -| `FLOCI_STORAGE_SERVICES_LAMBDA_MODE` | `memory` | Lambda storage mode | -| `FLOCI_STORAGE_SERVICES_LAMBDA_FLUSH_INTERVAL_MS` | `5000` | Lambda flush interval (ms) | -| `FLOCI_STORAGE_SERVICES_APIGATEWAY_MODE` | `memory` | API Gateway (v1) storage mode | -| `FLOCI_STORAGE_SERVICES_APIGATEWAY_FLUSH_INTERVAL_MS` | `5000` | API Gateway (v1) flush interval (ms) | -| `FLOCI_STORAGE_SERVICES_APIGATEWAYV2_MODE` | `memory` | API Gateway (v2) storage mode | -| `FLOCI_STORAGE_SERVICES_APIGATEWAYV2_FLUSH_INTERVAL_MS` | `5000` | API Gateway (v2) flush interval (ms) | -| `FLOCI_STORAGE_SERVICES_IAM_MODE` | `memory` | IAM storage mode | -| `FLOCI_STORAGE_SERVICES_IAM_FLUSH_INTERVAL_MS` | `5000` | IAM flush interval (ms) | -| `FLOCI_STORAGE_SERVICES_RDS_MODE` | `memory` | RDS storage mode | -| `FLOCI_STORAGE_SERVICES_RDS_FLUSH_INTERVAL_MS` | `5000` | RDS flush interval (ms) | -| `FLOCI_STORAGE_SERVICES_EVENTBRIDGE_MODE` | `memory` | EventBridge storage mode | -| `FLOCI_STORAGE_SERVICES_EVENTBRIDGE_FLUSH_INTERVAL_MS` | `5000` | EventBridge flush interval (ms) | -| `FLOCI_STORAGE_SERVICES_CLOUDWATCHLOGS_MODE` | `memory` | CloudWatch Logs storage mode | -| `FLOCI_STORAGE_SERVICES_CLOUDWATCHLOGS_FLUSH_INTERVAL_MS` | `5000` | CloudWatch Logs flush interval (ms) | -| `FLOCI_STORAGE_SERVICES_CLOUDWATCHMETRICS_MODE` | `memory` | CloudWatch Metrics storage mode | -| `FLOCI_STORAGE_SERVICES_CLOUDWATCHMETRICS_FLUSH_INTERVAL_MS` | `5000` | CloudWatch Metrics flush interval (ms) | -| `FLOCI_STORAGE_SERVICES_SECRETSMANAGER_MODE` | `memory` | Secrets Manager storage mode | -| `FLOCI_STORAGE_SERVICES_SECRETSMANAGER_FLUSH_INTERVAL_MS` | `5000` | Secrets Manager flush interval (ms) | +Override the global mode for individual services via environment variables. When not set, the service inherits `FLOCI_STORAGE_MODE`. + +| Variable | Default | Description | +|-----------------------------------------------------------------|----------------|----------------------------------------| +| `FLOCI_STORAGE_SERVICES_SSM_MODE` | global default | SSM storage mode | +| `FLOCI_STORAGE_SERVICES_SSM_FLUSH_INTERVAL_MS` | `5000` | SSM flush interval (ms) | +| `FLOCI_STORAGE_SERVICES_SQS_MODE` | global default | SQS storage mode | +| `FLOCI_STORAGE_SERVICES_S3_MODE` | global default | S3 storage mode | +| `FLOCI_STORAGE_SERVICES_DYNAMODB_MODE` | global default | DynamoDB storage mode | +| `FLOCI_STORAGE_SERVICES_DYNAMODB_FLUSH_INTERVAL_MS` | `5000` | DynamoDB flush interval (ms) | +| `FLOCI_STORAGE_SERVICES_SNS_MODE` | global default | SNS storage mode | +| `FLOCI_STORAGE_SERVICES_SNS_FLUSH_INTERVAL_MS` | `5000` | SNS flush interval (ms) | +| `FLOCI_STORAGE_SERVICES_LAMBDA_MODE` | global default | Lambda storage mode | +| `FLOCI_STORAGE_SERVICES_LAMBDA_FLUSH_INTERVAL_MS` | `5000` | Lambda flush interval (ms) | +| `FLOCI_STORAGE_SERVICES_CLOUDWATCHLOGS_MODE` | global default | CloudWatch Logs storage mode | +| `FLOCI_STORAGE_SERVICES_CLOUDWATCHLOGS_FLUSH_INTERVAL_MS` | `5000` | CloudWatch Logs flush interval (ms) | +| `FLOCI_STORAGE_SERVICES_CLOUDWATCHMETRICS_MODE` | global default | CloudWatch Metrics storage mode | +| `FLOCI_STORAGE_SERVICES_CLOUDWATCHMETRICS_FLUSH_INTERVAL_MS` | `5000` | CloudWatch Metrics flush interval (ms) | +| `FLOCI_STORAGE_SERVICES_SECRETSMANAGER_MODE` | global default | Secrets Manager storage mode | +| `FLOCI_STORAGE_SERVICES_SECRETSMANAGER_FLUSH_INTERVAL_MS` | `5000` | Secrets Manager flush interval (ms) | +| `FLOCI_STORAGE_SERVICES_ACM_MODE` | global default | ACM storage mode | +| `FLOCI_STORAGE_SERVICES_ACM_FLUSH_INTERVAL_MS` | `5000` | ACM flush interval (ms) | ## Environment Variable Override diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java index 89f25264..9fa01285 100644 --- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java +++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java @@ -75,74 +75,64 @@ interface ServiceStorageOverrides { } interface SsmStorageConfig { - @WithDefault("memory") - String mode(); + Optional mode(); @WithDefault("5000") long flushIntervalMs(); } interface SqsStorageConfig { - @WithDefault("memory") - String mode(); + Optional mode(); } interface S3StorageConfig { - @WithDefault("hybrid") - String mode(); + Optional mode(); } interface DynamoDbStorageConfig { - @WithDefault("memory") - String mode(); + Optional mode(); @WithDefault("5000") long flushIntervalMs(); } interface SnsStorageConfig { - @WithDefault("memory") - String mode(); + Optional mode(); @WithDefault("5000") long flushIntervalMs(); } interface LambdaStorageConfig { - @WithDefault("memory") - String mode(); + Optional mode(); @WithDefault("5000") long flushIntervalMs(); } interface CloudWatchLogsStorageConfig { - @WithDefault("memory") - String mode(); + Optional mode(); @WithDefault("5000") long flushIntervalMs(); } interface CloudWatchMetricsStorageConfig { - @WithDefault("memory") - String mode(); + Optional mode(); @WithDefault("5000") long flushIntervalMs(); } interface SecretsManagerStorageConfig { - @WithDefault("memory") - String mode(); + Optional mode(); @WithDefault("5000") long flushIntervalMs(); } interface AcmStorageConfig { - @WithDefault("memory") - String mode(); + Optional mode(); @WithDefault("5000") long flushIntervalMs(); diff --git a/src/main/java/io/github/hectorvent/floci/core/storage/StorageFactory.java b/src/main/java/io/github/hectorvent/floci/core/storage/StorageFactory.java index 1c217f4b..c245e3eb 100644 --- a/src/main/java/io/github/hectorvent/floci/core/storage/StorageFactory.java +++ b/src/main/java/io/github/hectorvent/floci/core/storage/StorageFactory.java @@ -98,17 +98,18 @@ public void shutdownAll() { } private String resolveMode(String serviceName) { + String globalMode = config.storage().mode(); return switch (serviceName) { - case "ssm" -> config.storage().services().ssm().mode(); - case "sqs" -> config.storage().services().sqs().mode(); - case "s3" -> config.storage().services().s3().mode(); - case "dynamodb" -> config.storage().services().dynamodb().mode(); - case "sns" -> config.storage().services().sns().mode(); - case "lambda" -> config.storage().services().lambda().mode(); - case "cloudwatchlogs" -> config.storage().services().cloudwatchlogs().mode(); - case "cloudwatchmetrics" -> config.storage().services().cloudwatchmetrics().mode(); - case "secretsmanager" -> config.storage().services().secretsmanager().mode(); - default -> config.storage().mode(); + case "ssm" -> config.storage().services().ssm().mode().orElse(globalMode); + case "sqs" -> config.storage().services().sqs().mode().orElse(globalMode); + case "s3" -> config.storage().services().s3().mode().orElse(globalMode); + case "dynamodb" -> config.storage().services().dynamodb().mode().orElse(globalMode); + case "sns" -> config.storage().services().sns().mode().orElse(globalMode); + case "lambda" -> config.storage().services().lambda().mode().orElse(globalMode); + case "cloudwatchlogs" -> config.storage().services().cloudwatchlogs().mode().orElse(globalMode); + case "cloudwatchmetrics" -> config.storage().services().cloudwatchmetrics().mode().orElse(globalMode); + case "secretsmanager" -> config.storage().services().secretsmanager().mode().orElse(globalMode); + default -> globalMode; }; } diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java index 806af7b3..6666e933 100644 --- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java +++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java @@ -37,6 +37,9 @@ public class S3Service { private final StorageBackend bucketStore; private final StorageBackend objectStore; private final Path dataRoot; + private final boolean inMemory; + private final ConcurrentHashMap memoryDataStore = new ConcurrentHashMap<>(); + private final ConcurrentHashMap> memoryMultipartStore = new ConcurrentHashMap<>(); private final ConcurrentHashMap multipartUploads = new ConcurrentHashMap<>(); private final SqsService sqsService; @@ -57,6 +60,7 @@ public S3Service(StorageFactory storageFactory, EmulatorConfig config, new TypeReference>() { }), Path.of(config.storage().persistentPath()).resolve("s3"), + "memory".equals(config.storage().services().s3().mode().orElse(config.storage().mode())), sqsService, snsService, regionResolver, config.effectiveBaseUrl(), objectMapper ); } @@ -67,26 +71,29 @@ public S3Service(StorageFactory storageFactory, EmulatorConfig config, S3Service(StorageBackend bucketStore, StorageBackend objectStore, Path dataRoot) { - this(bucketStore, objectStore, dataRoot, null, null, null, "http://localhost:4566", + this(bucketStore, objectStore, dataRoot, true, null, null, null, "http://localhost:4566", new ObjectMapper()); } private S3Service(StorageBackend bucketStore, StorageBackend objectStore, - Path dataRoot, SqsService sqsService, SnsService snsService, + Path dataRoot, boolean inMemory, SqsService sqsService, SnsService snsService, RegionResolver regionResolver, String baseUrl, ObjectMapper objectMapper) { this.bucketStore = bucketStore; this.objectStore = objectStore; this.dataRoot = dataRoot; + this.inMemory = inMemory; this.sqsService = sqsService; this.snsService = snsService; this.regionResolver = regionResolver; this.baseUrl = baseUrl; this.objectMapper = objectMapper; - try { - Files.createDirectories(dataRoot); - } catch (IOException e) { - throw new UncheckedIOException("Failed to create S3 data directory: " + dataRoot, e); + if (!inMemory) { + try { + Files.createDirectories(dataRoot); + } catch (IOException e) { + throw new UncheckedIOException("Failed to create S3 data directory: " + dataRoot, e); + } } } @@ -115,7 +122,12 @@ public void deleteBucket(String bucketName) { } bucketStore.delete(bucketName); - deleteDirectory(dataRoot.resolve(bucketName)); + if (inMemory) { + String prefix = bucketName + "/"; + memoryDataStore.keySet().removeIf(k -> k.startsWith(prefix)); + } else { + deleteDirectory(dataRoot.resolve(bucketName)); + } LOG.infov("Deleted bucket: {0}", bucketName); } @@ -729,10 +741,14 @@ public MultipartUpload initiateMultipartUpload(String bucket, String key, String } upload.setStorageClass(ObjectAttributeName.normalizeStorageClass(storageClass)); - try { - Files.createDirectories(dataRoot.resolve(".multipart").resolve(upload.getUploadId())); - } catch (IOException e) { - throw new UncheckedIOException("Failed to create multipart temp directory", e); + if (inMemory) { + memoryMultipartStore.put(upload.getUploadId(), new ConcurrentHashMap<>()); + } else { + try { + Files.createDirectories(dataRoot.resolve(".multipart").resolve(upload.getUploadId())); + } catch (IOException e) { + throw new UncheckedIOException("Failed to create multipart temp directory", e); + } } multipartUploads.put(upload.getUploadId(), upload); @@ -751,12 +767,15 @@ public String uploadPart(String bucket, String key, String uploadId, int partNum "Part number must be between 1 and 10000.", 400); } - // Write part to temp directory - Path partPath = dataRoot.resolve(".multipart").resolve(uploadId).resolve(String.valueOf(partNumber)); - try { - Files.write(partPath, data); - } catch (IOException e) { - throw new UncheckedIOException("Failed to write multipart part", e); + if (inMemory) { + memoryMultipartStore.get(uploadId).put(partNumber, data); + } else { + Path partPath = dataRoot.resolve(".multipart").resolve(uploadId).resolve(String.valueOf(partNumber)); + try { + Files.write(partPath, data); + } catch (IOException e) { + throw new UncheckedIOException("Failed to write multipart part", e); + } } String eTag = computeETag(data); @@ -788,8 +807,9 @@ public S3Object completeMultipartUpload(String bucket, String key, String upload MessageDigest md = MessageDigest.getInstance("MD5"); for (int num : partNumbers) { - Path partPath = dataRoot.resolve(".multipart").resolve(uploadId).resolve(String.valueOf(num)); - byte[] partData = Files.readAllBytes(partPath); + byte[] partData = inMemory + ? memoryMultipartStore.get(uploadId).get(num) + : Files.readAllBytes(dataRoot.resolve(".multipart").resolve(uploadId).resolve(String.valueOf(num))); combined.write(partData); // For composite ETag: hash each part's MD5 md.update(computeETagBytes(partData)); @@ -1104,7 +1124,11 @@ private String buildS3EventJson(String bucketName, String key, String eventName, private void cleanupMultipart(String uploadId) { multipartUploads.remove(uploadId); - deleteDirectory(dataRoot.resolve(".multipart").resolve(uploadId)); + if (inMemory) { + memoryMultipartStore.remove(uploadId); + } else { + deleteDirectory(dataRoot.resolve(".multipart").resolve(uploadId)); + } } private static S3Checksum buildChecksum(byte[] data, List parts, boolean multipartUpload) { @@ -1219,6 +1243,10 @@ private Path resolveVersionedPath(String bucketName, String key, String versionI } private void writeVersionedFile(String bucketName, String key, String versionId, byte[] data) { + if (inMemory) { + memoryDataStore.put(versionedKey(bucketName, key, versionId), data); + return; + } try { Path filePath = resolveVersionedPath(bucketName, key, versionId); Files.createDirectories(filePath.getParent()); @@ -1229,6 +1257,9 @@ private void writeVersionedFile(String bucketName, String key, String versionId, } private byte[] readVersionedFile(String bucketName, String key, String versionId) { + if (inMemory) { + return memoryDataStore.get(versionedKey(bucketName, key, versionId)); + } try { return Files.readAllBytes(resolveVersionedPath(bucketName, key, versionId)); } catch (IOException e) { @@ -1237,6 +1268,10 @@ private byte[] readVersionedFile(String bucketName, String key, String versionId } private void writeFile(String bucketName, String key, byte[] data) { + if (inMemory) { + memoryDataStore.put(objectKey(bucketName, key), data); + return; + } try { Path filePath = resolveObjectPath(bucketName, key); Files.createDirectories(filePath.getParent()); @@ -1247,6 +1282,9 @@ private void writeFile(String bucketName, String key, byte[] data) { } private byte[] readFile(String bucketName, String key) { + if (inMemory) { + return memoryDataStore.get(objectKey(bucketName, key)); + } try { return Files.readAllBytes(resolveObjectPath(bucketName, key)); } catch (IOException e) { @@ -1255,6 +1293,10 @@ private byte[] readFile(String bucketName, String key) { } private void deleteFile(String bucketName, String key) { + if (inMemory) { + memoryDataStore.remove(objectKey(bucketName, key)); + return; + } try { Files.deleteIfExists(resolveObjectPath(bucketName, key)); } catch (IOException e) { diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index f1c071a3..c1d72ca3 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -30,32 +30,20 @@ floci: compaction-interval-ms: 30000 services: ssm: - mode: memory flush-interval-ms: 5000 - sqs: - mode: memory - s3: - mode: memory dynamodb: - mode: memory flush-interval-ms: 5000 sns: - mode: memory flush-interval-ms: 5000 lambda: - mode: memory flush-interval-ms: 5000 cloudwatchlogs: - mode: memory flush-interval-ms: 5000 cloudwatchmetrics: - mode: memory flush-interval-ms: 5000 secretsmanager: - mode: memory flush-interval-ms: 5000 acm: - mode: memory flush-interval-ms: 5000 auth: diff --git a/src/test/resources/application.yml b/src/test/resources/application.yml index d3c0c033..04f74b11 100644 --- a/src/test/resources/application.yml +++ b/src/test/resources/application.yml @@ -16,29 +16,18 @@ floci: compaction-interval-ms: 60000 services: ssm: - mode: memory flush-interval-ms: 60000 - sqs: - mode: memory - s3: - mode: memory dynamodb: - mode: memory flush-interval-ms: 60000 sns: - mode: memory flush-interval-ms: 60000 lambda: - mode: memory flush-interval-ms: 60000 cloudwatchlogs: - mode: memory flush-interval-ms: 60000 cloudwatchmetrics: - mode: memory flush-interval-ms: 60000 secretsmanager: - mode: memory flush-interval-ms: 60000 auth: From 4756577308ae4e881ffead4f26ca7dc6b660d40d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20=C5=A0nuderl?= Date: Tue, 31 Mar 2026 06:17:07 +0200 Subject: [PATCH 02/32] feat(s3): add presigned POST upload support (#120) Implement S3 presigned POST (browser-based) uploads via multipart/form-data POST to /{bucket}. This is the standard AWS mechanism for browser-based file uploads using pre-signed policies. Changes: - S3Controller: detect multipart/form-data POST on /{bucket} and route to handlePresignedPost instead of returning InvalidArgument - Parse multipart body to extract form fields (key, policy, Content-Type, x-amz-credential, x-amz-signature, etc.) and file data - Validate content-length-range from the Base64-encoded policy JSON - Store object via S3Service.putObject with Content-Type from form field - Return 204 with ETag and Location headers (AWS-compatible response) Tests: - 11 new integration tests covering: basic upload, binary data, content-length validation (reject + accept), missing key/file fields, no-policy upload, Content-Type precedence (form field over file part), non-existent bucket Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../floci/services/s3/S3Controller.java | 257 ++++++++++++++++ .../s3/S3PresignedPostIntegrationTest.java | 291 ++++++++++++++++++ 2 files changed, 548 insertions(+) create mode 100644 src/test/java/io/github/hectorvent/floci/services/s3/S3PresignedPostIntegrationTest.java diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java index 71d0cc03..852f2739 100644 --- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java +++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java @@ -29,6 +29,7 @@ import java.time.Instant; import java.time.ZoneId; import java.time.format.DateTimeFormatter; +import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; @@ -552,12 +553,16 @@ public Response deleteObject(@PathParam("bucket") String bucket, @Path("/{bucket}") @Produces(MediaType.APPLICATION_XML) public Response handleBucketPost(@PathParam("bucket") String bucket, + @HeaderParam("Content-Type") String contentType, @Context UriInfo uriInfo, byte[] body) { try { if (hasQueryParam(uriInfo, "delete")) { return handleDeleteObjects(bucket, body); } + if (contentType != null && contentType.startsWith("multipart/form-data")) { + return handlePresignedPost(bucket, contentType, body); + } return xmlErrorResponse(new AwsException("InvalidArgument", "POST on bucket requires ?delete parameter.", 400)); } catch (AwsException e) { @@ -1256,6 +1261,258 @@ private Instant parseHttpDate(String dateStr) { } } + private Response handlePresignedPost(String bucket, String contentType, byte[] body) { + String boundary = extractBoundary(contentType); + if (boundary == null) { + throw new AwsException("InvalidArgument", + "Could not determine multipart boundary from Content-Type.", 400); + } + + Map fields = new LinkedHashMap<>(); + byte[] fileData = null; + String fileContentType = null; + + byte[] boundaryBytes = ("--" + boundary).getBytes(StandardCharsets.UTF_8); + List parts = splitMultipartParts(body, boundaryBytes); + + for (byte[] part : parts) { + int headerEnd = indexOfDoubleNewline(part); + if (headerEnd < 0) { + continue; + } + String headers = new String(part, 0, headerEnd, StandardCharsets.UTF_8); + int bodyStart = headerEnd + 4; // skip \r\n\r\n + byte[] partBody = Arrays.copyOfRange(part, bodyStart, part.length); + + // Trim trailing \r\n from part body + if (partBody.length >= 2 + && partBody[partBody.length - 2] == '\r' + && partBody[partBody.length - 1] == '\n') { + partBody = Arrays.copyOf(partBody, partBody.length - 2); + } + + String disposition = extractHeaderValue(headers, "Content-Disposition"); + if (disposition == null) { + continue; + } + String fieldName = extractDispositionParam(disposition, "name"); + if (fieldName == null) { + continue; + } + + String filename = extractDispositionParam(disposition, "filename"); + if (filename != null) { + fileData = partBody; + String partContentType = extractHeaderValue(headers, "Content-Type"); + if (partContentType != null) { + fileContentType = partContentType.trim(); + } + } else { + fields.put(fieldName, new String(partBody, StandardCharsets.UTF_8)); + } + } + + String key = fields.get("key"); + if (key == null || key.isEmpty()) { + throw new AwsException("InvalidArgument", + "Bucket POST must contain a field named 'key'.", 400); + } + + if (fileData == null) { + throw new AwsException("InvalidArgument", + "Bucket POST must contain a file field.", 400); + } + + // Validate content-length-range from policy if present + String policy = fields.get("policy"); + if (policy != null && !policy.isEmpty()) { + validateContentLengthRange(policy, fileData.length); + } + + // Use Content-Type from form fields, fall back to file part Content-Type + String objectContentType = fields.get("Content-Type"); + if (objectContentType == null || objectContentType.isEmpty()) { + objectContentType = fileContentType; + } + if (objectContentType == null || objectContentType.isEmpty()) { + objectContentType = "application/octet-stream"; + } + + S3Object obj = s3Service.putObject(bucket, key, fileData, objectContentType, null); + LOG.infov("Presigned POST upload: {0}/{1} ({2} bytes)", bucket, key, fileData.length); + + String xml = new XmlBuilder() + .raw("") + .start("PostResponse") + .elem("Location", bucket + "/" + key) + .elem("Bucket", bucket) + .elem("Key", key) + .elem("ETag", obj.getETag()) + .end("PostResponse") + .build(); + return Response.status(204) + .header("ETag", obj.getETag()) + .header("Location", bucket + "/" + key) + .build(); + } + + private void validateContentLengthRange(String policyBase64, int contentLength) { + try { + String decoded = new String(java.util.Base64.getDecoder().decode(policyBase64), StandardCharsets.UTF_8); + // Parse conditions array from the policy JSON to find content-length-range + int condIdx = decoded.indexOf("\"conditions\""); + if (condIdx < 0) { + return; + } + // Look for content-length-range condition: ["content-length-range", min, max] + String lower = decoded.toLowerCase(Locale.ROOT); + int rangeIdx = lower.indexOf("content-length-range"); + if (rangeIdx < 0) { + return; + } + // Find the enclosing array bracket + int bracketStart = decoded.lastIndexOf('[', rangeIdx); + int bracketEnd = decoded.indexOf(']', rangeIdx); + if (bracketStart < 0 || bracketEnd < 0) { + return; + } + String rangeArray = decoded.substring(bracketStart, bracketEnd + 1); + // Extract min and max values + String[] tokens = rangeArray.replaceAll("[\\[\\]\"]", "").split(","); + if (tokens.length >= 3) { + long min = Long.parseLong(tokens[1].trim()); + long max = Long.parseLong(tokens[2].trim()); + if (contentLength < min || contentLength > max) { + throw new AwsException("EntityTooLarge", + "Your proposed upload exceeds the maximum allowed size.", 400); + } + } + } catch (AwsException e) { + throw e; + } catch (Exception e) { + // If policy parsing fails, skip validation (match AWS lenient behavior for emulator) + LOG.debugv("Failed to parse presigned POST policy: {0}", e.getMessage()); + } + } + + private static String extractBoundary(String contentType) { + if (contentType == null) { + return null; + } + for (String part : contentType.split(";")) { + String trimmed = part.trim(); + if (trimmed.toLowerCase(Locale.ROOT).startsWith("boundary=")) { + String boundary = trimmed.substring("boundary=".length()).trim(); + if (boundary.startsWith("\"") && boundary.endsWith("\"")) { + boundary = boundary.substring(1, boundary.length() - 1); + } + return boundary; + } + } + return null; + } + + private static List splitMultipartParts(byte[] body, byte[] boundary) { + java.util.ArrayList parts = new java.util.ArrayList<>(); + int pos = indexOf(body, boundary, 0); + if (pos < 0) { + return parts; + } + // Skip past the first boundary line + pos += boundary.length; + // Skip the CRLF or -- after boundary + if (pos < body.length - 1 && body[pos] == '-' && body[pos + 1] == '-') { + return parts; // closing boundary immediately + } + if (pos < body.length - 1 && body[pos] == '\r' && body[pos + 1] == '\n') { + pos += 2; + } + + while (pos < body.length) { + int nextBoundary = indexOf(body, boundary, pos); + if (nextBoundary < 0) { + break; + } + parts.add(Arrays.copyOfRange(body, pos, nextBoundary)); + pos = nextBoundary + boundary.length; + // Check for closing boundary -- + if (pos < body.length - 1 && body[pos] == '-' && body[pos + 1] == '-') { + break; + } + // Skip CRLF after boundary + if (pos < body.length - 1 && body[pos] == '\r' && body[pos + 1] == '\n') { + pos += 2; + } + } + return parts; + } + + private static int indexOf(byte[] data, byte[] pattern, int fromIndex) { + outer: + for (int i = fromIndex; i <= data.length - pattern.length; i++) { + for (int j = 0; j < pattern.length; j++) { + if (data[i + j] != pattern[j]) { + continue outer; + } + } + return i; + } + return -1; + } + + private static int indexOfDoubleNewline(byte[] data) { + for (int i = 0; i < data.length - 3; i++) { + if (data[i] == '\r' && data[i + 1] == '\n' && data[i + 2] == '\r' && data[i + 3] == '\n') { + return i; + } + } + return -1; + } + + private static String extractHeaderValue(String headers, String headerName) { + String lowerHeaders = headers.toLowerCase(Locale.ROOT); + String lowerName = headerName.toLowerCase(Locale.ROOT) + ":"; + int idx = lowerHeaders.indexOf(lowerName); + if (idx < 0) { + return null; + } + int valueStart = idx + lowerName.length(); + int lineEnd = headers.indexOf('\r', valueStart); + if (lineEnd < 0) { + lineEnd = headers.indexOf('\n', valueStart); + } + if (lineEnd < 0) { + lineEnd = headers.length(); + } + return headers.substring(valueStart, lineEnd).trim(); + } + + private static String extractDispositionParam(String disposition, String paramName) { + String search = paramName + "="; + int idx = disposition.indexOf(search); + if (idx < 0) { + return null; + } + int valueStart = idx + search.length(); + if (valueStart >= disposition.length()) { + return null; + } + if (disposition.charAt(valueStart) == '"') { + valueStart++; + int valueEnd = disposition.indexOf('"', valueStart); + if (valueEnd < 0) { + return disposition.substring(valueStart); + } + return disposition.substring(valueStart, valueEnd); + } else { + int valueEnd = disposition.indexOf(';', valueStart); + if (valueEnd < 0) { + valueEnd = disposition.length(); + } + return disposition.substring(valueStart, valueEnd).trim(); + } + } + private boolean hasQueryParam(UriInfo uriInfo, String param) { if (uriInfo.getQueryParameters().containsKey(param)) return true; String query = uriInfo.getRequestUri().getQuery(); diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3PresignedPostIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3PresignedPostIntegrationTest.java new file mode 100644 index 00000000..c574d487 --- /dev/null +++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3PresignedPostIntegrationTest.java @@ -0,0 +1,291 @@ +package io.github.hectorvent.floci.services.s3; + +import io.quarkus.test.junit.QuarkusTest; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; + +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.Base64; + +import static io.restassured.RestAssured.given; +import static org.hamcrest.Matchers.*; + +@QuarkusTest +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +class S3PresignedPostIntegrationTest { + + private static final String BUCKET = "presigned-post-bucket"; + private static final DateTimeFormatter AMZ_DATE_FORMAT = + DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmss'Z'").withZone(ZoneOffset.UTC); + + @Test + @Order(1) + void createBucket() { + given() + .when() + .put("/" + BUCKET) + .then() + .statusCode(200); + } + + @Test + @Order(10) + void presignedPostUploadsObject() { + String key = "uploads/test-file.txt"; + String fileContent = "Hello from presigned POST!"; + String contentType = "text/plain"; + + String policy = buildPolicy(BUCKET, key, contentType, 0, 10485760); + String policyBase64 = Base64.getEncoder().encodeToString(policy.getBytes(StandardCharsets.UTF_8)); + + given() + .multiPart("key", key) + .multiPart("Content-Type", contentType) + .multiPart("policy", policyBase64) + .multiPart("x-amz-algorithm", "AWS4-HMAC-SHA256") + .multiPart("x-amz-credential", "AKIAIOSFODNN7EXAMPLE/20260330/us-east-1/s3/aws4_request") + .multiPart("x-amz-date", AMZ_DATE_FORMAT.format(Instant.now())) + .multiPart("x-amz-signature", "dummysignature") + .multiPart("file", "test-file.txt", fileContent.getBytes(StandardCharsets.UTF_8), contentType) + .when() + .post("/" + BUCKET) + .then() + .statusCode(204) + .header("ETag", notNullValue()); + + // Verify the object was stored correctly + given() + .when() + .get("/" + BUCKET + "/" + key) + .then() + .statusCode(200) + .header("Content-Type", equalTo(contentType)) + .body(equalTo(fileContent)); + } + + @Test + @Order(20) + void presignedPostWithBinaryData() { + String key = "uploads/binary-data.bin"; + byte[] binaryData = new byte[256]; + for (int i = 0; i < 256; i++) { + binaryData[i] = (byte) i; + } + + String policy = buildPolicy(BUCKET, key, "application/octet-stream", 0, 10485760); + String policyBase64 = Base64.getEncoder().encodeToString(policy.getBytes(StandardCharsets.UTF_8)); + + given() + .multiPart("key", key) + .multiPart("Content-Type", "application/octet-stream") + .multiPart("policy", policyBase64) + .multiPart("x-amz-algorithm", "AWS4-HMAC-SHA256") + .multiPart("x-amz-credential", "AKIAIOSFODNN7EXAMPLE/20260330/us-east-1/s3/aws4_request") + .multiPart("x-amz-date", AMZ_DATE_FORMAT.format(Instant.now())) + .multiPart("x-amz-signature", "dummysignature") + .multiPart("file", "binary-data.bin", binaryData, "application/octet-stream") + .when() + .post("/" + BUCKET) + .then() + .statusCode(204) + .header("ETag", notNullValue()); + + // Verify the binary object was stored correctly + byte[] retrieved = given() + .when() + .get("/" + BUCKET + "/" + key) + .then() + .statusCode(200) + .extract().body().asByteArray(); + + org.junit.jupiter.api.Assertions.assertArrayEquals(binaryData, retrieved); + } + + @Test + @Order(30) + void presignedPostRejectsExceedingContentLength() { + String key = "uploads/too-large.txt"; + // Create data that exceeds the max content-length-range of 10 bytes + String fileContent = "This content is definitely longer than 10 bytes"; + + String policy = buildPolicy(BUCKET, key, "text/plain", 0, 10); + String policyBase64 = Base64.getEncoder().encodeToString(policy.getBytes(StandardCharsets.UTF_8)); + + given() + .multiPart("key", key) + .multiPart("Content-Type", "text/plain") + .multiPart("policy", policyBase64) + .multiPart("x-amz-algorithm", "AWS4-HMAC-SHA256") + .multiPart("x-amz-credential", "AKIAIOSFODNN7EXAMPLE/20260330/us-east-1/s3/aws4_request") + .multiPart("x-amz-date", AMZ_DATE_FORMAT.format(Instant.now())) + .multiPart("x-amz-signature", "dummysignature") + .multiPart("file", "too-large.txt", fileContent.getBytes(StandardCharsets.UTF_8), "text/plain") + .when() + .post("/" + BUCKET) + .then() + .statusCode(400) + .body(containsString("EntityTooLarge")); + } + + @Test + @Order(40) + void presignedPostRequiresKeyField() { + given() + .multiPart("Content-Type", "text/plain") + .multiPart("policy", "dummypolicy") + .multiPart("file", "test.txt", "content".getBytes(StandardCharsets.UTF_8), "text/plain") + .when() + .post("/" + BUCKET) + .then() + .statusCode(400) + .body(containsString("InvalidArgument")); + } + + @Test + @Order(50) + void presignedPostRequiresFileField() { + given() + .multiPart("key", "uploads/no-file.txt") + .multiPart("Content-Type", "text/plain") + .multiPart("policy", "dummypolicy") + .when() + .post("/" + BUCKET) + .then() + .statusCode(400) + .body(containsString("InvalidArgument")); + } + + @Test + @Order(60) + void presignedPostWithoutPolicySkipsValidation() { + String key = "uploads/no-policy.txt"; + String fileContent = "Uploaded without policy"; + + given() + .multiPart("key", key) + .multiPart("Content-Type", "text/plain") + .multiPart("file", "no-policy.txt", fileContent.getBytes(StandardCharsets.UTF_8), "text/plain") + .when() + .post("/" + BUCKET) + .then() + .statusCode(204) + .header("ETag", notNullValue()); + + // Verify the object was stored + given() + .when() + .get("/" + BUCKET + "/" + key) + .then() + .statusCode(200) + .body(equalTo(fileContent)); + } + + @Test + @Order(70) + void presignedPostContentTypeFromFormField() { + String key = "uploads/typed-file.json"; + String fileContent = "{\"test\": true}"; + + String policy = buildPolicy(BUCKET, key, "application/json", 0, 10485760); + String policyBase64 = Base64.getEncoder().encodeToString(policy.getBytes(StandardCharsets.UTF_8)); + + given() + .multiPart("key", key) + .multiPart("Content-Type", "application/json") + .multiPart("policy", policyBase64) + .multiPart("x-amz-algorithm", "AWS4-HMAC-SHA256") + .multiPart("x-amz-credential", "AKIAIOSFODNN7EXAMPLE/20260330/us-east-1/s3/aws4_request") + .multiPart("x-amz-date", AMZ_DATE_FORMAT.format(Instant.now())) + .multiPart("x-amz-signature", "dummysignature") + .multiPart("file", "typed-file.json", fileContent.getBytes(StandardCharsets.UTF_8), "application/octet-stream") + .when() + .post("/" + BUCKET) + .then() + .statusCode(204); + + // Content-Type should come from the form field, not the file part + given() + .when() + .get("/" + BUCKET + "/" + key) + .then() + .statusCode(200) + .header("Content-Type", equalTo("application/json")); + } + + @Test + @Order(80) + void presignedPostToNonExistentBucketFails() { + given() + .multiPart("key", "test.txt") + .multiPart("file", "test.txt", "data".getBytes(StandardCharsets.UTF_8), "text/plain") + .when() + .post("/nonexistent-presigned-bucket") + .then() + .statusCode(404) + .body(containsString("NoSuchBucket")); + } + + @Test + @Order(90) + void presignedPostWithContentLengthWithinRange() { + String key = "uploads/within-range.txt"; + // Exactly 5 bytes, within range [1, 100] + String fileContent = "12345"; + + String policy = buildPolicy(BUCKET, key, "text/plain", 1, 100); + String policyBase64 = Base64.getEncoder().encodeToString(policy.getBytes(StandardCharsets.UTF_8)); + + given() + .multiPart("key", key) + .multiPart("Content-Type", "text/plain") + .multiPart("policy", policyBase64) + .multiPart("x-amz-algorithm", "AWS4-HMAC-SHA256") + .multiPart("x-amz-credential", "AKIAIOSFODNN7EXAMPLE/20260330/us-east-1/s3/aws4_request") + .multiPart("x-amz-date", AMZ_DATE_FORMAT.format(Instant.now())) + .multiPart("x-amz-signature", "dummysignature") + .multiPart("file", "within-range.txt", fileContent.getBytes(StandardCharsets.UTF_8), "text/plain") + .when() + .post("/" + BUCKET) + .then() + .statusCode(204); + } + + @Test + @Order(100) + void cleanupBucket() { + // Delete all objects + given().delete("/" + BUCKET + "/uploads/test-file.txt"); + given().delete("/" + BUCKET + "/uploads/binary-data.bin"); + given().delete("/" + BUCKET + "/uploads/no-policy.txt"); + given().delete("/" + BUCKET + "/uploads/typed-file.json"); + given().delete("/" + BUCKET + "/uploads/within-range.txt"); + + given() + .when() + .delete("/" + BUCKET) + .then() + .statusCode(204); + } + + private String buildPolicy(String bucket, String key, String contentType, long minSize, long maxSize) { + String expiration = Instant.now().plusSeconds(3600) + .atZone(ZoneOffset.UTC) + .format(DateTimeFormatter.ISO_INSTANT); + return """ + { + "expiration": "%s", + "conditions": [ + {"bucket": "%s"}, + {"key": "%s"}, + {"Content-Type": "%s"}, + ["content-length-range", %d, %d] + ] + } + """.formatted(expiration, bucket, key, contentType, minSize, maxSize); + } +} From 367af05981d57853e391723583a9e5506862f775 Mon Sep 17 00:00:00 2001 From: thegagne <1399002+thegagne@users.noreply.github.com> Date: Mon, 30 Mar 2026 23:20:14 -0500 Subject: [PATCH 03/32] feat(APIGW): add AWS integration type for API Gateway REST v1 (#108) Add support for the API Gateway REST v1 AWS (non-proxy) integration type, enabling direct service calls via VTL request/response mapping templates. New components: - VtlTemplateEngine: Apache Velocity engine with $input (body, json, path, params shorthand), $util (escapeJavaScript, urlEncode, base64, parseJson), $context, and $stageVariables - AwsServiceRouter: parses action-based integration URIs and dispatches to 13 internal service handlers (states, dynamodb, sqs, sns, events, etc.) Integration features: - Content-Type negotiation for request template selection - passthroughBehavior (WHEN_NO_MATCH, WHEN_NO_TEMPLATES, NEVER) - Request parameter mapping (method.request.* -> integration.request.*) - Response parameter mapping with full JSONPath body extraction - Integration response selection via selectionPattern regex matching - VTL directives: #set, #foreach, #if/#else Also fixes SFN timestamps to use epoch float (e.g. 1743263483.047) matching real AWS format. Signed-off-by: Hector Ventura Co-authored-by: Hector Ventura From a06dbe9d96d83212d000e3c3cbdd702b607dc3c8 Mon Sep 17 00:00:00 2001 From: Hector Ventura Date: Mon, 30 Mar 2026 23:48:41 -0500 Subject: [PATCH 04/32] feat: add opensearch service emulation (#85) (#132) --- README.md | 5 +- docs/configuration/application-yml.md | 16 + docs/configuration/storage.md | 2 + docs/services/index.md | 4 +- docs/services/opensearch.md | 215 +++++++ mkdocs.yml | 2 + .../floci/config/EmulatorConfig.java | 28 + .../floci/core/common/ServiceRegistry.java | 2 + .../floci/core/storage/StorageFactory.java | 2 + .../opensearch/OpenSearchController.java | 556 ++++++++++++++++++ .../opensearch/OpenSearchService.java | 190 ++++++ .../opensearch/model/ClusterConfig.java | 56 ++ .../services/opensearch/model/Domain.java | 137 +++++ .../services/opensearch/model/EbsOptions.java | 45 ++ src/main/resources/application.yml | 8 + .../opensearch/OpenSearchIntegrationTest.java | 413 +++++++++++++ src/test/resources/application.yml | 5 + 17 files changed, 1683 insertions(+), 3 deletions(-) create mode 100644 docs/services/opensearch.md create mode 100644 src/main/java/io/github/hectorvent/floci/services/opensearch/OpenSearchController.java create mode 100644 src/main/java/io/github/hectorvent/floci/services/opensearch/OpenSearchService.java create mode 100644 src/main/java/io/github/hectorvent/floci/services/opensearch/model/ClusterConfig.java create mode 100644 src/main/java/io/github/hectorvent/floci/services/opensearch/model/Domain.java create mode 100644 src/main/java/io/github/hectorvent/floci/services/opensearch/model/EbsOptions.java create mode 100644 src/test/java/io/github/hectorvent/floci/services/opensearch/OpenSearchIntegrationTest.java diff --git a/README.md b/README.md index 19d7aeb5..f7f6194f 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ | KMS (sign, verify, re-encrypt) | ✅ | ⚠️ Partial | | Native binary | ✅ ~40 MB | ❌ | -**24 services. 408/408 SDK tests passing. Free forever.** +**25 services. 408/408 SDK tests passing. Free forever.** ## Architecture Overview @@ -64,7 +64,7 @@ flowchart LR Router["HTTP Router\n(JAX-RS / Vert.x)"] subgraph Stateless ["Stateless Services"] - A["SSM · SQS · SNS\nIAM · STS · KMS\nSecrets Manager · SES\nCognito · Kinesis\nEventBridge · CloudWatch\nStep Functions · CloudFormation\nACM · API Gateway"] + A["SSM · SQS · SNS\nIAM · STS · KMS\nSecrets Manager · SES\nCognito · Kinesis · OpenSearch\nEventBridge · CloudWatch\nStep Functions · CloudFormation\nACM · API Gateway"] end subgraph Stateful ["Stateful Services"] @@ -114,6 +114,7 @@ flowchart LR | **RDS** | 14 | **Real Docker containers** | PostgreSQL & MySQL, IAM auth, JDBC-compatible | | **ACM** | 8 | In-process | Certificate issuance, validation lifecycle | | **SES** | 14 | In-process | Send email / raw email, identity verification, DKIM attributes | +| **OpenSearch** | 24 | In-process | Domain CRUD, tags, versions, instance types, upgrade stubs | > **Lambda, ElastiCache, and RDS** spin up real Docker containers and support IAM authentication and SigV4 request signing — the same auth flow as production AWS. diff --git a/docs/configuration/application-yml.md b/docs/configuration/application-yml.md index e4256d5f..62bf4ca9 100644 --- a/docs/configuration/application-yml.md +++ b/docs/configuration/application-yml.md @@ -40,6 +40,8 @@ floci: flush-interval-ms: 5000 secretsmanager: flush-interval-ms: 5000 + opensearch: + flush-interval-ms: 5000 # inherits global storage mode services: ssm: @@ -121,6 +123,20 @@ floci: cloudformation: enabled: true + + acm: + enabled: true + validation-wait-seconds: 0 # Seconds before transitioning PENDING_VALIDATION → ISSUED + + ses: + enabled: true + + opensearch: + enabled: true + mode: mock # mock | real + default-image: "opensearchproject/opensearch:2" + proxy-base-port: 9400 + proxy-max-port: 9499 ``` ## Service Limits diff --git a/docs/configuration/storage.md b/docs/configuration/storage.md index 8b9a54c8..044c0738 100644 --- a/docs/configuration/storage.md +++ b/docs/configuration/storage.md @@ -62,6 +62,8 @@ Override the global mode for individual services via environment variables. When | `FLOCI_STORAGE_SERVICES_SECRETSMANAGER_FLUSH_INTERVAL_MS` | `5000` | Secrets Manager flush interval (ms) | | `FLOCI_STORAGE_SERVICES_ACM_MODE` | global default | ACM storage mode | | `FLOCI_STORAGE_SERVICES_ACM_FLUSH_INTERVAL_MS` | `5000` | ACM flush interval (ms) | +| `FLOCI_STORAGE_SERVICES_OPENSEARCH_MODE` | global default | OpenSearch storage mode | +| `FLOCI_STORAGE_SERVICES_OPENSEARCH_FLUSH_INTERVAL_MS` | `5000` | OpenSearch flush interval (ms) | ## Environment Variable Override diff --git a/docs/services/index.md b/docs/services/index.md index abaa1f11..4db52f43 100644 --- a/docs/services/index.md +++ b/docs/services/index.md @@ -1,6 +1,6 @@ # Services Overview -Floci emulates 21+ AWS services on a single port (`4566`). All services use the real AWS wire protocol — your existing AWS CLI commands and SDK clients work without modification. +Floci emulates 25 AWS services on a single port (`4566`). All services use the real AWS wire protocol — your existing AWS CLI commands and SDK clients work without modification. ## Service Matrix @@ -29,6 +29,8 @@ Floci emulates 21+ AWS services on a single port (`4566`). All services use the | [CloudWatch Logs](cloudwatch.md) | `POST /` + `X-Amz-Target: Logs.*` | JSON 1.1 | 14 | | [CloudWatch Metrics](cloudwatch.md#metrics) | `POST /` with `Action=` or JSON 1.1 | Query / JSON | 8 | | [ACM](acm.md) | `POST /` + `X-Amz-Target: CertificateManager.*` | JSON 1.1 | 12 | +| [SES](ses.md) | `POST /` with `Action=` param | Query | 14 | +| [OpenSearch](opensearch.md) | `/2021-01-01/opensearch/...` | REST JSON | 24 | ## Common Setup diff --git a/docs/services/opensearch.md b/docs/services/opensearch.md new file mode 100644 index 00000000..8d33aca8 --- /dev/null +++ b/docs/services/opensearch.md @@ -0,0 +1,215 @@ +# OpenSearch Service + +**Protocol:** REST JSON +**Endpoint:** `http://localhost:4566/2021-01-01/...` +**Credential scope:** `es` + +## Implementation Mode + +OpenSearch supports two implementation modes controlled by `FLOCI_SERVICES_OPENSEARCH_MODE`: + +| Mode | Behaviour | +|---|---| +| `mock` *(default)* | Management API only. Domains are stored in the configured StorageBackend and appear `ACTIVE` immediately. No real search capability. | +| `real` | *(v2, coming soon)* Spins up a real OpenSearch Docker container per domain and proxies data-plane requests to it. | + +## Supported Operations + +### Domain Lifecycle + +| Operation | Method + Path | Description | +|---|---|---| +| `CreateDomain` | `POST /2021-01-01/opensearch/domain` | Create a new domain | +| `DescribeDomain` | `GET /2021-01-01/opensearch/domain/{name}` | Get domain details | +| `DescribeDomains` | `POST /2021-01-01/opensearch/domain-info` | Batch describe domains | +| `DescribeDomainConfig` | `GET /2021-01-01/opensearch/domain/{name}/config` | Get domain configuration | +| `UpdateDomainConfig` | `POST /2021-01-01/opensearch/domain/{name}/config` | Update cluster config, EBS options, engine version | +| `DeleteDomain` | `DELETE /2021-01-01/opensearch/domain/{name}` | Delete a domain | +| `ListDomainNames` | `GET /2021-01-01/domain` | List all domains (supports `?engineType=` filter) | + +### Tags + +| Operation | Method + Path | Description | +|---|---|---| +| `AddTags` | `POST /2021-01-01/tags` | Add tags to a domain by ARN | +| `ListTags` | `GET /2021-01-01/tags/?arn=` | List tags for a domain | +| `RemoveTags` | `POST /2021-01-01/tags-removal` | Remove tag keys from a domain | + +### Versions & Instance Types + +| Operation | Method + Path | Description | +|---|---|---| +| `ListVersions` | `GET /2021-01-01/opensearch/versions` | List supported engine versions | +| `GetCompatibleVersions` | `GET /2021-01-01/opensearch/compatibleVersions` | List valid upgrade paths | +| `ListInstanceTypeDetails` | `GET /2021-01-01/opensearch/instanceTypeDetails/{version}` | List available instance types | +| `DescribeInstanceTypeLimits` | `GET /2021-01-01/opensearch/instanceTypeLimits/{version}/{type}` | Get limits for an instance type | + +### Stubs (SDK-compatible, no-op responses) + +| Operation | Notes | +|---|---| +| `DescribeDomainChangeProgress` | Returns empty `ChangeProgressStatus` | +| `DescribeDomainAutoTunes` | Returns empty `AutoTunes` list | +| `DescribeDryRunProgress` | Returns empty `DryRunProgressStatus` | +| `DescribeDomainHealth` | Returns `ClusterHealth: Green` | +| `GetUpgradeHistory` | Returns empty list | +| `GetUpgradeStatus` | Returns `StepStatus: SUCCEEDED` | +| `UpgradeDomain` | Stores new engine version, returns immediately | +| `CancelDomainConfigChange` | Returns empty `CancelledChangeIds` | +| `StartServiceSoftwareUpdate` | Returns no-op `ServiceSoftwareOptions` | +| `CancelServiceSoftwareUpdate` | Returns no-op `ServiceSoftwareOptions` | + +## Configuration + +```yaml title="application.yml" +floci: + services: + opensearch: + enabled: true + mode: mock # mock | real (real is v2, not yet available) + default-image: "opensearchproject/opensearch:2" # used only when mode=real + proxy-base-port: 9400 # port range for real-mode containers + proxy-max-port: 9499 + + storage: + services: + opensearch: + flush-interval-ms: 5000 # flush interval when using hybrid/wal storage +``` + +### Environment Variables + +| Variable | Default | Description | +|---|---|---| +| `FLOCI_SERVICES_OPENSEARCH_ENABLED` | `true` | Enable/disable the service | +| `FLOCI_SERVICES_OPENSEARCH_MODE` | `mock` | `mock` or `real` | +| `FLOCI_SERVICES_OPENSEARCH_DEFAULT_IMAGE` | `opensearchproject/opensearch:2` | Docker image for `real` mode | +| `FLOCI_SERVICES_OPENSEARCH_PROXY_BASE_PORT` | `9400` | Port range start for `real` mode | +| `FLOCI_SERVICES_OPENSEARCH_PROXY_MAX_PORT` | `9499` | Port range end for `real` mode | +| `FLOCI_STORAGE_SERVICES_OPENSEARCH_MODE` | *(global default)* | Storage mode override | +| `FLOCI_STORAGE_SERVICES_OPENSEARCH_FLUSH_INTERVAL_MS` | `5000` | Flush interval (ms) | + +## Emulation Behaviour + +- **Domain name validation:** 3–28 characters, must start with a lowercase letter, only lowercase letters, digits, and hyphens. +- **ARN format:** `arn:aws:es:{region}:{accountId}:domain/{domainName}` +- **Domain ID format:** `{accountId}/{domainName}` +- **Processing:** Always `false` in `mock` mode — domains are `ACTIVE` immediately. +- **Engine version default:** `OpenSearch_2.11` +- **Cluster defaults:** `m5.large.search`, 1 instance, EBS enabled with 10 GiB `gp2` volume. + +## Examples + +```bash +export AWS_ENDPOINT_URL=http://localhost:4566 +export AWS_DEFAULT_REGION=us-east-1 +export AWS_ACCESS_KEY_ID=test +export AWS_SECRET_ACCESS_KEY=test + +# Create a domain +aws opensearch create-domain \ + --domain-name my-search \ + --engine-version "OpenSearch_2.11" \ + --cluster-config InstanceType=m5.large.search,InstanceCount=1 \ + --ebs-options EBSEnabled=true,VolumeType=gp2,VolumeSize=10 + +# Describe the domain +aws opensearch describe-domain --domain-name my-search + +# List all domains +aws opensearch list-domain-names + +# Update cluster config +aws opensearch update-domain-config \ + --domain-name my-search \ + --cluster-config InstanceCount=3 + +# Add tags +aws opensearch add-tags \ + --arn arn:aws:es:us-east-1:000000000000:domain/my-search \ + --tag-list Key=env,Value=dev + +# List tags +aws opensearch list-tags \ + --arn arn:aws:es:us-east-1:000000000000:domain/my-search + +# Delete domain +aws opensearch delete-domain --domain-name my-search +``` + +## SDK Example (Java) + +```java +OpenSearchClient os = OpenSearchClient.builder() + .endpointOverride(URI.create("http://localhost:4566")) + .region(Region.US_EAST_1) + .credentialsProvider(StaticCredentialsProvider.create( + AwsBasicCredentials.create("test", "test"))) + .build(); + +// Create a domain +CreateDomainResponse created = os.createDomain(req -> req + .domainName("my-search") + .engineVersion("OpenSearch_2.11") + .clusterConfig(c -> c + .instanceType(OpenSearchPartitionInstanceType.M5_LARGE_SEARCH) + .instanceCount(1)) + .ebsOptions(e -> e + .ebsEnabled(true) + .volumeType(VolumeType.GP2) + .volumeSize(10))); + +System.out.println("ARN: " + created.domainStatus().arn()); + +// Describe the domain +DescribeDomainResponse desc = os.describeDomain(req -> req + .domainName("my-search")); + +System.out.println("Version: " + desc.domainStatus().engineVersion()); + +// List domains +os.listDomainNames(req -> req.build()) + .domainNames() + .forEach(d -> System.out.println(d.domainName())); + +// Delete +os.deleteDomain(req -> req.domainName("my-search")); +``` + +## SDK Example (Python) + +```python +import boto3 + +os_client = boto3.client( + "opensearch", + endpoint_url="http://localhost:4566", + region_name="us-east-1", + aws_access_key_id="test", + aws_secret_access_key="test" +) + +# Create a domain +response = os_client.create_domain( + DomainName="my-search", + EngineVersion="OpenSearch_2.11", + ClusterConfig={"InstanceType": "m5.large.search", "InstanceCount": 1}, + EBSOptions={"EBSEnabled": True, "VolumeType": "gp2", "VolumeSize": 10} +) +print(response["DomainStatus"]["ARN"]) + +# List domains +domains = os_client.list_domain_names() +for d in domains["DomainNames"]: + print(d["DomainName"]) + +# Delete +os_client.delete_domain(DomainName="my-search") +``` + +## Limitations (v1) + +- No real search capability in `mock` mode — data-plane endpoints (`/_search`, `/_index`, etc.) are not proxied. +- No Elasticsearch-compatible endpoints (`/2015-01-01/es/domain/...`). +- VPC options, fine-grained access control, encryption-at-rest, and cross-cluster connections are accepted in the request but silently ignored. +- All 41 "not applicable" operations (VPC endpoints, reserved instances, packages, applications, data sources) return `UnsupportedOperationException`. diff --git a/mkdocs.yml b/mkdocs.yml index 1cbc49f9..e4b6a9b0 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -85,4 +85,6 @@ nav: - RDS: services/rds.md - EventBridge: services/eventbridge.md - CloudWatch: services/cloudwatch.md + - ACM: services/acm.md + - OpenSearch: services/opensearch.md - Contributing: contributing.md \ No newline at end of file diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java index 9fa01285..f3e8de4e 100644 --- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java +++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java @@ -72,6 +72,7 @@ interface ServiceStorageOverrides { CloudWatchMetricsStorageConfig cloudwatchmetrics(); SecretsManagerStorageConfig secretsmanager(); AcmStorageConfig acm(); + OpenSearchStorageConfig opensearch(); } interface SsmStorageConfig { @@ -138,6 +139,13 @@ interface AcmStorageConfig { long flushIntervalMs(); } + interface OpenSearchStorageConfig { + Optional mode(); + + @WithDefault("5000") + long flushIntervalMs(); + } + interface WalConfig { @WithDefault("30000") long compactionIntervalMs(); @@ -178,6 +186,7 @@ interface ServicesConfig { CloudFormationServiceConfig cloudformation(); AcmServiceConfig acm(); SesServiceConfig ses(); + OpenSearchServiceConfig opensearch(); } interface SsmServiceConfig { @@ -337,6 +346,25 @@ interface SesServiceConfig { boolean enabled(); } + interface OpenSearchServiceConfig { + @WithDefault("true") + boolean enabled(); + + @WithDefault("mock") + String mode(); + + @WithDefault("opensearchproject/opensearch:2") + String defaultImage(); + + @WithDefault("9400") + int proxyBasePort(); + + @WithDefault("9499") + int proxyMaxPort(); + + Optional dockerNetwork(); + } + interface LambdaServiceConfig { @WithDefault("true") boolean enabled(); diff --git a/src/main/java/io/github/hectorvent/floci/core/common/ServiceRegistry.java b/src/main/java/io/github/hectorvent/floci/core/common/ServiceRegistry.java index 9fd075f9..5da4e28f 100644 --- a/src/main/java/io/github/hectorvent/floci/core/common/ServiceRegistry.java +++ b/src/main/java/io/github/hectorvent/floci/core/common/ServiceRegistry.java @@ -47,6 +47,7 @@ public boolean isServiceEnabled(String serviceName) { case "cloudformation" -> config.services().cloudformation().enabled(); case "acm" -> config.services().acm().enabled(); case "email" -> config.services().ses().enabled(); + case "es" -> config.services().opensearch().enabled(); default -> true; }; } @@ -75,6 +76,7 @@ public List getEnabledServices() { if (config.services().cloudformation().enabled()) enabled.add("cloudformation"); if (config.services().acm().enabled()) enabled.add("acm"); if (config.services().ses().enabled()) enabled.add("email"); + if (config.services().opensearch().enabled()) enabled.add("es"); return enabled; } diff --git a/src/main/java/io/github/hectorvent/floci/core/storage/StorageFactory.java b/src/main/java/io/github/hectorvent/floci/core/storage/StorageFactory.java index c245e3eb..151f6b51 100644 --- a/src/main/java/io/github/hectorvent/floci/core/storage/StorageFactory.java +++ b/src/main/java/io/github/hectorvent/floci/core/storage/StorageFactory.java @@ -109,6 +109,7 @@ private String resolveMode(String serviceName) { case "cloudwatchlogs" -> config.storage().services().cloudwatchlogs().mode().orElse(globalMode); case "cloudwatchmetrics" -> config.storage().services().cloudwatchmetrics().mode().orElse(globalMode); case "secretsmanager" -> config.storage().services().secretsmanager().mode().orElse(globalMode); + case "opensearch" -> config.storage().services().opensearch().mode().orElse(globalMode); default -> globalMode; }; } @@ -122,6 +123,7 @@ private long resolveFlushInterval(String serviceName) { case "cloudwatchlogs" -> config.storage().services().cloudwatchlogs().flushIntervalMs(); case "cloudwatchmetrics" -> config.storage().services().cloudwatchmetrics().flushIntervalMs(); case "secretsmanager" -> config.storage().services().secretsmanager().flushIntervalMs(); + case "opensearch" -> config.storage().services().opensearch().flushIntervalMs(); default -> 5000L; }; } diff --git a/src/main/java/io/github/hectorvent/floci/services/opensearch/OpenSearchController.java b/src/main/java/io/github/hectorvent/floci/services/opensearch/OpenSearchController.java new file mode 100644 index 00000000..153abc23 --- /dev/null +++ b/src/main/java/io/github/hectorvent/floci/services/opensearch/OpenSearchController.java @@ -0,0 +1,556 @@ +package io.github.hectorvent.floci.services.opensearch; + +import io.github.hectorvent.floci.core.common.AwsException; +import io.github.hectorvent.floci.core.common.RegionResolver; +import io.github.hectorvent.floci.services.opensearch.model.ClusterConfig; +import io.github.hectorvent.floci.services.opensearch.model.Domain; +import io.github.hectorvent.floci.services.opensearch.model.EbsOptions; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import jakarta.inject.Inject; +import jakarta.ws.rs.*; +import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.HttpHeaders; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; +import org.jboss.logging.Logger; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Path("/2021-01-01") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +public class OpenSearchController { + + private static final Logger LOG = Logger.getLogger(OpenSearchController.class); + + private static final List SUPPORTED_VERSIONS = List.of( + "OpenSearch_2.13", "OpenSearch_2.11", "OpenSearch_2.9", "OpenSearch_2.7", + "OpenSearch_2.5", "OpenSearch_2.3", "OpenSearch_1.3", "OpenSearch_1.2", + "Elasticsearch_7.10", "Elasticsearch_7.9", "Elasticsearch_7.8" + ); + + private static final List INSTANCE_TYPES = List.of( + "t3.small.search", "t3.medium.search", + "m5.large.search", "m5.xlarge.search", "m5.2xlarge.search", + "r5.large.search", "r5.xlarge.search", "r5.2xlarge.search", + "c5.large.search", "c5.xlarge.search", "c5.2xlarge.search" + ); + + private final OpenSearchService service; + private final RegionResolver regionResolver; + private final ObjectMapper objectMapper; + + @Inject + public OpenSearchController(OpenSearchService service, RegionResolver regionResolver, + ObjectMapper objectMapper) { + this.service = service; + this.regionResolver = regionResolver; + this.objectMapper = objectMapper; + } + + @POST + @Path("/opensearch/domain") + public Response createDomain(@Context HttpHeaders headers, String body) { + String region = regionResolver.resolveRegion(headers); + try { + JsonNode req = objectMapper.readTree(body); + String domainName = req.path("DomainName").asText(null); + String engineVersion = req.path("EngineVersion").asText(null); + ClusterConfig clusterConfig = parseClusterConfig(req.path("ClusterConfig")); + EbsOptions ebsOptions = parseEbsOptions(req.path("EBSOptions")); + Map tags = parseTags(req.path("TagList")); + + Domain domain = service.createDomain(domainName, engineVersion, clusterConfig, + ebsOptions, tags, region); + + ObjectNode response = objectMapper.createObjectNode(); + response.set("DomainStatus", toDomainStatusNode(domain)); + return Response.ok(response).build(); + } catch (AwsException e) { + throw e; + } catch (IOException e) { + throw new AwsException("ValidationException", e.getMessage(), 400); + } + } + + @GET + @Path("/opensearch/domain/{domainName}") + public Response describeDomain(@Context HttpHeaders headers, + @PathParam("domainName") String domainName) { + Domain domain = service.describeDomain(domainName); + ObjectNode response = objectMapper.createObjectNode(); + response.set("DomainStatus", toDomainStatusNode(domain)); + return Response.ok(response).build(); + } + + @POST + @Path("/opensearch/domain-info") + public Response describeDomains(@Context HttpHeaders headers, String body) { + try { + JsonNode req = objectMapper.readTree(body); + List names = new ArrayList<>(); + req.path("DomainNames").forEach(n -> names.add(n.asText())); + List domains = service.describeDomains(names); + + ObjectNode response = objectMapper.createObjectNode(); + ArrayNode list = response.putArray("DomainStatusList"); + domains.forEach(d -> list.add(toDomainStatusNode(d))); + return Response.ok(response).build(); + } catch (AwsException e) { + throw e; + } catch (IOException e) { + throw new AwsException("ValidationException", e.getMessage(), 400); + } + } + + @GET + @Path("/domain") + public Response listDomainNames(@Context HttpHeaders headers, + @QueryParam("engineType") String engineType) { + List domains = service.listDomainNames(engineType); + ObjectNode response = objectMapper.createObjectNode(); + ArrayNode list = response.putArray("DomainNames"); + for (Domain d : domains) { + ObjectNode entry = objectMapper.createObjectNode(); + entry.put("DomainName", d.getDomainName()); + String ev = d.getEngineVersion(); + entry.put("EngineType", (ev != null && ev.startsWith("Elasticsearch")) ? "Elasticsearch" : "OpenSearch"); + list.add(entry); + } + return Response.ok(response).build(); + } + + @GET + @Path("/opensearch/domain/{domainName}/config") + public Response describeDomainConfig(@Context HttpHeaders headers, + @PathParam("domainName") String domainName) { + Domain domain = service.describeDomain(domainName); + long epochSeconds = domain.getCreatedAt() != null ? domain.getCreatedAt().getEpochSecond() : 0; + + ObjectNode response = objectMapper.createObjectNode(); + ObjectNode domainConfig = response.putObject("DomainConfig"); + + ObjectNode clusterSection = domainConfig.putObject("ClusterConfig"); + clusterSection.set("Options", toClusterConfigNode(domain.getClusterConfig())); + clusterSection.set("Status", configStatusNode(epochSeconds)); + + ObjectNode ebsSection = domainConfig.putObject("EBSOptions"); + ebsSection.set("Options", toEbsOptionsNode(domain.getEbsOptions())); + ebsSection.set("Status", configStatusNode(epochSeconds)); + + ObjectNode versionSection = domainConfig.putObject("EngineVersion"); + versionSection.put("Options", domain.getEngineVersion()); + versionSection.set("Status", configStatusNode(epochSeconds)); + + return Response.ok(response).build(); + } + + @POST + @Path("/opensearch/domain/{domainName}/config") + public Response updateDomainConfig(@Context HttpHeaders headers, + @PathParam("domainName") String domainName, + String body) { + String region = regionResolver.resolveRegion(headers); + try { + JsonNode req = objectMapper.readTree(body); + String engineVersion = req.path("EngineVersion").asText(null); + ClusterConfig clusterConfig = parseClusterConfig(req.path("ClusterConfig")); + EbsOptions ebsOptions = parseEbsOptions(req.path("EBSOptions")); + + Domain domain = service.updateDomainConfig(domainName, engineVersion, clusterConfig, + ebsOptions, region); + + long epochSeconds = domain.getCreatedAt() != null ? domain.getCreatedAt().getEpochSecond() : 0; + ObjectNode response = objectMapper.createObjectNode(); + ObjectNode domainConfig = response.putObject("DomainConfig"); + + ObjectNode clusterSection = domainConfig.putObject("ClusterConfig"); + clusterSection.set("Options", toClusterConfigNode(domain.getClusterConfig())); + clusterSection.set("Status", configStatusNode(epochSeconds)); + + ObjectNode ebsSection = domainConfig.putObject("EBSOptions"); + ebsSection.set("Options", toEbsOptionsNode(domain.getEbsOptions())); + ebsSection.set("Status", configStatusNode(epochSeconds)); + + ObjectNode versionSection = domainConfig.putObject("EngineVersion"); + versionSection.put("Options", domain.getEngineVersion()); + versionSection.set("Status", configStatusNode(epochSeconds)); + + return Response.ok(response).build(); + } catch (AwsException e) { + throw e; + } catch (IOException e) { + throw new AwsException("ValidationException", e.getMessage(), 400); + } + } + + @DELETE + @Path("/opensearch/domain/{domainName}") + public Response deleteDomain(@Context HttpHeaders headers, + @PathParam("domainName") String domainName) { + Domain domain = service.deleteDomain(domainName); + ObjectNode response = objectMapper.createObjectNode(); + response.set("DomainStatus", toDomainStatusNode(domain)); + return Response.ok(response).build(); + } + + // ── Tags ───────────────────────────────────────────────────────────────── + + @POST + @Path("/tags") + public Response addTags(@Context HttpHeaders headers, String body) { + try { + JsonNode req = objectMapper.readTree(body); + String arn = req.path("ARN").asText(null); + if (arn == null || arn.isBlank()) { + throw new AwsException("ValidationException", "ARN is required.", 400); + } + Map tags = parseTags(req.path("TagList")); + service.addTags(arn, tags); + return Response.ok("{}").build(); + } catch (AwsException e) { + throw e; + } catch (IOException e) { + throw new AwsException("ValidationException", e.getMessage(), 400); + } + } + + @GET + @Path("/tags/") + public Response listTags(@Context HttpHeaders headers, @QueryParam("arn") String arn) { + if (arn == null || arn.isBlank()) { + throw new AwsException("ValidationException", "ARN query parameter is required.", 400); + } + Map tags = service.listTags(arn); + ObjectNode response = objectMapper.createObjectNode(); + ArrayNode tagList = response.putArray("TagList"); + tags.forEach((k, v) -> { + ObjectNode tag = objectMapper.createObjectNode(); + tag.put("Key", k); + tag.put("Value", v); + tagList.add(tag); + }); + return Response.ok(response).build(); + } + + @POST + @Path("/tags-removal") + public Response removeTags(@Context HttpHeaders headers, String body) { + try { + JsonNode req = objectMapper.readTree(body); + String arn = req.path("ARN").asText(null); + if (arn == null || arn.isBlank()) { + throw new AwsException("ValidationException", "ARN is required.", 400); + } + List tagKeys = new ArrayList<>(); + req.path("TagKeys").forEach(n -> tagKeys.add(n.asText())); + service.removeTags(arn, tagKeys); + return Response.ok("{}").build(); + } catch (AwsException e) { + throw e; + } catch (IOException e) { + throw new AwsException("ValidationException", e.getMessage(), 400); + } + } + + @GET + @Path("/opensearch/versions") + public Response listVersions(@Context HttpHeaders headers) { + ObjectNode response = objectMapper.createObjectNode(); + ArrayNode versions = response.putArray("Versions"); + SUPPORTED_VERSIONS.forEach(versions::add); + return Response.ok(response).build(); + } + + @GET + @Path("/opensearch/compatibleVersions") + public Response getCompatibleVersions(@Context HttpHeaders headers, + @QueryParam("domainName") String domainName) { + ObjectNode response = objectMapper.createObjectNode(); + ArrayNode compatibleVersions = response.putArray("CompatibleVersions"); + + ObjectNode entry = objectMapper.createObjectNode(); + entry.put("SourceVersion", "OpenSearch_2.9"); + ArrayNode targets = entry.putArray("TargetVersions"); + targets.add("OpenSearch_2.11"); + targets.add("OpenSearch_2.13"); + compatibleVersions.add(entry); + + return Response.ok(response).build(); + } + + @GET + @Path("/opensearch/instanceTypeDetails/{engineVersion}") + public Response listInstanceTypeDetails(@Context HttpHeaders headers, + @PathParam("engineVersion") String engineVersion) { + ObjectNode response = objectMapper.createObjectNode(); + ArrayNode details = response.putArray("InstanceTypeDetails"); + for (String instanceType : INSTANCE_TYPES) { + ObjectNode detail = objectMapper.createObjectNode(); + detail.put("InstanceType", instanceType); + detail.put("EncryptionEnabled", true); + detail.put("CognitoEnabled", false); + detail.put("AppLogsEnabled", true); + detail.put("AdvancedSecurityEnabled", false); + ArrayNode roles = detail.putArray("InstanceRole"); + roles.add("Data"); + details.add(detail); + } + return Response.ok(response).build(); + } + + @GET + @Path("/opensearch/instanceTypeLimits/{engineVersion}/{instanceType}") + public Response describeInstanceTypeLimits(@Context HttpHeaders headers, + @PathParam("engineVersion") String engineVersion, + @PathParam("instanceType") String instanceType) { + ObjectNode response = objectMapper.createObjectNode(); + ObjectNode limitsByRole = response.putObject("LimitsByRole"); + ObjectNode dataRole = limitsByRole.putObject("data"); + + ArrayNode storageTypes = dataRole.putArray("StorageTypes"); + ObjectNode storageType = objectMapper.createObjectNode(); + storageType.put("StorageTypeName", "ebs"); + storageType.put("StorageSubTypeName", "standard"); + ArrayNode storageTypeLimits = storageType.putArray("StorageTypeLimits"); + ObjectNode minLimit = objectMapper.createObjectNode(); + minLimit.put("LimitName", "MinimumVolumeSize"); + minLimit.putArray("LimitValues").add("10"); + storageTypeLimits.add(minLimit); + ObjectNode maxLimit = objectMapper.createObjectNode(); + maxLimit.put("LimitName", "MaximumVolumeSize"); + maxLimit.putArray("LimitValues").add("3584"); + storageTypeLimits.add(maxLimit); + storageTypes.add(storageType); + + ObjectNode instanceLimits = dataRole.putObject("InstanceLimits"); + ObjectNode instanceCountLimits = instanceLimits.putObject("InstanceCountLimits"); + instanceCountLimits.put("MinimumInstanceCount", 1); + instanceCountLimits.put("MaximumInstanceCount", 20); + + dataRole.putArray("AdditionalLimits"); + + return Response.ok(response).build(); + } + + @GET + @Path("/opensearch/domain/{domainName}/progress") + public Response describeDomainChangeProgress(@PathParam("domainName") String domainName) { + ObjectNode response = objectMapper.createObjectNode(); + response.putObject("ChangeProgressStatus"); + return Response.ok(response).build(); + } + + @GET + @Path("/opensearch/domain/{domainName}/autoTunes") + public Response describeDomainAutoTunes(@PathParam("domainName") String domainName) { + ObjectNode response = objectMapper.createObjectNode(); + response.putArray("AutoTunes"); + return Response.ok(response).build(); + } + + @GET + @Path("/opensearch/domain/{domainName}/dryRun") + public Response describeDryRunProgress(@PathParam("domainName") String domainName) { + ObjectNode response = objectMapper.createObjectNode(); + response.putObject("DryRunProgressStatus"); + return Response.ok(response).build(); + } + + @GET + @Path("/opensearch/domain/{domainName}/health") + public Response describeDomainHealth(@PathParam("domainName") String domainName) { + ObjectNode response = objectMapper.createObjectNode(); + response.put("ClusterHealth", "Green"); + return Response.ok(response).build(); + } + + @GET + @Path("/opensearch/upgradeDomain/{domainName}/history") + public Response getUpgradeHistory(@PathParam("domainName") String domainName) { + ObjectNode response = objectMapper.createObjectNode(); + response.putArray("UpgradeHistories"); + return Response.ok(response).build(); + } + + @GET + @Path("/opensearch/upgradeDomain/{domainName}/status") + public Response getUpgradeStatus(@PathParam("domainName") String domainName) { + ObjectNode response = objectMapper.createObjectNode(); + response.put("UpgradeStep", "UPGRADE"); + response.put("StepStatus", "SUCCEEDED"); + response.put("UpgradeName", ""); + return Response.ok(response).build(); + } + + @POST + @Path("/opensearch/upgradeDomain") + public Response upgradeDomain(String body) { + try { + JsonNode req = objectMapper.readTree(body); + String domainName = req.path("DomainName").asText(null); + String targetVersion = req.path("TargetVersion").asText(null); + Domain domain = service.upgradeDomain(domainName, targetVersion); + + ObjectNode response = objectMapper.createObjectNode(); + response.put("DomainName", domain.getDomainName()); + response.put("TargetVersion", domain.getEngineVersion()); + response.put("PerformCheckOnly", false); + return Response.ok(response).build(); + } catch (AwsException e) { + throw e; + } catch (IOException e) { + throw new AwsException("ValidationException", e.getMessage(), 400); + } + } + + @POST + @Path("/opensearch/domain/{domainName}/config/cancel") + public Response cancelDomainConfigChange(@PathParam("domainName") String domainName) { + ObjectNode response = objectMapper.createObjectNode(); + response.put("DryRun", false); + response.putArray("CancelledChangeIds"); + return Response.ok(response).build(); + } + + @POST + @Path("/opensearch/serviceSoftwareUpdate/start") + public Response startServiceSoftwareUpdate(String body) { + ObjectNode response = objectMapper.createObjectNode(); + ObjectNode options = response.putObject("ServiceSoftwareOptions"); + options.put("UpdateAvailable", false); + options.put("Cancellable", false); + options.put("UpdateStatus", "COMPLETED"); + options.put("Description", "There is no software update available for this domain."); + options.put("AutomatedUpdateDate", 0); + options.put("OptionalDeployment", false); + return Response.ok(response).build(); + } + + @POST + @Path("/opensearch/serviceSoftwareUpdate/cancel") + public Response cancelServiceSoftwareUpdate(String body) { + ObjectNode response = objectMapper.createObjectNode(); + ObjectNode options = response.putObject("ServiceSoftwareOptions"); + options.put("UpdateAvailable", false); + options.put("Cancellable", false); + options.put("UpdateStatus", "COMPLETED"); + options.put("Description", "There is no software update available for this domain."); + options.put("AutomatedUpdateDate", 0); + options.put("OptionalDeployment", false); + return Response.ok(response).build(); + } + + // ── Helpers ─────────────────────────────────────────────────────────────── + + private ObjectNode toDomainStatusNode(Domain domain) { + ObjectNode node = objectMapper.createObjectNode(); + node.put("ARN", domain.getArn()); + node.put("DomainId", domain.getDomainId()); + node.put("DomainName", domain.getDomainName()); + node.put("EngineVersion", domain.getEngineVersion()); + node.put("Processing", domain.isProcessing()); + node.put("Deleted", domain.isDeleted()); + node.put("Endpoint", domain.getEndpoint() != null ? domain.getEndpoint() : ""); + node.set("ClusterConfig", toClusterConfigNode(domain.getClusterConfig())); + node.set("EBSOptions", toEbsOptionsNode(domain.getEbsOptions())); + return node; + } + + private ObjectNode toClusterConfigNode(ClusterConfig cc) { + ObjectNode node = objectMapper.createObjectNode(); + if (cc == null) { + node.put("InstanceType", "m5.large.search"); + node.put("InstanceCount", 1); + node.put("DedicatedMasterEnabled", false); + node.put("ZoneAwarenessEnabled", false); + } else { + node.put("InstanceType", cc.getInstanceType()); + node.put("InstanceCount", cc.getInstanceCount()); + node.put("DedicatedMasterEnabled", cc.isDedicatedMasterEnabled()); + node.put("ZoneAwarenessEnabled", cc.isZoneAwarenessEnabled()); + } + return node; + } + + private ObjectNode toEbsOptionsNode(EbsOptions ebs) { + ObjectNode node = objectMapper.createObjectNode(); + if (ebs == null) { + node.put("EBSEnabled", true); + node.put("VolumeType", "gp2"); + node.put("VolumeSize", 10); + } else { + node.put("EBSEnabled", ebs.isEbsEnabled()); + node.put("VolumeType", ebs.getVolumeType()); + node.put("VolumeSize", ebs.getVolumeSize()); + } + return node; + } + + private ObjectNode configStatusNode(long epochSeconds) { + ObjectNode status = objectMapper.createObjectNode(); + status.put("CreationDate", epochSeconds); + status.put("UpdateDate", epochSeconds); + status.put("State", "Active"); + return status; + } + + private ClusterConfig parseClusterConfig(JsonNode node) { + if (node == null || node.isMissingNode() || node.isNull()) { + return null; + } + ClusterConfig cc = new ClusterConfig(); + if (node.has("InstanceType")) { + cc.setInstanceType(node.get("InstanceType").asText()); + } + if (node.has("InstanceCount")) { + cc.setInstanceCount(node.get("InstanceCount").asInt()); + } + if (node.has("DedicatedMasterEnabled")) { + cc.setDedicatedMasterEnabled(node.get("DedicatedMasterEnabled").asBoolean()); + } + if (node.has("ZoneAwarenessEnabled")) { + cc.setZoneAwarenessEnabled(node.get("ZoneAwarenessEnabled").asBoolean()); + } + return cc; + } + + private EbsOptions parseEbsOptions(JsonNode node) { + if (node == null || node.isMissingNode() || node.isNull()) { + return null; + } + EbsOptions ebs = new EbsOptions(); + if (node.has("EBSEnabled")) { + ebs.setEbsEnabled(node.get("EBSEnabled").asBoolean()); + } + if (node.has("VolumeType")) { + ebs.setVolumeType(node.get("VolumeType").asText()); + } + if (node.has("VolumeSize")) { + ebs.setVolumeSize(node.get("VolumeSize").asInt()); + } + return ebs; + } + + private Map parseTags(JsonNode node) { + Map tags = new HashMap<>(); + if (node == null || node.isMissingNode() || node.isNull()) { + return tags; + } + node.forEach(tag -> { + String key = tag.path("Key").asText(null); + String value = tag.path("Value").asText(null); + if (key != null && value != null) { + tags.put(key, value); + } + }); + return tags; + } +} diff --git a/src/main/java/io/github/hectorvent/floci/services/opensearch/OpenSearchService.java b/src/main/java/io/github/hectorvent/floci/services/opensearch/OpenSearchService.java new file mode 100644 index 00000000..67557844 --- /dev/null +++ b/src/main/java/io/github/hectorvent/floci/services/opensearch/OpenSearchService.java @@ -0,0 +1,190 @@ +package io.github.hectorvent.floci.services.opensearch; + +import io.github.hectorvent.floci.config.EmulatorConfig; +import io.github.hectorvent.floci.core.common.AwsException; +import io.github.hectorvent.floci.core.storage.StorageBackend; +import io.github.hectorvent.floci.core.storage.StorageFactory; +import io.github.hectorvent.floci.services.opensearch.model.ClusterConfig; +import io.github.hectorvent.floci.services.opensearch.model.Domain; +import io.github.hectorvent.floci.services.opensearch.model.EbsOptions; +import com.fasterxml.jackson.core.type.TypeReference; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import org.jboss.logging.Logger; + +import java.time.Instant; +import java.util.List; +import java.util.Map; + +@ApplicationScoped +public class OpenSearchService { + + private static final Logger LOG = Logger.getLogger(OpenSearchService.class); + + private static final String DEFAULT_ENGINE_VERSION = "OpenSearch_2.11"; + + private final StorageBackend domainStore; + private final EmulatorConfig config; + + @Inject + public OpenSearchService(StorageFactory storageFactory, EmulatorConfig config) { + this.domainStore = storageFactory.create("opensearch", "opensearch-domains.json", + new TypeReference>() {}); + this.config = config; + } + + OpenSearchService(StorageBackend domainStore, EmulatorConfig config) { + this.domainStore = domainStore; + this.config = config; + } + + public Domain createDomain(String domainName, String engineVersion, ClusterConfig clusterConfig, + EbsOptions ebsOptions, Map tags, String region) { + validateDomainName(domainName); + + if (domainStore.get(domainName).isPresent()) { + throw new AwsException("ResourceAlreadyExistsException", + "Domain with name " + domainName + " already exists.", 409); + } + + String accountId = config.defaultAccountId(); + Domain domain = new Domain(); + domain.setDomainName(domainName); + domain.setDomainId(accountId + "/" + domainName); + domain.setArn("arn:aws:es:" + region + ":" + accountId + ":domain/" + domainName); + domain.setEngineVersion(engineVersion != null ? engineVersion : DEFAULT_ENGINE_VERSION); + domain.setProcessing(false); + domain.setDeleted(false); + domain.setEndpoint(""); + domain.setCreatedAt(Instant.now()); + + if (clusterConfig != null) { + domain.setClusterConfig(clusterConfig); + } + if (ebsOptions != null) { + domain.setEbsOptions(ebsOptions); + } + if (tags != null) { + domain.setTags(tags); + } + + domainStore.put(domainName, domain); + LOG.infov("Created OpenSearch domain: {0}", domainName); + return domain; + } + + public Domain describeDomain(String domainName) { + return domainStore.get(domainName) + .orElseThrow(() -> new AwsException("ResourceNotFoundException", + "Domain not found: " + domainName, 409)); + } + + public List describeDomains(List domainNames) { + return domainNames.stream() + .map(name -> domainStore.get(name) + .orElseThrow(() -> new AwsException("ResourceNotFoundException", + "Domain not found: " + name, 409))) + .toList(); + } + + public List listDomainNames(String engineType) { + return domainStore.scan(k -> true).stream() + .filter(d -> !d.isDeleted()) + .filter(d -> engineType == null || engineType.isBlank() + || matchesEngineType(d.getEngineVersion(), engineType)) + .toList(); + } + + public Domain updateDomainConfig(String domainName, String engineVersion, + ClusterConfig clusterConfig, EbsOptions ebsOptions, + String region) { + Domain domain = describeDomain(domainName); + + if (engineVersion != null && !engineVersion.isBlank()) { + domain.setEngineVersion(engineVersion); + } + if (clusterConfig != null) { + ClusterConfig existing = domain.getClusterConfig(); + if (clusterConfig.getInstanceType() != null) { + existing.setInstanceType(clusterConfig.getInstanceType()); + } + if (clusterConfig.getInstanceCount() > 0) { + existing.setInstanceCount(clusterConfig.getInstanceCount()); + } + existing.setDedicatedMasterEnabled(clusterConfig.isDedicatedMasterEnabled()); + existing.setZoneAwarenessEnabled(clusterConfig.isZoneAwarenessEnabled()); + } + if (ebsOptions != null) { + EbsOptions existing = domain.getEbsOptions(); + existing.setEbsEnabled(ebsOptions.isEbsEnabled()); + if (ebsOptions.getVolumeType() != null) { + existing.setVolumeType(ebsOptions.getVolumeType()); + } + if (ebsOptions.getVolumeSize() > 0) { + existing.setVolumeSize(ebsOptions.getVolumeSize()); + } + } + + domainStore.put(domainName, domain); + return domain; + } + + public Domain deleteDomain(String domainName) { + Domain domain = describeDomain(domainName); + domain.setDeleted(true); + domainStore.delete(domainName); + LOG.infov("Deleted OpenSearch domain: {0}", domainName); + return domain; + } + + public void addTags(String arn, Map tags) { + Domain domain = findByArn(arn); + domain.getTags().putAll(tags); + domainStore.put(domain.getDomainName(), domain); + } + + public Map listTags(String arn) { + return findByArn(arn).getTags(); + } + + public void removeTags(String arn, List tagKeys) { + Domain domain = findByArn(arn); + tagKeys.forEach(domain.getTags()::remove); + domainStore.put(domain.getDomainName(), domain); + } + + public Domain upgradeDomain(String domainName, String targetVersion) { + Domain domain = describeDomain(domainName); + if (targetVersion != null && !targetVersion.isBlank()) { + domain.setEngineVersion(targetVersion); + domainStore.put(domainName, domain); + } + return domain; + } + + private Domain findByArn(String arn) { + return domainStore.scan(k -> true).stream() + .filter(d -> arn.equals(d.getArn())) + .findFirst() + .orElseThrow(() -> new AwsException("ResourceNotFoundException", + "Domain not found for ARN: " + arn, 409)); + } + + private void validateDomainName(String name) { + if (name == null || name.length() < 3 || name.length() > 28) { + throw new AwsException("ValidationException", + "Domain name must be between 3 and 28 characters.", 400); + } + if (!name.matches("[a-z][a-z0-9\\-]*")) { + throw new AwsException("ValidationException", + "Domain name must start with a lowercase letter and contain only lowercase letters, numbers, and hyphens.", 400); + } + } + + private boolean matchesEngineType(String engineVersion, String engineType) { + if ("Elasticsearch".equalsIgnoreCase(engineType)) { + return engineVersion != null && engineVersion.startsWith("Elasticsearch"); + } + return engineVersion == null || engineVersion.startsWith("OpenSearch"); + } +} diff --git a/src/main/java/io/github/hectorvent/floci/services/opensearch/model/ClusterConfig.java b/src/main/java/io/github/hectorvent/floci/services/opensearch/model/ClusterConfig.java new file mode 100644 index 00000000..9ba6fb71 --- /dev/null +++ b/src/main/java/io/github/hectorvent/floci/services/opensearch/model/ClusterConfig.java @@ -0,0 +1,56 @@ +package io.github.hectorvent.floci.services.opensearch.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.quarkus.runtime.annotations.RegisterForReflection; + +@RegisterForReflection +@JsonIgnoreProperties(ignoreUnknown = true) +public class ClusterConfig { + + @JsonProperty("InstanceType") + private String instanceType = "m5.large.search"; + + @JsonProperty("InstanceCount") + private int instanceCount = 1; + + @JsonProperty("DedicatedMasterEnabled") + private boolean dedicatedMasterEnabled = false; + + @JsonProperty("ZoneAwarenessEnabled") + private boolean zoneAwarenessEnabled = false; + + public ClusterConfig() {} + + public String getInstanceType() { + return instanceType; + } + + public void setInstanceType(String instanceType) { + this.instanceType = instanceType; + } + + public int getInstanceCount() { + return instanceCount; + } + + public void setInstanceCount(int instanceCount) { + this.instanceCount = instanceCount; + } + + public boolean isDedicatedMasterEnabled() { + return dedicatedMasterEnabled; + } + + public void setDedicatedMasterEnabled(boolean dedicatedMasterEnabled) { + this.dedicatedMasterEnabled = dedicatedMasterEnabled; + } + + public boolean isZoneAwarenessEnabled() { + return zoneAwarenessEnabled; + } + + public void setZoneAwarenessEnabled(boolean zoneAwarenessEnabled) { + this.zoneAwarenessEnabled = zoneAwarenessEnabled; + } +} diff --git a/src/main/java/io/github/hectorvent/floci/services/opensearch/model/Domain.java b/src/main/java/io/github/hectorvent/floci/services/opensearch/model/Domain.java new file mode 100644 index 00000000..e3ffba64 --- /dev/null +++ b/src/main/java/io/github/hectorvent/floci/services/opensearch/model/Domain.java @@ -0,0 +1,137 @@ +package io.github.hectorvent.floci.services.opensearch.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.quarkus.runtime.annotations.RegisterForReflection; + +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; + +@RegisterForReflection +@JsonIgnoreProperties(ignoreUnknown = true) +public class Domain { + + @JsonProperty("DomainName") + private String domainName; + + @JsonProperty("DomainId") + private String domainId; + + @JsonProperty("ARN") + private String arn; + + @JsonProperty("EngineVersion") + private String engineVersion = "OpenSearch_2.11"; + + @JsonProperty("Processing") + private boolean processing = false; + + @JsonProperty("Deleted") + private boolean deleted = false; + + @JsonProperty("ClusterConfig") + private ClusterConfig clusterConfig = new ClusterConfig(); + + @JsonProperty("EBSOptions") + private EbsOptions ebsOptions = new EbsOptions(); + + @JsonProperty("Endpoint") + private String endpoint = ""; + + @JsonProperty("Tags") + private Map tags = new HashMap<>(); + + @JsonProperty("CreatedAt") + private Instant createdAt; + + public Domain() {} + + public String getDomainName() { + return domainName; + } + + public void setDomainName(String domainName) { + this.domainName = domainName; + } + + public String getDomainId() { + return domainId; + } + + public void setDomainId(String domainId) { + this.domainId = domainId; + } + + public String getArn() { + return arn; + } + + public void setArn(String arn) { + this.arn = arn; + } + + public String getEngineVersion() { + return engineVersion; + } + + public void setEngineVersion(String engineVersion) { + this.engineVersion = engineVersion; + } + + public boolean isProcessing() { + return processing; + } + + public void setProcessing(boolean processing) { + this.processing = processing; + } + + public boolean isDeleted() { + return deleted; + } + + public void setDeleted(boolean deleted) { + this.deleted = deleted; + } + + public ClusterConfig getClusterConfig() { + return clusterConfig; + } + + public void setClusterConfig(ClusterConfig clusterConfig) { + this.clusterConfig = clusterConfig; + } + + public EbsOptions getEbsOptions() { + return ebsOptions; + } + + public void setEbsOptions(EbsOptions ebsOptions) { + this.ebsOptions = ebsOptions; + } + + public String getEndpoint() { + return endpoint; + } + + public void setEndpoint(String endpoint) { + this.endpoint = endpoint; + } + + public Map getTags() { + return tags; + } + + public void setTags(Map tags) { + this.tags = tags != null ? new HashMap<>(tags) : new HashMap<>(); + } + + public Instant getCreatedAt() { + return createdAt; + } + + public void setCreatedAt(Instant createdAt) { + this.createdAt = createdAt; + } +} diff --git a/src/main/java/io/github/hectorvent/floci/services/opensearch/model/EbsOptions.java b/src/main/java/io/github/hectorvent/floci/services/opensearch/model/EbsOptions.java new file mode 100644 index 00000000..0725b09b --- /dev/null +++ b/src/main/java/io/github/hectorvent/floci/services/opensearch/model/EbsOptions.java @@ -0,0 +1,45 @@ +package io.github.hectorvent.floci.services.opensearch.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.quarkus.runtime.annotations.RegisterForReflection; + +@RegisterForReflection +@JsonIgnoreProperties(ignoreUnknown = true) +public class EbsOptions { + + @JsonProperty("EBSEnabled") + private boolean ebsEnabled = true; + + @JsonProperty("VolumeType") + private String volumeType = "gp2"; + + @JsonProperty("VolumeSize") + private int volumeSize = 10; + + public EbsOptions() {} + + public boolean isEbsEnabled() { + return ebsEnabled; + } + + public void setEbsEnabled(boolean ebsEnabled) { + this.ebsEnabled = ebsEnabled; + } + + public String getVolumeType() { + return volumeType; + } + + public void setVolumeType(String volumeType) { + this.volumeType = volumeType; + } + + public int getVolumeSize() { + return volumeSize; + } + + public void setVolumeSize(int volumeSize) { + this.volumeSize = volumeSize; + } +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index c1d72ca3..81f37964 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -45,6 +45,8 @@ floci: flush-interval-ms: 5000 acm: flush-interval-ms: 5000 + opensearch: + flush-interval-ms: 5000 auth: validate-signatures: false @@ -117,3 +119,9 @@ floci: validation-wait-seconds: 0 ses: enabled: true + opensearch: + enabled: true + mode: mock + default-image: "opensearchproject/opensearch:2" + proxy-base-port: 9400 + proxy-max-port: 9499 diff --git a/src/test/java/io/github/hectorvent/floci/services/opensearch/OpenSearchIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/opensearch/OpenSearchIntegrationTest.java new file mode 100644 index 00000000..a59b3fbe --- /dev/null +++ b/src/test/java/io/github/hectorvent/floci/services/opensearch/OpenSearchIntegrationTest.java @@ -0,0 +1,413 @@ +package io.github.hectorvent.floci.services.opensearch; + +import io.quarkus.test.junit.QuarkusTest; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; + +import static io.restassured.RestAssured.given; +import static org.hamcrest.Matchers.*; + +@QuarkusTest +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +class OpenSearchIntegrationTest { + + private static final String DOMAIN_NAME = "test-domain"; + private static final String AUTH_HEADER = "AWS4-HMAC-SHA256 Credential=AKID/20260101/us-east-1/es/aws4_request"; + + // ── Domain CRUD ────────────────────────────────────────────────────────── + + @Test + @Order(1) + void createDomain() { + given() + .contentType("application/json") + .header("Authorization", AUTH_HEADER) + .body("{\"DomainName\":\"" + DOMAIN_NAME + "\",\"EngineVersion\":\"OpenSearch_2.11\"}") + .when() + .post("/2021-01-01/opensearch/domain") + .then() + .statusCode(200) + .body("DomainStatus.DomainName", equalTo(DOMAIN_NAME)) + .body("DomainStatus.EngineVersion", equalTo("OpenSearch_2.11")) + .body("DomainStatus.Processing", equalTo(false)) + .body("DomainStatus.Deleted", equalTo(false)) + .body("DomainStatus.ARN", containsString("arn:aws:es:")) + .body("DomainStatus.ARN", containsString(DOMAIN_NAME)); + } + + @Test + @Order(2) + void createDuplicateDomainFails() { + given() + .contentType("application/json") + .header("Authorization", AUTH_HEADER) + .body("{\"DomainName\":\"" + DOMAIN_NAME + "\"}") + .when() + .post("/2021-01-01/opensearch/domain") + .then() + .statusCode(409); + } + + @Test + @Order(3) + void describeDomain() { + given() + .header("Authorization", AUTH_HEADER) + .when() + .get("/2021-01-01/opensearch/domain/" + DOMAIN_NAME) + .then() + .statusCode(200) + .body("DomainStatus.DomainName", equalTo(DOMAIN_NAME)) + .body("DomainStatus.EngineVersion", equalTo("OpenSearch_2.11")) + .body("DomainStatus.ClusterConfig.InstanceType", equalTo("m5.large.search")) + .body("DomainStatus.ClusterConfig.InstanceCount", equalTo(1)) + .body("DomainStatus.EBSOptions.EBSEnabled", equalTo(true)); + } + + @Test + @Order(4) + void describeDomains() { + given() + .contentType("application/json") + .header("Authorization", AUTH_HEADER) + .body("{\"DomainNames\":[\"" + DOMAIN_NAME + "\"]}") + .when() + .post("/2021-01-01/opensearch/domain-info") + .then() + .statusCode(200) + .body("DomainStatusList", hasSize(1)) + .body("DomainStatusList[0].DomainName", equalTo(DOMAIN_NAME)); + } + + @Test + @Order(5) + void listDomainNames() { + given() + .header("Authorization", AUTH_HEADER) + .when() + .get("/2021-01-01/domain") + .then() + .statusCode(200) + .body("DomainNames", hasSize(greaterThanOrEqualTo(1))) + .body("DomainNames.DomainName", hasItem(DOMAIN_NAME)); + } + + @Test + @Order(6) + void listDomainNamesFilteredByEngineType() { + given() + .header("Authorization", AUTH_HEADER) + .queryParam("engineType", "OpenSearch") + .when() + .get("/2021-01-01/domain") + .then() + .statusCode(200) + .body("DomainNames.DomainName", hasItem(DOMAIN_NAME)); + } + + @Test + @Order(7) + void describeDomainConfig() { + given() + .header("Authorization", AUTH_HEADER) + .when() + .get("/2021-01-01/opensearch/domain/" + DOMAIN_NAME + "/config") + .then() + .statusCode(200) + .body("DomainConfig.ClusterConfig.Options.InstanceType", equalTo("m5.large.search")) + .body("DomainConfig.ClusterConfig.Status.State", equalTo("Active")) + .body("DomainConfig.EBSOptions.Options.EBSEnabled", equalTo(true)) + .body("DomainConfig.EngineVersion.Options", equalTo("OpenSearch_2.11")); + } + + @Test + @Order(8) + void updateDomainConfig() { + given() + .contentType("application/json") + .header("Authorization", AUTH_HEADER) + .body("{\"ClusterConfig\":{\"InstanceCount\":3}}") + .when() + .post("/2021-01-01/opensearch/domain/" + DOMAIN_NAME + "/config") + .then() + .statusCode(200) + .body("DomainConfig.ClusterConfig.Options.InstanceCount", equalTo(3)); + } + + @Test + @Order(9) + void describeNonExistentDomain() { + given() + .header("Authorization", AUTH_HEADER) + .when() + .get("/2021-01-01/opensearch/domain/nonexistent-domain") + .then() + .statusCode(409); + } + + // ── Tags ───────────────────────────────────────────────────────────────── + + @Test + @Order(10) + void addTags() { + String arn = "arn:aws:es:us-east-1:000000000000:domain/" + DOMAIN_NAME; + given() + .contentType("application/json") + .header("Authorization", AUTH_HEADER) + .body("{\"ARN\":\"" + arn + "\",\"TagList\":[{\"Key\":\"env\",\"Value\":\"test\"},{\"Key\":\"owner\",\"Value\":\"team\"}]}") + .when() + .post("/2021-01-01/tags") + .then() + .statusCode(200); + } + + @Test + @Order(11) + void listTags() { + String arn = "arn:aws:es:us-east-1:000000000000:domain/" + DOMAIN_NAME; + given() + .header("Authorization", AUTH_HEADER) + .queryParam("arn", arn) + .when() + .get("/2021-01-01/tags/") + .then() + .statusCode(200) + .body("TagList.Key", hasItem("env")) + .body("TagList.Key", hasItem("owner")); + } + + @Test + @Order(12) + void removeTags() { + String arn = "arn:aws:es:us-east-1:000000000000:domain/" + DOMAIN_NAME; + given() + .contentType("application/json") + .header("Authorization", AUTH_HEADER) + .body("{\"ARN\":\"" + arn + "\",\"TagKeys\":[\"owner\"]}") + .when() + .post("/2021-01-01/tags-removal") + .then() + .statusCode(200); + + given() + .header("Authorization", AUTH_HEADER) + .queryParam("arn", arn) + .when() + .get("/2021-01-01/tags/") + .then() + .statusCode(200) + .body("TagList.Key", not(hasItem("owner"))) + .body("TagList.Key", hasItem("env")); + } + + // ── Versions & Instance Types ───────────────────────────────────────────── + + @Test + @Order(13) + void listVersions() { + given() + .header("Authorization", AUTH_HEADER) + .when() + .get("/2021-01-01/opensearch/versions") + .then() + .statusCode(200) + .body("Versions", not(empty())) + .body("Versions", hasItem("OpenSearch_2.11")); + } + + @Test + @Order(14) + void getCompatibleVersions() { + given() + .header("Authorization", AUTH_HEADER) + .when() + .get("/2021-01-01/opensearch/compatibleVersions") + .then() + .statusCode(200) + .body("CompatibleVersions", not(empty())); + } + + @Test + @Order(15) + void listInstanceTypeDetails() { + given() + .header("Authorization", AUTH_HEADER) + .when() + .get("/2021-01-01/opensearch/instanceTypeDetails/OpenSearch_2.11") + .then() + .statusCode(200) + .body("InstanceTypeDetails", not(empty())); + } + + @Test + @Order(16) + void describeInstanceTypeLimits() { + given() + .header("Authorization", AUTH_HEADER) + .when() + .get("/2021-01-01/opensearch/instanceTypeLimits/OpenSearch_2.11/m5.large.search") + .then() + .statusCode(200) + .body("LimitsByRole", notNullValue()); + } + + // ── Stubs ───────────────────────────────────────────────────────────────── + + @Test + @Order(17) + void describeDomainChangeProgress() { + given() + .header("Authorization", AUTH_HEADER) + .when() + .get("/2021-01-01/opensearch/domain/" + DOMAIN_NAME + "/progress") + .then() + .statusCode(200) + .body("ChangeProgressStatus", notNullValue()); + } + + @Test + @Order(18) + void describeDomainAutoTunes() { + given() + .header("Authorization", AUTH_HEADER) + .when() + .get("/2021-01-01/opensearch/domain/" + DOMAIN_NAME + "/autoTunes") + .then() + .statusCode(200) + .body("AutoTunes", empty()); + } + + @Test + @Order(19) + void describeDryRunProgress() { + given() + .header("Authorization", AUTH_HEADER) + .when() + .get("/2021-01-01/opensearch/domain/" + DOMAIN_NAME + "/dryRun") + .then() + .statusCode(200) + .body("DryRunProgressStatus", notNullValue()); + } + + @Test + @Order(20) + void describeDomainHealth() { + given() + .header("Authorization", AUTH_HEADER) + .when() + .get("/2021-01-01/opensearch/domain/" + DOMAIN_NAME + "/health") + .then() + .statusCode(200) + .body("ClusterHealth", equalTo("Green")); + } + + @Test + @Order(21) + void getUpgradeHistory() { + given() + .header("Authorization", AUTH_HEADER) + .when() + .get("/2021-01-01/opensearch/upgradeDomain/" + DOMAIN_NAME + "/history") + .then() + .statusCode(200) + .body("UpgradeHistories", empty()); + } + + @Test + @Order(22) + void getUpgradeStatus() { + given() + .header("Authorization", AUTH_HEADER) + .when() + .get("/2021-01-01/opensearch/upgradeDomain/" + DOMAIN_NAME + "/status") + .then() + .statusCode(200) + .body("UpgradeStep", equalTo("UPGRADE")) + .body("StepStatus", equalTo("SUCCEEDED")); + } + + @Test + @Order(23) + void upgradeDomain() { + given() + .contentType("application/json") + .header("Authorization", AUTH_HEADER) + .body("{\"DomainName\":\"" + DOMAIN_NAME + "\",\"TargetVersion\":\"OpenSearch_2.13\"}") + .when() + .post("/2021-01-01/opensearch/upgradeDomain") + .then() + .statusCode(200) + .body("DomainName", equalTo(DOMAIN_NAME)) + .body("TargetVersion", equalTo("OpenSearch_2.13")); + } + + @Test + @Order(24) + void cancelDomainConfigChange() { + given() + .contentType("application/json") + .header("Authorization", AUTH_HEADER) + .body("{}") + .when() + .post("/2021-01-01/opensearch/domain/" + DOMAIN_NAME + "/config/cancel") + .then() + .statusCode(200) + .body("CancelledChangeIds", empty()); + } + + @Test + @Order(25) + void startServiceSoftwareUpdate() { + given() + .contentType("application/json") + .header("Authorization", AUTH_HEADER) + .body("{\"DomainName\":\"" + DOMAIN_NAME + "\"}") + .when() + .post("/2021-01-01/opensearch/serviceSoftwareUpdate/start") + .then() + .statusCode(200) + .body("ServiceSoftwareOptions.UpdateStatus", equalTo("COMPLETED")); + } + + @Test + @Order(26) + void cancelServiceSoftwareUpdate() { + given() + .contentType("application/json") + .header("Authorization", AUTH_HEADER) + .body("{\"DomainName\":\"" + DOMAIN_NAME + "\"}") + .when() + .post("/2021-01-01/opensearch/serviceSoftwareUpdate/cancel") + .then() + .statusCode(200) + .body("ServiceSoftwareOptions.UpdateStatus", equalTo("COMPLETED")); + } + + // ── Cleanup ─────────────────────────────────────────────────────────────── + + @Test + @Order(30) + void deleteDomain() { + given() + .header("Authorization", AUTH_HEADER) + .when() + .delete("/2021-01-01/opensearch/domain/" + DOMAIN_NAME) + .then() + .statusCode(200) + .body("DomainStatus.DomainName", equalTo(DOMAIN_NAME)) + .body("DomainStatus.Deleted", equalTo(true)); + } + + @Test + @Order(31) + void deleteNonExistentDomain() { + given() + .header("Authorization", AUTH_HEADER) + .when() + .delete("/2021-01-01/opensearch/domain/" + DOMAIN_NAME) + .then() + .statusCode(409); + } +} diff --git a/src/test/resources/application.yml b/src/test/resources/application.yml index 04f74b11..da8aa025 100644 --- a/src/test/resources/application.yml +++ b/src/test/resources/application.yml @@ -29,6 +29,8 @@ floci: flush-interval-ms: 60000 secretsmanager: flush-interval-ms: 60000 + opensearch: + flush-interval-ms: 60000 auth: validate-signatures: false @@ -92,3 +94,6 @@ floci: enabled: true cloudformation: enabled: true + opensearch: + enabled: true + mode: mock \ No newline at end of file From d4cf75e1bdfefbb01dabc55998a2f2d5ffa220d2 Mon Sep 17 00:00:00 2001 From: Hector Ventura Date: Mon, 30 Mar 2026 23:56:46 -0500 Subject: [PATCH 05/32] fix: for no-such-key with non-ascii key (#112) --- .../floci/services/s3/S3Controller.java | 3 +- .../floci/services/s3/S3IntegrationTest.java | 310 ++++++++++-------- 2 files changed, 176 insertions(+), 137 deletions(-) diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java index 852f2739..92f24bb6 100644 --- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java +++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java @@ -25,6 +25,7 @@ import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.UriInfo; import java.io.ByteArrayOutputStream; +import java.net.URLDecoder; import java.nio.charset.StandardCharsets; import java.time.Instant; import java.time.ZoneId; @@ -1053,7 +1054,7 @@ private Response handleCopyObject(String copySource, String destBucket, String d throw new AwsException("InvalidArgument", "Invalid copy source: " + copySource, 400); } String sourceBucket = source.substring(0, slashIndex); - String sourceKey = source.substring(slashIndex + 1); + String sourceKey = URLDecoder.decode(source.substring(slashIndex + 1), StandardCharsets.UTF_8); S3Object copy = s3Service.copyObject(sourceBucket, sourceKey, destBucket, destKey, httpHeaders.getHeaderString("x-amz-metadata-directive"), diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java index 4dfff4bb..9ee2ddd1 100644 --- a/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java +++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java @@ -211,6 +211,180 @@ void deleteNonEmptyBucketFails() { .body(containsString("BucketNotEmpty")); } + @Test + @Order(15) + void getObjectAttributesRejectsUnknownSelector() { + given() + .header("x-amz-object-attributes", "ETag,UnknownThing") + .when() + .get("/test-bucket/greeting.txt?attributes") + .then() + .statusCode(400) + .body(containsString("InvalidArgument")); + } + + @Test + @Order(16) + void getNonExistentBucket() { + given() + .when() + .get("/nonexistent-bucket") + .then() + .statusCode(404) + .body(containsString("NoSuchBucket")); + } + + @Test + @Order(17) + void headBucketReturnsStoredRegionForLocationConstraintBucket() { + String bucket = "eu-head-bucket"; + String createBucketConfiguration = """ + + eu-central-1 + + """; + + given() + .contentType("application/xml") + .body(createBucketConfiguration) + .when() + .put("/" + bucket) + .then() + .statusCode(200) + .header("Location", equalTo("/" + bucket)); + + given() + .when() + .head("/" + bucket) + .then() + .statusCode(200) + .header("x-amz-bucket-region", equalTo("eu-central-1")); + + given() + .when() + .delete("/" + bucket) + .then() + .statusCode(204); + } + + @Test + @Order(18) + void createBucketUsesSigningRegionWhenBodyEmpty() { + String bucket = "signed-region-bucket"; + + given() + .header("Authorization", + "AWS4-HMAC-SHA256 Credential=test/20260325/eu-west-1/s3/aws4_request, SignedHeaders=host;x-amz-date, Signature=test") + .when() + .put("/" + bucket) + .then() + .statusCode(200) + .header("Location", equalTo("/" + bucket)); + + given() + .when() + .head("/" + bucket) + .then() + .statusCode(200) + .header("x-amz-bucket-region", equalTo("eu-west-1")); + + given() + .when() + .delete("/" + bucket) + .then() + .statusCode(204); + } + + @Test + @Order(19) + void createBucketRejectsUsEast1LocationConstraint() { + String createBucketConfiguration = """ + + us-east-1 + + """; + + given() + .contentType("application/xml") + .body(createBucketConfiguration) + .when() + .put("/invalid-location-bucket") + .then() + .statusCode(400) + .body(containsString("InvalidLocationConstraint")); + } + + @Test + @Order(20) + void copyObjectWithNonAsciiKeySucceeds() { + String bucket = "copy-nonascii-bucket"; + String srcKey = "src/テスト画像.png"; + String dstKey = "dst/テスト画像.png"; + String encodedSrcKey = "src/%E3%83%86%E3%82%B9%E3%83%88%E7%94%BB%E5%83%8F.png"; + + given().put("/" + bucket).then().statusCode(200); + + given() + .contentType("application/octet-stream") + .body("hello".getBytes()) + .when() + .put("/" + bucket + "/" + srcKey) + .then() + .statusCode(200); + + given() + .header("x-amz-copy-source", "/" + bucket + "/" + encodedSrcKey) + .when() + .put("/" + bucket + "/" + dstKey) + .then() + .statusCode(200) + .body(containsString("ETag")); + + given() + .when() + .get("/" + bucket + "/" + dstKey) + .then() + .statusCode(200) + .body(equalTo("hello")); + + given().delete("/" + bucket + "/" + srcKey); + given().delete("/" + bucket + "/" + dstKey); + given().delete("/" + bucket); + } + + @Test + @Order(21) + void putLargeObject() { + // 22 MB – exceeds the old Jackson 20 MB maxStringLength default + byte[] largeBody = new byte[22 * 1024 * 1024]; + Arrays.fill(largeBody, (byte) 'A'); + + given() + .when() + .put("/large-object-bucket") + .then() + .statusCode(200); + + given() + .contentType("application/octet-stream") + .body(largeBody) + .when() + .put("/large-object-bucket/large-file.bin") + .then() + .statusCode(200) + .header("ETag", notNullValue()); + + given() + .when() + .get("/large-object-bucket/large-file.bin") + .then() + .statusCode(200) + .header("Content-Length", String.valueOf(largeBody.length)); + + given().delete("/large-object-bucket/large-file.bin"); + given().delete("/large-object-bucket"); + } + @Test @Order(30) void getObjectWithFullRange() { @@ -558,140 +732,4 @@ void cleanupAndDeleteBucket() { .then() .statusCode(204); } - - @Test - @Order(16) - void getObjectAttributesRejectsUnknownSelector() { - given() - .header("x-amz-object-attributes", "ETag,UnknownThing") - .when() - .get("/test-bucket/greeting.txt?attributes") - .then() - .statusCode(400) - .body(containsString("InvalidArgument")); - } - - @Test - @Order(17) - void getNonExistentBucket() { - given() - .when() - .get("/nonexistent-bucket") - .then() - .statusCode(404) - .body(containsString("NoSuchBucket")); - } - - @Test - @Order(17) - void headBucketReturnsStoredRegionForLocationConstraintBucket() { - String bucket = "eu-head-bucket"; - String createBucketConfiguration = """ - - eu-central-1 - - """; - - given() - .contentType("application/xml") - .body(createBucketConfiguration) - .when() - .put("/" + bucket) - .then() - .statusCode(200) - .header("Location", equalTo("/" + bucket)); - - given() - .when() - .head("/" + bucket) - .then() - .statusCode(200) - .header("x-amz-bucket-region", equalTo("eu-central-1")); - - given() - .when() - .delete("/" + bucket) - .then() - .statusCode(204); - } - - @Test - @Order(18) - void createBucketUsesSigningRegionWhenBodyEmpty() { - String bucket = "signed-region-bucket"; - - given() - .header("Authorization", - "AWS4-HMAC-SHA256 Credential=test/20260325/eu-west-1/s3/aws4_request, SignedHeaders=host;x-amz-date, Signature=test") - .when() - .put("/" + bucket) - .then() - .statusCode(200) - .header("Location", equalTo("/" + bucket)); - - given() - .when() - .head("/" + bucket) - .then() - .statusCode(200) - .header("x-amz-bucket-region", equalTo("eu-west-1")); - - given() - .when() - .delete("/" + bucket) - .then() - .statusCode(204); - } - - @Test - @Order(19) - void createBucketRejectsUsEast1LocationConstraint() { - String createBucketConfiguration = """ - - us-east-1 - - """; - - given() - .contentType("application/xml") - .body(createBucketConfiguration) - .when() - .put("/invalid-location-bucket") - .then() - .statusCode(400) - .body(containsString("InvalidLocationConstraint")); - } - - @Test - @Order(20) - void putLargeObject() { - // 22 MB – exceeds the old Jackson 20 MB maxStringLength default - byte[] largeBody = new byte[22 * 1024 * 1024]; - Arrays.fill(largeBody, (byte) 'A'); - - given() - .when() - .put("/large-object-bucket") - .then() - .statusCode(200); - - given() - .contentType("application/octet-stream") - .body(largeBody) - .when() - .put("/large-object-bucket/large-file.bin") - .then() - .statusCode(200) - .header("ETag", notNullValue()); - - given() - .when() - .get("/large-object-bucket/large-file.bin") - .then() - .statusCode(200) - .header("Content-Length", String.valueOf(largeBody.length)); - - given().delete("/large-object-bucket/large-file.bin"); - given().delete("/large-object-bucket"); - } } From 012bf8fb34f77e7b5c5108414ebe2bf59db6ae88 Mon Sep 17 00:00:00 2001 From: Hector Ventura Date: Tue, 31 Mar 2026 00:20:06 -0500 Subject: [PATCH 06/32] chore: update GitHub Sponsors FUNDING.yml Signed-off-by: Hector Ventura --- .github/FUNDING.yml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000..cee6e69d --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,3 @@ +# These are supported funding model platforms + +github: [hectorvent] From b5fd6d41f38f03185a2735645042775e90bc79b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Diego=20L=C3=B3pez?= Date: Tue, 31 Mar 2026 01:45:41 -0400 Subject: [PATCH 07/32] fix(s3): expose inMemory flag in test constructor to fix S3 disk-persistence tests (#136) --- .../io/github/hectorvent/floci/services/s3/S3Service.java | 4 ++-- .../hectorvent/floci/services/s3/S3MultipartServiceTest.java | 2 +- .../io/github/hectorvent/floci/services/s3/S3ServiceTest.java | 2 +- .../hectorvent/floci/services/s3/S3VersioningServiceTest.java | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java index 6666e933..311552d7 100644 --- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java +++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java @@ -70,8 +70,8 @@ public S3Service(StorageFactory storageFactory, EmulatorConfig config, */ S3Service(StorageBackend bucketStore, StorageBackend objectStore, - Path dataRoot) { - this(bucketStore, objectStore, dataRoot, true, null, null, null, "http://localhost:4566", + Path dataRoot, boolean inMemory) { + this(bucketStore, objectStore, dataRoot, inMemory, null, null, null, "http://localhost:4566", new ObjectMapper()); } diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java index be17c625..31e34bf7 100644 --- a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java +++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java @@ -27,7 +27,7 @@ class S3MultipartServiceTest { @BeforeEach void setUp() { - s3Service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), tempDir); + s3Service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), tempDir, true); s3Service.createBucket("test-bucket", "us-east-1"); } diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java index 013c9ab2..6ddbc65e 100644 --- a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java +++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java @@ -29,7 +29,7 @@ class S3ServiceTest { @BeforeEach void setUp() { Path dataRoot = tempDir.resolve("s3"); - s3Service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), dataRoot); + s3Service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), dataRoot, false); } @Test diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3VersioningServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3VersioningServiceTest.java index c406dc79..e95ea801 100644 --- a/src/test/java/io/github/hectorvent/floci/services/s3/S3VersioningServiceTest.java +++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3VersioningServiceTest.java @@ -22,7 +22,7 @@ class S3VersioningServiceTest { @BeforeEach void setUp() { - s3Service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), tempDir); + s3Service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), tempDir, true); s3Service.createBucket("versioned-bucket", "us-east-1"); } From 6795f552e88744025940182f9f87b5f32b6e9b2a Mon Sep 17 00:00:00 2001 From: Cristian de la Hoz Date: Tue, 31 Mar 2026 07:18:13 -0400 Subject: [PATCH 08/32] docs: add stars chart history to `README.md` Signed-off-by: Cristian de la Hoz --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index f7f6194f..8d9c16e8 100644 --- a/README.md +++ b/README.md @@ -248,6 +248,10 @@ services: Without this, SQS returns `http://localhost:4566/...` in QueueUrl responses, which resolves to the wrong container. +## Star history + +[![Star History Chart](https://api.star-history.com/svg?repos=hectorvent/floci&type=Date)](https://star-history.com/#hectorvent/floci&Date) + ## Contributors From 3816005cbdec58dacc9aec2d9dba74262ba89140 Mon Sep 17 00:00:00 2001 From: Dixit R Jain Date: Wed, 1 Apr 2026 02:28:21 +0530 Subject: [PATCH 09/32] fix(core): globally inject aws request-id headers for sdk compatibility (#146) AWS SDKs rigorously depend on standard HTTP identification headers (like `x-amz-request-id` and `x-amzn-RequestId`) to completely and accurately construct the `ResponseMetadata` payload in their clients. Because Floci lacked these headers on generic API responses, the SDK failed to parse identifiers (resulting in `$metadata.requestId` resolving to `undefined`). This commit introduces a JAX-RS `AwsRequestIdFilter` to automatically and globally append these necessary structural headers to every HTTP response, mapped via a single uniformly-generated UUID. Preexisting explicitly set controllers (such as Lambda's Invoke) are bypassed to preserve context. Added `AwsRequestIdFilterIntegrationTest` verifying end-to-end protocol formats. Fixes #145 --- .../floci/core/common/AwsRequestIdFilter.java | 49 ++++++ .../AwsRequestIdFilterIntegrationTest.java | 156 ++++++++++++++++++ 2 files changed, 205 insertions(+) create mode 100644 src/main/java/io/github/hectorvent/floci/core/common/AwsRequestIdFilter.java create mode 100644 src/test/java/io/github/hectorvent/floci/core/common/AwsRequestIdFilterIntegrationTest.java diff --git a/src/main/java/io/github/hectorvent/floci/core/common/AwsRequestIdFilter.java b/src/main/java/io/github/hectorvent/floci/core/common/AwsRequestIdFilter.java new file mode 100644 index 00000000..bce8ad6c --- /dev/null +++ b/src/main/java/io/github/hectorvent/floci/core/common/AwsRequestIdFilter.java @@ -0,0 +1,49 @@ +package io.github.hectorvent.floci.core.common; + +import jakarta.ws.rs.container.ContainerRequestContext; +import jakarta.ws.rs.container.ContainerResponseContext; +import jakarta.ws.rs.container.ContainerResponseFilter; +import jakarta.ws.rs.ext.Provider; + +import java.util.UUID; + +/** + * Adds AWS request-id response headers to every HTTP response. + * + *

Real AWS services always return a request identifier so that SDKs can + * populate {@code $metadata.requestId}. The header name varies by protocol: + *

    + *
  • {@code x-amz-request-id} — REST XML (S3), REST JSON (Lambda), Query protocol
    • + *
  • {@code x-amzn-RequestId} — JSON 1.0 / 1.1 services (DynamoDB, SSM, …)
    • + *
  • {@code x-amz-id-2} — S3 extended request ID
    • + *
+ * + *

This filter emits all three so that every AWS SDK variant can find the + * header it expects. If a controller already set {@code x-amz-request-id} + * (e.g. Lambda invoke), the existing value is preserved. + */ +@Provider +public class AwsRequestIdFilter implements ContainerResponseFilter { + + private static final String AMZ_REQUEST_ID = "x-amz-request-id"; + private static final String AMZN_REQUEST_ID = "x-amzn-RequestId"; + private static final String AMZ_ID_2 = "x-amz-id-2"; + + @Override + public void filter(ContainerRequestContext requestContext, ContainerResponseContext responseContext) { + var headers = responseContext.getHeaders(); + + // Reuse the same ID across all header variants for this response + String requestId = UUID.randomUUID().toString(); + + if (!headers.containsKey(AMZ_REQUEST_ID)) { + headers.putSingle(AMZ_REQUEST_ID, requestId); + } + if (!headers.containsKey(AMZN_REQUEST_ID)) { + headers.putSingle(AMZN_REQUEST_ID, requestId); + } + if (!headers.containsKey(AMZ_ID_2)) { + headers.putSingle(AMZ_ID_2, requestId); + } + } +} diff --git a/src/test/java/io/github/hectorvent/floci/core/common/AwsRequestIdFilterIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/core/common/AwsRequestIdFilterIntegrationTest.java new file mode 100644 index 00000000..21044ff7 --- /dev/null +++ b/src/test/java/io/github/hectorvent/floci/core/common/AwsRequestIdFilterIntegrationTest.java @@ -0,0 +1,156 @@ +package io.github.hectorvent.floci.core.common; + +import io.quarkus.test.junit.QuarkusTest; +import org.junit.jupiter.api.Test; + +import static io.restassured.RestAssured.given; +import static org.hamcrest.Matchers.notNullValue; + +import io.restassured.RestAssured; +import io.restassured.config.EncoderConfig; +import io.restassured.http.ContentType; +import org.junit.jupiter.api.BeforeAll; + +/** + * Verifies that {@link AwsRequestIdFilter} injects {@code x-amz-request-id} and + * {@code x-amzn-RequestId} headers on every response, across all three AWS wire + * protocols supported by Floci: REST XML (S3), JSON 1.0 (DynamoDB), and Query (SQS). + * + *

These headers are the source from which the AWS SDK v3 populates + * {@code $metadata.requestId} and {@code $metadata.httpStatusCode} on every + * command output. + */ +@QuarkusTest +class AwsRequestIdFilterIntegrationTest { + + private static final String SSM_CONTENT_TYPE = "application/x-amz-json-1.1"; + private static final String DYNAMODB_CONTENT_TYPE = "application/x-amz-json-1.0"; + + @BeforeAll + static void configureRestAssured() { + RestAssured.config = RestAssured.config().encoderConfig( + EncoderConfig.encoderConfig() + .encodeContentTypeAs(SSM_CONTENT_TYPE, ContentType.TEXT) + .encodeContentTypeAs(DYNAMODB_CONTENT_TYPE, ContentType.TEXT)); + } + + // --- REST XML protocol (S3) --- + + @Test + void s3SuccessResponseContainsRequestIdHeaders() { + // Create a temporary bucket, verify headers, then clean it up + String bucket = "request-id-test-bucket"; + + given() + .when() + .put("/" + bucket) + .then() + .statusCode(200) + .header("x-amz-request-id", notNullValue()) + .header("x-amzn-RequestId", notNullValue()); + + given().delete("/" + bucket); + } + + @Test + void s3ErrorResponseContainsRequestIdHeaders() { + // Requesting a non-existent bucket produces a 404 error response — + // the headers must still be present so the SDK can surface the request ID. + given() + .when() + .get("/no-such-bucket-floci-test") + .then() + .statusCode(404) + .header("x-amz-request-id", notNullValue()) + .header("x-amzn-RequestId", notNullValue()); + } + + @Test + void s3CopyObjectResponseContainsRequestIdHeaders() { + String bucket = "request-id-copy-bucket"; + given().put("/" + bucket).then().statusCode(200); + + given() + .contentType("text/plain") + .body("hello") + .when() + .put("/" + bucket + "/src.txt") + .then() + .statusCode(200); + + // CopyObject is the operation the user reported as missing $metadata.requestId + given() + .header("x-amz-copy-source", "/" + bucket + "/src.txt") + .when() + .put("/" + bucket + "/dst.txt") + .then() + .statusCode(200) + .header("x-amz-request-id", notNullValue()) + .header("x-amzn-RequestId", notNullValue()); + + given().delete("/" + bucket + "/src.txt"); + given().delete("/" + bucket + "/dst.txt"); + given().delete("/" + bucket); + } + + // --- JSON 1.0 protocol (DynamoDB) --- + + @Test + void dynamoDbSuccessResponseContainsRequestIdHeaders() { + given() + .header("X-Amz-Target", "DynamoDB_20120810.ListTables") + .contentType("application/x-amz-json-1.0") + .body("{}") + .when() + .post("/") + .then() + .statusCode(200) + .header("x-amz-request-id", notNullValue()) + .header("x-amzn-RequestId", notNullValue()); + } + + @Test + void dynamoDbErrorResponseContainsRequestIdHeaders() { + given() + .header("X-Amz-Target", "DynamoDB_20120810.GetItem") + .contentType("application/x-amz-json-1.0") + .body("{\"TableName\": \"NonExistentTable\", \"Key\": {\"id\": {\"S\": \"1\"}}}") + .when() + .post("/") + .then() + .statusCode(400) + .header("x-amz-request-id", notNullValue()) + .header("x-amzn-RequestId", notNullValue()); + } + + // --- Query protocol (SQS) --- + + @Test + void sqsSuccessResponseContainsRequestIdHeaders() { + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "ListQueues") + .when() + .post("/") + .then() + .statusCode(200) + .header("x-amz-request-id", notNullValue()) + .header("x-amzn-RequestId", notNullValue()); + } + + // --- JSON 1.1 protocol (SSM) --- + + @Test + void ssmSuccessResponseContainsRequestIdHeaders() { + given() + .header("X-Amz-Target", "AmazonSSM.DescribeParameters") + .contentType("application/x-amz-json-1.1") + .body("{}") + .when() + .post("/") + .then() + .statusCode(200) + .header("x-amz-request-id", notNullValue()) + .header("x-amzn-RequestId", notNullValue()); + } +} From a529af8b84b543e3fcfab74529e466f65e0290e1 Mon Sep 17 00:00:00 2001 From: Masatoshi Tada Date: Wed, 1 Apr 2026 06:00:46 +0900 Subject: [PATCH 10/32] fix(sns) Allow to publish SMS using phone-number (#138) Co-authored-by: tada --- .../floci/services/sns/SnsJsonHandler.java | 3 ++- .../floci/services/sns/SnsQueryHandler.java | 3 ++- .../hectorvent/floci/services/sns/SnsService.java | 15 +++++++++++---- .../floci/services/sns/SnsServiceTest.java | 6 ++++++ 4 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/main/java/io/github/hectorvent/floci/services/sns/SnsJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/sns/SnsJsonHandler.java index 21df734b..508f18b5 100644 --- a/src/main/java/io/github/hectorvent/floci/services/sns/SnsJsonHandler.java +++ b/src/main/java/io/github/hectorvent/floci/services/sns/SnsJsonHandler.java @@ -150,6 +150,7 @@ private Response handleListSubscriptionsByTopic(JsonNode request, String region) private Response handlePublish(JsonNode request, String region) { String topicArn = request.path("TopicArn").asText(null); String targetArn = request.path("TargetArn").asText(null); + String phoneNumber = request.path("PhoneNumber").asText(null); String message = request.path("Message").asText(null); String subject = request.path("Subject").asText(null); @@ -161,7 +162,7 @@ private Response handlePublish(JsonNode request, String region) { }); } - String messageId = snsService.publish(topicArn, targetArn, message, subject, attributes, region); + String messageId = snsService.publish(topicArn, targetArn, phoneNumber, message, subject, attributes, region); ObjectNode response = objectMapper.createObjectNode(); response.put("MessageId", messageId); return Response.ok(response).build(); diff --git a/src/main/java/io/github/hectorvent/floci/services/sns/SnsQueryHandler.java b/src/main/java/io/github/hectorvent/floci/services/sns/SnsQueryHandler.java index 086b6370..39ffaef0 100644 --- a/src/main/java/io/github/hectorvent/floci/services/sns/SnsQueryHandler.java +++ b/src/main/java/io/github/hectorvent/floci/services/sns/SnsQueryHandler.java @@ -175,6 +175,7 @@ private Response buildSubscriptionListResponse(String action, List private Response handlePublish(MultivaluedMap params, String region) { String topicArn = getParam(params, "TopicArn"); String targetArn = getParam(params, "TargetArn"); + String phoneNumber = getParam(params, "PhoneNumber"); String message = getParam(params, "Message"); String subject = getParam(params, "Subject"); String messageGroupId = getParam(params, "MessageGroupId"); @@ -189,7 +190,7 @@ private Response handlePublish(MultivaluedMap params, String reg } try { - String messageId = snsService.publish(topicArn, targetArn, message, subject, + String messageId = snsService.publish(topicArn, targetArn, phoneNumber, message, subject, attributes, messageGroupId, messageDeduplicationId, region); String result = new XmlBuilder().elem("MessageId", messageId).build(); diff --git a/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.java b/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.java index 8499c79f..9e0b6c43 100644 --- a/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.java +++ b/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.java @@ -238,19 +238,26 @@ public List listSubscriptionsByTopic(String topicArn, String regio return subscriptionsByTopic(topicArn, region); } + // Since this method is called by S3 and EventBridge, this doesn't need "phoneNumber" parameter. public String publish(String topicArn, String targetArn, String message, String subject, String region) { - return publish(topicArn, targetArn, message, subject, null, null, null, region); + return publish(topicArn, targetArn, null, message, subject, null, null, null, region); } - public String publish(String topicArn, String targetArn, String message, + public String publish(String topicArn, String targetArn, String phoneNumber, String message, String subject, Map messageAttributes, String region) { - return publish(topicArn, targetArn, message, subject, messageAttributes, null, null, region); + return publish(topicArn, targetArn, phoneNumber, message, subject, messageAttributes, null, null, region); } - public String publish(String topicArn, String targetArn, String message, + public String publish(String topicArn, String targetArn, String phoneNumber, String message, String subject, Map messageAttributes, String messageGroupId, String messageDeduplicationId, String region) { + // Send SMS + if (phoneNumber != null) { + return UUID.randomUUID().toString(); + } + + // Send a message to topic or directly to a target ARN String effectiveArn = topicArn != null ? topicArn : targetArn; if (effectiveArn == null) { throw new AwsException("InvalidParameter", "TopicArn or TargetArn is required.", 400); diff --git a/src/test/java/io/github/hectorvent/floci/services/sns/SnsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/sns/SnsServiceTest.java index 79d7932e..fcf5cbad 100644 --- a/src/test/java/io/github/hectorvent/floci/services/sns/SnsServiceTest.java +++ b/src/test/java/io/github/hectorvent/floci/services/sns/SnsServiceTest.java @@ -153,6 +153,12 @@ void publish_withSqsSubscriber_returnsMessageId() { assertNotNull(messageId); } + @Test + void publish_withPhoneNumber_returnsMessageId() { + String messageId = snsService.publish(null, null, "+819012345678", "Hello phone!", null, null, REGION); + assertNotNull(messageId); + } + @Test void publish_requiresTopicArn() { assertThrows(AwsException.class, From 44101418a75f357c0428a79ed555faa8803766a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fred=20Pe=C3=B1a?= Date: Tue, 31 Mar 2026 17:02:34 -0400 Subject: [PATCH 11/32] docs: expand community section with GitHub Discussions (#150) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8d9c16e8..d69392aa 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@

- Join the community on Slack to ask questions, share feedback, and discuss Floci with other contributors and users. + Join the community on Slack to ask questions, share feedback, and discuss Floci with other contributors and users. You can also open any topic in GitHub Discussions — feature ideas, compatibility questions, design tradeoffs, wild proposals, or half-baked thoughts are all welcome. No idea is too small, too early, or too popcorn-fueled to start a good discussion.

--- From 68cb1ca1efc19902ec6da53d59f84404aafe6110 Mon Sep 17 00:00:00 2001 From: Roberto Perez Alcolea Date: Tue, 31 Mar 2026 14:49:02 -0700 Subject: [PATCH 12/32] fix(sns): enforce FilterPolicy on message delivery (#53) FilterPolicy subscription attribute was accepted and stored but never evaluated during publish. Messages were delivered to all confirmed subscriptions regardless of filter rules. Now evaluates filter policies supporting exact string matching, numeric comparisons, prefix matching, exists operator, and anything-but operator per SNS specification. Fixes #49 --- .../floci/services/sns/SnsService.java | 140 ++++++++++ .../services/sns/SnsIntegrationTest.java | 245 +++++++++++++++++- 2 files changed, 384 insertions(+), 1 deletion(-) diff --git a/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.java b/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.java index 9e0b6c43..c2d5da6a 100644 --- a/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.java +++ b/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.java @@ -12,12 +12,14 @@ import io.github.hectorvent.floci.services.sns.model.Topic; import io.github.hectorvent.floci.services.sqs.SqsService; import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import org.jboss.logging.Logger; +import java.math.BigDecimal; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; @@ -291,6 +293,9 @@ public String publish(String topicArn, String targetArn, String phoneNumber, Str LOG.debugv("Skipping delivery to pending subscription {0}", sub.getSubscriptionArn()); continue; } + if (!matchesFilterPolicy(sub, messageAttributes)) { + continue; + } deliverMessage(sub, message, subject, messageAttributes, messageId, effectiveArn, messageGroupId); } LOG.infov("Published message {0} to topic {1}", messageId, effectiveArn); @@ -359,6 +364,7 @@ public BatchPublishResult publishBatch(String topicArn, List Map attrs = (Map) entry.get("MessageAttributes"); for (Subscription sub : subscriptionsByTopic(topicArn, region)) { if ("true".equals(sub.getAttributes().get("PendingConfirmation"))) continue; + if (!matchesFilterPolicy(sub, attrs)) continue; deliverMessage(sub, message, subject, attrs, messageId, topicArn, messageGroupId); } LOG.debugv("Batch published message {0} (id={1}) to {2}", messageId, id, topicArn); @@ -393,6 +399,140 @@ public Map listTagsForResource(String resourceArn, String region return new java.util.LinkedHashMap<>(topic.getTags()); } + /** + * Evaluates whether a message satisfies the subscription's filter policy. + * Returns {@code true} if no filter policy is set. + * Returns {@code false} for malformed filter policies (fail closed). + *

+ * Only {@code FilterPolicyScope=MessageAttributes} is supported. When scope is + * {@code MessageBody}, filtering is skipped and the message is delivered (to avoid + * incorrectly dropping messages for an unsupported scope). + *

+ * All keys in the policy must match (AND logic). Within each key's rule array, + * any matching element is sufficient (OR logic). + */ + private boolean matchesFilterPolicy(Subscription sub, Map messageAttributes) { + String filterPolicyJson = sub.getAttributes().get("FilterPolicy"); + if (filterPolicyJson == null || filterPolicyJson.isBlank()) { + return true; + } + String scope = sub.getAttributes().getOrDefault("FilterPolicyScope", "MessageAttributes"); + if ("MessageBody".equals(scope)) { + return true; + } + try { + JsonNode filterPolicy = objectMapper.readTree(filterPolicyJson); + if (!filterPolicy.isObject()) { + LOG.warnv("Invalid FilterPolicy (not a JSON object) for {0}", sub.getSubscriptionArn()); + return false; + } + Map attrs = messageAttributes != null ? messageAttributes : Map.of(); + var fields = filterPolicy.fields(); + while (fields.hasNext()) { + var entry = fields.next(); + String key = entry.getKey(); + JsonNode rules = entry.getValue(); + String actualValue = attrs.get(key); + if (!matchesAttributeRules(actualValue, rules)) { + return false; + } + } + return true; + } catch (Exception e) { + LOG.warnv("Failed to parse filter policy for {0}: {1}", sub.getSubscriptionArn(), e.getMessage()); + return false; + } + } + + /** + * Checks if an attribute value matches a single filter policy rule set. + * Rules must be a JSON array where ANY element matching means the rule passes (OR logic). + * Non-array rules are treated as non-matching. + */ + private boolean matchesAttributeRules(String actualValue, JsonNode rules) { + if (!rules.isArray()) { + return false; + } + for (JsonNode rule : rules) { + if (rule.isTextual() && rule.asText().equals(actualValue)) { + return true; + } + if (rule.isNumber() && actualValue != null) { + try { + if (new BigDecimal(actualValue).compareTo(rule.decimalValue()) == 0) { + return true; + } + } catch (NumberFormatException ignored) { + } + } + if (rule.isObject() && matchesObjectRule(rule, actualValue)) { + return true; + } + } + return false; + } + + /** + * Evaluates a single object-type filter rule (exists, prefix, anything-but, numeric) + * against the actual attribute value. + */ + private boolean matchesObjectRule(JsonNode rule, String actualValue) { + if (rule.has("exists")) { + boolean shouldExist = rule.get("exists").asBoolean(); + return shouldExist ? actualValue != null : actualValue == null; + } + if (rule.has("prefix") && actualValue != null) { + return actualValue.startsWith(rule.get("prefix").asText()); + } + if (rule.has("anything-but") && actualValue != null) { + return !containsValue(rule.get("anything-but"), actualValue); + } + if (rule.has("numeric") && actualValue != null) { + try { + return evaluateNumericCondition(new BigDecimal(actualValue), rule.get("numeric")); + } catch (NumberFormatException ignored) { + } + } + return false; + } + + private boolean containsValue(JsonNode node, String value) { + if (node.isArray()) { + for (JsonNode element : node) { + if (element.asText().equals(value)) return true; + } + return false; + } + LOG.warnv("FilterPolicy 'anything-but' expected an array but got a scalar; treating as single-value list"); + return node.asText().equals(value); + } + + /** + * Evaluates a numeric condition array against a value. + * The conditions array contains alternating operator-target pairs (e.g. [">=", 100, "<", 200]). + * All pairs must match for the condition to pass (AND logic). + */ + private boolean evaluateNumericCondition(BigDecimal value, JsonNode conditions) { + if (!conditions.isArray() || conditions.size() % 2 != 0) { + return false; + } + for (int i = 0; i < conditions.size(); i += 2) { + String op = conditions.get(i).asText(); + BigDecimal target = conditions.get(i + 1).decimalValue(); + int cmp = value.compareTo(target); + boolean matches = switch (op) { + case "=" -> cmp == 0; + case ">" -> cmp > 0; + case ">=" -> cmp >= 0; + case "<" -> cmp < 0; + case "<=" -> cmp <= 0; + default -> false; + }; + if (!matches) return false; + } + return true; + } + private boolean isDuplicate(String topicArn, String deduplicationId) { String cacheKey = topicArn + ":" + deduplicationId; Instant now = Instant.now(); diff --git a/src/test/java/io/github/hectorvent/floci/services/sns/SnsIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/sns/SnsIntegrationTest.java index e70a2917..a41dd905 100644 --- a/src/test/java/io/github/hectorvent/floci/services/sns/SnsIntegrationTest.java +++ b/src/test/java/io/github/hectorvent/floci/services/sns/SnsIntegrationTest.java @@ -294,6 +294,11 @@ void setTopicAttributes() { .statusCode(200); } + private static String filterQueueUrlA; + private static String filterQueueUrlB; + private static String filterSubArnA; + private static String filterSubArnB; + @Test @Order(22) void getSubscriptionAttributes_jsonProtocol() { @@ -358,6 +363,230 @@ void getSubscriptionAttributes_jsonProtocol_notFound() { .statusCode(404); } + @Test + @Order(13) + void filterPolicy_createQueuesAndSubscribe() { + filterQueueUrlA = given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "CreateQueue") + .formParam("QueueName", "filter-queue-sports") + .when() + .post("/") + .then() + .statusCode(200) + .extract().xmlPath().getString("CreateQueueResponse.CreateQueueResult.QueueUrl"); + + filterQueueUrlB = given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "CreateQueue") + .formParam("QueueName", "filter-queue-weather") + .when() + .post("/") + .then() + .statusCode(200) + .extract().xmlPath().getString("CreateQueueResponse.CreateQueueResult.QueueUrl"); + + String sportsQueueArn = given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "GetQueueAttributes") + .formParam("QueueUrl", filterQueueUrlA) + .formParam("AttributeName.1", "QueueArn") + .when() + .post("/") + .then() + .statusCode(200) + .extract().xmlPath().getString("**.find { it.Name == 'QueueArn' }.Value"); + + String weatherQueueArn = given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "GetQueueAttributes") + .formParam("QueueUrl", filterQueueUrlB) + .formParam("AttributeName.1", "QueueArn") + .when() + .post("/") + .then() + .statusCode(200) + .extract().xmlPath().getString("**.find { it.Name == 'QueueArn' }.Value"); + + filterSubArnA = given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "Subscribe") + .formParam("TopicArn", topicArn) + .formParam("Protocol", "sqs") + .formParam("Endpoint", sportsQueueArn) + .when() + .post("/") + .then() + .statusCode(200) + .extract().xmlPath().getString("SubscribeResponse.SubscribeResult.SubscriptionArn"); + + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "SetSubscriptionAttributes") + .formParam("SubscriptionArn", filterSubArnA) + .formParam("AttributeName", "FilterPolicy") + .formParam("AttributeValue", "{\"category\":[\"sports\"]}") + .when() + .post("/") + .then() + .statusCode(200); + + filterSubArnB = given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "Subscribe") + .formParam("TopicArn", topicArn) + .formParam("Protocol", "sqs") + .formParam("Endpoint", weatherQueueArn) + .when() + .post("/") + .then() + .statusCode(200) + .extract().xmlPath().getString("SubscribeResponse.SubscribeResult.SubscriptionArn"); + + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "SetSubscriptionAttributes") + .formParam("SubscriptionArn", filterSubArnB) + .formParam("AttributeName", "FilterPolicy") + .formParam("AttributeValue", "{\"category\":[\"weather\"]}") + .when() + .post("/") + .then() + .statusCode(200); + } + + @Test + @Order(14) + void filterPolicy_routesMessageToMatchingSubscription() { + drainQueue(filterQueueUrlA); + drainQueue(filterQueueUrlB); + + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "Publish") + .formParam("TopicArn", topicArn) + .formParam("Message", "Goal scored!") + .formParam("MessageAttributes.entry.1.Name", "category") + .formParam("MessageAttributes.entry.1.Value.DataType", "String") + .formParam("MessageAttributes.entry.1.Value.StringValue", "sports") + .when() + .post("/") + .then() + .statusCode(200) + .body(containsString("")); + + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "ReceiveMessage") + .formParam("QueueUrl", filterQueueUrlA) + .formParam("MaxNumberOfMessages", "1") + .when() + .post("/") + .then() + .statusCode(200) + .body(containsString("Goal scored!")); + + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "ReceiveMessage") + .formParam("QueueUrl", filterQueueUrlB) + .formParam("MaxNumberOfMessages", "1") + .when() + .post("/") + .then() + .statusCode(200) + .body(not(containsString(""))); + } + + @Test + @Order(15) + void filterPolicy_noFilterPolicyReceivesAllMessages() { + drainQueue(sqsQueueUrl); + + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "Publish") + .formParam("TopicArn", topicArn) + .formParam("Message", "Unfiltered broadcast") + .formParam("MessageAttributes.entry.1.Name", "category") + .formParam("MessageAttributes.entry.1.Value.DataType", "String") + .formParam("MessageAttributes.entry.1.Value.StringValue", "weather") + .when() + .post("/") + .then() + .statusCode(200); + + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "ReceiveMessage") + .formParam("QueueUrl", sqsQueueUrl) + .formParam("MaxNumberOfMessages", "1") + .when() + .post("/") + .then() + .statusCode(200) + .body(containsString("Unfiltered broadcast")); + } + + @Test + @Order(16) + void filterPolicy_nonMatchingMessageNotDelivered() { + drainQueue(filterQueueUrlA); + drainQueue(filterQueueUrlB); + + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "Publish") + .formParam("TopicArn", topicArn) + .formParam("Message", "Stock update") + .formParam("MessageAttributes.entry.1.Name", "category") + .formParam("MessageAttributes.entry.1.Value.DataType", "String") + .formParam("MessageAttributes.entry.1.Value.StringValue", "finance") + .when() + .post("/") + .then() + .statusCode(200); + + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "ReceiveMessage") + .formParam("QueueUrl", filterQueueUrlA) + .formParam("MaxNumberOfMessages", "1") + .when() + .post("/") + .then() + .statusCode(200) + .body(not(containsString(""))); + + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "ReceiveMessage") + .formParam("QueueUrl", filterQueueUrlB) + .formParam("MaxNumberOfMessages", "1") + .when() + .post("/") + .then() + .statusCode(200) + .body(not(containsString(""))); + } + + @Test + @Order(17) + void filterPolicy_cleanup() { + given().contentType("application/x-www-form-urlencoded") + .formParam("Action", "Unsubscribe").formParam("SubscriptionArn", filterSubArnA) + .when().post("/"); + given().contentType("application/x-www-form-urlencoded") + .formParam("Action", "Unsubscribe").formParam("SubscriptionArn", filterSubArnB) + .when().post("/"); + given().contentType("application/x-www-form-urlencoded") + .formParam("Action", "DeleteQueue").formParam("QueueUrl", filterQueueUrlA) + .when().post("/"); + given().contentType("application/x-www-form-urlencoded") + .formParam("Action", "DeleteQueue").formParam("QueueUrl", filterQueueUrlB) + .when().post("/"); + } + @Test @Order(100) void unsubscribe() { @@ -404,7 +633,7 @@ void deleteTopic() { } @Test - @Order(15) + @Order(102) void unsupportedAction_returns400() { given() .contentType("application/x-www-form-urlencoded") @@ -415,4 +644,18 @@ void unsupportedAction_returns400() { .statusCode(400) .body(containsString("UnsupportedOperation")); } + + /** + * Drains all pending messages from the given SQS queue using PurgeQueue. + */ + private void drainQueue(String queueUrl) { + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "PurgeQueue") + .formParam("QueueUrl", queueUrl) + .when() + .post("/") + .then() + .statusCode(200); + } } From 995038096bf368fc2855c3da55139f0f4a0083fb Mon Sep 17 00:00:00 2001 From: Roberto Perez Alcolea Date: Tue, 31 Mar 2026 14:53:41 -0700 Subject: [PATCH 13/32] fix(s3): persist Content-Encoding header on S3 objects (#57) Add contentEncoding field to S3Object and thread it through putObject, copyObject, getObject, and headObject. aws-chunked is excluded from persistence as it is a transfer-protocol marker, not a real encoding. CopyObject respects x-amz-metadata-directive: REPLACE by adopting the new Content-Encoding from the request instead of the source object. --- .../floci/services/s3/S3Controller.java | 33 ++++- .../floci/services/s3/S3Service.java | 35 ++++- .../floci/services/s3/model/S3Object.java | 4 + .../floci/services/s3/S3IntegrationTest.java | 124 ++++++++++++++++++ 4 files changed, 192 insertions(+), 4 deletions(-) diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java index 92f24bb6..7ed30201 100644 --- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java +++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java @@ -344,8 +344,10 @@ public Response putObject(@PathParam("bucket") String bucket, byte[] data = decodeAwsChunked(body, contentEncoding, contentSha256); validateChecksumHeaders(httpHeaders, data); + String persistedEncoding = toPersistedContentEncoding(contentEncoding); S3Object obj = s3Service.putObject(bucket, key, data, contentType, extractUserMetadata(httpHeaders), httpHeaders.getHeaderString("x-amz-storage-class"), + persistedEncoding, lockMode, retainUntil, legalHold); var resp = Response.ok().header("ETag", obj.getETag()); if (obj.getVersionId() != null) { @@ -823,6 +825,30 @@ private Response handlePutBucketNotification(String bucket, byte[] body) { } } + /** + * Strips the {@code aws-chunked} token from a {@code Content-Encoding} value before persisting it. + * {@code aws-chunked} is a transfer-protocol marker used by AWS SDK v2 streaming uploads and is not + * a real content encoding. For example, {@code gzip,aws-chunked} persists as {@code gzip}; + * a value of only {@code aws-chunked} persists as {@code null}. + */ + private static String toPersistedContentEncoding(String contentEncoding) { + if (contentEncoding == null) { + return null; + } + String[] tokens = contentEncoding.split(","); + StringBuilder result = new StringBuilder(); + for (String token : tokens) { + String trimmed = token.trim(); + if (!trimmed.equalsIgnoreCase("aws-chunked")) { + if (!result.isEmpty()) { + result.append(","); + } + result.append(trimmed); + } + } + return result.isEmpty() ? null : result.toString(); + } + // --- AWS Chunked Decoding --- /** @@ -1022,6 +1048,9 @@ private void appendObjectHeaders(Response.ResponseBuilder resp, S3Object obj) { if (obj.getStorageClass() != null) { resp.header("x-amz-storage-class", obj.getStorageClass()); } + if (obj.getContentEncoding() != null) { + resp.header("Content-Encoding", obj.getContentEncoding()); + } if (obj.getMetadata() != null) { for (Map.Entry entry : obj.getMetadata().entrySet()) { resp.header("x-amz-meta-" + entry.getKey(), entry.getValue()); @@ -1056,11 +1085,13 @@ private Response handleCopyObject(String copySource, String destBucket, String d String sourceBucket = source.substring(0, slashIndex); String sourceKey = URLDecoder.decode(source.substring(slashIndex + 1), StandardCharsets.UTF_8); + String copyContentEncoding = toPersistedContentEncoding(httpHeaders.getHeaderString("Content-Encoding")); S3Object copy = s3Service.copyObject(sourceBucket, sourceKey, destBucket, destKey, httpHeaders.getHeaderString("x-amz-metadata-directive"), extractUserMetadata(httpHeaders), httpHeaders.getHeaderString("x-amz-storage-class"), - contentType); + contentType, + copyContentEncoding); String xml = new XmlBuilder() .raw("") .start("CopyObjectResult", AwsNamespaces.S3) diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java index 311552d7..b94fb607 100644 --- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java +++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java @@ -150,8 +150,16 @@ public S3Object putObject(String bucketName, String key, byte[] data, public S3Object putObject(String bucketName, String key, byte[] data, String contentType, Map metadata, String storageClass, String objectLockMode, Instant retainUntilDate, String legalHoldStatus) { - S3Object object = storeObject(bucketName, key, data, contentType, metadata, storageClass, null, null, + return putObject(bucketName, key, data, contentType, metadata, storageClass, null, objectLockMode, retainUntilDate, legalHoldStatus); + } + + public S3Object putObject(String bucketName, String key, byte[] data, + String contentType, Map metadata, String storageClass, + String contentEncoding, + String objectLockMode, Instant retainUntilDate, String legalHoldStatus) { + S3Object object = storeObject(bucketName, key, data, contentType, metadata, storageClass, null, null, + objectLockMode, retainUntilDate, legalHoldStatus, contentEncoding); fireNotifications(bucketName, key, "ObjectCreated:Put", object); return object; } @@ -162,13 +170,22 @@ public S3Object putObject(String bucketName, String key, byte[] data, private S3Object storeObject(String bucketName, String key, byte[] data, String contentType, Map metadata) { return storeObject(bucketName, key, data, contentType, metadata, null, null, null, - null, null, null); + null, null, null, null); } private S3Object storeObject(String bucketName, String key, byte[] data, String contentType, Map metadata, String storageClass, S3Checksum checksum, List parts, String objectLockMode, Instant retainUntilDate, String legalHoldStatus) { + return storeObject(bucketName, key, data, contentType, metadata, storageClass, checksum, parts, + objectLockMode, retainUntilDate, legalHoldStatus, null); + } + + private S3Object storeObject(String bucketName, String key, byte[] data, + String contentType, Map metadata, String storageClass, + S3Checksum checksum, List parts, + String objectLockMode, Instant retainUntilDate, String legalHoldStatus, + String contentEncoding) { Bucket bucket = bucketStore.get(bucketName) .orElseThrow(() -> new AwsException("NoSuchBucket", "The specified bucket does not exist.", 404)); @@ -180,6 +197,7 @@ private S3Object storeObject(String bucketName, String key, byte[] data, object.setStorageClass(ObjectAttributeName.normalizeStorageClass(storageClass)); object.setChecksum(checksum != null ? copyChecksum(checksum) : buildChecksum(data, parts, false)); object.setParts(copyParts(parts)); + object.setContentEncoding(contentEncoding); if (bucket.isVersioningEnabled()) { String versionId = UUID.randomUUID().toString(); @@ -462,6 +480,14 @@ public S3Object copyObject(String sourceBucket, String sourceKey, String destBucket, String destKey, String metadataDirective, Map replacementMetadata, String storageClass, String contentType) { + return copyObject(sourceBucket, sourceKey, destBucket, destKey, metadataDirective, + replacementMetadata, storageClass, contentType, null); + } + + public S3Object copyObject(String sourceBucket, String sourceKey, + String destBucket, String destKey, + String metadataDirective, Map replacementMetadata, + String storageClass, String contentType, String contentEncoding) { S3Object source = getObject(sourceBucket, sourceKey); ensureBucketExists(destBucket); @@ -473,8 +499,10 @@ public S3Object copyObject(String sourceBucket, String sourceKey, String effectiveContentType = replaceMetadata && contentType != null ? contentType : source.getContentType(); String effectiveStorageClass = storageClass != null ? storageClass : source.getStorageClass(); + String effectiveContentEncoding = replaceMetadata && contentEncoding != null ? contentEncoding : source.getContentEncoding(); S3Object copy = storeObject(destBucket, destKey, source.getData(), effectiveContentType, metadata, - effectiveStorageClass, source.getChecksum(), source.getParts(), null, null, null); + effectiveStorageClass, source.getChecksum(), source.getParts(), null, null, null, + effectiveContentEncoding); copy.setETag(source.getETag()); LOG.debugv("Copied object: {0}/{1} -> {2}/{3}", sourceBucket, sourceKey, destBucket, destKey); fireNotifications(destBucket, destKey, "ObjectCreated:Copy", copy); @@ -1148,6 +1176,7 @@ private static S3Object copyObject(S3Object source) { copy.setData(source.getData() != null ? Arrays.copyOf(source.getData(), source.getData().length) : null); copy.setMetadata(new HashMap<>(source.getMetadata())); copy.setContentType(source.getContentType()); + copy.setContentEncoding(source.getContentEncoding()); copy.setSize(source.getSize()); copy.setLastModified(source.getLastModified()); copy.setETag(source.getETag()); diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Object.java b/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Object.java index 08c422d5..facd71ec 100644 --- a/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Object.java +++ b/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Object.java @@ -21,6 +21,7 @@ public class S3Object { private byte[] data; private Map metadata; private String contentType; + private String contentEncoding; private long size; private Instant lastModified; private String eTag; @@ -77,6 +78,9 @@ public S3Object(String bucketName, String key, byte[] data, String contentType) public String getContentType() { return contentType; } public void setContentType(String contentType) { this.contentType = contentType; } + public String getContentEncoding() { return contentEncoding; } + public void setContentEncoding(String contentEncoding) { this.contentEncoding = contentEncoding; } + public long getSize() { return size; } public void setSize(long size) { this.size = size; } diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java index 9ee2ddd1..ce962863 100644 --- a/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java +++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java @@ -6,6 +6,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestMethodOrder; +import io.restassured.config.DecoderConfig; +import io.restassured.config.RestAssuredConfig; import java.util.Arrays; import static io.restassured.RestAssured.given; @@ -732,4 +734,126 @@ void cleanupAndDeleteBucket() { .then() .statusCode(204); } + + @Test + @Order(80) + void createEncodingTestBucket() { + given() + .when() + .put("/encoding-test-bucket") + .then() + .statusCode(200); + } + + @Test + @Order(81) + void putObjectWithContentEncoding() { + given() + .contentType("text/plain") + .header("Content-Encoding", "gzip") + .body("compressed-content") + .when() + .put("/encoding-test-bucket/encoded.txt") + .then() + .statusCode(200) + .header("ETag", notNullValue()); + } + + @Test + @Order(82) + void getObjectReturnsContentEncoding() { + RestAssuredConfig noDecompress = RestAssuredConfig.config() + .decoderConfig(DecoderConfig.decoderConfig().noContentDecoders()); + given() + .config(noDecompress) + .when() + .get("/encoding-test-bucket/encoded.txt") + .then() + .statusCode(200) + .header("Content-Encoding", equalTo("gzip")); + } + + @Test + @Order(83) + void headObjectReturnsContentEncoding() { + given() + .when() + .head("/encoding-test-bucket/encoded.txt") + .then() + .statusCode(200) + .header("Content-Encoding", equalTo("gzip")); + } + + @Test + @Order(84) + void copyObjectPreservesContentEncoding() { + given() + .header("x-amz-copy-source", "/encoding-test-bucket/encoded.txt") + .when() + .put("/encoding-test-bucket/encoded-copy.txt") + .then() + .statusCode(200) + .body(containsString("CopyObjectResult")); + + given() + .when() + .head("/encoding-test-bucket/encoded-copy.txt") + .then() + .statusCode(200) + .header("Content-Encoding", equalTo("gzip")); + } + + @Test + @Order(85) + void copyObjectReplaceContentEncoding() { + given() + .header("x-amz-copy-source", "/encoding-test-bucket/encoded.txt") + .header("x-amz-metadata-directive", "REPLACE") + .header("Content-Encoding", "identity") + .when() + .put("/encoding-test-bucket/encoded-replace.txt") + .then() + .statusCode(200) + .body(containsString("CopyObjectResult")); + + given() + .when() + .head("/encoding-test-bucket/encoded-replace.txt") + .then() + .statusCode(200) + .header("Content-Encoding", equalTo("identity")); + } + + @Test + @Order(86) + void putObjectWithCompositeEncoding_stripsAwsChunkedToken() { + RestAssuredConfig noDecompress = RestAssuredConfig.config() + .decoderConfig(DecoderConfig.decoderConfig().noContentDecoders()); + given() + .contentType("text/plain") + .header("Content-Encoding", "gzip,aws-chunked") + .body("compressed-chunked-content") + .when() + .put("/encoding-test-bucket/composite-encoded.txt") + .then() + .statusCode(200); + + given() + .config(noDecompress) + .when() + .head("/encoding-test-bucket/composite-encoded.txt") + .then() + .statusCode(200) + .header("Content-Encoding", equalTo("gzip")); + } + + @Test + @Order(88) + void cleanupContentEncodingBucket() { + given().delete("/encoding-test-bucket/encoded.txt"); + given().delete("/encoding-test-bucket/encoded-copy.txt"); + given().delete("/encoding-test-bucket/encoded-replace.txt"); + given().delete("/encoding-test-bucket/composite-encoded.txt"); + given().delete("/encoding-test-bucket"); + } } From b5c77371ab42e149628718d59af6ab52c502dd62 Mon Sep 17 00:00:00 2001 From: Roberto Perez Alcolea Date: Tue, 31 Mar 2026 15:10:18 -0700 Subject: [PATCH 14/32] fix(sqs): translate Query-protocol error codes to JSON __type equivalents (#59) The AWS SDK v2 uses the __type field in JSON error responses to instantiate typed exceptions. Query-protocol codes such as AWS.SimpleQueueService.NonExistentQueue were being forwarded verbatim into the JSON __type field, causing the SDK to fall back to a generic SqsException instead of QueueDoesNotExistException. Add a jsonType() method to AwsException that maps Query-protocol codes to their JSON-protocol equivalents (e.g. QueueDoesNotExist, QueueNameExists). Both AwsExceptionMapper and the inline catch in AwsJsonController now call jsonType() so the mapping is applied consistently across all code paths. --- .../floci/core/common/AwsException.java | 26 ++++++ .../floci/core/common/AwsExceptionMapper.java | 2 +- .../floci/core/common/AwsJsonController.java | 4 +- .../services/sqs/SqsIntegrationTest.java | 91 +++++++++++++++++++ 4 files changed, 120 insertions(+), 3 deletions(-) diff --git a/src/main/java/io/github/hectorvent/floci/core/common/AwsException.java b/src/main/java/io/github/hectorvent/floci/core/common/AwsException.java index 58616dbf..f792d3ac 100644 --- a/src/main/java/io/github/hectorvent/floci/core/common/AwsException.java +++ b/src/main/java/io/github/hectorvent/floci/core/common/AwsException.java @@ -1,11 +1,29 @@ package io.github.hectorvent.floci.core.common; +import java.util.Map; + /** * Base exception for AWS emulator errors. * Maps to AWS-style error responses with code, message, and HTTP status. + *

+ * Some services use different error code formats for Query (XML) and JSON protocols. + * {@link #jsonType()} returns the JSON-protocol {@code __type} value that the AWS SDK v2 + * uses to instantiate a specific typed exception rather than falling back to a generic one. */ public class AwsException extends RuntimeException { + /** + * Maps Query-protocol error codes to their JSON-protocol {@code __type} equivalents. + * Codes absent from this map are used as-is for both protocols. + */ + private static final Map JSON_TYPE_BY_QUERY_CODE = Map.of( + "AWS.SimpleQueueService.NonExistentQueue", "QueueDoesNotExist", + "QueueAlreadyExists", "QueueNameExists", + "ReceiptHandleIsInvalid", "ReceiptHandleIsInvalid", + "TooManyEntriesInBatchRequest", "TooManyEntriesInBatchRequest", + "BatchEntryIdNotUnique", "BatchEntryIdNotDistinct" + ); + private final String errorCode; private final int httpStatus; @@ -22,4 +40,12 @@ public String getErrorCode() { public int getHttpStatus() { return httpStatus; } + + /** + * Returns the JSON-protocol {@code __type} value for this error. + * The AWS SDK v2 uses this to map responses to typed exception classes. + */ + public String jsonType() { + return JSON_TYPE_BY_QUERY_CODE.getOrDefault(errorCode, errorCode); + } } diff --git a/src/main/java/io/github/hectorvent/floci/core/common/AwsExceptionMapper.java b/src/main/java/io/github/hectorvent/floci/core/common/AwsExceptionMapper.java index 161a65aa..7ddb33f8 100644 --- a/src/main/java/io/github/hectorvent/floci/core/common/AwsExceptionMapper.java +++ b/src/main/java/io/github/hectorvent/floci/core/common/AwsExceptionMapper.java @@ -18,8 +18,8 @@ public class AwsExceptionMapper implements ExceptionMapper { public Response toResponse(AwsException exception) { LOG.debugv("Mapping exception: {0} - {1}", exception.getErrorCode(), exception.getMessage()); return Response.status(exception.getHttpStatus()) - .entity(new AwsErrorResponse(exception.getErrorCode(), exception.getMessage())) .type(MediaType.APPLICATION_JSON) + .entity(new AwsErrorResponse(exception.jsonType(), exception.getMessage())) .build(); } } diff --git a/src/main/java/io/github/hectorvent/floci/core/common/AwsJsonController.java b/src/main/java/io/github/hectorvent/floci/core/common/AwsJsonController.java index cf34f05d..88a11f8b 100644 --- a/src/main/java/io/github/hectorvent/floci/core/common/AwsJsonController.java +++ b/src/main/java/io/github/hectorvent/floci/core/common/AwsJsonController.java @@ -135,7 +135,7 @@ public Response handleJsonRequest( } catch (AwsException e) { return Response.status(e.getHttpStatus()) .type(MediaType.APPLICATION_JSON) - .entity(new AwsErrorResponse(e.getErrorCode(), e.getMessage())) + .entity(new AwsErrorResponse(e.jsonType(), e.getMessage())) .build(); } catch (Exception e) { LOG.error("Error processing " + serviceName + " JSON request", e); @@ -329,7 +329,7 @@ private Response dispatchCbor(String serviceId, String operation, JsonNode reque private Response cborErrorResponse(AwsException e, String protocolHeader) { try { byte[] errBytes = CBOR_MAPPER.writeValueAsBytes( - new AwsErrorResponse(e.getErrorCode(), e.getMessage())); + new AwsErrorResponse(e.jsonType(), e.getMessage())); return Response.status(e.getHttpStatus()) .header(protocolHeader, "rpc-v2-cbor") .type("application/cbor") diff --git a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsIntegrationTest.java index b78afb18..3c570761 100644 --- a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsIntegrationTest.java +++ b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsIntegrationTest.java @@ -1,6 +1,9 @@ package io.github.hectorvent.floci.services.sqs; import io.quarkus.test.junit.QuarkusTest; +import io.restassured.RestAssured; +import io.restassured.config.EncoderConfig; +import io.restassured.http.ContentType; import org.junit.jupiter.api.MethodOrderer; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; @@ -215,4 +218,92 @@ void unsupportedAction() { .statusCode(400) .body(containsString("UnsupportedOperation")); } + + @Test + void createQueue_idempotent_sameAttributes() { + String queueName = "idempotent-test-queue"; + + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "CreateQueue") + .formParam("QueueName", queueName) + .formParam("Attribute.1.Name", "VisibilityTimeout") + .formParam("Attribute.1.Value", "60") + .when() + .post("/") + .then() + .statusCode(200) + .body(containsString(queueName)); + + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "CreateQueue") + .formParam("QueueName", queueName) + .formParam("Attribute.1.Name", "VisibilityTimeout") + .formParam("Attribute.1.Value", "60") + .when() + .post("/") + .then() + .statusCode(200) + .body(containsString(queueName)); + + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "DeleteQueue") + .formParam("QueueUrl", "http://localhost:4566/000000000000/" + queueName) + .when() + .post("/"); + } + + @Test + void createQueue_conflictingAttributes_returns400() { + String queueName = "conflict-test-queue"; + + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "CreateQueue") + .formParam("QueueName", queueName) + .formParam("Attribute.1.Name", "VisibilityTimeout") + .formParam("Attribute.1.Value", "30") + .when() + .post("/") + .then() + .statusCode(200); + + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "CreateQueue") + .formParam("QueueName", queueName) + .formParam("Attribute.1.Name", "VisibilityTimeout") + .formParam("Attribute.1.Value", "60") + .when() + .post("/") + .then() + .statusCode(400) + .body(containsString("QueueNameExists")); + + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "DeleteQueue") + .formParam("QueueUrl", "http://localhost:4566/000000000000/" + queueName) + .when() + .post("/"); + } + + @Test + void jsonProtocol_nonExistentQueue_returnsQueueDoesNotExist() { + given() + .config(RestAssured.config().encoderConfig( + EncoderConfig.encoderConfig() + .encodeContentTypeAs("application/x-amz-json-1.0", ContentType.TEXT))) + .contentType("application/x-amz-json-1.0") + .header("X-Amz-Target", "AmazonSQS.GetQueueUrl") + .body("{\"QueueName\": \"no-such-queue-xyz\"}") + .when() + .post("/") + .then() + .statusCode(400) + .body(containsString("QueueDoesNotExist")) + .body(not(containsString("AWS.SimpleQueueService.NonExistentQueue"))); + } } From 7a8a4bd8d705aee7d006603ca85e0ef3234e7e2f Mon Sep 17 00:00:00 2001 From: Cyril Schumacher Date: Wed, 1 Apr 2026 00:12:01 +0200 Subject: [PATCH 15/32] docs: improve initialization hooks documentation (#144) * docs: add initialization hooks to navigation * docs: improve link label for initialization hooks * docs: expand initialization hooks documentation with examples --- docs/configuration/docker-compose.md | 2 +- docs/configuration/initialization-hooks.md | 119 ++++++++++++++++----- mkdocs.yml | 1 + 3 files changed, 93 insertions(+), 29 deletions(-) diff --git a/docs/configuration/docker-compose.md b/docs/configuration/docker-compose.md index 4809f025..15f16088 100644 --- a/docs/configuration/docker-compose.md +++ b/docs/configuration/docker-compose.md @@ -55,7 +55,7 @@ services: - ./init/stop.d:/etc/floci/init/stop.d:ro ``` -See [`initialization-hooks.md`](./initialization-hooks.md) for execution behavior and configuration details. +See [Initialization Hooks](./initialization-hooks.md) for execution behavior and configuration details. ## Persistence diff --git a/docs/configuration/initialization-hooks.md b/docs/configuration/initialization-hooks.md index a8bc3fa8..920fd9ed 100644 --- a/docs/configuration/initialization-hooks.md +++ b/docs/configuration/initialization-hooks.md @@ -1,47 +1,110 @@ # Initialization Hooks -Floci can execute shell scripts during startup and shutdown. +Floci allows you to execute custom shell scripts when it starts and stops. These scripts can help set up your +environment (creating buckets, populating data, configuring resources, etc.) or tidy up during shutdown. -## Directories +Hook scripts ending with `.sh` are discovered in the following directories: -- Startup hooks are loaded from `/etc/floci/init/start.d` -- Shutdown hooks are loaded from `/etc/floci/init/stop.d` +- **Startup hooks** (`/etc/floci/init/start.d`) run after Floci services are initialized, but before the environment is marked as ready. +- **Shutdown hooks** (`/etc/floci/init/stop.d`) run when Floci is shutting down, after `destroy()` is triggered. -Only files ending with `.sh` are executed. +If a hook directory does not exist or contains no `.sh` scripts, Floci skips it and continues normally. +If the hook path exists but is not a directory, it is ignored. -## Execution Model +## Execution -- Scripts are executed in lexicographical order -- Hook scripts are executed sequentially -- Hook execution is fail-fast: execution stops at the first script that fails or times out +### Execution Environment + +Hooks run: + +- Inside the Floci runtime environment (same context as Floci services) +- Using the configured shell (default: `/bin/bash`) +- With access to configured services and their endpoints +- With the same environment variables as Floci + +Hooks can call Floci service endpoints directly from inside the container. If a hook depends on additional CLI tools, +make sure those tools are available in the runtime image. + +### Execution Behavior + +Scripts are executed: + +- In **lexicographical (alphabetical) order** +- **Sequentially** (one at a time) + +When execution order matters, prefix filenames with numbers such as `01-`, `02-`, and `03-`. + +Execution uses a fail-fast strategy: + +- If a script exits with a non-zero status, remaining hooks are not executed. +- If a script exceeds the configured timeout, it is terminated and remaining hooks are not executed. +- A hook failure marks the corresponding startup or shutdown phase as **failed**. + +## Examples + +The following examples assume the runtime image includes the AWS CLI and that Floci is reachable at +`http://localhost:4566`. + +### Startup Hook + +For example, a startup hook could look like this: + +```sh +#!/bin/sh +set -eu + +aws --endpoint-url http://localhost:4566 \ + ssm put-parameter \ + --name /demo/app/bootstrapped \ + --type String \ + --value true \ + --overwrite +``` + +This example assumes the script is stored at `/etc/floci/init/start.d/01-seed-parameter.sh`. +It seeds a known SSM parameter during startup so tests or local services can rely on it. + +### Shutdown Hook + +For example, a shutdown hook could look like this: + +```sh +#!/bin/sh +set -eu + +aws --endpoint-url http://localhost:4566 \ + ssm delete-parameter \ + --name /demo/app/bootstrapped +``` + +This example assumes the script is stored at `/etc/floci/init/stop.d/01-cleanup-parameter.sh`. +It removes the parameter during shutdown to leave the environment clean. ## Configuration -| Key | Default | Description | -|---|---|---| -| `floci.init-hooks.shell-executable` | `/bin/bash` | Shell executable used to run hook scripts | -| `floci.init-hooks.timeout-seconds` | `30` | Maximum execution time per hook script before it is considered failed | -| `floci.init-hooks.shutdown-grace-period-seconds` | `2` | Time to wait after `destroy()` before forcing process termination | +You can customize hook behavior via configuration: + +| Key | Default | Description | +|--------------------------------------------------|-------------|------------------------------------------------------------------------------------------------------------------| +| `floci.init-hooks.shell-executable` | `/bin/bash` | Shell executable used to run scripts | +| `floci.init-hooks.timeout-seconds` | `30` | Maximum execution time per script before it is terminated and considered failed | +| `floci.init-hooks.shutdown-grace-period-seconds` | `2` | Time to wait after calling `destroy()` before forcefully stopping the process (allows cleanup hooks to complete) | ### Example +The following configuration can be useful when startup hooks perform more in-depth setup work, such as seeding test +data or provisioning resources before an integration test suite starts. + ```yaml floci: init-hooks: - shell-executable: /bin/bash - timeout-seconds: 30 - shutdown-grace-period-seconds: 2 + shell-executable: /bin/sh + timeout-seconds: 60 + shutdown-grace-period-seconds: 10 ``` -## Docker Compose Example +In this example: -```yaml -services: - floci: - image: hectorvent/floci:latest - ports: - - "4566:4566" - volumes: - - ./init/start.d:/etc/floci/init/start.d:ro - - ./init/stop.d:/etc/floci/init/stop.d:ro -``` +- `shell-executable` uses `/bin/sh` for portable POSIX-compatible scripts. +- `timeout-seconds: 60` gives startup hooks more time to complete initialization tasks. +- `shutdown-grace-period-seconds: 10` gives shutdown hooks more time to finish cleanup before Floci stops. \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index e4b6a9b0..f9adc5e2 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -64,6 +64,7 @@ nav: - Ports Reference: configuration/ports.md - application.yml Reference: configuration/application-yml.md - Storage Modes: configuration/storage.md + - Initialization Hooks: configuration/initialization-hooks.md - Services: - Overview: services/index.md - SSM Parameter Store: services/ssm.md From 20618d32e6b11bd0682bd499f92dba54179bd5bd Mon Sep 17 00:00:00 2001 From: Hector Ventura Date: Tue, 31 Mar 2026 18:30:29 -0500 Subject: [PATCH 16/32] fix: adding aws-cli in its own floci image hectorvent/floci:x.y.z-aws (#151) --- .github/workflows/release.yml | 14 ++++++++++++ Dockerfile.awscli | 40 ----------------------------------- Dockerfile.jvm-package | 13 ++++++++---- 3 files changed, 23 insertions(+), 44 deletions(-) delete mode 100644 Dockerfile.awscli diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2b694be9..88833637 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -137,6 +137,20 @@ jobs: build-args: | VERSION=${{ steps.version.outputs.version }} + - name: Build and push JVM image (With AWS CLI) + uses: docker/build-push-action@v6 + with: + context: . + file: Dockerfile.jvm-package + platforms: linux/amd64,linux/arm64 + push: true + tags: | + ${{ secrets.DOCKERHUB_USERNAME }}/floci:${{ steps.version.outputs.version }}-aws + ${{ secrets.DOCKERHUB_USERNAME }}/floci:latest-aws + build-args: | + VERSION=${{ steps.version.outputs.version }} + INSTALL_AWS_CLI=true + # ── Push native Docker images (multi-arch manifest) ─────────────────────── push-native: name: Push native Docker images diff --git a/Dockerfile.awscli b/Dockerfile.awscli deleted file mode 100644 index 81d21168..00000000 --- a/Dockerfile.awscli +++ /dev/null @@ -1,40 +0,0 @@ -# Stage 1: Build -FROM eclipse-temurin:25-jdk AS build -WORKDIR /build - -RUN apt-get update && apt-get install -y maven && rm -rf /var/lib/apt/lists/* - -COPY pom.xml . -RUN mvn dependency:go-offline -q - -COPY src/ src/ -RUN mvn clean package -DskipTests -q - -# Stage 2: AWS CLI -FROM debian:stable-slim AS aws -RUN apt-get update && apt-get install -y --no-install-recommends curl unzip ca-certificates \ - && rm -rf /var/lib/apt/lists/* -RUN ARCH=$(uname -m) && \ - curl "https://awscli.amazonaws.com/awscli-exe-linux-${ARCH}.zip" -o awscliv2.zip && \ - unzip awscliv2.zip && \ - ./aws/install && \ - rm -rf awscliv2.zip aws - -# Stage 3: Runtime -FROM eclipse-temurin:25-jre -WORKDIR /app - -COPY --from=build /build/target/quarkus-app/ quarkus-app/ -COPY --from=aws /usr/local/aws-cli/ /usr/local/aws-cli/ -RUN ln -s /usr/local/aws-cli/v2/current/bin/aws /usr/local/bin/aws && \ - ln -s /usr/local/aws-cli/v2/current/bin/aws_completer /usr/local/bin/aws_completer - -RUN mkdir -p /app/data -VOLUME /app/data - -EXPOSE 4566 6379-6399 - -ARG VERSION=latest -ENV FLOCI_VERSION=${VERSION} - -ENTRYPOINT ["java", "-jar", "quarkus-app/quarkus-run.jar"] diff --git a/Dockerfile.jvm-package b/Dockerfile.jvm-package index feddf0a7..e3eee37a 100644 --- a/Dockerfile.jvm-package +++ b/Dockerfile.jvm-package @@ -1,11 +1,19 @@ FROM eclipse-temurin:25-jre-alpine +ARG VERSION=latest +ARG INSTALL_AWS_CLI=false + +ENV FLOCI_VERSION=${VERSION} + WORKDIR /app RUN mkdir -p /app/data \ && chown 1001:root /app \ && chmod "g+rwX" /app \ - && chown 1001:root /app/data + && chown 1001:root /app/data \ + && if [ "$INSTALL_AWS_CLI" = "true" ]; then \ + apk add --no-cache aws-cli; \ + fi VOLUME /app/data @@ -13,9 +21,6 @@ COPY --chown=1001:root target/quarkus-app quarkus-app/ EXPOSE 4566 6379-6399 -ARG VERSION=latest -ENV FLOCI_VERSION=${VERSION} - USER 1001 ENTRYPOINT ["java", "-jar", "quarkus-app/quarkus-run.jar", "-Dquarkus.http.host=0.0.0.0"] \ No newline at end of file From 6c2e214df109f8be550349e877f1485ca1f68aa2 Mon Sep 17 00:00:00 2001 From: Jeffen <85473293+aquanow-jeffen@users.noreply.github.com> Date: Tue, 31 Mar 2026 16:47:38 -0700 Subject: [PATCH 17/32] fix(Cognito): OAuth/OIDC parity for RS256/JWKS, /oauth2/token, and OAuth app-client settings (#97) * feat: add Cognito OAuth and resource server support * chore: set FLOCI_BASE_URL for compatibility tests --- .github/workflows/compatibility.yml | 1 + docs/services/cognito.md | 66 ++- .../core/common/ServiceEnabledFilter.java | 26 +- .../services/cognito/CognitoJsonHandler.java | 115 +++- .../cognito/CognitoOAuthController.java | 158 +++++ .../services/cognito/CognitoService.java | 402 ++++++++++++- .../cognito/CognitoWellKnownController.java | 36 +- .../cognito/model/ResourceServer.java | 42 ++ .../cognito/model/ResourceServerScope.java | 17 + .../services/cognito/model/UserPool.java | 12 + .../cognito/model/UserPoolClient.java | 25 + .../cognito/CognitoIntegrationTest.java | 230 ++++++++ .../CognitoOAuthTokenIntegrationTest.java | 557 ++++++++++++++++++ 13 files changed, 1645 insertions(+), 42 deletions(-) create mode 100644 src/main/java/io/github/hectorvent/floci/services/cognito/CognitoOAuthController.java create mode 100644 src/main/java/io/github/hectorvent/floci/services/cognito/model/ResourceServer.java create mode 100644 src/main/java/io/github/hectorvent/floci/services/cognito/model/ResourceServerScope.java create mode 100644 src/test/java/io/github/hectorvent/floci/services/cognito/CognitoIntegrationTest.java create mode 100644 src/test/java/io/github/hectorvent/floci/services/cognito/CognitoOAuthTokenIntegrationTest.java diff --git a/.github/workflows/compatibility.yml b/.github/workflows/compatibility.yml index 690465a0..6601f0f7 100644 --- a/.github/workflows/compatibility.yml +++ b/.github/workflows/compatibility.yml @@ -96,6 +96,7 @@ jobs: -p 4566:4566 \ -v /var/run/docker.sock:/var/run/docker.sock \ --group-add "$DOCKER_GID" \ + -e FLOCI_BASE_URL=http://floci:4566 \ -e FLOCI_SERVICES_DOCKER_NETWORK=compat-net \ -e FLOCI_HOSTNAME=floci \ floci:test diff --git a/docs/services/cognito.md b/docs/services/cognito.md index 356d5b12..827769e4 100644 --- a/docs/services/cognito.md +++ b/docs/services/cognito.md @@ -3,7 +3,7 @@ **Protocol:** JSON 1.1 (`X-Amz-Target: AWSCognitoIdentityProviderService.*`) **Endpoint:** `POST http://localhost:4566/` -Floci also serves OIDC well-known endpoints, making it compatible with JWT validation libraries. +Floci serves pool-specific discovery and JWKS endpoints, plus a relaxed OAuth token endpoint, so local clients can mint and validate Cognito-like access tokens against RS256 signing keys. ## Supported Actions @@ -11,17 +11,30 @@ Floci also serves OIDC well-known endpoints, making it compatible with JWT valid |---|---| | **User Pools** | CreateUserPool, DescribeUserPool, ListUserPools, DeleteUserPool | | **User Pool Clients** | CreateUserPoolClient, DescribeUserPoolClient, ListUserPoolClients, DeleteUserPoolClient | +| **Resource Servers** | CreateResourceServer, DescribeResourceServer, ListResourceServers, DeleteResourceServer | | **Admin User Management** | AdminCreateUser, AdminGetUser, AdminDeleteUser, AdminSetUserPassword, AdminUpdateUserAttributes | | **User Operations** | SignUp, ConfirmSignUp, GetUser, UpdateUserAttributes, ChangePassword, ForgotPassword, ConfirmForgotPassword | | **Authentication** | InitiateAuth, AdminInitiateAuth, RespondToAuthChallenge | | **User Listing** | ListUsers | -## OIDC Well-Known Endpoints +## Well-Known And OAuth Endpoints | Endpoint | Description | |---|---| -| `GET /.well-known/openid-configuration` | OIDC discovery document | -| `GET /.well-known/jwks.json` | JSON Web Key Set for JWT validation | +| `GET /{userPoolId}/.well-known/openid-configuration` | OpenID discovery document | +| `GET /{userPoolId}/.well-known/jwks.json` | JSON Web Key Set for JWT validation | +| `POST /cognito-idp/oauth2/token` | Relaxed OAuth token endpoint for `grant_type=client_credentials` | + +`POST /cognito-idp/oauth2/token` is intentionally emulator-friendly rather than full Cognito parity: + +- It requires an existing `client_id`. +- It accepts `client_id` and `client_secret` from the form body or Basic auth. +- It requires a confidential app client created with `GenerateSecret=true`. +- It requires `AllowedOAuthFlowsUserPoolClient=true` and `AllowedOAuthFlows=["client_credentials"]`. +- It doesn't require a Cognito domain. +- It returns only `access_token`, `token_type`, and `expires_in`. +- It validates requested OAuth scopes against the app client's `AllowedOAuthScopes` and the pool's registered resource-server scopes. +- It advertises the prefixed token endpoint in `/{userPoolId}/.well-known/openid-configuration`. ## Examples @@ -38,10 +51,28 @@ POOL_ID=$(aws cognito-idp create-user-pool \ CLIENT_ID=$(aws cognito-idp create-user-pool-client \ --user-pool-id $POOL_ID \ --client-name my-client \ - --explicit-auth-flows ALLOW_USER_PASSWORD_AUTH ALLOW_REFRESH_TOKEN_AUTH \ + --generate-secret \ + --allowed-o-auth-flows-user-pool-client \ + --allowed-o-auth-flows client_credentials \ + --allowed-o-auth-scopes notes/read notes/write \ --query UserPoolClient.ClientId --output text \ --endpoint-url $AWS_ENDPOINT) +# Retrieve the generated client secret +CLIENT_SECRET=$(aws cognito-idp describe-user-pool-client \ + --user-pool-id $POOL_ID \ + --client-id $CLIENT_ID \ + --query UserPoolClient.ClientSecret --output text \ + --endpoint-url $AWS_ENDPOINT) + +# Register a resource server and scopes +aws cognito-idp create-resource-server \ + --user-pool-id $POOL_ID \ + --identifier notes \ + --name "Notes API" \ + --scopes ScopeName=read,ScopeDescription="Read notes" ScopeName=write,ScopeDescription="Write notes" \ + --endpoint-url $AWS_ENDPOINT + # Create a user aws cognito-idp admin-create-user \ --user-pool-id $POOL_ID \ @@ -63,20 +94,35 @@ aws cognito-idp initiate-auth \ --client-id $CLIENT_ID \ --auth-parameters USERNAME=alice@example.com,PASSWORD=Perm1234! \ --endpoint-url $AWS_ENDPOINT + +# Fetch the pool discovery document +curl -s "$AWS_ENDPOINT/$POOL_ID/.well-known/openid-configuration" + +# Get a machine access token from the OAuth endpoint +curl -s \ + -X POST "$AWS_ENDPOINT/cognito-idp/oauth2/token" \ + -H "Content-Type: application/x-www-form-urlencoded" \ + -u "$CLIENT_ID:$CLIENT_SECRET" \ + --data-urlencode "grant_type=client_credentials" \ + --data-urlencode "scope=notes/read notes/write" ``` ## JWT Validation -Tokens issued by Floci can be validated using the JWKS endpoint: +Tokens issued by Floci can be validated using the discovery and JWKS endpoints: + +``` +http://localhost:4566/$POOL_ID/.well-known/openid-configuration +``` ``` -http://localhost:4566/.well-known/jwks.json +http://localhost:4566/$POOL_ID/.well-known/jwks.json ``` -The OIDC discovery endpoint returns: +Tokens issued by Cognito auth flows and the OAuth token endpoint use the emulator base URL plus the pool id: ``` -http://localhost:4566/.well-known/openid-configuration +http://localhost:4566/$POOL_ID ``` -This allows libraries like `jsonwebtoken`, `jose`, or Spring Security to validate tokens against Floci the same way they would against real Cognito. \ No newline at end of file +This keeps the issuer, discovery document, JWKS URL, and token endpoint internally consistent for local JWT validation while supporting LocalStack-style confidential clients and resource-server-backed scopes. diff --git a/src/main/java/io/github/hectorvent/floci/core/common/ServiceEnabledFilter.java b/src/main/java/io/github/hectorvent/floci/core/common/ServiceEnabledFilter.java index f6c054f8..1e63af3d 100644 --- a/src/main/java/io/github/hectorvent/floci/core/common/ServiceEnabledFilter.java +++ b/src/main/java/io/github/hectorvent/floci/core/common/ServiceEnabledFilter.java @@ -1,8 +1,12 @@ package io.github.hectorvent.floci.core.common; +import io.github.hectorvent.floci.services.cognito.CognitoOAuthController; +import io.github.hectorvent.floci.services.cognito.CognitoWellKnownController; import jakarta.inject.Inject; import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.container.ContainerRequestFilter; +import jakarta.ws.rs.container.ResourceInfo; +import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.ext.Provider; @@ -16,6 +20,9 @@ public class ServiceEnabledFilter implements ContainerRequestFilter { private static final Pattern AUTH_SERVICE_PATTERN = Pattern.compile("Credential=\\S+/\\d{8}/[^/]+/([^/]+)/"); + @Context + ResourceInfo resourceInfo; + private final ServiceRegistry serviceRegistry; @Inject @@ -48,7 +55,7 @@ private String resolveServiceKey(ContainerRequestContext ctx) { } } - return null; + return serviceKeyFromMatchedResource(); } private String serviceKeyFromTarget(String target) { @@ -75,12 +82,25 @@ private String mapCredentialScope(String scope) { }; } + private String serviceKeyFromMatchedResource() { + Class resourceClass = resourceInfo != null ? resourceInfo.getResourceClass() : null; + if (resourceClass == null) { + return null; + } + if (CognitoOAuthController.class.equals(resourceClass) + || CognitoWellKnownController.class.equals(resourceClass)) { + return "cognito-idp"; + } + return null; + } + private Response disabledResponse(ContainerRequestContext ctx, String serviceKey) { String message = "Service " + serviceKey + " is not enabled."; String target = ctx.getHeaderString("X-Amz-Target"); String contentType = ctx.getMediaType() != null ? ctx.getMediaType().toString() : ""; + boolean jsonEndpoint = serviceKeyFromMatchedResource() != null; - if (target != null || contentType.contains("json")) { + if (target != null || contentType.contains("json") || jsonEndpoint) { return Response.status(400) .type(MediaType.APPLICATION_JSON) .entity(new AwsErrorResponse("ServiceNotAvailableException", message)) @@ -99,4 +119,4 @@ private Response disabledResponse(ContainerRequestContext ctx, String serviceKey .build(); return Response.status(400).entity(xml).type(MediaType.APPLICATION_XML).build(); } -} \ No newline at end of file +} diff --git a/src/main/java/io/github/hectorvent/floci/services/cognito/CognitoJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/cognito/CognitoJsonHandler.java index 6e7acd74..abbc5aa4 100644 --- a/src/main/java/io/github/hectorvent/floci/services/cognito/CognitoJsonHandler.java +++ b/src/main/java/io/github/hectorvent/floci/services/cognito/CognitoJsonHandler.java @@ -6,6 +6,8 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import io.github.hectorvent.floci.services.cognito.model.CognitoUser; +import io.github.hectorvent.floci.services.cognito.model.ResourceServer; +import io.github.hectorvent.floci.services.cognito.model.ResourceServerScope; import io.github.hectorvent.floci.services.cognito.model.UserPool; import io.github.hectorvent.floci.services.cognito.model.UserPoolClient; import jakarta.enterprise.context.ApplicationScoped; @@ -38,6 +40,11 @@ public Response handle(String action, JsonNode request, String region) { case "DescribeUserPoolClient" -> handleDescribeUserPoolClient(request); case "ListUserPoolClients" -> handleListUserPoolClients(request); case "DeleteUserPoolClient" -> handleDeleteUserPoolClient(request); + case "CreateResourceServer" -> handleCreateResourceServer(request); + case "DescribeResourceServer" -> handleDescribeResourceServer(request); + case "ListResourceServers" -> handleListResourceServers(request); + case "UpdateResourceServer" -> handleUpdateResourceServer(request); + case "DeleteResourceServer" -> handleDeleteResourceServer(request); case "AdminCreateUser" -> handleAdminCreateUser(request); case "AdminGetUser" -> handleAdminGetUser(request); case "AdminDeleteUser" -> handleAdminDeleteUser(request); @@ -91,7 +98,11 @@ private Response handleDeleteUserPool(JsonNode request) { private Response handleCreateUserPoolClient(JsonNode request) { UserPoolClient client = service.createUserPoolClient( request.path("UserPoolId").asText(), - request.path("ClientName").asText() + request.path("ClientName").asText(), + request.path("GenerateSecret").asBoolean(false), + request.path("AllowedOAuthFlowsUserPoolClient").asBoolean(false), + readStringList(request.path("AllowedOAuthFlows")), + readStringList(request.path("AllowedOAuthScopes")) ); ObjectNode response = objectMapper.createObjectNode(); response.set("UserPoolClient", clientToNode(client)); @@ -124,6 +135,56 @@ private Response handleDeleteUserPoolClient(JsonNode request) { return Response.ok(objectMapper.createObjectNode()).build(); } + private Response handleCreateResourceServer(JsonNode request) { + ResourceServer server = service.createResourceServer( + request.path("UserPoolId").asText(), + request.path("Identifier").asText(), + request.path("Name").asText(), + parseScopes(request.path("Scopes")) + ); + ObjectNode response = objectMapper.createObjectNode(); + response.set("ResourceServer", resourceServerToNode(server)); + return Response.ok(response).build(); + } + + private Response handleDescribeResourceServer(JsonNode request) { + ResourceServer server = service.describeResourceServer( + request.path("UserPoolId").asText(), + request.path("Identifier").asText() + ); + ObjectNode response = objectMapper.createObjectNode(); + response.set("ResourceServer", resourceServerToNode(server)); + return Response.ok(response).build(); + } + + private Response handleListResourceServers(JsonNode request) { + List servers = service.listResourceServers(request.path("UserPoolId").asText()); + ObjectNode response = objectMapper.createObjectNode(); + ArrayNode items = response.putArray("ResourceServers"); + servers.forEach(server -> items.add(resourceServerToNode(server))); + return Response.ok(response).build(); + } + + private Response handleUpdateResourceServer(JsonNode request) { + ResourceServer server = service.updateResourceServer( + request.path("UserPoolId").asText(), + request.path("Identifier").asText(), + request.path("Name").asText(), + parseScopes(request.path("Scopes")) + ); + ObjectNode response = objectMapper.createObjectNode(); + response.set("ResourceServer", resourceServerToNode(server)); + return Response.ok(response).build(); + } + + private Response handleDeleteResourceServer(JsonNode request) { + service.deleteResourceServer( + request.path("UserPoolId").asText(), + request.path("Identifier").asText() + ); + return Response.ok(objectMapper.createObjectNode()).build(); + } + private Response handleAdminCreateUser(JsonNode request) { Map attrs = new HashMap<>(); request.path("UserAttributes").forEach(a -> attrs.put(a.path("Name").asText(), a.path("Value").asText())); @@ -321,11 +382,63 @@ private ObjectNode clientToNode(UserPoolClient c) { node.put("ClientId", c.getClientId()); node.put("UserPoolId", c.getUserPoolId()); node.put("ClientName", c.getClientName()); + if (c.getClientSecret() != null) { + node.put("ClientSecret", c.getClientSecret()); + } + node.put("GenerateSecret", c.isGenerateSecret()); + node.put("AllowedOAuthFlowsUserPoolClient", c.isAllowedOAuthFlowsUserPoolClient()); + ArrayNode flows = node.putArray("AllowedOAuthFlows"); + c.getAllowedOAuthFlows().forEach(flows::add); + ArrayNode scopes = node.putArray("AllowedOAuthScopes"); + c.getAllowedOAuthScopes().forEach(scopes::add); node.put("CreationDate", c.getCreationDate()); node.put("LastModifiedDate", c.getLastModifiedDate()); return node; } + private ObjectNode resourceServerToNode(ResourceServer server) { + ObjectNode node = objectMapper.createObjectNode(); + node.put("UserPoolId", server.getUserPoolId()); + node.put("Identifier", server.getIdentifier()); + node.put("Name", server.getName()); + node.put("CreationDate", server.getCreationDate()); + node.put("LastModifiedDate", server.getLastModifiedDate()); + ArrayNode scopes = node.putArray("Scopes"); + for (ResourceServerScope scope : server.getScopes()) { + ObjectNode item = scopes.addObject(); + item.put("ScopeName", scope.getScopeName()); + if (scope.getScopeDescription() != null) { + item.put("ScopeDescription", scope.getScopeDescription()); + } + } + return node; + } + + private List parseScopes(JsonNode scopesNode) { + if (scopesNode == null || !scopesNode.isArray()) { + return List.of(); + } + + List scopes = new java.util.ArrayList<>(); + scopesNode.forEach(item -> { + ResourceServerScope scope = new ResourceServerScope(); + scope.setScopeName(item.path("ScopeName").asText()); + scope.setScopeDescription(item.path("ScopeDescription").asText(null)); + scopes.add(scope); + }); + return scopes; + } + + private List readStringList(JsonNode node) { + if (node == null || !node.isArray()) { + return List.of(); + } + + List values = new java.util.ArrayList<>(); + node.forEach(item -> values.add(item.asText())); + return values; + } + private ObjectNode userToNode(CognitoUser u) { ObjectNode node = objectMapper.createObjectNode(); node.put("Username", u.getUsername()); diff --git a/src/main/java/io/github/hectorvent/floci/services/cognito/CognitoOAuthController.java b/src/main/java/io/github/hectorvent/floci/services/cognito/CognitoOAuthController.java new file mode 100644 index 00000000..f131f71b --- /dev/null +++ b/src/main/java/io/github/hectorvent/floci/services/cognito/CognitoOAuthController.java @@ -0,0 +1,158 @@ +package io.github.hectorvent.floci.services.cognito; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.github.hectorvent.floci.core.common.AwsException; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.HeaderParam; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.MultivaluedMap; +import jakarta.ws.rs.core.Response; +import org.jboss.logging.Logger; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.Map; + +@ApplicationScoped +@Path("/") +@Produces(MediaType.APPLICATION_JSON) +public class CognitoOAuthController { + + private static final Logger LOG = Logger.getLogger(CognitoOAuthController.class); + + private final CognitoService cognitoService; + private final ObjectMapper objectMapper; + + @Inject + public CognitoOAuthController(CognitoService cognitoService, ObjectMapper objectMapper) { + this.cognitoService = cognitoService; + this.objectMapper = objectMapper; + } + + @POST + @Path("/cognito-idp/oauth2/token") + @Consumes(MediaType.APPLICATION_FORM_URLENCODED) + public Response token(@HeaderParam("Authorization") String authorization, + MultivaluedMap formParams) { + return issueToken(authorization, formParams); + } + + private Response issueToken(String authorization, MultivaluedMap formParams) { + String grantType = trimToNull(formParams.getFirst("grant_type")); + if (grantType == null) { + return oauthError("invalid_request", "grant_type is required"); + } + if (!"client_credentials".equals(grantType)) { + return oauthError("unsupported_grant_type", "Only client_credentials is supported"); + } + + BasicCredentials basicCredentials; + try { + basicCredentials = parseBasicCredentials(authorization); + } catch (IllegalArgumentException e) { + return oauthError("invalid_request", e.getMessage()); + } + + String bodyClientId = trimToNull(formParams.getFirst("client_id")); + String bodyClientSecret = trimToNull(formParams.getFirst("client_secret")); + String basicClientId = basicCredentials != null ? basicCredentials.clientId() : null; + String basicClientSecret = basicCredentials != null ? basicCredentials.clientSecret() : null; + + if (bodyClientSecret != null && basicClientSecret != null && !bodyClientSecret.equals(basicClientSecret)) { + return oauthError("invalid_request", "client_secret does not match Authorization header"); + } + + if (bodyClientId != null && basicClientId != null && !bodyClientId.equals(basicClientId)) { + return oauthError("invalid_request", "client_id does not match Authorization header"); + } + + String clientId = bodyClientId != null ? bodyClientId : basicClientId; + if (clientId == null) { + return oauthError("invalid_request", "client_id is required"); + } + + String clientSecret = bodyClientSecret != null ? bodyClientSecret : basicClientSecret; + String scope = trimToNull(formParams.getFirst("scope")); + + try { + Map result = cognitoService.issueClientCredentialsToken(clientId, clientSecret, scope); + return Response.ok(objectMapper.valueToTree(result)) + .type(MediaType.APPLICATION_JSON) + .header("Cache-Control", "no-store") + .header("Pragma", "no-cache") + .build(); + } catch (AwsException e) { + if ("ResourceNotFoundException".equals(e.getErrorCode())) { + return oauthError("invalid_client", "Client not found"); + } + if ("InvalidClientException".equals(e.getErrorCode())) { + return oauthError("invalid_client", e.getMessage()); + } + if ("UnauthorizedClientException".equals(e.getErrorCode())) { + return oauthError("unauthorized_client", e.getMessage()); + } + if ("InvalidScopeException".equals(e.getErrorCode())) { + return oauthError("invalid_scope", e.getMessage()); + } + LOG.error("Failed to issue Cognito OAuth token", e); + return oauthError("invalid_request", e.getMessage()); + } + } + + private Response oauthError(String error, String description) { + ObjectNode body = objectMapper.createObjectNode(); + body.put("error", error); + body.put("error_description", description); + return Response.status(400) + .type(MediaType.APPLICATION_JSON) + .header("Cache-Control", "no-store") + .header("Pragma", "no-cache") + .entity(body) + .build(); + } + + private BasicCredentials parseBasicCredentials(String authorization) { + if (authorization == null || authorization.isBlank()) { + return null; + } + if (!authorization.regionMatches(true, 0, "Basic ", 0, 6)) { + return null; + } + + String encoded = authorization.substring(6).trim(); + if (encoded.isEmpty()) { + throw new IllegalArgumentException("Basic Authorization header is malformed"); + } + + try { + String decoded = new String(Base64.getDecoder().decode(encoded), StandardCharsets.UTF_8); + int separator = decoded.indexOf(':'); + if (separator < 0) { + throw new IllegalArgumentException("Basic Authorization header is malformed"); + } + return new BasicCredentials( + trimToNull(decoded.substring(0, separator)), + trimToNull(decoded.substring(separator + 1)) + ); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Basic Authorization header is malformed"); + } + } + + private String trimToNull(String value) { + if (value == null) { + return null; + } + String trimmed = value.trim(); + return trimmed.isEmpty() ? null : trimmed; + } + + private record BasicCredentials(String clientId, String clientSecret) { + } +} diff --git a/src/main/java/io/github/hectorvent/floci/services/cognito/CognitoService.java b/src/main/java/io/github/hectorvent/floci/services/cognito/CognitoService.java index 95e8f4cd..21a3e07c 100644 --- a/src/main/java/io/github/hectorvent/floci/services/cognito/CognitoService.java +++ b/src/main/java/io/github/hectorvent/floci/services/cognito/CognitoService.java @@ -1,20 +1,30 @@ package io.github.hectorvent.floci.services.cognito; import io.github.hectorvent.floci.core.common.AwsException; +import io.github.hectorvent.floci.config.EmulatorConfig; import io.github.hectorvent.floci.core.storage.StorageBackend; import io.github.hectorvent.floci.core.storage.StorageFactory; import com.fasterxml.jackson.core.type.TypeReference; import io.github.hectorvent.floci.services.cognito.model.CognitoUser; +import io.github.hectorvent.floci.services.cognito.model.ResourceServer; +import io.github.hectorvent.floci.services.cognito.model.ResourceServerScope; import io.github.hectorvent.floci.services.cognito.model.UserPool; import io.github.hectorvent.floci.services.cognito.model.UserPoolClient; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import org.jboss.logging.Logger; -import javax.crypto.Mac; -import javax.crypto.spec.SecretKeySpec; import java.nio.charset.StandardCharsets; +import java.security.KeyFactory; +import java.security.KeyPair; +import java.security.KeyPairGenerator; import java.security.MessageDigest; +import java.security.PrivateKey; +import java.security.PublicKey; +import java.security.Signature; +import java.security.interfaces.RSAPublicKey; +import java.security.spec.PKCS8EncodedKeySpec; +import java.security.spec.X509EncodedKeySpec; import java.util.*; @ApplicationScoped @@ -24,16 +34,21 @@ public class CognitoService { private final StorageBackend poolStore; private final StorageBackend clientStore; + private final StorageBackend resourceServerStore; private final StorageBackend userStore; + private final String baseUrl; @Inject - public CognitoService(StorageFactory storageFactory) { + public CognitoService(StorageFactory storageFactory, EmulatorConfig emulatorConfig) { this.poolStore = storageFactory.create("cognito", "cognito-pools.json", new TypeReference>() {}); this.clientStore = storageFactory.create("cognito", "cognito-clients.json", new TypeReference>() {}); + this.resourceServerStore = storageFactory.create("cognito", "cognito-resource-servers.json", + new TypeReference>() {}); this.userStore = storageFactory.create("cognito", "cognito-users.json", new TypeReference>() {}); + this.baseUrl = trimTrailingSlash(emulatorConfig.baseUrl()); } // ──────────────────────────── User Pools ──────────────────────────── @@ -43,14 +58,19 @@ public UserPool createUserPool(String name, String region) { UserPool pool = new UserPool(); pool.setId(id); pool.setName(name); + ensureJwtSigningKeys(pool); poolStore.put(id, pool); LOG.infov("Created User Pool: {0}", id); return pool; } public UserPool describeUserPool(String id) { - return poolStore.get(id) + UserPool pool = poolStore.get(id) .orElseThrow(() -> new AwsException("ResourceNotFoundException", "User pool not found", 404)); + if (ensureJwtSigningKeys(pool)) { + poolStore.put(id, pool); + } + return pool; } public List listUserPools() { @@ -63,13 +83,23 @@ public void deleteUserPool(String id) { // ──────────────────────────── User Pool Clients ──────────────────────────── - public UserPoolClient createUserPoolClient(String userPoolId, String clientName) { + public UserPoolClient createUserPoolClient(String userPoolId, String clientName, boolean generateSecret, + boolean allowedOAuthFlowsUserPoolClient, + List allowedOAuthFlows, + List allowedOAuthScopes) { describeUserPool(userPoolId); String clientId = UUID.randomUUID().toString().replace("-", "").substring(0, 26); UserPoolClient client = new UserPoolClient(); client.setClientId(clientId); client.setUserPoolId(userPoolId); client.setClientName(clientName); + client.setGenerateSecret(generateSecret); + client.setAllowedOAuthFlowsUserPoolClient(allowedOAuthFlowsUserPoolClient); + client.setAllowedOAuthFlows(normalizeStringList(allowedOAuthFlows)); + client.setAllowedOAuthScopes(normalizeStringList(allowedOAuthScopes)); + if (generateSecret) { + client.setClientSecret(generateSecretValue()); + } clientStore.put(clientId, client); LOG.infov("Created User Pool Client: {0} for pool {1}", clientId, userPoolId); return client; @@ -93,6 +123,69 @@ public void deleteUserPoolClient(String userPoolId, String clientId) { clientStore.delete(clientId); } + // ──────────────────────────── Resource Servers ──────────────────────────── + + public ResourceServer createResourceServer(String userPoolId, String identifier, String name, + List scopes) { + describeUserPool(userPoolId); + if (identifier == null || identifier.isBlank()) { + throw new AwsException("InvalidParameterException", "Identifier is required", 400); + } + if (name == null || name.isBlank()) { + throw new AwsException("InvalidParameterException", "Name is required", 400); + } + + String key = resourceServerKey(userPoolId, identifier); + if (resourceServerStore.get(key).isPresent()) { + throw new AwsException("ResourceConflictException", "Resource server already exists", 400); + } + + ResourceServer server = new ResourceServer(); + server.setUserPoolId(userPoolId); + server.setIdentifier(identifier); + server.setName(name); + server.setScopes(normalizeScopes(scopes)); + resourceServerStore.put(key, server); + return server; + } + + public ResourceServer describeResourceServer(String userPoolId, String identifier) { + describeUserPool(userPoolId); + return resourceServerStore.get(resourceServerKey(userPoolId, identifier)) + .orElseThrow(() -> new AwsException("ResourceNotFoundException", "Resource server not found", 404)); + } + + public List listResourceServers(String userPoolId) { + describeUserPool(userPoolId); + String prefix = userPoolId + "::"; + return resourceServerStore.scan(k -> k.startsWith(prefix)); + } + + public ResourceServer updateResourceServer(String userPoolId, String identifier, String name, + List scopes) { + if (userPoolId == null || userPoolId.isBlank()) { + throw new AwsException("InvalidParameterException", "UserPoolId is required", 400); + } + if (identifier == null || identifier.isBlank()) { + throw new AwsException("InvalidParameterException", "Identifier is required", 400); + } + if (name == null || name.isBlank()) { + throw new AwsException("InvalidParameterException", "Name is required", 400); + } + + ResourceServer server = describeResourceServer(userPoolId, identifier); + server.setName(name); + server.setScopes(normalizeScopes(scopes)); + server.setLastModifiedDate(System.currentTimeMillis() / 1000L); + resourceServerStore.put(resourceServerKey(userPoolId, identifier), server); + return server; + } + + public void deleteResourceServer(String userPoolId, String identifier) { + describeResourceServer(userPoolId, identifier); + resourceServerStore.delete(resourceServerKey(userPoolId, identifier)); + } + // ──────────────────────────── Users ──────────────────────────── public CognitoUser adminCreateUser(String userPoolId, String username, Map attributes, @@ -311,6 +404,33 @@ public void updateUserAttributes(String accessToken, Map attribu adminUpdateUserAttributes(poolId, username, attributes); } + public Map issueClientCredentialsToken(String clientId, String clientSecret, String scope) { + UserPoolClient client = clientStore.get(clientId) + .orElseThrow(() -> new AwsException("ResourceNotFoundException", "Client not found", 404)); + UserPool pool = describeUserPool(client.getUserPoolId()); + validateClientAllowsClientCredentials(client); + validateClientSecret(client, clientSecret); + String normalizedScope = resolveAuthorizedScopes(client, pool.getId(), scope); + + Map response = new LinkedHashMap<>(); + response.put("access_token", generateClientAccessToken(client, pool, normalizedScope)); + response.put("token_type", "Bearer"); + response.put("expires_in", 3600); + return response; + } + + public String getIssuer(String poolId) { + return baseUrl + "/" + poolId; + } + + public String getJwksUri(String poolId) { + return getIssuer(poolId) + "/.well-known/jwks.json"; + } + + public String getTokenEndpoint() { + return baseUrl + "/cognito-idp/oauth2/token"; + } + // ──────────────────────────── Private helpers ──────────────────────────── private Map authenticateWithPassword(UserPool pool, Map params, String clientId) { @@ -385,7 +505,9 @@ private Map generateAuthResult(CognitoUser user, UserPool pool) } private String generateSignedJwt(CognitoUser user, UserPool pool, String type) { - String headerJson = "{\"alg\":\"HS256\",\"typ\":\"JWT\"}"; + String headerJson = String.format( + "{\"alg\":\"RS256\",\"typ\":\"JWT\",\"kid\":\"%s\"}", + escapeJson(getSigningKeyId(pool))); String header = Base64.getUrlEncoder().withoutPadding() .encodeToString(headerJson.getBytes(StandardCharsets.UTF_8)); @@ -393,48 +515,265 @@ private String generateSignedJwt(CognitoUser user, UserPool pool, String type) { String email = user.getAttributes().getOrDefault("email", user.getUsername()); String payloadJson = String.format( "{\"sub\":\"%s\",\"event_id\":\"%s\",\"token_use\":\"%s\",\"auth_time\":%d," + - "\"iss\":\"https://cognito-idp.local/%s\",\"exp\":%d,\"iat\":%d," + + "\"iss\":\"%s\",\"exp\":%d,\"iat\":%d," + "\"username\":\"%s\",\"email\":\"%s\",\"cognito:username\":\"%s\"}", UUID.randomUUID(), UUID.randomUUID(), type, now, - pool.getId(), now + 3600, now, + escapeJson(getIssuer(pool.getId())), now + 3600, now, user.getUsername(), email, user.getUsername() ); String payload = Base64.getUrlEncoder().withoutPadding() .encodeToString(payloadJson.getBytes(StandardCharsets.UTF_8)); - - String signingInput = header + "." + payload; - String signature = hmacSha256(signingInput, pool.getSigningSecret()); - return signingInput + "." + signature; + return signJwt(header, payload, getSigningPrivateKey(pool)); } private String generateTokenString(String type, String username, UserPool pool) { long now = System.currentTimeMillis() / 1000L; - String headerJson = "{\"alg\":\"HS256\",\"typ\":\"JWT\"}"; + String headerJson = String.format( + "{\"alg\":\"RS256\",\"typ\":\"JWT\",\"kid\":\"%s\"}", + escapeJson(getSigningKeyId(pool))); String header = Base64.getUrlEncoder().withoutPadding() .encodeToString(headerJson.getBytes(StandardCharsets.UTF_8)); String payloadJson = String.format( - "{\"sub\":\"%s\",\"token_use\":\"%s\",\"iss\":\"https://cognito-idp.local/%s\"," + + "{\"sub\":\"%s\",\"token_use\":\"%s\",\"iss\":\"%s\"," + "\"exp\":%d,\"iat\":%d,\"username\":\"%s\"}", - UUID.randomUUID(), type, pool.getId(), now + 3600, now, username + UUID.randomUUID(), type, escapeJson(getIssuer(pool.getId())), now + 3600, now, username ); String payload = Base64.getUrlEncoder().withoutPadding() .encodeToString(payloadJson.getBytes(StandardCharsets.UTF_8)); + return signJwt(header, payload, getSigningPrivateKey(pool)); + } + + private String generateClientAccessToken(UserPoolClient client, UserPool pool, String scope) { + String headerJson = String.format( + "{\"alg\":\"RS256\",\"typ\":\"JWT\",\"kid\":\"%s\"}", + escapeJson(getSigningKeyId(pool))); + String header = Base64.getUrlEncoder().withoutPadding() + .encodeToString(headerJson.getBytes(StandardCharsets.UTF_8)); + + long now = System.currentTimeMillis() / 1000L; + StringBuilder payloadJson = new StringBuilder(); + payloadJson.append("{") + .append("\"iss\":\"").append(escapeJson(getIssuer(pool.getId()))).append("\",") + .append("\"version\":2,") + .append("\"sub\":\"").append(escapeJson(client.getClientId())).append("\",") + .append("\"client_id\":\"").append(escapeJson(client.getClientId())).append("\",") + .append("\"token_use\":\"access\",") + .append("\"exp\":").append(now + 3600).append(",") + .append("\"iat\":").append(now).append(",") + .append("\"jti\":\"").append(UUID.randomUUID()).append("\""); + if (scope != null && !scope.isBlank()) { + payloadJson.append(",\"scope\":\"").append(escapeJson(scope)).append("\""); + } + payloadJson.append("}"); + + String payload = Base64.getUrlEncoder().withoutPadding() + .encodeToString(payloadJson.toString().getBytes(StandardCharsets.UTF_8)); + return signJwt(header, payload, getSigningPrivateKey(pool)); + } + + private void validateClientSecret(UserPoolClient client, String clientSecret) { + String expectedSecret = client.getClientSecret(); + if (expectedSecret == null || expectedSecret.isBlank() || !client.isGenerateSecret()) { + throw new AwsException("InvalidClientException", "Client must have a secret for client_credentials", 400); + } + if (clientSecret == null || clientSecret.isBlank()) { + throw new AwsException("InvalidClientException", "Client secret is required", 400); + } + if (!expectedSecret.equals(clientSecret)) { + throw new AwsException("InvalidClientException", "Client secret is invalid", 400); + } + } + + private void validateClientAllowsClientCredentials(UserPoolClient client) { + if (!client.isAllowedOAuthFlowsUserPoolClient()) { + throw new AwsException("UnauthorizedClientException", "Client is not enabled for OAuth flows", 400); + } + if (!client.getAllowedOAuthFlows().contains("client_credentials")) { + throw new AwsException("UnauthorizedClientException", "Client is not allowed to use client_credentials", 400); + } + } + + private String resolveAuthorizedScopes(UserPoolClient client, String userPoolId, String requestedScope) { + List allowedScopes = normalizeStringList(client.getAllowedOAuthScopes()); + if (allowedScopes.isEmpty()) { + throw new AwsException("InvalidScopeException", "Client has no allowed OAuth scopes", 400); + } + + List effectiveScopes; + if (requestedScope == null || requestedScope.isBlank()) { + effectiveScopes = allowedScopes; + } else { + effectiveScopes = Arrays.asList(normalizeRequestedScope(requestedScope).split(" ")); + for (String scope : effectiveScopes) { + if (!allowedScopes.contains(scope)) { + throw new AwsException("InvalidScopeException", "Scope is not allowed for this client: " + scope, 400); + } + } + } + + Set validCustomScopes = new HashSet<>(); + for (ResourceServer server : listResourceServers(userPoolId)) { + for (ResourceServerScope serverScope : server.getScopes()) { + validCustomScopes.add(server.getIdentifier() + "/" + serverScope.getScopeName()); + } + } + + for (String scope : effectiveScopes) { + if (isBuiltInScope(scope)) { + continue; + } + if (!validCustomScopes.contains(scope)) { + throw new AwsException("InvalidScopeException", "Scope is invalid: " + scope, 400); + } + } + + return String.join(" ", effectiveScopes); + } + + private String normalizeRequestedScope(String scope) { + if (scope == null || scope.isBlank()) { + return null; + } + + List normalized = new ArrayList<>(); + for (String part : scope.trim().split("\\s+")) { + if (!part.isBlank()) { + normalized.add(part); + } + } + return normalized.isEmpty() ? null : String.join(" ", normalized); + } + + private List normalizeScopes(List scopes) { + if (scopes == null || scopes.isEmpty()) { + return List.of(); + } + + List normalized = new ArrayList<>(); + Set scopeNames = new HashSet<>(); + for (ResourceServerScope scope : scopes) { + if (scope == null || scope.getScopeName() == null || scope.getScopeName().isBlank()) { + throw new AwsException("InvalidParameterException", "ScopeName is required", 400); + } + if (!scopeNames.add(scope.getScopeName())) { + throw new AwsException("InvalidParameterException", "Duplicate scope name: " + scope.getScopeName(), 400); + } + ResourceServerScope normalizedScope = new ResourceServerScope(); + normalizedScope.setScopeName(scope.getScopeName()); + normalizedScope.setScopeDescription(scope.getScopeDescription()); + normalized.add(normalizedScope); + } + return normalized; + } + + private List normalizeStringList(List values) { + if (values == null || values.isEmpty()) { + return List.of(); + } + + List normalized = new ArrayList<>(); + Set seen = new LinkedHashSet<>(); + for (String value : values) { + if (value == null) { + continue; + } + String trimmed = value.trim(); + if (!trimmed.isEmpty() && seen.add(trimmed)) { + normalized.add(trimmed); + } + } + return normalized; + } + + private boolean isBuiltInScope(String scope) { + return switch (scope) { + case "phone", "email", "openid", "profile", "aws.cognito.signin.user.admin" -> true; + default -> false; + }; + } + + private String signJwt(String header, String payload, PrivateKey signingKey) { String signingInput = header + "." + payload; - String signature = hmacSha256(signingInput, pool.getSigningSecret()); + String signature = rsaSha256(signingInput, signingKey); return signingInput + "." + signature; } - private String hmacSha256(String data, String key) { + private String rsaSha256(String data, PrivateKey signingKey) { try { - Mac mac = Mac.getInstance("HmacSHA256"); - mac.init(new SecretKeySpec(key.getBytes(StandardCharsets.UTF_8), "HmacSHA256")); - byte[] sig = mac.doFinal(data.getBytes(StandardCharsets.UTF_8)); + Signature signature = Signature.getInstance("SHA256withRSA"); + signature.initSign(signingKey); + signature.update(data.getBytes(StandardCharsets.UTF_8)); + byte[] sig = signature.sign(); return Base64.getUrlEncoder().withoutPadding().encodeToString(sig); } catch (Exception e) { throw new RuntimeException("JWT signing failed", e); } } + String getSigningKeyId(UserPool pool) { + ensureJwtSigningKeys(pool); + return pool.getSigningKeyId(); + } + + RSAPublicKey getSigningPublicKey(UserPool pool) { + ensureJwtSigningKeys(pool); + + try { + byte[] encoded = Base64.getDecoder().decode(pool.getSigningPublicKey()); + X509EncodedKeySpec keySpec = new X509EncodedKeySpec(encoded); + PublicKey publicKey = KeyFactory.getInstance("RSA").generatePublic(keySpec); + return (RSAPublicKey) publicKey; + } catch (Exception e) { + throw new RuntimeException("Failed to load Cognito RSA public key", e); + } + } + + private PrivateKey getSigningPrivateKey(UserPool pool) { + ensureJwtSigningKeys(pool); + + try { + byte[] encoded = Base64.getDecoder().decode(pool.getSigningPrivateKey()); + PKCS8EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(encoded); + return KeyFactory.getInstance("RSA").generatePrivate(keySpec); + } catch (Exception e) { + throw new RuntimeException("Failed to load Cognito RSA private key", e); + } + } + + private boolean ensureJwtSigningKeys(UserPool pool) { + synchronized (pool) { + boolean changed = false; + + if (pool.getSigningKeyId() == null || pool.getSigningKeyId().isBlank()) { + pool.setSigningKeyId(pool.getId()); + changed = true; + } + + if (pool.getSigningPrivateKey() == null || pool.getSigningPrivateKey().isBlank() + || pool.getSigningPublicKey() == null || pool.getSigningPublicKey().isBlank()) { + try { + KeyPairGenerator generator = KeyPairGenerator.getInstance("RSA"); + generator.initialize(2048); + KeyPair keyPair = generator.generateKeyPair(); + + pool.setSigningPrivateKey( + Base64.getEncoder().encodeToString(keyPair.getPrivate().getEncoded())); + pool.setSigningPublicKey( + Base64.getEncoder().encodeToString(keyPair.getPublic().getEncoded())); + changed = true; + } catch (Exception e) { + throw new RuntimeException("Failed to generate Cognito RSA signing keypair", e); + } + } + + if (changed && pool.getId() != null) { + pool.setLastModifiedDate(System.currentTimeMillis() / 1000L); + } + + return changed; + } + } + String hashPassword(String password) { try { MessageDigest digest = MessageDigest.getInstance("SHA-256"); @@ -473,7 +812,6 @@ private String extractPoolIdFromToken(String token) { String payloadJson = new String(Base64.getUrlDecoder().decode(parts[1]), StandardCharsets.UTF_8); String iss = extractJsonField(payloadJson, "iss"); if (iss == null) return null; - // iss = "https://cognito-idp.local/POOL_ID" int lastSlash = iss.lastIndexOf('/'); return lastSlash >= 0 ? iss.substring(lastSlash + 1) : null; } catch (Exception e) { @@ -494,4 +832,26 @@ private String extractJsonField(String json, String field) { private String userKey(String poolId, String username) { return poolId + "::" + username; } + + private String resourceServerKey(String userPoolId, String identifier) { + return userPoolId + "::" + identifier; + } + + private String escapeJson(String value) { + return value + .replace("\\", "\\\\") + .replace("\"", "\\\""); + } + + private String generateSecretValue() { + return UUID.randomUUID().toString().replace("-", "") + + UUID.randomUUID().toString().replace("-", ""); + } + + private String trimTrailingSlash(String value) { + if (value.endsWith("/")) { + return value.substring(0, value.length() - 1); + } + return value; + } } diff --git a/src/main/java/io/github/hectorvent/floci/services/cognito/CognitoWellKnownController.java b/src/main/java/io/github/hectorvent/floci/services/cognito/CognitoWellKnownController.java index d8eacb0c..9f60676b 100644 --- a/src/main/java/io/github/hectorvent/floci/services/cognito/CognitoWellKnownController.java +++ b/src/main/java/io/github/hectorvent/floci/services/cognito/CognitoWellKnownController.java @@ -9,7 +9,7 @@ import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; -import java.nio.charset.StandardCharsets; +import java.math.BigInteger; import java.util.Base64; /** @@ -32,14 +32,36 @@ public CognitoWellKnownController(CognitoService cognitoService) { @Path("/{poolId}/.well-known/jwks.json") public Response getJwks(@PathParam("poolId") String poolId) { UserPool pool = cognitoService.describeUserPool(poolId); - String kid = pool.getId(); - // Encode signing secret bytes as Base64URL (no padding) for the JWK "k" parameter - byte[] secretBytes = pool.getSigningSecret().getBytes(StandardCharsets.UTF_8); - String k = Base64.getUrlEncoder().withoutPadding().encodeToString(secretBytes); + String kid = cognitoService.getSigningKeyId(pool); + var publicKey = cognitoService.getSigningPublicKey(pool); + String modulus = base64UrlEncodeUnsigned(publicKey.getModulus()); + String exponent = base64UrlEncodeUnsigned(publicKey.getPublicExponent()); String body = """ - {"keys":[{"kty":"oct","kid":"%s","alg":"HS256","k":"%s","use":"sig"}]} - """.formatted(kid, k).strip(); + {"keys":[{"kty":"RSA","kid":"%s","alg":"RS256","n":"%s","e":"%s","use":"sig"}]} + """.formatted(kid, modulus, exponent).strip(); return Response.ok(body).build(); } + + @GET + @Path("/{poolId}/.well-known/openid-configuration") + public Response getOpenIdConfiguration(@PathParam("poolId") String poolId) { + UserPool pool = cognitoService.describeUserPool(poolId); + String issuer = cognitoService.getIssuer(pool.getId()); + String jwksUri = cognitoService.getJwksUri(pool.getId()); + String tokenEndpoint = cognitoService.getTokenEndpoint(); + + String body = """ + {"issuer":"%s","jwks_uri":"%s","token_endpoint":"%s","subject_types_supported":["public"],"response_types_supported":[],"grant_types_supported":["client_credentials"],"token_endpoint_auth_methods_supported":["client_secret_basic","client_secret_post"],"id_token_signing_alg_values_supported":["RS256"]} + """.formatted(issuer, jwksUri, tokenEndpoint).strip(); + return Response.ok(body).build(); + } + + private String base64UrlEncodeUnsigned(BigInteger value) { + byte[] bytes = value.toByteArray(); + if (bytes.length > 1 && bytes[0] == 0) { + bytes = java.util.Arrays.copyOfRange(bytes, 1, bytes.length); + } + return Base64.getUrlEncoder().withoutPadding().encodeToString(bytes); + } } diff --git a/src/main/java/io/github/hectorvent/floci/services/cognito/model/ResourceServer.java b/src/main/java/io/github/hectorvent/floci/services/cognito/model/ResourceServer.java new file mode 100644 index 00000000..0e61f7c2 --- /dev/null +++ b/src/main/java/io/github/hectorvent/floci/services/cognito/model/ResourceServer.java @@ -0,0 +1,42 @@ +package io.github.hectorvent.floci.services.cognito.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import io.quarkus.runtime.annotations.RegisterForReflection; + +import java.util.ArrayList; +import java.util.List; + +@RegisterForReflection +@JsonIgnoreProperties(ignoreUnknown = true) +public class ResourceServer { + private String userPoolId; + private String identifier; + private String name; + private List scopes = new ArrayList<>(); + private long creationDate; + private long lastModifiedDate; + + public ResourceServer() { + long now = System.currentTimeMillis() / 1000L; + this.creationDate = now; + this.lastModifiedDate = now; + } + + public String getUserPoolId() { return userPoolId; } + public void setUserPoolId(String userPoolId) { this.userPoolId = userPoolId; } + + public String getIdentifier() { return identifier; } + public void setIdentifier(String identifier) { this.identifier = identifier; } + + public String getName() { return name; } + public void setName(String name) { this.name = name; } + + public List getScopes() { return scopes; } + public void setScopes(List scopes) { this.scopes = scopes; } + + public long getCreationDate() { return creationDate; } + public void setCreationDate(long creationDate) { this.creationDate = creationDate; } + + public long getLastModifiedDate() { return lastModifiedDate; } + public void setLastModifiedDate(long lastModifiedDate) { this.lastModifiedDate = lastModifiedDate; } +} diff --git a/src/main/java/io/github/hectorvent/floci/services/cognito/model/ResourceServerScope.java b/src/main/java/io/github/hectorvent/floci/services/cognito/model/ResourceServerScope.java new file mode 100644 index 00000000..28aa94c9 --- /dev/null +++ b/src/main/java/io/github/hectorvent/floci/services/cognito/model/ResourceServerScope.java @@ -0,0 +1,17 @@ +package io.github.hectorvent.floci.services.cognito.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import io.quarkus.runtime.annotations.RegisterForReflection; + +@RegisterForReflection +@JsonIgnoreProperties(ignoreUnknown = true) +public class ResourceServerScope { + private String scopeName; + private String scopeDescription; + + public String getScopeName() { return scopeName; } + public void setScopeName(String scopeName) { this.scopeName = scopeName; } + + public String getScopeDescription() { return scopeDescription; } + public void setScopeDescription(String scopeDescription) { this.scopeDescription = scopeDescription; } +} diff --git a/src/main/java/io/github/hectorvent/floci/services/cognito/model/UserPool.java b/src/main/java/io/github/hectorvent/floci/services/cognito/model/UserPool.java index b8e9db29..2518f59a 100644 --- a/src/main/java/io/github/hectorvent/floci/services/cognito/model/UserPool.java +++ b/src/main/java/io/github/hectorvent/floci/services/cognito/model/UserPool.java @@ -9,6 +9,9 @@ public class UserPool { private String id; private String name; private String signingSecret; + private String signingKeyId; + private String signingPublicKey; + private String signingPrivateKey; private long creationDate; private long lastModifiedDate; @@ -28,6 +31,15 @@ public UserPool() { public String getSigningSecret() { return signingSecret; } public void setSigningSecret(String signingSecret) { this.signingSecret = signingSecret; } + public String getSigningKeyId() { return signingKeyId; } + public void setSigningKeyId(String signingKeyId) { this.signingKeyId = signingKeyId; } + + public String getSigningPublicKey() { return signingPublicKey; } + public void setSigningPublicKey(String signingPublicKey) { this.signingPublicKey = signingPublicKey; } + + public String getSigningPrivateKey() { return signingPrivateKey; } + public void setSigningPrivateKey(String signingPrivateKey) { this.signingPrivateKey = signingPrivateKey; } + public long getCreationDate() { return creationDate; } public void setCreationDate(long creationDate) { this.creationDate = creationDate; } diff --git a/src/main/java/io/github/hectorvent/floci/services/cognito/model/UserPoolClient.java b/src/main/java/io/github/hectorvent/floci/services/cognito/model/UserPoolClient.java index 32e8b7bb..dee2bf88 100644 --- a/src/main/java/io/github/hectorvent/floci/services/cognito/model/UserPoolClient.java +++ b/src/main/java/io/github/hectorvent/floci/services/cognito/model/UserPoolClient.java @@ -3,12 +3,20 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import io.quarkus.runtime.annotations.RegisterForReflection; +import java.util.ArrayList; +import java.util.List; + @RegisterForReflection @JsonIgnoreProperties(ignoreUnknown = true) public class UserPoolClient { private String clientId; private String userPoolId; private String clientName; + private String clientSecret; + private boolean generateSecret; + private boolean allowedOAuthFlowsUserPoolClient; + private List allowedOAuthFlows = new ArrayList<>(); + private List allowedOAuthScopes = new ArrayList<>(); private long creationDate; private long lastModifiedDate; @@ -27,6 +35,23 @@ public UserPoolClient() { public String getClientName() { return clientName; } public void setClientName(String clientName) { this.clientName = clientName; } + public String getClientSecret() { return clientSecret; } + public void setClientSecret(String clientSecret) { this.clientSecret = clientSecret; } + + public boolean isGenerateSecret() { return generateSecret; } + public void setGenerateSecret(boolean generateSecret) { this.generateSecret = generateSecret; } + + public boolean isAllowedOAuthFlowsUserPoolClient() { return allowedOAuthFlowsUserPoolClient; } + public void setAllowedOAuthFlowsUserPoolClient(boolean allowedOAuthFlowsUserPoolClient) { + this.allowedOAuthFlowsUserPoolClient = allowedOAuthFlowsUserPoolClient; + } + + public List getAllowedOAuthFlows() { return allowedOAuthFlows; } + public void setAllowedOAuthFlows(List allowedOAuthFlows) { this.allowedOAuthFlows = allowedOAuthFlows; } + + public List getAllowedOAuthScopes() { return allowedOAuthScopes; } + public void setAllowedOAuthScopes(List allowedOAuthScopes) { this.allowedOAuthScopes = allowedOAuthScopes; } + public long getCreationDate() { return creationDate; } public void setCreationDate(long creationDate) { this.creationDate = creationDate; } diff --git a/src/test/java/io/github/hectorvent/floci/services/cognito/CognitoIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/cognito/CognitoIntegrationTest.java new file mode 100644 index 00000000..53f8ce59 --- /dev/null +++ b/src/test/java/io/github/hectorvent/floci/services/cognito/CognitoIntegrationTest.java @@ -0,0 +1,230 @@ +package io.github.hectorvent.floci.services.cognito; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.quarkus.test.junit.QuarkusTest; +import io.restassured.RestAssured; +import io.restassured.config.EncoderConfig; +import io.restassured.http.ContentType; +import io.restassured.response.Response; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; + +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.security.KeyFactory; +import java.security.PublicKey; +import java.security.Signature; +import java.security.spec.RSAPublicKeySpec; +import java.util.Base64; +import java.util.UUID; + +import static io.restassured.RestAssured.given; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@QuarkusTest +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +class CognitoIntegrationTest { + + private static final String COGNITO_CONTENT_TYPE = "application/x-amz-json-1.1"; + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private static String poolId; + private static String clientId; + private static final String username = "alice+" + UUID.randomUUID() + "@example.com"; + private static final String password = "Perm1234!"; + + @BeforeAll + static void configureRestAssured() { + RestAssured.config = RestAssured.config().encoderConfig( + EncoderConfig.encoderConfig() + .encodeContentTypeAs(COGNITO_CONTENT_TYPE, ContentType.TEXT)); + } + + @Test + @Order(1) + void createPoolClientAndUser() throws Exception { + JsonNode poolResponse = cognitoJson("CreateUserPool", """ + { + "PoolName": "JwtPool" + } + """); + poolId = poolResponse.path("UserPool").path("Id").asText(); + + JsonNode clientResponse = cognitoJson("CreateUserPoolClient", """ + { + "UserPoolId": "%s", + "ClientName": "jwt-client" + } + """.formatted(poolId)); + clientId = clientResponse.path("UserPoolClient").path("ClientId").asText(); + + cognitoAction("AdminCreateUser", """ + { + "UserPoolId": "%s", + "Username": "%s", + "UserAttributes": [ + { "Name": "email", "Value": "%s" } + ] + } + """.formatted(poolId, username, username)) + .then() + .statusCode(200); + + cognitoAction("AdminSetUserPassword", """ + { + "UserPoolId": "%s", + "Username": "%s", + "Password": "%s", + "Permanent": true + } + """.formatted(poolId, username, password)) + .then() + .statusCode(200); + } + + @Test + @Order(2) + void initiateAuthReturnsAuthenticationResult() { + cognitoAction("InitiateAuth", """ + { + "ClientId": "%s", + "AuthFlow": "USER_PASSWORD_AUTH", + "AuthParameters": { + "USERNAME": "%s", + "PASSWORD": "%s" + } + } + """.formatted(clientId, username, password)) + .then() + .statusCode(200) + .body("AuthenticationResult.AccessToken", org.hamcrest.Matchers.notNullValue()) + .body("AuthenticationResult.IdToken", org.hamcrest.Matchers.notNullValue()) + .body("AuthenticationResult.RefreshToken", org.hamcrest.Matchers.notNullValue()); + } + + @Test + @Order(3) + void authTokensAreSignedWithPublishedRsaJwksKey() throws Exception { + Response authResponse = cognitoAction("InitiateAuth", """ + { + "ClientId": "%s", + "AuthFlow": "USER_PASSWORD_AUTH", + "AuthParameters": { + "USERNAME": "%s", + "PASSWORD": "%s" + } + } + """.formatted(clientId, username, password)); + + authResponse.then().statusCode(200); + + String accessToken = authResponse.jsonPath().getString("AuthenticationResult.AccessToken"); + JsonNode header = decodeJwtHeader(accessToken); + JsonNode payload = decodeJwtPayload(accessToken); + assertEquals("RS256", header.path("alg").asText()); + assertEquals(poolId, header.path("kid").asText()); + assertEquals("http://localhost:4566/" + poolId, payload.path("iss").asText()); + assertEquals(username, payload.path("username").asText()); + assertEquals("access", payload.path("token_use").asText()); + + String jwksResponse = given() + .when() + .get("/" + poolId + "/.well-known/jwks.json") + .then() + .statusCode(200) + .extract() + .asString(); + + JsonNode jwks = OBJECT_MAPPER.readTree(jwksResponse); + JsonNode key = jwks.path("keys").get(0); + assertNotNull(key); + assertEquals("RSA", key.path("kty").asText()); + assertEquals("RS256", key.path("alg").asText()); + assertEquals("sig", key.path("use").asText()); + assertEquals(poolId, key.path("kid").asText()); + assertTrue(key.hasNonNull("n")); + assertTrue(key.hasNonNull("e")); + assertTrue(verifyJwtSignature(accessToken, key)); + } + + @Test + @Order(4) + void openIdConfigurationPublishesIssuerAndJwksUri() throws Exception { + String openIdResponse = given() + .when() + .get("/" + poolId + "/.well-known/openid-configuration") + .then() + .statusCode(200) + .extract() + .asString(); + + JsonNode document = OBJECT_MAPPER.readTree(openIdResponse); + assertEquals("http://localhost:4566/" + poolId, document.path("issuer").asText()); + assertEquals( + "http://localhost:4566/" + poolId + "/.well-known/jwks.json", + document.path("jwks_uri").asText()); + assertEquals("public", document.path("subject_types_supported").get(0).asText()); + assertEquals("RS256", document.path("id_token_signing_alg_values_supported").get(0).asText()); + } + + private static Response cognitoAction(String action, String body) { + return given() + .header("X-Amz-Target", "AWSCognitoIdentityProviderService." + action) + .contentType(COGNITO_CONTENT_TYPE) + .body(body) + .when() + .post("/"); + } + + private static JsonNode cognitoJson(String action, String body) throws Exception { + String response = cognitoAction(action, body) + .then() + .statusCode(200) + .extract() + .asString(); + return OBJECT_MAPPER.readTree(response); + } + + private static JsonNode decodeJwtPayload(String token) throws Exception { + return decodeJwtPart(token, 1); + } + + private static JsonNode decodeJwtHeader(String token) throws Exception { + return decodeJwtPart(token, 0); + } + + private static JsonNode decodeJwtPart(String token, int partIndex) throws Exception { + String[] parts = token.split("\\."); + assertEquals(3, parts.length); + return OBJECT_MAPPER.readTree(Base64.getUrlDecoder().decode(padBase64(parts[partIndex]))); + } + + private static boolean verifyJwtSignature(String token, JsonNode jwk) throws Exception { + String[] parts = token.split("\\."); + assertEquals(3, parts.length); + + BigInteger modulus = new BigInteger(1, Base64.getUrlDecoder().decode(padBase64(jwk.path("n").asText()))); + BigInteger exponent = new BigInteger(1, Base64.getUrlDecoder().decode(padBase64(jwk.path("e").asText()))); + RSAPublicKeySpec keySpec = new RSAPublicKeySpec(modulus, exponent); + PublicKey publicKey = KeyFactory.getInstance("RSA").generatePublic(keySpec); + + Signature signature = Signature.getInstance("SHA256withRSA"); + signature.initVerify(publicKey); + signature.update((parts[0] + "." + parts[1]).getBytes(StandardCharsets.UTF_8)); + return signature.verify(Base64.getUrlDecoder().decode(padBase64(parts[2]))); + } + + private static String padBase64(String value) { + int remainder = value.length() % 4; + if (remainder == 0) { + return value; + } + return value + "=".repeat(4 - remainder); + } +} diff --git a/src/test/java/io/github/hectorvent/floci/services/cognito/CognitoOAuthTokenIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/cognito/CognitoOAuthTokenIntegrationTest.java new file mode 100644 index 00000000..795ee045 --- /dev/null +++ b/src/test/java/io/github/hectorvent/floci/services/cognito/CognitoOAuthTokenIntegrationTest.java @@ -0,0 +1,557 @@ +package io.github.hectorvent.floci.services.cognito; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.quarkus.test.junit.QuarkusTest; +import io.restassured.RestAssured; +import io.restassured.config.EncoderConfig; +import io.restassured.http.ContentType; +import io.restassured.response.Response; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; + +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.security.KeyFactory; +import java.security.PublicKey; +import java.security.Signature; +import java.security.spec.RSAPublicKeySpec; +import java.util.Base64; + +import static io.restassured.RestAssured.given; +import static org.hamcrest.Matchers.equalTo; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@QuarkusTest +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +class CognitoOAuthTokenIntegrationTest { + + private static final String COGNITO_CONTENT_TYPE = "application/x-amz-json-1.1"; + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private static String poolId; + private static String clientId; + private static String limitedClientId; + private static String confidentialClientId; + private static String confidentialClientSecret; + + @BeforeAll + static void configureRestAssured() { + RestAssured.config = RestAssured.config().encoderConfig( + EncoderConfig.encoderConfig() + .encodeContentTypeAs(COGNITO_CONTENT_TYPE, ContentType.TEXT)); + } + + @Test + @Order(1) + void createPoolAndClients() throws Exception { + JsonNode poolResponse = cognitoJson("CreateUserPool", """ + { + "PoolName": "OAuthPool" + } + """); + poolId = poolResponse.path("UserPool").path("Id").asText(); + + JsonNode clientResponse = cognitoJson("CreateUserPoolClient", """ + { + "UserPoolId": "%s", + "ClientName": "oauth-client" + } + """.formatted(poolId)); + clientId = clientResponse.path("UserPoolClient").path("ClientId").asText(); + + JsonNode confidentialClientResponse = cognitoJson("CreateUserPoolClient", """ + { + "UserPoolId": "%s", + "ClientName": "confidential-oauth-client", + "GenerateSecret": true, + "AllowedOAuthFlowsUserPoolClient": true, + "AllowedOAuthFlows": ["client_credentials"], + "AllowedOAuthScopes": ["notes/read", "notes/write"] + } + """.formatted(poolId)); + confidentialClientId = confidentialClientResponse.path("UserPoolClient").path("ClientId").asText(); + confidentialClientSecret = confidentialClientResponse.path("UserPoolClient").path("ClientSecret").asText(); + + JsonNode limitedClientResponse = cognitoJson("CreateUserPoolClient", """ + { + "UserPoolId": "%s", + "ClientName": "limited-oauth-client", + "GenerateSecret": true, + "AllowedOAuthFlowsUserPoolClient": true, + "AllowedOAuthFlows": ["client_credentials"], + "AllowedOAuthScopes": ["notes/read"] + } + """.formatted(poolId)); + limitedClientId = limitedClientResponse.path("UserPoolClient").path("ClientId").asText(); + + JsonNode resourceServerResponse = cognitoJson("CreateResourceServer", """ + { + "UserPoolId": "%s", + "Identifier": "notes", + "Name": "Notes API", + "Scopes": [ + { + "ScopeName": "read", + "ScopeDescription": "Read notes" + }, + { + "ScopeName": "write", + "ScopeDescription": "Write notes" + } + ] + } + """.formatted(poolId)); + assertTrue(resourceServerResponse.path("ResourceServer").path("CreationDate").asLong() > 0); + assertTrue(resourceServerResponse.path("ResourceServer").path("LastModifiedDate").asLong() > 0); + } + + @Test + @Order(2) + void describeUserPoolClientReturnsGeneratedSecret() throws Exception { + JsonNode response = cognitoJson("DescribeUserPoolClient", """ + { + "UserPoolId": "%s", + "ClientId": "%s" + } + """.formatted(poolId, confidentialClientId)); + + assertEquals(confidentialClientId, response.path("UserPoolClient").path("ClientId").asText()); + assertEquals(confidentialClientSecret, response.path("UserPoolClient").path("ClientSecret").asText()); + assertTrue(response.path("UserPoolClient").path("GenerateSecret").asBoolean()); + assertTrue(response.path("UserPoolClient").path("AllowedOAuthFlowsUserPoolClient").asBoolean()); + assertEquals("client_credentials", + response.path("UserPoolClient").path("AllowedOAuthFlows").get(0).asText()); + } + + @Test + @Order(3) + void updateResourceServerReplacesNameAndScopes() throws Exception { + JsonNode before = cognitoJson("DescribeResourceServer", """ + { + "UserPoolId": "%s", + "Identifier": "notes" + } + """.formatted(poolId)); + long creationDate = before.path("ResourceServer").path("CreationDate").asLong(); + long previousLastModifiedDate = before.path("ResourceServer").path("LastModifiedDate").asLong(); + + JsonNode updateResponse = cognitoJson("UpdateResourceServer", """ + { + "UserPoolId": "%s", + "Identifier": "notes", + "Name": "Notes API v2", + "Scopes": [ + { + "ScopeName": "read", + "ScopeDescription": "Read notes v2" + }, + { + "ScopeName": "write", + "ScopeDescription": "Write notes v2" + } + ] + } + """.formatted(poolId)); + + JsonNode resourceServer = updateResponse.path("ResourceServer"); + assertEquals("notes", resourceServer.path("Identifier").asText()); + assertEquals("Notes API v2", resourceServer.path("Name").asText()); + assertEquals(creationDate, resourceServer.path("CreationDate").asLong()); + assertTrue(resourceServer.path("LastModifiedDate").asLong() >= previousLastModifiedDate); + assertEquals("read", resourceServer.path("Scopes").get(0).path("ScopeName").asText()); + assertEquals("Read notes v2", resourceServer.path("Scopes").get(0).path("ScopeDescription").asText()); + assertEquals("write", resourceServer.path("Scopes").get(1).path("ScopeName").asText()); + assertEquals("Write notes v2", resourceServer.path("Scopes").get(1).path("ScopeDescription").asText()); + + JsonNode described = cognitoJson("DescribeResourceServer", """ + { + "UserPoolId": "%s", + "Identifier": "notes" + } + """.formatted(poolId)); + assertEquals("Notes API v2", described.path("ResourceServer").path("Name").asText()); + assertEquals("write", described.path("ResourceServer").path("Scopes").get(1).path("ScopeName").asText()); + } + + @Test + @Order(4) + void updateResourceServerRequiresUserPoolId() { + cognitoAction("UpdateResourceServer", """ + { + "Identifier": "notes", + "Name": "Missing pool" + } + """) + .then() + .statusCode(400) + .body("__type", equalTo("InvalidParameterException")) + .body("message", equalTo("UserPoolId is required")); + } + + @Test + @Order(5) + void updateResourceServerRequiresIdentifier() { + cognitoAction("UpdateResourceServer", """ + { + "UserPoolId": "%s", + "Name": "Missing identifier" + } + """.formatted(poolId)) + .then() + .statusCode(400) + .body("__type", equalTo("InvalidParameterException")) + .body("message", equalTo("Identifier is required")); + } + + @Test + @Order(6) + void publicClientCannotUseClientCredentialsGrant() { + given() + .formParam("grant_type", "client_credentials") + .formParam("client_id", clientId) + .when() + .post("/cognito-idp/oauth2/token") + .then() + .statusCode(400) + .body("error", equalTo("unauthorized_client")); + } + + @Test + @Order(7) + void tokenEndpointReturnsAccessTokenFromBasicAuth() throws Exception { + String basic = Base64.getEncoder() + .encodeToString((confidentialClientId + ":" + confidentialClientSecret).getBytes(StandardCharsets.UTF_8)); + + Response response = given() + .header("Authorization", "Basic " + basic) + .formParam("grant_type", "client_credentials") + .when() + .post("/cognito-idp/oauth2/token"); + + response.then() + .statusCode(200) + .body("token_type", equalTo("Bearer")); + + JsonNode payload = decodeJwtPayload(response.jsonPath().getString("access_token")); + assertEquals(confidentialClientId, payload.path("client_id").asText()); + assertEquals("http://localhost:4566/" + poolId, payload.path("iss").asText()); + } + + @Test + @Order(8) + void tokenEndpointReturnsScopedAccessTokenForConfidentialClient() throws Exception { + String basic = Base64.getEncoder() + .encodeToString((confidentialClientId + ":" + confidentialClientSecret).getBytes(StandardCharsets.UTF_8)); + + Response response = given() + .header("Authorization", "Basic " + basic) + .formParam("grant_type", "client_credentials") + .formParam("scope", "notes/read notes/write") + .when() + .post("/cognito-idp/oauth2/token"); + + response.then().statusCode(200); + + JsonNode payload = decodeJwtPayload(response.jsonPath().getString("access_token")); + assertEquals("notes/read notes/write", payload.path("scope").asText()); + assertEquals(confidentialClientId, payload.path("client_id").asText()); + } + + @Test + @Order(9) + void tokenEndpointReturnsAllAllowedScopesWhenScopeOmitted() throws Exception { + String basic = Base64.getEncoder() + .encodeToString((confidentialClientId + ":" + confidentialClientSecret).getBytes(StandardCharsets.UTF_8)); + + Response response = given() + .header("Authorization", "Basic " + basic) + .formParam("grant_type", "client_credentials") + .when() + .post("/cognito-idp/oauth2/token"); + + response.then().statusCode(200); + + JsonNode payload = decodeJwtPayload(response.jsonPath().getString("access_token")); + assertEquals("notes/read notes/write", payload.path("scope").asText()); + } + + @Test + @Order(10) + void tokenEndpointAllowsClientSecretPostForConfidentialClient() { + given() + .formParam("grant_type", "client_credentials") + .formParam("client_id", confidentialClientId) + .formParam("client_secret", confidentialClientSecret) + .formParam("scope", "notes/read") + .when() + .post("/cognito-idp/oauth2/token") + .then() + .statusCode(200) + .body("token_type", equalTo("Bearer")); + } + + @Test + @Order(11) + void missingSecretForConfidentialClientReturnsInvalidClient() { + given() + .formParam("grant_type", "client_credentials") + .formParam("client_id", confidentialClientId) + .formParam("scope", "notes/read") + .when() + .post("/cognito-idp/oauth2/token") + .then() + .statusCode(400) + .body("error", equalTo("invalid_client")); + } + + @Test + @Order(12) + void invalidSecretForConfidentialClientReturnsInvalidClient() { + String basic = Base64.getEncoder() + .encodeToString((confidentialClientId + ":wrong-secret").getBytes(StandardCharsets.UTF_8)); + + given() + .header("Authorization", "Basic " + basic) + .formParam("grant_type", "client_credentials") + .formParam("scope", "notes/read") + .when() + .post("/cognito-idp/oauth2/token") + .then() + .statusCode(400) + .body("error", equalTo("invalid_client")); + } + + @Test + @Order(13) + void unknownScopeReturnsInvalidScope() { + String basic = Base64.getEncoder() + .encodeToString((confidentialClientId + ":" + confidentialClientSecret).getBytes(StandardCharsets.UTF_8)); + + given() + .header("Authorization", "Basic " + basic) + .formParam("grant_type", "client_credentials") + .formParam("scope", "notes/delete") + .when() + .post("/cognito-idp/oauth2/token") + .then() + .statusCode(400) + .body("error", equalTo("invalid_scope")); + } + + @Test + @Order(14) + void clientCannotRequestScopeThatIsNotAllowedForIt() { + String limitedClientSecret = cognitoDescribeClientSecret(limitedClientId); + String basic = Base64.getEncoder() + .encodeToString((limitedClientId + ":" + limitedClientSecret).getBytes(StandardCharsets.UTF_8)); + + given() + .header("Authorization", "Basic " + basic) + .formParam("grant_type", "client_credentials") + .formParam("scope", "notes/write") + .when() + .post("/cognito-idp/oauth2/token") + .then() + .statusCode(400) + .body("error", equalTo("invalid_scope")); + } + + @Test + @Order(15) + void missingGrantTypeReturnsInvalidRequest() { + given() + .formParam("client_id", clientId) + .when() + .post("/cognito-idp/oauth2/token") + .then() + .statusCode(400) + .body("error", equalTo("invalid_request")); + } + + @Test + @Order(16) + void unsupportedGrantTypeReturnsUnsupportedGrantType() { + given() + .formParam("grant_type", "refresh_token") + .formParam("client_id", clientId) + .when() + .post("/cognito-idp/oauth2/token") + .then() + .statusCode(400) + .body("error", equalTo("unsupported_grant_type")); + } + + @Test + @Order(17) + void missingClientIdReturnsInvalidRequest() { + given() + .formParam("grant_type", "client_credentials") + .when() + .post("/cognito-idp/oauth2/token") + .then() + .statusCode(400) + .body("error", equalTo("invalid_request")); + } + + @Test + @Order(18) + void unknownClientIdReturnsInvalidClient() { + given() + .formParam("grant_type", "client_credentials") + .formParam("client_id", "missing-client") + .when() + .post("/cognito-idp/oauth2/token") + .then() + .statusCode(400) + .body("error", equalTo("invalid_client")); + } + + @Test + @Order(19) + void mismatchedClientIdsReturnInvalidRequest() { + String basic = Base64.getEncoder() + .encodeToString((clientId + ":ignored-secret").getBytes(StandardCharsets.UTF_8)); + + given() + .header("Authorization", "Basic " + basic) + .formParam("grant_type", "client_credentials") + .formParam("client_id", "different-client-id") + .when() + .post("/cognito-idp/oauth2/token") + .then() + .statusCode(400) + .body("error", equalTo("invalid_request")); + } + + @Test + @Order(20) + void oauthTokensAreSignedWithPublishedRsaJwksKey() throws Exception { + Response tokenResponse = given() + .header("Authorization", "Basic " + Base64.getEncoder() + .encodeToString((confidentialClientId + ":" + confidentialClientSecret) + .getBytes(StandardCharsets.UTF_8))) + .formParam("grant_type", "client_credentials") + .when() + .post("/cognito-idp/oauth2/token"); + + tokenResponse.then().statusCode(200); + + String accessToken = tokenResponse.jsonPath().getString("access_token"); + JsonNode header = decodeJwtHeader(accessToken); + assertEquals("RS256", header.path("alg").asText()); + assertEquals(poolId, header.path("kid").asText()); + + String jwksResponse = given() + .when() + .get("/" + poolId + "/.well-known/jwks.json") + .then() + .statusCode(200) + .extract() + .asString(); + + JsonNode jwks = OBJECT_MAPPER.readTree(jwksResponse); + JsonNode key = jwks.path("keys").get(0); + assertNotNull(key); + assertEquals("RSA", key.path("kty").asText()); + assertEquals("RS256", key.path("alg").asText()); + assertEquals("sig", key.path("use").asText()); + assertEquals(poolId, key.path("kid").asText()); + assertTrue(key.hasNonNull("n")); + assertTrue(key.hasNonNull("e")); + assertTrue(verifyJwtSignature(accessToken, key)); + } + + @Test + @Order(21) + void openIdConfigurationIncludesTokenEndpointMetadata() throws Exception { + String openIdResponse = given() + .when() + .get("/" + poolId + "/.well-known/openid-configuration") + .then() + .statusCode(200) + .extract() + .asString(); + + JsonNode document = OBJECT_MAPPER.readTree(openIdResponse); + assertEquals( + "http://localhost:4566/cognito-idp/oauth2/token", + document.path("token_endpoint").asText()); + assertEquals("client_credentials", document.path("grant_types_supported").get(0).asText()); + assertEquals("client_secret_basic", document.path("token_endpoint_auth_methods_supported").get(0).asText()); + } + + private static Response cognitoAction(String action, String body) { + return given() + .header("X-Amz-Target", "AWSCognitoIdentityProviderService." + action) + .contentType(COGNITO_CONTENT_TYPE) + .body(body) + .when() + .post("/"); + } + + private static JsonNode cognitoJson(String action, String body) throws Exception { + String response = cognitoAction(action, body) + .then() + .statusCode(200) + .extract() + .asString(); + return OBJECT_MAPPER.readTree(response); + } + + private static String cognitoDescribeClientSecret(String clientId) { + return cognitoAction("DescribeUserPoolClient", """ + { + "UserPoolId": "%s", + "ClientId": "%s" + } + """.formatted(poolId, clientId)) + .then() + .statusCode(200) + .extract() + .jsonPath() + .getString("UserPoolClient.ClientSecret"); + } + + private static JsonNode decodeJwtPayload(String token) throws Exception { + return decodeJwtPart(token, 1); + } + + private static JsonNode decodeJwtHeader(String token) throws Exception { + return decodeJwtPart(token, 0); + } + + private static JsonNode decodeJwtPart(String token, int partIndex) throws Exception { + String[] parts = token.split("\\."); + assertEquals(3, parts.length); + return OBJECT_MAPPER.readTree(Base64.getUrlDecoder().decode(padBase64(parts[partIndex]))); + } + + private static boolean verifyJwtSignature(String token, JsonNode jwk) throws Exception { + String[] parts = token.split("\\."); + assertEquals(3, parts.length); + + BigInteger modulus = new BigInteger(1, Base64.getUrlDecoder().decode(padBase64(jwk.path("n").asText()))); + BigInteger exponent = new BigInteger(1, Base64.getUrlDecoder().decode(padBase64(jwk.path("e").asText()))); + RSAPublicKeySpec keySpec = new RSAPublicKeySpec(modulus, exponent); + PublicKey publicKey = KeyFactory.getInstance("RSA").generatePublic(keySpec); + + Signature signature = Signature.getInstance("SHA256withRSA"); + signature.initVerify(publicKey); + signature.update((parts[0] + "." + parts[1]).getBytes(StandardCharsets.UTF_8)); + return signature.verify(Base64.getUrlDecoder().decode(padBase64(parts[2]))); + } + + private static String padBase64(String value) { + int remainder = value.length() % 4; + if (remainder == 0) { + return value; + } + return value + "=".repeat(4 - remainder); + } +} From 29c27bb5ac97eac05ac7225e7010be4a24166907 Mon Sep 17 00:00:00 2001 From: Roberto Perez Alcolea Date: Tue, 31 Mar 2026 21:18:06 -0700 Subject: [PATCH 18/32] fix(sns): honor RawMessageDelivery attribute for SQS subscriptions (#54) * fix(sns): honor RawMessageDelivery attribute for SQS subscriptions * fix(sns): address PR review feedback on RawMessageDelivery - Preserve original SNS attribute DataType (Number, Binary, etc.) by changing the internal messageAttributes type from Map to Map throughout the publish pipeline - Update SnsQueryHandler and SnsJsonHandler to parse DataType from incoming requests and construct typed MessageAttributeValue objects - Return Collections.emptyMap() from toSqsMessageAttributes instead of null for safety on non-raw delivery paths - Use UUID-based queue names in raw delivery tests for isolation - Add test asserting message attributes (including Number type) are forwarded correctly to SQS in raw delivery mode --- .../floci/services/sns/SnsJsonHandler.java | 7 +- .../floci/services/sns/SnsQueryHandler.java | 6 +- .../floci/services/sns/SnsService.java | 53 +++-- .../services/sns/SnsIntegrationTest.java | 187 ++++++++++++++++++ 4 files changed, 232 insertions(+), 21 deletions(-) diff --git a/src/main/java/io/github/hectorvent/floci/services/sns/SnsJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/sns/SnsJsonHandler.java index 508f18b5..eb3b22e1 100644 --- a/src/main/java/io/github/hectorvent/floci/services/sns/SnsJsonHandler.java +++ b/src/main/java/io/github/hectorvent/floci/services/sns/SnsJsonHandler.java @@ -3,6 +3,7 @@ import io.github.hectorvent.floci.core.common.AwsErrorResponse; import io.github.hectorvent.floci.services.sns.model.Subscription; import io.github.hectorvent.floci.services.sns.model.Topic; +import io.github.hectorvent.floci.services.sqs.model.MessageAttributeValue; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; @@ -154,11 +155,13 @@ private Response handlePublish(JsonNode request, String region) { String message = request.path("Message").asText(null); String subject = request.path("Subject").asText(null); - Map attributes = new HashMap<>(); + Map attributes = new HashMap<>(); JsonNode attrsNode = request.path("MessageAttributes"); if (attrsNode.isObject()) { attrsNode.fields().forEachRemaining(entry -> { - attributes.put(entry.getKey(), entry.getValue().path("StringValue").asText()); + String dataType = entry.getValue().path("DataType").asText("String"); + String stringValue = entry.getValue().path("StringValue").asText(); + attributes.put(entry.getKey(), new MessageAttributeValue(stringValue, dataType)); }); } diff --git a/src/main/java/io/github/hectorvent/floci/services/sns/SnsQueryHandler.java b/src/main/java/io/github/hectorvent/floci/services/sns/SnsQueryHandler.java index 39ffaef0..e8c7b3ba 100644 --- a/src/main/java/io/github/hectorvent/floci/services/sns/SnsQueryHandler.java +++ b/src/main/java/io/github/hectorvent/floci/services/sns/SnsQueryHandler.java @@ -3,6 +3,7 @@ import io.github.hectorvent.floci.core.common.*; import io.github.hectorvent.floci.services.sns.model.Subscription; import io.github.hectorvent.floci.services.sns.model.Topic; +import io.github.hectorvent.floci.services.sqs.model.MessageAttributeValue; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import jakarta.ws.rs.core.MultivaluedMap; @@ -181,12 +182,13 @@ private Response handlePublish(MultivaluedMap params, String reg String messageGroupId = getParam(params, "MessageGroupId"); String messageDeduplicationId = getParam(params, "MessageDeduplicationId"); - Map attributes = new HashMap<>(); + Map attributes = new HashMap<>(); for (int i = 1; ; i++) { String name = params.getFirst("MessageAttributes.entry." + i + ".Name"); if (name == null) break; String value = params.getFirst("MessageAttributes.entry." + i + ".Value.StringValue"); - if (value != null) attributes.put(name, value); + String dataType = params.getFirst("MessageAttributes.entry." + i + ".Value.DataType"); + if (value != null) attributes.put(name, new MessageAttributeValue(value, dataType != null ? dataType : "String")); } try { diff --git a/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.java b/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.java index c2d5da6a..b71c4c8f 100644 --- a/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.java +++ b/src/main/java/io/github/hectorvent/floci/services/sns/SnsService.java @@ -11,6 +11,7 @@ import io.github.hectorvent.floci.services.sns.model.Subscription; import io.github.hectorvent.floci.services.sns.model.Topic; import io.github.hectorvent.floci.services.sqs.SqsService; +import io.github.hectorvent.floci.services.sqs.model.MessageAttributeValue; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -27,6 +28,7 @@ import java.time.Instant; import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.UUID; @@ -247,12 +249,12 @@ public String publish(String topicArn, String targetArn, String message, } public String publish(String topicArn, String targetArn, String phoneNumber, String message, - String subject, Map messageAttributes, String region) { + String subject, Map messageAttributes, String region) { return publish(topicArn, targetArn, phoneNumber, message, subject, messageAttributes, null, null, region); } public String publish(String topicArn, String targetArn, String phoneNumber, String message, - String subject, Map messageAttributes, + String subject, Map messageAttributes, String messageGroupId, String messageDeduplicationId, String region) { // Send SMS if (phoneNumber != null) { @@ -361,7 +363,7 @@ public BatchPublishResult publishBatch(String topicArn, List String messageId = UUID.randomUUID().toString(); @SuppressWarnings("unchecked") - Map attrs = (Map) entry.get("MessageAttributes"); + Map attrs = (Map) entry.get("MessageAttributes"); for (Subscription sub : subscriptionsByTopic(topicArn, region)) { if ("true".equals(sub.getAttributes().get("PendingConfirmation"))) continue; if (!matchesFilterPolicy(sub, attrs)) continue; @@ -411,7 +413,7 @@ public Map listTagsForResource(String resourceArn, String region * All keys in the policy must match (AND logic). Within each key's rule array, * any matching element is sufficient (OR logic). */ - private boolean matchesFilterPolicy(Subscription sub, Map messageAttributes) { + private boolean matchesFilterPolicy(Subscription sub, Map messageAttributes) { String filterPolicyJson = sub.getAttributes().get("FilterPolicy"); if (filterPolicyJson == null || filterPolicyJson.isBlank()) { return true; @@ -426,13 +428,14 @@ private boolean matchesFilterPolicy(Subscription sub, Map messag LOG.warnv("Invalid FilterPolicy (not a JSON object) for {0}", sub.getSubscriptionArn()); return false; } - Map attrs = messageAttributes != null ? messageAttributes : Map.of(); + Map attrs = messageAttributes != null ? messageAttributes : Map.of(); var fields = filterPolicy.fields(); while (fields.hasNext()) { var entry = fields.next(); String key = entry.getKey(); JsonNode rules = entry.getValue(); - String actualValue = attrs.get(key); + MessageAttributeValue attr = attrs.get(key); + String actualValue = attr != null ? attr.getStringValue() : null; if (!matchesAttributeRules(actualValue, rules)) { return false; } @@ -559,7 +562,7 @@ private List subscriptionsByTopic(String topicArn, String region) } private void deliverMessage(Subscription sub, String message, String subject, - Map messageAttributes, String messageId, + Map messageAttributes, String messageId, String topicArn, String messageGroupId) { try { switch (sub.getProtocol()) { @@ -569,10 +572,15 @@ private void deliverMessage(Subscription sub, String message, String subject, region = extractRegionFromArn(topicArn); } String queueUrl = sqsArnToUrl(sub.getEndpoint()); - String envelope = buildSnsEnvelope(message, subject, messageAttributes, topicArn, messageId); - sqsService.sendMessage(queueUrl, envelope, 0, messageGroupId, null, region); - LOG.debugv("Delivered SNS message to SQS: {0} ({1}) in {2}", - sub.getEndpoint(), queueUrl, region); + boolean rawDelivery = "true".equalsIgnoreCase(sub.getAttributes().get("RawMessageDelivery")); + String body = rawDelivery + ? message + : buildSnsEnvelope(message, subject, messageAttributes, topicArn, messageId); + Map sqsAttributes = rawDelivery + ? toSqsMessageAttributes(messageAttributes) + : Collections.emptyMap(); + sqsService.sendMessage(queueUrl, body, 0, messageGroupId, null, sqsAttributes, region); + LOG.debugv("Delivered SNS message to SQS: {0} ({1}) raw={2}", sub.getEndpoint(), queueUrl, rawDelivery); } case "lambda" -> { String fnName = extractFunctionName(sub.getEndpoint()); @@ -596,7 +604,7 @@ private void deliverMessage(Subscription sub, String message, String subject, } private String buildSnsLambdaEvent(String topicArn, String messageId, String message, - String subject, Map messageAttributes, + String subject, Map messageAttributes, String subscriptionArn) { try { String timestamp = DateTimeFormatter.ISO_INSTANT.format(Instant.now()); @@ -619,8 +627,8 @@ private String buildSnsLambdaEvent(String topicArn, String messageId, String mes if (messageAttributes != null) { for (var entry : messageAttributes.entrySet()) { ObjectNode attr = attrs.putObject(entry.getKey()); - attr.put("Type", "String"); - attr.put("Value", entry.getValue()); + attr.put("Type", entry.getValue().getDataType()); + attr.put("Value", entry.getValue().getStringValue()); } } ObjectNode record = objectMapper.createObjectNode(); @@ -647,8 +655,19 @@ private static String extractRegionFromArn(String arn) { return parts.length >= 4 ? parts[3] : null; } + /** + * Forwards SNS message attributes as SQS MessageAttributeValue objects + * when RawMessageDelivery is enabled, preserving the original DataType. + */ + private Map toSqsMessageAttributes(Map snsAttributes) { + if (snsAttributes == null || snsAttributes.isEmpty()) { + return Collections.emptyMap(); + } + return new java.util.HashMap<>(snsAttributes); + } + private String buildSnsEnvelope(String message, String subject, - Map messageAttributes, + Map messageAttributes, String topicArn, String messageId) { try { ObjectNode node = objectMapper.createObjectNode(); @@ -663,8 +682,8 @@ private String buildSnsEnvelope(String message, String subject, if (messageAttributes != null) { for (var entry : messageAttributes.entrySet()) { ObjectNode attr = attrs.putObject(entry.getKey()); - attr.put("Type", "String"); - attr.put("Value", entry.getValue()); + attr.put("Type", entry.getValue().getDataType()); + attr.put("Value", entry.getValue().getStringValue()); } } return objectMapper.writeValueAsString(node); diff --git a/src/test/java/io/github/hectorvent/floci/services/sns/SnsIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/sns/SnsIntegrationTest.java index a41dd905..51697910 100644 --- a/src/test/java/io/github/hectorvent/floci/services/sns/SnsIntegrationTest.java +++ b/src/test/java/io/github/hectorvent/floci/services/sns/SnsIntegrationTest.java @@ -10,6 +10,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestMethodOrder; +import java.util.UUID; + import static io.restassured.RestAssured.given; import static org.hamcrest.Matchers.*; @@ -32,6 +34,10 @@ static void configureRestAssured() { private static String topicArn; private static String subscriptionArn; private static String sqsQueueUrl; + private static String rawDeliveryQueueUrl; + private static String rawDeliverySubArn; + private static String envelopeQueueUrl; + private static String envelopeSubArn; @Test @Order(1) @@ -587,6 +593,187 @@ void filterPolicy_cleanup() { .when().post("/"); } + @Test + @Order(50) + void rawDelivery_createQueuesAndSubscribe() { + String suffix = UUID.randomUUID().toString().substring(0, 8); + + rawDeliveryQueueUrl = given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "CreateQueue") + .formParam("QueueName", "sns-raw-delivery-" + suffix) + .when() + .post("/") + .then() + .statusCode(200) + .extract().xmlPath().getString("CreateQueueResponse.CreateQueueResult.QueueUrl"); + + envelopeQueueUrl = given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "CreateQueue") + .formParam("QueueName", "sns-envelope-delivery-" + suffix) + .when() + .post("/") + .then() + .statusCode(200) + .extract().xmlPath().getString("CreateQueueResponse.CreateQueueResult.QueueUrl"); + + rawDeliverySubArn = given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "Subscribe") + .formParam("TopicArn", topicArn) + .formParam("Protocol", "sqs") + .formParam("Endpoint", rawDeliveryQueueUrl) + .when() + .post("/") + .then() + .statusCode(200) + .extract().xmlPath().getString("SubscribeResponse.SubscribeResult.SubscriptionArn"); + + envelopeSubArn = given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "Subscribe") + .formParam("TopicArn", topicArn) + .formParam("Protocol", "sqs") + .formParam("Endpoint", envelopeQueueUrl) + .when() + .post("/") + .then() + .statusCode(200) + .extract().xmlPath().getString("SubscribeResponse.SubscribeResult.SubscriptionArn"); + } + + @Test + @Order(51) + void rawDelivery_setSubscriptionAttribute() { + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "SetSubscriptionAttributes") + .formParam("SubscriptionArn", rawDeliverySubArn) + .formParam("AttributeName", "RawMessageDelivery") + .formParam("AttributeValue", "true") + .when() + .post("/") + .then() + .statusCode(200); + } + + @Test + @Order(52) + void rawDelivery_publishAndVerifyRawMessage() { + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "Publish") + .formParam("TopicArn", topicArn) + .formParam("Message", "Raw delivery test message") + .when() + .post("/") + .then() + .statusCode(200) + .body(containsString("")); + + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "ReceiveMessage") + .formParam("QueueUrl", rawDeliveryQueueUrl) + .formParam("MaxNumberOfMessages", "1") + .when() + .post("/") + .then() + .statusCode(200) + .body(containsString("Raw delivery test message")) + .body(not(containsString("Notification"))); + } + + @Test + @Order(53) + void rawDelivery_defaultSubscriptionWrapsInEnvelope() { + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "ReceiveMessage") + .formParam("QueueUrl", envelopeQueueUrl) + .formParam("MaxNumberOfMessages", "1") + .when() + .post("/") + .then() + .statusCode(200) + .body(containsString("Raw delivery test message")) + .body(containsString("Notification")); + } + + @Test + @Order(54) + void rawDelivery_messageAttributesForwardedOnRawDelivery() { + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "Publish") + .formParam("TopicArn", topicArn) + .formParam("Message", "Attribute forwarding test") + .formParam("MessageAttributes.entry.1.Name", "color") + .formParam("MessageAttributes.entry.1.Value.DataType", "String") + .formParam("MessageAttributes.entry.1.Value.StringValue", "blue") + .formParam("MessageAttributes.entry.2.Name", "count") + .formParam("MessageAttributes.entry.2.Value.DataType", "Number") + .formParam("MessageAttributes.entry.2.Value.StringValue", "42") + .when() + .post("/") + .then() + .statusCode(200) + .body(containsString("")); + + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "ReceiveMessage") + .formParam("QueueUrl", rawDeliveryQueueUrl) + .formParam("MaxNumberOfMessages", "1") + .formParam("MessageAttributeNames.member.1", "All") + .when() + .post("/") + .then() + .statusCode(200) + .body(containsString("Attribute forwarding test")) + .body(containsString("color")) + .body(containsString("blue")) + .body(containsString("count")) + .body(containsString("Number")); + } + + @Test + @Order(55) + void rawDelivery_cleanup() { + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "Unsubscribe") + .formParam("SubscriptionArn", rawDeliverySubArn) + .when() + .post("/") + .then() + .statusCode(200); + + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "Unsubscribe") + .formParam("SubscriptionArn", envelopeSubArn) + .when() + .post("/") + .then() + .statusCode(200); + + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "DeleteQueue") + .formParam("QueueUrl", rawDeliveryQueueUrl) + .when() + .post("/"); + + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "DeleteQueue") + .formParam("QueueUrl", envelopeQueueUrl) + .when() + .post("/"); + } + @Test @Order(100) void unsubscribe() { From e20ebe57e4cdccc154699926cc8da5020c820772 Mon Sep 17 00:00:00 2001 From: Ruan Leite <49437895+Ruanrls@users.noreply.github.com> Date: Wed, 1 Apr 2026 02:48:19 -0300 Subject: [PATCH 19/32] feat(cognito): add group management support (#149) * feat(cognito): add group management support * fix(cognito): handle null precedence in createGroup request * test(cognito): add tests for JWT escaping special characters in group names and user pool deletion cascading groups * feat(cognito): enhance group listing by sorting and improve group name validation * test(cognito): enhance group deletion test to verify group absence after user pool deletion * feat(cognito): update group and user models to set last modified date on changes * feat(cognito): update deleteGroup method to set last modified date for users when removing from group --- docs/services/cognito.md | 23 ++ .../services/cognito/CognitoJsonHandler.java | 83 ++++++ .../services/cognito/CognitoService.java | 163 +++++++++- .../services/cognito/model/CognitoGroup.java | 61 ++++ .../services/cognito/model/CognitoUser.java | 6 + .../cognito/CognitoIntegrationTest.java | 164 ++++++++++ .../services/cognito/CognitoServiceTest.java | 282 ++++++++++++++++++ 7 files changed, 780 insertions(+), 2 deletions(-) create mode 100644 src/main/java/io/github/hectorvent/floci/services/cognito/model/CognitoGroup.java create mode 100644 src/test/java/io/github/hectorvent/floci/services/cognito/CognitoServiceTest.java diff --git a/docs/services/cognito.md b/docs/services/cognito.md index 827769e4..7b114bbc 100644 --- a/docs/services/cognito.md +++ b/docs/services/cognito.md @@ -16,6 +16,7 @@ Floci serves pool-specific discovery and JWKS endpoints, plus a relaxed OAuth to | **User Operations** | SignUp, ConfirmSignUp, GetUser, UpdateUserAttributes, ChangePassword, ForgotPassword, ConfirmForgotPassword | | **Authentication** | InitiateAuth, AdminInitiateAuth, RespondToAuthChallenge | | **User Listing** | ListUsers | +| **Groups** | CreateGroup, GetGroup, ListGroups, DeleteGroup, AdminAddUserToGroup, AdminRemoveUserFromGroup, AdminListGroupsForUser | ## Well-Known And OAuth Endpoints @@ -95,6 +96,26 @@ aws cognito-idp initiate-auth \ --auth-parameters USERNAME=alice@example.com,PASSWORD=Perm1234! \ --endpoint-url $AWS_ENDPOINT +# Create a group +aws cognito-idp create-group \ + --user-pool-id $POOL_ID \ + --group-name admin \ + --description "Admin group" \ + --endpoint-url $AWS_ENDPOINT + +# Add user to group +aws cognito-idp admin-add-user-to-group \ + --user-pool-id $POOL_ID \ + --group-name admin \ + --username alice@example.com \ + --endpoint-url $AWS_ENDPOINT + +# List groups for user +aws cognito-idp admin-list-groups-for-user \ + --user-pool-id $POOL_ID \ + --username alice@example.com \ + --endpoint-url $AWS_ENDPOINT + # Fetch the pool discovery document curl -s "$AWS_ENDPOINT/$POOL_ID/.well-known/openid-configuration" @@ -119,6 +140,8 @@ http://localhost:4566/$POOL_ID/.well-known/openid-configuration http://localhost:4566/$POOL_ID/.well-known/jwks.json ``` +Tokens include the `cognito:groups` claim as a JSON array when the authenticated user belongs to one or more groups. + Tokens issued by Cognito auth flows and the OAuth token endpoint use the emulator base URL plus the pool id: ``` diff --git a/src/main/java/io/github/hectorvent/floci/services/cognito/CognitoJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/cognito/CognitoJsonHandler.java index abbc5aa4..0af2ca23 100644 --- a/src/main/java/io/github/hectorvent/floci/services/cognito/CognitoJsonHandler.java +++ b/src/main/java/io/github/hectorvent/floci/services/cognito/CognitoJsonHandler.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.github.hectorvent.floci.services.cognito.model.CognitoGroup; import io.github.hectorvent.floci.services.cognito.model.CognitoUser; import io.github.hectorvent.floci.services.cognito.model.ResourceServer; import io.github.hectorvent.floci.services.cognito.model.ResourceServerScope; @@ -61,6 +62,13 @@ public Response handle(String action, JsonNode request, String region) { case "ConfirmForgotPassword" -> handleConfirmForgotPassword(request); case "GetUser" -> handleGetUser(request); case "UpdateUserAttributes" -> handleUpdateUserAttributes(request); + case "CreateGroup" -> handleCreateGroup(request); + case "GetGroup" -> handleGetGroup(request); + case "ListGroups" -> handleListGroups(request); + case "DeleteGroup" -> handleDeleteGroup(request); + case "AdminAddUserToGroup" -> handleAdminAddUserToGroup(request); + case "AdminRemoveUserFromGroup" -> handleAdminRemoveUserFromGroup(request); + case "AdminListGroupsForUser" -> handleAdminListGroupsForUser(request); default -> Response.status(400) .entity(new AwsErrorResponse("UnsupportedOperation", "Operation " + action + " is not supported.")) .build(); @@ -455,4 +463,79 @@ private ObjectNode userToNode(CognitoUser u) { return node; } + private Response handleCreateGroup(JsonNode request) { + String userPoolId = request.path("UserPoolId").asText(); + String groupName = request.path("GroupName").asText(); + String description = request.path("Description").asText(null); + JsonNode precNode = request.path("Precedence"); + Integer precedence = precNode.isMissingNode() || precNode.isNull() ? null : precNode.asInt(); + String roleArn = request.path("RoleArn").asText(null); + CognitoGroup group = service.createGroup(userPoolId, groupName, description, precedence, roleArn); + ObjectNode response = objectMapper.createObjectNode(); + response.set("Group", groupToNode(group)); + return Response.ok(response).build(); + } + + private Response handleGetGroup(JsonNode request) { + CognitoGroup group = service.getGroup( + request.path("UserPoolId").asText(), + request.path("GroupName").asText()); + ObjectNode response = objectMapper.createObjectNode(); + response.set("Group", groupToNode(group)); + return Response.ok(response).build(); + } + + private Response handleListGroups(JsonNode request) { + List groups = service.listGroups(request.path("UserPoolId").asText()); + ObjectNode response = objectMapper.createObjectNode(); + ArrayNode items = response.putArray("Groups"); + groups.forEach(g -> items.add(groupToNode(g))); + return Response.ok(response).build(); + } + + private Response handleDeleteGroup(JsonNode request) { + service.deleteGroup( + request.path("UserPoolId").asText(), + request.path("GroupName").asText()); + return Response.ok(objectMapper.createObjectNode()).build(); + } + + private Response handleAdminAddUserToGroup(JsonNode request) { + service.adminAddUserToGroup( + request.path("UserPoolId").asText(), + request.path("GroupName").asText(), + request.path("Username").asText()); + return Response.ok(objectMapper.createObjectNode()).build(); + } + + private Response handleAdminRemoveUserFromGroup(JsonNode request) { + service.adminRemoveUserFromGroup( + request.path("UserPoolId").asText(), + request.path("GroupName").asText(), + request.path("Username").asText()); + return Response.ok(objectMapper.createObjectNode()).build(); + } + + private Response handleAdminListGroupsForUser(JsonNode request) { + List groups = service.adminListGroupsForUser( + request.path("UserPoolId").asText(), + request.path("Username").asText()); + ObjectNode response = objectMapper.createObjectNode(); + ArrayNode items = response.putArray("Groups"); + groups.forEach(g -> items.add(groupToNode(g))); + return Response.ok(response).build(); + } + + private ObjectNode groupToNode(CognitoGroup g) { + ObjectNode node = objectMapper.createObjectNode(); + node.put("GroupName", g.getGroupName()); + node.put("UserPoolId", g.getUserPoolId()); + if (g.getDescription() != null) node.put("Description", g.getDescription()); + if (g.getPrecedence() != null) node.put("Precedence", g.getPrecedence()); + if (g.getRoleArn() != null) node.put("RoleArn", g.getRoleArn()); + node.put("CreationDate", g.getCreationDate()); + node.put("LastModifiedDate", g.getLastModifiedDate()); + return node; + } + } diff --git a/src/main/java/io/github/hectorvent/floci/services/cognito/CognitoService.java b/src/main/java/io/github/hectorvent/floci/services/cognito/CognitoService.java index 21a3e07c..f7364cf1 100644 --- a/src/main/java/io/github/hectorvent/floci/services/cognito/CognitoService.java +++ b/src/main/java/io/github/hectorvent/floci/services/cognito/CognitoService.java @@ -5,6 +5,7 @@ import io.github.hectorvent.floci.core.storage.StorageBackend; import io.github.hectorvent.floci.core.storage.StorageFactory; import com.fasterxml.jackson.core.type.TypeReference; +import io.github.hectorvent.floci.services.cognito.model.CognitoGroup; import io.github.hectorvent.floci.services.cognito.model.CognitoUser; import io.github.hectorvent.floci.services.cognito.model.ResourceServer; import io.github.hectorvent.floci.services.cognito.model.ResourceServerScope; @@ -26,6 +27,7 @@ import java.security.spec.PKCS8EncodedKeySpec; import java.security.spec.X509EncodedKeySpec; import java.util.*; +import java.util.stream.Collectors; @ApplicationScoped public class CognitoService { @@ -36,6 +38,7 @@ public class CognitoService { private final StorageBackend clientStore; private final StorageBackend resourceServerStore; private final StorageBackend userStore; + private final StorageBackend groupStore; private final String baseUrl; @Inject @@ -48,9 +51,25 @@ public CognitoService(StorageFactory storageFactory, EmulatorConfig emulatorConf new TypeReference>() {}); this.userStore = storageFactory.create("cognito", "cognito-users.json", new TypeReference>() {}); + this.groupStore = storageFactory.create("cognito", "cognito-groups.json", + new TypeReference>() {}); this.baseUrl = trimTrailingSlash(emulatorConfig.baseUrl()); } + CognitoService(StorageBackend poolStore, + StorageBackend clientStore, + StorageBackend resourceServerStore, + StorageBackend userStore, + StorageBackend groupStore, + String baseUrl) { + this.poolStore = poolStore; + this.clientStore = clientStore; + this.resourceServerStore = resourceServerStore; + this.userStore = userStore; + this.groupStore = groupStore; + this.baseUrl = baseUrl; + } + // ──────────────────────────── User Pools ──────────────────────────── public UserPool createUserPool(String name, String region) { @@ -78,6 +97,9 @@ public List listUserPools() { } public void deleteUserPool(String id) { + String prefix = id + "::"; + groupStore.scan(k -> k.startsWith(prefix)) + .forEach(g -> groupStore.delete(groupKey(id, g.getGroupName()))); poolStore.delete(id); } @@ -220,6 +242,14 @@ public CognitoUser adminGetUser(String userPoolId, String username) { } public void adminDeleteUser(String userPoolId, String username) { + CognitoUser user = adminGetUser(userPoolId, username); + for (String groupName : new ArrayList<>(user.getGroupNames())) { + groupStore.get(groupKey(userPoolId, groupName)).ifPresent(group -> { + group.removeUserName(username); + group.setLastModifiedDate(System.currentTimeMillis() / 1000L); + groupStore.put(groupKey(userPoolId, groupName), group); + }); + } userStore.delete(userKey(userPoolId, username)); } @@ -245,6 +275,95 @@ public List listUsers(String userPoolId) { return userStore.scan(k -> k.startsWith(prefix)); } + // ──────────────────────────── Groups ──────────────────────────── + + public CognitoGroup createGroup(String userPoolId, String groupName, String description, + Integer precedence, String roleArn) { + describeUserPool(userPoolId); + validateGroupName(groupName); + if (groupStore.get(groupKey(userPoolId, groupName)).isPresent()) { + throw new AwsException("GroupExistsException", + "A group with the name " + groupName + " already exists.", 400); + } + CognitoGroup group = new CognitoGroup(); + group.setGroupName(groupName); + group.setUserPoolId(userPoolId); + group.setDescription(description); + group.setPrecedence(precedence); + group.setRoleArn(roleArn); + groupStore.put(groupKey(userPoolId, groupName), group); + LOG.infov("Created Cognito group: {0} in pool {1}", groupName, userPoolId); + return group; + } + + public CognitoGroup getGroup(String userPoolId, String groupName) { + describeUserPool(userPoolId); + validateGroupName(groupName); + return groupStore.get(groupKey(userPoolId, groupName)) + .orElseThrow(() -> new AwsException("ResourceNotFoundException", + "Group not found: " + groupName, 404)); + } + + public List listGroups(String userPoolId) { + describeUserPool(userPoolId); + String prefix = userPoolId + "::"; + List groups = new ArrayList<>(groupStore.scan(k -> k.startsWith(prefix))); + groups.sort(Comparator.comparing(CognitoGroup::getGroupName)); + return groups; + } + + public void deleteGroup(String userPoolId, String groupName) { + CognitoGroup group = getGroup(userPoolId, groupName); + long now = System.currentTimeMillis() / 1000L; + for (String username : new ArrayList<>(group.getUserNames())) { + userStore.get(userKey(userPoolId, username)).ifPresent(user -> { + if (user.getGroupNames().remove(groupName)) { + user.setLastModifiedDate(now); + userStore.put(userKey(userPoolId, username), user); + } + }); + } + groupStore.delete(groupKey(userPoolId, groupName)); + LOG.infov("Deleted Cognito group: {0} from pool {1}", groupName, userPoolId); + } + + public void adminAddUserToGroup(String userPoolId, String groupName, String username) { + CognitoGroup group = getGroup(userPoolId, groupName); + CognitoUser user = adminGetUser(userPoolId, username); + long now = System.currentTimeMillis() / 1000L; + if (group.addUserName(username)) { + group.setLastModifiedDate(now); + groupStore.put(groupKey(userPoolId, groupName), group); + } + if (!user.getGroupNames().contains(groupName)) { + user.getGroupNames().add(groupName); + user.setLastModifiedDate(now); + userStore.put(userKey(userPoolId, username), user); + } + } + + public void adminRemoveUserFromGroup(String userPoolId, String groupName, String username) { + CognitoGroup group = getGroup(userPoolId, groupName); + CognitoUser user = adminGetUser(userPoolId, username); + long now = System.currentTimeMillis() / 1000L; + if (group.removeUserName(username)) { + group.setLastModifiedDate(now); + groupStore.put(groupKey(userPoolId, groupName), group); + } + if (user.getGroupNames().remove(groupName)) { + user.setLastModifiedDate(now); + userStore.put(userKey(userPoolId, username), user); + } + } + + public List adminListGroupsForUser(String userPoolId, String username) { + describeUserPool(userPoolId); + CognitoUser user = adminGetUser(userPoolId, username); + return user.getGroupNames().stream() + .flatMap(gn -> groupStore.get(groupKey(userPoolId, gn)).stream()) + .toList(); + } + // ──────────────────────────── Self-Service Registration ──────────────────────────── public CognitoUser signUp(String clientId, String username, String password, Map attributes) { @@ -513,13 +632,20 @@ private String generateSignedJwt(CognitoUser user, UserPool pool, String type) { long now = System.currentTimeMillis() / 1000L; String email = user.getAttributes().getOrDefault("email", user.getUsername()); + String groupsFragment = ""; + if (!user.getGroupNames().isEmpty()) { + String groupsJson = user.getGroupNames().stream() + .map(g -> "\"" + escapeJsonString(g) + "\"") + .collect(Collectors.joining(",", "[", "]")); + groupsFragment = ",\"cognito:groups\":" + groupsJson; + } String payloadJson = String.format( "{\"sub\":\"%s\",\"event_id\":\"%s\",\"token_use\":\"%s\",\"auth_time\":%d," + "\"iss\":\"%s\",\"exp\":%d,\"iat\":%d," + - "\"username\":\"%s\",\"email\":\"%s\",\"cognito:username\":\"%s\"}", + "\"username\":\"%s\",\"email\":\"%s\",\"cognito:username\":\"%s\"%s}", UUID.randomUUID(), UUID.randomUUID(), type, now, escapeJson(getIssuer(pool.getId())), now + 3600, now, - user.getUsername(), email, user.getUsername() + user.getUsername(), email, user.getUsername(), groupsFragment ); String payload = Base64.getUrlEncoder().withoutPadding() .encodeToString(payloadJson.getBytes(StandardCharsets.UTF_8)); @@ -819,6 +945,35 @@ private String extractPoolIdFromToken(String token) { } } + private void validateGroupName(String groupName) { + if (groupName == null || groupName.isBlank()) { + throw new AwsException("InvalidParameterException", "GroupName is required", 400); + } + } + + private String escapeJsonString(String s) { + StringBuilder sb = new StringBuilder(); + for (char c : s.toCharArray()) { + switch (c) { + case '"' -> sb.append("\\\""); + case '\\' -> sb.append("\\\\"); + case '\b' -> sb.append("\\b"); + case '\f' -> sb.append("\\f"); + case '\n' -> sb.append("\\n"); + case '\r' -> sb.append("\\r"); + case '\t' -> sb.append("\\t"); + default -> { + if (c < 0x20) { + sb.append(String.format("\\u%04x", (int) c)); + } else { + sb.append(c); + } + } + } + } + return sb.toString(); + } + private String extractJsonField(String json, String field) { String search = "\"" + field + "\":\""; int start = json.indexOf(search); @@ -833,6 +988,10 @@ private String userKey(String poolId, String username) { return poolId + "::" + username; } + private String groupKey(String poolId, String groupName) { + return poolId + "::" + groupName; + } + private String resourceServerKey(String userPoolId, String identifier) { return userPoolId + "::" + identifier; } diff --git a/src/main/java/io/github/hectorvent/floci/services/cognito/model/CognitoGroup.java b/src/main/java/io/github/hectorvent/floci/services/cognito/model/CognitoGroup.java new file mode 100644 index 00000000..6992c37f --- /dev/null +++ b/src/main/java/io/github/hectorvent/floci/services/cognito/model/CognitoGroup.java @@ -0,0 +1,61 @@ +package io.github.hectorvent.floci.services.cognito.model; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import io.quarkus.runtime.annotations.RegisterForReflection; + +@RegisterForReflection +@JsonIgnoreProperties(ignoreUnknown = true) +public class CognitoGroup { + private String groupName; + private String userPoolId; + private String description; + private Integer precedence; + private String roleArn; + private long creationDate; + private long lastModifiedDate; + private List userNames; + + public CognitoGroup() { + long now = System.currentTimeMillis() / 1000L; + this.creationDate = now; + this.lastModifiedDate = now; + this.userNames = new ArrayList<>(); + } + + public String getGroupName() { return groupName; } + public void setGroupName(String groupName) { this.groupName = groupName; } + + public String getUserPoolId() { return userPoolId; } + public void setUserPoolId(String userPoolId) { this.userPoolId = userPoolId; } + + public String getDescription() { return description; } + public void setDescription(String description) { this.description = description; } + + public Integer getPrecedence() { return precedence; } + public void setPrecedence(Integer precedence) { this.precedence = precedence; } + + public String getRoleArn() { return roleArn; } + public void setRoleArn(String roleArn) { this.roleArn = roleArn; } + + public long getCreationDate() { return creationDate; } + public void setCreationDate(long creationDate) { this.creationDate = creationDate; } + + public long getLastModifiedDate() { return lastModifiedDate; } + public void setLastModifiedDate(long lastModifiedDate) { this.lastModifiedDate = lastModifiedDate; } + + public List getUserNames() { return Collections.unmodifiableList(userNames); } + public void setUserNames(List userNames) { this.userNames = userNames == null ? new ArrayList<>() : new ArrayList<>(userNames); } + + public boolean addUserName(String name) { + if (userNames.contains(name)) return false; + return userNames.add(name); + } + + public boolean removeUserName(String name) { + return userNames.remove(name); + } +} diff --git a/src/main/java/io/github/hectorvent/floci/services/cognito/model/CognitoUser.java b/src/main/java/io/github/hectorvent/floci/services/cognito/model/CognitoUser.java index 73b0ecd0..3506dc23 100644 --- a/src/main/java/io/github/hectorvent/floci/services/cognito/model/CognitoUser.java +++ b/src/main/java/io/github/hectorvent/floci/services/cognito/model/CognitoUser.java @@ -3,7 +3,9 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import io.quarkus.runtime.annotations.RegisterForReflection; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; @RegisterForReflection @@ -18,6 +20,7 @@ public class CognitoUser { private long lastModifiedDate; private String passwordHash; private boolean temporaryPassword; + private List groupNames = new ArrayList<>(); public CognitoUser() { long now = System.currentTimeMillis() / 1000L; @@ -53,4 +56,7 @@ public CognitoUser() { public boolean isTemporaryPassword() { return temporaryPassword; } public void setTemporaryPassword(boolean temporaryPassword) { this.temporaryPassword = temporaryPassword; } + + public List getGroupNames() { return groupNames; } + public void setGroupNames(List groupNames) { this.groupNames = groupNames == null ? new ArrayList<>() : new ArrayList<>(groupNames); } } diff --git a/src/test/java/io/github/hectorvent/floci/services/cognito/CognitoIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/cognito/CognitoIntegrationTest.java index 53f8ce59..51c063ea 100644 --- a/src/test/java/io/github/hectorvent/floci/services/cognito/CognitoIntegrationTest.java +++ b/src/test/java/io/github/hectorvent/floci/services/cognito/CognitoIntegrationTest.java @@ -173,6 +173,170 @@ void openIdConfigurationPublishesIssuerAndJwksUri() throws Exception { assertEquals("RS256", document.path("id_token_signing_alg_values_supported").get(0).asText()); } + // ── Groups ──────────────────────────────────────────────────────── + + @Test + @Order(10) + void createGroup() throws Exception { + JsonNode resp = cognitoJson("CreateGroup", """ + { + "UserPoolId": "%s", + "GroupName": "admin", + "Description": "Admin group", + "Precedence": 1 + } + """.formatted(poolId)); + assertEquals("admin", resp.path("Group").path("GroupName").asText()); + assertEquals(poolId, resp.path("Group").path("UserPoolId").asText()); + assertEquals("Admin group", resp.path("Group").path("Description").asText()); + assertEquals(1, resp.path("Group").path("Precedence").asInt()); + } + + @Test + @Order(11) + void createGroupDuplicate() { + cognitoAction("CreateGroup", """ + { + "UserPoolId": "%s", + "GroupName": "admin", + "Description": "Admin group", + "Precedence": 1 + } + """.formatted(poolId)) + .then() + .statusCode(400); + } + + @Test + @Order(12) + void getGroup() throws Exception { + JsonNode resp = cognitoJson("GetGroup", """ + { + "UserPoolId": "%s", + "GroupName": "admin" + } + """.formatted(poolId)); + assertEquals("admin", resp.path("Group").path("GroupName").asText()); + } + + @Test + @Order(13) + void listGroups() throws Exception { + JsonNode resp = cognitoJson("ListGroups", """ + { + "UserPoolId": "%s" + } + """.formatted(poolId)); + assertEquals(1, resp.path("Groups").size()); + assertEquals("admin", resp.path("Groups").get(0).path("GroupName").asText()); + } + + @Test + @Order(14) + void adminAddUserToGroup() { + cognitoAction("AdminAddUserToGroup", """ + { + "UserPoolId": "%s", + "GroupName": "admin", + "Username": "%s" + } + """.formatted(poolId, username)) + .then() + .statusCode(200); + } + + @Test + @Order(15) + void adminListGroupsForUser() throws Exception { + JsonNode resp = cognitoJson("AdminListGroupsForUser", """ + { + "UserPoolId": "%s", + "Username": "%s" + } + """.formatted(poolId, username)); + assertEquals(1, resp.path("Groups").size()); + assertEquals("admin", resp.path("Groups").get(0).path("GroupName").asText()); + } + + @Test + @Order(16) + void authenticateAndVerifyGroupsInToken() throws Exception { + Response authResponse = cognitoAction("InitiateAuth", """ + { + "ClientId": "%s", + "AuthFlow": "USER_PASSWORD_AUTH", + "AuthParameters": { + "USERNAME": "%s", + "PASSWORD": "%s" + } + } + """.formatted(clientId, username, password)); + + authResponse.then().statusCode(200); + + String accessToken = authResponse.jsonPath().getString("AuthenticationResult.AccessToken"); + JsonNode payload = decodeJwtPayload(accessToken); + + assertTrue(payload.has("cognito:groups"), + "JWT payload should contain cognito:groups claim"); + assertTrue(payload.path("cognito:groups").toString().contains("\"admin\""), + "JWT payload should contain admin group"); + } + + @Test + @Order(17) + void adminRemoveUserFromGroup() { + cognitoAction("AdminRemoveUserFromGroup", """ + { + "UserPoolId": "%s", + "GroupName": "admin", + "Username": "%s" + } + """.formatted(poolId, username)) + .then() + .statusCode(200); + } + + @Test + @Order(18) + void adminListGroupsForUserEmpty() throws Exception { + JsonNode resp = cognitoJson("AdminListGroupsForUser", """ + { + "UserPoolId": "%s", + "Username": "%s" + } + """.formatted(poolId, username)); + assertEquals(0, resp.path("Groups").size()); + } + + @Test + @Order(19) + void deleteGroup() { + cognitoAction("DeleteGroup", """ + { + "UserPoolId": "%s", + "GroupName": "admin" + } + """.formatted(poolId)) + .then() + .statusCode(200); + } + + @Test + @Order(20) + void getGroupNotFound() { + cognitoAction("GetGroup", """ + { + "UserPoolId": "%s", + "GroupName": "admin" + } + """.formatted(poolId)) + .then() + .statusCode(404); + } + + // ── Helpers ─────────────────────────────────────────────────────── + private static Response cognitoAction(String action, String body) { return given() .header("X-Amz-Target", "AWSCognitoIdentityProviderService." + action) diff --git a/src/test/java/io/github/hectorvent/floci/services/cognito/CognitoServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/cognito/CognitoServiceTest.java new file mode 100644 index 00000000..3f89404a --- /dev/null +++ b/src/test/java/io/github/hectorvent/floci/services/cognito/CognitoServiceTest.java @@ -0,0 +1,282 @@ +package io.github.hectorvent.floci.services.cognito; + +import io.github.hectorvent.floci.core.common.AwsException; +import io.github.hectorvent.floci.core.storage.InMemoryStorage; +import io.github.hectorvent.floci.services.cognito.model.CognitoGroup; +import io.github.hectorvent.floci.services.cognito.model.CognitoUser; +import io.github.hectorvent.floci.services.cognito.model.UserPool; +import io.github.hectorvent.floci.services.cognito.model.UserPoolClient; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +class CognitoServiceTest { + + private CognitoService service; + private InMemoryStorage groupStore; + + @BeforeEach + void setUp() { + groupStore = new InMemoryStorage<>(); + service = new CognitoService( + new InMemoryStorage<>(), + new InMemoryStorage<>(), + new InMemoryStorage<>(), + new InMemoryStorage<>(), + groupStore, + "http://localhost:4566" + ); + } + + private UserPool createPoolAndUser() { + UserPool pool = service.createUserPool("TestPool", "us-east-1"); + service.adminCreateUser(pool.getId(), "alice", Map.of("email", "alice@example.com"), "TempPass1!"); + service.adminSetUserPassword(pool.getId(), "alice", "Perm1234!", true); + return pool; + } + + // ========================================================================= + // Groups + // ========================================================================= + + @Test + void createGroup() { + UserPool pool = service.createUserPool("TestPool", "us-east-1"); + CognitoGroup group = service.createGroup(pool.getId(), "admins", "Admin group", 1, null); + + assertEquals("admins", group.getGroupName()); + assertEquals(pool.getId(), group.getUserPoolId()); + assertEquals("Admin group", group.getDescription()); + assertEquals(1, group.getPrecedence()); + assertNull(group.getRoleArn()); + assertTrue(group.getCreationDate() > 0); + assertTrue(group.getLastModifiedDate() > 0); + } + + @Test + void createGroupDuplicateThrows() { + UserPool pool = service.createUserPool("TestPool", "us-east-1"); + service.createGroup(pool.getId(), "admins", "Admin group", 1, null); + + assertThrows(AwsException.class, () -> + service.createGroup(pool.getId(), "admins", "Another desc", 2, null)); + } + + @Test + void getGroup() { + UserPool pool = service.createUserPool("TestPool", "us-east-1"); + service.createGroup(pool.getId(), "admins", "Admin group", 1, null); + + CognitoGroup fetched = service.getGroup(pool.getId(), "admins"); + assertEquals("admins", fetched.getGroupName()); + assertEquals(pool.getId(), fetched.getUserPoolId()); + assertEquals("Admin group", fetched.getDescription()); + assertEquals(1, fetched.getPrecedence()); + } + + @Test + void getGroupNotFoundThrows() { + UserPool pool = service.createUserPool("TestPool", "us-east-1"); + + assertThrows(AwsException.class, () -> + service.getGroup(pool.getId(), "nonexistent")); + } + + @Test + void listGroups() { + UserPool pool = service.createUserPool("TestPool", "us-east-1"); + service.createGroup(pool.getId(), "admins", "Admin group", 1, null); + service.createGroup(pool.getId(), "editors", "Editor group", 2, null); + + List groups = service.listGroups(pool.getId()); + assertEquals(2, groups.size()); + } + + @Test + void deleteGroup() { + UserPool pool = service.createUserPool("TestPool", "us-east-1"); + service.createGroup(pool.getId(), "admins", "Admin group", 1, null); + + service.deleteGroup(pool.getId(), "admins"); + + assertThrows(AwsException.class, () -> + service.getGroup(pool.getId(), "admins")); + } + + @Test + void deleteGroupCleansUpUserMembership() { + UserPool pool = createPoolAndUser(); + service.createGroup(pool.getId(), "admins", "Admin group", 1, null); + service.adminAddUserToGroup(pool.getId(), "admins", "alice"); + + service.deleteGroup(pool.getId(), "admins"); + + CognitoUser user = service.adminGetUser(pool.getId(), "alice"); + assertTrue(user.getGroupNames().isEmpty()); + } + + @Test + void adminDeleteUserCleansUpGroupMembership() { + UserPool pool = createPoolAndUser(); + service.createGroup(pool.getId(), "admins", "Admin group", 1, null); + service.adminAddUserToGroup(pool.getId(), "admins", "alice"); + + service.adminDeleteUser(pool.getId(), "alice"); + + CognitoGroup group = service.getGroup(pool.getId(), "admins"); + assertFalse(group.getUserNames().contains("alice")); + } + + // ========================================================================= + // Group membership + // ========================================================================= + + @Test + void adminAddUserToGroup() { + UserPool pool = createPoolAndUser(); + service.createGroup(pool.getId(), "admins", "Admin group", 1, null); + + service.adminAddUserToGroup(pool.getId(), "admins", "alice"); + + CognitoGroup group = service.getGroup(pool.getId(), "admins"); + assertTrue(group.getUserNames().contains("alice")); + + CognitoUser user = service.adminGetUser(pool.getId(), "alice"); + assertTrue(user.getGroupNames().contains("admins")); + } + + @Test + void adminAddUserToGroupIdempotent() { + UserPool pool = createPoolAndUser(); + service.createGroup(pool.getId(), "admins", "Admin group", 1, null); + + service.adminAddUserToGroup(pool.getId(), "admins", "alice"); + service.adminAddUserToGroup(pool.getId(), "admins", "alice"); + + CognitoGroup group = service.getGroup(pool.getId(), "admins"); + assertEquals(1, group.getUserNames().size()); + } + + @Test + void adminRemoveUserFromGroup() { + UserPool pool = createPoolAndUser(); + service.createGroup(pool.getId(), "admins", "Admin group", 1, null); + service.adminAddUserToGroup(pool.getId(), "admins", "alice"); + + service.adminRemoveUserFromGroup(pool.getId(), "admins", "alice"); + + CognitoGroup group = service.getGroup(pool.getId(), "admins"); + assertFalse(group.getUserNames().contains("alice")); + + CognitoUser user = service.adminGetUser(pool.getId(), "alice"); + assertFalse(user.getGroupNames().contains("admins")); + } + + @Test + void adminListGroupsForUser() { + UserPool pool = createPoolAndUser(); + service.createGroup(pool.getId(), "admins", "Admin group", 1, null); + service.createGroup(pool.getId(), "editors", "Editor group", 2, null); + service.adminAddUserToGroup(pool.getId(), "admins", "alice"); + service.adminAddUserToGroup(pool.getId(), "editors", "alice"); + + List groups = service.adminListGroupsForUser(pool.getId(), "alice"); + assertEquals(2, groups.size()); + } + + @Test + void adminAddUserToGroupNonexistentGroupThrows() { + UserPool pool = createPoolAndUser(); + + assertThrows(AwsException.class, () -> + service.adminAddUserToGroup(pool.getId(), "nonexistent", "alice")); + } + + @Test + void adminAddUserToGroupNonexistentUserThrows() { + UserPool pool = service.createUserPool("TestPool", "us-east-1"); + service.createGroup(pool.getId(), "admins", "Admin group", 1, null); + + assertThrows(AwsException.class, () -> + service.adminAddUserToGroup(pool.getId(), "admins", "nonexistent")); + } + + // ========================================================================= + // JWT groups claim + // ========================================================================= + + @Test + @SuppressWarnings("unchecked") + void jwtContainsGroupsClaim() { + UserPool pool = createPoolAndUser(); + UserPoolClient client = service.createUserPoolClient(pool.getId(), "test-client", false, false, List.of(), List.of()); + String clientId = client.getClientId(); + + service.createGroup(pool.getId(), "admins", "Admin group", 1, null); + service.adminAddUserToGroup(pool.getId(), "admins", "alice"); + + Map authResult = service.initiateAuth( + clientId, "USER_PASSWORD_AUTH", + Map.of("USERNAME", "alice", "PASSWORD", "Perm1234!")); + + Map authenticationResult = (Map) authResult.get("AuthenticationResult"); + String accessToken = (String) authenticationResult.get("AccessToken"); + + // Decode the JWT payload (second segment) + String[] parts = accessToken.split("\\."); + String payloadJson = new String( + Base64.getUrlDecoder().decode(parts[1]), StandardCharsets.UTF_8); + + assertTrue(payloadJson.contains("\"cognito:groups\":[\"admins\"]"), + "JWT payload should contain cognito:groups claim with the group name"); + } + + @Test + @SuppressWarnings("unchecked") + void jwtEscapesSpecialCharsInGroupName() { + UserPool pool = createPoolAndUser(); + UserPoolClient client = service.createUserPoolClient(pool.getId(), "test-client", false, false, List.of(), List.of()); + + String specialGroup = "group\"with\\special\nchars"; + service.createGroup(pool.getId(), specialGroup, null, null, null); + service.adminAddUserToGroup(pool.getId(), specialGroup, "alice"); + + Map authResult = service.initiateAuth( + client.getClientId(), "USER_PASSWORD_AUTH", + Map.of("USERNAME", "alice", "PASSWORD", "Perm1234!")); + + Map auth = (Map) authResult.get("AuthenticationResult"); + String token = (String) auth.get("AccessToken"); + String payloadJson = new String( + Base64.getUrlDecoder().decode(token.split("\\.")[1]), StandardCharsets.UTF_8); + + assertTrue(payloadJson.contains("cognito:groups"), + "JWT should contain cognito:groups claim"); + assertTrue(payloadJson.contains("group\\\"with\\\\special\\nchars"), + "Group name should be properly JSON-escaped in JWT payload"); + } + + // ========================================================================= + // deleteUserPool cascades groups + // ========================================================================= + + @Test + void deleteUserPoolCascadesGroups() { + UserPool pool = service.createUserPool("TestPool", "us-east-1"); + service.createGroup(pool.getId(), "admins", "Admin group", 1, null); + service.createGroup(pool.getId(), "editors", "Editor group", 2, null); + + String prefix = pool.getId() + "::"; + assertEquals(2, groupStore.scan(k -> k.startsWith(prefix)).size()); + + service.deleteUserPool(pool.getId()); + + assertEquals(0, groupStore.scan(k -> k.startsWith(prefix)).size()); + } +} From 9fbe53f37ad7920a4e5cce7ca18c3833be7d4e82 Mon Sep 17 00:00:00 2001 From: Hector Ventura Date: Wed, 1 Apr 2026 00:49:51 -0500 Subject: [PATCH 20/32] fix(sqs): route queue URL path requests to SQS handler (#153) * fix: route SQS JSON 1.0 requests sent to queue URL path correctly (#99) * fix: route SQS JSON 1.0 requests sent to queue URL path correctly Newer AWS SDKs (e.g. aws-sdk-sqs Ruby gem >= 1.71) send JSON 1.0 requests to the queue URL path (/{accountId}/{queueName}) rather than POST /. These were matched by S3Controller's /{bucket}/{key:.+} handler, returning NoSuchBucket or InvalidArgument XML errors. Added a @PreMatching ContainerRequestFilter (SqsQueueUrlFilter) that rewrites the request URI to POST / before JAX-RS routing when both Content-Type is application/x-amz-json-1.0 and X-Amz-Target starts with AmazonSQS. The existing AwsJsonController dispatcher then handles the request normally. A @Consumes-based approach was attempted first but is ineffective in RESTEasy Reactive: @Consumes only affects method selection within a class, not between competing resource classes, so S3Controller was still selected at the class-routing stage. Closes #17 --- .gitignore | 1 + .../core/common/SqsQueueUrlRouterFilter.java | 65 +++++++ .../services/sqs/SqsJsonProtocolTest.java | 166 ++++++++++++++++++ 3 files changed, 232 insertions(+) create mode 100644 src/main/java/io/github/hectorvent/floci/core/common/SqsQueueUrlRouterFilter.java create mode 100644 src/test/java/io/github/hectorvent/floci/services/sqs/SqsJsonProtocolTest.java diff --git a/.gitignore b/.gitignore index 2fe912b9..8583daf2 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ COPILOT.md .claude posts +local \ No newline at end of file diff --git a/src/main/java/io/github/hectorvent/floci/core/common/SqsQueueUrlRouterFilter.java b/src/main/java/io/github/hectorvent/floci/core/common/SqsQueueUrlRouterFilter.java new file mode 100644 index 00000000..d093e308 --- /dev/null +++ b/src/main/java/io/github/hectorvent/floci/core/common/SqsQueueUrlRouterFilter.java @@ -0,0 +1,65 @@ +package io.github.hectorvent.floci.core.common; + +import jakarta.ws.rs.container.ContainerRequestContext; +import jakarta.ws.rs.container.ContainerRequestFilter; +import jakarta.ws.rs.container.PreMatching; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.ext.Provider; + +import java.net.URI; +import java.util.regex.Pattern; + +/** + * Pre-matching filter that rewrites SQS JSON 1.0 requests sent to the queue URL path + * (/{accountId}/{queueName}) to POST / so they are handled by AwsJsonController. + *

+ * Newer AWS SDKs (e.g. aws-sdk-sqs Ruby gem >= 1.71) route operations to the queue URL + * rather than POST /. Without this filter, those requests match S3Controller's + * /{bucket}/{key:.+} handler and return NoSuchBucket errors. + */ +@Provider +@PreMatching +public class SqsQueueUrlRouterFilter implements ContainerRequestFilter { + + private static final Pattern QUEUE_PATH = Pattern.compile("^/([^/]+)/([^/]+)$"); + + @Override + public void filter(ContainerRequestContext ctx) { + + if (!"POST".equals(ctx.getMethod())) { + return; + } + + String path = ctx.getUriInfo().getPath(); + if (!QUEUE_PATH.matcher(path).matches()) { + return; + } + + MediaType mt = ctx.getMediaType(); + if (mt == null) { + return; + } + + boolean isSqsJson = "application".equals(mt.getType()) + && "x-amz-json-1.0".equals(mt.getSubtype()) + && isSqsTarget(ctx.getHeaderString("X-Amz-Target")); + + // S3 never receives form-encoded POSTs to /{bucket}/{key} paths — + // S3 presigned POST always goes to /{bucket}, not /{bucket}/{key}. + boolean isSqsQuery = "application".equals(mt.getType()) + && "x-www-form-urlencoded".equals(mt.getSubtype()); + + if (!isSqsJson && !isSqsQuery) { + return; + } + + URI rewritten = ctx.getUriInfo().getRequestUriBuilder() + .replacePath("/") + .build(); + ctx.setRequestUri(rewritten); + } + + private boolean isSqsTarget(String target) { + return target != null && target.startsWith("AmazonSQS."); + } +} diff --git a/src/test/java/io/github/hectorvent/floci/services/sqs/SqsJsonProtocolTest.java b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsJsonProtocolTest.java new file mode 100644 index 00000000..5f82011f --- /dev/null +++ b/src/test/java/io/github/hectorvent/floci/services/sqs/SqsJsonProtocolTest.java @@ -0,0 +1,166 @@ +package io.github.hectorvent.floci.services.sqs; + +import io.quarkus.test.junit.QuarkusTest; +import io.restassured.RestAssured; +import io.restassured.config.EncoderConfig; +import io.restassured.http.ContentType; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; + +import static io.restassured.RestAssured.given; +import static org.hamcrest.Matchers.*; + +/** + * Integration tests for the SQS JSON 1.0 protocol (application/x-amz-json-1.0). + * + * Covers two routing modes used by AWS SDKs: + * - Root path: POST / with X-Amz-Target header (older SDKs) + * - Queue URL path: POST /{accountId}/{queueName} with X-Amz-Target header + * (newer SDKs, e.g. aws-sdk-sqs Ruby gem >= 1.71) + */ +@QuarkusTest +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +class SqsJsonProtocolTest { + + private static final String CONTENT_TYPE = "application/x-amz-json-1.0"; + private static final String ACCOUNT_ID = "000000000000"; + private static final String QUEUE_NAME = "json-protocol-test-queue"; + + private static String queueUrl; + private static String receiptHandle; + + @BeforeAll + static void configureRestAssured() { + RestAssured.config = RestAssured.config().encoderConfig( + EncoderConfig.encoderConfig() + .encodeContentTypeAs(CONTENT_TYPE, ContentType.TEXT) + ); + } + + // --- Root-path JSON 1.0 (POST /) --- + + @Test + @Order(1) + void createQueueViaRootPath() { + String body = "{\"QueueName\":\"" + QUEUE_NAME + "\"}"; + + queueUrl = given() + .contentType(CONTENT_TYPE) + .header("X-Amz-Target", "AmazonSQS.CreateQueue") + .body(body) + .when() + .post("/") + .then() + .statusCode(200) + .body("QueueUrl", containsString(QUEUE_NAME)) + .extract().jsonPath().getString("QueueUrl"); + } + + @Test + @Order(2) + void getQueueAttributesViaRootPath() { + String body = "{\"QueueUrl\":\"" + queueUrl + "\",\"AttributeNames\":[\"All\"]}"; + + given() + .contentType(CONTENT_TYPE) + .header("X-Amz-Target", "AmazonSQS.GetQueueAttributes") + .body(body) + .when() + .post("/") + .then() + .statusCode(200) + .body("Attributes.QueueArn", notNullValue()); + } + + // --- Queue-URL-path JSON 1.0 (POST /{accountId}/{queueName}) --- + // Regression: these requests were previously routed to S3Controller, + // returning NoSuchBucket errors. + + @Test + @Order(3) + void sendMessageViaQueueUrlPath() { + String body = "{\"QueueUrl\":\"" + queueUrl + "\"," + + "\"MessageBody\":\"hello from json protocol test\"}"; + + receiptHandle = null; + + given() + .contentType(CONTENT_TYPE) + .header("X-Amz-Target", "AmazonSQS.SendMessage") + .body(body) + .when() + .post("/" + ACCOUNT_ID + "/" + QUEUE_NAME) + .then() + .statusCode(200) + .body("MessageId", notNullValue()) + .body("MD5OfMessageBody", notNullValue()); + } + + @Test + @Order(4) + void receiveMessageViaQueueUrlPath() { + String body = "{\"QueueUrl\":\"" + queueUrl + "\",\"MaxNumberOfMessages\":1}"; + + receiptHandle = given() + .contentType(CONTENT_TYPE) + .header("X-Amz-Target", "AmazonSQS.ReceiveMessage") + .body(body) + .when() + .post("/" + ACCOUNT_ID + "/" + QUEUE_NAME) + .then() + .statusCode(200) + .body("Messages", hasSize(1)) + .body("Messages[0].Body", equalTo("hello from json protocol test")) + .extract().jsonPath().getString("Messages[0].ReceiptHandle"); + } + + @Test + @Order(5) + void deleteMessageViaQueueUrlPath() { + String body = "{\"QueueUrl\":\"" + queueUrl + "\"," + + "\"ReceiptHandle\":\"" + receiptHandle + "\"}"; + + given() + .contentType(CONTENT_TYPE) + .header("X-Amz-Target", "AmazonSQS.DeleteMessage") + .body(body) + .when() + .post("/" + ACCOUNT_ID + "/" + QUEUE_NAME) + .then() + .statusCode(200); + } + + @Test + @Order(6) + void getQueueAttributesViaQueueUrlPath() { + String body = "{\"QueueUrl\":\"" + queueUrl + "\",\"AttributeNames\":[\"All\"]}"; + + given() + .contentType(CONTENT_TYPE) + .header("X-Amz-Target", "AmazonSQS.GetQueueAttributes") + .body(body) + .when() + .post("/" + ACCOUNT_ID + "/" + QUEUE_NAME) + .then() + .statusCode(200) + .body("Attributes.QueueArn", notNullValue()); + } + + @Test + @Order(7) + void deleteQueueViaQueueUrlPath() { + String body = "{\"QueueUrl\":\"" + queueUrl + "\"}"; + + given() + .contentType(CONTENT_TYPE) + .header("X-Amz-Target", "AmazonSQS.DeleteQueue") + .body(body) + .when() + .post("/" + ACCOUNT_ID + "/" + QUEUE_NAME) + .then() + .statusCode(200); + } +} From 2bb534f5fb86b175ad24fce22c8d884cb9292128 Mon Sep 17 00:00:00 2001 From: nakano16180 <36945685+nakano16180@users.noreply.github.com> Date: Wed, 1 Apr 2026 14:59:04 +0900 Subject: [PATCH 21/32] fix(dynamodb): fix FilterExpression for BOOL types, List/Set contains, and nested attribute paths (#137) * fix(dynamodb): fix FilterExpression handling for BOOL, List/Set contains, and nested attribute paths - Add BOOL type support to extractScalarValue() so comparison operators like <> work correctly with boolean attributes - Extend contains() to support L (List), SS (String Set), and NS (Number Set) types instead of only scalar string substring matching - Support dotted attribute paths in attribute_exists/attribute_not_exists by navigating through DynamoDB Map (M) structure Fixes #126 --- .../services/dynamodb/DynamoDbService.java | 150 ++++++++++++- .../dynamodb/DynamoDbServiceTest.java | 209 ++++++++++++++++++ 2 files changed, 350 insertions(+), 9 deletions(-) diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java index 41c8a0c5..cd823efd 100644 --- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java +++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java @@ -1072,15 +1072,13 @@ private boolean evaluateSingleCondition(JsonNode item, String condition, if (condLower.startsWith("attribute_exists")) { String attr = extractFunctionArg(condition); - String attrName = resolveAttributeName(attr, exprAttrNames); - // If item is null, attribute cannot exist - return item != null && item.has(attrName); + String resolvedPath = resolveAttributePath(attr, exprAttrNames); + return item != null && resolveNestedAttribute(item, resolvedPath) != null; } if (condLower.startsWith("attribute_not_exists")) { String attr = extractFunctionArg(condition); - String attrName = resolveAttributeName(attr, exprAttrNames); - // If item is null, attribute definitely doesn't exist - return item == null || !item.has(attrName); + String resolvedPath = resolveAttributePath(attr, exprAttrNames); + return item == null || resolveNestedAttribute(item, resolvedPath) == null; } if (condLower.startsWith("begins_with")) { String[] args = extractFunctionArgs(condition); @@ -1096,9 +1094,54 @@ private boolean evaluateSingleCondition(JsonNode item, String condition, String[] args = extractFunctionArgs(condition); if (args.length == 2) { String attrName = resolveAttributeName(args[0], exprAttrNames); - String substring = resolveExprValue(args[1], exprAttrValues); - String actual = item != null ? extractScalarValue(item.get(attrName)) : null; - return actual != null && substring != null && actual.contains(substring); + if (item == null) return false; + JsonNode attrNode = item.get(attrName); + if (attrNode == null) return false; + // Resolve the raw AttributeValue node for type-aware comparisons + JsonNode searchAttrValue = exprAttrValues != null + ? exprAttrValues.get(args[1].trim()) : null; + if (searchAttrValue == null) return false; + // List type: type-aware element membership check + if (attrNode.has("L")) { + for (JsonNode element : attrNode.get("L")) { + if (attributeValuesEqual(element, searchAttrValue)) return true; + } + return false; + } + // SS (String Set): operand must be S type + if (attrNode.has("SS")) { + if (!searchAttrValue.has("S")) return false; + String target = searchAttrValue.get("S").asText(); + for (JsonNode element : attrNode.get("SS")) { + if (target.equals(element.asText())) return true; + } + return false; + } + // NS (Number Set): operand must be N type, compare numerically + if (attrNode.has("NS")) { + if (!searchAttrValue.has("N")) return false; + try { + java.math.BigDecimal target = new java.math.BigDecimal(searchAttrValue.get("N").asText()); + for (JsonNode element : attrNode.get("NS")) { + if (target.compareTo(new java.math.BigDecimal(element.asText())) == 0) return true; + } + } catch (NumberFormatException ignored) {} + return false; + } + // BS (Binary Set): operand must be B type + if (attrNode.has("BS")) { + if (!searchAttrValue.has("B")) return false; + String target = searchAttrValue.get("B").asText(); + for (JsonNode element : attrNode.get("BS")) { + if (target.equals(element.asText())) return true; + } + return false; + } + // String type: operand must be S type, check substring + if (attrNode.has("S") && searchAttrValue.has("S")) { + return attrNode.get("S").asText().contains(searchAttrValue.get("S").asText()); + } + return false; } return false; } @@ -1139,6 +1182,52 @@ private String resolveExprValue(String placeholder, JsonNode exprAttrValues) { return placeholder; } + private boolean attributeValuesEqual(JsonNode a, JsonNode b) { + if (a == null || b == null) return a == b; + // Scalar types: S, B, BOOL, NULL + for (String type : new String[]{"S", "B", "BOOL", "NULL"}) { + if (a.has(type) && b.has(type)) { + return a.get(type).asText().equals(b.get(type).asText()); + } + if (a.has(type) || b.has(type)) return false; // type mismatch + } + // Numeric comparison with normalization + if (a.has("N") && b.has("N")) { + try { + return new java.math.BigDecimal(a.get("N").asText()) + .compareTo(new java.math.BigDecimal(b.get("N").asText())) == 0; + } catch (NumberFormatException e) { + return false; + } + } + if (a.has("N") || b.has("N")) return false; + // Map type: compare all entries recursively + if (a.has("M") && b.has("M")) { + JsonNode aMap = a.get("M"); + JsonNode bMap = b.get("M"); + if (aMap.size() != bMap.size()) return false; + var fields = aMap.fields(); + while (fields.hasNext()) { + var entry = fields.next(); + if (!bMap.has(entry.getKey())) return false; + if (!attributeValuesEqual(entry.getValue(), bMap.get(entry.getKey()))) return false; + } + return true; + } + // List type: compare element by element + if (a.has("L") && b.has("L")) { + JsonNode aList = a.get("L"); + JsonNode bList = b.get("L"); + if (aList.size() != bList.size()) return false; + for (int i = 0; i < aList.size(); i++) { + if (!attributeValuesEqual(aList.get(i), bList.get(i))) return false; + } + return true; + } + // Different types are never equal + return false; + } + private int compareValues(String a, String b) { // Try numeric comparison first try { @@ -1148,6 +1237,48 @@ private int compareValues(String a, String b) { } } + private static final String DOT_ESCAPE = "\uFF0E"; + + private String resolveAttributePath(String path, JsonNode exprAttrNames) { + // Resolve each segment of a dotted path, e.g. "passengerInformation.#name" + // ExpressionAttributeNames may resolve to names containing dots (e.g. "#a" -> "foo.bar"). + // Escape those dots so resolveNestedAttribute treats them as single keys. + String[] segments = path.split("\\."); + StringBuilder resolved = new StringBuilder(); + for (int i = 0; i < segments.length; i++) { + if (i > 0) resolved.append("."); + String original = segments[i]; + String resolvedSegment = resolveAttributeName(original, exprAttrNames); + if (original.startsWith("#") && resolvedSegment != null) { + resolvedSegment = resolvedSegment.replace(".", DOT_ESCAPE); + } + resolved.append(resolvedSegment); + } + return resolved.toString(); + } + + private JsonNode resolveNestedAttribute(JsonNode item, String path) { + // Navigate a dotted path through DynamoDB's {"M": {...}} structure + String[] segments = path.split("\\."); + JsonNode current = item; + for (int i = 0; i < segments.length; i++) { + if (current == null) return null; + String segment = segments[i].replace(DOT_ESCAPE, "."); + if (i == 0) { + // First segment: resolve against the top-level item map + current = current.get(segment); + } else { + // Subsequent segments: descend into DynamoDB Map type + if (current.has("M")) { + current = current.get("M").get(segment); + } else { + current = current.get(segment); + } + } + } + return current; + } + private String extractFunctionArg(String funcCall) { int open = funcCall.indexOf('('); int close = funcCall.lastIndexOf(')'); @@ -1230,6 +1361,7 @@ private String extractScalarValue(JsonNode attrValue) { if (attrValue.has("S")) return attrValue.get("S").asText(); if (attrValue.has("N")) return attrValue.get("N").asText(); if (attrValue.has("B")) return attrValue.get("B").asText(); + if (attrValue.has("BOOL")) return attrValue.get("BOOL").asText(); return attrValue.asText(); } diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java index efd103a7..bc2dfa71 100644 --- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java +++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java @@ -475,4 +475,213 @@ void updateItemSetIfNotExistsUsesFallbackWhenCheckAttrAbsentAndAttrNameDiffers() assertEquals("fallback", stored.get("target").get("S").asText(), "target should receive the fallback value when source is absent"); } + + @Test + void scanWithBoolFilterExpression() { + createUsersTable(); + ObjectNode u1 = item("userId", "u1"); + u1.set("deleted", boolAttributeValue(false)); + service.putItem("Users", u1); + + ObjectNode u2 = item("userId", "u2"); + u2.set("deleted", boolAttributeValue(true)); + service.putItem("Users", u2); + + ObjectNode u3 = item("userId", "u3"); + u3.set("deleted", boolAttributeValue(false)); + service.putItem("Users", u3); + + ObjectNode exprValues = mapper.createObjectNode(); + exprValues.set(":d", boolAttributeValue(true)); + + DynamoDbService.ScanResult result = service.scan("Users", "deleted <> :d", null, exprValues, null, null); + assertEquals(2, result.items().size()); + } + + @Test + void scanContainsOnListAttribute() { + createUsersTable(); + ObjectNode u1 = item("userId", "u1"); + u1.set("tags", listAttributeValue("a", "b")); + service.putItem("Users", u1); + + ObjectNode u2 = item("userId", "u2"); + u2.set("tags", listAttributeValue("a", "c")); + service.putItem("Users", u2); + + ObjectNode u3 = item("userId", "u3"); + u3.set("tags", listAttributeValue("b", "c")); + service.putItem("Users", u3); + + ObjectNode exprValues = mapper.createObjectNode(); + exprValues.set(":v", attributeValue("S", "a")); + + DynamoDbService.ScanResult result = service.scan("Users", "contains(tags, :v)", null, exprValues, null, null); + assertEquals(2, result.items().size()); + } + + @Test + void scanContainsOnStringSetAttribute() { + createUsersTable(); + ObjectNode u1 = item("userId", "u1"); + u1.set("roles", stringSetAttributeValue("admin", "user")); + service.putItem("Users", u1); + + ObjectNode u2 = item("userId", "u2"); + u2.set("roles", stringSetAttributeValue("user")); + service.putItem("Users", u2); + + ObjectNode exprValues = mapper.createObjectNode(); + exprValues.set(":r", attributeValue("S", "admin")); + + DynamoDbService.ScanResult result = service.scan("Users", "contains(roles, :r)", null, exprValues, null, null); + assertEquals(1, result.items().size()); + } + + @Test + void scanAttributeExistsOnNestedMapPath() { + createUsersTable(); + ObjectNode u1 = item("userId", "u1"); + u1.set("info", mapAttributeValue("name", "Alice")); + service.putItem("Users", u1); + + ObjectNode u2 = item("userId", "u2"); + ObjectNode emptyMap = mapper.createObjectNode(); + ObjectNode mapWrapper = mapper.createObjectNode(); + mapWrapper.set("M", emptyMap); + u2.set("info", mapWrapper); + service.putItem("Users", u2); + + ObjectNode u3 = item("userId", "u3"); + u3.set("info", mapAttributeValue("name", "Bob")); + service.putItem("Users", u3); + + ObjectNode exprNames = mapper.createObjectNode(); + exprNames.put("#n", "name"); + + DynamoDbService.ScanResult result = service.scan("Users", "attribute_exists(info.#n)", exprNames, null, null, null); + assertEquals(2, result.items().size()); + + DynamoDbService.ScanResult result2 = service.scan("Users", "attribute_not_exists(info.#n)", exprNames, null, null, null); + assertEquals(1, result2.items().size()); + } + + private ObjectNode boolAttributeValue(boolean value) { + ObjectNode node = mapper.createObjectNode(); + node.put("BOOL", value); + return node; + } + + private ObjectNode listAttributeValue(String... values) { + ObjectNode node = mapper.createObjectNode(); + var arrayNode = mapper.createArrayNode(); + for (String v : values) { + arrayNode.add(attributeValue("S", v)); + } + node.set("L", arrayNode); + return node; + } + + private ObjectNode stringSetAttributeValue(String... values) { + ObjectNode node = mapper.createObjectNode(); + var arrayNode = mapper.createArrayNode(); + for (String v : values) { + arrayNode.add(v); + } + node.set("SS", arrayNode); + return node; + } + + private ObjectNode mapAttributeValue(String key, String value) { + ObjectNode inner = mapper.createObjectNode(); + inner.set(key, attributeValue("S", value)); + ObjectNode node = mapper.createObjectNode(); + node.set("M", inner); + return node; + } + + private ObjectNode numberSetAttributeValue(String... values) { + ObjectNode node = mapper.createObjectNode(); + var arrayNode = mapper.createArrayNode(); + for (String v : values) { + arrayNode.add(v); + } + node.set("NS", arrayNode); + return node; + } + + private ObjectNode binarySetAttributeValue(String... base64Values) { + ObjectNode node = mapper.createObjectNode(); + var arrayNode = mapper.createArrayNode(); + for (String v : base64Values) { + arrayNode.add(v); + } + node.set("BS", arrayNode); + return node; + } + + @Test + void scanContainsOnNumberSetWithNumericNormalization() { + createUsersTable(); + ObjectNode u1 = item("userId", "u1"); + u1.set("scores", numberSetAttributeValue("1", "2", "3")); + service.putItem("Users", u1); + + ObjectNode u2 = item("userId", "u2"); + u2.set("scores", numberSetAttributeValue("4", "5")); + service.putItem("Users", u2); + + // Search for "1.0" — should match "1" via numeric comparison + ObjectNode exprValues = mapper.createObjectNode(); + exprValues.set(":v", attributeValue("N", "1.0")); + + DynamoDbService.ScanResult result = service.scan("Users", "contains(scores, :v)", null, exprValues, null, null); + assertEquals(1, result.items().size(), "contains() on NS should match 1.0 == 1 numerically"); + } + + @Test + void scanContainsOnBinarySet() { + createUsersTable(); + ObjectNode u1 = item("userId", "u1"); + u1.set("bins", binarySetAttributeValue("AQID", "BAUG")); // base64 for [1,2,3] and [4,5,6] + service.putItem("Users", u1); + + ObjectNode u2 = item("userId", "u2"); + u2.set("bins", binarySetAttributeValue("BwgJ")); + service.putItem("Users", u2); + + ObjectNode exprValues = mapper.createObjectNode(); + exprValues.set(":v", attributeValue("B", "AQID")); + + DynamoDbService.ScanResult result = service.scan("Users", "contains(bins, :v)", null, exprValues, null, null); + assertEquals(1, result.items().size()); + } + + @Test + void scanContainsOnListWithNumericElements() { + createUsersTable(); + ObjectNode u1 = item("userId", "u1"); + var list = mapper.createArrayNode(); + list.add(attributeValue("N", "10")); + list.add(attributeValue("N", "20")); + ObjectNode listNode = mapper.createObjectNode(); + listNode.set("L", list); + u1.set("values", listNode); + service.putItem("Users", u1); + + ObjectNode u2 = item("userId", "u2"); + var list2 = mapper.createArrayNode(); + list2.add(attributeValue("N", "30")); + ObjectNode listNode2 = mapper.createObjectNode(); + listNode2.set("L", list2); + u2.set("values", listNode2); + service.putItem("Users", u2); + + // Search for N:10.0 — should match N:10 via type-aware comparison + ObjectNode exprValues = mapper.createObjectNode(); + exprValues.set(":v", attributeValue("N", "10.0")); + + DynamoDbService.ScanResult result = service.scan("Users", "contains(values, :v)", null, exprValues, null, null); + assertEquals(1, result.items().size(), "contains() on List with N elements should use type-aware numeric comparison"); + } } From 06a4e6ae35bfa640810fc1ffce10529465f7efc9 Mon Sep 17 00:00:00 2001 From: yoyo Date: Wed, 1 Apr 2026 15:10:03 +0900 Subject: [PATCH 22/32] feat: support GSI and LSI in CloudFormation DynamoDB table provisioning (#125) --- .../CloudFormationResourceProvisioner.java | 46 +++++++++- .../CloudFormationIntegrationTest.java | 87 +++++++++++++++++++ .../dynamodb/DynamoDbIntegrationTest.java | 60 +++++++++++++ 3 files changed, 192 insertions(+), 1 deletion(-) diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationResourceProvisioner.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationResourceProvisioner.java index 66719c85..7efafb3e 100644 --- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationResourceProvisioner.java +++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationResourceProvisioner.java @@ -3,7 +3,9 @@ import io.github.hectorvent.floci.services.cloudformation.model.StackResource; import io.github.hectorvent.floci.services.dynamodb.DynamoDbService; import io.github.hectorvent.floci.services.dynamodb.model.AttributeDefinition; +import io.github.hectorvent.floci.services.dynamodb.model.GlobalSecondaryIndex; import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement; +import io.github.hectorvent.floci.services.dynamodb.model.LocalSecondaryIndex; import io.github.hectorvent.floci.services.iam.IamService; import io.github.hectorvent.floci.services.kms.KmsService; import io.github.hectorvent.floci.services.lambda.LambdaService; @@ -190,6 +192,8 @@ private void provisionDynamoTable(StackResource r, JsonNode props, CloudFormatio List keySchema = new ArrayList<>(); List attrDefs = new ArrayList<>(); + List gsis = new ArrayList<>(); + List lsis = new ArrayList<>(); if (props != null && props.has("KeySchema")) { for (JsonNode ks : props.get("KeySchema")) { @@ -206,12 +210,52 @@ private void provisionDynamoTable(StackResource r, JsonNode props, CloudFormatio } } + if (props != null && props.has("GlobalSecondaryIndexes")) { + for (JsonNode gsiNode : props.get("GlobalSecondaryIndexes")) { + String indexName = engine.resolve(gsiNode.get("IndexName")); + List gsiKeySchema = new ArrayList<>(); + if (gsiNode.has("KeySchema")) { + for (JsonNode ks : gsiNode.get("KeySchema")) { + String attrName = engine.resolve(ks.get("AttributeName")); + String keyType = engine.resolve(ks.get("KeyType")); + gsiKeySchema.add(new KeySchemaElement(attrName, keyType)); + } + } + String projectionType = "ALL"; + JsonNode projection = gsiNode.get("Projection"); + if (projection != null && projection.has("ProjectionType")) { + projectionType = engine.resolve(projection.get("ProjectionType")); + } + gsis.add(new GlobalSecondaryIndex(indexName, gsiKeySchema, null, projectionType)); + } + } + + if (props != null && props.has("LocalSecondaryIndexes")) { + for (JsonNode lsiNode : props.get("LocalSecondaryIndexes")) { + String indexName = engine.resolve(lsiNode.get("IndexName")); + List lsiKeySchema = new ArrayList<>(); + if (lsiNode.has("KeySchema")) { + for (JsonNode ks : lsiNode.get("KeySchema")) { + String attrName = engine.resolve(ks.get("AttributeName")); + String keyType = engine.resolve(ks.get("KeyType")); + lsiKeySchema.add(new KeySchemaElement(attrName, keyType)); + } + } + String projectionType = "ALL"; + JsonNode projection = lsiNode.get("Projection"); + if (projection != null && projection.has("ProjectionType")) { + projectionType = engine.resolve(projection.get("ProjectionType")); + } + lsis.add(new LocalSecondaryIndex(indexName, lsiKeySchema, null, projectionType)); + } + } + if (keySchema.isEmpty()) { keySchema.add(new KeySchemaElement("id", "HASH")); attrDefs.add(new AttributeDefinition("id", "S")); } - var table = dynamoDbService.createTable(tableName, keySchema, attrDefs, null, null, region); + var table = dynamoDbService.createTable(tableName, keySchema, attrDefs, null, null, gsis, lsis, region); r.setPhysicalId(tableName); r.getAttributes().put("Arn", table.getTableArn()); r.getAttributes().put("StreamArn", table.getTableArn() + "/stream/2024-01-01T00:00:00.000"); diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationIntegrationTest.java index 0756a765..3d4a3b02 100644 --- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationIntegrationTest.java +++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationIntegrationTest.java @@ -1,15 +1,29 @@ package io.github.hectorvent.floci.services.cloudformation; import io.quarkus.test.junit.QuarkusTest; +import io.restassured.RestAssured; +import io.restassured.config.EncoderConfig; +import io.restassured.http.ContentType; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import static io.restassured.RestAssured.given; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; @QuarkusTest class CloudFormationIntegrationTest { + private static final String DYNAMODB_CONTENT_TYPE = "application/x-amz-json-1.0"; + + @BeforeAll + static void configureRestAssured() { + RestAssured.config = RestAssured.config().encoderConfig( + EncoderConfig.encoderConfig() + .encodeContentTypeAs(DYNAMODB_CONTENT_TYPE, ContentType.TEXT)); + } + @Test void createStack_withS3AndSqs() { String template = """ @@ -75,6 +89,79 @@ void createStack_withS3AndSqs() { .body(containsString("CREATE_COMPLETE")); } + @Test + void createStack_withDynamoDbGsiAndLsi() { + String template = """ + { + "Resources": { + "MyTable": { + "Type": "AWS::DynamoDB::Table", + "Properties": { + "TableName": "cf-index-table", + "AttributeDefinitions": [ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + {"AttributeName": "gsiPk", "AttributeType": "S"} + ], + "KeySchema": [ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"} + ], + "GlobalSecondaryIndexes": [ + { + "IndexName": "gsi-1", + "KeySchema": [ + {"AttributeName": "gsiPk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"} + ], + "Projection": {"ProjectionType": "ALL"} + } + ], + "LocalSecondaryIndexes": [ + { + "IndexName": "lsi-1", + "KeySchema": [ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "gsiPk", "KeyType": "RANGE"} + ], + "Projection": {"ProjectionType": "KEYS_ONLY"} + } + ] + } + } + } + } + """; + + // 1. Create Stack + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "CreateStack") + .formParam("StackName", "test-dynamo-index-stack") + .formParam("TemplateBody", template) + .when() + .post("/") + .then() + .statusCode(200) + .body(containsString("")); + + // 2. Verify GSI and LSI via DescribeTable + given() + .header("X-Amz-Target", "DynamoDB_20120810.DescribeTable") + .contentType(DYNAMODB_CONTENT_TYPE) + .body(""" + {"TableName": "cf-index-table"} + """) + .when() + .post("/") + .then() + .statusCode(200) + .body("Table.GlobalSecondaryIndexes.size()", equalTo(1)) + .body("Table.GlobalSecondaryIndexes[0].IndexName", equalTo("gsi-1")) + .body("Table.LocalSecondaryIndexes.size()", equalTo(1)) + .body("Table.LocalSecondaryIndexes[0].IndexName", equalTo("lsi-1")); + } + @Test void deleteChangeSet_removesChangeSet() { String template = """ diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbIntegrationTest.java index 11d8fa7f..f2d22778 100644 --- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbIntegrationTest.java +++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbIntegrationTest.java @@ -78,6 +78,66 @@ void createDuplicateTableFails() { .body("__type", equalTo("ResourceInUseException")); } + @Test + void createTableWithGsiAndLsi() { + given() + .header("X-Amz-Target", "DynamoDB_20120810.CreateTable") + .contentType(DYNAMODB_CONTENT_TYPE) + .body(""" + { + "TableName": "IndexTable", + "KeySchema": [ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"} + ], + "AttributeDefinitions": [ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + {"AttributeName": "gsiPk", "AttributeType": "S"} + ], + "GlobalSecondaryIndexes": [ + { + "IndexName": "gsi-1", + "KeySchema": [ + {"AttributeName": "gsiPk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"} + ], + "Projection": {"ProjectionType": "ALL"} + } + ], + "LocalSecondaryIndexes": [ + { + "IndexName": "lsi-1", + "KeySchema": [ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "gsiPk", "KeyType": "RANGE"} + ], + "Projection": {"ProjectionType": "KEYS_ONLY"} + } + ] + } + """) + .when() + .post("/") + .then() + .statusCode(200) + .body("TableDescription.GlobalSecondaryIndexes.size()", equalTo(1)) + .body("TableDescription.GlobalSecondaryIndexes[0].IndexName", equalTo("gsi-1")) + .body("TableDescription.LocalSecondaryIndexes.size()", equalTo(1)) + .body("TableDescription.LocalSecondaryIndexes[0].IndexName", equalTo("lsi-1")); + + given() + .header("X-Amz-Target", "DynamoDB_20120810.DeleteTable") + .contentType(DYNAMODB_CONTENT_TYPE) + .body(""" + {"TableName": "IndexTable"} + """) + .when() + .post("/") + .then() + .statusCode(200); + } + @Test @Order(3) void describeTable() { From 2df8015650951cce704c8abd63f54e53a5c2fc02 Mon Sep 17 00:00:00 2001 From: Ilya Bryzgalov Date: Wed, 1 Apr 2026 08:24:47 +0200 Subject: [PATCH 23/32] feat: add support of Cloudformation mapping and Fn::FindInMap function (#101) --- .../cloudformation/CloudFormationService.java | 8 ++++-- .../CloudFormationTemplateEngine.java | 26 +++++++++++++++++++ .../CloudFormationIntegrationTest.java | 15 ++++++++++- 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java index 9093a628..4330cfc4 100644 --- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java +++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java @@ -200,6 +200,10 @@ private void executeTemplate(Stack stack, String templateBody, Map conditions = resolveConditions(template, resolvedParams, stack, region); + // Mappings + Map mappings = new HashMap<>(); + template.path("Mappings").fields().forEachRemaining(e -> mappings.put(e.getKey(), e.getValue())); + // Process resources in order JsonNode resources = template.path("Resources"); Map physicalIds = new LinkedHashMap<>(); @@ -224,7 +228,7 @@ private void executeTemplate(Stack stack, String templateBody, Map physicalIds; private final Map> resourceAttributes; private final Map conditions; + private final Map mappings; private final ObjectMapper objectMapper; CloudFormationTemplateEngine(String accountId, String region, String stackName, String stackId, @@ -34,6 +35,7 @@ public class CloudFormationTemplateEngine { Map physicalIds, Map> resourceAttributes, Map conditions, + Map mappings, ObjectMapper objectMapper) { this.accountId = accountId; this.region = region; @@ -43,6 +45,7 @@ public class CloudFormationTemplateEngine { this.physicalIds = physicalIds; this.resourceAttributes = resourceAttributes; this.conditions = conditions; + this.mappings = mappings; this.objectMapper = objectMapper; } @@ -87,6 +90,9 @@ public String resolve(JsonNode node) { if (node.has("Fn::ImportValue")) { return resolve(node.get("Fn::ImportValue")); } + if (node.has("Fn::FindInMap")) { + return resolveFindInMap(node.get("Fn::FindInMap")); + } } return node.asText(); } @@ -245,4 +251,24 @@ private String resolveGetAttParts(String logicalId, String attrName) { LOG.debugv("Unresolved GetAtt: {0}.{1}", logicalId, attrName); return logicalId + "." + attrName; } + + private String resolveFindInMap(JsonNode node) { + if (node.isArray()) { + String mapName = resolve(node.get(0)); + String topLvlName = resolve(node.get(1)); + String secondLvlName = resolve(node.get(2)); + + JsonNode map = mappings.get(mapName); + if (map != null && map.isObject()) { + JsonNode topLvl = map.get(topLvlName); + if (topLvl != null && topLvl.isObject()) { + JsonNode secondLvl = topLvl.get(secondLvlName); + if (secondLvl != null) { + return resolve(secondLvl); + } + } + } + } + return ""; + } } diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationIntegrationTest.java index 3d4a3b02..de037446 100644 --- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationIntegrationTest.java +++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationIntegrationTest.java @@ -28,11 +28,24 @@ static void configureRestAssured() { void createStack_withS3AndSqs() { String template = """ { + "Mappings": { + "Env": { + "us-east-1": { + "Name": "test" + } + } + }, "Resources": { "MyBucket": { "Type": "AWS::S3::Bucket", "Properties": { - "BucketName": "cf-test-bucket" + "BucketName": { + "Fn::Sub": ["cf-${env}-bucket", { + "env": { + "Fn::FindInMap": ["Env", { "Ref" : "AWS::Region" }, "Name"] + } + }] + } } }, "MyQueue": { From 1767c0c89371920a776174c68f81e7aee1555504 Mon Sep 17 00:00:00 2001 From: Jakub Date: Wed, 1 Apr 2026 00:27:19 -0600 Subject: [PATCH 24/32] feat: implement UploadPartCopy for S3 multipart uploads (#98) --- .../floci/services/s3/S3Controller.java | 25 +++++++ .../floci/services/s3/S3Service.java | 20 ++++++ .../s3/S3MultipartIntegrationTest.java | 65 +++++++++++++++++++ 3 files changed, 110 insertions(+) diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java index 7ed30201..3d4d67fa 100644 --- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java +++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java @@ -327,6 +327,9 @@ public Response putObject(@PathParam("bucket") String bucket, } if (uploadId != null && partNumber != null) { + if (copySource != null && !copySource.isEmpty()) { + return handleUploadPartCopy(copySource, bucket, key, uploadId, partNumber, httpHeaders); + } byte[] partData = decodeAwsChunked(body, contentEncoding, contentSha256); validateChecksumHeaders(httpHeaders, partData); String eTag = s3Service.uploadPart(bucket, key, uploadId, partNumber, partData); @@ -1102,6 +1105,28 @@ private Response handleCopyObject(String copySource, String destBucket, String d return Response.ok(xml).build(); } + private Response handleUploadPartCopy(String copySource, String destBucket, String destKey, + String uploadId, int partNumber, HttpHeaders httpHeaders) { + String source = copySource.startsWith("/") ? copySource.substring(1) : copySource; + int slashIndex = source.indexOf('/'); + if (slashIndex < 0) { + throw new AwsException("InvalidArgument", "Invalid copy source: " + copySource, 400); + } + String sourceBucket = source.substring(0, slashIndex); + String sourceKey = source.substring(slashIndex + 1); + String copySourceRange = httpHeaders.getHeaderString("x-amz-copy-source-range"); + String eTag = s3Service.uploadPartCopy(destBucket, destKey, uploadId, partNumber, + sourceBucket, sourceKey, copySourceRange); + String xml = new XmlBuilder() + .raw("") + .start("CopyPartResult", AwsNamespaces.S3) + .elem("LastModified", ISO_FORMAT.format(java.time.Instant.now())) + .elem("ETag", eTag) + .end("CopyPartResult") + .build(); + return Response.ok(xml).type(MediaType.APPLICATION_XML).build(); + } + private Response handleGetObjectAttributes(String bucket, String key, String versionId, String objectAttributesHeader, Integer maxParts, Integer partNumberMarker) { diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java index b94fb607..fa4381fd 100644 --- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java +++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java @@ -814,6 +814,26 @@ public String uploadPart(String bucket, String key, String uploadId, int partNum return eTag; } + public String uploadPartCopy(String destBucket, String destKey, String uploadId, int partNumber, + String sourceBucket, String sourceKey, String copySourceRange) { + S3Object source = getObject(sourceBucket, sourceKey); + byte[] data = source.getData(); + + if (copySourceRange != null && !copySourceRange.isBlank()) { + // format: "bytes=START-END" (inclusive on both ends) + String range = copySourceRange.startsWith("bytes=") ? copySourceRange.substring(6) : copySourceRange; + int dash = range.indexOf('-'); + if (dash < 0) { + throw new AwsException("InvalidArgument", "Invalid x-amz-copy-source-range: " + copySourceRange, 400); + } + int start = Integer.parseInt(range.substring(0, dash).trim()); + int end = Integer.parseInt(range.substring(dash + 1).trim()); + data = Arrays.copyOfRange(data, start, end + 1); + } + + return uploadPart(destBucket, destKey, uploadId, partNumber, data); + } + public S3Object completeMultipartUpload(String bucket, String key, String uploadId, List partNumbers) { MultipartUpload upload = multipartUploads.get(uploadId); if (upload == null || !upload.getBucket().equals(bucket) || !upload.getKey().equals(key)) { diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartIntegrationTest.java index eb678d8d..4c376a40 100644 --- a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartIntegrationTest.java +++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartIntegrationTest.java @@ -178,8 +178,73 @@ void abortMultipartUpload() { @Test @Order(11) + void uploadPartCopy() { + // Put a source object + given() + .body("ABCDEFGHIJ") + .when() + .put("/" + BUCKET + "/source-for-copy.bin") + .then() + .statusCode(200); + + // Initiate multipart upload for destination + String copyUploadId = given() + .when() + .post("/" + BUCKET + "/copy-dest.bin?uploads") + .then() + .statusCode(200) + .extract().xmlPath().getString("InitiateMultipartUploadResult.UploadId"); + + // UploadPartCopy full source + given() + .header("x-amz-copy-source", "/" + BUCKET + "/source-for-copy.bin") + .when() + .put("/" + BUCKET + "/copy-dest.bin?uploadId=" + copyUploadId + "&partNumber=1") + .then() + .statusCode(200) + .body(containsString("")); + + // UploadPartCopy with range (bytes 2-5 → "CDEF") + given() + .header("x-amz-copy-source", "/" + BUCKET + "/source-for-copy.bin") + .header("x-amz-copy-source-range", "bytes=2-5") + .when() + .put("/" + BUCKET + "/copy-dest.bin?uploadId=" + copyUploadId + "&partNumber=2") + .then() + .statusCode(200) + .body(containsString("")); + + // Complete the upload + String completeXml = """ + + 1etag1 + 2etag2 + """; + given() + .contentType("application/xml") + .body(completeXml) + .when() + .post("/" + BUCKET + "/copy-dest.bin?uploadId=" + copyUploadId) + .then() + .statusCode(200); + + // Verify contents: full source + ranged slice + given() + .when() + .get("/" + BUCKET + "/copy-dest.bin") + .then() + .statusCode(200) + .body(equalTo("ABCDEFGHIJCDEF")); + } + + @Test + @Order(12) void cleanUp() { given().when().delete("/" + BUCKET + "/" + KEY).then().statusCode(204); + given().when().delete("/" + BUCKET + "/source-for-copy.bin").then().statusCode(204); + given().when().delete("/" + BUCKET + "/copy-dest.bin").then().statusCode(204); given().when().delete("/" + BUCKET).then().statusCode(204); } } From a2b37c692a0cc3dfe8275dc1b69bc61c7bd52f8b Mon Sep 17 00:00:00 2001 From: Simone Paolo Petta <74307062+Simi24@users.noreply.github.com> Date: Wed, 1 Apr 2026 14:36:01 +0200 Subject: [PATCH 25/32] fix: defer startup hooks until HTTP server is ready (#157) (#159) Startup hooks that make HTTP calls back to floci (e.g. aws cli) deadlocked because they ran synchronously during StartupEvent, before the HTTP server bound to port 4566. Hooks now run on a virtual thread that polls for port readiness first. When no hooks are configured, behavior is unchanged. --- docs/configuration/initialization-hooks.md | 16 +++-- .../floci/lifecycle/EmulatorLifecycle.java | 41 ++++++++++- .../inithook/InitializationHooksRunner.java | 4 ++ .../lifecycle/EmulatorLifecycleTest.java | 72 +++++++++++++++++++ .../InitializationHooksRunnerTest.java | 34 +++++++++ 5 files changed, 160 insertions(+), 7 deletions(-) create mode 100644 src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java diff --git a/docs/configuration/initialization-hooks.md b/docs/configuration/initialization-hooks.md index 920fd9ed..eebe1a82 100644 --- a/docs/configuration/initialization-hooks.md +++ b/docs/configuration/initialization-hooks.md @@ -5,7 +5,7 @@ environment (creating buckets, populating data, configuring resources, etc.) or Hook scripts ending with `.sh` are discovered in the following directories: -- **Startup hooks** (`/etc/floci/init/start.d`) run after Floci services are initialized, but before the environment is marked as ready. +- **Startup hooks** (`/etc/floci/init/start.d`) run after the HTTP server is ready and accepting connections on port 4566. This means hooks can safely make HTTP calls back to Floci (e.g. using the AWS CLI). - **Shutdown hooks** (`/etc/floci/init/stop.d`) run when Floci is shutting down, after `destroy()` is triggered. If a hook directory does not exist or contains no `.sh` scripts, Floci skips it and continues normally. @@ -22,8 +22,15 @@ Hooks run: - With access to configured services and their endpoints - With the same environment variables as Floci -Hooks can call Floci service endpoints directly from inside the container. If a hook depends on additional CLI tools, -make sure those tools are available in the runtime image. +Hooks can call Floci service endpoints directly from inside the container (e.g. `http://localhost:4566`). +The published Docker image does not include the AWS CLI. If your hooks require it, extend the image: + +```dockerfile +FROM ghcr.io/hectorvent/floci:latest +RUN apk add --no-cache aws-cli +``` + +If a hook depends on additional CLI tools, make sure those tools are available in the runtime image. ### Execution Behavior @@ -38,7 +45,8 @@ Execution uses a fail-fast strategy: - If a script exits with a non-zero status, remaining hooks are not executed. - If a script exceeds the configured timeout, it is terminated and remaining hooks are not executed. -- A hook failure marks the corresponding startup or shutdown phase as **failed**. +- A startup hook failure triggers application shutdown. +- A shutdown hook failure is logged but does not prevent the shutdown from completing. ## Examples diff --git a/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java b/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java index 13fb7fbd..9e1aa5bc 100644 --- a/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java +++ b/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java @@ -7,6 +7,7 @@ import io.github.hectorvent.floci.lifecycle.inithook.InitializationHooksRunner; import io.github.hectorvent.floci.services.elasticache.proxy.ElastiCacheProxyManager; import io.github.hectorvent.floci.services.rds.proxy.RdsProxyManager; +import io.quarkus.runtime.Quarkus; import io.quarkus.runtime.ShutdownEvent; import io.quarkus.runtime.StartupEvent; import jakarta.enterprise.context.ApplicationScoped; @@ -15,11 +16,17 @@ import org.jboss.logging.Logger; import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Socket; @ApplicationScoped public class EmulatorLifecycle { private static final Logger LOG = Logger.getLogger(EmulatorLifecycle.class); + private static final int HTTP_PORT = 4566; + private static final int PORT_POLL_TIMEOUT_MS = 100; + private static final int PORT_POLL_INTERVAL_MS = 50; + private static final int PORT_POLL_MAX_RETRIES = 100; private final StorageFactory storageFactory; private final ServiceRegistry serviceRegistry; @@ -40,16 +47,44 @@ public EmulatorLifecycle(StorageFactory storageFactory, ServiceRegistry serviceR this.initializationHooksRunner = initializationHooksRunner; } - void onStart(@Observes StartupEvent ignored) throws IOException, InterruptedException { + void onStart(@Observes StartupEvent ignored) throws IOException { LOG.info("=== AWS Local Emulator Starting ==="); LOG.infov("Storage mode: {0}", config.storage().mode()); LOG.infov("Persistent path: {0}", config.storage().persistentPath()); serviceRegistry.logEnabledServices(); storageFactory.loadAll(); - initializationHooksRunner.run(InitializationHook.START); - LOG.info("=== AWS Local Emulator Ready ==="); + if (initializationHooksRunner.hasHooks(InitializationHook.START)) { + LOG.info("Startup hooks detected — deferring execution until HTTP server is ready"); + Thread.ofVirtual().name("init-hooks-runner").start(this::runStartupHooksAfterReady); + } else { + LOG.info("=== AWS Local Emulator Ready ==="); + } + } + + private void runStartupHooksAfterReady() { + try { + waitForHttpPort(); + initializationHooksRunner.run(InitializationHook.START); + LOG.info("=== AWS Local Emulator Ready ==="); + } catch (Exception e) { + LOG.error("Startup hook execution failed — shutting down", e); + Quarkus.asyncExit(); + } + } + + private static void waitForHttpPort() throws InterruptedException { + for (int attempt = 1; attempt <= PORT_POLL_MAX_RETRIES; attempt++) { + try (Socket socket = new Socket()) { + socket.connect(new InetSocketAddress("localhost", HTTP_PORT), PORT_POLL_TIMEOUT_MS); + LOG.debugv("HTTP port {0} is ready (attempt {1})", HTTP_PORT, attempt); + return; + } catch (IOException ignored) { + Thread.sleep(PORT_POLL_INTERVAL_MS); + } + } + throw new IllegalStateException("HTTP port " + HTTP_PORT + " did not become ready in time"); } void onStop(@Observes ShutdownEvent ignored) throws IOException, InterruptedException { diff --git a/src/main/java/io/github/hectorvent/floci/lifecycle/inithook/InitializationHooksRunner.java b/src/main/java/io/github/hectorvent/floci/lifecycle/inithook/InitializationHooksRunner.java index df8d1a2b..d6af7f70 100644 --- a/src/main/java/io/github/hectorvent/floci/lifecycle/inithook/InitializationHooksRunner.java +++ b/src/main/java/io/github/hectorvent/floci/lifecycle/inithook/InitializationHooksRunner.java @@ -44,6 +44,10 @@ private static String[] findScriptFileNames(final String hookName, final File ho return scriptFileNames; } + public boolean hasHooks(final InitializationHook hook) { + return findScriptFileNames(hook.getName(), hook.getPath()).length > 0; + } + public void run(final InitializationHook hook) throws IOException, InterruptedException { final String hookName = hook.getName(); final File hookDirectory = hook.getPath(); diff --git a/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java b/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java new file mode 100644 index 00000000..61b572d3 --- /dev/null +++ b/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java @@ -0,0 +1,72 @@ +package io.github.hectorvent.floci.lifecycle; + +import io.github.hectorvent.floci.config.EmulatorConfig; +import io.github.hectorvent.floci.core.common.ServiceRegistry; +import io.github.hectorvent.floci.core.storage.StorageFactory; +import io.github.hectorvent.floci.lifecycle.inithook.InitializationHook; +import io.github.hectorvent.floci.lifecycle.inithook.InitializationHooksRunner; +import io.github.hectorvent.floci.services.elasticache.proxy.ElastiCacheProxyManager; +import io.github.hectorvent.floci.services.rds.proxy.RdsProxyManager; +import io.quarkus.runtime.StartupEvent; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.io.IOException; + +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class EmulatorLifecycleTest { + + @Mock private StorageFactory storageFactory; + @Mock private ServiceRegistry serviceRegistry; + @Mock private EmulatorConfig config; + @Mock private EmulatorConfig.StorageConfig storageConfig; + @Mock private ElastiCacheProxyManager elastiCacheProxyManager; + @Mock private RdsProxyManager rdsProxyManager; + @Mock private InitializationHooksRunner initializationHooksRunner; + + private EmulatorLifecycle emulatorLifecycle; + + @BeforeEach + void setUp() { + when(config.storage()).thenReturn(storageConfig); + when(storageConfig.mode()).thenReturn("in-memory"); + when(storageConfig.persistentPath()).thenReturn("/app/data"); + emulatorLifecycle = new EmulatorLifecycle( + storageFactory, serviceRegistry, config, + elastiCacheProxyManager, rdsProxyManager, initializationHooksRunner); + } + + @Test + @DisplayName("Should log Ready immediately when no startup hooks exist") + void shouldLogReadyImmediatelyWhenNoHooksExist() throws IOException, InterruptedException { + when(initializationHooksRunner.hasHooks(InitializationHook.START)).thenReturn(false); + + emulatorLifecycle.onStart(Mockito.mock(StartupEvent.class)); + + verify(storageFactory).loadAll(); + verify(initializationHooksRunner).hasHooks(InitializationHook.START); + verify(initializationHooksRunner, never()).run(InitializationHook.START); + } + + @Test + @DisplayName("Should defer hook execution when startup hooks exist") + void shouldDeferHookExecutionWhenHooksExist() throws IOException, InterruptedException { + when(initializationHooksRunner.hasHooks(InitializationHook.START)).thenReturn(true); + + emulatorLifecycle.onStart(Mockito.mock(StartupEvent.class)); + + verify(storageFactory).loadAll(); + verify(initializationHooksRunner).hasHooks(InitializationHook.START); + // run() is NOT called synchronously — it will be called by the virtual thread + verify(initializationHooksRunner, never()).run(InitializationHook.START); + } +} diff --git a/src/test/java/io/github/hectorvent/floci/lifecycle/inithook/InitializationHooksRunnerTest.java b/src/test/java/io/github/hectorvent/floci/lifecycle/inithook/InitializationHooksRunnerTest.java index 7d1f5c02..9239ab36 100644 --- a/src/test/java/io/github/hectorvent/floci/lifecycle/inithook/InitializationHooksRunnerTest.java +++ b/src/test/java/io/github/hectorvent/floci/lifecycle/inithook/InitializationHooksRunnerTest.java @@ -4,6 +4,7 @@ import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; import org.mockito.ArgumentMatchers; import org.mockito.InjectMocks; import org.mockito.Mock; @@ -12,6 +13,8 @@ import java.io.File; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; @ExtendWith(MockitoExtension.class) class InitializationHooksRunnerTest { @@ -152,4 +155,35 @@ void shouldPropagateInterruptedExceptionFromHookScriptExecutor() throws IOExcept Assertions.assertSame(interruptedException, exception); } + @Test + @DisplayName("hasHooks should return true when scripts exist in hook directory") + void hasHooksShouldReturnTrueWhenScriptsExist(@TempDir Path tempDir) throws IOException { + Files.createFile(tempDir.resolve("01-setup.sh")); + InitializationHook hook = Mockito.mock(InitializationHook.class); + Mockito.when(hook.getName()).thenReturn("startup"); + Mockito.when(hook.getPath()).thenReturn(tempDir.toFile()); + + Assertions.assertTrue(initializationHooksRunner.hasHooks(hook)); + } + + @Test + @DisplayName("hasHooks should return false when hook directory is empty") + void hasHooksShouldReturnFalseWhenDirectoryIsEmpty(@TempDir Path tempDir) { + InitializationHook hook = Mockito.mock(InitializationHook.class); + Mockito.when(hook.getName()).thenReturn("startup"); + Mockito.when(hook.getPath()).thenReturn(tempDir.toFile()); + + Assertions.assertFalse(initializationHooksRunner.hasHooks(hook)); + } + + @Test + @DisplayName("hasHooks should return false when hook directory does not exist") + void hasHooksShouldReturnFalseWhenDirectoryDoesNotExist() { + InitializationHook hook = Mockito.mock(InitializationHook.class); + Mockito.when(hook.getName()).thenReturn("startup"); + Mockito.when(hook.getPath()).thenReturn(new File("/nonexistent/path")); + + Assertions.assertFalse(initializationHooksRunner.hasHooks(hook)); + } + } From 985783c13f498f4a18f9588f804cace2db3b1c33 Mon Sep 17 00:00:00 2001 From: Dixit R Jain Date: Wed, 1 Apr 2026 19:26:38 +0530 Subject: [PATCH 26/32] feat: officially support Docker named volumes for Native images (#155) * feat: officially support Docker named volumes for Native images * Pre-creates /app/data and assigns 1001 ownership in Dockerfile.native and Dockerfile.native-package before declaring VOLUME and dropping to USER 1001. This natively fixes the AccessDeniedException when utilizing Docker named volumes instead of local bind mounts, aligning container architecture flawlessly with tools like LocalStack. * Standardizes named volumes as officially documented optional alternatives across docker-compose templates in quick-start, index, and README documentation for a cleaner repository drop-in replacement footprint. * feat: natively support Docker named volumes via recursive permissions * Pre-creates /app/data and applies recursive 1001:root ownership and g+rwX permissions in Dockerfile.native and Dockerfile.native-package before declaring VOLUME and dropping to USER 1001. This elegantly resolves AccessDeniedException crashes when mapping Docker named volumes instead of local binds, natively supporting frictionless LocalStack-style drop-in architecture. * Standardizes named volumes as officially documented optional alternatives across docker-compose templates in quick-start, index, and README documentation for a cleaner repository drop-in replacement footprint. --- Dockerfile.native | 5 +++++ Dockerfile.native-package | 8 ++++---- README.md | 7 +++++++ docs/configuration/docker-compose.md | 23 ++++++++++++++++++++++- docs/getting-started/quick-start.md | 14 ++++++++++++++ docs/index.md | 7 +++++++ 6 files changed, 59 insertions(+), 5 deletions(-) diff --git a/Dockerfile.native b/Dockerfile.native index 6adf4de6..0c84c732 100644 --- a/Dockerfile.native +++ b/Dockerfile.native @@ -16,7 +16,11 @@ RUN mvn clean package -Dnative -DskipTests -B # Stage 2: Minimal runtime FROM quay.io/quarkus/quarkus-micro-image:2.0 +USER root WORKDIR /app +RUN mkdir -p /app/data \ + && chown -R 1001:root /app \ + && chmod -R g+rwX /app VOLUME /app/data EXPOSE 4566 @@ -27,4 +31,5 @@ ENV FLOCI_VERSION=${VERSION} COPY --from=build /app/target/*-runner /app/application RUN chmod +x /app/application +USER 1001 ENTRYPOINT ["/app/application"] diff --git a/Dockerfile.native-package b/Dockerfile.native-package index 3590ee36..10e60812 100644 --- a/Dockerfile.native-package +++ b/Dockerfile.native-package @@ -7,11 +7,11 @@ ENV FLOCI_VERSION=${VERSION} WORKDIR /app -VOLUME /app/data +RUN mkdir -p /app/data \ + && chown -R 1001:root /app \ + && chmod -R g+rwX /app -RUN chown 1001 /app \ - && chmod "g+rwX" /app \ - && chown 1001:root /app +VOLUME /app/data COPY --chown=1001:root target/*.properties target/*.so /app/ COPY --chown=1001:root target/*-runner /app/application diff --git a/README.md b/README.md index d69392aa..61ce8bd2 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,14 @@ services: ports: - "4566:4566" volumes: + # Local directory bind mount (default) - ./data:/app/data + + # OR named volume (optional): + # - floci-data:/app/data + +#volumes: +# floci-data: ``` ```bash diff --git a/docs/configuration/docker-compose.md b/docs/configuration/docker-compose.md index 15f16088..bb074dbf 100644 --- a/docs/configuration/docker-compose.md +++ b/docs/configuration/docker-compose.md @@ -11,9 +11,16 @@ services: ports: - "4566:4566" volumes: + # Local directory bind mount (default) - ./data:/app/data + + # OR named volume (optional): + # - floci-data:/app/data - ./init/start.d:/etc/floci/init/start.d:ro - ./init/stop.d:/etc/floci/init/stop.d:ro + +#volumes: +# floci-data: ``` ## Full Setup (with ElastiCache and RDS) @@ -29,10 +36,17 @@ services: - "6379-6399:6379-6399" # ElastiCache / Redis proxy ports - "7001-7099:7001-7099" # RDS / PostgreSQL + MySQL proxy ports volumes: - - /var/run/docker.sock:/var/run/docker.sock # required for Lambda, ElastiCache, RDS + # Local directory bind mount (default) - ./data:/app/data + + # OR named volume (optional): + # - floci-data:/app/data + - /var/run/docker.sock:/var/run/docker.sock # required for Lambda, ElastiCache, RDS environment: FLOCI_SERVICES_DOCKER_NETWORK: my-project_default # (1) + +#volumes: +# floci-data: ``` 1. Set this to the Docker network name that your compose project creates (usually `_default`). Floci uses it to attach spawned Lambda / ElastiCache / RDS containers to the same network. @@ -68,10 +82,17 @@ services: ports: - "4566:4566" volumes: + # Local directory bind mount (default) - ./data:/app/data + + # OR named volume (optional): + # - floci-data:/app/data environment: FLOCI_STORAGE_MODE: persistent FLOCI_STORAGE_PERSISTENT_PATH: /app/data + +#volumes: +# floci-data: ``` ## Environment Variables Reference diff --git a/docs/getting-started/quick-start.md b/docs/getting-started/quick-start.md index 85fc98be..845ce900 100644 --- a/docs/getting-started/quick-start.md +++ b/docs/getting-started/quick-start.md @@ -15,7 +15,14 @@ This guide gets Floci running and verifies that AWS CLI commands work against it ports: - "4566:4566" volumes: + # Local directory bind mount (default) - ./data:/app/data + + # OR named volume (optional): + # - floci-data:/app/data + +# volumes: +# floci-data: ``` ```bash @@ -33,7 +40,14 @@ This guide gets Floci running and verifies that AWS CLI commands work against it ports: - "4566:4566" volumes: + # Local directory bind mount (default) - ./data:/app/data + + # OR named volume (optional): + # - floci-data:/app/data + +# volumes: +# floci-data: ``` ```bash diff --git a/docs/index.md b/docs/index.md index 5ae2397a..af025b06 100644 --- a/docs/index.md +++ b/docs/index.md @@ -53,7 +53,14 @@ services: ports: - "4566:4566" volumes: + # Local directory bind mount (default) - ./data:/app/data + + # OR named volume (optional): + # - floci-data:/app/data + +#volumes: +# floci-data: ``` ```bash From 375f05a7e14dd29df8c9ef991b9f2b959d742617 Mon Sep 17 00:00:00 2001 From: yoyo Date: Thu, 2 Apr 2026 06:17:52 +0900 Subject: [PATCH 27/32] Feat/cfn auto physical name generation (#163) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor(cfn): add stackName param and generatePhysicalName utility Add stackName parameter to provision() method signature and pass it from CloudFormationService.executeTemplate(). Add generatePhysicalName helper that generates AWS-like names: {stackName}-{logicalId}-{suffix}. No resource naming behavior changed yet — this commit only introduces the infrastructure for the next step. * feat(cfn): use generatePhysicalName for all unnamed resources Replace cfn-xxx random fallback names with AWS-like naming pattern {stackName}-{logicalId}-{suffix} for 12 resource types: - S3 Bucket (lowercase, max 63) - SQS Queue (max 80) - SNS Topic (max 256) - DynamoDB Table/GlobalTable (max 255) - Lambda Function (max 64) - IAM Role (max 64), User (max 64), Policy (max 128), ManagedPolicy (max 128), InstanceProfile (max 128) - SSM Parameter (path-based: /cfn/{stackName}/{logicalId}) - SecretsManager Secret (max 512) - ECR Repository (lowercase, max 256) --- .../CloudFormationResourceProvisioner.java | 97 ++-- .../cloudformation/CloudFormationService.java | 2 +- .../CloudFormationIntegrationTest.java | 497 +++++++++++++++++- 3 files changed, 555 insertions(+), 41 deletions(-) diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationResourceProvisioner.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationResourceProvisioner.java index 7efafb3e..5776d493 100644 --- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationResourceProvisioner.java +++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationResourceProvisioner.java @@ -61,34 +61,35 @@ public CloudFormationResourceProvisioner(S3Service s3Service, SqsService sqsServ * Returns null and logs a warning for unsupported types. */ public StackResource provision(String logicalId, String resourceType, JsonNode properties, - CloudFormationTemplateEngine engine, String region, String accountId) { + CloudFormationTemplateEngine engine, String region, String accountId, + String stackName) { StackResource resource = new StackResource(); resource.setLogicalId(logicalId); resource.setResourceType(resourceType); try { switch (resourceType) { - case "AWS::S3::Bucket" -> provisionS3Bucket(resource, properties, engine, region, accountId); - case "AWS::SQS::Queue" -> provisionSqsQueue(resource, properties, engine, region, accountId); - case "AWS::SNS::Topic" -> provisionSnsTopic(resource, properties, engine, region, accountId); + case "AWS::S3::Bucket" -> provisionS3Bucket(resource, properties, engine, region, accountId, stackName); + case "AWS::SQS::Queue" -> provisionSqsQueue(resource, properties, engine, region, accountId, stackName); + case "AWS::SNS::Topic" -> provisionSnsTopic(resource, properties, engine, region, accountId, stackName); case "AWS::DynamoDB::Table", "AWS::DynamoDB::GlobalTable" -> - provisionDynamoTable(resource, properties, engine, region, accountId); - case "AWS::Lambda::Function" -> provisionLambda(resource, properties, engine, region, accountId); - case "AWS::IAM::Role" -> provisionIamRole(resource, properties, engine, accountId); - case "AWS::IAM::User" -> provisionIamUser(resource, properties, engine); + provisionDynamoTable(resource, properties, engine, region, accountId, stackName); + case "AWS::Lambda::Function" -> provisionLambda(resource, properties, engine, region, accountId, stackName); + case "AWS::IAM::Role" -> provisionIamRole(resource, properties, engine, accountId, stackName); + case "AWS::IAM::User" -> provisionIamUser(resource, properties, engine, stackName); case "AWS::IAM::AccessKey" -> provisionIamAccessKey(resource, properties, engine); case "AWS::IAM::Policy", "AWS::IAM::ManagedPolicy" -> - provisionIamPolicy(resource, properties, engine, accountId); - case "AWS::IAM::InstanceProfile" -> provisionInstanceProfile(resource, properties, engine, accountId); - case "AWS::SSM::Parameter" -> provisionSsmParameter(resource, properties, engine, region); + provisionIamPolicy(resource, properties, engine, accountId, stackName); + case "AWS::IAM::InstanceProfile" -> provisionInstanceProfile(resource, properties, engine, accountId, stackName); + case "AWS::SSM::Parameter" -> provisionSsmParameter(resource, properties, engine, region, stackName); case "AWS::KMS::Key" -> provisionKmsKey(resource, properties, engine, region, accountId); case "AWS::KMS::Alias" -> provisionKmsAlias(resource, properties, engine, region); - case "AWS::SecretsManager::Secret" -> provisionSecret(resource, properties, engine, region, accountId); + case "AWS::SecretsManager::Secret" -> provisionSecret(resource, properties, engine, region, accountId, stackName); case "AWS::CloudFormation::Stack" -> provisionNestedStack(resource, properties, engine, region); case "AWS::CDK::Metadata" -> provisionCdkMetadata(resource); case "AWS::S3::BucketPolicy" -> provisionS3BucketPolicy(resource, properties, engine); case "AWS::SQS::QueuePolicy" -> provisionSqsQueuePolicy(resource, properties, engine); - case "AWS::ECR::Repository" -> provisionEcrRepository(resource, properties, engine); + case "AWS::ECR::Repository" -> provisionEcrRepository(resource, properties, engine, stackName); case "AWS::Route53::HostedZone" -> provisionRoute53HostedZone(resource, properties, engine); case "AWS::Route53::RecordSet" -> provisionRoute53RecordSet(resource, properties, engine); default -> { @@ -133,10 +134,10 @@ public void delete(String resourceType, String physicalId, String region) { // ── S3 ──────────────────────────────────────────────────────────────────── private void provisionS3Bucket(StackResource r, JsonNode props, CloudFormationTemplateEngine engine, - String region, String accountId) { + String region, String accountId, String stackName) { String bucketName = resolveOptional(props, "BucketName", engine); if (bucketName == null || bucketName.isBlank()) { - bucketName = "cfn-" + UUID.randomUUID().toString().substring(0, 12).toLowerCase(); + bucketName = generatePhysicalName(stackName, r.getLogicalId(), 63, true); } s3Service.createBucket(bucketName, region); r.setPhysicalId(bucketName); @@ -150,10 +151,10 @@ private void provisionS3Bucket(StackResource r, JsonNode props, CloudFormationTe // ── SQS ─────────────────────────────────────────────────────────────────── private void provisionSqsQueue(StackResource r, JsonNode props, CloudFormationTemplateEngine engine, - String region, String accountId) { + String region, String accountId, String stackName) { String queueName = resolveOptional(props, "QueueName", engine); if (queueName == null || queueName.isBlank()) { - queueName = "cfn-" + UUID.randomUUID().toString().substring(0, 12); + queueName = generatePhysicalName(stackName, r.getLogicalId(), 80, false); } Map attrs = new HashMap<>(); if (props != null && props.has("VisibilityTimeout")) { @@ -170,10 +171,10 @@ private void provisionSqsQueue(StackResource r, JsonNode props, CloudFormationTe // ── SNS ─────────────────────────────────────────────────────────────────── private void provisionSnsTopic(StackResource r, JsonNode props, CloudFormationTemplateEngine engine, - String region, String accountId) { + String region, String accountId, String stackName) { String topicName = resolveOptional(props, "TopicName", engine); if (topicName == null || topicName.isBlank()) { - topicName = "cfn-" + UUID.randomUUID().toString().substring(0, 12); + topicName = generatePhysicalName(stackName, r.getLogicalId(), 256, false); } var topic = snsService.createTopic(topicName, Map.of(), Map.of(), region); r.setPhysicalId(topic.getTopicArn()); @@ -184,10 +185,10 @@ private void provisionSnsTopic(StackResource r, JsonNode props, CloudFormationTe // ── DynamoDB ────────────────────────────────────────────────────────────── private void provisionDynamoTable(StackResource r, JsonNode props, CloudFormationTemplateEngine engine, - String region, String accountId) { + String region, String accountId, String stackName) { String tableName = resolveOptional(props, "TableName", engine); if (tableName == null || tableName.isBlank()) { - tableName = "cfn-" + UUID.randomUUID().toString().substring(0, 12); + tableName = generatePhysicalName(stackName, r.getLogicalId(), 255, false); } List keySchema = new ArrayList<>(); @@ -264,10 +265,10 @@ private void provisionDynamoTable(StackResource r, JsonNode props, CloudFormatio // ── Lambda ──────────────────────────────────────────────────────────────── private void provisionLambda(StackResource r, JsonNode props, CloudFormationTemplateEngine engine, - String region, String accountId) { + String region, String accountId, String stackName) { String funcName = resolveOptional(props, "FunctionName", engine); if (funcName == null || funcName.isBlank()) { - funcName = "cfn-func-" + UUID.randomUUID().toString().substring(0, 8); + funcName = generatePhysicalName(stackName, r.getLogicalId(), 64, false); } Map req = new HashMap<>(); req.put("FunctionName", funcName); @@ -287,10 +288,10 @@ private void provisionLambda(StackResource r, JsonNode props, CloudFormationTemp // ── IAM Role ────────────────────────────────────────────────────────────── private void provisionIamRole(StackResource r, JsonNode props, CloudFormationTemplateEngine engine, - String accountId) { + String accountId, String stackName) { String roleName = resolveOptional(props, "RoleName", engine); if (roleName == null || roleName.isBlank()) { - roleName = "cfn-role-" + UUID.randomUUID().toString().substring(0, 8); + roleName = generatePhysicalName(stackName, r.getLogicalId(), 64, false); } String assumeDoc = props != null && props.has("AssumeRolePolicyDocument") ? props.get("AssumeRolePolicyDocument").toString() @@ -328,10 +329,10 @@ private void provisionIamRole(StackResource r, JsonNode props, CloudFormationTem // ── IAM Policy ──────────────────────────────────────────────────────────── private void provisionIamPolicy(StackResource r, JsonNode props, CloudFormationTemplateEngine engine, - String accountId) { + String accountId, String stackName) { String policyName = resolveOptional(props, "PolicyName", engine); if (policyName == null || policyName.isBlank()) { - policyName = "cfn-policy-" + UUID.randomUUID().toString().substring(0, 8); + policyName = generatePhysicalName(stackName, r.getLogicalId(), 128, false); } String document = props != null && props.has("PolicyDocument") ? props.get("PolicyDocument").toString() @@ -353,17 +354,17 @@ private void provisionIamPolicy(StackResource r, JsonNode props, CloudFormationT } private void provisionIamManagedPolicy(StackResource r, JsonNode props, CloudFormationTemplateEngine engine, - String accountId) { - provisionIamPolicy(r, props, engine, accountId); + String accountId, String stackName) { + provisionIamPolicy(r, props, engine, accountId, stackName); } // ── IAM Instance Profile ────────────────────────────────────────────────── private void provisionInstanceProfile(StackResource r, JsonNode props, CloudFormationTemplateEngine engine, - String accountId) { + String accountId, String stackName) { String name = resolveOptional(props, "InstanceProfileName", engine); if (name == null || name.isBlank()) { - name = "cfn-profile-" + UUID.randomUUID().toString().substring(0, 8); + name = generatePhysicalName(stackName, r.getLogicalId(), 128, false); } try { var profile = iamService.createInstanceProfile(name, "/"); @@ -378,10 +379,10 @@ private void provisionInstanceProfile(StackResource r, JsonNode props, CloudForm // ── SSM Parameter ───────────────────────────────────────────────────────── private void provisionSsmParameter(StackResource r, JsonNode props, CloudFormationTemplateEngine engine, - String region) { + String region, String stackName) { String name = resolveOptional(props, "Name", engine); if (name == null || name.isBlank()) { - name = "/cfn/" + UUID.randomUUID().toString().substring(0, 12); + name = generatePhysicalName(stackName, r.getLogicalId(), 2048, false); } String value = resolveOptional(props, "Value", engine); if (value == null) { @@ -419,10 +420,10 @@ private void provisionKmsAlias(StackResource r, JsonNode props, CloudFormationTe // ── Secrets Manager ─────────────────────────────────────────────────────── private void provisionSecret(StackResource r, JsonNode props, CloudFormationTemplateEngine engine, - String region, String accountId) { + String region, String accountId, String stackName) { String name = resolveOptional(props, "Name", engine); if (name == null || name.isBlank()) { - name = "cfn-secret-" + UUID.randomUUID().toString().substring(0, 8); + name = generatePhysicalName(stackName, r.getLogicalId(), 512, false); } String value = resolveOptional(props, "SecretString", engine); if (value == null) { @@ -460,10 +461,11 @@ private void provisionSqsQueuePolicy(StackResource r, JsonNode props, CloudForma r.setPhysicalId("queue-policy-" + UUID.randomUUID().toString().substring(0, 8)); } - private void provisionIamUser(StackResource r, JsonNode props, CloudFormationTemplateEngine engine) { + private void provisionIamUser(StackResource r, JsonNode props, CloudFormationTemplateEngine engine, + String stackName) { String userName = resolveOptional(props, "UserName", engine); if (userName == null || userName.isBlank()) { - userName = "cfn-user-" + UUID.randomUUID().toString().substring(0, 8); + userName = generatePhysicalName(stackName, r.getLogicalId(), 64, false); } var user = iamService.createUser(userName, "/"); r.setPhysicalId(userName); @@ -479,10 +481,11 @@ private void provisionIamAccessKey(StackResource r, JsonNode props, CloudFormati } } - private void provisionEcrRepository(StackResource r, JsonNode props, CloudFormationTemplateEngine engine) { + private void provisionEcrRepository(StackResource r, JsonNode props, CloudFormationTemplateEngine engine, + String stackName) { String repoName = resolveOptional(props, "RepositoryName", engine); if (repoName == null || repoName.isBlank()) { - repoName = "cfn-repo-" + UUID.randomUUID().toString().substring(0, 8); + repoName = generatePhysicalName(stackName, r.getLogicalId(), 256, true); } r.setPhysicalId(repoName); r.getAttributes().put("Arn", "arn:aws:ecr:us-east-1:000000000000:repository/" + repoName); @@ -527,4 +530,20 @@ private void deletePolicySafe(String policyArn) { LOG.debugv("Could not delete policy {0}: {1}", policyArn, e.getMessage()); } } + + /** + * Generate an AWS-like physical name: {stackName}-{logicalId}-{randomSuffix}. + * Mirrors the naming pattern AWS CloudFormation uses when no explicit name is provided. + */ + private String generatePhysicalName(String stackName, String logicalId, int maxLength, boolean lowercase) { + String suffix = UUID.randomUUID().toString().replace("-", "").substring(0, 12); + String name = stackName + "-" + logicalId + "-" + suffix; + if (lowercase) { + name = name.toLowerCase(); + } + if (maxLength > 0 && name.length() > maxLength) { + name = name.substring(0, maxLength); + } + return name; + } } diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java index 4330cfc4..e72f93be 100644 --- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java +++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationService.java @@ -240,7 +240,7 @@ private void executeTemplate(Stack stack, String templateBody, Map")); + + // 2. Verify stack completed and the auto-generated table name follows the pattern + var describeResponse = given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "DescribeStackResources") + .formParam("StackName", "auto-name-stack") + .when() + .post("/") + .then() + .statusCode(200) + .body(containsString("AWS::DynamoDB::Table")) + .body(containsString("auto-name-stack-MyTable-")) + .extract().asString(); + + // 3. Verify SSM Parameter was created with the auto-generated table name as value + given() + .header("X-Amz-Target", "AmazonSSM.GetParameter") + .contentType(SSM_CONTENT_TYPE) + .body(""" + {"Name": "/app/auto-table-name", "WithDecryption": true} + """) + .when() + .post("/") + .then() + .statusCode(200) + .body("Parameter.Name", equalTo("/app/auto-table-name")) + .body("Parameter.Value", startsWith("auto-name-stack-MyTable-")); + } + + @Test + void createStack_explicitNamesPreserved() { + // When explicit names are provided, CloudFormation uses them as-is. + // See: https://docs.aws.amazon.com/AWSCloudFormation/latest/TemplateReference/aws-properties-name.html + String template = """ + { + "Resources": { + "Bucket": { + "Type": "AWS::S3::Bucket", + "Properties": { + "BucketName": "my-explicit-bucket-name" + } + }, + "Queue": { + "Type": "AWS::SQS::Queue", + "Properties": { + "QueueName": "MyExplicitQueueName" + } + }, + "Table": { + "Type": "AWS::DynamoDB::Table", + "Properties": { + "TableName": "MyExplicitTableName", + "AttributeDefinitions": [ + {"AttributeName": "id", "AttributeType": "S"} + ], + "KeySchema": [ + {"AttributeName": "id", "KeyType": "HASH"} + ] + } + } + } + } + """; + + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "CreateStack") + .formParam("StackName", "explicit-names-stack") + .formParam("TemplateBody", template) + .when() + .post("/") + .then() + .statusCode(200) + .body(containsString("")); + + // Verify explicit names are used as-is in DescribeStackResources + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "DescribeStackResources") + .formParam("StackName", "explicit-names-stack") + .when() + .post("/") + .then() + .statusCode(200) + .body(containsString("my-explicit-bucket-name")) + .body(containsString("MyExplicitQueueName")) + .body(containsString("MyExplicitTableName")) + // Must NOT contain auto-generated pattern + .body(not(containsString("explicit-names-stack-Bucket-"))) + .body(not(containsString("explicit-names-stack-Queue-"))) + .body(not(containsString("explicit-names-stack-Table-"))); + } + + @Test + void createStack_s3AutoName_isLowercase() { + // S3 bucket names must be lowercase letters, numbers, periods, and hyphens (max 63 chars). + // See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html + String template = """ + { + "Resources": { + "MyUpperCaseBucket": { + "Type": "AWS::S3::Bucket", + "Properties": {} + } + } + } + """; + + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "CreateStack") + .formParam("StackName", "S3LowerCase-Stack") + .formParam("TemplateBody", template) + .when() + .post("/") + .then() + .statusCode(200); + + // The auto-generated name should be all lowercase: s3lowercase-stack-myuppercasebucket-... + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "DescribeStackResources") + .formParam("StackName", "S3LowerCase-Stack") + .when() + .post("/") + .then() + .statusCode(200) + .body(containsString("s3lowercase-stack-myuppercasebucket-")) + // Must not contain uppercase variants + .body(not(containsString("S3LowerCase-Stack-MyUpperCaseBucket-"))); + } + + @Test + void createStack_sqsAutoName_preservesCase() { + // SQS queue names preserve case. AWS example: mystack-myqueue-1VF9BKQH5BJVI + // See: https://docs.aws.amazon.com/AWSCloudFormation/latest/TemplateReference/aws-resource-sqs-queue.html + String template = """ + { + "Resources": { + "MyMixedCaseQueue": { + "Type": "AWS::SQS::Queue", + "Properties": {} + } + } + } + """; + + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "CreateStack") + .formParam("StackName", "CaseSensitive-Stack") + .formParam("TemplateBody", template) + .when() + .post("/") + .then() + .statusCode(200); + + // The SQS auto-generated name should preserve case: CaseSensitive-Stack-MyMixedCaseQueue-... + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "DescribeStackResources") + .formParam("StackName", "CaseSensitive-Stack") + .when() + .post("/") + .then() + .statusCode(200) + .body(containsString("CaseSensitive-Stack-MyMixedCaseQueue-")); + } + + @Test + void createStack_multipleUnnamedResources_uniqueNames() { + // Multiple resources of same type without names get unique auto-generated names + String template = """ + { + "Resources": { + "TableA": { + "Type": "AWS::DynamoDB::Table", + "Properties": { + "AttributeDefinitions": [ + {"AttributeName": "id", "AttributeType": "S"} + ], + "KeySchema": [ + {"AttributeName": "id", "KeyType": "HASH"} + ] + } + }, + "TableB": { + "Type": "AWS::DynamoDB::Table", + "Properties": { + "AttributeDefinitions": [ + {"AttributeName": "id", "AttributeType": "S"} + ], + "KeySchema": [ + {"AttributeName": "id", "KeyType": "HASH"} + ] + } + } + } + } + """; + + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "CreateStack") + .formParam("StackName", "multi-table-stack") + .formParam("TemplateBody", template) + .when() + .post("/") + .then() + .statusCode(200); + + // Both tables should have distinct names derived from their logical IDs + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "DescribeStackResources") + .formParam("StackName", "multi-table-stack") + .when() + .post("/") + .then() + .statusCode(200) + .body(containsString("multi-table-stack-TableA-")) + .body(containsString("multi-table-stack-TableB-")); + } + + @Test + void createStack_ssmAutoName_followsAwsPattern() { + // AWS SSM Parameter auto-name pattern: {stackName}-{logicalId}-{suffix} + // See: https://docs.aws.amazon.com/AWSCloudFormation/latest/TemplateReference/aws-resource-ssm-parameter.html + String template = """ + { + "Resources": { + "MyParam": { + "Type": "AWS::SSM::Parameter", + "Properties": { + "Type": "String", + "Value": "test-value" + } + } + } + } + """; + + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "CreateStack") + .formParam("StackName", "ssm-auto-stack") + .formParam("TemplateBody", template) + .when() + .post("/") + .then() + .statusCode(200); + + // SSM Parameter physical ID should follow {stackName}-{logicalId}-{suffix} pattern + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "DescribeStackResources") + .formParam("StackName", "ssm-auto-stack") + .when() + .post("/") + .then() + .statusCode(200) + .body(containsString("ssm-auto-stack-MyParam-")); + + // Verify SSM Parameter name via SSM API using DescribeStackResources physical ID + // We extract the auto-generated name from the stack resource and verify it's accessible + var ssmResourceXml = given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "DescribeStackResources") + .formParam("StackName", "ssm-auto-stack") + .when() + .post("/") + .then() + .statusCode(200) + .extract().asString(); + + // Extract the auto-generated parameter name from the XML response + String paramName = ssmResourceXml + .split("")[1] + .split("")[0]; + + given() + .header("X-Amz-Target", "AmazonSSM.GetParameter") + .contentType(SSM_CONTENT_TYPE) + .body("{\"Name\": \"" + paramName + "\", \"WithDecryption\": true}") + .when() + .post("/") + .then() + .statusCode(200) + .body("Parameter.Value", equalTo("test-value")); + } + + @Test + void createStack_getAttOnAutoNamedResource() { + // Fn::GetAtt should work on auto-named resources (e.g. DynamoDB Arn) + String template = """ + { + "Resources": { + "AutoTable": { + "Type": "AWS::DynamoDB::Table", + "Properties": { + "AttributeDefinitions": [ + {"AttributeName": "pk", "AttributeType": "S"} + ], + "KeySchema": [ + {"AttributeName": "pk", "KeyType": "HASH"} + ] + } + }, + "ArnParam": { + "Type": "AWS::SSM::Parameter", + "Properties": { + "Name": "/app/auto-table-arn", + "Type": "String", + "Value": {"Fn::GetAtt": ["AutoTable", "Arn"]} + } + } + }, + "Outputs": { + "TableArn": { + "Value": {"Fn::GetAtt": ["AutoTable", "Arn"]} + }, + "TableName": { + "Value": {"Ref": "AutoTable"} + } + } + } + """; + + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "CreateStack") + .formParam("StackName", "getatt-stack") + .formParam("TemplateBody", template) + .when() + .post("/") + .then() + .statusCode(200); + + // Verify Outputs contain the auto-generated name and ARN + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "DescribeStacks") + .formParam("StackName", "getatt-stack") + .when() + .post("/") + .then() + .statusCode(200) + .body(containsString("TableArn")) + .body(containsString("TableName")) + .body(containsString("getatt-stack-AutoTable-")); + + // Verify SSM Parameter received the Arn via GetAtt + given() + .header("X-Amz-Target", "AmazonSSM.GetParameter") + .contentType(SSM_CONTENT_TYPE) + .body(""" + {"Name": "/app/auto-table-arn", "WithDecryption": true} + """) + .when() + .post("/") + .then() + .statusCode(200) + .body("Parameter.Value", startsWith("arn:aws:dynamodb:")); + } + + @Test + void createStack_snsAutoName_refReturnsArn() { + // SNS Ref returns TopicArn. AWS example: arn:aws:sns:us-east-1:123456789012:mystack-mytopic-NZJ5JSMVGFIE + // See: https://docs.aws.amazon.com/AWSCloudFormation/latest/TemplateReference/aws-resource-sns-topic.html + String template = """ + { + "Resources": { + "MyTopic": { + "Type": "AWS::SNS::Topic", + "Properties": {} + } + }, + "Outputs": { + "TopicRef": { + "Value": {"Ref": "MyTopic"} + }, + "TopicArn": { + "Value": {"Fn::GetAtt": ["MyTopic", "TopicName"]} + } + } + } + """; + + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "CreateStack") + .formParam("StackName", "sns-auto-stack") + .formParam("TemplateBody", template) + .when() + .post("/") + .then() + .statusCode(200); + + // SNS Ref returns ARN (which contains the auto-generated topic name) + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "DescribeStacks") + .formParam("StackName", "sns-auto-stack") + .when() + .post("/") + .then() + .statusCode(200) + // Ref returns ARN containing the auto-generated name + .body(containsString("arn:aws:sns:")) + .body(containsString("sns-auto-stack-MyTopic-")); + } + + @Test + void createStack_ecrAutoName_isLowercase() { + // ECR repository names must be lowercase. + // See: https://docs.aws.amazon.com/AWSCloudFormation/latest/TemplateReference/aws-resource-ecr-repository.html + String template = """ + { + "Resources": { + "MyRepo": { + "Type": "AWS::ECR::Repository", + "Properties": {} + } + } + } + """; + + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "CreateStack") + .formParam("StackName", "ECR-Upper-Stack") + .formParam("TemplateBody", template) + .when() + .post("/") + .then() + .statusCode(200); + + // ECR auto-name should be lowercase + given() + .contentType("application/x-www-form-urlencoded") + .formParam("Action", "DescribeStackResources") + .formParam("StackName", "ECR-Upper-Stack") + .when() + .post("/") + .then() + .statusCode(200) + .body(containsString("ecr-upper-stack-myrepo-")) + .body(not(containsString("ECR-Upper-Stack-MyRepo-"))); + } } From 2c59c83d7081f38f89d671b582f54d315354d839 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fred=20Pe=C3=B1a?= Date: Wed, 1 Apr 2026 17:55:51 -0400 Subject: [PATCH 28/32] fix: support DynamoDB Query BETWEEN and ScanIndexForward=false (#160) * docs: expand community section with GitHub Discussions * docs: simplify supported services table * fix: support DynamoDB Query BETWEEN and ScanIndexForward=false * docs: simplify supported services table --- .../dynamodb/DynamoDbJsonHandler.java | 4 +- .../services/dynamodb/DynamoDbService.java | 27 ++++++++- .../dynamodb/DynamoDbIntegrationTest.java | 60 +++++++++++++++++-- .../dynamodb/DynamoDbServiceTest.java | 41 ++++++++++++- 4 files changed, 122 insertions(+), 10 deletions(-) diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java index 2af73357..4e5cddb6 100644 --- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java +++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java @@ -282,12 +282,14 @@ private Response handleQuery(JsonNode request, String region) { String filterExpr = request.has("FilterExpression") ? request.get("FilterExpression").asText() : null; Integer limit = request.has("Limit") ? request.get("Limit").asInt() : null; + Boolean scanIndexForward = request.has("ScanIndexForward") + ? request.get("ScanIndexForward").asBoolean() : null; String indexName = request.has("IndexName") ? request.get("IndexName").asText() : null; JsonNode exclusiveStartKey = request.has("ExclusiveStartKey") ? request.get("ExclusiveStartKey") : null; DynamoDbService.QueryResult result = dynamoDbService.query(tableName, keyConditions, - exprAttrValues, keyConditionExpr, filterExpr, limit, indexName, + exprAttrValues, keyConditionExpr, filterExpr, limit, scanIndexForward, indexName, exclusiveStartKey, exprAttrNames, region); ObjectNode response = objectMapper.createObjectNode(); diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java index cd823efd..e5f242f1 100644 --- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java +++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java @@ -18,6 +18,7 @@ import java.time.Instant; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -379,19 +380,19 @@ public QueryResult query(String tableName, JsonNode keyConditions, JsonNode expressionAttrValues, String keyConditionExpression, String filterExpression, Integer limit) { return query(tableName, keyConditions, expressionAttrValues, keyConditionExpression, - filterExpression, limit, null, null, null, regionResolver.getDefaultRegion()); + filterExpression, limit, null, null, null, null, regionResolver.getDefaultRegion()); } public QueryResult query(String tableName, JsonNode keyConditions, JsonNode expressionAttrValues, String keyConditionExpression, String filterExpression, Integer limit, String region) { return query(tableName, keyConditions, expressionAttrValues, keyConditionExpression, - filterExpression, limit, null, null, null, region); + filterExpression, limit, null, null, null, null, region); } public QueryResult query(String tableName, JsonNode keyConditions, JsonNode expressionAttrValues, String keyConditionExpression, - String filterExpression, Integer limit, String indexName, + String filterExpression, Integer limit, Boolean scanIndexForward, String indexName, JsonNode exclusiveStartKey, JsonNode exprAttrNames, String region) { String storageKey = regionKey(region, tableName); TableDefinition table = tableStore.get(storageKey) @@ -467,6 +468,9 @@ public QueryResult query(String tableName, JsonNode keyConditions, if (bVal == null) return 1; return compareValues(aVal, bVal); }); + if (Boolean.FALSE.equals(scanIndexForward)) { + Collections.reverse(results); + } } // Apply ExclusiveStartKey offset @@ -1459,6 +1463,23 @@ private boolean matchesSkExpression(JsonNode skValue, String expression, JsonNod return prefix != null && actual.startsWith(prefix); } + if (exprLower.contains(" between ")) { + int betweenIdx = exprLower.indexOf(" between "); + int andIdx = exprLower.indexOf(" and ", betweenIdx + " between ".length()); + if (andIdx < 0) return false; + + String lowerExpr = expression.substring(betweenIdx + " between ".length(), andIdx).trim(); + String upperExpr = expression.substring(andIdx + " and ".length()).trim(); + String lowerPlaceholder = lowerExpr.startsWith(":") ? lowerExpr.split("\\s+")[0] : null; + String upperPlaceholder = upperExpr.startsWith(":") ? upperExpr.split("\\s+")[0] : null; + String lower = lowerPlaceholder != null && exprValues != null + ? extractScalarValue(exprValues.get(lowerPlaceholder)) : null; + String upper = upperPlaceholder != null && exprValues != null + ? extractScalarValue(exprValues.get(upperPlaceholder)) : null; + if (lower == null || upper == null) return false; + return compareValues(actual, lower) >= 0 && compareValues(actual, upper) <= 0; + } + // Detect comparison operator String[] operators = {"<>", "<=", ">=", "=", "<", ">"}; for (String op : operators) { diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbIntegrationTest.java index f2d22778..c34aca70 100644 --- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbIntegrationTest.java +++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbIntegrationTest.java @@ -311,6 +311,58 @@ void queryWithBeginsWith() { @Test @Order(11) + void queryWithBetweenOnSortKey() { + given() + .header("X-Amz-Target", "DynamoDB_20120810.Query") + .contentType(DYNAMODB_CONTENT_TYPE) + .body(""" + { + "TableName": "TestTable", + "KeyConditionExpression": "pk = :pk AND sk BETWEEN :from AND :to", + "ExpressionAttributeValues": { + ":pk": {"S": "user-1"}, + ":from": {"S": "order-001"}, + ":to": {"S": "order-002"} + } + } + """) + .when() + .post("/") + .then() + .statusCode(200) + .body("Count", equalTo(2)) + .body("Items[0].sk.S", equalTo("order-001")) + .body("Items[1].sk.S", equalTo("order-002")); + } + + @Test + @Order(12) + void queryWithScanIndexForwardFalse() { + given() + .header("X-Amz-Target", "DynamoDB_20120810.Query") + .contentType(DYNAMODB_CONTENT_TYPE) + .body(""" + { + "TableName": "TestTable", + "KeyConditionExpression": "pk = :pk AND begins_with(sk, :prefix)", + "ScanIndexForward": false, + "ExpressionAttributeValues": { + ":pk": {"S": "user-1"}, + ":prefix": {"S": "order"} + } + } + """) + .when() + .post("/") + .then() + .statusCode(200) + .body("Count", equalTo(2)) + .body("Items[0].sk.S", equalTo("order-002")) + .body("Items[1].sk.S", equalTo("order-001")); + } + + @Test + @Order(13) void queryWithFilterExpression() { given() .header("X-Amz-Target", "DynamoDB_20120810.Query") @@ -336,7 +388,7 @@ void queryWithFilterExpression() { } @Test - @Order(12) + @Order(14) void queryWithFilterExpressionAndLimitReturnsLastEvaluatedKey() { given() .header("X-Amz-Target", "DynamoDB_20120810.Query") @@ -365,7 +417,7 @@ void queryWithFilterExpressionAndLimitReturnsLastEvaluatedKey() { } @Test - @Order(13) + @Order(15) void scan() { given() .header("X-Amz-Target", "DynamoDB_20120810.Scan") @@ -382,7 +434,7 @@ void scan() { } @Test - @Order(14) + @Order(16) void deleteItem() { given() .header("X-Amz-Target", "DynamoDB_20120810.DeleteItem") @@ -422,7 +474,7 @@ void deleteItem() { } @Test - @Order(15) + @Order(17) void deleteTable() { given() .header("X-Amz-Target", "DynamoDB_20120810.DeleteTable") diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java index bc2dfa71..f4973977 100644 --- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java +++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java @@ -220,6 +220,43 @@ void queryWithBeginsWith() { assertEquals(2, results.items().size()); } + @Test + void queryWithBetweenOnSortKey() { + createOrdersTable(); + service.putItem("Orders", item("customerId", "c1", "orderId", "2024-01-01")); + service.putItem("Orders", item("customerId", "c1", "orderId", "2024-01-15")); + service.putItem("Orders", item("customerId", "c1", "orderId", "2024-02-01")); + + ObjectNode exprValues = mapper.createObjectNode(); + exprValues.set(":pk", attributeValue("S", "c1")); + exprValues.set(":from", attributeValue("S", "2024-01-10")); + exprValues.set(":to", attributeValue("S", "2024-01-31")); + + DynamoDbService.QueryResult results = service.query("Orders", null, exprValues, + "customerId = :pk AND orderId BETWEEN :from AND :to", null, null); + + assertEquals(1, results.items().size()); + assertEquals("2024-01-15", results.items().getFirst().get("orderId").get("S").asText()); + } + + @Test + void queryWithScanIndexForwardFalseReturnsDescendingOrder() { + createOrdersTable(); + service.putItem("Orders", item("customerId", "c1", "orderId", "o1")); + service.putItem("Orders", item("customerId", "c1", "orderId", "o2")); + service.putItem("Orders", item("customerId", "c1", "orderId", "o3")); + + ObjectNode exprValues = mapper.createObjectNode(); + exprValues.set(":pk", attributeValue("S", "c1")); + + DynamoDbService.QueryResult results = service.query("Orders", null, exprValues, + "customerId = :pk", null, null, false, null, null, null, "us-east-1"); + + assertEquals(List.of("o3", "o2", "o1"), results.items().stream() + .map(result -> result.get("orderId").get("S").asText()) + .toList()); + } + @Test void queryAppliesFilterExpressionAfterKeyCondition() { createOrdersTable(); @@ -271,7 +308,7 @@ void queryWithFilterExpressionAndLimitUsesPreFilterPageState() { exprValues.set(":min", attributeValue("N", "100")); DynamoDbService.QueryResult firstPage = service.query("Orders", null, exprValues, - "customerId = :pk", "total >= :min", 2, null, null, null, "us-east-1"); + "customerId = :pk", "total >= :min", 2, null, null, null, null, "us-east-1"); assertEquals(1, firstPage.items().size()); assertEquals("o1", firstPage.items().get(0).get("orderId").get("S").asText()); @@ -280,7 +317,7 @@ void queryWithFilterExpressionAndLimitUsesPreFilterPageState() { assertEquals("o2", firstPage.lastEvaluatedKey().get("orderId").get("S").asText()); DynamoDbService.QueryResult secondPage = service.query("Orders", null, exprValues, - "customerId = :pk", "total >= :min", 2, null, + "customerId = :pk", "total >= :min", 2, null, null, firstPage.lastEvaluatedKey(), null, "us-east-1"); assertEquals(1, secondPage.items().size()); From 8c156816e50ca271404f46b5ecebaaaf6075c44b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=86=B0=E6=B0=B4=E6=B3=A1=E6=9E=B8=E6=9D=9E?= <87858694+xingzihai@users.noreply.github.com> Date: Thu, 2 Apr 2026 01:36:40 +0000 Subject: [PATCH 29/32] docs: fix initialization hooks Docker image names and AWS CLI instructions (#169) - Fix incorrect ghcr.io image reference to correct Docker Hub names - Add explanation of different image variants (native, JVM, AWS) - Provide clear options for users needing AWS CLI - Fix #167 --- docs/configuration/initialization-hooks.md | 27 ++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/docs/configuration/initialization-hooks.md b/docs/configuration/initialization-hooks.md index eebe1a82..31130aeb 100644 --- a/docs/configuration/initialization-hooks.md +++ b/docs/configuration/initialization-hooks.md @@ -23,13 +23,36 @@ Hooks run: - With the same environment variables as Floci Hooks can call Floci service endpoints directly from inside the container (e.g. `http://localhost:4566`). -The published Docker image does not include the AWS CLI. If your hooks require it, extend the image: + +The published Docker images are available on Docker Hub: + +- `hectorvent/floci:latest` — native image (minimal, no apk) +- `hectorvent/floci:latest-jvm` — JVM image (Alpine-based, has apk) +- `hectorvent/floci:latest-aws` — JVM image with AWS CLI pre-installed + +If your hooks require the AWS CLI, use one of these options: + +**Option 1: Use the pre-built AWS CLI image** + +```dockerfile +FROM hectorvent/floci:latest-aws +# AWS CLI is already installed +``` + +**Option 2: Extend the JVM image (Alpine-based)** ```dockerfile -FROM ghcr.io/hectorvent/floci:latest +FROM hectorvent/floci:latest-jvm RUN apk add --no-cache aws-cli ``` +**Option 3: Extend the JVM image with additional tools** + +```dockerfile +FROM hectorvent/floci:latest-jvm +RUN apk add --no-cache aws-cli jq curl +``` + If a hook depends on additional CLI tools, make sure those tools are available in the runtime image. ### Execution Behavior From e3e5897861810bca9ca1f4db82031bbf757a49bb Mon Sep 17 00:00:00 2001 From: Niklas Herder Date: Thu, 2 Apr 2026 15:49:03 +0200 Subject: [PATCH 30/32] docs: clean up named volume examples in docker-compose and quick-start docs (#161) Improve the Docker Compose documentation for named volumes and persistence section for clarity and consistency. Fix indentation in quick-start.md that broke MkDocs tabbed content rendering. --- docs/configuration/docker-compose.md | 43 ++++++++++++++-------------- docs/getting-started/quick-start.md | 16 +++++------ 2 files changed, 30 insertions(+), 29 deletions(-) diff --git a/docs/configuration/docker-compose.md b/docs/configuration/docker-compose.md index bb074dbf..b4055c21 100644 --- a/docs/configuration/docker-compose.md +++ b/docs/configuration/docker-compose.md @@ -11,16 +11,9 @@ services: ports: - "4566:4566" volumes: - # Local directory bind mount (default) - ./data:/app/data - - # OR named volume (optional): - # - floci-data:/app/data - ./init/start.d:/etc/floci/init/start.d:ro - ./init/stop.d:/etc/floci/init/stop.d:ro - -#volumes: -# floci-data: ``` ## Full Setup (with ElastiCache and RDS) @@ -36,17 +29,10 @@ services: - "6379-6399:6379-6399" # ElastiCache / Redis proxy ports - "7001-7099:7001-7099" # RDS / PostgreSQL + MySQL proxy ports volumes: - # Local directory bind mount (default) - - ./data:/app/data - - # OR named volume (optional): - # - floci-data:/app/data - /var/run/docker.sock:/var/run/docker.sock # required for Lambda, ElastiCache, RDS + - ./data:/app/data environment: FLOCI_SERVICES_DOCKER_NETWORK: my-project_default # (1) - -#volumes: -# floci-data: ``` 1. Set this to the Docker network name that your compose project creates (usually `_default`). Floci uses it to attach spawned Lambda / ElastiCache / RDS containers to the same network. @@ -82,19 +68,34 @@ services: ports: - "4566:4566" volumes: - # Local directory bind mount (default) - ./data:/app/data - - # OR named volume (optional): - # - floci-data:/app/data environment: FLOCI_STORAGE_MODE: persistent FLOCI_STORAGE_PERSISTENT_PATH: /app/data +``` -#volumes: -# floci-data: +### Using Named Volumes + +Instead of bind-mounting a local directory, you can use Docker named volumes to keep your project directory clean: + +```yaml +services: + floci: + image: hectorvent/floci:latest + ports: + - "4566:4566" + volumes: + - floci-data:/app/data + environment: + FLOCI_STORAGE_MODE: persistent + FLOCI_STORAGE_PERSISTENT_PATH: /app/data + +volumes: + floci-data: ``` +Named volumes are managed entirely by Docker and won't create files in your repository. This works with both the JVM and native images. + ## Environment Variables Reference All `application.yml` options can be overridden via environment variables using the `FLOCI_` prefix with underscores replacing dots and dashes: diff --git a/docs/getting-started/quick-start.md b/docs/getting-started/quick-start.md index 845ce900..3d298a8c 100644 --- a/docs/getting-started/quick-start.md +++ b/docs/getting-started/quick-start.md @@ -17,12 +17,12 @@ This guide gets Floci running and verifies that AWS CLI commands work against it volumes: # Local directory bind mount (default) - ./data:/app/data - + # OR named volume (optional): # - floci-data:/app/data - -# volumes: -# floci-data: + + # volumes: + # floci-data: ``` ```bash @@ -42,12 +42,12 @@ This guide gets Floci running and verifies that AWS CLI commands work against it volumes: # Local directory bind mount (default) - ./data:/app/data - + # OR named volume (optional): # - floci-data:/app/data - -# volumes: -# floci-data: + + # volumes: + # floci-data: ``` ```bash From 60ed53b7decd724ca668ab45a5ab7ca0d269fa6e Mon Sep 17 00:00:00 2001 From: Andreas Caravella Date: Thu, 2 Apr 2026 16:11:23 +0200 Subject: [PATCH 31/32] feat(dynamodb): add ScanFilter support for Scan operation Support the legacy ScanFilter parameter in DynamoDB Scan requests. ScanFilter is used by older AWS SDKs and provides per-attribute condition filtering (EQ, NE, LT, LE, GT, GE, BEGINS_WITH, CONTAINS, NOT_CONTAINS, IN, BETWEEN, NOT_NULL, NULL) via ComparisonOperator + AttributeValueList, as an alternative to FilterExpression. --- .../dynamodb/DynamoDbJsonHandler.java | 4 +- .../services/dynamodb/DynamoDbService.java | 37 ++++++++----- .../dynamodb/DynamoDbIntegrationTest.java | 52 ++++++++++++++++++- .../dynamodb/DynamoDbServiceTest.java | 49 +++++++++++++++-- 4 files changed, 124 insertions(+), 18 deletions(-) diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java index 4e5cddb6..bfe30901 100644 --- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java +++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java @@ -312,12 +312,14 @@ private Response handleScan(JsonNode request, String region) { ? request.get("ExpressionAttributeNames") : null; JsonNode exprAttrValues = request.has("ExpressionAttributeValues") ? request.get("ExpressionAttributeValues") : null; + JsonNode scanFilter = request.has("ScanFilter") + ? request.get("ScanFilter") : null; Integer limit = request.has("Limit") ? request.get("Limit").asInt() : null; JsonNode exclusiveStartKey = request.has("ExclusiveStartKey") ? request.get("ExclusiveStartKey") : null; DynamoDbService.ScanResult result = dynamoDbService.scan( - tableName, filterExpr, exprAttrNames, exprAttrValues, limit, exclusiveStartKey, region); + tableName, filterExpr, exprAttrNames, exprAttrValues, scanFilter, limit, exclusiveStartKey, region); ObjectNode response = objectMapper.createObjectNode(); ArrayNode itemsArray = objectMapper.createArrayNode(); diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java index e5f242f1..867f85c4 100644 --- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java +++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java @@ -513,21 +513,14 @@ public QueryResult query(String tableName, JsonNode keyConditions, public ScanResult scan(String tableName, String filterExpression, JsonNode expressionAttrNames, JsonNode expressionAttrValues, - Integer limit, String startKey) { + JsonNode scanFilter, Integer limit, JsonNode exclusiveStartKey) { return scan(tableName, filterExpression, expressionAttrNames, expressionAttrValues, - limit, (JsonNode) null, regionResolver.getDefaultRegion()); + scanFilter, limit, exclusiveStartKey, regionResolver.getDefaultRegion()); } public ScanResult scan(String tableName, String filterExpression, JsonNode expressionAttrNames, JsonNode expressionAttrValues, - Integer limit, String startKey, String region) { - return scan(tableName, filterExpression, expressionAttrNames, expressionAttrValues, - limit, (JsonNode) null, region); - } - - public ScanResult scan(String tableName, String filterExpression, - JsonNode expressionAttrNames, JsonNode expressionAttrValues, - Integer limit, JsonNode exclusiveStartKey, String region) { + JsonNode scanFilter, Integer limit, JsonNode exclusiveStartKey, String region) { String storageKey = regionKey(region, tableName); TableDefinition table = tableStore.get(storageKey) .orElseThrow(() -> resourceNotFoundException(tableName)); @@ -551,10 +544,14 @@ public ScanResult scan(String tableName, String filterExpression, if (isExpired(item, table)) { continue; } - if (filterExpression == null - || matchesFilterExpression(item, filterExpression, expressionAttrNames, expressionAttrValues)) { - results.add(item); + if (filterExpression != null + && !matchesFilterExpression(item, filterExpression, expressionAttrNames, expressionAttrValues)) { + continue; + } + if (scanFilter != null && !matchesScanFilter(item, scanFilter)) { + continue; } + results.add(item); } JsonNode lastEvaluatedKey = null; @@ -567,6 +564,20 @@ public ScanResult scan(String tableName, String filterExpression, return new ScanResult(results, totalScanned, lastEvaluatedKey); } + private boolean matchesScanFilter(JsonNode item, JsonNode scanFilter) { + Iterator> fields = scanFilter.fields(); + while (fields.hasNext()) { + var entry = fields.next(); + String attrName = entry.getKey(); + JsonNode condition = entry.getValue(); + JsonNode attrValue = item.get(attrName); + if (!matchesKeyCondition(attrValue, condition)) { + return false; + } + } + return true; + } + // --- Batch Operations --- public record BatchWriteResult(Map> unprocessedItems) {} diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbIntegrationTest.java index c34aca70..39bdcc0a 100644 --- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbIntegrationTest.java +++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbIntegrationTest.java @@ -435,6 +435,56 @@ void scan() { @Test @Order(16) + void scanWithScanFilter() { + given() + .header("X-Amz-Target", "DynamoDB_20120810.Scan") + .contentType(DYNAMODB_CONTENT_TYPE) + .body(""" + { + "TableName": "TestTable", + "ScanFilter": { + "name": { + "AttributeValueList": [{"S": "Alice"}], + "ComparisonOperator": "EQ" + } + } + } + """) + .when() + .post("/") + .then() + .statusCode(200) + .body("Count", equalTo(1)) + .body("Items[0].name.S", equalTo("Alice")); + } + + @Test + @Order(17) + void scanWithScanFilterGE() { + given() + .header("X-Amz-Target", "DynamoDB_20120810.Scan") + .contentType(DYNAMODB_CONTENT_TYPE) + .body(""" + { + "TableName": "TestTable", + "ScanFilter": { + "age": { + "AttributeValueList": [{"N": "30"}], + "ComparisonOperator": "GE" + } + } + } + """) + .when() + .post("/") + .then() + .statusCode(200) + .body("Count", equalTo(1)) + .body("Items[0].name.S", equalTo("Alice")); + } + + @Test + @Order(18) void deleteItem() { given() .header("X-Amz-Target", "DynamoDB_20120810.DeleteItem") @@ -474,7 +524,7 @@ void deleteItem() { } @Test - @Order(17) + @Order(19) void deleteTable() { given() .header("X-Amz-Target", "DynamoDB_20120810.DeleteTable") diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java index f4973977..8651eca8 100644 --- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java +++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java @@ -333,10 +333,53 @@ void scan() { service.putItem("Users", item("userId", "u2", "name", "Bob")); service.putItem("Users", item("userId", "u3", "name", "Charlie")); - DynamoDbService.ScanResult result = service.scan("Users", null, null, null, null, null); + DynamoDbService.ScanResult result = service.scan("Users", null, null, null, null, null, null); assertEquals(3, result.items().size()); } + @Test + void scanWithScanFilter() { + createUsersTable(); + service.putItem("Users", item("userId", "u1", "name", "Alice")); + service.putItem("Users", item("userId", "u2", "name", "Bob")); + service.putItem("Users", item("userId", "u3", "name", "Charlie")); + + ObjectNode scanFilter = mapper.createObjectNode(); + ObjectNode condition = mapper.createObjectNode(); + condition.put("ComparisonOperator", "EQ"); + var attrList = mapper.createArrayNode(); + ObjectNode val = mapper.createObjectNode(); + val.put("S", "Alice"); + attrList.add(val); + condition.set("AttributeValueList", attrList); + scanFilter.set("name", condition); + + DynamoDbService.ScanResult result = service.scan("Users", null, null, null, scanFilter, null, null); + assertEquals(1, result.items().size()); + assertEquals("Alice", result.items().get(0).get("name").get("S").asText()); + } + + @Test + void scanWithScanFilterGE() { + createUsersTable(); + service.putItem("Users", item("userId", "u1", "name", "Alice")); + service.putItem("Users", item("userId", "u2", "name", "Bob")); + service.putItem("Users", item("userId", "u3", "name", "Charlie")); + + ObjectNode scanFilter = mapper.createObjectNode(); + ObjectNode condition = mapper.createObjectNode(); + condition.put("ComparisonOperator", "GE"); + var attrList = mapper.createArrayNode(); + ObjectNode val = mapper.createObjectNode(); + val.put("S", "Bob"); + attrList.add(val); + condition.set("AttributeValueList", attrList); + scanFilter.set("name", condition); + + DynamoDbService.ScanResult result = service.scan("Users", null, null, null, scanFilter, null, null); + assertEquals(2, result.items().size()); + } + @Test void scanWithLimit() { createUsersTable(); @@ -344,7 +387,7 @@ void scanWithLimit() { service.putItem("Users", item("userId", "u2")); service.putItem("Users", item("userId", "u3")); - DynamoDbService.ScanResult result = service.scan("Users", null, null, null, 2, null); + DynamoDbService.ScanResult result = service.scan("Users", null, null, null, null, 2, null); assertEquals(2, result.items().size()); } @@ -354,7 +397,7 @@ void operationsOnNonExistentTableThrow() { assertThrows(AwsException.class, () -> service.getItem("NoTable", item("id", "1"))); assertThrows(AwsException.class, () -> service.deleteItem("NoTable", item("id", "1"))); assertThrows(AwsException.class, () -> service.query("NoTable", null, null, null, null, null)); - assertThrows(AwsException.class, () -> service.scan("NoTable", null, null, null, null, null)); + assertThrows(AwsException.class, () -> service.scan("NoTable", null, null, null, null, null, null, null)); } @Test From bdde1c420d2ba5ea86f7834612593d1db9741513 Mon Sep 17 00:00:00 2001 From: Andreas Caravella Date: Fri, 3 Apr 2026 10:55:12 +0200 Subject: [PATCH 32/32] fix(dynamodb): address review feedback for ScanFilter PR --- .../services/dynamodb/DynamoDbService.java | 1 + .../services/dynamodb/DynamoDbServiceTest.java | 18 +++++++++--------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java index 867f85c4..fb261b73 100644 --- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java +++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java @@ -1395,6 +1395,7 @@ private String extractComparisonValue(JsonNode condition) { return null; } + // NE, CONTAINS, NOT_CONTAINS, IN, NULL, NOT_NULL not yet supported private boolean matchesKeyCondition(JsonNode attrValue, JsonNode condition) { if (condition == null) return true; String op = condition.has("ComparisonOperator") ? condition.get("ComparisonOperator").asText() : "EQ"; diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java index 8651eca8..03f1ae72 100644 --- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java +++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java @@ -397,7 +397,7 @@ void operationsOnNonExistentTableThrow() { assertThrows(AwsException.class, () -> service.getItem("NoTable", item("id", "1"))); assertThrows(AwsException.class, () -> service.deleteItem("NoTable", item("id", "1"))); assertThrows(AwsException.class, () -> service.query("NoTable", null, null, null, null, null)); - assertThrows(AwsException.class, () -> service.scan("NoTable", null, null, null, null, null, null, null)); + assertThrows(AwsException.class, () -> service.scan("NoTable", null, null, null, null, null, null)); } @Test @@ -574,7 +574,7 @@ void scanWithBoolFilterExpression() { ObjectNode exprValues = mapper.createObjectNode(); exprValues.set(":d", boolAttributeValue(true)); - DynamoDbService.ScanResult result = service.scan("Users", "deleted <> :d", null, exprValues, null, null); + DynamoDbService.ScanResult result = service.scan("Users", "deleted <> :d", null, exprValues, null, null, null); assertEquals(2, result.items().size()); } @@ -596,7 +596,7 @@ void scanContainsOnListAttribute() { ObjectNode exprValues = mapper.createObjectNode(); exprValues.set(":v", attributeValue("S", "a")); - DynamoDbService.ScanResult result = service.scan("Users", "contains(tags, :v)", null, exprValues, null, null); + DynamoDbService.ScanResult result = service.scan("Users", "contains(tags, :v)", null, exprValues, null, null, null); assertEquals(2, result.items().size()); } @@ -614,7 +614,7 @@ void scanContainsOnStringSetAttribute() { ObjectNode exprValues = mapper.createObjectNode(); exprValues.set(":r", attributeValue("S", "admin")); - DynamoDbService.ScanResult result = service.scan("Users", "contains(roles, :r)", null, exprValues, null, null); + DynamoDbService.ScanResult result = service.scan("Users", "contains(roles, :r)", null, exprValues, null, null, null); assertEquals(1, result.items().size()); } @@ -639,10 +639,10 @@ void scanAttributeExistsOnNestedMapPath() { ObjectNode exprNames = mapper.createObjectNode(); exprNames.put("#n", "name"); - DynamoDbService.ScanResult result = service.scan("Users", "attribute_exists(info.#n)", exprNames, null, null, null); + DynamoDbService.ScanResult result = service.scan("Users", "attribute_exists(info.#n)", exprNames, null, null, null, null); assertEquals(2, result.items().size()); - DynamoDbService.ScanResult result2 = service.scan("Users", "attribute_not_exists(info.#n)", exprNames, null, null, null); + DynamoDbService.ScanResult result2 = service.scan("Users", "attribute_not_exists(info.#n)", exprNames, null, null, null, null); assertEquals(1, result2.items().size()); } @@ -715,7 +715,7 @@ void scanContainsOnNumberSetWithNumericNormalization() { ObjectNode exprValues = mapper.createObjectNode(); exprValues.set(":v", attributeValue("N", "1.0")); - DynamoDbService.ScanResult result = service.scan("Users", "contains(scores, :v)", null, exprValues, null, null); + DynamoDbService.ScanResult result = service.scan("Users", "contains(scores, :v)", null, exprValues, null, null, null); assertEquals(1, result.items().size(), "contains() on NS should match 1.0 == 1 numerically"); } @@ -733,7 +733,7 @@ void scanContainsOnBinarySet() { ObjectNode exprValues = mapper.createObjectNode(); exprValues.set(":v", attributeValue("B", "AQID")); - DynamoDbService.ScanResult result = service.scan("Users", "contains(bins, :v)", null, exprValues, null, null); + DynamoDbService.ScanResult result = service.scan("Users", "contains(bins, :v)", null, exprValues, null, null, null); assertEquals(1, result.items().size()); } @@ -761,7 +761,7 @@ void scanContainsOnListWithNumericElements() { ObjectNode exprValues = mapper.createObjectNode(); exprValues.set(":v", attributeValue("N", "10.0")); - DynamoDbService.ScanResult result = service.scan("Users", "contains(values, :v)", null, exprValues, null, null); + DynamoDbService.ScanResult result = service.scan("Users", "contains(values, :v)", null, exprValues, null, null, null); assertEquals(1, result.items().size(), "contains() on List with N elements should use type-aware numeric comparison"); } }