diff --git a/core/pom.xml b/core/pom.xml index 68d0353b..4cd10b47 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -98,6 +98,11 @@ com.google.classpath-explorer classpath-explorer + + com.amazonaws + aws-java-sdk-s3 + 1.12.500 + com.google.inject guice diff --git a/core/src/main/java/org/jsmart/zerocode/core/di/main/ApplicationMainModule.java b/core/src/main/java/org/jsmart/zerocode/core/di/main/ApplicationMainModule.java index f25460c9..907b0c11 100644 --- a/core/src/main/java/org/jsmart/zerocode/core/di/main/ApplicationMainModule.java +++ b/core/src/main/java/org/jsmart/zerocode/core/di/main/ApplicationMainModule.java @@ -74,6 +74,7 @@ public void configure() { bind(ZeroCodeExternalFileProcessor.class).to(ZeroCodeExternalFileProcessorImpl.class); bind(ZeroCodeParameterizedProcessor.class).to(ZeroCodeParameterizedProcessorImpl.class); bind(ZeroCodeSorter.class).to(ZeroCodeSorterImpl.class); + bind(org.jsmart.zerocode.core.s3.S3Client.class).to(org.jsmart.zerocode.core.s3.BasicS3Client.class); // ------------------------------------------------ // Bind properties for localhost, CI, DIT, SIT etc diff --git a/core/src/main/java/org/jsmart/zerocode/core/engine/executor/ApiServiceExecutor.java b/core/src/main/java/org/jsmart/zerocode/core/engine/executor/ApiServiceExecutor.java index 0351546e..354894f3 100644 --- a/core/src/main/java/org/jsmart/zerocode/core/engine/executor/ApiServiceExecutor.java +++ b/core/src/main/java/org/jsmart/zerocode/core/engine/executor/ApiServiceExecutor.java @@ -32,4 +32,14 @@ public interface ApiServiceExecutor { */ String executeKafkaService(String kafkaServers, String kafkaTopic, String methodName, String requestJson, ScenarioExecutionState scenarioExecutionState); + /** + * + * @param bucketName The name of the S3 bucket extracted from the url + * @param operation An S3 operation e.g. upload, download, list + * @param requestJson A json with s3 parameters + * @param scenarioExecutionState The state of the scenario execution + * @return String The S3 operation result in JSON + */ + String executeS3Service(String bucketName, String operation, String requestJson, ScenarioExecutionState scenarioExecutionState); + } diff --git a/core/src/main/java/org/jsmart/zerocode/core/engine/executor/ApiServiceExecutorImpl.java b/core/src/main/java/org/jsmart/zerocode/core/engine/executor/ApiServiceExecutorImpl.java index 8c8d36c1..d61c8e34 100644 --- a/core/src/main/java/org/jsmart/zerocode/core/engine/executor/ApiServiceExecutorImpl.java +++ b/core/src/main/java/org/jsmart/zerocode/core/engine/executor/ApiServiceExecutorImpl.java @@ -21,6 +21,9 @@ public class ApiServiceExecutorImpl implements ApiServiceExecutor { @Inject private BasicKafkaClient kafkaClient; + @Inject + private org.jsmart.zerocode.core.s3.S3Client s3Client; + @Inject(optional = true) @Named("mock.api.port") private int mockPort; @@ -58,4 +61,9 @@ public String executeJavaOperation(String className, String methodName, String r public String executeKafkaService(String kafkaServers, String kafkaTopic, String operation, String requestJson, ScenarioExecutionState scenarioExecutionState) { return kafkaClient.execute(kafkaServers, kafkaTopic, operation, requestJson, scenarioExecutionState); } + + @Override + public String executeS3Service(String bucketName, String operation, String requestJson, ScenarioExecutionState scenarioExecutionState) { + return s3Client.execute(bucketName, operation, requestJson, scenarioExecutionState); + } } diff --git a/core/src/main/java/org/jsmart/zerocode/core/runner/ZeroCodeMultiStepsScenarioRunnerImpl.java b/core/src/main/java/org/jsmart/zerocode/core/runner/ZeroCodeMultiStepsScenarioRunnerImpl.java index 2580c9cc..4a9e274c 100644 --- a/core/src/main/java/org/jsmart/zerocode/core/runner/ZeroCodeMultiStepsScenarioRunnerImpl.java +++ b/core/src/main/java/org/jsmart/zerocode/core/runner/ZeroCodeMultiStepsScenarioRunnerImpl.java @@ -489,6 +489,20 @@ private String executeApi(String logPrefixRelationshipId, executionResult = apiExecutor.executeKafkaService(kafkaServers, topicName, operationName, resolvedRequestJsonMaskRemoved, scenarioExecutionState); break; + case S3_CALL: + correlLogger.aRequestBuilder() + .relationshipId(logPrefixRelationshipId) + .requestTimeStamp(requestTimeStamp) + .step(thisStepName) + .url(url) + .method(operationName) + .id(stepId) + .request(prettyPrintJson(resolvedRequestJsonMaskApplied)); + + String bucketName = url.substring("s3-bucket:".length()); + executionResult = apiExecutor.executeS3Service(bucketName, operationName, resolvedRequestJsonMaskRemoved, scenarioExecutionState); + break; + case NONE: correlLogger.aRequestBuilder() .relationshipId(logPrefixRelationshipId) diff --git a/core/src/main/java/org/jsmart/zerocode/core/s3/BasicS3Client.java b/core/src/main/java/org/jsmart/zerocode/core/s3/BasicS3Client.java new file mode 100644 index 00000000..448b6ada --- /dev/null +++ b/core/src/main/java/org/jsmart/zerocode/core/s3/BasicS3Client.java @@ -0,0 +1,236 @@ +package org.jsmart.zerocode.core.s3; + +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.auth.BasicSessionCredentials; +import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.ListObjectsV2Request; +import com.amazonaws.services.s3.model.ListObjectsV2Result; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.inject.Inject; +import com.google.inject.name.Named; +import org.apache.commons.lang3.StringUtils; +import org.jsmart.zerocode.core.engine.preprocessor.ScenarioExecutionState; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.net.URL; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class BasicS3Client implements S3Client { + private static final Logger LOGGER = LoggerFactory.getLogger(BasicS3Client.class); + private static final String FAILED = "Failed"; + + @Inject + private ObjectMapper objectMapper; + + @Inject(optional = true) + @Named("s3.accessKey") + private String s3AccessKey; + + @Inject(optional = true) + @Named("s3.secretKey") + private String s3SecretKey; + + @Inject(optional = true) + @Named("s3.region") + private String s3Region; + + @Inject(optional = true) + @Named("s3.token") + private String s3Token; + + @Inject(optional = true) + @Named("s3.endpoint") + private String s3Endpoint; + + public BasicS3Client() { + } + + private AmazonS3 buildS3Client() { + AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard(); + + String envAccessKey = System.getenv("AWS_ACCESS_KEY_ID"); + String envSecretKey = System.getenv("AWS_SECRET_ACCESS_KEY"); + String envSessionToken = System.getenv("AWS_SESSION_TOKEN"); + String envRegion = System.getenv("AWS_REGION"); + + String finalAccessKey = StringUtils.isNotBlank(envAccessKey) ? envAccessKey : s3AccessKey; + String finalSecretKey = StringUtils.isNotBlank(envSecretKey) ? envSecretKey : s3SecretKey; + String finalToken = StringUtils.isNotBlank(envSessionToken) ? envSessionToken : s3Token; + String finalRegion = StringUtils.isNotBlank(envRegion) ? envRegion : s3Region; + + if (StringUtils.isNotBlank(finalAccessKey) && StringUtils.isNotBlank(finalSecretKey)) { + AWSCredentialsProvider credentialsProvider; + if (StringUtils.isNotBlank(finalToken)) { + credentialsProvider = new AWSStaticCredentialsProvider(new BasicSessionCredentials(finalAccessKey, finalSecretKey, finalToken)); + } else { + credentialsProvider = new AWSStaticCredentialsProvider(new BasicAWSCredentials(finalAccessKey, finalSecretKey)); + } + builder.withCredentials(credentialsProvider); + } else { + // fallback to default provider chain + builder.withCredentials(new DefaultAWSCredentialsProviderChain()); + } + + if (StringUtils.isNotBlank(s3Endpoint)) { + builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(s3Endpoint, finalRegion)); + } else if (StringUtils.isNotBlank(finalRegion)) { + builder.withRegion(finalRegion); + } + + return builder.build(); + } + + @Override + public String execute(String bucketName, String operation, String requestJson, ScenarioExecutionState scenarioExecutionState) { + LOGGER.debug("S3 operation: {}, bucket: {}, request: {}", operation, bucketName, requestJson); + + try { + AmazonS3 s3Client = buildS3Client(); + + String actionName = operation; + if (actionName.contains(".")) { + actionName = actionName.substring(actionName.indexOf('.') + 1); + } + S3Action s3Action = S3Action.fromString(actionName); + + JsonNode requestNode = null; + if (StringUtils.isNotBlank(requestJson)) { + requestNode = objectMapper.readTree(requestJson); + } + + switch (s3Action) { + case UPLOAD: + return handleUpload(s3Client, bucketName, requestNode); + case DOWNLOAD: + return handleDownload(s3Client, bucketName, requestNode); + case LIST: + return handleList(s3Client, bucketName, requestNode); + default: + throw new RuntimeException("Unsupported S3 operation: " + operation); + } + } catch (Throwable e) { + LOGGER.error("Error executing S3 operation: {}", e.getMessage(), e); + try { + Map errorResponse = new HashMap<>(); + errorResponse.put("status", FAILED); + errorResponse.put("message", e.getMessage()); + return objectMapper.writeValueAsString(errorResponse); + } catch (JsonProcessingException ex) { + throw new RuntimeException(e); + } + } + } + + private String handleUpload(AmazonS3 s3Client, String bucket, JsonNode requestNode) throws JsonProcessingException { + validateRequestNode(requestNode); + String key = getRequiredString(requestNode, "key"); + String filePath = getRequiredString(requestNode, "filePath"); + + File file = validateAndGetFile(filePath); + + s3Client.putObject(bucket, key, file); + + Map result = new HashMap<>(); + result.put("status", 200); + result.put("bucket", bucket); + result.put("key", key); + + return objectMapper.writeValueAsString(result); + } + + private String handleDownload(AmazonS3 s3Client, String bucket, JsonNode requestNode) throws JsonProcessingException { + validateRequestNode(requestNode); + String key = getRequiredString(requestNode, "key"); + String saveAs = getRequiredString(requestNode, "saveAs"); + + File outFile = new File(saveAs); + if (!outFile.isAbsolute() && !saveAs.startsWith("/")) { + outFile = new File("target/" + saveAs); + } + + File parent = outFile.getParentFile(); + if (parent != null && !parent.exists()) { + parent.mkdirs(); + } + + s3Client.getObject(new com.amazonaws.services.s3.model.GetObjectRequest(bucket, key), outFile); + + Map result = new HashMap<>(); + result.put("downloaded", true); + result.put("savedTo", outFile.getAbsolutePath()); + + return objectMapper.writeValueAsString(result); + } + + private String handleList(AmazonS3 s3Client, String bucket, JsonNode requestNode) throws JsonProcessingException { + String folder = null; + if (requestNode != null && requestNode.has("folder") && !requestNode.get("folder").isNull()) { + folder = requestNode.get("folder").asText(); + } + + ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(bucket); + if (StringUtils.isNotBlank(folder)) { + req.withPrefix(folder); + } + + ListObjectsV2Result objectListing = s3Client.listObjectsV2(req); + List> files = new ArrayList<>(); + + for (S3ObjectSummary summary : objectListing.getObjectSummaries()) { + Map fileInfo = new HashMap<>(); + fileInfo.put("key", summary.getKey()); + fileInfo.put("size", summary.getSize()); + fileInfo.put("lastModified", summary.getLastModified().getTime()); + files.add(fileInfo); + } + + Map result = new HashMap<>(); + result.put("files", files); + result.put("files.SIZE", files.size()); + + return objectMapper.writeValueAsString(result); + } + + private void validateRequestNode(JsonNode requestNode) { + if (requestNode == null) { + throw new IllegalArgumentException("Missing required request body for S3 operation."); + } + } + + private String getRequiredString(JsonNode node, String fieldName) { + if (!node.has(fieldName) || node.get(fieldName).isNull()) { + throw new IllegalArgumentException("Missing required field: " + fieldName); + } + return node.get(fieldName).asText(); + } + + private File validateAndGetFile(String fileName) { + File absoluteFile = new File(fileName); + if (absoluteFile.exists()) { + return absoluteFile; + } + + try { + URL resource = getClass().getClassLoader().getResource(fileName); + if (resource == null) { + throw new IllegalArgumentException("File does not exist: " + fileName); + } + return new File(resource.getFile()); + } catch (Exception ex) { + throw new RuntimeException("Error accessing file: `" + fileName + "' - " + ex); + } + } +} diff --git a/core/src/main/java/org/jsmart/zerocode/core/s3/S3Action.java b/core/src/main/java/org/jsmart/zerocode/core/s3/S3Action.java new file mode 100644 index 00000000..c45c6741 --- /dev/null +++ b/core/src/main/java/org/jsmart/zerocode/core/s3/S3Action.java @@ -0,0 +1,26 @@ +package org.jsmart.zerocode.core.s3; + +public enum S3Action { + UPLOAD("UPLOAD"), + DOWNLOAD("DOWNLOAD"), + LIST("LIST"); + + private final String action; + + S3Action(String action) { + this.action = action; + } + + public String getAction() { + return action; + } + + public static S3Action fromString(String action) { + for (S3Action s3Action : values()) { + if (s3Action.action.equalsIgnoreCase(action)) { + return s3Action; + } + } + throw new IllegalArgumentException("Unknown S3 action: " + action); + } +} diff --git a/core/src/main/java/org/jsmart/zerocode/core/s3/S3Client.java b/core/src/main/java/org/jsmart/zerocode/core/s3/S3Client.java new file mode 100644 index 00000000..6d2c8689 --- /dev/null +++ b/core/src/main/java/org/jsmart/zerocode/core/s3/S3Client.java @@ -0,0 +1,7 @@ +package org.jsmart.zerocode.core.s3; + +import org.jsmart.zerocode.core.engine.preprocessor.ScenarioExecutionState; + +public interface S3Client { + String execute(String bucketName, String operation, String requestJson, ScenarioExecutionState scenarioExecutionState); +} diff --git a/core/src/main/java/org/jsmart/zerocode/core/utils/ApiType.java b/core/src/main/java/org/jsmart/zerocode/core/utils/ApiType.java index 1cf8430a..0813604e 100644 --- a/core/src/main/java/org/jsmart/zerocode/core/utils/ApiType.java +++ b/core/src/main/java/org/jsmart/zerocode/core/utils/ApiType.java @@ -3,6 +3,7 @@ public enum ApiType { REST_CALL, KAFKA_CALL, + S3_CALL, JAVA_CALL, NONE; } diff --git a/core/src/main/java/org/jsmart/zerocode/core/utils/ApiTypeUtils.java b/core/src/main/java/org/jsmart/zerocode/core/utils/ApiTypeUtils.java index 293f62bb..e6e4b07c 100644 --- a/core/src/main/java/org/jsmart/zerocode/core/utils/ApiTypeUtils.java +++ b/core/src/main/java/org/jsmart/zerocode/core/utils/ApiTypeUtils.java @@ -42,6 +42,9 @@ public static ApiType apiType(String serviceName, String methodName) { } else if (serviceName != null && serviceName.contains(KAFKA)) { apiType = ApiType.KAFKA_CALL; + } else if (serviceName != null && serviceName.startsWith("s3-bucket:")) { + apiType = ApiType.S3_CALL; + } else { apiType = ApiType.JAVA_CALL; diff --git a/core/src/test/java/org/jsmart/zerocode/core/s3/BasicS3ClientTest.java b/core/src/test/java/org/jsmart/zerocode/core/s3/BasicS3ClientTest.java new file mode 100644 index 00000000..2f72d82e --- /dev/null +++ b/core/src/test/java/org/jsmart/zerocode/core/s3/BasicS3ClientTest.java @@ -0,0 +1,26 @@ +package org.jsmart.zerocode.core.s3; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.jsmart.zerocode.core.di.main.ApplicationMainModule; +import org.jsmart.zerocode.core.engine.preprocessor.ScenarioExecutionState; +import org.junit.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; + +public class BasicS3ClientTest { + + @Test + public void testS3ActionParsing() { + S3Action upload = S3Action.fromString("UPLOAD"); + assertThat(upload, is(S3Action.UPLOAD)); + + S3Action list = S3Action.fromString("list"); + assertThat(list, is(S3Action.LIST)); + } + + @Test(expected = IllegalArgumentException.class) + public void testS3ActionParsing_Invalid() { + S3Action.fromString("INVALID"); + } +} diff --git a/core/src/test/java/org/jsmart/zerocode/core/utils/ApiTypeUtilsTest.java b/core/src/test/java/org/jsmart/zerocode/core/utils/ApiTypeUtilsTest.java index 6efbd98a..54f2ccb4 100644 --- a/core/src/test/java/org/jsmart/zerocode/core/utils/ApiTypeUtilsTest.java +++ b/core/src/test/java/org/jsmart/zerocode/core/utils/ApiTypeUtilsTest.java @@ -46,4 +46,16 @@ public void testJavaApiProtoMappings_nullMappings() { String qualifiedClass = apiTypeUtils.getQualifiedJavaApi("foo://v1/s1"); } + @Test + public void testApiTypeS3() { + ApiType apiType = ApiTypeUtils.apiType("s3-bucket:test-bucket", "upload"); + assertThat(apiType, is(ApiType.S3_CALL)); + + apiType = ApiTypeUtils.apiType("s3-bucket:another", "download"); + assertThat(apiType, is(ApiType.S3_CALL)); + + apiType = ApiTypeUtils.apiType("s3-bucket:", "list"); + assertThat(apiType, is(ApiType.S3_CALL)); + } + } \ No newline at end of file diff --git a/core/src/test/java/org/jsmart/zerocode/integrationtests/s3/S3IntegrationTest.java b/core/src/test/java/org/jsmart/zerocode/integrationtests/s3/S3IntegrationTest.java new file mode 100644 index 00000000..c5abdeee --- /dev/null +++ b/core/src/test/java/org/jsmart/zerocode/integrationtests/s3/S3IntegrationTest.java @@ -0,0 +1,19 @@ +package org.jsmart.zerocode.integrationtests.s3; + +import org.jsmart.zerocode.core.domain.Scenario; +import org.jsmart.zerocode.core.domain.TargetEnv; +import org.jsmart.zerocode.core.runner.ZeroCodeUnitRunner; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; + +@TargetEnv("s3_host.properties") +@RunWith(ZeroCodeUnitRunner.class) +public class S3IntegrationTest { + + @Ignore("Requires active AWS credentials and a valid S3 bucket to run") + @Test + @Scenario("integration_test_files/s3/s3_operations.json") + public void testS3Operations() throws Exception { + } +} diff --git a/core/src/test/resources/integration_test_files/s3/input.txt b/core/src/test/resources/integration_test_files/s3/input.txt new file mode 100644 index 00000000..9ea6bb37 --- /dev/null +++ b/core/src/test/resources/integration_test_files/s3/input.txt @@ -0,0 +1 @@ +This is a test file for S3 upload. diff --git a/core/src/test/resources/integration_test_files/s3/s3_operations.json b/core/src/test/resources/integration_test_files/s3/s3_operations.json new file mode 100644 index 00000000..d6ca0fdd --- /dev/null +++ b/core/src/test/resources/integration_test_files/s3/s3_operations.json @@ -0,0 +1,41 @@ +{ + "scenarioName": "S3 Operations", + "steps": [ + { + "name": "upload_to_S3", + "url": "s3-bucket:test-bucket", + "operation": "upload", + "request": { + "key": "folder/file.txt", + "filePath": "src/test/resources/integration_test_files/s3/input.txt" + }, + "verify": { + "status": 200, + "bucket": "test-bucket" + } + }, + { + "name": "list_files_in_S3", + "url": "s3-bucket:test-bucket", + "operation": "list", + "request": { + "folder": "folder/" + }, + "verify": { + "files.SIZE": 1 + } + }, + { + "name": "download_from_S3", + "url": "s3-bucket:test-bucket", + "operation": "download", + "request": { + "key": "folder/file.txt", + "saveAs": "target/downloaded.txt" + }, + "verify": { + "downloaded": true + } + } + ] +} diff --git a/core/src/test/resources/s3_host.properties b/core/src/test/resources/s3_host.properties new file mode 100644 index 00000000..b7f5dd59 --- /dev/null +++ b/core/src/test/resources/s3_host.properties @@ -0,0 +1,4 @@ +s3.endpoint=https://s3.amazonaws.com +s3.accessKey=mock-access-key +s3.secretKey=mock-secret-key +s3.region=us-east-1