diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 3b1af03d7a..6d47d776c1 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -119,6 +119,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version + - name: Check out the repo uses: actions/checkout@v3 with: @@ -156,3 +157,16 @@ jobs: run: | cfn-lint --version cfn-lint -I -t ./deploy/cloudformation/elastic-agent-ec2.yml + + terraform-linter: + name: terraform-lint + runs-on: ubuntu-20.04 + steps: + - name: Checkout Repository + uses: actions/checkout@v3 + + - name: Init Hermit + run: ./bin/hermit env -r >> $GITHUB_ENV + + - name: Terraform fmt + run: terraform fmt -check -recursive diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml new file mode 100644 index 0000000000..4c5af30075 --- /dev/null +++ b/.github/workflows/weekly-enviroment.yml @@ -0,0 +1,126 @@ +name: Weekly environment deployment + +on: + workflow_dispatch: + inputs: + environment: + description: 'Environment to deploy' + type: choice + options: + - weekly environment + logLevel: + description: 'Log level' + required: true + default: 'INFO' + type: choice + options: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + +env: + WORKING_DIR: deploy/weekly-environment + SCRIPTS_DIR: deploy/weekly-environment/scripts/benchmarks/kspm_vanilla + TF_VAR_ec_api_key: ${{ secrets.WEEKLY_ENVIRONMENT_EC_API_KEY }} + TF_VAR_environment: ${{ github.event.inputs.logLevel }} + TF_LOG: ${{ github.event.inputs.logLevel }} + TF_VAR_stack_version: 8.6.1 + +jobs: + terraform: + name: Deploy KSPM cloud environment + runs-on: ubuntu-latest + defaults: + run: + working-directory: ${{ env.WORKING_DIR }} + steps: + - name: Check out the repo + uses: actions/checkout@v2 + + - name: Init Hermit + run: ./bin/hermit env -r >> $GITHUB_ENV + working-directory: ./ + + - name: Terraform Init + run: terraform init -no-color + + - name: Terraform Validate + run: terraform validate -no-color + + - name: Deploy Elastic Cloud + run: terraform apply --auto-approve + + - name: Set terraform output as env variable + run: | + echo "KIBANA_URL=$(terraform output kibana_url)" >> $GITHUB_ENV + + - name: Set sensitive terraform output as env variable + run: | + export ELASTICSEARCH_USERNAME=$(terraform output elasticsearch_username) + echo "::add-mask::$ELASTICSEARCH_USERNAME" + echo "ELASTICSEARCH_USERNAME=$ELASTICSEARCH_USERNAME" >> $GITHUB_ENV + + export ELASTICSEARCH_PASSWORD=$(terraform output elasticsearch_password) + echo "::add-mask::$ELASTICSEARCH_PASSWORD" + echo "ELASTICSEARCH_PASSWORD=$ELASTICSEARCH_PASSWORD" >> $GITHUB_ENV + + - name: Install KSPM vanilla integration + working-directory: ${{ env.SCRIPTS_DIR }} + run: | + ./install-kspm-vanilla-integration.sh ${{ env.KIBANA_URL }} ${{ env.ELASTICSEARCH_PASSWORD }} + + - name: Deploy agent on EC2 + working-directory: ${{ env.SCRIPTS_DIR }} + run: | + echo -e "${{ secrets.WEEKLY_ENVIRONMENT_EC2_PRIVATE_KEY }}" > weekly-key.pem + chmod 600 weekly-key.pem + # Copy the manifest file to the EC2 instance + scp -o StrictHostKeyChecking=no -v -i weekly-key.pem manifest.yaml "ubuntu@${{ secrets.WEEKLY_ENVIRONMENT_EC2_PUBLIC_IP }}:~/." + # Apply the manifest file + ssh -o StrictHostKeyChecking=no -v -i weekly-key.pem "ubuntu@${{ secrets.WEEKLY_ENVIRONMENT_EC2_PUBLIC_IP }}" "kubectl apply -f manifest.yaml" + + # Once https://github.com/slackapi/slack-github-action/issues/84 will be resolved we can push the payload to a different file + - name: Send custom JSON data to Slack workflow + uses: slackapi/slack-github-action@v1.23.0 + with: + payload: | + { + "text": "A new deployment job has been triggered", + "attachments": [ + { + "color": "#36a64f", + "fields": [ + { + "title": "Environment", + "value": "${{ github.event.inputs.environment }}", + "short": true + }, + { + "title": "Log level", + "value": "${{ github.event.inputs.logLevel }}", + "short": true + }, + { + "title": "Kibana URL", + "value": ${{ env.KIBANA_URL }}, + "short": false + }, + { + "title": "ElasticSearch username", + "value": ${{ env.ELASTICSEARCH_USERNAME }}, + "short": false + }, + { + "title": "ElasticSearch password", + "value": ${{ env.ELASTICSEARCH_PASSWORD }}, + "short": false + } + ] + } + ] + } + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK diff --git a/.gitignore b/.gitignore index 0bcfa0ed72..ac28157667 100644 --- a/.gitignore +++ b/.gitignore @@ -41,7 +41,31 @@ bundle.tar.gz /deploy/k8s/cloudbeat-ds.yaml # terraform -deploy/cloud/.terraform -deploy/cloud/.terraform.lock.hcl -deploy/cloud/terraform.tfstate -deploy/cloud/terraform.tfstate.backup +.terraform +.terraform.lock.hcl +.tfstate.backup + +# .tfstate files +*.tfstate +*.tfstate.* +*.tfplan + +# Crash log files +crash.log + +# Exclude all .tfvars files, which are likely to contain sentitive data, such as +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject +# to change depending on the environment. +*.tfvars + +# Ignore override files as they are usually used to override resources locally and so +# are not checked in +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Ignore CLI configuration files +.terraformrc +terraform.rc diff --git a/deploy/cloud/.gitignore b/deploy/cloud/.gitignore deleted file mode 100644 index 6665869f80..0000000000 --- a/deploy/cloud/.gitignore +++ /dev/null @@ -1,27 +0,0 @@ -# Local .terraform directories -**/.terraform/* - -# .tfstate files -*.tfstate -*.tfstate.* -*.tfplan - -# Crash log files -crash.log - -# Exclude all .tfvars files, which are likely to contain sentitive data, such as -# password, private keys, and other secrets. These should not be part of version -# control as they are data points which are potentially sensitive and subject -# to change depending on the environment. -*.tfvars - -# Ignore override files as they are usually used to override resources locally and so -# are not checked in -override.tf -override.tf.json -*_override.tf -*_override.tf.json - -# Ignore CLI configuration files -.terraformrc -terraform.rc diff --git a/deploy/cloud/modules/api/terraform.tf b/deploy/cloud/modules/api/terraform.tf index dec6d6e92a..0d4352222f 100644 --- a/deploy/cloud/modules/api/terraform.tf +++ b/deploy/cloud/modules/api/terraform.tf @@ -1,11 +1,11 @@ terraform { required_providers { restapi = { - source = "mastercard/restapi" + source = "mastercard/restapi" version = "~> 1.18.0" } http = { - source = "hashicorp/http" + source = "hashicorp/http" version = "~> 3.2.1" } } diff --git a/deploy/cloud/modules/provision-apps/aws-ebs-csi-driver.tf b/deploy/cloud/modules/provision-apps/aws-ebs-csi-driver.tf index d2b0a6431c..c357cf1de3 100644 --- a/deploy/cloud/modules/provision-apps/aws-ebs-csi-driver.tf +++ b/deploy/cloud/modules/provision-apps/aws-ebs-csi-driver.tf @@ -1,12 +1,12 @@ resource "helm_release" "aws_ebs_csi_driver" { - chart = "aws-ebs-csi-driver" - name = "aws-ebs-csi-driver" - namespace = var.namespace + chart = "aws-ebs-csi-driver" + name = "aws-ebs-csi-driver" + namespace = var.namespace repository = "https://kubernetes-sigs.github.io/aws-ebs-csi-driver" set { - name = "controller.serviceAccount.name" + name = "controller.serviceAccount.name" value = "ebs-csi-controller-sa" } diff --git a/deploy/cloud/modules/provision-apps/nginx-ingress.tf b/deploy/cloud/modules/provision-apps/nginx-ingress.tf index 7e7571d748..d9ed770093 100644 --- a/deploy/cloud/modules/provision-apps/nginx-ingress.tf +++ b/deploy/cloud/modules/provision-apps/nginx-ingress.tf @@ -1,10 +1,10 @@ resource "helm_release" "nginx_ingress" { - chart = "nginx-ingress-controller" - name = "nginx-ingress-controller" + chart = "nginx-ingress-controller" + name = "nginx-ingress-controller" repository = "https://charts.bitnami.com/bitnami" - timeout = 600 - namespace = var.namespace + timeout = 600 + namespace = var.namespace set { name = "service.type" @@ -12,7 +12,7 @@ resource "helm_release" "nginx_ingress" { } set { - name = "replicaCount" + name = "replicaCount" value = var.replica_count } } diff --git a/deploy/cloud/modules/provision-apps/terraform.tf b/deploy/cloud/modules/provision-apps/terraform.tf index d436ae8bb5..995559fe6f 100644 --- a/deploy/cloud/modules/provision-apps/terraform.tf +++ b/deploy/cloud/modules/provision-apps/terraform.tf @@ -1,7 +1,7 @@ terraform { required_providers { helm = { - source = "hashicorp/helm" + source = "hashicorp/helm" version = ">=2.8.0" } diff --git a/deploy/cloud/modules/provision-apps/variables.tf b/deploy/cloud/modules/provision-apps/variables.tf index 4a927ff926..09377378a6 100644 --- a/deploy/cloud/modules/provision-apps/variables.tf +++ b/deploy/cloud/modules/provision-apps/variables.tf @@ -1,9 +1,9 @@ variable "namespace" { - type = string + type = string default = "default" } variable "replica_count" { - type = string + type = string default = "2" } diff --git a/deploy/weekly-environment/main.tf b/deploy/weekly-environment/main.tf new file mode 100644 index 0000000000..9262b1a142 --- /dev/null +++ b/deploy/weekly-environment/main.tf @@ -0,0 +1,25 @@ +provider "ec" { + apikey = var.ec_api_key +} + +module "ec_deployment" { + source = "github.com/elastic/apm-server/testing/infra/terraform/modules/ec_deployment" + + region = var.ess_region + stack_version = var.stack_version + + deployment_template = var.deployment_template + deployment_name_prefix = "${var.deployment_name_prefix}-${formatdate("MMM DD, YYYY", timestamp())}" + + integrations_server = true + + elasticsearch_size = var.elasticsearch_size + elasticsearch_zone_count = var.elasticsearch_zone_count + + docker_image = var.docker_image_override + docker_image_tag_override = { + "elasticsearch" : "", + "kibana" : "", + "apm" : "" + } +} diff --git a/deploy/weekly-environment/outputs.tf b/deploy/weekly-environment/outputs.tf new file mode 100644 index 0000000000..0c55271150 --- /dev/null +++ b/deploy/weekly-environment/outputs.tf @@ -0,0 +1,26 @@ +output "elasticsearch_url" { + value = module.ec_deployment.elasticsearch_url + description = "The secure Elasticsearch URL" +} + +output "elasticsearch_username" { + value = module.ec_deployment.elasticsearch_username + description = "The Elasticsearch username" + sensitive = true +} + +output "elasticsearch_password" { + value = module.ec_deployment.elasticsearch_password + description = "The Elasticsearch password" + sensitive = true +} + +output "kibana_url" { + value = module.ec_deployment.kibana_url + description = "The secure Kibana URL" +} + +output "admin_console_url" { + value = module.ec_deployment.admin_console_url + description = "The admin console URL" +} diff --git a/deploy/weekly-environment/scripts/benchmarks/kspm_vanilla/data/agent_policy_vanilla.json b/deploy/weekly-environment/scripts/benchmarks/kspm_vanilla/data/agent_policy_vanilla.json new file mode 100644 index 0000000000..8ceaf9e3aa --- /dev/null +++ b/deploy/weekly-environment/scripts/benchmarks/kspm_vanilla/data/agent_policy_vanilla.json @@ -0,0 +1,8 @@ +{ + "name": "weekly-environment-vanilla-policy", + "namespace": "default", + "monitoring_enabled": [ + "logs", + "metrics" + ] +} diff --git a/deploy/weekly-environment/scripts/benchmarks/kspm_vanilla/data/package_policy_vanilla.json b/deploy/weekly-environment/scripts/benchmarks/kspm_vanilla/data/package_policy_vanilla.json new file mode 100644 index 0000000000..452a5c06ea --- /dev/null +++ b/deploy/weekly-environment/scripts/benchmarks/kspm_vanilla/data/package_policy_vanilla.json @@ -0,0 +1,64 @@ +{ + "name": "weekly-environment-vanilla-integration-package", + "description": "", + "namespace": "default", + "policy_id": "${agent_policy_id}", + "enabled": true, + "inputs": [ + { + "type": "cloudbeat/cis_k8s", + "policy_template": "kspm", + "enabled": true, + "streams": [ + { + "enabled": true, + "data_stream": { + "type": "logs", + "dataset": "cloud_security_posture.findings" + }, + "release": "ga" + } + ] + }, + { + "type": "cloudbeat/cis_eks", + "policy_template": "kspm", + "enabled": false, + "streams": [ + { + "enabled": false, + "data_stream": { + "type": "logs", + "dataset": "cloud_security_posture.findings" + }, + "release": "ga", + "vars": { + "access_key_id": { + "type": "text" + }, + "secret_access_key": { + "type": "text" + }, + "session_token": { + "type": "text" + }, + "shared_credential_file": { + "type": "text" + }, + "credential_profile_name": { + "type": "text" + }, + "role_arn": { + "type": "text" + } + } + } + ] + } + ], + "package": { + "name": "cloud_security_posture", + "title": "Kubernetes Security Posture Management (KSPM)", + "version": "1.1.1" + } +} diff --git a/deploy/weekly-environment/scripts/benchmarks/kspm_vanilla/install-kspm-vanilla-integration.sh b/deploy/weekly-environment/scripts/benchmarks/kspm_vanilla/install-kspm-vanilla-integration.sh new file mode 100755 index 0000000000..1787a68cb4 --- /dev/null +++ b/deploy/weekly-environment/scripts/benchmarks/kspm_vanilla/install-kspm-vanilla-integration.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +source ../../utils.sh + +# This script is used to install a vanilla integration for the KSPM vanilla benchmark. +# It will create a new agent policy, a new vanilla integration and a new vanilla integration manifest file. +# The script requires two arguments: +# 1. Kibana URL +# 2. Kibana password + +KIBANA_URL=$1 +KIBANA_PASSWORD=$2 +KIBANA_AUTH=elastic:${KIBANA_PASSWORD} + +readonly AGENT_POLICY=data/agent_policy_vanilla.json +readonly INTEGRATION_POLICY=data/package_policy_vanilla.json + +# Check if input is empty +if [ -z "$KIBANA_URL" ] || [ -z "$KIBANA_PASSWORD" ]; then + echo "Kibana URL or Kibana password is empty" + exit 1 +fi + +## Create a new agent policy And get the agent id +create_a_new_agent_policy "$KIBANA_URL" "$KIBANA_AUTH" "$AGENT_POLICY" +if [ -z "$POLICY_ID" ]; then + echo "Agent policy id is empty" + exit 1 +fi + +# Create a new vanilla integration +create_a_new_vanilla_integration "$KIBANA_URL" "$KIBANA_AUTH" "$POLICY_ID" "$INTEGRATION_POLICY" + +# Create a new agent policy +create_new_vanilla_integration_manifest_file "$KIBANA_URL" "$KIBANA_AUTH" "$POLICY_ID" +if [ -z "$MANIFEST_FILE" ]; then + echo "Manifest file is empty" + exit 1 +fi diff --git a/deploy/weekly-environment/scripts/utils.sh b/deploy/weekly-environment/scripts/utils.sh new file mode 100755 index 0000000000..2d3268556d --- /dev/null +++ b/deploy/weekly-environment/scripts/utils.sh @@ -0,0 +1,155 @@ +#!/bin/bash + +# This utility script contains functions that are used by the benchmark scripts. + +####################################### +# Creates a new agent policy and set the new POLICY_ID as the new integration policy id +# Globals: +# POLICY_ID +# Arguments: +# $1: Kibana URL +# $2: Kibana auth +# $3: Agent policy +# Returns: +# None +####################################### +create_a_new_agent_policy() { + local kibana_url=$1 + local kibana_auth=$2 + local agent_policy=$3 + + # Install Agent policy + local install_agent_response + install_agent_response="$(curl -X POST \ + --url "${kibana_url}/api/fleet/agent_policies?sys_monitoring=true" \ + -u "${kibana_auth}" \ + -H 'Cache-Control: no-cache' \ + -H 'Connection: keep-alive' \ + -H 'Content-Type: application/json' \ + -H 'kbn-xsrf: true' \ + -d "@${agent_policy}")" + + echo "Install agent response: ${install_agent_response}" + check_status_code_of_curl "${install_agent_response}" + + POLICY_ID=$(echo "${install_agent_response}" | jq -r '.item.id') + echo "Creating a new agent policy has completed successfully: New policy id: ${POLICY_ID}" +} + +####################################### +# Creates a new vanilla integration on the given policy id +# Arguments: +# $1: Kibana URL +# $2: Kibana auth +# $3: Policy id +# $4: Integration policy +# Returns: +# None +####################################### +create_a_new_vanilla_integration() { + local kibana_url=$1 + local kibana_auth=$2 + local policy_id=$3 + local integration_policy=$4 + + # Updating the new integration policy with the policy id + local updated_policy + updated_policy="$(jq --arg policy_id "${policy_id}" '.policy_id |= $policy_id' "${integration_policy}")" + echo "New integration policy: ${updated_policy}" + + package_policy_response="$(curl -X POST \ + --url "${kibana_url}/api/fleet/package_policies" \ + -u "${kibana_auth}" \ + -H 'Cache-Control: no-cache' \ + -H 'Connection: keep-alive' \ + -H 'Content-Type: application/json' \ + -H 'kbn-xsrf: true' \ + -d "${updated_policy}")" + + check_status_code_of_curl "${package_policy_response}" + + echo "Creating a new a new vanilla integration with policy id: ${policy_id} has completed successfully.Integration policy: ${updated_policy}" +} + +####################################### +# Creates a new vanilla integration manifest file manifest.yaml +# Globals: +# MANIFEST_FILE +# Arguments: +# $1: Kibana URL +# $2: Kibana auth +# $3: Policy id +# Returns: +# None +####################################### +create_new_vanilla_integration_manifest_file() { + local kibana_url=$1 + local kibana_auth=$2 + local policy_id=$3 + + local enrolment_token_response + enrolment_token_response="$(curl -X GET \ + --url "${kibana_url}/api/fleet/enrollment_api_keys" \ + -u "${kibana_auth}" \ + -H 'Cache-Control: no-cache' \ + -H 'Connection: keep-alive' \ + -H 'Content-Type: application/json' \ + -H 'kbn-xsrf: true')" + + check_status_code_of_curl "${enrolment_token_response}" + + local enrolment_token + enrolment_token="$(echo "${enrolment_token_response}" | jq --arg policy "${policy_id}" '.list[] | select(.policy_id == $policy)' | jq -r '.api_key')" + echo "enrolment_token: ${enrolment_token}" + + local fleet_data_response + fleet_data_response="$(curl -X GET \ + --url "${kibana_url}/api/fleet/settings" \ + -u "${kibana_auth}" \ + -H 'Cache-Control: no-cache' \ + -H 'Connection: keep-alive' \ + -H 'Content-Type: application/json' \ + -H 'kbn-xsrf: true')" + + check_status_code_of_curl "${fleet_data_response}" + + local fleet_server_host + fleet_server_host="$(echo "${fleet_data_response}" | jq -r '.item.fleet_server_hosts[0]')" + echo "fleet_server_host: ${fleet_server_host}" + + # Create the manifest file + local manifest_creation_response + manifest_creation_response="$(curl -X GET \ + --url "${kibana_url}/api/fleet/kubernetes?fleetServer=${fleet_server_host}&enrolToken=${enrolment_token}" \ + -u "${kibana_auth}" \ + -H 'Cache-Control: no-cache' \ + -H 'Connection: keep-alive' \ + -H 'Content-Type: application/json' \ + -H 'kbn-xsrf: true')" + + check_status_code_of_curl "${manifest_creation_response}" + + # write the manifest file to the file system + # get the item field from the response + MANIFEST_FILE=$(echo "$manifest_creation_response" | jq -r '.item') + echo "$MANIFEST_FILE" > manifest.yaml +} + +####################################### +# Checks the status code of the curl response and exits if the status code is not 200 +# Globals: +# Arguments: +# $1: Curl response +# Returns: +# None +####################################### +check_status_code_of_curl() { + local curl_response=$1 + error_code=$(echo "$curl_response" | jq -r '.statusCode') + if [ "$error_code" != "null" ] && [ "$error_code" != "200" ]; then + echo "Error code: $error_code" + echo "Error message: $(echo "$curl_response" | jq -r '.message')" + echo "Error full response: $curl_response" + exit 1 + fi +} diff --git a/deploy/weekly-environment/terraform.tf b/deploy/weekly-environment/terraform.tf new file mode 100644 index 0000000000..94ff40d981 --- /dev/null +++ b/deploy/weekly-environment/terraform.tf @@ -0,0 +1,8 @@ +terraform { + required_providers { + ec = { + source = "elastic/ec" + version = ">=0.5.0" + } + } +} diff --git a/deploy/weekly-environment/variables.tf b/deploy/weekly-environment/variables.tf new file mode 100644 index 0000000000..0dac9f064a --- /dev/null +++ b/deploy/weekly-environment/variables.tf @@ -0,0 +1,60 @@ +## Deployment configuration + +variable "ec_api_key" { + description = "Elastic cloud API key" + type = string +} + +variable "ess_region" { + default = "gcp-us-central1" + description = "Optional ESS region where the deployment will be created. Defaults to gcp-us-west2" + type = string +} + +variable "deployment_template" { + default = "gcp-compute-optimized-v2" + description = "Optional deployment template. Defaults to the CPU optimized template for GCP" + type = string +} + +variable "stack_version" { + default = "latest" + description = "Optional stack version" + type = string +} + +variable "elasticsearch_size" { + default = "8g" + type = string + description = "Optional Elasticsearch instance size" +} + +variable "elasticsearch_zone_count" { + default = 1 + type = number + description = "Optional Elasticsearch zone count" +} + +variable "docker_image_tag_override" { + default = { + "elasticsearch" : "", + "kibana" : "", + "apm" : "", + } + description = "Optional docker image tag override" + type = map(string) +} + +variable "docker_image_override" { + default = { + "elasticsearch" : "docker.elastic.co/cloud-release/elasticsearch-cloud-ess", + "kibana" : "docker.elastic.co/cloud-release/kibana-cloud", + "apm" : "docker.elastic.co/cloud-release/elastic-agent-cloud", + } + type = map(string) +} + +variable "deployment_name_prefix" { + default = "weekly-environment" + description = "Optional set a prefix of the deployment. Defaults to weekly-environment" +}