From 0b1b5f643e1f7d970e7820a4272de0c6dee9354a Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Wed, 25 Jan 2023 11:29:02 +0200 Subject: [PATCH 001/127] first CI test --- .github/workflows/weekly-enviroment.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .github/workflows/weekly-enviroment.yml diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml new file mode 100644 index 0000000000..144de135c4 --- /dev/null +++ b/.github/workflows/weekly-enviroment.yml @@ -0,0 +1,13 @@ +name: Weekly environment deployment + +on: + push: + tags: + - weekly-deployment + +jobs: + Explore-GitHub-Actions: + runs-on: ubuntu-latest + steps: + - name: Echo test completed + run: echo "test completed" From 36fa1b9ffacab3c2153cc929a73a5567580aef85 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Wed, 25 Jan 2023 12:04:59 +0200 Subject: [PATCH 002/127] changing the CI --- .github/workflows/weekly-enviroment.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 144de135c4..1712d80463 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -1,6 +1,10 @@ name: Weekly environment deployment on: + pull_request: + types: [labeled] + labels: + - weekly deployment push: tags: - weekly-deployment From 6c7693806e17969136e272fdae044e342f2c178e Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Wed, 25 Jan 2023 12:05:40 +0200 Subject: [PATCH 003/127] changing the CI --- .github/workflows/weekly-enviroment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 1712d80463..3128a2e2bc 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -10,7 +10,7 @@ on: - weekly-deployment jobs: - Explore-GitHub-Actions: + deploy: runs-on: ubuntu-latest steps: - name: Echo test completed From ea3aa8980bed09e9b5caf06227628749eb5ee4b6 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Wed, 25 Jan 2023 12:16:25 +0200 Subject: [PATCH 004/127] adding log message --- .github/workflows/weekly-enviroment.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 3128a2e2bc..d3de8d8f76 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -15,3 +15,13 @@ jobs: steps: - name: Echo test completed run: echo "test completed" + + - name: Comment test success result + uses: marocchino/sticky-pull-request-comment@v2 + with: + header: Test Results + number: ${{ env.PR_NUMBER }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + message: | + ## Cloudbeat CI :robot: + Hello from Ofir From 429be715eb7b9ade1b4d121c5368d4e962d76105 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Wed, 25 Jan 2023 12:31:18 +0200 Subject: [PATCH 005/127] adding log message --- .github/workflows/weekly-enviroment.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index d3de8d8f76..e668dd1435 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -8,6 +8,14 @@ on: push: tags: - weekly-deployment +env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + CONTAINER_SUFFIX: ${{ github.run_id }} + AWS_REGION: eu-west-2 + REPORTS_DIR: tests/allure/results/ + KUBE_NAMESPACE: kube-system jobs: deploy: @@ -23,5 +31,8 @@ jobs: number: ${{ env.PR_NUMBER }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} message: | - ## Cloudbeat CI :robot: - Hello from Ofir + ## Weekly environment :face_with_cowboy_hat: + A new environment has been deployed to the elastic cloud. :tada: + You can access it at _______. + :warning: This environment will be deleted in 7 days. + From b2faace9285e93c4277c419dc2c4bde4a72d027c Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Wed, 25 Jan 2023 12:34:59 +0200 Subject: [PATCH 006/127] fixing message to have a proper emoji --- .github/workflows/weekly-enviroment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index e668dd1435..0147926141 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -31,7 +31,7 @@ jobs: number: ${{ env.PR_NUMBER }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} message: | - ## Weekly environment :face_with_cowboy_hat: + ## Weekly environment :man_dancing: A new environment has been deployed to the elastic cloud. :tada: You can access it at _______. :warning: This environment will be deleted in 7 days. From 0b5b59972ffa22ee6d110a8f236d9ad79061b987 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Wed, 25 Jan 2023 12:49:49 +0200 Subject: [PATCH 007/127] adding working directory --- .github/workflows/weekly-enviroment.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 0147926141..3905b12ed1 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -20,10 +20,20 @@ env: jobs: deploy: runs-on: ubuntu-latest + defaults: + run: + working-directory: ../deploy/cloud steps: - name: Echo test completed run: echo "test completed" + - name: List all files + run: ls -la + + - name: Terraform Format + id: fmt + run: terraform fmt -check + - name: Comment test success result uses: marocchino/sticky-pull-request-comment@v2 with: From ea4fb2356a04598131740c71bfc1d8ddb6a0b4a3 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Wed, 25 Jan 2023 13:53:05 +0200 Subject: [PATCH 008/127] adding working directory --- .github/workflows/weekly-enviroment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 3905b12ed1..ab0a5ce090 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-latest defaults: run: - working-directory: ../deploy/cloud + working-directory: ./deploy/cloud steps: - name: Echo test completed run: echo "test completed" From 011b3b08ac7524e346c6b11ff0b6bf934f206f47 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Wed, 25 Jan 2023 13:59:42 +0200 Subject: [PATCH 009/127] adding working directory --- .github/workflows/weekly-enviroment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index ab0a5ce090..e37c555edb 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-latest defaults: run: - working-directory: ./deploy/cloud + working-directory: deploy/cloud steps: - name: Echo test completed run: echo "test completed" From b972dfec262bda04f0232f5ee0c1e925563ce6f1 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Wed, 25 Jan 2023 14:04:58 +0200 Subject: [PATCH 010/127] skip-ci-tag --- .github/workflows/cloudbeat-ci.yml | 1 + .github/workflows/packaging.yml | 2 ++ .github/workflows/unit-test.yml | 2 ++ 3 files changed, 5 insertions(+) diff --git a/.github/workflows/cloudbeat-ci.yml b/.github/workflows/cloudbeat-ci.yml index 02c6ab3ce6..948473c892 100644 --- a/.github/workflows/cloudbeat-ci.yml +++ b/.github/workflows/cloudbeat-ci.yml @@ -19,6 +19,7 @@ jobs: # for more information see .pre-commit-config.yaml name: Lint runs-on: ubuntu-latest + if: contains(github.ref, 'refs/tags/') != true || !contains(github.ref, 'skip-ci') timeout-minutes: 10 steps: - name: Check out the repo diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml index b9b1eb7982..3ec45d1c09 100644 --- a/.github/workflows/packaging.yml +++ b/.github/workflows/packaging.yml @@ -6,6 +6,8 @@ on: - main - '[0-9]+.[0-9]+' types: [opened, synchronize, reopened] + tags: + - '!skip-ci' env: DEV: true diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index b03c5a03c5..8cd543f132 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -6,6 +6,8 @@ on: - main - '[0-9]+.[0-9]+' types: [opened, synchronize, reopened] + tags: + - '!skip-ci' env: K8S_MANIFEST_DIR: deploy From 4c86c2b23bc147ea52a92bee87d2a982ac90a236 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Wed, 25 Jan 2023 14:11:29 +0200 Subject: [PATCH 011/127] skip-ci-tag --- .github/workflows/weekly-enviroment.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index e37c555edb..3395d33a1a 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -20,13 +20,16 @@ env: jobs: deploy: runs-on: ubuntu-latest - defaults: - run: - working-directory: deploy/cloud +# defaults: +# run: +# working-directory: deploy/cloud steps: - name: Echo test completed run: echo "test completed" + - name: Check out the repo + uses: actions/checkout@v2 + - name: List all files run: ls -la From 464ff80b025ca225199396a7273b164c7263b350 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Wed, 25 Jan 2023 14:16:12 +0200 Subject: [PATCH 012/127] updating job --- .github/workflows/weekly-enviroment.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 3395d33a1a..7590086241 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -20,9 +20,9 @@ env: jobs: deploy: runs-on: ubuntu-latest -# defaults: -# run: -# working-directory: deploy/cloud + defaults: + run: + working-directory: deploy/cloud steps: - name: Echo test completed run: echo "test completed" @@ -41,7 +41,6 @@ jobs: uses: marocchino/sticky-pull-request-comment@v2 with: header: Test Results - number: ${{ env.PR_NUMBER }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} message: | ## Weekly environment :man_dancing: From 082469eb62edf23a90fa436ff1c357cd5b6dac9a Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Wed, 25 Jan 2023 14:35:15 +0200 Subject: [PATCH 013/127] remove all ci --- .github/workflows/Periodic-CI.yml | 16 +--------------- .github/workflows/cloudbeat-ci.yml | 8 ++------ .github/workflows/eks-ci.yml | 10 ++-------- .github/workflows/packaging.yml | 10 ++-------- .github/workflows/publish-test-results.yml | 7 ++----- .github/workflows/unit-test.yml | 10 +--------- 6 files changed, 10 insertions(+), 51 deletions(-) diff --git a/.github/workflows/Periodic-CI.yml b/.github/workflows/Periodic-CI.yml index 8aaaa76ced..421267354f 100644 --- a/.github/workflows/Periodic-CI.yml +++ b/.github/workflows/Periodic-CI.yml @@ -1,20 +1,6 @@ name: Periodic-CI -on: - # On demand execution of workflow will run all suites. - workflow_dispatch: - inputs: - test-targets: - required: true - description: 'Specify test markers to run' - default: '[\"pre_merge\", \"file_system_rules\", \"k8s_object_rules\", \"process_api_server_rules\", \"process_controller_manager_rules\", \"process_etcd_rules\", \"process_kubelet_rules\", \"process_scheduler_rules\"]' - - schedule: - # Nightly job. Running every day at 2 am - - cron: '00 02 * * 0-4' - - # Weekly job running every saturday at 2 am - - cron: '00 02 * * 6' + # TODO - revert env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} diff --git a/.github/workflows/cloudbeat-ci.yml b/.github/workflows/cloudbeat-ci.yml index 948473c892..d627219912 100644 --- a/.github/workflows/cloudbeat-ci.yml +++ b/.github/workflows/cloudbeat-ci.yml @@ -1,11 +1,7 @@ name: Cloudbeat-CI -on: - pull_request: - branches: - - main - - '[0-9]+.[0-9]+' - types: [opened, synchronize, reopened] +# TODO - revert + env: CONTAINER_SUFFIX: ${{ github.run_id }} diff --git a/.github/workflows/eks-ci.yml b/.github/workflows/eks-ci.yml index 451f5dae63..b248337342 100644 --- a/.github/workflows/eks-ci.yml +++ b/.github/workflows/eks-ci.yml @@ -1,13 +1,7 @@ name: EKS-CI -on: - # On demand execution of workflow will run all suites. - workflow_dispatch: - inputs: - test-targets: - required: true - description: "Specify test markers to run" - default: '[\"pre_merge\"]' +# TODO - revert + env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml index 3ec45d1c09..9240f51a1c 100644 --- a/.github/workflows/packaging.yml +++ b/.github/workflows/packaging.yml @@ -1,13 +1,7 @@ name: Packaging -on: - pull_request: - branches: - - main - - '[0-9]+.[0-9]+' - types: [opened, synchronize, reopened] - tags: - - '!skip-ci' +# TODO - revert + env: DEV: true diff --git a/.github/workflows/publish-test-results.yml b/.github/workflows/publish-test-results.yml index 94acfb3397..615a8c61f0 100644 --- a/.github/workflows/publish-test-results.yml +++ b/.github/workflows/publish-test-results.yml @@ -1,10 +1,7 @@ name: Publish-Test-Results -on: - workflow_run: - workflows: ["Cloudbeat-CI"] - types: - - completed +# TODO - revert + jobs: publish_results: diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 8cd543f132..34393ec976 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -1,14 +1,6 @@ name: UnitTests -on: - pull_request: - branches: - - main - - '[0-9]+.[0-9]+' - types: [opened, synchronize, reopened] - tags: - - '!skip-ci' - +# TODO - revert env: K8S_MANIFEST_DIR: deploy From f89148f632520117fafd6961e0ce6e654cdaf5eb Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Wed, 25 Jan 2023 14:51:02 +0200 Subject: [PATCH 014/127] update ci to work with the correct working directory --- .github/workflows/weekly-enviroment.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 7590086241..3d426ef75f 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -20,9 +20,6 @@ env: jobs: deploy: runs-on: ubuntu-latest - defaults: - run: - working-directory: deploy/cloud steps: - name: Echo test completed run: echo "test completed" @@ -30,6 +27,12 @@ jobs: - name: Check out the repo uses: actions/checkout@v2 + - name: List all files before setting directory + run: ls -la + + - name: Set working directory + working-directory: cloudbeat/deploy/cloud + - name: List all files run: ls -la From 1ad1d5a2f5a3942e8d8de2ed5234519ac7a809a0 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Wed, 25 Jan 2023 14:56:42 +0200 Subject: [PATCH 015/127] add run on demand --- .github/workflows/eks-ci.yml | 10 ++++++++-- .github/workflows/weekly-enviroment.yml | 6 ++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/eks-ci.yml b/.github/workflows/eks-ci.yml index b248337342..451f5dae63 100644 --- a/.github/workflows/eks-ci.yml +++ b/.github/workflows/eks-ci.yml @@ -1,7 +1,13 @@ name: EKS-CI -# TODO - revert - +on: + # On demand execution of workflow will run all suites. + workflow_dispatch: + inputs: + test-targets: + required: true + description: "Specify test markers to run" + default: '[\"pre_merge\"]' env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 3d426ef75f..ae89625ff4 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -8,6 +8,12 @@ on: push: tags: - weekly-deployment + workflow_dispatch: + inputs: + test-targets: + required: true + description: "Run it on demand" + default: '[\"pre_merge\"]' env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} From 65aec6e32d02045f947ab6bb5e9395af8b5feadc Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Wed, 25 Jan 2023 14:59:37 +0200 Subject: [PATCH 016/127] remove all workflows --- .github/workflows/Periodic-CI.yml | 235 --------------- .github/workflows/cloudbeat-ci.yml | 317 --------------------- .github/workflows/eks-ci.yml | 238 ---------------- .github/workflows/packaging.yml | 46 --- .github/workflows/publish-test-results.yml | 82 ------ .github/workflows/unit-test.yml | 128 --------- 6 files changed, 1046 deletions(-) delete mode 100644 .github/workflows/Periodic-CI.yml delete mode 100644 .github/workflows/cloudbeat-ci.yml delete mode 100644 .github/workflows/eks-ci.yml delete mode 100644 .github/workflows/packaging.yml delete mode 100644 .github/workflows/publish-test-results.yml delete mode 100644 .github/workflows/unit-test.yml diff --git a/.github/workflows/Periodic-CI.yml b/.github/workflows/Periodic-CI.yml deleted file mode 100644 index 421267354f..0000000000 --- a/.github/workflows/Periodic-CI.yml +++ /dev/null @@ -1,235 +0,0 @@ -name: Periodic-CI - - # TODO - revert - -env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - CONTAINER_SUFFIX: ${{ github.run_id }} - TEST_TARGETS_DEFAULT: '[\"pre_merge\", \"file_system_rules\", \"k8s_object_rules\", \"process_api_server_rules\", \"process_controller_manager_rules\", \"process_etcd_rules\", \"process_kubelet_rules\", \"process_scheduler_rules\"]' - -permissions: - actions: read - checks: write - statuses: read -jobs: - Setup: - runs-on: ubuntu-20.04 - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - timeout-minutes: 15 - steps: - - id: Default - env: - TEST_TARGETS: ${{ env.TEST_TARGETS_DEFAULT }} - run: | - echo "TEST_MATRIX=$TEST_TARGETS" >> $GITHUB_ENV - - - id: Nightly - if: github.event_name == 'schedule' && github.event.schedule == '00 02 * * 0-4' - env: - TEST_TARGETS: ${{ env.TEST_TARGETS_DEFAULT }} - run: echo "TEST_MATRIX=${TEST_TARGETS}" >> $GITHUB_ENV - - - id: Weekly - if: github.event_name == 'schedule' && github.event.schedule == '00 02 * * 6' - env: - TEST_TARGETS: ${{ env.TEST_TARGETS_DEFAULT }} - run: echo "TEST_MATRIX=${TEST_TARGETS}" >> $GITHUB_ENV - - - id: Push - if: github.event_name == 'push' - env: - TEST_TARGETS: '[\"pre_merge\", \"file_system_rules\"]' - run: | - echo "TEST_MATRIX=$TEST_TARGETS" >> $GITHUB_ENV - - - id: Dispatch - if: github.event_name == 'workflow_dispatch' - env: - TEST_TARGETS: ${{ github.event.inputs.test-targets }} - run: echo "TEST_MATRIX=${TEST_TARGETS}" >> $GITHUB_ENV - - - id: set-matrix - run: echo "::set-output name=matrix::${{ env.TEST_MATRIX}}" - - Build: - name: Build - runs-on: ubuntu-20.04 - timeout-minutes: 15 - steps: - - name: Check out the repo - uses: actions/checkout@v3 - - - name: Set up Go - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - - name: Fetch OPA version - run: echo "OPA_VERSION=$(go list -m -f {{.Version}} github.com/open-policy-agent/opa | sed 's/v//')" >> $GITHUB_ENV - - - name: Setup OPA - uses: open-policy-agent/setup-opa@v1 - with: - version: ${{ env.OPA_VERSION }} - - - name: cache go dependencies - uses: actions/cache@v2 - with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - key: ${{ runner.os }}-go-${{ matrix.go-version }}-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-${{ matrix.go-version }}- - - - name: build cloudbeat - run: make - - - name: build opa bundle - uses: magefile/mage-action@v2 - with: - version: latest - args: BuildOpaBundle - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Cache Build dependencies - uses: actions/cache@v2 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.workflow }} - restore-keys: | - ${{ runner.os }}-buildx- - - - name: Build cloudbeat-docker - uses: docker/build-push-action@v2 - with: - context: . - push: false - tags: cloudbeat:latest - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new - outputs: type=docker,dest=/tmp/cloudbeat-${{ env.CONTAINER_SUFFIX }}.tar - - - name: Build pytest-docker - uses: docker/build-push-action@v2 - with: - context: ./tests/. - push: false - tags: cloudbeat-test:latest - cache-from: type=local,mode=max,src=/tmp/.buildx-cache - cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-new - outputs: type=docker,dest=/tmp/pytest-${{ env.CONTAINER_SUFFIX }}.tar - - - name: Cache docker images - uses: actions/cache@v2 - with: - path: /tmp/*.tar - key: ${{ runner.os }}-dockers-cache-${{ env.CONTAINER_SUFFIX }} - restore-keys: | - ${{ runner.os }}-dockers-cache-${{ env.CONTAINER_SUFFIX }} - - - name: Move cache - run: | - rm -rf /tmp/.buildx-cache - mv /tmp/.buildx-cache-new /tmp/.buildx-cache - - Test_Matrix: - needs: ['Build', 'Setup'] - strategy: - fail-fast: false - matrix: - test-target: ${{fromJson(needs.Setup.outputs.matrix)}} - name: ${{ matrix.test-target }}-tests - runs-on: ubuntu-20.04 - timeout-minutes: 90 - steps: - - name: Check out the repo - uses: actions/checkout@v2 - - - uses: azure/setup-helm@v1 - id: install - - - uses: extractions/setup-just@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Create k8s Kind Cluster - uses: helm/kind-action@v1.2.0 - with: - config: deploy/k8s/kind/kind-mono.yml - - - name: Cache docker images - uses: actions/cache@v2 - with: - path: /tmp/*.tar - key: ${{ runner.os }}-dockers-cache-${{ env.CONTAINER_SUFFIX }} - restore-keys: | - ${{ runner.os }}-dockers-cache- - - - name: Load images to kind - run: | - kind load image-archive /tmp/cloudbeat-${{ env.CONTAINER_SUFFIX }}.tar --name=kind-mono & kind load image-archive /tmp/pytest-${{ env.CONTAINER_SUFFIX }}.tar --name=kind-mono - - - name: Deploy tests Helm chart - id: deploy_helm - run: | - just deploy-tests-helm-ci ${{ matrix.test-target }} - - - name: Run Tests - id: run_tests - run: | - just run-tests-ci - - - name: Upload Test Results - if: always() - uses: actions/upload-artifact@v2 - with: - name: allure_results-${{ matrix.test-target}} - path: tests/allure/results/ - - - name: clear cache - id: clear_cache - if: success() - run: rm -rf /tmp/${{ env.CONTAINER_SUFFIX }}}.tar - - publish_results: - name: Publish Results - needs: Test_Matrix - if: always() - runs-on: ubuntu-20.04 - steps: - - name: Download Artifacts - uses: actions/download-artifact@v3 - with: - path: artifacts - - - name: Extract Artifacts - run: | - mkdir -p tests/allure/results - find artifacts/ -type f -print0 | xargs -0 mv -t tests/allure/results - - - name: Publish allure report - if: always() - uses: andrcuns/allure-publish-action@v1.0.1 - with: - storageType: s3 - resultsGlob: "tests/allure/results/*" - updatePr: actions - collapseSummary: false - summary: suites - summaryTableType: markdown - copyLatest: true - bucket: csp-allure-reports - prefix: allure_reports/cloudbeat/${{ github.ref_name }} - ignoreMissingResults: true - - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - if: failure() - with: - limit-access-to-actor: true diff --git a/.github/workflows/cloudbeat-ci.yml b/.github/workflows/cloudbeat-ci.yml deleted file mode 100644 index d627219912..0000000000 --- a/.github/workflows/cloudbeat-ci.yml +++ /dev/null @@ -1,317 +0,0 @@ -name: Cloudbeat-CI - -# TODO - revert - - -env: - CONTAINER_SUFFIX: ${{ github.run_id }} - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - Lint: - # for more information see .pre-commit-config.yaml - name: Lint - runs-on: ubuntu-latest - if: contains(github.ref, 'refs/tags/') != true || !contains(github.ref, 'skip-ci') - timeout-minutes: 10 - steps: - - name: Check out the repo - uses: actions/checkout@v2 - - - name: Init Hermit - run: ./bin/hermit env -r >> $GITHUB_ENV - - - name: Python lints - run: | - git ls-files -- '*.py' | xargs pre-commit run --file - shell: bash - - - name: Go lints - run: | - git ls-files -- '*.go' | xargs pre-commit run --file - shell: bash - - Build: - name: Build - runs-on: ubuntu-20.04 - timeout-minutes: 15 - steps: - - name: Check out the repo - uses: actions/checkout@v2 - - - name: Init Hermit - run: ./bin/hermit env -r >> $GITHUB_ENV - - - name: cache go dependencies - uses: actions/cache@v2 - with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - key: ${{ runner.os }}-go-${{ matrix.go-version }}-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-${{ matrix.go-version }}- - - - name: build cloudbeat binary - uses: magefile/mage-action@v2 - with: - version: latest - args: build - - - name: build opa bundle - uses: magefile/mage-action@v2 - with: - version: latest - args: BuildOpaBundle - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Cache Build dependencies - uses: actions/cache@v2 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.workflow }} - restore-keys: | - ${{ runner.os }}-buildx- - - - name: Build cloudbeat-docker - uses: docker/build-push-action@v2 - with: - context: . - push: false - tags: cloudbeat:latest - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new - outputs: type=docker,dest=/tmp/cloudbeat-${{ env.CONTAINER_SUFFIX }}.tar - - - name: Build elastic-agent - run: | - make build_elastic_agent_docker_image - - - name: Build pytest-docker - uses: docker/build-push-action@v2 - with: - context: ./tests/. - push: false - tags: cloudbeat-test:latest - cache-from: type=local,mode=max,src=/tmp/.buildx-cache - cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-new - outputs: type=docker,dest=/tmp/pytest-${{ env.CONTAINER_SUFFIX }}.tar - - - name: Cache docker images - uses: actions/cache@v2 - with: - path: /tmp/*.tar - key: ${{ runner.os }}-dockers-cache-${{ env.CONTAINER_SUFFIX }} - - - name: Move cache - run: | - ./.ci/scripts/gh-cache.sh - shell: bash - - Test_Matrix: - name: ${{ matrix.test-target }}-${{ matrix.range }}-tests - needs: [Build] - runs-on: ubuntu-20.04 - timeout-minutes: 55 - strategy: - matrix: - include: - - test-target: pre_merge_agent - range: '' - values_file: tests/deploy/values/ci-sa-agent.yml - - test-target: pre_merge - range: '' - values_file: tests/deploy/values/ci.yml - - test-target: file_system_rules - range: '0..5' - values_file: tests/deploy/values/ci.yml - - test-target: file_system_rules - range: '5..10' - values_file: tests/deploy/values/ci.yml - - test-target: file_system_rules - range: '10..15' - values_file: tests/deploy/values/ci.yml - - test-target: file_system_rules - range: '15..20' - values_file: tests/deploy/values/ci.yml - - test-target: file_system_rules - range: '20..25' - values_file: tests/deploy/values/ci.yml - - test-target: file_system_rules - range: '25..30' - values_file: tests/deploy/values/ci.yml - - test-target: file_system_rules - range: '30..35' - values_file: tests/deploy/values/ci.yml - - test-target: file_system_rules - range: '35..40' - values_file: tests/deploy/values/ci.yml - - test-target: file_system_rules - range: '40..45' - values_file: tests/deploy/values/ci.yml - - test-target: file_system_rules - range: '45..50' - values_file: tests/deploy/values/ci.yml - - test-target: file_system_rules - range: '50..55' - values_file: tests/deploy/values/ci.yml - - test-target: file_system_rules - range: '55..60' - values_file: tests/deploy/values/ci.yml - - test-target: file_system_rules - range: '60..65' - values_file: tests/deploy/values/ci.yml - - test-target: file_system_rules - range: '65..70' - values_file: tests/deploy/values/ci.yml - - test-target: file_system_rules - range: '70..' - values_file: tests/deploy/values/ci.yml - - test-target: k8s_object_rules - range: '0..6' - values_file: tests/deploy/values/ci.yml - - test-target: k8s_object_rules - range: '6..12' - values_file: tests/deploy/values/ci.yml - - test-target: k8s_object_rules - range: '12..18' - values_file: tests/deploy/values/ci.yml - - test-target: k8s_object_rules - range: '18..' - values_file: tests/deploy/values/ci.yml - - test-target: process_scheduler_rules - range: '0..3' - values_file: tests/deploy/values/ci.yml - - test-target: process_scheduler_rules - range: '3..' - values_file: tests/deploy/values/ci.yml - - test-target: process_api_server_rules - range: '0..5' - values_file: tests/deploy/values/ci.yml - - test-target: process_api_server_rules - range: '5..10' - values_file: tests/deploy/values/ci.yml - - test-target: process_api_server_rules - range: '10..15' - values_file: tests/deploy/values/ci.yml - - test-target: process_api_server_rules - range: '15..20' - values_file: tests/deploy/values/ci.yml - - test-target: process_api_server_rules - range: '20..24' - values_file: tests/deploy/values/ci.yml - - test-target: process_api_server_rules - range: '24..28' - values_file: tests/deploy/values/ci.yml - - test-target: process_api_server_rules - range: '28..32' - values_file: tests/deploy/values/ci.yml - - test-target: process_api_server_rules - range: '32..36' - values_file: tests/deploy/values/ci.yml - - test-target: process_api_server_rules - range: '36..40' - values_file: tests/deploy/values/ci.yml - - test-target: process_api_server_rules - range: '40..44' - values_file: tests/deploy/values/ci.yml - - test-target: process_api_server_rules - range: '44..48' - values_file: tests/deploy/values/ci.yml - - test-target: process_api_server_rules - range: '48..52' - values_file: tests/deploy/values/ci.yml - - test-target: process_api_server_rules - range: '52..56' - values_file: tests/deploy/values/ci.yml - - test-target: process_api_server_rules - range: '56..' - values_file: tests/deploy/values/ci.yml - - test-target: process_controller_manager_rules - range: '0..4' - values_file: tests/deploy/values/ci.yml - - test-target: process_controller_manager_rules - range: '4..8' - values_file: tests/deploy/values/ci.yml - - test-target: process_controller_manager_rules - range: '8..12' - values_file: tests/deploy/values/ci.yml - - test-target: process_controller_manager_rules - range: '12..' - values_file: tests/deploy/values/ci.yml - - test-target: process_etcd_rules - range: '0..4' - values_file: tests/deploy/values/ci.yml - - test-target: process_etcd_rules - range: '4..8' - values_file: tests/deploy/values/ci.yml - - test-target: process_etcd_rules - range: '8..' - values_file: tests/deploy/values/ci.yml - - test-target: process_kubelet_rules - range: '0..4' - values_file: tests/deploy/values/ci.yml - - test-target: process_kubelet_rules - range: '4..8' - values_file: tests/deploy/values/ci.yml - - test-target: process_kubelet_rules - range: '8..12' - values_file: tests/deploy/values/ci.yml - - test-target: process_kubelet_rules - range: '12..16' - values_file: tests/deploy/values/ci.yml - - test-target: process_kubelet_rules - range: '16..20' - values_file: tests/deploy/values/ci.yml - - test-target: process_kubelet_rules - range: '20..' - values_file: tests/deploy/values/ci.yml - fail-fast: false - steps: - - name: Check out the repo - uses: actions/checkout@v2 - - - name: Init Hermit - run: ./bin/hermit env -r >> $GITHUB_ENV - - - name: Create k8s Kind Cluster - run: | - just create-kind-cluster - - - name: Cache docker images - uses: actions/cache@v2 - with: - path: /tmp/*.tar - key: ${{ runner.os }}-dockers-cache-${{ env.CONTAINER_SUFFIX }} - - - name: Load images to kind - run: | - ./.ci/scripts/kind-images.sh ${{ env.CONTAINER_SUFFIX }} kind-multi - shell: bash - - - name: Deploy tests Helm chart - id: deploy_helm - run: | - just deploy-tests-helm ${{ matrix.test-target }} ${{ matrix.values_file }} ${{ matrix.range }} - - - name: Run Tests - id: run_tests - run: | - just run-tests ${{ matrix.test-target }} - - - name: Upload Test Results - if: always() - uses: actions/upload-artifact@v2 - with: - name: allure_results-${{ matrix.test-target}} - path: tests/allure/results/ - - - name: clear cache - id: clear_cache - if: always() - run: rm -rf /tmp/*-${{ env.CONTAINER_SUFFIX }}.tar diff --git a/.github/workflows/eks-ci.yml b/.github/workflows/eks-ci.yml deleted file mode 100644 index 451f5dae63..0000000000 --- a/.github/workflows/eks-ci.yml +++ /dev/null @@ -1,238 +0,0 @@ -name: EKS-CI - -on: - # On demand execution of workflow will run all suites. - workflow_dispatch: - inputs: - test-targets: - required: true - description: "Specify test markers to run" - default: '[\"pre_merge\"]' - -env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - CONTAINER_SUFFIX: ${{ github.run_id }} - TEST_TARGETS_DEFAULT: '[\"pre_merge\"]' - AWS_REGION: eu-west-2 - REPORTS_DIR: tests/allure/results/ - KUBE_NAMESPACE: kube-system - -jobs: - Setup: - runs-on: ubuntu-20.04 - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - timeout-minutes: 15 - steps: - - id: set-default-matrix - env: - TEST_TARGETS: ${{ env.TEST_TARGETS_DEFAULT }} - run: | - echo "TEST_MATRIX=$TEST_TARGETS" >> $GITHUB_ENV - - - id: Dispatch - if: github.event_name == 'workflow_dispatch' - env: - TEST_TARGETS: ${{ github.event.inputs.test-targets }} - run: echo "TEST_MATRIX=${TEST_TARGETS}" >> $GITHUB_ENV - - - id: set-matrix - run: echo "matrix=${{ env.TEST_MATRIX}}" >> $GITHUB_OUTPUT - - Build: - name: Build - runs-on: ubuntu-20.04 - timeout-minutes: 15 - steps: - - name: Check out the repo - uses: actions/checkout@v3 - - - name: Init Hermit - run: ./bin/hermit env -r >> $GITHUB_ENV - - - name: cache go dependencies - uses: actions/cache@v3 - with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - key: ${{ runner.os }}-go-${{ matrix.go-version }}-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-${{ matrix.go-version }}- - - - name: build cloudbeat binary - uses: magefile/mage-action@v2 - with: - version: latest - args: build - - - name: build opa bundle - uses: magefile/mage-action@v2 - with: - version: latest - args: BuildOpaBundle - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Cache Build dependencies - uses: actions/cache@v3 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.workflow }} - restore-keys: | - ${{ runner.os }}-buildx- - - - name: Build cloudbeat-docker - uses: docker/build-push-action@v2 - with: - context: . - push: false - tags: cloudbeat:latest - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new - outputs: type=docker,dest=/tmp/cloudbeat-${{ env.CONTAINER_SUFFIX }}.tar - - - name: Build elastic-agent - run: | - make build_elastic_agent_docker_image - - - name: Build pytest-docker - uses: docker/build-push-action@v3 - with: - context: ./tests/. - push: false - tags: cloudbeat-test:latest - cache-from: type=local,mode=max,src=/tmp/.buildx-cache - cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-new - outputs: type=docker,dest=/tmp/pytest-${{ env.CONTAINER_SUFFIX }}.tar - - - name: Cache docker images - uses: actions/cache@v3 - with: - path: /tmp/*.tar - key: ${{ runner.os }}-dockers-cache-${{ env.CONTAINER_SUFFIX }} - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ env.AWS_REGION }} - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - - name: Load images cloudbeat and cloudbeat-test to ECR - id: load-to-ecr - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - run: | - ./.ci/scripts/ecr-images.sh ${{ env.CONTAINER_SUFFIX }} ${{ env.ECR_REGISTRY}} - shell: bash - - - - name: Move cache - run: | - ./.ci/scripts/gh-cache.sh - shell: bash - - Test_Matrix: - needs: ["Setup", "Build"] - strategy: - fail-fast: false - matrix: - include: - - test-target: pre_merge - range: "" - values_file: tests/deploy/values/ci-eks.yml - k8s_context: "test-eks-config-1" - name: ${{ matrix.test-target }}-tests - runs-on: ubuntu-20.04 - timeout-minutes: 90 - steps: - - name: Check out the repo - uses: actions/checkout@v3 - - # Setup all required tools - - name: Init Hermit - run: ./bin/hermit env -r >> $GITHUB_ENV - - - name: Get kubeconfig and select context - if: success() - run: | - aws eks update-kubeconfig --name ${{ matrix.k8s_context }} --region ${{ env.AWS_REGION }} --alias ${{ matrix.k8s_context }} - echo 'KUBE_CONFIG_DATA=$(cat ~/.kube/config | base64)' >> $GITHUB_ENV - kubectl config use-context ${{ matrix.k8s_context }} - kubectl config set-context --current --namespace=${{ env.KUBE_NAMESPACE }} - - - name: Deploy tests Helm chart - id: deploy_helm - if: success() - run: | - just deploy-tests-helm ${{ matrix.test-target }} ${{ matrix.values_file }} ${{ matrix.range }} - - - name: Execute tests - id: run_tests - if: success() - run: | - just run-tests ${{ matrix.test-target }} ${{ matrix.k8s_context }} - - - name: Copy test results - id: copy_results - if: always() - run: | - NODE=$(kubectl get po test-pod-v1 -o jsonpath='{.spec.nodeName}') - RESULTS_POD=$(kubectl get po -l k8s-app=eks-results --field-selector spec.nodeName=$NODE -o jsonpath='{.items[*].metadata.name}') - kubectl cp $RESULTS_POD:/reports ${{ env.REPORTS_DIR }} - kubectl exec $RESULTS_POD -- sh -c 'rm -rf /reports/*' - - - name: Upload Test Results - if: success() - uses: actions/upload-artifact@v3 - with: - name: allure_results-${{ matrix.test-target}} - path: ${{ env.REPORTS_DIR }} - - - name: Purge tests environment - id: purge_helm - if: always() - run: | - just purge-tests - - publish_results: - name: Publish Results - needs: Test_Matrix - if: always() - runs-on: ubuntu-20.04 - steps: - - name: Download Artifacts - uses: actions/download-artifact@v3 - with: - path: artifacts - - - name: Extract Artifacts - run: | - mkdir -p tests/allure/results - find artifacts/ -type f -print0 | xargs -0 mv -t tests/allure/results - - - name: Publish allure report - if: always() - uses: andrcuns/allure-publish-action@v1.0.1 - env: - GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - AWS_REGION: eu-west-1 - with: - storageType: s3 - resultsGlob: "${{ env.REPORTS_DIR }}*" - updatePr: actions - collapseSummary: false - summary: suites - summaryTableType: markdown - copyLatest: true - bucket: csp-allure-reports - prefix: allure_reports/cloudbeat/${{ github.ref_name }} - ignoreMissingResults: true diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml deleted file mode 100644 index 9240f51a1c..0000000000 --- a/.github/workflows/packaging.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: Packaging - -# TODO - revert - - -env: - DEV: true - SNAPSHOT: true - PLATFORMS: linux/amd64 - GOPATH: /home/runner/go - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - packag_beat: - name: Package Cloudbeat - runs-on: ubuntu-20.04 - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - types: [ tar.gz, docker ] - steps: - - name: Check out the repo - uses: actions/checkout@v2 - - - name: Init Hermit - run: ./bin/hermit env -r >> $GITHUB_ENV - - - name: cache go dependencies - uses: actions/cache@v2 - with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - key: ${{ runner.os }}-go-${{ matrix.go-version }}-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-${{ matrix.go-version }}- - - - name: Packaging - run: | - mage -v package - env: - TYPES: ${{ matrix.types }} diff --git a/.github/workflows/publish-test-results.yml b/.github/workflows/publish-test-results.yml deleted file mode 100644 index 615a8c61f0..0000000000 --- a/.github/workflows/publish-test-results.yml +++ /dev/null @@ -1,82 +0,0 @@ -name: Publish-Test-Results - -# TODO - revert - - -jobs: - publish_results: - timeout-minutes: 15 - name: Publish Results - runs-on: ubuntu-20.04 - if: github.event.workflow_run.conclusion != 'skipped' - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - permissions: - checks: write - # needed unless run with comment_mode: off - pull-requests: write - # required by download step to access artifacts API - actions: read - steps: - - - name: Download and Extract Artifacts - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - mkdir -p tests/allure/results - mkdir -p artifacts && cd artifacts - - artifacts_url=${{ github.event.workflow_run.artifacts_url }} - - gh api "$artifacts_url" -q '.artifacts[] | [.name, .archive_download_url] | @tsv' | while read artifact - do - IFS=$'\t' read name url <<< "$artifact" - gh api $url > "$name.zip" - unzip -o "$name.zip" -d "../tests/allure/results" - done - - - name: Get pr info - id: pr_info - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - PR_NUM=$(gh api -X GET search/issues -f q='elastic/cloudbeat is:open is:pr is:unmerged ${{ github.event.workflow_run.head_sha }}' | jq '.items[0].number' ) - echo "PR_NUMBER=$PR_NUM" >> $GITHUB_ENV - - - name: Publish allure report - if: always() - uses: andrcuns/allure-publish-action@v1.0.1 - env: - GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - storageType: s3 - resultsGlob: "tests/allure/results/*" - updatePr: actions - collapseSummary: false - summary: suites - summaryTableType: markdown - copyLatest: true - bucket: csp-allure-reports - prefix: allure_reports/cloudbeat/prs/${{ env.PR_NUMBER }} - ignoreMissingResults: true - - - name: Output to Job Summary - if: ${{ github.event.workflow_run.event == 'pull_request' }} - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - echo "### Run Summary :rocket:" >> $GITHUB_STEP_SUMMARY - echo "Allure Report: http://csp-allure-reports.s3.amazonaws.com/allure_reports/cloudbeat/prs/${{ env.PR_NUMBER }}/index.html" >> $GITHUB_STEP_SUMMARY - - - name: Comment test success result - uses: marocchino/sticky-pull-request-comment@v2 - if: ${{ github.event.workflow_run.event == 'pull_request' }} - with: - header: Test Results - number: ${{ env.PR_NUMBER }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - message: | - ## Cloudbeat CI :robot: - Allure Report: http://csp-allure-reports.s3.amazonaws.com/allure_reports/cloudbeat/prs/${{ env.PR_NUMBER }}/index.html diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml deleted file mode 100644 index 34393ec976..0000000000 --- a/.github/workflows/unit-test.yml +++ /dev/null @@ -1,128 +0,0 @@ -name: UnitTests - -# TODO - revert -env: - K8S_MANIFEST_DIR: deploy - - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - unit_tests: - name: Unit Tests - runs-on: ubuntu-20.04 - timeout-minutes: 15 - steps: - - name: Check out the repo - uses: actions/checkout@v2 - - - name: Init Hermit - run: ./bin/hermit env -r >> $GITHUB_ENV - - - name: Mage Check - run: mage check - - - name: Mage checkLicenseHeaders - run: mage checkLicenseHeaders - - - name: Mage build bundle before unit tests - uses: magefile/mage-action@v1.6.0 - with: - args: buildOpaBundle - - - name: Copy bundle to required dirs - run: | - cp bundle.tar.gz evaluator/ - - - name: Unit-Test Cloudbeat - run: | - GOOS=linux go test -v -coverpkg=./... -coverprofile=cover.out.tmp ./... - cat cover.out.tmp | grep -v "_mock.go" > cover.out # remove mock files from coverage report - - - name: Upload coverage artifact - uses: actions/upload-artifact@v3 - with: - name: coverage-file - path: cover.out - - coverage: - name: Coverage report - needs: unit_tests - runs-on: ubuntu-20.04 - timeout-minutes: 10 - steps: - - name: Check out the repo - uses: actions/checkout@v2 - - - name: Init Hermit - run: ./bin/hermit env -r >> $GITHUB_ENV - - - name: Install goveralls - run: go install github.com/mattn/goveralls@latest - - - name: Download coverage artifact - uses: actions/download-artifact@v3 - with: - name: coverage-file - - - name: Send coverage - env: - COVERALLS_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: goveralls -coverprofile=cover.out -service=github - - manifest_tests: - name: Manifest Tests - runs-on: ubuntu-20.04 - timeout-minutes: 15 - steps: - - name: Check out the repo - uses: actions/checkout@v2 - - - name: Get diff k8s manifests - uses: technote-space/get-diff-action@v4 - with: - PATTERNS: ${{ env.K8S_MANIFEST_DIR }}/** - - - name: Run K8s manifest tests - if: env.GIT_DIFF - uses: stefanprodan/kube-tools@v1 - with: - kubectl: 1.18.4 - kubeval: v0.16.1 - command: | - kubeval --strict -d ${{ env.K8S_MANIFEST_DIR }} --ignore-missing-schemas --ignored-filename-patterns kustomization,kind,patch,cloudbeat,cloudformation --skip-kinds ClusterConfig - - golangci: - name: golangci-lint - runs-on: ubuntu-20.04 - timeout-minutes: 15 - steps: - - name: Check out the repo - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - name: Check out the repo - uses: actions/checkout@v3 - with: - token: ${{ secrets.GITHUB_TOKEN }} - repository: ${{ (github.event.pull_request_target || github.event.pull_request).head.repo.full_name }} - ref: ${{ (github.event.pull_request_target || github.event.pull_request).head.sha }} - - - name: Go linter - uses: golangci/golangci-lint-action@v3 - with: - version: latest - args: --timeout=10m --whole-files - - dependency-review: - runs-on: ubuntu-20.04 - if: ${{ github.event_name == 'pull_request' }} - steps: - - name: Checkout Repository - uses: actions/checkout@v3 - - - name: Dependency Review - uses: actions/dependency-review-action@v2 From a9995b4b24e599d9e1bc00f81194698a0c391c12 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Wed, 25 Jan 2023 15:01:49 +0200 Subject: [PATCH 017/127] update workflow --- .github/workflows/weekly-enviroment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index ae89625ff4..4cf92d6fa1 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -36,8 +36,8 @@ jobs: - name: List all files before setting directory run: ls -la - - name: Set working directory - working-directory: cloudbeat/deploy/cloud + - name: Change working directory + run: cd cloudbeat/deploy/cloud - name: List all files run: ls -la From 4c57305a6ccedc0629c12b1e04eaa772557b0d11 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Wed, 25 Jan 2023 15:09:25 +0200 Subject: [PATCH 018/127] update workflow --- .github/workflows/weekly-enviroment.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 4cf92d6fa1..3b10b19ce3 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -1,13 +1,13 @@ name: Weekly environment deployment on: - pull_request: - types: [labeled] - labels: - - weekly deployment - push: - tags: - - weekly-deployment +# pull_request: +# types: [labeled] +# labels: +# - weekly deployment +# push: +# tags: +# - weekly-deployment workflow_dispatch: inputs: test-targets: @@ -37,7 +37,7 @@ jobs: run: ls -la - name: Change working directory - run: cd cloudbeat/deploy/cloud + run: cd deploy/cloud - name: List all files run: ls -la From 4693c7a8a7ac106e6e284e0ae748610e8085d969 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Wed, 25 Jan 2023 15:13:07 +0200 Subject: [PATCH 019/127] update workflow --- .github/workflows/weekly-enviroment.yml | 27 ++++++++++++++++--------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 3b10b19ce3..6c0e2dca89 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -1,19 +1,26 @@ name: Weekly environment deployment on: -# pull_request: -# types: [labeled] -# labels: -# - weekly deployment -# push: -# tags: -# - weekly-deployment workflow_dispatch: inputs: - test-targets: + logLevel: + description: 'Log level' required: true - description: "Run it on demand" - default: '[\"pre_merge\"]' + default: 'warning' + type: choice + options: + - info + - warning + - debug + tags: + description: 'Test scenario tags' + required: false + type: boolean + environment: + description: 'Environment to run tests against' + type: environment + required: true + env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} From c3d921aac3ebecf00c2a98ab18bd7e43629355ca Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Wed, 25 Jan 2023 15:18:05 +0200 Subject: [PATCH 020/127] update workflow --- .github/workflows/weekly-enviroment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 6c0e2dca89..e5c616ab88 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -46,7 +46,7 @@ jobs: - name: Change working directory run: cd deploy/cloud - - name: List all files + - name: List all files in deploy/cloud run: ls -la - name: Terraform Format From 939140a78d34fa2342300562ec50e2c3b73132a6 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Wed, 25 Jan 2023 15:22:52 +0200 Subject: [PATCH 021/127] update enviroment with choices --- .github/workflows/weekly-enviroment.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index e5c616ab88..33e673b58e 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -18,8 +18,10 @@ on: type: boolean environment: description: 'Environment to run tests against' - type: environment - required: true + type: choice + options: + - weekly environment + - local environment env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} From 2e62231fc6f7e92e50ea2e8233b1919deb2f8690 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Wed, 25 Jan 2023 15:43:06 +0200 Subject: [PATCH 022/127] adding terraform init --- .github/workflows/weekly-enviroment.yml | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 33e673b58e..ad3e915871 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -31,6 +31,7 @@ env: AWS_REGION: eu-west-2 REPORTS_DIR: tests/allure/results/ KUBE_NAMESPACE: kube-system + WORKING_DIR: deploy/cloud jobs: deploy: @@ -42,18 +43,19 @@ jobs: - name: Check out the repo uses: actions/checkout@v2 - - name: List all files before setting directory - run: ls -la - - - name: Change working directory - run: cd deploy/cloud - - - name: List all files in deploy/cloud - run: ls -la + - name: 'Terraform Init' + uses: hashicorp/terraform-github-actions@master + with: + tf_actions_version: 1.3.5 + tf_actions_subcommand: 'init' + tf_actions_working_dir: ${{ env.WORKING_DIR }} + tf_actions_comment: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Terraform Format - id: fmt - run: terraform fmt -check +# - name: Terraform Format +# id: fmt +# run: terraform fmt -check - name: Comment test success result uses: marocchino/sticky-pull-request-comment@v2 From e8bdf48c188b5298cebfe20c921ba85ca1ceca37 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Wed, 25 Jan 2023 15:48:15 +0200 Subject: [PATCH 023/127] adding terraform plan --- .github/workflows/weekly-enviroment.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index ad3e915871..14cbc9dc2e 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -53,9 +53,13 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -# - name: Terraform Format -# id: fmt -# run: terraform fmt -check + - name: 'Terraform Plan' + uses: hashicorp/terraform-github-actions@master + with: + tf_actions_version: 1.3.5 + tf_actions_subcommand: 'plan' + tf_actions_working_dir: ${{ env.WORKING_DIR }} + tf_actions_comment: true - name: Comment test success result uses: marocchino/sticky-pull-request-comment@v2 From 03bcb1cadea062592ee52008599248add7fdaf51 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Thu, 26 Jan 2023 10:11:20 +0200 Subject: [PATCH 024/127] updating job with plan target --- .github/workflows/weekly-enviroment.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 14cbc9dc2e..72998ffacd 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -37,9 +37,6 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - name: Echo test completed - run: echo "test completed" - - name: Check out the repo uses: actions/checkout@v2 @@ -57,7 +54,7 @@ jobs: uses: hashicorp/terraform-github-actions@master with: tf_actions_version: 1.3.5 - tf_actions_subcommand: 'plan' + tf_actions_subcommand: 'plan -target=module.ec_deployment' tf_actions_working_dir: ${{ env.WORKING_DIR }} tf_actions_comment: true @@ -72,3 +69,4 @@ jobs: You can access it at _______. :warning: This environment will be deleted in 7 days. +# terraform apply --auto-approve -target "module.ec_deployment" From 3b20d5a381f444a4c250bbb2376fd92e5c2afcaa Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Thu, 26 Jan 2023 10:34:23 +0200 Subject: [PATCH 025/127] updating job with plan target --- .github/workflows/weekly-enviroment.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 72998ffacd..6ccaabdf73 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -54,9 +54,10 @@ jobs: uses: hashicorp/terraform-github-actions@master with: tf_actions_version: 1.3.5 - tf_actions_subcommand: 'plan -target=module.ec_deployment' + tf_actions_subcommand: 'plan' tf_actions_working_dir: ${{ env.WORKING_DIR }} tf_actions_comment: true + tf_actions_args: '-target=module.ec_deployment' - name: Comment test success result uses: marocchino/sticky-pull-request-comment@v2 From dcbc1bbe07798fcb16a79ae4270c950ed7330160 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Thu, 26 Jan 2023 10:37:59 +0200 Subject: [PATCH 026/127] updating terraform --- .github/workflows/weekly-enviroment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 6ccaabdf73..6db6bc42c9 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -57,7 +57,7 @@ jobs: tf_actions_subcommand: 'plan' tf_actions_working_dir: ${{ env.WORKING_DIR }} tf_actions_comment: true - tf_actions_args: '-target=module.ec_deployment' + args: '-target=module.ec_deployment' - name: Comment test success result uses: marocchino/sticky-pull-request-comment@v2 From b4d6579f0a0dd89841cdc62e17984acff9ed8cc0 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Thu, 26 Jan 2023 10:52:06 +0200 Subject: [PATCH 027/127] updating terraform --- .github/workflows/weekly-enviroment.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 6db6bc42c9..dd5ba8504b 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -58,6 +58,8 @@ jobs: tf_actions_working_dir: ${{ env.WORKING_DIR }} tf_actions_comment: true args: '-target=module.ec_deployment' + env: + TF_VAR_ec_api_key: ${{ secrets.TF_VAR_EC_API_KEY }} - name: Comment test success result uses: marocchino/sticky-pull-request-comment@v2 From 99100938edb11c22ac1a42b997aa18eca6f6904f Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Thu, 26 Jan 2023 11:02:26 +0200 Subject: [PATCH 028/127] adding terraform apply --- .github/workflows/weekly-enviroment.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index dd5ba8504b..2f9980c894 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -61,6 +61,17 @@ jobs: env: TF_VAR_ec_api_key: ${{ secrets.TF_VAR_EC_API_KEY }} + - name: 'Terraform Apply' + uses: hashicorp/terraform-github-actions@master + with: + tf_actions_version: 1.3.5 + tf_actions_subcommand: 'apply' + tf_actions_working_dir: ${{ env.WORKING_DIR }} + tf_actions_comment: true + args: '-target=module.ec_deployment -auto-approve' + env: + TF_VAR_ec_api_key: ${{ secrets.TF_VAR_EC_API_KEY }} + - name: Comment test success result uses: marocchino/sticky-pull-request-comment@v2 with: From 2e6d40a1976c9f93f6cc5444df62a97ce4e5109c Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Thu, 26 Jan 2023 11:54:10 +0200 Subject: [PATCH 029/127] updating github actions variables --- .github/workflows/weekly-enviroment.yml | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 2f9980c894..ca199a96e5 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -3,6 +3,20 @@ name: Weekly environment deployment on: workflow_dispatch: inputs: + environment: + description: 'Environment to run tests against' + type: choice + options: + - weekly environment + benchmark: + description: 'The benchmark you wish to install' + type: choice + options: + - Kubernetes Vanilla(k8s-vanilla) + alerts: + description: 'Install alerts' + required: false + type: boolean logLevel: description: 'Log level' required: true @@ -12,16 +26,6 @@ on: - info - warning - debug - tags: - description: 'Test scenario tags' - required: false - type: boolean - environment: - description: 'Environment to run tests against' - type: choice - options: - - weekly environment - - local environment env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} From a4a9f427e294dbd4cd268b7ba5c10ea77e612a2d Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Thu, 26 Jan 2023 12:17:05 +0200 Subject: [PATCH 030/127] adding terraform output --- .github/workflows/weekly-enviroment.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index ca199a96e5..bf996a7217 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -66,6 +66,7 @@ jobs: TF_VAR_ec_api_key: ${{ secrets.TF_VAR_EC_API_KEY }} - name: 'Terraform Apply' + id: terraform_apply uses: hashicorp/terraform-github-actions@master with: tf_actions_version: 1.3.5 @@ -76,6 +77,9 @@ jobs: env: TF_VAR_ec_api_key: ${{ secrets.TF_VAR_EC_API_KEY }} + - name: 'Output Terraform Output' + run: echo ${{ steps.terraform_apply.outputs.kibana_url }} + - name: Comment test success result uses: marocchino/sticky-pull-request-comment@v2 with: From 08f17ff0a36bf26287962ebb5436c61d5c9c8093 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Thu, 26 Jan 2023 12:46:21 +0200 Subject: [PATCH 031/127] adding terraform output --- .github/workflows/weekly-enviroment.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index bf996a7217..878ae4ecf0 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -77,8 +77,15 @@ jobs: env: TF_VAR_ec_api_key: ${{ secrets.TF_VAR_EC_API_KEY }} - - name: 'Output Terraform Output' - run: echo ${{ steps.terraform_apply.outputs.kibana_url }} + - name: Call terraform output url parameter + run: | + output=$(terraform output elasticsearch_url) + echo $elasticsearch_url + + - name: Call terraform output no parameter + run: | + output1=$(terraform output ) + echo $elasticsearch_url1 - name: Comment test success result uses: marocchino/sticky-pull-request-comment@v2 From ed05aedeb269fc56eae58a17e78c2ef173fac37c Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Thu, 26 Jan 2023 13:39:29 +0200 Subject: [PATCH 032/127] adding terraform setup --- .github/workflows/weekly-enviroment.yml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 878ae4ecf0..c8899871e7 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -38,26 +38,30 @@ env: WORKING_DIR: deploy/cloud jobs: - deploy: + terraform: + name: Deploy KSPM/CSPM cloud environment runs-on: ubuntu-latest steps: - name: Check out the repo uses: actions/checkout@v2 - - name: 'Terraform Init' + - name: Setup Terraform + uses: hashicorp/setup-terraform@v1 + with: + terraform_version: 1.3.5 + + - name: Terraform Init uses: hashicorp/terraform-github-actions@master with: - tf_actions_version: 1.3.5 tf_actions_subcommand: 'init' tf_actions_working_dir: ${{ env.WORKING_DIR }} tf_actions_comment: true env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: 'Terraform Plan' + - name: Terraform Plan uses: hashicorp/terraform-github-actions@master with: - tf_actions_version: 1.3.5 tf_actions_subcommand: 'plan' tf_actions_working_dir: ${{ env.WORKING_DIR }} tf_actions_comment: true @@ -65,11 +69,9 @@ jobs: env: TF_VAR_ec_api_key: ${{ secrets.TF_VAR_EC_API_KEY }} - - name: 'Terraform Apply' - id: terraform_apply + - name: Terraform Apply uses: hashicorp/terraform-github-actions@master with: - tf_actions_version: 1.3.5 tf_actions_subcommand: 'apply' tf_actions_working_dir: ${{ env.WORKING_DIR }} tf_actions_comment: true From 7c1c9195d627f8f177b0825676cec4ea8d244d88 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Thu, 26 Jan 2023 13:59:44 +0200 Subject: [PATCH 033/127] using the latest hasicorp teraform github action package --- .github/workflows/weekly-enviroment.yml | 42 ++++++++++++------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index c8899871e7..b72aad56d6 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -41,43 +41,41 @@ jobs: terraform: name: Deploy KSPM/CSPM cloud environment runs-on: ubuntu-latest + defaults: + run: + working-directory: ${{ env.WORKING_DIR }} steps: - name: Check out the repo uses: actions/checkout@v2 - name: Setup Terraform - uses: hashicorp/setup-terraform@v1 + uses: hashicorp/setup-terraform@v2 with: terraform_version: 1.3.5 + terraform_wrapper: false - - name: Terraform Init - uses: hashicorp/terraform-github-actions@master - with: - tf_actions_subcommand: 'init' - tf_actions_working_dir: ${{ env.WORKING_DIR }} - tf_actions_comment: true - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} +# - name: Terraform fmt +# id: fmt +# run: terraform fmt -check +# continue-on-error: true +# +# - name: Terraform Validate +# id: validate +# run: terraform validate -no-color - name: Terraform Plan - uses: hashicorp/terraform-github-actions@master - with: - tf_actions_subcommand: 'plan' - tf_actions_working_dir: ${{ env.WORKING_DIR }} - tf_actions_comment: true - args: '-target=module.ec_deployment' + id: plan + run: terraform plan -target=module.ec_deployment -no-color + continue-on-error: true env: TF_VAR_ec_api_key: ${{ secrets.TF_VAR_EC_API_KEY }} - name: Terraform Apply - uses: hashicorp/terraform-github-actions@master - with: - tf_actions_subcommand: 'apply' - tf_actions_working_dir: ${{ env.WORKING_DIR }} - tf_actions_comment: true - args: '-target=module.ec_deployment -auto-approve' + id: apply + run: terraform apply -auto-approve -target=module.ec_deployment -no-color + continue-on-error: true env: - TF_VAR_ec_api_key: ${{ secrets.TF_VAR_EC_API_KEY }} + TF_VAR_ec_api_key: ${{ secrets.TF_VAR_EC_API_KEY }} - name: Call terraform output url parameter run: | From d7bff9981fa4e95925efcc89719e26dd3a1e035c Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Thu, 26 Jan 2023 14:02:24 +0200 Subject: [PATCH 034/127] using the latest hasicorp teraform github action package --- .github/workflows/weekly-enviroment.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index b72aad56d6..b3712ce3ea 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -62,6 +62,9 @@ jobs: # - name: Terraform Validate # id: validate # run: terraform validate -no-color + - name: Terraform Init + id: init + run: terraform init -no-color - name: Terraform Plan id: plan From b041ac84121bf1e85959b5772e0921091fcfed2b Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Thu, 26 Jan 2023 14:03:32 +0200 Subject: [PATCH 035/127] using the latest hasicorp teraform github action package --- .github/workflows/weekly-enviroment.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index b3712ce3ea..fd6153a014 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -69,14 +69,12 @@ jobs: - name: Terraform Plan id: plan run: terraform plan -target=module.ec_deployment -no-color - continue-on-error: true env: TF_VAR_ec_api_key: ${{ secrets.TF_VAR_EC_API_KEY }} - name: Terraform Apply id: apply run: terraform apply -auto-approve -target=module.ec_deployment -no-color - continue-on-error: true env: TF_VAR_ec_api_key: ${{ secrets.TF_VAR_EC_API_KEY }} From fcbe9baf59a8120d28770ddb2fbad88816fb5e00 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Thu, 26 Jan 2023 14:14:39 +0200 Subject: [PATCH 036/127] updating terraform output --- .github/workflows/weekly-enviroment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index fd6153a014..cdc85baec9 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -86,7 +86,7 @@ jobs: - name: Call terraform output no parameter run: | output1=$(terraform output ) - echo $elasticsearch_url1 + echo $output1 - name: Comment test success result uses: marocchino/sticky-pull-request-comment@v2 From 3b167ddd9edbcaefaee52170bf26585c772b418b Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Thu, 26 Jan 2023 14:41:26 +0200 Subject: [PATCH 037/127] updating output --- .github/workflows/weekly-enviroment.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index cdc85baec9..22278db231 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -33,7 +33,6 @@ env: GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} CONTAINER_SUFFIX: ${{ github.run_id }} AWS_REGION: eu-west-2 - REPORTS_DIR: tests/allure/results/ KUBE_NAMESPACE: kube-system WORKING_DIR: deploy/cloud @@ -80,8 +79,8 @@ jobs: - name: Call terraform output url parameter run: | - output=$(terraform output elasticsearch_url) - echo $elasticsearch_url + output=$(terraform output kibana_url) + echo $output - name: Call terraform output no parameter run: | From 1043ea118de8bfe2b82140e77adb9ad87043e039 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Thu, 26 Jan 2023 15:06:06 +0200 Subject: [PATCH 038/127] updating comment message --- .github/workflows/weekly-enviroment.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 22278db231..53709aa0b7 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -85,6 +85,14 @@ jobs: - name: Call terraform output no parameter run: | output1=$(terraform output ) + echo "::set-output name=tf_output::$tf_output" + echo $output1 + + - name: Get terraform output + id: terraform_output + run: | + output1=$(terraform output) + echo "::set-output name=tf_output::$tf_output" echo $output1 - name: Comment test success result @@ -95,7 +103,9 @@ jobs: message: | ## Weekly environment :man_dancing: A new environment has been deployed to the elastic cloud. :tada: - You can access it at _______. + You can access it at $output1. + Or you can use ${{ steps.terraform_output.outputs.output1 }} + Or you can use 2 ${{ steps.terraform_output.outputs.tf_output }} :warning: This environment will be deleted in 7 days. # terraform apply --auto-approve -target "module.ec_deployment" From 755569a1e3d9810a4ec1f8addfbc1d8fc1501048 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 09:45:27 +0200 Subject: [PATCH 039/127] updating comment message --- .github/workflows/weekly-enviroment.yml | 46 ++++++++++++------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 53709aa0b7..12e4ea4c6d 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -71,29 +71,29 @@ jobs: env: TF_VAR_ec_api_key: ${{ secrets.TF_VAR_EC_API_KEY }} - - name: Terraform Apply - id: apply - run: terraform apply -auto-approve -target=module.ec_deployment -no-color - env: - TF_VAR_ec_api_key: ${{ secrets.TF_VAR_EC_API_KEY }} - - - name: Call terraform output url parameter - run: | - output=$(terraform output kibana_url) - echo $output - - - name: Call terraform output no parameter - run: | - output1=$(terraform output ) - echo "::set-output name=tf_output::$tf_output" - echo $output1 - - - name: Get terraform output - id: terraform_output - run: | - output1=$(terraform output) - echo "::set-output name=tf_output::$tf_output" - echo $output1 +# - name: Terraform Apply +# id: apply +# run: terraform apply -auto-approve -target=module.ec_deployment -no-color +# env: +# TF_VAR_ec_api_key: ${{ secrets.TF_VAR_EC_API_KEY }} +# +# - name: Call terraform output url parameter +# run: | +# output=$(terraform output kibana_url) +# echo $output +# +# - name: Call terraform output no parameter +# run: | +# output1=$(terraform output ) +# echo "::set-output name=tf_output::$tf_output" +# echo $output1 +# +# - name: Get terraform output +# id: terraform_output +# run: | +# output1=$(terraform output) +# echo "::set-output name=tf_output::$tf_output" +# echo $output1 - name: Comment test success result uses: marocchino/sticky-pull-request-comment@v2 From 83fcf356d8961a579b83347c20196c54268886ce Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 09:46:02 +0200 Subject: [PATCH 040/127] updating comment message --- .github/workflows/weekly-enviroment.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 12e4ea4c6d..5eff48f094 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -103,9 +103,5 @@ jobs: message: | ## Weekly environment :man_dancing: A new environment has been deployed to the elastic cloud. :tada: - You can access it at $output1. - Or you can use ${{ steps.terraform_output.outputs.output1 }} - Or you can use 2 ${{ steps.terraform_output.outputs.tf_output }} - :warning: This environment will be deleted in 7 days. -# terraform apply --auto-approve -target "module.ec_deployment" + From a6e41562dc0466f843c0bdfd917fa703d6fe941f Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 09:49:51 +0200 Subject: [PATCH 041/127] updating comment message --- .github/workflows/weekly-enviroment.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 5eff48f094..046f62d8f9 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -99,7 +99,6 @@ jobs: uses: marocchino/sticky-pull-request-comment@v2 with: header: Test Results - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} message: | ## Weekly environment :man_dancing: A new environment has been deployed to the elastic cloud. :tada: From 60552c2f0100c6605566b913c33cae1795ba0886 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 09:57:04 +0200 Subject: [PATCH 042/127] updating comment message --- .github/workflows/weekly-enviroment.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 046f62d8f9..36aaef050c 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -95,12 +95,12 @@ jobs: # echo "::set-output name=tf_output::$tf_output" # echo $output1 - - name: Comment test success result - uses: marocchino/sticky-pull-request-comment@v2 + - uses: marocchino/sticky-pull-request-comment@v2 with: - header: Test Results + append: true message: | - ## Weekly environment :man_dancing: - A new environment has been deployed to the elastic cloud. :tada: - + Test with ${{ github.sha }}. + ``` + ${{ steps.test.outputs.result }} + ``` From ea82c386c5caf3398bff1862b48b44e3888ba2c9 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 10:00:35 +0200 Subject: [PATCH 043/127] updating comment message --- .github/workflows/weekly-enviroment.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 36aaef050c..5971556853 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -95,12 +95,12 @@ jobs: # echo "::set-output name=tf_output::$tf_output" # echo $output1 - - uses: marocchino/sticky-pull-request-comment@v2 + - name: Comment test success result + uses: marocchino/sticky-pull-request-comment@v2 + if: ${{ github.event.workflow_run.event == 'pull_request' }} with: - append: true + header: Test Results + number: ${{ env.PR_NUMBER }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} message: | - Test with ${{ github.sha }}. - ``` - ${{ steps.test.outputs.result }} - ``` - + ## Cloudbeat CI :robot: From 7b4227353888ec8f8ca1e799143112437f06eed3 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 10:02:58 +0200 Subject: [PATCH 044/127] updating comment message --- .github/workflows/weekly-enviroment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 5971556853..6e8109e137 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -97,7 +97,7 @@ jobs: - name: Comment test success result uses: marocchino/sticky-pull-request-comment@v2 - if: ${{ github.event.workflow_run.event == 'pull_request' }} +# if: ${{ github.event.workflow_run.event == 'pull_request' }} with: header: Test Results number: ${{ env.PR_NUMBER }} From 4919b25a3cae608faf3720a7e8b4caf1a3f627c8 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 10:28:29 +0200 Subject: [PATCH 045/127] updating comment message --- .github/workflows/weekly-enviroment.yml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 6e8109e137..dd1be100bb 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -95,12 +95,15 @@ jobs: # echo "::set-output name=tf_output::$tf_output" # echo $output1 - - name: Comment test success result - uses: marocchino/sticky-pull-request-comment@v2 -# if: ${{ github.event.workflow_run.event == 'pull_request' }} + - name: Send custom JSON data to Slack workflow + id: slack + uses: slackapi/slack-github-action@v1.23.0 with: - header: Test Results - number: ${{ env.PR_NUMBER }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - message: | - ## Cloudbeat CI :robot: + # This data can be any valid JSON from a previous step in the GitHub Action + payload: | + { + "key": "value", + "foo": "bar" + } + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} From fee8efe001c59e8fbb553d43b65ba1de56720406 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 10:35:14 +0200 Subject: [PATCH 046/127] updating slack message --- .github/workflows/weekly-enviroment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index dd1be100bb..9c5f77170e 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -102,6 +102,7 @@ jobs: # This data can be any valid JSON from a previous step in the GitHub Action payload: | { + "text": "GitHub Action build result:", "key": "value", "foo": "bar" } From 80fa310ebafe7e5fa292ca1b629eeb27146e43f0 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 10:37:57 +0200 Subject: [PATCH 047/127] updating slack message --- .github/workflows/weekly-enviroment.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 9c5f77170e..4b4e01a990 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -100,11 +100,12 @@ jobs: uses: slackapi/slack-github-action@v1.23.0 with: # This data can be any valid JSON from a previous step in the GitHub Action - payload: | - { - "text": "GitHub Action build result:", - "key": "value", - "foo": "bar" - } + slack-message: "GitHub build result 111:" +# payload: | +# { +# "text": "GitHub Action build result:", +# "key": "value", +# "foo": "bar" +# } env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} From 4198fbcead5d9ee70bdddb38a95f2beba9bf8871 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 10:42:03 +0200 Subject: [PATCH 048/127] updating slack message --- .github/workflows/weekly-enviroment.yml | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 4b4e01a990..aaaab036f2 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -100,12 +100,18 @@ jobs: uses: slackapi/slack-github-action@v1.23.0 with: # This data can be any valid JSON from a previous step in the GitHub Action - slack-message: "GitHub build result 111:" -# payload: | -# { -# "text": "GitHub Action build result:", -# "key": "value", -# "foo": "bar" -# } + payload: | + { + "text": "GitHub Action build result: ${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head_commit.url }}", + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "GitHub Action build result: ${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head_commit.url }}" + } + } + ] + } env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} From 4504c58a424df8a251201c802398ee7bdae12ea2 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 10:47:33 +0200 Subject: [PATCH 049/127] updating slack message --- .github/workflows/weekly-enviroment.yml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index aaaab036f2..4747f21a87 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -102,14 +102,18 @@ jobs: # This data can be any valid JSON from a previous step in the GitHub Action payload: | { - "text": "GitHub Action build result: ${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head_commit.url }}", - "blocks": [ + "text": "Deployment started (In Progress)", + "attachments": [ { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "GitHub Action build result: ${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head_commit.url }}" - } + "pretext": "Deployment started", + "color": "dbab09", + "fields": [ + { + "title": "Status", + "short": true, + "value": "In Progress" + } + ] } ] } From 568b19343f8e9da46cd8fba21fd4a4719e5c81e5 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 10:55:20 +0200 Subject: [PATCH 050/127] updating slack message --- .github/workflows/weekly-enviroment.yml | 38 ++++++++++++++++++++----- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 4747f21a87..d116c3e2d5 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -102,20 +102,44 @@ jobs: # This data can be any valid JSON from a previous step in the GitHub Action payload: | { - "text": "Deployment started (In Progress)", - "attachments": [ + "blocks": [ { - "pretext": "Deployment started", - "color": "dbab09", + "type": "section", + "text": { + "type": "mrkdwn", + "text": ":rocket: *New Deployment Created!* :rocket:" + } + }, + { + "type": "section", "fields": [ { - "title": "Status", - "short": true, - "value": "In Progress" + "type": "mrkdwn", + "text": "*Environment:*\n Production" + }, + { + "type": "mrkdwn", + "text": "*URL:*\n https://example.com" + }, + { + "type": "mrkdwn", + "text": "*Username:*\n deployuser" + }, + { + "type": "mrkdwn", + "text": "*Password:*\n *****" } ] + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": ":warning: *Access these credentials securely and keep them confidential!* :warning:" + } } ] } + env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} From 75dd2fbcb2f492b2a3cbfc15b8172e2483d22ec7 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 11:15:28 +0200 Subject: [PATCH 051/127] updating slack message --- .github/workflows/weekly-enviroment.yml | 38 +++++++++++-------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index d116c3e2d5..832c181d18 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -102,41 +102,37 @@ jobs: # This data can be any valid JSON from a previous step in the GitHub Action payload: | { + "text": "Danny Torrence left a 1 star review for your property.", "blocks": [ { "type": "section", "text": { "type": "mrkdwn", - "text": ":rocket: *New Deployment Created!* :rocket:" + "text": "Danny Torrence left the following review for your property:" } }, { "type": "section", + "block_id": "section567", + "text": { + "type": "mrkdwn", + "text": " \n :star: \n Doors had too many axe holes, guest in room 237 was far too rowdy, whole place felt stuck in the 1920s." + }, + "accessory": { + "type": "image", + "image_url": "https://is5-ssl.mzstatic.com/image/thumb/Purple3/v4/d3/72/5c/d3725c8f-c642-5d69-1904-aa36e4297885/source/256x256bb.jpg", + "alt_text": "Haunted hotel image" + } + }, + { + "type": "section", + "block_id": "section789", "fields": [ { "type": "mrkdwn", - "text": "*Environment:*\n Production" - }, - { - "type": "mrkdwn", - "text": "*URL:*\n https://example.com" - }, - { - "type": "mrkdwn", - "text": "*Username:*\n deployuser" - }, - { - "type": "mrkdwn", - "text": "*Password:*\n *****" + "text": "*Average Rating*\n1.0" } ] - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": ":warning: *Access these credentials securely and keep them confidential!* :warning:" - } } ] } From 24e979ef49bd28871beb65cf4c85e58ea256d6a0 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 11:38:40 +0200 Subject: [PATCH 052/127] adding slack message file --- .../weekly-enviroment-slack-payload.json | 40 +++++++++++++++++++ .github/workflows/weekly-enviroment.yml | 39 +----------------- 2 files changed, 41 insertions(+), 38 deletions(-) create mode 100644 .github/workflows/weekly-enviroment-slack-payload.json diff --git a/.github/workflows/weekly-enviroment-slack-payload.json b/.github/workflows/weekly-enviroment-slack-payload.json new file mode 100644 index 0000000000..37eb33e80e --- /dev/null +++ b/.github/workflows/weekly-enviroment-slack-payload.json @@ -0,0 +1,40 @@ +{ + "text": ":rocket: *A new Deployment was Created!* :rocket:", + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Danny Torrence left the following review for your property: \n" + } + }, + { + "type": "section", + "block_id": "section567", + "text": { + "type": "mrkdwn", + "text": "Kibana URL: https://needs_to_be_added /n" + } + }, + { + "type": "section", + "block_id": "section789", + "fields": [ + { + "type": "mrkdwn", + "text": "Kibana username: needs_to_be_added /n" + } + ] + }, + { + "type": "section", + "block_id": "section789", + "fields": [ + { + "type": "mrkdwn", + "text": "kibana password: needs_to_be_added /n" + } + ] + } + ] +} diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 832c181d18..5c24d40115 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -99,43 +99,6 @@ jobs: id: slack uses: slackapi/slack-github-action@v1.23.0 with: - # This data can be any valid JSON from a previous step in the GitHub Action - payload: | - { - "text": "Danny Torrence left a 1 star review for your property.", - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "Danny Torrence left the following review for your property:" - } - }, - { - "type": "section", - "block_id": "section567", - "text": { - "type": "mrkdwn", - "text": " \n :star: \n Doors had too many axe holes, guest in room 237 was far too rowdy, whole place felt stuck in the 1920s." - }, - "accessory": { - "type": "image", - "image_url": "https://is5-ssl.mzstatic.com/image/thumb/Purple3/v4/d3/72/5c/d3725c8f-c642-5d69-1904-aa36e4297885/source/256x256bb.jpg", - "alt_text": "Haunted hotel image" - } - }, - { - "type": "section", - "block_id": "section789", - "fields": [ - { - "type": "mrkdwn", - "text": "*Average Rating*\n1.0" - } - ] - } - ] - } - + payload-file-path: "./weekly-enviroment-slack-payload.json" env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} From 252b88a55a9c3e394f919f548bddc1443a27abd9 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 11:43:40 +0200 Subject: [PATCH 053/127] adding payload json to env --- .github/workflows/weekly-enviroment.yml | 3 ++- ...lack-payload.json => weekly-environment-slack-payload.json} | 0 2 files changed, 2 insertions(+), 1 deletion(-) rename .github/workflows/{weekly-enviroment-slack-payload.json => weekly-environment-slack-payload.json} (100%) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 5c24d40115..d95dc0dd41 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -35,6 +35,7 @@ env: AWS_REGION: eu-west-2 KUBE_NAMESPACE: kube-system WORKING_DIR: deploy/cloud + SLACK_MESSAGE_PAYLOAD_PATH: ".github/workflows/weekly-environment-slack-payload.json" jobs: terraform: @@ -99,6 +100,6 @@ jobs: id: slack uses: slackapi/slack-github-action@v1.23.0 with: - payload-file-path: "./weekly-enviroment-slack-payload.json" + payload-file-path: ${{ env.SLACK_MESSAGE_PAYLOAD_PATH }} env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/weekly-enviroment-slack-payload.json b/.github/workflows/weekly-environment-slack-payload.json similarity index 100% rename from .github/workflows/weekly-enviroment-slack-payload.json rename to .github/workflows/weekly-environment-slack-payload.json From 421c2777f79d5359ca0565e2c254def326fcf61e Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 12:06:12 +0200 Subject: [PATCH 054/127] updating slack message --- .../weekly-environment-slack-payload.json | 33 ++++++++----------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/.github/workflows/weekly-environment-slack-payload.json b/.github/workflows/weekly-environment-slack-payload.json index 37eb33e80e..ec34da35fa 100644 --- a/.github/workflows/weekly-environment-slack-payload.json +++ b/.github/workflows/weekly-environment-slack-payload.json @@ -1,38 +1,31 @@ { - "text": ":rocket: *A new Deployment was Created!* :rocket:", + "text": "", "blocks": [ { "type": "section", "text": { "type": "mrkdwn", - "text": "Danny Torrence left the following review for your property: \n" + "text": ":rocket: *A new Deployment was Created!* :rocket:" } }, { "type": "section", - "block_id": "section567", - "text": { - "type": "mrkdwn", - "text": "Kibana URL: https://needs_to_be_added /n" - } - }, - { - "type": "section", - "block_id": "section789", "fields": [ { "type": "mrkdwn", - "text": "Kibana username: needs_to_be_added /n" - } - ] - }, - { - "type": "section", - "block_id": "section789", - "fields": [ + "text": "*Environment:*\n Weekly enviroment" + }, + { + "type": "mrkdwn", + "text": "*Kibana URL:*\n https://example.com" + }, + { + "type": "mrkdwn", + "text": "*Username:*\n deployuser" + }, { "type": "mrkdwn", - "text": "kibana password: needs_to_be_added /n" + "text": "*Password:*\n *****" } ] } From ed8a506d423e4c8fdec8a0f60f3be7e5846b2ad3 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 12:10:31 +0200 Subject: [PATCH 055/127] updating slack message --- .../weekly-environment-slack-payload.json | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/.github/workflows/weekly-environment-slack-payload.json b/.github/workflows/weekly-environment-slack-payload.json index ec34da35fa..4b2f1c0656 100644 --- a/.github/workflows/weekly-environment-slack-payload.json +++ b/.github/workflows/weekly-environment-slack-payload.json @@ -8,26 +8,5 @@ "text": ":rocket: *A new Deployment was Created!* :rocket:" } }, - { - "type": "section", - "fields": [ - { - "type": "mrkdwn", - "text": "*Environment:*\n Weekly enviroment" - }, - { - "type": "mrkdwn", - "text": "*Kibana URL:*\n https://example.com" - }, - { - "type": "mrkdwn", - "text": "*Username:*\n deployuser" - }, - { - "type": "mrkdwn", - "text": "*Password:*\n *****" - } - ] - } ] } From 7e3c050a6eb23e2bf19c18177bb75f9367f45ba6 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 12:11:53 +0200 Subject: [PATCH 056/127] updating slack message --- .github/workflows/weekly-environment-slack-payload.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/weekly-environment-slack-payload.json b/.github/workflows/weekly-environment-slack-payload.json index 4b2f1c0656..b769c9ec24 100644 --- a/.github/workflows/weekly-environment-slack-payload.json +++ b/.github/workflows/weekly-environment-slack-payload.json @@ -7,6 +7,6 @@ "type": "mrkdwn", "text": ":rocket: *A new Deployment was Created!* :rocket:" } - }, + } ] } From 10d0b6b2835942e17e98d060266cc5919498b0e9 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 12:13:41 +0200 Subject: [PATCH 057/127] updating slack message --- .github/workflows/weekly-environment-slack-payload.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/weekly-environment-slack-payload.json b/.github/workflows/weekly-environment-slack-payload.json index b769c9ec24..315d399458 100644 --- a/.github/workflows/weekly-environment-slack-payload.json +++ b/.github/workflows/weekly-environment-slack-payload.json @@ -1,5 +1,5 @@ { - "text": "", + "text": "some text", "blocks": [ { "type": "section", From 1a1afc5bcf03efaf7cd91b95f272ac8392e9abb7 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 12:15:36 +0200 Subject: [PATCH 058/127] updating slack message --- .../weekly-environment-slack-payload.json | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.github/workflows/weekly-environment-slack-payload.json b/.github/workflows/weekly-environment-slack-payload.json index 315d399458..17cacc41b2 100644 --- a/.github/workflows/weekly-environment-slack-payload.json +++ b/.github/workflows/weekly-environment-slack-payload.json @@ -7,6 +7,27 @@ "type": "mrkdwn", "text": ":rocket: *A new Deployment was Created!* :rocket:" } + }, + { + "type": "section", + "fields": [ + { + "type": "mrkdwn", + "text": "*Environment:*\n Weekly enviroment" + }, + { + "type": "mrkdwn", + "text": "*Kibana URL:*\n https://example.com" + }, + { + "type": "mrkdwn", + "text": "*Username:*\n deployuser" + }, + { + "type": "mrkdwn", + "text": "*Password:*\n *****" + } + ] } ] } From 8e5a720267ee67a48ce49b03737c2f225ca4bb8c Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 12:19:52 +0200 Subject: [PATCH 059/127] updating slack message --- .../weekly-environment-slack-payload.json | 51 +++++++++++-------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/.github/workflows/weekly-environment-slack-payload.json b/.github/workflows/weekly-environment-slack-payload.json index 17cacc41b2..949a7abe8a 100644 --- a/.github/workflows/weekly-environment-slack-payload.json +++ b/.github/workflows/weekly-environment-slack-payload.json @@ -1,31 +1,40 @@ { - "text": "some text", - "blocks": [ + "attachments": [ { - "type": "section", - "text": { - "type": "mrkdwn", - "text": ":rocket: *A new Deployment was Created!* :rocket:" - } - }, - { - "type": "section", - "fields": [ - { - "type": "mrkdwn", - "text": "*Environment:*\n Weekly enviroment" - }, + "color": "#d90000", + "blocks": [ { - "type": "mrkdwn", - "text": "*Kibana URL:*\n https://example.com" + "type": "section", + "text": { + "type": "mrkdwn", + "text": "${{ github.workflow }} workflow failed on ${{ env.DEFAULT_BRANCH }} branch in the ${{ github.event.repository.name }} repository!" + } }, { - "type": "mrkdwn", - "text": "*Username:*\n deployuser" + "type": "divider" }, { - "type": "mrkdwn", - "text": "*Password:*\n *****" + "type": "actions", + "elements": [ + { + "type": "button", + "text": { + "type": "plain_text", + "text": "Build", + "emoji": true + }, + "url": "${{ env.REPO_URL }}/actions/runs/${{ github.run_id }}" + }, + { + "type": "button", + "text": { + "type": "plain_text", + "text": "Commit", + "emoji": true + }, + "url": "${{ env.REPO_URL }}/commit/${{ github.sha }}" + } + ] } ] } From fc0818dfb83d687ceea9e6167be9a8d59d969ba5 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 12:27:30 +0200 Subject: [PATCH 060/127] updating slack message --- .github/workflows/weekly-enviroment.yml | 1 + .../weekly-environment-slack-payload.json | 51 ++++++++----------- 2 files changed, 22 insertions(+), 30 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index d95dc0dd41..d79a5b6b9b 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -103,3 +103,4 @@ jobs: payload-file-path: ${{ env.SLACK_MESSAGE_PAYLOAD_PATH }} env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK diff --git a/.github/workflows/weekly-environment-slack-payload.json b/.github/workflows/weekly-environment-slack-payload.json index 949a7abe8a..d7aac6d8b5 100644 --- a/.github/workflows/weekly-environment-slack-payload.json +++ b/.github/workflows/weekly-environment-slack-payload.json @@ -1,40 +1,31 @@ { - "attachments": [ + "text": ":rocket: *A new Deployment was Created!* :rocket:", + "blocks": [ { - "color": "#d90000", - "blocks": [ + "type": "section", + "text": { + "type": "mrkdwn", + "text": ":rocket: *A new Deployment was Created!* :rocket:" + } + }, + { + "type": "section", + "fields": [ + { + "type": "mrkdwn", + "text": "*Environment:*\n Weekly enviroment" + }, { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "${{ github.workflow }} workflow failed on ${{ env.DEFAULT_BRANCH }} branch in the ${{ github.event.repository.name }} repository!" - } + "type": "mrkdwn", + "text": "*Kibana URL:*\n https://example.com" }, { - "type": "divider" + "type": "mrkdwn", + "text": "*Username:*\n deployuser" }, { - "type": "actions", - "elements": [ - { - "type": "button", - "text": { - "type": "plain_text", - "text": "Build", - "emoji": true - }, - "url": "${{ env.REPO_URL }}/actions/runs/${{ github.run_id }}" - }, - { - "type": "button", - "text": { - "type": "plain_text", - "text": "Commit", - "emoji": true - }, - "url": "${{ env.REPO_URL }}/commit/${{ github.sha }}" - } - ] + "type": "mrkdwn", + "text": "*Password:*\n *****" } ] } From ed5d393e4c80b13f320acd88602083f6885d323e Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 13:27:10 +0200 Subject: [PATCH 061/127] updating slack message --- .github/workflows/weekly-environment-slack-payload.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/weekly-environment-slack-payload.json b/.github/workflows/weekly-environment-slack-payload.json index d7aac6d8b5..ec34da35fa 100644 --- a/.github/workflows/weekly-environment-slack-payload.json +++ b/.github/workflows/weekly-environment-slack-payload.json @@ -1,5 +1,5 @@ { - "text": ":rocket: *A new Deployment was Created!* :rocket:", + "text": "", "blocks": [ { "type": "section", From 81aa3c5230c625cfa66da4eea57366e58eec6abd Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 13:28:59 +0200 Subject: [PATCH 062/127] updating slack message --- .github/workflows/weekly-environment-slack-payload.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/weekly-environment-slack-payload.json b/.github/workflows/weekly-environment-slack-payload.json index ec34da35fa..7667299fd2 100644 --- a/.github/workflows/weekly-environment-slack-payload.json +++ b/.github/workflows/weekly-environment-slack-payload.json @@ -13,7 +13,7 @@ "fields": [ { "type": "mrkdwn", - "text": "*Environment:*\n Weekly enviroment" + "text": "*Environment:*\n Weekly environment" }, { "type": "mrkdwn", From e7a61148273a30f0b597908f366ceef702fd44f6 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 13:51:47 +0200 Subject: [PATCH 063/127] updating slack message --- .github/workflows/weekly-enviroment.yml | 3 +++ .github/workflows/weekly-environment-slack-payload.json | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index d79a5b6b9b..8da23c907e 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -96,6 +96,9 @@ jobs: # echo "::set-output name=tf_output::$tf_output" # echo $output1 + - name: Terraform Output + run: terraform output -json > output.json + - name: Send custom JSON data to Slack workflow id: slack uses: slackapi/slack-github-action@v1.23.0 diff --git a/.github/workflows/weekly-environment-slack-payload.json b/.github/workflows/weekly-environment-slack-payload.json index 7667299fd2..1ea5760bf0 100644 --- a/.github/workflows/weekly-environment-slack-payload.json +++ b/.github/workflows/weekly-environment-slack-payload.json @@ -13,15 +13,15 @@ "fields": [ { "type": "mrkdwn", - "text": "*Environment:*\n Weekly environment" + "text": "*Environment:*\n Weekly environment ${{terraform output kibana_url}}" }, { "type": "mrkdwn", - "text": "*Kibana URL:*\n https://example.com" + "text": "*Kibana URL:*\n ${{terraform output kibana_url}}" }, { "type": "mrkdwn", - "text": "*Username:*\n deployuser" + "text": "*Username:*\n ${{ job.status }}" }, { "type": "mrkdwn", From 10d31b14908489c667d392a8d65bb3709d9604db Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 14:02:28 +0200 Subject: [PATCH 064/127] updating slack message --- .github/workflows/weekly-environment-slack-payload.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/weekly-environment-slack-payload.json b/.github/workflows/weekly-environment-slack-payload.json index 1ea5760bf0..52e0d26358 100644 --- a/.github/workflows/weekly-environment-slack-payload.json +++ b/.github/workflows/weekly-environment-slack-payload.json @@ -17,7 +17,7 @@ }, { "type": "mrkdwn", - "text": "*Kibana URL:*\n ${{terraform output kibana_url}}" + "text": "*Kibana URL:*\n ${{PATH}}" }, { "type": "mrkdwn", From 482f335e63808b1cca09a1c23b3ba7a62d52bf73 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 14:05:00 +0200 Subject: [PATCH 065/127] updating slack message --- .github/workflows/weekly-enviroment.yml | 1 + .github/workflows/weekly-environment-slack-payload.json | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 8da23c907e..8254c334a5 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -107,3 +107,4 @@ jobs: env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + Ofir: 'Lapid11' diff --git a/.github/workflows/weekly-environment-slack-payload.json b/.github/workflows/weekly-environment-slack-payload.json index 52e0d26358..9e92bdad35 100644 --- a/.github/workflows/weekly-environment-slack-payload.json +++ b/.github/workflows/weekly-environment-slack-payload.json @@ -17,7 +17,7 @@ }, { "type": "mrkdwn", - "text": "*Kibana URL:*\n ${{PATH}}" + "text": "*Kibana URL:*\n ${{Ofir}}" }, { "type": "mrkdwn", From 118c5e866f0157c0aeeaa11451936e183bbf0660 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 14:08:37 +0200 Subject: [PATCH 066/127] updating slack message --- .github/workflows/weekly-enviroment.yml | 37 ++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 8254c334a5..f50e3996b9 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -103,7 +103,42 @@ jobs: id: slack uses: slackapi/slack-github-action@v1.23.0 with: - payload-file-path: ${{ env.SLACK_MESSAGE_PAYLOAD_PATH }} + payload: | + { + "text": "Weekly environment deployed", + "attachments": [ + { + "color": "#36a64f", + "fields": [ + { + "title": "Environment", + "value": "${{ github.event.inputs.environment }}", + "short": true + }, + { + "title": "Benchmark", + "value": "${{ github.event.inputs.benchmark }}", + "short": true + }, + { + "title": "Alerts", + "value": "${{ github.event.inputs.alerts }}", + "short": true + }, + { + "title": "Log level", + "value": "${{ github.event.inputs.logLevel }}", + "short": true + }, + { + "title": "Kibana URL", + "value": "${{ steps.terraform_output.outputs.tf_output }}", + "short": true + } + ] + } + ] + } env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK From 515d87eebaf180d8d479669fcd2c8369657430b7 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 14:11:19 +0200 Subject: [PATCH 067/127] updating slack message --- .github/workflows/weekly-enviroment.yml | 37 +----------------- .../weekly-environment-slack-payload.json | 38 ++++++++++--------- 2 files changed, 21 insertions(+), 54 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index f50e3996b9..8254c334a5 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -103,42 +103,7 @@ jobs: id: slack uses: slackapi/slack-github-action@v1.23.0 with: - payload: | - { - "text": "Weekly environment deployed", - "attachments": [ - { - "color": "#36a64f", - "fields": [ - { - "title": "Environment", - "value": "${{ github.event.inputs.environment }}", - "short": true - }, - { - "title": "Benchmark", - "value": "${{ github.event.inputs.benchmark }}", - "short": true - }, - { - "title": "Alerts", - "value": "${{ github.event.inputs.alerts }}", - "short": true - }, - { - "title": "Log level", - "value": "${{ github.event.inputs.logLevel }}", - "short": true - }, - { - "title": "Kibana URL", - "value": "${{ steps.terraform_output.outputs.tf_output }}", - "short": true - } - ] - } - ] - } + payload-file-path: ${{ env.SLACK_MESSAGE_PAYLOAD_PATH }} env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK diff --git a/.github/workflows/weekly-environment-slack-payload.json b/.github/workflows/weekly-environment-slack-payload.json index 9e92bdad35..3d6187ab3c 100644 --- a/.github/workflows/weekly-environment-slack-payload.json +++ b/.github/workflows/weekly-environment-slack-payload.json @@ -1,31 +1,33 @@ { - "text": "", - "blocks": [ + "text": "Weekly environment deployed", + "attachments": [ { - "type": "section", - "text": { - "type": "mrkdwn", - "text": ":rocket: *A new Deployment was Created!* :rocket:" - } - }, - { - "type": "section", + "color": "#36a64f", "fields": [ { - "type": "mrkdwn", - "text": "*Environment:*\n Weekly environment ${{terraform output kibana_url}}" + "title": "Environment", + "value": "${{ github.event.inputs.environment }}", + "short": true + }, + { + "title": "Benchmark", + "value": "${{ github.event.inputs.benchmark }}", + "short": true }, { - "type": "mrkdwn", - "text": "*Kibana URL:*\n ${{Ofir}}" + "title": "Alerts", + "value": "${{ github.event.inputs.alerts }}", + "short": true }, { - "type": "mrkdwn", - "text": "*Username:*\n ${{ job.status }}" + "title": "Log level", + "value": "${{ github.event.inputs.logLevel }}", + "short": true }, { - "type": "mrkdwn", - "text": "*Password:*\n *****" + "title": "Kibana URL", + "value": "${{ steps.terraform_output.outputs.tf_output }}", + "short": true } ] } From 6cb5d67558172f0308c0cc4b7b2a14002e3fa53c Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 14:19:48 +0200 Subject: [PATCH 068/127] updating slack message --- .github/workflows/weekly-enviroment.yml | 50 ++++++++++++++++++- .../weekly-environment-slack-payload.json | 38 +++++++------- 2 files changed, 67 insertions(+), 21 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 8254c334a5..d14f79705f 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -99,11 +99,59 @@ jobs: - name: Terraform Output run: terraform output -json > output.json +# Once https://github.com/slackapi/slack-github-action/issues/84 will be resolved we can push the payload to a different file - name: Send custom JSON data to Slack workflow id: slack uses: slackapi/slack-github-action@v1.23.0 with: - payload-file-path: ${{ env.SLACK_MESSAGE_PAYLOAD_PATH }} + payload: | + { + "text": ":rocket: *A new Deployment was Created!* :rocket:", + "attachments": [ + { + "color": "#36a64f", + "fields": [ + { + "title": "Environment", + "value": "${{ github.event.inputs.environment }}", + "short": true + }, + { + "title": "Benchmark", + "value": "${{ github.event.inputs.benchmark }}", + "short": true + }, + { + "title": "Alerts", + "value": "${{ github.event.inputs.alerts }}", + "short": true + }, + { + "title": "Log level", + "value": "${{ github.event.inputs.logLevel }}", + "short": true + }, + { + "title": "Kibana URL", + "value": "${{ steps.terraform_output.outputs.tf_output }}", + "short": true + } + ] + } + ] + } + { + "text": "GitHub Action build result: ${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head_commit.url }}", + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "GitHub Action build result: ${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head_commit.url }}" + } + } + ] + } env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK diff --git a/.github/workflows/weekly-environment-slack-payload.json b/.github/workflows/weekly-environment-slack-payload.json index 3d6187ab3c..9e92bdad35 100644 --- a/.github/workflows/weekly-environment-slack-payload.json +++ b/.github/workflows/weekly-environment-slack-payload.json @@ -1,33 +1,31 @@ { - "text": "Weekly environment deployed", - "attachments": [ + "text": "", + "blocks": [ { - "color": "#36a64f", + "type": "section", + "text": { + "type": "mrkdwn", + "text": ":rocket: *A new Deployment was Created!* :rocket:" + } + }, + { + "type": "section", "fields": [ { - "title": "Environment", - "value": "${{ github.event.inputs.environment }}", - "short": true - }, - { - "title": "Benchmark", - "value": "${{ github.event.inputs.benchmark }}", - "short": true + "type": "mrkdwn", + "text": "*Environment:*\n Weekly environment ${{terraform output kibana_url}}" }, { - "title": "Alerts", - "value": "${{ github.event.inputs.alerts }}", - "short": true + "type": "mrkdwn", + "text": "*Kibana URL:*\n ${{Ofir}}" }, { - "title": "Log level", - "value": "${{ github.event.inputs.logLevel }}", - "short": true + "type": "mrkdwn", + "text": "*Username:*\n ${{ job.status }}" }, { - "title": "Kibana URL", - "value": "${{ steps.terraform_output.outputs.tf_output }}", - "short": true + "type": "mrkdwn", + "text": "*Password:*\n *****" } ] } From 55794c00103532819f7783132d7b4a180ea1eecb Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 14:24:50 +0200 Subject: [PATCH 069/127] updating slack message --- .github/workflows/weekly-enviroment.yml | 86 +++++++++---------- .../weekly-environment-slack-payload.json | 47 ++++++---- 2 files changed, 71 insertions(+), 62 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index d14f79705f..f6c05f9b0d 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -105,53 +105,51 @@ jobs: uses: slackapi/slack-github-action@v1.23.0 with: payload: | - { - "text": ":rocket: *A new Deployment was Created!* :rocket:", - "attachments": [ - { - "color": "#36a64f", - "fields": [ { - "title": "Environment", - "value": "${{ github.event.inputs.environment }}", - "short": true - }, - { - "title": "Benchmark", - "value": "${{ github.event.inputs.benchmark }}", - "short": true - }, - { - "title": "Alerts", - "value": "${{ github.event.inputs.alerts }}", - "short": true - }, - { - "title": "Log level", - "value": "${{ github.event.inputs.logLevel }}", - "short": true - }, - { - "title": "Kibana URL", - "value": "${{ steps.terraform_output.outputs.tf_output }}", - "short": true - } - ] - } - ] - } - { - "text": "GitHub Action build result: ${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head_commit.url }}", - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "GitHub Action build result: ${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head_commit.url }}" + "text": ":rocket: *A new Deployment was Created!* :rocket:", + "attachments": [ + { + "color": "#36a64f", + "fields": [ + { + "title": "Environment", + "value": "${{ github.event.inputs.environment }}", + "short": true + }, + { + "title": "Benchmark", + "value": "${{ github.event.inputs.benchmark }}", + "short": true + }, + { + "title": "Alerts", + "value": "${{ github.event.inputs.alerts }}", + "short": true + }, + { + "title": "Log level", + "value": "${{ github.event.inputs.logLevel }}", + "short": true + }, + { + "title": "Kibana URL", + "value": "${{ steps.terraform_output.outputs.tf_output }}", + "short": true } + ] + } + ], + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "GitHub Action build result: ${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head_commit.url }}" } - ] - } + } + ] + } + env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK diff --git a/.github/workflows/weekly-environment-slack-payload.json b/.github/workflows/weekly-environment-slack-payload.json index 9e92bdad35..6e6a6e0757 100644 --- a/.github/workflows/weekly-environment-slack-payload.json +++ b/.github/workflows/weekly-environment-slack-payload.json @@ -1,33 +1,44 @@ { - "text": "", - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": ":rocket: *A new Deployment was Created!* :rocket:" - } - }, + "text": ":rocket: *A new Deployment was Created!* :rocket:", + "attachments": [ { - "type": "section", + "color": "#36a64f", "fields": [ { - "type": "mrkdwn", - "text": "*Environment:*\n Weekly environment ${{terraform output kibana_url}}" + "title": "Environment", + "value": "${{ github.event.inputs.environment }}", + "short": true + }, + { + "title": "Benchmark", + "value": "${{ github.event.inputs.benchmark }}", + "short": true }, { - "type": "mrkdwn", - "text": "*Kibana URL:*\n ${{Ofir}}" + "title": "Alerts", + "value": "${{ github.event.inputs.alerts }}", + "short": true }, { - "type": "mrkdwn", - "text": "*Username:*\n ${{ job.status }}" + "title": "Log level", + "value": "${{ github.event.inputs.logLevel }}", + "short": true }, { - "type": "mrkdwn", - "text": "*Password:*\n *****" + "title": "Kibana URL", + "value": "${{ steps.terraform_output.outputs.tf_output }}", + "short": true } ] } + ], + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "GitHub Action build result: ${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head_commit.url }}" + } + } ] } From 3648f603923bb74740bc8ef1f2662949aef926f7 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 14:27:47 +0200 Subject: [PATCH 070/127] updating slack message --- .github/workflows/weekly-enviroment.yml | 76 +++++++++++-------------- 1 file changed, 33 insertions(+), 43 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index f6c05f9b0d..106cc02e00 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -105,51 +105,41 @@ jobs: uses: slackapi/slack-github-action@v1.23.0 with: payload: | + { + "text": "Weekly environment deployed", + "attachments": [ + { + "color": "#36a64f", + "fields": [ { - "text": ":rocket: *A new Deployment was Created!* :rocket:", - "attachments": [ - { - "color": "#36a64f", - "fields": [ - { - "title": "Environment", - "value": "${{ github.event.inputs.environment }}", - "short": true - }, - { - "title": "Benchmark", - "value": "${{ github.event.inputs.benchmark }}", - "short": true - }, - { - "title": "Alerts", - "value": "${{ github.event.inputs.alerts }}", - "short": true - }, - { - "title": "Log level", - "value": "${{ github.event.inputs.logLevel }}", - "short": true - }, - { - "title": "Kibana URL", - "value": "${{ steps.terraform_output.outputs.tf_output }}", - "short": true - } - ] - } - ], - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "GitHub Action build result: ${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head_commit.url }}" + "title": "Environment", + "value": "${{ github.event.inputs.environment }}", + "short": true + }, + { + "title": "Benchmark", + "value": "${{ github.event.inputs.benchmark }}", + "short": true + }, + { + "title": "Alerts", + "value": "${{ github.event.inputs.alerts }}", + "short": true + }, + { + "title": "Log level", + "value": "${{ github.event.inputs.logLevel }}", + "short": true + }, + { + "title": "Kibana URL", + "value": "${{ steps.terraform_output.outputs.tf_output }}", + "short": true + } + ] } - } - ] - } - + ] + } env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK From 6a704300ceff0e1a17adb0f962aa0251bb7fb418 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 14:38:20 +0200 Subject: [PATCH 071/127] updating slack message --- .github/workflows/weekly-enviroment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 106cc02e00..89b50e3415 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -106,7 +106,7 @@ jobs: with: payload: | { - "text": "Weekly environment deployed", + "text": "A new deployment job has been triggered", "attachments": [ { "color": "#36a64f", @@ -133,7 +133,7 @@ jobs: }, { "title": "Kibana URL", - "value": "${{ steps.terraform_output.outputs.tf_output }}", + "value": "${{ terraform output }}", "short": true } ] From 01965e9c93c8679a4aff1c35394f02e4f41c92bd Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 14:41:24 +0200 Subject: [PATCH 072/127] updating slack message --- .github/workflows/weekly-enviroment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 89b50e3415..1239d15790 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -133,7 +133,7 @@ jobs: }, { "title": "Kibana URL", - "value": "${{ terraform output }}", + "value": "${{ jq -r '.kibana_url' output.json }}", "short": true } ] From e9e55388f5691ef2473e62f7c8a47d8b23913f55 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 14:42:50 +0200 Subject: [PATCH 073/127] updating slack message --- .github/workflows/weekly-enviroment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 1239d15790..7902f4475e 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -133,7 +133,7 @@ jobs: }, { "title": "Kibana URL", - "value": "${{ jq -r '.kibana_url' output.json }}", + "value": "${ jq -r '.kibana_url' output.json }", "short": true } ] From 24fa4f5036231b8c9592ee6555f469ddc1f24790 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 14:44:41 +0200 Subject: [PATCH 074/127] updating slack message --- .github/workflows/weekly-enviroment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 7902f4475e..5027247bc7 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -133,7 +133,7 @@ jobs: }, { "title": "Kibana URL", - "value": "${ jq -r '.kibana_url' output.json }", + "value": "$( jq -r '.kibana_url' output.json )", "short": true } ] From 1e249900f06b9f6f0764840ba0cbc09ce5e61a08 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 15:00:19 +0200 Subject: [PATCH 075/127] updating slack message --- .github/workflows/weekly-enviroment.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 5027247bc7..54925aecac 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -99,6 +99,13 @@ jobs: - name: Terraform Output run: terraform output -json > output.json + - name: Get Kibana URL + id: kibana_url + run: | + kibana_url=$(jq -r '.kibana_url.value' output.json) + echo "::set-output name=kibana_url::$kibana_url" + echo $kibana_url + # Once https://github.com/slackapi/slack-github-action/issues/84 will be resolved we can push the payload to a different file - name: Send custom JSON data to Slack workflow id: slack @@ -133,7 +140,7 @@ jobs: }, { "title": "Kibana URL", - "value": "$( jq -r '.kibana_url' output.json )", + "value": "${{ kibana_url }}", "short": true } ] From 4aac1f96298bb9dce19b0f6c0ceb5cbe65f49438 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 15:05:52 +0200 Subject: [PATCH 076/127] updating slack message --- .github/workflows/weekly-enviroment.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 54925aecac..46d9d3841b 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -102,9 +102,7 @@ jobs: - name: Get Kibana URL id: kibana_url run: | - kibana_url=$(jq -r '.kibana_url.value' output.json) - echo "::set-output name=kibana_url::$kibana_url" - echo $kibana_url + echo "KIBANA_URL=$(jq -r '.kibana_url' output.json)" >> $GITHUB_ENV # Once https://github.com/slackapi/slack-github-action/issues/84 will be resolved we can push the payload to a different file - name: Send custom JSON data to Slack workflow @@ -140,7 +138,7 @@ jobs: }, { "title": "Kibana URL", - "value": "${{ kibana_url }}", + "value": "${{ env.KIBANA_URL }}", "short": true } ] From b42aea7e450b3e178b6b1e5c8fb90a2245cec1e4 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 15:36:42 +0200 Subject: [PATCH 077/127] updating slack message --- .github/workflows/weekly-enviroment.yml | 15 ++++- deploy/cloud/output.json | 77 +++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 deploy/cloud/output.json diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 46d9d3841b..8cc80934ae 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -102,7 +102,9 @@ jobs: - name: Get Kibana URL id: kibana_url run: | - echo "KIBANA_URL=$(jq -r '.kibana_url' output.json)" >> $GITHUB_ENV + echo "KIBANA_URL=$(jq -r '.kibana_url.value' output.json)" >> $GITHUB_ENV + echo "OFIR_VAR=LAPID" >> $GITHUB_ENV + # Once https://github.com/slackapi/slack-github-action/issues/84 will be resolved we can push the payload to a different file - name: Send custom JSON data to Slack workflow @@ -140,7 +142,16 @@ jobs: "title": "Kibana URL", "value": "${{ env.KIBANA_URL }}", "short": true - } + }, + { + "title": "Region", + "value": "${{ env.AWS_REGION }}", + "short": true + }, + { + title: "Ofir variable", + value: "${{ env.OFIR_VAR }}", + "short": true ] } ] diff --git a/deploy/cloud/output.json b/deploy/cloud/output.json new file mode 100644 index 0000000000..864dc5123f --- /dev/null +++ b/deploy/cloud/output.json @@ -0,0 +1,77 @@ +{ + "admin_console_url": { + "sensitive": false, + "type": "string", + "value": "https://admin.found.no/deployments/4d3712fc8797a9b955c7b1e6b38ae167/integrations_server" + }, + "agent_policy_id": { + "sensitive": false, + "type": "string", + "value": "146e8810-86c9-11ed-9237-0d7253b6dee7" + }, + "eks_cluster_endpoint": { + "sensitive": false, + "type": "string", + "value": "https://BD429AB6593519FFA2E4F7E65E5957B7.gr7.eu-west-1.eks.amazonaws.com" + }, + "eks_cluster_id": { + "sensitive": false, + "type": "string", + "value": "cloudbeat-tf-Gfp" + }, + "eks_cluster_name": { + "sensitive": false, + "type": "string", + "value": "cloudbeat-tf-Gfp" + }, + "eks_cluster_security_group_id": { + "sensitive": false, + "type": "string", + "value": "sg-031ac0d1f1d2e00e2" + }, + "eks_region": { + "sensitive": false, + "type": "string", + "value": "eu-west-1" + }, + "elasticsearch_password": { + "sensitive": true, + "type": "string", + "value": "F0s0zcmPV2a9l9CM7aTHmYFF" + }, + "elasticsearch_url": { + "sensitive": false, + "type": "string", + "value": "https://13613f4ad86b4106923a8ca48dd62154.us-west2.gcp.elastic-cloud.com:443" + }, + "elasticsearch_username": { + "sensitive": true, + "type": "string", + "value": "elastic" + }, + "enrollment_token": { + "sensitive": false, + "type": "string", + "value": "Vi1GNFdZVUJta253Mjk4LTM4ME86Vjg1eGMxQWlUSENuYjh0X1lBYV9XUQ==" + }, + "fleet_url": { + "sensitive": false, + "type": "string", + "value": "https://4d3712fc8797a9b955c7b1e6b38ae167.fleet.us-west2.gcp.elastic-cloud.com:443" + }, + "kibana_url": { + "sensitive": false, + "type": "string", + "value": "https://52bce8740a3047ce98acbdd32dfd94b6.us-west2.gcp.elastic-cloud.com:9243" + }, + "role_arn": { + "sensitive": false, + "type": "string", + "value": "arn:aws:iam::704479110758:role/cloudbeat-tf-Gfp" + }, + "yaml": { + "sensitive": false, + "type": "string", + "value": "---\n# For more information refer to https://www.elastic.co/guide/en/fleet/current/running-on-kubernetes-managed-by-fleet.html\napiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n name: elastic-agent\n namespace: kube-system\n labels:\n app: elastic-agent\nspec:\n selector:\n matchLabels:\n app: elastic-agent\n template:\n metadata:\n labels:\n app: elastic-agent\n spec:\n # Tolerations are needed to run Elastic Agent on Kubernetes control-plane nodes.\n # Agents running on control-plane nodes collect metrics from the control plane components (scheduler, controller manager) of Kubernetes\n tolerations:\n - key: node-role.kubernetes.io/control-plane\n effect: NoSchedule\n - key: node-role.kubernetes.io/master\n effect: NoSchedule\n serviceAccountName: elastic-agent\n hostNetwork: true\n # 'hostPID: true' enables the Elastic Security integration to observe all process exec events on the host.\n # Sharing the host process ID namespace gives visibility of all processes running on the same host.\n hostPID: true\n dnsPolicy: ClusterFirstWithHostNet\n containers:\n - name: elastic-agent\n image: docker.elastic.co/beats/elastic-agent:8.5.3\n env:\n # Set to 1 for enrollment into Fleet server. If not set, Elastic Agent is run in standalone mode\n - name: FLEET_ENROLL\n value: \"1\"\n # Set to true to communicate with Fleet with either insecure HTTP or unverified HTTPS\n - name: FLEET_INSECURE\n value: \"true\"\n # Fleet Server URL to enroll the Elastic Agent into\n # FLEET_URL can be found in Kibana, go to Management \u003e Fleet \u003e Settings\n - name: FLEET_URL\n value: \"https://4d3712fc8797a9b955c7b1e6b38ae167.fleet.us-west2.gcp.elastic-cloud.com:443\"\n # Elasticsearch API key used to enroll Elastic Agents in Fleet (https://www.elastic.co/guide/en/fleet/current/fleet-enrollment-tokens.html#fleet-enrollment-tokens)\n # If FLEET_ENROLLMENT_TOKEN is empty then KIBANA_HOST, KIBANA_FLEET_USERNAME, KIBANA_FLEET_PASSWORD are needed\n - name: FLEET_ENROLLMENT_TOKEN\n value: \"Vi1GNFdZVUJta253Mjk4LTM4ME86Vjg1eGMxQWlUSENuYjh0X1lBYV9XUQ==\"\n - name: KIBANA_HOST\n value: \"http://kibana:5601\"\n # The basic authentication username used to connect to Kibana and retrieve a service_token to enable Fleet\n - name: KIBANA_FLEET_USERNAME\n value: \"elastic\"\n # The basic authentication password used to connect to Kibana and retrieve a service_token to enable Fleet\n - name: KIBANA_FLEET_PASSWORD\n value: \"changeme\"\n - name: NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: spec.nodeName\n - name: POD_NAME\n valueFrom:\n fieldRef:\n fieldPath: metadata.name\n securityContext:\n runAsUser: 0\n resources:\n limits:\n memory: 500Mi\n requests:\n cpu: 100m\n memory: 200Mi\n volumeMounts:\n - name: proc\n mountPath: /hostfs/proc\n readOnly: true\n - name: cgroup\n mountPath: /hostfs/sys/fs/cgroup\n readOnly: true\n - name: varlibdockercontainers\n mountPath: /var/lib/docker/containers\n readOnly: true\n - name: varlog\n mountPath: /var/log\n readOnly: true\n - name: etc-kubernetes\n mountPath: /hostfs/etc/kubernetes\n readOnly: true\n - name: var-lib\n mountPath: /hostfs/var/lib\n readOnly: true\n - name: passwd\n mountPath: /hostfs/etc/passwd\n readOnly: true\n - name: group\n mountPath: /hostfs/etc/group\n readOnly: true\n - name: etcsysmd\n mountPath: /hostfs/etc/systemd\n readOnly: true\n - name: etc-mid\n mountPath: /etc/machine-id\n readOnly: true\n volumes:\n - name: proc\n hostPath:\n path: /proc\n - name: cgroup\n hostPath:\n path: /sys/fs/cgroup\n - name: varlibdockercontainers\n hostPath:\n path: /var/lib/docker/containers\n - name: varlog\n hostPath:\n path: /var/log\n # Needed for cloudbeat\n - name: etc-kubernetes\n hostPath:\n path: /etc/kubernetes\n # Needed for cloudbeat\n - name: var-lib\n hostPath:\n path: /var/lib\n # Needed for cloudbeat\n - name: passwd\n hostPath:\n path: /etc/passwd\n # Needed for cloudbeat\n - name: group\n hostPath:\n path: /etc/group\n # Needed for cloudbeat\n - name: etcsysmd\n hostPath:\n path: /etc/systemd\n # Mount /etc/machine-id from the host to determine host ID\n # Needed for Elastic Security integration\n - name: etc-mid\n hostPath:\n path: /etc/machine-id\n type: File\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: elastic-agent\nsubjects:\n - kind: ServiceAccount\n name: elastic-agent\n namespace: kube-system\nroleRef:\n kind: ClusterRole\n name: elastic-agent\n apiGroup: rbac.authorization.k8s.io\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: RoleBinding\nmetadata:\n namespace: kube-system\n name: elastic-agent\nsubjects:\n - kind: ServiceAccount\n name: elastic-agent\n namespace: kube-system\nroleRef:\n kind: Role\n name: elastic-agent\n apiGroup: rbac.authorization.k8s.io\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: RoleBinding\nmetadata:\n name: elastic-agent-kubeadm-config\n namespace: kube-system\nsubjects:\n - kind: ServiceAccount\n name: elastic-agent\n namespace: kube-system\nroleRef:\n kind: Role\n name: elastic-agent-kubeadm-config\n apiGroup: rbac.authorization.k8s.io\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n name: elastic-agent\n labels:\n k8s-app: elastic-agent\nrules:\n - apiGroups: [\"\"]\n resources:\n - nodes\n - namespaces\n - events\n - pods\n - services\n - configmaps\n # Needed for cloudbeat\n - serviceaccounts\n - persistentvolumes\n - persistentvolumeclaims\n verbs: [\"get\", \"list\", \"watch\"]\n # Enable this rule only if planing to use kubernetes_secrets provider\n #- apiGroups: [\"\"]\n # resources:\n # - secrets\n # verbs: [\"get\"]\n - apiGroups: [\"extensions\"]\n resources:\n - replicasets\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"apps\"]\n resources:\n - statefulsets\n - deployments\n - replicasets\n - daemonsets\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups:\n - \"\"\n resources:\n - nodes/stats\n verbs:\n - get\n - apiGroups: [ \"batch\" ]\n resources:\n - jobs\n - cronjobs\n verbs: [ \"get\", \"list\", \"watch\" ]\n # Needed for apiserver\n - nonResourceURLs:\n - \"/metrics\"\n verbs:\n - get\n # Needed for cloudbeat\n - apiGroups: [\"rbac.authorization.k8s.io\"]\n resources:\n - clusterrolebindings\n - clusterroles\n - rolebindings\n - roles\n verbs: [\"get\", \"list\", \"watch\"]\n # Needed for cloudbeat\n - apiGroups: [\"policy\"]\n resources:\n - podsecuritypolicies\n verbs: [\"get\", \"list\", \"watch\"]\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: Role\nmetadata:\n name: elastic-agent\n # Should be the namespace where elastic-agent is running\n namespace: kube-system\n labels:\n k8s-app: elastic-agent\nrules:\n - apiGroups:\n - coordination.k8s.io\n resources:\n - leases\n verbs: [\"get\", \"create\", \"update\"]\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: Role\nmetadata:\n name: elastic-agent-kubeadm-config\n namespace: kube-system\n labels:\n k8s-app: elastic-agent\nrules:\n - apiGroups: [\"\"]\n resources:\n - configmaps\n resourceNames:\n - kubeadm-config\n verbs: [\"get\"]\n---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: elastic-agent\n namespace: kube-system\n labels:\n k8s-app: elastic-agent\n---\n" + } +} From e28b21a0a2a47d3037f1f30123f5688bdb49fa37 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 15:38:07 +0200 Subject: [PATCH 078/127] updating slack message --- .github/workflows/weekly-enviroment.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 8cc80934ae..401b317b87 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -72,11 +72,11 @@ jobs: env: TF_VAR_ec_api_key: ${{ secrets.TF_VAR_EC_API_KEY }} -# - name: Terraform Apply -# id: apply -# run: terraform apply -auto-approve -target=module.ec_deployment -no-color -# env: -# TF_VAR_ec_api_key: ${{ secrets.TF_VAR_EC_API_KEY }} + - name: Terraform Apply + id: apply + run: terraform apply -auto-approve -target=module.ec_deployment -no-color + env: + TF_VAR_ec_api_key: ${{ secrets.TF_VAR_EC_API_KEY }} # # - name: Call terraform output url parameter # run: | From f8b4b6e65d6f47e1bb9187173d1b3340a58acf45 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 15:39:17 +0200 Subject: [PATCH 079/127] updating slack message --- .github/workflows/weekly-enviroment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 401b317b87..02819e92ab 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -152,6 +152,7 @@ jobs: title: "Ofir variable", value: "${{ env.OFIR_VAR }}", "short": true + } ] } ] From 5a832063677ec4a047fcd2e1de834fab321e2d3c Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 17:24:22 +0200 Subject: [PATCH 080/127] updating slack message --- .github/workflows/weekly-enviroment.yml | 12 ------------ .../workflows/weekly-environment-slack-payload.json | 13 ++----------- 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 02819e92ab..0b6a5204e8 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -103,7 +103,6 @@ jobs: id: kibana_url run: | echo "KIBANA_URL=$(jq -r '.kibana_url.value' output.json)" >> $GITHUB_ENV - echo "OFIR_VAR=LAPID" >> $GITHUB_ENV # Once https://github.com/slackapi/slack-github-action/issues/84 will be resolved we can push the payload to a different file @@ -142,16 +141,6 @@ jobs: "title": "Kibana URL", "value": "${{ env.KIBANA_URL }}", "short": true - }, - { - "title": "Region", - "value": "${{ env.AWS_REGION }}", - "short": true - }, - { - title: "Ofir variable", - value: "${{ env.OFIR_VAR }}", - "short": true } ] } @@ -160,4 +149,3 @@ jobs: env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK - Ofir: 'Lapid11' diff --git a/.github/workflows/weekly-environment-slack-payload.json b/.github/workflows/weekly-environment-slack-payload.json index 6e6a6e0757..e9bc74812f 100644 --- a/.github/workflows/weekly-environment-slack-payload.json +++ b/.github/workflows/weekly-environment-slack-payload.json @@ -1,5 +1,5 @@ { - "text": ":rocket: *A new Deployment was Created!* :rocket:", + "text": "A new deployment job has been triggered", "attachments": [ { "color": "#36a64f", @@ -26,19 +26,10 @@ }, { "title": "Kibana URL", - "value": "${{ steps.terraform_output.outputs.tf_output }}", + "value": "${{ env.KIBANA_URL }}", "short": true } ] } - ], - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "GitHub Action build result: ${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head_commit.url }}" - } - } ] } From f138853cc0d6898bce2b5dd4542c2a5e9720a5aa Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 17:47:31 +0200 Subject: [PATCH 081/127] updating slack message --- .github/workflows/weekly-enviroment.yml | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 0b6a5204e8..354ad28bcc 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -72,11 +72,11 @@ jobs: env: TF_VAR_ec_api_key: ${{ secrets.TF_VAR_EC_API_KEY }} - - name: Terraform Apply - id: apply - run: terraform apply -auto-approve -target=module.ec_deployment -no-color - env: - TF_VAR_ec_api_key: ${{ secrets.TF_VAR_EC_API_KEY }} +# - name: Terraform Apply +# id: apply +# run: terraform apply -auto-approve -target=module.ec_deployment -no-color +# env: +# TF_VAR_ec_api_key: ${{ secrets.TF_VAR_EC_API_KEY }} # # - name: Call terraform output url parameter # run: | @@ -104,6 +104,13 @@ jobs: run: | echo "KIBANA_URL=$(jq -r '.kibana_url.value' output.json)" >> $GITHUB_ENV + - name: Set sensitive output + id: set_sensitive_output + run: | + SWA_DEPLOYMENT_TOKEN="Lapid" + echo "::add-mask::$SWA_DEPLOYMENT_TOKEN" + echo SWA_DEPLOYMENT_TOKEN=$SWA_DEPLOYMENT_TOKEN >> $GITHUB_ENV + # Once https://github.com/slackapi/slack-github-action/issues/84 will be resolved we can push the payload to a different file - name: Send custom JSON data to Slack workflow @@ -141,6 +148,11 @@ jobs: "title": "Kibana URL", "value": "${{ env.KIBANA_URL }}", "short": true + }, + { + "title": "Sensetive output", + "value": "${{ env.SWA_DEPLOYMENT_TOKEN }}", + "short": true } ] } From e8a2b5348aa8d41138b9c912a62c9adc457328db Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 17:56:38 +0200 Subject: [PATCH 082/127] updating slack message --- .github/workflows/weekly-enviroment.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 354ad28bcc..c850c5f9aa 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -99,12 +99,13 @@ jobs: - name: Terraform Output run: terraform output -json > output.json - - name: Get Kibana URL + - name: Set terraform output as env variable id: kibana_url run: | - echo "KIBANA_URL=$(jq -r '.kibana_url.value' output.json)" >> $GITHUB_ENV +# echo "KIBANA_URL=$(jq -r '.kibana_url.value' output.json)" >> $GITHUB_ENV + echo "::add-mask::KIBANA_URL=$(jq -r '.elasticsearch_username.value' output.json)" >> $GITHUB_ENV - - name: Set sensitive output + - name: Set sensitive terraform output as env variable id: set_sensitive_output run: | SWA_DEPLOYMENT_TOKEN="Lapid" From 5a185169d612da5fe52dad64656bcc15b394ed33 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 17:57:23 +0200 Subject: [PATCH 083/127] updating slack message --- .github/workflows/weekly-enviroment.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index c850c5f9aa..49cd724d01 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -102,7 +102,6 @@ jobs: - name: Set terraform output as env variable id: kibana_url run: | -# echo "KIBANA_URL=$(jq -r '.kibana_url.value' output.json)" >> $GITHUB_ENV echo "::add-mask::KIBANA_URL=$(jq -r '.elasticsearch_username.value' output.json)" >> $GITHUB_ENV - name: Set sensitive terraform output as env variable From 580e0e60f0f044eb8bfa1cb79a98b189449022b5 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 18:21:57 +0200 Subject: [PATCH 084/127] updating slack message --- .github/workflows/weekly-enviroment.yml | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 49cd724d01..2cc9d21e6a 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -107,10 +107,12 @@ jobs: - name: Set sensitive terraform output as env variable id: set_sensitive_output run: | - SWA_DEPLOYMENT_TOKEN="Lapid" - echo "::add-mask::$SWA_DEPLOYMENT_TOKEN" - echo SWA_DEPLOYMENT_TOKEN=$SWA_DEPLOYMENT_TOKEN >> $GITHUB_ENV - + ELASTICSEARCH_USERNAME= $(jq -r '.elasticsearch_username.value' output.json) + echo "::add-mask::ELASTICSEARCH_USERNAME" + echo ELASTICSEARCH_USERNAME=$ELASTICSEARCH_USERNAME >> $GITHUB_ENV + ELASTIC_SEARCH_PASSWORD= $(jq -r '.elasticsearch_password.value' output.json) + echo "::add-mask::ELASTIC_SEARCH_PASSWORD" + echo ELASTIC_SEARCH_PASSWORD=$ELASTIC_SEARCH_PASSWORD >> $GITHUB_ENV # Once https://github.com/slackapi/slack-github-action/issues/84 will be resolved we can push the payload to a different file - name: Send custom JSON data to Slack workflow @@ -150,8 +152,13 @@ jobs: "short": true }, { - "title": "Sensetive output", - "value": "${{ env.SWA_DEPLOYMENT_TOKEN }}", + "title": "ElasticSearch username", + "value": "${{ env.ELASTICSEARCH_USERNAME }}", + "short": true + }, + { + "title": "ElasticSearch password", + "value": "${{ env.ELASTICSEARCH_PASSWORD }}", "short": true } ] From 4dbfcd6a63277c5d102299a0f1be7988ddae9771 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 18:27:59 +0200 Subject: [PATCH 085/127] updating slack message --- .github/workflows/weekly-enviroment.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 2cc9d21e6a..93f4a4392f 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -160,6 +160,16 @@ jobs: "title": "ElasticSearch password", "value": "${{ env.ELASTICSEARCH_PASSWORD }}", "short": true + }, + { + "title": "Some property", + "value": "${{ terraform output rgname }}", + "short": true + }, + { + "title": "Some property", + "value": "${{ jq -r '.elasticsearch_username.value' output.json }}", + "short": true } ] } From a1eed7d45df70a9f2d9722c82bb38170c8466f1c Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 18:31:00 +0200 Subject: [PATCH 086/127] updating slack message --- .github/workflows/weekly-enviroment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 93f4a4392f..36667262e1 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -107,7 +107,7 @@ jobs: - name: Set sensitive terraform output as env variable id: set_sensitive_output run: | - ELASTICSEARCH_USERNAME= $(jq -r '.elasticsearch_username.value' output.json) + ELASTICSEARCH_USERNAME=$(jq -r '.elasticsearch_username.value' output.json) echo "::add-mask::ELASTICSEARCH_USERNAME" echo ELASTICSEARCH_USERNAME=$ELASTICSEARCH_USERNAME >> $GITHUB_ENV ELASTIC_SEARCH_PASSWORD= $(jq -r '.elasticsearch_password.value' output.json) From 8b69d4bd64ff77013df4472a609f8bfbae2c7717 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 18:31:47 +0200 Subject: [PATCH 087/127] updating slack message --- .github/workflows/weekly-enviroment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 36667262e1..e7a768115b 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -163,7 +163,7 @@ jobs: }, { "title": "Some property", - "value": "${{ terraform output rgname }}", + "value": "$( terraform output rgname )", "short": true }, { From dd8c1989dd66f3815a58ca9f4df6a6bcaea7088b Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 18:34:36 +0200 Subject: [PATCH 088/127] updating slack message --- .github/workflows/weekly-enviroment.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index e7a768115b..b7bbc85673 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -102,15 +102,15 @@ jobs: - name: Set terraform output as env variable id: kibana_url run: | - echo "::add-mask::KIBANA_URL=$(jq -r '.elasticsearch_username.value' output.json)" >> $GITHUB_ENV + echo "KIBANA_URL=$(jq -r '.elasticsearch_username.value' output.json)" >> $GITHUB_ENV - name: Set sensitive terraform output as env variable id: set_sensitive_output run: | - ELASTICSEARCH_USERNAME=$(jq -r '.elasticsearch_username.value' output.json) + export ELASTICSEARCH_USERNAME=$(jq -r '.elasticsearch_username.value' output.json) echo "::add-mask::ELASTICSEARCH_USERNAME" echo ELASTICSEARCH_USERNAME=$ELASTICSEARCH_USERNAME >> $GITHUB_ENV - ELASTIC_SEARCH_PASSWORD= $(jq -r '.elasticsearch_password.value' output.json) + export ELASTIC_SEARCH_PASSWORD= $(jq -r '.elasticsearch_password.value' output.json) echo "::add-mask::ELASTIC_SEARCH_PASSWORD" echo ELASTIC_SEARCH_PASSWORD=$ELASTIC_SEARCH_PASSWORD >> $GITHUB_ENV From b0786e52c3ba787e6ec6823bfbcf5f20901da8d0 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 18:41:31 +0200 Subject: [PATCH 089/127] updating slack message --- .github/workflows/weekly-enviroment.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index b7bbc85673..d48fcf0310 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -108,11 +108,8 @@ jobs: id: set_sensitive_output run: | export ELASTICSEARCH_USERNAME=$(jq -r '.elasticsearch_username.value' output.json) - echo "::add-mask::ELASTICSEARCH_USERNAME" - echo ELASTICSEARCH_USERNAME=$ELASTICSEARCH_USERNAME >> $GITHUB_ENV - export ELASTIC_SEARCH_PASSWORD= $(jq -r '.elasticsearch_password.value' output.json) - echo "::add-mask::ELASTIC_SEARCH_PASSWORD" - echo ELASTIC_SEARCH_PASSWORD=$ELASTIC_SEARCH_PASSWORD >> $GITHUB_ENV + echo "::add-mask::$ELASTICSEARCH_USERNAME" + echo "ELASTICSEARCH_USERNAME=$ELASTICSEARCH_USERNAME" >> $GITHUB_ENV # Once https://github.com/slackapi/slack-github-action/issues/84 will be resolved we can push the payload to a different file - name: Send custom JSON data to Slack workflow From 8dad967694837ff3d5551981f9e7e1782bbed581 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 18:42:41 +0200 Subject: [PATCH 090/127] updating slack message --- .github/workflows/weekly-enviroment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index d48fcf0310..48faff3553 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -107,7 +107,7 @@ jobs: - name: Set sensitive terraform output as env variable id: set_sensitive_output run: | - export ELASTICSEARCH_USERNAME=$(jq -r '.elasticsearch_username.value' output.json) + export ELASTICSEARCH_USERNAME="ofir" echo "::add-mask::$ELASTICSEARCH_USERNAME" echo "ELASTICSEARCH_USERNAME=$ELASTICSEARCH_USERNAME" >> $GITHUB_ENV From 9a657c6ef2c52d0c97951435f5c94d920814ab77 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 18:45:34 +0200 Subject: [PATCH 091/127] updating slack message --- .github/workflows/weekly-enviroment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 48faff3553..3cafa31b9a 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -165,7 +165,7 @@ jobs: }, { "title": "Some property", - "value": "${{ jq -r '.elasticsearch_username.value' output.json }}", + "value": "$(jq -r '.elasticsearch_username.value' output.json)", "short": true } ] From 9ad7e0d69ed519d6221ce7a17afc9010b148a2fd Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 18:47:59 +0200 Subject: [PATCH 092/127] updating slack message --- .github/workflows/weekly-enviroment.yml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 3cafa31b9a..8ad15eca98 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -107,7 +107,7 @@ jobs: - name: Set sensitive terraform output as env variable id: set_sensitive_output run: | - export ELASTICSEARCH_USERNAME="ofir" + export ELASTICSEARCH_USERNAME=$(jq -r '.elasticsearch_username.value' output.json)" echo "::add-mask::$ELASTICSEARCH_USERNAME" echo "ELASTICSEARCH_USERNAME=$ELASTICSEARCH_USERNAME" >> $GITHUB_ENV @@ -157,16 +157,6 @@ jobs: "title": "ElasticSearch password", "value": "${{ env.ELASTICSEARCH_PASSWORD }}", "short": true - }, - { - "title": "Some property", - "value": "$( terraform output rgname )", - "short": true - }, - { - "title": "Some property", - "value": "$(jq -r '.elasticsearch_username.value' output.json)", - "short": true } ] } From d66da1ba41263460c725c8aeeb8ff501ae2da2e6 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 18:51:32 +0200 Subject: [PATCH 093/127] updating slack message --- .github/workflows/weekly-enviroment.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 8ad15eca98..68f02237b5 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -102,14 +102,17 @@ jobs: - name: Set terraform output as env variable id: kibana_url run: | - echo "KIBANA_URL=$(jq -r '.elasticsearch_username.value' output.json)" >> $GITHUB_ENV + echo "KIBANA_URL=$(jq -r '.kibana_url.value' output.json)" >> $GITHUB_ENV - name: Set sensitive terraform output as env variable id: set_sensitive_output run: | - export ELASTICSEARCH_USERNAME=$(jq -r '.elasticsearch_username.value' output.json)" + export ELASTICSEARCH_USERNAME=$(jq -r '.elasticsearch_username.value' output.json) echo "::add-mask::$ELASTICSEARCH_USERNAME" echo "ELASTICSEARCH_USERNAME=$ELASTICSEARCH_USERNAME" >> $GITHUB_ENV + export ELASTICSEARCH_PASSWORD=$(jq -r '.elasticsearch_password.value' output.json) + echo "::add-mask::ELASTICSEARCH_PASSWORD" + echo "ELASTICSEARCH_PASSWORD=ELASTICSEARCH_PASSWORD" >> $GITHUB_ENV # Once https://github.com/slackapi/slack-github-action/issues/84 will be resolved we can push the payload to a different file - name: Send custom JSON data to Slack workflow From b0cd8cf540a4af6aa75024ef780db7eb6043ae49 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 18:52:46 +0200 Subject: [PATCH 094/127] updating slack message --- .github/workflows/weekly-enviroment.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 68f02237b5..b30374dd0d 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -72,11 +72,11 @@ jobs: env: TF_VAR_ec_api_key: ${{ secrets.TF_VAR_EC_API_KEY }} -# - name: Terraform Apply -# id: apply -# run: terraform apply -auto-approve -target=module.ec_deployment -no-color -# env: -# TF_VAR_ec_api_key: ${{ secrets.TF_VAR_EC_API_KEY }} + - name: Terraform Apply + id: apply + run: terraform apply -auto-approve -target=module.ec_deployment -no-color + env: + TF_VAR_ec_api_key: ${{ secrets.TF_VAR_EC_API_KEY }} # # - name: Call terraform output url parameter # run: | From f38ffb71f22016d1849f5882f673fa97b0a22ec7 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 30 Jan 2023 19:07:34 +0200 Subject: [PATCH 095/127] updating slack message --- .github/workflows/weekly-enviroment.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index b30374dd0d..a692c5f724 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -96,23 +96,20 @@ jobs: # echo "::set-output name=tf_output::$tf_output" # echo $output1 - - name: Terraform Output - run: terraform output -json > output.json - - name: Set terraform output as env variable id: kibana_url run: | - echo "KIBANA_URL=$(jq -r '.kibana_url.value' output.json)" >> $GITHUB_ENV + echo "KIBANA_URL=$(terraform output kibana_url)" >> $GITHUB_ENV - name: Set sensitive terraform output as env variable id: set_sensitive_output run: | - export ELASTICSEARCH_USERNAME=$(jq -r '.elasticsearch_username.value' output.json) + export ELASTICSEARCH_USERNAME=$(terraform output elasticsearch_username) echo "::add-mask::$ELASTICSEARCH_USERNAME" echo "ELASTICSEARCH_USERNAME=$ELASTICSEARCH_USERNAME" >> $GITHUB_ENV - export ELASTICSEARCH_PASSWORD=$(jq -r '.elasticsearch_password.value' output.json) + export ELASTICSEARCH_PASSWORD=$(terraform output elasticsearch_password) echo "::add-mask::ELASTICSEARCH_PASSWORD" - echo "ELASTICSEARCH_PASSWORD=ELASTICSEARCH_PASSWORD" >> $GITHUB_ENV + echo "ELASTICSEARCH_PASSWORD=$ELASTICSEARCH_PASSWORD" >> $GITHUB_ENV # Once https://github.com/slackapi/slack-github-action/issues/84 will be resolved we can push the payload to a different file - name: Send custom JSON data to Slack workflow From 523d894655922ad1dd9fb5e470ba130d7682b7b7 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Tue, 31 Jan 2023 09:47:47 +0200 Subject: [PATCH 096/127] updating slack message --- .github/workflows/weekly-enviroment.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index a692c5f724..1d381659ca 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -145,17 +145,17 @@ jobs: }, { "title": "Kibana URL", - "value": "${{ env.KIBANA_URL }}", + "value": ${{ env.KIBANA_URL }}, "short": true }, { "title": "ElasticSearch username", - "value": "${{ env.ELASTICSEARCH_USERNAME }}", + "value": ${{ env.ELASTICSEARCH_USERNAME }}, "short": true }, { "title": "ElasticSearch password", - "value": "${{ env.ELASTICSEARCH_PASSWORD }}", + "value": ${{ env.ELASTICSEARCH_PASSWORD }}, "short": true } ] From 581296c8ca4d0af8b5c2248d985f7be536cbab2f Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Tue, 31 Jan 2023 14:58:32 +0200 Subject: [PATCH 097/127] adding the EC2 machine --- .github/workflows/weekly-enviroment.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 1d381659ca..e92ccd231f 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -111,6 +111,13 @@ jobs: echo "::add-mask::ELASTICSEARCH_PASSWORD" echo "ELASTICSEARCH_PASSWORD=$ELASTICSEARCH_PASSWORD" >> $GITHUB_ENV + - name: Deploy EC2 instance + id: deploy_ec2_instance + run: | + terraform apply -auto-approve -target=module.aws_ec2_with_agent -no-color + env: + TF_VAR_ec_api_key: ${{ secrets.TF_VAR_EC_API_KEY }} + # Once https://github.com/slackapi/slack-github-action/issues/84 will be resolved we can push the payload to a different file - name: Send custom JSON data to Slack workflow id: slack From 931d3369eabd26fab91de9181482b3924dc8d75f Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Tue, 31 Jan 2023 15:22:35 +0200 Subject: [PATCH 098/127] adding the EC2 machine --- .github/workflows/weekly-enviroment.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index e92ccd231f..5e2977c3d3 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -117,8 +117,11 @@ jobs: terraform apply -auto-approve -target=module.aws_ec2_with_agent -no-color env: TF_VAR_ec_api_key: ${{ secrets.TF_VAR_EC_API_KEY }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} -# Once https://github.com/slackapi/slack-github-action/issues/84 will be resolved we can push the payload to a different file + + # Once https://github.com/slackapi/slack-github-action/issues/84 will be resolved we can push the payload to a different file - name: Send custom JSON data to Slack workflow id: slack uses: slackapi/slack-github-action@v1.23.0 From e27e35a6de5cb9daf5dfffefd4568ca35f5652b8 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Thu, 2 Feb 2023 11:55:21 +0200 Subject: [PATCH 099/127] Adding SSH key to the slack message --- .github/workflows/weekly-enviroment.yml | 28 +++++++++---------------- deploy/cloud/modules/ec2/output.tf | 5 +++++ deploy/cloud/outputs.tf | 6 ++++++ 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 5e2977c3d3..fa81441a54 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -77,24 +77,6 @@ jobs: run: terraform apply -auto-approve -target=module.ec_deployment -no-color env: TF_VAR_ec_api_key: ${{ secrets.TF_VAR_EC_API_KEY }} -# -# - name: Call terraform output url parameter -# run: | -# output=$(terraform output kibana_url) -# echo $output -# -# - name: Call terraform output no parameter -# run: | -# output1=$(terraform output ) -# echo "::set-output name=tf_output::$tf_output" -# echo $output1 -# -# - name: Get terraform output -# id: terraform_output -# run: | -# output1=$(terraform output) -# echo "::set-output name=tf_output::$tf_output" -# echo $output1 - name: Set terraform output as env variable id: kibana_url @@ -107,10 +89,16 @@ jobs: export ELASTICSEARCH_USERNAME=$(terraform output elasticsearch_username) echo "::add-mask::$ELASTICSEARCH_USERNAME" echo "ELASTICSEARCH_USERNAME=$ELASTICSEARCH_USERNAME" >> $GITHUB_ENV + export ELASTICSEARCH_PASSWORD=$(terraform output elasticsearch_password) echo "::add-mask::ELASTICSEARCH_PASSWORD" echo "ELASTICSEARCH_PASSWORD=$ELASTICSEARCH_PASSWORD" >> $GITHUB_ENV + export EC2_PRIVATE_KEY=$(terraform output ec2_private_key) + echo "::add-mask::EC2_PRIVATE_KEY" + echo "EC2_PRIVATE_KEY=$EC2_PRIVATE_KEY" >> $GITHUB_ENV + + - name: Deploy EC2 instance id: deploy_ec2_instance run: | @@ -167,6 +155,10 @@ jobs: "title": "ElasticSearch password", "value": ${{ env.ELASTICSEARCH_PASSWORD }}, "short": true + }, + { + "title": "EC2 private key", + "value": ${{ env.EC2_PRIVATE_KEY }}, } ] } diff --git a/deploy/cloud/modules/ec2/output.tf b/deploy/cloud/modules/ec2/output.tf index b63c0535b2..686f59bd06 100644 --- a/deploy/cloud/modules/ec2/output.tf +++ b/deploy/cloud/modules/ec2/output.tf @@ -8,3 +8,8 @@ output "cloudbeat_ssh_cmd" { value = "ssh -i ${local.cloudbeat_private_key_file} ${local.ec2_username}@${aws_instance.cloudbeat.public_ip}" } +output "ec2_private_key" { + description = "Use this private key to SSH into the ec2 instance" + value = file(local.cloudbeat_private_key_file) + sensitive = true +} diff --git a/deploy/cloud/outputs.tf b/deploy/cloud/outputs.tf index bd0d6f212c..1578331ec0 100644 --- a/deploy/cloud/outputs.tf +++ b/deploy/cloud/outputs.tf @@ -76,4 +76,10 @@ output "role_arn" { output "cloudbeat_ssh_cmd" { value = module.aws_ec2_with_agent.cloudbeat_ssh_cmd + sensitive = true +} + +output "ec2_private_key" { + value = module.aws_ec2_with_agent.ec2_private_key + sensitive = true } From dcef2337753960d6b2af5bc06aaa1baf355a5f76 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Thu, 2 Feb 2023 13:30:36 +0200 Subject: [PATCH 100/127] Adding Environment variable --- .github/workflows/weekly-enviroment.yml | 5 +++++ deploy/cloud/main.tf | 6 ++++++ deploy/cloud/variables.tf | 6 ++++++ 3 files changed, 17 insertions(+) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index fa81441a54..065a81947b 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -36,6 +36,7 @@ env: KUBE_NAMESPACE: kube-system WORKING_DIR: deploy/cloud SLACK_MESSAGE_PAYLOAD_PATH: ".github/workflows/weekly-environment-slack-payload.json" + ENVIRONMENT: "weekly-environment" jobs: terraform: @@ -71,12 +72,15 @@ jobs: run: terraform plan -target=module.ec_deployment -no-color env: TF_VAR_ec_api_key: ${{ secrets.TF_VAR_EC_API_KEY }} + TF_VAR_environment: ${{ env.ENVIRONMENT }} + - name: Terraform Apply id: apply run: terraform apply -auto-approve -target=module.ec_deployment -no-color env: TF_VAR_ec_api_key: ${{ secrets.TF_VAR_EC_API_KEY }} + TF_VAR_environment: ${{ env.ENVIRONMENT }} - name: Set terraform output as env variable id: kibana_url @@ -105,6 +109,7 @@ jobs: terraform apply -auto-approve -target=module.aws_ec2_with_agent -no-color env: TF_VAR_ec_api_key: ${{ secrets.TF_VAR_EC_API_KEY }} + TF_VAR_environment: ${{ env.ENVIRONMENT }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/deploy/cloud/main.tf b/deploy/cloud/main.tf index 55fc1f1e23..f8651e91a9 100644 --- a/deploy/cloud/main.tf +++ b/deploy/cloud/main.tf @@ -34,6 +34,12 @@ module "eks" { # Retrieve EKS cluster information provider "aws" { region = module.eks.region + + default_tags { + tags = { + Environment = var.environment + } + } } data "aws_eks_cluster" "cluster" { diff --git a/deploy/cloud/variables.tf b/deploy/cloud/variables.tf index 570c73ee11..93dd46c39a 100644 --- a/deploy/cloud/variables.tf +++ b/deploy/cloud/variables.tf @@ -69,3 +69,9 @@ variable "eks_region" { type = string } +variable environment { + default = "local environment" + description = "Optional environment name" + type = string +} + From 802a1a12ca5e12163daa6809ac4637f7090bfdab Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Thu, 2 Feb 2023 14:31:50 +0200 Subject: [PATCH 101/127] Adding enviroment tag --- .github/workflows/weekly-enviroment.yml | 2 +- deploy/cloud/main.tf | 8 ++------ deploy/cloud/modules/ec2/main.tf | 1 + deploy/cloud/modules/ec2/variables.tf | 5 +++++ deploy/cloud/variables.tf | 4 ++-- 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 065a81947b..a005c7367f 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -109,7 +109,7 @@ jobs: terraform apply -auto-approve -target=module.aws_ec2_with_agent -no-color env: TF_VAR_ec_api_key: ${{ secrets.TF_VAR_EC_API_KEY }} - TF_VAR_environment: ${{ env.ENVIRONMENT }} + TF_VAR_environment_tag_name: ${{ env.ENVIRONMENT }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/deploy/cloud/main.tf b/deploy/cloud/main.tf index f8651e91a9..23d66b63d7 100644 --- a/deploy/cloud/main.tf +++ b/deploy/cloud/main.tf @@ -34,12 +34,6 @@ module "eks" { # Retrieve EKS cluster information provider "aws" { region = module.eks.region - - default_tags { - tags = { - Environment = var.environment - } - } } data "aws_eks_cluster" "cluster" { @@ -206,6 +200,8 @@ module "aws_ec2_with_agent" { source = "./modules/ec2" providers = { aws : aws } yml = module.api.yaml_vanilla + environment_tag_name = var.environment_tag_name + depends_on = [ module.ec_deployment, module.api, diff --git a/deploy/cloud/modules/ec2/main.tf b/deploy/cloud/modules/ec2/main.tf index 39c44bda57..1becefb4f4 100644 --- a/deploy/cloud/modules/ec2/main.tf +++ b/deploy/cloud/modules/ec2/main.tf @@ -4,6 +4,7 @@ locals { common_tags = { id = "${random_id.id.hex}" provisioner = "terraform" + environment = var.environment_tag_name } } resource "tls_private_key" "cloudbeat_key" { diff --git a/deploy/cloud/modules/ec2/variables.tf b/deploy/cloud/modules/ec2/variables.tf index 345474a9be..8c5745d949 100644 --- a/deploy/cloud/modules/ec2/variables.tf +++ b/deploy/cloud/modules/ec2/variables.tf @@ -15,3 +15,8 @@ variable "yml" { type = string } + +variable "environment_tag_name" { + description = "Environment tag name to be used" + type = string +} diff --git a/deploy/cloud/variables.tf b/deploy/cloud/variables.tf index 93dd46c39a..0519176142 100644 --- a/deploy/cloud/variables.tf +++ b/deploy/cloud/variables.tf @@ -69,9 +69,9 @@ variable "eks_region" { type = string } -variable environment { +variable "environment_tag_name" { default = "local environment" - description = "Optional environment name" + description = "Environment tag name to be used" type = string } From ec09a97019ce86864f91e2cfb4d35ce790885912 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Thu, 2 Feb 2023 15:12:44 +0200 Subject: [PATCH 102/127] updating region for the job --- .github/workflows/weekly-enviroment.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index a005c7367f..0c5993901c 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -32,7 +32,7 @@ env: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} CONTAINER_SUFFIX: ${{ github.run_id }} - AWS_REGION: eu-west-2 + AWS_REGION: eu-north-1 KUBE_NAMESPACE: kube-system WORKING_DIR: deploy/cloud SLACK_MESSAGE_PAYLOAD_PATH: ".github/workflows/weekly-environment-slack-payload.json" @@ -73,6 +73,7 @@ jobs: env: TF_VAR_ec_api_key: ${{ secrets.TF_VAR_EC_API_KEY }} TF_VAR_environment: ${{ env.ENVIRONMENT }} + TF_VAR_eks_region: ${{ env.AWS_REGION }} - name: Terraform Apply @@ -81,6 +82,7 @@ jobs: env: TF_VAR_ec_api_key: ${{ secrets.TF_VAR_EC_API_KEY }} TF_VAR_environment: ${{ env.ENVIRONMENT }} + TF_VAR_eks_region: ${{ env.AWS_REGION }} - name: Set terraform output as env variable id: kibana_url @@ -112,6 +114,7 @@ jobs: TF_VAR_environment_tag_name: ${{ env.ENVIRONMENT }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + TF_VAR_eks_region: ${{ env.AWS_REGION }} # Once https://github.com/slackapi/slack-github-action/issues/84 will be resolved we can push the payload to a different file From a0bd0815850e6dcbbcea0f2c21d42aa434729324 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Thu, 2 Feb 2023 16:59:37 +0200 Subject: [PATCH 103/127] remove output.json --- deploy/cloud/output.json | 77 ---------------------------------------- 1 file changed, 77 deletions(-) delete mode 100644 deploy/cloud/output.json diff --git a/deploy/cloud/output.json b/deploy/cloud/output.json deleted file mode 100644 index 864dc5123f..0000000000 --- a/deploy/cloud/output.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "admin_console_url": { - "sensitive": false, - "type": "string", - "value": "https://admin.found.no/deployments/4d3712fc8797a9b955c7b1e6b38ae167/integrations_server" - }, - "agent_policy_id": { - "sensitive": false, - "type": "string", - "value": "146e8810-86c9-11ed-9237-0d7253b6dee7" - }, - "eks_cluster_endpoint": { - "sensitive": false, - "type": "string", - "value": "https://BD429AB6593519FFA2E4F7E65E5957B7.gr7.eu-west-1.eks.amazonaws.com" - }, - "eks_cluster_id": { - "sensitive": false, - "type": "string", - "value": "cloudbeat-tf-Gfp" - }, - "eks_cluster_name": { - "sensitive": false, - "type": "string", - "value": "cloudbeat-tf-Gfp" - }, - "eks_cluster_security_group_id": { - "sensitive": false, - "type": "string", - "value": "sg-031ac0d1f1d2e00e2" - }, - "eks_region": { - "sensitive": false, - "type": "string", - "value": "eu-west-1" - }, - "elasticsearch_password": { - "sensitive": true, - "type": "string", - "value": "F0s0zcmPV2a9l9CM7aTHmYFF" - }, - "elasticsearch_url": { - "sensitive": false, - "type": "string", - "value": "https://13613f4ad86b4106923a8ca48dd62154.us-west2.gcp.elastic-cloud.com:443" - }, - "elasticsearch_username": { - "sensitive": true, - "type": "string", - "value": "elastic" - }, - "enrollment_token": { - "sensitive": false, - "type": "string", - "value": "Vi1GNFdZVUJta253Mjk4LTM4ME86Vjg1eGMxQWlUSENuYjh0X1lBYV9XUQ==" - }, - "fleet_url": { - "sensitive": false, - "type": "string", - "value": "https://4d3712fc8797a9b955c7b1e6b38ae167.fleet.us-west2.gcp.elastic-cloud.com:443" - }, - "kibana_url": { - "sensitive": false, - "type": "string", - "value": "https://52bce8740a3047ce98acbdd32dfd94b6.us-west2.gcp.elastic-cloud.com:9243" - }, - "role_arn": { - "sensitive": false, - "type": "string", - "value": "arn:aws:iam::704479110758:role/cloudbeat-tf-Gfp" - }, - "yaml": { - "sensitive": false, - "type": "string", - "value": "---\n# For more information refer to https://www.elastic.co/guide/en/fleet/current/running-on-kubernetes-managed-by-fleet.html\napiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n name: elastic-agent\n namespace: kube-system\n labels:\n app: elastic-agent\nspec:\n selector:\n matchLabels:\n app: elastic-agent\n template:\n metadata:\n labels:\n app: elastic-agent\n spec:\n # Tolerations are needed to run Elastic Agent on Kubernetes control-plane nodes.\n # Agents running on control-plane nodes collect metrics from the control plane components (scheduler, controller manager) of Kubernetes\n tolerations:\n - key: node-role.kubernetes.io/control-plane\n effect: NoSchedule\n - key: node-role.kubernetes.io/master\n effect: NoSchedule\n serviceAccountName: elastic-agent\n hostNetwork: true\n # 'hostPID: true' enables the Elastic Security integration to observe all process exec events on the host.\n # Sharing the host process ID namespace gives visibility of all processes running on the same host.\n hostPID: true\n dnsPolicy: ClusterFirstWithHostNet\n containers:\n - name: elastic-agent\n image: docker.elastic.co/beats/elastic-agent:8.5.3\n env:\n # Set to 1 for enrollment into Fleet server. If not set, Elastic Agent is run in standalone mode\n - name: FLEET_ENROLL\n value: \"1\"\n # Set to true to communicate with Fleet with either insecure HTTP or unverified HTTPS\n - name: FLEET_INSECURE\n value: \"true\"\n # Fleet Server URL to enroll the Elastic Agent into\n # FLEET_URL can be found in Kibana, go to Management \u003e Fleet \u003e Settings\n - name: FLEET_URL\n value: \"https://4d3712fc8797a9b955c7b1e6b38ae167.fleet.us-west2.gcp.elastic-cloud.com:443\"\n # Elasticsearch API key used to enroll Elastic Agents in Fleet (https://www.elastic.co/guide/en/fleet/current/fleet-enrollment-tokens.html#fleet-enrollment-tokens)\n # If FLEET_ENROLLMENT_TOKEN is empty then KIBANA_HOST, KIBANA_FLEET_USERNAME, KIBANA_FLEET_PASSWORD are needed\n - name: FLEET_ENROLLMENT_TOKEN\n value: \"Vi1GNFdZVUJta253Mjk4LTM4ME86Vjg1eGMxQWlUSENuYjh0X1lBYV9XUQ==\"\n - name: KIBANA_HOST\n value: \"http://kibana:5601\"\n # The basic authentication username used to connect to Kibana and retrieve a service_token to enable Fleet\n - name: KIBANA_FLEET_USERNAME\n value: \"elastic\"\n # The basic authentication password used to connect to Kibana and retrieve a service_token to enable Fleet\n - name: KIBANA_FLEET_PASSWORD\n value: \"changeme\"\n - name: NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: spec.nodeName\n - name: POD_NAME\n valueFrom:\n fieldRef:\n fieldPath: metadata.name\n securityContext:\n runAsUser: 0\n resources:\n limits:\n memory: 500Mi\n requests:\n cpu: 100m\n memory: 200Mi\n volumeMounts:\n - name: proc\n mountPath: /hostfs/proc\n readOnly: true\n - name: cgroup\n mountPath: /hostfs/sys/fs/cgroup\n readOnly: true\n - name: varlibdockercontainers\n mountPath: /var/lib/docker/containers\n readOnly: true\n - name: varlog\n mountPath: /var/log\n readOnly: true\n - name: etc-kubernetes\n mountPath: /hostfs/etc/kubernetes\n readOnly: true\n - name: var-lib\n mountPath: /hostfs/var/lib\n readOnly: true\n - name: passwd\n mountPath: /hostfs/etc/passwd\n readOnly: true\n - name: group\n mountPath: /hostfs/etc/group\n readOnly: true\n - name: etcsysmd\n mountPath: /hostfs/etc/systemd\n readOnly: true\n - name: etc-mid\n mountPath: /etc/machine-id\n readOnly: true\n volumes:\n - name: proc\n hostPath:\n path: /proc\n - name: cgroup\n hostPath:\n path: /sys/fs/cgroup\n - name: varlibdockercontainers\n hostPath:\n path: /var/lib/docker/containers\n - name: varlog\n hostPath:\n path: /var/log\n # Needed for cloudbeat\n - name: etc-kubernetes\n hostPath:\n path: /etc/kubernetes\n # Needed for cloudbeat\n - name: var-lib\n hostPath:\n path: /var/lib\n # Needed for cloudbeat\n - name: passwd\n hostPath:\n path: /etc/passwd\n # Needed for cloudbeat\n - name: group\n hostPath:\n path: /etc/group\n # Needed for cloudbeat\n - name: etcsysmd\n hostPath:\n path: /etc/systemd\n # Mount /etc/machine-id from the host to determine host ID\n # Needed for Elastic Security integration\n - name: etc-mid\n hostPath:\n path: /etc/machine-id\n type: File\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: elastic-agent\nsubjects:\n - kind: ServiceAccount\n name: elastic-agent\n namespace: kube-system\nroleRef:\n kind: ClusterRole\n name: elastic-agent\n apiGroup: rbac.authorization.k8s.io\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: RoleBinding\nmetadata:\n namespace: kube-system\n name: elastic-agent\nsubjects:\n - kind: ServiceAccount\n name: elastic-agent\n namespace: kube-system\nroleRef:\n kind: Role\n name: elastic-agent\n apiGroup: rbac.authorization.k8s.io\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: RoleBinding\nmetadata:\n name: elastic-agent-kubeadm-config\n namespace: kube-system\nsubjects:\n - kind: ServiceAccount\n name: elastic-agent\n namespace: kube-system\nroleRef:\n kind: Role\n name: elastic-agent-kubeadm-config\n apiGroup: rbac.authorization.k8s.io\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n name: elastic-agent\n labels:\n k8s-app: elastic-agent\nrules:\n - apiGroups: [\"\"]\n resources:\n - nodes\n - namespaces\n - events\n - pods\n - services\n - configmaps\n # Needed for cloudbeat\n - serviceaccounts\n - persistentvolumes\n - persistentvolumeclaims\n verbs: [\"get\", \"list\", \"watch\"]\n # Enable this rule only if planing to use kubernetes_secrets provider\n #- apiGroups: [\"\"]\n # resources:\n # - secrets\n # verbs: [\"get\"]\n - apiGroups: [\"extensions\"]\n resources:\n - replicasets\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"apps\"]\n resources:\n - statefulsets\n - deployments\n - replicasets\n - daemonsets\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups:\n - \"\"\n resources:\n - nodes/stats\n verbs:\n - get\n - apiGroups: [ \"batch\" ]\n resources:\n - jobs\n - cronjobs\n verbs: [ \"get\", \"list\", \"watch\" ]\n # Needed for apiserver\n - nonResourceURLs:\n - \"/metrics\"\n verbs:\n - get\n # Needed for cloudbeat\n - apiGroups: [\"rbac.authorization.k8s.io\"]\n resources:\n - clusterrolebindings\n - clusterroles\n - rolebindings\n - roles\n verbs: [\"get\", \"list\", \"watch\"]\n # Needed for cloudbeat\n - apiGroups: [\"policy\"]\n resources:\n - podsecuritypolicies\n verbs: [\"get\", \"list\", \"watch\"]\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: Role\nmetadata:\n name: elastic-agent\n # Should be the namespace where elastic-agent is running\n namespace: kube-system\n labels:\n k8s-app: elastic-agent\nrules:\n - apiGroups:\n - coordination.k8s.io\n resources:\n - leases\n verbs: [\"get\", \"create\", \"update\"]\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: Role\nmetadata:\n name: elastic-agent-kubeadm-config\n namespace: kube-system\n labels:\n k8s-app: elastic-agent\nrules:\n - apiGroups: [\"\"]\n resources:\n - configmaps\n resourceNames:\n - kubeadm-config\n verbs: [\"get\"]\n---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: elastic-agent\n namespace: kube-system\n labels:\n k8s-app: elastic-agent\n---\n" - } -} From b8861b55d54e5eeaabc1a3e7c8d2552f52cbfcce Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Thu, 2 Feb 2023 18:00:28 +0200 Subject: [PATCH 104/127] Refactoring --- .github/workflows/weekly-enviroment.yml | 22 +++++------- .../weekly-environment-slack-payload.json | 35 ------------------- 2 files changed, 8 insertions(+), 49 deletions(-) delete mode 100644 .github/workflows/weekly-environment-slack-payload.json diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 0c5993901c..1c7fc69ebc 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -8,24 +8,17 @@ on: type: choice options: - weekly environment - benchmark: - description: 'The benchmark you wish to install' - type: choice - options: - - Kubernetes Vanilla(k8s-vanilla) - alerts: - description: 'Install alerts' - required: false - type: boolean logLevel: description: 'Log level' required: true default: 'warning' type: choice options: - - info - - warning - - debug + - TRACE + - DEBUG + - INFO + - WARN + - ERROR env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} @@ -74,6 +67,7 @@ jobs: TF_VAR_ec_api_key: ${{ secrets.TF_VAR_EC_API_KEY }} TF_VAR_environment: ${{ env.ENVIRONMENT }} TF_VAR_eks_region: ${{ env.AWS_REGION }} + TF_LOG: ${{ github.event.inputs.logLevel }} - name: Terraform Apply @@ -105,10 +99,10 @@ jobs: echo "EC2_PRIVATE_KEY=$EC2_PRIVATE_KEY" >> $GITHUB_ENV - - name: Deploy EC2 instance + - name: Deploy Agents id: deploy_ec2_instance run: | - terraform apply -auto-approve -target=module.aws_ec2_with_agent -no-color + terraform apply -auto-approve -no-color env: TF_VAR_ec_api_key: ${{ secrets.TF_VAR_EC_API_KEY }} TF_VAR_environment_tag_name: ${{ env.ENVIRONMENT }} diff --git a/.github/workflows/weekly-environment-slack-payload.json b/.github/workflows/weekly-environment-slack-payload.json deleted file mode 100644 index e9bc74812f..0000000000 --- a/.github/workflows/weekly-environment-slack-payload.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "text": "A new deployment job has been triggered", - "attachments": [ - { - "color": "#36a64f", - "fields": [ - { - "title": "Environment", - "value": "${{ github.event.inputs.environment }}", - "short": true - }, - { - "title": "Benchmark", - "value": "${{ github.event.inputs.benchmark }}", - "short": true - }, - { - "title": "Alerts", - "value": "${{ github.event.inputs.alerts }}", - "short": true - }, - { - "title": "Log level", - "value": "${{ github.event.inputs.logLevel }}", - "short": true - }, - { - "title": "Kibana URL", - "value": "${{ env.KIBANA_URL }}", - "short": true - } - ] - } - ] -} From 00080676baa94ad2e622f66f77df31418bd8c5c6 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Thu, 2 Feb 2023 18:11:30 +0200 Subject: [PATCH 105/127] Trying to use environment variables in the outer scope of the flow --- .github/workflows/weekly-enviroment.yml | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 1c7fc69ebc..91a36a352c 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -28,8 +28,11 @@ env: AWS_REGION: eu-north-1 KUBE_NAMESPACE: kube-system WORKING_DIR: deploy/cloud - SLACK_MESSAGE_PAYLOAD_PATH: ".github/workflows/weekly-environment-slack-payload.json" ENVIRONMENT: "weekly-environment" + TF_VAR_ec_api_key: ${{ secrets.TF_VAR_EC_API_KEY }} + TF_VAR_environment: ${{ github.event.inputs.logLevel }} + TF_VAR_eks_region: eu-north-1 + TF_LOG: ${{ github.event.inputs.logLevel }} jobs: terraform: @@ -63,20 +66,9 @@ jobs: - name: Terraform Plan id: plan run: terraform plan -target=module.ec_deployment -no-color - env: - TF_VAR_ec_api_key: ${{ secrets.TF_VAR_EC_API_KEY }} - TF_VAR_environment: ${{ env.ENVIRONMENT }} - TF_VAR_eks_region: ${{ env.AWS_REGION }} - TF_LOG: ${{ github.event.inputs.logLevel }} - - - name: Terraform Apply id: apply run: terraform apply -auto-approve -target=module.ec_deployment -no-color - env: - TF_VAR_ec_api_key: ${{ secrets.TF_VAR_EC_API_KEY }} - TF_VAR_environment: ${{ env.ENVIRONMENT }} - TF_VAR_eks_region: ${{ env.AWS_REGION }} - name: Set terraform output as env variable id: kibana_url @@ -103,15 +95,8 @@ jobs: id: deploy_ec2_instance run: | terraform apply -auto-approve -no-color - env: - TF_VAR_ec_api_key: ${{ secrets.TF_VAR_EC_API_KEY }} - TF_VAR_environment_tag_name: ${{ env.ENVIRONMENT }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - TF_VAR_eks_region: ${{ env.AWS_REGION }} - - # Once https://github.com/slackapi/slack-github-action/issues/84 will be resolved we can push the payload to a different file + # Once https://github.com/slackapi/slack-github-action/issues/84 will be resolved we can push the payload to a different file - name: Send custom JSON data to Slack workflow id: slack uses: slackapi/slack-github-action@v1.23.0 From c47917a29b5e1230999fc4c8da1b290bcb9bc64f Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Thu, 2 Feb 2023 22:21:21 +0200 Subject: [PATCH 106/127] Trying to follow the readme --- .github/workflows/weekly-enviroment.yml | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 91a36a352c..397f92f6a8 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -63,12 +63,21 @@ jobs: id: init run: terraform init -no-color - - name: Terraform Plan - id: plan - run: terraform plan -target=module.ec_deployment -no-color - - name: Terraform Apply + - name: Deploy EKS cluster and Elastic Cloud id: apply - run: terraform apply -auto-approve -target=module.ec_deployment -no-color + run: terraform apply --auto-approve -target "module.ec_deployment" -target "null_resource.rules" -target "null_resource.store_local_dashboard" -target "module.eks" + + - name: Create IAM role for EKS + id: kspm + run: terraform apply --auto-approve -target "module.api" -target "module.iam_eks_role" + + - name : Deploy EC2 instance to run Cloudbeat on vanilla cluster + id: ec2 + run: terraform apply --auto-approve -target "module.aws_ec2_with_agent" + + - name: Deploy agent on EKS + id: eks + run: terraform apply --auto-approve - name: Set terraform output as env variable id: kibana_url From 2cb05c1dfa08bfe2ac968397558f5583926fbece Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Fri, 3 Feb 2023 09:45:43 +0200 Subject: [PATCH 107/127] updating ami for the job --- deploy/cloud/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/deploy/cloud/main.tf b/deploy/cloud/main.tf index 23d66b63d7..ca6303f925 100644 --- a/deploy/cloud/main.tf +++ b/deploy/cloud/main.tf @@ -201,6 +201,7 @@ module "aws_ec2_with_agent" { providers = { aws : aws } yml = module.api.yaml_vanilla environment_tag_name = var.environment_tag_name + aws_ami = "ami-03260a5267db32f55" depends_on = [ module.ec_deployment, From 3b6b0f5e4ae416ceb303fedc305b8637ed3f6c85 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Fri, 3 Feb 2023 14:58:06 +0200 Subject: [PATCH 108/127] updating private key for the job --- deploy/cloud/modules/ec2/output.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/cloud/modules/ec2/output.tf b/deploy/cloud/modules/ec2/output.tf index 686f59bd06..5f0a509482 100644 --- a/deploy/cloud/modules/ec2/output.tf +++ b/deploy/cloud/modules/ec2/output.tf @@ -10,6 +10,6 @@ output "cloudbeat_ssh_cmd" { output "ec2_private_key" { description = "Use this private key to SSH into the ec2 instance" - value = file(local.cloudbeat_private_key_file) + value = local.cloudbeat_private_key_file sensitive = true } From 0505d3a379a8dd9bbd64a16461257b49fa5afda4 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Fri, 3 Feb 2023 19:01:51 +0200 Subject: [PATCH 109/127] fixing slack message --- .github/workflows/weekly-enviroment.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 397f92f6a8..f74cb426dd 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -92,11 +92,11 @@ jobs: echo "ELASTICSEARCH_USERNAME=$ELASTICSEARCH_USERNAME" >> $GITHUB_ENV export ELASTICSEARCH_PASSWORD=$(terraform output elasticsearch_password) - echo "::add-mask::ELASTICSEARCH_PASSWORD" + echo "::add-mask::$ELASTICSEARCH_PASSWORD" echo "ELASTICSEARCH_PASSWORD=$ELASTICSEARCH_PASSWORD" >> $GITHUB_ENV export EC2_PRIVATE_KEY=$(terraform output ec2_private_key) - echo "::add-mask::EC2_PRIVATE_KEY" + echo "::add-mask::$EC2_PRIVATE_KEY" echo "EC2_PRIVATE_KEY=$EC2_PRIVATE_KEY" >> $GITHUB_ENV @@ -155,6 +155,7 @@ jobs: { "title": "EC2 private key", "value": ${{ env.EC2_PRIVATE_KEY }}, + "short": true } ] } From bdb8b1fee781b74a9756deaf55432f7975ef1dea Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Fri, 3 Feb 2023 19:09:58 +0200 Subject: [PATCH 110/127] adding private key to message --- deploy/cloud/modules/ec2/output.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/cloud/modules/ec2/output.tf b/deploy/cloud/modules/ec2/output.tf index 5f0a509482..70cf78a426 100644 --- a/deploy/cloud/modules/ec2/output.tf +++ b/deploy/cloud/modules/ec2/output.tf @@ -10,6 +10,6 @@ output "cloudbeat_ssh_cmd" { output "ec2_private_key" { description = "Use this private key to SSH into the ec2 instance" - value = local.cloudbeat_private_key_file + value = tls_private_key.cloudbeat_key.private_key_pem sensitive = true } From 403421cbad27496a6d2f7dcbee1018fcb1afa0f2 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Fri, 3 Feb 2023 19:19:28 +0200 Subject: [PATCH 111/127] adding eks region and eks cluster name to the slack message --- .github/workflows/weekly-enviroment.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index f74cb426dd..40cc91b996 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -83,6 +83,8 @@ jobs: id: kibana_url run: | echo "KIBANA_URL=$(terraform output kibana_url)" >> $GITHUB_ENV + echo "EKS_CLUSTER_NAME=$(terraform output eks_cluster_name)" >> $GITHUB_ENV + echo "EKS_REGION=$(terraform output eks_region)" >> $GITHUB_ENV - name: Set sensitive terraform output as env variable id: set_sensitive_output @@ -99,6 +101,9 @@ jobs: echo "::add-mask::$EC2_PRIVATE_KEY" echo "EC2_PRIVATE_KEY=$EC2_PRIVATE_KEY" >> $GITHUB_ENV + export SSH_CMD=$(terraform output cloudbeat_ssh_cmd) + echo "::add-mask::$SSH_CMD" + echo "SSH_CMD=$SSH_CMD" >> $GITHUB_ENV - name: Deploy Agents id: deploy_ec2_instance @@ -156,6 +161,21 @@ jobs: "title": "EC2 private key", "value": ${{ env.EC2_PRIVATE_KEY }}, "short": true + }, + { + "title": "SSH command", + "value": ${{ env.SSH_CMD }}, + "short": true + }, + { + "title": "EKS cluster name", + "value": ${{ env.EKS_CLUSTER_NAME }}, + "short": true + }, + { + "title": "EKS region", + "value": ${{ env.EKS_REGION }}, + "short": true } ] } From f4771b7f2a45833fea7cbc0bdfb001792229f69a Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Sat, 4 Feb 2023 11:28:16 +0200 Subject: [PATCH 112/127] Upgrade all providers --- deploy/cloud/modules/ec2/terraform.tf | 2 +- deploy/cloud/modules/provision-apps/terraform.tf | 6 +++--- deploy/cloud/modules/provision-eks-cluster/eks-cluster.tf | 2 +- deploy/cloud/terraform.tf | 8 ++++---- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/deploy/cloud/modules/ec2/terraform.tf b/deploy/cloud/modules/ec2/terraform.tf index a61fb73fa6..5985dc93f6 100644 --- a/deploy/cloud/modules/ec2/terraform.tf +++ b/deploy/cloud/modules/ec2/terraform.tf @@ -2,7 +2,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 4.15.0" + version = "~> 4.53.0" } } } diff --git a/deploy/cloud/modules/provision-apps/terraform.tf b/deploy/cloud/modules/provision-apps/terraform.tf index 4eb3879e2b..73a338054f 100644 --- a/deploy/cloud/modules/provision-apps/terraform.tf +++ b/deploy/cloud/modules/provision-apps/terraform.tf @@ -4,12 +4,12 @@ terraform { source = "hashicorp/helm" version = ">=2.8.0" } - + kubernetes = { source = "hashicorp/kubernetes" - version = "~> 2.12.1" + version = "~> 2.17.0" } } required_version = ">= 1.3, <2.0.0" -} \ No newline at end of file +} diff --git a/deploy/cloud/modules/provision-eks-cluster/eks-cluster.tf b/deploy/cloud/modules/provision-eks-cluster/eks-cluster.tf index 817869aea7..94ad1c2db3 100644 --- a/deploy/cloud/modules/provision-eks-cluster/eks-cluster.tf +++ b/deploy/cloud/modules/provision-eks-cluster/eks-cluster.tf @@ -1,6 +1,6 @@ module "eks" { source = "terraform-aws-modules/eks/aws" - version = "18.26.6" + version = "19.6.0" cluster_name = local.cluster_name cluster_version = "1.24" diff --git a/deploy/cloud/terraform.tf b/deploy/cloud/terraform.tf index fbad04921c..ff46b92b73 100644 --- a/deploy/cloud/terraform.tf +++ b/deploy/cloud/terraform.tf @@ -2,17 +2,17 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 4.15.0" + version = "~> 4.53.0" } random = { source = "hashicorp/random" - version = "~> 3.1.0" + version = "~> 3.4.3" } tls = { source = "hashicorp/tls" - version = "~> 3.4.0" + version = "~> 4.0.4" } cloudinit = { @@ -22,7 +22,7 @@ terraform { kubernetes = { source = "hashicorp/kubernetes" - version = "~> 2.12.1" + version = "~> 2.17.0" } ec = { From d50501a58af0c94f76fffd227c4de616da9e0c54 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Sat, 4 Feb 2023 17:27:51 +0200 Subject: [PATCH 113/127] fixing terraform --- deploy/cloud/README.md | 2 +- deploy/cloud/main.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/cloud/README.md b/deploy/cloud/README.md index 41f2dc705c..1f491c72a8 100644 --- a/deploy/cloud/README.md +++ b/deploy/cloud/README.md @@ -14,7 +14,7 @@ Create environment 1. Create an [API token](https://cloud.elastic.co/deployment-features/keys) from your cloud console account. - 1.1 use the token `export TF_VAR_ec_api_key={TOKEN}` + 1.1 use the token `export TF_VAR_ec_api_key={T OKEN}` 2. In case you want to deploy a specific stack version, set the `TF_VAR_stack_version` variable to the desired version. diff --git a/deploy/cloud/main.tf b/deploy/cloud/main.tf index ca6303f925..9b1f91d100 100644 --- a/deploy/cloud/main.tf +++ b/deploy/cloud/main.tf @@ -37,7 +37,7 @@ provider "aws" { } data "aws_eks_cluster" "cluster" { - name = module.eks.cluster_id + name = module.eks.cluster_name } module "iam_eks_role" { From c071a7008bddc3431b988713fb0256a898722dbd Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Sat, 4 Feb 2023 19:08:16 +0200 Subject: [PATCH 114/127] Revert "Upgrade all providers" This reverts commit f4771b7f2a45833fea7cbc0bdfb001792229f69a. --- deploy/cloud/modules/ec2/terraform.tf | 2 +- deploy/cloud/modules/provision-apps/terraform.tf | 6 +++--- deploy/cloud/modules/provision-eks-cluster/eks-cluster.tf | 2 +- deploy/cloud/terraform.tf | 8 ++++---- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/deploy/cloud/modules/ec2/terraform.tf b/deploy/cloud/modules/ec2/terraform.tf index 5985dc93f6..a61fb73fa6 100644 --- a/deploy/cloud/modules/ec2/terraform.tf +++ b/deploy/cloud/modules/ec2/terraform.tf @@ -2,7 +2,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 4.53.0" + version = "~> 4.15.0" } } } diff --git a/deploy/cloud/modules/provision-apps/terraform.tf b/deploy/cloud/modules/provision-apps/terraform.tf index 73a338054f..4eb3879e2b 100644 --- a/deploy/cloud/modules/provision-apps/terraform.tf +++ b/deploy/cloud/modules/provision-apps/terraform.tf @@ -4,12 +4,12 @@ terraform { source = "hashicorp/helm" version = ">=2.8.0" } - + kubernetes = { source = "hashicorp/kubernetes" - version = "~> 2.17.0" + version = "~> 2.12.1" } } required_version = ">= 1.3, <2.0.0" -} +} \ No newline at end of file diff --git a/deploy/cloud/modules/provision-eks-cluster/eks-cluster.tf b/deploy/cloud/modules/provision-eks-cluster/eks-cluster.tf index 94ad1c2db3..817869aea7 100644 --- a/deploy/cloud/modules/provision-eks-cluster/eks-cluster.tf +++ b/deploy/cloud/modules/provision-eks-cluster/eks-cluster.tf @@ -1,6 +1,6 @@ module "eks" { source = "terraform-aws-modules/eks/aws" - version = "19.6.0" + version = "18.26.6" cluster_name = local.cluster_name cluster_version = "1.24" diff --git a/deploy/cloud/terraform.tf b/deploy/cloud/terraform.tf index ff46b92b73..fbad04921c 100644 --- a/deploy/cloud/terraform.tf +++ b/deploy/cloud/terraform.tf @@ -2,17 +2,17 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 4.53.0" + version = "~> 4.15.0" } random = { source = "hashicorp/random" - version = "~> 3.4.3" + version = "~> 3.1.0" } tls = { source = "hashicorp/tls" - version = "~> 4.0.4" + version = "~> 3.4.0" } cloudinit = { @@ -22,7 +22,7 @@ terraform { kubernetes = { source = "hashicorp/kubernetes" - version = "~> 2.17.0" + version = "~> 2.12.1" } ec = { From 01658a0ea6bf4e9cebb3578c96b6e69d929ca644 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Sat, 4 Feb 2023 19:08:48 +0200 Subject: [PATCH 115/127] Revert "fixing terraform" This reverts commit d50501a58af0c94f76fffd227c4de616da9e0c54. --- deploy/cloud/README.md | 2 +- deploy/cloud/main.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/cloud/README.md b/deploy/cloud/README.md index 1f491c72a8..41f2dc705c 100644 --- a/deploy/cloud/README.md +++ b/deploy/cloud/README.md @@ -14,7 +14,7 @@ Create environment 1. Create an [API token](https://cloud.elastic.co/deployment-features/keys) from your cloud console account. - 1.1 use the token `export TF_VAR_ec_api_key={T OKEN}` + 1.1 use the token `export TF_VAR_ec_api_key={TOKEN}` 2. In case you want to deploy a specific stack version, set the `TF_VAR_stack_version` variable to the desired version. diff --git a/deploy/cloud/main.tf b/deploy/cloud/main.tf index 9b1f91d100..ca6303f925 100644 --- a/deploy/cloud/main.tf +++ b/deploy/cloud/main.tf @@ -37,7 +37,7 @@ provider "aws" { } data "aws_eks_cluster" "cluster" { - name = module.eks.cluster_name + name = module.eks.cluster_id } module "iam_eks_role" { From 16911dd3f8266654464054c5f95a9309944d0ac3 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Sun, 5 Feb 2023 11:17:22 +0200 Subject: [PATCH 116/127] adding base64 encoding to ssh key --- .github/workflows/weekly-enviroment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 40cc91b996..3b446d5a1e 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -97,7 +97,7 @@ jobs: echo "::add-mask::$ELASTICSEARCH_PASSWORD" echo "ELASTICSEARCH_PASSWORD=$ELASTICSEARCH_PASSWORD" >> $GITHUB_ENV - export EC2_PRIVATE_KEY=$(terraform output ec2_private_key) + export EC2_PRIVATE_KEY=$(echo $(terraform output cloudbeat_private_key) | base64 -w0) echo "::add-mask::$EC2_PRIVATE_KEY" echo "EC2_PRIVATE_KEY=$EC2_PRIVATE_KEY" >> $GITHUB_ENV From 6fc0c489c300783b3d9f8c4e65c31b5784d28329 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Sun, 5 Feb 2023 13:49:41 +0200 Subject: [PATCH 117/127] trying to fix slack message --- .github/workflows/a.json | 58 +++++++++++++++++++++++++ .github/workflows/weekly-enviroment.yml | 14 +----- 2 files changed, 60 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/a.json diff --git a/.github/workflows/a.json b/.github/workflows/a.json new file mode 100644 index 0000000000..a94069dddc --- /dev/null +++ b/.github/workflows/a.json @@ -0,0 +1,58 @@ +{ + "text": "A new deployment job has been triggered", + "attachments": [ + { + "color": "#36a64f", + "fields": [ + { + "title": "Environment", + "value": "${{ github.event.inputs.environment }}", + "short": true + }, + { + "title": "Benchmark", + "value": "${{ github.event.inputs.benchmark }}", + "short": true + }, + { + "title": "Alerts", + "value": "${{ github.event.inputs.alerts }}", + "short": true + }, + { + "title": "Log level", + "value": "${{ github.event.inputs.logLevel }}", + "short": true + }, + { + "title": "Kibana URL", + "short": true + }, + { + "title": "ElasticSearch username", + "short": true + }, + { + "title": "ElasticSearch password", + "short": true + }, + { + "title": "EC2 private key", + "short": true + }, + { + "title": "SSH command", + "short": true + }, + { + "title": "EKS cluster name", + "short": true + }, + { + "title": "EKS region", + "short": true + } + ] + } + ] +} diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 3b446d5a1e..38239aa6e4 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -127,16 +127,6 @@ jobs: "value": "${{ github.event.inputs.environment }}", "short": true }, - { - "title": "Benchmark", - "value": "${{ github.event.inputs.benchmark }}", - "short": true - }, - { - "title": "Alerts", - "value": "${{ github.event.inputs.alerts }}", - "short": true - }, { "title": "Log level", "value": "${{ github.event.inputs.logLevel }}", @@ -159,12 +149,12 @@ jobs: }, { "title": "EC2 private key", - "value": ${{ env.EC2_PRIVATE_KEY }}, + "value": "${{ env.EC2_PRIVATE_KEY }}", "short": true }, { "title": "SSH command", - "value": ${{ env.SSH_CMD }}, + "value": "${{ env.SSH_CMD }}", "short": true }, { From 9eeb08e73ea45195b1b4579e17910a1d0bdbda37 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Sun, 5 Feb 2023 15:29:37 +0200 Subject: [PATCH 118/127] removing fields from slack message --- .github/workflows/weekly-enviroment.yml | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 38239aa6e4..456b1b3f1e 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -146,26 +146,6 @@ jobs: "title": "ElasticSearch password", "value": ${{ env.ELASTICSEARCH_PASSWORD }}, "short": true - }, - { - "title": "EC2 private key", - "value": "${{ env.EC2_PRIVATE_KEY }}", - "short": true - }, - { - "title": "SSH command", - "value": "${{ env.SSH_CMD }}", - "short": true - }, - { - "title": "EKS cluster name", - "value": ${{ env.EKS_CLUSTER_NAME }}, - "short": true - }, - { - "title": "EKS region", - "value": ${{ env.EKS_REGION }}, - "short": true } ] } From 6b6f6c34c5b4792f92edf85f53df1e386d0b1153 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Sun, 5 Feb 2023 16:50:53 +0200 Subject: [PATCH 119/127] editing fields from slack message --- .github/workflows/weekly-enviroment.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 456b1b3f1e..67370f5052 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -146,6 +146,16 @@ jobs: "title": "ElasticSearch password", "value": ${{ env.ELASTICSEARCH_PASSWORD }}, "short": true + }, + { + "title": "EKS cluster name", + "value": ${{ env.EKS_CLUSTER_NAME }}, + "short": true + }, + { + "title": "EKS region", + "value": ${{ env.EKS_REGION }}, + "short": true } ] } From d7dfe06dd39fd634dc659000e65509f43d40c630 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Sun, 5 Feb 2023 18:17:15 +0200 Subject: [PATCH 120/127] adding the ssh command --- .github/workflows/weekly-enviroment.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 67370f5052..5afcec06ef 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -147,6 +147,11 @@ jobs: "value": ${{ env.ELASTICSEARCH_PASSWORD }}, "short": true }, + { + "title": "SSH command", + "value": "${{ env.SSH_CMD }}", + "short": true + }, { "title": "EKS cluster name", "value": ${{ env.EKS_CLUSTER_NAME }}, From 885cc0bd9e48c7fdf1d796f185dbffdbfc15e2b1 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 6 Feb 2023 12:40:41 +0200 Subject: [PATCH 121/127] Update policies to 1.1.1 --- deploy/cloud/data/package_policy.json | 9 ++------- deploy/cloud/data/package_policy_vanilla.json | 11 +++-------- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/deploy/cloud/data/package_policy.json b/deploy/cloud/data/package_policy.json index d4387d1c40..c81a761f2c 100644 --- a/deploy/cloud/data/package_policy.json +++ b/deploy/cloud/data/package_policy.json @@ -59,12 +59,7 @@ ], "package": { "name": "cloud_security_posture", - "title": "Kubernetes Security Posture Management", - "version": "1.0.5" - }, - "vars": { - "runtimeCfg": { - "type": "yaml" - } + "title": "Kubernetes Security Posture Management (KSPM)", + "version": "1.1.1" } } diff --git a/deploy/cloud/data/package_policy_vanilla.json b/deploy/cloud/data/package_policy_vanilla.json index 33345e39d9..8724da43f3 100644 --- a/deploy/cloud/data/package_policy_vanilla.json +++ b/deploy/cloud/data/package_policy_vanilla.json @@ -58,12 +58,7 @@ ], "package": { "name": "cloud_security_posture", - "title": "Kubernetes Security Posture Management", - "version": "1.0.5" - }, - "vars": { - "runtimeCfg": { - "type": "yaml" - } + "title": "Kubernetes Security Posture Management (KSPM)", + "version": "1.1.1" } -} \ No newline at end of file +} From 3a611c426c02cde8e7fd38ad0399507cb3bab239 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 6 Feb 2023 12:53:40 +0200 Subject: [PATCH 122/127] Adding EC2 key to slack message --- .github/workflows/weekly-enviroment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 5afcec06ef..862d0248e7 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -148,8 +148,8 @@ jobs: "short": true }, { - "title": "SSH command", - "value": "${{ env.SSH_CMD }}", + "title": "EC2 private key", + "value": "${{ env.EC2_PRIVATE_KEY }}", "short": true }, { From 3a02b2647df05c52a6d3418cef8ae12217ebc929 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 6 Feb 2023 13:37:09 +0200 Subject: [PATCH 123/127] remove short from fields --- .github/workflows/weekly-enviroment.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 862d0248e7..95c66b55fc 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -135,22 +135,22 @@ jobs: { "title": "Kibana URL", "value": ${{ env.KIBANA_URL }}, - "short": true + "short": false }, { "title": "ElasticSearch username", "value": ${{ env.ELASTICSEARCH_USERNAME }}, - "short": true + "short": false }, { "title": "ElasticSearch password", "value": ${{ env.ELASTICSEARCH_PASSWORD }}, - "short": true + "short": false }, { "title": "EC2 private key", "value": "${{ env.EC2_PRIVATE_KEY }}", - "short": true + "short": false }, { "title": "EKS cluster name", From 6b36d818c7cc3adace1fb0702e9986418d3e43a5 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Mon, 6 Feb 2023 13:56:10 +0200 Subject: [PATCH 124/127] Adding instance ID --- .github/workflows/weekly-enviroment.yml | 18 ++++++++++++------ deploy/cloud/modules/ec2/output.tf | 6 ++++++ deploy/cloud/outputs.tf | 11 +++++++++-- 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 95c66b55fc..52fa1ee2ef 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -85,6 +85,7 @@ jobs: echo "KIBANA_URL=$(terraform output kibana_url)" >> $GITHUB_ENV echo "EKS_CLUSTER_NAME=$(terraform output eks_cluster_name)" >> $GITHUB_ENV echo "EKS_REGION=$(terraform output eks_region)" >> $GITHUB_ENV + echo "EC2_INSTANCE_ID=$(terraform output ec2_instance_id)" >> $GITHUB_ENV - name: Set sensitive terraform output as env variable id: set_sensitive_output @@ -148,9 +149,9 @@ jobs: "short": false }, { - "title": "EC2 private key", - "value": "${{ env.EC2_PRIVATE_KEY }}", - "short": false + "title": "EKS region", + "value": ${{ env.EKS_REGION }}, + "short": true }, { "title": "EKS cluster name", @@ -158,9 +159,14 @@ jobs: "short": true }, { - "title": "EKS region", - "value": ${{ env.EKS_REGION }}, - "short": true + "title": "Instance ID", + "value": "${{ env.EC2_INSTANCE_ID }}", + "short": false + }, + { + "title": "EC2 private key", + "value": "${{ env.EC2_PRIVATE_KEY }}", + "short": false } ] } diff --git a/deploy/cloud/modules/ec2/output.tf b/deploy/cloud/modules/ec2/output.tf index 70cf78a426..0f0817c27d 100644 --- a/deploy/cloud/modules/ec2/output.tf +++ b/deploy/cloud/modules/ec2/output.tf @@ -13,3 +13,9 @@ output "ec2_private_key" { value = tls_private_key.cloudbeat_key.private_key_pem sensitive = true } + +output "ec2_instance_id" { + description = "Use this private key to SSH into the ec2 instance" + value = aws_instance.cloudbeat.id + sensitive = true +} diff --git a/deploy/cloud/outputs.tf b/deploy/cloud/outputs.tf index 1578331ec0..e03a52c14e 100644 --- a/deploy/cloud/outputs.tf +++ b/deploy/cloud/outputs.tf @@ -75,11 +75,18 @@ output "role_arn" { } output "cloudbeat_ssh_cmd" { - value = module.aws_ec2_with_agent.cloudbeat_ssh_cmd + value = module.aws_ec2_with_agent.cloudbeat_ssh_cmd sensitive = true } output "ec2_private_key" { - value = module.aws_ec2_with_agent.ec2_private_key + value = module.aws_ec2_with_agent.ec2_private_key sensitive = true } + +output "ec2_instance_id" { + description = "EC2 instance ID" + value = module.aws_ec2_with_agent.ec2_instance_id +} + + From 9a9c4f782839d0c17d05921e43d4ea4e194f6489 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Tue, 7 Feb 2023 14:42:59 +0200 Subject: [PATCH 125/127] base64 encryption --- .github/workflows/weekly-enviroment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 52fa1ee2ef..a3dcfc87be 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -98,7 +98,7 @@ jobs: echo "::add-mask::$ELASTICSEARCH_PASSWORD" echo "ELASTICSEARCH_PASSWORD=$ELASTICSEARCH_PASSWORD" >> $GITHUB_ENV - export EC2_PRIVATE_KEY=$(echo $(terraform output cloudbeat_private_key) | base64 -w0) + export EC2_PRIVATE_KEY=$(echo $(terraform output cloudbeat_private_key) | base64) echo "::add-mask::$EC2_PRIVATE_KEY" echo "EC2_PRIVATE_KEY=$EC2_PRIVATE_KEY" >> $GITHUB_ENV From a83cd3c27ae3a173a0ac25ba817a57e8f94f6853 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Tue, 7 Feb 2023 14:45:45 +0200 Subject: [PATCH 126/127] Reverting the rest of the CI files --- .github/workflows/Periodic-CI.yml | 249 ++++++++++++++++ .github/workflows/a.json | 58 ---- .github/workflows/cloudbeat-ci.yml | 320 +++++++++++++++++++++ .github/workflows/packaging.yml | 50 ++++ .github/workflows/publish-test-results.yml | 85 ++++++ .github/workflows/unit-test.yml | 134 +++++++++ 6 files changed, 838 insertions(+), 58 deletions(-) create mode 100644 .github/workflows/Periodic-CI.yml delete mode 100644 .github/workflows/a.json create mode 100644 .github/workflows/cloudbeat-ci.yml create mode 100644 .github/workflows/packaging.yml create mode 100644 .github/workflows/publish-test-results.yml create mode 100644 .github/workflows/unit-test.yml diff --git a/.github/workflows/Periodic-CI.yml b/.github/workflows/Periodic-CI.yml new file mode 100644 index 0000000000..8aaaa76ced --- /dev/null +++ b/.github/workflows/Periodic-CI.yml @@ -0,0 +1,249 @@ +name: Periodic-CI + +on: + # On demand execution of workflow will run all suites. + workflow_dispatch: + inputs: + test-targets: + required: true + description: 'Specify test markers to run' + default: '[\"pre_merge\", \"file_system_rules\", \"k8s_object_rules\", \"process_api_server_rules\", \"process_controller_manager_rules\", \"process_etcd_rules\", \"process_kubelet_rules\", \"process_scheduler_rules\"]' + + schedule: + # Nightly job. Running every day at 2 am + - cron: '00 02 * * 0-4' + + # Weekly job running every saturday at 2 am + - cron: '00 02 * * 6' + +env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + CONTAINER_SUFFIX: ${{ github.run_id }} + TEST_TARGETS_DEFAULT: '[\"pre_merge\", \"file_system_rules\", \"k8s_object_rules\", \"process_api_server_rules\", \"process_controller_manager_rules\", \"process_etcd_rules\", \"process_kubelet_rules\", \"process_scheduler_rules\"]' + +permissions: + actions: read + checks: write + statuses: read +jobs: + Setup: + runs-on: ubuntu-20.04 + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + timeout-minutes: 15 + steps: + - id: Default + env: + TEST_TARGETS: ${{ env.TEST_TARGETS_DEFAULT }} + run: | + echo "TEST_MATRIX=$TEST_TARGETS" >> $GITHUB_ENV + + - id: Nightly + if: github.event_name == 'schedule' && github.event.schedule == '00 02 * * 0-4' + env: + TEST_TARGETS: ${{ env.TEST_TARGETS_DEFAULT }} + run: echo "TEST_MATRIX=${TEST_TARGETS}" >> $GITHUB_ENV + + - id: Weekly + if: github.event_name == 'schedule' && github.event.schedule == '00 02 * * 6' + env: + TEST_TARGETS: ${{ env.TEST_TARGETS_DEFAULT }} + run: echo "TEST_MATRIX=${TEST_TARGETS}" >> $GITHUB_ENV + + - id: Push + if: github.event_name == 'push' + env: + TEST_TARGETS: '[\"pre_merge\", \"file_system_rules\"]' + run: | + echo "TEST_MATRIX=$TEST_TARGETS" >> $GITHUB_ENV + + - id: Dispatch + if: github.event_name == 'workflow_dispatch' + env: + TEST_TARGETS: ${{ github.event.inputs.test-targets }} + run: echo "TEST_MATRIX=${TEST_TARGETS}" >> $GITHUB_ENV + + - id: set-matrix + run: echo "::set-output name=matrix::${{ env.TEST_MATRIX}}" + + Build: + name: Build + runs-on: ubuntu-20.04 + timeout-minutes: 15 + steps: + - name: Check out the repo + uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version-file: .go-version + + - name: Fetch OPA version + run: echo "OPA_VERSION=$(go list -m -f {{.Version}} github.com/open-policy-agent/opa | sed 's/v//')" >> $GITHUB_ENV + + - name: Setup OPA + uses: open-policy-agent/setup-opa@v1 + with: + version: ${{ env.OPA_VERSION }} + + - name: cache go dependencies + uses: actions/cache@v2 + with: + path: | + ~/go/pkg/mod + ~/.cache/go-build + key: ${{ runner.os }}-go-${{ matrix.go-version }}-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-${{ matrix.go-version }}- + + - name: build cloudbeat + run: make + + - name: build opa bundle + uses: magefile/mage-action@v2 + with: + version: latest + args: BuildOpaBundle + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Cache Build dependencies + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.workflow }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Build cloudbeat-docker + uses: docker/build-push-action@v2 + with: + context: . + push: false + tags: cloudbeat:latest + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new + outputs: type=docker,dest=/tmp/cloudbeat-${{ env.CONTAINER_SUFFIX }}.tar + + - name: Build pytest-docker + uses: docker/build-push-action@v2 + with: + context: ./tests/. + push: false + tags: cloudbeat-test:latest + cache-from: type=local,mode=max,src=/tmp/.buildx-cache + cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-new + outputs: type=docker,dest=/tmp/pytest-${{ env.CONTAINER_SUFFIX }}.tar + + - name: Cache docker images + uses: actions/cache@v2 + with: + path: /tmp/*.tar + key: ${{ runner.os }}-dockers-cache-${{ env.CONTAINER_SUFFIX }} + restore-keys: | + ${{ runner.os }}-dockers-cache-${{ env.CONTAINER_SUFFIX }} + + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache + + Test_Matrix: + needs: ['Build', 'Setup'] + strategy: + fail-fast: false + matrix: + test-target: ${{fromJson(needs.Setup.outputs.matrix)}} + name: ${{ matrix.test-target }}-tests + runs-on: ubuntu-20.04 + timeout-minutes: 90 + steps: + - name: Check out the repo + uses: actions/checkout@v2 + + - uses: azure/setup-helm@v1 + id: install + + - uses: extractions/setup-just@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Create k8s Kind Cluster + uses: helm/kind-action@v1.2.0 + with: + config: deploy/k8s/kind/kind-mono.yml + + - name: Cache docker images + uses: actions/cache@v2 + with: + path: /tmp/*.tar + key: ${{ runner.os }}-dockers-cache-${{ env.CONTAINER_SUFFIX }} + restore-keys: | + ${{ runner.os }}-dockers-cache- + + - name: Load images to kind + run: | + kind load image-archive /tmp/cloudbeat-${{ env.CONTAINER_SUFFIX }}.tar --name=kind-mono & kind load image-archive /tmp/pytest-${{ env.CONTAINER_SUFFIX }}.tar --name=kind-mono + + - name: Deploy tests Helm chart + id: deploy_helm + run: | + just deploy-tests-helm-ci ${{ matrix.test-target }} + + - name: Run Tests + id: run_tests + run: | + just run-tests-ci + + - name: Upload Test Results + if: always() + uses: actions/upload-artifact@v2 + with: + name: allure_results-${{ matrix.test-target}} + path: tests/allure/results/ + + - name: clear cache + id: clear_cache + if: success() + run: rm -rf /tmp/${{ env.CONTAINER_SUFFIX }}}.tar + + publish_results: + name: Publish Results + needs: Test_Matrix + if: always() + runs-on: ubuntu-20.04 + steps: + - name: Download Artifacts + uses: actions/download-artifact@v3 + with: + path: artifacts + + - name: Extract Artifacts + run: | + mkdir -p tests/allure/results + find artifacts/ -type f -print0 | xargs -0 mv -t tests/allure/results + + - name: Publish allure report + if: always() + uses: andrcuns/allure-publish-action@v1.0.1 + with: + storageType: s3 + resultsGlob: "tests/allure/results/*" + updatePr: actions + collapseSummary: false + summary: suites + summaryTableType: markdown + copyLatest: true + bucket: csp-allure-reports + prefix: allure_reports/cloudbeat/${{ github.ref_name }} + ignoreMissingResults: true + + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + if: failure() + with: + limit-access-to-actor: true diff --git a/.github/workflows/a.json b/.github/workflows/a.json deleted file mode 100644 index a94069dddc..0000000000 --- a/.github/workflows/a.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "text": "A new deployment job has been triggered", - "attachments": [ - { - "color": "#36a64f", - "fields": [ - { - "title": "Environment", - "value": "${{ github.event.inputs.environment }}", - "short": true - }, - { - "title": "Benchmark", - "value": "${{ github.event.inputs.benchmark }}", - "short": true - }, - { - "title": "Alerts", - "value": "${{ github.event.inputs.alerts }}", - "short": true - }, - { - "title": "Log level", - "value": "${{ github.event.inputs.logLevel }}", - "short": true - }, - { - "title": "Kibana URL", - "short": true - }, - { - "title": "ElasticSearch username", - "short": true - }, - { - "title": "ElasticSearch password", - "short": true - }, - { - "title": "EC2 private key", - "short": true - }, - { - "title": "SSH command", - "short": true - }, - { - "title": "EKS cluster name", - "short": true - }, - { - "title": "EKS region", - "short": true - } - ] - } - ] -} diff --git a/.github/workflows/cloudbeat-ci.yml b/.github/workflows/cloudbeat-ci.yml new file mode 100644 index 0000000000..02c6ab3ce6 --- /dev/null +++ b/.github/workflows/cloudbeat-ci.yml @@ -0,0 +1,320 @@ +name: Cloudbeat-CI + +on: + pull_request: + branches: + - main + - '[0-9]+.[0-9]+' + types: [opened, synchronize, reopened] + +env: + CONTAINER_SUFFIX: ${{ github.run_id }} + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + Lint: + # for more information see .pre-commit-config.yaml + name: Lint + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check out the repo + uses: actions/checkout@v2 + + - name: Init Hermit + run: ./bin/hermit env -r >> $GITHUB_ENV + + - name: Python lints + run: | + git ls-files -- '*.py' | xargs pre-commit run --file + shell: bash + + - name: Go lints + run: | + git ls-files -- '*.go' | xargs pre-commit run --file + shell: bash + + Build: + name: Build + runs-on: ubuntu-20.04 + timeout-minutes: 15 + steps: + - name: Check out the repo + uses: actions/checkout@v2 + + - name: Init Hermit + run: ./bin/hermit env -r >> $GITHUB_ENV + + - name: cache go dependencies + uses: actions/cache@v2 + with: + path: | + ~/go/pkg/mod + ~/.cache/go-build + key: ${{ runner.os }}-go-${{ matrix.go-version }}-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-${{ matrix.go-version }}- + + - name: build cloudbeat binary + uses: magefile/mage-action@v2 + with: + version: latest + args: build + + - name: build opa bundle + uses: magefile/mage-action@v2 + with: + version: latest + args: BuildOpaBundle + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Cache Build dependencies + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.workflow }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Build cloudbeat-docker + uses: docker/build-push-action@v2 + with: + context: . + push: false + tags: cloudbeat:latest + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new + outputs: type=docker,dest=/tmp/cloudbeat-${{ env.CONTAINER_SUFFIX }}.tar + + - name: Build elastic-agent + run: | + make build_elastic_agent_docker_image + + - name: Build pytest-docker + uses: docker/build-push-action@v2 + with: + context: ./tests/. + push: false + tags: cloudbeat-test:latest + cache-from: type=local,mode=max,src=/tmp/.buildx-cache + cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-new + outputs: type=docker,dest=/tmp/pytest-${{ env.CONTAINER_SUFFIX }}.tar + + - name: Cache docker images + uses: actions/cache@v2 + with: + path: /tmp/*.tar + key: ${{ runner.os }}-dockers-cache-${{ env.CONTAINER_SUFFIX }} + + - name: Move cache + run: | + ./.ci/scripts/gh-cache.sh + shell: bash + + Test_Matrix: + name: ${{ matrix.test-target }}-${{ matrix.range }}-tests + needs: [Build] + runs-on: ubuntu-20.04 + timeout-minutes: 55 + strategy: + matrix: + include: + - test-target: pre_merge_agent + range: '' + values_file: tests/deploy/values/ci-sa-agent.yml + - test-target: pre_merge + range: '' + values_file: tests/deploy/values/ci.yml + - test-target: file_system_rules + range: '0..5' + values_file: tests/deploy/values/ci.yml + - test-target: file_system_rules + range: '5..10' + values_file: tests/deploy/values/ci.yml + - test-target: file_system_rules + range: '10..15' + values_file: tests/deploy/values/ci.yml + - test-target: file_system_rules + range: '15..20' + values_file: tests/deploy/values/ci.yml + - test-target: file_system_rules + range: '20..25' + values_file: tests/deploy/values/ci.yml + - test-target: file_system_rules + range: '25..30' + values_file: tests/deploy/values/ci.yml + - test-target: file_system_rules + range: '30..35' + values_file: tests/deploy/values/ci.yml + - test-target: file_system_rules + range: '35..40' + values_file: tests/deploy/values/ci.yml + - test-target: file_system_rules + range: '40..45' + values_file: tests/deploy/values/ci.yml + - test-target: file_system_rules + range: '45..50' + values_file: tests/deploy/values/ci.yml + - test-target: file_system_rules + range: '50..55' + values_file: tests/deploy/values/ci.yml + - test-target: file_system_rules + range: '55..60' + values_file: tests/deploy/values/ci.yml + - test-target: file_system_rules + range: '60..65' + values_file: tests/deploy/values/ci.yml + - test-target: file_system_rules + range: '65..70' + values_file: tests/deploy/values/ci.yml + - test-target: file_system_rules + range: '70..' + values_file: tests/deploy/values/ci.yml + - test-target: k8s_object_rules + range: '0..6' + values_file: tests/deploy/values/ci.yml + - test-target: k8s_object_rules + range: '6..12' + values_file: tests/deploy/values/ci.yml + - test-target: k8s_object_rules + range: '12..18' + values_file: tests/deploy/values/ci.yml + - test-target: k8s_object_rules + range: '18..' + values_file: tests/deploy/values/ci.yml + - test-target: process_scheduler_rules + range: '0..3' + values_file: tests/deploy/values/ci.yml + - test-target: process_scheduler_rules + range: '3..' + values_file: tests/deploy/values/ci.yml + - test-target: process_api_server_rules + range: '0..5' + values_file: tests/deploy/values/ci.yml + - test-target: process_api_server_rules + range: '5..10' + values_file: tests/deploy/values/ci.yml + - test-target: process_api_server_rules + range: '10..15' + values_file: tests/deploy/values/ci.yml + - test-target: process_api_server_rules + range: '15..20' + values_file: tests/deploy/values/ci.yml + - test-target: process_api_server_rules + range: '20..24' + values_file: tests/deploy/values/ci.yml + - test-target: process_api_server_rules + range: '24..28' + values_file: tests/deploy/values/ci.yml + - test-target: process_api_server_rules + range: '28..32' + values_file: tests/deploy/values/ci.yml + - test-target: process_api_server_rules + range: '32..36' + values_file: tests/deploy/values/ci.yml + - test-target: process_api_server_rules + range: '36..40' + values_file: tests/deploy/values/ci.yml + - test-target: process_api_server_rules + range: '40..44' + values_file: tests/deploy/values/ci.yml + - test-target: process_api_server_rules + range: '44..48' + values_file: tests/deploy/values/ci.yml + - test-target: process_api_server_rules + range: '48..52' + values_file: tests/deploy/values/ci.yml + - test-target: process_api_server_rules + range: '52..56' + values_file: tests/deploy/values/ci.yml + - test-target: process_api_server_rules + range: '56..' + values_file: tests/deploy/values/ci.yml + - test-target: process_controller_manager_rules + range: '0..4' + values_file: tests/deploy/values/ci.yml + - test-target: process_controller_manager_rules + range: '4..8' + values_file: tests/deploy/values/ci.yml + - test-target: process_controller_manager_rules + range: '8..12' + values_file: tests/deploy/values/ci.yml + - test-target: process_controller_manager_rules + range: '12..' + values_file: tests/deploy/values/ci.yml + - test-target: process_etcd_rules + range: '0..4' + values_file: tests/deploy/values/ci.yml + - test-target: process_etcd_rules + range: '4..8' + values_file: tests/deploy/values/ci.yml + - test-target: process_etcd_rules + range: '8..' + values_file: tests/deploy/values/ci.yml + - test-target: process_kubelet_rules + range: '0..4' + values_file: tests/deploy/values/ci.yml + - test-target: process_kubelet_rules + range: '4..8' + values_file: tests/deploy/values/ci.yml + - test-target: process_kubelet_rules + range: '8..12' + values_file: tests/deploy/values/ci.yml + - test-target: process_kubelet_rules + range: '12..16' + values_file: tests/deploy/values/ci.yml + - test-target: process_kubelet_rules + range: '16..20' + values_file: tests/deploy/values/ci.yml + - test-target: process_kubelet_rules + range: '20..' + values_file: tests/deploy/values/ci.yml + fail-fast: false + steps: + - name: Check out the repo + uses: actions/checkout@v2 + + - name: Init Hermit + run: ./bin/hermit env -r >> $GITHUB_ENV + + - name: Create k8s Kind Cluster + run: | + just create-kind-cluster + + - name: Cache docker images + uses: actions/cache@v2 + with: + path: /tmp/*.tar + key: ${{ runner.os }}-dockers-cache-${{ env.CONTAINER_SUFFIX }} + + - name: Load images to kind + run: | + ./.ci/scripts/kind-images.sh ${{ env.CONTAINER_SUFFIX }} kind-multi + shell: bash + + - name: Deploy tests Helm chart + id: deploy_helm + run: | + just deploy-tests-helm ${{ matrix.test-target }} ${{ matrix.values_file }} ${{ matrix.range }} + + - name: Run Tests + id: run_tests + run: | + just run-tests ${{ matrix.test-target }} + + - name: Upload Test Results + if: always() + uses: actions/upload-artifact@v2 + with: + name: allure_results-${{ matrix.test-target}} + path: tests/allure/results/ + + - name: clear cache + id: clear_cache + if: always() + run: rm -rf /tmp/*-${{ env.CONTAINER_SUFFIX }}.tar diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml new file mode 100644 index 0000000000..b9b1eb7982 --- /dev/null +++ b/.github/workflows/packaging.yml @@ -0,0 +1,50 @@ +name: Packaging + +on: + pull_request: + branches: + - main + - '[0-9]+.[0-9]+' + types: [opened, synchronize, reopened] + +env: + DEV: true + SNAPSHOT: true + PLATFORMS: linux/amd64 + GOPATH: /home/runner/go + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + packag_beat: + name: Package Cloudbeat + runs-on: ubuntu-20.04 + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + types: [ tar.gz, docker ] + steps: + - name: Check out the repo + uses: actions/checkout@v2 + + - name: Init Hermit + run: ./bin/hermit env -r >> $GITHUB_ENV + + - name: cache go dependencies + uses: actions/cache@v2 + with: + path: | + ~/go/pkg/mod + ~/.cache/go-build + key: ${{ runner.os }}-go-${{ matrix.go-version }}-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-${{ matrix.go-version }}- + + - name: Packaging + run: | + mage -v package + env: + TYPES: ${{ matrix.types }} diff --git a/.github/workflows/publish-test-results.yml b/.github/workflows/publish-test-results.yml new file mode 100644 index 0000000000..94acfb3397 --- /dev/null +++ b/.github/workflows/publish-test-results.yml @@ -0,0 +1,85 @@ +name: Publish-Test-Results + +on: + workflow_run: + workflows: ["Cloudbeat-CI"] + types: + - completed + +jobs: + publish_results: + timeout-minutes: 15 + name: Publish Results + runs-on: ubuntu-20.04 + if: github.event.workflow_run.conclusion != 'skipped' + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + permissions: + checks: write + # needed unless run with comment_mode: off + pull-requests: write + # required by download step to access artifacts API + actions: read + steps: + + - name: Download and Extract Artifacts + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + mkdir -p tests/allure/results + mkdir -p artifacts && cd artifacts + + artifacts_url=${{ github.event.workflow_run.artifacts_url }} + + gh api "$artifacts_url" -q '.artifacts[] | [.name, .archive_download_url] | @tsv' | while read artifact + do + IFS=$'\t' read name url <<< "$artifact" + gh api $url > "$name.zip" + unzip -o "$name.zip" -d "../tests/allure/results" + done + + - name: Get pr info + id: pr_info + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + PR_NUM=$(gh api -X GET search/issues -f q='elastic/cloudbeat is:open is:pr is:unmerged ${{ github.event.workflow_run.head_sha }}' | jq '.items[0].number' ) + echo "PR_NUMBER=$PR_NUM" >> $GITHUB_ENV + + - name: Publish allure report + if: always() + uses: andrcuns/allure-publish-action@v1.0.1 + env: + GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + storageType: s3 + resultsGlob: "tests/allure/results/*" + updatePr: actions + collapseSummary: false + summary: suites + summaryTableType: markdown + copyLatest: true + bucket: csp-allure-reports + prefix: allure_reports/cloudbeat/prs/${{ env.PR_NUMBER }} + ignoreMissingResults: true + + - name: Output to Job Summary + if: ${{ github.event.workflow_run.event == 'pull_request' }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + echo "### Run Summary :rocket:" >> $GITHUB_STEP_SUMMARY + echo "Allure Report: http://csp-allure-reports.s3.amazonaws.com/allure_reports/cloudbeat/prs/${{ env.PR_NUMBER }}/index.html" >> $GITHUB_STEP_SUMMARY + + - name: Comment test success result + uses: marocchino/sticky-pull-request-comment@v2 + if: ${{ github.event.workflow_run.event == 'pull_request' }} + with: + header: Test Results + number: ${{ env.PR_NUMBER }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + message: | + ## Cloudbeat CI :robot: + Allure Report: http://csp-allure-reports.s3.amazonaws.com/allure_reports/cloudbeat/prs/${{ env.PR_NUMBER }}/index.html diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml new file mode 100644 index 0000000000..b03c5a03c5 --- /dev/null +++ b/.github/workflows/unit-test.yml @@ -0,0 +1,134 @@ +name: UnitTests + +on: + pull_request: + branches: + - main + - '[0-9]+.[0-9]+' + types: [opened, synchronize, reopened] + +env: + K8S_MANIFEST_DIR: deploy + + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + unit_tests: + name: Unit Tests + runs-on: ubuntu-20.04 + timeout-minutes: 15 + steps: + - name: Check out the repo + uses: actions/checkout@v2 + + - name: Init Hermit + run: ./bin/hermit env -r >> $GITHUB_ENV + + - name: Mage Check + run: mage check + + - name: Mage checkLicenseHeaders + run: mage checkLicenseHeaders + + - name: Mage build bundle before unit tests + uses: magefile/mage-action@v1.6.0 + with: + args: buildOpaBundle + + - name: Copy bundle to required dirs + run: | + cp bundle.tar.gz evaluator/ + + - name: Unit-Test Cloudbeat + run: | + GOOS=linux go test -v -coverpkg=./... -coverprofile=cover.out.tmp ./... + cat cover.out.tmp | grep -v "_mock.go" > cover.out # remove mock files from coverage report + + - name: Upload coverage artifact + uses: actions/upload-artifact@v3 + with: + name: coverage-file + path: cover.out + + coverage: + name: Coverage report + needs: unit_tests + runs-on: ubuntu-20.04 + timeout-minutes: 10 + steps: + - name: Check out the repo + uses: actions/checkout@v2 + + - name: Init Hermit + run: ./bin/hermit env -r >> $GITHUB_ENV + + - name: Install goveralls + run: go install github.com/mattn/goveralls@latest + + - name: Download coverage artifact + uses: actions/download-artifact@v3 + with: + name: coverage-file + + - name: Send coverage + env: + COVERALLS_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: goveralls -coverprofile=cover.out -service=github + + manifest_tests: + name: Manifest Tests + runs-on: ubuntu-20.04 + timeout-minutes: 15 + steps: + - name: Check out the repo + uses: actions/checkout@v2 + + - name: Get diff k8s manifests + uses: technote-space/get-diff-action@v4 + with: + PATTERNS: ${{ env.K8S_MANIFEST_DIR }}/** + + - name: Run K8s manifest tests + if: env.GIT_DIFF + uses: stefanprodan/kube-tools@v1 + with: + kubectl: 1.18.4 + kubeval: v0.16.1 + command: | + kubeval --strict -d ${{ env.K8S_MANIFEST_DIR }} --ignore-missing-schemas --ignored-filename-patterns kustomization,kind,patch,cloudbeat,cloudformation --skip-kinds ClusterConfig + + golangci: + name: golangci-lint + runs-on: ubuntu-20.04 + timeout-minutes: 15 + steps: + - name: Check out the repo + uses: actions/checkout@v3 + - uses: actions/setup-go@v3 + with: + go-version-file: .go-version + - name: Check out the repo + uses: actions/checkout@v3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + repository: ${{ (github.event.pull_request_target || github.event.pull_request).head.repo.full_name }} + ref: ${{ (github.event.pull_request_target || github.event.pull_request).head.sha }} + + - name: Go linter + uses: golangci/golangci-lint-action@v3 + with: + version: latest + args: --timeout=10m --whole-files + + dependency-review: + runs-on: ubuntu-20.04 + if: ${{ github.event_name == 'pull_request' }} + steps: + - name: Checkout Repository + uses: actions/checkout@v3 + + - name: Dependency Review + uses: actions/dependency-review-action@v2 From e57f103c2fe32bbd6a542d89ef9d4fd613041642 Mon Sep 17 00:00:00 2001 From: ofiriro3 Date: Tue, 7 Feb 2023 14:46:28 +0200 Subject: [PATCH 127/127] Reverting the rest of the CI files --- .github/workflows/eks-ci.yml | 238 +++++++++++++++++++++++++++++++++++ 1 file changed, 238 insertions(+) create mode 100644 .github/workflows/eks-ci.yml diff --git a/.github/workflows/eks-ci.yml b/.github/workflows/eks-ci.yml new file mode 100644 index 0000000000..451f5dae63 --- /dev/null +++ b/.github/workflows/eks-ci.yml @@ -0,0 +1,238 @@ +name: EKS-CI + +on: + # On demand execution of workflow will run all suites. + workflow_dispatch: + inputs: + test-targets: + required: true + description: "Specify test markers to run" + default: '[\"pre_merge\"]' + +env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + CONTAINER_SUFFIX: ${{ github.run_id }} + TEST_TARGETS_DEFAULT: '[\"pre_merge\"]' + AWS_REGION: eu-west-2 + REPORTS_DIR: tests/allure/results/ + KUBE_NAMESPACE: kube-system + +jobs: + Setup: + runs-on: ubuntu-20.04 + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + timeout-minutes: 15 + steps: + - id: set-default-matrix + env: + TEST_TARGETS: ${{ env.TEST_TARGETS_DEFAULT }} + run: | + echo "TEST_MATRIX=$TEST_TARGETS" >> $GITHUB_ENV + + - id: Dispatch + if: github.event_name == 'workflow_dispatch' + env: + TEST_TARGETS: ${{ github.event.inputs.test-targets }} + run: echo "TEST_MATRIX=${TEST_TARGETS}" >> $GITHUB_ENV + + - id: set-matrix + run: echo "matrix=${{ env.TEST_MATRIX}}" >> $GITHUB_OUTPUT + + Build: + name: Build + runs-on: ubuntu-20.04 + timeout-minutes: 15 + steps: + - name: Check out the repo + uses: actions/checkout@v3 + + - name: Init Hermit + run: ./bin/hermit env -r >> $GITHUB_ENV + + - name: cache go dependencies + uses: actions/cache@v3 + with: + path: | + ~/go/pkg/mod + ~/.cache/go-build + key: ${{ runner.os }}-go-${{ matrix.go-version }}-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-${{ matrix.go-version }}- + + - name: build cloudbeat binary + uses: magefile/mage-action@v2 + with: + version: latest + args: build + + - name: build opa bundle + uses: magefile/mage-action@v2 + with: + version: latest + args: BuildOpaBundle + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Cache Build dependencies + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.workflow }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Build cloudbeat-docker + uses: docker/build-push-action@v2 + with: + context: . + push: false + tags: cloudbeat:latest + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new + outputs: type=docker,dest=/tmp/cloudbeat-${{ env.CONTAINER_SUFFIX }}.tar + + - name: Build elastic-agent + run: | + make build_elastic_agent_docker_image + + - name: Build pytest-docker + uses: docker/build-push-action@v3 + with: + context: ./tests/. + push: false + tags: cloudbeat-test:latest + cache-from: type=local,mode=max,src=/tmp/.buildx-cache + cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-new + outputs: type=docker,dest=/tmp/pytest-${{ env.CONTAINER_SUFFIX }}.tar + + - name: Cache docker images + uses: actions/cache@v3 + with: + path: /tmp/*.tar + key: ${{ runner.os }}-dockers-cache-${{ env.CONTAINER_SUFFIX }} + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + + - name: Load images cloudbeat and cloudbeat-test to ECR + id: load-to-ecr + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + run: | + ./.ci/scripts/ecr-images.sh ${{ env.CONTAINER_SUFFIX }} ${{ env.ECR_REGISTRY}} + shell: bash + + + - name: Move cache + run: | + ./.ci/scripts/gh-cache.sh + shell: bash + + Test_Matrix: + needs: ["Setup", "Build"] + strategy: + fail-fast: false + matrix: + include: + - test-target: pre_merge + range: "" + values_file: tests/deploy/values/ci-eks.yml + k8s_context: "test-eks-config-1" + name: ${{ matrix.test-target }}-tests + runs-on: ubuntu-20.04 + timeout-minutes: 90 + steps: + - name: Check out the repo + uses: actions/checkout@v3 + + # Setup all required tools + - name: Init Hermit + run: ./bin/hermit env -r >> $GITHUB_ENV + + - name: Get kubeconfig and select context + if: success() + run: | + aws eks update-kubeconfig --name ${{ matrix.k8s_context }} --region ${{ env.AWS_REGION }} --alias ${{ matrix.k8s_context }} + echo 'KUBE_CONFIG_DATA=$(cat ~/.kube/config | base64)' >> $GITHUB_ENV + kubectl config use-context ${{ matrix.k8s_context }} + kubectl config set-context --current --namespace=${{ env.KUBE_NAMESPACE }} + + - name: Deploy tests Helm chart + id: deploy_helm + if: success() + run: | + just deploy-tests-helm ${{ matrix.test-target }} ${{ matrix.values_file }} ${{ matrix.range }} + + - name: Execute tests + id: run_tests + if: success() + run: | + just run-tests ${{ matrix.test-target }} ${{ matrix.k8s_context }} + + - name: Copy test results + id: copy_results + if: always() + run: | + NODE=$(kubectl get po test-pod-v1 -o jsonpath='{.spec.nodeName}') + RESULTS_POD=$(kubectl get po -l k8s-app=eks-results --field-selector spec.nodeName=$NODE -o jsonpath='{.items[*].metadata.name}') + kubectl cp $RESULTS_POD:/reports ${{ env.REPORTS_DIR }} + kubectl exec $RESULTS_POD -- sh -c 'rm -rf /reports/*' + + - name: Upload Test Results + if: success() + uses: actions/upload-artifact@v3 + with: + name: allure_results-${{ matrix.test-target}} + path: ${{ env.REPORTS_DIR }} + + - name: Purge tests environment + id: purge_helm + if: always() + run: | + just purge-tests + + publish_results: + name: Publish Results + needs: Test_Matrix + if: always() + runs-on: ubuntu-20.04 + steps: + - name: Download Artifacts + uses: actions/download-artifact@v3 + with: + path: artifacts + + - name: Extract Artifacts + run: | + mkdir -p tests/allure/results + find artifacts/ -type f -print0 | xargs -0 mv -t tests/allure/results + + - name: Publish allure report + if: always() + uses: andrcuns/allure-publish-action@v1.0.1 + env: + GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + AWS_REGION: eu-west-1 + with: + storageType: s3 + resultsGlob: "${{ env.REPORTS_DIR }}*" + updatePr: actions + collapseSummary: false + summary: suites + summaryTableType: markdown + copyLatest: true + bucket: csp-allure-reports + prefix: allure_reports/cloudbeat/${{ github.ref_name }} + ignoreMissingResults: true