Weekly enviroment vanilla work with snapshot

.github/workflows/unit-test.yml

@@ -4,7 +4,7 @@ on:
   pull_request:
     branches:
       - main
-      - '[0-9]+.[0-9]+'
+      - "[0-9]+.[0-9]+"
     types: [opened, synchronize, reopened]
   push:
     branches:
@@ -13,7 +13,6 @@ on:
 env:
   K8S_MANIFEST_DIR: deploy
 
-
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
@@ -22,7 +21,7 @@ jobs:
   unit_tests:
     name: Unit Tests
     runs-on: ubuntu-20.04
-    timeout-minutes: 15
+    timeout-minutes: 20
     steps:
       - name: Check out the repo
         uses: actions/checkout@v3
@@ -50,7 +49,7 @@ jobs:
 
       - name: Unit-Test Cloudbeat
         run: |
-          GOOS=linux go test -v -coverpkg=./... -coverprofile=cover.out.tmp ./...
+          GOOS=linux go test -v -race -coverpkg=./... -coverprofile=cover.out.tmp ./...
           cat cover.out.tmp | grep -v "mock_.*.go" > cover.out # remove mock files from coverage report
 
       - name: Upload coverage artifact
@@ -116,6 +115,7 @@ jobs:
       - uses: actions/setup-go@v3
         with:
           go-version-file: .go-version
+
       - name: Check out the repo
         uses: actions/checkout@v3
         with:
@@ -153,3 +153,16 @@ jobs:
         run: |
           cfn-lint --version
           cfn-lint -I -t ./deploy/cloudformation/elastic-agent-ec2.yml
+
+  terraform-linter:
+    name: terraform-lint
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v3
+
+      - name: Init Hermit
+        run: ./bin/hermit env -r >> $GITHUB_ENV
+
+      - name: Terraform fmt
+        run: terraform fmt -check -recursive

.github/workflows/weekly-enviroment.yml

@@ -26,11 +26,11 @@ env:
   TF_VAR_ec_api_key: ${{ secrets.WEEKLY_ENVIRONMENT_EC_API_KEY }}
   TF_VAR_environment: ${{ github.event.inputs.logLevel }}
   TF_LOG: ${{ github.event.inputs.logLevel }}
-  TF_VAR: 8.6.1
+  TF_VAR_stack_version: 8.7.0-SNAPSHOT
 
 jobs:
   terraform:
-    name: Deploy KSPM/CSPM cloud environment
+    name: Deploy KSPM cloud environment
     runs-on: ubuntu-latest
     defaults:
       run:
@@ -39,36 +39,24 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v2
 
-      - name: Setup Terraform
-        uses: hashicorp/setup-terraform@v2
-        with:
-          terraform_version: 1.3.5
-          terraform_wrapper: false
-
-      - name: Terraform fmt
-        id: fmt
-        run: terraform fmt -check
-        continue-on-error: true
+      - name: Init Hermit
+        run: ./bin/hermit env -r >> $GITHUB_ENV
+        working-directory: ./
 
       - name: Terraform Init
-        id: init
         run: terraform init -no-color
 
       - name: Terraform Validate
-        id: validate
         run: terraform validate -no-color
 
       - name: Deploy Elastic Cloud
-        id: apply
         run: terraform apply --auto-approve
 
       - name: Set terraform output as env variable
-        id: kibana_url
         run: |
           echo "KIBANA_URL=$(terraform output kibana_url)" >> $GITHUB_ENV
 
       - name: Set sensitive terraform output as env variable
-        id: set_sensitive_output
         run: |
           export ELASTICSEARCH_USERNAME=$(terraform output elasticsearch_username)
           echo "::add-mask::$ELASTICSEARCH_USERNAME"
@@ -79,13 +67,11 @@ jobs:
           echo "ELASTICSEARCH_PASSWORD=$ELASTICSEARCH_PASSWORD" >> $GITHUB_ENV
 
       - name: Install KSPM vanilla integration
-        id: install_vanilla_integration
         working-directory: ${{ env.SCRIPTS_DIR }}
         run: |
           ./install-kspm-vanilla-integration.sh ${{ env.KIBANA_URL }} ${{ env.ELASTICSEARCH_PASSWORD }}
 
       - name: Deploy agent on EC2
-        id: deploy_agent_on_ec2
         working-directory: ${{ env.SCRIPTS_DIR }}
         run: |
           echo -e "${{ secrets.WEEKLY_ENVIRONMENT_EC2_PRIVATE_KEY }}" > weekly-key.pem
@@ -97,7 +83,6 @@ jobs:
 
       # Once https://github.com/slackapi/slack-github-action/issues/84 will be resolved we can push the payload to a different file
       - name: Send custom JSON data to Slack workflow
-        id: slack
         uses: slackapi/slack-github-action@v1.23.0
         with:
           payload: |

.go-version

@@ -1 +1 @@
-1.18.6
+1.19

beater/beater.go

@@ -0,0 +1,59 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package beater
+
+import (
+	"fmt"
+
+	"github.com/elastic/cloudbeat/config"
+	"github.com/elastic/cloudbeat/flavors"
+	"github.com/elastic/cloudbeat/launcher"
+
+	"github.com/elastic/beats/v7/libbeat/beat"
+	"github.com/elastic/beats/v7/libbeat/common/reload"
+	agentconfig "github.com/elastic/elastic-agent-libs/config"
+	"github.com/elastic/elastic-agent-libs/logp"
+)
+
+func New(b *beat.Beat, cfg *agentconfig.C) (beat.Beater, error) {
+	log := logp.NewLogger("launcher")
+	reloader := launcher.NewListener(log)
+	validator := &validator{}
+
+	s, err := launcher.New(log, "Cloudbeat", reloader, validator, NewBeater, cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	reload.RegisterV2.MustRegisterInput(reloader)
+	return s, nil
+}
+
+// NewBeater creates an instance of beater.
+func NewBeater(b *beat.Beat, cfg *agentconfig.C) (beat.Beater, error) {
+	c, err := config.New(cfg)
+	if err != nil {
+		return nil, fmt.Errorf("NewBeater: could not parse configuration %v, skipping with error: %w", cfg.FlattenedKeys(), err)
+	}
+	switch c.Deployment {
+	case config.VulnerabilityType:
+		return flavors.NewVulnerability(b, cfg)
+	default:
+		return flavors.NewPosture(b, cfg)
+	}
+}

bin/go

@@ -1 +1 @@
-.go-1.18.6.pkg
+.go-1.19.pkg

bin/gofmt

@@ -1 +1 @@
-.go-1.18.6.pkg
+.go-1.19.pkg

bin/hermit.hcl

@@ -1,4 +1,4 @@
 env = {
-  "CLOUDBEAT_VERSION": "8.7.0",
+  "CLOUDBEAT_VERSION": "8.8.0",
   "ELK_VERSION": "${CLOUDBEAT_VERSION}-SNAPSHOT",
 }

config/config.go

@@ -31,9 +31,11 @@ import (
 	"github.com/elastic/elastic-agent-libs/config"
 )
 
-const DefaultNamespace = "default"
-
-const ResultsDatastreamIndexPrefix = "logs-cloud_security_posture.findings"
+const (
+	DefaultNamespace             = "default"
+	VulnerabilityType            = "cloudbeat/vuln_mgmt_aws"
+	ResultsDatastreamIndexPrefix = "logs-cloud_security_posture.findings"
+)
 
 var ErrBenchmarkNotSupported = launcher.NewUnhealthyError("benchmark is not supported")
 
@@ -43,6 +45,7 @@ type Fetcher struct {
 
 type Config struct {
 	Benchmark   string                  `config:"config.v1.benchmark"`
+	Deployment  string                  `config:"config.v1.deployment"`
 	CloudConfig CloudConfig             `config:"config.v1"`
 	Fetchers    []*config.C             `config:"fetchers"`
 	KubeConfig  string                  `config:"kube_config"`

deploy/cloud/modules/api/terraform.tf

@@ -1,11 +1,11 @@
 terraform {
   required_providers {
     restapi = {
-      source = "mastercard/restapi"
+      source  = "mastercard/restapi"
       version = "~> 1.18.0"
     }
     http = {
-      source = "hashicorp/http"
+      source  = "hashicorp/http"
       version = "~> 3.2.1"
     }
   }

deploy/cloud/modules/provision-apps/aws-ebs-csi-driver.tf

@@ -1,12 +1,12 @@
 resource "helm_release" "aws_ebs_csi_driver" {
-  chart = "aws-ebs-csi-driver"
-  name = "aws-ebs-csi-driver"
-  namespace = var.namespace
+  chart      = "aws-ebs-csi-driver"
+  name       = "aws-ebs-csi-driver"
+  namespace  = var.namespace
   repository = "https://kubernetes-sigs.github.io/aws-ebs-csi-driver"
 
 
   set {
-    name = "controller.serviceAccount.name"
+    name  = "controller.serviceAccount.name"
     value = "ebs-csi-controller-sa"
   }
 

deploy/cloud/modules/provision-apps/nginx-ingress.tf

@@ -1,18 +1,18 @@
 resource "helm_release" "nginx_ingress" {
-  chart      = "nginx-ingress-controller"
-  name       = "nginx-ingress-controller"
+  chart = "nginx-ingress-controller"
+  name  = "nginx-ingress-controller"
 
   repository = "https://charts.bitnami.com/bitnami"
-  timeout = 600
-  namespace = var.namespace
+  timeout    = 600
+  namespace  = var.namespace
 
   set {
     name  = "service.type"
     value = "ClusterIP"
   }
 
   set {
-    name = "replicaCount"
+    name  = "replicaCount"
     value = var.replica_count
   }
 }

deploy/cloud/modules/provision-apps/terraform.tf

@@ -1,7 +1,7 @@
 terraform {
   required_providers {
     helm = {
-      source = "hashicorp/helm"
+      source  = "hashicorp/helm"
       version = ">=2.8.0"
     }
 

deploy/cloud/modules/provision-apps/variables.tf

@@ -1,9 +1,9 @@
 variable "namespace" {
-  type = string
+  type    = string
   default = "default"
 }
 
 variable "replica_count" {
-  type = string
+  type    = string
   default = "2"
 }

deploy/k8s/kind/kind-mono.yml

@@ -7,5 +7,12 @@ nodes:
   extraMounts:
   - hostPath: ./tests/allure/results
     containerPath: /tmp/data
+  extraPortMappings:
+  - containerPort: 9200
+    hostPort: 9200
+    listenAddress: "127.0.0.1"
+  - containerPort: 5601
+    hostPort: 5601
+    listenAddress: "127.0.0.1"
 
 # Todo Enable EphemeralContainers on kind config for debug

deploy/weekly-environment/main.tf

@@ -1,5 +1,6 @@
 provider "ec" {
-  apikey = var.ec_api_key
+  apikey   = var.ec_api_key
+  endpoint = var.endpoint
 }
 
 module "ec_deployment" {

deploy/weekly-environment/scripts/benchmarks/kspm_vanilla/data/package_policy_vanilla.json

@@ -15,50 +15,24 @@
                     "data_stream": {
                         "type": "logs",
                         "dataset": "cloud_security_posture.findings"
-                    },
-                    "release": "ga"
-                }
-            ]
-        },
-        {
-            "type": "cloudbeat/cis_eks",
-            "policy_template": "kspm",
-            "enabled": false,
-            "streams": [
-                {
-                    "enabled": false,
-                    "data_stream": {
-                        "type": "logs",
-                        "dataset": "cloud_security_posture.findings"
-                    },
-                    "release": "ga",
-                    "vars": {
-                        "access_key_id": {
-                            "type": "text"
-                        },
-                        "secret_access_key": {
-                            "type": "text"
-                        },
-                        "session_token": {
-                            "type": "text"
-                        },
-                        "shared_credential_file": {
-                            "type": "text"
-                        },
-                        "credential_profile_name": {
-                            "type": "text"
-                        },
-                        "role_arn": {
-                            "type": "text"
-                        }
                     }
                 }
             ]
         }
     ],
     "package": {
         "name": "cloud_security_posture",
-        "title": "Kubernetes Security Posture Management (KSPM)",
-        "version": "1.1.1"
+        "title": "Security Posture Management (CSPM/KSPM)",
+        "version": "1.2.10"
+    },
+    "vars": {
+        "posture": {
+            "value": "kspm",
+            "type": "text"
+        },
+        "deployment": {
+            "value": "cloudbeat/cis_k8s",
+            "type": "text"
+        }
     }
 }