From ffb79510be3f82c12a0a46f1e31bc8dfebcb1764 Mon Sep 17 00:00:00 2001 From: rams3sh Date: Thu, 15 Jun 2023 12:20:50 +0530 Subject: [PATCH 1/7] added docker support --- .dockerignore | 6 ++ .github/image_build.yaml | 37 ++++++++++++ .gitignore | 4 +- Dockerfile | 20 +++++++ README.md | 124 +++++++++++++++++++++++++++++++-------- 5 files changed, 164 insertions(+), 27 deletions(-) create mode 100644 .dockerignore create mode 100644 .github/image_build.yaml create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..4383690 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +*.md +.dockerignore +.gitignore +.github/** +.git/** +Dockerfile \ No newline at end of file diff --git a/.github/image_build.yaml b/.github/image_build.yaml new file mode 100644 index 0000000..dd3883b --- /dev/null +++ b/.github/image_build.yaml @@ -0,0 +1,37 @@ +name: Log test image builder + +on: + + push: + paths-ignore: + - '*.md' + - '.gitignore' + branches: + - main + +jobs: + logtest-image-build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Build Container Image + id: container_image_build + run: | + docker build -t logtest . + + - name: Push image to Github Registry + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + CONTAINER_IMAGE_NAME="logtest" + # As of now latest image tag will be used + IMAGE_TAG="latest" && \ + GITHUB_CONTAINER_REGISTRY="docker.pkg.github.com" && \ + GH_REPO_ORG=$(echo $GITHUB_REPOSITORY | cut -f1 -d\/) && \ + GH_REPO_NAME=$(echo $GITHUB_REPOSITORY | cut -f2 -d\/) && \ + GITHUB_CONTAINER_REPOSITORY_NAME="$GH_REPO_ORG/$GH_REPO_NAME/$CONTAINER_IMAGE_NAME" && \ + echo $GITHUB_TOKEN | docker login -u $GITHUB_ACTOR --password-stdin $GITHUB_CONTAINER_REGISTRY && \ + docker tag $CONTAINER_IMAGE_NAME $GITHUB_CONTAINER_REGISTRY/$GITHUB_CONTAINER_REPOSITORY_NAME:$IMAGE_TAG && \ + docker push $GITHUB_CONTAINER_REGISTRY/$GITHUB_CONTAINER_REPOSITORY_NAME:$IMAGE_TAG && \ + docker logout $GITHUB_CONTAINER_REGISTRY \ No newline at end of file diff --git a/.gitignore b/.gitignore index 1f4b6b6..4872e6f 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,6 @@ vrl-web .DS_Store log_source_generated.yml.go **/*log_source_generated.yml.go -.vscode \ No newline at end of file +.vscode +env +.vscode/** \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ca457a5 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,20 @@ +FROM python:3.11-bullseye AS builder +WORKDIR /logtest +RUN apt-get update && apt-get install wget tar -y && \ + wget https://github.com/Wilfred/difftastic/releases/download/0.47.0/difft-x86_64-unknown-linux-gnu.tar.gz && \ + tar -xf difft-x86_64-unknown-linux-gnu.tar.gz && \ + rm difft-x86_64-unknown-linux-gnu.tar.gz && mv difft /usr/bin/ +COPY . /logtest/ +RUN pip3 install -r requirements.txt --target . && \ + rm requirements.txt +# Removing python intermediate bytecodes +RUN find . -regex '^.*\(__pycache__\|\.py[co]\)$' -delete + + +FROM cgr.dev/chainguard/python:latest +USER 0 +WORKDIR /logtest +COPY --from=builder /logtest /logtest +COPY --from=builder /usr/bin/difft /usr/bin/difft +#RUN mv difft /usr/bin/ +ENTRYPOINT ["python3", "-u", "main.py" ] diff --git a/README.md b/README.md index ce80329..e40ba07 100644 --- a/README.md +++ b/README.md @@ -4,37 +4,92 @@ ## Installation #### Prerequisites +--- +##### For usage from Source + - **1) Install [difftastic](https://difftastic.wilfred.me.uk/) (`difft` cli)** + ###### macOS / Linux + ``` + brew install difftastic + ``` + **Other:** See other installation options for difftastic [here](https://difftastic.wilfred.me.uk/installation.html). + + + - **2) Clone / start the local VRL webserver (for testing)** + ###### Clone + ``` + git clone https://github.com/shaeqahmed/vrl-web.git + ``` + ###### Start the server and keep it running (in a new terminal tab) + ``` + cd vrl-web/vrl-web-server-warp + cargo run --release + ``` + + - **3) Install dependencies** + + ``` + python3 -m pip install -r requirements.txt + ``` +--- +##### Using docker + + - **1) Clone / start the local VRL webserver (for testing)** + + ###### Clone + ``` + git clone https://github.com/shaeqahmed/vrl-web.git + ``` + ###### Start the server and keep it running (in a new terminal tab) + ``` + cd vrl-web/vrl-web-server-warp + cargo run --release + ``` + + - **2) Ensure docker is installed in your system.** + + Refer installation steps [here](https://docs.docker.com/engine/install/) -- **1) Install [difftastic](https://difftastic.wilfred.me.uk/) (`difft` cli)** - ##### macOS / Linux - ``` - brew install difftastic - ``` - **Other:** See other installation options for difftastic [here](https://difftastic.wilfred.me.uk/installation.html). - +## Usage -- **2) Clone / start the local VRL webserver (for testing)** - ##### Clone - ``` - git clone https://github.com/shaeqahmed/vrl-web.git - ``` - ##### Start the server and keep it running (in a new terminal tab) - ``` - cd vrl-web/vrl-web-server-warp - cargo run --release - ``` +### Example +--- +#### Usage from source -#### Install dependencies +##### Directory structure -``` -python3 -m pip install -r requirements.txt +```python3 +examples (an example directory) +└── aws_vpcflow # (create a folder with the logsource name) + ├── fields # (place files containing ECS / custom fields specified in the Filebeat-style format in this sub directory) + │ ├── agent.yml + │ ├── base-fields.yml + │ ├── ecs.yml + │ └── fields.yml + ├── log_source.yml # (the edited log source file) + ├── log_source_generated.yml.go # (the generated log source file, use this to update^) + └── test # (directory containing test + expected files, also according to the Filebeat style (e.g. '-expected.json' for assertions) + ├── test-extra-samples.log + ├── test-extra-samples.log-expected.json + ├── test-tcp-flag-sequence.log + ├── test-tcp-flag-sequence.log-expected.json + ├── test-v5-all-fields.log + ├── test-v5-all-fields.log-expected.json + ├── test-with-message-field.log + └── test-with-message-field.log-expected.json ``` -## Usage +##### Running the CLI -### Example +```bash +python3 main.py --logsource-dir examples/minimal # simple working example -### Directory sturcture +python3 main.py --logsource-dir examples/aws_vpcflow # example for devloping a new log source for AWS VPC Flow logs using an existing test case suite / schema +``` + +--- +#### Usage with docker + +Just like the previous section, you can clone this repo and have the examples folder in your path. ```python3 examples (an example directory) @@ -56,17 +111,34 @@ examples (an example directory) ├── test-with-message-field.log └── test-with-message-field.log-expected.json ``` +##### Running the CLI -### Running the CLI +Set the alias of logtest with below commmand. ```bash -python3 main.py --logsource-dir examples/minimal # simple working example +alias logtest=`docker run --network=host -v `pwd`:/logtest/test/ -it +``` -python3 main.py --logsource-dir examples/aws_vpcflow # example for devloping a new log source for AWS VPC Flow logs using an existing test case suite / schema + +The above alias mounts the current directory from where it is run within the logtest's container. + +Once the alias is set, you may use the cli as given below :- + +```bash +logtest --logsource-dir test/examples/minimal # simple working example + +logtest --logsource-dir test/examples/aws_vpcflow # example for devloping a new log source for AWS VPC Flow logs using an existing test case suite / schema ``` +Note: + Prefix `test` before using the relative paths in your current directory. It is so because the current directory is mounted within test folder relative to the logtest's folder. + +--- + Running this command will help you create a valid `log_source.yml` file for a log source name `aws_vpcflow`. It will use the fields (schema) and tests you have provided in the `fields/` and `test/` subdirectories respectively. + + For inspiration on example test cases, fields to start with, and how to structure ECS compatible tables, you can reference the approach from Filebeat / Elastic e.g.: ##### Log source https://github.com/elastic/integrations/tree/main/packages/aws/data_stream/vpcflow From 3d07dd5fc484f05bf492e37354963e3c5eb8dd3c Mon Sep 17 00:00:00 2001 From: rams3sh Date: Thu, 15 Jun 2023 12:22:22 +0530 Subject: [PATCH 2/7] triggering workflow --- .github/image_build.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/image_build.yaml b/.github/image_build.yaml index dd3883b..a1ba4e5 100644 --- a/.github/image_build.yaml +++ b/.github/image_build.yaml @@ -1,7 +1,6 @@ name: Log test image builder on: - push: paths-ignore: - '*.md' From 38acb9ddb94b917629ad7a03e3416193258ab962 Mon Sep 17 00:00:00 2001 From: rams3sh Date: Thu, 15 Jun 2023 12:28:03 +0530 Subject: [PATCH 3/7] triggering workflow --- .github/image_build.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/image_build.yaml b/.github/image_build.yaml index a1ba4e5..9d3976c 100644 --- a/.github/image_build.yaml +++ b/.github/image_build.yaml @@ -1,24 +1,24 @@ -name: Log test image builder +name: Logtest image builder on: + workflow-dispatch: + push: paths-ignore: - '*.md' - '.gitignore' branches: - main - + jobs: logtest-image-build: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Build Container Image id: container_image_build run: | docker build -t logtest . - - name: Push image to Github Registry env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 541d0c9d60729f8e78985c2285e12e7f1c1594d9 Mon Sep 17 00:00:00 2001 From: rams3sh Date: Thu, 15 Jun 2023 12:29:34 +0530 Subject: [PATCH 4/7] triggering workflow --- .github/{ => workflows}/image_build.yaml | 2 -- 1 file changed, 2 deletions(-) rename .github/{ => workflows}/image_build.yaml (98%) diff --git a/.github/image_build.yaml b/.github/workflows/image_build.yaml similarity index 98% rename from .github/image_build.yaml rename to .github/workflows/image_build.yaml index 9d3976c..5356968 100644 --- a/.github/image_build.yaml +++ b/.github/workflows/image_build.yaml @@ -1,8 +1,6 @@ name: Logtest image builder on: - workflow-dispatch: - push: paths-ignore: - '*.md' From d7ed430ea9c9426c7736cde7553c9d9fd60de2d8 Mon Sep 17 00:00:00 2001 From: rams3sh Date: Thu, 15 Jun 2023 12:33:04 +0530 Subject: [PATCH 5/7] triggering workflow --- .github/workflows/image_build.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/image_build.yaml b/.github/workflows/image_build.yaml index 5356968..689bd81 100644 --- a/.github/workflows/image_build.yaml +++ b/.github/workflows/image_build.yaml @@ -7,7 +7,11 @@ on: - '.gitignore' branches: - main - + +permissions: + contents: read + packages: write + jobs: logtest-image-build: runs-on: ubuntu-latest From 83dfce5598d14eae7962ad84263598957beabe24 Mon Sep 17 00:00:00 2001 From: rams3sh Date: Thu, 15 Jun 2023 12:41:43 +0530 Subject: [PATCH 6/7] added workflow dispatch in github actions --- .github/workflows/image_build.yaml | 3 +++ README.md | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/image_build.yaml b/.github/workflows/image_build.yaml index 689bd81..5cb6b3f 100644 --- a/.github/workflows/image_build.yaml +++ b/.github/workflows/image_build.yaml @@ -1,6 +1,9 @@ name: Logtest image builder on: + # Just in case to upgrade chainguard base image + workflow_dispatch: + push: paths-ignore: - '*.md' diff --git a/README.md b/README.md index e40ba07..73713a0 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,7 @@ examples (an example directory) Set the alias of logtest with below commmand. ```bash -alias logtest=`docker run --network=host -v `pwd`:/logtest/test/ -it +alias logtest=`docker run --network=host -v `pwd`:/logtest/test/ -it ``` @@ -131,7 +131,7 @@ logtest --logsource-dir test/examples/aws_vpcflow # example for devloping a new ``` Note: - Prefix `test` before using the relative paths in your current directory. It is so because the current directory is mounted within test folder relative to the logtest's folder. + Prefix `test` before using the relative paths in your current directory. It is so because the current directory is mounted within `test` folder relative to the logtest's folder. --- From 8fb0615fd17176cd9aa162bd9a387ec2dbfe1c58 Mon Sep 17 00:00:00 2001 From: rams3sh Date: Thu, 15 Jun 2023 13:03:57 +0530 Subject: [PATCH 7/7] removed commented code --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ca457a5..b816603 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,5 +16,4 @@ USER 0 WORKDIR /logtest COPY --from=builder /logtest /logtest COPY --from=builder /usr/bin/difft /usr/bin/difft -#RUN mv difft /usr/bin/ ENTRYPOINT ["python3", "-u", "main.py" ]