diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..4383690 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +*.md +.dockerignore +.gitignore +.github/** +.git/** +Dockerfile \ No newline at end of file diff --git a/.github/workflows/image_build.yaml b/.github/workflows/image_build.yaml new file mode 100644 index 0000000..5cb6b3f --- /dev/null +++ b/.github/workflows/image_build.yaml @@ -0,0 +1,41 @@ +name: Logtest image builder + +on: + # Just in case to upgrade chainguard base image + workflow_dispatch: + + push: + paths-ignore: + - '*.md' + - '.gitignore' + branches: + - main + +permissions: + contents: read + packages: write + +jobs: + logtest-image-build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Build Container Image + id: container_image_build + run: | + docker build -t logtest . + - name: Push image to Github Registry + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + CONTAINER_IMAGE_NAME="logtest" + # As of now latest image tag will be used + IMAGE_TAG="latest" && \ + GITHUB_CONTAINER_REGISTRY="docker.pkg.github.com" && \ + GH_REPO_ORG=$(echo $GITHUB_REPOSITORY | cut -f1 -d\/) && \ + GH_REPO_NAME=$(echo $GITHUB_REPOSITORY | cut -f2 -d\/) && \ + GITHUB_CONTAINER_REPOSITORY_NAME="$GH_REPO_ORG/$GH_REPO_NAME/$CONTAINER_IMAGE_NAME" && \ + echo $GITHUB_TOKEN | docker login -u $GITHUB_ACTOR --password-stdin $GITHUB_CONTAINER_REGISTRY && \ + docker tag $CONTAINER_IMAGE_NAME $GITHUB_CONTAINER_REGISTRY/$GITHUB_CONTAINER_REPOSITORY_NAME:$IMAGE_TAG && \ + docker push $GITHUB_CONTAINER_REGISTRY/$GITHUB_CONTAINER_REPOSITORY_NAME:$IMAGE_TAG && \ + docker logout $GITHUB_CONTAINER_REGISTRY \ No newline at end of file diff --git a/.gitignore b/.gitignore index 1f4b6b6..4872e6f 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,6 @@ vrl-web .DS_Store log_source_generated.yml.go **/*log_source_generated.yml.go -.vscode \ No newline at end of file +.vscode +env +.vscode/** \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b816603 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,19 @@ +FROM python:3.11-bullseye AS builder +WORKDIR /logtest +RUN apt-get update && apt-get install wget tar -y && \ + wget https://github.com/Wilfred/difftastic/releases/download/0.47.0/difft-x86_64-unknown-linux-gnu.tar.gz && \ + tar -xf difft-x86_64-unknown-linux-gnu.tar.gz && \ + rm difft-x86_64-unknown-linux-gnu.tar.gz && mv difft /usr/bin/ +COPY . /logtest/ +RUN pip3 install -r requirements.txt --target . && \ + rm requirements.txt +# Removing python intermediate bytecodes +RUN find . -regex '^.*\(__pycache__\|\.py[co]\)$' -delete + + +FROM cgr.dev/chainguard/python:latest +USER 0 +WORKDIR /logtest +COPY --from=builder /logtest /logtest +COPY --from=builder /usr/bin/difft /usr/bin/difft +ENTRYPOINT ["python3", "-u", "main.py" ] diff --git a/README.md b/README.md index ce80329..73713a0 100644 --- a/README.md +++ b/README.md @@ -4,37 +4,92 @@ ## Installation #### Prerequisites +--- +##### For usage from Source + - **1) Install [difftastic](https://difftastic.wilfred.me.uk/) (`difft` cli)** + ###### macOS / Linux + ``` + brew install difftastic + ``` + **Other:** See other installation options for difftastic [here](https://difftastic.wilfred.me.uk/installation.html). + + + - **2) Clone / start the local VRL webserver (for testing)** + ###### Clone + ``` + git clone https://github.com/shaeqahmed/vrl-web.git + ``` + ###### Start the server and keep it running (in a new terminal tab) + ``` + cd vrl-web/vrl-web-server-warp + cargo run --release + ``` + + - **3) Install dependencies** + + ``` + python3 -m pip install -r requirements.txt + ``` +--- +##### Using docker + + - **1) Clone / start the local VRL webserver (for testing)** + + ###### Clone + ``` + git clone https://github.com/shaeqahmed/vrl-web.git + ``` + ###### Start the server and keep it running (in a new terminal tab) + ``` + cd vrl-web/vrl-web-server-warp + cargo run --release + ``` + + - **2) Ensure docker is installed in your system.** + + Refer installation steps [here](https://docs.docker.com/engine/install/) -- **1) Install [difftastic](https://difftastic.wilfred.me.uk/) (`difft` cli)** - ##### macOS / Linux - ``` - brew install difftastic - ``` - **Other:** See other installation options for difftastic [here](https://difftastic.wilfred.me.uk/installation.html). - +## Usage -- **2) Clone / start the local VRL webserver (for testing)** - ##### Clone - ``` - git clone https://github.com/shaeqahmed/vrl-web.git - ``` - ##### Start the server and keep it running (in a new terminal tab) - ``` - cd vrl-web/vrl-web-server-warp - cargo run --release - ``` +### Example +--- +#### Usage from source -#### Install dependencies +##### Directory structure -``` -python3 -m pip install -r requirements.txt +```python3 +examples (an example directory) +└── aws_vpcflow # (create a folder with the logsource name) + ├── fields # (place files containing ECS / custom fields specified in the Filebeat-style format in this sub directory) + │ ├── agent.yml + │ ├── base-fields.yml + │ ├── ecs.yml + │ └── fields.yml + ├── log_source.yml # (the edited log source file) + ├── log_source_generated.yml.go # (the generated log source file, use this to update^) + └── test # (directory containing test + expected files, also according to the Filebeat style (e.g. '-expected.json' for assertions) + ├── test-extra-samples.log + ├── test-extra-samples.log-expected.json + ├── test-tcp-flag-sequence.log + ├── test-tcp-flag-sequence.log-expected.json + ├── test-v5-all-fields.log + ├── test-v5-all-fields.log-expected.json + ├── test-with-message-field.log + └── test-with-message-field.log-expected.json ``` -## Usage +##### Running the CLI -### Example +```bash +python3 main.py --logsource-dir examples/minimal # simple working example -### Directory sturcture +python3 main.py --logsource-dir examples/aws_vpcflow # example for devloping a new log source for AWS VPC Flow logs using an existing test case suite / schema +``` + +--- +#### Usage with docker + +Just like the previous section, you can clone this repo and have the examples folder in your path. ```python3 examples (an example directory) @@ -56,17 +111,34 @@ examples (an example directory) ├── test-with-message-field.log └── test-with-message-field.log-expected.json ``` +##### Running the CLI -### Running the CLI +Set the alias of logtest with below commmand. ```bash -python3 main.py --logsource-dir examples/minimal # simple working example +alias logtest=`docker run --network=host -v `pwd`:/logtest/test/ -it +``` -python3 main.py --logsource-dir examples/aws_vpcflow # example for devloping a new log source for AWS VPC Flow logs using an existing test case suite / schema + +The above alias mounts the current directory from where it is run within the logtest's container. + +Once the alias is set, you may use the cli as given below :- + +```bash +logtest --logsource-dir test/examples/minimal # simple working example + +logtest --logsource-dir test/examples/aws_vpcflow # example for devloping a new log source for AWS VPC Flow logs using an existing test case suite / schema ``` +Note: + Prefix `test` before using the relative paths in your current directory. It is so because the current directory is mounted within `test` folder relative to the logtest's folder. + +--- + Running this command will help you create a valid `log_source.yml` file for a log source name `aws_vpcflow`. It will use the fields (schema) and tests you have provided in the `fields/` and `test/` subdirectories respectively. + + For inspiration on example test cases, fields to start with, and how to structure ECS compatible tables, you can reference the approach from Filebeat / Elastic e.g.: ##### Log source https://github.com/elastic/integrations/tree/main/packages/aws/data_stream/vpcflow