From cf9f4892fc10dbc4100fa57cbe72f8d09e510ca8 Mon Sep 17 00:00:00 2001 From: Brendan Shaklovitz Date: Wed, 7 Oct 2020 14:28:24 -0500 Subject: [PATCH 1/5] Use YAML anchors in docker-compose example --- docker-compose.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 53efcd108..5c3d6e1cc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,3 +1,4 @@ +--- version: '2' services: mysql: @@ -5,7 +6,7 @@ services: volumes: - .:/tmp command: mysqld --datadir=/tmp/mysqldata --slow_query_log=1 --log_output=TABLE --log-queries-not-using-indexes --event-scheduler=ON - environment: + environment: &mysql-default-environment MYSQL_ROOT_PASSWORD: toor MYSQL_DATABASE: kolide MYSQL_USER: kolide @@ -17,11 +18,7 @@ services: image: mysql:5.7 command: mysqld --datadir=/tmpfs --slow_query_log=1 --log_output=TABLE --log-queries-not-using-indexes --event-scheduler=ON tmpfs: /tmpfs - environment: - MYSQL_ROOT_PASSWORD: toor - MYSQL_DATABASE: kolide - MYSQL_USER: kolide - MYSQL_PASSWORD: kolide + environment: *mysql-default-environment ports: - "3307:3306" From 9fcacae4bd1ea915a43510642473700358d90b52 Mon Sep 17 00:00:00 2001 From: Brendan Shaklovitz Date: Wed, 7 Oct 2020 14:30:34 -0500 Subject: [PATCH 2/5] Use YAML anchors for osquery docker-compose --- tools/osquery/docker-compose.yml | 54 +++++++++++--------------------- 1 file changed, 18 insertions(+), 36 deletions(-) diff --git a/tools/osquery/docker-compose.yml b/tools/osquery/docker-compose.yml index 72e1ad01a..06c919e5e 100644 --- a/tools/osquery/docker-compose.yml +++ b/tools/osquery/docker-compose.yml @@ -1,55 +1,37 @@ - +--- version: '2' services: ubuntu14-osquery: image: "kolide/osquery:${KOLIDE_OSQUERY_VERSION}" - volumes: + volumes: &default-volumes - ./kolide.crt:/etc/osquery/kolide.crt - ./example_osquery.flags:/etc/osquery/osquery.flags - environment: - ENROLL_SECRET: "${ENROLL_SECRET}" - command: osqueryd --flagfile=/etc/osquery/osquery.flags - ulimits: + environment: &default-environment + ENROLL_SECRET: "${ENROLL_SECRET:?ENROLL_SECRET must be set for server authentication}" + command: &default-command osqueryd --flagfile=/etc/osquery/osquery.flags + ulimits: &default-ulimits core: hard: 1000000000 soft: 1000000000 ubuntu16-osquery: image: "kolide/ubuntu16-osquery:${KOLIDE_OSQUERY_VERSION}" - volumes: - - ./kolide.crt:/etc/osquery/kolide.crt - - ./example_osquery.flags:/etc/osquery/osquery.flags - environment: - ENROLL_SECRET: "${ENROLL_SECRET}" - command: osqueryd --flagfile=/etc/osquery/osquery.flags - ulimits: - core: - hard: 1000000000 - soft: 1000000000 + volumes: *default-volumes + environment: *default-environment + command: *default-command + ulimits: *default-ulimits centos7-osquery: image: "kolide/centos7-osquery:${KOLIDE_OSQUERY_VERSION}" - volumes: - - ./kolide.crt:/etc/osquery/kolide.crt - - ./example_osquery.flags:/etc/osquery/osquery.flags - environment: - ENROLL_SECRET: "${ENROLL_SECRET}" - command: osqueryd --flagfile=/etc/osquery/osquery.flags - ulimits: - core: - hard: 1000000000 - soft: 1000000000 + volumes: *default-volumes + environment: *default-environment + command: *default-command + ulimits: *default-ulimits centos6-osquery: image: "kolide/centos6-osquery:${KOLIDE_OSQUERY_VERSION}" - volumes: - - ./kolide.crt:/etc/osquery/kolide.crt - - ./example_osquery.flags:/etc/osquery/osquery.flags - environment: - ENROLL_SECRET: "${ENROLL_SECRET}" - command: osqueryd --flagfile=/etc/osquery/osquery.flags - ulimits: - core: - hard: 1000000000 - soft: 1000000000 + volumes: *default-volumes + environment: *default-environment + command: *default-command + ulimits: *default-ulimits From eeea84e5b7d684fd7fa0b6cd0b37b04cb10d94bf Mon Sep 17 00:00:00 2001 From: Brendan Shaklovitz Date: Wed, 7 Oct 2020 14:32:38 -0500 Subject: [PATCH 3/5] Support alternate fleet addresses * Allow users to specify addresses other than `host.docker.internal` for the `tls_hostname` in the example config. --- tools/osquery/README.md | 3 +++ tools/osquery/docker-compose.yml | 26 ++++++++++++++++---------- tools/osquery/example_osquery.flags | 1 - 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/tools/osquery/README.md b/tools/osquery/README.md index 3977ec5f3..e7c5f59df 100644 --- a/tools/osquery/README.md +++ b/tools/osquery/README.md @@ -25,6 +25,9 @@ Set the environment variable `ENROLL_SECRET` to the value of your Fleet enroll s (Optionally) Set `KOLIDE_OSQUERY_VERSION` if you want to run an osquery container besides `latest`. +(Optionally) Set `FLEET_SERVER` if you want to connect to a fleet server +besides `host.docker.internal:8080`. + ### Running osqueryd The osqueryd instances are configured to use the TLS plugins at `host.docker.internal:8080`. Using the `example_osquery.flags` in this directory should configure Fleet with the appropriate settings for these `osqueryd` containers to connect. diff --git a/tools/osquery/docker-compose.yml b/tools/osquery/docker-compose.yml index 06c919e5e..61c84fd44 100644 --- a/tools/osquery/docker-compose.yml +++ b/tools/osquery/docker-compose.yml @@ -1,19 +1,25 @@ --- version: '2' +x-default-settings: + volumes: &default-volumes + - ./kolide.crt:/etc/osquery/kolide.crt + - ./example_osquery.flags:/etc/osquery/osquery.flags + environment: &default-environment + ENROLL_SECRET: "${ENROLL_SECRET:?ENROLL_SECRET must be set for server authentication}" + command: &default-command osqueryd --flagfile=/etc/osquery/osquery.flags --tls_hostname=${FLEET_SERVER:-host.docker.internal:8080} + ulimits: &default-ulimits + core: + hard: 1000000000 + soft: 1000000000 + services: ubuntu14-osquery: image: "kolide/osquery:${KOLIDE_OSQUERY_VERSION}" - volumes: &default-volumes - - ./kolide.crt:/etc/osquery/kolide.crt - - ./example_osquery.flags:/etc/osquery/osquery.flags - environment: &default-environment - ENROLL_SECRET: "${ENROLL_SECRET:?ENROLL_SECRET must be set for server authentication}" - command: &default-command osqueryd --flagfile=/etc/osquery/osquery.flags - ulimits: &default-ulimits - core: - hard: 1000000000 - soft: 1000000000 + volumes: *default-volumes + environment: *default-environment + command: *default-command + ulimits: *default-ulimits ubuntu16-osquery: image: "kolide/ubuntu16-osquery:${KOLIDE_OSQUERY_VERSION}" diff --git a/tools/osquery/example_osquery.flags b/tools/osquery/example_osquery.flags index 9fe2b02c1..9f868c7ac 100644 --- a/tools/osquery/example_osquery.flags +++ b/tools/osquery/example_osquery.flags @@ -4,7 +4,6 @@ --debug --tls_dump=true ---tls_hostname=host.docker.internal:8080 --tls_server_certs=/etc/osquery/kolide.crt --enroll_secret_env=ENROLL_SECRET From 9e9898b2b4383ef0b9eb50a60e6df07629353738 Mon Sep 17 00:00:00 2001 From: Brendan Shaklovitz Date: Wed, 7 Oct 2020 14:36:24 -0500 Subject: [PATCH 4/5] Use persistent volume for mysql container * Using the current working directory can result in permissions issues if the UID / GID of the host don't match that of the mysql user in the container. Using a docker volume allows us to persist the data and avoid these permissions issues. --- docker-compose.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5c3d6e1cc..9b61c698a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,7 @@ services: mysql: image: mysql:5.7 volumes: - - .:/tmp + - mysql-persistent-volume:/tmp command: mysqld --datadir=/tmp/mysqldata --slow_query_log=1 --log_output=TABLE --log-queries-not-using-indexes --event-scheduler=ON environment: &mysql-default-environment MYSQL_ROOT_PASSWORD: toor @@ -44,3 +44,6 @@ services: - /var/run:/var/run:rw - /sys:/sys:ro - /var/lib/docker/:/var/lib/docker:ro + +volumes: + mysql-persistent-volume: From 254b09b3d50c2d0becabd0c39b32163e6041a286 Mon Sep 17 00:00:00 2001 From: Brendan Shaklovitz Date: Wed, 7 Oct 2020 17:51:35 -0500 Subject: [PATCH 5/5] Support adding host.docker.internal dns on Linux * Fixes #2319 by allowing Linux users to add `extra_hosts` settings, which create an `/etc/hosts` entry for `host.docker.internal`. --- tools/osquery/README.md | 10 ++++++++- .../docker-compose.linux-overrides.yml | 21 +++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 tools/osquery/docker-compose.linux-overrides.yml diff --git a/tools/osquery/README.md b/tools/osquery/README.md index e7c5f59df..db870b5f0 100644 --- a/tools/osquery/README.md +++ b/tools/osquery/README.md @@ -32,12 +32,20 @@ besides `host.docker.internal:8080`. The osqueryd instances are configured to use the TLS plugins at `host.docker.internal:8080`. Using the `example_osquery.flags` in this directory should configure Fleet with the appropriate settings for these `osqueryd` containers to connect. -To start one instance each of Centos and Ubuntu `osqueryd`, use: +To start one instance each of Centos 6, Centos 7, Ubuntu 14, and Ubuntu 16 +`osqueryd`, use: ``` docker-compose up ``` +Linux users should use the overrides (which add DNS entries for +`host.docker.internal` based on the `DOCKER_HOST` env var): + +``` +docker-compose -f docker-compose.yml -f docker-compose.linux-overrides.yml up +``` + The logs will be displayed on the host shell. Note that `docker-compose up` will reuse containers (so the state of `osqueryd` will be maintained across calls). To remove the containers and start from a fresh state on the next call to `up`, use: ``` diff --git a/tools/osquery/docker-compose.linux-overrides.yml b/tools/osquery/docker-compose.linux-overrides.yml new file mode 100644 index 000000000..852672aee --- /dev/null +++ b/tools/osquery/docker-compose.linux-overrides.yml @@ -0,0 +1,21 @@ +--- +version: '2' + +x-default-settings: + extra_hosts: &linux-extra-hosts + # Add host.docker.internal record to /etc/hosts of the containers. This is + # added on Docker for Mac by default, but needs to be added by Linux users. + - "host.docker.internal:${DOCKER_HOST:-172.17.0.1}" + +services: + ubuntu14-osquery: + extra_hosts: *linux-extra-hosts + + ubuntu16-osquery: + extra_hosts: *linux-extra-hosts + + centos7-osquery: + extra_hosts: *linux-extra-hosts + + centos6-osquery: + extra_hosts: *linux-extra-hosts