From 12002aa36138c49a897c57bbe28082f63d483940 Mon Sep 17 00:00:00 2001 From: Christian Peron Date: Thu, 28 Nov 2019 13:07:37 -0600 Subject: [PATCH 1/8] Add TLS support to the docker container - Introduce two options --port and --protocol which will allow users to override the YAML specification and test using TLS (or on alternate ports) - Properly handle exceptions when we are expecting gzip'ed data but actually get clear text (or invalid compressed data) --- docker/docker_entry.sh | 10 +++++++++- ftw/http.py | 10 +++++++++- ftw/testrunner.py | 6 +++++- tools/build_journal.py | 12 +++++++++--- 4 files changed, 32 insertions(+), 6 deletions(-) diff --git a/docker/docker_entry.sh b/docker/docker_entry.sh index 6a99e70..e41748d 100644 --- a/docker/docker_entry.sh +++ b/docker/docker_entry.sh @@ -5,7 +5,7 @@ destaddr="127.0.0.1" ruledir=/CRS/tests cmd_args="--ruledir_recurse " -while getopts "Dd:f:F" opt; do +while getopts "Dd:f:FP:p:" opt; do case $opt in F) cmd_args="$cmd_args --destaddr_as_host " @@ -29,6 +29,14 @@ while getopts "Dd:f:F" opt; do destaddr=$OPTARG cmd_args="$cmd_args --destaddr $destaddr " ;; + p) + port=$OPTARG + cmd_args="$cmd_args --port $port" + ;; + P) + proto=$OPTARG + cmd_args="$cmd_args --protocol $proto" + ;; esac done diff --git a/ftw/http.py b/ftw/http.py index 34aa7f2..07c481f 100644 --- a/ftw/http.py +++ b/ftw/http.py @@ -46,7 +46,15 @@ def parse_content_encoding(self, response_headers, response_data): if response_headers['content-encoding'] == 'gzip': buf = StringIO.StringIO(response_data) zipbuf = gzip.GzipFile(fileobj=buf) - response_data = zipbuf.read() + try: + response_data = zipbuf.read() + except IOError: + raise errors.TestError( + 'Content encoding gzip but no compressed data', + { + 'response_data': str(response_data), + 'function': 'http.HttpResponse.parse_content_encoding' + }) elif response_headers['content-encoding'] == 'deflate': data = StringIO.StringIO(zlib.decompress(response_data)) response_data = data.read() diff --git a/ftw/testrunner.py b/ftw/testrunner.py index 0ddf4bf..5dc7c93 100644 --- a/ftw/testrunner.py +++ b/ftw/testrunner.py @@ -122,7 +122,7 @@ def run_stage_with_journal(self, rule_id, test, journal_file, tablename, logger_ if stage.output.status: self.test_status(stage.output.status, status) - def run_test_build_journal(self, rule_id, test, journal_file, tablename, destaddr, callback, headers = {}): + def run_test_build_journal(self, rule_id, test, journal_file, tablename, destaddr, callback, proto, port, headers = {}): """ Build journal entries from a test within a specified rule_id Pass in the rule_id, test object, and path to journal_file @@ -139,6 +139,10 @@ def run_test_build_journal(self, rule_id, test, journal_file, tablename, destadd callback(test, rule_id) if destaddr is not None: stage.input.dest_addr = destaddr + if proto: + stage.input.protocol = proto + if port: + stage.input.port = port ''' Merge in/override the headers that were passed in by the caller. diff --git a/tools/build_journal.py b/tools/build_journal.py index ad6cc39..cf8d8fb 100644 --- a/tools/build_journal.py +++ b/tools/build_journal.py @@ -4,13 +4,13 @@ def diag_print(test, rule_id): print 'Running test %s from rule file %s' % (test.test_title, rule_id) -def build_journal(journal_file, ruledir, ruledir_recurse, tablename, destaddr, headers): +def build_journal(journal_file, ruledir, ruledir_recurse, tablename, destaddr, headers, protocol, port): util.instantiate_database(journal_file) rulesets = util.get_rulesets(ruledir, ruledir_recurse) for rule in rulesets: for test in rule.tests: runner = testrunner.TestRunner() - runner.run_test_build_journal(test.ruleset_meta['name'], test, journal_file, tablename, destaddr, diag_print, headers) + runner.run_test_build_journal(test.ruleset_meta['name'], test, journal_file, tablename, destaddr, diag_print, protocol, port, headers) def main(): parser = argparse.ArgumentParser(description='Build FTW Journal database') @@ -26,6 +26,10 @@ def main(): help='Destination host for the payloads') parser.add_argument('--destaddr_as_host', action='store_true', help='Use destination address as the Host header') + parser.add_argument('--protocol', default=None, + help='Specify protocol: http or https (default http)') + parser.add_argument('--port', default=None, + help='Specify port number (default 80)') args = parser.parse_args() destaddr = args.destaddr journal_file = args.journal @@ -33,9 +37,11 @@ def main(): ruledir_recurse = args.ruledir_recurse tablename = args.tablename headers = {} + protocol = args.protocol + port = int(args.port) if args.destaddr_as_host: headers['Host'] = destaddr = args.destaddr - build_journal(journal_file, ruledir, ruledir_recurse, tablename, destaddr, headers) + build_journal(journal_file, ruledir, ruledir_recurse, tablename, destaddr, headers, protocol, port) if __name__ == '__main__': main() From baa4fec43394958199a766c3e7f152cc4bbbc726 Mon Sep 17 00:00:00 2001 From: Christian Peron Date: Thu, 28 Nov 2019 14:09:08 -0600 Subject: [PATCH 2/8] Fix test fixture - The content of example.com changed, the string be searched for is no longer there. Fix this test to identify a string that is present as of this commit --- test/integration/test_htmlcontains.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/test_htmlcontains.py b/test/integration/test_htmlcontains.py index 1b2b1e8..bca021b 100644 --- a/test/integration/test_htmlcontains.py +++ b/test/integration/test_htmlcontains.py @@ -32,7 +32,7 @@ def test_search3(): x = ruleset.Input(dest_addr="example.com",headers={"Host":"example.com"}) http_ua = http.HttpUA() http_ua.send_request(x) - runner.test_response(http_ua.response_object,re.compile('established to be used for')) + runner.test_response(http_ua.response_object,re.compile('for use in illustrative examples in documents')) # Should return a success because we found our regex def test_search4(): From 7f167da8273e0e6dfda265d6ba85ed9384f80730 Mon Sep 17 00:00:00 2001 From: Christian Peron Date: Thu, 28 Nov 2019 14:15:27 -0600 Subject: [PATCH 3/8] Update YAML fixture to reflect unit test fix --- test/integration/HTMLCONTAINSFIXTURE.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/HTMLCONTAINSFIXTURE.yaml b/test/integration/HTMLCONTAINSFIXTURE.yaml index f60c510..6a8dac5 100644 --- a/test/integration/HTMLCONTAINSFIXTURE.yaml +++ b/test/integration/HTMLCONTAINSFIXTURE.yaml @@ -21,7 +21,7 @@ uri: "/" output: status: 200 - response_contains: "established to be used for" + response_contains: "for use in illustrative examples in documents" - test_title: "response_contains(2)" stages: From bfbd78136bbb2035a410cd5ea5985897f3d1b66d Mon Sep 17 00:00:00 2001 From: Christian Peron Date: Thu, 28 Nov 2019 16:31:21 -0600 Subject: [PATCH 4/8] Fix conditional initialization of port --- ftw/testrunner.py | 2 +- tools/build_journal.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ftw/testrunner.py b/ftw/testrunner.py index 5dc7c93..87fc7e1 100644 --- a/ftw/testrunner.py +++ b/ftw/testrunner.py @@ -141,7 +141,7 @@ def run_test_build_journal(self, rule_id, test, journal_file, tablename, destadd stage.input.dest_addr = destaddr if proto: stage.input.protocol = proto - if port: + if port != 0: stage.input.port = port ''' Merge in/override the headers that were passed in by diff --git a/tools/build_journal.py b/tools/build_journal.py index cf8d8fb..4e19f5a 100644 --- a/tools/build_journal.py +++ b/tools/build_journal.py @@ -38,7 +38,9 @@ def main(): tablename = args.tablename headers = {} protocol = args.protocol - port = int(args.port) + port = 0 + if args.port: + port = int(args.port) if args.destaddr_as_host: headers['Host'] = destaddr = args.destaddr build_journal(journal_file, ruledir, ruledir_recurse, tablename, destaddr, headers, protocol, port) From 8fa7138c140ef2bd63b181331e13e5e24251b1d3 Mon Sep 17 00:00:00 2001 From: Christian Peron Date: Thu, 28 Nov 2019 16:40:35 -0600 Subject: [PATCH 5/8] Add information about -P and -p --- docs/Docker.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/Docker.md b/docs/Docker.md index eec3045..c7146c9 100644 --- a/docs/Docker.md +++ b/docs/Docker.md @@ -28,3 +28,11 @@ If you are testing through the CDN, you can use `-F` to use the target specifica ``` % docker run -i ftw-test -F -d -f - < mytest.yaml ``` + +## Connecting using TLS + +If you wan't to connect using TLS, you need to change the port and protocol: + +``` + % docker run -i ftw-test -P https -p 443 -F -d -f - < mytest.yaml +``` From 5c9726554a81cd5d7e1f4956ec6abfd19fa187a4 Mon Sep 17 00:00:00 2001 From: Christian Peron Date: Fri, 29 Nov 2019 11:25:06 -0600 Subject: [PATCH 6/8] Do not translate escape characters - When loading files from the host environment, they could contain escaped characters. The backslashes from the YAML need to be preserved --- docker/docker_entry.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/docker_entry.sh b/docker/docker_entry.sh index e41748d..c09a83f 100644 --- a/docker/docker_entry.sh +++ b/docker/docker_entry.sh @@ -15,8 +15,8 @@ while getopts "Dd:f:FP:p:" opt; do ruledir=$OPTARG else T=`mktemp -d -t rules.XXXXXX` - while IFS= read LINE; do - echo "$LINE" >> $T/rules.yaml + while IFS= read -r LINE; do + echo -E "$LINE" >> /$T/rules.yaml done ruledir=$T fi From 1e799ed7d68c621f21ac5c8bdb0b44185ae76280 Mon Sep 17 00:00:00 2001 From: Christian Peron Date: Fri, 29 Nov 2019 11:28:07 -0600 Subject: [PATCH 7/8] Remove leading forward slash --- docker/docker_entry.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker_entry.sh b/docker/docker_entry.sh index c09a83f..ca5b540 100644 --- a/docker/docker_entry.sh +++ b/docker/docker_entry.sh @@ -16,7 +16,7 @@ while getopts "Dd:f:FP:p:" opt; do else T=`mktemp -d -t rules.XXXXXX` while IFS= read -r LINE; do - echo -E "$LINE" >> /$T/rules.yaml + echo -E "$LINE" >> $T/rules.yaml done ruledir=$T fi From 0cc16707bae9b3d37caa6e41fd49b087fa22f8fd Mon Sep 17 00:00:00 2001 From: Christian Peron Date: Mon, 2 Dec 2019 10:22:11 -0600 Subject: [PATCH 8/8] Fix typo --- docs/Docker.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Docker.md b/docs/Docker.md index c7146c9..5aaf52a 100644 --- a/docs/Docker.md +++ b/docs/Docker.md @@ -31,7 +31,7 @@ If you are testing through the CDN, you can use `-F` to use the target specifica ## Connecting using TLS -If you wan't to connect using TLS, you need to change the port and protocol: +If you want to connect using TLS, you need to change the port and protocol: ``` % docker run -i ftw-test -P https -p 443 -F -d -f - < mytest.yaml