diff --git a/Makefile b/Makefile index e7f6d427..5d212215 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ BINEXT ?= SOLIBNAME = libhttp_parser SOMAJOR = 2 SOMINOR = 9 -SOREV = 2 +SOREV = 4 ifeq (darwin,$(PLATFORM)) SOEXT ?= dylib SONAME ?= $(SOLIBNAME).$(SOMAJOR).$(SOMINOR).$(SOEXT) @@ -133,14 +133,14 @@ tags: http_parser.c http_parser.h test.c install: library $(INSTALL) -D http_parser.h $(DESTDIR)$(INCLUDEDIR)/http_parser.h $(INSTALL) -D $(LIBNAME) $(DESTDIR)$(LIBDIR)/$(LIBNAME) - ln -s $(LIBNAME) $(DESTDIR)$(LIBDIR)/$(SONAME) - ln -s $(LIBNAME) $(DESTDIR)$(LIBDIR)/$(SOLIBNAME).$(SOEXT) + ln -sf $(LIBNAME) $(DESTDIR)$(LIBDIR)/$(SONAME) + ln -sf $(LIBNAME) $(DESTDIR)$(LIBDIR)/$(SOLIBNAME).$(SOEXT) install-strip: library $(INSTALL) -D http_parser.h $(DESTDIR)$(INCLUDEDIR)/http_parser.h $(INSTALL) -D -s $(LIBNAME) $(DESTDIR)$(LIBDIR)/$(LIBNAME) - ln -s $(LIBNAME) $(DESTDIR)$(LIBDIR)/$(SONAME) - ln -s $(LIBNAME) $(DESTDIR)$(LIBDIR)/$(SOLIBNAME).$(SOEXT) + ln -sf $(LIBNAME) $(DESTDIR)$(LIBDIR)/$(SONAME) + ln -sf $(LIBNAME) $(DESTDIR)$(LIBDIR)/$(SOLIBNAME).$(SOEXT) uninstall: rm $(DESTDIR)$(INCLUDEDIR)/http_parser.h diff --git a/README.md b/README.md index b265d717..e38d3a57 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,9 @@ HTTP Parser =========== +http-parser is [**not** actively maintained](https://github.com/nodejs/http-parser/issues/522). +New projects and projects looking to migrate should consider [llhttp](https://github.com/nodejs/llhttp). + [![Build Status](https://api.travis-ci.org/nodejs/http-parser.svg?branch=master)](https://travis-ci.org/nodejs/http-parser) This is a parser for HTTP messages written in C. It parses both requests and diff --git a/fuzzers/fuzz_parser.c b/fuzzers/fuzz_parser.c new file mode 100644 index 00000000..1a8442c9 --- /dev/null +++ b/fuzzers/fuzz_parser.c @@ -0,0 +1,26 @@ +#include +#include +#include +#include "http_parser.h" + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + static const http_parser_settings settings_null = { + .on_message_begin = 0 + , .on_header_field = 0 + ,.on_header_value = 0 + ,.on_url = 0 + ,.on_status = 0 + ,.on_body = 0 + ,.on_headers_complete = 0 + ,.on_message_complete = 0 + ,.on_chunk_header = 0 + ,.on_chunk_complete = 0 + }; + + http_parser parser; + http_parser_init(&parser, HTTP_BOTH); + http_parser_execute(&parser, &settings_null, (char*)data, size); + + return 0; +} diff --git a/fuzzers/fuzz_url.c b/fuzzers/fuzz_url.c new file mode 100644 index 00000000..eca11a2e --- /dev/null +++ b/fuzzers/fuzz_url.c @@ -0,0 +1,14 @@ +#include +#include +#include +#include "http_parser.h" + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + struct http_parser_url u; + http_parser_url_init(&u); + http_parser_parse_url((char*)data, size, 0, &u); + http_parser_parse_url((char*)data, size, 1, &u); + + return 0; +} diff --git a/http_parser.c b/http_parser.c index 48963853..9be003e7 100644 --- a/http_parser.c +++ b/http_parser.c @@ -381,7 +381,10 @@ enum header_states , h_transfer_encoding , h_upgrade + , h_matching_transfer_encoding_token_start , h_matching_transfer_encoding_chunked + , h_matching_transfer_encoding_token + , h_matching_connection_token_start , h_matching_connection_keep_alive , h_matching_connection_close @@ -650,6 +653,8 @@ size_t http_parser_execute (http_parser *parser, const char *status_mark = 0; enum state p_state = (enum state) parser->state; const unsigned int lenient = parser->lenient_http_headers; + const unsigned int allow_chunked_length = parser->allow_chunked_length; + uint32_t nread = parser->nread; /* We're in an error state. Don't bother doing anything. */ @@ -728,6 +733,7 @@ size_t http_parser_execute (http_parser *parser, if (ch == CR || ch == LF) break; parser->flags = 0; + parser->uses_transfer_encoding = 0; parser->content_length = ULLONG_MAX; if (ch == 'H') { @@ -765,6 +771,7 @@ size_t http_parser_execute (http_parser *parser, if (ch == CR || ch == LF) break; parser->flags = 0; + parser->uses_transfer_encoding = 0; parser->content_length = ULLONG_MAX; if (ch == 'H') { @@ -922,6 +929,7 @@ size_t http_parser_execute (http_parser *parser, if (ch == CR || ch == LF) break; parser->flags = 0; + parser->uses_transfer_encoding = 0; parser->content_length = ULLONG_MAX; if (UNLIKELY(!IS_ALPHA(ch))) { @@ -1335,6 +1343,7 @@ size_t http_parser_execute (http_parser *parser, parser->header_state = h_general; } else if (parser->index == sizeof(TRANSFER_ENCODING)-2) { parser->header_state = h_transfer_encoding; + parser->uses_transfer_encoding = 1; } break; @@ -1416,10 +1425,14 @@ size_t http_parser_execute (http_parser *parser, if ('c' == c) { parser->header_state = h_matching_transfer_encoding_chunked; } else { - parser->header_state = h_general; + parser->header_state = h_matching_transfer_encoding_token; } break; + /* Multi-value `Transfer-Encoding` header */ + case h_matching_transfer_encoding_token_start: + break; + case h_content_length: if (UNLIKELY(!IS_NUM(ch))) { SET_ERRNO(HPE_INVALID_CONTENT_LENGTH); @@ -1563,16 +1576,41 @@ size_t http_parser_execute (http_parser *parser, goto error; /* Transfer-Encoding: chunked */ + case h_matching_transfer_encoding_token_start: + /* looking for 'Transfer-Encoding: chunked' */ + if ('c' == c) { + h_state = h_matching_transfer_encoding_chunked; + } else if (STRICT_TOKEN(c)) { + /* TODO(indutny): similar code below does this, but why? + * At the very least it seems to be inconsistent given that + * h_matching_transfer_encoding_token does not check for + * `STRICT_TOKEN` + */ + h_state = h_matching_transfer_encoding_token; + } else if (c == ' ' || c == '\t') { + /* Skip lws */ + } else { + h_state = h_general; + } + break; + case h_matching_transfer_encoding_chunked: parser->index++; if (parser->index > sizeof(CHUNKED)-1 || c != CHUNKED[parser->index]) { - h_state = h_general; + h_state = h_matching_transfer_encoding_token; } else if (parser->index == sizeof(CHUNKED)-2) { h_state = h_transfer_encoding_chunked; } break; + case h_matching_transfer_encoding_token: + if (ch == ',') { + h_state = h_matching_transfer_encoding_token_start; + parser->index = 0; + } + break; + case h_matching_connection_token_start: /* looking for 'Connection: keep-alive' */ if (c == 'k') { @@ -1631,7 +1669,7 @@ size_t http_parser_execute (http_parser *parser, break; case h_transfer_encoding_chunked: - if (ch != ' ') h_state = h_general; + if (ch != ' ') h_state = h_matching_transfer_encoding_token; break; case h_connection_keep_alive: @@ -1765,12 +1803,22 @@ size_t http_parser_execute (http_parser *parser, REEXECUTE(); } - /* Cannot use chunked encoding and a content-length header together - per the HTTP specification. */ - if ((parser->flags & F_CHUNKED) && + /* Cannot use transfer-encoding and a content-length header together + per the HTTP specification. (RFC 7230 Section 3.3.3) */ + if ((parser->uses_transfer_encoding == 1) && (parser->flags & F_CONTENTLENGTH)) { - SET_ERRNO(HPE_UNEXPECTED_CONTENT_LENGTH); - goto error; + /* Allow it for lenient parsing as long as `Transfer-Encoding` is + * not `chunked` or allow_length_with_encoding is set + */ + if (parser->flags & F_CHUNKED) { + if (!allow_chunked_length) { + SET_ERRNO(HPE_UNEXPECTED_CONTENT_LENGTH); + goto error; + } + } else if (!lenient) { + SET_ERRNO(HPE_UNEXPECTED_CONTENT_LENGTH); + goto error; + } } UPDATE_STATE(s_headers_done); @@ -1845,8 +1893,31 @@ size_t http_parser_execute (http_parser *parser, UPDATE_STATE(NEW_MESSAGE()); CALLBACK_NOTIFY(message_complete); } else if (parser->flags & F_CHUNKED) { - /* chunked encoding - ignore Content-Length header */ + /* chunked encoding - ignore Content-Length header, + * prepare for a chunk */ UPDATE_STATE(s_chunk_size_start); + } else if (parser->uses_transfer_encoding == 1) { + if (parser->type == HTTP_REQUEST && !lenient) { + /* RFC 7230 3.3.3 */ + + /* If a Transfer-Encoding header field + * is present in a request and the chunked transfer coding is not + * the final encoding, the message body length cannot be determined + * reliably; the server MUST respond with the 400 (Bad Request) + * status code and then close the connection. + */ + SET_ERRNO(HPE_INVALID_TRANSFER_ENCODING); + RETURN(p - data); /* Error */ + } else { + /* RFC 7230 3.3.3 */ + + /* If a Transfer-Encoding header field is present in a response and + * the chunked transfer coding is not the final encoding, the + * message body length is determined by reading the connection until + * it is closed by the server. + */ + UPDATE_STATE(s_body_identity_eof); + } } else { if (parser->content_length == 0) { /* Content-Length header given but zero: Content-Length: 0\r\n */ @@ -2100,6 +2171,12 @@ http_message_needs_eof (const http_parser *parser) return 0; } + /* RFC 7230 3.3.3, see `s_headers_almost_done` */ + if ((parser->uses_transfer_encoding == 1) && + (parser->flags & F_CHUNKED) == 0) { + return 1; + } + if ((parser->flags & F_CHUNKED) || parser->content_length != ULLONG_MAX) { return 0; } @@ -2447,7 +2524,7 @@ http_parser_parse_url(const char *buf, size_t buflen, int is_connect, end = buf + off + len; /* NOTE: The characters are already validated and are in the [0-9] range */ - assert(off + len <= buflen && "Port number overflow"); + assert((size_t) (off + len) <= buflen && "Port number overflow"); v = 0; for (p = buf + off; p < end; p++) { v *= 10; diff --git a/http_parser.h b/http_parser.h index 16b5281d..3772b399 100644 --- a/http_parser.h +++ b/http_parser.h @@ -27,7 +27,7 @@ extern "C" { /* Also update SONAME in the Makefile whenever you change these. */ #define HTTP_PARSER_VERSION_MAJOR 2 #define HTTP_PARSER_VERSION_MINOR 9 -#define HTTP_PARSER_VERSION_PATCH 2 +#define HTTP_PARSER_VERSION_PATCH 4 #include #if defined(_WIN32) && !defined(__MINGW32__) && \ @@ -41,6 +41,8 @@ typedef __int32 int32_t; typedef unsigned __int32 uint32_t; typedef __int64 int64_t; typedef unsigned __int64 uint64_t; +#elif (defined(__sun) || defined(__sun__)) && defined(__SunOS_5_9) +#include #else #include #endif @@ -275,7 +277,9 @@ enum flags XX(INVALID_INTERNAL_STATE, "encountered unexpected internal state")\ XX(STRICT, "strict mode assertion failed") \ XX(PAUSED, "parser is paused") \ - XX(UNKNOWN, "an unknown error occurred") + XX(UNKNOWN, "an unknown error occurred") \ + XX(INVALID_TRANSFER_ENCODING, \ + "request has invalid transfer-encoding") \ /* Define HPE_* values for each errno value above */ @@ -293,14 +297,20 @@ enum http_errno { struct http_parser { /** PRIVATE **/ unsigned int type : 2; /* enum http_parser_type */ - unsigned int flags : 8; /* F_* values from 'flags' enum; semi-public */ + unsigned int flags : 8; /* F_* values from 'flags' enum; semi-public */ unsigned int state : 7; /* enum state from http_parser.c */ unsigned int header_state : 7; /* enum header_state from http_parser.c */ - unsigned int index : 7; /* index into current matcher */ + unsigned int index : 5; /* index into current matcher */ + unsigned int uses_transfer_encoding : 1; /* Transfer-Encoding header is present */ + unsigned int allow_chunked_length : 1; /* Allow headers with both + * `Content-Length` and + * `Transfer-Encoding: chunked` set */ unsigned int lenient_http_headers : 1; uint32_t nread; /* # bytes read in various scenarios */ - uint64_t content_length; /* # bytes in body (0 if no Content-Length header) */ + uint64_t content_length; /* # bytes in body. `(uint64_t) -1` (all bits one) + * if no Content-Length header. + */ /** READ-ONLY **/ unsigned short http_major; diff --git a/test.c b/test.c index 0140a18b..3f7c77b3 100644 --- a/test.c +++ b/test.c @@ -74,6 +74,7 @@ struct message { unsigned short http_major; unsigned short http_minor; + uint64_t content_length; int message_begin_cb_called; int headers_complete_cb_called; @@ -81,6 +82,7 @@ struct message { int status_cb_called; int message_complete_on_eof; int body_is_final; + int allow_chunked_length; }; static int currently_parsing_eof; @@ -108,6 +110,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/test" ,.request_url= "/test" + ,.content_length= -1 ,.num_headers= 3 ,.headers= { { "User-Agent", "curl/7.18.0 (i486-pc-linux-gnu) libcurl/7.18.0 OpenSSL/0.9.8g zlib/1.2.3.3 libidn/1.1" } @@ -139,6 +142,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/favicon.ico" ,.request_url= "/favicon.ico" + ,.content_length= -1 ,.num_headers= 8 ,.headers= { { "Host", "0.0.0.0=5000" } @@ -168,6 +172,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/dumbluck" ,.request_url= "/dumbluck" + ,.content_length= -1 ,.num_headers= 1 ,.headers= { { "aaaaaaaaaaaaa", "++++++++++" } @@ -190,6 +195,7 @@ const struct message requests[] = ,.request_path= "/forums/1/topics/2375" /* XXX request url does include fragment? */ ,.request_url= "/forums/1/topics/2375?page=1#posts-17408" + ,.content_length= -1 ,.num_headers= 0 ,.body= "" } @@ -208,6 +214,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/get_no_headers_no_body/world" ,.request_url= "/get_no_headers_no_body/world" + ,.content_length= -1 ,.num_headers= 0 ,.body= "" } @@ -227,6 +234,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/get_one_header_no_body" ,.request_url= "/get_one_header_no_body" + ,.content_length= -1 ,.num_headers= 1 ,.headers= { { "Accept" , "*/*" } @@ -250,6 +258,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/get_funky_content_length_body_hello" ,.request_url= "/get_funky_content_length_body_hello" + ,.content_length= 5 ,.num_headers= 1 ,.headers= { { "conTENT-Length" , "5" } @@ -262,7 +271,6 @@ const struct message requests[] = ,.type= HTTP_REQUEST ,.raw= "POST /post_identity_body_world?q=search#hey HTTP/1.1\r\n" "Accept: */*\r\n" - "Transfer-Encoding: identity\r\n" "Content-Length: 5\r\n" "\r\n" "World" @@ -275,10 +283,10 @@ const struct message requests[] = ,.fragment= "hey" ,.request_path= "/post_identity_body_world" ,.request_url= "/post_identity_body_world?q=search#hey" - ,.num_headers= 3 + ,.content_length= 5 + ,.num_headers= 2 ,.headers= { { "Accept", "*/*" } - , { "Transfer-Encoding", "identity" } , { "Content-Length", "5" } } ,.body= "World" @@ -302,6 +310,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/post_chunked_all_your_base" ,.request_url= "/post_chunked_all_your_base" + ,.content_length= -1 ,.num_headers= 1 ,.headers= { { "Transfer-Encoding" , "chunked" } @@ -330,6 +339,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/two_chunks_mult_zero_end" ,.request_url= "/two_chunks_mult_zero_end" + ,.content_length= -1 ,.num_headers= 1 ,.headers= { { "Transfer-Encoding", "chunked" } @@ -360,6 +370,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/chunked_w_trailing_headers" ,.request_url= "/chunked_w_trailing_headers" + ,.content_length= -1 ,.num_headers= 3 ,.headers= { { "Transfer-Encoding", "chunked" } @@ -390,6 +401,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/chunked_w_nonsense_after_length" ,.request_url= "/chunked_w_nonsense_after_length" + ,.content_length= -1 ,.num_headers= 1 ,.headers= { { "Transfer-Encoding", "chunked" } @@ -412,6 +424,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/with_\"stupid\"_quotes" ,.request_url= "/with_\"stupid\"_quotes?foo=\"bar\"" + ,.content_length= -1 ,.num_headers= 0 ,.headers= { } ,.body= "" @@ -438,6 +451,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/test" ,.request_url= "/test" + ,.content_length= -1 ,.num_headers= 3 ,.headers= { { "Host", "0.0.0.0:5000" } , { "User-Agent", "ApacheBench/2.3" } @@ -461,6 +475,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/test.cgi" ,.request_url= "/test.cgi?foo=bar?baz" + ,.content_length= -1 ,.num_headers= 0 ,.headers= {} ,.body= "" @@ -482,6 +497,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/test" ,.request_url= "/test" + ,.content_length= -1 ,.num_headers= 0 ,.headers= { } ,.body= "" @@ -509,6 +525,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/demo" ,.request_url= "/demo" + ,.content_length= -1 ,.num_headers= 7 ,.upgrade="Hot diggity dogg" ,.headers= { { "Host", "example.com" } @@ -540,6 +557,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "" ,.request_url= "0-home0.netscape.com:443" + ,.content_length= -1 ,.num_headers= 2 ,.upgrade="some data\r\nand yet even more data" ,.headers= { { "User-agent", "Mozilla/1.1N" } @@ -562,6 +580,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/test" ,.request_url= "/test" + ,.content_length= -1 ,.num_headers= 0 ,.headers= {} ,.body= "" @@ -581,6 +600,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/" ,.request_url= "/" + ,.content_length= -1 ,.num_headers= 0 ,.headers= {} ,.body= "" @@ -603,6 +623,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "*" ,.request_url= "*" + ,.content_length= -1 ,.num_headers= 3 ,.headers= { { "HOST", "239.255.255.250:1900" } , { "MAN", "\"ssdp:discover\"" } @@ -638,6 +659,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/" ,.request_url= "/" + ,.content_length= -1 ,.num_headers= 5 ,.headers= { { "Line1", "abc\tdef ghi\t\tjkl mno \t \tqrs" } , { "Line2", "line2\t" } @@ -664,6 +686,7 @@ const struct message requests[] = ,.request_path= "" ,.request_url= "http://hypnotoad.org?hail=all" ,.host= "hypnotoad.org" + ,.content_length= -1 ,.num_headers= 0 ,.headers= { } ,.body= "" @@ -685,6 +708,7 @@ const struct message requests[] = ,.request_url= "http://hypnotoad.org:1234?hail=all" ,.host= "hypnotoad.org" ,.port= 1234 + ,.content_length= -1 ,.num_headers= 0 ,.headers= { } ,.body= "" @@ -706,6 +730,7 @@ const struct message requests[] = ,.request_url= "http://hypnotoad.org:1234" ,.host= "hypnotoad.org" ,.port= 1234 + ,.content_length= -1 ,.num_headers= 0 ,.headers= { } ,.body= "" @@ -730,6 +755,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/file.txt" ,.request_url= "/file.txt" + ,.content_length= 10 ,.num_headers= 4 ,.headers= { { "Host", "www.example.com" } , { "Content-Type", "application/example" } @@ -755,6 +781,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "" ,.request_url= "HOME0.NETSCAPE.COM:443" + ,.content_length= -1 ,.num_headers= 2 ,.upgrade="" ,.headers= { { "User-agent", "Mozilla/1.1N" } @@ -779,6 +806,7 @@ const struct message requests[] = ,.fragment= "narf" ,.request_path= "/δ¶/δt/pope" ,.request_url= "/δ¶/δt/pope?q=1#narf" + ,.content_length= -1 ,.num_headers= 1 ,.headers= { {"Host", "github.com" } } @@ -801,6 +829,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "" ,.request_url= "home_0.netscape.com:443" + ,.content_length= -1 ,.num_headers= 2 ,.upgrade="" ,.headers= { { "User-agent", "Mozilla/1.1N" } @@ -828,6 +857,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/" ,.request_url= "/" + ,.content_length= 4 ,.num_headers= 3 ,.upgrade= 0 ,.headers= { { "Host", "www.example.com" } @@ -856,6 +886,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/" ,.request_url= "/" + ,.content_length= 4 ,.num_headers= 4 ,.upgrade= 0 ,.headers= { { "Host", "www.example.com" } @@ -881,6 +912,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/file.txt" ,.request_url= "/file.txt" + ,.content_length= -1 ,.num_headers= 1 ,.headers= { { "Host", "www.example.com" } } ,.body= "" @@ -901,6 +933,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/" ,.request_url= "/" + ,.content_length= -1 ,.num_headers= 1 ,.headers= { { "Host", "www.example.com" } } ,.body= "" @@ -922,6 +955,7 @@ const struct message requests[] = ,.host= "hypnotoad.org" ,.userinfo= "a%12:b!&*$" ,.port= 1234 + ,.content_length= -1 ,.num_headers= 0 ,.headers= { } ,.body= "" @@ -954,6 +988,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/" ,.request_url= "/" + ,.content_length= -1 ,.num_headers= 5 ,.headers= { { "Line1", "abc\tdef ghi\t\tjkl mno \t \tqrs" } , { "Line2", "line2\t" } @@ -987,6 +1022,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/demo" ,.request_url= "/demo" + ,.content_length= -1 ,.num_headers= 7 ,.upgrade="Hot diggity dogg" ,.headers= { { "Host", "example.com" } @@ -1017,6 +1053,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/demo" ,.request_url= "/demo" + ,.content_length= -1 ,.num_headers= 2 ,.upgrade="Hot diggity dogg" ,.headers= { { "Connection", "keep-alive, upgrade" } @@ -1042,6 +1079,7 @@ const struct message requests[] = ,.fragment= "" ,.request_path= "/demo" ,.request_url= "/demo" + ,.content_length= -1 ,.num_headers= 2 ,.upgrade="Hot diggity dogg" ,.headers= { { "Connection", "keep-alive, upgrade" } @@ -1068,6 +1106,7 @@ const struct message requests[] = ,.method= HTTP_POST ,.request_path= "/demo" ,.request_url= "/demo" + ,.content_length= 15 ,.num_headers= 4 ,.upgrade="Hot diggity dogg" ,.headers= { { "Host", "example.com" } @@ -1093,6 +1132,7 @@ const struct message requests[] = ,.http_minor= 0 ,.method= HTTP_CONNECT ,.request_url= "foo.bar.com:443" + ,.content_length= 10 ,.num_headers= 3 ,.upgrade="blarfcicle" ,.headers= { { "User-agent", "Mozilla/1.1N" } @@ -1123,6 +1163,7 @@ const struct message requests[] = ,.request_url= "/images/my_dog.jpg" ,.query_string= "" ,.fragment= "" + ,.content_length= -1 ,.num_headers= 3 ,.headers= { { "Host", "example.com" } , { "Link", "; rel=\"tag\"" } @@ -1147,6 +1188,7 @@ const struct message requests[] = ,.request_url= "/images/my_dog.jpg" ,.query_string= "" ,.fragment= "" + ,.content_length= -1 ,.num_headers= 2 ,.headers= { { "Host", "example.com" } , { "Link", "; rel=\"tag\"" } @@ -1169,12 +1211,13 @@ const struct message requests[] = ,.request_url= "/music/sweet/music" ,.query_string= "" ,.fragment= "" + ,.content_length= -1 ,.num_headers= 1 ,.headers= { { "Host", "example.com" } } ,.body= "" } -#define SOURCE_ICE_REQUEST 42 +#define SOURCE_ICE_REQUEST 43 , {.name = "source request" ,.type= HTTP_REQUEST ,.raw= "SOURCE /music/sweet/music ICE/1.0\r\n" @@ -1189,10 +1232,99 @@ const struct message requests[] = ,.request_url= "/music/sweet/music" ,.query_string= "" ,.fragment= "" + ,.content_length= -1 ,.num_headers= 1 ,.headers= { { "Host", "example.com" } } ,.body= "" } + +#define POST_MULTI_TE_LAST_CHUNKED 44 +, {.name= "post - multi coding transfer-encoding chunked body" + ,.type= HTTP_REQUEST + ,.raw= "POST / HTTP/1.1\r\n" + "Transfer-Encoding: deflate, chunked\r\n" + "\r\n" + "1e\r\nall your base are belong to us\r\n" + "0\r\n" + "\r\n" + ,.should_keep_alive= TRUE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 1 + ,.method= HTTP_POST + ,.query_string= "" + ,.fragment= "" + ,.request_path= "/" + ,.request_url= "/" + ,.content_length= -1 + ,.num_headers= 1 + ,.headers= + { { "Transfer-Encoding" , "deflate, chunked" } + } + ,.body= "all your base are belong to us" + ,.num_chunks_complete= 2 + ,.chunk_lengths= { 0x1e } + } + +#define POST_MULTI_LINE_TE_LAST_CHUNKED 45 +, {.name= "post - multi line coding transfer-encoding chunked body" + ,.type= HTTP_REQUEST + ,.raw= "POST / HTTP/1.1\r\n" + "Transfer-Encoding: deflate,\r\n" + " chunked\r\n" + "\r\n" + "1e\r\nall your base are belong to us\r\n" + "0\r\n" + "\r\n" + ,.should_keep_alive= TRUE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 1 + ,.method= HTTP_POST + ,.query_string= "" + ,.fragment= "" + ,.request_path= "/" + ,.request_url= "/" + ,.content_length= -1 + ,.num_headers= 1 + ,.headers= + { { "Transfer-Encoding" , "deflate, chunked" } + } + ,.body= "all your base are belong to us" + ,.num_chunks_complete= 2 + ,.chunk_lengths= { 0x1e } + } + +#define CHUNKED_CONTENT_LENGTH 46 +, {.name= "chunked with content-length set, allow_chunked_length flag is set" + ,.type= HTTP_REQUEST + ,.raw= "POST /chunked_w_content_length HTTP/1.1\r\n" + "Content-Length: 10\r\n" + "Transfer-Encoding: chunked\r\n" + "\r\n" + "5; ilovew3;whattheluck=aretheseparametersfor\r\nhello\r\n" + "6; blahblah; blah\r\n world\r\n" + "0\r\n" + "\r\n" + ,.allow_chunked_length = 1 + ,.should_keep_alive= TRUE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 1 + ,.method= HTTP_POST + ,.query_string= "" + ,.fragment= "" + ,.request_path= "/chunked_w_content_length" + ,.request_url= "/chunked_w_content_length" + ,.content_length= 10 + ,.num_headers= 2 + ,.headers={ { "Content-Length", "10"} + , { "Transfer-Encoding", "chunked" } + } + ,.body= "hello world" + ,.num_chunks_complete= 3 + ,.chunk_lengths= { 5, 6 } + } }; /* * R E S P O N S E S * */ @@ -1222,6 +1354,7 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 301 ,.response_status= "Moved Permanently" + ,.content_length= 219 ,.num_headers= 8 ,.headers= { { "Location", "http://www.google.com/" } @@ -1271,6 +1404,7 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 200 ,.response_status= "OK" + ,.content_length= -1 ,.num_headers= 5 ,.headers= { { "Date", "Tue, 04 Aug 2009 07:59:32 GMT" } @@ -1300,6 +1434,7 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 404 ,.response_status= "Not Found" + ,.content_length= -1 ,.num_headers= 0 ,.headers= {} ,.body_size= 0 @@ -1315,6 +1450,7 @@ const struct message responses[] = ,.http_major= 1 ,.http_minor= 1 ,.status_code= 301 + ,.content_length= -1 ,.response_status= "" ,.num_headers= 0 ,.headers= {} @@ -1342,6 +1478,7 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 200 ,.response_status= "OK" + ,.content_length= -1 ,.num_headers= 2 ,.headers= { {"Content-Type", "text/plain" } @@ -1369,6 +1506,7 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 200 ,.response_status= "OK" + ,.content_length= -1 ,.num_headers= 2 ,.headers= { {"Content-Type", "text/html; charset=utf-8" } @@ -1393,6 +1531,7 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 200 ,.response_status= "OK" + ,.content_length= 11 ,.num_headers= 4 ,.headers= { {"Content-Type", "text/html; charset=UTF-8" } @@ -1419,6 +1558,7 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 200 ,.response_status= "OK" + ,.content_length= 0 ,.num_headers= 4 ,.headers= { {"Server", "DCLK-AdSvr" } @@ -1452,6 +1592,7 @@ const struct message responses[] = ,.http_minor= 0 ,.status_code= 301 ,.response_status= "Moved Permanently" + ,.content_length= 0 ,.num_headers= 9 ,.headers= { { "Date", "Thu, 03 Jun 2010 09:56:32 GMT" } @@ -1491,6 +1632,7 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 200 ,.response_status= "OK" + ,.content_length= -1 ,.num_headers= 11 ,.headers= { { "Date", "Tue, 28 Sep 2010 01:14:13 GMT" } @@ -1525,6 +1667,7 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 500 ,.response_status= "Oriëntatieprobleem" + ,.content_length= 0 ,.num_headers= 3 ,.headers= { { "Date", "Fri, 5 Nov 2010 23:07:12 GMT+2" } @@ -1546,6 +1689,7 @@ const struct message responses[] = ,.http_minor= 9 ,.status_code= 200 ,.response_status= "OK" + ,.content_length= -1 ,.num_headers= 0 ,.headers= {} @@ -1569,6 +1713,7 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 200 ,.response_status= "OK" + ,.content_length= -1 ,.num_headers= 1 ,.headers= { { "Content-Type", "text/plain" } @@ -1588,6 +1733,7 @@ const struct message responses[] = ,.http_minor= 0 ,.status_code= 200 ,.response_status= "OK" + ,.content_length= -1 ,.num_headers= 1 ,.headers= { { "Connection", "keep-alive" } @@ -1608,6 +1754,7 @@ const struct message responses[] = ,.http_minor= 0 ,.status_code= 204 ,.response_status= "No content" + ,.content_length= -1 ,.num_headers= 1 ,.headers= { { "Connection", "keep-alive" } @@ -1627,6 +1774,7 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 200 ,.response_status= "OK" + ,.content_length= -1 ,.num_headers= 0 ,.headers={} ,.body_size= 0 @@ -1644,6 +1792,7 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 204 ,.response_status= "No content" + ,.content_length= -1 ,.num_headers= 0 ,.headers={} ,.body_size= 0 @@ -1662,6 +1811,7 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 204 ,.response_status= "No content" + ,.content_length= -1 ,.num_headers= 1 ,.headers= { { "Connection", "close" } @@ -1684,6 +1834,7 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 200 ,.response_status= "OK" + ,.content_length= -1 ,.num_headers= 1 ,.headers= { { "Transfer-Encoding", "chunked" } @@ -1714,6 +1865,7 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 200 ,.response_status= "OK" + ,.content_length= 16 ,.num_headers= 7 ,.headers= { { "Server", "Microsoft-IIS/6.0" } @@ -1752,6 +1904,7 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 301 ,.response_status= "MovedPermanently" + ,.content_length= -1 ,.num_headers= 9 ,.headers= { { "Date", "Wed, 15 May 2013 17:06:33 GMT" } , { "Server", "Server" } @@ -1768,7 +1921,7 @@ const struct message responses[] = ,.chunk_lengths= { 1 } } -#define EMPTY_REASON_PHRASE_AFTER_SPACE 20 +#define EMPTY_REASON_PHRASE_AFTER_SPACE 21 , {.name= "empty reason phrase after space" ,.type= HTTP_RESPONSE ,.raw= "HTTP/1.1 200 \r\n" @@ -1779,12 +1932,13 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 200 ,.response_status= "" + ,.content_length= -1 ,.num_headers= 0 ,.headers= {} ,.body= "" } -#define CONTENT_LENGTH_X 21 +#define CONTENT_LENGTH_X 22 , {.name= "Content-Length-X" ,.type= HTTP_RESPONSE ,.raw= "HTTP/1.1 200 OK\r\n" @@ -1801,6 +1955,7 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 200 ,.response_status= "OK" + ,.content_length= -1 ,.num_headers= 2 ,.headers= { { "Content-Length-X", "0" } , { "Transfer-Encoding", "chunked" } @@ -1810,7 +1965,7 @@ const struct message responses[] = ,.chunk_lengths= { 2 } } -#define HTTP_101_RESPONSE_WITH_UPGRADE_HEADER 22 +#define HTTP_101_RESPONSE_WITH_UPGRADE_HEADER 23 , {.name= "HTTP 101 response with Upgrade header" ,.type= HTTP_RESPONSE ,.raw= "HTTP/1.1 101 Switching Protocols\r\n" @@ -1825,6 +1980,7 @@ const struct message responses[] = ,.status_code= 101 ,.response_status= "Switching Protocols" ,.upgrade= "proto" + ,.content_length= -1 ,.num_headers= 2 ,.headers= { { "Connection", "upgrade" } @@ -1832,7 +1988,7 @@ const struct message responses[] = } } -#define HTTP_101_RESPONSE_WITH_UPGRADE_HEADER_AND_CONTENT_LENGTH 23 +#define HTTP_101_RESPONSE_WITH_UPGRADE_HEADER_AND_CONTENT_LENGTH 24 , {.name= "HTTP 101 response with Upgrade and Content-Length header" ,.type= HTTP_RESPONSE ,.raw= "HTTP/1.1 101 Switching Protocols\r\n" @@ -1850,6 +2006,7 @@ const struct message responses[] = ,.response_status= "Switching Protocols" ,.body= "body" ,.upgrade= "proto" + ,.content_length= 4 ,.num_headers= 3 ,.headers= { { "Connection", "upgrade" } @@ -1858,7 +2015,7 @@ const struct message responses[] = } } -#define HTTP_101_RESPONSE_WITH_UPGRADE_HEADER_AND_TRANSFER_ENCODING 24 +#define HTTP_101_RESPONSE_WITH_UPGRADE_HEADER_AND_TRANSFER_ENCODING 25 , {.name= "HTTP 101 response with Upgrade and Transfer-Encoding header" ,.type= HTTP_RESPONSE ,.raw= "HTTP/1.1 101 Switching Protocols\r\n" @@ -1881,6 +2038,7 @@ const struct message responses[] = ,.response_status= "Switching Protocols" ,.body= "body" ,.upgrade= "proto" + ,.content_length= -1 ,.num_headers= 3 ,.headers= { { "Connection", "upgrade" } @@ -1891,7 +2049,7 @@ const struct message responses[] = ,.chunk_lengths= { 2, 2 } } -#define HTTP_200_RESPONSE_WITH_UPGRADE_HEADER 25 +#define HTTP_200_RESPONSE_WITH_UPGRADE_HEADER 26 , {.name= "HTTP 200 response with Upgrade header" ,.type= HTTP_RESPONSE ,.raw= "HTTP/1.1 200 OK\r\n" @@ -1907,6 +2065,7 @@ const struct message responses[] = ,.response_status= "OK" ,.body= "body" ,.upgrade= NULL + ,.content_length= -1 ,.num_headers= 2 ,.headers= { { "Connection", "upgrade" } @@ -1914,7 +2073,7 @@ const struct message responses[] = } } -#define HTTP_200_RESPONSE_WITH_UPGRADE_HEADER_AND_CONTENT_LENGTH 26 +#define HTTP_200_RESPONSE_WITH_UPGRADE_HEADER_AND_CONTENT_LENGTH 27 , {.name= "HTTP 200 response with Upgrade and Content-Length header" ,.type= HTTP_RESPONSE ,.raw= "HTTP/1.1 200 OK\r\n" @@ -1929,6 +2088,7 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 200 ,.response_status= "OK" + ,.content_length= 4 ,.num_headers= 3 ,.body= "body" ,.upgrade= NULL @@ -1939,7 +2099,7 @@ const struct message responses[] = } } -#define HTTP_200_RESPONSE_WITH_UPGRADE_HEADER_AND_TRANSFER_ENCODING 27 +#define HTTP_200_RESPONSE_WITH_UPGRADE_HEADER_AND_TRANSFER_ENCODING 28 , {.name= "HTTP 200 response with Upgrade and Transfer-Encoding header" ,.type= HTTP_RESPONSE ,.raw= "HTTP/1.1 200 OK\r\n" @@ -1959,6 +2119,7 @@ const struct message responses[] = ,.http_minor= 1 ,.status_code= 200 ,.response_status= "OK" + ,.content_length= -1 ,.num_headers= 3 ,.body= "body" ,.upgrade= NULL @@ -1970,6 +2131,29 @@ const struct message responses[] = ,.num_chunks_complete= 3 ,.chunk_lengths= { 2, 2 } } +#define HTTP_200_MULTI_TE_NOT_LAST_CHUNKED 29 +, {.name= "HTTP 200 response with `chunked` being *not last* Transfer-Encoding" + ,.type= HTTP_RESPONSE + ,.raw= "HTTP/1.1 200 OK\r\n" + "Transfer-Encoding: chunked, identity\r\n" + "\r\n" + "2\r\n" + "OK\r\n" + "0\r\n" + "\r\n" + ,.should_keep_alive= FALSE + ,.message_complete_on_eof= TRUE + ,.http_major= 1 + ,.http_minor= 1 + ,.status_code= 200 + ,.response_status= "OK" + ,.content_length= -1 + ,.num_headers= 1 + ,.headers= { { "Transfer-Encoding", "chunked, identity" } + } + ,.body= "2\r\nOK\r\n0\r\n\r\n" + ,.num_chunks_complete= 0 + } }; /* strnlen() is a POSIX.2008 addition. Can't rely on it being available so @@ -2127,6 +2311,7 @@ headers_complete_cb (http_parser *p) messages[num_messages].status_code = parser.status_code; messages[num_messages].http_major = parser.http_major; messages[num_messages].http_minor = parser.http_minor; + messages[num_messages].content_length = parser.content_length; messages[num_messages].headers_complete_cb_called = TRUE; messages[num_messages].should_keep_alive = http_should_keep_alive(&parser); return 0; @@ -2575,6 +2760,7 @@ message_eq (int index, int connect, const struct message *expected) MESSAGE_CHECK_NUM_EQ(expected, m, http_major); MESSAGE_CHECK_NUM_EQ(expected, m, http_minor); + MESSAGE_CHECK_NUM_EQ(expected, m, content_length); if (expected->type == HTTP_REQUEST) { MESSAGE_CHECK_NUM_EQ(expected, m, method); @@ -3428,6 +3614,9 @@ test_message (const struct message *message) size_t msg1len; for (msg1len = 0; msg1len < raw_len; msg1len++) { parser_init(message->type); + if (message->allow_chunked_length) { + parser.allow_chunked_length = 1; + } size_t read; const char *msg1 = message->raw; @@ -3663,7 +3852,7 @@ test_chunked_content_length_error (int req) parsed = http_parser_execute(&parser, &settings_null, buf, strlen(buf)); assert(parsed == strlen(buf)); - buf = "Transfer-Encoding: chunked\r\nContent-Length: 1\r\n\r\n"; + buf = "Transfer-Encoding: anything\r\nContent-Length: 1\r\n\r\n"; size_t buflen = strlen(buf); parsed = http_parser_execute(&parser, &settings_null, buf, buflen); @@ -3869,6 +4058,11 @@ test_multiple3 (const struct message *r1, const struct message *r2, const struct strcat(total, r3->raw); parser_init(r1->type); + if (r1->allow_chunked_length || + r2->allow_chunked_length || + r3->allow_chunked_length) { + parser.allow_chunked_length = 1; + } size_t read; @@ -4071,6 +4265,9 @@ test_message_pause (const struct message *msg) size_t nread; parser_init(msg->type); + if (msg->allow_chunked_length) { + parser.allow_chunked_length = 1; + } do { nread = parse_pause(buf, buflen); @@ -4147,6 +4344,13 @@ main (void) printf("sizeof(http_parser) = %u\n", (unsigned int)sizeof(http_parser)); +#if defined(__i386__) || defined(__x86_64__) + /* Should be 32 on both 32 bits and 64 bits x86 because of struct padding, + * see https://github.com/nodejs/http-parser/issues/507. + */ + assert(sizeof(http_parser) == 32); +#endif + //// API test_preserve_data(); test_parse_url(); @@ -4275,6 +4479,7 @@ main (void) ,.http_minor= 0 ,.status_code= 200 ,.response_status= "OK" + ,.content_length= -1 ,.num_headers= 2 ,.headers= { { "Transfer-Encoding", "chunked" } @@ -4332,6 +4537,12 @@ main (void) "fooba", HPE_OK); + // Unknown Transfer-Encoding in request + test_simple("GET / HTTP/1.1\r\n" + "Transfer-Encoding: unknown\r\n" + "\r\n", + HPE_INVALID_TRANSFER_ENCODING); + static const char *all_methods[] = { "DELETE", "GET",