commit
8d46aca41f
@ -0,0 +1,133 @@
|
|||||||
|
From 95f9f188665618759d8d1a27c96b3dacc3ed89be Mon Sep 17 00:00:00 2001
|
||||||
|
From: Renata Ravanelli <renata.ravanelli@gmail.com>
|
||||||
|
Date: Fri, 15 Sep 2023 12:22:48 -0300
|
||||||
|
Subject: [PATCH 1/6] This patch is a backport of commit: e75b0d9
|
||||||
|
|
||||||
|
From: Bert JW Regeer <bertjw@regeer.org>
|
||||||
|
Date: Sat, 12 Mar 2022 18:30:30 -0700
|
||||||
|
Subject: [PATCH 1/8] Add new regular expressions for Chunked Encoding
|
||||||
|
|
||||||
|
This also moves some regular expressions for QUOTED_PAIR/QUOTED_STRING
|
||||||
|
into this module from utilities so that they may be reused.
|
||||||
|
|
||||||
|
Backport:
|
||||||
|
* Patch refresh - no functional change.
|
||||||
|
|
||||||
|
Signed-off-by: Renata Ravanelli <renata.ravanelli@gmail.com>
|
||||||
|
---
|
||||||
|
src/waitress/rfc7230.py | 27 ++++++++++++++++++++++++++-
|
||||||
|
src/waitress/utilities.py | 28 +++-------------------------
|
||||||
|
2 files changed, 29 insertions(+), 26 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/src/waitress/rfc7230.py b/src/waitress/rfc7230.py
|
||||||
|
index cd33c90..0b76a38 100644
|
||||||
|
--- a/src/waitress/rfc7230.py
|
||||||
|
+++ b/src/waitress/rfc7230.py
|
||||||
|
@@ -7,6 +7,9 @@ import re
|
||||||
|
|
||||||
|
from .compat import tobytes
|
||||||
|
|
||||||
|
+HEXDIG = "[0-9a-fA-F]"
|
||||||
|
+DIGIT = "[0-9]"
|
||||||
|
+
|
||||||
|
WS = "[ \t]"
|
||||||
|
OWS = WS + "{0,}?"
|
||||||
|
RWS = WS + "{1,}?"
|
||||||
|
@@ -27,6 +30,12 @@ TOKEN = TCHAR + "{1,}"
|
||||||
|
# ; visible (printing) characters
|
||||||
|
VCHAR = r"\x21-\x7e"
|
||||||
|
|
||||||
|
+# The '\\' between \x5b and \x5d is needed to escape \x5d (']')
|
||||||
|
+QDTEXT = "[\t \x21\x23-\x5b\\\x5d-\x7e" + OBS_TEXT + "]"
|
||||||
|
+
|
||||||
|
+QUOTED_PAIR = r"\\" + "([\t " + VCHAR + OBS_TEXT + "])"
|
||||||
|
+QUOTED_STRING = '"(?:(?:' + QDTEXT + ")|(?:" + QUOTED_PAIR + '))*"'
|
||||||
|
+
|
||||||
|
# header-field = field-name ":" OWS field-value OWS
|
||||||
|
# field-name = token
|
||||||
|
# field-value = *( field-content / obs-fold )
|
||||||
|
@@ -45,8 +54,24 @@ FIELD_CONTENT = FIELD_VCHAR + "+(?:[ \t]+" + FIELD_VCHAR + "+)*"
|
||||||
|
# Which allows the field value here to just see if there is even a value in the first place
|
||||||
|
FIELD_VALUE = "(?:" + FIELD_CONTENT + ")?"
|
||||||
|
|
||||||
|
-HEADER_FIELD = re.compile(
|
||||||
|
+# chunk-ext = *( ";" chunk-ext-name [ "=" chunk-ext-val ] )
|
||||||
|
+# chunk-ext-name = token
|
||||||
|
+# chunk-ext-val = token / quoted-string
|
||||||
|
+
|
||||||
|
+CHUNK_EXT_NAME = TOKEN
|
||||||
|
+CHUNK_EXT_VAL = "(?:" + TOKEN + ")|(?:" + QUOTED_STRING + ")"
|
||||||
|
+CHUNK_EXT = (
|
||||||
|
+ "(?:;(?P<extension>" + CHUNK_EXT_NAME + ")(?:=(?P<value>" + CHUNK_EXT_VAL + "))?)*"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+# Pre-compiled regular expressions for use elsewhere
|
||||||
|
+ONLY_HEXDIG_RE = re.compile(("^" + HEXDIG + "+$").encode("latin-1"))
|
||||||
|
+ONLY_DIGIT_RE = re.compile(("^" + DIGIT + "+$").encode("latin-1"))
|
||||||
|
+HEADER_FIELD_RE = re.compile(
|
||||||
|
tobytes(
|
||||||
|
"^(?P<name>" + TOKEN + "):" + OWS + "(?P<value>" + FIELD_VALUE + ")" + OWS + "$"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
+QUOTED_PAIR_RE = re.compile(QUOTED_PAIR)
|
||||||
|
+QUOTED_STRING_RE = re.compile(QUOTED_STRING)
|
||||||
|
+CHUNK_EXT_RE = re.compile(("^" + CHUNK_EXT + "$").encode("latin-1"))
|
||||||
|
diff --git a/src/waitress/utilities.py b/src/waitress/utilities.py
|
||||||
|
index 556bed2..fa59657 100644
|
||||||
|
--- a/src/waitress/utilities.py
|
||||||
|
+++ b/src/waitress/utilities.py
|
||||||
|
@@ -22,7 +22,7 @@ import re
|
||||||
|
import stat
|
||||||
|
import time
|
||||||
|
|
||||||
|
-from .rfc7230 import OBS_TEXT, VCHAR
|
||||||
|
+from .rfc7230 import QUOTED_PAIR_RE, QUOTED_STRING_RE
|
||||||
|
|
||||||
|
logger = logging.getLogger("waitress")
|
||||||
|
queue_logger = logging.getLogger("waitress.queue")
|
||||||
|
@@ -216,32 +216,10 @@ def parse_http_date(d):
|
||||||
|
return retval
|
||||||
|
|
||||||
|
|
||||||
|
-# RFC 5234 Appendix B.1 "Core Rules":
|
||||||
|
-# VCHAR = %x21-7E
|
||||||
|
-# ; visible (printing) characters
|
||||||
|
-vchar_re = VCHAR
|
||||||
|
-
|
||||||
|
-# RFC 7230 Section 3.2.6 "Field Value Components":
|
||||||
|
-# quoted-string = DQUOTE *( qdtext / quoted-pair ) DQUOTE
|
||||||
|
-# qdtext = HTAB / SP /%x21 / %x23-5B / %x5D-7E / obs-text
|
||||||
|
-# obs-text = %x80-FF
|
||||||
|
-# quoted-pair = "\" ( HTAB / SP / VCHAR / obs-text )
|
||||||
|
-obs_text_re = OBS_TEXT
|
||||||
|
-
|
||||||
|
-# The '\\' between \x5b and \x5d is needed to escape \x5d (']')
|
||||||
|
-qdtext_re = "[\t \x21\x23-\x5b\\\x5d-\x7e" + obs_text_re + "]"
|
||||||
|
-
|
||||||
|
-quoted_pair_re = r"\\" + "([\t " + vchar_re + obs_text_re + "])"
|
||||||
|
-quoted_string_re = '"(?:(?:' + qdtext_re + ")|(?:" + quoted_pair_re + '))*"'
|
||||||
|
-
|
||||||
|
-quoted_string = re.compile(quoted_string_re)
|
||||||
|
-quoted_pair = re.compile(quoted_pair_re)
|
||||||
|
-
|
||||||
|
-
|
||||||
|
def undquote(value):
|
||||||
|
if value.startswith('"') and value.endswith('"'):
|
||||||
|
# So it claims to be DQUOTE'ed, let's validate that
|
||||||
|
- matches = quoted_string.match(value)
|
||||||
|
+ matches = QUOTED_STRING_RE.match(value)
|
||||||
|
|
||||||
|
if matches and matches.end() == len(value):
|
||||||
|
# Remove the DQUOTE's from the value
|
||||||
|
@@ -249,7 +227,7 @@ def undquote(value):
|
||||||
|
|
||||||
|
# Remove all backslashes that are followed by a valid vchar or
|
||||||
|
# obs-text
|
||||||
|
- value = quoted_pair.sub(r"\1", value)
|
||||||
|
+ value = QUOTED_PAIR_RE.sub(r"\1", value)
|
||||||
|
|
||||||
|
return value
|
||||||
|
elif not value.startswith('"') and not value.endswith('"'):
|
||||||
|
--
|
||||||
|
2.39.2 (Apple Git-143)
|
||||||
|
|
@ -0,0 +1,108 @@
|
|||||||
|
From c2188f39de0df7fc488703ebe0ed6e224f7be820 Mon Sep 17 00:00:00 2001
|
||||||
|
From: Renata Ravanelli <renata.ravanelli@gmail.com>
|
||||||
|
Date: Fri, 15 Sep 2023 12:26:52 -0300
|
||||||
|
Subject: [PATCH 2/6] This patch is a backport of commit: 1f6059f
|
||||||
|
|
||||||
|
From: Bert JW Regeer <bertjw@regeer.org>
|
||||||
|
Date: Sat, 12 Mar 2022 18:32:24 -0700
|
||||||
|
Subject: [PATCH] Be more strict in parsing Content-Length
|
||||||
|
|
||||||
|
Validate that we are only parsing digits and nothing else. RFC7230 is
|
||||||
|
explicit in that the Content-Length can only exist of 1*DIGIT and may
|
||||||
|
not include any additional sign information.
|
||||||
|
|
||||||
|
The Python int() function parses `+10` as `10` which means we were more
|
||||||
|
lenient than the standard intended.
|
||||||
|
|
||||||
|
Backport:
|
||||||
|
* Patch refresh - no functional change.
|
||||||
|
|
||||||
|
Signed-off-by: Renata Ravanelli <renata.ravanelli@gmail.com>
|
||||||
|
---
|
||||||
|
src/waitress/parser.py | 11 ++++++-----
|
||||||
|
tests/test_parser.py | 24 ++++++++++++++++++++++++
|
||||||
|
2 files changed, 30 insertions(+), 5 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/src/waitress/parser.py b/src/waitress/parser.py
|
||||||
|
index 765fe59..4c6ebeb 100644
|
||||||
|
--- a/src/waitress/parser.py
|
||||||
|
+++ b/src/waitress/parser.py
|
||||||
|
@@ -22,6 +22,7 @@ from io import BytesIO
|
||||||
|
from waitress.buffers import OverflowableBuffer
|
||||||
|
from waitress.compat import tostr, unquote_bytes_to_wsgi, urlparse
|
||||||
|
from waitress.receiver import ChunkedReceiver, FixedStreamReceiver
|
||||||
|
+from waitress.rfc7230 import HEADER_FIELD_RE, ONLY_DIGIT_RE
|
||||||
|
from waitress.utilities import (
|
||||||
|
BadRequest,
|
||||||
|
RequestEntityTooLarge,
|
||||||
|
@@ -29,7 +30,6 @@ from waitress.utilities import (
|
||||||
|
ServerNotImplemented,
|
||||||
|
find_double_newline,
|
||||||
|
)
|
||||||
|
-from .rfc7230 import HEADER_FIELD
|
||||||
|
|
||||||
|
|
||||||
|
class ParsingError(Exception):
|
||||||
|
@@ -209,7 +209,7 @@ class HTTPRequestParser(object):
|
||||||
|
|
||||||
|
headers = self.headers
|
||||||
|
for line in lines:
|
||||||
|
- header = HEADER_FIELD.match(line)
|
||||||
|
+ header = HEADER_FIELD_RE.match(line)
|
||||||
|
|
||||||
|
if not header:
|
||||||
|
raise ParsingError("Invalid header")
|
||||||
|
@@ -299,11 +299,12 @@ class HTTPRequestParser(object):
|
||||||
|
self.connection_close = True
|
||||||
|
|
||||||
|
if not self.chunked:
|
||||||
|
- try:
|
||||||
|
- cl = int(headers.get("CONTENT_LENGTH", 0))
|
||||||
|
- except ValueError:
|
||||||
|
+ cl = headers.get("CONTENT_LENGTH", "0")
|
||||||
|
+
|
||||||
|
+ if not ONLY_DIGIT_RE.match(cl.encode("latin-1")):
|
||||||
|
raise ParsingError("Content-Length is invalid")
|
||||||
|
|
||||||
|
+ cl = int(cl)
|
||||||
|
self.content_length = cl
|
||||||
|
if cl > 0:
|
||||||
|
buf = OverflowableBuffer(self.adj.inbuf_overflow)
|
||||||
|
diff --git a/tests/test_parser.py b/tests/test_parser.py
|
||||||
|
index 91837c7..eabf353 100644
|
||||||
|
--- a/tests/test_parser.py
|
||||||
|
+++ b/tests/test_parser.py
|
||||||
|
@@ -194,6 +194,30 @@ class TestHTTPRequestParser(unittest.TestCase):
|
||||||
|
else: # pragma: nocover
|
||||||
|
self.assertTrue(False)
|
||||||
|
|
||||||
|
+ def test_parse_header_bad_content_length_plus(self):
|
||||||
|
+ from waitress.parser import ParsingError
|
||||||
|
+
|
||||||
|
+ data = b"GET /foobar HTTP/8.4\r\ncontent-length: +10\r\n"
|
||||||
|
+
|
||||||
|
+ try:
|
||||||
|
+ self.parser.parse_header(data)
|
||||||
|
+ except ParsingError as e:
|
||||||
|
+ self.assertIn("Content-Length is invalid", e.args[0])
|
||||||
|
+ else: # pragma: nocover
|
||||||
|
+ self.assertTrue(False)
|
||||||
|
+
|
||||||
|
+ def test_parse_header_bad_content_length_minus(self):
|
||||||
|
+ from waitress.parser import ParsingError
|
||||||
|
+
|
||||||
|
+ data = b"GET /foobar HTTP/8.4\r\ncontent-length: -10\r\n"
|
||||||
|
+
|
||||||
|
+ try:
|
||||||
|
+ self.parser.parse_header(data)
|
||||||
|
+ except ParsingError as e:
|
||||||
|
+ self.assertIn("Content-Length is invalid", e.args[0])
|
||||||
|
+ else: # pragma: nocover
|
||||||
|
+ self.assertTrue(False)
|
||||||
|
+
|
||||||
|
def test_parse_header_multiple_content_length(self):
|
||||||
|
from waitress.parser import ParsingError
|
||||||
|
|
||||||
|
--
|
||||||
|
2.39.2 (Apple Git-143)
|
||||||
|
|
@ -0,0 +1,77 @@
|
|||||||
|
From 82003049b2b8053d74504c4e6b3e14528a8b38ff Mon Sep 17 00:00:00 2001
|
||||||
|
From: Renata Ravanelli <renata.ravanelli@gmail.com>
|
||||||
|
Date: Fri, 15 Sep 2023 12:32:19 -0300
|
||||||
|
Subject: [PATCH 3/6] This patch is a backport of commit 884bed1
|
||||||
|
|
||||||
|
From: Bert JW Regeer <bertjw@regeer.org>
|
||||||
|
Date: Sat, 12 Mar 2022 18:35:01 -0700
|
||||||
|
Subject: [PATCH] Update tests to remove invalid chunked encoding chunk-size
|
||||||
|
|
||||||
|
RFC7230 states the following:
|
||||||
|
|
||||||
|
chunk = chunk-size [ chunk-ext ] CRLF
|
||||||
|
chunk-data CRLF
|
||||||
|
chunk-size = 1*HEXDIG
|
||||||
|
|
||||||
|
Where chunk-ext is:
|
||||||
|
|
||||||
|
chunk-ext = *( ";" chunk-ext-name [ "=" chunk-ext-val ] )
|
||||||
|
|
||||||
|
Only if there is a chunk-ext should there be a `;` after the 1*HEXDIG.
|
||||||
|
And a chunk-ext that is empty is invalid.
|
||||||
|
|
||||||
|
Backport:
|
||||||
|
* Patch refresh - no functional change.
|
||||||
|
Signed-off-by: Renata Ravanelli <renata.ravanelli@gmail.com>
|
||||||
|
---
|
||||||
|
tests/test_functional.py | 6 +++---
|
||||||
|
tests/test_parser.py | 2 +-
|
||||||
|
2 files changed, 4 insertions(+), 4 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/tests/test_functional.py b/tests/test_functional.py
|
||||||
|
index e894497..7a54b22 100644
|
||||||
|
--- a/tests/test_functional.py
|
||||||
|
+++ b/tests/test_functional.py
|
||||||
|
@@ -302,7 +302,7 @@ class EchoTests(object):
|
||||||
|
self.assertFalse("transfer-encoding" in headers)
|
||||||
|
|
||||||
|
def test_chunking_request_with_content(self):
|
||||||
|
- control_line = b"20;\r\n" # 20 hex = 32 dec
|
||||||
|
+ control_line = b"20\r\n" # 20 hex = 32 dec
|
||||||
|
s = b"This string has 32 characters.\r\n"
|
||||||
|
expected = s * 12
|
||||||
|
header = tobytes("GET / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n")
|
||||||
|
@@ -321,7 +321,7 @@ class EchoTests(object):
|
||||||
|
self.assertFalse("transfer-encoding" in headers)
|
||||||
|
|
||||||
|
def test_broken_chunked_encoding(self):
|
||||||
|
- control_line = "20;\r\n" # 20 hex = 32 dec
|
||||||
|
+ control_line = "20\r\n" # 20 hex = 32 dec
|
||||||
|
s = "This string has 32 characters.\r\n"
|
||||||
|
to_send = "GET / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"
|
||||||
|
to_send += control_line + s + "\r\n"
|
||||||
|
@@ -346,7 +346,7 @@ class EchoTests(object):
|
||||||
|
self.assertRaises(ConnectionClosed, read_http, fp)
|
||||||
|
|
||||||
|
def test_broken_chunked_encoding_missing_chunk_end(self):
|
||||||
|
- control_line = "20;\r\n" # 20 hex = 32 dec
|
||||||
|
+ control_line = "20\r\n" # 20 hex = 32 dec
|
||||||
|
s = "This string has 32 characters.\r\n"
|
||||||
|
to_send = "GET / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"
|
||||||
|
to_send += control_line + s
|
||||||
|
diff --git a/tests/test_parser.py b/tests/test_parser.py
|
||||||
|
index eabf353..420f280 100644
|
||||||
|
--- a/tests/test_parser.py
|
||||||
|
+++ b/tests/test_parser.py
|
||||||
|
@@ -152,7 +152,7 @@ class TestHTTPRequestParser(unittest.TestCase):
|
||||||
|
b"Transfer-Encoding: chunked\r\n"
|
||||||
|
b"X-Foo: 1\r\n"
|
||||||
|
b"\r\n"
|
||||||
|
- b"1d;\r\n"
|
||||||
|
+ b"1d\r\n"
|
||||||
|
b"This string has 29 characters\r\n"
|
||||||
|
b"0\r\n\r\n"
|
||||||
|
)
|
||||||
|
--
|
||||||
|
2.39.2 (Apple Git-143)
|
||||||
|
|
@ -0,0 +1,152 @@
|
|||||||
|
From 86a7f4d2ea10ab96a3597f64b8662fbd741e2031 Mon Sep 17 00:00:00 2001
|
||||||
|
From: Renata Ravanelli <renata.ravanelli@gmail.com>
|
||||||
|
Date: Fri, 15 Sep 2023 12:40:31 -0300
|
||||||
|
Subject: [PATCH 4/6] This patch is a backport of commit: d032a66
|
||||||
|
|
||||||
|
From: Bert JW Regeer <bertjw@regeer.org>
|
||||||
|
|
||||||
|
Date: Sat, 12 Mar 2022 18:42:51 -0700
|
||||||
|
Subject: [PATCH] Error when receiving back Chunk Extension
|
||||||
|
|
||||||
|
Waitress discards chunked extensions and does no further processing on
|
||||||
|
them, however it failed to validate that the chunked encoding extension
|
||||||
|
did not contain invalid data.
|
||||||
|
|
||||||
|
We now validate that if there are any chunked extensions that they are
|
||||||
|
well-formed, if they are not and contain invalid characters, then
|
||||||
|
Waitress will now correctly return a Bad Request and stop any further
|
||||||
|
processing of the request
|
||||||
|
|
||||||
|
Signed-off-by: Renata Ravanelli <renata.ravanelli@gmail.com>
|
||||||
|
---
|
||||||
|
src/waitress/receiver.py | 11 ++++++++++-
|
||||||
|
tests/test_functional.py | 22 ++++++++++++++++++++++
|
||||||
|
tests/test_receiver.py | 37 +++++++++++++++++++++++++++++++++++++
|
||||||
|
3 files changed, 69 insertions(+), 1 deletion(-)
|
||||||
|
|
||||||
|
diff --git a/src/waitress/receiver.py b/src/waitress/receiver.py
|
||||||
|
index 5d1568d..106dbc7 100644
|
||||||
|
--- a/src/waitress/receiver.py
|
||||||
|
+++ b/src/waitress/receiver.py
|
||||||
|
@@ -14,6 +14,7 @@
|
||||||
|
"""Data Chunk Receiver
|
||||||
|
"""
|
||||||
|
|
||||||
|
+from waitress.rfc7230 import CHUNK_EXT_RE, ONLY_HEXDIG_RE
|
||||||
|
from waitress.utilities import BadRequest, find_double_newline
|
||||||
|
|
||||||
|
|
||||||
|
@@ -110,6 +111,7 @@ class ChunkedReceiver(object):
|
||||||
|
s = b""
|
||||||
|
else:
|
||||||
|
self.chunk_end = b""
|
||||||
|
+
|
||||||
|
if pos == 0:
|
||||||
|
# Chop off the terminating CR LF from the chunk
|
||||||
|
s = s[2:]
|
||||||
|
@@ -140,7 +142,14 @@ class ChunkedReceiver(object):
|
||||||
|
semi = line.find(b";")
|
||||||
|
|
||||||
|
if semi >= 0:
|
||||||
|
- # discard extension info.
|
||||||
|
+ extinfo = line[semi:]
|
||||||
|
+ valid_ext_info = CHUNK_EXT_RE.match(extinfo)
|
||||||
|
+
|
||||||
|
+ if not valid_ext_info:
|
||||||
|
+ self.error = BadRequest("Invalid chunk extension")
|
||||||
|
+ self.all_chunks_received = True
|
||||||
|
+
|
||||||
|
+ break
|
||||||
|
line = line[:semi]
|
||||||
|
try:
|
||||||
|
sz = int(line.strip(), 16) # hexadecimal
|
||||||
|
diff --git a/tests/test_functional.py b/tests/test_functional.py
|
||||||
|
index 7a54b22..853942c 100644
|
||||||
|
--- a/tests/test_functional.py
|
||||||
|
+++ b/tests/test_functional.py
|
||||||
|
@@ -345,6 +345,28 @@ class EchoTests(object):
|
||||||
|
self.send_check_error(to_send)
|
||||||
|
self.assertRaises(ConnectionClosed, read_http, fp)
|
||||||
|
|
||||||
|
+ def test_broken_chunked_encoding_invalid_extension(self):
|
||||||
|
+ control_line = b"20;invalid=\r\n" # 20 hex = 32 dec
|
||||||
|
+ s = b"This string has 32 characters.\r\n"
|
||||||
|
+ to_send = b"GET / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"
|
||||||
|
+ to_send += control_line + s + b"\r\n"
|
||||||
|
+ self.connect()
|
||||||
|
+ self.sock.send(to_send)
|
||||||
|
+ with self.sock.makefile("rb", 0) as fp:
|
||||||
|
+ line, headers, response_body = read_http(fp)
|
||||||
|
+ self.assertline(line, "400", "Bad Request", "HTTP/1.1")
|
||||||
|
+ cl = int(headers["content-length"])
|
||||||
|
+ self.assertEqual(cl, len(response_body))
|
||||||
|
+ self.assertIn(b"Invalid chunk extension", response_body)
|
||||||
|
+ self.assertEqual(
|
||||||
|
+ sorted(headers.keys()),
|
||||||
|
+ ["connection", "content-length", "content-type", "date", "server"],
|
||||||
|
+ )
|
||||||
|
+ self.assertEqual(headers["content-type"], "text/plain")
|
||||||
|
+ # connection has been closed
|
||||||
|
+ self.send_check_error(to_send)
|
||||||
|
+ self.assertRaises(ConnectionClosed, read_http, fp)
|
||||||
|
+
|
||||||
|
def test_broken_chunked_encoding_missing_chunk_end(self):
|
||||||
|
control_line = "20\r\n" # 20 hex = 32 dec
|
||||||
|
s = "This string has 32 characters.\r\n"
|
||||||
|
diff --git a/tests/test_receiver.py b/tests/test_receiver.py
|
||||||
|
index b4910bb..a6261ea 100644
|
||||||
|
--- a/tests/test_receiver.py
|
||||||
|
+++ b/tests/test_receiver.py
|
||||||
|
@@ -1,5 +1,7 @@
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
+import pytest
|
||||||
|
+
|
||||||
|
|
||||||
|
class TestFixedStreamReceiver(unittest.TestCase):
|
||||||
|
def _makeOne(self, cl, buf):
|
||||||
|
@@ -226,6 +228,41 @@ class TestChunkedReceiver(unittest.TestCase):
|
||||||
|
self.assertEqual(inst.error, None)
|
||||||
|
|
||||||
|
|
||||||
|
+class TestChunkedReceiverParametrized:
|
||||||
|
+ def _makeOne(self, buf):
|
||||||
|
+ from waitress.receiver import ChunkedReceiver
|
||||||
|
+
|
||||||
|
+ return ChunkedReceiver(buf)
|
||||||
|
+
|
||||||
|
+ @pytest.mark.parametrize(
|
||||||
|
+ "invalid_extension", [b"\n", b"invalid=", b"\r", b"invalid = true"]
|
||||||
|
+ )
|
||||||
|
+ def test_received_invalid_extensions(self, invalid_extension):
|
||||||
|
+ from waitress.utilities import BadRequest
|
||||||
|
+
|
||||||
|
+ buf = DummyBuffer()
|
||||||
|
+ inst = self._makeOne(buf)
|
||||||
|
+ data = b"4;" + invalid_extension + b"\r\ntest\r\n"
|
||||||
|
+ result = inst.received(data)
|
||||||
|
+ assert result == len(data)
|
||||||
|
+ assert inst.error.__class__ == BadRequest
|
||||||
|
+ assert inst.error.body == "Invalid chunk extension"
|
||||||
|
+
|
||||||
|
+ @pytest.mark.parametrize(
|
||||||
|
+ "valid_extension", [b"test", b"valid=true", b"valid=true;other=true"]
|
||||||
|
+ )
|
||||||
|
+ def test_received_valid_extensions(self, valid_extension):
|
||||||
|
+ # While waitress may ignore extensions in Chunked Encoding, we do want
|
||||||
|
+ # to make sure that we don't fail when we do encounter one that is
|
||||||
|
+ # valid
|
||||||
|
+ buf = DummyBuffer()
|
||||||
|
+ inst = self._makeOne(buf)
|
||||||
|
+ data = b"4;" + valid_extension + b"\r\ntest\r\n"
|
||||||
|
+ result = inst.received(data)
|
||||||
|
+ assert result == len(data)
|
||||||
|
+ assert inst.error == None
|
||||||
|
+
|
||||||
|
+
|
||||||
|
class DummyBuffer(object):
|
||||||
|
def __init__(self, data=None):
|
||||||
|
if data is None:
|
||||||
|
--
|
||||||
|
2.39.2 (Apple Git-143)
|
||||||
|
|
@ -0,0 +1,116 @@
|
|||||||
|
From b0ae7e3e156ac6f4a30ac4a54af0bffb707b008d Mon Sep 17 00:00:00 2001
|
||||||
|
From: Renata Ravanelli <renata.ravanelli@gmail.com>
|
||||||
|
Date: Fri, 15 Sep 2023 12:41:06 -0300
|
||||||
|
Subject: [PATCH 5/6] This patch is a backport of commit d9bdfa0
|
||||||
|
|
||||||
|
From: Bert JW Regeer <bertjw@regeer.org>
|
||||||
|
Date: Sat, 12 Mar 2022 18:48:26 -0700
|
||||||
|
Subject: [PATCH] Validate chunk size in Chunked Encoding are HEXDIG
|
||||||
|
|
||||||
|
RFC7230 states that a chunk-size should be 1*HEXDIG, this is now
|
||||||
|
validated before passing the resulting string to int() which would also
|
||||||
|
parse other formats for hex, such as: `0x01` as `1` and `+0x01` as `1`.
|
||||||
|
This would lead to a potential for a frontend proxy server and waitress
|
||||||
|
to disagree on where a chunk started and ended, thereby potentially
|
||||||
|
leading to request smuggling.
|
||||||
|
|
||||||
|
With the increased validation if the size is not just hex digits,
|
||||||
|
Waitress now returns a Bad Request and stops processing the request.
|
||||||
|
|
||||||
|
Signed-off-by: Renata Ravanelli <renata.ravanelli@gmail.com>
|
||||||
|
---
|
||||||
|
src/waitress/receiver.py | 19 ++++++++++++++-----
|
||||||
|
tests/test_functional.py | 22 ++++++++++++++++++++++
|
||||||
|
tests/test_receiver.py | 12 ++++++++++++
|
||||||
|
3 files changed, 48 insertions(+), 5 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/src/waitress/receiver.py b/src/waitress/receiver.py
|
||||||
|
index 106dbc7..9e4bffe 100644
|
||||||
|
--- a/src/waitress/receiver.py
|
||||||
|
+++ b/src/waitress/receiver.py
|
||||||
|
@@ -150,12 +150,21 @@ class ChunkedReceiver(object):
|
||||||
|
self.all_chunks_received = True
|
||||||
|
|
||||||
|
break
|
||||||
|
+
|
||||||
|
line = line[:semi]
|
||||||
|
- try:
|
||||||
|
- sz = int(line.strip(), 16) # hexadecimal
|
||||||
|
- except ValueError: # garbage in input
|
||||||
|
- self.error = BadRequest("garbage in chunked encoding input")
|
||||||
|
- sz = 0
|
||||||
|
+
|
||||||
|
+ # Remove any whitespace
|
||||||
|
+ line = line.strip()
|
||||||
|
+
|
||||||
|
+ if not ONLY_HEXDIG_RE.match(line):
|
||||||
|
+ self.error = BadRequest("Invalid chunk size")
|
||||||
|
+ self.all_chunks_received = True
|
||||||
|
+
|
||||||
|
+ break
|
||||||
|
+
|
||||||
|
+ # Can not fail due to matching against the regular
|
||||||
|
+ # expression above
|
||||||
|
+ sz = int(line.strip(), 16) # hexadecimal
|
||||||
|
|
||||||
|
if sz > 0:
|
||||||
|
# Start a new chunk.
|
||||||
|
diff --git a/tests/test_functional.py b/tests/test_functional.py
|
||||||
|
index 853942c..448e0c0 100644
|
||||||
|
--- a/tests/test_functional.py
|
||||||
|
+++ b/tests/test_functional.py
|
||||||
|
@@ -345,6 +345,28 @@ class EchoTests(object):
|
||||||
|
self.send_check_error(to_send)
|
||||||
|
self.assertRaises(ConnectionClosed, read_http, fp)
|
||||||
|
|
||||||
|
+ def test_broken_chunked_encoding_invalid_hex(self):
|
||||||
|
+ control_line = b"0x20\r\n" # 20 hex = 32 dec
|
||||||
|
+ s = b"This string has 32 characters.\r\n"
|
||||||
|
+ to_send = b"GET / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"
|
||||||
|
+ to_send += control_line + s + b"\r\n"
|
||||||
|
+ self.connect()
|
||||||
|
+ self.sock.send(to_send)
|
||||||
|
+ with self.sock.makefile("rb", 0) as fp:
|
||||||
|
+ line, headers, response_body = read_http(fp)
|
||||||
|
+ self.assertline(line, "400", "Bad Request", "HTTP/1.1")
|
||||||
|
+ cl = int(headers["content-length"])
|
||||||
|
+ self.assertEqual(cl, len(response_body))
|
||||||
|
+ self.assertIn(b"Invalid chunk size", response_body)
|
||||||
|
+ self.assertEqual(
|
||||||
|
+ sorted(headers.keys()),
|
||||||
|
+ ["connection", "content-length", "content-type", "date", "server"],
|
||||||
|
+ )
|
||||||
|
+ self.assertEqual(headers["content-type"], "text/plain")
|
||||||
|
+ # connection has been closed
|
||||||
|
+ self.send_check_error(to_send)
|
||||||
|
+ self.assertRaises(ConnectionClosed, read_http, fp)
|
||||||
|
+
|
||||||
|
def test_broken_chunked_encoding_invalid_extension(self):
|
||||||
|
control_line = b"20;invalid=\r\n" # 20 hex = 32 dec
|
||||||
|
s = b"This string has 32 characters.\r\n"
|
||||||
|
diff --git a/tests/test_receiver.py b/tests/test_receiver.py
|
||||||
|
index a6261ea..17328d4 100644
|
||||||
|
--- a/tests/test_receiver.py
|
||||||
|
+++ b/tests/test_receiver.py
|
||||||
|
@@ -262,6 +262,18 @@ class TestChunkedReceiverParametrized:
|
||||||
|
assert result == len(data)
|
||||||
|
assert inst.error == None
|
||||||
|
|
||||||
|
+ @pytest.mark.parametrize("invalid_size", [b"0x04", b"+0x04", b"x04", b"+04"])
|
||||||
|
+ def test_received_invalid_size(self, invalid_size):
|
||||||
|
+ from waitress.utilities import BadRequest
|
||||||
|
+
|
||||||
|
+ buf = DummyBuffer()
|
||||||
|
+ inst = self._makeOne(buf)
|
||||||
|
+ data = invalid_size + b"\r\ntest\r\n"
|
||||||
|
+ result = inst.received(data)
|
||||||
|
+ assert result == len(data)
|
||||||
|
+ assert inst.error.__class__ == BadRequest
|
||||||
|
+ assert inst.error.body == "Invalid chunk size"
|
||||||
|
+
|
||||||
|
|
||||||
|
class DummyBuffer(object):
|
||||||
|
def __init__(self, data=None):
|
||||||
|
--
|
||||||
|
2.39.2 (Apple Git-143)
|
||||||
|
|
@ -0,0 +1,74 @@
|
|||||||
|
From ef0b3d7cb9f532c062052082f71174ef94d4a3e3 Mon Sep 17 00:00:00 2001
|
||||||
|
From: Renata Ravanelli <renata.ravanelli@gmail.com>
|
||||||
|
Date: Fri, 15 Sep 2023 12:41:52 -0300
|
||||||
|
Subject: [PATCH 6/6] This patch is a backport of commit bd22869
|
||||||
|
|
||||||
|
From bd22869 Mon Sep 17 00:00:00 2001
|
||||||
|
From: Bert JW Regeer <bertjw@regeer.org>
|
||||||
|
Date: Sat, 12 Mar 2022 19:16:23 -0700
|
||||||
|
Subject: [PATCH] Remove extraneous calls to .strip() in Chunked Encoding
|
||||||
|
|
||||||
|
To be valid chunked encoding we should not be removing any whitespace as
|
||||||
|
the standard does not allow for optional whitespace.
|
||||||
|
|
||||||
|
If whitespace is encountered in the wrong place, it should lead to a 400
|
||||||
|
Bad Request instead.
|
||||||
|
|
||||||
|
Backport:
|
||||||
|
* Patch refresh - no functional change.
|
||||||
|
|
||||||
|
Signed-off-by: Renata Ravanelli <renata.ravanelli@gmail.com>
|
||||||
|
---
|
||||||
|
src/waitress/receiver.py | 6 +-----
|
||||||
|
tests/test_receiver.py | 4 +++-
|
||||||
|
2 files changed, 4 insertions(+), 6 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/src/waitress/receiver.py b/src/waitress/receiver.py
|
||||||
|
index 9e4bffe..806ff87 100644
|
||||||
|
--- a/src/waitress/receiver.py
|
||||||
|
+++ b/src/waitress/receiver.py
|
||||||
|
@@ -135,7 +135,6 @@ class ChunkedReceiver(object):
|
||||||
|
line = s[:pos]
|
||||||
|
s = s[pos + 2 :]
|
||||||
|
self.control_line = b""
|
||||||
|
- line = line.strip()
|
||||||
|
|
||||||
|
if line:
|
||||||
|
# Begin a new chunk.
|
||||||
|
@@ -153,9 +152,6 @@ class ChunkedReceiver(object):
|
||||||
|
|
||||||
|
line = line[:semi]
|
||||||
|
|
||||||
|
- # Remove any whitespace
|
||||||
|
- line = line.strip()
|
||||||
|
-
|
||||||
|
if not ONLY_HEXDIG_RE.match(line):
|
||||||
|
self.error = BadRequest("Invalid chunk size")
|
||||||
|
self.all_chunks_received = True
|
||||||
|
@@ -164,7 +160,7 @@ class ChunkedReceiver(object):
|
||||||
|
|
||||||
|
# Can not fail due to matching against the regular
|
||||||
|
# expression above
|
||||||
|
- sz = int(line.strip(), 16) # hexadecimal
|
||||||
|
+ sz = int(line, 16) # hexadecimal
|
||||||
|
|
||||||
|
if sz > 0:
|
||||||
|
# Start a new chunk.
|
||||||
|
diff --git a/tests/test_receiver.py b/tests/test_receiver.py
|
||||||
|
index 17328d4..014f785 100644
|
||||||
|
--- a/tests/test_receiver.py
|
||||||
|
+++ b/tests/test_receiver.py
|
||||||
|
@@ -262,7 +262,9 @@ class TestChunkedReceiverParametrized:
|
||||||
|
assert result == len(data)
|
||||||
|
assert inst.error == None
|
||||||
|
|
||||||
|
- @pytest.mark.parametrize("invalid_size", [b"0x04", b"+0x04", b"x04", b"+04"])
|
||||||
|
+ @pytest.mark.parametrize(
|
||||||
|
+ "invalid_size", [b"0x04", b"+0x04", b"x04", b"+04", b" 04", b" 0x04"]
|
||||||
|
+ )
|
||||||
|
def test_received_invalid_size(self, invalid_size):
|
||||||
|
from waitress.utilities import BadRequest
|
||||||
|
|
||||||
|
--
|
||||||
|
2.39.2 (Apple Git-143)
|
||||||
|
|
Loading…
Reference in new issue