Compare commits

...

No commits in common. 'c9-beta' and 'i9' have entirely different histories.
c9-beta ... i9

3
.gitignore vendored

@ -1 +1,2 @@
SOURCES/Python-3.11.9.tar.xz SOURCES/Python-3.11.2.tar.xz
SOURCES/pgp_keys.asc

@ -1 +1,2 @@
926cd6a577b2e8dcbb17671b30eda04019328ada SOURCES/Python-3.11.9.tar.xz ae1c199ecb7a969588b15354e19e7b60cb65d1b9 SOURCES/Python-3.11.2.tar.xz
befe131eceffa73877ba3adceca3b7b1da1d2319 SOURCES/pgp_keys.asc

@ -1,4 +1,4 @@
From 4345f8ea8a56a58ef8a48439c0e201702d1012a2 Mon Sep 17 00:00:00 2001 From c96f1bea2ffc5c0ca849d5406236c07ea229a64f Mon Sep 17 00:00:00 2001
From: Charalampos Stratakis <cstratak@redhat.com> From: Charalampos Stratakis <cstratak@redhat.com>
Date: Thu, 12 Dec 2019 16:58:31 +0100 Date: Thu, 12 Dec 2019 16:58:31 +0100
Subject: [PATCH 1/7] Expose blake2b and blake2s hashes from OpenSSL Subject: [PATCH 1/7] Expose blake2b and blake2s hashes from OpenSSL
@ -29,10 +29,10 @@ index 67becdd..6607ef7 100644
computed = m.hexdigest() if not shake else m.hexdigest(length) computed = m.hexdigest() if not shake else m.hexdigest(length)
self.assertEqual( self.assertEqual(
diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c
index 57d64bd..d0c3b9e 100644 index 3c40f09..e819d02 100644
--- a/Modules/_hashopenssl.c --- a/Modules/_hashopenssl.c
+++ b/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c
@@ -1078,6 +1078,41 @@ _hashlib_openssl_sha512_impl(PyObject *module, PyObject *data_obj, @@ -1077,6 +1077,41 @@ _hashlib_openssl_sha512_impl(PyObject *module, PyObject *data_obj,
} }
@ -74,7 +74,7 @@ index 57d64bd..d0c3b9e 100644
#ifdef PY_OPENSSL_HAS_SHA3 #ifdef PY_OPENSSL_HAS_SHA3
/*[clinic input] /*[clinic input]
@@ -2066,6 +2101,8 @@ static struct PyMethodDef EVP_functions[] = { @@ -2065,6 +2100,8 @@ static struct PyMethodDef EVP_functions[] = {
_HASHLIB_OPENSSL_SHA256_METHODDEF _HASHLIB_OPENSSL_SHA256_METHODDEF
_HASHLIB_OPENSSL_SHA384_METHODDEF _HASHLIB_OPENSSL_SHA384_METHODDEF
_HASHLIB_OPENSSL_SHA512_METHODDEF _HASHLIB_OPENSSL_SHA512_METHODDEF
@ -205,10 +205,10 @@ index 5d84f4a..011026a 100644
-/*[clinic end generated code: output=69f2374071bff707 input=a9049054013a1b77]*/ -/*[clinic end generated code: output=69f2374071bff707 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=c6a9af5563972eda input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c6a9af5563972eda input=a9049054013a1b77]*/
-- --
2.45.0 2.39.1
From 1f79be1a11ad6811913c239da980c5bab0f1c538 Mon Sep 17 00:00:00 2001 From 9a7e164840aa35602e1c6dddadd461fafc666a63 Mon Sep 17 00:00:00 2001
From: Petr Viktorin <pviktori@redhat.com> From: Petr Viktorin <pviktori@redhat.com>
Date: Thu, 1 Aug 2019 17:57:05 +0200 Date: Thu, 1 Aug 2019 17:57:05 +0200
Subject: [PATCH 2/7] Use a stronger hash in multiprocessing handshake Subject: [PATCH 2/7] Use a stronger hash in multiprocessing handshake
@ -220,10 +220,10 @@ https://bugs.python.org/issue17258
1 file changed, 6 insertions(+), 2 deletions(-) 1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py
index 59c61d2..7fc594e 100644 index b08144f..0497557 100644
--- a/Lib/multiprocessing/connection.py --- a/Lib/multiprocessing/connection.py
+++ b/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py
@@ -43,6 +43,10 @@ BUFSIZE = 8192 @@ -42,6 +42,10 @@ BUFSIZE = 8192
# A very generous timeout when it comes to local connections... # A very generous timeout when it comes to local connections...
CONNECTION_TIMEOUT = 20. CONNECTION_TIMEOUT = 20.
@ -234,7 +234,7 @@ index 59c61d2..7fc594e 100644
_mmap_counter = itertools.count() _mmap_counter = itertools.count()
default_family = 'AF_INET' default_family = 'AF_INET'
@@ -753,7 +757,7 @@ def deliver_challenge(connection, authkey): @@ -735,7 +739,7 @@ def deliver_challenge(connection, authkey):
"Authkey must be bytes, not {0!s}".format(type(authkey))) "Authkey must be bytes, not {0!s}".format(type(authkey)))
message = os.urandom(MESSAGE_LENGTH) message = os.urandom(MESSAGE_LENGTH)
connection.send_bytes(CHALLENGE + message) connection.send_bytes(CHALLENGE + message)
@ -243,7 +243,7 @@ index 59c61d2..7fc594e 100644
response = connection.recv_bytes(256) # reject large message response = connection.recv_bytes(256) # reject large message
if response == digest: if response == digest:
connection.send_bytes(WELCOME) connection.send_bytes(WELCOME)
@@ -769,7 +773,7 @@ def answer_challenge(connection, authkey): @@ -751,7 +755,7 @@ def answer_challenge(connection, authkey):
message = connection.recv_bytes(256) # reject large message message = connection.recv_bytes(256) # reject large message
assert message[:len(CHALLENGE)] == CHALLENGE, 'message = %r' % message assert message[:len(CHALLENGE)] == CHALLENGE, 'message = %r' % message
message = message[len(CHALLENGE):] message = message[len(CHALLENGE):]
@ -253,10 +253,10 @@ index 59c61d2..7fc594e 100644
response = connection.recv_bytes(256) # reject large message response = connection.recv_bytes(256) # reject large message
if response != WELCOME: if response != WELCOME:
-- --
2.45.0 2.39.1
From e069ed2dcd0edf0de489eb387267fb35a92ed506 Mon Sep 17 00:00:00 2001 From 10b91783a2f22153738c5658a98daf7475ad9a8c Mon Sep 17 00:00:00 2001
From: Petr Viktorin <pviktori@redhat.com> From: Petr Viktorin <pviktori@redhat.com>
Date: Thu, 25 Jul 2019 17:19:06 +0200 Date: Thu, 25 Jul 2019 17:19:06 +0200
Subject: [PATCH 3/7] Disable Python's hash implementations in FIPS mode, Subject: [PATCH 3/7] Disable Python's hash implementations in FIPS mode,
@ -359,7 +359,7 @@ index c2cac98..55b1677 100644
if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE)
diff --git a/Modules/_blake2/blake2module.c b/Modules/_blake2/blake2module.c diff --git a/Modules/_blake2/blake2module.c b/Modules/_blake2/blake2module.c
index 93478f5..e3a024d 100644 index 44d783b..d247e44 100644
--- a/Modules/_blake2/blake2module.c --- a/Modules/_blake2/blake2module.c
+++ b/Modules/_blake2/blake2module.c +++ b/Modules/_blake2/blake2module.c
@@ -13,6 +13,7 @@ @@ -13,6 +13,7 @@
@ -370,7 +370,7 @@ index 93478f5..e3a024d 100644
#include "blake2module.h" #include "blake2module.h"
extern PyType_Spec blake2b_type_spec; extern PyType_Spec blake2b_type_spec;
@@ -83,6 +84,7 @@ _blake2_free(void *module) @@ -77,6 +78,7 @@ _blake2_free(void *module)
static int static int
blake2_exec(PyObject *m) blake2_exec(PyObject *m)
{ {
@ -378,7 +378,7 @@ index 93478f5..e3a024d 100644
Blake2State* st = blake2_get_state(m); Blake2State* st = blake2_get_state(m);
st->blake2b_type = (PyTypeObject *)PyType_FromModuleAndSpec( st->blake2b_type = (PyTypeObject *)PyType_FromModuleAndSpec(
@@ -154,5 +156,6 @@ static struct PyModuleDef blake2_module = { @@ -145,5 +147,6 @@ static struct PyModuleDef blake2_module = {
PyMODINIT_FUNC PyMODINIT_FUNC
PyInit__blake2(void) PyInit__blake2(void)
{ {
@ -446,10 +446,10 @@ index 56ae7a5..45fb403 100644
+ if (_Py_hashlib_fips_error(exc, name)) return NULL; \ + if (_Py_hashlib_fips_error(exc, name)) return NULL; \
+} while (0) +} while (0)
diff --git a/configure.ac b/configure.ac diff --git a/configure.ac b/configure.ac
index 7b4000f..8e2f0ad 100644 index c62a565..861f7a0 100644
--- a/configure.ac --- a/configure.ac
+++ b/configure.ac +++ b/configure.ac
@@ -7070,7 +7070,8 @@ PY_STDLIB_MOD([_sha512], [test "$with_builtin_sha512" = yes]) @@ -7044,7 +7044,8 @@ PY_STDLIB_MOD([_sha512], [test "$with_builtin_sha512" = yes])
PY_STDLIB_MOD([_sha3], [test "$with_builtin_sha3" = yes]) PY_STDLIB_MOD([_sha3], [test "$with_builtin_sha3" = yes])
PY_STDLIB_MOD([_blake2], PY_STDLIB_MOD([_blake2],
[test "$with_builtin_blake2" = yes], [], [test "$with_builtin_blake2" = yes], [],
@ -460,10 +460,10 @@ index 7b4000f..8e2f0ad 100644
PY_STDLIB_MOD([_crypt], PY_STDLIB_MOD([_crypt],
[], [test "$ac_cv_crypt_crypt" = yes], [], [test "$ac_cv_crypt_crypt" = yes],
-- --
2.45.0 2.39.1
From 2e0c5086f4a52803595e19795111278c3c80ee2f Mon Sep 17 00:00:00 2001 From e26066b1c05c9768e38cb6f45d6a01058de55b3f Mon Sep 17 00:00:00 2001
From: Charalampos Stratakis <cstratak@redhat.com> From: Charalampos Stratakis <cstratak@redhat.com>
Date: Fri, 29 Jan 2021 14:16:21 +0100 Date: Fri, 29 Jan 2021 14:16:21 +0100
Subject: [PATCH 4/7] Use python's fall back crypto implementations only if we Subject: [PATCH 4/7] Use python's fall back crypto implementations only if we
@ -623,10 +623,10 @@ index 01d12f5..a7cdb07 100644
def test_pbkdf2_hmac_py(self): def test_pbkdf2_hmac_py(self):
with warnings_helper.check_warnings(): with warnings_helper.check_warnings():
-- --
2.45.0 2.39.1
From 0e1d2a67ef66cccc9afa4a515dc34ce587946f22 Mon Sep 17 00:00:00 2001 From 9ccbd22b8538fee379717c8b2916dc1ff8b96f07 Mon Sep 17 00:00:00 2001
From: Charalampos Stratakis <cstratak@redhat.com> From: Charalampos Stratakis <cstratak@redhat.com>
Date: Wed, 31 Jul 2019 15:43:43 +0200 Date: Wed, 31 Jul 2019 15:43:43 +0200
Subject: [PATCH 5/7] Test equivalence of hashes for the various digests with Subject: [PATCH 5/7] Test equivalence of hashes for the various digests with
@ -783,21 +783,21 @@ index a7cdb07..c071f28 100644
class KDFTests(unittest.TestCase): class KDFTests(unittest.TestCase):
-- --
2.45.0 2.39.1
From f1c9ecbb2e2f08d792fb0557058824eed23abb7b Mon Sep 17 00:00:00 2001 From c3b8d6ecc76c87e8b05fd2cb212d5dece50ce0b1 Mon Sep 17 00:00:00 2001
From: Petr Viktorin <pviktori@redhat.com> From: Petr Viktorin <pviktori@redhat.com>
Date: Mon, 26 Aug 2019 19:39:48 +0200 Date: Mon, 26 Aug 2019 19:39:48 +0200
Subject: [PATCH 6/7] Guard against Python HMAC in FIPS mode Subject: [PATCH 6/7] Guard against Python HMAC in FIPS mode
--- ---
Lib/hmac.py | 12 +++++++++--- Lib/hmac.py | 13 +++++++++----
Lib/test/test_hmac.py | 10 ++++++++++ Lib/test/test_hmac.py | 10 ++++++++++
2 files changed, 19 insertions(+), 3 deletions(-) 2 files changed, 19 insertions(+), 4 deletions(-)
diff --git a/Lib/hmac.py b/Lib/hmac.py diff --git a/Lib/hmac.py b/Lib/hmac.py
index 8b4eb2f..8930bda 100644 index 8b4f920..20ef96c 100644
--- a/Lib/hmac.py --- a/Lib/hmac.py
+++ b/Lib/hmac.py +++ b/Lib/hmac.py
@@ -16,8 +16,9 @@ else: @@ -16,8 +16,9 @@ else:
@ -812,9 +812,16 @@ index 8b4eb2f..8930bda 100644
# The size of the digests returned by HMAC depends on the underlying # The size of the digests returned by HMAC depends on the underlying
# hashing module used. Use digest_size from the instance of HMAC instead. # hashing module used. Use digest_size from the instance of HMAC instead.
@@ -55,10 +56,12 @@ class HMAC: @@ -48,17 +49,18 @@ class HMAC:
msg argument. Passing it as a keyword argument is
recommended, though not required for legacy API reasons.
"""
-
if not isinstance(key, (bytes, bytearray)):
raise TypeError("key: expected bytes or bytearray, but got %r" % type(key).__name__)
if not digestmod: if not digestmod:
raise TypeError("Missing required argument 'digestmod'.") raise TypeError("Missing required parameter 'digestmod'.")
- if _hashopenssl and isinstance(digestmod, (str, _functype)): - if _hashopenssl and isinstance(digestmod, (str, _functype)):
+ if _hashopenssl.get_fips_mode() or (_hashopenssl and isinstance(digestmod, (str, _functype))): + if _hashopenssl.get_fips_mode() or (_hashopenssl and isinstance(digestmod, (str, _functype))):
@ -826,7 +833,7 @@ index 8b4eb2f..8930bda 100644
self._init_old(key, msg, digestmod) self._init_old(key, msg, digestmod)
else: else:
self._init_old(key, msg, digestmod) self._init_old(key, msg, digestmod)
@@ -69,6 +72,9 @@ class HMAC: @@ -69,6 +71,9 @@ class HMAC:
self.block_size = self._hmac.block_size self.block_size = self._hmac.block_size
def _init_old(self, key, msg, digestmod): def _init_old(self, key, msg, digestmod):
@ -837,7 +844,7 @@ index 8b4eb2f..8930bda 100644
digest_cons = digestmod digest_cons = digestmod
elif isinstance(digestmod, str): elif isinstance(digestmod, str):
diff --git a/Lib/test/test_hmac.py b/Lib/test/test_hmac.py diff --git a/Lib/test/test_hmac.py b/Lib/test/test_hmac.py
index 1502fba..e40ca4b 100644 index 7cf9973..a9e4e39 100644
--- a/Lib/test/test_hmac.py --- a/Lib/test/test_hmac.py
+++ b/Lib/test/test_hmac.py +++ b/Lib/test/test_hmac.py
@@ -5,6 +5,7 @@ import hashlib @@ -5,6 +5,7 @@ import hashlib
@ -868,7 +875,7 @@ index 1502fba..e40ca4b 100644
with warnings.catch_warnings(): with warnings.catch_warnings():
warnings.simplefilter('error', RuntimeWarning) warnings.simplefilter('error', RuntimeWarning)
with self.assertRaises(RuntimeWarning): with self.assertRaises(RuntimeWarning):
@@ -453,6 +460,7 @@ class ConstructorTestCase(unittest.TestCase): @@ -443,6 +450,7 @@ class ConstructorTestCase(unittest.TestCase):
with self.assertRaisesRegex(TypeError, "immutable type"): with self.assertRaisesRegex(TypeError, "immutable type"):
C_HMAC.value = None C_HMAC.value = None
@ -876,7 +883,7 @@ index 1502fba..e40ca4b 100644
@unittest.skipUnless(sha256_module is not None, 'need _sha256') @unittest.skipUnless(sha256_module is not None, 'need _sha256')
def test_with_sha256_module(self): def test_with_sha256_module(self):
h = hmac.HMAC(b"key", b"hash this!", digestmod=sha256_module.sha256) h = hmac.HMAC(b"key", b"hash this!", digestmod=sha256_module.sha256)
@@ -489,6 +497,7 @@ class UpdateTestCase(unittest.TestCase): @@ -471,6 +479,7 @@ class SanityTestCase(unittest.TestCase):
class CopyTestCase(unittest.TestCase): class CopyTestCase(unittest.TestCase):
@ -884,7 +891,7 @@ index 1502fba..e40ca4b 100644
@hashlib_helper.requires_hashdigest('sha256') @hashlib_helper.requires_hashdigest('sha256')
def test_attributes_old(self): def test_attributes_old(self):
# Testing if attributes are of same type. # Testing if attributes are of same type.
@@ -500,6 +509,7 @@ class CopyTestCase(unittest.TestCase): @@ -482,6 +491,7 @@ class CopyTestCase(unittest.TestCase):
self.assertEqual(type(h1._outer), type(h2._outer), self.assertEqual(type(h1._outer), type(h2._outer),
"Types of outer don't match.") "Types of outer don't match.")
@ -893,43 +900,290 @@ index 1502fba..e40ca4b 100644
def test_realcopy_old(self): def test_realcopy_old(self):
# Testing if the copy method created a real copy. # Testing if the copy method created a real copy.
-- --
2.45.0 2.39.1
From 2b06ee89344e8735cdc8435aadbdf83fe289e934 Mon Sep 17 00:00:00 2001
From: Petr Viktorin <encukou@gmail.com>
Date: Wed, 25 Aug 2021 16:44:43 +0200
Subject: [PATCH 7/7] Disable hash-based PYCs in FIPS mode
From a0c3f9ac5a4e60ab22418a3196ae46ba34e9477b Mon Sep 17 00:00:00 2001 If FIPS mode is on, we can't use siphash-based HMAC
From: Nikita Sobolev <mail@sobolevn.me> (_Py_KeyedHash), so:
Date: Thu, 24 Nov 2022 01:47:31 +0300
Subject: [PATCH 7/7] closes gh-99508: fix `TypeError` in
`Lib/importlib/_bootstrap_external.py` (GH-99635)
- Unchecked hash PYCs can be imported, but not created
- Checked hash PYCs can not be imported nor created
- The default mode is timestamp-based PYCs, even if
SOURCE_DATE_EPOCH is set.
If FIPS mode is off, there are no changes in behavior.
Resolves: https://bugzilla.redhat.com/show_bug.cgi?id=1835169
--- ---
Lib/importlib/_bootstrap_external.py | 3 ++- Lib/py_compile.py | 2 ++
.../next/Library/2022-11-21-10-45-54.gh-issue-99508.QqVbby.rst | 2 ++ Lib/test/support/__init__.py | 14 +++++++++++++
2 files changed, 4 insertions(+), 1 deletion(-) Lib/test/test_cmd_line_script.py | 2 ++
create mode 100644 Misc/NEWS.d/next/Library/2022-11-21-10-45-54.gh-issue-99508.QqVbby.rst Lib/test/test_compileall.py | 11 +++++++++-
Lib/test/test_imp.py | 2 ++
.../test_importlib/source/test_file_loader.py | 6 ++++++
Lib/test/test_py_compile.py | 11 ++++++++--
Lib/test/test_zipimport.py | 2 ++
Python/import.c | 20 +++++++++++++++++++
9 files changed, 67 insertions(+), 3 deletions(-)
diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py diff --git a/Lib/py_compile.py b/Lib/py_compile.py
index e53f6ac..bdc491e 100644 index db52725..5fca65e 100644
--- a/Lib/importlib/_bootstrap_external.py --- a/Lib/py_compile.py
+++ b/Lib/importlib/_bootstrap_external.py +++ b/Lib/py_compile.py
@@ -1077,7 +1077,8 @@ class SourceLoader(_LoaderBasics): @@ -70,7 +70,9 @@ class PycInvalidationMode(enum.Enum):
source_mtime is not None):
if hash_based:
if source_hash is None: def _get_default_invalidation_mode():
- source_hash = _imp.source_hash(source_bytes) + import _hashlib
+ source_hash = _imp.source_hash(_RAW_MAGIC_NUMBER, if (os.environ.get('SOURCE_DATE_EPOCH') and not
+ source_bytes) + _hashlib.get_fips_mode() and not
data = _code_to_hash_pyc(code_object, source_hash, check_source) os.environ.get('RPM_BUILD_ROOT')):
else: return PycInvalidationMode.CHECKED_HASH
data = _code_to_timestamp_pyc(code_object, source_mtime, else:
diff --git a/Misc/NEWS.d/next/Library/2022-11-21-10-45-54.gh-issue-99508.QqVbby.rst b/Misc/NEWS.d/next/Library/2022-11-21-10-45-54.gh-issue-99508.QqVbby.rst diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
new file mode 100644 index c33f90d..7d40540 100644
index 0000000..82720d1 --- a/Lib/test/support/__init__.py
--- /dev/null +++ b/Lib/test/support/__init__.py
+++ b/Misc/NEWS.d/next/Library/2022-11-21-10-45-54.gh-issue-99508.QqVbby.rst @@ -2225,6 +2225,20 @@ def requires_venv_with_pip():
@@ -0,0 +1,2 @@ return unittest.skipUnless(ctypes, 'venv: pip requires ctypes')
+Fix ``TypeError`` in ``Lib/importlib/_bootstrap_external.py`` while calling
+``_imp.source_hash()``.
+def fails_in_fips_mode(expected_error):
+ import _hashlib
+ if _hashlib.get_fips_mode():
+ def _decorator(func):
+ def _wrapper(self, *args, **kwargs):
+ with self.assertRaises(expected_error):
+ func(self, *args, **kwargs)
+ return _wrapper
+ else:
+ def _decorator(func):
+ return func
+ return _decorator
+
+
@contextlib.contextmanager
def adjust_int_max_str_digits(max_digits):
"""Temporarily change the integer string conversion length limit."""
diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py
index 4dadbc0..7dc7e51 100644
--- a/Lib/test/test_cmd_line_script.py
+++ b/Lib/test/test_cmd_line_script.py
@@ -286,6 +286,7 @@ class CmdLineTest(unittest.TestCase):
self._check_script(zip_name, run_name, zip_name, zip_name, '',
zipimport.zipimporter)
+ @support.fails_in_fips_mode(ImportError)
def test_zipfile_compiled_checked_hash(self):
with os_helper.temp_dir() as script_dir:
script_name = _make_test_script(script_dir, '__main__')
@@ -296,6 +297,7 @@ class CmdLineTest(unittest.TestCase):
self._check_script(zip_name, run_name, zip_name, zip_name, '',
zipimport.zipimporter)
+ @support.fails_in_fips_mode(ImportError)
def test_zipfile_compiled_unchecked_hash(self):
with os_helper.temp_dir() as script_dir:
script_name = _make_test_script(script_dir, '__main__')
diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py
index 05154c8..c678d4a 100644
--- a/Lib/test/test_compileall.py
+++ b/Lib/test/test_compileall.py
@@ -800,14 +800,23 @@ class CommandLineTestsBase:
out = self.assertRunOK('badfilename')
self.assertRegex(out, b"Can't list 'badfilename'")
- def test_pyc_invalidation_mode(self):
+ @support.fails_in_fips_mode(AssertionError)
+ def test_pyc_invalidation_mode_checked(self):
script_helper.make_script(self.pkgdir, 'f1', '')
pyc = importlib.util.cache_from_source(
os.path.join(self.pkgdir, 'f1.py'))
+
self.assertRunOK('--invalidation-mode=checked-hash', self.pkgdir)
with open(pyc, 'rb') as fp:
data = fp.read()
self.assertEqual(int.from_bytes(data[4:8], 'little'), 0b11)
+
+ @support.fails_in_fips_mode(AssertionError)
+ def test_pyc_invalidation_mode_unchecked(self):
+ script_helper.make_script(self.pkgdir, 'f1', '')
+ pyc = importlib.util.cache_from_source(
+ os.path.join(self.pkgdir, 'f1.py'))
+
self.assertRunOK('--invalidation-mode=unchecked-hash', self.pkgdir)
with open(pyc, 'rb') as fp:
data = fp.read()
diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py
index 4bb0390..ff62483 100644
--- a/Lib/test/test_imp.py
+++ b/Lib/test/test_imp.py
@@ -350,6 +350,7 @@ class ImportTests(unittest.TestCase):
import _frozen_importlib
self.assertEqual(_frozen_importlib.__spec__.origin, "frozen")
+ @support.fails_in_fips_mode(ImportError)
def test_source_hash(self):
self.assertEqual(_imp.source_hash(42, b'hi'), b'\xfb\xd9G\x05\xaf$\x9b~')
self.assertEqual(_imp.source_hash(43, b'hi'), b'\xd0/\x87C\xccC\xff\xe2')
@@ -369,6 +370,7 @@ class ImportTests(unittest.TestCase):
res = script_helper.assert_python_ok(*args)
self.assertEqual(res.out.strip().decode('utf-8'), expected)
+ @support.fails_in_fips_mode(ImportError)
def test_find_and_load_checked_pyc(self):
# issue 34056
with os_helper.temp_cwd():
diff --git a/Lib/test/test_importlib/source/test_file_loader.py b/Lib/test/test_importlib/source/test_file_loader.py
index 378dcbe..7b223a1 100644
--- a/Lib/test/test_importlib/source/test_file_loader.py
+++ b/Lib/test/test_importlib/source/test_file_loader.py
@@ -16,6 +16,7 @@ import types
import unittest
import warnings
+from test import support
from test.support.import_helper import make_legacy_pyc, unload
from test.test_py_compile import without_source_date_epoch
@@ -238,6 +239,7 @@ class SimpleTest(abc.LoaderTests):
loader.load_module('bad name')
@util.writes_bytecode_files
+ @support.fails_in_fips_mode(ImportError)
def test_checked_hash_based_pyc(self):
with util.create_modules('_temp') as mapping:
source = mapping['_temp']
@@ -269,6 +271,7 @@ class SimpleTest(abc.LoaderTests):
)
@util.writes_bytecode_files
+ @support.fails_in_fips_mode(ImportError)
def test_overridden_checked_hash_based_pyc(self):
with util.create_modules('_temp') as mapping, \
unittest.mock.patch('_imp.check_hash_based_pycs', 'never'):
@@ -294,6 +297,7 @@ class SimpleTest(abc.LoaderTests):
self.assertEqual(mod.state, 'old')
@util.writes_bytecode_files
+ @support.fails_in_fips_mode(ImportError)
def test_unchecked_hash_based_pyc(self):
with util.create_modules('_temp') as mapping:
source = mapping['_temp']
@@ -324,6 +328,7 @@ class SimpleTest(abc.LoaderTests):
)
@util.writes_bytecode_files
+ @support.fails_in_fips_mode(ImportError)
def test_overridden_unchecked_hash_based_pyc(self):
with util.create_modules('_temp') as mapping, \
unittest.mock.patch('_imp.check_hash_based_pycs', 'always'):
@@ -433,6 +438,7 @@ class BadBytecodeTest:
del_source=del_source)
test('_temp', mapping, bc_path)
+ @support.fails_in_fips_mode(ImportError)
def _test_partial_hash(self, test, *, del_source=False):
with util.create_modules('_temp') as mapping:
bc_path = self.manipulate_bytecode(
diff --git a/Lib/test/test_py_compile.py b/Lib/test/test_py_compile.py
index e53f5d9..7266212 100644
--- a/Lib/test/test_py_compile.py
+++ b/Lib/test/test_py_compile.py
@@ -141,13 +141,16 @@ class PyCompileTestsBase:
importlib.util.cache_from_source(bad_coding)))
def test_source_date_epoch(self):
+ import _hashlib
py_compile.compile(self.source_path, self.pyc_path)
self.assertTrue(os.path.exists(self.pyc_path))
self.assertFalse(os.path.exists(self.cache_path))
with open(self.pyc_path, 'rb') as fp:
flags = importlib._bootstrap_external._classify_pyc(
fp.read(), 'test', {})
- if os.environ.get('SOURCE_DATE_EPOCH'):
+ if _hashlib.get_fips_mode():
+ expected_flags = 0b00
+ elif os.environ.get('SOURCE_DATE_EPOCH'):
expected_flags = 0b11
else:
expected_flags = 0b00
@@ -178,7 +181,8 @@ class PyCompileTestsBase:
# Specifying optimized bytecode should lead to a path reflecting that.
self.assertIn('opt-2', py_compile.compile(self.source_path, optimize=2))
- def test_invalidation_mode(self):
+ @support.fails_in_fips_mode(ImportError)
+ def test_invalidation_mode_checked(self):
py_compile.compile(
self.source_path,
invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH,
@@ -187,6 +191,9 @@ class PyCompileTestsBase:
flags = importlib._bootstrap_external._classify_pyc(
fp.read(), 'test', {})
self.assertEqual(flags, 0b11)
+
+ @support.fails_in_fips_mode(ImportError)
+ def test_invalidation_mode_unchecked(self):
py_compile.compile(
self.source_path,
invalidation_mode=py_compile.PycInvalidationMode.UNCHECKED_HASH,
diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py
index 59a5200..81fadb3 100644
--- a/Lib/test/test_zipimport.py
+++ b/Lib/test/test_zipimport.py
@@ -190,6 +190,7 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase):
TESTMOD + pyc_ext: (NOW, test_pyc)}
self.doTest(pyc_ext, files, TESTMOD)
+ @support.fails_in_fips_mode(ImportError)
def testUncheckedHashBasedPyc(self):
source = b"state = 'old'"
source_hash = importlib.util.source_hash(source)
@@ -204,6 +205,7 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase):
self.assertEqual(mod.state, 'old')
self.doTest(None, files, TESTMOD, call=check)
+ @support.fails_in_fips_mode(ImportError)
@unittest.mock.patch('_imp.check_hash_based_pycs', 'always')
def test_checked_hash_based_change_pyc(self):
source = b"state = 'old'"
diff --git a/Python/import.c b/Python/import.c
index 07a8b90..e97b47b 100644
--- a/Python/import.c
+++ b/Python/import.c
@@ -2437,6 +2437,26 @@ static PyObject *
_imp_source_hash_impl(PyObject *module, long key, Py_buffer *source)
/*[clinic end generated code: output=edb292448cf399ea input=9aaad1e590089789]*/
{
+ PyObject *_hashlib = PyImport_ImportModule("_hashlib");
+ if (_hashlib == NULL) {
+ return NULL;
+ }
+ PyObject *fips_mode_obj = PyObject_CallMethod(_hashlib, "get_fips_mode", NULL);
+ Py_DECREF(_hashlib);
+ if (fips_mode_obj == NULL) {
+ return NULL;
+ }
+ int fips_mode = PyObject_IsTrue(fips_mode_obj);
+ Py_DECREF(fips_mode_obj);
+ if (fips_mode < 0) {
+ return NULL;
+ }
+ if (fips_mode) {
+ PyErr_SetString(
+ PyExc_ImportError,
+ "hash-based PYC validation (siphash24) not available in FIPS mode");
+ return NULL;
+ };
union {
uint64_t x;
char data[sizeof(uint64_t)];
-- --
2.45.0 2.39.1

@ -1,250 +0,0 @@
From 0181d677dd7fd11bc19a211b3eb735ac3ad3d7fb Mon Sep 17 00:00:00 2001
From: Petr Viktorin <encukou@gmail.com>
Date: Mon, 6 Mar 2023 17:24:24 +0100
Subject: [PATCH] CVE-2007-4559, PEP-706: Add filters for tarfile extraction
(downstream)
Add and test RHEL-specific ways of configuring the default behavior: environment
variable and config file.
---
Lib/tarfile.py | 42 +++++++++++++
Lib/test/test_shutil.py | 3 +-
Lib/test/test_tarfile.py | 127 ++++++++++++++++++++++++++++++++++++++-
3 files changed, 168 insertions(+), 4 deletions(-)
diff --git a/Lib/tarfile.py b/Lib/tarfile.py
index 612217b..dc59fc6 100755
--- a/Lib/tarfile.py
+++ b/Lib/tarfile.py
@@ -72,6 +72,13 @@ __all__ = ["TarFile", "TarInfo", "is_tarfile", "TarError", "ReadError",
"ENCODING", "USTAR_FORMAT", "GNU_FORMAT", "PAX_FORMAT",
"DEFAULT_FORMAT", "open"]
+# If true, use the safer (but backwards-incompatible) 'tar' extraction filter,
+# rather than 'fully_trusted', by default.
+# The emitted warning is changed to match.
+_RH_SAFER_DEFAULT = True
+
+# System-wide configuration file
+_CONFIG_FILENAME = '/etc/python/tarfile.cfg'
#---------------------------------------------------------
# tar constants
@@ -2219,6 +2226,41 @@ class TarFile(object):
if filter is None:
filter = self.extraction_filter
if filter is None:
+ name = os.environ.get('PYTHON_TARFILE_EXTRACTION_FILTER')
+ if name is None:
+ try:
+ file = bltn_open(_CONFIG_FILENAME)
+ except FileNotFoundError:
+ pass
+ else:
+ import configparser
+ conf = configparser.ConfigParser(
+ interpolation=None,
+ comment_prefixes=('#', ),
+ )
+ with file:
+ conf.read_file(file)
+ name = conf.get('tarfile',
+ 'PYTHON_TARFILE_EXTRACTION_FILTER',
+ fallback='')
+ if name:
+ try:
+ filter = _NAMED_FILTERS[name]
+ except KeyError:
+ raise ValueError(f"filter {filter!r} not found") from None
+ self.extraction_filter = filter
+ return filter
+ if _RH_SAFER_DEFAULT:
+ warnings.warn(
+ 'The default behavior of tarfile extraction has been '
+ + 'changed to disallow common exploits '
+ + '(including CVE-2007-4559). '
+ + 'By default, absolute/parent paths are disallowed '
+ + 'and some mode bits are cleared. '
+ + 'See https://access.redhat.com/articles/7004769 '
+ + 'for more details.',
+ RuntimeWarning)
+ return tar_filter
return fully_trusted_filter
if isinstance(filter, str):
raise TypeError(
diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py
index 6728d30..2338b63 100644
--- a/Lib/test/test_shutil.py
+++ b/Lib/test/test_shutil.py
@@ -1774,7 +1774,8 @@ class TestArchives(BaseTest, unittest.TestCase):
def check_unpack_tarball(self, format):
self.check_unpack_archive(format, filter='fully_trusted')
self.check_unpack_archive(format, filter='data')
- with warnings_helper.check_no_warnings(self):
+ with warnings_helper.check_warnings(
+ ('.*CVE-2007-4559', RuntimeWarning)):
self.check_unpack_archive(format)
def test_unpack_archive_tar(self):
diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py
index 389da7b..5a43f9d 100644
--- a/Lib/test/test_tarfile.py
+++ b/Lib/test/test_tarfile.py
@@ -3,7 +3,7 @@ import sys
import os
import io
from hashlib import sha256
-from contextlib import contextmanager
+from contextlib import contextmanager, ExitStack
from random import Random
import pathlib
import shutil
@@ -3049,7 +3049,11 @@ class NoneInfoExtractTests(ReadTest):
tar = tarfile.open(tarname, mode='r', encoding="iso8859-1")
cls.control_dir = pathlib.Path(TEMPDIR) / "extractall_ctrl"
tar.errorlevel = 0
- tar.extractall(cls.control_dir, filter=cls.extraction_filter)
+ with ExitStack() as cm:
+ if cls.extraction_filter is None:
+ cm.enter_context(warnings.catch_warnings())
+ warnings.simplefilter(action="ignore", category=RuntimeWarning)
+ tar.extractall(cls.control_dir, filter=cls.extraction_filter)
tar.close()
cls.control_paths = set(
p.relative_to(cls.control_dir)
@@ -3868,7 +3872,8 @@ class TestExtractionFilters(unittest.TestCase):
"""Ensure the default filter does not warn (like in 3.12)"""
with ArchiveMaker() as arc:
arc.add('foo')
- with warnings_helper.check_no_warnings(self):
+ with warnings_helper.check_warnings(
+ ('.*CVE-2007-4559', RuntimeWarning)):
with self.check_context(arc.open(), None):
self.expect_file('foo')
@@ -4037,6 +4042,122 @@ class TestExtractionFilters(unittest.TestCase):
with self.check_context(arc.open(errorlevel='boo!'), filtererror_filter):
self.expect_exception(TypeError) # errorlevel is not int
+ @contextmanager
+ def rh_config_context(self, config_lines=None):
+ """Set up for testing various ways of overriding the default filter
+
+ return a triple with:
+ - temporary directory
+ - EnvironmentVarGuard()
+ - a test archive for use with check_* methods below
+
+ If config_lines is given, write them to the config file. Otherwise
+ the config file is missing.
+ """
+ tempdir = pathlib.Path(TEMPDIR) / 'tmp'
+ configfile = tempdir / 'tarfile.cfg'
+ with ArchiveMaker() as arc:
+ arc.add('good')
+ arc.add('ugly', symlink_to='/etc/passwd')
+ arc.add('../bad')
+ with (
+ os_helper.temp_dir(tempdir),
+ support.swap_attr(tarfile, '_CONFIG_FILENAME', str(configfile)),
+ os_helper.EnvironmentVarGuard() as env,
+ arc.open() as tar,
+ ):
+ if config_lines is not None:
+ with configfile.open('w') as f:
+ for line in config_lines:
+ print(line, file=f)
+ yield tempdir, env, tar
+
+ def check_rh_default_behavior(self, tar, tempdir):
+ """Check RH default: warn and refuse to extract dangerous files."""
+ with (
+ warnings_helper.check_warnings(
+ ('.*CVE-2007-4559', RuntimeWarning)),
+ self.assertRaises(tarfile.OutsideDestinationError),
+ ):
+ tar.extractall(tempdir / 'outdir')
+
+ def check_trusted_default(self, tar, tempdir):
+ """Check 'fully_trusted' is configured as the default filter."""
+ with (
+ warnings_helper.check_no_warnings(self),
+ ):
+ tar.extractall(tempdir / 'outdir')
+ self.assertTrue((tempdir / 'outdir/good').exists())
+ self.assertEqual((tempdir / 'outdir/ugly').readlink(),
+ pathlib.Path('/etc/passwd'))
+ self.assertTrue((tempdir / 'bad').exists())
+
+ def test_rh_default_no_conf(self):
+ with self.rh_config_context() as (tempdir, env, tar):
+ self.check_rh_default_behavior(tar, tempdir)
+
+ def test_rh_default_from_file(self):
+ lines = ['[tarfile]', 'PYTHON_TARFILE_EXTRACTION_FILTER=fully_trusted']
+ with self.rh_config_context(lines) as (tempdir, env, tar):
+ self.check_trusted_default(tar, tempdir)
+
+ def test_rh_empty_config_file(self):
+ """Empty config file -> default behavior"""
+ lines = []
+ with self.rh_config_context(lines) as (tempdir, env, tar):
+ self.check_rh_default_behavior(tar, tempdir)
+
+ def test_empty_config_section(self):
+ """Empty section in config file -> default behavior"""
+ lines = ['[tarfile]']
+ with self.rh_config_context(lines) as (tempdir, env, tar):
+ self.check_rh_default_behavior(tar, tempdir)
+
+ def test_rh_default_empty_config_option(self):
+ """Empty option value in config file -> default behavior"""
+ lines = ['[tarfile]', 'PYTHON_TARFILE_EXTRACTION_FILTER=']
+ with self.rh_config_context(lines) as (tempdir, env, tar):
+ self.check_rh_default_behavior(tar, tempdir)
+
+ def test_bad_config_option(self):
+ """Bad option value in config file -> ValueError"""
+ lines = ['[tarfile]', 'PYTHON_TARFILE_EXTRACTION_FILTER=unknown!']
+ with self.rh_config_context(lines) as (tempdir, env, tar):
+ with self.assertRaises(ValueError):
+ tar.extractall(tempdir / 'outdir')
+
+ def test_default_from_envvar(self):
+ with self.rh_config_context() as (tempdir, env, tar):
+ env['PYTHON_TARFILE_EXTRACTION_FILTER'] = 'fully_trusted'
+ self.check_trusted_default(tar, tempdir)
+
+ def test_empty_envvar(self):
+ """Empty env variable -> default behavior"""
+ with self.rh_config_context() as (tempdir, env, tar):
+ env['PYTHON_TARFILE_EXTRACTION_FILTER'] = ''
+ self.check_rh_default_behavior(tar, tempdir)
+
+ def test_bad_envvar(self):
+ with self.rh_config_context() as (tempdir, env, tar):
+ env['PYTHON_TARFILE_EXTRACTION_FILTER'] = 'unknown!'
+ with self.assertRaises(ValueError):
+ tar.extractall(tempdir / 'outdir')
+
+ def test_envvar_overrides_file(self):
+ lines = ['[tarfile]', 'PYTHON_TARFILE_EXTRACTION_FILTER=data']
+ with self.rh_config_context(lines) as (tempdir, env, tar):
+ env['PYTHON_TARFILE_EXTRACTION_FILTER'] = 'fully_trusted'
+ self.check_trusted_default(tar, tempdir)
+
+ def test_monkeypatch_overrides_envvar(self):
+ with self.rh_config_context(None) as (tempdir, env, tar):
+ env['PYTHON_TARFILE_EXTRACTION_FILTER'] = 'data'
+ with support.swap_attr(
+ tarfile.TarFile, 'extraction_filter',
+ staticmethod(tarfile.fully_trusted_filter)
+ ):
+ self.check_trusted_default(tar, tempdir)
+
class OverwriteTests(archiver_tests.OverwriteTests, unittest.TestCase):
testdir = os.path.join(TEMPDIR, "testoverwrite")
--
2.44.0

@ -1,748 +0,0 @@
From 642f28679e04c7b4ec7731f0c8872103f21a76f8 Mon Sep 17 00:00:00 2001
From: Victor Stinner <vstinner@python.org>
Date: Fri, 15 Dec 2023 16:10:40 +0100
Subject: [PATCH 1/2] 00415: [CVE-2023-27043] gh-102988: Reject malformed
addresses in email.parseaddr() (#111116)
Detect email address parsing errors and return empty tuple to
indicate the parsing error (old API). Add an optional 'strict'
parameter to getaddresses() and parseaddr() functions. Patch by
Thomas Dwyer.
Co-Authored-By: Thomas Dwyer <github@tomd.tel>
---
Doc/library/email.utils.rst | 19 +-
Lib/email/utils.py | 150 ++++++++++++-
Lib/test/test_email/test_email.py | 204 +++++++++++++++++-
...-10-20-15-28-08.gh-issue-102988.dStNO7.rst | 8 +
4 files changed, 360 insertions(+), 21 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2023-10-20-15-28-08.gh-issue-102988.dStNO7.rst
diff --git a/Doc/library/email.utils.rst b/Doc/library/email.utils.rst
index 0e266b6..6723dc4 100644
--- a/Doc/library/email.utils.rst
+++ b/Doc/library/email.utils.rst
@@ -60,13 +60,18 @@ of the new API.
begins with angle brackets, they are stripped off.
-.. function:: parseaddr(address)
+.. function:: parseaddr(address, *, strict=True)
Parse address -- which should be the value of some address-containing field such
as :mailheader:`To` or :mailheader:`Cc` -- into its constituent *realname* and
*email address* parts. Returns a tuple of that information, unless the parse
fails, in which case a 2-tuple of ``('', '')`` is returned.
+ If *strict* is true, use a strict parser which rejects malformed inputs.
+
+ .. versionchanged:: 3.13
+ Add *strict* optional parameter and reject malformed inputs by default.
+
.. function:: formataddr(pair, charset='utf-8')
@@ -84,12 +89,15 @@ of the new API.
Added the *charset* option.
-.. function:: getaddresses(fieldvalues)
+.. function:: getaddresses(fieldvalues, *, strict=True)
This method returns a list of 2-tuples of the form returned by ``parseaddr()``.
*fieldvalues* is a sequence of header field values as might be returned by
- :meth:`Message.get_all <email.message.Message.get_all>`. Here's a simple
- example that gets all the recipients of a message::
+ :meth:`Message.get_all <email.message.Message.get_all>`.
+
+ If *strict* is true, use a strict parser which rejects malformed inputs.
+
+ Here's a simple example that gets all the recipients of a message::
from email.utils import getaddresses
@@ -99,6 +107,9 @@ of the new API.
resent_ccs = msg.get_all('resent-cc', [])
all_recipients = getaddresses(tos + ccs + resent_tos + resent_ccs)
+ .. versionchanged:: 3.13
+ Add *strict* optional parameter and reject malformed inputs by default.
+
.. function:: parsedate(date)
diff --git a/Lib/email/utils.py b/Lib/email/utils.py
index 8993858..41bb3c9 100644
--- a/Lib/email/utils.py
+++ b/Lib/email/utils.py
@@ -106,12 +106,127 @@ def formataddr(pair, charset='utf-8'):
return address
+def _iter_escaped_chars(addr):
+ pos = 0
+ escape = False
+ for pos, ch in enumerate(addr):
+ if escape:
+ yield (pos, '\\' + ch)
+ escape = False
+ elif ch == '\\':
+ escape = True
+ else:
+ yield (pos, ch)
+ if escape:
+ yield (pos, '\\')
+
+
+def _strip_quoted_realnames(addr):
+ """Strip real names between quotes."""
+ if '"' not in addr:
+ # Fast path
+ return addr
+
+ start = 0
+ open_pos = None
+ result = []
+ for pos, ch in _iter_escaped_chars(addr):
+ if ch == '"':
+ if open_pos is None:
+ open_pos = pos
+ else:
+ if start != open_pos:
+ result.append(addr[start:open_pos])
+ start = pos + 1
+ open_pos = None
+
+ if start < len(addr):
+ result.append(addr[start:])
+
+ return ''.join(result)
+
+
+supports_strict_parsing = True
+
+def getaddresses(fieldvalues, *, strict=True):
+ """Return a list of (REALNAME, EMAIL) or ('','') for each fieldvalue.
+
+ When parsing fails for a fieldvalue, a 2-tuple of ('', '') is returned in
+ its place.
-def getaddresses(fieldvalues):
- """Return a list of (REALNAME, EMAIL) for each fieldvalue."""
- all = COMMASPACE.join(str(v) for v in fieldvalues)
- a = _AddressList(all)
- return a.addresslist
+ If strict is true, use a strict parser which rejects malformed inputs.
+ """
+
+ # If strict is true, if the resulting list of parsed addresses is greater
+ # than the number of fieldvalues in the input list, a parsing error has
+ # occurred and consequently a list containing a single empty 2-tuple [('',
+ # '')] is returned in its place. This is done to avoid invalid output.
+ #
+ # Malformed input: getaddresses(['alice@example.com <bob@example.com>'])
+ # Invalid output: [('', 'alice@example.com'), ('', 'bob@example.com')]
+ # Safe output: [('', '')]
+
+ if not strict:
+ all = COMMASPACE.join(str(v) for v in fieldvalues)
+ a = _AddressList(all)
+ return a.addresslist
+
+ fieldvalues = [str(v) for v in fieldvalues]
+ fieldvalues = _pre_parse_validation(fieldvalues)
+ addr = COMMASPACE.join(fieldvalues)
+ a = _AddressList(addr)
+ result = _post_parse_validation(a.addresslist)
+
+ # Treat output as invalid if the number of addresses is not equal to the
+ # expected number of addresses.
+ n = 0
+ for v in fieldvalues:
+ # When a comma is used in the Real Name part it is not a deliminator.
+ # So strip those out before counting the commas.
+ v = _strip_quoted_realnames(v)
+ # Expected number of addresses: 1 + number of commas
+ n += 1 + v.count(',')
+ if len(result) != n:
+ return [('', '')]
+
+ return result
+
+
+def _check_parenthesis(addr):
+ # Ignore parenthesis in quoted real names.
+ addr = _strip_quoted_realnames(addr)
+
+ opens = 0
+ for pos, ch in _iter_escaped_chars(addr):
+ if ch == '(':
+ opens += 1
+ elif ch == ')':
+ opens -= 1
+ if opens < 0:
+ return False
+ return (opens == 0)
+
+
+def _pre_parse_validation(email_header_fields):
+ accepted_values = []
+ for v in email_header_fields:
+ if not _check_parenthesis(v):
+ v = "('', '')"
+ accepted_values.append(v)
+
+ return accepted_values
+
+
+def _post_parse_validation(parsed_email_header_tuples):
+ accepted_values = []
+ # The parser would have parsed a correctly formatted domain-literal
+ # The existence of an [ after parsing indicates a parsing failure
+ for v in parsed_email_header_tuples:
+ if '[' in v[1]:
+ v = ('', '')
+ accepted_values.append(v)
+
+ return accepted_values
def _format_timetuple_and_zone(timetuple, zone):
@@ -205,16 +320,33 @@ def parsedate_to_datetime(data):
tzinfo=datetime.timezone(datetime.timedelta(seconds=tz)))
-def parseaddr(addr):
+def parseaddr(addr, *, strict=True):
"""
Parse addr into its constituent realname and email address parts.
Return a tuple of realname and email address, unless the parse fails, in
which case return a 2-tuple of ('', '').
+
+ If strict is True, use a strict parser which rejects malformed inputs.
"""
- addrs = _AddressList(addr).addresslist
- if not addrs:
- return '', ''
+ if not strict:
+ addrs = _AddressList(addr).addresslist
+ if not addrs:
+ return ('', '')
+ return addrs[0]
+
+ if isinstance(addr, list):
+ addr = addr[0]
+
+ if not isinstance(addr, str):
+ return ('', '')
+
+ addr = _pre_parse_validation([addr])[0]
+ addrs = _post_parse_validation(_AddressList(addr).addresslist)
+
+ if not addrs or len(addrs) > 1:
+ return ('', '')
+
return addrs[0]
diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py
index 785696e..ad60ed3 100644
--- a/Lib/test/test_email/test_email.py
+++ b/Lib/test/test_email/test_email.py
@@ -17,6 +17,7 @@ from unittest.mock import patch
import email
import email.policy
+import email.utils
from email.charset import Charset
from email.generator import Generator, DecodedGenerator, BytesGenerator
@@ -3336,15 +3337,154 @@ Foo
[('Al Person', 'aperson@dom.ain'),
('Bud Person', 'bperson@dom.ain')])
+ def test_getaddresses_comma_in_name(self):
+ """GH-106669 regression test."""
+ self.assertEqual(
+ utils.getaddresses(
+ [
+ '"Bud, Person" <bperson@dom.ain>',
+ 'aperson@dom.ain (Al Person)',
+ '"Mariusz Felisiak" <to@example.com>',
+ ]
+ ),
+ [
+ ('Bud, Person', 'bperson@dom.ain'),
+ ('Al Person', 'aperson@dom.ain'),
+ ('Mariusz Felisiak', 'to@example.com'),
+ ],
+ )
+
+ def test_parsing_errors(self):
+ """Test for parsing errors from CVE-2023-27043 and CVE-2019-16056"""
+ alice = 'alice@example.org'
+ bob = 'bob@example.com'
+ empty = ('', '')
+
+ # Test utils.getaddresses() and utils.parseaddr() on malformed email
+ # addresses: default behavior (strict=True) rejects malformed address,
+ # and strict=False which tolerates malformed address.
+ for invalid_separator, expected_non_strict in (
+ ('(', [(f'<{bob}>', alice)]),
+ (')', [('', alice), empty, ('', bob)]),
+ ('<', [('', alice), empty, ('', bob), empty]),
+ ('>', [('', alice), empty, ('', bob)]),
+ ('[', [('', f'{alice}[<{bob}>]')]),
+ (']', [('', alice), empty, ('', bob)]),
+ ('@', [empty, empty, ('', bob)]),
+ (';', [('', alice), empty, ('', bob)]),
+ (':', [('', alice), ('', bob)]),
+ ('.', [('', alice + '.'), ('', bob)]),
+ ('"', [('', alice), ('', f'<{bob}>')]),
+ ):
+ address = f'{alice}{invalid_separator}<{bob}>'
+ with self.subTest(address=address):
+ self.assertEqual(utils.getaddresses([address]),
+ [empty])
+ self.assertEqual(utils.getaddresses([address], strict=False),
+ expected_non_strict)
+
+ self.assertEqual(utils.parseaddr([address]),
+ empty)
+ self.assertEqual(utils.parseaddr([address], strict=False),
+ ('', address))
+
+ # Comma (',') is treated differently depending on strict parameter.
+ # Comma without quotes.
+ address = f'{alice},<{bob}>'
+ self.assertEqual(utils.getaddresses([address]),
+ [('', alice), ('', bob)])
+ self.assertEqual(utils.getaddresses([address], strict=False),
+ [('', alice), ('', bob)])
+ self.assertEqual(utils.parseaddr([address]),
+ empty)
+ self.assertEqual(utils.parseaddr([address], strict=False),
+ ('', address))
+
+ # Real name between quotes containing comma.
+ address = '"Alice, alice@example.org" <bob@example.com>'
+ expected_strict = ('Alice, alice@example.org', 'bob@example.com')
+ self.assertEqual(utils.getaddresses([address]), [expected_strict])
+ self.assertEqual(utils.getaddresses([address], strict=False), [expected_strict])
+ self.assertEqual(utils.parseaddr([address]), expected_strict)
+ self.assertEqual(utils.parseaddr([address], strict=False),
+ ('', address))
+
+ # Valid parenthesis in comments.
+ address = 'alice@example.org (Alice)'
+ expected_strict = ('Alice', 'alice@example.org')
+ self.assertEqual(utils.getaddresses([address]), [expected_strict])
+ self.assertEqual(utils.getaddresses([address], strict=False), [expected_strict])
+ self.assertEqual(utils.parseaddr([address]), expected_strict)
+ self.assertEqual(utils.parseaddr([address], strict=False),
+ ('', address))
+
+ # Invalid parenthesis in comments.
+ address = 'alice@example.org )Alice('
+ self.assertEqual(utils.getaddresses([address]), [empty])
+ self.assertEqual(utils.getaddresses([address], strict=False),
+ [('', 'alice@example.org'), ('', ''), ('', 'Alice')])
+ self.assertEqual(utils.parseaddr([address]), empty)
+ self.assertEqual(utils.parseaddr([address], strict=False),
+ ('', address))
+
+ # Two addresses with quotes separated by comma.
+ address = '"Jane Doe" <jane@example.net>, "John Doe" <john@example.net>'
+ self.assertEqual(utils.getaddresses([address]),
+ [('Jane Doe', 'jane@example.net'),
+ ('John Doe', 'john@example.net')])
+ self.assertEqual(utils.getaddresses([address], strict=False),
+ [('Jane Doe', 'jane@example.net'),
+ ('John Doe', 'john@example.net')])
+ self.assertEqual(utils.parseaddr([address]), empty)
+ self.assertEqual(utils.parseaddr([address], strict=False),
+ ('', address))
+
+ # Test email.utils.supports_strict_parsing attribute
+ self.assertEqual(email.utils.supports_strict_parsing, True)
+
def test_getaddresses_nasty(self):
- eq = self.assertEqual
- eq(utils.getaddresses(['foo: ;']), [('', '')])
- eq(utils.getaddresses(
- ['[]*-- =~$']),
- [('', ''), ('', ''), ('', '*--')])
- eq(utils.getaddresses(
- ['foo: ;', '"Jason R. Mastaler" <jason@dom.ain>']),
- [('', ''), ('Jason R. Mastaler', 'jason@dom.ain')])
+ for addresses, expected in (
+ (['"Sürname, Firstname" <to@example.com>'],
+ [('Sürname, Firstname', 'to@example.com')]),
+
+ (['foo: ;'],
+ [('', '')]),
+
+ (['foo: ;', '"Jason R. Mastaler" <jason@dom.ain>'],
+ [('', ''), ('Jason R. Mastaler', 'jason@dom.ain')]),
+
+ ([r'Pete(A nice \) chap) <pete(his account)@silly.test(his host)>'],
+ [('Pete (A nice ) chap his account his host)', 'pete@silly.test')]),
+
+ (['(Empty list)(start)Undisclosed recipients :(nobody(I know))'],
+ [('', '')]),
+
+ (['Mary <@machine.tld:mary@example.net>, , jdoe@test . example'],
+ [('Mary', 'mary@example.net'), ('', ''), ('', 'jdoe@test.example')]),
+
+ (['John Doe <jdoe@machine(comment). example>'],
+ [('John Doe (comment)', 'jdoe@machine.example')]),
+
+ (['"Mary Smith: Personal Account" <smith@home.example>'],
+ [('Mary Smith: Personal Account', 'smith@home.example')]),
+
+ (['Undisclosed recipients:;'],
+ [('', '')]),
+
+ ([r'<boss@nil.test>, "Giant; \"Big\" Box" <bob@example.net>'],
+ [('', 'boss@nil.test'), ('Giant; "Big" Box', 'bob@example.net')]),
+ ):
+ with self.subTest(addresses=addresses):
+ self.assertEqual(utils.getaddresses(addresses),
+ expected)
+ self.assertEqual(utils.getaddresses(addresses, strict=False),
+ expected)
+
+ addresses = ['[]*-- =~$']
+ self.assertEqual(utils.getaddresses(addresses),
+ [('', '')])
+ self.assertEqual(utils.getaddresses(addresses, strict=False),
+ [('', ''), ('', ''), ('', '*--')])
def test_getaddresses_embedded_comment(self):
"""Test proper handling of a nested comment"""
@@ -3535,6 +3675,54 @@ multipart/report
m = cls(*constructor, policy=email.policy.default)
self.assertIs(m.policy, email.policy.default)
+ def test_iter_escaped_chars(self):
+ self.assertEqual(list(utils._iter_escaped_chars(r'a\\b\"c\\"d')),
+ [(0, 'a'),
+ (2, '\\\\'),
+ (3, 'b'),
+ (5, '\\"'),
+ (6, 'c'),
+ (8, '\\\\'),
+ (9, '"'),
+ (10, 'd')])
+ self.assertEqual(list(utils._iter_escaped_chars('a\\')),
+ [(0, 'a'), (1, '\\')])
+
+ def test_strip_quoted_realnames(self):
+ def check(addr, expected):
+ self.assertEqual(utils._strip_quoted_realnames(addr), expected)
+
+ check('"Jane Doe" <jane@example.net>, "John Doe" <john@example.net>',
+ ' <jane@example.net>, <john@example.net>')
+ check(r'"Jane \"Doe\"." <jane@example.net>',
+ ' <jane@example.net>')
+
+ # special cases
+ check(r'before"name"after', 'beforeafter')
+ check(r'before"name"', 'before')
+ check(r'b"name"', 'b') # single char
+ check(r'"name"after', 'after')
+ check(r'"name"a', 'a') # single char
+ check(r'"name"', '')
+
+ # no change
+ for addr in (
+ 'Jane Doe <jane@example.net>, John Doe <john@example.net>',
+ 'lone " quote',
+ ):
+ self.assertEqual(utils._strip_quoted_realnames(addr), addr)
+
+
+ def test_check_parenthesis(self):
+ addr = 'alice@example.net'
+ self.assertTrue(utils._check_parenthesis(f'{addr} (Alice)'))
+ self.assertFalse(utils._check_parenthesis(f'{addr} )Alice('))
+ self.assertFalse(utils._check_parenthesis(f'{addr} (Alice))'))
+ self.assertFalse(utils._check_parenthesis(f'{addr} ((Alice)'))
+
+ # Ignore real name between quotes
+ self.assertTrue(utils._check_parenthesis(f'")Alice((" {addr}'))
+
# Test the iterator/generators
class TestIterators(TestEmailBase):
diff --git a/Misc/NEWS.d/next/Library/2023-10-20-15-28-08.gh-issue-102988.dStNO7.rst b/Misc/NEWS.d/next/Library/2023-10-20-15-28-08.gh-issue-102988.dStNO7.rst
new file mode 100644
index 0000000..3d0e9e4
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2023-10-20-15-28-08.gh-issue-102988.dStNO7.rst
@@ -0,0 +1,8 @@
+:func:`email.utils.getaddresses` and :func:`email.utils.parseaddr` now
+return ``('', '')`` 2-tuples in more situations where invalid email
+addresses are encountered instead of potentially inaccurate values. Add
+optional *strict* parameter to these two functions: use ``strict=False`` to
+get the old behavior, accept malformed inputs.
+``getattr(email.utils, 'supports_strict_parsing', False)`` can be use to check
+if the *strict* paramater is available. Patch by Thomas Dwyer and Victor
+Stinner to improve the CVE-2023-27043 fix.
--
2.44.0
From d371679e7c485551c10380ac11e5039a9fb4515b Mon Sep 17 00:00:00 2001
From: Lumir Balhar <lbalhar@redhat.com>
Date: Wed, 10 Jan 2024 08:53:53 +0100
Subject: [PATCH 2/2] Make it possible to disable strict parsing in email
module
---
Doc/library/email.utils.rst | 26 +++++++++++
Lib/email/utils.py | 55 ++++++++++++++++++++++-
Lib/test/test_email/test_email.py | 74 ++++++++++++++++++++++++++++++-
3 files changed, 151 insertions(+), 4 deletions(-)
diff --git a/Doc/library/email.utils.rst b/Doc/library/email.utils.rst
index 6723dc4..c89602d 100644
--- a/Doc/library/email.utils.rst
+++ b/Doc/library/email.utils.rst
@@ -69,6 +69,19 @@ of the new API.
If *strict* is true, use a strict parser which rejects malformed inputs.
+ The default setting for *strict* is set to ``True``, but you can override
+ it by setting the environment variable ``PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING``
+ to non-empty string.
+
+ Additionally, you can permanently set the default value for *strict* to
+ ``False`` by creating the configuration file ``/etc/python/email.cfg``
+ with the following content:
+
+ .. code-block:: ini
+
+ [email_addr_parsing]
+ PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING = true
+
.. versionchanged:: 3.13
Add *strict* optional parameter and reject malformed inputs by default.
@@ -97,6 +110,19 @@ of the new API.
If *strict* is true, use a strict parser which rejects malformed inputs.
+ The default setting for *strict* is set to ``True``, but you can override
+ it by setting the environment variable ``PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING``
+ to non-empty string.
+
+ Additionally, you can permanently set the default value for *strict* to
+ ``False`` by creating the configuration file ``/etc/python/email.cfg``
+ with the following content:
+
+ .. code-block:: ini
+
+ [email_addr_parsing]
+ PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING = true
+
Here's a simple example that gets all the recipients of a message::
from email.utils import getaddresses
diff --git a/Lib/email/utils.py b/Lib/email/utils.py
index 41bb3c9..09a414c 100644
--- a/Lib/email/utils.py
+++ b/Lib/email/utils.py
@@ -48,6 +48,47 @@ TICK = "'"
specialsre = re.compile(r'[][\\()<>@,:;".]')
escapesre = re.compile(r'[\\"]')
+_EMAIL_CONFIG_FILE = "/etc/python/email.cfg"
+_cached_strict_addr_parsing = None
+
+
+def _use_strict_email_parsing():
+ """"Cache implementation for _cached_strict_addr_parsing"""
+ global _cached_strict_addr_parsing
+ if _cached_strict_addr_parsing is None:
+ _cached_strict_addr_parsing = _use_strict_email_parsing_impl()
+ return _cached_strict_addr_parsing
+
+
+def _use_strict_email_parsing_impl():
+ """Returns True if strict email parsing is not disabled by
+ config file or env variable.
+ """
+ disabled = bool(os.environ.get("PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING"))
+ if disabled:
+ return False
+
+ try:
+ file = open(_EMAIL_CONFIG_FILE)
+ except FileNotFoundError:
+ pass
+ else:
+ with file:
+ import configparser
+ config = configparser.ConfigParser(
+ interpolation=None,
+ comment_prefixes=('#', ),
+
+ )
+ config.read_file(file)
+ disabled = config.getboolean('email_addr_parsing', "PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING", fallback=None)
+
+ if disabled:
+ return False
+
+ return True
+
+
def _has_surrogates(s):
"""Return True if s may contain surrogate-escaped binary data."""
# This check is based on the fact that unless there are surrogates, utf8
@@ -148,7 +189,7 @@ def _strip_quoted_realnames(addr):
supports_strict_parsing = True
-def getaddresses(fieldvalues, *, strict=True):
+def getaddresses(fieldvalues, *, strict=None):
"""Return a list of (REALNAME, EMAIL) or ('','') for each fieldvalue.
When parsing fails for a fieldvalue, a 2-tuple of ('', '') is returned in
@@ -157,6 +198,11 @@ def getaddresses(fieldvalues, *, strict=True):
If strict is true, use a strict parser which rejects malformed inputs.
"""
+ # If default is used, it's True unless disabled
+ # by env variable or config file.
+ if strict == None:
+ strict = _use_strict_email_parsing()
+
# If strict is true, if the resulting list of parsed addresses is greater
# than the number of fieldvalues in the input list, a parsing error has
# occurred and consequently a list containing a single empty 2-tuple [('',
@@ -320,7 +366,7 @@ def parsedate_to_datetime(data):
tzinfo=datetime.timezone(datetime.timedelta(seconds=tz)))
-def parseaddr(addr, *, strict=True):
+def parseaddr(addr, *, strict=None):
"""
Parse addr into its constituent realname and email address parts.
@@ -329,6 +375,11 @@ def parseaddr(addr, *, strict=True):
If strict is True, use a strict parser which rejects malformed inputs.
"""
+ # If default is used, it's True unless disabled
+ # by env variable or config file.
+ if strict == None:
+ strict = _use_strict_email_parsing()
+
if not strict:
addrs = _AddressList(addr).addresslist
if not addrs:
diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py
index ad60ed3..f85da56 100644
--- a/Lib/test/test_email/test_email.py
+++ b/Lib/test/test_email/test_email.py
@@ -8,6 +8,9 @@ import base64
import unittest
import textwrap
import warnings
+import contextlib
+import tempfile
+import os
from io import StringIO, BytesIO
from itertools import chain
@@ -41,8 +44,8 @@ from email import quoprimime
from email import utils
from test import support
-from test.support import threading_helper
-from test.support.os_helper import unlink
+from test.support import threading_helper, swap_attr
+from test.support.os_helper import unlink, EnvironmentVarGuard
from test.test_email import openfile, TestEmailBase
# These imports are documented to work, but we are testing them using a
@@ -3442,6 +3445,73 @@ Foo
# Test email.utils.supports_strict_parsing attribute
self.assertEqual(email.utils.supports_strict_parsing, True)
+ def test_parsing_errors_strict_set_via_env_var(self):
+ address = 'alice@example.org )Alice('
+ empty = ('', '')
+
+ # Reset cached default value to make the function
+ # reload the config file provided below.
+ utils._cached_strict_addr_parsing = None
+
+ # Strict disabled via env variable, old behavior expected
+ with EnvironmentVarGuard() as environ:
+ environ["PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING"] = "1"
+
+ self.assertEqual(utils.getaddresses([address]),
+ [('', 'alice@example.org'), ('', ''), ('', 'Alice')])
+ self.assertEqual(utils.parseaddr([address]), ('', address))
+
+ # Clear cache again
+ utils._cached_strict_addr_parsing = None
+
+ # Default strict=True, empty result expected
+ self.assertEqual(utils.getaddresses([address]), [empty])
+ self.assertEqual(utils.parseaddr([address]), empty)
+
+ # Clear cache again
+ utils._cached_strict_addr_parsing = None
+
+ # Empty string in env variable = strict parsing enabled (default)
+ with EnvironmentVarGuard() as environ:
+ environ["PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING"] = ""
+
+ # Default strict=True, empty result expected
+ self.assertEqual(utils.getaddresses([address]), [empty])
+ self.assertEqual(utils.parseaddr([address]), empty)
+
+ @contextlib.contextmanager
+ def _email_strict_parsing_conf(self):
+ """Context for the given email strict parsing configured in config file"""
+ with tempfile.TemporaryDirectory() as tmpdirname:
+ filename = os.path.join(tmpdirname, 'conf.cfg')
+ with swap_attr(utils, "_EMAIL_CONFIG_FILE", filename):
+ with open(filename, 'w') as file:
+ file.write('[email_addr_parsing]\n')
+ file.write('PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING = true')
+ utils._EMAIL_CONFIG_FILE = filename
+ yield
+
+ def test_parsing_errors_strict_disabled_via_config_file(self):
+ address = 'alice@example.org )Alice('
+ empty = ('', '')
+
+ # Reset cached default value to make the function
+ # reload the config file provided below.
+ utils._cached_strict_addr_parsing = None
+
+ # Strict disabled via config file, old results expected
+ with self._email_strict_parsing_conf():
+ self.assertEqual(utils.getaddresses([address]),
+ [('', 'alice@example.org'), ('', ''), ('', 'Alice')])
+ self.assertEqual(utils.parseaddr([address]), ('', address))
+
+ # Clear cache again
+ utils._cached_strict_addr_parsing = None
+
+ # Default strict=True, empty result expected
+ self.assertEqual(utils.getaddresses([address]), [empty])
+ self.assertEqual(utils.parseaddr([address]), empty)
+
def test_getaddresses_nasty(self):
for addresses, expected in (
(['"Sürname, Firstname" <to@example.com>'],
--
2.44.0

@ -1,75 +0,0 @@
From 670984c96eea60488c5355b4cf535c1ee3cf081a Mon Sep 17 00:00:00 2001
From: rpm-build <rpm-build>
Date: Wed, 24 Apr 2024 04:24:16 +0200
Subject: [PATCH] Fix xml tests
---
Lib/test/test_pyexpat.py | 3 +++
Lib/test/test_sax.py | 2 ++
Lib/test/test_xml_etree.py | 6 ++++++
3 files changed, 11 insertions(+)
diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py
index 44bd1de..5976fa0 100644
--- a/Lib/test/test_pyexpat.py
+++ b/Lib/test/test_pyexpat.py
@@ -3,6 +3,7 @@
import os
import platform
+import pyexpat
import sys
import sysconfig
import unittest
@@ -793,6 +794,8 @@ class ReparseDeferralTest(unittest.TestCase):
self.assertEqual(started, ['doc'])
+ @unittest.skipIf(pyexpat.version_info < (2, 6, 0),
+ "Reparse deferral not defined for libexpat < 2.6.0")
def test_reparse_deferral_disabled(self):
started = []
diff --git a/Lib/test/test_sax.py b/Lib/test/test_sax.py
index 9b3014a..5960de1 100644
--- a/Lib/test/test_sax.py
+++ b/Lib/test/test_sax.py
@@ -1240,6 +1240,8 @@ class ExpatReaderTest(XmlTestBase):
self.assertEqual(result.getvalue(), start + b"<doc></doc>")
+ @unittest.skipIf(pyexpat.version_info < (2, 6, 0),
+ "Reparse deferral not defined for libexpat < 2.6.0")
def test_flush_reparse_deferral_disabled(self):
result = BytesIO()
xmlgen = XMLGenerator(result)
diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py
index 8becafb..5e9b6b5 100644
--- a/Lib/test/test_xml_etree.py
+++ b/Lib/test/test_xml_etree.py
@@ -1424,9 +1424,13 @@ class XMLPullParserTest(unittest.TestCase):
self.assert_event_tags(parser, [('end', 'root')])
self.assertIsNone(parser.close())
+ @unittest.skipIf(pyexpat.version_info < (2, 6, 0),
+ "test not compatible with the latest expat security release")
def test_simple_xml_chunk_1(self):
self.test_simple_xml(chunk_size=1, flush=True)
+ @unittest.skipIf(pyexpat.version_info < (2, 6, 0),
+ "test not compatible with the latest expat security release")
def test_simple_xml_chunk_5(self):
self.test_simple_xml(chunk_size=5, flush=True)
@@ -1651,6 +1655,8 @@ class XMLPullParserTest(unittest.TestCase):
self.assert_event_tags(parser, [('end', 'doc')])
+ @unittest.skipIf(pyexpat.version_info < (2, 6, 0),
+ "Reparse deferral not defined for libexpat < 2.6.0")
def test_flush_reparse_deferral_disabled(self):
parser = ET.XMLPullParser(events=('start', 'end'))
--
2.44.0

@ -1,346 +0,0 @@
From 2963bbab04546f5aef6a37a3b027ae7a484deec1 Mon Sep 17 00:00:00 2001
From: Petr Viktorin <encukou@gmail.com>
Date: Thu, 25 Apr 2024 14:45:48 +0200
Subject: [PATCH] gh-113171: gh-65056: Fix "private" (non-global) IP address
ranges (GH-113179) (GH-113186) (GH-118177) (#118227)
---
Doc/library/ipaddress.rst | 43 +++++++-
Doc/whatsnew/3.11.rst | 9 ++
Lib/ipaddress.py | 99 +++++++++++++++----
Lib/test/test_ipaddress.py | 21 +++-
...-03-14-01-38-44.gh-issue-113171.VFnObz.rst | 9 ++
5 files changed, 157 insertions(+), 24 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-03-14-01-38-44.gh-issue-113171.VFnObz.rst
diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst
index 03dc956..f57fa15 100644
--- a/Doc/library/ipaddress.rst
+++ b/Doc/library/ipaddress.rst
@@ -178,18 +178,53 @@ write code that handles both IP versions correctly. Address objects are
.. attribute:: is_private
- ``True`` if the address is allocated for private networks. See
+ ``True`` if the address is defined as not globally reachable by
iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_
- (for IPv6).
+ (for IPv6) with the following exceptions:
+
+ * ``is_private`` is ``False`` for the shared address space (``100.64.0.0/10``)
+ * For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the
+ semantics of the underlying IPv4 addresses and the following condition holds
+ (see :attr:`IPv6Address.ipv4_mapped`)::
+
+ address.is_private == address.ipv4_mapped.is_private
+
+ ``is_private`` has value opposite to :attr:`is_global`, except for the shared address space
+ (``100.64.0.0/10`` range) where they are both ``False``.
+
+ .. versionchanged:: 3.11.10
+
+ Fixed some false positives and false negatives.
+
+ * ``192.0.0.0/24`` is considered private with the exception of ``192.0.0.9/32`` and
+ ``192.0.0.10/32`` (previously: only the ``192.0.0.0/29`` sub-range was considered private).
+ * ``64:ff9b:1::/48`` is considered private.
+ * ``2002::/16`` is considered private.
+ * There are exceptions within ``2001::/23`` (otherwise considered private): ``2001:1::1/128``,
+ ``2001:1::2/128``, ``2001:3::/32``, ``2001:4:112::/48``, ``2001:20::/28``, ``2001:30::/28``.
+ The exceptions are not considered private.
.. attribute:: is_global
- ``True`` if the address is allocated for public networks. See
+ ``True`` if the address is defined as globally reachable by
iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_
- (for IPv6).
+ (for IPv6) with the following exception:
+
+ For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the
+ semantics of the underlying IPv4 addresses and the following condition holds
+ (see :attr:`IPv6Address.ipv4_mapped`)::
+
+ address.is_global == address.ipv4_mapped.is_global
+
+ ``is_global`` has value opposite to :attr:`is_private`, except for the shared address space
+ (``100.64.0.0/10`` range) where they are both ``False``.
.. versionadded:: 3.4
+ .. versionchanged:: 3.11.10
+
+ Fixed some false positives and false negatives, see :attr:`is_private` for details.
+
.. attribute:: is_unspecified
``True`` if the address is unspecified. See :RFC:`5735` (for IPv4)
diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst
index f670fa1..42b61c7 100644
--- a/Doc/whatsnew/3.11.rst
+++ b/Doc/whatsnew/3.11.rst
@@ -2727,3 +2727,12 @@ OpenSSL
* Windows builds and macOS installers from python.org now use OpenSSL 3.0.
.. _libb2: https://www.blake2.net/
+
+Notable changes in 3.11.10
+==========================
+
+ipaddress
+---------
+
+* Fixed ``is_global`` and ``is_private`` behavior in ``IPv4Address``,
+ ``IPv6Address``, ``IPv4Network`` and ``IPv6Network``.
diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py
index 16ba16c..567beb3 100644
--- a/Lib/ipaddress.py
+++ b/Lib/ipaddress.py
@@ -1086,7 +1086,11 @@ class _BaseNetwork(_IPAddressBase):
"""
return any(self.network_address in priv_network and
self.broadcast_address in priv_network
- for priv_network in self._constants._private_networks)
+ for priv_network in self._constants._private_networks) and all(
+ self.network_address not in network and
+ self.broadcast_address not in network
+ for network in self._constants._private_networks_exceptions
+ )
@property
def is_global(self):
@@ -1333,18 +1337,41 @@ class IPv4Address(_BaseV4, _BaseAddress):
@property
@functools.lru_cache()
def is_private(self):
- """Test if this address is allocated for private networks.
+ """``True`` if the address is defined as not globally reachable by
+ iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_
+ (for IPv6) with the following exceptions:
- Returns:
- A boolean, True if the address is reserved per
- iana-ipv4-special-registry.
+ * ``is_private`` is ``False`` for ``100.64.0.0/10``
+ * For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the
+ semantics of the underlying IPv4 addresses and the following condition holds
+ (see :attr:`IPv6Address.ipv4_mapped`)::
+ address.is_private == address.ipv4_mapped.is_private
+
+ ``is_private`` has value opposite to :attr:`is_global`, except for the ``100.64.0.0/10``
+ IPv4 range where they are both ``False``.
"""
- return any(self in net for net in self._constants._private_networks)
+ return (
+ any(self in net for net in self._constants._private_networks)
+ and all(self not in net for net in self._constants._private_networks_exceptions)
+ )
@property
@functools.lru_cache()
def is_global(self):
+ """``True`` if the address is defined as globally reachable by
+ iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_
+ (for IPv6) with the following exception:
+
+ For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the
+ semantics of the underlying IPv4 addresses and the following condition holds
+ (see :attr:`IPv6Address.ipv4_mapped`)::
+
+ address.is_global == address.ipv4_mapped.is_global
+
+ ``is_global`` has value opposite to :attr:`is_private`, except for the ``100.64.0.0/10``
+ IPv4 range where they are both ``False``.
+ """
return self not in self._constants._public_network and not self.is_private
@property
@@ -1548,13 +1575,15 @@ class _IPv4Constants:
_public_network = IPv4Network('100.64.0.0/10')
+ # Not globally reachable address blocks listed on
+ # https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml
_private_networks = [
IPv4Network('0.0.0.0/8'),
IPv4Network('10.0.0.0/8'),
IPv4Network('127.0.0.0/8'),
IPv4Network('169.254.0.0/16'),
IPv4Network('172.16.0.0/12'),
- IPv4Network('192.0.0.0/29'),
+ IPv4Network('192.0.0.0/24'),
IPv4Network('192.0.0.170/31'),
IPv4Network('192.0.2.0/24'),
IPv4Network('192.168.0.0/16'),
@@ -1565,6 +1594,11 @@ class _IPv4Constants:
IPv4Network('255.255.255.255/32'),
]
+ _private_networks_exceptions = [
+ IPv4Network('192.0.0.9/32'),
+ IPv4Network('192.0.0.10/32'),
+ ]
+
_reserved_network = IPv4Network('240.0.0.0/4')
_unspecified_address = IPv4Address('0.0.0.0')
@@ -2010,27 +2044,42 @@ class IPv6Address(_BaseV6, _BaseAddress):
@property
@functools.lru_cache()
def is_private(self):
- """Test if this address is allocated for private networks.
+ """``True`` if the address is defined as not globally reachable by
+ iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_
+ (for IPv6) with the following exceptions:
- Returns:
- A boolean, True if the address is reserved per
- iana-ipv6-special-registry, or is ipv4_mapped and is
- reserved in the iana-ipv4-special-registry.
+ * ``is_private`` is ``False`` for ``100.64.0.0/10``
+ * For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the
+ semantics of the underlying IPv4 addresses and the following condition holds
+ (see :attr:`IPv6Address.ipv4_mapped`)::
+ address.is_private == address.ipv4_mapped.is_private
+
+ ``is_private`` has value opposite to :attr:`is_global`, except for the ``100.64.0.0/10``
+ IPv4 range where they are both ``False``.
"""
ipv4_mapped = self.ipv4_mapped
if ipv4_mapped is not None:
return ipv4_mapped.is_private
- return any(self in net for net in self._constants._private_networks)
+ return (
+ any(self in net for net in self._constants._private_networks)
+ and all(self not in net for net in self._constants._private_networks_exceptions)
+ )
@property
def is_global(self):
- """Test if this address is allocated for public networks.
+ """``True`` if the address is defined as globally reachable by
+ iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_
+ (for IPv6) with the following exception:
- Returns:
- A boolean, true if the address is not reserved per
- iana-ipv6-special-registry.
+ For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the
+ semantics of the underlying IPv4 addresses and the following condition holds
+ (see :attr:`IPv6Address.ipv4_mapped`)::
+
+ address.is_global == address.ipv4_mapped.is_global
+ ``is_global`` has value opposite to :attr:`is_private`, except for the ``100.64.0.0/10``
+ IPv4 range where they are both ``False``.
"""
return not self.is_private
@@ -2271,19 +2320,31 @@ class _IPv6Constants:
_multicast_network = IPv6Network('ff00::/8')
+ # Not globally reachable address blocks listed on
+ # https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml
_private_networks = [
IPv6Network('::1/128'),
IPv6Network('::/128'),
IPv6Network('::ffff:0:0/96'),
+ IPv6Network('64:ff9b:1::/48'),
IPv6Network('100::/64'),
IPv6Network('2001::/23'),
- IPv6Network('2001:2::/48'),
IPv6Network('2001:db8::/32'),
- IPv6Network('2001:10::/28'),
+ # IANA says N/A, let's consider it not globally reachable to be safe
+ IPv6Network('2002::/16'),
IPv6Network('fc00::/7'),
IPv6Network('fe80::/10'),
]
+ _private_networks_exceptions = [
+ IPv6Network('2001:1::1/128'),
+ IPv6Network('2001:1::2/128'),
+ IPv6Network('2001:3::/32'),
+ IPv6Network('2001:4:112::/48'),
+ IPv6Network('2001:20::/28'),
+ IPv6Network('2001:30::/28'),
+ ]
+
_reserved_networks = [
IPv6Network('::/8'), IPv6Network('100::/8'),
IPv6Network('200::/7'), IPv6Network('400::/6'),
diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py
index fc27628..16c3416 100644
--- a/Lib/test/test_ipaddress.py
+++ b/Lib/test/test_ipaddress.py
@@ -2269,6 +2269,10 @@ class IpaddrUnitTest(unittest.TestCase):
self.assertEqual(True, ipaddress.ip_address(
'172.31.255.255').is_private)
self.assertEqual(False, ipaddress.ip_address('172.32.0.0').is_private)
+ self.assertFalse(ipaddress.ip_address('192.0.0.0').is_global)
+ self.assertTrue(ipaddress.ip_address('192.0.0.9').is_global)
+ self.assertTrue(ipaddress.ip_address('192.0.0.10').is_global)
+ self.assertFalse(ipaddress.ip_address('192.0.0.255').is_global)
self.assertEqual(True,
ipaddress.ip_address('169.254.100.200').is_link_local)
@@ -2294,6 +2298,7 @@ class IpaddrUnitTest(unittest.TestCase):
self.assertEqual(True, ipaddress.ip_network("169.254.0.0/16").is_private)
self.assertEqual(True, ipaddress.ip_network("172.16.0.0/12").is_private)
self.assertEqual(True, ipaddress.ip_network("192.0.0.0/29").is_private)
+ self.assertEqual(False, ipaddress.ip_network("192.0.0.9/32").is_private)
self.assertEqual(True, ipaddress.ip_network("192.0.0.170/31").is_private)
self.assertEqual(True, ipaddress.ip_network("192.0.2.0/24").is_private)
self.assertEqual(True, ipaddress.ip_network("192.168.0.0/16").is_private)
@@ -2310,8 +2315,8 @@ class IpaddrUnitTest(unittest.TestCase):
self.assertEqual(True, ipaddress.ip_network("::/128").is_private)
self.assertEqual(True, ipaddress.ip_network("::ffff:0:0/96").is_private)
self.assertEqual(True, ipaddress.ip_network("100::/64").is_private)
- self.assertEqual(True, ipaddress.ip_network("2001::/23").is_private)
self.assertEqual(True, ipaddress.ip_network("2001:2::/48").is_private)
+ self.assertEqual(False, ipaddress.ip_network("2001:3::/48").is_private)
self.assertEqual(True, ipaddress.ip_network("2001:db8::/32").is_private)
self.assertEqual(True, ipaddress.ip_network("2001:10::/28").is_private)
self.assertEqual(True, ipaddress.ip_network("fc00::/7").is_private)
@@ -2390,6 +2395,20 @@ class IpaddrUnitTest(unittest.TestCase):
self.assertEqual(True, ipaddress.ip_address('0::0').is_unspecified)
self.assertEqual(False, ipaddress.ip_address('::1').is_unspecified)
+ self.assertFalse(ipaddress.ip_address('64:ff9b:1::').is_global)
+ self.assertFalse(ipaddress.ip_address('2001::').is_global)
+ self.assertTrue(ipaddress.ip_address('2001:1::1').is_global)
+ self.assertTrue(ipaddress.ip_address('2001:1::2').is_global)
+ self.assertFalse(ipaddress.ip_address('2001:2::').is_global)
+ self.assertTrue(ipaddress.ip_address('2001:3::').is_global)
+ self.assertFalse(ipaddress.ip_address('2001:4::').is_global)
+ self.assertTrue(ipaddress.ip_address('2001:4:112::').is_global)
+ self.assertFalse(ipaddress.ip_address('2001:10::').is_global)
+ self.assertTrue(ipaddress.ip_address('2001:20::').is_global)
+ self.assertTrue(ipaddress.ip_address('2001:30::').is_global)
+ self.assertFalse(ipaddress.ip_address('2001:40::').is_global)
+ self.assertFalse(ipaddress.ip_address('2002::').is_global)
+
# some generic IETF reserved addresses
self.assertEqual(True, ipaddress.ip_address('100::').is_reserved)
self.assertEqual(True, ipaddress.ip_network('4000::1/128').is_reserved)
diff --git a/Misc/NEWS.d/next/Library/2024-03-14-01-38-44.gh-issue-113171.VFnObz.rst b/Misc/NEWS.d/next/Library/2024-03-14-01-38-44.gh-issue-113171.VFnObz.rst
new file mode 100644
index 0000000..f9a7247
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-03-14-01-38-44.gh-issue-113171.VFnObz.rst
@@ -0,0 +1,9 @@
+Fixed various false positives and false negatives in
+
+* :attr:`ipaddress.IPv4Address.is_private` (see these docs for details)
+* :attr:`ipaddress.IPv4Address.is_global`
+* :attr:`ipaddress.IPv6Address.is_private`
+* :attr:`ipaddress.IPv6Address.is_global`
+
+Also in the corresponding :class:`ipaddress.IPv4Network` and :class:`ipaddress.IPv6Network`
+attributes.
--
2.45.2

@ -1,365 +0,0 @@
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: Petr Viktorin <encukou@gmail.com>
Date: Wed, 31 Jul 2024 00:19:48 +0200
Subject: [PATCH] 00435: gh-121650: Encode newlines in headers, and verify
headers are sound (GH-122233)
Per RFC 2047:
> [...] these encoding schemes allow the
> encoding of arbitrary octet values, mail readers that implement this
> decoding should also ensure that display of the decoded data on the
> recipient's terminal will not cause unwanted side-effects
It seems that the "quoted-word" scheme is a valid way to include
a newline character in a header value, just like we already allow
undecodable bytes or control characters.
They do need to be properly quoted when serialized to text, though.
This should fail for custom fold() implementations that aren't careful
about newlines.
(cherry picked from commit 097633981879b3c9de9a1dd120d3aa585ecc2384)
Co-authored-by: Petr Viktorin <encukou@gmail.com>
Co-authored-by: Bas Bloemsaat <bas@bloemsaat.org>
Co-authored-by: Serhiy Storchaka <storchaka@gmail.com>
---
Doc/library/email.errors.rst | 7 +++
Doc/library/email.policy.rst | 18 ++++++
Doc/whatsnew/3.11.rst | 13 ++++
Lib/email/_header_value_parser.py | 12 +++-
Lib/email/_policybase.py | 8 +++
Lib/email/errors.py | 4 ++
Lib/email/generator.py | 13 +++-
Lib/test/test_email/test_generator.py | 62 +++++++++++++++++++
Lib/test/test_email/test_policy.py | 26 ++++++++
...-07-27-16-10-41.gh-issue-121650.nf6oc9.rst | 5 ++
10 files changed, 164 insertions(+), 4 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-07-27-16-10-41.gh-issue-121650.nf6oc9.rst
diff --git a/Doc/library/email.errors.rst b/Doc/library/email.errors.rst
index 56aea6598b..27b0481a85 100644
--- a/Doc/library/email.errors.rst
+++ b/Doc/library/email.errors.rst
@@ -58,6 +58,13 @@ The following exception classes are defined in the :mod:`email.errors` module:
:class:`~email.mime.nonmultipart.MIMENonMultipart` (e.g.
:class:`~email.mime.image.MIMEImage`).
+
+.. exception:: HeaderWriteError()
+
+ Raised when an error occurs when the :mod:`~email.generator` outputs
+ headers.
+
+
.. exception:: MessageDefect()
This is the base class for all defects found when parsing email messages.
diff --git a/Doc/library/email.policy.rst b/Doc/library/email.policy.rst
index bb406c5a56..3edba4028b 100644
--- a/Doc/library/email.policy.rst
+++ b/Doc/library/email.policy.rst
@@ -228,6 +228,24 @@ added matters. To illustrate::
.. versionadded:: 3.6
+
+ .. attribute:: verify_generated_headers
+
+ If ``True`` (the default), the generator will raise
+ :exc:`~email.errors.HeaderWriteError` instead of writing a header
+ that is improperly folded or delimited, such that it would
+ be parsed as multiple headers or joined with adjacent data.
+ Such headers can be generated by custom header classes or bugs
+ in the ``email`` module.
+
+ As it's a security feature, this defaults to ``True`` even in the
+ :class:`~email.policy.Compat32` policy.
+ For backwards compatible, but unsafe, behavior, it must be set to
+ ``False`` explicitly.
+
+ .. versionadded:: 3.11.10
+
+
The following :class:`Policy` method is intended to be called by code using
the email library to create policy instances with custom settings:
diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst
index 42b61c75c7..f12c871998 100644
--- a/Doc/whatsnew/3.11.rst
+++ b/Doc/whatsnew/3.11.rst
@@ -2728,6 +2728,7 @@ OpenSSL
.. _libb2: https://www.blake2.net/
+
Notable changes in 3.11.10
==========================
@@ -2736,3 +2737,15 @@ ipaddress
* Fixed ``is_global`` and ``is_private`` behavior in ``IPv4Address``,
``IPv6Address``, ``IPv4Network`` and ``IPv6Network``.
+
+email
+-----
+
+* Headers with embedded newlines are now quoted on output.
+
+ The :mod:`~email.generator` will now refuse to serialize (write) headers
+ that are improperly folded or delimited, such that they would be parsed as
+ multiple headers or joined with adjacent data.
+ If you need to turn this safety feature off,
+ set :attr:`~email.policy.Policy.verify_generated_headers`.
+ (Contributed by Bas Bloemsaat and Petr Viktorin in :gh:`121650`.)
diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py
index 8cb8852cf0..255a953092 100644
--- a/Lib/email/_header_value_parser.py
+++ b/Lib/email/_header_value_parser.py
@@ -92,6 +92,8 @@
ASPECIALS = TSPECIALS | set("*'%")
ATTRIBUTE_ENDS = ASPECIALS | WSP
EXTENDED_ATTRIBUTE_ENDS = ATTRIBUTE_ENDS - set('%')
+NLSET = {'\n', '\r'}
+SPECIALSNL = SPECIALS | NLSET
def quote_string(value):
return '"'+str(value).replace('\\', '\\\\').replace('"', r'\"')+'"'
@@ -2780,9 +2782,13 @@ def _refold_parse_tree(parse_tree, *, policy):
wrap_as_ew_blocked -= 1
continue
tstr = str(part)
- if part.token_type == 'ptext' and set(tstr) & SPECIALS:
- # Encode if tstr contains special characters.
- want_encoding = True
+ if not want_encoding:
+ if part.token_type == 'ptext':
+ # Encode if tstr contains special characters.
+ want_encoding = not SPECIALSNL.isdisjoint(tstr)
+ else:
+ # Encode if tstr contains newlines.
+ want_encoding = not NLSET.isdisjoint(tstr)
try:
tstr.encode(encoding)
charset = encoding
diff --git a/Lib/email/_policybase.py b/Lib/email/_policybase.py
index c9cbadd2a8..d1f48211f9 100644
--- a/Lib/email/_policybase.py
+++ b/Lib/email/_policybase.py
@@ -157,6 +157,13 @@ class Policy(_PolicyBase, metaclass=abc.ABCMeta):
message_factory -- the class to use to create new message objects.
If the value is None, the default is Message.
+ verify_generated_headers
+ -- if true, the generator verifies that each header
+ they are properly folded, so that a parser won't
+ treat it as multiple headers, start-of-body, or
+ part of another header.
+ This is a check against custom Header & fold()
+ implementations.
"""
raise_on_defect = False
@@ -165,6 +172,7 @@ class Policy(_PolicyBase, metaclass=abc.ABCMeta):
max_line_length = 78
mangle_from_ = False
message_factory = None
+ verify_generated_headers = True
def handle_defect(self, obj, defect):
"""Based on policy, either raise defect or call register_defect.
diff --git a/Lib/email/errors.py b/Lib/email/errors.py
index 3ad0056554..02aa5eced6 100644
--- a/Lib/email/errors.py
+++ b/Lib/email/errors.py
@@ -29,6 +29,10 @@ class CharsetError(MessageError):
"""An illegal charset was given."""
+class HeaderWriteError(MessageError):
+ """Error while writing headers."""
+
+
# These are parsing defects which the parser was able to work around.
class MessageDefect(ValueError):
"""Base class for a message defect."""
diff --git a/Lib/email/generator.py b/Lib/email/generator.py
index eb597de76d..563ca17072 100644
--- a/Lib/email/generator.py
+++ b/Lib/email/generator.py
@@ -14,12 +14,14 @@
from copy import deepcopy
from io import StringIO, BytesIO
from email.utils import _has_surrogates
+from email.errors import HeaderWriteError
UNDERSCORE = '_'
NL = '\n' # XXX: no longer used by the code below.
NLCRE = re.compile(r'\r\n|\r|\n')
fcre = re.compile(r'^From ', re.MULTILINE)
+NEWLINE_WITHOUT_FWSP = re.compile(r'\r\n[^ \t]|\r[^ \n\t]|\n[^ \t]')
class Generator:
@@ -222,7 +224,16 @@ def _dispatch(self, msg):
def _write_headers(self, msg):
for h, v in msg.raw_items():
- self.write(self.policy.fold(h, v))
+ folded = self.policy.fold(h, v)
+ if self.policy.verify_generated_headers:
+ linesep = self.policy.linesep
+ if not folded.endswith(self.policy.linesep):
+ raise HeaderWriteError(
+ f'folded header does not end with {linesep!r}: {folded!r}')
+ if NEWLINE_WITHOUT_FWSP.search(folded.removesuffix(linesep)):
+ raise HeaderWriteError(
+ f'folded header contains newline: {folded!r}')
+ self.write(folded)
# A blank line always separates headers from body
self.write(self._NL)
diff --git a/Lib/test/test_email/test_generator.py b/Lib/test/test_email/test_generator.py
index 89e7edeb63..d29400f0ed 100644
--- a/Lib/test/test_email/test_generator.py
+++ b/Lib/test/test_email/test_generator.py
@@ -6,6 +6,7 @@
from email.generator import Generator, BytesGenerator
from email.headerregistry import Address
from email import policy
+import email.errors
from test.test_email import TestEmailBase, parameterize
@@ -216,6 +217,44 @@ def test_rfc2231_wrapping_switches_to_default_len_if_too_narrow(self):
g.flatten(msg)
self.assertEqual(s.getvalue(), self.typ(expected))
+ def test_keep_encoded_newlines(self):
+ msg = self.msgmaker(self.typ(textwrap.dedent("""\
+ To: nobody
+ Subject: Bad subject=?UTF-8?Q?=0A?=Bcc: injection@example.com
+
+ None
+ """)))
+ expected = textwrap.dedent("""\
+ To: nobody
+ Subject: Bad subject=?UTF-8?Q?=0A?=Bcc: injection@example.com
+
+ None
+ """)
+ s = self.ioclass()
+ g = self.genclass(s, policy=self.policy.clone(max_line_length=80))
+ g.flatten(msg)
+ self.assertEqual(s.getvalue(), self.typ(expected))
+
+ def test_keep_long_encoded_newlines(self):
+ msg = self.msgmaker(self.typ(textwrap.dedent("""\
+ To: nobody
+ Subject: Bad subject=?UTF-8?Q?=0A?=Bcc: injection@example.com
+
+ None
+ """)))
+ expected = textwrap.dedent("""\
+ To: nobody
+ Subject: Bad subject
+ =?utf-8?q?=0A?=Bcc:
+ injection@example.com
+
+ None
+ """)
+ s = self.ioclass()
+ g = self.genclass(s, policy=self.policy.clone(max_line_length=30))
+ g.flatten(msg)
+ self.assertEqual(s.getvalue(), self.typ(expected))
+
class TestGenerator(TestGeneratorBase, TestEmailBase):
@@ -224,6 +263,29 @@ class TestGenerator(TestGeneratorBase, TestEmailBase):
ioclass = io.StringIO
typ = str
+ def test_verify_generated_headers(self):
+ """gh-121650: by default the generator prevents header injection"""
+ class LiteralHeader(str):
+ name = 'Header'
+ def fold(self, **kwargs):
+ return self
+
+ for text in (
+ 'Value\r\nBad Injection\r\n',
+ 'NoNewLine'
+ ):
+ with self.subTest(text=text):
+ message = message_from_string(
+ "Header: Value\r\n\r\nBody",
+ policy=self.policy,
+ )
+
+ del message['Header']
+ message['Header'] = LiteralHeader(text)
+
+ with self.assertRaises(email.errors.HeaderWriteError):
+ message.as_string()
+
class TestBytesGenerator(TestGeneratorBase, TestEmailBase):
diff --git a/Lib/test/test_email/test_policy.py b/Lib/test/test_email/test_policy.py
index c6b9c80efe..baa35fd68e 100644
--- a/Lib/test/test_email/test_policy.py
+++ b/Lib/test/test_email/test_policy.py
@@ -26,6 +26,7 @@ class PolicyAPITests(unittest.TestCase):
'raise_on_defect': False,
'mangle_from_': True,
'message_factory': None,
+ 'verify_generated_headers': True,
}
# These default values are the ones set on email.policy.default.
# If any of these defaults change, the docs must be updated.
@@ -294,6 +295,31 @@ def test_short_maxlen_error(self):
with self.assertRaises(email.errors.HeaderParseError):
policy.fold("Subject", subject)
+ def test_verify_generated_headers(self):
+ """Turning protection off allows header injection"""
+ policy = email.policy.default.clone(verify_generated_headers=False)
+ for text in (
+ 'Header: Value\r\nBad: Injection\r\n',
+ 'Header: NoNewLine'
+ ):
+ with self.subTest(text=text):
+ message = email.message_from_string(
+ "Header: Value\r\n\r\nBody",
+ policy=policy,
+ )
+ class LiteralHeader(str):
+ name = 'Header'
+ def fold(self, **kwargs):
+ return self
+
+ del message['Header']
+ message['Header'] = LiteralHeader(text)
+
+ self.assertEqual(
+ message.as_string(),
+ f"{text}\nBody",
+ )
+
# XXX: Need subclassing tests.
# For adding subclassed objects, make sure the usual rules apply (subclass
# wins), but that the order still works (right overrides left).
diff --git a/Misc/NEWS.d/next/Library/2024-07-27-16-10-41.gh-issue-121650.nf6oc9.rst b/Misc/NEWS.d/next/Library/2024-07-27-16-10-41.gh-issue-121650.nf6oc9.rst
new file mode 100644
index 0000000000..83dd28d4ac
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-07-27-16-10-41.gh-issue-121650.nf6oc9.rst
@@ -0,0 +1,5 @@
+:mod:`email` headers with embedded newlines are now quoted on output. The
+:mod:`~email.generator` will now refuse to serialize (write) headers that
+are unsafely folded or delimited; see
+:attr:`~email.policy.Policy.verify_generated_headers`. (Contributed by Bas
+Bloemsaat and Petr Viktorin in :gh:`121650`.)

@ -1,128 +0,0 @@
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: "Jason R. Coombs" <jaraco@jaraco.com>
Date: Mon, 19 Aug 2024 19:28:20 -0400
Subject: [PATCH] 00436: [CVE-2024-8088] gh-122905: Sanitize names in
zipfile.Path.
Co-authored-by: Jason R. Coombs <jaraco@jaraco.com>
---
Lib/test/test_zipfile.py | 17 ++++++
Lib/zipfile.py | 61 ++++++++++++++++++-
...-08-11-14-08-04.gh-issue-122905.7tDsxA.rst | 1 +
3 files changed, 78 insertions(+), 1 deletion(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-08-11-14-08-04.gh-issue-122905.7tDsxA.rst
diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py
index 4de6f379a4..8bdc7a1b7d 100644
--- a/Lib/test/test_zipfile.py
+++ b/Lib/test/test_zipfile.py
@@ -3651,6 +3651,23 @@ def test_extract_orig_with_implied_dirs(self, alpharep):
zipfile.Path(zf)
zf.extractall(source_path.parent)
+ def test_malformed_paths(self):
+ """
+ Path should handle malformed paths.
+ """
+ data = io.BytesIO()
+ zf = zipfile.ZipFile(data, "w")
+ zf.writestr("/one-slash.txt", b"content")
+ zf.writestr("//two-slash.txt", b"content")
+ zf.writestr("../parent.txt", b"content")
+ zf.filename = ''
+ root = zipfile.Path(zf)
+ assert list(map(str, root.iterdir())) == [
+ 'one-slash.txt',
+ 'two-slash.txt',
+ 'parent.txt',
+ ]
+
class EncodedMetadataTests(unittest.TestCase):
file_names = ['\u4e00', '\u4e8c', '\u4e09'] # Han 'one', 'two', 'three'
diff --git a/Lib/zipfile.py b/Lib/zipfile.py
index 86829abce4..b7bf9ef7e3 100644
--- a/Lib/zipfile.py
+++ b/Lib/zipfile.py
@@ -9,6 +9,7 @@
import itertools
import os
import posixpath
+import re
import shutil
import stat
import struct
@@ -2243,7 +2244,65 @@ def _difference(minuend, subtrahend):
return itertools.filterfalse(set(subtrahend).__contains__, minuend)
-class CompleteDirs(ZipFile):
+class SanitizedNames:
+ """
+ ZipFile mix-in to ensure names are sanitized.
+ """
+
+ def namelist(self):
+ return list(map(self._sanitize, super().namelist()))
+
+ @staticmethod
+ def _sanitize(name):
+ r"""
+ Ensure a relative path with posix separators and no dot names.
+ Modeled after
+ https://github.com/python/cpython/blob/bcc1be39cb1d04ad9fc0bd1b9193d3972835a57c/Lib/zipfile/__init__.py#L1799-L1813
+ but provides consistent cross-platform behavior.
+ >>> san = SanitizedNames._sanitize
+ >>> san('/foo/bar')
+ 'foo/bar'
+ >>> san('//foo.txt')
+ 'foo.txt'
+ >>> san('foo/.././bar.txt')
+ 'foo/bar.txt'
+ >>> san('foo../.bar.txt')
+ 'foo../.bar.txt'
+ >>> san('\\foo\\bar.txt')
+ 'foo/bar.txt'
+ >>> san('D:\\foo.txt')
+ 'D/foo.txt'
+ >>> san('\\\\server\\share\\file.txt')
+ 'server/share/file.txt'
+ >>> san('\\\\?\\GLOBALROOT\\Volume3')
+ '?/GLOBALROOT/Volume3'
+ >>> san('\\\\.\\PhysicalDrive1\\root')
+ 'PhysicalDrive1/root'
+ Retain any trailing slash.
+ >>> san('abc/')
+ 'abc/'
+ Raises a ValueError if the result is empty.
+ >>> san('../..')
+ Traceback (most recent call last):
+ ...
+ ValueError: Empty filename
+ """
+
+ def allowed(part):
+ return part and part not in {'..', '.'}
+
+ # Remove the drive letter.
+ # Don't use ntpath.splitdrive, because that also strips UNC paths
+ bare = re.sub('^([A-Z]):', r'\1', name, flags=re.IGNORECASE)
+ clean = bare.replace('\\', '/')
+ parts = clean.split('/')
+ joined = '/'.join(filter(allowed, parts))
+ if not joined:
+ raise ValueError("Empty filename")
+ return joined + '/' * name.endswith('/')
+
+
+class CompleteDirs(SanitizedNames, ZipFile):
"""
A ZipFile subclass that ensures that implied directories
are always included in the namelist.
diff --git a/Misc/NEWS.d/next/Library/2024-08-11-14-08-04.gh-issue-122905.7tDsxA.rst b/Misc/NEWS.d/next/Library/2024-08-11-14-08-04.gh-issue-122905.7tDsxA.rst
new file mode 100644
index 0000000000..1be44c906c
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-08-11-14-08-04.gh-issue-122905.7tDsxA.rst
@@ -0,0 +1 @@
+:class:`zipfile.Path` objects now sanitize names from the zipfile.

@ -0,0 +1,16 @@
-----BEGIN PGP SIGNATURE-----
iQIzBAABCAAdFiEEz9yiRbEEPPKl+Xhl/+h0BBaL2EcFAmPiV84ACgkQ/+h0BBaL
2EeZ1xAAwBi0AEjUlZ9oeC54VuqC/XLuVwc3xWf+Irw/5mJA2/weJHoQqG9aEDkB
ph1pDJ6G/vDyKdjh8NZKkKftIL9pggRpAcA4mQ3XcDMKI/J+EQe5P/BwsTGClLhK
cZg6IcQKZvo9djfyRz48w9wfKs34NasBgoFQP+hOzmU10UMrcR7gUSB2ZgMVMDID
0rK1w2aPmZmDLUltBhf6Xb2voUYo+3jINLHWmQC6tdDOBxtxv222dhxS1mvpV7Zu
Xw8do9OsQxonc+owkpciMKDLcFoVmkdQPz9bmvHJKovMXT2RY7FEam9H7ukr35fC
xA6BKnyMgvWIWQVTwjBhcz3C85adzAz/ypHNTbJOuPxp1ZP8qO3D6vPlhZIFyTeJ
7LhagUBUkIKKtbz7u3ERJgvA6tn3UVyLOXM1DnaKkXQ1FgSymgWPRU7BsxanQ8FD
QkfTjC8fatZLCewNfGInkeAdLue+rMwZc8Q6vw2CAmcVdOKsQ98Db/FLF5sC+Kjz
D3brUESEX1ELcVk7vumUI0/z+MECF11dpv5hPOZ4cZDoInsNu846TfU0rzOeVe7H
gGO6Ae/Lu5gG09TNqepbFGA/dWR8V3zdLs5ZShTT4FsNFrHh7GDAEAMZSwT3AsVZ
TjOdU3+xEGsEfrYWRXOkhVIQdJtuovwv9+me5YWeyC4Puzp0Zwk=
=8/cW
-----END PGP SIGNATURE-----

@ -1,16 +0,0 @@
-----BEGIN PGP SIGNATURE-----
iQIzBAABCAAdFiEEz9yiRbEEPPKl+Xhl/+h0BBaL2EcFAmYNMEcACgkQ/+h0BBaL
2EeHhxAAuuIM9bl0dgAWOjbgRjCeXR8aFdfcI4dkO7bZrUy8eKbM+XCvPUUvloRJ
vzGkxYyTmI4kcNPOHfscUwH7AVVij8nGv7WeaXBUZGIXNwfHwvqOxvYvSsNNNFnr
70yJB7Df8/2s0XqFx3X1aWcnyMDerWKpfJ/VI/NPmCVxkYXGshuTTSFcCMTSFBQB
sNrIb5NWAsBF4R85uRQDlCg1AoyaKOdJNQkPo1Nrjol1ExJ+MHE7+E+QL9pQkUWG
SBISPUhJySBAegxolw6YR5dz1L4nukueQDJz3NizUeQGDvH7h1ImY8cypRi44U61
SUUHhBfmUBiC2dS/tTQawySULWcgbkV4GJ6cJZfDd95uffd4S/GDJCa2wCE2UTlA
XzQHwbcnIeoL064gX7ruBuFHJ6n/Oz7nZkFqbH2aqLTAWgLiUq31xH3HY734sL6X
zIJQRbcK1EM7cnNjKMVPlnHpAeKbsbHbU6yzWwZ7reIoyWlZ7vEGrfXO7Kmul93K
wVaWu0AiOY566ugekdDx4cKV+FQN6oppAN63yTfPJ2Ddcmxs4KNrtozw9OAgDTPE
GTPFD6V1CMuyQj/jOpAmbj+4bRD4Mx3u2PSittvrIeopxrXPsGGSZ5kdl62Xa2+A
DzKyYNXzcmxqS9lGdFb+OWCTyAIXxwZrdz1Q61g5xDvR9z/wZiI=
=Br9/
-----END PGP SIGNATURE-----

@ -16,9 +16,9 @@ LEVELS = (None, 1, 2)
# list of globs of test and other files that we expect not to have bytecode # list of globs of test and other files that we expect not to have bytecode
not_compiled = [ not_compiled = [
'/usr/bin/*', '/usr/bin/*',
'*/test/*/bad_coding.py', '*/test/bad_coding.py',
'*/test/*/bad_coding2.py', '*/test/bad_coding2.py',
'*/test/*/badsyntax_*.py', '*/test/badsyntax_*.py',
'*/lib2to3/tests/data/bom.py', '*/lib2to3/tests/data/bom.py',
'*/lib2to3/tests/data/crlf.py', '*/lib2to3/tests/data/crlf.py',
'*/lib2to3/tests/data/different_encoding.py', '*/lib2to3/tests/data/different_encoding.py',

@ -1,109 +0,0 @@
-----BEGIN PGP PUBLIC KEY BLOCK-----
mQINBFq+ToQBEADRYvIVtbK6owynD3j3nxwpW2KEk/p+aDvtXmc2SR2dBcZ8sFW2
R5vEsG8d3/D3wgv5pcL3KfNNXQYUnXVbobrFUUWQYc79qIsE3MgiPf5NVOtwKPUR
i5g9YJgKvpBxkQfqp3LYGm9ZBtwo3DVLA3yn7KsazCmAgTNFJYw7ku1XxgmIzY6K
5J30DfbJiqDqj4f9GslCCCCH3qiPnuLG/HUyVLHMpbWlaiy9NI0GcaLxjJewHj9w
W2D2lydkxe5JGo7egUkV3ILcuLVSVKA35SKY27dYqfuyqp9tAzaRbjDYjsYdHA6G
BqrNrKBn/GwlFDPrVdcvN3ZSY2wMLTxWE3Axc/FweuHxFnou/80FwX7F3JD+oEQ6
rofmcxOBCC7J98I7HZAhP9jBn88XIS2hztbLq8d6rZJZRtcz0k61VR0ddO+TrFmf
9rMYCPgCckRtVxeFIVIabrN1IzKynLFeo040h8hSGswd6YKDOVwjJY6Oa6EmVefZ
a8QSt4+M65RSzH6SEPY008F3nJUAK6MEkzTak+tFltZNrVWu8p2xd1j9nmxAwEhZ
/lgbxLqzYgaUWmfyHeZ8yVA0MhHzdiAL8nVUEdG3KecIq0RWCJLGLWWIjd6KAJl1
yAmhRYKK/sjPDsL3elHsFACfZbyx3o5GGQNlas1FYoPLWbaNGaJtgFTF2QARAQAB
tCtQYWJsbyBHYWxpbmRvIFNhbGdhZG8gPHBhYmxvZ3NhbEBnbWFpbC5jb20+iQJO
BBMBCgA4FiEEoDXIwZIZuoIezqhrZOYo+NaEaW0FAlq+ToQCGwMFCwkIBwMFFQoJ
CAsFFgIDAQACHgECF4AACgkQZOYo+NaEaW2bmA/+PXIap2udLoUVOHxnsIBdqYwp
sv1Aj5lfIJmNhmxPbHShwp1Jg+w4urxe+2Dj5ofKVlIo1i83bQkvnKJMDXDVuc/K
P6zqhBJ3rT4Q3qx2mzX8bIfQoJ2JHuH4lkP+I7doDcHHRyeNASyk72VdQmU4twNw
Ibn8nSNV6ThKHdoPYzVnO2rZUFcGIqH5HNsvR+B7cc1MBCHsgURYwSVhSePIFGlZ
iasdBD6QQkDSe4QWi7AcJFWFElw4kbOKJWxAWsrEk+tMXJVGRjnmL289EmPCx/vx
BqKy7Mse0yWCSRR3vB+O6TB1S5SgEyEgqlYsfGNv1qf/rfRD4KkyCbNU3LhY1Aim
vJP4pDW+KFxTk2Ks8vrx8gOSd2aFqPeO/pFDrpsF7PD62XwsfoXu4xc5V0Giw7r1
Nai0nax7kOrldNF8TbbtRjW0jmoC7wLIDujAkwDIOroZ0CXA3N4HVHdSbrHm/urX
nyxJXupXAQNwGx64JCBcbF2fp3Kvu1VAXBEFnd01KaopthHcbG5pA50Kl2Vhe+98
OdezUX42fHkQpQkB7HgtXfm6W1bw6YRBamrNvs1OoHBYmUjlECpe566IIu25Hc8s
x3qA+6eca7iqizyLG+WyMT8ZIYTWGAS59jxwR4esqGczbbZPSAPHFwLbGv7Wr0Rd
TPu5B0FcKpDkTd4IxQW5Ag0EWr5O2gEQAMjLe4CtbSfofmJrz5wfNkMVsZ81Gbqe
MoYd3dtkJnQYERUj8flzBj3ucaxGJ+Cuf7ybh3naPopKvEI1q0vkcgCDqrEgXK//
jKJbP28uPSMGhOG28q4PbamG55gy5FtM3ezzAxPWWKe9qBpV65GMmFy7eBQx2iJs
yiDIOOQQ4kraS+cTqNFimEXAGLCOQRNLcwIZzwAAHoW7HEpNUfVwaBD9kMlbo1ND
I60IKcNrNcmcmRxhJqfxjj8YBMwcKHO6GBE3AVpaE/+UO9zyr4TH+0YuQUgxKlPW
Dkg5XlkDo0S1GyLY5e9ckIDIlkTdDa2pOkoE2yB5MQCEga3YiHrKUVTTWaxn9XVJ
6x5ZjUF6bgSWGkrG5dUqSYoO1iDMuNVjtiujNyf/rvfj5cNxS7/lgxchhQKZHZXL
WVqxlneeVJ6s0P4+ROVG9ga2Sve7aUJ6wXIewZwulBcV2sE/W/DgxHgLBi53CUQt
vEzFzKvo48GnDqL5VYjA7l0HMYHd4GksCLi8E8U6Cgj+imXiM8voL7pHRZfs8mY8
udR+UT4e1Scl2MYP2qBJ9/17B/X52B3s1EZdqI/r+hfOyqrhPs+dbAN0mtMPn68+
nrvY1+nscvrSYEP6ZBlc9Hp2mgJdb6IcTvINXBEeLRjgc3pjViva443pkiFp9Axm
ecOckMKP3uSlABEBAAGJBGwEGAEKACAWIQSgNcjBkhm6gh7OqGtk5ij41oRpbQUC
Wr5O2gIbAgJACRBk5ij41oRpbcF0IAQZAQoAHRYhBM/cokWxBDzypfl4Zf/odAQW
i9hHBQJavk7aAAoJEP/odAQWi9hHr7YP/RCLre1CmOoWYpAtoa1yVCeYMDV6eQgL
B488/BEZHQE1zbrYy16XkhORob3JF/kUMjmJW7XaFF8FrWvRcdj/xaUGbOOEulKg
v+8zWfswYQRiZ4/JlwER4vRLi6fTE89MVER6Fkj2ASD4D2cifY+EztD4flV3sq3s
vIogGFaN9IvdrdeptOVGXs1RmAyoTsiS2mKQ6xsGh8B9ZAm55W8fBOGiSzLX21Xk
Ofdw53BrFQxn3cu/JgIKpdeZxgukcvEAI62B6X+YL6Na4j0eqEGLzsNtU1+xeJlo
WtVvmRwnRHGSxF6fzIZ3mk/p/aFiXAEq/xITCTY6tDv7x7pFE/RpdlJZyNJ+R5Y4
SQiuDsylxNCa/4G5EB6q+7iVYtbEQ9MnZg2phowEE42tlj0rz8/rvDK3LH3xibot
KHIodCWKlWByxH99u2PuHUQ0c1oCVBUE1KkruMpvI236DpU/dvdq4JLSg/fWrys/
VIjqLZgsIE5g/KO9XqngWHkLcBLh4CNAmHJ8Iia+s+/rfgsejQWB5uJb6eYg2JjB
4WP1EI0rULM6fdrCNB+MJ36wE2Lnb4bfT0phOMgjjH5/Ki7ZCbkxkOsBs4SRjiS+
weCsmpAtMqodWY/Cnw9pWSA/qLSRD5/mKeb9SO6OZ/OPfAatwnGHsvZ2sAueC6rR
04W5BfXZWrnJUXQP/id/EKE1Ksp5fKoxSCbkKTCig+Sf5Afwe36yFN+niZBqzn5b
BgL/HIKaZM97oDHersPPANeEgS+JVlBf95iKIYnQbZP43FLVbvOuaINhBIVtFO54
2Y7EYwl41kP7ILDElVy36KAmdQyBAfrjnZiRA70xShOxApLug1L0lxhR3YfmLwNi
RJ0V6KnYDKf0pfdhO9VFyFFWUojX1usn2SmSsXNizsNtvRqHXzPnX0rbJzZ9+N4O
9k1nxygYFG/2R/jGonVmTjRzcAHrAkNJETMWXMA7/8wRMDwluz8j+cCldey9x8Vk
JwgLGnZSbQtVpcFAnm5r/36Gt+9wc1VWMyrUrVr6Z679aqAbG7PMaeR5h5ygMj1k
VqRTYAUPSk1f8bZKRssQkQwEbp9dVIjm9SsR8VT7/tB+UuB85dABxgHfv3psJRT+
tL8g9V7kSZqQfcLNGmvEVvr2Zl9NtxwXtsFM2OBprxCenwb+e9Ppm1LjfJG/NE72
mAnOERfDaiLt4bqNo36Ei5sGCJ4Fx61phzNBXzkdRNM47i8J5UZRKFkE91c99BVM
HKUaY61NRK24fR0zP98ftDU82YFw0VRFJpTeBrO5ivN1MlQxUPzUWxKxMxO+20wa
UOXroEw11Tb4SRLGOla1pCl6lCUPJRy9IzadPDgTr/OTMkob/snt/XLdnV5/uQIN
BFq+TvoBEAC8Oy1g6pPWBbrCMhIq7VWY2fjylJ1fwg5BPXkOKVK1dsGYO4QD7oW9
L0aSqcFSNFGF9Cl0Ri4TFXZC3hnG4HeSXUWApuKdBLn21H3jba36Ay1oGcGfdm0v
Zght4c6BlMVBpGCw2wIkJbUNEy6InMM+O8CCbbaH3iJkJ4141P7pODHignx5AmZI
conMui4YOhC+IXQXynVEv1Juk7erB1Nh1RcRvsA4lb44HWx49lIwe85ejOmoZ0O3
6f9NJRer6bV0+rHWmg4IV5Q9h/Gn4IhEDZxA0DZl1RQI7dMgaMbIFbXGq7Kgzstz
EUnOoy29hXodxVmwIsMrAiQUYtwJ9hW+ESsw47+W2iPHVgviGWl7r/SgcgMYmf6m
5kiTBtwU7BQPS9G3zwwP2Rm3AA/6g39Q+tQKjOwi1I8+GZsY2On44Zly7BreBNg5
4gJgdAGcMOYU9etr050clH3UpTYcAEtX++ahtOKhJgLIPNcIAQNlnifqvU0VYpgw
R4YpZ7hgg+AVDzC73PIM0lFI0XiDuqChbxE+K1jmLXWe5iJF0dzgVTwP+PmsifNZ
Wg3+YxSsS+hDMPQ2xPiQN49gT4JJDHcDuyhHyCGYgyMiVJCsku9KrkubbfVRivyN
ZF2Zfo3f+nbrRxsftz0yjAq8byCvb0V0XOpt4pJ/ddlug9ytRxALNwARAQABiQI2
BBgBCgAgFiEEoDXIwZIZuoIezqhrZOYo+NaEaW0FAlq+TvoCGwwACgkQZOYo+NaE
aW3urA//UQ/cKQ7HvWjcLphzQOZc+6m5YL0wxvZkSjemU7mqjZdpacteIvRAoers
EqXHc208liIBtNfRzoreXdcXNzie65xXkrRnWoHVH/fTWy4lOnHr2CMXLeHjUgg/
M6PYi8+sARm05YFB8nsYhlhx3IdLhcfeVVbJedQKO0yL3CK1okT30DUVq5Lq6X/K
DC6AxuJR3D6UMSoT0WLaoX8qbhAp88qLynInfBVL18d97h916WPLTPeP0eHwhwND
bYtKDCMDuKQ9XX5+QsNH0RmbxlX274LHrUMMvkLKxcfCBvP+iuqrBeIuoeVzXYJZ
j7ZJtEH79bW44eecl/CY/STFYgSQ2XGTp2BI2q60wAmtKlNhwxY5ena0FgyFl6Tm
5OBHW/Pwo+ndQJGfbrCyWkTgRay9c8er3gl3GQYIBH6X0kCiG7h/Epj0b5CHOPU5
hCw0kEB8MB4poTIjeiY+Q01472/lQ68CL3DX158hR5d3XaPSIxAN+qFsfB1o316p
yjxhfK1MD/IfrOgjlggPPnc/KmLkCzpgdwKcZwLCdZq9hYBvF1Zs34HbaVMYbWTK
uxLowtXGU43vatCXXqmPOvl4/g4tZD6rysJDgOrHQnEHzT+Napn07s0BRC0IbbNn
FynUrkr5KMSuRz7Hg7xMApENOrb0nqdHSUJ914ZpuMIS6RhJgGu5Ag0EWr5PIAEQ
ALfh9vPD2B+miHDTMADI8aRZ7g9tnzynZYkk3+2sCiiusetsQQ+HIPJ/ASEJB7On
ane9dyT/LTRhrK9qaxgVMimk2COXB/xyh7Mnw7nJgFU0aRSbtX0vbvQz2suSzrQ6
9mPKzan28JGoClqB0bw1vwf3VjjxHV2dgD57CmqFPv7kAC/2a56dE+etzXattZAL
+2JWTpmfQ0ePRRadtBm0VahQhnU8x0+jvAVrEawqpVW83ozYFyW/0WInM2J7jHgQ
16OosY4lj5L/DxpVxaArhRFoRfWPXfC37iE8Mou/I95isvPQIhp1wTo4jG0KM02B
oIVbp/QRNBQ6WtpOzvJs1gqQiJJTfqbKJXQ3NDEY9crpVS83HJ+Zv99PNsyNkFjG
QpU84U3ZhsI4ygjdY45mpZueqI1RVcRQdu8Hgvoo/78Q/Sir6gMGop3mVdVo2guI
kFcJrXh0Xk3ech4aVqrmKx/mPXGwOAQU0DAul4RW3fKg1QxQE7Tlw3+95Ee/+q5j
HARL0uDbCJpRO8Sl8NDEuL32n/2Ot6kQeCSHrU7KJRYAkTxkKvr8zNow7hFhHFPE
SnHvTnskI6noh0VY6NwMhmLvhm0wKkRxZPzUNc3sgLvbK1NymIZ9aKCZamzhZrmG
vnblEz/OSLwGUua465H3hM1vvBQiartj7+6ZqWIkSmBPABEBAAGJAjYEGAEKACAW
IQSgNcjBkhm6gh7OqGtk5ij41oRpbQUCWr5PIAIbIAAKCRBk5ij41oRpbWmeEACG
+axtDC8UoNp9ORiYwEWLzZWDuugE+ah7DYYGD4Vs633FXVZW3SgM/bFtJ/0Lg8CF
74jI4LMHyIjDzEjcoItwnhBLix+kUoJTvrY58GPydwekLuw1p4KXLqtRs4fsZbNQ
YTknl4jYtRWoxO98x7tun7Gq2gqmJkIB2uj630fKz5cBk6p6oDFKjzyrHe+V7BiK
3okQPaD4x7hq8OnTy7lOy92ZZAqztS4tNEb4DkYW1MpuwsJ7hbBZitc1siI+FVVb
GjVVGZz6ssXoW67Tz8+VxdWJxNLXlv27eMcj4sme5S0th/YYNA5fRRv6zuzqZAru
YNGLpYYU7JLvZJ+3lCwa5j5ycOGBF0GvsGs6gj6h+CHkjR/BgzAgWC+GgUgslt6q
aH04rWtV6rVz+Y91LcrX5P6OM4anmXD3Gp3kl35AypXb4KyASF19+11RUziD4Z7q
wQEWfbwOltNyZv2lD8s2jPr7P02axWRQUbZAEhxRmvOQev/FZPyCF6gqUo/HxRbQ
y3bzmnipyHSv1DlXNfCFCHvN8kGyZnRWARqIKRg+j9ediJgOUqlLhg6KmrTVxd5v
3Dfv52PW2UODDTM20s3cQGuX/UswzMRwPI/+P44iCMwEKdm7duM/5oisZT9Vhy7g
P15MreFZLcZvUVgjqgy0u57cstyGK1Bo9e2sFcK2fA==
=6Zb4
-----END PGP PUBLIC KEY BLOCK-----

@ -16,11 +16,11 @@ URL: https://www.python.org/
# WARNING When rebasing to a new Python version, # WARNING When rebasing to a new Python version,
# remember to update the python3-docs package as well # remember to update the python3-docs package as well
%global general_version %{pybasever}.9 %global general_version %{pybasever}.2
#global prerel ... #global prerel ...
%global upstream_version %{general_version}%{?prerel} %global upstream_version %{general_version}%{?prerel}
Version: %{general_version}%{?prerel:~%{prerel}} Version: %{general_version}%{?prerel:~%{prerel}}
Release: 7%{?dist} Release: 2%{?dist}
License: Python License: Python
@ -63,7 +63,7 @@ License: Python
# If the rpmwheels condition is disabled, we use the bundled wheel packages # If the rpmwheels condition is disabled, we use the bundled wheel packages
# from Python with the versions below. # from Python with the versions below.
# This needs to be manually updated when we update Python. # This needs to be manually updated when we update Python.
%global pip_version 24.0 %global pip_version 22.3.1
%global setuptools_version 65.5.0 %global setuptools_version 65.5.0
# Expensive optimizations (mainly, profile-guided optimizations) # Expensive optimizations (mainly, profile-guided optimizations)
@ -145,13 +145,6 @@ License: Python
%global py_INSTSONAME_optimized libpython%{LDVERSION_optimized}.so.%{py_SOVERSION} %global py_INSTSONAME_optimized libpython%{LDVERSION_optimized}.so.%{py_SOVERSION}
%global py_INSTSONAME_debug libpython%{LDVERSION_debug}.so.%{py_SOVERSION} %global py_INSTSONAME_debug libpython%{LDVERSION_debug}.so.%{py_SOVERSION}
# The -O flag for the compiler, optimized builds
# https://fedoraproject.org/wiki/Changes/Python_built_with_gcc_O3
%global optflags_optimized -O3
# The -O flag for the compiler, debug builds
# -Wno-cpp avoids some warnings with -O0
%global optflags_debug -O0 -Wno-cpp
# Disable automatic bytecompilation. The python3 binary is not yet be # Disable automatic bytecompilation. The python3 binary is not yet be
# available in /usr/bin when Python is built. Also, the bytecompilation fails # available in /usr/bin when Python is built. Also, the bytecompilation fails
# on files that test invalid syntax. # on files that test invalid syntax.
@ -339,65 +332,6 @@ Patch329: 00329-fips.patch
# https://github.com/GrahamDumpleton/mod_wsgi/issues/730 # https://github.com/GrahamDumpleton/mod_wsgi/issues/730
Patch371: 00371-revert-bpo-1596321-fix-threading-_shutdown-for-the-main-thread-gh-28549-gh-28589.patch Patch371: 00371-revert-bpo-1596321-fix-threading-_shutdown-for-the-main-thread-gh-28549-gh-28589.patch
# 00397 #
# Filters for tarfile extraction (CVE-2007-4559, PEP-706)
# First patch fixes determination of symlink targets, which were treated
# as relative to the root of the archive,
# rather than the directory containing the symlink.
# Not yet upstream as of this writing.
# The second patch is Red Hat configuration, see KB for documentation:
# - https://access.redhat.com/articles/7004769
Patch397: 00397-tarfile-filter.patch
# 00415 #
# [CVE-2023-27043] gh-102988: Reject malformed addresses in email.parseaddr() (#111116)
#
# Detect email address parsing errors and return empty tuple to
# indicate the parsing error (old API). Add an optional 'strict'
# parameter to getaddresses() and parseaddr() functions. Patch by
# Thomas Dwyer.
#
# Upstream PR: https://github.com/python/cpython/pull/111116
#
# Second patch implmenets the possibility to restore the old behavior via
# config file or environment variable.
Patch415: 00415-cve-2023-27043-gh-102988-reject-malformed-addresses-in-email-parseaddr-111116.patch
# 00422 #
# Fix the test suite for releases of expat < 2.6.0
# which backport the CVE-2023-52425 fix.
# Downstream only.
Patch422: 00422-fix-expat-tests.patch
# 00431 #
# Security fix for CVE-2024-4032: incorrect IPv4 and IPv6 private ranges
# Resolved upstream: https://github.com/python/cpython/issues/113171
# Tracking bugzilla: https://bugzilla.redhat.com/show_bug.cgi?id=2292921
Patch431: 00431-CVE-2024-4032.patch
# 00435 # d33a3c90daa3d5d2d7e67f6e9264e5438d9608a0
# gh-121650: Encode newlines in headers, and verify headers are sound (GH-122233)
#
# Per RFC 2047:
#
# > [...] these encoding schemes allow the
# > encoding of arbitrary octet values, mail readers that implement this
# > decoding should also ensure that display of the decoded data on the
# > recipient's terminal will not cause unwanted side-effects
#
# It seems that the "quoted-word" scheme is a valid way to include
# a newline character in a header value, just like we already allow
# undecodable bytes or control characters.
# They do need to be properly quoted when serialized to text, though.
#
# This should fail for custom fold() implementations that aren't careful
# about newlines.
Patch435: 00435-gh-121650-encode-newlines-in-headers-and-verify-headers-are-sound-gh-122233.patch
# 00436 # 1acd6db660ad1124ab7ae449a841608dd9d9062d
# [CVE-2024-8088] gh-122905: Sanitize names in zipfile.Path.
Patch436: 00436-cve-2024-8088-gh-122905-sanitize-names-in-zipfile-path.patch
# (New patches go here ^^^) # (New patches go here ^^^)
# #
# When adding new patches to "python" and "python3" in Fedora, EL, etc., # When adding new patches to "python" and "python3" in Fedora, EL, etc.,
@ -768,7 +702,6 @@ BuildPython() {
ConfName=$1 ConfName=$1
ExtraConfigArgs=$2 ExtraConfigArgs=$2
MoreCFlags=$3 MoreCFlags=$3
MoreCFlagsNodist=$4
# Each build is done in its own directory # Each build is done in its own directory
ConfDir=build/$ConfName ConfDir=build/$ConfName
@ -808,7 +741,7 @@ BuildPython() {
$ExtraConfigArgs \ $ExtraConfigArgs \
%{nil} %{nil}
%global flags_override EXTRA_CFLAGS="$MoreCFlags" CFLAGS_NODIST="$CFLAGS_NODIST $MoreCFlags $MoreCFlagsNodist" %global flags_override EXTRA_CFLAGS="$MoreCFlags" CFLAGS_NODIST="$CFLAGS_NODIST $MoreCFlags"
%if %{without bootstrap} %if %{without bootstrap}
# Regenerate generated files (needs python3) # Regenerate generated files (needs python3)
@ -831,14 +764,12 @@ BuildPython() {
# See also: https://bugzilla.redhat.com/show_bug.cgi?id=1818857 # See also: https://bugzilla.redhat.com/show_bug.cgi?id=1818857
BuildPython debug \ BuildPython debug \
"--without-ensurepip --with-pydebug" \ "--without-ensurepip --with-pydebug" \
"%{optflags_debug}" \ "-O0 -Wno-cpp"
""
%endif # with debug_build %endif # with debug_build
BuildPython optimized \ BuildPython optimized \
"--without-ensurepip %{optimizations_flag}" \ "--without-ensurepip %{optimizations_flag}" \
"" \ ""
"%{optflags_optimized}"
# ====================================================== # ======================================================
# Installing the built code: # Installing the built code:
@ -937,7 +868,7 @@ EOF
%if %{with debug_build} %if %{with debug_build}
InstallPython debug \ InstallPython debug \
%{py_INSTSONAME_debug} \ %{py_INSTSONAME_debug} \
"%{optflags_debug}" \ -O0 \
%{LDVERSION_debug} %{LDVERSION_debug}
%endif # with debug_build %endif # with debug_build
@ -996,10 +927,6 @@ for tool in pygettext msgfmt; do
ln -s ${tool}%{pybasever}.py %{buildroot}%{_bindir}/${tool}3.py ln -s ${tool}%{pybasever}.py %{buildroot}%{_bindir}/${tool}3.py
done done
# Install missing test data
# Fixed upstream: https://github.com/python/cpython/pull/112784
cp -rp Lib/test/regrtestdata/ %{buildroot}%{pylibdir}/test/
# Switch all shebangs to refer to the specific Python version. # Switch all shebangs to refer to the specific Python version.
# This currently only covers files matching ^[a-zA-Z0-9_]+\.py$, # This currently only covers files matching ^[a-zA-Z0-9_]+\.py$,
# so handle files named using other naming scheme separately. # so handle files named using other naming scheme separately.
@ -1158,14 +1085,10 @@ CheckPython() {
# test_freeze_simple_script is skipped, because it fails when bundled wheels # test_freeze_simple_script is skipped, because it fails when bundled wheels
# are removed in Fedora. # are removed in Fedora.
# upstream report: https://bugs.python.org/issue45783 # upstream report: https://bugs.python.org/issue45783
# test_check_probes is failing since it was introduced in 3.11.5,
# the test is skipped until it is fixed in upstream.
# see: https://github.com/python/cpython/issues/104280#issuecomment-1669249980
LD_LIBRARY_PATH=$ConfDir $ConfDir/python -m test.regrtest \ LD_LIBRARY_PATH=$ConfDir $ConfDir/python -m test.regrtest \
-wW --slowest -j0 --timeout=1800 \ -wW --slowest -j0 --timeout=1800 \
-i test_freeze_simple_script \ -i test_freeze_simple_script \
-i test_check_probes \
%if %{with bootstrap} %if %{with bootstrap}
-x test_distutils \ -x test_distutils \
%endif %endif
@ -1676,60 +1599,8 @@ CheckPython optimized
# ====================================================== # ======================================================
%changelog %changelog
* Fri Aug 23 2024 Charalampos Stratakis <cstratak@redhat.com> - 3.11.9-7 * Wed May 24 2023 MSVSphere Packaging Team <packager@msvsphere.ru> - 3.11.2-2
- Security fix for CVE-2024-8088 - Rebuilt for MSVSphere 9.2
Resolves: RHEL-55959
* Thu Aug 15 2024 Charalampos Stratakis <cstratak@redhat.com> - 3.11.9-6
- Security fix for CVE-2024-6923
Resolves: RHEL-53038
* Thu Jul 25 2024 Charalampos Stratakis <cstratak@redhat.com> - 3.11.9-5
- Properly propagate the optimization flags to C extensions
* Thu Jul 18 2024 Charalampos Stratakis <cstratak@redhat.com> - 3.11.9-4
- Build Python with -O3
- https://fedoraproject.org/wiki/Changes/Python_built_with_gcc_O3
* Thu Jul 18 2024 Charalampos Stratakis <cstratak@redhat.com> - 3.11.9-3
- Security fix for CVE-2024-4032
Resolves: RHEL-44099
* Tue Jun 11 2024 Charalampos Stratakis <cstratak@redhat.com> - 3.11.9-2
- Enable importing of hash-based .pyc files under FIPS mode
Resolves: RHEL-40779
* Mon Apr 22 2024 Charalampos Stratakis <cstratak@redhat.com> - 3.11.9-1
- Rebase to 3.11.9
- Security fixes for CVE-2023-6597 and CVE-2024-0450
- Fix expat tests for the latest expat security release
Resolves: RHEL-33677, RHEL-33689
* Mon Jan 22 2024 Charalampos Stratakis <cstratak@redhat.com> - 3.11.7-1
- Rebase to 3.11.7
Resolves: RHEL-20233
* Tue Jan 09 2024 Lumír Balhar <lbalhar@redhat.com> - 3.11.5-2
- Security fix for CVE-2023-27043
Resolves: RHEL-21325
* Thu Sep 07 2023 Charalampos Stratakis <cstratak@redhat.com> - 3.11.5-1
- Rebase to 3.11.5
- Security fixes for CVE-2023-40217 and CVE-2023-41105
Resolves: RHEL-3045, RHEL-3269
* Wed Aug 09 2023 Petr Viktorin <pviktori@redhat.com> - 3.11.4-3
- Fix symlink handling in the fix for CVE-2023-24329
Resolves: rhbz#263261
* Fri Jun 30 2023 Charalampos Stratakis <cstratak@redhat.com> - 3.11.4-2
- Security fix for CVE-2007-4559
Resolves: rhbz#263261
* Mon Jun 26 2023 Charalampos Stratakis <cstratak@redhat.com> - 3.11.4-1
- Update to 3.11.4
- Security fix for CVE-2023-24329
Resolves: rhbz#2173917
* Thu Feb 16 2023 Charalampos Stratakis <cstratak@redhat.com> - 3.11.2-2 * Thu Feb 16 2023 Charalampos Stratakis <cstratak@redhat.com> - 3.11.2-2
- Support OpenSSL FIPS mode - Support OpenSSL FIPS mode

Loading…
Cancel
Save