diff --git a/SOURCES/00329-fips.patch b/SOURCES/00329-fips.patch index cec8740..6038ccb 100644 --- a/SOURCES/00329-fips.patch +++ b/SOURCES/00329-fips.patch @@ -1,4 +1,4 @@ -From ccb2659fa0ec259d4161ed84345553bf3f216531 Mon Sep 17 00:00:00 2001 +From 4cef6c756055a15dc33a475c1f405676fa69410c Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 11 Aug 2021 16:51:03 +0200 Subject: [PATCH 01/10] Backport PyModule_AddObjectRef as @@ -71,10 +71,10 @@ index 13482c6..fca1083 100644 PyModule_AddIntConstant(PyObject *m, const char *name, long value) { -- -2.37.2 +2.45.0 -From 794c37495d91823bd820b96382b999d84dcad58d Mon Sep 17 00:00:00 2001 +From a115773e979f968edaed8a3419f3ccc34eef8320 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 13 Aug 2021 13:16:43 +0200 Subject: [PATCH 02/10] _hashopenssl: Uncomment and use initialization function @@ -144,10 +144,10 @@ index 4db058c..56dfff9 100644 return m; -- -2.37.2 +2.45.0 -From 94b56c82b459474c3e0f9e5421fa7becbf5a1c70 Mon Sep 17 00:00:00 2001 +From 28506af69b2b005bb7a8931624f97273269458a1 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Sat, 27 Mar 2021 14:55:03 +0100 Subject: [PATCH 03/10] bpo-40645: use C implementation of HMAC (GH-24920, @@ -927,10 +927,10 @@ index 68aa765..4466ec4 100644 -/*[clinic end generated code: output=b6b280e46bf0b139 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7ff9aad0bd53e7ce input=a9049054013a1b77]*/ -- -2.37.2 +2.45.0 -From b63e3fbd7c0506b5a6c00c1bb0d255054e38bbe8 Mon Sep 17 00:00:00 2001 +From 1f5f7d3892febb68cf96ef151beab06eae1792ce Mon Sep 17 00:00:00 2001 From: Charalampos Stratakis Date: Thu, 12 Dec 2019 16:58:31 +0100 Subject: [PATCH 04/10] Expose blake2b and blake2s hashes from OpenSSL @@ -944,7 +944,7 @@ used under FIPS. 3 files changed, 148 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py -index f845c7a..7aaeb76 100644 +index bc11a8d..9a07499 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -363,6 +363,12 @@ class HashLibTestCase(unittest.TestCase): @@ -1137,10 +1137,10 @@ index 4466ec4..54c22b2 100644 -/*[clinic end generated code: output=7ff9aad0bd53e7ce input=a9049054013a1b77]*/ +/*[clinic end generated code: output=fab05055e982f112 input=a9049054013a1b77]*/ -- -2.37.2 +2.45.0 -From dc8ad7b98d6d9bf14cae439acb3a99fa8f4f5020 Mon Sep 17 00:00:00 2001 +From 842bd1c8c8a62c342fed848e5a5f0f1d97daeaba Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 1 Aug 2019 17:57:05 +0200 Subject: [PATCH 05/10] Use a stronger hash in multiprocessing handshake @@ -1152,7 +1152,7 @@ https://bugs.python.org/issue17258 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py -index 510e4b5..b68f2fb 100644 +index 8e2facf..bb4acb6 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -42,6 +42,10 @@ BUFSIZE = 8192 @@ -1166,7 +1166,7 @@ index 510e4b5..b68f2fb 100644 _mmap_counter = itertools.count() default_family = 'AF_INET' -@@ -741,7 +745,7 @@ def deliver_challenge(connection, authkey): +@@ -736,7 +740,7 @@ def deliver_challenge(connection, authkey): "Authkey must be bytes, not {0!s}".format(type(authkey))) message = os.urandom(MESSAGE_LENGTH) connection.send_bytes(CHALLENGE + message) @@ -1175,7 +1175,7 @@ index 510e4b5..b68f2fb 100644 response = connection.recv_bytes(256) # reject large message if response == digest: connection.send_bytes(WELCOME) -@@ -757,7 +761,7 @@ def answer_challenge(connection, authkey): +@@ -752,7 +756,7 @@ def answer_challenge(connection, authkey): message = connection.recv_bytes(256) # reject large message assert message[:len(CHALLENGE)] == CHALLENGE, 'message = %r' % message message = message[len(CHALLENGE):] @@ -1185,10 +1185,10 @@ index 510e4b5..b68f2fb 100644 response = connection.recv_bytes(256) # reject large message if response != WELCOME: -- -2.37.2 +2.45.0 -From af0c88c9d5bc4f9c127e49ed80d14e25d18813f2 Mon Sep 17 00:00:00 2001 +From 0390f1ea33f8a604467829733541791a29cee4da Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 25 Jul 2019 17:19:06 +0200 Subject: [PATCH 06/10] Disable Python's hash implementations in FIPS mode, @@ -1231,7 +1231,7 @@ index ffa3be0..3e3f4dd 100644 def __get_builtin_constructor(name): cache = __builtin_constructor_cache diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py -index 7aaeb76..fa4a8d7 100644 +index 9a07499..56dfbaa 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -35,14 +35,15 @@ else: @@ -1446,10 +1446,10 @@ index 0bec170..479f4b5 100644 )) -- -2.37.2 +2.45.0 -From 9bc3d493a3508fb82df7d24cc62315c072d9eca8 Mon Sep 17 00:00:00 2001 +From 667781e01425308ae95f062f8596866a5af76f77 Mon Sep 17 00:00:00 2001 From: Charalampos Stratakis Date: Fri, 29 Jan 2021 14:16:21 +0100 Subject: [PATCH 07/10] Use python's fall back crypto implementations only if @@ -1565,7 +1565,7 @@ index 3e3f4dd..b842f5f 100644 for __func_name in __always_supported: diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py -index fa4a8d7..ec6c883 100644 +index 56dfbaa..05f4a54 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -171,7 +171,13 @@ class HashLibTestCase(unittest.TestCase): @@ -1604,7 +1604,7 @@ index fa4a8d7..ec6c883 100644 def test_get_builtin_constructor(self): get_builtin_constructor = getattr(hashlib, '__get_builtin_constructor') -@@ -1061,6 +1081,7 @@ class KDFTests(unittest.TestCase): +@@ -1070,6 +1090,7 @@ class KDFTests(unittest.TestCase): iterations=1, dklen=None) self.assertEqual(out, self.pbkdf2_results['sha1'][0][0]) @@ -1613,10 +1613,10 @@ index fa4a8d7..ec6c883 100644 def test_pbkdf2_hmac_py(self): self._test_pbkdf2_hmac(builtin_hashlib.pbkdf2_hmac, builtin_hashes) -- -2.37.2 +2.45.0 -From 331c0d39cbc9c4df266c375bae8c1a0d27dd78d9 Mon Sep 17 00:00:00 2001 +From e36df0297eca09ca1826163f39a812b45558f690 Mon Sep 17 00:00:00 2001 From: Charalampos Stratakis Date: Wed, 31 Jul 2019 15:43:43 +0200 Subject: [PATCH 08/10] Test equivalence of hashes for the various digests with @@ -1659,7 +1659,7 @@ index 0000000..1f99dd7 +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py -index ec6c883..0fd036f 100644 +index 05f4a54..980c773 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -20,6 +20,7 @@ import warnings @@ -1755,7 +1755,7 @@ index ec6c883..0fd036f 100644 return m = hash_object_constructor(data, **kwargs) -@@ -974,6 +989,15 @@ class HashLibTestCase(unittest.TestCase): +@@ -983,6 +998,15 @@ class HashLibTestCase(unittest.TestCase): ): HASHXOF() @@ -1772,10 +1772,10 @@ index ec6c883..0fd036f 100644 class KDFTests(unittest.TestCase): -- -2.37.2 +2.45.0 -From 1a3df28f95710925bc80018bcf22b7f37bbb1e17 Mon Sep 17 00:00:00 2001 +From 8fc0216da1d6e148bf086c8c137ddc19a33ab642 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 26 Aug 2019 19:39:48 +0200 Subject: [PATCH 09/10] Guard against Python HMAC in FIPS mode @@ -1889,290 +1889,43 @@ index adf52ad..41e6a14 100644 def test_realcopy_old(self): # Testing if the copy method created a real copy. -- -2.37.2 +2.45.0 -From dded0e09dd3e51998a2aa54d2ae8464d73987e51 Mon Sep 17 00:00:00 2001 -From: Petr Viktorin -Date: Wed, 25 Aug 2021 16:44:43 +0200 -Subject: [PATCH 10/10] Disable hash-based PYCs in FIPS mode - -If FIPS mode is on, we can't use siphash-based HMAC -(_Py_KeyedHash), so: - -- Unchecked hash PYCs can be imported, but not created -- Checked hash PYCs can not be imported nor created -- The default mode is timestamp-based PYCs, even if - SOURCE_DATE_EPOCH is set. +From 4271e404c9aa918368d397654d60e4e845dfc844 Mon Sep 17 00:00:00 2001 +From: Nikita Sobolev +Date: Thu, 24 Nov 2022 01:47:31 +0300 +Subject: [PATCH 10/10] closes gh-99508: fix `TypeError` in + `Lib/importlib/_bootstrap_external.py` (GH-99635) -If FIPS mode is off, there are no changes in behavior. - -Resolves: https://bugzilla.redhat.com/show_bug.cgi?id=1835169 --- - Lib/py_compile.py | 2 ++ - Lib/test/support/__init__.py | 14 +++++++++++++ - Lib/test/test_cmd_line_script.py | 2 ++ - Lib/test/test_compileall.py | 11 +++++++++- - Lib/test/test_imp.py | 2 ++ - .../test_importlib/source/test_file_loader.py | 6 ++++++ - Lib/test/test_py_compile.py | 11 ++++++++-- - Lib/test/test_zipimport.py | 2 ++ - Python/import.c | 20 +++++++++++++++++++ - 9 files changed, 67 insertions(+), 3 deletions(-) + Lib/importlib/_bootstrap_external.py | 3 ++- + .../next/Library/2022-11-21-10-45-54.gh-issue-99508.QqVbby.rst | 2 ++ + 2 files changed, 4 insertions(+), 1 deletion(-) + create mode 100644 Misc/NEWS.d/next/Library/2022-11-21-10-45-54.gh-issue-99508.QqVbby.rst -diff --git a/Lib/py_compile.py b/Lib/py_compile.py -index bba3642..02db901 100644 ---- a/Lib/py_compile.py -+++ b/Lib/py_compile.py -@@ -70,7 +70,9 @@ class PycInvalidationMode(enum.Enum): - - - def _get_default_invalidation_mode(): -+ import _hashlib - if (os.environ.get('SOURCE_DATE_EPOCH') and not -+ _hashlib.get_fips_mode() and not - os.environ.get('RPM_BUILD_ROOT')): - return PycInvalidationMode.CHECKED_HASH - else: -diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py -index 6dc0813..b9d5f9a 100644 ---- a/Lib/test/support/__init__.py -+++ b/Lib/test/support/__init__.py -@@ -3296,6 +3296,20 @@ def clear_ignored_deprecations(*tokens: object) -> None: - warnings._filters_mutated() - - -+def fails_in_fips_mode(expected_error): -+ import _hashlib -+ if _hashlib.get_fips_mode(): -+ def _decorator(func): -+ def _wrapper(self, *args, **kwargs): -+ with self.assertRaises(expected_error): -+ func(self, *args, **kwargs) -+ return _wrapper -+ else: -+ def _decorator(func): -+ return func -+ return _decorator -+ -+ - @contextlib.contextmanager - def adjust_int_max_str_digits(max_digits): - """Temporarily change the integer string conversion length limit.""" -diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py -index 7cb1370..61df232 100644 ---- a/Lib/test/test_cmd_line_script.py -+++ b/Lib/test/test_cmd_line_script.py -@@ -282,6 +282,7 @@ class CmdLineTest(unittest.TestCase): - self._check_script(zip_name, run_name, zip_name, zip_name, '', - zipimport.zipimporter) - -+ @support.fails_in_fips_mode(ImportError) - def test_zipfile_compiled_checked_hash(self): - with support.temp_dir() as script_dir: - script_name = _make_test_script(script_dir, '__main__') -@@ -292,6 +293,7 @@ class CmdLineTest(unittest.TestCase): - self._check_script(zip_name, run_name, zip_name, zip_name, '', - zipimport.zipimporter) - -+ @support.fails_in_fips_mode(ImportError) - def test_zipfile_compiled_unchecked_hash(self): - with support.temp_dir() as script_dir: - script_name = _make_test_script(script_dir, '__main__') -diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py -index ab647d6..7d50f07 100644 ---- a/Lib/test/test_compileall.py -+++ b/Lib/test/test_compileall.py -@@ -758,14 +758,23 @@ class CommandLineTestsBase: - out = self.assertRunOK('badfilename') - self.assertRegex(out, b"Can't list 'badfilename'") - -- def test_pyc_invalidation_mode(self): -+ @support.fails_in_fips_mode(AssertionError) -+ def test_pyc_invalidation_mode_checked(self): - script_helper.make_script(self.pkgdir, 'f1', '') - pyc = importlib.util.cache_from_source( - os.path.join(self.pkgdir, 'f1.py')) -+ - self.assertRunOK('--invalidation-mode=checked-hash', self.pkgdir) - with open(pyc, 'rb') as fp: - data = fp.read() - self.assertEqual(int.from_bytes(data[4:8], 'little'), 0b11) -+ -+ @support.fails_in_fips_mode(AssertionError) -+ def test_pyc_invalidation_mode_unchecked(self): -+ script_helper.make_script(self.pkgdir, 'f1', '') -+ pyc = importlib.util.cache_from_source( -+ os.path.join(self.pkgdir, 'f1.py')) -+ - self.assertRunOK('--invalidation-mode=unchecked-hash', self.pkgdir) - with open(pyc, 'rb') as fp: - data = fp.read() -diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py -index fe394dc..802f0e8 100644 ---- a/Lib/test/test_imp.py -+++ b/Lib/test/test_imp.py -@@ -343,6 +343,7 @@ class ImportTests(unittest.TestCase): - import _frozen_importlib - self.assertEqual(_frozen_importlib.__spec__.origin, "frozen") - -+ @support.fails_in_fips_mode(ImportError) - def test_source_hash(self): - self.assertEqual(_imp.source_hash(42, b'hi'), b'\xc6\xe7Z\r\x03:}\xab') - self.assertEqual(_imp.source_hash(43, b'hi'), b'\x85\x9765\xf8\x9a\x8b9') -@@ -362,6 +363,7 @@ class ImportTests(unittest.TestCase): - res = script_helper.assert_python_ok(*args) - self.assertEqual(res.out.strip().decode('utf-8'), expected) - -+ @support.fails_in_fips_mode(ImportError) - def test_find_and_load_checked_pyc(self): - # issue 34056 - with support.temp_cwd(): -diff --git a/Lib/test/test_importlib/source/test_file_loader.py b/Lib/test/test_importlib/source/test_file_loader.py -index ab44722..480cc81 100644 ---- a/Lib/test/test_importlib/source/test_file_loader.py -+++ b/Lib/test/test_importlib/source/test_file_loader.py -@@ -17,6 +17,7 @@ import types - import unittest - import warnings - -+from test import support - from test.support import make_legacy_pyc, unload - - from test.test_py_compile import without_source_date_epoch -@@ -239,6 +240,7 @@ class SimpleTest(abc.LoaderTests): - loader.load_module('bad name') - - @util.writes_bytecode_files -+ @support.fails_in_fips_mode(ImportError) - def test_checked_hash_based_pyc(self): - with util.create_modules('_temp') as mapping: - source = mapping['_temp'] -@@ -270,6 +272,7 @@ class SimpleTest(abc.LoaderTests): - ) - - @util.writes_bytecode_files -+ @support.fails_in_fips_mode(ImportError) - def test_overridden_checked_hash_based_pyc(self): - with util.create_modules('_temp') as mapping, \ - unittest.mock.patch('_imp.check_hash_based_pycs', 'never'): -@@ -295,6 +298,7 @@ class SimpleTest(abc.LoaderTests): - self.assertEqual(mod.state, 'old') - - @util.writes_bytecode_files -+ @support.fails_in_fips_mode(ImportError) - def test_unchecked_hash_based_pyc(self): - with util.create_modules('_temp') as mapping: - source = mapping['_temp'] -@@ -325,6 +329,7 @@ class SimpleTest(abc.LoaderTests): - ) - - @util.writes_bytecode_files -+ @support.fails_in_fips_mode(ImportError) - def test_overridden_unchecked_hash_based_pyc(self): - with util.create_modules('_temp') as mapping, \ - unittest.mock.patch('_imp.check_hash_based_pycs', 'always'): -@@ -434,6 +439,7 @@ class BadBytecodeTest: - del_source=del_source) - test('_temp', mapping, bc_path) - -+ @support.fails_in_fips_mode(ImportError) - def _test_partial_hash(self, test, *, del_source=False): - with util.create_modules('_temp') as mapping: - bc_path = self.manipulate_bytecode( -diff --git a/Lib/test/test_py_compile.py b/Lib/test/test_py_compile.py -index b2d3dcf..7e4b0c5 100644 ---- a/Lib/test/test_py_compile.py -+++ b/Lib/test/test_py_compile.py -@@ -141,13 +141,16 @@ class PyCompileTestsBase: - importlib.util.cache_from_source(bad_coding))) - - def test_source_date_epoch(self): -+ import _hashlib - py_compile.compile(self.source_path, self.pyc_path) - self.assertTrue(os.path.exists(self.pyc_path)) - self.assertFalse(os.path.exists(self.cache_path)) - with open(self.pyc_path, 'rb') as fp: - flags = importlib._bootstrap_external._classify_pyc( - fp.read(), 'test', {}) -- if os.environ.get('SOURCE_DATE_EPOCH'): -+ if _hashlib.get_fips_mode(): -+ expected_flags = 0b00 -+ elif os.environ.get('SOURCE_DATE_EPOCH'): - expected_flags = 0b11 - else: - expected_flags = 0b00 -@@ -178,7 +181,8 @@ class PyCompileTestsBase: - # Specifying optimized bytecode should lead to a path reflecting that. - self.assertIn('opt-2', py_compile.compile(self.source_path, optimize=2)) - -- def test_invalidation_mode(self): -+ @support.fails_in_fips_mode(ImportError) -+ def test_invalidation_mode_checked(self): - py_compile.compile( - self.source_path, - invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH, -@@ -187,6 +191,9 @@ class PyCompileTestsBase: - flags = importlib._bootstrap_external._classify_pyc( - fp.read(), 'test', {}) - self.assertEqual(flags, 0b11) -+ -+ @support.fails_in_fips_mode(ImportError) -+ def test_invalidation_mode_unchecked(self): - py_compile.compile( - self.source_path, - invalidation_mode=py_compile.PycInvalidationMode.UNCHECKED_HASH, -diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py -index b7347a3..09ea990 100644 ---- a/Lib/test/test_zipimport.py -+++ b/Lib/test/test_zipimport.py -@@ -186,6 +186,7 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase): - TESTMOD + pyc_ext: (NOW, test_pyc)} - self.doTest(pyc_ext, files, TESTMOD) - -+ @support.fails_in_fips_mode(ImportError) - def testUncheckedHashBasedPyc(self): - source = b"state = 'old'" - source_hash = importlib.util.source_hash(source) -@@ -200,6 +201,7 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase): - self.assertEqual(mod.state, 'old') - self.doTest(None, files, TESTMOD, call=check) - -+ @support.fails_in_fips_mode(ImportError) - @unittest.mock.patch('_imp.check_hash_based_pycs', 'always') - def test_checked_hash_based_change_pyc(self): - source = b"state = 'old'" -diff --git a/Python/import.c b/Python/import.c -index 8358d70..1b7fb85 100644 ---- a/Python/import.c -+++ b/Python/import.c -@@ -2354,6 +2354,26 @@ static PyObject * - _imp_source_hash_impl(PyObject *module, long key, Py_buffer *source) - /*[clinic end generated code: output=edb292448cf399ea input=9aaad1e590089789]*/ - { -+ PyObject *_hashlib = PyImport_ImportModule("_hashlib"); -+ if (_hashlib == NULL) { -+ return NULL; -+ } -+ PyObject *fips_mode_obj = PyObject_CallMethod(_hashlib, "get_fips_mode", NULL); -+ Py_DECREF(_hashlib); -+ if (fips_mode_obj == NULL) { -+ return NULL; -+ } -+ int fips_mode = PyObject_IsTrue(fips_mode_obj); -+ Py_DECREF(fips_mode_obj); -+ if (fips_mode < 0) { -+ return NULL; -+ } -+ if (fips_mode) { -+ PyErr_SetString( -+ PyExc_ImportError, -+ "hash-based PYC validation (siphash24) not available in FIPS mode"); -+ return NULL; -+ }; - union { - uint64_t x; - char data[sizeof(uint64_t)]; +diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py +index f0c9f8e..cccf6b2 100644 +--- a/Lib/importlib/_bootstrap_external.py ++++ b/Lib/importlib/_bootstrap_external.py +@@ -986,7 +986,8 @@ class SourceLoader(_LoaderBasics): + source_mtime is not None): + if hash_based: + if source_hash is None: +- source_hash = _imp.source_hash(source_bytes) ++ source_hash = _imp.source_hash(_RAW_MAGIC_NUMBER, ++ source_bytes) + data = _code_to_hash_pyc(code_object, source_hash, check_source) + else: + data = _code_to_timestamp_pyc(code_object, source_mtime, +diff --git a/Misc/NEWS.d/next/Library/2022-11-21-10-45-54.gh-issue-99508.QqVbby.rst b/Misc/NEWS.d/next/Library/2022-11-21-10-45-54.gh-issue-99508.QqVbby.rst +new file mode 100644 +index 0000000..82720d1 +--- /dev/null ++++ b/Misc/NEWS.d/next/Library/2022-11-21-10-45-54.gh-issue-99508.QqVbby.rst +@@ -0,0 +1,2 @@ ++Fix ``TypeError`` in ``Lib/importlib/_bootstrap_external.py`` while calling ++``_imp.source_hash()``. -- -2.37.2 +2.45.0 diff --git a/SOURCES/00431-CVE-2024-4032.patch b/SOURCES/00431-CVE-2024-4032.patch new file mode 100644 index 0000000..03bc928 --- /dev/null +++ b/SOURCES/00431-CVE-2024-4032.patch @@ -0,0 +1,402 @@ +From f647bd8884bc89767914a5e0dea9ae099a8b50b5 Mon Sep 17 00:00:00 2001 +From: Petr Viktorin +Date: Tue, 7 May 2024 11:57:58 +0200 +Subject: [PATCH] gh-113171: gh-65056: Fix "private" (non-global) IP address + ranges (GH-113179) (GH-113186) (GH-118177) (GH-118472) + +The _private_networks variables, used by various is_private +implementations, were missing some ranges and at the same time had +overly strict ranges (where there are more specific ranges considered +globally reachable by the IANA registries). + +This patch updates the ranges with what was missing or otherwise +incorrect. + +100.64.0.0/10 is left alone, for now, as it's been made special in [1]. + +The _address_exclude_many() call returns 8 networks for IPv4, 121 +networks for IPv6. + +[1] https://github.com/python/cpython/issues/61602 + +In 3.10 and below, is_private checks whether the network and broadcast +address are both private. +In later versions (where the test wss backported from), it checks +whether they both are in the same private network. + +For 0.0.0.0/0, both 0.0.0.0 and 255.225.255.255 are private, +but one is in 0.0.0.0/8 ("This network") and the other in +255.255.255.255/32 ("Limited broadcast"). + +--------- + +Co-authored-by: Jakub Stasiak +--- + Doc/library/ipaddress.rst | 43 ++++++++- + Doc/tools/susp-ignored.csv | 8 ++ + Doc/whatsnew/3.9.rst | 9 ++ + Lib/ipaddress.py | 95 +++++++++++++++---- + Lib/test/test_ipaddress.py | 52 ++++++++++ + ...-03-14-01-38-44.gh-issue-113171.VFnObz.rst | 9 ++ + 6 files changed, 195 insertions(+), 21 deletions(-) + create mode 100644 Misc/NEWS.d/next/Library/2024-03-14-01-38-44.gh-issue-113171.VFnObz.rst + +diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst +index 9c2dff5..f9c1ebf 100644 +--- a/Doc/library/ipaddress.rst ++++ b/Doc/library/ipaddress.rst +@@ -188,18 +188,53 @@ write code that handles both IP versions correctly. Address objects are + + .. attribute:: is_private + +- ``True`` if the address is allocated for private networks. See ++ ``True`` if the address is defined as not globally reachable by + iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ +- (for IPv6). ++ (for IPv6) with the following exceptions: ++ ++ * ``is_private`` is ``False`` for the shared address space (``100.64.0.0/10``) ++ * For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the ++ semantics of the underlying IPv4 addresses and the following condition holds ++ (see :attr:`IPv6Address.ipv4_mapped`):: ++ ++ address.is_private == address.ipv4_mapped.is_private ++ ++ ``is_private`` has value opposite to :attr:`is_global`, except for the shared address space ++ (``100.64.0.0/10`` range) where they are both ``False``. ++ ++ .. versionchanged:: 3.9.20 ++ ++ Fixed some false positives and false negatives. ++ ++ * ``192.0.0.0/24`` is considered private with the exception of ``192.0.0.9/32`` and ++ ``192.0.0.10/32`` (previously: only the ``192.0.0.0/29`` sub-range was considered private). ++ * ``64:ff9b:1::/48`` is considered private. ++ * ``2002::/16`` is considered private. ++ * There are exceptions within ``2001::/23`` (otherwise considered private): ``2001:1::1/128``, ++ ``2001:1::2/128``, ``2001:3::/32``, ``2001:4:112::/48``, ``2001:20::/28``, ``2001:30::/28``. ++ The exceptions are not considered private. + + .. attribute:: is_global + +- ``True`` if the address is allocated for public networks. See ++ ``True`` if the address is defined as globally reachable by + iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ +- (for IPv6). ++ (for IPv6) with the following exception: ++ ++ For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the ++ semantics of the underlying IPv4 addresses and the following condition holds ++ (see :attr:`IPv6Address.ipv4_mapped`):: ++ ++ address.is_global == address.ipv4_mapped.is_global ++ ++ ``is_global`` has value opposite to :attr:`is_private`, except for the shared address space ++ (``100.64.0.0/10`` range) where they are both ``False``. + + .. versionadded:: 3.4 + ++ .. versionchanged:: 3.9.20 ++ ++ Fixed some false positives and false negatives, see :attr:`is_private` for details. ++ + .. attribute:: is_unspecified + + ``True`` if the address is unspecified. See :RFC:`5735` (for IPv4) +diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv +index 3eb3d79..de91a50 100644 +--- a/Doc/tools/susp-ignored.csv ++++ b/Doc/tools/susp-ignored.csv +@@ -169,6 +169,14 @@ library/ipaddress,,:db00,2001:db00::0/24 + library/ipaddress,,::,2001:db00::0/24 + library/ipaddress,,:db00,2001:db00::0/ffff:ff00:: + library/ipaddress,,::,2001:db00::0/ffff:ff00:: ++library/ipaddress,,:ff9b,64:ff9b:1::/48 ++library/ipaddress,,::,64:ff9b:1::/48 ++library/ipaddress,,::,2001:: ++library/ipaddress,,::,2001:1:: ++library/ipaddress,,::,2001:3:: ++library/ipaddress,,::,2001:4:112:: ++library/ipaddress,,::,2001:20:: ++library/ipaddress,,::,2001:30:: + library/itertools,,:step,elements from seq[start:stop:step] + library/itertools,,:stop,elements from seq[start:stop:step] + library/itertools,,::,kernel = tuple(kernel)[::-1] +diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst +index 0064e07..1756a37 100644 +--- a/Doc/whatsnew/3.9.rst ++++ b/Doc/whatsnew/3.9.rst +@@ -1616,3 +1616,12 @@ tarfile + :exc:`DeprecationWarning`. + In Python 3.14, the default will switch to ``'data'``. + (Contributed by Petr Viktorin in :pep:`706`.) ++ ++Notable changes in 3.9.20 ++========================= ++ ++ipaddress ++--------- ++ ++* Fixed ``is_global`` and ``is_private`` behavior in ``IPv4Address``, ++ ``IPv6Address``, ``IPv4Network`` and ``IPv6Network``. +diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py +index 25f373a..9b35340 100644 +--- a/Lib/ipaddress.py ++++ b/Lib/ipaddress.py +@@ -1322,18 +1322,41 @@ class IPv4Address(_BaseV4, _BaseAddress): + @property + @functools.lru_cache() + def is_private(self): +- """Test if this address is allocated for private networks. ++ """``True`` if the address is defined as not globally reachable by ++ iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ ++ (for IPv6) with the following exceptions: + +- Returns: +- A boolean, True if the address is reserved per +- iana-ipv4-special-registry. ++ * ``is_private`` is ``False`` for ``100.64.0.0/10`` ++ * For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the ++ semantics of the underlying IPv4 addresses and the following condition holds ++ (see :attr:`IPv6Address.ipv4_mapped`):: ++ ++ address.is_private == address.ipv4_mapped.is_private + ++ ``is_private`` has value opposite to :attr:`is_global`, except for the ``100.64.0.0/10`` ++ IPv4 range where they are both ``False``. + """ +- return any(self in net for net in self._constants._private_networks) ++ return ( ++ any(self in net for net in self._constants._private_networks) ++ and all(self not in net for net in self._constants._private_networks_exceptions) ++ ) + + @property + @functools.lru_cache() + def is_global(self): ++ """``True`` if the address is defined as globally reachable by ++ iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ ++ (for IPv6) with the following exception: ++ ++ For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the ++ semantics of the underlying IPv4 addresses and the following condition holds ++ (see :attr:`IPv6Address.ipv4_mapped`):: ++ ++ address.is_global == address.ipv4_mapped.is_global ++ ++ ``is_global`` has value opposite to :attr:`is_private`, except for the ``100.64.0.0/10`` ++ IPv4 range where they are both ``False``. ++ """ + return self not in self._constants._public_network and not self.is_private + + @property +@@ -1537,13 +1560,15 @@ class _IPv4Constants: + + _public_network = IPv4Network('100.64.0.0/10') + ++ # Not globally reachable address blocks listed on ++ # https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml + _private_networks = [ + IPv4Network('0.0.0.0/8'), + IPv4Network('10.0.0.0/8'), + IPv4Network('127.0.0.0/8'), + IPv4Network('169.254.0.0/16'), + IPv4Network('172.16.0.0/12'), +- IPv4Network('192.0.0.0/29'), ++ IPv4Network('192.0.0.0/24'), + IPv4Network('192.0.0.170/31'), + IPv4Network('192.0.2.0/24'), + IPv4Network('192.168.0.0/16'), +@@ -1554,6 +1579,11 @@ class _IPv4Constants: + IPv4Network('255.255.255.255/32'), + ] + ++ _private_networks_exceptions = [ ++ IPv4Network('192.0.0.9/32'), ++ IPv4Network('192.0.0.10/32'), ++ ] ++ + _reserved_network = IPv4Network('240.0.0.0/4') + + _unspecified_address = IPv4Address('0.0.0.0') +@@ -1995,23 +2025,42 @@ class IPv6Address(_BaseV6, _BaseAddress): + @property + @functools.lru_cache() + def is_private(self): +- """Test if this address is allocated for private networks. ++ """``True`` if the address is defined as not globally reachable by ++ iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ ++ (for IPv6) with the following exceptions: + +- Returns: +- A boolean, True if the address is reserved per +- iana-ipv6-special-registry. ++ * ``is_private`` is ``False`` for ``100.64.0.0/10`` ++ * For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the ++ semantics of the underlying IPv4 addresses and the following condition holds ++ (see :attr:`IPv6Address.ipv4_mapped`):: ++ ++ address.is_private == address.ipv4_mapped.is_private + ++ ``is_private`` has value opposite to :attr:`is_global`, except for the ``100.64.0.0/10`` ++ IPv4 range where they are both ``False``. + """ +- return any(self in net for net in self._constants._private_networks) ++ ipv4_mapped = self.ipv4_mapped ++ if ipv4_mapped is not None: ++ return ipv4_mapped.is_private ++ return ( ++ any(self in net for net in self._constants._private_networks) ++ and all(self not in net for net in self._constants._private_networks_exceptions) ++ ) + + @property + def is_global(self): +- """Test if this address is allocated for public networks. ++ """``True`` if the address is defined as globally reachable by ++ iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ ++ (for IPv6) with the following exception: + +- Returns: +- A boolean, true if the address is not reserved per +- iana-ipv6-special-registry. ++ For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the ++ semantics of the underlying IPv4 addresses and the following condition holds ++ (see :attr:`IPv6Address.ipv4_mapped`):: ++ ++ address.is_global == address.ipv4_mapped.is_global + ++ ``is_global`` has value opposite to :attr:`is_private`, except for the ``100.64.0.0/10`` ++ IPv4 range where they are both ``False``. + """ + return not self.is_private + +@@ -2252,19 +2301,31 @@ class _IPv6Constants: + + _multicast_network = IPv6Network('ff00::/8') + ++ # Not globally reachable address blocks listed on ++ # https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml + _private_networks = [ + IPv6Network('::1/128'), + IPv6Network('::/128'), + IPv6Network('::ffff:0:0/96'), ++ IPv6Network('64:ff9b:1::/48'), + IPv6Network('100::/64'), + IPv6Network('2001::/23'), +- IPv6Network('2001:2::/48'), + IPv6Network('2001:db8::/32'), +- IPv6Network('2001:10::/28'), ++ # IANA says N/A, let's consider it not globally reachable to be safe ++ IPv6Network('2002::/16'), + IPv6Network('fc00::/7'), + IPv6Network('fe80::/10'), + ] + ++ _private_networks_exceptions = [ ++ IPv6Network('2001:1::1/128'), ++ IPv6Network('2001:1::2/128'), ++ IPv6Network('2001:3::/32'), ++ IPv6Network('2001:4:112::/48'), ++ IPv6Network('2001:20::/28'), ++ IPv6Network('2001:30::/28'), ++ ] ++ + _reserved_networks = [ + IPv6Network('::/8'), IPv6Network('100::/8'), + IPv6Network('200::/7'), IPv6Network('400::/6'), +diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py +index 90897f6..bd14f04 100644 +--- a/Lib/test/test_ipaddress.py ++++ b/Lib/test/test_ipaddress.py +@@ -2263,6 +2263,10 @@ class IpaddrUnitTest(unittest.TestCase): + self.assertEqual(True, ipaddress.ip_address( + '172.31.255.255').is_private) + self.assertEqual(False, ipaddress.ip_address('172.32.0.0').is_private) ++ self.assertFalse(ipaddress.ip_address('192.0.0.0').is_global) ++ self.assertTrue(ipaddress.ip_address('192.0.0.9').is_global) ++ self.assertTrue(ipaddress.ip_address('192.0.0.10').is_global) ++ self.assertFalse(ipaddress.ip_address('192.0.0.255').is_global) + + self.assertEqual(True, + ipaddress.ip_address('169.254.100.200').is_link_local) +@@ -2278,6 +2282,40 @@ class IpaddrUnitTest(unittest.TestCase): + self.assertEqual(False, ipaddress.ip_address('128.0.0.0').is_loopback) + self.assertEqual(True, ipaddress.ip_network('0.0.0.0').is_unspecified) + ++ def testPrivateNetworks(self): ++ self.assertEqual(True, ipaddress.ip_network("0.0.0.0/0").is_private) ++ self.assertEqual(False, ipaddress.ip_network("1.0.0.0/8").is_private) ++ ++ self.assertEqual(True, ipaddress.ip_network("0.0.0.0/8").is_private) ++ self.assertEqual(True, ipaddress.ip_network("10.0.0.0/8").is_private) ++ self.assertEqual(True, ipaddress.ip_network("127.0.0.0/8").is_private) ++ self.assertEqual(True, ipaddress.ip_network("169.254.0.0/16").is_private) ++ self.assertEqual(True, ipaddress.ip_network("172.16.0.0/12").is_private) ++ self.assertEqual(True, ipaddress.ip_network("192.0.0.0/29").is_private) ++ self.assertEqual(False, ipaddress.ip_network("192.0.0.9/32").is_private) ++ self.assertEqual(True, ipaddress.ip_network("192.0.0.170/31").is_private) ++ self.assertEqual(True, ipaddress.ip_network("192.0.2.0/24").is_private) ++ self.assertEqual(True, ipaddress.ip_network("192.168.0.0/16").is_private) ++ self.assertEqual(True, ipaddress.ip_network("198.18.0.0/15").is_private) ++ self.assertEqual(True, ipaddress.ip_network("198.51.100.0/24").is_private) ++ self.assertEqual(True, ipaddress.ip_network("203.0.113.0/24").is_private) ++ self.assertEqual(True, ipaddress.ip_network("240.0.0.0/4").is_private) ++ self.assertEqual(True, ipaddress.ip_network("255.255.255.255/32").is_private) ++ ++ self.assertEqual(False, ipaddress.ip_network("::/0").is_private) ++ self.assertEqual(False, ipaddress.ip_network("::ff/128").is_private) ++ ++ self.assertEqual(True, ipaddress.ip_network("::1/128").is_private) ++ self.assertEqual(True, ipaddress.ip_network("::/128").is_private) ++ self.assertEqual(True, ipaddress.ip_network("::ffff:0:0/96").is_private) ++ self.assertEqual(True, ipaddress.ip_network("100::/64").is_private) ++ self.assertEqual(True, ipaddress.ip_network("2001:2::/48").is_private) ++ self.assertEqual(False, ipaddress.ip_network("2001:3::/48").is_private) ++ self.assertEqual(True, ipaddress.ip_network("2001:db8::/32").is_private) ++ self.assertEqual(True, ipaddress.ip_network("2001:10::/28").is_private) ++ self.assertEqual(True, ipaddress.ip_network("fc00::/7").is_private) ++ self.assertEqual(True, ipaddress.ip_network("fe80::/10").is_private) ++ + def testReservedIpv6(self): + + self.assertEqual(True, ipaddress.ip_network('ffff::').is_multicast) +@@ -2351,6 +2389,20 @@ class IpaddrUnitTest(unittest.TestCase): + self.assertEqual(True, ipaddress.ip_address('0::0').is_unspecified) + self.assertEqual(False, ipaddress.ip_address('::1').is_unspecified) + ++ self.assertFalse(ipaddress.ip_address('64:ff9b:1::').is_global) ++ self.assertFalse(ipaddress.ip_address('2001::').is_global) ++ self.assertTrue(ipaddress.ip_address('2001:1::1').is_global) ++ self.assertTrue(ipaddress.ip_address('2001:1::2').is_global) ++ self.assertFalse(ipaddress.ip_address('2001:2::').is_global) ++ self.assertTrue(ipaddress.ip_address('2001:3::').is_global) ++ self.assertFalse(ipaddress.ip_address('2001:4::').is_global) ++ self.assertTrue(ipaddress.ip_address('2001:4:112::').is_global) ++ self.assertFalse(ipaddress.ip_address('2001:10::').is_global) ++ self.assertTrue(ipaddress.ip_address('2001:20::').is_global) ++ self.assertTrue(ipaddress.ip_address('2001:30::').is_global) ++ self.assertFalse(ipaddress.ip_address('2001:40::').is_global) ++ self.assertFalse(ipaddress.ip_address('2002::').is_global) ++ + # some generic IETF reserved addresses + self.assertEqual(True, ipaddress.ip_address('100::').is_reserved) + self.assertEqual(True, ipaddress.ip_network('4000::1/128').is_reserved) +diff --git a/Misc/NEWS.d/next/Library/2024-03-14-01-38-44.gh-issue-113171.VFnObz.rst b/Misc/NEWS.d/next/Library/2024-03-14-01-38-44.gh-issue-113171.VFnObz.rst +new file mode 100644 +index 0000000..f9a7247 +--- /dev/null ++++ b/Misc/NEWS.d/next/Library/2024-03-14-01-38-44.gh-issue-113171.VFnObz.rst +@@ -0,0 +1,9 @@ ++Fixed various false positives and false negatives in ++ ++* :attr:`ipaddress.IPv4Address.is_private` (see these docs for details) ++* :attr:`ipaddress.IPv4Address.is_global` ++* :attr:`ipaddress.IPv6Address.is_private` ++* :attr:`ipaddress.IPv6Address.is_global` ++ ++Also in the corresponding :class:`ipaddress.IPv4Network` and :class:`ipaddress.IPv6Network` ++attributes. +-- +2.45.2 + diff --git a/SOURCES/00435-gh-121650-encode-newlines-in-headers-and-verify-headers-are-sound-gh-122233.patch b/SOURCES/00435-gh-121650-encode-newlines-in-headers-and-verify-headers-are-sound-gh-122233.patch new file mode 100644 index 0000000..432920d --- /dev/null +++ b/SOURCES/00435-gh-121650-encode-newlines-in-headers-and-verify-headers-are-sound-gh-122233.patch @@ -0,0 +1,356 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Petr Viktorin +Date: Wed, 31 Jul 2024 00:19:48 +0200 +Subject: [PATCH] 00435: gh-121650: Encode newlines in headers, and verify + headers are sound (GH-122233) + +Per RFC 2047: + +> [...] these encoding schemes allow the +> encoding of arbitrary octet values, mail readers that implement this +> decoding should also ensure that display of the decoded data on the +> recipient's terminal will not cause unwanted side-effects + +It seems that the "quoted-word" scheme is a valid way to include +a newline character in a header value, just like we already allow +undecodable bytes or control characters. +They do need to be properly quoted when serialized to text, though. + +This should fail for custom fold() implementations that aren't careful +about newlines. + +(cherry picked from commit 097633981879b3c9de9a1dd120d3aa585ecc2384) + +Co-authored-by: Petr Viktorin +Co-authored-by: Bas Bloemsaat +Co-authored-by: Serhiy Storchaka +--- + Doc/library/email.errors.rst | 6 ++ + Doc/library/email.policy.rst | 18 ++++++ + Doc/whatsnew/3.9.rst | 12 ++++ + Lib/email/_header_value_parser.py | 12 +++- + Lib/email/_policybase.py | 8 +++ + Lib/email/errors.py | 4 ++ + Lib/email/generator.py | 13 +++- + Lib/test/test_email/test_generator.py | 62 +++++++++++++++++++ + Lib/test/test_email/test_policy.py | 26 ++++++++ + ...-07-27-16-10-41.gh-issue-121650.nf6oc9.rst | 5 ++ + 10 files changed, 162 insertions(+), 4 deletions(-) + create mode 100644 Misc/NEWS.d/next/Library/2024-07-27-16-10-41.gh-issue-121650.nf6oc9.rst + +diff --git a/Doc/library/email.errors.rst b/Doc/library/email.errors.rst +index f4b9f52509..878c09bb04 100644 +--- a/Doc/library/email.errors.rst ++++ b/Doc/library/email.errors.rst +@@ -59,6 +59,12 @@ The following exception classes are defined in the :mod:`email.errors` module: + :class:`~email.mime.image.MIMEImage`). + + ++.. exception:: HeaderWriteError() ++ ++ Raised when an error occurs when the :mod:`~email.generator` outputs ++ headers. ++ ++ + Here is the list of the defects that the :class:`~email.parser.FeedParser` + can find while parsing messages. Note that the defects are added to the message + where the problem was found, so for example, if a message nested inside a +diff --git a/Doc/library/email.policy.rst b/Doc/library/email.policy.rst +index bf53b9520f..57a75ce452 100644 +--- a/Doc/library/email.policy.rst ++++ b/Doc/library/email.policy.rst +@@ -229,6 +229,24 @@ added matters. To illustrate:: + + .. versionadded:: 3.6 + ++ ++ .. attribute:: verify_generated_headers ++ ++ If ``True`` (the default), the generator will raise ++ :exc:`~email.errors.HeaderWriteError` instead of writing a header ++ that is improperly folded or delimited, such that it would ++ be parsed as multiple headers or joined with adjacent data. ++ Such headers can be generated by custom header classes or bugs ++ in the ``email`` module. ++ ++ As it's a security feature, this defaults to ``True`` even in the ++ :class:`~email.policy.Compat32` policy. ++ For backwards compatible, but unsafe, behavior, it must be set to ++ ``False`` explicitly. ++ ++ .. versionadded:: 3.9.20 ++ ++ + The following :class:`Policy` method is intended to be called by code using + the email library to create policy instances with custom settings: + +diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst +index 1756a37338..eeda4e6955 100644 +--- a/Doc/whatsnew/3.9.rst ++++ b/Doc/whatsnew/3.9.rst +@@ -1625,3 +1625,15 @@ ipaddress + + * Fixed ``is_global`` and ``is_private`` behavior in ``IPv4Address``, + ``IPv6Address``, ``IPv4Network`` and ``IPv6Network``. ++ ++email ++----- ++ ++* Headers with embedded newlines are now quoted on output. ++ ++ The :mod:`~email.generator` will now refuse to serialize (write) headers ++ that are improperly folded or delimited, such that they would be parsed as ++ multiple headers or joined with adjacent data. ++ If you need to turn this safety feature off, ++ set :attr:`~email.policy.Policy.verify_generated_headers`. ++ (Contributed by Bas Bloemsaat and Petr Viktorin in :gh:`121650`.) +diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py +index 8a8fb8bc42..e394cfd2e1 100644 +--- a/Lib/email/_header_value_parser.py ++++ b/Lib/email/_header_value_parser.py +@@ -92,6 +92,8 @@ TOKEN_ENDS = TSPECIALS | WSP + ASPECIALS = TSPECIALS | set("*'%") + ATTRIBUTE_ENDS = ASPECIALS | WSP + EXTENDED_ATTRIBUTE_ENDS = ATTRIBUTE_ENDS - set('%') ++NLSET = {'\n', '\r'} ++SPECIALSNL = SPECIALS | NLSET + + def quote_string(value): + return '"'+str(value).replace('\\', '\\\\').replace('"', r'\"')+'"' +@@ -2778,9 +2780,13 @@ def _refold_parse_tree(parse_tree, *, policy): + wrap_as_ew_blocked -= 1 + continue + tstr = str(part) +- if part.token_type == 'ptext' and set(tstr) & SPECIALS: +- # Encode if tstr contains special characters. +- want_encoding = True ++ if not want_encoding: ++ if part.token_type == 'ptext': ++ # Encode if tstr contains special characters. ++ want_encoding = not SPECIALSNL.isdisjoint(tstr) ++ else: ++ # Encode if tstr contains newlines. ++ want_encoding = not NLSET.isdisjoint(tstr) + try: + tstr.encode(encoding) + charset = encoding +diff --git a/Lib/email/_policybase.py b/Lib/email/_policybase.py +index c9cbadd2a8..d1f48211f9 100644 +--- a/Lib/email/_policybase.py ++++ b/Lib/email/_policybase.py +@@ -157,6 +157,13 @@ class Policy(_PolicyBase, metaclass=abc.ABCMeta): + message_factory -- the class to use to create new message objects. + If the value is None, the default is Message. + ++ verify_generated_headers ++ -- if true, the generator verifies that each header ++ they are properly folded, so that a parser won't ++ treat it as multiple headers, start-of-body, or ++ part of another header. ++ This is a check against custom Header & fold() ++ implementations. + """ + + raise_on_defect = False +@@ -165,6 +172,7 @@ class Policy(_PolicyBase, metaclass=abc.ABCMeta): + max_line_length = 78 + mangle_from_ = False + message_factory = None ++ verify_generated_headers = True + + def handle_defect(self, obj, defect): + """Based on policy, either raise defect or call register_defect. +diff --git a/Lib/email/errors.py b/Lib/email/errors.py +index d28a680010..1a0d5c63e6 100644 +--- a/Lib/email/errors.py ++++ b/Lib/email/errors.py +@@ -29,6 +29,10 @@ class CharsetError(MessageError): + """An illegal charset was given.""" + + ++class HeaderWriteError(MessageError): ++ """Error while writing headers.""" ++ ++ + # These are parsing defects which the parser was able to work around. + class MessageDefect(ValueError): + """Base class for a message defect.""" +diff --git a/Lib/email/generator.py b/Lib/email/generator.py +index c9b121624e..89224ae41c 100644 +--- a/Lib/email/generator.py ++++ b/Lib/email/generator.py +@@ -14,12 +14,14 @@ import random + from copy import deepcopy + from io import StringIO, BytesIO + from email.utils import _has_surrogates ++from email.errors import HeaderWriteError + + UNDERSCORE = '_' + NL = '\n' # XXX: no longer used by the code below. + + NLCRE = re.compile(r'\r\n|\r|\n') + fcre = re.compile(r'^From ', re.MULTILINE) ++NEWLINE_WITHOUT_FWSP = re.compile(r'\r\n[^ \t]|\r[^ \n\t]|\n[^ \t]') + + + +@@ -223,7 +225,16 @@ class Generator: + + def _write_headers(self, msg): + for h, v in msg.raw_items(): +- self.write(self.policy.fold(h, v)) ++ folded = self.policy.fold(h, v) ++ if self.policy.verify_generated_headers: ++ linesep = self.policy.linesep ++ if not folded.endswith(self.policy.linesep): ++ raise HeaderWriteError( ++ f'folded header does not end with {linesep!r}: {folded!r}') ++ if NEWLINE_WITHOUT_FWSP.search(folded.removesuffix(linesep)): ++ raise HeaderWriteError( ++ f'folded header contains newline: {folded!r}') ++ self.write(folded) + # A blank line always separates headers from body + self.write(self._NL) + +diff --git a/Lib/test/test_email/test_generator.py b/Lib/test/test_email/test_generator.py +index 89e7edeb63..d29400f0ed 100644 +--- a/Lib/test/test_email/test_generator.py ++++ b/Lib/test/test_email/test_generator.py +@@ -6,6 +6,7 @@ from email.message import EmailMessage + from email.generator import Generator, BytesGenerator + from email.headerregistry import Address + from email import policy ++import email.errors + from test.test_email import TestEmailBase, parameterize + + +@@ -216,6 +217,44 @@ class TestGeneratorBase: + g.flatten(msg) + self.assertEqual(s.getvalue(), self.typ(expected)) + ++ def test_keep_encoded_newlines(self): ++ msg = self.msgmaker(self.typ(textwrap.dedent("""\ ++ To: nobody ++ Subject: Bad subject=?UTF-8?Q?=0A?=Bcc: injection@example.com ++ ++ None ++ """))) ++ expected = textwrap.dedent("""\ ++ To: nobody ++ Subject: Bad subject=?UTF-8?Q?=0A?=Bcc: injection@example.com ++ ++ None ++ """) ++ s = self.ioclass() ++ g = self.genclass(s, policy=self.policy.clone(max_line_length=80)) ++ g.flatten(msg) ++ self.assertEqual(s.getvalue(), self.typ(expected)) ++ ++ def test_keep_long_encoded_newlines(self): ++ msg = self.msgmaker(self.typ(textwrap.dedent("""\ ++ To: nobody ++ Subject: Bad subject=?UTF-8?Q?=0A?=Bcc: injection@example.com ++ ++ None ++ """))) ++ expected = textwrap.dedent("""\ ++ To: nobody ++ Subject: Bad subject ++ =?utf-8?q?=0A?=Bcc: ++ injection@example.com ++ ++ None ++ """) ++ s = self.ioclass() ++ g = self.genclass(s, policy=self.policy.clone(max_line_length=30)) ++ g.flatten(msg) ++ self.assertEqual(s.getvalue(), self.typ(expected)) ++ + + class TestGenerator(TestGeneratorBase, TestEmailBase): + +@@ -224,6 +263,29 @@ class TestGenerator(TestGeneratorBase, TestEmailBase): + ioclass = io.StringIO + typ = str + ++ def test_verify_generated_headers(self): ++ """gh-121650: by default the generator prevents header injection""" ++ class LiteralHeader(str): ++ name = 'Header' ++ def fold(self, **kwargs): ++ return self ++ ++ for text in ( ++ 'Value\r\nBad Injection\r\n', ++ 'NoNewLine' ++ ): ++ with self.subTest(text=text): ++ message = message_from_string( ++ "Header: Value\r\n\r\nBody", ++ policy=self.policy, ++ ) ++ ++ del message['Header'] ++ message['Header'] = LiteralHeader(text) ++ ++ with self.assertRaises(email.errors.HeaderWriteError): ++ message.as_string() ++ + + class TestBytesGenerator(TestGeneratorBase, TestEmailBase): + +diff --git a/Lib/test/test_email/test_policy.py b/Lib/test/test_email/test_policy.py +index e87c275549..ff1ddf7d7a 100644 +--- a/Lib/test/test_email/test_policy.py ++++ b/Lib/test/test_email/test_policy.py +@@ -26,6 +26,7 @@ class PolicyAPITests(unittest.TestCase): + 'raise_on_defect': False, + 'mangle_from_': True, + 'message_factory': None, ++ 'verify_generated_headers': True, + } + # These default values are the ones set on email.policy.default. + # If any of these defaults change, the docs must be updated. +@@ -277,6 +278,31 @@ class PolicyAPITests(unittest.TestCase): + with self.assertRaises(email.errors.HeaderParseError): + policy.fold("Subject", subject) + ++ def test_verify_generated_headers(self): ++ """Turning protection off allows header injection""" ++ policy = email.policy.default.clone(verify_generated_headers=False) ++ for text in ( ++ 'Header: Value\r\nBad: Injection\r\n', ++ 'Header: NoNewLine' ++ ): ++ with self.subTest(text=text): ++ message = email.message_from_string( ++ "Header: Value\r\n\r\nBody", ++ policy=policy, ++ ) ++ class LiteralHeader(str): ++ name = 'Header' ++ def fold(self, **kwargs): ++ return self ++ ++ del message['Header'] ++ message['Header'] = LiteralHeader(text) ++ ++ self.assertEqual( ++ message.as_string(), ++ f"{text}\nBody", ++ ) ++ + # XXX: Need subclassing tests. + # For adding subclassed objects, make sure the usual rules apply (subclass + # wins), but that the order still works (right overrides left). +diff --git a/Misc/NEWS.d/next/Library/2024-07-27-16-10-41.gh-issue-121650.nf6oc9.rst b/Misc/NEWS.d/next/Library/2024-07-27-16-10-41.gh-issue-121650.nf6oc9.rst +new file mode 100644 +index 0000000000..83dd28d4ac +--- /dev/null ++++ b/Misc/NEWS.d/next/Library/2024-07-27-16-10-41.gh-issue-121650.nf6oc9.rst +@@ -0,0 +1,5 @@ ++:mod:`email` headers with embedded newlines are now quoted on output. The ++:mod:`~email.generator` will now refuse to serialize (write) headers that ++are unsafely folded or delimited; see ++:attr:`~email.policy.Policy.verify_generated_headers`. (Contributed by Bas ++Bloemsaat and Petr Viktorin in :gh:`121650`.) diff --git a/SOURCES/00436-cve-2024-8088-gh-122905-sanitize-names-in-zipfile-path.patch b/SOURCES/00436-cve-2024-8088-gh-122905-sanitize-names-in-zipfile-path.patch new file mode 100644 index 0000000..fed0497 --- /dev/null +++ b/SOURCES/00436-cve-2024-8088-gh-122905-sanitize-names-in-zipfile-path.patch @@ -0,0 +1,128 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: "Jason R. Coombs" +Date: Mon, 19 Aug 2024 19:28:20 -0400 +Subject: [PATCH] 00436: [CVE-2024-8088] gh-122905: Sanitize names in + zipfile.Path. + +Co-authored-by: Jason R. Coombs +--- + Lib/test/test_zipfile.py | 17 ++++++ + Lib/zipfile.py | 61 ++++++++++++++++++- + ...-08-11-14-08-04.gh-issue-122905.7tDsxA.rst | 1 + + 3 files changed, 78 insertions(+), 1 deletion(-) + create mode 100644 Misc/NEWS.d/next/Library/2024-08-11-14-08-04.gh-issue-122905.7tDsxA.rst + +diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py +index 17e95eb862..9a72152357 100644 +--- a/Lib/test/test_zipfile.py ++++ b/Lib/test/test_zipfile.py +@@ -3054,6 +3054,23 @@ class TestPath(unittest.TestCase): + data = ['/'.join(string.ascii_lowercase + str(n)) for n in range(10000)] + zipfile.CompleteDirs._implied_dirs(data) + ++ def test_malformed_paths(self): ++ """ ++ Path should handle malformed paths. ++ """ ++ data = io.BytesIO() ++ zf = zipfile.ZipFile(data, "w") ++ zf.writestr("/one-slash.txt", b"content") ++ zf.writestr("//two-slash.txt", b"content") ++ zf.writestr("../parent.txt", b"content") ++ zf.filename = '' ++ root = zipfile.Path(zf) ++ assert list(map(str, root.iterdir())) == [ ++ 'one-slash.txt', ++ 'two-slash.txt', ++ 'parent.txt', ++ ] ++ + + if __name__ == "__main__": + unittest.main() +diff --git a/Lib/zipfile.py b/Lib/zipfile.py +index 95f95ee112..2e9b2868cd 100644 +--- a/Lib/zipfile.py ++++ b/Lib/zipfile.py +@@ -9,6 +9,7 @@ import io + import itertools + import os + import posixpath ++import re + import shutil + import stat + import struct +@@ -2177,7 +2178,65 @@ def _difference(minuend, subtrahend): + return itertools.filterfalse(set(subtrahend).__contains__, minuend) + + +-class CompleteDirs(ZipFile): ++class SanitizedNames: ++ """ ++ ZipFile mix-in to ensure names are sanitized. ++ """ ++ ++ def namelist(self): ++ return list(map(self._sanitize, super().namelist())) ++ ++ @staticmethod ++ def _sanitize(name): ++ r""" ++ Ensure a relative path with posix separators and no dot names. ++ Modeled after ++ https://github.com/python/cpython/blob/bcc1be39cb1d04ad9fc0bd1b9193d3972835a57c/Lib/zipfile/__init__.py#L1799-L1813 ++ but provides consistent cross-platform behavior. ++ >>> san = SanitizedNames._sanitize ++ >>> san('/foo/bar') ++ 'foo/bar' ++ >>> san('//foo.txt') ++ 'foo.txt' ++ >>> san('foo/.././bar.txt') ++ 'foo/bar.txt' ++ >>> san('foo../.bar.txt') ++ 'foo../.bar.txt' ++ >>> san('\\foo\\bar.txt') ++ 'foo/bar.txt' ++ >>> san('D:\\foo.txt') ++ 'D/foo.txt' ++ >>> san('\\\\server\\share\\file.txt') ++ 'server/share/file.txt' ++ >>> san('\\\\?\\GLOBALROOT\\Volume3') ++ '?/GLOBALROOT/Volume3' ++ >>> san('\\\\.\\PhysicalDrive1\\root') ++ 'PhysicalDrive1/root' ++ Retain any trailing slash. ++ >>> san('abc/') ++ 'abc/' ++ Raises a ValueError if the result is empty. ++ >>> san('../..') ++ Traceback (most recent call last): ++ ... ++ ValueError: Empty filename ++ """ ++ ++ def allowed(part): ++ return part and part not in {'..', '.'} ++ ++ # Remove the drive letter. ++ # Don't use ntpath.splitdrive, because that also strips UNC paths ++ bare = re.sub('^([A-Z]):', r'\1', name, flags=re.IGNORECASE) ++ clean = bare.replace('\\', '/') ++ parts = clean.split('/') ++ joined = '/'.join(filter(allowed, parts)) ++ if not joined: ++ raise ValueError("Empty filename") ++ return joined + '/' * name.endswith('/') ++ ++ ++class CompleteDirs(SanitizedNames, ZipFile): + """ + A ZipFile subclass that ensures that implied directories + are always included in the namelist. +diff --git a/Misc/NEWS.d/next/Library/2024-08-11-14-08-04.gh-issue-122905.7tDsxA.rst b/Misc/NEWS.d/next/Library/2024-08-11-14-08-04.gh-issue-122905.7tDsxA.rst +new file mode 100644 +index 0000000000..1be44c906c +--- /dev/null ++++ b/Misc/NEWS.d/next/Library/2024-08-11-14-08-04.gh-issue-122905.7tDsxA.rst +@@ -0,0 +1 @@ ++:class:`zipfile.Path` objects now sanitize names from the zipfile. diff --git a/SPECS/python39.spec b/SPECS/python39.spec index 294de70..a89bbf4 100644 --- a/SPECS/python39.spec +++ b/SPECS/python39.spec @@ -17,7 +17,7 @@ URL: https://www.python.org/ #global prerel ... %global upstream_version %{general_version}%{?prerel} Version: %{general_version}%{?prerel:~%{prerel}} -Release: 1%{?dist} +Release: 7%{?dist} License: Python # Exclude i686 arch. Due to a modularity issue it's being added to the @@ -182,6 +182,13 @@ ExcludeArch: i686 %global py_INSTSONAME_optimized libpython%{LDVERSION_optimized}.so.%{py_SOVERSION} %global py_INSTSONAME_debug libpython%{LDVERSION_debug}.so.%{py_SOVERSION} +# The -O flag for the compiler, optimized builds +# https://fedoraproject.org/wiki/Changes/Python_built_with_gcc_O3 +%global optflags_optimized -O3 +# The -O flag for the compiler, debug builds +# -Wno-cpp avoids some warnings with -O0 +%global optflags_debug -O0 -Wno-cpp + # Disable automatic bytecompilation. The python3 binary is not yet be # available in /usr/bin when Python is built. Also, the bytecompilation fails # on files that test invalid syntax. @@ -452,6 +459,35 @@ Patch415: 00415-cve-2023-27043-gh-102988-reject-malformed-addresses-in-email-par # CVE-2023-52425. Future versions of Expat may be more reactive. Patch422: 00422-fix-tests-for-xmlpullparser-with-expat-2-6-0.patch +# 00431 # +# Security fix for CVE-2024-4032: incorrect IPv4 and IPv6 private ranges +# Resolved upstream: https://github.com/python/cpython/issues/113171 +# Tracking bugzilla: https://bugzilla.redhat.com/show_bug.cgi?id=2292921 +Patch431: 00431-CVE-2024-4032.patch + +# 00435 # f2924d30f4dd44804219c10410a57dd96764d297 +# gh-121650: Encode newlines in headers, and verify headers are sound (GH-122233) +# +# Per RFC 2047: +# +# > [...] these encoding schemes allow the +# > encoding of arbitrary octet values, mail readers that implement this +# > decoding should also ensure that display of the decoded data on the +# > recipient's terminal will not cause unwanted side-effects +# +# It seems that the "quoted-word" scheme is a valid way to include +# a newline character in a header value, just like we already allow +# undecodable bytes or control characters. +# They do need to be properly quoted when serialized to text, though. +# +# This should fail for custom fold() implementations that aren't careful +# about newlines. +Patch435: 00435-gh-121650-encode-newlines-in-headers-and-verify-headers-are-sound-gh-122233.patch + +# 00436 # 506dd77b7132f69ada7185b8bb91eba0e1296aa8 +# [CVE-2024-8088] gh-122905: Sanitize names in zipfile.Path. +Patch436: 00436-cve-2024-8088-gh-122905-sanitize-names-in-zipfile-path.patch + # (New patches go here ^^^) # # When adding new patches to "python" and "python3" in Fedora, EL, etc., @@ -867,6 +903,9 @@ rm Lib/ensurepip/_bundled/*.whl %apply_patch -q %{PATCH414} %apply_patch -q %{PATCH415} %apply_patch -q %{PATCH422} +%apply_patch -q %{PATCH431} +%apply_patch -q %{PATCH435} +%apply_patch -q %{PATCH436} # Remove all exe files to ensure we are not shipping prebuilt binaries # note that those are only used to create Microsoft Windows installers @@ -943,6 +982,7 @@ BuildPython() { ConfName=$1 ExtraConfigArgs=$2 MoreCFlags=$3 + MoreCFlagsNodist=$4 # Each build is done in its own directory ConfDir=build/$ConfName @@ -977,7 +1017,7 @@ BuildPython() { $ExtraConfigArgs \ %{nil} -%global flags_override EXTRA_CFLAGS="$MoreCFlags" CFLAGS_NODIST="$CFLAGS_NODIST $MoreCFlags" +%global flags_override EXTRA_CFLAGS="$MoreCFlags" CFLAGS_NODIST="$CFLAGS_NODIST $MoreCFlags $MoreCFlagsNodist" %if %{without bootstrap} # Regenerate generated files (needs python3) @@ -1000,12 +1040,14 @@ BuildPython() { # See also: https://bugzilla.redhat.com/show_bug.cgi?id=1818857 BuildPython debug \ "--without-ensurepip --with-pydebug" \ - "-O0 -Wno-cpp" + "%{optflags_debug}" \ + "" %endif # with debug_build BuildPython optimized \ "--without-ensurepip %{optimizations_flag}" \ - "" + "" \ + "%{optflags_optimized}" # ====================================================== # Installing the built code: @@ -1104,7 +1146,7 @@ EOF %if %{with debug_build} InstallPython debug \ %{py_INSTSONAME_debug} \ - -O0 \ + "%{optflags_debug}" \ %{LDVERSION_debug} %endif # with debug_build @@ -2038,6 +2080,29 @@ fi # ====================================================== %changelog +* Fri Aug 23 2024 Charalampos Stratakis - 3.9.19-7 +- Security fix for CVE-2024-8088 +Resolves: RHEL-55954 + +* Tue Aug 13 2024 Lumír Balhar - 3.9.19-6 +- Security fix for CVE-2024-6923 +Resolves: RHEL-53102 + +* Thu Jul 25 2024 Charalampos Stratakis - 3.9.19-5 +- Properly propagate the optimization flags to C extensions + +* Thu Jul 18 2024 Charalampos Stratakis - 3.9.19-4 +- Build Python with -O3 +- https://fedoraproject.org/wiki/Changes/Python_built_with_gcc_O3 + +* Thu Jul 18 2024 Charalampos Stratakis - 3.9.19-3 +- Security fix for CVE-2024-4032 +Resolves: RHEL-44094 + +* Tue Jun 11 2024 Charalampos Stratakis - 3.9.19-2 +- Enable importing of hash-based .pyc files under FIPS mode +Resolves: RHEL-40786 + * Mon Apr 22 2024 Charalampos Stratakis - 3.9.19-1 - Update to 3.9.19 - Security fixes for CVE-2023-6597 and CVE-2024-0450