From 13e0fc90d394d10abf44399cb1675b04fed18814 Mon Sep 17 00:00:00 2001 From: Adam Williamson Date: Mon, 17 Jun 2024 11:25:08 -0700 Subject: [PATCH] array tests: handle different hexdigests from zlib-ng (#1678) As explained in the issue, zlib-ng produces different hex digests from original zlib. This adjusts the tests slightly to allow for this. Signed-off-by: Adam Williamson --- zarr/tests/test_core.py | 458 ++++++++++++++++++++++++---------------- 1 file changed, 274 insertions(+), 184 deletions(-) diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index 77b9415d..861aeefc 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -619,13 +619,25 @@ class TestArray: def expected(self): # tests for array without path will not be run for v3 stores assert self.version == 2 - return [ - "063b02ff8d9d3bab6da932ad5828b506ef0a6578", - "f97b84dc9ffac807415f750100108764e837bb82", - "c7190ad2bea1e9d2e73eaa2d3ca9187be1ead261", - "14470724dca6c1837edddedc490571b6a7f270bc", - "2a1046dd99b914459b3e86be9dde05027a07d209", - ] + return ( + # zlib + [ + "063b02ff8d9d3bab6da932ad5828b506ef0a6578", + "f97b84dc9ffac807415f750100108764e837bb82", + "c7190ad2bea1e9d2e73eaa2d3ca9187be1ead261", + "14470724dca6c1837edddedc490571b6a7f270bc", + "2a1046dd99b914459b3e86be9dde05027a07d209", + ], + # zlib-ng + [ + "063b02ff8d9d3bab6da932ad5828b506ef0a6578", + "f97b84dc9ffac807415f750100108764e837bb82", + "c7190ad2bea1e9d2e73eaa2d3ca9187be1ead261", + "f3f04f0e30844739d34ef8a9eee6c949a47840b8", + "2a1046dd99b914459b3e86be9dde05027a07d209", + ], + ) + def test_hexdigest(self): found = [] @@ -664,7 +676,7 @@ class TestArray: found.append(z.hexdigest()) z.store.close() - assert self.expected() == found + assert found in self.expected() def test_resize_1d(self): @@ -1622,13 +1634,15 @@ class TestArrayWithPath(TestArray): pass def expected(self): - return [ - "f710da18d45d38d4aaf2afd7fb822fdd73d02957", - "1437428e69754b1e1a38bd7fc9e43669577620db", - "6c530b6b9d73e108cc5ee7b6be3d552cc994bdbe", - "4c0a76fb1222498e09dcd92f7f9221d6cea8b40e", - "05b0663ffe1785f38d3a459dec17e57a18f254af", - ] + return ( + [ + "f710da18d45d38d4aaf2afd7fb822fdd73d02957", + "1437428e69754b1e1a38bd7fc9e43669577620db", + "6c530b6b9d73e108cc5ee7b6be3d552cc994bdbe", + "4c0a76fb1222498e09dcd92f7f9221d6cea8b40e", + "05b0663ffe1785f38d3a459dec17e57a18f254af", + ], + ) def test_nbytes_stored(self): @@ -1656,13 +1670,15 @@ class TestArrayWithChunkStore(TestArray): return KVStore(dict()) def expected(self): - return [ - "f710da18d45d38d4aaf2afd7fb822fdd73d02957", - "1437428e69754b1e1a38bd7fc9e43669577620db", - "6c530b6b9d73e108cc5ee7b6be3d552cc994bdbe", - "4c0a76fb1222498e09dcd92f7f9221d6cea8b40e", - "05b0663ffe1785f38d3a459dec17e57a18f254af", - ] + return ( + [ + "f710da18d45d38d4aaf2afd7fb822fdd73d02957", + "1437428e69754b1e1a38bd7fc9e43669577620db", + "6c530b6b9d73e108cc5ee7b6be3d552cc994bdbe", + "4c0a76fb1222498e09dcd92f7f9221d6cea8b40e", + "05b0663ffe1785f38d3a459dec17e57a18f254af", + ], + ) def test_nbytes_stored(self): @@ -1734,13 +1750,24 @@ class TestArrayWithNestedDirectoryStore(TestArrayWithDirectoryStore): return store def expected(self): - return [ - "d174aa384e660eb51c6061fc8d20850c1159141f", - "125f00eea40032f16016b292f6767aa3928c00a7", - "1b52ead0ed889a781ebd4db077a29e35d513c1f3", - "719a88b34e362ff65df30e8f8810c1146ab72bc1", - "6e0abf30daf45de51593c227fb907759ca725551", - ] + return ( + # zlib + [ + "d174aa384e660eb51c6061fc8d20850c1159141f", + "125f00eea40032f16016b292f6767aa3928c00a7", + "1b52ead0ed889a781ebd4db077a29e35d513c1f3", + "719a88b34e362ff65df30e8f8810c1146ab72bc1", + "6e0abf30daf45de51593c227fb907759ca725551", + ], + # zlib-ng + [ + "d174aa384e660eb51c6061fc8d20850c1159141f", + "125f00eea40032f16016b292f6767aa3928c00a7", + "1b52ead0ed889a781ebd4db077a29e35d513c1f3", + "42d9c96e60ed22346c4671bc5bec32a2078ce25c", + "6e0abf30daf45de51593c227fb907759ca725551", + ], + ) class TestArrayWithN5Store(TestArrayWithDirectoryStore): @@ -2028,13 +2055,24 @@ class TestArrayWithN5Store(TestArrayWithDirectoryStore): assert np.all(a2[:] == 1) def expected(self): - return [ - "8811a77d54caaa1901d5cc4452d946ae433c8d90", - "d880b007d9779db5f2cdbe13274eb1cbac4a425a", - "d80eb66d5521744f051e816ab368d8ccfc2e3edf", - "568f9f837e4b682a3819cb122988e2eebeb6572b", - "4fdf4475d786d6694110db5619acd30c80dfc372", - ] + return ( + # zlib + [ + "8811a77d54caaa1901d5cc4452d946ae433c8d90", + "d880b007d9779db5f2cdbe13274eb1cbac4a425a", + "d80eb66d5521744f051e816ab368d8ccfc2e3edf", + "568f9f837e4b682a3819cb122988e2eebeb6572b", + "4fdf4475d786d6694110db5619acd30c80dfc372", + ], + # zlib-ng + [ + "8811a77d54caaa1901d5cc4452d946ae433c8d90", + "d880b007d9779db5f2cdbe13274eb1cbac4a425a", + "d80eb66d5521744f051e816ab368d8ccfc2e3edf", + "ea7d9e80211679291141840b111775b088e51480", + "4fdf4475d786d6694110db5619acd30c80dfc372", + ], + ) @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") @@ -2112,13 +2150,15 @@ class TestArrayWithNoCompressor(TestArray): compressor = None def expected(self): - return [ - "d3da3d485de4a5fcc6d91f9dfc6a7cba9720c561", - "443b8dee512e42946cb63ff01d28e9bee8105a5f", - "b75eb90f68aa8ee1e29f2c542e851d3945066c54", - "42b6ae0d50ec361628736ab7e68fe5fefca22136", - "a0535f31c130f5e5ac66ba0713d1c1ceaebd089b", - ] + return ( + [ + "d3da3d485de4a5fcc6d91f9dfc6a7cba9720c561", + "443b8dee512e42946cb63ff01d28e9bee8105a5f", + "b75eb90f68aa8ee1e29f2c542e851d3945066c54", + "42b6ae0d50ec361628736ab7e68fe5fefca22136", + "a0535f31c130f5e5ac66ba0713d1c1ceaebd089b", + ], + ) class TestArrayWithBZ2Compressor(TestArray): @@ -2126,13 +2166,15 @@ class TestArrayWithBZ2Compressor(TestArray): compressor = BZ2(level=1) def expected(self): - return [ - "33141032439fb1df5e24ad9891a7d845b6c668c8", - "44d719da065c88a412d609a5500ff41e07b331d6", - "37c7c46e5730bba37da5e518c9d75f0d774c5098", - "1e1bcaac63e4ef3c4a68f11672537131c627f168", - "86d7b9bf22dccbeaa22f340f38be506b55e76ff2", - ] + return ( + [ + "33141032439fb1df5e24ad9891a7d845b6c668c8", + "44d719da065c88a412d609a5500ff41e07b331d6", + "37c7c46e5730bba37da5e518c9d75f0d774c5098", + "1e1bcaac63e4ef3c4a68f11672537131c627f168", + "86d7b9bf22dccbeaa22f340f38be506b55e76ff2", + ], + ) class TestArrayWithBloscCompressor(TestArray): @@ -2140,13 +2182,15 @@ class TestArrayWithBloscCompressor(TestArray): compressor = Blosc(cname="zstd", clevel=1, shuffle=1) def expected(self): - return [ - "7ff2ae8511eac915fad311647c168ccfe943e788", - "962705c861863495e9ccb7be7735907aa15e85b5", - "74ed339cfe84d544ac023d085ea0cd6a63f56c4b", - "90e30bdab745a9641cd0eb605356f531bc8ec1c3", - "95d40c391f167db8b1290e3c39d9bf741edacdf6", - ] + return ( + [ + "7ff2ae8511eac915fad311647c168ccfe943e788", + "962705c861863495e9ccb7be7735907aa15e85b5", + "74ed339cfe84d544ac023d085ea0cd6a63f56c4b", + "90e30bdab745a9641cd0eb605356f531bc8ec1c3", + "95d40c391f167db8b1290e3c39d9bf741edacdf6", + ], + ) try: @@ -2161,13 +2205,15 @@ class TestArrayWithLZMACompressor(TestArray): compressor = LZMA(preset=1) def expected(self): - return [ - "93ecaa530a1162a9d48a3c1dcee4586ccfc59bae", - "04a9755a0cd638683531b7816c7fa4fbb6f577f2", - "9de97b5c49b38e68583ed701d7e8f4c94b6a8406", - "cde499f3dc945b4e97197ff8e3cf8188a1262c35", - "e2cf3afbf66ad0e28a2b6b68b1b07817c69aaee2", - ] + return ( + [ + "93ecaa530a1162a9d48a3c1dcee4586ccfc59bae", + "04a9755a0cd638683531b7816c7fa4fbb6f577f2", + "9de97b5c49b38e68583ed701d7e8f4c94b6a8406", + "cde499f3dc945b4e97197ff8e3cf8188a1262c35", + "e2cf3afbf66ad0e28a2b6b68b1b07817c69aaee2", + ], + ) class TestArrayWithFilters(TestArray): @@ -2181,13 +2227,24 @@ class TestArrayWithFilters(TestArray): ) def expected(self): - return [ - "b80367c5599d47110d42bd8886240c2f46620dba", - "95a7b2471225e73199c9716d21e8d3dd6e5f6f2a", - "7300f1eb130cff5891630038fd99c28ef23d3a01", - "c649ad229bc5720258b934ea958570c2f354c2eb", - "62fc9236d78af18a5ec26c12eea1d33bce52501e", - ] + return ( + # zlib + [ + "b80367c5599d47110d42bd8886240c2f46620dba", + "95a7b2471225e73199c9716d21e8d3dd6e5f6f2a", + "7300f1eb130cff5891630038fd99c28ef23d3a01", + "c649ad229bc5720258b934ea958570c2f354c2eb", + "62fc9236d78af18a5ec26c12eea1d33bce52501e", + ], + # zlib-ng + [ + "b80367c5599d47110d42bd8886240c2f46620dba", + "95a7b2471225e73199c9716d21e8d3dd6e5f6f2a", + "7300f1eb130cff5891630038fd99c28ef23d3a01", + "1e053b6ad7dc58de7b1f5dad7fb45851f6b7b3ee", + "62fc9236d78af18a5ec26c12eea1d33bce52501e", + ], + ) def test_astype_no_filters(self): shape = (100,) @@ -2312,7 +2369,8 @@ class TestArrayWithCustomMapping(TestArray): z = self.create_array(shape=1000, chunks=100) assert 245 == z.nbytes_stored z[:] = 42 - assert 515 == z.nbytes_stored + # 515 is zlib, 485 is zlib-ng + assert z.nbytes_stored in (515, 485) class TestArrayNoCache(TestArray): @@ -2407,13 +2465,15 @@ class TestArrayWithFSStore(TestArray): return store def expected(self): - return [ - "ab753fc81df0878589535ca9bad2816ba88d91bc", - "c16261446f9436b1e9f962e57ce3e8f6074abe8a", - "c2ef3b2fb2bc9dcace99cd6dad1a7b66cc1ea058", - "6e52f95ac15b164a8e96843a230fcee0e610729b", - "091fa99bc60706095c9ce30b56ce2503e0223f56", - ] + return ( + [ + "ab753fc81df0878589535ca9bad2816ba88d91bc", + "c16261446f9436b1e9f962e57ce3e8f6074abe8a", + "c2ef3b2fb2bc9dcace99cd6dad1a7b66cc1ea058", + "6e52f95ac15b164a8e96843a230fcee0e610729b", + "091fa99bc60706095c9ce30b56ce2503e0223f56", + ], + ) @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") @@ -2439,13 +2499,15 @@ class TestArrayWithFSStoreFromFilesystem(TestArray): return store def expected(self): - return [ - "ab753fc81df0878589535ca9bad2816ba88d91bc", - "c16261446f9436b1e9f962e57ce3e8f6074abe8a", - "c2ef3b2fb2bc9dcace99cd6dad1a7b66cc1ea058", - "6e52f95ac15b164a8e96843a230fcee0e610729b", - "091fa99bc60706095c9ce30b56ce2503e0223f56", - ] + return ( + [ + "ab753fc81df0878589535ca9bad2816ba88d91bc", + "c16261446f9436b1e9f962e57ce3e8f6074abe8a", + "c2ef3b2fb2bc9dcace99cd6dad1a7b66cc1ea058", + "6e52f95ac15b164a8e96843a230fcee0e610729b", + "091fa99bc60706095c9ce30b56ce2503e0223f56", + ], + ) @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") @@ -2460,13 +2522,15 @@ class TestArrayWithFSStorePartialRead(TestArray): return store def expected(self): - return [ - "dd7577d645c38767cf6f6d1ef8fd64002883a014", - "aa0de9892cf1ed3cda529efbf3233720b84489b7", - "e6191c44cf958576c29c41cef0f55b028a4dbdff", - "88adeeabb819feecccadf50152293dbb42f9107e", - "1426e084427f9920e29c9ec81b663d1005849455", - ] + return ( + [ + "dd7577d645c38767cf6f6d1ef8fd64002883a014", + "aa0de9892cf1ed3cda529efbf3233720b84489b7", + "e6191c44cf958576c29c41cef0f55b028a4dbdff", + "88adeeabb819feecccadf50152293dbb42f9107e", + "1426e084427f9920e29c9ec81b663d1005849455", + ], + ) def test_non_cont(self): z = self.create_array(shape=(500, 500, 500), chunks=(50, 50, 50), dtype="