From fd3e105dde6f4c14b4d85a325cd70c79d7b452cc Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Fri, 22 May 2020 14:25:48 +0200 Subject: [PATCH] rpm wheels --- src/virtualenv/seed/embed/base_embed.py | 5 ++++ src/virtualenv/seed/embed/pip_invoke.py | 5 ++-- src/virtualenv/seed/embed/wheels/__init__.py | 3 +++ src/virtualenv/seed/embed/wheels/acquire.py | 24 ++++++++++++------- .../seed/via_app_data/via_app_data.py | 1 + src/virtualenv/util/path/_system_wheels.py | 22 +++++++++++++++++ 6 files changed, 49 insertions(+), 11 deletions(-) create mode 100644 src/virtualenv/util/path/_system_wheels.py diff --git a/src/virtualenv/seed/embed/base_embed.py b/src/virtualenv/seed/embed/base_embed.py index bffd494..349e5ff 100644 --- a/src/virtualenv/seed/embed/base_embed.py +++ b/src/virtualenv/seed/embed/base_embed.py @@ -6,6 +6,7 @@ from six import add_metaclass from virtualenv.util.path import Path from virtualenv.util.six import ensure_str, ensure_text +from virtualenv.util.path._system_wheels import get_system_wheels_paths from ..seeder import Seeder @@ -101,3 +102,7 @@ class BaseEmbed(Seeder): def __repr__(self): return ensure_str(self.__unicode__()) + + def insert_system_wheels_paths(self, creator): + system_wheels_paths = get_system_wheels_paths(creator.interpreter.executable) + self.extra_search_dir = list(system_wheels_paths) + self.extra_search_dir diff --git a/src/virtualenv/seed/embed/pip_invoke.py b/src/virtualenv/seed/embed/pip_invoke.py index 25be493..f49a420 100644 --- a/src/virtualenv/seed/embed/pip_invoke.py +++ b/src/virtualenv/seed/embed/pip_invoke.py @@ -23,8 +23,9 @@ class PipInvoke(BaseEmbed): def run(self, creator): if not self.enabled: return + self.insert_system_wheels_paths(creator) with self.get_pip_install_cmd(creator.exe, creator.interpreter.version_release_str) as cmd: - with pip_wheel_env_run(creator.interpreter.version_release_str, self.app_data) as env: + with pip_wheel_env_run(creator.interpreter.version_release_str, creator.interpreter.executable, self.app_data) as env: self._execute(cmd, env) @staticmethod @@ -46,8 +47,6 @@ class PipInvoke(BaseEmbed): cmd.append("{}{}".format(key, "=={}".format(ver) if ver is not None else "")) with ExitStack() as stack: folders = set() - for context in (ensure_file_on_disk(get_bundled_wheel(p, version), self.app_data) for p in pkg_versions): - folders.add(stack.enter_context(context).parent) folders.update(set(self.extra_search_dir)) for folder in folders: cmd.extend(["--find-links", str(folder)]) diff --git a/src/virtualenv/seed/embed/wheels/__init__.py b/src/virtualenv/seed/embed/wheels/__init__.py index 90fea02..99cc251 100644 --- a/src/virtualenv/seed/embed/wheels/__init__.py +++ b/src/virtualenv/seed/embed/wheels/__init__.py @@ -38,3 +38,6 @@ BUNDLE_SUPPORT = { }, } MAX = "3.9" + +# Redefined here because bundled wheels are removed in RPM build +BUNDLE_SUPPORT = None diff --git a/src/virtualenv/seed/embed/wheels/acquire.py b/src/virtualenv/seed/embed/wheels/acquire.py index 91b630d..4067f0e 100644 --- a/src/virtualenv/seed/embed/wheels/acquire.py +++ b/src/virtualenv/seed/embed/wheels/acquire.py @@ -12,6 +12,7 @@ from zipfile import ZipFile from virtualenv.info import IS_ZIPAPP from virtualenv.util.path import Path +from virtualenv.util.path._system_wheels import get_system_wheels_paths from virtualenv.util.six import ensure_str, ensure_text from virtualenv.util.subprocess import Popen, subprocess from virtualenv.util.zipapp import ensure_file_on_disk @@ -33,8 +34,9 @@ class WheelDownloadFail(ValueError): def get_wheels(for_py_version, wheel_cache_dir, extra_search_dir, packages, app_data, download): # not all wheels are compatible with all python versions, so we need to py version qualify it processed = copy(packages) - # 1. acquire from bundle - acquire_from_bundle(processed, for_py_version, wheel_cache_dir) + # Do not use bundled wheels, they are removed in rpmbuild anyway + # acquire_from_bundle(processed, for_py_version, wheel_cache_dir) + # 2. acquire from extra search dir acquire_from_dir(processed, for_py_version, wheel_cache_dir, extra_search_dir) # 3. download from the internet @@ -47,6 +49,7 @@ def get_wheels(for_py_version, wheel_cache_dir, extra_search_dir, packages, app_ def acquire_from_bundle(packages, for_py_version, to_folder): + raise NotImplementedError("Bundled wheels are not available") for pkg, version in list(packages.items()): bundle = get_bundled_wheel(pkg, for_py_version) if bundle is not None: @@ -67,6 +70,7 @@ def acquire_from_bundle(packages, for_py_version, to_folder): def get_bundled_wheel(package, version_release): + raise NotImplementedError("Bundled wheels are not available") # and BUNDLE_SUPPORT == None anyway return BUNDLE_FOLDER / (BUNDLE_SUPPORT.get(version_release, {}) or BUNDLE_SUPPORT[MAX]).get(package) @@ -156,7 +160,7 @@ def download_wheel(packages, for_py_version, to_folder, app_data): cmd.extend(to_download) # pip has no interface in python - must be a new sub-process - with pip_wheel_env_run("{}.{}".format(*sys.version_info[0:2]), app_data) as env: + with pip_wheel_env_run("{}.{}".format(*sys.version_info[0:2]), sys.executable, app_data) as env: process = Popen(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) out, err = process.communicate() if process.returncode != 0: @@ -164,7 +168,7 @@ def download_wheel(packages, for_py_version, to_folder, app_data): @contextmanager -def pip_wheel_env_run(version, app_data): +def pip_wheel_env_run(version, executable, app_data): env = os.environ.copy() env.update( { @@ -172,7 +176,11 @@ def pip_wheel_env_run(version, app_data): for k, v in {"PIP_USE_WHEEL": "1", "PIP_USER": "0", "PIP_NO_INPUT": "1"}.items() }, ) - with ensure_file_on_disk(get_bundled_wheel("pip", version), app_data) as pip_wheel_path: - # put the bundled wheel onto the path, and use it to do the bootstrap operation - env[str("PYTHONPATH")] = str(pip_wheel_path) - yield env + + paths = list(get_system_wheels_paths(executable)) + pip_wheels = [] + for path in paths: + pip_wheels.extend([str(wheel) for wheel in path.glob("pip-*")]) + env[str("PYTHONPATH")] = pip_wheels[0] # Use first pip in the list (ensurepip, if exists) + + yield env diff --git a/src/virtualenv/seed/via_app_data/via_app_data.py b/src/virtualenv/seed/via_app_data/via_app_data.py index de3757d..db7c6d9 100644 --- a/src/virtualenv/seed/via_app_data/via_app_data.py +++ b/src/virtualenv/seed/via_app_data/via_app_data.py @@ -38,6 +38,7 @@ class FromAppData(BaseEmbed): def run(self, creator): if not self.enabled: return + self.insert_system_wheels_paths(creator) base_cache = self.base_cache / creator.interpreter.version_release_str with self._get_seed_wheels(creator, base_cache) as name_to_whl: pip_version = name_to_whl["pip"].stem.split("-")[1] if "pip" in name_to_whl else None diff --git a/src/virtualenv/util/path/_system_wheels.py b/src/virtualenv/util/path/_system_wheels.py new file mode 100644 index 0000000..a968dee --- /dev/null +++ b/src/virtualenv/util/path/_system_wheels.py @@ -0,0 +1,22 @@ +from subprocess import check_output, CalledProcessError + +from virtualenv.util.path import Path + + +def get_system_wheels_paths(executable): + # ensurepip wheels + # We need subprocess here to check ensurepip with the Python we are creating + # a new virtual environment for + try: + ensurepip_path = check_output((executable, "-u", "-c", 'import ensurepip; print(ensurepip.__path__[0])'), universal_newlines=True) + ensurepip_path = Path(ensurepip_path.strip()) / "_bundled" + except CalledProcessError: + pass + else: + if ensurepip_path.is_dir(): + yield ensurepip_path + + # Standard wheels path + wheels_dir = Path("/usr/share/python-wheels") + if wheels_dir.exists(): + yield wheels_dir -- 2.26.2