import contextlib import itertools import json import os import re import shutil from leapp.exceptions import StopActorExecutionError from leapp.libraries.common import dnfconfig, guards, mounting, overlaygen, rhsm, utils from leapp.libraries.common.config import get_env from leapp.libraries.common.config.version import get_target_major_version, get_target_version from leapp.libraries.stdlib import api, CalledProcessError, config from leapp.models import DNFWorkaround DNF_PLUGIN_NAME = 'rhel_upgrade.py' _DEDICATED_URL = 'https://access.redhat.com/solutions/7011704' class _DnfPluginPathStr(str): _PATHS = { "8": os.path.join('/lib/python3.6/site-packages/dnf-plugins', DNF_PLUGIN_NAME), "9": os.path.join('/lib/python3.9/site-packages/dnf-plugins', DNF_PLUGIN_NAME), } def __init__(self): # noqa: W0231; pylint: disable=super-init-not-called self.data = "" def _feed(self): major = get_target_major_version() if major not in _DnfPluginPathStr._PATHS: raise KeyError('{} is not a supported target version of RHEL'.format(major)) self.data = _DnfPluginPathStr._PATHS[major] def __str__(self): self._feed() return str(self.data) def __repr__(self): self._feed() return repr(self.data) def lstrip(self, chars=None): self._feed() return self.data.lstrip(chars) # Deprecated DNF_PLUGIN_PATH = _DnfPluginPathStr() DNF_PLUGIN_DATA_NAME = 'dnf-plugin-data.txt' DNF_PLUGIN_DATA_PATH = os.path.join('/var/lib/leapp', DNF_PLUGIN_DATA_NAME) DNF_PLUGIN_DATA_LOG_PATH = os.path.join('/var/log/leapp', DNF_PLUGIN_DATA_NAME) DNF_DEBUG_DATA_PATH = '/var/log/leapp/dnf-debugdata/' def install(target_basedir): """ Installs our plugin to the DNF plugins. """ try: shutil.copy2( api.get_file_path(DNF_PLUGIN_NAME), os.path.join(target_basedir, DNF_PLUGIN_PATH.lstrip('/'))) except EnvironmentError as e: api.current_logger().debug('Failed to install DNF plugin', exc_info=True) raise StopActorExecutionError( message='Failed to install DNF plugin. Error: {}'.format(str(e)) ) def _rebuild_rpm_db(context, root=None): """ Convert rpmdb from BerkeleyDB to Sqlite """ base_cmd = ['rpmdb', '--rebuilddb'] cmd = base_cmd if not root else base_cmd + ['-r', root] context.call(cmd) def _the_nogpgcheck_option_used(): return get_env('LEAPP_NOGPGCHECK', '0') == '1' def build_plugin_data(target_repoids, debug, test, tasks, on_aws): """ Generates a dictionary with the DNF plugin data. """ # get list of repo IDs of target repositories that should be used for upgrade data = { 'pkgs_info': { 'local_rpms': [os.path.join('/installroot', pkg.lstrip('/')) for pkg in tasks.local_rpms], 'to_install': tasks.to_install, 'to_remove': tasks.to_remove, 'to_upgrade': tasks.to_upgrade, 'modules_to_enable': ['{}:{}'.format(m.name, m.stream) for m in tasks.modules_to_enable], }, 'dnf_conf': { 'allow_erasing': True, 'best': True, 'debugsolver': debug, 'disable_repos': True, 'enable_repos': target_repoids, 'gpgcheck': not _the_nogpgcheck_option_used(), 'platform_id': 'platform:el{}'.format(get_target_major_version()), 'releasever': get_target_version(), 'installroot': '/installroot', 'test_flag': test }, 'rhui': { 'aws': { 'on_aws': on_aws, 'region': None, } } } return data def create_config(context, target_repoids, debug, test, tasks, on_aws=False): """ Creates the configuration data file for our DNF plugin. """ context.makedirs(os.path.dirname(DNF_PLUGIN_DATA_PATH), exists_ok=True) with context.open(DNF_PLUGIN_DATA_PATH, 'w+') as f: config_data = build_plugin_data( target_repoids=target_repoids, debug=debug, test=test, tasks=tasks, on_aws=on_aws ) json.dump(config_data, f, sort_keys=True, indent=2) def backup_config(context): """ Backs up the configuration data used for the plugin. """ context.copy_from(DNF_PLUGIN_DATA_PATH, DNF_PLUGIN_DATA_LOG_PATH) def backup_debug_data(context): """ Performs the backup of DNF debug data """ if config.is_debug(): # The debugdata is a folder generated by dnf when using the --debugsolver dnf option. We switch on the # debug_solver dnf config parameter in our rhel-upgrade dnf plugin when LEAPP_DEBUG env var set to 1. try: context.copytree_from('/debugdata', DNF_DEBUG_DATA_PATH) except OSError as e: api.current_logger().warning('Failed to copy debugdata. Message: {}'.format(str(e)), exc_info=True) def _handle_transaction_err_msg_old(stage, xfs_info, err): # NOTE(pstodulk): This is going to be removed in future! message = 'DNF execution failed with non zero exit code.' details = {'STDOUT': err.stdout, 'STDERR': err.stderr} if 'more space needed on the' in err.stderr and stage != 'upgrade': # Disk Requirements: # At least more space needed on the filesystem. # article_section = 'Generic case' if xfs_info.present and xfs_info.without_ftype: article_section = 'XFS ftype=0 case' message = ('There is not enough space on the file system hosting /var/lib/leapp directory ' 'to extract the packages.') details = {'hint': "Please follow the instructions in the '{}' section of the article at: " "link: https://access.redhat.com/solutions/5057391".format(article_section)} raise StopActorExecutionError(message=message, details=details) def _handle_transaction_err_msg(stage, xfs_info, err, is_container=False): # ignore the fallback when the error is related to the container issue # e.g. installation of packages inside the container; so it's unrelated # to the upgrade transactions. if get_env('LEAPP_OVL_LEGACY', '0') == '1' and not is_container: _handle_transaction_err_msg_old(stage, xfs_info, err) return # not needed actually as the above function raises error, but for visibility NO_SPACE_STR = 'more space needed on the' message = 'DNF execution failed with non zero exit code.' details = {'STDOUT': err.stdout, 'STDERR': err.stderr} if NO_SPACE_STR not in err.stderr: raise StopActorExecutionError(message=message, details=details) # Disk Requirements: # At least more space needed on the filesystem. # missing_space = [line.strip() for line in err.stderr.split('\n') if NO_SPACE_STR in line] if is_container: size_str = re.match(r'At least (.*) more space needed', missing_space[0]).group(1) message = 'There is not enough space on the file system hosting /var/lib/leapp.' hint = ( 'Increase the free space on the filesystem hosting' ' /var/lib/leapp by {} at minimum. It is suggested to provide' ' reasonably more space to be able to perform all planned actions' ' (e.g. when 200MB is missing, add 1700MB or more).\n\n' 'It is also a good practice to create dedicated partition' ' for /var/lib/leapp when more space is needed, which can be' ' dropped after the system upgrade is fully completed' ' For more info, see: {}' .format(size_str, _DEDICATED_URL) ) # we do not want to confuse customers by the orig msg speaking about # missing space on '/'. Skip the Disk Requirements section. # The information is part of the hint. details = {'hint': hint} else: message = 'There is not enough space on some file systems to perform the upgrade transaction.' hint = ( 'Increase the free space on listed filesystems. Presented values' ' are required minimum calculated by RPM and it is suggested to' ' provide reasonably more free space (e.g. when 200 MB is missing' ' on /usr, add 1200MB or more).' ) details = {'hint': hint, 'Disk Requirements': '\n'.join(missing_space)} raise StopActorExecutionError(message=message, details=details) def _transaction(context, stage, target_repoids, tasks, plugin_info, xfs_info, test=False, cmd_prefix=None, on_aws=False): """ Perform the actual DNF rpm download via our DNF plugin """ # we do not want if stage not in ['dry-run', 'upgrade']: create_config( context=context, target_repoids=target_repoids, debug=config.is_debug(), test=test, tasks=tasks, on_aws=on_aws ) backup_config(context=context) # FIXME: rhsm with guards.guarded_execution(guards.connection_guard(), guards.space_guard()): cmd_prefix = cmd_prefix or [] common_params = [] if config.is_verbose(): common_params.append('-v') if rhsm.skip_rhsm(): common_params += ['--disableplugin', 'subscription-manager'] if plugin_info: for info in plugin_info: if stage in info.disable_in: common_params += ['--disableplugin', info.name] env = {} if get_target_major_version() == '9': # allow handling new RHEL 9 syscalls by systemd-nspawn env = {'SYSTEMD_SECCOMP': '0'} # We need to reset modules twice, once before we check, and the second time before we actually perform # the upgrade. Not more often as the modules will be reset already. if stage in ('check', 'upgrade') and tasks.modules_to_reset: # We shall only reset modules that are not going to be enabled # This will make sure it is so modules_to_reset = {(module.name, module.stream) for module in tasks.modules_to_reset} modules_to_enable = {(module.name, module.stream) for module in tasks.modules_to_enable} module_reset_list = [module[0] for module in modules_to_reset - modules_to_enable] # Perform module reset cmd = ['/usr/bin/dnf', 'module', 'reset', '--enabled', ] + module_reset_list cmd += ['--disablerepo', '*', '-y', '--installroot', '/installroot'] try: context.call( cmd=cmd_prefix + cmd + common_params, callback_raw=utils.logging_handler, env=env ) except (CalledProcessError, OSError): api.current_logger().debug('Failed to reset modules via dnf with an error. Ignoring.', exc_info=True) cmd = [ '/usr/bin/dnf', 'rhel-upgrade', stage, DNF_PLUGIN_DATA_PATH ] try: context.call( cmd=cmd_prefix + cmd + common_params, callback_raw=utils.logging_handler, env=env ) except OSError as e: api.current_logger().error('Could not call dnf command: Message: %s', str(e), exc_info=True) raise StopActorExecutionError( message='Failed to execute dnf. Reason: {}'.format(str(e)) ) except CalledProcessError as e: api.current_logger().error('Cannot calculate, check, test, or perform the upgrade transaction.') _handle_transaction_err_msg(stage, xfs_info, e, is_container=False) finally: if stage == 'check': backup_debug_data(context=context) @contextlib.contextmanager def _prepare_transaction(used_repos, target_userspace_info, binds=()): """ Creates the transaction environment needed for the target userspace DNF execution """ target_repoids = set() for message in used_repos: target_repoids.update([repo.repoid for repo in message.repos]) with mounting.NspawnActions(base_dir=target_userspace_info.path, binds=binds) as context: yield context, list(target_repoids), target_userspace_info def apply_workarounds(context=None): """ Apply registered workarounds in the given context environment """ context = context or mounting.NotIsolatedActions(base_dir='/') for workaround in api.consume(DNFWorkaround): try: api.show_message('Applying transaction workaround - {}'.format(workaround.display_name)) if workaround.script_args: cmd_str = '{script} {args}'.format( script=workaround.script_path, args=' '.join(workaround.script_args) ) else: cmd_str = workaround.script_path context.call(['/bin/bash', '-c', cmd_str]) except (OSError, CalledProcessError) as e: raise StopActorExecutionError( message=('Failed to execute script to apply transaction workaround {display_name}.' ' Message: {error}'.format(error=str(e), display_name=workaround.display_name)) ) def install_initramdisk_requirements(packages, target_userspace_info, used_repos): """ Performs the installation of packages into the initram disk """ with _prepare_transaction(used_repos=used_repos, target_userspace_info=target_userspace_info) as (context, target_repoids, _unused): if get_target_major_version() == '9': _rebuild_rpm_db(context) repos_opt = [['--enablerepo', repo] for repo in target_repoids] repos_opt = list(itertools.chain(*repos_opt)) cmd = [ 'dnf', 'install', '-y'] if _the_nogpgcheck_option_used(): cmd.append('--nogpgcheck') cmd += [ '--setopt=module_platform_id=platform:el{}'.format(get_target_major_version()), '--setopt=keepcache=1', '--releasever', api.current_actor().configuration.version.target, '--disablerepo', '*' ] + repos_opt + list(packages) if config.is_verbose(): cmd.append('-v') if rhsm.skip_rhsm(): cmd += ['--disableplugin', 'subscription-manager'] env = {} if get_target_major_version() == '9': # allow handling new RHEL 9 syscalls by systemd-nspawn env = {'SYSTEMD_SECCOMP': '0'} try: context.call(cmd, env=env) except CalledProcessError as e: api.current_logger().error( 'Cannot install packages in the target container required to build the upgrade initramfs.' ) _handle_transaction_err_msg('', None, e, is_container=True) def perform_transaction_install(target_userspace_info, storage_info, used_repos, tasks, plugin_info, xfs_info): """ Performs the actual installation with the DNF rhel-upgrade plugin using the target userspace """ stage = 'upgrade' # These bind mounts are performed by systemd-nspawn --bind parameters bind_mounts = [ '/:/installroot', '/dev:/installroot/dev', '/proc:/installroot/proc', '/run/udev:/installroot/run/udev', ] if get_target_major_version() == '8': bind_mounts.append('/sys:/installroot/sys') else: # the target major version is RHEL 9+ # we are bindmounting host's "/sys" to the intermediate "/hostsys" # in the upgrade initramdisk to avoid cgroups tree layout clash bind_mounts.append('/hostsys:/installroot/sys') already_mounted = {entry.split(':')[0] for entry in bind_mounts} for entry in storage_info.fstab: mp = entry.fs_file if not os.path.isdir(mp): continue if mp not in already_mounted: bind_mounts.append('{}:{}'.format(mp, os.path.join('/installroot', mp.lstrip('/')))) if os.path.ismount('/boot'): bind_mounts.append('/boot:/installroot/boot') if os.path.ismount('/boot/efi'): bind_mounts.append('/boot/efi:/installroot/boot/efi') with _prepare_transaction(used_repos=used_repos, target_userspace_info=target_userspace_info, binds=bind_mounts ) as (context, target_repoids, _unused): # the below nsenter command is important as we need to enter sysvipc namespace on the host so we can # communicate with udev cmd_prefix = ['nsenter', '--ipc=/installroot/proc/1/ns/ipc'] disable_plugins = [] if plugin_info: for info in plugin_info: if stage in info.disable_in: disable_plugins += [info.name] # we have to ensure the leapp packages will stay untouched # Note: this is the most probably duplicate action - it should be already # set like that, however seatbelt is a good thing. dnfconfig.exclude_leapp_rpms(context, disable_plugins) if get_target_major_version() == '9': _rebuild_rpm_db(context, root='/installroot') _transaction( context=context, stage='upgrade', target_repoids=target_repoids, plugin_info=plugin_info, xfs_info=xfs_info, tasks=tasks, cmd_prefix=cmd_prefix ) # we have to ensure the leapp packages will stay untouched even after the # upgrade is fully finished (it cannot be done before the upgrade # on the host as the config-manager plugin is available since rhel-8) dnfconfig.exclude_leapp_rpms(mounting.NotIsolatedActions(base_dir='/'), disable_plugins=disable_plugins) @contextlib.contextmanager def _prepare_perform(used_repos, target_userspace_info, xfs_info, storage_info, target_iso=None): reserve_space = overlaygen.get_recommended_leapp_free_space(target_userspace_info.path) with _prepare_transaction(used_repos=used_repos, target_userspace_info=target_userspace_info ) as (context, target_repoids, userspace_info): with overlaygen.create_source_overlay(mounts_dir=userspace_info.mounts, scratch_dir=userspace_info.scratch, xfs_info=xfs_info, storage_info=storage_info, mount_target=os.path.join(context.base_dir, 'installroot'), scratch_reserve=reserve_space) as overlay: with mounting.mount_upgrade_iso_to_root_dir(target_userspace_info.path, target_iso): yield context, overlay, target_repoids def perform_transaction_check(target_userspace_info, used_repos, tasks, xfs_info, storage_info, plugin_info, target_iso=None): """ Perform DNF transaction check using our plugin """ stage = 'check' with _prepare_perform(used_repos=used_repos, target_userspace_info=target_userspace_info, xfs_info=xfs_info, storage_info=storage_info, target_iso=target_iso) as (context, overlay, target_repoids): apply_workarounds(overlay.nspawn()) disable_plugins = [] if plugin_info: for info in plugin_info: if stage in info.disable_in: disable_plugins += [info.name] dnfconfig.exclude_leapp_rpms(context, disable_plugins) _transaction( context=context, stage='check', target_repoids=target_repoids, plugin_info=plugin_info, xfs_info=xfs_info, tasks=tasks ) def perform_rpm_download(target_userspace_info, used_repos, tasks, xfs_info, storage_info, plugin_info, target_iso=None, on_aws=False): """ Perform RPM download including the transaction test using dnf with our plugin """ stage = 'download' with _prepare_perform(used_repos=used_repos, target_userspace_info=target_userspace_info, xfs_info=xfs_info, storage_info=storage_info, target_iso=target_iso) as (context, overlay, target_repoids): disable_plugins = [] if plugin_info: for info in plugin_info: if stage in info.disable_in: disable_plugins += [info.name] apply_workarounds(overlay.nspawn()) dnfconfig.exclude_leapp_rpms(context, disable_plugins) _transaction( context=context, stage='download', target_repoids=target_repoids, plugin_info=plugin_info, tasks=tasks, test=True, on_aws=on_aws, xfs_info=xfs_info ) def perform_dry_run(target_userspace_info, used_repos, tasks, xfs_info, storage_info, plugin_info, target_iso=None, on_aws=False): """ Perform the dnf transaction test / dry-run using only cached data. """ with _prepare_perform(used_repos=used_repos, target_userspace_info=target_userspace_info, xfs_info=xfs_info, storage_info=storage_info, target_iso=target_iso) as (context, overlay, target_repoids): apply_workarounds(overlay.nspawn()) _transaction( context=context, stage='dry-run', target_repoids=target_repoids, plugin_info=plugin_info, tasks=tasks, test=True, on_aws=on_aws, xfs_info=xfs_info )