diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index 25b26f0..91258a3 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -1,14 +1,19 @@ # -*- coding: utf-8 -*- ''' -Support for YUM +Support for YUM/DNF .. note:: - This module makes heavy use of the **repoquery** utility, from the - yum-utils_ package. This package will be installed as a dependency if salt - is installed via EPEL. However, if salt has been installed using pip, or a + This module makes use of the **repoquery** utility, from the yum-utils_ + package. This package will be installed as a dependency if salt is + installed via EPEL. However, if salt has been installed using pip, or a host is being managed using salt-ssh, then as of version 2014.7.0 yum-utils_ will be installed automatically to satisfy this dependency. + DNF is fully supported as of version 2015.5.10 and 2015.8.4 (partial + support for DNF was initially added in 2015.8.0), and DNF is used + automatically in place of YUM in Fedora 22 and newer. For these versions, + repoquery is available from the ``dnf-plugins-core`` package. + .. _yum-utils: http://yum.baseurl.org/wiki/YumUtils ''' @@ -16,16 +21,18 @@ Support for YUM # Import python libs from __future__ import absolute_import import copy +import fnmatch +import itertools import logging import os import re +import string from distutils.version import LooseVersion as _LooseVersion # pylint: disable=no-name-in-module,import-error # Import 3rd-party libs # pylint: disable=import-error,redefined-builtin -import salt.ext.six as six -from salt.ext.six import string_types -from salt.ext.six.moves import shlex_quote as _cmd_quote, range +from salt.ext import six +from salt.ext.six.moves import zip try: import yum @@ -39,36 +46,20 @@ try: HAS_RPMUTILS = True except ImportError: HAS_RPMUTILS = False -# pylint: enable=import-error +# pylint: enable=import-error,redefined-builtin # Import salt libs import salt.utils +import salt.utils.itertools import salt.utils.decorators as decorators +import salt.utils.pkg.rpm from salt.exceptions import ( CommandExecutionError, MinionError, SaltInvocationError ) log = logging.getLogger(__name__) -__QUERYFORMAT = '%{NAME}_|-%{VERSION}_|-%{RELEASE}_|-%{ARCH}_|-%{REPOID}' - -# These arches compiled from the rpmUtils.arch python module source -__ARCHES_64 = ('x86_64', 'athlon', 'amd64', 'ia32e', 'ia64', 'geode') -__ARCHES_32 = ('i386', 'i486', 'i586', 'i686') -__ARCHES_PPC = ('ppc', 'ppc64', 'ppc64iseries', 'ppc64pseries') -__ARCHES_S390 = ('s390', 's390x') -__ARCHES_SPARC = ( - 'sparc', 'sparcv8', 'sparcv9', 'sparcv9v', 'sparc64', 'sparc64v' -) -__ARCHES_ALPHA = ( - 'alpha', 'alphaev4', 'alphaev45', 'alphaev5', 'alphaev56', - 'alphapca56', 'alphaev6', 'alphaev67', 'alphaev68', 'alphaev7' -) -__ARCHES_ARM = ('armv5tel', 'armv5tejl', 'armv6l', 'armv7l') -__ARCHES_SH = ('sh3', 'sh4', 'sh4a') - -__ARCHES = __ARCHES_64 + __ARCHES_32 + __ARCHES_PPC + __ARCHES_S390 + \ - __ARCHES_ALPHA + __ARCHES_ARM + __ARCHES_SH +__HOLD_PATTERN = r'\w+(?:[.-][^-]+)*' # Define the module's virtual name __virtualname__ = 'pkg' @@ -93,32 +84,63 @@ def __virtual__(): return False -def _parse_pkginfo(line): - ''' - A small helper to parse a repoquery; returns a namedtuple +def _strip_headers(output, *args): + if not args: + args_lc = ('installed packages', + 'available packages', + 'updated packages', + 'upgraded packages') + else: + args_lc = [x.lower() for x in args] + ret = '' + for line in salt.utils.itertools.split(output, '\n'): + if line.lower() not in args_lc: + ret += line + '\n' + return ret + + +def _get_hold(line, pattern=__HOLD_PATTERN, full=True): ''' - # Importing `collections` here since this function is re-namespaced into - # another module - import collections - pkginfo = collections.namedtuple( - 'PkgInfo', - ('name', 'version', 'arch', 'repoid') - ) + Resolve a package name from a line containing the hold expression. If the + regex is not matched, None is returned. - try: - name, pkg_version, release, arch, repoid = line.split('_|-') - # Handle unpack errors (should never happen with the queryformat we are - # using, but can't hurt to be careful). - except ValueError: - return None + yum ==> 2:vim-enhanced-7.4.629-5.el6.* + dnf ==> vim-enhanced-2:7.4.827-1.fc22.* + ''' + if full: + if _yum() == 'dnf': + lock_re = r'({0}-\S+)'.format(pattern) + else: + lock_re = r'(\d+:{0}-\S+)'.format(pattern) + else: + if _yum() == 'dnf': + lock_re = r'({0}-\S+)'.format(pattern) + else: + lock_re = r'\d+:({0}-\S+)'.format(pattern) + + match = re.search(lock_re, line) + if match: + if not full: + woarch = match.group(1).rsplit('.', 1)[0] + worel = woarch.rsplit('-', 1)[0] + return worel.rsplit('-', 1)[0] + else: + return match.group(1) + return None - if not _check_32(arch): - if arch not in (__grains__['osarch'], 'noarch'): - name += '.{0}'.format(arch) - if release: - pkg_version += '-{0}'.format(release) - return pkginfo(name, pkg_version, arch, repoid) +def _yum(): + ''' + return yum or dnf depending on version + ''' + contextkey = 'yum_bin' + if contextkey not in __context__: + if 'fedora' in __grains__['os'].lower() \ + and int(__grains__['osrelease']) >= 22: + __context__[contextkey] = 'dnf' + else: + __context__[contextkey] = 'yum' + return __context__[contextkey] def _repoquery_pkginfo(repoquery_args): @@ -126,8 +148,11 @@ def _repoquery_pkginfo(repoquery_args): Wrapper to call repoquery and parse out all the tuples ''' ret = [] - for line in _repoquery(repoquery_args): - pkginfo = _parse_pkginfo(line) + for line in _repoquery(repoquery_args, ignore_stderr=True): + pkginfo = salt.utils.pkg.rpm.parse_pkginfo( + line, + osarch=__grains__['osarch'] + ) if pkginfo is not None: ret.append(pkginfo) return ret @@ -138,38 +163,141 @@ def _check_repoquery(): Check for existence of repoquery and install yum-utils if it is not present. ''' - if not salt.utils.which('repoquery'): - __salt__['cmd.run']( - ['yum', '-y', 'install', 'yum-utils'], - python_shell=False, - output_loglevel='trace' + contextkey = 'yumpkg.has_repoquery' + if contextkey in __context__: + # We don't really care about the return value, we're just using this + # context key as a marker so that we know that repoquery is available, + # and we don't have to continue to repeat the check if this function is + # called more than once per run. If repoquery is not available, we + # raise an exception. + return + + if _yum() == 'dnf': + # For some silly reason the core plugins and their manpages are in + # separate packages. The dnf-plugins-core package contains the manpages + # and depends on python-dnf-plugins-core (which contains the actual + # plugins). + def _check_plugins(): + out = __salt__['cmd.run_all']( + ['rpm', '-q', '--queryformat', '%{VERSION}\n', + 'dnf-plugins-core'], + python_shell=False, + ignore_retcode=True + ) + if out['retcode'] != 0: + return False + if salt.utils.compare_versions(ver1=out['stdout'], + oper='<', + ver2='0.1.15', + cmp_func=version_cmp): + return False + __context__[contextkey] = True + return True + + if not _check_plugins(): + __salt__['cmd.run']( + ['dnf', '-y', 'install', 'dnf-plugins-core >= 0.1.15'], + python_shell=False, + output_loglevel='trace' + ) + # Check again now that we've installed dnf-plugins-core + if not _check_plugins(): + raise CommandExecutionError('Unable to install dnf-plugins-core') + else: + if salt.utils.which('repoquery'): + __context__[contextkey] = True + else: + __salt__['cmd.run']( + ['yum', '-y', 'install', 'yum-utils'], + python_shell=False, + output_loglevel='trace' + ) + # Check again now that we've installed yum-utils + if salt.utils.which('repoquery'): + __context__[contextkey] = True + else: + raise CommandExecutionError('Unable to install yum-utils') + + +def _yum_pkginfo(output): + ''' + Parse yum/dnf output (which could contain irregular line breaks if package + names are long) retrieving the name, version, etc., and return a list of + pkginfo namedtuples. + ''' + cur = {} + keys = itertools.cycle(('name', 'version', 'repoid')) + values = salt.utils.itertools.split(_strip_headers(output)) + osarch = __grains__['osarch'] + for (key, value) in zip(keys, values): + if key == 'name': + try: + cur['name'], cur['arch'] = value.rsplit('.', 1) + except ValueError: + cur['name'] = value + cur['arch'] = osarch + cur['name'] = salt.utils.pkg.rpm.resolve_name(cur['name'], + cur['arch'], + osarch) + else: + if key == 'repoid': + # Installed packages show a '@' at the beginning + value = value.lstrip('@') + cur[key] = value + if key == 'repoid': + # We're done with this package, create the pkginfo namedtuple + pkginfo = salt.utils.pkg.rpm.pkginfo(**cur) + # Clear the dict for the next package + cur = {} + # Yield the namedtuple + if pkginfo is not None: + yield pkginfo + + +def _check_versionlock(): + ''' + Ensure that the appropriate versionlock plugin is present + ''' + if _yum() == 'dnf': + vl_plugin = 'python-dnf-plugins-extras-versionlock' + else: + vl_plugin = 'yum-versionlock' \ + if __grains__.get('osmajorrelease') == '5' \ + else 'yum-plugin-versionlock' + + if vl_plugin not in list_pkgs(): + raise SaltInvocationError( + 'Cannot proceed, {0} is not installed.'.format(vl_plugin) ) - # Check again now that we've installed yum-utils - if not salt.utils.which('repoquery'): - raise CommandExecutionError('Unable to install yum-utils') -def _repoquery(repoquery_args, query_format=__QUERYFORMAT): +def _repoquery(repoquery_args, + query_format=salt.utils.pkg.rpm.QUERYFORMAT, + ignore_stderr=False): ''' Runs a repoquery command and returns a list of namedtuples ''' _check_repoquery() - cmd = 'repoquery --plugins --queryformat {0} {1}'.format( - _cmd_quote(query_format), repoquery_args - ) + if _yum() == 'dnf': + cmd = ['dnf', 'repoquery', '--quiet', '--queryformat', query_format] + else: + cmd = ['repoquery', '--plugins', '--queryformat', query_format] + + cmd.extend(repoquery_args) call = __salt__['cmd.run_all'](cmd, output_loglevel='trace') if call['retcode'] != 0: comment = '' - if 'stderr' in call: + # When checking for packages some yum modules return data via stderr + # that don't cause non-zero return codes. A perfect example of this is + # when spacewalk is installed but not yet registered. We should ignore + # those when getting pkginfo. + if 'stderr' in call and not salt.utils.is_true(ignore_stderr): comment += call['stderr'] if 'stdout' in call: comment += call['stdout'] - raise CommandExecutionError( - '{0}'.format(comment) - ) + raise CommandExecutionError(comment) else: - out = call['stdout'] - return out.splitlines() + return call['stdout'].splitlines() def _get_repo_options(**kwargs): @@ -187,20 +315,38 @@ def _get_repo_options(**kwargs): if repo and not fromrepo: fromrepo = repo - repo_arg = '' + use_dnf_repoquery = kwargs.get('repoquery', False) and _yum() == 'dnf' + ret = [] if fromrepo: - log.info('Restricting to repo {0!r}'.format(fromrepo)) - repo_arg = ('--disablerepo={0!r} --enablerepo={1!r}' - .format('*', fromrepo)) + log.info('Restricting to repo \'%s\'', fromrepo) + if use_dnf_repoquery: + # dnf-plugins-core renamed --repoid to --repo in version 0.1.7, but + # still keeps it as a hidden option for backwards compatibility. + # This is good, because --repo does not work at all (see + # https://bugzilla.redhat.com/show_bug.cgi?id=1299261 for more + # information). Using --repoid here so this will actually work. + ret.append('--repoid={0}'.format(fromrepo)) + else: + ret.extend(['--disablerepo=*', + '--enablerepo={0}'.format(fromrepo)]) else: - repo_arg = '' if disablerepo: - log.info('Disabling repo {0!r}'.format(disablerepo)) - repo_arg += '--disablerepo={0!r}'.format(disablerepo) + if use_dnf_repoquery: + log.warning( + 'ignoring disablerepo, not supported in dnf repoquery' + ) + else: + log.info('Disabling repo \'%s\'', disablerepo) + ret.append('--disablerepo={0}'.format(disablerepo)) if enablerepo: - log.info('Enabling repo {0!r}'.format(enablerepo)) - repo_arg += '--enablerepo={0!r}'.format(enablerepo) - return repo_arg + if use_dnf_repoquery: + log.warning( + 'ignoring enablerepo, not supported in dnf repoquery' + ) + else: + log.info('Enabling repo \'%s\'', enablerepo) + ret.append('--enablerepo={0}'.format(enablerepo)) + return ret def _get_excludes_option(**kwargs): @@ -208,14 +354,18 @@ def _get_excludes_option(**kwargs): Returns a string of '--disableexcludes' option to be used in the yum command, based on the kwargs. ''' - disable_excludes_arg = '' disable_excludes = kwargs.get('disableexcludes', '') if disable_excludes: - log.info('Disabling excludes for {0!r}'.format(disable_excludes)) - disable_excludes_arg = ('--disableexcludes={0!r}'.format(disable_excludes)) - - return disable_excludes_arg + if kwargs.get('repoquery', False) and _yum() == 'dnf': + log.warning( + 'Ignoring disableexcludes, not supported in dnf repoquery' + ) + return [] + else: + log.info('Disabling excludes for \'%s\'', disable_excludes) + return ['--disableexcludes=\'{0}\''.format(disable_excludes)] + return [] def _get_branch_option(**kwargs): @@ -226,45 +376,11 @@ def _get_branch_option(**kwargs): # Get branch option from the kwargs branch = kwargs.get('branch', '') - branch_arg = '' + ret = [] if branch: - log.info('Adding branch {0!r}'.format(branch)) - branch_arg = ('--branch={0!r}'.format(branch)) - return branch_arg - - -def _check_32(arch): - ''' - Returns True if both the OS arch and the passed arch are 32-bit - ''' - return all(x in __ARCHES_32 for x in (__grains__['osarch'], arch)) - - -def _rpm_pkginfo(name): - ''' - Parses RPM metadata and returns a pkginfo namedtuple - ''' - # REPOID is not a valid tag for the rpm command. Remove it and replace it - # with 'none' - queryformat = __QUERYFORMAT.replace('%{REPOID}', 'none') - output = __salt__['cmd.run_stdout']( - 'rpm -qp --queryformat {0!r} {1}'.format(_cmd_quote(queryformat), name), - output_loglevel='trace', - ignore_retcode=True - ) - return _parse_pkginfo(output) - - -def _rpm_installed(name): - ''' - Parses RPM metadata to determine if the RPM target is already installed. - Returns the name of the installed package if found, otherwise None. - ''' - pkg = _rpm_pkginfo(name) - try: - return pkg.name if pkg.name in list_pkgs() else None - except AttributeError: - return None + log.info('Adding branch \'%s\'', branch) + ret.append('--branch=\'{0}\''.format(branch)) + return ret def _get_yum_config(): @@ -275,9 +391,9 @@ def _get_yum_config(): This is currently only used to get the reposdir settings, but could be used for other things if needed. - If the yum python library is available, use that, which will give us - all of the options, including all of the defaults not specified in the - yum config. Additionally, they will all be of the correct object type. + If the yum python library is available, use that, which will give us all of + the options, including all of the defaults not specified in the yum config. + Additionally, they will all be of the correct object type. If the yum library is not available, we try to read the yum.conf directly ourselves with a minimal set of "defaults". @@ -291,7 +407,7 @@ def _get_yum_config(): try: yb = yum.YumBase() yb.preconf.init_plugins = False - for name, value in yb.conf.iteritems(): + for name, value in six.iteritems(yb.conf): conf[name] = value except (AttributeError, yum.Errors.ConfigError) as exc: raise CommandExecutionError( @@ -324,11 +440,16 @@ def _get_yum_config(): for opt in cp.options('main'): if opt in ('reposdir', 'commands', 'excludes'): # these options are expected to be lists - conf[opt] = [x.strip() for x in cp.get('main', opt).split(',')] + conf[opt] = [x.strip() + for x in cp.get('main', opt).split(',')] else: conf[opt] = cp.get('main', opt) else: - log.warning('Could not find [main] section in {0}, using internal defaults'.format(fn)) + log.warning( + 'Could not find [main] section in %s, using internal ' + 'defaults', + fn + ) return conf @@ -350,13 +471,13 @@ def _normalize_basedir(basedir=None): Returns a list of directories. ''' - if basedir is None: - basedir = [] - # if we are passed a string (for backward compatibility), convert to a list - if isinstance(basedir, basestring): + if isinstance(basedir, six.string_types): basedir = [x.strip() for x in basedir.split(',')] + if basedir is None: + basedir = [] + # nothing specified, so use the reposdir option as the default if not basedir: basedir = _get_yum_config_value('reposdir') @@ -383,11 +504,12 @@ def normalize_name(name): ''' try: arch = name.rsplit('.', 1)[-1] - if arch not in __ARCHES + ('noarch',): + if arch not in salt.utils.pkg.rpm.ARCHES + ('noarch',): return name except ValueError: return name - if arch in (__grains__['osarch'], 'noarch') or _check_32(arch): + if arch in (__grains__['osarch'], 'noarch') \ + or salt.utils.pkg.rpm.check_32(arch, osarch=__grains__['osarch']): return name[:-(len(arch) + 1)] return name @@ -423,25 +545,20 @@ def latest_version(*names, **kwargs): # Initialize the return dict with empty strings, and populate namearch_map. # namearch_map will provide a means of distinguishing between multiple # matches for the same package name, for example a target of 'glibc' on an - # x86_64 arch would return both x86_64 and i686 versions when searched - # using repoquery: - # - # $ repoquery --all --pkgnarrow=available glibc - # glibc-0:2.12-1.132.el6.i686 - # glibc-0:2.12-1.132.el6.x86_64 + # x86_64 arch would return both x86_64 and i686 versions. # # Note that the logic in the for loop below would place the osarch into the # map for noarch packages, but those cases are accounted for when iterating - # through the repoquery results later on. If the repoquery match for that - # package is a noarch, then the package is assumed to be noarch, and the - # namearch_map is ignored. + # through the 'yum list' results later on. If the match for that package is + # a noarch, then the package is assumed to be noarch, and the namearch_map + # is ignored. ret = {} namearch_map = {} for name in names: ret[name] = '' try: arch = name.rsplit('.', 1)[-1] - if arch not in __ARCHES: + if arch not in salt.utils.pkg.rpm.ARCHES: arch = __grains__['osarch'] except ValueError: arch = __grains__['osarch'] @@ -454,19 +571,46 @@ def latest_version(*names, **kwargs): if refresh: refresh_db(**kwargs) - # Get updates for specified package(s) - updates = _repoquery_pkginfo( - '{0} {1} --pkgnarrow=available {2}' - .format(repo_arg, exclude_arg, ' '.join(names)) - ) + # Get available versions for specified package(s) + cmd = [_yum(), '--quiet'] + cmd.extend(repo_arg) + cmd.extend(exclude_arg) + cmd.extend(['list', 'available']) + cmd.extend(names) + out = __salt__['cmd.run_all'](cmd, + output_loglevel='trace', + ignore_retcode=True, + python_shell=False) + if out['retcode'] != 0: + if out['stderr']: + # Check first if this is just a matter of the packages being + # up-to-date. + cur_pkgs = list_pkgs() + if not all([x in cur_pkgs for x in names]): + log.error( + 'Problem encountered getting latest version for the ' + 'following package(s): %s. Stderr follows: \n%s', + ', '.join(names), + out['stderr'] + ) + updates = [] + else: + # Sort by version number (highest to lowest) for loop below + updates = sorted( + _yum_pkginfo(out['stdout']), + key=lambda pkginfo: _LooseVersion(pkginfo.version), + reverse=True + ) for name in names: for pkg in (x for x in updates if x.name == name): if pkg.arch == 'noarch' or pkg.arch == namearch_map[name] \ - or _check_32(pkg.arch): + or salt.utils.pkg.rpm.check_32(pkg.arch): ret[name] = pkg.version # no need to check another match, if there was one break + else: + ret[name] = '' # Return a string if only one package name passed if len(names) == 1: @@ -533,8 +677,8 @@ def version_cmp(pkg1, pkg2): return cmp_result except Exception as exc: log.warning( - 'Failed to compare version \'{0}\' to \'{1}\' using ' - 'rpmUtils: {2}'.format(pkg1, pkg2, exc) + 'Failed to compare version \'%s\' to \'%s\' using ' + 'rpmUtils: %s', pkg1, pkg2, exc ) # Fall back to distutils.version.LooseVersion (should only need to do # this for RHEL5, or if an exception is raised when attempting to compare @@ -569,10 +713,20 @@ def list_pkgs(versions_as_list=False, **kwargs): return ret ret = {} - for pkginfo in _repoquery_pkginfo('--all --pkgnarrow=installed'): - if pkginfo is None: - continue - __salt__['pkg_resource.add_pkg'](ret, pkginfo.name, pkginfo.version) + cmd = ['rpm', '-qa', '--queryformat', + salt.utils.pkg.rpm.QUERYFORMAT.replace('%{REPOID}', '(none)\n')] + output = __salt__['cmd.run'](cmd, + python_shell=False, + output_loglevel='trace') + for line in output.splitlines(): + pkginfo = salt.utils.pkg.rpm.parse_pkginfo( + line, + osarch=__grains__['osarch'] + ) + if pkginfo is not None: + __salt__['pkg_resource.add_pkg'](ret, + pkginfo.name, + pkginfo.version) __salt__['pkg_resource.sort_pkglist'](ret) __context__['pkg.list_pkgs'] = copy.deepcopy(ret) @@ -593,6 +747,14 @@ def list_repo_pkgs(*args, **kwargs): can be passed and the results will be filtered to packages matching those names. This is recommended as it speeds up the function considerably. + .. warning:: + Running this function on RHEL/CentOS 6 and earlier will be more + resource-intensive, as the version of yum that ships with older + RHEL/CentOS has no yum subcommand for listing packages from a + repository. Thus, a ``yum list installed`` and ``yum list available`` + are run, which generates a lot of output, which must then be analyzed + to determine which package information to include in the return data. + This function can be helpful in discovering the version or repo to specify in a :mod:`pkg.installed ` state. @@ -643,17 +805,70 @@ def list_repo_pkgs(*args, **kwargs): ) ret = {} - for repo in repos: - repoquery_cmd = '--all --repoid="{0}" --show-duplicates'.format(repo) + + def _check_args(args, name): + ''' + Do glob matching on args and return True if a match was found. + Otherwise, return False + ''' for arg in args: - repoquery_cmd += ' "{0}"'.format(arg) - all_pkgs = _repoquery_pkginfo(repoquery_cmd) - for pkg in all_pkgs: + if fnmatch.fnmatch(name, arg): + return True + return False + + def _no_repository_packages(): + ''' + Check yum version, the repository-packages subcommand is only in + 3.4.3 and newer. + ''' + if _yum() == 'yum': + yum_version = _LooseVersion( + __salt__['cmd.run']( + ['yum', '--version'], + python_shell=False + ).splitlines()[0].strip() + ) + return yum_version < _LooseVersion('3.4.3') + return False + + def _parse_output(output, strict=False): + for pkg in _yum_pkginfo(output): + if strict and (pkg.repoid not in repos + or not _check_args(args, pkg.name)): + continue repo_dict = ret.setdefault(pkg.repoid, {}) - version_list = repo_dict.setdefault(pkg.name, []) - version_list.append(pkg.version) + version_list = repo_dict.setdefault(pkg.name, set()) + version_list.add(pkg.version) + + if _no_repository_packages(): + cmd_prefix = ['yum', '--quiet', 'list'] + for pkg_src in ('installed', 'available'): + # Check installed packages first + out = __salt__['cmd.run_all']( + cmd_prefix + [pkg_src], + output_loglevel='trace', + ignore_retcode=True, + python_shell=False + ) + if out['retcode'] == 0: + _parse_output(out['stdout'], strict=True) + else: + for repo in repos: + cmd = [_yum(), '--quiet', 'repository-packages', repo, + 'list', '--showduplicates'] + # Can't concatenate because args is a tuple, using list.extend() + cmd.extend(args) + + out = __salt__['cmd.run_all'](cmd, + output_loglevel='trace', + ignore_retcode=True, + python_shell=False) + if out['retcode'] != 0 and 'Error:' in out['stdout']: + continue + _parse_output(out['stdout']) for reponame in ret: + # Sort versions newest to oldest for pkgname in ret[reponame]: sorted_versions = sorted( [_LooseVersion(x) for x in ret[reponame][pkgname]], @@ -669,7 +884,8 @@ def list_upgrades(refresh=True, **kwargs): The ``fromrepo``, ``enablerepo``, and ``disablerepo`` arguments are supported, as used in pkg states, and the ``disableexcludes`` option is - also supported. + also supported. However, in Fedora 22 and newer all of these but + ``fromrepo`` is ignored. .. versionadded:: 2014.7.0 Support for the ``disableexcludes`` option @@ -685,10 +901,19 @@ def list_upgrades(refresh=True, **kwargs): if salt.utils.is_true(refresh): refresh_db(**kwargs) - updates = _repoquery_pkginfo( - '{0} {1} --all --pkgnarrow=updates'.format(repo_arg, exclude_arg) - ) - return dict([(x.name, x.version) for x in updates]) + + cmd = [_yum(), '--quiet'] + cmd.extend(repo_arg) + cmd.extend(exclude_arg) + cmd.extend(['list', 'upgrades' if _yum() == 'dnf' else 'updates']) + out = __salt__['cmd.run_all'](cmd, + output_loglevel='trace', + ignore_retcode=True, + python_shell=False) + if out['retcode'] != 0 and 'Error:' in out: + return {} + + return dict([(x.name, x.version) for x in _yum_pkginfo(out['stdout'])]) def check_db(*names, **kwargs): @@ -705,7 +930,8 @@ def check_db(*names, **kwargs): The ``fromrepo``, ``enablerepo`` and ``disablerepo`` arguments are supported, as used in pkg states, and the ``disableexcludes`` option is - also supported. + also supported. However, in Fedora 22 and newer all of these but + ``fromrepo`` is ignored. .. versionadded:: 2014.7.0 Support for the ``disableexcludes`` option @@ -718,41 +944,44 @@ def check_db(*names, **kwargs): salt '*' pkg.check_db fromrepo=epel-testing salt '*' pkg.check_db disableexcludes=main ''' - normalize = kwargs.pop('normalize') if kwargs.get('normalize') else False - repo_arg = _get_repo_options(**kwargs) - exclude_arg = _get_excludes_option(**kwargs) - repoquery_base = \ - '{0} {1} --all --quiet --whatprovides'.format(repo_arg, exclude_arg) + normalize = kwargs.pop('normalize', True) + repo_arg = _get_repo_options(repoquery=True, **kwargs) + exclude_arg = _get_excludes_option(repoquery=True, **kwargs) + + if _yum() == 'dnf': + repoquery_base = repo_arg + ['--whatprovides'] + avail_cmd = repo_arg + else: + repoquery_base = repo_arg + exclude_arg + repoquery_base.extend(['--all', '--quiet', '--whatprovides']) + avail_cmd = repo_arg + ['--pkgnarrow=all', '--all'] if 'pkg._avail' in __context__: avail = __context__['pkg._avail'] else: # get list of available packages - avail = [] - lines = _repoquery( - '{0} --pkgnarrow=all --all'.format(repo_arg), - query_format='%{NAME}_|-%{ARCH}' - ) + avail = set() + lines = _repoquery(avail_cmd, query_format='%{NAME}_|-%{ARCH}') for line in lines: try: name, arch = line.split('_|-') except ValueError: continue if normalize: - avail.append(normalize_name('.'.join((name, arch)))) + avail.add(normalize_name('.'.join((name, arch)))) else: - avail.append('.'.join((name, arch))) + avail.add('.'.join((name, arch))) + avail = sorted(avail) __context__['pkg._avail'] = avail ret = {} - if names: - repoquery_cmd = repoquery_base + ' {0}'.format(" ".join(names)) - provides = sorted( - set(x.name for x in _repoquery_pkginfo(repoquery_cmd)) - ) for name in names: ret.setdefault(name, {})['found'] = name in avail if not ret[name]['found']: + repoquery_cmd = repoquery_base + [name] + provides = sorted( + set(x.name for x in _repoquery_pkginfo(repoquery_cmd)) + ) if name in provides: # Package was not in avail but was found by the repoquery_cmd ret[name]['found'] = True @@ -807,22 +1036,19 @@ def refresh_db(**kwargs): exclude_arg = _get_excludes_option(**kwargs) branch_arg = _get_branch_option(**kwargs) - clean_cmd = 'yum -q clean expire-cache {repo} {exclude} {branch}'.format( - repo=repo_arg, - exclude=exclude_arg, - branch=branch_arg - ) - update_cmd = 'yum -q check-update {repo} {exclude} {branch}'.format( - repo=repo_arg, - exclude=exclude_arg, - branch=branch_arg - ) + clean_cmd = [_yum(), '--quiet', 'clean', 'expire-cache'] + update_cmd = [_yum(), '--quiet', 'check-update'] + for args in (repo_arg, exclude_arg, branch_arg): + if args: + clean_cmd.extend(args) + update_cmd.extend(args) - __salt__['cmd.run'](clean_cmd) - return retcodes.get( - __salt__['cmd.retcode'](update_cmd, ignore_retcode=True), - False - ) + __salt__['cmd.run'](clean_cmd, python_shell=False) + result = __salt__['cmd.retcode'](update_cmd, + output_loglevel='trace', + ignore_retcode=True, + python_shell=False) + return retcodes.get(result, False) def clean_metadata(**kwargs): @@ -841,102 +1067,6 @@ def clean_metadata(**kwargs): return refresh_db(**kwargs) -def group_install(name, - skip=(), - include=(), - **kwargs): - ''' - .. versionadded:: 2014.1.0 - - Install the passed package group(s). This is basically a wrapper around - pkg.install, which performs package group resolution for the user. This - function is currently considered experimental, and should be expected to - undergo changes. - - name - Package group to install. To install more than one group, either use a - comma-separated list or pass the value as a python list. - - CLI Examples: - - .. code-block:: bash - - salt '*' pkg.group_install 'Group 1' - salt '*' pkg.group_install 'Group 1,Group 2' - salt '*' pkg.group_install '["Group 1", "Group 2"]' - - skip - The name(s), in a list, of any packages that would normally be - installed by the package group ("default" packages), which should not - be installed. Can be passed either as a comma-separated list or a - python list. - - CLI Examples: - - .. code-block:: bash - - salt '*' pkg.group_install 'My Group' skip='foo,bar' - salt '*' pkg.group_install 'My Group' skip='["foo", "bar"]' - - include - The name(s), in a list, of any packages which are included in a group, - which would not normally be installed ("optional" packages). Note that - this will not enforce group membership; if you include packages which - are not members of the specified groups, they will still be installed. - Can be passed either as a comma-separated list or a python list. - - CLI Examples: - - .. code-block:: bash - - salt '*' pkg.group_install 'My Group' include='foo,bar' - salt '*' pkg.group_install 'My Group' include='["foo", "bar"]' - - .. note:: - - Because this is essentially a wrapper around pkg.install, any argument - which can be passed to pkg.install may also be included here, and it - will be passed along wholesale. - ''' - groups = name.split(',') if isinstance(name, string_types) else name - - if not groups: - raise SaltInvocationError('no groups specified') - elif not isinstance(groups, list): - raise SaltInvocationError('\'groups\' must be a list') - - # pylint: disable=maybe-no-member - if isinstance(skip, string_types): - skip = skip.split(',') - if not isinstance(skip, (list, tuple)): - raise SaltInvocationError('\'skip\' must be a list') - - if isinstance(include, string_types): - include = include.split(',') - if not isinstance(include, (list, tuple)): - raise SaltInvocationError('\'include\' must be a list') - # pylint: enable=maybe-no-member - - targets = [] - for group in groups: - group_detail = group_info(group) - targets.extend(group_detail.get('mandatory packages', [])) - targets.extend( - [pkg for pkg in group_detail.get('default packages', []) - if pkg not in skip] - ) - if include: - targets.extend(include) - - # Don't install packages that are already installed, install() isn't smart - # enough to make this distinction. - pkgs = [x for x in targets if x not in list_pkgs()] - if not pkgs: - return {} - - return install(pkgs=pkgs, **kwargs) - - def install(name=None, refresh=False, skip_verify=False, @@ -977,7 +1107,7 @@ def install(name=None, ``yum reinstall`` will only be used if the installed version matches the requested version. - Works with sources when the package header of the source can be + Works with ``sources`` when the package header of the source can be matched to the name and version of an installed package. .. versionadded:: 2014.7.0 @@ -1037,19 +1167,17 @@ def install(name=None, salt '*' pkg.install sources='[{"foo": "salt://foo.rpm"}, {"bar": "salt://bar.rpm"}]' - normalize - Normalize the package name by removing the architecture. Default is True. - This is useful for poorly created packages which might include the - architecture as an actual part of the name such as kernel modules - which match a specific kernel version. - - .. versionadded:: 2014.7.0 + normalize : True + Normalize the package name by removing the architecture. This is useful + for poorly created packages which might include the architecture as an + actual part of the name such as kernel modules which match a specific + kernel version. - Example: + .. code-block:: bash - .. code-block:: bash + salt -G role:nsd pkg.install gpfs.gplbin-2.6.32-279.31.1.el6.x86_64 normalize=False - salt -G role:nsd pkg.install gpfs.gplbin-2.6.32-279.31.1.el6.x86_64 normalize=False + .. versionadded:: 2014.7.0 Returns a dict containing the new package names and versions:: @@ -1093,36 +1221,48 @@ def install(name=None, else: pkg_params_items = [] for pkg_source in pkg_params: - rpm_info = _rpm_pkginfo(pkg_source) - if rpm_info is not None: - pkg_params_items.append([rpm_info.name, rpm_info.version, pkg_source]) - else: - pkg_params_items.append([pkg_source, None, pkg_source]) + pkg_params_items.append([pkg_source]) for pkg_item_list in pkg_params_items: - pkgname = pkg_item_list[0] - version_num = pkg_item_list[1] - if version_num is None: - if reinstall and pkg_type == 'repository' and pkgname in old: - to_reinstall[pkgname] = pkgname - else: - targets.append(pkgname) + if pkg_type == 'repository': + pkgname, version_num = pkg_item_list else: - cver = old.get(pkgname, '') - arch = '' try: - namepart, archpart = pkgname.rsplit('.', 1) + pkgname, pkgpath, version_num = pkg_item_list except ValueError: - pass - else: - if archpart in __ARCHES: - arch = '.' + archpart - pkgname = namepart + pkgname = None + pkgpath = pkg_item_list[0] + version_num = None + if version_num is None: + if pkg_type == 'repository': + if reinstall and pkgname in old: + to_reinstall[pkgname] = pkgname + else: + targets.append(pkgname) + else: + targets.append(pkgpath) + else: + # If we are installing a package file and not one from the repo, + # and version_num is not None, then we can assume that pkgname is + # not None, since the only way version_num is not None is if RPM + # metadata parsing was successful. if pkg_type == 'repository': - pkgstr = '"{0}-{1}{2}"'.format(pkgname, version_num, arch) + arch = '' + try: + namepart, archpart = pkgname.rsplit('.', 1) + except ValueError: + pass + else: + if archpart in salt.utils.pkg.rpm.ARCHES: + arch = '.' + archpart + pkgname = namepart + + pkgstr = '{0}-{1}{2}'.format(pkgname, version_num, arch) else: - pkgstr = pkg_item_list[2] + pkgstr = pkgpath + + cver = old.get(pkgname, '') if reinstall and cver \ and salt.utils.compare_versions(ver1=version_num, oper='==', @@ -1137,47 +1277,63 @@ def install(name=None, else: downgrade.append(pkgstr) + def _add_common_args(cmd): + ''' + DRY function to add args common to all yum/dnf commands + ''' + for args in (repo_arg, exclude_arg, branch_arg): + if args: + cmd.extend(args) + if skip_verify: + cmd.append('--nogpgcheck') + if targets: - cmd = 'yum -y {repo} {exclude} {branch} {gpgcheck} install {pkg}'.format( - repo=repo_arg, - exclude=exclude_arg, - branch=branch_arg, - gpgcheck='--nogpgcheck' if skip_verify else '', - pkg=' '.join(targets), + cmd = [_yum(), '-y'] + if _yum() == 'dnf': + cmd.extend(['--best', '--allowerasing']) + _add_common_args(cmd) + cmd.append('install') + cmd.extend(targets) + __salt__['cmd.run_all']( + cmd, + output_loglevel='trace', + python_shell=False, + redirect_stderr=True ) - __salt__['cmd.run'](cmd, output_loglevel='trace') if downgrade: - cmd = 'yum -y {repo} {exclude} {branch} {gpgcheck} downgrade {pkg}'.format( - repo=repo_arg, - exclude=exclude_arg, - branch=branch_arg, - gpgcheck='--nogpgcheck' if skip_verify else '', - pkg=' '.join(downgrade), + cmd = [_yum(), '-y'] + _add_common_args(cmd) + cmd.append('downgrade') + cmd.extend(downgrade) + __salt__['cmd.run_all']( + cmd, + output_loglevel='trace', + python_shell=False, + redirect_stderr=True ) - __salt__['cmd.run'](cmd, output_loglevel='trace') if to_reinstall: - cmd = 'yum -y {repo} {exclude} {branch} {gpgcheck} reinstall {pkg}'.format( - repo=repo_arg, - exclude=exclude_arg, - branch=branch_arg, - gpgcheck='--nogpgcheck' if skip_verify else '', - pkg=' '.join(six.itervalues(to_reinstall)), + cmd = [_yum(), '-y'] + _add_common_args(cmd) + cmd.append('reinstall') + cmd.extend(six.itervalues(to_reinstall)) + __salt__['cmd.run_all']( + cmd, + output_loglevel='trace', + python_shell=False, + redirect_stderr=True ) - __salt__['cmd.run'](cmd, output_loglevel='trace') __context__.pop('pkg.list_pkgs', None) new = list_pkgs() + ret = salt.utils.compare_dicts(old, new) + for pkgname in to_reinstall: - if not pkgname not in old: + if pkgname not in ret or pkgname in old: ret.update({pkgname: {'old': old.get(pkgname, ''), 'new': new.get(pkgname, '')}}) - else: - if pkgname not in ret: - ret.update({pkgname: {'old': old.get(pkgname, ''), - 'new': new.get(pkgname, '')}}) if ret: __context__.pop('pkg._avail', None) return ret @@ -1228,13 +1384,15 @@ def upgrade(refresh=True, skip_verify=False, **kwargs): refresh_db(**kwargs) old = list_pkgs() - cmd = 'yum -q -y {repo} {exclude} {branch} {gpgcheck} upgrade'.format( - repo=repo_arg, - exclude=exclude_arg, - branch=branch_arg, - gpgcheck='--nogpgcheck' if skip_verify else '') - - __salt__['cmd.run'](cmd, output_loglevel='trace') + cmd = [_yum(), '--quiet', '-y'] + for args in (repo_arg, exclude_arg, branch_arg): + if args: + cmd.extend(args) + if skip_verify: + cmd.append('--nogpgcheck') + cmd.append('upgrade') + + __salt__['cmd.run'](cmd, output_loglevel='trace', python_shell=False) __context__.pop('pkg.list_pkgs', None) new = list_pkgs() ret = salt.utils.compare_dicts(old, new) @@ -1245,10 +1403,10 @@ def upgrade(refresh=True, skip_verify=False, **kwargs): def remove(name=None, pkgs=None, **kwargs): # pylint: disable=W0613 ''' - Remove packages with ``yum -q -y remove``. + Remove packages name - The name of the package to be deleted. + The name of the package to be removed Multiple Package Options: @@ -1279,8 +1437,7 @@ def remove(name=None, pkgs=None, **kwargs): # pylint: disable=W0613 targets = [x for x in pkg_params if x in old] if not targets: return {} - quoted_targets = [_cmd_quote(target) for target in targets] - cmd = 'yum -q -y remove {0}'.format(' '.join(quoted_targets)) + cmd = [_yum(), '-y', 'remove'] + targets __salt__['cmd.run'](cmd, output_loglevel='trace') __context__.pop('pkg.list_pkgs', None) new = list_pkgs() @@ -1296,7 +1453,7 @@ def purge(name=None, pkgs=None, **kwargs): # pylint: disable=W0613 :mod:`pkg.remove `. name - The name of the package to be deleted. + The name of the package to be purged Multiple Package Options: @@ -1325,7 +1482,15 @@ def hold(name=None, pkgs=None, sources=None, **kwargs): # pylint: disable=W0613 ''' .. versionadded:: 2014.7.0 - Hold packages with ``yum -q versionlock``. + Version-lock packages + + .. note:: + Requires the appropriate ``versionlock`` plugin package to be installed: + + - On RHEL 5: ``yum-versionlock`` + - On RHEL 6 & 7: ``yum-plugin-versionlock`` + - On Fedora: ``python-dnf-plugins-extras-versionlock`` + name The name of the package to be held. @@ -1345,13 +1510,8 @@ def hold(name=None, pkgs=None, sources=None, **kwargs): # pylint: disable=W0613 salt '*' pkg.hold salt '*' pkg.hold pkgs='["foo", "bar"]' ''' + _check_versionlock() - on_redhat_5 = __grains__.get('osmajorrelease', None) == '5' - lock_pkg = 'yum-versionlock' if on_redhat_5 else 'yum-plugin-versionlock' - if lock_pkg not in list_pkgs(): - raise SaltInvocationError( - 'Packages cannot be held, {0} is not installed.'.format(lock_pkg) - ) if not name and not pkgs and not sources: raise SaltInvocationError( 'One of name, pkgs, or sources must be specified.' @@ -1363,29 +1523,18 @@ def hold(name=None, pkgs=None, sources=None, **kwargs): # pylint: disable=W0613 targets = [] if pkgs: - for pkg in salt.utils.repack_dictlist(pkgs): - ret = check_db(pkg) - if not ret[pkg]['found']: - raise SaltInvocationError( - 'Package {0} not available in repository.'.format(name) - ) targets.extend(pkgs) elif sources: for source in sources: - targets.append(next(iter(source))) + targets.append(next(six.iterkeys(source))) else: - ret = check_db(name) - if not ret[name]['found']: - raise SaltInvocationError( - 'Package {0} not available in repository.'.format(name) - ) targets.append(name) - current_locks = get_locked_packages(full=False) + current_locks = list_holds(full=False) ret = {} for target in targets: if isinstance(target, dict): - target = next(iter(target)) + target = next(six.iterkeys(target)) ret[target] = {'name': target, 'changes': {}, @@ -1398,8 +1547,8 @@ def hold(name=None, pkgs=None, sources=None, **kwargs): # pylint: disable=W0613 ret[target]['comment'] = ('Package {0} is set to be held.' .format(target)) else: - cmd = 'yum -q versionlock {0}'.format(target) - out = __salt__['cmd.run_all'](cmd) + cmd = [_yum(), 'versionlock', target] + out = __salt__['cmd.run_all'](cmd, python_shell=False) if out['retcode'] == 0: ret[target].update(result=True) @@ -1421,10 +1570,18 @@ def unhold(name=None, pkgs=None, sources=None, **kwargs): # pylint: disable=W06 ''' .. versionadded:: 2014.7.0 - Hold packages with ``yum -q versionlock``. + Remove version locks + + .. note:: + Requires the appropriate ``versionlock`` plugin package to be installed: + + - On RHEL 5: ``yum-versionlock`` + - On RHEL 6 & 7: ``yum-plugin-versionlock`` + - On Fedora: ``python-dnf-plugins-extras-versionlock`` + name - The name of the package to be deleted. + The name of the package to be unheld Multiple Package Options: @@ -1441,13 +1598,8 @@ def unhold(name=None, pkgs=None, sources=None, **kwargs): # pylint: disable=W06 salt '*' pkg.unhold salt '*' pkg.unhold pkgs='["foo", "bar"]' ''' + _check_versionlock() - on_redhat_5 = __grains__.get('osmajorrelease', None) == '5' - lock_pkg = 'yum-versionlock' if on_redhat_5 else 'yum-plugin-versionlock' - if lock_pkg not in list_pkgs(): - raise SaltInvocationError( - 'Packages cannot be unheld, {0} is not installed.'.format(lock_pkg) - ) if not name and not pkgs and not sources: raise SaltInvocationError( 'One of name, pkgs, or sources must be specified.' @@ -1467,30 +1619,43 @@ def unhold(name=None, pkgs=None, sources=None, **kwargs): # pylint: disable=W06 else: targets.append(name) - current_locks = get_locked_packages(full=True) + # Yum's versionlock plugin doesn't support passing just the package name + # when removing a lock, so we need to get the full list and then use + # fnmatch below to find the match. + current_locks = list_holds(full=_yum() == 'yum') + ret = {} for target in targets: if isinstance(target, dict): - target = next(iter(target)) + target = next(six.iterkeys(target)) ret[target] = {'name': target, 'changes': {}, 'result': False, 'comment': ''} - search_locks = [lock for lock in current_locks - if target in lock] + if _yum() == 'dnf': + search_locks = [x for x in current_locks if x == target] + else: + # To accommodate yum versionlock's lack of support for removing + # locks using just the package name, we have to use fnmatch to do + # glob matching on the target name, and then for each matching + # expression double-check that the package name (obtained via + # _get_hold()) matches the targeted package. + search_locks = [ + x for x in current_locks + if fnmatch.fnmatch(x, '*{0}*'.format(target)) + and target == _get_hold(x, full=False) + ] + if search_locks: - if 'test' in __opts__ and __opts__['test']: + if __opts__['test']: ret[target].update(result=None) ret[target]['comment'] = ('Package {0} is set to be unheld.' .format(target)) else: - quoted_targets = [_cmd_quote(item) for item in search_locks] - cmd = 'yum -q versionlock delete {0}'.format( - ' '.join(quoted_targets) - ) - out = __salt__['cmd.run_all'](cmd) + cmd = [_yum(), 'versionlock', 'delete'] + search_locks + out = __salt__['cmd.run_all'](cmd, python_shell=False) if out['retcode'] == 0: ret[target].update(result=True) @@ -1508,45 +1673,47 @@ def unhold(name=None, pkgs=None, sources=None, **kwargs): # pylint: disable=W06 return ret -def get_locked_packages(pattern=None, full=True): - ''' - Get packages that are currently locked - ``yum -q versionlock list``. +def list_holds(pattern=__HOLD_PATTERN, full=True): + r''' + .. versionchanged:: Boron,2015.8.4,2015.5.10 + Function renamed from ``pkg.get_locked_pkgs`` to ``pkg.list_holds``. + + List information on locked packages + + .. note:: + Requires the appropriate ``versionlock`` plugin package to be installed: + + - On RHEL 5: ``yum-versionlock`` + - On RHEL 6 & 7: ``yum-plugin-versionlock`` + - On Fedora: ``python-dnf-plugins-extras-versionlock`` + + pattern : \w+(?:[.-][^-]+)* + Regular expression used to match the package name + + full : True + Show the full hold definition including version and epoch. Set to + ``False`` to return just the name of the package(s) being held. + CLI Example: .. code-block:: bash - salt '*' pkg.get_locked_packages + salt '*' pkg.list_holds + salt '*' pkg.list_holds full=False ''' - cmd = 'yum -q versionlock list' - ret = __salt__['cmd.run'](cmd).split('\n') + _check_versionlock() - if pattern: - if full: - _pat = r'(\d\:{0}\-\S+)'.format(pattern) - else: - _pat = r'\d\:({0}\-\S+)'.format(pattern) - else: - if full: - _pat = r'(\d\:\w+(?:[\.\-][^\-]+)*-\S+)' - else: - _pat = r'\d\:(\w+(?:[\.\-][^\-]+)*-\S+)' - pat = re.compile(_pat) - - current_locks = [] - for item in ret: - match = pat.search(item) - if match: - if not full: - woarch = match.group(1).rsplit('.', 1)[0] - worel = woarch.rsplit('-', 1)[0] - wover = worel.rsplit('-', 1)[0] - _match = wover - else: - _match = match.group(1) - current_locks.append(_match) - return current_locks + out = __salt__['cmd.run']([_yum(), 'versionlock', 'list'], + python_shell=False) + ret = [] + for line in out.splitlines(): + match = _get_hold(line, pattern=pattern, full=full) + if match is not None: + ret.append(match) + return ret + +get_locked_packages = list_holds def verify(*names, **kwargs): @@ -1582,42 +1749,62 @@ def group_list(): salt '*' pkg.group_list ''' - ret = {'installed': [], 'available': [], 'available languages': {}} - cmd = 'yum grouplist' - out = __salt__['cmd.run_stdout'](cmd, output_loglevel='trace').splitlines() + ret = {'installed': [], + 'available': [], + 'installed environments': [], + 'available environments': [], + 'available languages': {}} + + section_map = { + 'installed groups:': 'installed', + 'available groups:': 'available', + 'installed environment groups:': 'installed environments', + 'available environment groups:': 'available environments', + 'available language groups:': 'available languages', + } + + out = __salt__['cmd.run_stdout']( + [_yum(), 'grouplist', 'hidden'], + output_loglevel='trace', + python_shell=False + ) key = None - for idx in range(len(out)): - if out[idx] == 'Installed Groups:': - key = 'installed' - continue - elif out[idx] == 'Available Groups:': - key = 'available' - continue - elif out[idx] == 'Available Language Groups:': - key = 'available languages' - continue - elif out[idx] == 'Done': + for line in out.splitlines(): + line_lc = line.lower() + if line_lc == 'done': + break + + section_lookup = section_map.get(line_lc) + if section_lookup is not None and section_lookup != key: + key = section_lookup continue + # Ignore any administrative comments (plugin info, repo info, etc.) if key is None: continue + line = line.strip() if key != 'available languages': - ret[key].append(out[idx].strip()) + ret[key].append(line) else: - line = out[idx].strip() - try: - name, lang = re.match(r'(.+) \[(.+)\]', line).groups() - except AttributeError: - pass - else: + match = re.match(r'(.+) \[(.+)\]', line) + if match: + name, lang = match.groups() ret[key][line] = {'name': name, 'language': lang} return ret -def group_info(name): +def group_info(name, expand=False): ''' .. versionadded:: 2014.1.0 + .. versionchanged:: Boron,2015.8.4,2015.5.10 + The return data has changed. A new key ``type`` has been added to + distinguish environment groups from package groups. Also, keys for the + group name and group ID have been added. The ``mandatory packages``, + ``optional packages``, and ``default packages`` keys have been renamed + to ``mandatory``, ``optional``, and ``default`` for accuracy, as + environment groups include other groups, and not packages. Finally, + this function now properly identifies conditional packages. Lists packages belonging to a certain group @@ -1627,44 +1814,67 @@ def group_info(name): salt '*' pkg.group_info 'Perl Support' ''' - # Not using _repoquery_pkginfo() here because group queries are handled - # differently, and ignore the '--queryformat' param - ret = { - 'mandatory packages': [], - 'optional packages': [], - 'default packages': [], - 'description': '' - } - cmd_template = 'repoquery --plugins --group --grouppkgs={0} --list {1}' - - cmd = cmd_template.format('all', _cmd_quote(name)) - out = __salt__['cmd.run_stdout'](cmd, output_loglevel='trace') - all_pkgs = set(out.splitlines()) - - if not all_pkgs: - raise CommandExecutionError('Group {0!r} not found'.format(name)) + pkgtypes = ('mandatory', 'optional', 'default', 'conditional') + ret = {} + for pkgtype in pkgtypes: + ret[pkgtype] = set() - for pkgtype in ('mandatory', 'optional', 'default'): - cmd = cmd_template.format(pkgtype, _cmd_quote(name)) - packages = set( - __salt__['cmd.run_stdout']( - cmd, output_loglevel='trace' - ).splitlines() - ) - ret['{0} packages'.format(pkgtype)].extend(sorted(packages)) - all_pkgs -= packages + cmd = [_yum(), '--quiet', 'groupinfo', name] + out = __salt__['cmd.run_stdout']( + cmd, + output_loglevel='trace', + python_shell=False + ) - # 'contitional' is not a valid --grouppkgs value. Any pkgs that show up - # in '--grouppkgs=all' that aren't in mandatory, optional, or default are - # considered to be conditional packages. - ret['conditional packages'] = sorted(all_pkgs) + g_info = {} + for line in out.splitlines(): + try: + key, value = [x.strip() for x in line.split(':')] + g_info[key.lower()] = value + except ValueError: + continue - cmd = 'repoquery --plugins --group --info {0}'.format(_cmd_quote(name)) - out = __salt__['cmd.run_stdout']( - cmd, output_loglevel='trace' + if 'environment group' in g_info: + ret['type'] = 'environment group' + elif 'group' in g_info: + ret['type'] = 'package group' + + ret['group'] = g_info.get('environment group') or g_info.get('group') + ret['id'] = g_info.get('environment-id') or g_info.get('group-id') + if not ret['group'] and not ret['id']: + raise CommandExecutionError('Group \'{0}\' not found'.format(name)) + + ret['description'] = g_info.get('description', '') + + pkgtypes_capturegroup = '(' + '|'.join(pkgtypes) + ')' + for pkgtype in pkgtypes: + target_found = False + for line in out.splitlines(): + line = line.strip().lstrip(string.punctuation) + match = re.match( + pkgtypes_capturegroup + r' (?:groups|packages):\s*$', + line.lower() ) - if out: - ret['description'] = '\n'.join(out.splitlines()[1:]).strip() + if match: + if target_found: + # We've reached a new section, break from loop + break + else: + if match.group(1) == pkgtype: + # We've reached the targeted section + target_found = True + continue + if target_found: + if expand and ret['type'] == 'environment group': + expanded = group_info(line, expand=True) + # Don't shadow the pkgtype variable from the outer loop + for p_type in pkgtypes: + ret[p_type].update(set(expanded[p_type])) + else: + ret[pkgtype].add(line) + + for pkgtype in pkgtypes: + ret[pkgtype] = sorted(ret[pkgtype]) return ret @@ -1672,6 +1882,10 @@ def group_info(name): def group_diff(name): ''' .. versionadded:: 2014.1.0 + .. versionchanged:: Boron,2015.8.4,2015.5.10 + Environment groups are now supported. The key names have been renamed, + similar to the changes made in :py:func:`pkg.group_info + `. Lists packages belonging to a certain group, and which are installed @@ -1681,24 +1895,117 @@ def group_diff(name): salt '*' pkg.group_diff 'Perl Support' ''' - ret = { - 'mandatory packages': {'installed': [], 'not installed': []}, - 'optional packages': {'installed': [], 'not installed': []}, - 'default packages': {'installed': [], 'not installed': []}, - 'conditional packages': {'installed': [], 'not installed': []}, - } + pkgtypes = ('mandatory', 'optional', 'default', 'conditional') + ret = {} + for pkgtype in pkgtypes: + ret[pkgtype] = {'installed': [], 'not installed': []} + pkgs = list_pkgs() - group_pkgs = group_info(name) - for pkgtype in ('mandatory', 'optional', 'default', 'conditional'): - for member in group_pkgs.get('{0} packages'.format(pkgtype), []): - key = '{0} packages'.format(pkgtype) + group_pkgs = group_info(name, expand=True) + for pkgtype in pkgtypes: + for member in group_pkgs.get(pkgtype, []): if member in pkgs: - ret[key]['installed'].append(member) + ret[pkgtype]['installed'].append(member) else: - ret[key]['not installed'].append(member) + ret[pkgtype]['not installed'].append(member) return ret +def group_install(name, + skip=(), + include=(), + **kwargs): + ''' + .. versionadded:: 2014.1.0 + + Install the passed package group(s). This is basically a wrapper around + :py:func:`pkg.install `, which performs + package group resolution for the user. This function is currently + considered experimental, and should be expected to undergo changes. + + name + Package group to install. To install more than one group, either use a + comma-separated list or pass the value as a python list. + + CLI Examples: + + .. code-block:: bash + + salt '*' pkg.group_install 'Group 1' + salt '*' pkg.group_install 'Group 1,Group 2' + salt '*' pkg.group_install '["Group 1", "Group 2"]' + + skip + Packages that would normally be installed by the package group + ("default" packages), which should not be installed. Can be passed + either as a comma-separated list or a python list. + + CLI Examples: + + .. code-block:: bash + + salt '*' pkg.group_install 'My Group' skip='foo,bar' + salt '*' pkg.group_install 'My Group' skip='["foo", "bar"]' + + include + Packages which are included in a group, which would not normally be + installed by a ``yum groupinstall`` ("optional" packages). Note that + this will not enforce group membership; if you include packages which + are not members of the specified groups, they will still be installed. + Can be passed either as a comma-separated list or a python list. + + CLI Examples: + + .. code-block:: bash + + salt '*' pkg.group_install 'My Group' include='foo,bar' + salt '*' pkg.group_install 'My Group' include='["foo", "bar"]' + + .. note:: + + Because this is essentially a wrapper around pkg.install, any argument + which can be passed to pkg.install may also be included here, and it + will be passed along wholesale. + ''' + groups = name.split(',') if isinstance(name, six.string_types) else name + + if not groups: + raise SaltInvocationError('no groups specified') + elif not isinstance(groups, list): + raise SaltInvocationError('\'groups\' must be a list') + + # pylint: disable=maybe-no-member + if isinstance(skip, six.string_types): + skip = skip.split(',') + if not isinstance(skip, (list, tuple)): + raise SaltInvocationError('\'skip\' must be a list') + + if isinstance(include, six.string_types): + include = include.split(',') + if not isinstance(include, (list, tuple)): + raise SaltInvocationError('\'include\' must be a list') + # pylint: enable=maybe-no-member + + targets = [] + for group in groups: + group_detail = group_info(group) + targets.extend(group_detail.get('mandatory packages', [])) + targets.extend( + [pkg for pkg in group_detail.get('default packages', []) + if pkg not in skip] + ) + if include: + targets.extend(include) + + # Don't install packages that are already installed, install() isn't smart + # enough to make this distinction. + pkgs = [x for x in targets if x not in list_pkgs()] + if not pkgs: + return {} + + return install(pkgs=pkgs, **kwargs) + + def list_repos(basedir=None): ''' Lists all repos in (default: all dirs in `reposdir` yum option). @@ -1714,7 +2021,7 @@ def list_repos(basedir=None): basedirs = _normalize_basedir(basedir) repos = {} - log.debug('Searching for repos in {0}'.format(basedirs)) + log.debug('Searching for repos in %s', basedirs) for bdir in basedirs: if not os.path.exists(bdir): continue @@ -1732,7 +2039,8 @@ def list_repos(basedir=None): def get_repo(repo, basedir=None, **kwargs): # pylint: disable=W0613 ''' - Display a repo from (default basedir: all dirs in `reposdir` yum option). + Display a repo from (default basedir: all dirs in ``reposdir`` + yum option). CLI Examples: @@ -1746,7 +2054,7 @@ def get_repo(repo, basedir=None, **kwargs): # pylint: disable=W0613 # Find out what file the repo lives in repofile = '' - for arepo in repos.keys(): + for arepo in six.iterkeys(repos): if arepo == repo: repofile = repos[arepo]['file'] @@ -1788,7 +2096,7 @@ def del_repo(repo, basedir=None, **kwargs): # pylint: disable=W0613 # See if the repo is the only one in the file onlyrepo = True - for arepo in repos.keys(): + for arepo in six.iterkeys(repos): if arepo == repo: continue if repos[arepo]['file'] == repofile: @@ -1803,7 +2111,7 @@ def del_repo(repo, basedir=None, **kwargs): # pylint: disable=W0613 # There must be other repos in this file, write the file with them header, filerepos = _parse_repo_file(repofile) content = header - for stanza in filerepos.keys(): + for stanza in six.iterkeys(filerepos): if stanza == repo: continue comments = '' @@ -1928,32 +2236,32 @@ def mod_repo(repo, basedir=None, **kwargs): # Error out if they tried to delete baseurl or mirrorlist improperly if 'baseurl' in todelete: if 'mirrorlist' not in repo_opts and 'mirrorlist' \ - not in filerepos[repo].keys(): + not in filerepos[repo]: raise SaltInvocationError( 'Cannot delete baseurl without specifying mirrorlist' ) if 'mirrorlist' in todelete: if 'baseurl' not in repo_opts and 'baseurl' \ - not in filerepos[repo].keys(): + not in filerepos[repo]: raise SaltInvocationError( 'Cannot delete mirrorlist without specifying baseurl' ) # Delete anything in the todelete list for key in todelete: - if key in filerepos[repo].keys(): + if key in six.iterkeys(filerepos[repo].copy()): del filerepos[repo][key] # Old file or new, write out the repos(s) filerepos[repo].update(repo_opts) content = header - for stanza in filerepos.keys(): + for stanza in six.iterkeys(filerepos): comments = '' - if 'comments' in filerepos[stanza].keys(): + if 'comments' in six.iterkeys(filerepos[stanza]): comments = '\n'.join(filerepos[stanza]['comments']) del filerepos[stanza]['comments'] content += '\n[{0}]'.format(stanza) - for line in filerepos[stanza].keys(): + for line in six.iterkeys(filerepos[stanza]): content += '\n{0}={1}'.format(line, filerepos[stanza][line]) content += '\n{0}\n'.format(comments) @@ -1997,8 +2305,8 @@ def _parse_repo_file(filename): repos[repo][comps[0].strip()] = '='.join(comps[1:]) except KeyError: log.error( - 'Failed to parse line in {0}, offending line was ' - '\'{1}\''.format(filename, line.rstrip()) + 'Failed to parse line in %s, offending line was ' + '\'%s\'', filename, line.rstrip() ) return (header, repos) @@ -2077,18 +2385,15 @@ def owner(*paths): return '' ret = {} for path in paths: - cmd = 'rpm -qf --queryformat {0} {1!r}'.format( - _cmd_quote('%{{NAME}}'), - path - ) ret[path] = __salt__['cmd.run_stdout']( - cmd.format(path), - output_loglevel='trace' - ) + ['rpm', '-qf', '--queryformat', '%{NAME}', path], + output_loglevel='trace', + python_shell=False + ) if 'not owned' in ret[path].lower(): ret[path] = '' if len(ret) == 1: - return next(ret.itervalues()) + return next(six.itervalues(ret)) return ret @@ -2123,10 +2428,12 @@ def modified(*packages, **flags): Include only files where group has been changed. time - Include only files where modification time of the file has been changed. + Include only files where modification time of the file has been + changed. capabilities - Include only files where capabilities differ or not. Note: supported only on newer RPM versions. + Include only files where capabilities differ or not. Note: supported + only on newer RPM versions. CLI Examples: @@ -2174,11 +2481,11 @@ def download(*packages): for x in cached_pkgs if x.startswith('{0}-'.format(pkg))]) for purge_target in set(to_purge): - log.debug('Removing cached package {0}'.format(purge_target)) + log.debug('Removing cached package %s', purge_target) try: os.unlink(purge_target) except OSError as exc: - log.error('Unable to remove {0}: {1}'.format(purge_target, exc)) + log.error('Unable to remove %s: %s', purge_target, exc) __salt__['cmd.run']( 'yumdownloader -q {0} --destdir={1}'.format( @@ -2244,6 +2551,7 @@ def diff(*paths): local_pkgs = __salt__['pkg.download'](*pkg_to_paths.keys()) for pkg, files in pkg_to_paths.items(): for path in files: - ret[path] = __salt__['lowpkg.diff'](local_pkgs[pkg]['path'], path) or 'Unchanged' + ret[path] = __salt__['lowpkg.diff']( + local_pkgs[pkg]['path'], path) or 'Unchanged' return ret diff --git a/salt/states/pkg.py b/salt/states/pkg.py index 15c669d..3811404 100644 --- a/salt/states/pkg.py +++ b/salt/states/pkg.py @@ -327,27 +327,34 @@ def _find_install_targets(name=None, if not (name in cur_pkgs and version in (None, cur_pkgs[name])) ]) if not_installed: - problems = _preflight_check(not_installed, **kwargs) - comments = [] - if problems.get('no_suggest'): - comments.append( - 'The following package(s) were not found, and no possible ' - 'matches were found in the package db: ' - '{0}'.format(', '.join(sorted(problems['no_suggest']))) - ) - if problems.get('suggest'): - for pkgname, suggestions in six.iteritems(problems['suggest']): + try: + problems = _preflight_check(not_installed, **kwargs) + except CommandExecutionError: + pass + else: + comments = [] + if problems.get('no_suggest'): comments.append( - 'Package \'{0}\' not found (possible matches: {1})' - .format(pkgname, ', '.join(suggestions)) + 'The following package(s) were not found, and no ' + 'possible matches were found in the package db: ' + '{0}'.format( + ', '.join(sorted(problems['no_suggest'])) + ) ) - if comments: - if len(comments) > 1: - comments.append('') - return {'name': name, - 'changes': {}, - 'result': False, - 'comment': '. '.join(comments).rstrip()} + if problems.get('suggest'): + for pkgname, suggestions in \ + six.iteritems(problems['suggest']): + comments.append( + 'Package \'{0}\' not found (possible matches: ' + '{1})'.format(pkgname, ', '.join(suggestions)) + ) + if comments: + if len(comments) > 1: + comments.append('') + return {'name': name, + 'changes': {}, + 'result': False, + 'comment': '. '.join(comments).rstrip()} # Check current versions against desired versions targets = {} diff --git a/salt/utils/itertools.py b/salt/utils/itertools.py new file mode 100644 index 0000000..f824adb --- /dev/null +++ b/salt/utils/itertools.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +''' +Helpful generators and other tools +''' + +# Import python libs +from __future__ import absolute_import +import re + + +def split(orig, sep=None): + ''' + Generator function for iterating through large strings, particularly useful + as a replacement for str.splitlines(). + + See http://stackoverflow.com/a/3865367 + ''' + exp = re.compile(r'\s+' if sep is None else re.escape(sep)) + pos = 0 + length = len(orig) + while True: + match = exp.search(orig, pos) + if not match: + if pos < length or sep is not None: + val = orig[pos:] + if val: + # Only yield a value if the slice was not an empty string, + # because if it is then we've reached the end. This keeps + # us from yielding an extra blank value at the end. + yield val + break + if pos < match.start() or sep is not None: + yield orig[pos:match.start()] + pos = match.end() diff --git a/salt/utils/pkg/__init__.py b/salt/utils/pkg/__init__.py new file mode 100644 index 0000000..e316343 --- /dev/null +++ b/salt/utils/pkg/__init__.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- +''' +Helper modules used by lowpkg modules +''' diff --git a/salt/utils/pkg/rpm.py b/salt/utils/pkg/rpm.py new file mode 100644 index 0000000..9c01d5b --- /dev/null +++ b/salt/utils/pkg/rpm.py @@ -0,0 +1,102 @@ +# -*- coding: utf-8 -*- +''' +Common functions for working with RPM packages +''' + +# Import python libs +from __future__ import absolute_import +import collections +import logging + +# Import salt libs +from salt._compat import subprocess + +log = logging.getLogger(__name__) + +# These arches compiled from the rpmUtils.arch python module source +ARCHES_64 = ('x86_64', 'athlon', 'amd64', 'ia32e', 'ia64', 'geode') +ARCHES_32 = ('i386', 'i486', 'i586', 'i686') +ARCHES_PPC = ('ppc', 'ppc64', 'ppc64iseries', 'ppc64pseries') +ARCHES_S390 = ('s390', 's390x') +ARCHES_SPARC = ( + 'sparc', 'sparcv8', 'sparcv9', 'sparcv9v', 'sparc64', 'sparc64v' +) +ARCHES_ALPHA = ( + 'alpha', 'alphaev4', 'alphaev45', 'alphaev5', 'alphaev56', + 'alphapca56', 'alphaev6', 'alphaev67', 'alphaev68', 'alphaev7' +) +ARCHES_ARM = ('armv5tel', 'armv5tejl', 'armv6l', 'armv7l') +ARCHES_SH = ('sh3', 'sh4', 'sh4a') + +ARCHES = ARCHES_64 + ARCHES_32 + ARCHES_PPC + ARCHES_S390 + \ + ARCHES_ALPHA + ARCHES_ARM + ARCHES_SH + +# EPOCHNUM can't be used until RHEL5 is EOL as it is not present +QUERYFORMAT = '%{NAME}_|-%{EPOCH}_|-%{VERSION}_|-%{RELEASE}_|-%{ARCH}_|-%{REPOID}' + + +def get_osarch(): + ''' + Get the os architecture using rpm --eval + ''' + ret = subprocess.Popen( + 'rpm --eval "%{_host_cpu}"', + shell=True, + close_fds=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate()[0] + return ret or 'unknown' + + +def check_32(arch, osarch=None): + ''' + Returns True if both the OS arch and the passed arch are 32-bit + ''' + if osarch is None: + osarch = get_osarch() + return all(x in ARCHES_32 for x in (osarch, arch)) + + +def pkginfo(name, version, arch, repoid): + ''' + Build and return a pkginfo namedtuple + ''' + pkginfo_tuple = collections.namedtuple( + 'PkgInfo', + ('name', 'version', 'arch', 'repoid') + ) + return pkginfo_tuple(name, version, arch, repoid) + + +def resolve_name(name, arch, osarch=None): + ''' + Resolve the package name and arch into a unique name referred to by salt. + For example, on a 64-bit OS, a 32-bit package will be pkgname.i386. + ''' + if osarch is None: + osarch = get_osarch() + + if not check_32(arch, osarch) and arch not in (osarch, 'noarch'): + name += '.{0}'.format(arch) + return name + + +def parse_pkginfo(line, osarch=None): + ''' + A small helper to parse an rpm/repoquery command's output. Returns a + pkginfo namedtuple. + ''' + try: + name, epoch, version, release, arch, repoid = line.split('_|-') + # Handle unpack errors (should never happen with the queryformat we are + # using, but can't hurt to be careful). + except ValueError: + return None + + name = resolve_name(name, arch, osarch) + if release: + version += '-{0}'.format(release) + if epoch not in ('(none)', '0'): + version = ':'.join((epoch, version)) + + return pkginfo(name, version, arch, repoid) diff --git a/salt/modules/debian_ip.py b/salt/modules/debian_ip.py index d0d6db1..275cb97 100644 --- a/salt/modules/debian_ip.py +++ b/salt/modules/debian_ip.py @@ -819,9 +819,12 @@ def _parse_settings_bond_0(opts, iface, bond_def): if 'arp_ip_target' in opts: if isinstance(opts['arp_ip_target'], list): if 1 <= len(opts['arp_ip_target']) <= 16: - bond.update({'arp_ip_target': []}) + bond.update({'arp_ip_target': ''}) for ip in opts['arp_ip_target']: # pylint: disable=C0103 - bond['arp_ip_target'].append(ip) + if len(bond['arp_ip_target']) > 0: + bond['arp_ip_target'] = bond['arp_ip_target'] + ',' + ip + else: + bond['arp_ip_target'] = ip else: _raise_error_iface(iface, 'arp_ip_target', valid) else: @@ -892,9 +895,12 @@ def _parse_settings_bond_2(opts, iface, bond_def): if 'arp_ip_target' in opts: if isinstance(opts['arp_ip_target'], list): if 1 <= len(opts['arp_ip_target']) <= 16: - bond.update({'arp_ip_target': []}) + bond.update({'arp_ip_target': ''}) for ip in opts['arp_ip_target']: # pylint: disable=C0103 - bond['arp_ip_target'].append(ip) + if len(bond['arp_ip_target']) > 0: + bond['arp_ip_target'] = bond['arp_ip_target'] + ',' + ip + else: + bond['arp_ip_target'] = ip else: _raise_error_iface(iface, 'arp_ip_target', valid) else: diff --git a/salt/modules/rh_ip.py b/salt/modules/rh_ip.py index 2762125..cd362e4 100644 --- a/salt/modules/rh_ip.py +++ b/salt/modules/rh_ip.py @@ -276,9 +276,12 @@ def _parse_settings_bond_0(opts, iface, bond_def): if 'arp_ip_target' in opts: if isinstance(opts['arp_ip_target'], list): if 1 <= len(opts['arp_ip_target']) <= 16: - bond.update({'arp_ip_target': []}) + bond.update({'arp_ip_target': ''}) for ip in opts['arp_ip_target']: # pylint: disable=C0103 - bond['arp_ip_target'].append(ip) + if len(bond['arp_ip_target']) > 0: + bond['arp_ip_target'] = bond['arp_ip_target'] + ',' + ip + else: + bond['arp_ip_target'] = ip else: _raise_error_iface(iface, 'arp_ip_target', valid) else: @@ -349,9 +352,12 @@ def _parse_settings_bond_2(opts, iface, bond_def): if 'arp_ip_target' in opts: if isinstance(opts['arp_ip_target'], list): if 1 <= len(opts['arp_ip_target']) <= 16: - bond.update({'arp_ip_target': []}) + bond.update({'arp_ip_target': ''}) for ip in opts['arp_ip_target']: # pylint: disable=C0103 - bond['arp_ip_target'].append(ip) + if len(bond['arp_ip_target']) > 0: + bond['arp_ip_target'] = bond['arp_ip_target'] + ',' + ip + else: + bond['arp_ip_target'] = ip else: _raise_error_iface(iface, 'arp_ip_target', valid) else: