From bf1c33fd4d1c1a9cb3e1ef9afee52697ff75ba2b Mon Sep 17 00:00:00 2001 From: MSVSphere Packaging Team Date: Wed, 20 Mar 2024 01:21:43 +0300 Subject: [PATCH] import koji-1.34.0-2.el9 --- .gitignore | 2 +- .koji.metadata | 2 +- ...961929a45ccae1d635b7e85a9d8826baf72d.patch | 29 +++ ...5927aeb3e3c052fc9208ca71133d9d097fc0.patch | 229 ++++++++++++++++++ ...18fa356f1aa2a1b5099e55e0af1c89ae4163.patch | 26 ++ SOURCES/3974.patch | 43 ++++ SOURCES/4013.patch | 52 ++++ SOURCES/4026.patch | 38 +++ SPECS/koji.spec | 50 +++- 9 files changed, 465 insertions(+), 6 deletions(-) create mode 100644 SOURCES/0251961929a45ccae1d635b7e85a9d8826baf72d.patch create mode 100644 SOURCES/1d515927aeb3e3c052fc9208ca71133d9d097fc0.patch create mode 100644 SOURCES/2a6e18fa356f1aa2a1b5099e55e0af1c89ae4163.patch create mode 100644 SOURCES/3974.patch create mode 100644 SOURCES/4013.patch create mode 100644 SOURCES/4026.patch diff --git a/.gitignore b/.gitignore index 68269f8..0f21b56 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1 @@ -SOURCES/koji-1.33.0.tar.bz2 +SOURCES/koji-1.34.0.tar.bz2 diff --git a/.koji.metadata b/.koji.metadata index e721bdf..9228bfb 100644 --- a/.koji.metadata +++ b/.koji.metadata @@ -1 +1 @@ -bab50aad5ad51badf99ab1532af2e9be94b9c366 SOURCES/koji-1.33.0.tar.bz2 +5f67829ac737963d05fc5c337a42e5d6439d1004 SOURCES/koji-1.34.0.tar.bz2 diff --git a/SOURCES/0251961929a45ccae1d635b7e85a9d8826baf72d.patch b/SOURCES/0251961929a45ccae1d635b7e85a9d8826baf72d.patch new file mode 100644 index 0000000..c910262 --- /dev/null +++ b/SOURCES/0251961929a45ccae1d635b7e85a9d8826baf72d.patch @@ -0,0 +1,29 @@ +From 0251961929a45ccae1d635b7e85a9d8826baf72d Mon Sep 17 00:00:00 2001 +From: Tomas Kopecek +Date: Feb 07 2024 13:45:08 +0000 +Subject: PR#3989: Oz: don't hardcode the image size unit as 'G' + + +Merges #3989 +https://pagure.io/koji/pull-request/3989 + +Fixes: #3993 +https://pagure.io/koji/issue/3993 + Oz: don't hardcode the image size unit as 'G' + +--- + +diff --git a/builder/kojid b/builder/kojid +index fe35e47..b4536dd 100755 +--- a/builder/kojid ++++ b/builder/kojid +@@ -4101,7 +4101,7 @@ class OzImageTask(BaseTaskHandler): + template += """ + %s OS + +- %sG ++ %s + + + """ % (name, self.opts.get('disk_size')) # noqa: E501 + diff --git a/SOURCES/1d515927aeb3e3c052fc9208ca71133d9d097fc0.patch b/SOURCES/1d515927aeb3e3c052fc9208ca71133d9d097fc0.patch new file mode 100644 index 0000000..9be3dcc --- /dev/null +++ b/SOURCES/1d515927aeb3e3c052fc9208ca71133d9d097fc0.patch @@ -0,0 +1,229 @@ +From 1d515927aeb3e3c052fc9208ca71133d9d097fc0 Mon Sep 17 00:00:00 2001 +From: Tomas Kopecek +Date: Thu, 13 Apr 2023 11:12:40 +0200 +Subject: [PATCH] scmpolicy plugin + +--- + docs/source/defining_hub_policies.rst | 10 +++- + docs/source/plugins.rst | 30 ++++++++++- + koji/policy.py | 53 +++++++++++++++++++- + plugins/builder/scmpolicy.py | 72 +++++++++++++++++++++++++++ + 4 files changed, 162 insertions(+), 3 deletions(-) + create mode 100644 plugins/builder/scmpolicy.py + +diff --git a/docs/source/defining_hub_policies.rst b/docs/source/defining_hub_policies.rst +index a0b67eed..8f9cf2cd 100644 +--- a/docs/source/defining_hub_policies.rst ++++ b/docs/source/defining_hub_policies.rst +@@ -341,5 +341,13 @@ Available tests + * the user matched is the user performing the action + + ``match`` +- * matches a field in the data against glob patterns ++ * matches a field in the data against glob patterns + * true if any pattern matches ++ ++``match_any`` ++ * matches a field (of list/tuple/set type) in the data against glob patterns ++ * true if any field item matches all patterns ++ ++``match_all`` ++ * matches a field (of list/tuple/set type) in the data against glob patterns ++ * true if all field items match any pattern +diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst +index c370709a..d5b2d13f 100644 +--- a/docs/source/plugins.rst ++++ b/docs/source/plugins.rst +@@ -223,7 +223,7 @@ The ``[message]`` section sets parameters for how messages are formed. + Currently only one field is understood: + + * ``extra_limit`` -- the maximum allowed size for ``build.extra`` fields that +- appear in messages. If the ``build.extra`` field is longer (in terms of ++ appear in messages. If the ``build.extra`` field is longer (in terms of + json-encoded length), then it will be omitted. The default value is ``0`` + which means no limit. + +@@ -441,3 +441,31 @@ For example: + For each RPM in the tag, Koji will use the first signed copy that it finds. In other words, + Koji will try the first key (`45719a39`), and if Koji does not have the first key's signature + for that RPM, then it will try the second key (`9867c58f`), third key (`38ab71f4`), and so on. ++ ++Scm Policies ++============ ++ ++Basic filtering of allowed scms normally happens via standard ++``build_from_scm`` hub policy. Nevertheless, some relevant information can be ++only gathered after cloning the repo. Typical case is that admin would like to ++build content only from some set of allowed branches. If user specify the ++commit via hash, we don't have that information in moment of task creation. ++Just after cloning we can check existing branches and if the given commit is on ++some of the relevant ones. For this purpose there is special ++``postSCMCheckout`` plugin ``scmpolicy``. ++ ++Installation happens only on builder via editing ``/etc/kojid.conf`` by adding ++``plugin = scmpolicy`` there. Plugin itself is not configured but uses hub ++policy ``scm``. Policy data provided there are composed of two parts. First one ++are ``scm_*`` values which are same as in ``build_from_scm``. ++ ++.. code:: ++ ++ scm = ++ # allow scratch builds from any commits ++ bool scratch :: allow ++ # very safe scm, allow anything from there, but only to special target ++ match scm_host very.safe.git.org && buildtag testing-build-tag :: allow ++ match_all branches * !! deny Commit must be present on some branch ++ match_all branches private-* test-* :: deny Private/testing branches are not allowed ++ all :: allow +diff --git a/koji/policy.py b/koji/policy.py +index 729e02e5..8a570575 100644 +--- a/koji/policy.py ++++ b/koji/policy.py +@@ -25,7 +25,7 @@ import logging + import six + + import koji +-from koji.util import to_list ++from koji.util import to_list, multi_fnmatch + + + class BaseSimpleTest(object): +@@ -141,6 +141,57 @@ class MatchTest(BaseSimpleTest): + return False + + ++class MatchAnyTest(BaseSimpleTest): ++ """Matches any item of a list/tuple/set value in the data against glob patterns ++ ++ True if any of the expressions matches any item in the list/tuple/set, else False. ++ If the field doesn't exist or isn't a list/tuple/set, the test returns False ++ ++ Syntax: ++ find field pattern1 [pattern2 ...] ++ ++ """ ++ name = 'match_any' ++ field = None ++ ++ def run(self, data): ++ args = self.str.split()[1:] ++ self.field = args[0] ++ args = args[1:] ++ tgt = data.get(self.field) ++ if tgt and isinstance(tgt, (list, tuple, set)): ++ for i in tgt: ++ if i is not None and multi_fnmatch(str(i), args): ++ return True ++ return False ++ ++ ++class MatchAllTest(BaseSimpleTest): ++ """Matches all items of a list/tuple/set value in the data against glob patterns ++ ++ True if any of the expressions matches all items in the list/tuple/set, else False. ++ If the field doesn't exist or isn't a list/tuple/set, the test returns False ++ ++ Syntax: ++ match_all field pattern1 [pattern2 ...] ++ ++ """ ++ name = 'match_all' ++ field = None ++ ++ def run(self, data): ++ args = self.str.split()[1:] ++ self.field = args[0] ++ args = args[1:] ++ tgt = data.get(self.field) ++ if tgt and isinstance(tgt, (list, tuple, set)): ++ for i in tgt: ++ if i is None or not multi_fnmatch(str(i), args): ++ return False ++ return True ++ return False ++ ++ + class TargetTest(MatchTest): + """Matches target in the data against glob patterns + +diff --git a/plugins/builder/scmpolicy.py b/plugins/builder/scmpolicy.py +new file mode 100644 +index 00000000..f120e33b +--- /dev/null ++++ b/plugins/builder/scmpolicy.py +@@ -0,0 +1,72 @@ ++import logging ++import re ++import subprocess ++ ++import six ++ ++from koji import ActionNotAllowed, GenericError ++from koji.plugin import callback ++ ++ ++logger = logging.getLogger('koji.plugins.scmpolicy') ++ ++ ++@callback('postSCMCheckout') ++def assert_scm_policy(clb_type, *args, **kwargs): ++ taskinfo = kwargs['taskinfo'] ++ session = kwargs['session'] ++ build_tag = kwargs['build_tag'] ++ scminfo = kwargs['scminfo'] ++ srcdir = kwargs['srcdir'] ++ scratch = kwargs['scratch'] ++ ++ method = get_task_method(session, taskinfo) ++ ++ policy_data = { ++ 'build_tag': build_tag, ++ 'method': method, ++ 'scratch': scratch, ++ 'branches': get_branches(srcdir) ++ } ++ ++ # Merge scminfo into data with "scm_" prefix. And "scm*" are changed to "scm_*". ++ for k, v in six.iteritems(scminfo): ++ policy_data[re.sub(r'^(scm_?)?', 'scm_', k)] = v ++ ++ logger.info("Checking SCM policy for task %s", taskinfo['id']) ++ logger.debug("Policy data: %r", policy_data) ++ ++ # check the policy ++ try: ++ session.host.assertPolicy('scm', policy_data) ++ logger.info("SCM policy check for task %s: PASSED", taskinfo['id']) ++ except ActionNotAllowed: ++ logger.warning("SCM policy check for task %s: DENIED", taskinfo['id']) ++ raise ++ ++ ++def get_task_method(session, taskinfo): ++ """Get the Task method from taskinfo""" ++ method = None ++ if isinstance(taskinfo, six.integer_types): ++ taskinfo = session.getTaskInfo(taskinfo, strict=True) ++ if isinstance(taskinfo, dict): ++ method = taskinfo.get('method') ++ if method is None: ++ raise GenericError("Invalid taskinfo: %s" % taskinfo) ++ return method ++ ++ ++def get_branches(srcdir): ++ """Determine which remote branches contain the current checkout""" ++ cmd = ['git', 'branch', '-r', '--contains', 'HEAD'] ++ proc = subprocess.Popen(cmd, cwd=srcdir, stdout=subprocess.PIPE) ++ (out, _) = proc.communicate() ++ status = proc.wait() ++ if status != 0: ++ raise Exception('Error getting branches for git checkout') ++ ++ # cut off origin/ prefix ++ branches = [b.strip() for b in out.decode().split('\n') if 'origin/HEAD' not in b and b] ++ branches = [re.sub('^origin/', '', b) for b in branches] ++ return branches +-- +GitLab + diff --git a/SOURCES/2a6e18fa356f1aa2a1b5099e55e0af1c89ae4163.patch b/SOURCES/2a6e18fa356f1aa2a1b5099e55e0af1c89ae4163.patch new file mode 100644 index 0000000..71f07d5 --- /dev/null +++ b/SOURCES/2a6e18fa356f1aa2a1b5099e55e0af1c89ae4163.patch @@ -0,0 +1,26 @@ +From 2a6e18fa356f1aa2a1b5099e55e0af1c89ae4163 Mon Sep 17 00:00:00 2001 +From: Mike McLean +Date: Feb 05 2024 10:28:43 +0000 +Subject: typo in set_refusal + + +Fixes https://pagure.io/koji/issue/3997 + +--- + +diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py +index 815b0f1..961ef39 100644 +--- a/kojihub/scheduler.py ++++ b/kojihub/scheduler.py +@@ -91,8 +91,8 @@ def get_tasks_for_host(hostID, retry=True): + + def set_refusal(hostID, taskID, soft=True, by_host=False, msg=''): + data = { +- 'task_id': kojihub.convert_value(hostID, cast=int), +- 'host_id': kojihub.convert_value(taskID, cast=int), ++ 'host_id': kojihub.convert_value(hostID, cast=int), ++ 'task_id': kojihub.convert_value(taskID, cast=int), + 'soft': kojihub.convert_value(soft, cast=bool), + 'by_host': kojihub.convert_value(by_host, cast=bool), + 'msg': kojihub.convert_value(msg, cast=str), + diff --git a/SOURCES/3974.patch b/SOURCES/3974.patch new file mode 100644 index 0000000..7096d54 --- /dev/null +++ b/SOURCES/3974.patch @@ -0,0 +1,43 @@ +From 36953540662aa39ff1b85218cefededfa4e529a9 Mon Sep 17 00:00:00 2001 +From: Tomas Kopecek +Date: Jan 15 2024 12:48:16 +0000 +Subject: Use dnf5-compatible "group install" command + + + * yum knows only "yum groupinstall" + * dnf < 5 knows both + * dnf-5 only "dnf5 group install" + +It is reasonable to assume that dnf is used in most setups, so changing +default to "group install". If yum is specified *explicitly* via +tags's extra "mock.package_manager", "groupinstall" is used instead. + +Related: https://pagure.io/koji/issue/3971 + +--- + +diff --git a/koji/__init__.py b/koji/__init__.py +index 334b403..9b222cd 100644 +--- a/koji/__init__.py ++++ b/koji/__init__.py +@@ -1804,7 +1804,7 @@ def genMockConfig(name, arch, managed=False, repoid=None, tag_name=None, **opts) + 'target_arch': opts.get('target_arch', arch), + 'chroothome': '/builddir', + # Use the group data rather than a generated rpm +- 'chroot_setup_cmd': 'groupinstall %s' % opts.get('install_group', 'build'), ++ 'chroot_setup_cmd': 'group install %s' % opts.get('install_group', 'build'), + # don't encourage network access from the chroot + 'rpmbuild_networking': opts.get('use_host_resolv', False), + 'use_host_resolv': opts.get('use_host_resolv', False), +@@ -1817,6 +1817,10 @@ def genMockConfig(name, arch, managed=False, repoid=None, tag_name=None, **opts) + config_opts['forcearch'] = opts['forcearch'] + if opts.get('package_manager'): + config_opts['package_manager'] = opts['package_manager'] ++ if opts['package_manager'].endswith('yum'): ++ # backward compatibility with yum (doesn't have separate "group") ++ config_opts['chroot_setup_cmd'] = \ ++ 'groupinstall %s' % opts.get('install_group', 'build') + if opts.get('bootstrap_image'): + config_opts['use_bootstrap_image'] = True + config_opts['bootstrap_image'] = opts['bootstrap_image'] + diff --git a/SOURCES/4013.patch b/SOURCES/4013.patch new file mode 100644 index 0000000..9b3fae8 --- /dev/null +++ b/SOURCES/4013.patch @@ -0,0 +1,52 @@ +From bc8e6253f519eeb78fbc8740bba25e8a34490814 Mon Sep 17 00:00:00 2001 +From: Mike McLean +Date: Feb 14 2024 06:40:24 +0000 +Subject: let tag.extra override tag arches for noarch + + +--- + +diff --git a/builder/kojid b/builder/kojid +index b4536dd..8b81d66 100755 +--- a/builder/kojid ++++ b/builder/kojid +@@ -1339,23 +1339,33 @@ class BuildTask(BaseTaskHandler): + exclusivearch = koji.get_header_field(h, 'exclusivearch') + excludearch = koji.get_header_field(h, 'excludearch') + +- if exclusivearch or excludearch: ++ buildconfig = self.session.getBuildConfig(build_tag, event=self.event_id) ++ noarch_arches = buildconfig.get('extra', {}).get('noarch_arches') ++ ++ if exclusivearch or excludearch or noarch_arches: + # if one of the tag arches is filtered out, then we can't use a + # noarch task +- buildconfig = self.session.getBuildConfig(build_tag, event=self.event_id) + arches = buildconfig['arches'] + tag_arches = [koji.canonArch(a) for a in arches.split()] + exclusivearch = [koji.canonArch(a) for a in exclusivearch] + excludearch = [koji.canonArch(a) for a in excludearch] +- archlist = list(tag_arches) ++ # tag.extra overrides tag arches for noarch ++ if noarch_arches: ++ archlist = [koji.canonArch(a) for a in noarch_arches.split()] ++ archlist = [a for a in archlist if a in tag_arches] ++ else: ++ archlist = list(tag_arches) + if exclusivearch: + archlist = [a for a in archlist if a in exclusivearch] + if excludearch: + archlist = [a for a in archlist if a not in excludearch] ++ self.logger.info('Filtering arches for noarch subtask. Choices: %r', archlist) + if not archlist: +- raise koji.BuildError("No valid arches were found. tag %r, " +- "exclusive %r, exclude %r" % (tag_arches, ++ raise koji.BuildError("No valid arches were found. tag %r, extra %r," ++ "exclusive %r, exclude %r" % (tag_arches, noarch_arches, + exclusivearch, excludearch)) ++ self.logger.debug('tag: %r, extra: %r, exclusive: %r, exclude: %r', ++ tag_arches, noarch_arches, exclusivearch, excludearch) + if set(archlist) != set(tag_arches): + return random.choice(archlist) + else: + diff --git a/SOURCES/4026.patch b/SOURCES/4026.patch new file mode 100644 index 0000000..edf0b36 --- /dev/null +++ b/SOURCES/4026.patch @@ -0,0 +1,38 @@ +From f453092d308605707ba1fb3fa314e05b515e7a25 Mon Sep 17 00:00:00 2001 +From: Tomas Kopecek +Date: Feb 29 2024 10:45:55 +0000 +Subject: Better index for rpm lookup + + +Related: https://pagure.io/koji/issue/4022 + +--- + +diff --git a/schemas/schema-upgrade-1.34-1.35.sql b/schemas/schema-upgrade-1.34-1.35.sql +new file mode 100644 +index 0000000..3cc82bc +--- /dev/null ++++ b/schemas/schema-upgrade-1.34-1.35.sql +@@ -0,0 +1,9 @@ ++-- upgrade script to migrate the Koji database schema ++-- from version 1.33 to 1.34 ++ ++BEGIN; ++ ++CREATE INDEX CONCURRENTLY IF NOT EXISTS rpminfo_nvra ++ ON rpminfo(name,version,release,arch,external_repo_id); ++ ++COMMIT; +diff --git a/schemas/schema.sql b/schemas/schema.sql +index e5f3462..7e3298c 100644 +--- a/schemas/schema.sql ++++ b/schemas/schema.sql +@@ -752,6 +752,7 @@ CREATE TABLE rpminfo ( + CREATE INDEX rpminfo_build ON rpminfo(build_id); + CREATE UNIQUE INDEX rpminfo_unique_nvra_not_draft ON rpminfo(name,version,release,arch,external_repo_id) + WHERE draft IS NOT TRUE; ++CREATE INDEX rpminfo_nvra ON rpminfo(name,version,release,arch,external_repo_id); + -- index for default search method for rpms, PG11+ can benefit from new include method + DO $$ + DECLARE version integer; + diff --git a/SPECS/koji.spec b/SPECS/koji.spec index 059932a..56100e0 100644 --- a/SPECS/koji.spec +++ b/SPECS/koji.spec @@ -8,16 +8,31 @@ %{?!python3_pkgversion:%global python3_pkgversion 3} Name: koji -Version: 1.33.0 -Release: 1%{?dist} +Version: 1.34.0 +Release: 2%{?dist} # the included arch lib from yum's rpmUtils is GPLv2+ License: LGPLv2 and GPLv2+ Summary: Build system tools URL: https://pagure.io/koji/ Source0: https://releases.pagure.org/koji/koji-%{version}.tar.bz2 +# scm policy plugin - already upstreamed +Patch1: 1d515927aeb3e3c052fc9208ca71133d9d097fc0.patch # Not upstreamable -Patch1000: fedora-config.patch +Patch100: fedora-config.patch +# Use dnf5-compatible "group install" command +# This should work on yum/dnf-4/dnf5 +Patch102: https://pagure.io/koji/pull-request/3974.patch +# noarch builds only happen on some arches +# allows picking what arches will do noarch builds +Patch103: https://pagure.io/koji/pull-request/4013.patch +# oz size patch +# Drop passing a unit to oz so it can determine GiB vs GB +Patch104: https://pagure.io/koji/c/0251961929a45ccae1d635b7e85a9d8826baf72d.patch +# fix typo in refusal - already upstreamed +Patch105: https://pagure.io/koji/c/2a6e18fa356f1aa2a1b5099e55e0af1c89ae4163.patch +# Add index for rpminfo +Patch106: https://pagure.io/koji/pull-request/4026.patch BuildArch: noarch Requires: python%{python3_pkgversion}-%{name} = %{version}-%{release} @@ -44,6 +59,7 @@ Requires: python%{python3_pkgversion}-requests Requires: python%{python3_pkgversion}-requests-gssapi Requires: python%{python3_pkgversion}-dateutil Requires: python%{python3_pkgversion}-six +Requires: python%{python3_pkgversion}-defusedxml %description -n python%{python3_pkgversion}-%{name} Koji is a system for building and tracking RPMS. @@ -348,9 +364,35 @@ done %systemd_postun kojira.service %changelog -* Mon Oct 16 2023 Sergey Cherevko - 1.33.0-1 +* Mon Mar 18 2024 Kevin Fenzi - 1.34.0-2 +- Carry scm policy plugin for hub, it's already upstream +- Use dnf5 compatible 'group install' command +- Allow specifying with a tag value what arches noarch builds happen on. +- Fix image-build to not pass units to oz (to avoid GB/GiB issues) +- Fix a typo in scheduler (already upstreamed) +- Add back index for rpminfo table that was mistakenly dropped. + +* Thu Jan 25 2024 Kevin Fenzi - 1.34.0-1 +- Update to 1.34.0. Fixes rhbz#2260055 + +* Thu Jan 25 2024 Fedora Release Engineering - 1.33.1-4 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_40_Mass_Rebuild + +* Sun Jan 21 2024 Fedora Release Engineering - 1.33.1-3 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_40_Mass_Rebuild + +* Mon Oct 16 2023 Sergey Cherevko - 1.33.1-2 - Rebuilt for MSVSphere 9.2 +* Thu Jul 20 2023 Fedora Release Engineering - 1.33.1-2 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_39_Mass_Rebuild + +* Fri Jul 14 2023 Kevin Fenzi - 1.33.1-1 +- Update to 1.31.1. Fixes rhbz#2222032 + +* Tue Jun 13 2023 Python Maint - 1.33.0-2 +- Rebuilt for Python 3.12 + * Wed May 24 2023 Kevin Fenzi - 1.33.0-1 - Update to 1.33.0. Fixes rhbz#2209371