MSVSphere Packaging Team
11 months ago
parent
ade0cdd3ce
commit
bf1c33fd4d
@ -1 +1 @@
|
||||
SOURCES/koji-1.33.0.tar.bz2
|
||||
SOURCES/koji-1.34.0.tar.bz2
|
||||
|
||||
@ -1 +1 @@
|
||||
bab50aad5ad51badf99ab1532af2e9be94b9c366 SOURCES/koji-1.33.0.tar.bz2
|
||||
5f67829ac737963d05fc5c337a42e5d6439d1004 SOURCES/koji-1.34.0.tar.bz2
|
||||
|
||||
@ -0,0 +1,29 @@
|
||||
From 0251961929a45ccae1d635b7e85a9d8826baf72d Mon Sep 17 00:00:00 2001
|
||||
From: Tomas Kopecek <tkopecek@redhat.com>
|
||||
Date: Feb 07 2024 13:45:08 +0000
|
||||
Subject: PR#3989: Oz: don't hardcode the image size unit as 'G'
|
||||
|
||||
|
||||
Merges #3989
|
||||
https://pagure.io/koji/pull-request/3989
|
||||
|
||||
Fixes: #3993
|
||||
https://pagure.io/koji/issue/3993
|
||||
Oz: don't hardcode the image size unit as 'G'
|
||||
|
||||
---
|
||||
|
||||
diff --git a/builder/kojid b/builder/kojid
|
||||
index fe35e47..b4536dd 100755
|
||||
--- a/builder/kojid
|
||||
+++ b/builder/kojid
|
||||
@@ -4101,7 +4101,7 @@ class OzImageTask(BaseTaskHandler):
|
||||
template += """</os>
|
||||
<description>%s OS</description>
|
||||
<disk>
|
||||
- <size>%sG</size>
|
||||
+ <size>%s</size>
|
||||
</disk>
|
||||
</template>
|
||||
""" % (name, self.opts.get('disk_size')) # noqa: E501
|
||||
|
||||
@ -0,0 +1,229 @@
|
||||
From 1d515927aeb3e3c052fc9208ca71133d9d097fc0 Mon Sep 17 00:00:00 2001
|
||||
From: Tomas Kopecek <tkopecek@redhat.com>
|
||||
Date: Thu, 13 Apr 2023 11:12:40 +0200
|
||||
Subject: [PATCH] scmpolicy plugin
|
||||
|
||||
---
|
||||
docs/source/defining_hub_policies.rst | 10 +++-
|
||||
docs/source/plugins.rst | 30 ++++++++++-
|
||||
koji/policy.py | 53 +++++++++++++++++++-
|
||||
plugins/builder/scmpolicy.py | 72 +++++++++++++++++++++++++++
|
||||
4 files changed, 162 insertions(+), 3 deletions(-)
|
||||
create mode 100644 plugins/builder/scmpolicy.py
|
||||
|
||||
diff --git a/docs/source/defining_hub_policies.rst b/docs/source/defining_hub_policies.rst
|
||||
index a0b67eed..8f9cf2cd 100644
|
||||
--- a/docs/source/defining_hub_policies.rst
|
||||
+++ b/docs/source/defining_hub_policies.rst
|
||||
@@ -341,5 +341,13 @@ Available tests
|
||||
* the user matched is the user performing the action
|
||||
|
||||
``match``
|
||||
- * matches a field in the data against glob patterns
|
||||
+ * matches a field in the data against glob patterns
|
||||
* true if any pattern matches
|
||||
+
|
||||
+``match_any``
|
||||
+ * matches a field (of list/tuple/set type) in the data against glob patterns
|
||||
+ * true if any field item matches all patterns
|
||||
+
|
||||
+``match_all``
|
||||
+ * matches a field (of list/tuple/set type) in the data against glob patterns
|
||||
+ * true if all field items match any pattern
|
||||
diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst
|
||||
index c370709a..d5b2d13f 100644
|
||||
--- a/docs/source/plugins.rst
|
||||
+++ b/docs/source/plugins.rst
|
||||
@@ -223,7 +223,7 @@ The ``[message]`` section sets parameters for how messages are formed.
|
||||
Currently only one field is understood:
|
||||
|
||||
* ``extra_limit`` -- the maximum allowed size for ``build.extra`` fields that
|
||||
- appear in messages. If the ``build.extra`` field is longer (in terms of
|
||||
+ appear in messages. If the ``build.extra`` field is longer (in terms of
|
||||
json-encoded length), then it will be omitted. The default value is ``0``
|
||||
which means no limit.
|
||||
|
||||
@@ -441,3 +441,31 @@ For example:
|
||||
For each RPM in the tag, Koji will use the first signed copy that it finds. In other words,
|
||||
Koji will try the first key (`45719a39`), and if Koji does not have the first key's signature
|
||||
for that RPM, then it will try the second key (`9867c58f`), third key (`38ab71f4`), and so on.
|
||||
+
|
||||
+Scm Policies
|
||||
+============
|
||||
+
|
||||
+Basic filtering of allowed scms normally happens via standard
|
||||
+``build_from_scm`` hub policy. Nevertheless, some relevant information can be
|
||||
+only gathered after cloning the repo. Typical case is that admin would like to
|
||||
+build content only from some set of allowed branches. If user specify the
|
||||
+commit via hash, we don't have that information in moment of task creation.
|
||||
+Just after cloning we can check existing branches and if the given commit is on
|
||||
+some of the relevant ones. For this purpose there is special
|
||||
+``postSCMCheckout`` plugin ``scmpolicy``.
|
||||
+
|
||||
+Installation happens only on builder via editing ``/etc/kojid.conf`` by adding
|
||||
+``plugin = scmpolicy`` there. Plugin itself is not configured but uses hub
|
||||
+policy ``scm``. Policy data provided there are composed of two parts. First one
|
||||
+are ``scm_*`` values which are same as in ``build_from_scm``.
|
||||
+
|
||||
+.. code::
|
||||
+
|
||||
+ scm =
|
||||
+ # allow scratch builds from any commits
|
||||
+ bool scratch :: allow
|
||||
+ # very safe scm, allow anything from there, but only to special target
|
||||
+ match scm_host very.safe.git.org && buildtag testing-build-tag :: allow
|
||||
+ match_all branches * !! deny Commit must be present on some branch
|
||||
+ match_all branches private-* test-* :: deny Private/testing branches are not allowed
|
||||
+ all :: allow
|
||||
diff --git a/koji/policy.py b/koji/policy.py
|
||||
index 729e02e5..8a570575 100644
|
||||
--- a/koji/policy.py
|
||||
+++ b/koji/policy.py
|
||||
@@ -25,7 +25,7 @@ import logging
|
||||
import six
|
||||
|
||||
import koji
|
||||
-from koji.util import to_list
|
||||
+from koji.util import to_list, multi_fnmatch
|
||||
|
||||
|
||||
class BaseSimpleTest(object):
|
||||
@@ -141,6 +141,57 @@ class MatchTest(BaseSimpleTest):
|
||||
return False
|
||||
|
||||
|
||||
+class MatchAnyTest(BaseSimpleTest):
|
||||
+ """Matches any item of a list/tuple/set value in the data against glob patterns
|
||||
+
|
||||
+ True if any of the expressions matches any item in the list/tuple/set, else False.
|
||||
+ If the field doesn't exist or isn't a list/tuple/set, the test returns False
|
||||
+
|
||||
+ Syntax:
|
||||
+ find field pattern1 [pattern2 ...]
|
||||
+
|
||||
+ """
|
||||
+ name = 'match_any'
|
||||
+ field = None
|
||||
+
|
||||
+ def run(self, data):
|
||||
+ args = self.str.split()[1:]
|
||||
+ self.field = args[0]
|
||||
+ args = args[1:]
|
||||
+ tgt = data.get(self.field)
|
||||
+ if tgt and isinstance(tgt, (list, tuple, set)):
|
||||
+ for i in tgt:
|
||||
+ if i is not None and multi_fnmatch(str(i), args):
|
||||
+ return True
|
||||
+ return False
|
||||
+
|
||||
+
|
||||
+class MatchAllTest(BaseSimpleTest):
|
||||
+ """Matches all items of a list/tuple/set value in the data against glob patterns
|
||||
+
|
||||
+ True if any of the expressions matches all items in the list/tuple/set, else False.
|
||||
+ If the field doesn't exist or isn't a list/tuple/set, the test returns False
|
||||
+
|
||||
+ Syntax:
|
||||
+ match_all field pattern1 [pattern2 ...]
|
||||
+
|
||||
+ """
|
||||
+ name = 'match_all'
|
||||
+ field = None
|
||||
+
|
||||
+ def run(self, data):
|
||||
+ args = self.str.split()[1:]
|
||||
+ self.field = args[0]
|
||||
+ args = args[1:]
|
||||
+ tgt = data.get(self.field)
|
||||
+ if tgt and isinstance(tgt, (list, tuple, set)):
|
||||
+ for i in tgt:
|
||||
+ if i is None or not multi_fnmatch(str(i), args):
|
||||
+ return False
|
||||
+ return True
|
||||
+ return False
|
||||
+
|
||||
+
|
||||
class TargetTest(MatchTest):
|
||||
"""Matches target in the data against glob patterns
|
||||
|
||||
diff --git a/plugins/builder/scmpolicy.py b/plugins/builder/scmpolicy.py
|
||||
new file mode 100644
|
||||
index 00000000..f120e33b
|
||||
--- /dev/null
|
||||
+++ b/plugins/builder/scmpolicy.py
|
||||
@@ -0,0 +1,72 @@
|
||||
+import logging
|
||||
+import re
|
||||
+import subprocess
|
||||
+
|
||||
+import six
|
||||
+
|
||||
+from koji import ActionNotAllowed, GenericError
|
||||
+from koji.plugin import callback
|
||||
+
|
||||
+
|
||||
+logger = logging.getLogger('koji.plugins.scmpolicy')
|
||||
+
|
||||
+
|
||||
+@callback('postSCMCheckout')
|
||||
+def assert_scm_policy(clb_type, *args, **kwargs):
|
||||
+ taskinfo = kwargs['taskinfo']
|
||||
+ session = kwargs['session']
|
||||
+ build_tag = kwargs['build_tag']
|
||||
+ scminfo = kwargs['scminfo']
|
||||
+ srcdir = kwargs['srcdir']
|
||||
+ scratch = kwargs['scratch']
|
||||
+
|
||||
+ method = get_task_method(session, taskinfo)
|
||||
+
|
||||
+ policy_data = {
|
||||
+ 'build_tag': build_tag,
|
||||
+ 'method': method,
|
||||
+ 'scratch': scratch,
|
||||
+ 'branches': get_branches(srcdir)
|
||||
+ }
|
||||
+
|
||||
+ # Merge scminfo into data with "scm_" prefix. And "scm*" are changed to "scm_*".
|
||||
+ for k, v in six.iteritems(scminfo):
|
||||
+ policy_data[re.sub(r'^(scm_?)?', 'scm_', k)] = v
|
||||
+
|
||||
+ logger.info("Checking SCM policy for task %s", taskinfo['id'])
|
||||
+ logger.debug("Policy data: %r", policy_data)
|
||||
+
|
||||
+ # check the policy
|
||||
+ try:
|
||||
+ session.host.assertPolicy('scm', policy_data)
|
||||
+ logger.info("SCM policy check for task %s: PASSED", taskinfo['id'])
|
||||
+ except ActionNotAllowed:
|
||||
+ logger.warning("SCM policy check for task %s: DENIED", taskinfo['id'])
|
||||
+ raise
|
||||
+
|
||||
+
|
||||
+def get_task_method(session, taskinfo):
|
||||
+ """Get the Task method from taskinfo"""
|
||||
+ method = None
|
||||
+ if isinstance(taskinfo, six.integer_types):
|
||||
+ taskinfo = session.getTaskInfo(taskinfo, strict=True)
|
||||
+ if isinstance(taskinfo, dict):
|
||||
+ method = taskinfo.get('method')
|
||||
+ if method is None:
|
||||
+ raise GenericError("Invalid taskinfo: %s" % taskinfo)
|
||||
+ return method
|
||||
+
|
||||
+
|
||||
+def get_branches(srcdir):
|
||||
+ """Determine which remote branches contain the current checkout"""
|
||||
+ cmd = ['git', 'branch', '-r', '--contains', 'HEAD']
|
||||
+ proc = subprocess.Popen(cmd, cwd=srcdir, stdout=subprocess.PIPE)
|
||||
+ (out, _) = proc.communicate()
|
||||
+ status = proc.wait()
|
||||
+ if status != 0:
|
||||
+ raise Exception('Error getting branches for git checkout')
|
||||
+
|
||||
+ # cut off origin/ prefix
|
||||
+ branches = [b.strip() for b in out.decode().split('\n') if 'origin/HEAD' not in b and b]
|
||||
+ branches = [re.sub('^origin/', '', b) for b in branches]
|
||||
+ return branches
|
||||
--
|
||||
GitLab
|
||||
|
||||
@ -0,0 +1,26 @@
|
||||
From 2a6e18fa356f1aa2a1b5099e55e0af1c89ae4163 Mon Sep 17 00:00:00 2001
|
||||
From: Mike McLean <mikem@redhat.com>
|
||||
Date: Feb 05 2024 10:28:43 +0000
|
||||
Subject: typo in set_refusal
|
||||
|
||||
|
||||
Fixes https://pagure.io/koji/issue/3997
|
||||
|
||||
---
|
||||
|
||||
diff --git a/kojihub/scheduler.py b/kojihub/scheduler.py
|
||||
index 815b0f1..961ef39 100644
|
||||
--- a/kojihub/scheduler.py
|
||||
+++ b/kojihub/scheduler.py
|
||||
@@ -91,8 +91,8 @@ def get_tasks_for_host(hostID, retry=True):
|
||||
|
||||
def set_refusal(hostID, taskID, soft=True, by_host=False, msg=''):
|
||||
data = {
|
||||
- 'task_id': kojihub.convert_value(hostID, cast=int),
|
||||
- 'host_id': kojihub.convert_value(taskID, cast=int),
|
||||
+ 'host_id': kojihub.convert_value(hostID, cast=int),
|
||||
+ 'task_id': kojihub.convert_value(taskID, cast=int),
|
||||
'soft': kojihub.convert_value(soft, cast=bool),
|
||||
'by_host': kojihub.convert_value(by_host, cast=bool),
|
||||
'msg': kojihub.convert_value(msg, cast=str),
|
||||
|
||||
@ -0,0 +1,43 @@
|
||||
From 36953540662aa39ff1b85218cefededfa4e529a9 Mon Sep 17 00:00:00 2001
|
||||
From: Tomas Kopecek <tkopecek@redhat.com>
|
||||
Date: Jan 15 2024 12:48:16 +0000
|
||||
Subject: Use dnf5-compatible "group install" command
|
||||
|
||||
|
||||
* yum knows only "yum groupinstall"
|
||||
* dnf < 5 knows both
|
||||
* dnf-5 only "dnf5 group install"
|
||||
|
||||
It is reasonable to assume that dnf is used in most setups, so changing
|
||||
default to "group install". If yum is specified *explicitly* via
|
||||
tags's extra "mock.package_manager", "groupinstall" is used instead.
|
||||
|
||||
Related: https://pagure.io/koji/issue/3971
|
||||
|
||||
---
|
||||
|
||||
diff --git a/koji/__init__.py b/koji/__init__.py
|
||||
index 334b403..9b222cd 100644
|
||||
--- a/koji/__init__.py
|
||||
+++ b/koji/__init__.py
|
||||
@@ -1804,7 +1804,7 @@ def genMockConfig(name, arch, managed=False, repoid=None, tag_name=None, **opts)
|
||||
'target_arch': opts.get('target_arch', arch),
|
||||
'chroothome': '/builddir',
|
||||
# Use the group data rather than a generated rpm
|
||||
- 'chroot_setup_cmd': 'groupinstall %s' % opts.get('install_group', 'build'),
|
||||
+ 'chroot_setup_cmd': 'group install %s' % opts.get('install_group', 'build'),
|
||||
# don't encourage network access from the chroot
|
||||
'rpmbuild_networking': opts.get('use_host_resolv', False),
|
||||
'use_host_resolv': opts.get('use_host_resolv', False),
|
||||
@@ -1817,6 +1817,10 @@ def genMockConfig(name, arch, managed=False, repoid=None, tag_name=None, **opts)
|
||||
config_opts['forcearch'] = opts['forcearch']
|
||||
if opts.get('package_manager'):
|
||||
config_opts['package_manager'] = opts['package_manager']
|
||||
+ if opts['package_manager'].endswith('yum'):
|
||||
+ # backward compatibility with yum (doesn't have separate "group")
|
||||
+ config_opts['chroot_setup_cmd'] = \
|
||||
+ 'groupinstall %s' % opts.get('install_group', 'build')
|
||||
if opts.get('bootstrap_image'):
|
||||
config_opts['use_bootstrap_image'] = True
|
||||
config_opts['bootstrap_image'] = opts['bootstrap_image']
|
||||
|
||||
@ -0,0 +1,52 @@
|
||||
From bc8e6253f519eeb78fbc8740bba25e8a34490814 Mon Sep 17 00:00:00 2001
|
||||
From: Mike McLean <mikem@redhat.com>
|
||||
Date: Feb 14 2024 06:40:24 +0000
|
||||
Subject: let tag.extra override tag arches for noarch
|
||||
|
||||
|
||||
---
|
||||
|
||||
diff --git a/builder/kojid b/builder/kojid
|
||||
index b4536dd..8b81d66 100755
|
||||
--- a/builder/kojid
|
||||
+++ b/builder/kojid
|
||||
@@ -1339,23 +1339,33 @@ class BuildTask(BaseTaskHandler):
|
||||
exclusivearch = koji.get_header_field(h, 'exclusivearch')
|
||||
excludearch = koji.get_header_field(h, 'excludearch')
|
||||
|
||||
- if exclusivearch or excludearch:
|
||||
+ buildconfig = self.session.getBuildConfig(build_tag, event=self.event_id)
|
||||
+ noarch_arches = buildconfig.get('extra', {}).get('noarch_arches')
|
||||
+
|
||||
+ if exclusivearch or excludearch or noarch_arches:
|
||||
# if one of the tag arches is filtered out, then we can't use a
|
||||
# noarch task
|
||||
- buildconfig = self.session.getBuildConfig(build_tag, event=self.event_id)
|
||||
arches = buildconfig['arches']
|
||||
tag_arches = [koji.canonArch(a) for a in arches.split()]
|
||||
exclusivearch = [koji.canonArch(a) for a in exclusivearch]
|
||||
excludearch = [koji.canonArch(a) for a in excludearch]
|
||||
- archlist = list(tag_arches)
|
||||
+ # tag.extra overrides tag arches for noarch
|
||||
+ if noarch_arches:
|
||||
+ archlist = [koji.canonArch(a) for a in noarch_arches.split()]
|
||||
+ archlist = [a for a in archlist if a in tag_arches]
|
||||
+ else:
|
||||
+ archlist = list(tag_arches)
|
||||
if exclusivearch:
|
||||
archlist = [a for a in archlist if a in exclusivearch]
|
||||
if excludearch:
|
||||
archlist = [a for a in archlist if a not in excludearch]
|
||||
+ self.logger.info('Filtering arches for noarch subtask. Choices: %r', archlist)
|
||||
if not archlist:
|
||||
- raise koji.BuildError("No valid arches were found. tag %r, "
|
||||
- "exclusive %r, exclude %r" % (tag_arches,
|
||||
+ raise koji.BuildError("No valid arches were found. tag %r, extra %r,"
|
||||
+ "exclusive %r, exclude %r" % (tag_arches, noarch_arches,
|
||||
exclusivearch, excludearch))
|
||||
+ self.logger.debug('tag: %r, extra: %r, exclusive: %r, exclude: %r',
|
||||
+ tag_arches, noarch_arches, exclusivearch, excludearch)
|
||||
if set(archlist) != set(tag_arches):
|
||||
return random.choice(archlist)
|
||||
else:
|
||||
|
||||
@ -0,0 +1,38 @@
|
||||
From f453092d308605707ba1fb3fa314e05b515e7a25 Mon Sep 17 00:00:00 2001
|
||||
From: Tomas Kopecek <tkopecek@redhat.com>
|
||||
Date: Feb 29 2024 10:45:55 +0000
|
||||
Subject: Better index for rpm lookup
|
||||
|
||||
|
||||
Related: https://pagure.io/koji/issue/4022
|
||||
|
||||
---
|
||||
|
||||
diff --git a/schemas/schema-upgrade-1.34-1.35.sql b/schemas/schema-upgrade-1.34-1.35.sql
|
||||
new file mode 100644
|
||||
index 0000000..3cc82bc
|
||||
--- /dev/null
|
||||
+++ b/schemas/schema-upgrade-1.34-1.35.sql
|
||||
@@ -0,0 +1,9 @@
|
||||
+-- upgrade script to migrate the Koji database schema
|
||||
+-- from version 1.33 to 1.34
|
||||
+
|
||||
+BEGIN;
|
||||
+
|
||||
+CREATE INDEX CONCURRENTLY IF NOT EXISTS rpminfo_nvra
|
||||
+ ON rpminfo(name,version,release,arch,external_repo_id);
|
||||
+
|
||||
+COMMIT;
|
||||
diff --git a/schemas/schema.sql b/schemas/schema.sql
|
||||
index e5f3462..7e3298c 100644
|
||||
--- a/schemas/schema.sql
|
||||
+++ b/schemas/schema.sql
|
||||
@@ -752,6 +752,7 @@ CREATE TABLE rpminfo (
|
||||
CREATE INDEX rpminfo_build ON rpminfo(build_id);
|
||||
CREATE UNIQUE INDEX rpminfo_unique_nvra_not_draft ON rpminfo(name,version,release,arch,external_repo_id)
|
||||
WHERE draft IS NOT TRUE;
|
||||
+CREATE INDEX rpminfo_nvra ON rpminfo(name,version,release,arch,external_repo_id);
|
||||
-- index for default search method for rpms, PG11+ can benefit from new include method
|
||||
DO $$
|
||||
DECLARE version integer;
|
||||
|
||||
Loading…
Reference in new issue