From c4bba768d42ef0a9c1e58c05421c9256668b0ddf Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 28 May 2019 12:08:02 -0700 Subject: [PATCH] - Switch kojid back to python3 as imagefactory and oz have moved. - Backport patch to download only repomd.xml instead of all repodata. - Backport patch to allow 'bare' repo merging for modularity. - Backport patch to allow for seperate srpm repos in buildroot repos. Signed-off-by: Kevin Fenzi --- 1273.patch | 224 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1398.patch | 30 +++++++ 1411.patch | 61 +++++++++++++++ koji.spec | 31 ++++++-- 4 files changed, 338 insertions(+), 8 deletions(-) create mode 100644 1273.patch create mode 100644 1398.patch create mode 100644 1411.patch diff --git a/1273.patch b/1273.patch new file mode 100644 index 0000000..7c46251 --- /dev/null +++ b/1273.patch @@ -0,0 +1,224 @@ +From 3590ab0a355adaf63c7a5f068f7a6ac0eb5e8b05 Mon Sep 17 00:00:00 2001 +From: Mikolaj Izdebski +Date: Feb 21 2019 12:42:18 +0000 +Subject: Allow generating separate src repo for build repos + + +Fixes #1266 + +--- + +diff --git a/builder/kojid b/builder/kojid +index 27fe523..0fffb89 100755 +--- a/builder/kojid ++++ b/builder/kojid +@@ -4990,13 +4990,15 @@ class NewRepoTask(BaseTaskHandler): + Methods = ['newRepo'] + _taskWeight = 0.1 + +- def handler(self, tag, event=None, src=False, debuginfo=False): ++ def handler(self, tag, event=None, src=False, debuginfo=False, separate_src=False): + tinfo = self.session.getTag(tag, strict=True, event=event) + kwargs = {} + if event is not None: + kwargs['event'] = event + if src: + kwargs['with_src'] = True ++ if separate_src: ++ kwargs['with_separate_src'] = True + if debuginfo: + kwargs['with_debuginfo'] = True + repo_id, event_id = self.session.host.repoInit(tinfo['id'], **kwargs) +diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py +index a2b2b2b..1a2ace6 100644 +--- a/cli/koji_cli/commands.py ++++ b/cli/koji_cli/commands.py +@@ -7011,7 +7011,8 @@ def handle_regen_repo(options, session, args): + parser.add_option("--target", action="store_true", help=_("Interpret the argument as a build target name")) + parser.add_option("--nowait", action="store_true", help=_("Don't wait on for regen to finish")) + parser.add_option("--debuginfo", action="store_true", help=_("Include debuginfo rpms in repo")) +- parser.add_option("--source", "--src", action="store_true", help=_("Include source rpms in the repo")) ++ parser.add_option("--source", "--src", action="store_true", help=_("Include source rpms in each of repos")) ++ parser.add_option("--separate-source", "--separate-src", action="store_true", help=_("Include source rpms in separate src repo")) + (suboptions, args) = parser.parse_args(args) + if len(args) == 0: + parser.error(_("A tag name must be specified")) +@@ -7047,6 +7048,8 @@ def handle_regen_repo(options, session, args): + repo_opts['debuginfo'] = True + if suboptions.source: + repo_opts['src'] = True ++ if suboptions.separate_source: ++ repo_opts['separate_src'] = True + task_id = session.newRepo(tag, **repo_opts) + print("Regenerating repo for tag: %s" % tag) + print("Created task: %d" % task_id) +diff --git a/hub/kojihub.py b/hub/kojihub.py +index 46b8f3e..9844861 100644 +--- a/hub/kojihub.py ++++ b/hub/kojihub.py +@@ -2345,7 +2345,7 @@ def maven_tag_archives(tag_id, event_id=None, inherit=True): + yield archive + return _iter_archives() + +-def repo_init(tag, with_src=False, with_debuginfo=False, event=None): ++def repo_init(tag, with_src=False, with_debuginfo=False, event=None, with_separate_src=False): + """Create a new repo entry in the INIT state, return full repo data + + Returns a dictionary containing +@@ -2355,9 +2355,11 @@ def repo_init(tag, with_src=False, with_debuginfo=False, event=None): + state = koji.REPO_INIT + tinfo = get_tag(tag, strict=True, event=event) + koji.plugin.run_callbacks('preRepoInit', tag=tinfo, with_src=with_src, with_debuginfo=with_debuginfo, +- event=event, repo_id=None) ++ event=event, repo_id=None, with_separate_src=with_separate_src) + tag_id = tinfo['id'] + repo_arches = {} ++ if with_separate_src: ++ repo_arches['src'] = 1 + if tinfo['arches']: + for arch in tinfo['arches'].split(): + arch = koji.canonArch(arch) +@@ -2420,6 +2422,8 @@ def repo_init(tag, with_src=False, with_debuginfo=False, event=None): + if with_src: + for repoarch in repo_arches: + pkglist[repoarch].write(relpath) ++ if with_separate_src: ++ pkglist[arch].write(relpath) + elif arch == 'noarch': + for repoarch in repo_arches: + pkglist[repoarch].write(relpath) +@@ -2474,7 +2478,7 @@ def repo_init(tag, with_src=False, with_debuginfo=False, event=None): + _write_maven_repo_metadata(artifact_dir, artifacts) + + koji.plugin.run_callbacks('postRepoInit', tag=tinfo, with_src=with_src, with_debuginfo=with_debuginfo, +- event=event, repo_id=repo_id) ++ event=event, repo_id=repo_id, with_separate_src=with_separate_src) + return [repo_id, event_id] + + def _write_maven_repo_metadata(destdir, artifacts): +@@ -2519,7 +2523,8 @@ def dist_repo_init(tag, keys, task_opts): + # note: we need to match args from the other preRepoInit callback + koji.plugin.run_callbacks('preRepoInit', tag=tinfo, with_src=False, + with_debuginfo=False, event=event, repo_id=None, +- dist=True, keys=keys, arches=arches, task_opts=task_opts) ++ dist=True, keys=keys, arches=arches, task_opts=task_opts, ++ with_separate_src=False) + if not event: + event = get_event() + repo_id = nextval('repo_id_seq') +@@ -2540,7 +2545,7 @@ def dist_repo_init(tag, keys, task_opts): + koji.plugin.run_callbacks('postRepoInit', tag=tinfo, with_src=False, + with_debuginfo=False, event=event, repo_id=repo_id, + dist=True, keys=keys, arches=arches, task_opts=task_opts, +- repodir=repodir) ++ repodir=repodir, with_reparate_src=False) + return repo_id, event + + +@@ -10521,7 +10526,7 @@ class RootExports(object): + task_opts['event'] = event_id + return make_task('distRepo', [tag, repo_id, keys, task_opts], priority=15, channel='createrepo') + +- def newRepo(self, tag, event=None, src=False, debuginfo=False): ++ def newRepo(self, tag, event=None, src=False, debuginfo=False, separate_src=False): + """Create a newRepo task. returns task id""" + if context.session.hasPerm('regen-repo'): + pass +@@ -10532,6 +10537,8 @@ class RootExports(object): + opts['event'] = event + if src: + opts['src'] = True ++ if separate_src: ++ opts['separate_src'] = True + if debuginfo: + opts['debuginfo'] = True + args = koji.encode_args(tag, **opts) +@@ -12699,11 +12706,11 @@ class HostExports(object): + + return br.updateArchiveList(archives, project) + +- def repoInit(self, tag, with_src=False, with_debuginfo=False, event=None): ++ def repoInit(self, tag, with_src=False, with_debuginfo=False, event=None, with_separate_src=False): + """Initialize a new repo for tag""" + host = Host() + host.verify() +- return repo_init(tag, with_src=with_src, with_debuginfo=with_debuginfo, event=event) ++ return repo_init(tag, with_src=with_src, with_debuginfo=with_debuginfo, event=event, with_separate_src=with_separate_src) + + def repoAddRPM(self, repo_id, path): + """Add an uploaded rpm to a repo""" +diff --git a/koji/tasks.py b/koji/tasks.py +index ba4cd31..90656b8 100644 +--- a/koji/tasks.py ++++ b/koji/tasks.py +@@ -200,7 +200,7 @@ LEGACY_SIGNATURES = { + [['builds', 'target', 'opts'], None, None, (None,)], + ], + 'newRepo' : [ +- [['tag', 'event', 'src', 'debuginfo'], None, None, (None, False, False)], ++ [['tag', 'event', 'src', 'debuginfo', 'separate_src'], None, None, (None, False, False, False)], + ], + 'createImage' : [ + [['name', 'version', 'release', 'arch', 'target_info', 'build_tag', 'repo_info', 'inst_tree', 'opts'], None, None, (None,)], +diff --git a/tests/test_cli/test_regen_repo.py b/tests/test_cli/test_regen_repo.py +index 43eff0b..90f16b1 100644 +--- a/tests/test_cli/test_regen_repo.py ++++ b/tests/test_cli/test_regen_repo.py +@@ -168,11 +168,13 @@ class TestRegenRepo(utils.CliTestCase): + (Specify the --help global option for a list of other help options) + + Options: +- -h, --help show this help message and exit +- --target Interpret the argument as a build target name +- --nowait Don't wait on for regen to finish +- --debuginfo Include debuginfo rpms in repo +- --source, --src Include source rpms in the repo ++ -h, --help show this help message and exit ++ --target Interpret the argument as a build target name ++ --nowait Don't wait on for regen to finish ++ --debuginfo Include debuginfo rpms in repo ++ --source, --src Include source rpms in each of repos ++ --separate-source, --separate-src ++ Include source rpms in separate src repo + """ % self.progname) + + +diff --git a/util/kojira b/util/kojira +index 8e68cfd..a93d966 100755 +--- a/util/kojira ++++ b/util/kojira +@@ -671,6 +671,7 @@ class RepoManager(object): + + debuginfo_pat = self.options.debuginfo_tags.split() + src_pat = self.options.source_tags.split() ++ separate_src_pat = self.options.separate_source_tags.split() + order = sorted(self.needed_tags.values(), key=lambda t: t['score']) + for tag in order: + if running_tasks >= self.options.max_repo_tasks: +@@ -699,6 +700,8 @@ class RepoManager(object): + taskopts['debuginfo'] = True + if koji.util.multi_fnmatch(tagname, src_pat): + taskopts['src'] = True ++ if koji.util.multi_fnmatch(tagname, separate_src_pat): ++ taskopts['separate_src'] = True + maven = tag['taginfo']['maven_support'] + if maven: + if running_tasks_maven >= self.options.max_repo_tasks_maven: +@@ -836,6 +839,7 @@ def get_options(): + defaults = {'with_src': False, + 'debuginfo_tags': '', + 'source_tags': '', ++ 'separate_source_tags': '', + 'ignore_tags': '', + 'verbose': False, + 'debug': False, +@@ -876,7 +880,7 @@ def get_options(): + 'recent_tasks_lifetime') + str_opts = ('topdir', 'server', 'user', 'password', 'logfile', 'principal', 'keytab', 'krbservice', + 'cert', 'ca', 'serverca', 'debuginfo_tags', +- 'source_tags', 'ignore_tags') # FIXME: remove ca here ++ 'source_tags', 'separate_source_tags', 'ignore_tags') # FIXME: remove ca here + bool_opts = ('with_src','verbose','debug','ignore_stray_repos', 'offline_retry', + 'krb_rdns', 'krb_canon_host', 'no_ssl_verify') + for name in config.options(section): + diff --git a/1398.patch b/1398.patch new file mode 100644 index 0000000..6fbcbaa --- /dev/null +++ b/1398.patch @@ -0,0 +1,30 @@ +From a66b4ddfb4399cdc03ccc6f66acd634c692ee88f Mon Sep 17 00:00:00 2001 +From: Igor Gnatenko +Date: Apr 17 2019 05:38:03 +0000 +Subject: kojid: Download only 'repomd.xml' + + +We have pretty slow connection from s390x koji which helped to uncover +this part. Kojid downloads all files from repomd.xml (incl. filelists) +which is really big. What we really want is just find location of +'origin' (used later in the code). + +Signed-off-by: Igor Gnatenko + +--- + +diff --git a/builder/kojid b/builder/kojid +index 75f8a7d..5110d04 100755 +--- a/builder/kojid ++++ b/builder/kojid +@@ -739,6 +739,9 @@ class BuildRoot(object): + h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO) + h.setopt(librepo.LRO_URLS, [repo_url]) + h.setopt(librepo.LRO_DESTDIR, tmpdir) ++ # We are using this just to find out location of 'origin', ++ # we don't even need to download it since we use openRemoteFile ++ h.setopt(librepo.LRO_YUMDLIST, []) + h.perform(r) + pkgorigins = r.getinfo(librepo.LRR_YUM_REPOMD)['origin']['location_href'] + koji.util.rmtree(tmpdir) + diff --git a/1411.patch b/1411.patch new file mode 100644 index 0000000..2957c4e --- /dev/null +++ b/1411.patch @@ -0,0 +1,61 @@ +From 3c5ddb8dab9a5736f03b6d60593adc9170304af3 Mon Sep 17 00:00:00 2001 +From: Mike McLean +Date: May 03 2019 14:57:43 +0000 +Subject: [PATCH 1/2] handle bare merge mode + + +Fixes https://pagure.io/koji/issue/1404 + +--- + +diff --git a/builder/kojid b/builder/kojid +index 5110d04..e176266 100755 +--- a/builder/kojid ++++ b/builder/kojid +@@ -5186,6 +5186,10 @@ class CreaterepoTask(BaseTaskHandler): + cmd = ['/usr/libexec/kojid/mergerepos', + '--mode', 'simple', + '--tempdir', self.workdir] ++ elif merge_mode == 'bare': ++ # "bare" merge mode for repos with modular metadata ++ # forces use of mergerepo_c ++ cmd = ['/usr/bin/mergerepo_c', '--pkgorigins'] + elif self.options.use_createrepo_c: + cmd = ['/usr/bin/mergerepo_c', '--koji'] + else: +diff --git a/koji/__init__.py b/koji/__init__.py +index 6e667ce..9c7101c 100644 +--- a/koji/__init__.py ++++ b/koji/__init__.py +@@ -236,7 +236,7 @@ REPO_EXPIRED = REPO_STATES['EXPIRED'] + REPO_DELETED = REPO_STATES['DELETED'] + REPO_PROBLEM = REPO_STATES['PROBLEM'] + +-REPO_MERGE_MODES = set(['koji', 'simple']) ++REPO_MERGE_MODES = set(['koji', 'simple', 'dumb']) + + # buildroot states + BR_STATES = Enum(( + +From 0da20019456c386aa44589c3f0529926fb426541 Mon Sep 17 00:00:00 2001 +From: Mike McLean +Date: May 03 2019 16:54:54 +0000 +Subject: [PATCH 2/2] fix mode name + + +--- + +diff --git a/koji/__init__.py b/koji/__init__.py +index 9c7101c..0b9b5bc 100644 +--- a/koji/__init__.py ++++ b/koji/__init__.py +@@ -236,7 +236,7 @@ REPO_EXPIRED = REPO_STATES['EXPIRED'] + REPO_DELETED = REPO_STATES['DELETED'] + REPO_PROBLEM = REPO_STATES['PROBLEM'] + +-REPO_MERGE_MODES = set(['koji', 'simple', 'dumb']) ++REPO_MERGE_MODES = set(['koji', 'simple', 'bare']) + + # buildroot states + BR_STATES = Enum(( + diff --git a/koji.spec b/koji.spec index b6d3e61..f537d24 100644 --- a/koji.spec +++ b/koji.spec @@ -44,9 +44,6 @@ %endif %endif -# Until imgfac is ported, keep kojid at py2 -%bcond_without py2_kojid - # Lastly enforce the bcond parameters %if %{without python2} %define py2_support 0 @@ -81,7 +78,7 @@ Name: koji Version: 1.17.0 -Release: 5%{?dist} +Release: 6%{?dist} # the included arch lib from yum's rpmUtils is GPLv2+ License: LGPLv2 and GPLv2+ Summary: Build system tools @@ -93,6 +90,15 @@ Source0: https://releases.pagure.org/koji/koji-%{version}.tar.bz2 ## From: https://pagure.io/koji/pull-request/1278 Patch10: koji-PR1278-use-createrepo_c-by-default.patch +# Download only the repomd.xml instead of all the repodata +Patch11: https://pagure.io/koji/pull-request/1398.patch + +# Allow generating seperate srpm repos in buildroot repos +Patch12: https://pagure.io/koji/pull-request/1273.patch + +# Handle 'bare' merge mode for repos +Patch13: https://pagure.io/koji/pull-request/1411.patch + # Not upstreamable Patch100: fedora-config.patch @@ -297,7 +303,7 @@ Requires: /usr/bin/cvs Requires: /usr/bin/svn Requires: /usr/bin/git Requires: createrepo_c >= 0.10.0 -%if 0%{py3_support} > 1 && (! %{with py2_kojid}) +%if 0%{py3_support} > 1 Requires: python%{python3_pkgversion}-%{name} = %{version}-%{release} Requires: python%{python3_pkgversion}-librepo Requires: python%{python3_pkgversion}-multilib @@ -412,9 +418,12 @@ koji-web is a web UI to the Koji system. %prep %setup -q -%patch10 -p1 -b .createrepo_c +%patch10 -p1 +%patch11 -p1 +%patch12 -p1 +%patch13 -p1 -%patch100 -p1 -b .fedoraconfig +%patch100 -p1 %build @@ -445,7 +454,7 @@ done %if 0%{py3_support} > 1 make DESTDIR=$RPM_BUILD_ROOT PYTHON=%{__python3} %{?install_opt} install # alter python interpreter in koji CLI -scripts='%{_bindir}/koji %{!?with_py2_kojid:%{_sbindir}/kojid} %{_sbindir}/kojira %{_sbindir}/koji-shadow +scripts='%{_bindir}/koji %{_sbindir}/kojid %{_sbindir}/kojira %{_sbindir}/koji-shadow %{_sbindir}/koji-gc %{_sbindir}/kojivmd' for fn in $scripts ; do sed -i 's|#!/usr/bin/python2|#!/usr/bin/python3|' $RPM_BUILD_ROOT$fn @@ -690,6 +699,12 @@ fi %endif %changelog +* Tue May 28 2019 Kevin Fenzi - 1.17.0-6 +- Switch kojid back to python3 as imagefactory and oz have moved. +- Backport patch to download only repomd.xml instead of all repodata. +- Backport patch to allow 'bare' repo merging for modularity. +- Backport patch to allow for seperate srpm repos in buildroot repos. + * Mon Mar 11 2019 Neal Gompa - 1.17.0-5 - Switch kojid back to Python 2 so that imgfac doesn't get disabled