- Backport patch to download only repomd.xml instead of all repodata. - Backport patch to allow 'bare' repo merging for modularity. - Backport patch to allow for seperate srpm repos in buildroot repos. Signed-off-by: Kevin Fenzi <kevin@scrye.com>epel9
Kevin Fenzi
6 years ago
parent
c350e2ec04
commit
c4bba768d4
@ -0,0 +1,224 @@
|
|||||||
|
From 3590ab0a355adaf63c7a5f068f7a6ac0eb5e8b05 Mon Sep 17 00:00:00 2001
|
||||||
|
From: Mikolaj Izdebski <mizdebsk@redhat.com>
|
||||||
|
Date: Feb 21 2019 12:42:18 +0000
|
||||||
|
Subject: Allow generating separate src repo for build repos
|
||||||
|
|
||||||
|
|
||||||
|
Fixes #1266
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
diff --git a/builder/kojid b/builder/kojid
|
||||||
|
index 27fe523..0fffb89 100755
|
||||||
|
--- a/builder/kojid
|
||||||
|
+++ b/builder/kojid
|
||||||
|
@@ -4990,13 +4990,15 @@ class NewRepoTask(BaseTaskHandler):
|
||||||
|
Methods = ['newRepo']
|
||||||
|
_taskWeight = 0.1
|
||||||
|
|
||||||
|
- def handler(self, tag, event=None, src=False, debuginfo=False):
|
||||||
|
+ def handler(self, tag, event=None, src=False, debuginfo=False, separate_src=False):
|
||||||
|
tinfo = self.session.getTag(tag, strict=True, event=event)
|
||||||
|
kwargs = {}
|
||||||
|
if event is not None:
|
||||||
|
kwargs['event'] = event
|
||||||
|
if src:
|
||||||
|
kwargs['with_src'] = True
|
||||||
|
+ if separate_src:
|
||||||
|
+ kwargs['with_separate_src'] = True
|
||||||
|
if debuginfo:
|
||||||
|
kwargs['with_debuginfo'] = True
|
||||||
|
repo_id, event_id = self.session.host.repoInit(tinfo['id'], **kwargs)
|
||||||
|
diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py
|
||||||
|
index a2b2b2b..1a2ace6 100644
|
||||||
|
--- a/cli/koji_cli/commands.py
|
||||||
|
+++ b/cli/koji_cli/commands.py
|
||||||
|
@@ -7011,7 +7011,8 @@ def handle_regen_repo(options, session, args):
|
||||||
|
parser.add_option("--target", action="store_true", help=_("Interpret the argument as a build target name"))
|
||||||
|
parser.add_option("--nowait", action="store_true", help=_("Don't wait on for regen to finish"))
|
||||||
|
parser.add_option("--debuginfo", action="store_true", help=_("Include debuginfo rpms in repo"))
|
||||||
|
- parser.add_option("--source", "--src", action="store_true", help=_("Include source rpms in the repo"))
|
||||||
|
+ parser.add_option("--source", "--src", action="store_true", help=_("Include source rpms in each of repos"))
|
||||||
|
+ parser.add_option("--separate-source", "--separate-src", action="store_true", help=_("Include source rpms in separate src repo"))
|
||||||
|
(suboptions, args) = parser.parse_args(args)
|
||||||
|
if len(args) == 0:
|
||||||
|
parser.error(_("A tag name must be specified"))
|
||||||
|
@@ -7047,6 +7048,8 @@ def handle_regen_repo(options, session, args):
|
||||||
|
repo_opts['debuginfo'] = True
|
||||||
|
if suboptions.source:
|
||||||
|
repo_opts['src'] = True
|
||||||
|
+ if suboptions.separate_source:
|
||||||
|
+ repo_opts['separate_src'] = True
|
||||||
|
task_id = session.newRepo(tag, **repo_opts)
|
||||||
|
print("Regenerating repo for tag: %s" % tag)
|
||||||
|
print("Created task: %d" % task_id)
|
||||||
|
diff --git a/hub/kojihub.py b/hub/kojihub.py
|
||||||
|
index 46b8f3e..9844861 100644
|
||||||
|
--- a/hub/kojihub.py
|
||||||
|
+++ b/hub/kojihub.py
|
||||||
|
@@ -2345,7 +2345,7 @@ def maven_tag_archives(tag_id, event_id=None, inherit=True):
|
||||||
|
yield archive
|
||||||
|
return _iter_archives()
|
||||||
|
|
||||||
|
-def repo_init(tag, with_src=False, with_debuginfo=False, event=None):
|
||||||
|
+def repo_init(tag, with_src=False, with_debuginfo=False, event=None, with_separate_src=False):
|
||||||
|
"""Create a new repo entry in the INIT state, return full repo data
|
||||||
|
|
||||||
|
Returns a dictionary containing
|
||||||
|
@@ -2355,9 +2355,11 @@ def repo_init(tag, with_src=False, with_debuginfo=False, event=None):
|
||||||
|
state = koji.REPO_INIT
|
||||||
|
tinfo = get_tag(tag, strict=True, event=event)
|
||||||
|
koji.plugin.run_callbacks('preRepoInit', tag=tinfo, with_src=with_src, with_debuginfo=with_debuginfo,
|
||||||
|
- event=event, repo_id=None)
|
||||||
|
+ event=event, repo_id=None, with_separate_src=with_separate_src)
|
||||||
|
tag_id = tinfo['id']
|
||||||
|
repo_arches = {}
|
||||||
|
+ if with_separate_src:
|
||||||
|
+ repo_arches['src'] = 1
|
||||||
|
if tinfo['arches']:
|
||||||
|
for arch in tinfo['arches'].split():
|
||||||
|
arch = koji.canonArch(arch)
|
||||||
|
@@ -2420,6 +2422,8 @@ def repo_init(tag, with_src=False, with_debuginfo=False, event=None):
|
||||||
|
if with_src:
|
||||||
|
for repoarch in repo_arches:
|
||||||
|
pkglist[repoarch].write(relpath)
|
||||||
|
+ if with_separate_src:
|
||||||
|
+ pkglist[arch].write(relpath)
|
||||||
|
elif arch == 'noarch':
|
||||||
|
for repoarch in repo_arches:
|
||||||
|
pkglist[repoarch].write(relpath)
|
||||||
|
@@ -2474,7 +2478,7 @@ def repo_init(tag, with_src=False, with_debuginfo=False, event=None):
|
||||||
|
_write_maven_repo_metadata(artifact_dir, artifacts)
|
||||||
|
|
||||||
|
koji.plugin.run_callbacks('postRepoInit', tag=tinfo, with_src=with_src, with_debuginfo=with_debuginfo,
|
||||||
|
- event=event, repo_id=repo_id)
|
||||||
|
+ event=event, repo_id=repo_id, with_separate_src=with_separate_src)
|
||||||
|
return [repo_id, event_id]
|
||||||
|
|
||||||
|
def _write_maven_repo_metadata(destdir, artifacts):
|
||||||
|
@@ -2519,7 +2523,8 @@ def dist_repo_init(tag, keys, task_opts):
|
||||||
|
# note: we need to match args from the other preRepoInit callback
|
||||||
|
koji.plugin.run_callbacks('preRepoInit', tag=tinfo, with_src=False,
|
||||||
|
with_debuginfo=False, event=event, repo_id=None,
|
||||||
|
- dist=True, keys=keys, arches=arches, task_opts=task_opts)
|
||||||
|
+ dist=True, keys=keys, arches=arches, task_opts=task_opts,
|
||||||
|
+ with_separate_src=False)
|
||||||
|
if not event:
|
||||||
|
event = get_event()
|
||||||
|
repo_id = nextval('repo_id_seq')
|
||||||
|
@@ -2540,7 +2545,7 @@ def dist_repo_init(tag, keys, task_opts):
|
||||||
|
koji.plugin.run_callbacks('postRepoInit', tag=tinfo, with_src=False,
|
||||||
|
with_debuginfo=False, event=event, repo_id=repo_id,
|
||||||
|
dist=True, keys=keys, arches=arches, task_opts=task_opts,
|
||||||
|
- repodir=repodir)
|
||||||
|
+ repodir=repodir, with_reparate_src=False)
|
||||||
|
return repo_id, event
|
||||||
|
|
||||||
|
|
||||||
|
@@ -10521,7 +10526,7 @@ class RootExports(object):
|
||||||
|
task_opts['event'] = event_id
|
||||||
|
return make_task('distRepo', [tag, repo_id, keys, task_opts], priority=15, channel='createrepo')
|
||||||
|
|
||||||
|
- def newRepo(self, tag, event=None, src=False, debuginfo=False):
|
||||||
|
+ def newRepo(self, tag, event=None, src=False, debuginfo=False, separate_src=False):
|
||||||
|
"""Create a newRepo task. returns task id"""
|
||||||
|
if context.session.hasPerm('regen-repo'):
|
||||||
|
pass
|
||||||
|
@@ -10532,6 +10537,8 @@ class RootExports(object):
|
||||||
|
opts['event'] = event
|
||||||
|
if src:
|
||||||
|
opts['src'] = True
|
||||||
|
+ if separate_src:
|
||||||
|
+ opts['separate_src'] = True
|
||||||
|
if debuginfo:
|
||||||
|
opts['debuginfo'] = True
|
||||||
|
args = koji.encode_args(tag, **opts)
|
||||||
|
@@ -12699,11 +12706,11 @@ class HostExports(object):
|
||||||
|
|
||||||
|
return br.updateArchiveList(archives, project)
|
||||||
|
|
||||||
|
- def repoInit(self, tag, with_src=False, with_debuginfo=False, event=None):
|
||||||
|
+ def repoInit(self, tag, with_src=False, with_debuginfo=False, event=None, with_separate_src=False):
|
||||||
|
"""Initialize a new repo for tag"""
|
||||||
|
host = Host()
|
||||||
|
host.verify()
|
||||||
|
- return repo_init(tag, with_src=with_src, with_debuginfo=with_debuginfo, event=event)
|
||||||
|
+ return repo_init(tag, with_src=with_src, with_debuginfo=with_debuginfo, event=event, with_separate_src=with_separate_src)
|
||||||
|
|
||||||
|
def repoAddRPM(self, repo_id, path):
|
||||||
|
"""Add an uploaded rpm to a repo"""
|
||||||
|
diff --git a/koji/tasks.py b/koji/tasks.py
|
||||||
|
index ba4cd31..90656b8 100644
|
||||||
|
--- a/koji/tasks.py
|
||||||
|
+++ b/koji/tasks.py
|
||||||
|
@@ -200,7 +200,7 @@ LEGACY_SIGNATURES = {
|
||||||
|
[['builds', 'target', 'opts'], None, None, (None,)],
|
||||||
|
],
|
||||||
|
'newRepo' : [
|
||||||
|
- [['tag', 'event', 'src', 'debuginfo'], None, None, (None, False, False)],
|
||||||
|
+ [['tag', 'event', 'src', 'debuginfo', 'separate_src'], None, None, (None, False, False, False)],
|
||||||
|
],
|
||||||
|
'createImage' : [
|
||||||
|
[['name', 'version', 'release', 'arch', 'target_info', 'build_tag', 'repo_info', 'inst_tree', 'opts'], None, None, (None,)],
|
||||||
|
diff --git a/tests/test_cli/test_regen_repo.py b/tests/test_cli/test_regen_repo.py
|
||||||
|
index 43eff0b..90f16b1 100644
|
||||||
|
--- a/tests/test_cli/test_regen_repo.py
|
||||||
|
+++ b/tests/test_cli/test_regen_repo.py
|
||||||
|
@@ -168,11 +168,13 @@ class TestRegenRepo(utils.CliTestCase):
|
||||||
|
(Specify the --help global option for a list of other help options)
|
||||||
|
|
||||||
|
Options:
|
||||||
|
- -h, --help show this help message and exit
|
||||||
|
- --target Interpret the argument as a build target name
|
||||||
|
- --nowait Don't wait on for regen to finish
|
||||||
|
- --debuginfo Include debuginfo rpms in repo
|
||||||
|
- --source, --src Include source rpms in the repo
|
||||||
|
+ -h, --help show this help message and exit
|
||||||
|
+ --target Interpret the argument as a build target name
|
||||||
|
+ --nowait Don't wait on for regen to finish
|
||||||
|
+ --debuginfo Include debuginfo rpms in repo
|
||||||
|
+ --source, --src Include source rpms in each of repos
|
||||||
|
+ --separate-source, --separate-src
|
||||||
|
+ Include source rpms in separate src repo
|
||||||
|
""" % self.progname)
|
||||||
|
|
||||||
|
|
||||||
|
diff --git a/util/kojira b/util/kojira
|
||||||
|
index 8e68cfd..a93d966 100755
|
||||||
|
--- a/util/kojira
|
||||||
|
+++ b/util/kojira
|
||||||
|
@@ -671,6 +671,7 @@ class RepoManager(object):
|
||||||
|
|
||||||
|
debuginfo_pat = self.options.debuginfo_tags.split()
|
||||||
|
src_pat = self.options.source_tags.split()
|
||||||
|
+ separate_src_pat = self.options.separate_source_tags.split()
|
||||||
|
order = sorted(self.needed_tags.values(), key=lambda t: t['score'])
|
||||||
|
for tag in order:
|
||||||
|
if running_tasks >= self.options.max_repo_tasks:
|
||||||
|
@@ -699,6 +700,8 @@ class RepoManager(object):
|
||||||
|
taskopts['debuginfo'] = True
|
||||||
|
if koji.util.multi_fnmatch(tagname, src_pat):
|
||||||
|
taskopts['src'] = True
|
||||||
|
+ if koji.util.multi_fnmatch(tagname, separate_src_pat):
|
||||||
|
+ taskopts['separate_src'] = True
|
||||||
|
maven = tag['taginfo']['maven_support']
|
||||||
|
if maven:
|
||||||
|
if running_tasks_maven >= self.options.max_repo_tasks_maven:
|
||||||
|
@@ -836,6 +839,7 @@ def get_options():
|
||||||
|
defaults = {'with_src': False,
|
||||||
|
'debuginfo_tags': '',
|
||||||
|
'source_tags': '',
|
||||||
|
+ 'separate_source_tags': '',
|
||||||
|
'ignore_tags': '',
|
||||||
|
'verbose': False,
|
||||||
|
'debug': False,
|
||||||
|
@@ -876,7 +880,7 @@ def get_options():
|
||||||
|
'recent_tasks_lifetime')
|
||||||
|
str_opts = ('topdir', 'server', 'user', 'password', 'logfile', 'principal', 'keytab', 'krbservice',
|
||||||
|
'cert', 'ca', 'serverca', 'debuginfo_tags',
|
||||||
|
- 'source_tags', 'ignore_tags') # FIXME: remove ca here
|
||||||
|
+ 'source_tags', 'separate_source_tags', 'ignore_tags') # FIXME: remove ca here
|
||||||
|
bool_opts = ('with_src','verbose','debug','ignore_stray_repos', 'offline_retry',
|
||||||
|
'krb_rdns', 'krb_canon_host', 'no_ssl_verify')
|
||||||
|
for name in config.options(section):
|
||||||
|
|
||||||
@ -0,0 +1,30 @@
|
|||||||
|
From a66b4ddfb4399cdc03ccc6f66acd634c692ee88f Mon Sep 17 00:00:00 2001
|
||||||
|
From: Igor Gnatenko <ignatenkobrain@fedoraproject.org>
|
||||||
|
Date: Apr 17 2019 05:38:03 +0000
|
||||||
|
Subject: kojid: Download only 'repomd.xml'
|
||||||
|
|
||||||
|
|
||||||
|
We have pretty slow connection from s390x koji which helped to uncover
|
||||||
|
this part. Kojid downloads all files from repomd.xml (incl. filelists)
|
||||||
|
which is really big. What we really want is just find location of
|
||||||
|
'origin' (used later in the code).
|
||||||
|
|
||||||
|
Signed-off-by: Igor Gnatenko <ignatenkobrain@fedoraproject.org>
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
diff --git a/builder/kojid b/builder/kojid
|
||||||
|
index 75f8a7d..5110d04 100755
|
||||||
|
--- a/builder/kojid
|
||||||
|
+++ b/builder/kojid
|
||||||
|
@@ -739,6 +739,9 @@ class BuildRoot(object):
|
||||||
|
h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO)
|
||||||
|
h.setopt(librepo.LRO_URLS, [repo_url])
|
||||||
|
h.setopt(librepo.LRO_DESTDIR, tmpdir)
|
||||||
|
+ # We are using this just to find out location of 'origin',
|
||||||
|
+ # we don't even need to download it since we use openRemoteFile
|
||||||
|
+ h.setopt(librepo.LRO_YUMDLIST, [])
|
||||||
|
h.perform(r)
|
||||||
|
pkgorigins = r.getinfo(librepo.LRR_YUM_REPOMD)['origin']['location_href']
|
||||||
|
koji.util.rmtree(tmpdir)
|
||||||
|
|
||||||
@ -0,0 +1,61 @@
|
|||||||
|
From 3c5ddb8dab9a5736f03b6d60593adc9170304af3 Mon Sep 17 00:00:00 2001
|
||||||
|
From: Mike McLean <mikem@redhat.com>
|
||||||
|
Date: May 03 2019 14:57:43 +0000
|
||||||
|
Subject: [PATCH 1/2] handle bare merge mode
|
||||||
|
|
||||||
|
|
||||||
|
Fixes https://pagure.io/koji/issue/1404
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
diff --git a/builder/kojid b/builder/kojid
|
||||||
|
index 5110d04..e176266 100755
|
||||||
|
--- a/builder/kojid
|
||||||
|
+++ b/builder/kojid
|
||||||
|
@@ -5186,6 +5186,10 @@ class CreaterepoTask(BaseTaskHandler):
|
||||||
|
cmd = ['/usr/libexec/kojid/mergerepos',
|
||||||
|
'--mode', 'simple',
|
||||||
|
'--tempdir', self.workdir]
|
||||||
|
+ elif merge_mode == 'bare':
|
||||||
|
+ # "bare" merge mode for repos with modular metadata
|
||||||
|
+ # forces use of mergerepo_c
|
||||||
|
+ cmd = ['/usr/bin/mergerepo_c', '--pkgorigins']
|
||||||
|
elif self.options.use_createrepo_c:
|
||||||
|
cmd = ['/usr/bin/mergerepo_c', '--koji']
|
||||||
|
else:
|
||||||
|
diff --git a/koji/__init__.py b/koji/__init__.py
|
||||||
|
index 6e667ce..9c7101c 100644
|
||||||
|
--- a/koji/__init__.py
|
||||||
|
+++ b/koji/__init__.py
|
||||||
|
@@ -236,7 +236,7 @@ REPO_EXPIRED = REPO_STATES['EXPIRED']
|
||||||
|
REPO_DELETED = REPO_STATES['DELETED']
|
||||||
|
REPO_PROBLEM = REPO_STATES['PROBLEM']
|
||||||
|
|
||||||
|
-REPO_MERGE_MODES = set(['koji', 'simple'])
|
||||||
|
+REPO_MERGE_MODES = set(['koji', 'simple', 'dumb'])
|
||||||
|
|
||||||
|
# buildroot states
|
||||||
|
BR_STATES = Enum((
|
||||||
|
|
||||||
|
From 0da20019456c386aa44589c3f0529926fb426541 Mon Sep 17 00:00:00 2001
|
||||||
|
From: Mike McLean <mikem@redhat.com>
|
||||||
|
Date: May 03 2019 16:54:54 +0000
|
||||||
|
Subject: [PATCH 2/2] fix mode name
|
||||||
|
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
diff --git a/koji/__init__.py b/koji/__init__.py
|
||||||
|
index 9c7101c..0b9b5bc 100644
|
||||||
|
--- a/koji/__init__.py
|
||||||
|
+++ b/koji/__init__.py
|
||||||
|
@@ -236,7 +236,7 @@ REPO_EXPIRED = REPO_STATES['EXPIRED']
|
||||||
|
REPO_DELETED = REPO_STATES['DELETED']
|
||||||
|
REPO_PROBLEM = REPO_STATES['PROBLEM']
|
||||||
|
|
||||||
|
-REPO_MERGE_MODES = set(['koji', 'simple', 'dumb'])
|
||||||
|
+REPO_MERGE_MODES = set(['koji', 'simple', 'bare'])
|
||||||
|
|
||||||
|
# buildroot states
|
||||||
|
BR_STATES = Enum((
|
||||||
|
|
||||||
Loading…
Reference in new issue